def mu2eFiles(spec, fileInfo=False, limit=10000000 ): ''' a function to make a list of files using a file name, a datatset, a dataset definition, or a samweb dimension statement ''' # number of all types of fields nf = len(spec.replace(":"," ").replace("="," ").split()) # number of fields delimited by dots nd = len(spec.split(".")) samweb = samweb_client.SAMWebClient() if nf==1 and nd==6 : dims = "file_name="+spec else : if nf==1 and nd==5: # just a dataset name dims = "dh.dataset="+spec elif nf==1 and nd<5: # probably a dataset def dims = "defname : "+spec else: # dataset def or more complicated samweb selection dims = spec dims = dims+" with limit {0:d}".format(limit) # get the list list = samweb.listFiles(dims, fileinfo=fileInfo, stream=False) return list
def run(self, args ): ''' execute the on-tape command, to count files on tape args = list of the args to the command ''' if( len(args) != 1 ): print >> sys.stderr, "ERROR: on-tape expects one argument as input" return 1 aa = args[0] nd = len(aa.split(".")) if( nd != 5 ): print >> sys.stderr, "ERROR: on-tape expects a dataset name as input" return 1 self.samweb = samweb_client.SAMWebClient() dims = "dh.dataset="+args[0] # get the full list of files summary = self.samweb.listFilesSummary(dims) dims = dims+" and tape_label %" # now the ones on tape summary_tape = self.samweb.listFilesSummary(dims) nt = summary["file_count"] no = summary_tape["file_count"] print("{0:6d} total {1:6d} on tape {2:6d} not on tape".format(nt,no,nt-no)) return 0
def Decode(Run): # This function interacts with Samweb to locate the run file for the requested run. # Either the run has already been decoded, in which case the path to the decoded file # is returned, or the run file needs to be decoded, in which case the function launches # a LArSoft job to decode the file. In either case the function returns the path to # a decoded file for the requested run. logging.debug('[ Decode() ]: Decode() called with Run = ' + Run) # Configure the samweb_client object, which serves as a python interface for Samweb. # We need to tell it which experiment (ICARUS) the data files are as part of the # configuration process. Then we list the data files for the requested run number. samweb = samweb_client.SAMWebClient(experiment='icarus') Files = np.array(samweb.listFiles('run_number='+Run)) logging.debug('[ Decode() ]: Files returned by Samweb: \n' + str(Files)) # By default we select the first file for each run, which is facilitated by the boolean # mask. In theory a file from each DataLogger could pass the mask, but as of now the only # DataLogger being sent to tape is DataLogger1. Then we use Samweb to find the locations # of the files and append to the paths the file name. This creates a list with the full # path of each of the files. Mask = [ '_1_' in x for x in Files ] logging.debug('[ Decode() ]: Selected file: ' + str(Files[Mask])) Locations = [ samweb.locateFile(x)[0]['full_path'].split(':')[1] for x in Files[Mask] ] ToProcess = [ Locations[x] + '/' + Files[Mask][x] for x in range(len(Files[Mask])) ] logging.debug('[ Decode() ]: Full path of file: ' + str(ToProcess)) # The decoder tends to produce Supplemental files, which we should tidy up. if len(glob.glob('Supplemental*.root')) > 0: os.system('rm Supplemental*.root') logging.debug('[ Decode() ]: Removing supplemental file.') # Now we check if there are any already decoded files for the run, and if not we launch a # a decoder job on the first file in ToProcess. MatchingFiles = glob.glob('*' + Run + '_1_*-decode.root') logging.debug('[ Decode() ]: Matching glob for run: ' + str(MatchingFiles)) if len(MatchingFiles) < 1: logging.debug('[ Decode() ]: No decoded file found for run. Begin decode job.') os.system('lar -c decoder.fcl -n 50 ' + ToProcess[0]) os.system('rm Supplemental*.root') # We want to double check that the decoder job was successful. If we find a decoded file, # then return the full path. Else tell the user that no file has been found and exit. MatchingFiles = glob.glob('*' + Run + '_1_*-decode.root') if len(MatchingFiles) > 0: logging.debug('[ Decode() ]: Successfully returning file: ' + MatchingFiles[0]) return MatchingFiles[0] else: logging.debug('[ Decode() ]: No matching file found. Returning None.') print('No matching files.') sys.exit(0)
def mu2eLocateList(list): ''' a utility function to find pnfs locations of a list of files if a file doesn't have a location, don't return an entry for it ''' samweb = samweb_client.SAMWebClient() ret = [] for fn in list: # locList is a list of locations locList = samweb.locateFile(fn) # find location in enstore - if more than one, take first for loc in locList: if loc['system'] == "enstore": lstr = loc['location'].split(':')[-1].split('(')[0] ret.append(lstr + "/" + fn) break return ret
def getTimeStampFromSAM(fname): #pull json from adjacent fname+".json" file jfname = fname + ".json" with open(jfname) as data_file: data = json.load(data_file) #print data parentfname = data['parents'][0]['file_name'] print "Found parent filename:", parentfname print "Querying SAM metadata..." samweb = samweb_client.SAMWebClient(experiment='uboone') mdata = samweb.getMetadata(parentfname) print "Found parent start,end:", mdata['start_time'], ",", mdata[ 'end_time'] dstart = datetime.datetime.strptime(mdata['start_time'], "%Y-%m-%dT%H:%M:%S+00:00") dend = datetime.datetime.strptime(mdata['end_time'], "%Y-%m-%dT%H:%M:%S+00:00") run = data['runs'][0][0] start = time.mktime(dstart.timetuple()) end = time.mktime(dend.timetuple()) return start, end, run
def run(self, options, args): if not (len(args) == 1): raise CmdError("wrong number of arguments") aa = args[0] nd = len(aa.split(".")) if (nd != 5): raise CmdError("on-tape expects a dataset name as input") self.samweb = samweb_client.SAMWebClient() dims = "dh.dataset=" + args[0] + " with availability anylocation" # get the full list of files summary = self.samweb.listFilesSummary(dims) dims = "dh.dataset=" + args[0] + " and tape_label %" # now the ones on tape summary_tape = self.samweb.listFilesSummary(dims) nt = summary["file_count"] no = summary_tape["file_count"] print("{0:6d} total {1:6d} on tape {2:6d} not on tape".format( nt, no, nt - no)) return 0
import samweb_client import datetime from slowmondb import * from FixedOffset import * samweb = samweb_client.SAMWebClient(experiment='uboone') files = samweb.listFiles("run_number 3702") fsmcdb = smcdb() for f in files: print f #get sam meta data for this file mdata = samweb.getMetadata(f) #dates from sam are formatted like 2015-11-12T12:00:06+00:00 #put start and end into datetimes start = parseISOTime(mdata['start_time']) end = parseISOTime(mdata['end_time']) print "querying for %s to %s" % (start, end) #print fsmcdb.ChannelStatsInTimeRange('uB_ArPurity_PM01_0/LIFETIME',start,end) print fsmcdb.GetClosestChannelValue('uB_TPCDrift_HV01_1_0/voltage', start) #print fsmcdb.GetClosestChannelValue('uB_ArPurity_PM01_0/LIFETIME',end) #print fsmcdb.GetClosestChannelValue('uB_TPCDrift_HV01_1_0/voltage',datetime.datetime.now()) exit()
import samweb_client as sw import os samweb = sw.SAMWebClient(experiment='uboone') x = 6120 while x < 6340: arg1 = "run_number = %i and (ub_project.stage = mergeext_unbiased or ub_project.stage = mergebnb_unbiased or ub_project.stage = nubnb_unbiased)" %(x) rawData = samweb.listFilesSummary(arg1) num_events = rawData['total_event_count'] if type(num_events) != int: sys_out2 = "sed -i -e 's/runnum_%i/runnum_%i/g' SingleRunGain.xml"%(x,x+1) os.system(sys_out2) x+=1 continue sys_out="samweb create-definition runnum_%i 'run_number = %i and (ub_project.stage = mergeext_unbiased or ub_project.stage = mergebnb_unbiased or ub_project.stage = nubnb_unbiased)'" %(x,x) os.system(sys_out) os.system("project.py --xml /uboone/app/users/moon/OpticalStudies/v05_01_01/workdir/GainsDatabase/SingleRunGain.xml --stage RunSingleGain --clean") os.system("project.py --xml /uboone/app/users/moon/OpticalStudies/v05_01_01/workdir/GainsDatabase/SingleRunGain.xml --stage RunSingleGain --submit") sys_out2 = "sed -i -e 's/runnum_%i/runnum_%i/g' SingleRunGain.xml"%(x,x+1) os.system(sys_out2) x+=1
def setUp(self): self.samweb = samweb_client.SAMWebClient(experiment=self.experiment)
backupCount=10, ) #handler = logging.StreamHandler() handler.setFormatter(formatter) handler.setLevel(logging.DEBUG) logger = logging.getLogger(logger_name) logger.addHandler(handler) logger.setLevel(logging.DEBUG) #/////////////////////////////////////////////////////////////////////// # configure samweb #/////////////////////////////////////////////////////////////////////// samweb = samweb_client.SAMWebClient(experiment='lariat') def raw_file_path_from_sam(file_name): """ Returns file path of raw data art ROOT file from SAM given a file name. An exception is raised of the file does not exist. """ file_path_dict = samweb.locateFile(file_name) system = file_path_dict[0]['system'] file_path = \ file_path_dict[0]['full_path'].split(system + ':')[1] \ + '/' + file_name return file_path
def __init__(self): self.samweb = samweb_client.SAMWebClient()
def run(self, args ): ''' execute the toPnfs command, to make a list of filespecs args = list of the args to the command ''' #print(len(args),args) #options, remainder = getopt.getopt(sys.argv[1:], 'h', #['help']) if( len(args) != 1 ): print >> sys.stderr, "ERROR: to-pnfs expects one argument as input" return 1 aa = args[0] # number of all types of fields nf = len(aa.replace(":"," ").replace("="," ").split()) # number of fields delimited by dots nd = len(aa.split(".")) #print(na," ",aa) self.samweb = samweb_client.SAMWebClient() if nf==1 and nd == 6 : # a single file, run samweb locate-file ret = self.locateOneLocalFile(aa, True) if ret[0] == 0: print(ret[1]+"/"+aa) else : if nf==1 and nd == 5: # just a dataset name dims = "dh.dataset="+aa elif nf==1 and nd<5: # probably a dataset def dims = "defname : "+aa else: # dataset def or more complicated samweb selection dims = aa # get the full list of files list = self.samweb.listFiles(dims, fileinfo=True, stream=False) # locate the first only first = list[0] ret = self.locateOneLocalFile(first[0], True) if ret[0] != 0 : return ret[0] # now loop over file and print calculated file base = "/".join(ret[1].split('/')[0:-2]) test = ret[1].split('/')[3] oldStyle = True if test in ['tape','disk','persistent','scratch']: oldStyle = False for line in list: name = line[0] if oldStyle: x = int(line[1]) x1 = x/1000000 x2 = x/1000 spr = "{0:03d}/{1:03d}".format(x1,x2) else: hasher = hashlib.sha256() hasher.update(name) hh = hasher.hexdigest() spr = hh[0:2]+'/'+hh[2:4] print(base+'/'+spr+'/'+name) return 0
def run(self, args ): ''' execute the on-tape command, to count files on tape args = list of the args to the command ''' #print(len(args),args) options, remainder = getopt.getopt(args, 'n:d:', ["limit=","directory="]) if( len(remainder) != 1 ): print >> sys.stderr, "ERROR: get-files expects one argument as input" return 1 nlimit = 100 diry = "." for opt in options: if opt[0]=="-n" or opt[0]=="--limit": nlimit = int(opt[1]) if opt[0]=="-d" or opt[0]=="--directory": diry = opt[1] if nlimit>100: nlimit = 100 # use utility function to interpret the filespec # this returns sam names list = mu2eFiles(remainder[0],False,nlimit) #print(list) #return 0 # now locate them cmdlist = [] samweb = samweb_client.SAMWebClient() for fn in list: loca = samweb.locateFile(fn) if len(loca) == 0 : print >> sys.stderr, "ERROR: {0:s} has no location".format(fn) else: loc = loca[0] lstr = loc['location'].split(':')[-1].split('(')[0] cmdlist.append(lstr+"/"+fn+" "+diry+"/"+fn+"\n") # write out the commands to a temp file ret = ShellCommand("mktemp", 1) if ret[0] or ret[1]!=0: print >> sys.stderr, "ERROR: could not make a temp file" return 1 #print(ret[2]) # mktemp comes back with a teminal \n tempFileName = ret[2].replace("\n","") tempFile = open(tempFileName, "w") for line in cmdlist: tempFile.write(line) tempFile.close() #print("cat ",tempFile.name) #return 1 # execute a bulk ifdh transfer cmd = "source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups" cmd = cmd+ " ; " cmd = cmd+ "[ -n \"$SETUP_PYTHON\" ] && unsetup python" cmd = cmd+ " ; " cmd = cmd+ "setup ifdhc" cmd = cmd+ " ; " cmd = cmd+ "ifdh cp -f "+tempFileName ret = ShellCommand(cmd, 1) #print(cmd) #print(ret[2]) if ret[0] or ret[1]!=0 : print >> sys.stderr, "ERROR: could not run ifdh cp to move files" return 2 # remove temp file cmd = "rm -f "+tempFileName ret = ShellCommand(cmd, 1) return 0