def mu2eFiles(spec, fileInfo=False, limit=10000000 ):
'''
a function to make a list of files using a file name, a datatset,
a dataset definition, or a samweb dimension statement
'''
# number of all types of fields
nf = len(spec.replace(":"," ").replace("="," ").split())
# number of fields delimited by dots
nd = len(spec.split("."))
samweb = samweb_client.SAMWebClient()
if nf==1 and nd==6 :
dims = "file_name="+spec
else :
if nf==1 and nd==5:
# just a dataset name
dims = "dh.dataset="+spec
elif nf==1 and nd<5:
# probably a dataset def
dims = "defname : "+spec
else: # dataset def or more complicated samweb selection
dims = spec
dims = dims+" with limit {0:d}".format(limit)
# get the list
list = samweb.listFiles(dims, fileinfo=fileInfo, stream=False)
return list
def run(self, args ):
'''
execute the on-tape command, to count files on tape
args = list of the args to the command
'''
if( len(args) != 1 ):
print >> sys.stderr, "ERROR: on-tape expects one argument as input"
return 1
aa = args[0]
nd = len(aa.split("."))
if( nd != 5 ):
print >> sys.stderr, "ERROR: on-tape expects a dataset name as input"
return 1
self.samweb = samweb_client.SAMWebClient()
dims = "dh.dataset="+args[0]
# get the full list of files
summary = self.samweb.listFilesSummary(dims)
dims = dims+" and tape_label %"
# now the ones on tape
summary_tape = self.samweb.listFilesSummary(dims)
nt = summary["file_count"]
no = summary_tape["file_count"]
print("{0:6d} total {1:6d} on tape {2:6d} not on tape".format(nt,no,nt-no))
return 0
def Decode(Run):
# This function interacts with Samweb to locate the run file for the requested run.
# Either the run has already been decoded, in which case the path to the decoded file
# is returned, or the run file needs to be decoded, in which case the function launches
# a LArSoft job to decode the file. In either case the function returns the path to
# a decoded file for the requested run.
logging.debug('[ Decode() ]: Decode() called with Run = ' + Run)
# Configure the samweb_client object, which serves as a python interface for Samweb.
# We need to tell it which experiment (ICARUS) the data files are as part of the
# configuration process. Then we list the data files for the requested run number.
samweb = samweb_client.SAMWebClient(experiment='icarus')
Files = np.array(samweb.listFiles('run_number='+Run))
logging.debug('[ Decode() ]: Files returned by Samweb: \n' + str(Files))
# By default we select the first file for each run, which is facilitated by the boolean
# mask. In theory a file from each DataLogger could pass the mask, but as of now the only
# DataLogger being sent to tape is DataLogger1. Then we use Samweb to find the locations
# of the files and append to the paths the file name. This creates a list with the full
# path of each of the files.
Mask = [ '_1_' in x for x in Files ]
logging.debug('[ Decode() ]: Selected file: ' + str(Files[Mask]))
Locations = [ samweb.locateFile(x)[0]['full_path'].split(':')[1] for x in Files[Mask] ]
ToProcess = [ Locations[x] + '/' + Files[Mask][x] for x in range(len(Files[Mask])) ]
logging.debug('[ Decode() ]: Full path of file: ' + str(ToProcess))
# The decoder tends to produce Supplemental files, which we should tidy up.
if len(glob.glob('Supplemental*.root')) > 0:
os.system('rm Supplemental*.root')
logging.debug('[ Decode() ]: Removing supplemental file.')
# Now we check if there are any already decoded files for the run, and if not we launch a
# a decoder job on the first file in ToProcess.
MatchingFiles = glob.glob('*' + Run + '_1_*-decode.root')
logging.debug('[ Decode() ]: Matching glob for run: ' + str(MatchingFiles))
if len(MatchingFiles) < 1:
logging.debug('[ Decode() ]: No decoded file found for run. Begin decode job.')
os.system('lar -c decoder.fcl -n 50 ' + ToProcess[0])
os.system('rm Supplemental*.root')
# We want to double check that the decoder job was successful. If we find a decoded file,
# then return the full path. Else tell the user that no file has been found and exit.
MatchingFiles = glob.glob('*' + Run + '_1_*-decode.root')
if len(MatchingFiles) > 0:
logging.debug('[ Decode() ]: Successfully returning file: ' + MatchingFiles[0])
return MatchingFiles[0]
else:
logging.debug('[ Decode() ]: No matching file found. Returning None.')
print('No matching files.')
sys.exit(0)
def mu2eLocateList(list):
'''
a utility function to find pnfs locations of a list of files
if a file doesn't have a location, don't return an entry for it
'''
samweb = samweb_client.SAMWebClient()
ret = []
for fn in list:
# locList is a list of locations
locList = samweb.locateFile(fn)
# find location in enstore - if more than one, take first
for loc in locList:
if loc['system'] == "enstore":
lstr = loc['location'].split(':')[-1].split('(')[0]
ret.append(lstr + "/" + fn)
break
return ret
def getTimeStampFromSAM(fname):
#pull json from adjacent fname+".json" file
jfname = fname + ".json"
with open(jfname) as data_file:
data = json.load(data_file)
#print data
parentfname = data['parents'][0]['file_name']
print "Found parent filename:", parentfname
print "Querying SAM metadata..."
samweb = samweb_client.SAMWebClient(experiment='uboone')
mdata = samweb.getMetadata(parentfname)
print "Found parent start,end:", mdata['start_time'], ",", mdata[
'end_time']
dstart = datetime.datetime.strptime(mdata['start_time'],
"%Y-%m-%dT%H:%M:%S+00:00")
dend = datetime.datetime.strptime(mdata['end_time'],
"%Y-%m-%dT%H:%M:%S+00:00")
run = data['runs'][0][0]
start = time.mktime(dstart.timetuple())
end = time.mktime(dend.timetuple())
return start, end, run
def run(self, options, args):
if not (len(args) == 1):
raise CmdError("wrong number of arguments")
aa = args[0]
nd = len(aa.split("."))
if (nd != 5):
raise CmdError("on-tape expects a dataset name as input")
self.samweb = samweb_client.SAMWebClient()
dims = "dh.dataset=" + args[0] + " with availability anylocation"
# get the full list of files
summary = self.samweb.listFilesSummary(dims)
dims = "dh.dataset=" + args[0] + " and tape_label %"
# now the ones on tape
summary_tape = self.samweb.listFilesSummary(dims)
nt = summary["file_count"]
no = summary_tape["file_count"]
print("{0:6d} total {1:6d} on tape {2:6d} not on tape".format(
nt, no, nt - no))
return 0
import samweb_client
import datetime
from slowmondb import *
from FixedOffset import *
samweb = samweb_client.SAMWebClient(experiment='uboone')
files = samweb.listFiles("run_number 3702")
fsmcdb = smcdb()
for f in files:
print f
#get sam meta data for this file
mdata = samweb.getMetadata(f)
#dates from sam are formatted like 2015-11-12T12:00:06+00:00
#put start and end into datetimes
start = parseISOTime(mdata['start_time'])
end = parseISOTime(mdata['end_time'])
print "querying for %s to %s" % (start, end)
#print fsmcdb.ChannelStatsInTimeRange('uB_ArPurity_PM01_0/LIFETIME',start,end)
print fsmcdb.GetClosestChannelValue('uB_TPCDrift_HV01_1_0/voltage', start)
#print fsmcdb.GetClosestChannelValue('uB_ArPurity_PM01_0/LIFETIME',end)
#print fsmcdb.GetClosestChannelValue('uB_TPCDrift_HV01_1_0/voltage',datetime.datetime.now())
exit()
import samweb_client as sw
import os
samweb = sw.SAMWebClient(experiment='uboone')
x = 6120
while x < 6340:
arg1 = "run_number = %i and (ub_project.stage = mergeext_unbiased or ub_project.stage = mergebnb_unbiased or ub_project.stage = nubnb_unbiased)" %(x)
rawData = samweb.listFilesSummary(arg1)
num_events = rawData['total_event_count']
if type(num_events) != int:
sys_out2 = "sed -i -e 's/runnum_%i/runnum_%i/g' SingleRunGain.xml"%(x,x+1)
os.system(sys_out2)
x+=1
continue
sys_out="samweb create-definition runnum_%i 'run_number = %i and (ub_project.stage = mergeext_unbiased or ub_project.stage = mergebnb_unbiased or ub_project.stage = nubnb_unbiased)'" %(x,x)
os.system(sys_out)
os.system("project.py --xml /uboone/app/users/moon/OpticalStudies/v05_01_01/workdir/GainsDatabase/SingleRunGain.xml --stage RunSingleGain --clean")
os.system("project.py --xml /uboone/app/users/moon/OpticalStudies/v05_01_01/workdir/GainsDatabase/SingleRunGain.xml --stage RunSingleGain --submit")
sys_out2 = "sed -i -e 's/runnum_%i/runnum_%i/g' SingleRunGain.xml"%(x,x+1)
os.system(sys_out2)
x+=1
def setUp(self):
self.samweb = samweb_client.SAMWebClient(experiment=self.experiment)
backupCount=10,
)
#handler = logging.StreamHandler()
handler.setFormatter(formatter)
handler.setLevel(logging.DEBUG)
logger = logging.getLogger(logger_name)
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
#///////////////////////////////////////////////////////////////////////
# configure samweb
#///////////////////////////////////////////////////////////////////////
samweb = samweb_client.SAMWebClient(experiment='lariat')
def raw_file_path_from_sam(file_name):
"""
Returns file path of raw data art ROOT file from SAM given a
file name. An exception is raised of the file does not exist.
"""
file_path_dict = samweb.locateFile(file_name)
system = file_path_dict[0]['system']
file_path = \
file_path_dict[0]['full_path'].split(system + ':')[1] \
+ '/' + file_name
return file_path
def __init__(self):
self.samweb = samweb_client.SAMWebClient()
def run(self, args ):
'''
execute the toPnfs command, to make a list of filespecs
args = list of the args to the command
'''
#print(len(args),args)
#options, remainder = getopt.getopt(sys.argv[1:], 'h',
#['help'])
if( len(args) != 1 ):
print >> sys.stderr, "ERROR: to-pnfs expects one argument as input"
return 1
aa = args[0]
# number of all types of fields
nf = len(aa.replace(":"," ").replace("="," ").split())
# number of fields delimited by dots
nd = len(aa.split("."))
#print(na," ",aa)
self.samweb = samweb_client.SAMWebClient()
if nf==1 and nd == 6 :
# a single file, run samweb locate-file
ret = self.locateOneLocalFile(aa, True)
if ret[0] == 0:
print(ret[1]+"/"+aa)
else :
if nf==1 and nd == 5:
# just a dataset name
dims = "dh.dataset="+aa
elif nf==1 and nd<5:
# probably a dataset def
dims = "defname : "+aa
else: # dataset def or more complicated samweb selection
dims = aa
# get the full list of files
list = self.samweb.listFiles(dims, fileinfo=True, stream=False)
# locate the first only
first = list[0]
ret = self.locateOneLocalFile(first[0], True)
if ret[0] != 0 :
return ret[0]
# now loop over file and print calculated file
base = "/".join(ret[1].split('/')[0:-2])
test = ret[1].split('/')[3]
oldStyle = True
if test in ['tape','disk','persistent','scratch']:
oldStyle = False
for line in list:
name = line[0]
if oldStyle:
x = int(line[1])
x1 = x/1000000
x2 = x/1000
spr = "{0:03d}/{1:03d}".format(x1,x2)
else:
hasher = hashlib.sha256()
hasher.update(name)
hh = hasher.hexdigest()
spr = hh[0:2]+'/'+hh[2:4]
print(base+'/'+spr+'/'+name)
return 0
def run(self, args ):
'''
execute the on-tape command, to count files on tape
args = list of the args to the command
'''
#print(len(args),args)
options, remainder = getopt.getopt(args, 'n:d:',
["limit=","directory="])
if( len(remainder) != 1 ):
print >> sys.stderr, "ERROR: get-files expects one argument as input"
return 1
nlimit = 100
diry = "."
for opt in options:
if opt[0]=="-n" or opt[0]=="--limit":
nlimit = int(opt[1])
if opt[0]=="-d" or opt[0]=="--directory":
diry = opt[1]
if nlimit>100:
nlimit = 100
# use utility function to interpret the filespec
# this returns sam names
list = mu2eFiles(remainder[0],False,nlimit)
#print(list)
#return 0
# now locate them
cmdlist = []
samweb = samweb_client.SAMWebClient()
for fn in list:
loca = samweb.locateFile(fn)
if len(loca) == 0 :
print >> sys.stderr, "ERROR: {0:s} has no location".format(fn)
else:
loc = loca[0]
lstr = loc['location'].split(':')[-1].split('(')[0]
cmdlist.append(lstr+"/"+fn+" "+diry+"/"+fn+"\n")
# write out the commands to a temp file
ret = ShellCommand("mktemp", 1)
if ret[0] or ret[1]!=0:
print >> sys.stderr, "ERROR: could not make a temp file"
return 1
#print(ret[2])
# mktemp comes back with a teminal \n
tempFileName = ret[2].replace("\n","")
tempFile = open(tempFileName, "w")
for line in cmdlist:
tempFile.write(line)
tempFile.close()
#print("cat ",tempFile.name)
#return 1
# execute a bulk ifdh transfer
cmd = "source /cvmfs/fermilab.opensciencegrid.org/products/common/etc/setups"
cmd = cmd+ " ; "
cmd = cmd+ "[ -n \"$SETUP_PYTHON\" ] && unsetup python"
cmd = cmd+ " ; "
cmd = cmd+ "setup ifdhc"
cmd = cmd+ " ; "
cmd = cmd+ "ifdh cp -f "+tempFileName
ret = ShellCommand(cmd, 1)
#print(cmd)
#print(ret[2])
if ret[0] or ret[1]!=0 :
print >> sys.stderr, "ERROR: could not run ifdh cp to move files"
return 2
# remove temp file
cmd = "rm -f "+tempFileName
ret = ShellCommand(cmd, 1)
return 0