def getfiles(self, refresh=False, url=True, verb=0):
"""Get list of files from DAS."""
files = self.files
if self.refreshable and (not files or refresh):
files = []
for path in self.paths:
if self.storage: # get files from storage system
sepath = repkey(self.storage, PATH=path).replace('//', '/')
storage = getstorage(sepath, verb=verb - 1)
outlist = storage.getfiles(url=url, verb=verb - 1)
else: # get files from DAS
dascmd = 'dasgoclient --query="file dataset=%s instance=%s"' % (
path, self.instance) #--limit=0
LOG.verb(repr(dascmd), verb)
cmdout = execute(dascmd, verb=verb - 1)
outlist = cmdout.split(os.linesep)
for line in outlist: # filter root files
line = line.strip()
if line.endswith('.root') and not any(
f.endswith(line) for f in self.blacklist):
if url and self.url not in line and 'root://' not in line:
line = self.url + line
files.append(line)
files.sort() # for consistent list order
self.files = files
return files
print ">>> %-12s = %r"%('outdir',outdir)
print ">>> %-12s = %r"%('copydir',copydir)
print ">>> %-12s = %r"%('postfix',postfix)
print ">>> %-12s = %s"%('infiles',infiles)
print ">>> %-12s = %r"%('branchsel',branchsel)
print ">>> %-12s = %r"%('json',json)
print ">>> %-12s = %s"%('modules',modules)
print ">>> %-12s = %s"%('prefetch',prefetch)
print ">>> %-12s = %s"%('doJEC',doJEC)
print ">>> %-12s = %s"%('doJECSys',doJECSys)
print ">>> %-12s = %s"%('cwd',os.getcwd())
print '-'*80
# RUN
p = PostProcessor(outdir,infiles,cut=None,branchsel=None,outputbranchsel=branchsel,noOut=False,
modules=modules,jsonInput=json,postfix=postfix,maxEntries=maxevts,prefetch=prefetch)
p.run()
# COPY
if copydir and outdir!=copydir:
from TauFW.PicoProducer.storage.utils import getstorage
from TauFW.PicoProducer.tools.file import rmfile
store = getstorage(copydir,verb=2)
outfiles = os.path.join(outdir,"*%s.root"%postfix)
store.cp(outfiles)
print ">>> Removing %s..."%(outfiles)
rmfile(outfiles)
# DONE
print ">>> Done after %.1f seconds"%(time.time()-time0)
def __init__(self, group, name, *paths, **kwargs):
"""Container class for CMSSW samples, e.g.:
- group: DY (used to group similar samples in final output)
- name: DYJetsToLL_M-50 (used as shorthand and jobname)
- path: /DYJetsToLL_M-50_TuneCP5_13TeV-madgraphMLM-pythia8/RunIIAutumn18NanoAODv6_Nano25Oct2019_102X_mcRun2/NANOAODSIM
- dtype: 'mc', 'data', 'embed'
"""
# PATH
LOG.insist(
len(paths) >= 1, "Need at least one path to create a sample...")
if len(paths) == 1 and isinstance(paths[0], list):
paths = paths[0]
for path in paths:
LOG.insist(
path.count('/') >= 3 and path.startswith('/'),
"DAS path %r has wrong format. Need /SAMPLE/CAMPAIGN/FORMAT." %
(path))
#sample = '/'.join(line.split('/')[-3:])
# DATA TYPE
dtype = kwargs.get('dtype', None)
dtypes = ['mc', 'data', 'embed']
if dtype == None: # automatic recognition
path = paths[0]
if 'Embed' in path:
dtype = 'embed'
elif path.endswith('SIM') or any(g in path
for g in ['pythia', 'madgraph']):
dtype = 'mc'
elif re.search(r"/Run20\d\d", path):
dtype = 'data'
dtype = 'mc' # TODO: remove
LOG.insist(
dtype in dtypes,
"Given data type '%s' is not recongized! Please choose from %s..."
% (dtype, ', '.join(dtypes)))
# ATTRIBUTES
self.group = group
self.name = name
self.paths = paths # DAS dataset path
self.dtype = dtype
self.channels = kwargs.get('channel', None)
self.channels = kwargs.get('channels', self.channels)
self.storage = None
self.storepath = kwargs.get('store',
None) # if stored elsewhere than DAS
self.url = kwargs.get('url', None) # URL if stored elsewhere
self.dasurl = kwargs.get(
'dasurl', None) or "root://cms-xrd-global.cern.ch/" # URL for DAS
self.blacklist = kwargs.get('blacklist', []) # black list file
self.instance = kwargs.get(
'instance', 'prod/phys03' if path.endswith('USER') else
'prod/global') # if None, does not exist in DAS
self.nfilesperjob = kwargs.get('nfilesperjob',
-1) # number of nanoAOD files per job
self.maxevts = kwargs.get(
'maxevtsperjob', -1) # maximum number of events processed per job
self.maxevts = kwargs.get(
'maxevts',
self.maxevts) # maximum number of events processed per job
self.extraopts = kwargs.get(
'opts', []
) # extra options for analysis module, e.g. ['doZpt=1','tes=1.1']
self.subtry = kwargs.get('subtry',
0) # to help keep track of resubmission
self.jobcfg = kwargs.get('jobcfg',
{}) # to help keep track of resubmission
self.nevents = kwargs.get(
'nevts', 0) # number of nanoAOD events that can be processed
self.nevents = kwargs.get('nevents',
self.nevents) # cache of number of events
self.files = kwargs.get(
'files', []) # list of ROOT files, OR text file with list of files
self.filenevts = {} # cache of number of events for each file
self.postfix = kwargs.get(
'postfix',
None) or "" # post-fix (before '.root') for stored ROOT files
self.era = kwargs.get('era', "") # for expansion of $ERA variable
self.dosplit = kwargs.get(
'split',
len(self.paths) >= 2) # allow splitting (if multiple DAS datasets)
self.verbosity = kwargs.get('verbosity',
0) # verbosity level for debugging
self.refreshable = not self.files # allow refresh on file list in getfiles()
# ENSURE LIST
if self.channels != None and not isinstance(self.channels, list):
self.channels = [self.channels]
if isinstance(self.extraopts, str):
if ',' in self.extraopts:
self.extraopts = self.extraopts.split(',')
self.extraopts = [self.extraopts]
# STORAGE & URL DEFAULTS
if self.storepath:
self.storepath = repkey(self.storepath,
USER=_user,
ERA=self.era,
GROUP=self.group,
SAMPLE=self.name)
self.storage = getstorage(repkey(self.storepath,
PATH=self.paths[0],
DAS=self.paths[0]),
ensure=False)
if not self.dasurl:
self.dasurl = self.url if (self.url in dasurls) else dasurls[0]
if not self.url:
if self.storepath:
if self.storage.__class__.__name__ == 'Local':
self.url = "" #root://cms-xrd-global.cern.ch/
else:
self.url = self.storage.fileurl
else:
self.url = self.dasurl
# GET FILE LIST FROM TEXT FILE
if isinstance(self.files, str):
self.loadfiles(self.files)
def __init__(self, group, name, *paths, **kwargs):
"""Container class for CMSSW samples, e.g.:
- group: DY (used to group similar samples in final output)
- name: DYJetsToLL_M-50 (used as shorthand and jobname)
- path: /DYJetsToLL_M-50_TuneCP5_13TeV-madgraphMLM-pythia8/RunIIAutumn18NanoAODv6_Nano25Oct2019_102X_mcRun2/NANOAODSIM
- dtype: 'mc', 'data', 'embed'
"""
# PATH
assert len(paths) >= 1, "Need at least one path to create a sample..."
if len(paths) == 1 and isinstance(paths[0], list):
paths = paths[0]
for path in paths:
assert path.count('/') >= 3 and path.startswith(
'/'), "Path %s has wrong format. Need /SAMPLE/CAMPAIGN/FORMAT."
#sample = '/'.join(line.split('/')[-3:])
# DATA TYPE
dtype = kwargs.get('dtype', None)
dtypes = ['mc', 'data', 'embed']
if dtype == None: # automatic recognition
path = paths[0]
if 'Embed' in path:
dtype = 'embed'
elif path.endswith('SIM') or any(g in path
for g in ['pythia', 'madgraph']):
dtype = 'mc'
elif re.search(r"/Run20\d\d", path):
dtype = 'data'
assert dtype in dtypes, "Given data type '%s' is not recongized! Please choose from %s..." % (
dtype, ', '.join(dtypes))
# ATTRIBUTES
self.group = group
self.name = name
self.paths = paths # DAS path
self.dtype = dtype
self.channels = kwargs.get('channels', None)
self.storage = kwargs.get('store',
None) # if stored elsewhere than DAS
self.url = kwargs.get('url', None)
self.blacklist = kwargs.get('blacklist', []) # black list file
self.instance = kwargs.get(
'instance',
'prod/phys03' if path.endswith('USER') else 'prod/global')
self.nfilesperjob = kwargs.get('nfilesperjob', -1)
self.subtry = kwargs.get('subtry',
0) # to help keep track of resubmission
self.jobcfg = kwargs.get('jobcfg',
{}) # to help keep track of resubmission
self.nevents = kwargs.get('nevents', 0)
self.files = kwargs.get(
'files', []) # list of ROOT files, OR text file with list of files
self.era = kwargs.get('era', "") # for expansion of $ERA variable
self.verbosity = kwargs.get('verbosity',
0) # verbosity level for debugging
self.refreshable = not self.files # allow refresh on file list in getfiles()
# STORAGE & URL DEFAULTS
if self.storage:
self.storage = repkey(self.storage,
ERA=self.era,
GROUP=self.group,
SAMPLE=self.name)
if not self.url:
if self.storage:
from TauFW.PicoProducer.storage.StorageSystem import Local
storage = getstorage(repkey(self.storage, PATH=self.paths[0]))
if isinstance(storage, Local):
self.url = "root://cms-xrd-global.cern.ch/"
else:
self.url = storage.fileurl
else:
self.url = "root://cms-xrd-global.cern.ch/"
# GET FILE LIST FROM TEXT FILE
if isinstance(self.files, str):
filename = repkey(self.files,
ERA=self.era,
GROUP=self.group,
SAMPLE=self.name)
if self.verbosity >= 1:
print ">>> Loading sample files from '%r'" % (filename)
if self.verbosity >= 2:
print ">>> %-14s = %s" % ('filelist', self.files)
print ">>> %-14s = %s" % ('filename', filename)
filelist = []
with open(filename, 'r') as file:
for line in file:
line = line.strip().split()
if not line: continue
infile = line[0].strip()
if infile[0] == '#': continue
if infile.endswith('.root'):
filelist.append(infile)
self.files = filelist
self.files.sort()
def main_status(args):
"""Check status of jobs (succesful/pending/failed/missing), or hadd job output."""
if args.verbosity>=1:
print ">>> main_status", args
# SETTING
eras = args.eras
channels = args.channels
tag = args.tag
checkdas = args.checkdas
checkqueue = args.checkqueue
dtypes = args.dtypes
filters = args.samples
vetoes = args.vetoes
force = args.force
hadd = args.subcommand=='hadd'
cleanup = args.cleanup if hadd else False
dryrun = args.dryrun
verbosity = args.verbosity
cmdverb = max(1,verbosity)
outdirformat = CONFIG.outdir
jobdirformat = CONFIG.jobdir
storedirformat = CONFIG.picodir
jobs = [ ]
# LOOP over ERAS
for era in eras:
# LOOP over CHANNELS
for channel in channels:
print header("%s, %s"%(era,channel))
# GET SAMPLES
jobcfgs = repkey(os.path.join(jobdirformat,"config/jobconfig_$CHANNEL$TAG_try[0-9]*.json"),
ERA=era,SAMPLE='*',GROUP='*',CHANNEL=channel,TAG=tag)
if verbosity>=1:
print ">>> %-12s = %s"%('cwd',os.getcwd())
print ">>> %-12s = %s"%('jobcfgs',jobcfgs)
print ">>> %-12s = %s"%('filters',filters)
print ">>> %-12s = %s"%('vetoes',vetoes)
print ">>> %-12s = %s"%('dtypes',dtypes)
samples = getcfgsamples(jobcfgs,filter=filters,veto=vetoes,dtype=dtypes,verb=verbosity)
if verbosity>=2:
print ">>> Found samples: "+", ".join(repr(s.name) for s in samples)
if hadd and 'skim' in channel:
LOG.warning("Hadding into one file not available for skimming...")
print
continue
# SAMPLE over SAMPLES
found = False
for sample in samples:
if sample.channels and channel not in sample.channels: continue
found = True
print ">>> %s"%(bold(sample.name))
for path in sample.paths:
print ">>> %s"%(bold(path))
# CHECK JOBS ONLY ONCE
if checkqueue==1 and not jobs:
batch = getbatch(CONFIG,verb=verbosity)
jobs = batch.jobs(verb=verbosity-1)
# HADD
if hadd:
jobdir = sample.jobcfg['jobdir']
outdir = sample.jobcfg['outdir']
storedir = repkey(storedirformat,ERA=era,CHANNEL=channel,TAG=tag,SAMPLE=sample.name,
DAS=sample.paths[0].strip('/'),GROUP=sample.group)
storage = getstorage(storedir,ensure=True,verb=verbosity)
outfile = '%s_%s%s.root'%(sample.name,channel,tag)
infiles = os.path.join(outdir,'*_%s%s_[0-9]*.root'%(channel,tag))
cfgfiles = os.path.join(sample.jobcfg['cfgdir'],'job*_%s%s_try[0-9]*.*'%(channel,tag))
logfiles = os.path.join(sample.jobcfg['logdir'],'*_%s%s_try[0-9]*.*.*.log'%(channel,tag))
if verbosity>=1:
print ">>> Hadd'ing job output for '%s'"%(sample.name)
print ">>> %-12s = %r"%('jobdir',jobdir)
print ">>> %-12s = %r"%('outdir',outdir)
print ">>> %-12s = %r"%('storedir',storedir)
print ">>> %-12s = %s"%('infiles',infiles)
print ">>> %-12s = %r"%('outfile',outfile)
resubfiles, chunkdict = checkchuncks(sample,channel=channel,tag=tag,jobs=jobs,
checkqueue=checkqueue,das=checkdas,verb=verbosity)
if len(resubfiles)>0 and not force:
LOG.warning("Cannot hadd job output because %d chunks need to be resubmitted..."%(len(resubfiles))+
"Please use -f or --force to hadd anyway.")
continue
#haddcmd = 'hadd -f %s %s'%(outfile,infiles)
#haddout = execute(haddcmd,dry=dryrun,verb=max(1,verbosity))
haddout = storage.hadd(infiles,outfile,dry=dryrun,verb=cmdverb)
#os.system(haddcmd)
# CLEAN UP
# TODO: check if hadd was succesful with isvalid
if cleanup:
rmfiles = ""
rmfileset = [infiles,cfgfiles,logfiles]
for files in rmfileset:
if len(glob.glob(files))>0:
rmfiles += ' '+files
if verbosity>=2:
print ">>> %-12s = %s"%('rmfileset',rmfileset)
print ">>> %-12s = %s"%('rmfiles',rmfiles)
if rmfiles:
rmcmd = "rm %s"%(rmfiles)
rmout = execute(rmcmd,dry=dryrun,verb=cmdverb)
# ONLY CHECK STATUS
else:
outdir = sample.jobcfg['outdir']
if verbosity>=1:
print ">>> Checking job status for '%s'"%(sample.name)
print ">>> %-12s = %r"%('outdir',outdir)
checkchuncks(sample,channel=channel,tag=tag,jobs=jobs,
checkqueue=checkqueue,das=checkdas,verb=verbosity)
print
if not found:
print ">>> Did not find any samples."
print
def checkchuncks(sample,**kwargs):
"""Help function to check jobs status: success, pending, failed or missing.
Return list of files to be resubmitted, and a dictionary between chunk index and input files."""
outdir = kwargs.get('outdir', None)
channel = kwargs.get('channel', None)
tag = kwargs.get('tag', None)
checkqueue = kwargs.get('checkqueue', False)
pendjobs = kwargs.get('jobs', [ ])
checkdas = kwargs.get('das', True)
verbosity = kwargs.get('verb', 0)
oldjobcfg = sample.jobcfg
oldcfgname = oldjobcfg['config']
chunkdict = oldjobcfg['chunkdict'] # filenames
jobids = oldjobcfg['jobids']
joblist = oldjobcfg['joblist']
postfix = oldjobcfg['postfix']
nfilesperjob = oldjobcfg['nfilesperjob']
if outdir==None:
outdir = oldjobcfg['outdir']
storage = getstorage(outdir,ensure=True)
if channel==None:
channel = oldjobcfg['channel']
if tag==None:
tag = oldjobcfg['tag']
noldchunks = len(chunkdict) # = number of jobs
goodchunks = [ ] # good job output
pendchunks = [ ] # pending or running jobs
badchunks = [ ] # corrupted job output
misschunks = [ ] # missing job output
resubfiles = [ ] # files to resubmit (if bad or missing)
# NUMBER OF EVENTS
nprocevents = 0 # total number of processed events
ndasevents = oldjobcfg['nevents'] # total number of available events
if checkdas and oldjobcfg['nevents']==0:
ndasevents = sample.getnevents()
oldjobcfg['nevents'] = ndasevents
if verbosity>=2:
print ">>> %-12s = %s"%('ndasevents',ndasevents)
if verbosity>=3:
print ">>> %-12s = %s"%('chunkdict',chunkdict)
# CHECK PENDING JOBS
if checkqueue<0 or pendjobs:
batch = getbatch(CONFIG,verb=verbosity)
if checkqueue!=1 or not pendjobs:
pendjobs = batch.jobs(jobids,verb=verbosity-1) # get refreshed job list
else:
pendjobs = [j for j in pendjobs if j.jobid in jobids] # get new job list with right job id
###########################################################################
# CHECK SKIMMED OUTPUT: nanoAOD format, one or more output files per job
if 'skim' in channel: # and nfilesperjob>1:
flagexp = re.compile(r"-i (.+\.root)") #r"-i ((?:(?<! -).)+\.root[, ])"
fpattern = "*%s.root"%(postfix)
chunkexp = re.compile(r".+%s\.root"%(postfix))
if verbosity>=2:
print ">>> %-12s = %r"%('flagexp',flagexp.pattern)
print ">>> %-12s = %r"%('fpattern',fpattern)
print ">>> %-12s = %r"%('chunkexp',chunkexp.pattern)
print ">>> %-12s = %s"%('checkqueue',checkqueue)
print ">>> %-12s = %s"%('pendjobs',pendjobs)
print ">>> %-12s = %s"%('jobids',jobids)
# CHECK PENDING JOBS
pendfiles = [ ]
for job in pendjobs:
if verbosity>=3:
print ">>> Found job %r, status=%r, args=%r"%(job,job.getstatus(),job.args.rstrip())
if job.getstatus() in ['q','r']:
if CONFIG.batch=='HTCondor':
jobarg = str(job.args)
matches = flagexp.findall(jobarg)
else:
jobarg = getline(joblist,job.taskid-1)
matches = flagexp.findall(jobarg)
if verbosity>=3:
print ">>> matches = ",matches
if not matches:
continue
infiles = [ ]
for file in matches[0].split():
if not file.endswith('.root'):
break
infiles.append(file)
LOG.insist(infiles,"Did not find any root files in %r, matches=%r"%(jobarg,matches))
ichunk = -1
for i in chunkdict:
if all(f in chunkdict[i] for f in infiles):
ichunk = i
break
LOG.insist(ichunk>=0,
"Did not find to which the input files of jobids %s belong! "%(jobids)+
"\nichunk=%s,\ninfiles=%s,\nchunkdict=%s"%(ichunk,infiles,chunkdict))
LOG.insist(len(chunkdict[i])==len(infiles),
"Mismatch between input files of jobids %s and chunkdict! "%(jobids)+
"\nichunk=%s,\ninfiles=%s,\nchunkdict[%s]=%s"%(ichunk,infiles,ichunk,chunkdict[ichunk]))
pendchunks.append(ichunk)
# CHECK OUTPUT FILES
badfiles = [ ]
goodfiles = [ ]
fnames = storage.getfiles(filter=fpattern,verb=verbosity-1)
if verbosity>=2:
print ">>> %-12s = %s"%('pendchunks',pendchunks)
print ">>> %-12s = %s"%('fnames',fnames)
for fname in fnames:
if verbosity>=2:
print ">>> Checking job output '%s'..."%(fname)
infile = os.path.basename(fname.replace(postfix+".root",".root")) # reconstruct input file
nevents = isvalid(fname) # check for corruption
ichunk = -1
fmatch = None
for i in chunkdict:
if fmatch:
break
for chunkfile in chunkdict[i]:
if infile in chunkfile: # find chunk input file belongs to
ichunk = i
fmatch = chunkfile
break
if ichunk<0:
if verbosity>=2:
print ">>> => No match..."
#LOG.warning("Did not recognize output file '%s'!"%(fname))
continue
if ichunk in pendchunks:
if verbosity>=2:
print ">>> => Pending..."
continue
if nevents<0:
if verbosity>=2:
print ">>> => Bad nevents=%s..."%(nevents)
badfiles.append(fmatch)
else:
if verbosity>=2:
print ">>> => Good, nevents=%s"%(nevents)
nprocevents += nevents
goodfiles.append(fmatch)
# GET FILES for RESUBMISSION + sanity checks
for ichunk in chunkdict.keys():
if ichunk in pendchunks:
continue
chunkfiles = chunkdict[ichunk]
if all(f in goodfiles for f in chunkfiles): # all files succesful
goodchunks.append(ichunk)
continue
bad = False # count each chunk only once: bad, else missing
for fname in chunkfiles:
LOG.insist(fname not in resubfiles,"Found file for chunk '%d' more than once: %s "%(ichunk,fname)+
"Possible overcounting or conflicting job output file format!")
if fname in badfiles:
bad = True
resubfiles.append(fname)
elif fname not in goodfiles:
resubfiles.append(fname)
if bad:
badchunks.append(ichunk)
else:
misschunks.append(ichunk)
chunkdict.pop(ichunk)
###########################################################################
# CHECK ANALYSIS OUTPUT: custom tree format, one output file per job
else:
flagexp = re.compile(r"-t \w*(\d+)")
fpattern = "*%s_[0-9]*.root"%(postfix)
chunkexp = re.compile(r".+%s_(\d+)\.root"%(postfix))
if verbosity>=2:
print ">>> %-12s = %r"%('flagexp',flagexp.pattern)
print ">>> %-12s = %r"%('fpattern',fpattern)
print ">>> %-12s = %r"%('chunkexp',chunkexp.pattern)
print ">>> %-12s = %s"%('checkqueue',checkqueue)
print ">>> %-12s = %s"%('pendjobs',pendjobs)
print ">>> %-12s = %s"%('jobids',jobids)
# CHECK PENDING JOBS
for job in pendjobs:
if verbosity>=3:
print ">>> Found job %r, status=%r, args=%r"%(job,job.getstatus(),job.args.rstrip())
if job.getstatus() in ['q','r']:
if CONFIG.batch=='HTCondor':
jobarg = str(job.args)
matches = flagexp.findall(jobarg)
else:
jobarg = getline(joblist,job.taskid-1)
matches = flagexp.findall(jobarg)
if verbosity>=3:
print ">>> matches = ",matches
if not matches:
continue
ichunk = int(matches[0])
LOG.insist(ichunk in chunkdict,"Found an impossible chunk %d for job %s.%s! "%(ichunk,job.jobid,job.taskid)+
"Possible overcounting!")
pendchunks.append(ichunk)
# CHECK OUTPUT FILES
fnames = storage.getfiles(filter=fpattern,verb=verbosity-1)
if verbosity>=2:
print ">>> %-12s = %s"%('pendchunks',pendchunks)
print ">>> %-12s = %s"%('fnames',fnames)
for fname in fnames:
if verbosity>=2:
print ">>> Checking job output '%s'..."%(fname)
match = chunkexp.search(fname)
if match:
ichunk = int(match.group(1))
LOG.insist(ichunk in chunkdict,"Found an impossible chunk %d for file %s!"%(ichunk,fname)+
"Possible overcounting or conflicting job output file format!")
if ichunk in pendchunks:
continue
else:
#LOG.warning("Did not recognize output file '%s'!"%(fname))
continue
nevents = isvalid(fname) # check for corruption
if nevents<0:
if verbosity>=2:
print ">>> => Bad, nevents=%s"%(nevents)
badchunks.append(ichunk)
# TODO: remove file from outdir?
else:
if verbosity>=2:
print ">>> => Good, nevents=%s"%(nevents)
nprocevents += nevents
goodchunks.append(ichunk)
# GET FILES for RESUBMISSION + sanity checks
if verbosity>=2:
print ">>> %-12s = %s"%('nprocevents',nprocevents)
for ichunk in chunkdict.keys():
count = goodchunks.count(ichunk)+pendchunks.count(ichunk)+badchunks.count(ichunk)
LOG.insist(count in [0,1],"Found %d times chunk '%d' (good=%d, pending=%d, bad=%d). "%(
count,ichunk,goodchunks.count(ichunk),pendchunks.count(ichunk),badchunks.count(ichunk))+
"Possible overcounting or conflicting job output file format!")
if count==0: # missing chunk
misschunks.append(ichunk)
elif ichunk not in badchunks: # good or pending chunk
continue
fchunk = chunkdict[ichunk]
for fname in fchunk:
LOG.insist(fname not in resubfiles,"Found file for chunk '%d' more than once: %s "%(ichunk,fname)+
"Possible overcounting or conflicting job output file format!")
resubfiles.extend(chunkdict[ichunk])
chunkdict.pop(ichunk) # only save good chunks
###########################################################################
goodchunks.sort()
pendchunks.sort()
badchunks.sort()
misschunks.sort()
# PRINT
def printchunks(jobden,label,text,col,show=False):
if jobden:
ratio = color("%4d/%d"%(len(jobden),noldchunks),col,bold=False)
label = color(label,col,bold=True)
jlist = (": "+', '.join(str(j) for j in jobden)) if show else ""
print ">>> %s %s - %s%s"%(ratio,label,text,jlist)
#else:
# print ">>> %2d/%d %s - %s"%(len(jobden),len(jobs),label,text)
rtext = ""
if ndasevents>0:
ratio = 100.0*nprocevents/ndasevents
rcol = 'green' if ratio>90. else 'yellow' if ratio>80. else 'red'
rtext = ": "+color("%d/%d (%d%%)"%(nprocevents,ndasevents,ratio),rcol,bold=True)
printchunks(goodchunks,'SUCCES', "Chunks with output in outdir"+rtext,'green')
printchunks(pendchunks,'PEND',"Chunks with pending or running jobs",'white',True)
printchunks(badchunks, 'FAIL', "Chunks with corrupted output in outdir",'red',True)
printchunks(misschunks,'MISS',"Chunks with no output in outdir",'red',True)
return resubfiles, chunkdict
def preparejobs(args):
"""Help function to iterate over samples per given channel and era and prepare job config and list."""
if args.verbosity>=1:
print ">>> preparejobs", args
resubmit = args.subcommand=='resubmit'
eras = args.eras
channels = args.channels
tag = args.tag
dtypes = args.dtypes
filters = args.samples
vetoes = args.vetoes
checkdas = args.checkdas
checkqueue = args.checkqueue
prefetch = args.prefetch
nfilesperjob = args.nfilesperjob
split_nfpj = args.split_nfpj
verbosity = args.verbosity
jobs = [ ]
# LOOP over ERAS
for era in eras:
moddict = { } # save time by loading samples and get their file list only once
# LOOP over CHANNELS
for channel in channels:
print header("%s, %s"%(era,channel))
# CHANNEL -> MODULE
assert channel in CONFIG.channels, "Channel '%s' not found in the configuration file. Available: %s"%(channel,CONFIG.channels)
module = CONFIG.channels[channel]
if channel!='test' and 'skim' not in channel:
ensuremodule(module)
if verbosity>=1:
print '-'*80
print ">>> %-12s = %r"%('channel',channel)
print ">>> %-12s = %r"%('module',module)
print ">>> %-12s = %s"%('filters',filters)
print ">>> %-12s = %s"%('vetoes',vetoes)
print ">>> %-12s = %r"%('dtypes',dtypes)
# PROCESSOR
if 'skim' in channel:
processor = module
elif channel=='test':
processor = module
else:
processor = "picojob.py"
procpath = os.path.join("python/processors",processor)
if not os.path.isfile(procpath):
LOG.throw(IOError,"Processor '%s' does not exist in '%s'..."%(processor,procpath))
processor = os.path.abspath(procpath)
if verbosity>=1:
print ">>> %-12s = %r"%('processor',processor)
print '-'*80
# GET SAMPLES
jobdirformat = CONFIG.jobdir # for job config & log files
outdirformat = CONFIG.nanodir if 'skim' in channel else CONFIG.outdir # for job output
if resubmit:
# TODO: allow user to resubmit given config file
jobcfgs = repkey(os.path.join(jobdirformat,"config/jobconfig_$SAMPLE$TAG_try[0-9]*.json"),
ERA=era,SAMPLE='*',CHANNEL=channel,TAG=tag)
if verbosity>=2:
print ">>> %-12s = %s"%('cwd',os.getcwd())
print ">>> %-12s = %s"%('jobcfgs',jobcfgs)
samples = getcfgsamples(jobcfgs,filter=filters,veto=vetoes,dtype=dtypes,verb=verbosity)
else:
assert era in CONFIG.eras, "Era '%s' not found in the configuration file. Available: %s"%(era,CONFIG.eras)
samples = getsamples(era,channel=channel,tag=tag,dtype=dtypes,filter=filters,veto=vetoes,moddict=moddict,verb=verbosity)
if verbosity>=2:
print ">>> Found samples: "+", ".join(repr(s.name) for s in samples)
# SAMPLE over SAMPLES
found = False
for sample in samples:
if sample.channels and channel not in sample.channels: continue
found = True
print ">>> %s"%(bold(sample.name))
for path in sample.paths:
print ">>> %s"%(bold(path))
# DIRECTORIES
subtry = sample.subtry+1 if resubmit else 1
jobids = sample.jobcfg.get('jobids',[ ])
postfix = "_%s%s"%(channel,tag)
jobtag = '_%s%s_try%d'%(channel,tag,subtry)
jobname = sample.name+jobtag.rstrip('try1').rstrip('_')
nfilesperjob_ = sample.nfilesperjob if sample.nfilesperjob>0 else nfilesperjob
if split_nfpj>1:
nfilesperjob_ = min(1,nfilesperjob_/split_nfpj)
outdir = repkey(outdirformat,ERA=era,CHANNEL=channel,TAG=tag,SAMPLE=sample.name,
DAS=sample.paths[0].strip('/'),GROUP=sample.group)
jobdir = ensuredir(repkey(jobdirformat,ERA=era,CHANNEL=channel,TAG=tag,SAMPLE=sample.name,
DAS=sample.paths[0].strip('/'),GROUP=sample.group))
cfgdir = ensuredir(jobdir,"config")
logdir = ensuredir(jobdir,"log")
cfgname = "%s/jobconfig%s.json"%(cfgdir,jobtag)
joblist = '%s/jobarglist%s.txt'%(cfgdir,jobtag)
if verbosity==1:
print ">>> %-12s = %s"%('cfgname',cfgname)
print ">>> %-12s = %s"%('joblist',joblist)
elif verbosity>=2:
print '-'*80
print ">>> Preparing job %ssubmission for '%s'"%("re" if resubmit else "",sample.name)
print ">>> %-12s = %r"%('processor',processor)
print ">>> %-12s = %r"%('jobname',jobname)
print ">>> %-12s = %r"%('jobtag',jobtag)
print ">>> %-12s = %r"%('postfix',postfix)
print ">>> %-12s = %r"%('outdir',outdir)
print ">>> %-12s = %r"%('cfgdir',cfgdir)
print ">>> %-12s = %r"%('logdir',logdir)
print ">>> %-12s = %r"%('cfgname',cfgname)
print ">>> %-12s = %r"%('joblist',joblist)
print ">>> %-12s = %s"%('try',subtry)
print ">>> %-12s = %r"%('jobids',jobids)
# CHECKS
if os.path.isfile(cfgname):
# TODO: check for running jobs
LOG.warning("Job configuration '%s' already exists and will be overwritten! "+
"Beware of conflicting job output!"%(cfgname))
if not resubmit:
cfgpattern = re.sub(r"(?<=try)\d+(?=.json$)",r"*",cfgname)
cfgnames = [f for f in glob.glob(cfgpattern) if not f.endswith("_try1.json")]
if cfgnames:
LOG.warning("Job configurations for resubmission already exists! This can cause conflicting job output!"+
"If you are sure you want to submit from scratch, please remove these files:\n>>> "+"\n>>> ".join(cfgnames))
storage = getstorage(outdir,verb=verbosity,ensure=True)
# GET FILES
nevents = 0
if resubmit: # resubmission
if checkqueue==0 and not jobs: # check jobs only once
batch = getbatch(CONFIG,verb=verbosity)
jobs = batch.jobs(verb=verbosity-1)
infiles, chunkdict = checkchuncks(sample,outdir=outdir,channel=channel,tag=tag,jobs=jobs,
checkqueue=checkqueue,das=checkdas,verb=verbosity)
nevents = sample.jobcfg['nevents'] # updated in checkchuncks
else: # first-time submission
infiles = sample.getfiles(verb=verbosity-1)
if checkdas:
nevents = sample.getnevents()
chunkdict = { }
if args.testrun:
infiles = infiles[:2]
if verbosity==1:
print ">>> %-12s = %s"%('nfilesperjob',nfilesperjob_)
print ">>> %-12s = %s"%('nfiles',len(infiles))
elif verbosity>=2:
print ">>> %-12s = %s"%('nfilesperjob',nfilesperjob_)
print ">>> %-12s = %s"%('nfiles',len(infiles))
print ">>> %-12s = [ "%('infiles')
for file in infiles:
print ">>> %r"%file
print ">>> ]"
print ">>> %-12s = %s"%('nevents',nevents)
# CHUNKS
infiles.sort() # to have consistent order with resubmission
chunks = [ ] # chunk indices
fchunks = chunkify(infiles,nfilesperjob_) # file chunks
nfiles = len(infiles)
nchunks = len(fchunks)
if verbosity>=1:
print ">>> %-12s = %s"%('nchunks',nchunks)
if verbosity>=2:
print '-'*80
# WRITE JOB LIST with arguments per job
if args.verbosity>=1:
print ">>> Creating job list %s..."%(joblist)
with open(joblist,'w') as listfile:
ichunk = 0
for fchunk in fchunks:
while ichunk in chunkdict:
ichunk += 1 # allows for different nfilesperjob on resubmission
continue
jobfiles = ' '.join(fchunk) # list of input files
filetag = postfix
if 'skim' not in channel:
filetag += "_%d"%(ichunk)
jobcmd = processor
if 'skim' in channel:
jobcmd += " -y %s --copydir %s -t %s --jec-sys"%(era,outdir,filetag)
elif 'test' in channel:
jobcmd += " -o %s -t %s -i %s"%(outdir,filetag)
else:
jobcmd += " -y %s -c %s -M %s --copydir %s -t %s"%(era,channel,module,outdir,filetag)
if prefetch:
jobcmd += " -p"
jobcmd += " -i %s"%(jobfiles) # add last
if args.verbosity>=1:
print jobcmd
listfile.write(jobcmd+'\n')
chunkdict[ichunk] = fchunk
chunks.append(ichunk)
# JSON CONFIG
jobcfg = OrderedDict([
('time',str(datetime.now())),
('group',sample.group), ('paths',sample.paths), ('name',sample.name), ('nevents',nevents),
('channel',channel), ('module',module),
('jobname',jobname), ('jobtag',jobtag), ('tag',tag), ('postfix',postfix),
('try',subtry), ('jobids',jobids),
('outdir',outdir), ('jobdir',jobdir), ('cfgdir',cfgdir), ('logdir',logdir),
('cfgname',cfgname), ('joblist',joblist),
('nfiles',nfiles), ('files',infiles), ('nfilesperjob',nfilesperjob_), #('nchunks',nchunks),
('nchunks',nchunks), ('chunks',chunks), ('chunkdict',chunkdict),
])
# YIELD
yield jobcfg
print
#if args.testrun:
# break # only run one sample
if not found:
print ">>> Did not find any samples."
if verbosity>=1:
print ">>> %-8s = %s"%('filters',filters)
print ">>> %-8s = %s"%('vetoes',vetoes)
def testStorage(path, readonly=False, hadd=True, verb=0):
# INITIALIZE
LOG.header("__init__")
#storage = ensuremodule(system,"PicoProducer.storage"
storage = getstorage(path, ensure=True, verb=verb)
print ">>> %r" % (storage)
print ">>> %-10s = %s" % ('path', storage.path)
print ">>> %-10s = %s" % ('rmcmd', storage.rmcmd)
print ">>> %-10s = %s" % ('lscmd', storage.lscmd)
print ">>> %-10s = %s" % ('mkdrcmd', storage.mkdrcmd)
print ">>> %-10s = %s" % ('cpcmd', storage.cpcmd)
print ">>> %-10s = %s" % ('tmpdir', storage.tmpdir)
print ">>> "
# EXPAND PATH
LOG.header("expandpath")
pathargs = [
('test.py', ),
('$PATH/test.py', ),
(
'foo',
'bar',
),
]
pathkwargs = [
{
'here': True
},
{
'here': False
},
]
for patharg in pathargs:
for pathkwarg in pathkwargs:
LOG.color("storage.expandpath(%s,%s)" %
(','.join(repr(a) for a in patharg), ','.join(
"%s=%r" % (k, v) for k, v in pathkwarg.iteritems())))
result = storage.expandpath(*patharg, **pathkwarg)
print ">>> %r" % (result)
# LS
LOG.header("ls")
LOG.color("storage.ls(verb=%d)" % (verb))
contents = storage.ls(verb=verb)
print ">>> Found %d items" % (len(contents))
print ">>> Contents: %s" % (contents)
# FILES
LOG.header("getfiles")
LOG.color("storage.getfiles(verb=%d)" % (verb))
contents = storage.getfiles(verb=verb)
print ">>> Found %d items" % (len(contents))
print ">>> Contents: %s" % (contents)
print ">>> "
LOG.color("storage.getfiles(filter='*.*',verb=%d)" % (verb))
contents = storage.getfiles(filter='*.*', verb=verb)
print ">>> Found %d files" % (len(contents))
print ">>> Contents: %s" % (contents)
print ">>> "
LOG.color("storage.getfiles(filter='*.*',url=None,verb=%d)" % (verb))
contents = storage.getfiles(filter='*.*', url=None, verb=verb)
print ">>> Found %d files" % (len(contents))
print ">>> Contents: %s" % (contents)
if readonly:
print ">>> Read only. Skip test for cp, rm, mkdir, hadd..."
return
# CP
LOG.header("cp")
fname = createdummy("testStorage.txt")
LOG.color("storage.cp(%r,verb=%d)" % (fname, verb))
storage.cp(fname, verb=verb)
storage.ls(verb=verb)
# EXISTS
LOG.header("exists")
LOG.color("storage.exists(%r,verb=%d)" % (fname, verb))
result = storage.exists(fname, verb=verb)
print ">>> Exists: %r" % (result)
storage.ls(verb=verb)
# RM
LOG.header("rm")
LOG.color("storage.rm(%r,verb=%d)" % (fname, verb))
try:
storage.rm(fname, verb=verb)
except Exception as error:
print error
storage.ls(verb=verb)
# MKDIR
LOG.header("mkdir")
dirname = 'test'
LOG.color("storage.mkdir(%r.verb=%d)" % (dirname, verb))
try:
storage.mkdir(dirname, verb=verb)
storage.ls(verb=verb)
storage.ls(dirname, here=True, verb=verb)
result = storage.exists(dirname, verb=verb)
print ">>> Exists: %r" % (result)
except Exception as error:
print error
# RM DIRECTORY
LOG.header("rm directory")
submit = raw_input(">>> Careful! Do you really want to remove %r? [y/n] " %
(storage.expandpath(dirname, here=True)))
if submit == 'y':
LOG.color("storage.rm(%r,verb=%d)" % (dirname, verb))
try:
storage.rm(dirname, verb=verb)
storage.ls(verb=verb)
except Exception as error:
print error
# HADD
if hadd:
LOG.header("hadd")
infiles = [
createdummyroot("testStorage1.root"),
createdummyroot("testStorage2.root")
]
outfile = "testStorage.root"
for tmpdir in [
True,
]: #False
LOG.color("storage.hadd(%r,%r,tmpdir=%s,verb=%d)" %
(infiles, outfile, tmpdir, verb))
try:
storage.hadd(infiles, outfile, tmpdir=tmpdir, verb=verb)
storage.ls(verb=verb)
storage.rm(outfile, verb=verb)
except Exception as error:
print error
def testStorage(path,verb=0):
# INITIALIZE
LOG.header("__init__")
#storage = ensuremodule(system,"PicoProducer.storage"
storage = getstorage(path,ensure=True,verb=verb)
print ">>> %r"%(storage)
print ">>> %-10s = %s"%('path',storage.path)
print ">>> %-10s = %s"%('rmcmd',storage.rmcmd)
print ">>> %-10s = %s"%('lscmd',storage.lscmd)
print ">>> %-10s = %s"%('mkdrcmd',storage.mkdrcmd)
print ">>> %-10s = %s"%('cpcmd',storage.cpcmd)
print ">>> %-10s = %s"%('tmpdir',storage.tmpdir)
print ">>> "
# EXPAND PATH
LOG.header("expandpath")
pathargs = [
('test.py',),
('$PATH/test.py',),
('foo','bar',),
]
pathkwargs = [
{'here':True},
{'here':False},
]
for patharg in pathargs:
for pathkwarg in pathkwargs:
LOG.color("storage.expandpath(%s,%s)"%(','.join(repr(a) for a in patharg),','.join("%s=%r"%(k,v) for k,v in pathkwarg.iteritems())))
result = storage.expandpath(*patharg,**pathkwarg)
print ">>> %r"%(result)
# LS
LOG.header("ls")
LOG.color("storage.ls(verb=%d)"%(verb))
storage.ls(verb=verb)
# CP
LOG.header("cp")
fname = createdummy("testStorage.txt")
LOG.color("storage.cp(%r,verb=%d)"%(fname,verb))
storage.cp(fname,verb=verb)
storage.ls(verb=verb)
# EXISTS
LOG.header("exists")
LOG.color("storage.exists(%r,verb=%d)"%(fname,verb))
result = storage.exists(fname,verb=verb)
print ">>> %r"%(result)
storage.ls(verb=verb)
# RM
LOG.header("rm")
LOG.color("storage.rm(%r,verb=%d)"%(fname,verb))
storage.rm(fname,verb=verb)
storage.ls(verb=verb)
# MKDIR
LOG.header("mkdir")
dirname = 'test'
LOG.color("storage.mkdir(%r.verb=%d)"%(dirname,verb))
storage.mkdir(dirname,verb=verb)
storage.ls(verb=verb)
storage.ls(dirname,verb=verb)
# RM DIRECTORY
LOG.header("rm directory")
submit = raw_input(">>> Careful! Do you really want to remove %r? [y/n] "%(storage.expandpath(dirname,here=True)))
if submit=='y':
LOG.color("storage.rm(%r,verb=%d)"%(dirname,verb))
storage.rm(dirname,verb=verb)
storage.ls(verb=verb)
# HADD
LOG.header("hadd")
infiles = [createdummyroot("testStorage1.root"),createdummyroot("testStorage2.root")]
outfile = "testStorage.root"
for tmpdir in [True,]: #False
LOG.color("storage.hadd(%r,%r,tmpdir=%s,verb=%d)"%(infiles,outfile,tmpdir,verb))
storage.hadd(infiles,outfile,tmpdir=tmpdir,verb=verb)
storage.ls(verb=verb)
storage.rm(outfile,verb=verb)