def main():
channels = args.channels
years = args.years
tes = args.tes
ltf = args.ltf
jtf = args.jtf
Zmass = args.Zmass
prefetch = args.prefetch
batchscript = 'submit_SGE.sh'
tag = args.tag
if tag and tag[0]!='_': tag = '_'+tag
if tes!=1.: tag += "_TES%.3f"%(tes)
if ltf!=1.: tag += "_LTF%.3f"%(ltf)
if jtf!=1.: tag += "_JTF%.3f"%(jtf)
if Zmass: tag += "_Zmass"
tag = tag.replace('.','p')
for year in years:
# READ SAMPLES
directories = [ ]
samplelist = "samples_%s.cfg"%(year)
with open(samplelist, 'r') as file:
for line in file:
line = line.rstrip().lstrip().split(' ')[0].rstrip('/')
if line[:2].count('#')>0: continue
if line=='': continue
if line.count('/')<2: continue
sample = '/'.join(line.split('/')[-3:])
if args.samples and not matchSampleToPattern(sample,args.samples): continue
if args.vetoes and matchSampleToPattern(sample,args.vetoes): continue
if args.type=='mc' and any(s in sample[:len(s)+2] for s in ['SingleMuon','SingleElectron','Tau','EGamma']): continue
if args.type=='data' and not any(s in sample[:len(s)+2] for s in ['SingleMuon','SingleElectron','Tau','EGamma']): continue
directories.append(line)
if args.testrun:
directories = directories[:1]
#print directories
blacklist = getBlackList("filelist/blacklist.txt")
tasks = []
for channel in channels:
print header(year,channel,tag)
# SUBMIT SAMPLES
for directory in directories:
if args.verbose:
print "\ndirectory =",directory
# FILTER
if 'SingleMuon' in directory and channel not in ['mutau','mumu','elemu']: continue
if ('SingleElectron' in directory or 'EGamma' in directory) and channel!='eletau': continue
if 'Tau' in directory[:5] and channel!='tautau': continue
if 'LQ3' in directory[:5] and channel not in ['mutau','eletau','tautau']: continue
# GET SKIMMED
if args.useSkim and isSkimmed(directory,year):
directory = isSkimmed(directory,year)
if args.verbose:
print "skimmed:", directory
print bcolors.BOLD + bcolors.OKGREEN + directory + bcolors.ENDC
# FILE LIST
files = [ ]
if not args.useDAS:
files = getFileListLocal(directory,blacklist=blacklist)
if not files:
if not args.useDAS:
print "Getting file list from DAS/PNFS..."
files = getFileList(directory,blacklist=blacklist)
if files:
saveFileListLocal(directory,files,blacklist=blacklist)
if not files:
print bcolors.BOLD + bcolors.WARNING + "Warning! EMPTY filelist for " + directory + bcolors.ENDC
continue
elif args.verbose:
print "FILELIST = "+files[0]
for file in files[1:]:
print " "+file
if args.testrun:
files = files[:1]
# JOB LIST
sample = '__'.join(directory.split('/')[-3:])
ensureDirectory('joblist')
jobList = 'joblist/joblist_%s_%s%s.txt'%(sample,channel,tag)
print "Creating job file %s..."%(jobList)
jobName = getSampleShortName(directory)[1]
jobName += "_%s_%s"%(channel,year)+tag
jobs = open(jobList,'w')
# outdir = ensureDirectory("output_%s/%s"%(year,sample))
outdir = "$PWD"
ensureDirectory(outdir+'/logs/')
# NFILESPERJOBS
nFilesPerJob = args.nFilesPerJob
if nFilesPerJob<1:
for default, patterns in nFilesPerJob_defaults:
if matchSampleToPattern(sample,patterns):
nFilesPerJob = default
break
else:
nFilesPerJob = nFilesPerJob_default
if args.verbose:
print "nFilesPerJob = %s"%nFilesPerJob
filelists = chunkify(files,nFilesPerJob)
# CREATE JOBS
nChunks = 0
checkExistingFiles(outdir,channel,len(filelists))
#filelists = chunkify(files,1)
for file in filelists:
#print "FILES = ",f
createJobs(jobs,file,outdir,sample,nChunks,channel,year=year,tes=tes,ltf=ltf,jtf=jtf,Zmass=Zmass,tag=tag,prefetch=prefetch)
nChunks = nChunks+1
jobs.close()
# SUBMIT
if args.force:
submitJobs_gc(jobName,jobList,nChunks,outdir,batchscript, args.workdir, year, channel)
else:
submit = raw_input("Do you also want to submit %d jobs to the batch system? [y/n] "%(nChunks))
if submit.lower()=='force':
submit = 'y'
args.force = True
if submit.lower()=='quit':
exit(0)
if submit.lower()=='y':
submitJobs_gc(jobName,jobList,nChunks,outdir,batchscript, args.workdir, year, channel)
else:
print "Not submitting jobs"
print
tasks.append("_".join([jobName, str(year), channel]))
# Assemble while script to start every gc task.
while_temp = open(os.environ["CMSSW_BASE"]+"/src/NanoTreeProducer/while_temp.sh", "r").read()
task_list = ["go.py {}gc_conf/{}.conf".format(args.workdir,task) for task in tasks]
while_temp = while_temp.format(TASK_COMMANDS="\n".join(task_list))
with open(os.path.join(args.workdir, "while.sh"), "w") as out:
out.write(while_temp)
print 'Submit samples with: "bash {}"'.format(os.path.join(args.workdir, "while.sh"))
def main():
channels = args.channels
years = args.years
tes = args.tes
ltf = args.ltf
jtf = args.jtf
batchSystem = 'psibatch_runner.sh'
tag = ""
if tes != 1.:
tag += "_TES%.3f" % (tes)
if ltf != 1.:
tag += "_LTF%.3f" % (ltf)
if jtf != 1.:
tag += "_JTF%.3f" % (jtf)
tag = tag.replace('.', 'p')
for year in years:
# READ SAMPLES
directories = []
samplelist = "samples_%s.cfg" % (year)
with open(samplelist, 'r') as file:
for line in file:
line = line.rstrip().lstrip().split(' ')[0].rstrip('/')
if line[:2].count('#') > 0: continue
if line == '': continue
if args.samples and not matchSampleToPattern(
line, args.samples):
continue
if args.vetos and matchSampleToPattern(line, args.vetos):
continue
if args.type == 'mc' and any(
s in line[:len(s) + 2] for s in
['SingleMuon', 'SingleElectron', 'EGamma', 'MET']):
continue
if args.type == 'data' and not any(
s in line[:len(s) + 2] for s in
['SingleMuon', 'SingleElectron', 'EGamma', 'MET']):
continue
directories.append(line)
#print directories
for channel in channels:
print header(year, channel, tag)
# SUBMIT SAMPLES
for directory in directories:
if args.verbose:
print "\ndirectory =", directory
# FILTER
print bcolors.BOLD + bcolors.OKGREEN + directory + bcolors.ENDC
# FILE LIST
files = []
name = directory.split('/')[-3].replace(
'/', '') + '__' + directory.split('/')[-2].replace(
'/', '') + '__' + directory.split('/')[-1].replace(
'/', '')
if not args.useDAS:
files = getFileListLocal(directory)
if not files:
if not args.useDAS:
print "Getting file list from DAS..."
if 'pnfs' in directory:
files = getFileListPNFS(directory)
else:
files = getFileListDAS(directory)
if files:
saveFileListLocal(name, files)
if not files:
print bcolors.BOLD + bcolors.WARNING + "Warning!!! FILELIST empty" + bcolors.ENDC
continue
elif args.verbose:
print "FILELIST = " + files[0]
for file in files[1:]:
print " " + file
# JOB LIST
ensureDirectory('joblist_skim')
jobList = 'joblist_skim/joblist_%s_%s%s.txt' % (name, channel,
tag)
print "Creating job file %s..." % (jobList)
jobName = getSampleShortName(directory, year)[1]
jobName += "_%s_%s" % (channel, year) + tag
jobs = open(jobList, 'w')
outdir = ensureDirectory(
"/work/pbaertsc/heavy_resonance/output_skim_%s/%s" %
(year, name))
ensureDirectory(outdir + '/logs/')
# NFILESPERJOBS
nFilesPerJob = 1
if nFilesPerJob < 1:
for default, patterns in nFilesPerJob_defaults:
if matchSampleToPattern(directory, patterns):
nFilesPerJob = default
break
else:
nFilesPerJob = 4 # default
if args.verbose:
print "nFilesPerJob = %s" % nFilesPerJob
filelists = list(split_seq(files, nFilesPerJob))
# CREATE JOBS
nChunks = 0
checkExistingFiles(outdir, channel, len(filelists))
#filelists = list(split_seq(files,1))
for file in filelists:
#print "FILES = ",f
createJobs(jobs,
file,
outdir,
name,
nChunks,
channel,
year=year,
tes=tes,
ltf=ltf,
jtf=jtf)
nChunks = nChunks + 1
jobs.close()
# SUBMIT
if args.force:
submitJobs(jobName, jobList, nChunks, outdir, batchSystem)
else:
submit = raw_input(
"Do you also want to submit %d jobs to the batch system? [y/n] "
% (nChunks))
if submit.lower() == 'force':
submit = 'y'
args.force = True
if submit.lower() == 'quit':
exit(0)
if submit.lower() == 'y':
submitJobs(jobName, jobList, nChunks, outdir,
batchSystem)
else:
print "Not submitting jobs"
print
outfile = args.outfile
nchunck = args.nchunck
year = args.year
tes = args.tes
kwargs = {
'year': year,
'tes': args.tes,
'ltf': args.ltf,
'jtf': args.jtf,
'ZmassWindow': args.Zmass,
}
if isinstance(infiles, str):
infiles = infiles.split(',')
ensureDirectory(outdir)
dataType = 'mc'
if 'SingleMuon' in infiles[0] or "/Tau/" in infiles[
0] or 'SingleElectron' in infiles[0] or 'EGamma' in infiles[0]:
dataType = 'data'
JSON = '/shome/ineuteli/analysis/LQ_legacy/NanoTreeProducer/json/'
if year == 2016:
json = JSON + 'Cert_271036-284044_13TeV_PromptReco_Collisions16_JSON.txt'
#json = '/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions16/13TeV/Final/Cert_271036-284044_13TeV_PromptReco_Collisions16_JSON.txt'
elif year == 2017:
json = JSON + 'Cert_294927-306462_13TeV_PromptReco_Collisions17_JSON.txt'
#json = '/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions17/13TeV/Final/Cert_294927-306462_13TeV_PromptReco_Collisions17_JSON.txt'
else:
json = JSON + 'Cert_314472-325175_13TeV_PromptReco_Collisions18_JSON.txt'
def main(args):
years = args.years
tag = args.tag
checkEvents = args.checkEvents
outbasedir = "/scratch/ineuteli"
batchscript = 'submit_SGE.sh'
director = "root://t3dcachedb.psi.ch:1094/" #"root://xrootd-cms.infn.it/"
for year in years:
samplesdir = args.outdir if args.outdir else "/pnfs/psi.ch/cms/trivcat/store/user/ineuteli/samples/NANOAOD_%s" % (
year)
# GET LIST
samplelist = []
if args.instance == None:
directories = glob.glob(samplesdir +
"/*/*/NANOAOD*") + glob.glob(samplesdir +
"/*/*/USER*")
else:
directories = glob.glob(samplesdir + "/*/*/" + args.instance)
for directory in sorted(directories):
sample = '/'.join(directory.split('/')[-3:])
if args.samples and not matchSampleToPattern(sample, args.samples):
continue
if args.vetos and matchSampleToPattern(sample, args.vetos):
continue
if args.type == 'mc' and any(
s in sample[:len(s) + 2]
for s in ['SingleMuon', 'SingleElectron', 'Tau', 'EGamma'
]):
continue
if args.type == 'data' and not any(
s in sample[:len(s) + 2]
for s in ['SingleMuon', 'SingleElectron', 'Tau', 'EGamma'
]):
continue
if not os.path.isdir(directory): continue
samplelist.append(directory)
if not samplelist:
print "No samples found in %s!" % (samplesdir)
if args.verbose:
print 'samplelist = %s\n' % (samplelist)
# CHECK samples
print header(year, tag)
for directory in samplelist:
sample = '__'.join(directory.split('/')[-3:])
filelist = '%s/*_skim%s*.root' % (directory, tag)
if args.verbose:
print "directory = %s" % (directory)
print "filelist = %s" % (filelist)
print "sample = %s" % (sample)
# FILE LIST ON SE
filelist = [
director + d
for d in sorted(glob.glob(filelist), key=naturalSort)
]
if not filelist:
print bcolors.BOLD + bcolors.WARNING + "[WN] %s empty filelist" % directory + bcolors.ENDC
elif args.verbose:
print "filelist = %s" % (filelist[0])
for file in filelist[1:]:
print " " + file
# FILE LIST ON DAS
infilelist = []
if args.useLocal:
infilelist = getFileListLocal(sample)
if not infilelist:
if args.useLocal:
print "Getting file list from DAS/PNFS..."
infilelist = getFileListDAS(sample)
if not infilelist:
print bcolors.BOLD + bcolors.WARNING + "Warning! EMPTY filelist for " + directory + bcolors.ENDC
elif args.verbose:
print "infilelist = %s" % (infilelist[0])
for file in infilelist[1:]:
print " " + file
# FILE LIST FOR RESUBMISSION
nevents, resubmitfiles = checkFiles(
filelist,
infilelist,
directory,
clean=args.removeBadFiles,
force=args.force,
cleanBug=args.removeBuggedFiles,
checkEvents=checkEvents)
if len(resubmitfiles) == 0:
print bcolors.BOLD + bcolors.OKGREEN + '[OK] %s is complete ! ' % sample + bcolors.ENDC
elif len(resubmitfiles) > len(infilelist):
print bcolors.BOLD + bcolors.FAIL + 'WARNING! %s has more output files %d than %d input files from DAS!' % (
sample, len(resubmitfiles), len(infilelist)) + bcolors.ENDC
else:
print bcolors.BOLD + bcolors.WARNING + '[WN] %d / %d of %s need to be resubmitted...' % (
len(resubmitfiles), len(infilelist), sample) + bcolors.ENDC
if checkEvents and not any(s in directory for s in ['LQ3']):
compareEventsToDAS(nevents, sample, treename='Events')
if len(resubmitfiles) == 0:
print
continue
# JOB LIST
ensureDirectory('joblist')
jobList = 'joblist/joblist_%s_skim%s_retry.txt' % (sample, tag)
print "Creating job file %s..." % (jobList)
jobName = getSampleShortName(directory)[1]
jobName += "_%s_skim" % (year) + tag
outdir = "%s/output_%s/%s" % (outbasedir, year, sample)
logdir = ensureDirectory("skim_logs_%s/%s" % (year, sample))
# NFILESPERJOBS
nFilesPerJob = args.nFilesPerJob
if nFilesPerJob < 1:
for default, patterns in nFilesPerJob_defaults:
if matchSampleToPattern(directory, patterns):
nFilesPerJob = default
break
else:
nFilesPerJob = 1 # default
if args.verbose:
print "nFilesPerJob = %s" % nFilesPerJob
filelists = chunkify(resubmitfiles, nFilesPerJob)
# CREATE JOBS
with open(jobList, 'w') as jobs:
nChunks = 0
for filelist in filelists:
createSkimJobs(jobs,
year,
sample,
filelist,
outdir,
prefetch=args.prefetch)
nChunks = nChunks + 1
# SUBMIT
if args.force:
submitSkimJobs(jobName, jobList, nChunks, logdir, batchscript)
else:
submit = raw_input(
"Do you also want to submit %d jobs to the batch system? [y/n] "
% (nChunks))
if submit.lower() == 'force':
submit = 'y'
args.force = True
if submit.lower() == 'quit':
exit(0)
if submit.lower() == 'y':
submitSkimJobs(jobName, jobList, nChunks, logdir,
batchscript)
else:
print "Not submitting jobs"
print
def main():
channels = args.channels
years = args.years
tes = args.tes
batchSystem = 'psibatch_runner.sh'
for year in years:
# READ SAMPLES
directories = []
samplelist = "samples_%s.cfg" % (year)
for line in open(samplelist, 'r'):
line = line.rstrip().lstrip().split(' ')[0]
if line[:2].count('#') > 0: continue
if line == '': continue
if args.samples and not matchSampleToPattern(line, args.samples):
continue
if args.veto and matchSampleToPattern(line, args.veto): continue
if args.type == 'mc' and any(
s in line[:len(s) + 2]
for s in ['SingleMuon', 'SingleElectron', 'Tau']):
continue
if args.type == 'data' and not any(
s in line[:len(s) + 2]
for s in ['SingleMuon', 'SingleElectron', 'Tau']):
continue
directories.append(line)
#print directories
for channel in channels:
print header(year, channel)
# SUBMIT SAMPLES
for directory in directories:
if args.verbose:
print "\ndirectory =", directory
# FILTER
if 'SingleMuon' in directory and channel not in [
'mutau', 'mumu'
]:
continue
if 'SingleElectron' in directory and channel != 'eletau':
continue
if 'Tau' in directory[:5] and channel != 'tautau': continue
if 'LQ3' in directory[:5] and channel not in [
'mutau', 'eletau', 'tautau'
]:
continue
print bcolors.BOLD + bcolors.OKGREEN + directory + bcolors.ENDC
files = None
name = None
if 'pnfs' in directory:
name = directory.split('/')[8].replace(
'/', '') + '__' + directory.split('/')[9].replace(
'/', '') + '__' + directory.split('/')[10].replace(
'/', '')
#files = getFileListPNFS(directory)
files = getFileListPNFS(name)
else:
files = getFileListDAS(directory)
name = directory.split('/')[1].replace(
'/', '') + '__' + directory.split('/')[2].replace(
'/', '') + '__' + directory.split('/')[3].replace(
'/', '')
if not files:
print bcolors.BOLD + bcolors.WARNING + "Warning!!! FILELIST empty" + bcolors.ENDC
continue
elif args.verbose:
print "FILELIST = " + files[0]
for file in files[1:]:
print " " + file
# JOBLIST
ensureDirectory('joblist')
jobList = 'joblist/joblist%s_%s.txt' % (name, channel)
print "Creating job file %s..." % (jobList)
jobName = getSampleShortName(directory)[1]
jobs = open(jobList, 'w')
nFilesPerJob = args.nFilesPerJob
outdir = ensureDirectory("output_%s/%s" % (year, name))
# NFILESPERJOBS CHECKS
# Diboson (WW, WZ, ZZ) have very large files and acceptance,
# and the jet-binned DY and WJ files need to be run separately because of a bug affecting LHE_Njets
if nFilesPerJob > 1 and any(vv in jobName[:8] for vv in [
'WW', 'WZ', 'ZZ', 'DY', 'WJ', 'W1J', 'W2J', 'W3J',
'W4J', 'Single', 'Tau'
]):
print bcolors.BOLD + bcolors.WARNING + "[WN] setting number of files per job from %s to 1 for %s" % (
nFilesPerJob, jobName) + bcolors.ENDC
nFilesPerJob = 1
try:
os.stat(outdir)
except:
os.mkdir(outdir)
try:
os.stat(outdir + '/logs/')
except:
os.mkdir(outdir + '/logs/')
# CREATE JOBS
nChunks = 0
filelists = list(split_seq(files, nFilesPerJob))
checkExistingFiles(outdir, channel, len(filelists))
#filelists = list(split_seq(files,1))
for file in filelists:
#print "FILES = ",f
createJobs(jobs,
file,
outdir,
name,
nChunks,
channel,
year=year)
nChunks = nChunks + 1
jobs.close()
# SUBMIT
jobName += "_%s_%s" % (channel, year)
if args.force:
submitJobs(jobName, jobList, nChunks, outdir, batchSystem)
else:
submit = raw_input(
"Do you also want to submit %d jobs to the batch system? [y/n] "
% (nChunks))
if submit.lower() == 'force':
submit = 'y'
args.force = True
if submit.lower() == 'quit':
exit(0)
if submit.lower() == 'y':
submitJobs(jobName, jobList, nChunks, outdir,
batchSystem)
else:
print "Not submitting jobs"
print
def main():
years = args.years
outbasedir = "/scratch/ineuteli"
batchscript = 'submit_SGE.sh'
tag = ""
for year in years:
# READ SAMPLES
directories = []
samplelist = "samples_%s.cfg" % (year)
with open(samplelist, 'r') as file:
for line in file:
line = line.rstrip().lstrip().split(' ')[0].rstrip('/')
if line[:2].count('#') > 0: continue
if line == '': continue
if '/pnfs/' in line: continue
if args.samples and not matchSampleToPattern(
line, args.samples):
continue
if args.vetos and matchSampleToPattern(line, args.vetos):
continue
if args.type == 'mc' and any(
s in line[:len(s) + 2] for s in
['SingleMuon', 'SingleElectron', 'Tau', 'EGamma']):
continue
if args.type == 'data' and not any(
s in line[:len(s) + 2] for s in
['SingleMuon', 'SingleElectron', 'Tau', 'EGamma']):
continue
directories.append(line)
if args.testrun:
directories = directories[:1]
blacklist = [] #getBlackList("filelist/blacklist.txt")
print header(year, tag)
# SUBMIT SAMPLES
for directory in directories:
if args.verbose:
print "\ndirectory =", directory
print bcolors.BOLD + bcolors.OKGREEN + directory + bcolors.ENDC
# FILE LIST
files = []
name = '__'.join(directory.split('/')[-3:])
sample = '/' + name.replace('__', '/')
###if not args.useDAS:
### files = getFileListLocal(directory,blacklist=blacklist)
###if not files:
###if not args.useDAS:
### print "Getting file list from DAS..."
files = getFileListDAS(directory, blacklist=blacklist)
### if files:
### saveFileListLocal(name,files,blacklist=blacklist)
if not files:
print bcolors.BOLD + bcolors.WARNING + "Warning! EMPTY filelist for " + directory + bcolors.ENDC
continue
elif args.verbose:
print "FILELIST = " + files[0]
for file in files[1:]:
print " " + file
if args.testrun:
files = files[:1]
# JOB LIST
ensureDirectory('joblist')
jobList = 'joblist/joblist_%s_skim%s.txt' % (name, tag)
print "Creating job file %s..." % (jobList)
jobName = getSampleShortName(directory)[1]
jobName += "_%s_skim" % (year) + tag
outdir = "%s/output_%s/%s" % (outbasedir, year, name)
logdir = ensureDirectory("skim_logs_%s/%s" % (year, name))
# NFILESPERJOBS
nFilesPerJob = args.nFilesPerJob
if nFilesPerJob < 1:
for default, patterns in nFilesPerJob_defaults:
if matchSampleToPattern(directory, patterns):
nFilesPerJob = default
break
else:
nFilesPerJob = nFilesPerJob_default # default
if args.verbose:
print "nFilesPerJob = %s" % nFilesPerJob
filelists = chunkify(files, nFilesPerJob)
# CREATE JOBS
with open(jobList, 'w') as jobs:
nChunks = 0
for filelist in filelists:
createSkimJobs(jobs,
year,
name,
filelist,
outdir,
prefetch=args.prefetch)
nChunks = nChunks + 1
# SUBMIT
if args.force:
submitSkimJobs(jobName, jobList, nChunks, logdir, batchscript)
else:
submit = raw_input(
"Do you also want to submit %d jobs to the batch system? [y/n] "
% (nChunks))
if submit.lower() == 'force':
submit = 'y'
args.force = True
if submit.lower() == 'quit':
exit(0)
if submit.lower() == 'y':
submitSkimJobs(jobName, jobList, nChunks, logdir,
batchscript)
else:
print "Not submitting jobs"
print