def GetLPClist():
if os.environ['CMSSW_BASE'] == "":
return False
import sys
import shutil
sys.path.append("%s/src/SusyAnaTools/Tools/condor/" % os.environ["CMSSW_BASE"])
from samples import SampleCollection
sc = SampleCollection()
for ds in sc.sampleCollectionList():
ds = ds.strip()
print(ds)
for s, n in sc.sampleList(ds):
if not os.path.isfile(s):
continue
shutil.copyfile(s, n+".list")
def GetLPClist():
if os.environ['CMSSW_BASE'] == "":
return False
import sys
import shutil
sys.path.append("%s/src/SusyAnaTools/Tools/condor/" %
os.environ["CMSSW_BASE"])
from samples import SampleCollection
sc = SampleCollection()
for ds in sc.sampleCollectionList():
ds = ds.strip()
print(ds)
for s, n in sc.sampleList(ds):
if not os.path.isfile(s):
continue
shutil.copyfile(s, n + ".list")
makeExeAndFriendsTarrball(filestoTransferTT, "TT")
else:
exeName = "makePlots"
submitFile = submitFileGMP
makeExeAndFriendsTarrball(filestoTransferGMP, "gmp")
nFilesPerJob = options.numfile
fileParts = [submitFile]
sc = SampleCollection("../sampleSets.cfg", "../sampleCollections.cfg")
datasets = []
if options.dataCollections or options.dataCollectionslong:
scl = sc.sampleCollectionList()
for sampleCollection in scl:
sl = sc.sampleList(sampleCollection)
print sampleCollection
if options.dataCollectionslong:
sys.stdout.write("\t")
for sample in sl:
sys.stdout.write("%s " % sample[1])
print ""
print ""
exit(0)
if options.datasets:
datasets = options.datasets.split(',')
else:
print "No dataset specified"
exit(0)
if options.datasets:
datasets = options.datasets.split(',')
else:
print "No dataset pecified"
exit(0)
lumis = sc.sampleCollectionLumiList()
lumi = sc.getFixedLumi()
if options.refLumi != None:
lumi = lumis[options.refLumi]
print "Normalizing to %s pb-1" % (lumi)
for ds in datasets:
ds = ds.strip()
for s, n in sc.sampleList(ds):
print n
f = open(s)
if not f == None:
count = 0
for l in f:
if '.root' in l and not 'failed' in l:
count = count + 1
for startFileNum in xrange(0, count, nFilesPerJob):
fileParts.append(
"Arguments = %s $ENV(CMSSW_BASE) %i %i %f %s\nQueue\n\n" %
(n, nFilesPerJob, startFileNum, lumi, s))
f.close()
fout = open("condor_submit.txt", "w")
fout.write(''.join(fileParts))
fileParts = [submitFile]
sc = SampleCollection()
datasets = []
if options.datasets:
datasets = options.datasets.split(',')
else:
print "No dataset pecified"
exit(0)
for ds in datasets:
ds = ds.strip()
for s, n in sc.sampleList(ds):
# if n == skip1 or n == skip2:
#continue
print n
f = open(s)
if not f == None:
count = 0
for l in f:
if '.root' in l and not 'failed' in l:
count = count + 1
for startFileNum in xrange(0, count, nFilesPerJob):
fileParts.append("Arguments = %s $ENV(CMSSW_BASE) %i %i %s\nQueue\n\n"%(n, startFileNum, nFilesPerJob, s))
f.close()
fout = open("condorCS_submit.txt", "w")
fout.write(''.join(fileParts))
def submit(datasets,
refLumi,
era,
numfile=5,
noSubmit=False,
verbose=False,
dataCollections=False,
dataCollectionslong=False,
goMakeEff=False,
goMakeEffPhoton=False,
goMakeBeff=False,
goMakeSigEff=False,
goMakeTopPlots=False,
goTTPlots=False):
print "# ---------- Submitting condor jobs for {0} ---------- #".format(
era)
# split string into list
if datasets:
datasets = datasets.split(',')
else:
datasets = []
print "ERROR: datasets not found"
# sample config files
eras = [
"2016", "2017", "2017_BE", "2017_F", "2018", "2018_PreHEM",
"2018_PostHEM"
]
if era not in eras:
print "Please use -y to enter era (2016, 2017, 2017_BE, 2017_F, 2018, 2018_PreHEM, or 2018_PostHEM)."
exit(1)
# era may include period
# year does not include period
year = era[0:4]
sampleSetsFile = "sampleSets_PostProcessed_" + year + ".cfg"
sampleCollectionsFile = "sampleCollections_" + year + ".cfg"
# TopTagger.cfg
mvaFileName = ""
with file(environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/TopTagger_Tensorflow_2016.cfg"
) as meowttcfgFile:
for line in meowttcfgFile:
line = line.split("#")[0]
if "modelFile" in line:
mvaFileName = line.split("=")[1].strip().strip("\"")
break
if verbose:
print "mvaFileName = {0}".format(mvaFileName)
#here I hack in the tarball for GMP, this needs to be generalized to the other options
filestoTransferGMP = [
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/makePlots",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/shapes_njets.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2016_trigger_eff.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2017_trigger_eff.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2018_trigger_eff.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/lepEffHists.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/effhists_GJets.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/njetWgtHists.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/dataMCweights.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/dataMCreweight.root",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/dataMCreweight_allJets.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_bTagEff.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/ISRWeights.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_ISRJets.root",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/PileupHistograms_0121_69p2mb_pm4p6.root",
environ["CMSSW_BASE"] +
"/src/TopTagger/TopTagger/test/libTopTagger.so",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagSF_AltTWP.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2016.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2017.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2018.root",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/TopTagger_Tensorflow_2016.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/TopTagger_Tensorflow_2017.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/TopTagger_Tensorflow_2018.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2016.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2017.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2018.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2016.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2017.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2018.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/dc_BkgPred_BinMaps_master.json",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/systematics.json"
]
if mvaFileName:
filestoTransferGMP += [
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/%(trainingFile)s" % {
"trainingFile": mvaFileName
}
]
filestoTransferGMEP = [
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/calcEffPhoton",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2016_trigger_eff.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2017_trigger_eff.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2018_trigger_eff.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_bTagEff.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/ISRWeights.root",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_ISRJets.root",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/PileupHistograms_0121_69p2mb_pm4p6.root",
environ["CMSSW_BASE"] +
"/src/TopTagger/TopTagger/test/libTopTagger.so",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2016.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2017.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2018.cfg"
]
if mvaFileName:
filestoTransferGMEP += [
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/%(trainingFile)s" % {
"trainingFile": mvaFileName
}
]
if sampleSetsFile:
filestoTransferGMEP += [
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/" + sampleSetsFile
]
if sampleCollectionsFile:
filestoTransferGMEP += [
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/" +
sampleCollectionsFile
]
#go make plots!
# old version for submitting from condor directory
#submitFileGMP = """universe = vanilla
#Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh
#Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
#Should_Transfer_Files = YES
#WhenToTransferOutput = ON_EXIT
#Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gmp.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz
#Output = logs/makePlots_$(Process).stdout
#Error = logs/makePlots_$(Process).stderr
#Log = logs/makePlots_$(Process).log
#notify_user = ${LOGNAME}@FNAL.GOV
#x509userproxy = $ENV(X509_USER_PROXY)
#+maxWallTime = 2880
#
#"""
# new version for submitting from specific directory for submission (use relative instead of full paths for tar files)
# note that tabs and spaces will appear in condor_submit.txt if you have them here
submitFileGMP = """universe = vanilla
Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh
Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT
Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,gmp.tar.gz,$ENV(CMSSW_VERSION).tar.gz
Output = logs/makePlots_$(Process).stdout
Error = logs/makePlots_$(Process).stderr
Log = logs/makePlots_$(Process).log
notify_user = ${LOGNAME}@FNAL.GOV
x509userproxy = $ENV(X509_USER_PROXY)
+maxWallTime = 2880
"""
#Here is the configuration for the Data/MC validation of the TopTagger
filestoTransferTT = [
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/simpleAnalyzer",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root",
environ["CMSSW_BASE"] +
"/src/SusyAnaTools/Tools/data/allINone_bTagEff.root",
environ["CMSSW_BASE"] +
"/src/SusyAnaTools/Tools/ISR_Root_Files/ISRWeights.root",
environ["CMSSW_BASE"] +
"/src/SusyAnaTools/Tools/ISR_Root_Files/allINone_ISRJets.root",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv",
environ["CMSSW_BASE"] +
"/src/SusyAnaTools/Tools/data/PileupHistograms_0121_69p2mb_pm4p6.root",
environ["CMSSW_BASE"] +
"/src/TopTagger/TopTagger/test/libTopTagger.so",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2016.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2017.cfg",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/sampleCollections_2018.cfg"
]
if mvaFileName:
filestoTransferTT += [
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/%(trainingFile)s" % {
"trainingFile": mvaFileName
}
]
#go make TTopTagger plots!
submitFileTT = """universe = vanilla
Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goTTplots.sh
Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT
Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goTTplots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/TT.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz
Output = logs/TT_$(Process).stdout
Error = logs/TT_$(Process).stderr
Log = logs/TT_$(Process).log
notify_user = ${LOGNAME}@FNAL.GOV
x509userproxy = $ENV(X509_USER_PROXY)
+maxWallTime = 2880
"""
#go make top plots!
filestoTransferGTP = [
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/makeTopPlots",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root",
environ["CMSSW_BASE"] +
"/src/SusyAnaTools/Tools/data/allINone_bTagEff.root",
environ["CMSSW_BASE"] +
"/src/SusyAnaTools/Tools/ISR_Root_Files/ISRWeights.root",
environ["CMSSW_BASE"] +
"/src/SusyAnaTools/Tools/ISR_Root_Files/allINone_ISRJets.root",
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv",
environ["CMSSW_BASE"] +
"/src/SusyAnaTools/Tools/data/PileupHistograms_0121_69p2mb_pm4p6.root",
environ["CMSSW_BASE"] +
"/src/TopTagger/TopTagger/test/libTopTagger.so",
environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg"
]
if mvaFileName:
filestoTransferTT += [
environ["CMSSW_BASE"] +
"/src/ZInvisible/Tools/%(trainingFile)s" % {
"trainingFile": mvaFileName
}
]
submitFileGTP = """universe = vanilla
Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeTopPlots.sh
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT
Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gtp.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz
Output = logs/makePlots_$(Process).stdout
Error = logs/makePlots_$(Process).stderr
Log = logs/makePlots_$(Process).log
notify_user = ${LOGNAME}@FNAL.GOV
x509userproxy = $ENV(X509_USER_PROXY)
+maxWallTime = 2880
"""
#go make lepton efficiency
submitFileGME = """universe = vanilla
Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEff.sh
Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT
Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/calcEff, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEff.sh, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/zRes.root
notify_user = ${LOGNAME}@FNAL.GOV
x509userproxy = $ENV(X509_USER_PROXY)
"""
#go make photon efficiency
submitFileGMEP = """universe = vanilla
Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEffPhoton.sh
Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT
Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEffPhoton.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gmep.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz
Output = logs/calcEffPhoton_$(Process).stdout
Error = logs/calcEffPhoton_$(Process).stderr
Log = logs/calcEffPhoton_$(Process).log
notify_user = ${LOGNAME}@FNAL.GOV
x509userproxy = $ENV(X509_USER_PROXY)
+maxWallTime = 2880
"""
#go B Efficiency calc
submitFileGBE = """universe = vanilla
Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeBeff.sh
Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT
Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/beffCalc, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeBeff.sh
Output = logs/makePlots_$(Process).stdout
Error = logs/makePlots_$(Process).stderr
Log = logs/makePlots_$(Process).log
notify_user = ${LOGNAME}@FNAL.GOV
x509userproxy = $ENV(X509_USER_PROXY)
"""
#go Signal Efficiency
submitFileGSE = """universe = vanilla
Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeSigEff.sh
Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" )
Should_Transfer_Files = YES
WhenToTransferOutput = ON_EXIT
Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/makeSignalHistograms, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeSigEff.sh
Output = logs/makePlots_$(Process).stdout
Error = logs/makePlots_$(Process).stderr
Log = logs/makePlots_$(Process).log
notify_user = ${LOGNAME}@FNAL.GOV
x509userproxy = $ENV(X509_USER_PROXY)
"""
submitFile = ""
exeName = ""
# load samples
sc = SampleCollection("../" + sampleSetsFile,
"../" + sampleCollectionsFile)
# make directory for condor submission
now = datetime.datetime.now()
dirName = "DataMC_%s_submission_%s" % (era,
now.strftime("%Y-%m-%d_%H-%M-%S"))
os.system("mkdir %s" % dirName)
#os.chdir(dirName)
# use class to cd into directory and then return to previous directory
with cd(dirName):
def makeExeAndFriendsTarrball(filestoTransfer, fname):
if not dataCollections and not dataCollectionslong:
#WORLDSWORSESOLUTIONTOAPROBLEM
system("mkdir -p WORLDSWORSESOLUTIONTOAPROBLEM")
for fn in filestoTransfer:
system("cd WORLDSWORSESOLUTIONTOAPROBLEM; ln -s %s" % fn)
tarallinputs = "tar czf %s.tar.gz WORLDSWORSESOLUTIONTOAPROBLEM --dereference" % fname
if verbose:
# Use v option if verbose is set
tarallinputs = "tar czvf %s.tar.gz WORLDSWORSESOLUTIONTOAPROBLEM --dereference" % fname
print "Create tarball {0}.tag.gz".format(fname)
print tarallinputs
system(tarallinputs)
system("rm -r WORLDSWORSESOLUTIONTOAPROBLEM")
if not dataCollections and not dataCollectionslong:
if verbose:
print "Create tarball ${CMSSW_VERSION}.tar.gz"
system(
"tar --exclude-caches-all --exclude-vcs -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION} --exclude=src --exclude=tmp"
)
# makeExeAndFriendsTarrball() is necessary now to apply WORLDSWORSESOLUTIONTOAPROBLEM
if goMakeEff:
exeName = "calcEff"
submitFile = submitFileGME
elif goMakeEffPhoton:
exeName = "calcEffPhoton"
submitFile = submitFileGMEP
makeExeAndFriendsTarrball(filestoTransferGMEP, "gmep")
elif goMakeBeff:
exeName = "beffCalc"
submitFile = submitFileGBE
elif goMakeSigEff:
exeName = "makeSignalHistograms"
submitFile = submitFileGSE
elif goMakeTopPlots:
exeName = "makeTopPlots"
submitFile = submitFileGTP
makeExeAndFriendsTarrball(filestoTransferGTP, "gtp")
elif goTTPlots:
exeName = "simpleAnalyzer"
submitFile = submitFileTT
makeExeAndFriendsTarrball(filestoTransferTT, "TT")
else:
exeName = "makePlots"
submitFile = submitFileGMP
makeExeAndFriendsTarrball(filestoTransferGMP, "gmp")
nFilesPerJob = numfile
fileParts = [submitFile]
if dataCollections or dataCollectionslong:
scl = sc.sampleCollectionList()
for sampleCollection in scl:
sl = sc.sampleList(sampleCollection)
print sampleCollection
if dataCollectionslong:
sys.stdout.write("\t")
for sample in sl:
sys.stdout.write("%s " % sample[1])
print ""
print ""
exit(1)
lumis = sc.sampleCollectionLumiList()
lumi = sc.getFixedLumi()
if refLumi != None:
lumi = lumis[refLumi]
print "Sample for Ref. Lumi: {0}; normalizing to lumi = {1} pb-1".format(
refLumi, lumi)
debug = False
total_files = 0
total_events = 0
jobMap = {}
for ds in datasets:
sample_files = 0
sample_events = 0
ds = ds.strip()
if verbose:
print "# --- {0} --- #".format(ds)
if "Data" in ds:
print "Lumi: {0}".format(lumis[ds])
# s: file, n:name, e:nEvts
for s, n, e in sc.sampleList(ds):
if debug:
print "s={0}, n={1}, e={2}".format(s, n, e)
nFiles = sum(1 for line in open(s))
nJobs = divideRoundUp(nFiles, nFilesPerJob)
sample_files += nFiles
sample_events += e
total_files += nFiles
total_events += e
jobMap[n] = {}
jobMap[n]["nFilesPerJob"] = nFilesPerJob
jobMap[n]["nFiles"] = nFiles
jobMap[n]["nJobs"] = nJobs
if verbose:
print "\t{0}: nFiles={1}, nJobs={2}, n_events={3}".format(
n, nFiles, nJobs, e)
try:
f = open(s)
except IOError:
fShort = s.split("/")[-1]
if (os.path.isfile(fShort)):
os.remove(fShort)
system(
"xrdcp root://cmseos.fnal.gov/$(echo %s | sed 's|/eos/uscms||') ."
% s)
if verbose:
print "fShort = {0}".format(fShort)
f = open(fShort)
if not f == None:
count = 0
for l in f:
if '.root' in l and not 'failed' in l:
count = count + 1
for startFileNum in xrange(0, count, nFilesPerJob):
fileParts.append(
"Arguments = %s $ENV(CMSSW_VERSION) %i %i %f %s %s\n"
% (n, nFilesPerJob, startFileNum, lumi, s, era))
fileParts.append(
"Output = logs/$(Cluster)_$(Process)_%s_%s_%i.stdout\n"
% (exeName, n, startFileNum))
fileParts.append(
"Error = logs/$(Cluster)_$(Process)_%s_%s_%i.stderr\n"
% (exeName, n, startFileNum))
fileParts.append(
"Log = logs/$(Cluster)_$(Process)_%s_%s_%i.log\n" %
(exeName, n, startFileNum))
fileParts.append("Queue\n\n")
f.close()
# number of files and events in sample
if verbose:
print "\t{0}: sample_files={1}, sample_events={2}".format(
ds, sample_files, sample_events)
# total number of files and events
if verbose:
print "# --- Totals --- #"
print "total_files={0}, total_events={1}".format(
total_files, total_events)
with open("nJobs.json", "w") as outfile:
json.dump(jobMap, outfile, sort_keys=True, indent=4)
fout = open("condor_submit.txt", "w")
fout.write(''.join(fileParts))
fout.close()
if not noSubmit:
system('mkdir -p logs')
system("echo 'condor_submit condor_submit.txt'")
system('condor_submit condor_submit.txt')
print "Condor submission directory: {0}".format(dirName)
return dirName
def main():
# Parse command line arguments
parser = optparse.OptionParser("usage: %prog [options]\n")
parser.add_option('-d',
dest='datasets',
type='string',
default='',
help="Lists of datasets, comma separated")
parser.add_option('-H',
dest='outDir',
type='string',
default='rootfiles',
help="Can pass in the output directory name")
parser.add_option('-p',
dest='inPath',
type='string',
default='output-files',
help="Can pass in the input directory name")
parser.add_option('-y',
dest='year',
type='string',
default='',
help="Can pass in the year for this data")
parser.add_option('-o',
action='store_true',
help="Overwrite output directory")
parser.add_option('--noHadd',
action='store_true',
help="Dont hadd the the root files")
parser.add_option('--haddOther',
action='store_true',
help="Do the hack to make BG_OTHER.root")
parser.add_option('--haddData',
action='store_true',
help="Do the hack to make Data.root")
options, args = parser.parse_args()
# Get input directory path
inPath = options.inPath
# Checks if user specified a dataset(s)
datasets = []
if options.datasets:
datasets = options.datasets.split(',')
else:
datasets = getDataSets(inPath)
# Check if output directory exits and makes it if not
outDir = options.outDir
overwrite = options.o
if os.path.exists(outDir):
if overwrite:
print red("Warning: Overwriting output directory")
shutil.rmtree(outDir)
os.makedirs(outDir)
else:
print red("Error: Output directory %s already exits" %
('"' + outDir + '"'))
exit(0)
else:
os.makedirs(outDir)
# Loop over all sample options to find files to hadd
log = []
sc = SampleCollection("../sampleSets.cfg", "../sampleCollections.cfg")
scl = sc.sampleCollectionList()
for sampleCollection in scl:
sl = sc.sampleList(sampleCollection)
if sampleCollection in datasets:
directory = sampleCollection
files = ""
print "-----------------------------------------------------------"
print sampleCollection
print "-----------------------------------------------------------"
# hadd signal root files
sampleSetsToHadd = [
"2016_AllSignal", "2017_AllSignal", "2017_AllSignal_CP5",
"2018pre_AllSignal", "2018post_AllSignal"
]
if sampleCollection in sampleSetsToHadd:
for sample in sl:
files = " " + " ".join(
glob("%s/%s/MyAnalysis_%s_*.root" %
(inPath, directory, sample[1])))
outfile = "%s/%s.root" % (outDir, sample[1])
command = "hadd %s/%s.root %s" % (outDir, sample[1], files)
if not options.noHadd: system(command)
log = checkNumEvents(nEvents=float(sample[2]),
rootFile=outfile,
sampleCollection=sample[1],
log=log)
# hadd other condor jobs
else:
nEvents = 0.0
for sample in sl:
files += " " + " ".join(
glob("%s/%s/MyAnalysis_%s_*.root" %
(inPath, directory, sample[1])))
nEvents += float(sample[2])
outfile = "%s/%s.root" % (outDir, sampleCollection)
command = "hadd %s %s" % (outfile, files)
try:
if not options.noHadd:
process = subprocess.Popen(command, shell=True)
process.wait()
except:
print red(
"Warning: Too many files to hadd, using the exception setup"
)
command = "hadd %s/%s.root %s/%s/*" % (
outDir, sampleCollection, inPath, sampleCollection)
if not options.noHadd: system(command)
pass
log = checkNumEvents(nEvents=nEvents,
rootFile=outfile,
sampleCollection=sampleCollection,
log=log)
#Print log of hadd at the end
if len(log) > 0:
print red(
"------------------------------------------------------------------------------------------------"
)
print red(
"There was some jobs that didn't match the epected number of events, see summary below"
)
for l in log:
print red(l)
print red(
"------------------------------------------------------------------------------------------------"
)
if options.haddOther:
# Hack to make the BG_OTHER.root file
sigNttbar_old = [
"AllSignal", "TT", "TTJets", "Data_SingleMuon",
"Data_SingleElectron"
]
sigNttbar_2016 = [
"2016_AllSignal", "2016_TT", "2016_TTJets", "2016_Data_SingleMuon",
"2016_Data_SingleElectron", "2016_TT_isrUp", "2016_TT_isrDown",
"2016_TT_fsrUp", "2016_TT_fsrDown", "2016_TTX", "2016_QCD",
"2016_TT_erdOn", "2016_TT_hdampUp", "2016_TT_hdampDown",
"2016_TT_underlyingEvtUp", "2016_TT_underlyingEvtDown"
]
sigNttbar_2017 = [
"2017_AllSignal", "2017_AllSignal_CP5", "2017_TT", "2017_TTJets",
"2017_Data_SingleMuon", "2017_Data_SingleElectron", "2017_TTX",
"2017_QCD", "2017_TT_erdOn", "2017_TT_hdampUp",
"2017_TT_hdampDown", "2017_TT_underlyingEvtUp",
"2017_TT_underlyingEvtDown"
]
sigNttbar_2018pre = [
"2018pre_AllSignal", "2018pre_TT", "2018pre_TTJets",
"2018pre_Data_SingleMuon", "2018pre_Data_SingleElectron",
"2018pre_TTX", "2018pre_QCD", "2018pre_TT_erdOn",
"2018pre_TT_hdampUp", "2018pre_TT_hdampDown",
"2018pre_TT_underlyingEvtUp", "2018pre_TT_underlyingEvtDown"
]
sigNttbar_2018post = [
"2018post_AllSignal", "2018post_TT", "2018post_TTJets",
"2018post_Data_SingleMuon", "2018post_Data_SingleElectron",
"2018post_TTX", "2018post_QCD", "2018post_TT_erdOn",
"2018post_TT_hdampUp", "2018post_TT_hdampDown",
"2018post_TT_underlyingEvtUp", "2018post_TT_underlyingEvtDown"
]
sigNttbar = sigNttbar_old + sigNttbar_2016 + sigNttbar_2017 + sigNttbar_2018pre + sigNttbar_2018post
files = ""
for sampleCollection in scl:
sl = sc.sampleList(sampleCollection)
if sampleCollection in datasets:
if sampleCollection not in sigNttbar:
directory = sampleCollection
files += " %s/%s.root " % (outDir, directory)
if options.year:
command = "hadd %s/%s_BG_OTHER.root %s" % (outDir, options.year,
files)
else:
command = "hadd %s/BG_OTHER.root %s" % (outDir, files)
print "-----------------------------------------------------------"
print command
print "-----------------------------------------------------------"
system(command)
if options.haddData:
# Hack to make the Data.root file (hadd all the data together)
dataFiles = [
"Data_SingleMuon.root", "Data_SingleElectron.root",
"2016_Data_SingleMuon.root", "2016_Data_SingleElectron.root",
"2017_Data_SingleMuon.root", "2017_Data_SingleElectron.root",
"2018pre_Data_SingleMuon.root", "2018pre_Data_SingleElectron.root",
"2018post_Data_SingleMuon.root",
"2018post_Data_SingleElectron.root"
]
if options.year:
command = "hadd %s/%s_Data.root " % (outDir, options.year)
else:
command = "hadd %s/Data.root " % outDir
for f in dataFiles:
if os.path.exists(outDir + "/" + f):
command += " %s/%s" % (outDir, f)
print "-----------------------------------------------------------"
print command
print "-----------------------------------------------------------"
system(command)
def main():
repo = "Analyzer/Analyzer"
# Parse command line arguments
parser = optparse.OptionParser("usage: %prog [options]\n")
parser.add_option('-n',
dest='numfile',
type='int',
default=10,
help="number of files per job")
parser.add_option('-d',
dest='datasets',
type='string',
default='',
help="List of datasets, comma separated")
parser.add_option('-l',
dest='dataCollections',
action='store_true',
default=False,
help="List all datacollections")
parser.add_option('-L',
dest='dataCollectionslong',
action='store_true',
default=False,
help="List all datacollections and sub collections")
parser.add_option(
'-c',
dest='noSubmit',
action='store_true',
default=False,
help="Do not submit jobs. Only create condor_submit.txt.")
parser.add_option(
'--output',
dest='outPath',
type='string',
default='.',
help="Name of directory where output of each condor job goes")
parser.add_option(
'--analyze',
dest='analyze',
default='Analyze1Lep',
help=
"AnalyzeBackground, AnalyzeEventSelection, Analyze0Lep, Analyze1Lep, MakeNJetDists"
)
options, args = parser.parse_args()
# Prepare the list of files to transfer
mvaFileName2016 = getTopTaggerTrainingFile(
environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2016.cfg" % repo)
mvaFileName2017 = getTopTaggerTrainingFile(
environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2017.cfg" % repo)
mvaFileName2018 = getTopTaggerTrainingFile(
environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2018.cfg" % repo)
filestoTransfer = [
environ["CMSSW_BASE"] + "/src/%s/test/MyAnalysis" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/%s" % (repo, mvaFileName2016),
environ["CMSSW_BASE"] + "/src/%s/test/%s" % (repo, mvaFileName2017),
environ["CMSSW_BASE"] + "/src/%s/test/%s" % (repo, mvaFileName2018),
environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2016.cfg" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2017.cfg" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2018.cfg" % repo,
environ["CMSSW_BASE"] +
"/src/TopTagger/TopTagger/test/libTopTagger.so",
environ["CMSSW_BASE"] + "/src/%s/test/sampleSets.cfg" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/sampleCollections.cfg" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_2016.cfg" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_2017.cfg" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/DeepEventShape_2018pre.cfg" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/DeepEventShape_2018post.cfg" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/DeepEventShape_NonIsoMuon_2016.cfg" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/DeepEventShape_NonIsoMuon_2017.cfg" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/DeepEventShape_NonIsoMuon_2018pre.cfg" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/DeepEventShape_NonIsoMuon_2018post.cfg" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/keras_frozen_2016.pb" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/keras_frozen_2017.pb" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/keras_frozen_2018pre.pb" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/keras_frozen_2018post.pb" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/allInOne_BTagEff.root" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/allInOne_SFMean.root" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/allInOne_leptonSF_2016.root" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/allInOne_leptonSF_2017.root" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/allInOne_leptonSF_2018.root" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/PileupHistograms_0121_69p2mb_pm4p6.root" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/pu_ratio.root" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/PileupHistograms_2018_69mb_pm5.root" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/CSVv2_Moriond17_B_H.csv" % repo,
environ["CMSSW_BASE"] + "/src/%s/test/DeepCSV_102XSF_WP_V1.csv" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/DeepCSV_2016LegacySF_WP_V1.csv" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/DeepCSV_94XSF_WP_V4_B_F.csv" % repo,
environ["CMSSW_BASE"] +
"/src/%s/test/L1prefiring_jetpt_2017BtoF.root" % repo,
]
print "--------------Files to Transfer-----------------"
for i in filestoTransfer:
print i
print "------------------------------------------------"
sc = SampleCollection("../sampleSets.cfg", "../sampleCollections.cfg")
if options.dataCollections or options.dataCollectionslong:
scl = sc.sampleCollectionList()
for sampleCollection in scl:
sl = sc.sampleList(sampleCollection)
print sampleCollection
if options.dataCollectionslong:
sys.stdout.write("\t")
for sample in sl:
sys.stdout.write("%s " % sample[1])
print ""
print ""
exit(0)
datasets = []
if options.datasets:
datasets = options.datasets.split(',')
else:
print "No dataset specified"
exit(0)
fileParts = []
fileParts.append("Universe = vanilla\n")
fileParts.append("Executable = run_Analyzer_condor.sh\n")
fileParts.append(
"Transfer_Input_Files = %s/%s.tar.gz, %s/exestuff.tar.gz\n" %
(options.outPath, environ["CMSSW_VERSION"], options.outPath))
fileParts.append("Should_Transfer_Files = YES\n")
fileParts.append("WhenToTransferOutput = ON_EXIT\n")
fileParts.append("x509userproxy = $ENV(X509_USER_PROXY)\n\n")
nFilesPerJob = options.numfile
numberOfJobs = 0
for ds in datasets:
ds = ds.strip()
# create the directory
if not os.path.isdir("%s/output-files/%s" % (options.outPath, ds)):
os.makedirs("%s/output-files/%s" % (options.outPath, ds))
for s, n, e in sc.sampleList(ds):
print "SampleSet:", n, ", nEvents:", e
f = open(s)
if not f == None:
count = 0
for l in f:
if '.root' in l:
count = count + 1
for startFileNum in xrange(0, count, nFilesPerJob):
numberOfJobs += 1
fileParts.append(
"transfer_output_remaps = \"MyAnalysis_%s_%s.root = %s/output-files/%s/MyAnalysis_%s_%s.root\"\n"
% (n, startFileNum, options.outPath, ds, n,
startFileNum))
fileParts.append(
"Arguments = %s %i %i %s %s %s\n" %
(n, nFilesPerJob, startFileNum, s, options.analyze,
environ["CMSSW_VERSION"]))
fileParts.append(
"Output = %s/log-files/MyAnalysis_%s_%i.stdout\n" %
(options.outPath, n, startFileNum))
fileParts.append(
"Error = %s/log-files/MyAnalysis_%s_%i.stderr\n" %
(options.outPath, n, startFileNum))
fileParts.append(
"Log = %s/log-files/MyAnalysis_%s_%i.log\n" %
(options.outPath, n, startFileNum))
fileParts.append("Queue\n\n")
f.close()
fout = open("condor_submit.txt", "w")
fout.write(''.join(fileParts))
fout.close()
if not options.dataCollections and not options.dataCollectionslong:
makeExeAndFriendsTarball(filestoTransfer, "exestuff", options.outPath)
system(
"tar --exclude-caches-all --exclude-vcs -zcf %s/${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION} --exclude=src --exclude=tmp"
% options.outPath)
if not options.noSubmit:
system('mkdir -p %s/log-files' % options.outPath)
system("echo 'condor_submit condor_submit.txt'")
system('condor_submit condor_submit.txt')
else:
print "------------------------------------------"
print "Number of Jobs:", numberOfJobs
print "------------------------------------------"
exeName = "makeTrainingTuples"
submitFile = submitFileGTP
makeExeAndFriendsTarrball(filestoTransferGTP, "gtp")
nFilesPerJob = options.numfile
fileParts = [submitFile]
sc = SampleCollection("../sampleSets.txt", "../sampleCollections.txt")
datasets = []
if options.dataCollections or options.dataCollectionslong:
scl = sc.sampleCollectionList()
for sampleCollection in scl:
sl = sc.sampleList(sampleCollection)
print sampleCollection
if options.dataCollectionslong:
sys.stdout.write("\t")
for sample in sl:
sys.stdout.write("%s "%sample[1])
print ""
print ""
exit(0)
if options.datasets:
datasets = options.datasets.split(',')
else:
print "No dataset specified"
exit(0)