def GetLPClist(): if os.environ['CMSSW_BASE'] == "": return False import sys import shutil sys.path.append("%s/src/SusyAnaTools/Tools/condor/" % os.environ["CMSSW_BASE"]) from samples import SampleCollection sc = SampleCollection() for ds in sc.sampleCollectionList(): ds = ds.strip() print(ds) for s, n in sc.sampleList(ds): if not os.path.isfile(s): continue shutil.copyfile(s, n + ".list")
exeName = "makeTopPlots" submitFile = submitFileGTP makeExeAndFriendsTarrball(filestoTransferGTP, "gtp") elif options.goTTPlots: exeName = "simpleAnalyzer" submitFile = submitFileTT makeExeAndFriendsTarrball(filestoTransferTT, "TT") else: exeName = "makePlots" submitFile = submitFileGMP makeExeAndFriendsTarrball(filestoTransferGMP, "gmp") nFilesPerJob = options.numfile fileParts = [submitFile] sc = SampleCollection("../sampleSets.cfg", "../sampleCollections.cfg") datasets = [] if options.dataCollections or options.dataCollectionslong: scl = sc.sampleCollectionList() for sampleCollection in scl: sl = sc.sampleList(sampleCollection) print sampleCollection if options.dataCollectionslong: sys.stdout.write("\t") for sample in sl: sys.stdout.write("%s " % sample[1]) print "" print "" exit(0)
default=None, help= "Data collection to define lumi (uses default lumi if no reference data collection is defined)" ) parser.add_option('-c', dest='noSubmit', action='store_true', default=False, help="Do not submit jobs. Only create condor_submit.txt.") options, args = parser.parse_args() nFilesPerJob = options.numfile fileParts = [submitFile] sc = SampleCollection() datasets = [] if options.dataCollections: print sc.sampleCollectionList() exit(0) if options.datasets: datasets = options.datasets.split(',') else: print "No dataset pecified" exit(0) lumis = sc.sampleCollectionLumiList() lumi = sc.getFixedLumi() if options.refLumi != None:
def submit(datasets, refLumi, era, numfile=5, noSubmit=False, verbose=False, dataCollections=False, dataCollectionslong=False, goMakeEff=False, goMakeEffPhoton=False, goMakeBeff=False, goMakeSigEff=False, goMakeTopPlots=False, goTTPlots=False): print "# ---------- Submitting condor jobs for {0} ---------- #".format( era) # split string into list if datasets: datasets = datasets.split(',') else: datasets = [] print "ERROR: datasets not found" # sample config files eras = [ "2016", "2017", "2017_BE", "2017_F", "2018", "2018_PreHEM", "2018_PostHEM" ] if era not in eras: print "Please use -y to enter era (2016, 2017, 2017_BE, 2017_F, 2018, 2018_PreHEM, or 2018_PostHEM)." exit(1) # era may include period # year does not include period year = era[0:4] sampleSetsFile = "sampleSets_PostProcessed_" + year + ".cfg" sampleCollectionsFile = "sampleCollections_" + year + ".cfg" # TopTagger.cfg mvaFileName = "" with file(environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger_Tensorflow_2016.cfg" ) as meowttcfgFile: for line in meowttcfgFile: line = line.split("#")[0] if "modelFile" in line: mvaFileName = line.split("=")[1].strip().strip("\"") break if verbose: print "mvaFileName = {0}".format(mvaFileName) #here I hack in the tarball for GMP, this needs to be generalized to the other options filestoTransferGMP = [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/makePlots", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/shapes_njets.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2016_trigger_eff.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2017_trigger_eff.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2018_trigger_eff.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/lepEffHists.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/effhists_GJets.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/njetWgtHists.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/dataMCweights.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/dataMCreweight.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/dataMCreweight_allJets.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_bTagEff.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/ISRWeights.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_ISRJets.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/PileupHistograms_0121_69p2mb_pm4p6.root", environ["CMSSW_BASE"] + "/src/TopTagger/TopTagger/test/libTopTagger.so", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagSF_AltTWP.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2016.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2017.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/tTagEff_2018.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger_Tensorflow_2016.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger_Tensorflow_2017.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger_Tensorflow_2018.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2016.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2017.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger_DiscriminatorFilter_2018.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2016.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2017.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2018.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/dc_BkgPred_BinMaps_master.json", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/systematics.json" ] if mvaFileName: filestoTransferGMP += [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/%(trainingFile)s" % { "trainingFile": mvaFileName } ] filestoTransferGMEP = [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/calcEffPhoton", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2016_trigger_eff.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2017_trigger_eff.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/2018_trigger_eff.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_bTagEff.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/ISRWeights.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/allINone_ISRJets.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/PileupHistograms_0121_69p2mb_pm4p6.root", environ["CMSSW_BASE"] + "/src/TopTagger/TopTagger/test/libTopTagger.so", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2016.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2017.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2018.cfg" ] if mvaFileName: filestoTransferGMEP += [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/%(trainingFile)s" % { "trainingFile": mvaFileName } ] if sampleSetsFile: filestoTransferGMEP += [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/" + sampleSetsFile ] if sampleCollectionsFile: filestoTransferGMEP += [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/" + sampleCollectionsFile ] #go make plots! # old version for submitting from condor directory #submitFileGMP = """universe = vanilla #Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh #Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" ) #Should_Transfer_Files = YES #WhenToTransferOutput = ON_EXIT #Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gmp.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz #Output = logs/makePlots_$(Process).stdout #Error = logs/makePlots_$(Process).stderr #Log = logs/makePlots_$(Process).log #notify_user = ${LOGNAME}@FNAL.GOV #x509userproxy = $ENV(X509_USER_PROXY) #+maxWallTime = 2880 # #""" # new version for submitting from specific directory for submission (use relative instead of full paths for tar files) # note that tabs and spaces will appear in condor_submit.txt if you have them here submitFileGMP = """universe = vanilla Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" ) Should_Transfer_Files = YES WhenToTransferOutput = ON_EXIT Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,gmp.tar.gz,$ENV(CMSSW_VERSION).tar.gz Output = logs/makePlots_$(Process).stdout Error = logs/makePlots_$(Process).stderr Log = logs/makePlots_$(Process).log notify_user = ${LOGNAME}@FNAL.GOV x509userproxy = $ENV(X509_USER_PROXY) +maxWallTime = 2880 """ #Here is the configuration for the Data/MC validation of the TopTagger filestoTransferTT = [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/simpleAnalyzer", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root", environ["CMSSW_BASE"] + "/src/SusyAnaTools/Tools/data/allINone_bTagEff.root", environ["CMSSW_BASE"] + "/src/SusyAnaTools/Tools/ISR_Root_Files/ISRWeights.root", environ["CMSSW_BASE"] + "/src/SusyAnaTools/Tools/ISR_Root_Files/allINone_ISRJets.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv", environ["CMSSW_BASE"] + "/src/SusyAnaTools/Tools/data/PileupHistograms_0121_69p2mb_pm4p6.root", environ["CMSSW_BASE"] + "/src/TopTagger/TopTagger/test/libTopTagger.so", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2016.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2017.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleSets_PostProcessed_2018.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2016.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2017.cfg", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/sampleCollections_2018.cfg" ] if mvaFileName: filestoTransferTT += [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/%(trainingFile)s" % { "trainingFile": mvaFileName } ] #go make TTopTagger plots! submitFileTT = """universe = vanilla Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goTTplots.sh Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" ) Should_Transfer_Files = YES WhenToTransferOutput = ON_EXIT Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goTTplots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/TT.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz Output = logs/TT_$(Process).stdout Error = logs/TT_$(Process).stderr Log = logs/TT_$(Process).log notify_user = ${LOGNAME}@FNAL.GOV x509userproxy = $ENV(X509_USER_PROXY) +maxWallTime = 2880 """ #go make top plots! filestoTransferGTP = [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/makeTopPlots", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/puppiCorr.root", environ["CMSSW_BASE"] + "/src/SusyAnaTools/Tools/data/allINone_bTagEff.root", environ["CMSSW_BASE"] + "/src/SusyAnaTools/Tools/ISR_Root_Files/ISRWeights.root", environ["CMSSW_BASE"] + "/src/SusyAnaTools/Tools/ISR_Root_Files/allINone_ISRJets.root", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/CSVv2_Moriond17_B_H.csv", environ["CMSSW_BASE"] + "/src/SusyAnaTools/Tools/data/PileupHistograms_0121_69p2mb_pm4p6.root", environ["CMSSW_BASE"] + "/src/TopTagger/TopTagger/test/libTopTagger.so", environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/TopTagger.cfg" ] if mvaFileName: filestoTransferTT += [ environ["CMSSW_BASE"] + "/src/ZInvisible/Tools/%(trainingFile)s" % { "trainingFile": mvaFileName } ] submitFileGTP = """universe = vanilla Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeTopPlots.sh Should_Transfer_Files = YES WhenToTransferOutput = ON_EXIT Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakePlots.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gtp.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz Output = logs/makePlots_$(Process).stdout Error = logs/makePlots_$(Process).stderr Log = logs/makePlots_$(Process).log notify_user = ${LOGNAME}@FNAL.GOV x509userproxy = $ENV(X509_USER_PROXY) +maxWallTime = 2880 """ #go make lepton efficiency submitFileGME = """universe = vanilla Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEff.sh Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" ) Should_Transfer_Files = YES WhenToTransferOutput = ON_EXIT Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/calcEff, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEff.sh, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/zRes.root notify_user = ${LOGNAME}@FNAL.GOV x509userproxy = $ENV(X509_USER_PROXY) """ #go make photon efficiency submitFileGMEP = """universe = vanilla Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEffPhoton.sh Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" ) Should_Transfer_Files = YES WhenToTransferOutput = ON_EXIT Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeEffPhoton.sh,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/gmep.tar.gz,$ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/$ENV(CMSSW_VERSION).tar.gz Output = logs/calcEffPhoton_$(Process).stdout Error = logs/calcEffPhoton_$(Process).stderr Log = logs/calcEffPhoton_$(Process).log notify_user = ${LOGNAME}@FNAL.GOV x509userproxy = $ENV(X509_USER_PROXY) +maxWallTime = 2880 """ #go B Efficiency calc submitFileGBE = """universe = vanilla Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeBeff.sh Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" ) Should_Transfer_Files = YES WhenToTransferOutput = ON_EXIT Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/beffCalc, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeBeff.sh Output = logs/makePlots_$(Process).stdout Error = logs/makePlots_$(Process).stderr Log = logs/makePlots_$(Process).log notify_user = ${LOGNAME}@FNAL.GOV x509userproxy = $ENV(X509_USER_PROXY) """ #go Signal Efficiency submitFileGSE = """universe = vanilla Executable = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeSigEff.sh Requirements = OpSys == "LINUX"&& (Arch != "DUMMY" ) Should_Transfer_Files = YES WhenToTransferOutput = ON_EXIT Transfer_Input_Files = $ENV(CMSSW_BASE)/src/ZInvisible/Tools/makeSignalHistograms, $ENV(CMSSW_BASE)/src/ZInvisible/Tools/condor/goMakeSigEff.sh Output = logs/makePlots_$(Process).stdout Error = logs/makePlots_$(Process).stderr Log = logs/makePlots_$(Process).log notify_user = ${LOGNAME}@FNAL.GOV x509userproxy = $ENV(X509_USER_PROXY) """ submitFile = "" exeName = "" # load samples sc = SampleCollection("../" + sampleSetsFile, "../" + sampleCollectionsFile) # make directory for condor submission now = datetime.datetime.now() dirName = "DataMC_%s_submission_%s" % (era, now.strftime("%Y-%m-%d_%H-%M-%S")) os.system("mkdir %s" % dirName) #os.chdir(dirName) # use class to cd into directory and then return to previous directory with cd(dirName): def makeExeAndFriendsTarrball(filestoTransfer, fname): if not dataCollections and not dataCollectionslong: #WORLDSWORSESOLUTIONTOAPROBLEM system("mkdir -p WORLDSWORSESOLUTIONTOAPROBLEM") for fn in filestoTransfer: system("cd WORLDSWORSESOLUTIONTOAPROBLEM; ln -s %s" % fn) tarallinputs = "tar czf %s.tar.gz WORLDSWORSESOLUTIONTOAPROBLEM --dereference" % fname if verbose: # Use v option if verbose is set tarallinputs = "tar czvf %s.tar.gz WORLDSWORSESOLUTIONTOAPROBLEM --dereference" % fname print "Create tarball {0}.tag.gz".format(fname) print tarallinputs system(tarallinputs) system("rm -r WORLDSWORSESOLUTIONTOAPROBLEM") if not dataCollections and not dataCollectionslong: if verbose: print "Create tarball ${CMSSW_VERSION}.tar.gz" system( "tar --exclude-caches-all --exclude-vcs -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION} --exclude=src --exclude=tmp" ) # makeExeAndFriendsTarrball() is necessary now to apply WORLDSWORSESOLUTIONTOAPROBLEM if goMakeEff: exeName = "calcEff" submitFile = submitFileGME elif goMakeEffPhoton: exeName = "calcEffPhoton" submitFile = submitFileGMEP makeExeAndFriendsTarrball(filestoTransferGMEP, "gmep") elif goMakeBeff: exeName = "beffCalc" submitFile = submitFileGBE elif goMakeSigEff: exeName = "makeSignalHistograms" submitFile = submitFileGSE elif goMakeTopPlots: exeName = "makeTopPlots" submitFile = submitFileGTP makeExeAndFriendsTarrball(filestoTransferGTP, "gtp") elif goTTPlots: exeName = "simpleAnalyzer" submitFile = submitFileTT makeExeAndFriendsTarrball(filestoTransferTT, "TT") else: exeName = "makePlots" submitFile = submitFileGMP makeExeAndFriendsTarrball(filestoTransferGMP, "gmp") nFilesPerJob = numfile fileParts = [submitFile] if dataCollections or dataCollectionslong: scl = sc.sampleCollectionList() for sampleCollection in scl: sl = sc.sampleList(sampleCollection) print sampleCollection if dataCollectionslong: sys.stdout.write("\t") for sample in sl: sys.stdout.write("%s " % sample[1]) print "" print "" exit(1) lumis = sc.sampleCollectionLumiList() lumi = sc.getFixedLumi() if refLumi != None: lumi = lumis[refLumi] print "Sample for Ref. Lumi: {0}; normalizing to lumi = {1} pb-1".format( refLumi, lumi) debug = False total_files = 0 total_events = 0 jobMap = {} for ds in datasets: sample_files = 0 sample_events = 0 ds = ds.strip() if verbose: print "# --- {0} --- #".format(ds) if "Data" in ds: print "Lumi: {0}".format(lumis[ds]) # s: file, n:name, e:nEvts for s, n, e in sc.sampleList(ds): if debug: print "s={0}, n={1}, e={2}".format(s, n, e) nFiles = sum(1 for line in open(s)) nJobs = divideRoundUp(nFiles, nFilesPerJob) sample_files += nFiles sample_events += e total_files += nFiles total_events += e jobMap[n] = {} jobMap[n]["nFilesPerJob"] = nFilesPerJob jobMap[n]["nFiles"] = nFiles jobMap[n]["nJobs"] = nJobs if verbose: print "\t{0}: nFiles={1}, nJobs={2}, n_events={3}".format( n, nFiles, nJobs, e) try: f = open(s) except IOError: fShort = s.split("/")[-1] if (os.path.isfile(fShort)): os.remove(fShort) system( "xrdcp root://cmseos.fnal.gov/$(echo %s | sed 's|/eos/uscms||') ." % s) if verbose: print "fShort = {0}".format(fShort) f = open(fShort) if not f == None: count = 0 for l in f: if '.root' in l and not 'failed' in l: count = count + 1 for startFileNum in xrange(0, count, nFilesPerJob): fileParts.append( "Arguments = %s $ENV(CMSSW_VERSION) %i %i %f %s %s\n" % (n, nFilesPerJob, startFileNum, lumi, s, era)) fileParts.append( "Output = logs/$(Cluster)_$(Process)_%s_%s_%i.stdout\n" % (exeName, n, startFileNum)) fileParts.append( "Error = logs/$(Cluster)_$(Process)_%s_%s_%i.stderr\n" % (exeName, n, startFileNum)) fileParts.append( "Log = logs/$(Cluster)_$(Process)_%s_%s_%i.log\n" % (exeName, n, startFileNum)) fileParts.append("Queue\n\n") f.close() # number of files and events in sample if verbose: print "\t{0}: sample_files={1}, sample_events={2}".format( ds, sample_files, sample_events) # total number of files and events if verbose: print "# --- Totals --- #" print "total_files={0}, total_events={1}".format( total_files, total_events) with open("nJobs.json", "w") as outfile: json.dump(jobMap, outfile, sort_keys=True, indent=4) fout = open("condor_submit.txt", "w") fout.write(''.join(fileParts)) fout.close() if not noSubmit: system('mkdir -p logs') system("echo 'condor_submit condor_submit.txt'") system('condor_submit condor_submit.txt') print "Condor submission directory: {0}".format(dirName) return dirName
def main(): # Parse command line arguments parser = optparse.OptionParser("usage: %prog [options]\n") parser.add_option('-d', dest='datasets', type='string', default='', help="Lists of datasets, comma separated") parser.add_option('-H', dest='outDir', type='string', default='rootfiles', help="Can pass in the output directory name") parser.add_option('-p', dest='inPath', type='string', default='output-files', help="Can pass in the input directory name") parser.add_option('-y', dest='year', type='string', default='', help="Can pass in the year for this data") parser.add_option('-o', action='store_true', help="Overwrite output directory") parser.add_option('--noHadd', action='store_true', help="Dont hadd the the root files") parser.add_option('--haddOther', action='store_true', help="Do the hack to make BG_OTHER.root") parser.add_option('--haddData', action='store_true', help="Do the hack to make Data.root") options, args = parser.parse_args() # Get input directory path inPath = options.inPath # Checks if user specified a dataset(s) datasets = [] if options.datasets: datasets = options.datasets.split(',') else: datasets = getDataSets(inPath) # Check if output directory exits and makes it if not outDir = options.outDir overwrite = options.o if os.path.exists(outDir): if overwrite: print red("Warning: Overwriting output directory") shutil.rmtree(outDir) os.makedirs(outDir) else: print red("Error: Output directory %s already exits" % ('"' + outDir + '"')) exit(0) else: os.makedirs(outDir) # Loop over all sample options to find files to hadd log = [] sc = SampleCollection("../sampleSets.cfg", "../sampleCollections.cfg") scl = sc.sampleCollectionList() for sampleCollection in scl: sl = sc.sampleList(sampleCollection) if sampleCollection in datasets: directory = sampleCollection files = "" print "-----------------------------------------------------------" print sampleCollection print "-----------------------------------------------------------" # hadd signal root files sampleSetsToHadd = [ "2016_AllSignal", "2017_AllSignal", "2017_AllSignal_CP5", "2018pre_AllSignal", "2018post_AllSignal" ] if sampleCollection in sampleSetsToHadd: for sample in sl: files = " " + " ".join( glob("%s/%s/MyAnalysis_%s_*.root" % (inPath, directory, sample[1]))) outfile = "%s/%s.root" % (outDir, sample[1]) command = "hadd %s/%s.root %s" % (outDir, sample[1], files) if not options.noHadd: system(command) log = checkNumEvents(nEvents=float(sample[2]), rootFile=outfile, sampleCollection=sample[1], log=log) # hadd other condor jobs else: nEvents = 0.0 for sample in sl: files += " " + " ".join( glob("%s/%s/MyAnalysis_%s_*.root" % (inPath, directory, sample[1]))) nEvents += float(sample[2]) outfile = "%s/%s.root" % (outDir, sampleCollection) command = "hadd %s %s" % (outfile, files) try: if not options.noHadd: process = subprocess.Popen(command, shell=True) process.wait() except: print red( "Warning: Too many files to hadd, using the exception setup" ) command = "hadd %s/%s.root %s/%s/*" % ( outDir, sampleCollection, inPath, sampleCollection) if not options.noHadd: system(command) pass log = checkNumEvents(nEvents=nEvents, rootFile=outfile, sampleCollection=sampleCollection, log=log) #Print log of hadd at the end if len(log) > 0: print red( "------------------------------------------------------------------------------------------------" ) print red( "There was some jobs that didn't match the epected number of events, see summary below" ) for l in log: print red(l) print red( "------------------------------------------------------------------------------------------------" ) if options.haddOther: # Hack to make the BG_OTHER.root file sigNttbar_old = [ "AllSignal", "TT", "TTJets", "Data_SingleMuon", "Data_SingleElectron" ] sigNttbar_2016 = [ "2016_AllSignal", "2016_TT", "2016_TTJets", "2016_Data_SingleMuon", "2016_Data_SingleElectron", "2016_TT_isrUp", "2016_TT_isrDown", "2016_TT_fsrUp", "2016_TT_fsrDown", "2016_TTX", "2016_QCD", "2016_TT_erdOn", "2016_TT_hdampUp", "2016_TT_hdampDown", "2016_TT_underlyingEvtUp", "2016_TT_underlyingEvtDown" ] sigNttbar_2017 = [ "2017_AllSignal", "2017_AllSignal_CP5", "2017_TT", "2017_TTJets", "2017_Data_SingleMuon", "2017_Data_SingleElectron", "2017_TTX", "2017_QCD", "2017_TT_erdOn", "2017_TT_hdampUp", "2017_TT_hdampDown", "2017_TT_underlyingEvtUp", "2017_TT_underlyingEvtDown" ] sigNttbar_2018pre = [ "2018pre_AllSignal", "2018pre_TT", "2018pre_TTJets", "2018pre_Data_SingleMuon", "2018pre_Data_SingleElectron", "2018pre_TTX", "2018pre_QCD", "2018pre_TT_erdOn", "2018pre_TT_hdampUp", "2018pre_TT_hdampDown", "2018pre_TT_underlyingEvtUp", "2018pre_TT_underlyingEvtDown" ] sigNttbar_2018post = [ "2018post_AllSignal", "2018post_TT", "2018post_TTJets", "2018post_Data_SingleMuon", "2018post_Data_SingleElectron", "2018post_TTX", "2018post_QCD", "2018post_TT_erdOn", "2018post_TT_hdampUp", "2018post_TT_hdampDown", "2018post_TT_underlyingEvtUp", "2018post_TT_underlyingEvtDown" ] sigNttbar = sigNttbar_old + sigNttbar_2016 + sigNttbar_2017 + sigNttbar_2018pre + sigNttbar_2018post files = "" for sampleCollection in scl: sl = sc.sampleList(sampleCollection) if sampleCollection in datasets: if sampleCollection not in sigNttbar: directory = sampleCollection files += " %s/%s.root " % (outDir, directory) if options.year: command = "hadd %s/%s_BG_OTHER.root %s" % (outDir, options.year, files) else: command = "hadd %s/BG_OTHER.root %s" % (outDir, files) print "-----------------------------------------------------------" print command print "-----------------------------------------------------------" system(command) if options.haddData: # Hack to make the Data.root file (hadd all the data together) dataFiles = [ "Data_SingleMuon.root", "Data_SingleElectron.root", "2016_Data_SingleMuon.root", "2016_Data_SingleElectron.root", "2017_Data_SingleMuon.root", "2017_Data_SingleElectron.root", "2018pre_Data_SingleMuon.root", "2018pre_Data_SingleElectron.root", "2018post_Data_SingleMuon.root", "2018post_Data_SingleElectron.root" ] if options.year: command = "hadd %s/%s_Data.root " % (outDir, options.year) else: command = "hadd %s/Data.root " % outDir for f in dataFiles: if os.path.exists(outDir + "/" + f): command += " %s/%s" % (outDir, f) print "-----------------------------------------------------------" print command print "-----------------------------------------------------------" system(command)
def main(): repo = "Analyzer/Analyzer" # Parse command line arguments parser = optparse.OptionParser("usage: %prog [options]\n") parser.add_option('-n', dest='numfile', type='int', default=10, help="number of files per job") parser.add_option('-d', dest='datasets', type='string', default='', help="List of datasets, comma separated") parser.add_option('-l', dest='dataCollections', action='store_true', default=False, help="List all datacollections") parser.add_option('-L', dest='dataCollectionslong', action='store_true', default=False, help="List all datacollections and sub collections") parser.add_option( '-c', dest='noSubmit', action='store_true', default=False, help="Do not submit jobs. Only create condor_submit.txt.") parser.add_option( '--output', dest='outPath', type='string', default='.', help="Name of directory where output of each condor job goes") parser.add_option( '--analyze', dest='analyze', default='Analyze1Lep', help= "AnalyzeBackground, AnalyzeEventSelection, Analyze0Lep, Analyze1Lep, MakeNJetDists" ) options, args = parser.parse_args() # Prepare the list of files to transfer mvaFileName2016 = getTopTaggerTrainingFile( environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2016.cfg" % repo) mvaFileName2017 = getTopTaggerTrainingFile( environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2017.cfg" % repo) mvaFileName2018 = getTopTaggerTrainingFile( environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2018.cfg" % repo) filestoTransfer = [ environ["CMSSW_BASE"] + "/src/%s/test/MyAnalysis" % repo, environ["CMSSW_BASE"] + "/src/%s/test/%s" % (repo, mvaFileName2016), environ["CMSSW_BASE"] + "/src/%s/test/%s" % (repo, mvaFileName2017), environ["CMSSW_BASE"] + "/src/%s/test/%s" % (repo, mvaFileName2018), environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2016.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2017.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/TopTaggerCfg_2018.cfg" % repo, environ["CMSSW_BASE"] + "/src/TopTagger/TopTagger/test/libTopTagger.so", environ["CMSSW_BASE"] + "/src/%s/test/sampleSets.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/sampleCollections.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_2016.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_2017.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_2018pre.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_2018post.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_NonIsoMuon_2016.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_NonIsoMuon_2017.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_NonIsoMuon_2018pre.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepEventShape_NonIsoMuon_2018post.cfg" % repo, environ["CMSSW_BASE"] + "/src/%s/test/keras_frozen_2016.pb" % repo, environ["CMSSW_BASE"] + "/src/%s/test/keras_frozen_2017.pb" % repo, environ["CMSSW_BASE"] + "/src/%s/test/keras_frozen_2018pre.pb" % repo, environ["CMSSW_BASE"] + "/src/%s/test/keras_frozen_2018post.pb" % repo, environ["CMSSW_BASE"] + "/src/%s/test/allInOne_BTagEff.root" % repo, environ["CMSSW_BASE"] + "/src/%s/test/allInOne_SFMean.root" % repo, environ["CMSSW_BASE"] + "/src/%s/test/allInOne_leptonSF_2016.root" % repo, environ["CMSSW_BASE"] + "/src/%s/test/allInOne_leptonSF_2017.root" % repo, environ["CMSSW_BASE"] + "/src/%s/test/allInOne_leptonSF_2018.root" % repo, environ["CMSSW_BASE"] + "/src/%s/test/PileupHistograms_0121_69p2mb_pm4p6.root" % repo, environ["CMSSW_BASE"] + "/src/%s/test/pu_ratio.root" % repo, environ["CMSSW_BASE"] + "/src/%s/test/PileupHistograms_2018_69mb_pm5.root" % repo, environ["CMSSW_BASE"] + "/src/%s/test/CSVv2_Moriond17_B_H.csv" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepCSV_102XSF_WP_V1.csv" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepCSV_2016LegacySF_WP_V1.csv" % repo, environ["CMSSW_BASE"] + "/src/%s/test/DeepCSV_94XSF_WP_V4_B_F.csv" % repo, environ["CMSSW_BASE"] + "/src/%s/test/L1prefiring_jetpt_2017BtoF.root" % repo, ] print "--------------Files to Transfer-----------------" for i in filestoTransfer: print i print "------------------------------------------------" sc = SampleCollection("../sampleSets.cfg", "../sampleCollections.cfg") if options.dataCollections or options.dataCollectionslong: scl = sc.sampleCollectionList() for sampleCollection in scl: sl = sc.sampleList(sampleCollection) print sampleCollection if options.dataCollectionslong: sys.stdout.write("\t") for sample in sl: sys.stdout.write("%s " % sample[1]) print "" print "" exit(0) datasets = [] if options.datasets: datasets = options.datasets.split(',') else: print "No dataset specified" exit(0) fileParts = [] fileParts.append("Universe = vanilla\n") fileParts.append("Executable = run_Analyzer_condor.sh\n") fileParts.append( "Transfer_Input_Files = %s/%s.tar.gz, %s/exestuff.tar.gz\n" % (options.outPath, environ["CMSSW_VERSION"], options.outPath)) fileParts.append("Should_Transfer_Files = YES\n") fileParts.append("WhenToTransferOutput = ON_EXIT\n") fileParts.append("x509userproxy = $ENV(X509_USER_PROXY)\n\n") nFilesPerJob = options.numfile numberOfJobs = 0 for ds in datasets: ds = ds.strip() # create the directory if not os.path.isdir("%s/output-files/%s" % (options.outPath, ds)): os.makedirs("%s/output-files/%s" % (options.outPath, ds)) for s, n, e in sc.sampleList(ds): print "SampleSet:", n, ", nEvents:", e f = open(s) if not f == None: count = 0 for l in f: if '.root' in l: count = count + 1 for startFileNum in xrange(0, count, nFilesPerJob): numberOfJobs += 1 fileParts.append( "transfer_output_remaps = \"MyAnalysis_%s_%s.root = %s/output-files/%s/MyAnalysis_%s_%s.root\"\n" % (n, startFileNum, options.outPath, ds, n, startFileNum)) fileParts.append( "Arguments = %s %i %i %s %s %s\n" % (n, nFilesPerJob, startFileNum, s, options.analyze, environ["CMSSW_VERSION"])) fileParts.append( "Output = %s/log-files/MyAnalysis_%s_%i.stdout\n" % (options.outPath, n, startFileNum)) fileParts.append( "Error = %s/log-files/MyAnalysis_%s_%i.stderr\n" % (options.outPath, n, startFileNum)) fileParts.append( "Log = %s/log-files/MyAnalysis_%s_%i.log\n" % (options.outPath, n, startFileNum)) fileParts.append("Queue\n\n") f.close() fout = open("condor_submit.txt", "w") fout.write(''.join(fileParts)) fout.close() if not options.dataCollections and not options.dataCollectionslong: makeExeAndFriendsTarball(filestoTransfer, "exestuff", options.outPath) system( "tar --exclude-caches-all --exclude-vcs -zcf %s/${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION} --exclude=src --exclude=tmp" % options.outPath) if not options.noSubmit: system('mkdir -p %s/log-files' % options.outPath) system("echo 'condor_submit condor_submit.txt'") system('condor_submit condor_submit.txt') else: print "------------------------------------------" print "Number of Jobs:", numberOfJobs print "------------------------------------------"
system(tarallinputs) system("rm -r WORLDSWORSESOLUTIONTOAPROBLEM") if not options.dataCollections and not options.dataCollectionslong: system("tar --exclude-caches-all --exclude-vcs -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION} --exclude=src --exclude=tmp") exeName = "makeTrainingTuples" submitFile = submitFileGTP makeExeAndFriendsTarrball(filestoTransferGTP, "gtp") nFilesPerJob = options.numfile fileParts = [submitFile] sc = SampleCollection("../sampleSets.txt", "../sampleCollections.txt") datasets = [] if options.dataCollections or options.dataCollectionslong: scl = sc.sampleCollectionList() for sampleCollection in scl: sl = sc.sampleList(sampleCollection) print sampleCollection if options.dataCollectionslong: sys.stdout.write("\t") for sample in sl: sys.stdout.write("%s "%sample[1]) print "" print "" exit(0)
def makeExeAndFriendsTarrball(filestoTransfer, fname): #WORLDSWORSESOLUTIONTOAPROBLEM system("mkdir -p WORLDSWORSESOLUTIONTOAPROBLEM") for fn in filestoTransfer: system("cd WORLDSWORSESOLUTIONTOAPROBLEM; ln -s %s" % fn) print "Create tarball {0}.tag.gz".format(fname) tarallinputs = "tar czvf %s.tar.gz WORLDSWORSESOLUTIONTOAPROBLEM --dereference" % fname print tarallinputs system(tarallinputs) system("rm -r WORLDSWORSESOLUTIONTOAPROBLEM") submitFile = submitFileTT fileParts = [submitFile] sc = SampleCollection(options.sampleSetCfg, options.sampleCollectionsFile) ss = SampleSet(options.sampleSetCfg) datasets = [] now = datetime.datetime.now() dirName = "submission_%s" % now.strftime("%Y-%m-%d_%H-%M-%S") os.system("mkdir %s" % dirName) os.chdir(dirName) print "Create tarball ${CMSSW_VERSION}.tar.gz" system( "tar --exclude-caches-all --exclude-vcs -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION} --exclude=src --exclude=tmp" ) # makeExeAndFriendsTarball() is necessary now to apply WORLDSWORSESOLUTIONTOAPROBLEM
system("rm -r WORLDSWORSESOLUTIONTOAPROBLEM") if not options.dataCollections and not options.dataCollectionslong: system( "tar --exclude-caches-all --exclude-vcs -zcf ${CMSSW_VERSION}.tar.gz -C ${CMSSW_BASE}/.. ${CMSSW_VERSION} --exclude=src --exclude=tmp" ) exeName = "makeTrainingTuples" submitFile = submitFileGTP makeExeAndFriendsTarrball(filestoTransferGTP, "gtp") nFilesPerJob = options.numfile fileParts = [submitFile] sc = SampleCollection("../sampleSets_PostProcessed_2016.cfg", "../sampleCollections_2016.cfg") datasets = [] if options.dataCollections or options.dataCollectionslong: scl = sc.sampleCollectionList() for sampleCollection in scl: sl = sc.sampleList(sampleCollection) print sampleCollection if options.dataCollectionslong: sys.stdout.write("\t") for sample in sl: sys.stdout.write("%s " % sample[1]) print "" print "" exit(0)