def merge(paths, script, subjob_list, subjobargs, argparser=None, istest=False, groups_to_merge=None, additionalArgs=None): try: if not istest and not checkShouldMerge( script, argparser, additionalArgs=additionalArgs): print "Nothing to merge" return except: should_abort = raw_input( "The log files you are asking for do not exist. Would you like to skip merging? (y/n) \n" ) if should_abort in ['y', 'Y']: return if not istest: if argparser is None: pass else: failed_jobs = checkCompletedJobs(script, subjob_list, argparser, additionalArgs=additionalArgs) print failed_jobs if failed_jobs is not None and len(failed_jobs) != 0: should_resubmit = raw_input( "Would you like to resubmit the failed jobs? (y/n) \n") if should_resubmit == 'y' or should_resubmit == 'Y': print 'resubmitting:' submitJobs(script, subjobargs, failed_jobs, argparser, resubmission=True) else: pass if should_resubmit != 'skip': exit(0) cleanJobFiles(argparser, script) for f in paths: if '.txt' in f or '.root' in f: continue mergeSinglePath(f, groups_to_merge=groups_to_merge) if not istest: disableShouldMerge(script, argparser, additionalArgs=additionalArgs)
output_name += '/' + sample.name + '_events_' + subjobAppendix + '.root' makeDirIfNeeded(output_name) return output_name # # Code to process events if we are not making plots # if not args.makePlots: # # Submit subjobs # if not args.isChild: from HNL.Tools.jobSubmitter import submitJobs submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel='tightToLoose') exit(0) # # Load in sample and chain # sample = sample_manager.getSample(args.sample) if not args.inData and 'Data' in sample.name: exit(0) chain = sample.initTree(needhcount=False) chain.HNLmass = sample.getMass() chain.year = int(args.year) # # Initialize reweighter #
sample_manager = SampleManager(args.year, 'noskim', 'compareTauIdList_' + str(args.year)) # # Submit subjobs # if args.runOnCream and not args.isChild: from HNL.Tools.jobSubmitter import submitJobs jobs = [] for sample_name in sample_manager.sample_names: sample = sample_manager.getSample(sample_name) for njob in xrange(sample.split_jobs): jobs += [(sample.name, str(njob))] submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel='plotTau') exit(0) #Start a loop over samples sample_names = [] for sample in sample_manager.sample_list: if sample.name not in sample_manager.sample_names: continue if args.runOnCream and sample.name != args.sample: continue if args.sample and sample.name != args.sample: continue list_of_hist[sample.name] = {} # # Load in sample and chain
# # Submit Jobs # TODO: Rewrite so that it checks all ID's in a single event loop instead of using more resources than needed by having jobs for every algorithm # if not args.isChild: from HNL.Tools.jobSubmitter import submitJobs jobs = [] for sample_name in sample_manager.sample_names: sample = sample_manager.getSample(sample_name) for njob in xrange(sample.split_jobs): jobs += [(sample.name, str(njob))] submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel='compareTauID') exit(0) # #Initialize chain # sample = sample_manager.getSample(args.sample) chain = sample.initTree(False) chain.year = int(args.year) isBkgr = not 'HNL' in sample.name # #Set output dir # from HNL.Tools.helpers import makeDirIfNeeded
log = getLogger('INFO') args.isChild = True if args.sample is None: args.sample = 'HNL-tau-m20' if args.subJob is None: args.subJob = '0' if not args.isChild: from HNL.Tools.jobSubmitter import submitJobs jobs = [] for sample_name in sample_manager.sample_names: print sample_name sample = sample_manager.getSample(sample_name) for njob in xrange(sample.returnSplitJobs()): jobs += [(sample.name, str(njob))] submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel='compression') exit(0) #Initialize chain print 'Initializing chain' sample = sample_manager.getSample(args.sample) chain = sample.initTree() chain.HNLmass = sample.getMass() chain.year = int(args.year) print 'Chain initialized' #Set output dir #Since number of subjobs was set to be 1 (HNL samples are small), this name was chosen since no overlap possible #If this changes, this needs to be changed as well from HNL.Tools.helpers import makeDirIfNeeded
sample = sample_manager.getSample(sample_name) for njob in xrange(sample.returnSplitJobs()): jobs += [(sample.name, str(njob))] from HNL.EventSelection.eventCategorization import SUPER_CATEGORIES if not args.merge: # # Submit subjobs # if not args.isChild: from HNL.Tools.jobSubmitter import submitJobs submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel='prepareTMVA') exit(0) # #Get specific sample for this subjob # sample = sample_manager.getSample(args.sample) chain = sample.initTree(needhcount=False) chain.year = int(args.year) chain.is_signal = 'HNL' in sample.name chain.selection = args.selection chain.strategy = args.strategy reweighter = Reweighter(sample, sample_manager)
from HNL.Samples.sampleManager import SampleManager sample_manager = SampleManager(args.year, 'noskim', 'Triggers/triggerlist_'+str(args.year)) jobs = [] for sample_name in sample_manager.sample_names: sample = sample_manager.getSample(sample_name) for njob in xrange(sample.returnSplitJobs()): jobs += [(sample.name, str(njob))] # # Submit subjobs # if not args.isChild: from HNL.Tools.jobSubmitter import submitJobs submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel = 'trigger') exit(0) # # Load in sample and chain # sample = sample_manager.getSample(args.sample) chain = sample.initTree() chain.year = int(args.year) chain.is_signal = 'HNL' in sample.name chain.HNLmass = sample.getMass() # # Does an event pass the triggers #
# Submit subjobs # if not args.isChild: from HNL.Tools.jobSubmitter import submitJobs jobs = [] for sample in sample_list: if args.sample and args.sample not in sample.name: continue if not 'HNLtau' in sample.name: continue if args.masses is not None and 'HNL' in sample.name and not any( [str(m) in sample.name for m in args.masses]): continue for njob in xrange(sample.split_jobs): jobs += [(sample.name, str(njob))] submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel='calcSignalEfficiency') exit(0) # # Load in sample and chain # sample = getSampleFromList(sample_list, args.sample) chain = sample.initTree(needhcount=False) # # Import and create cutter to provide cut flow # from HNL.EventSelection.cutter import Cutter cutter = Cutter(chain=chain)
help='Choose the selection region', choices=[ 'baseline', 'highMassSR', 'lowMassSR', 'ZZCR', 'WZCR', 'ConversionCR' ]) args = argParser.parse_args() if not args.isChild: abort_script = raw_input("Did you check if all log files are ok? (y/n) \n") if abort_script == 'n' or abort_script == 'N': exit(0) if args.batchSystem == 'HTCondor' and not args.isChild and not args.isTest and args.region is None: jobs = [[0]] submitJobs(__file__, ('subJob'), jobs, argParser, jobLabel='mergeSkim') exit(0) from HNL.Tools.helpers import fileSize, makeDirIfNeeded, isValidRootFile def mergeSmallFile(merge_file): path, name = merge_file.rsplit('/', 1) if not args.isTest: os.system('hadd -f -v ' + path + '/' + name.split('_', 1)[1] + '.root ' + merge_file + '/*root') os.system('rm -r -f ' + merge_file) else: makeDirIfNeeded(path + '/testArea/' + name.split('_', 1)[1] + '.root') os.system('hadd -f -v ' + path + '/testArea/' + name.split('_', 1)[1] + '.root ' + merge_file + '/*root')
efficiency[c][ptcuts] = Efficiency('efficiency_'+str(c)+'_l1_'+str(ptcuts[0])+'_l2_'+str(ptcuts[1])+'_l3_'+str(ptcuts[2]), var_for_eff['HNLmass'][0], var_for_eff['HNLmass'][2], output_name, bins=bins_for_eff, subdirs=['efficiency_'+str(c), 'l1_'+str(ptcuts[0])+'_l2_'+str(ptcuts[1])+'_l3_'+str(ptcuts[2])]) return efficiency from HNL.EventSelection.eventCategorization import CATEGORIES, SUPER_CATEGORIES, CATEGORY_NAMES if not args.makePlots: # # Submit subjobs # if not args.isChild: from HNL.Tools.jobSubmitter import submitJobs flavors = ['tau', 'e', 'mu', '2l'] if args.flavor is None else [args.flavor] for flavor in flavors: submitJobs(__file__, ('sample', 'subJob'), jobs[flavor], argParser, jobLabel = 'calcSignalEfficiency', additionalArgs= [('flavor', flavor)]) exit(0) if args.isChild and args.flavor is None: raise RuntimeError("Flavor should be set at this point") # # Load in sample and chain # sample = sample_manager.getSample(args.sample) chain = sample.initTree() output_name = getOutputBase(args.flavor) if args.isChild: output_name += '/tmp_'+args.flavor
for sample_name in sample_manager.sample_names: if args.sample and args.sample not in sample_name: continue if args.signalOnly and not 'HNL' in sample_name: continue if args.backgroundOnly and 'HNL' in sample_name: continue if 'HNL' in sample_name and not 'HNL-' + args.flavor in sample_name: continue if not args.includeData and sample_name == 'Data': continue sample = sample_manager.getSample(sample_name) if args.masses is not None and 'HNL' in sample.name and not any( [str(m) in sample.name for m in args.masses]): continue for njob in xrange(sample.split_jobs): jobs += [(sample.name, str(njob))] submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel='calcYields') exit(0) # # Load in sample and chain # if not args.includeData and args.sample == 'Data': raise RuntimeError( 'Trying to run data while it is not allowed. Stopping the program') sample = sample_manager.getSample(args.sample) chain = sample.initTree(needhcount=False) # # Import and create cutter to provide cut flow #
# maxdepth = ['4'] # boosttypes=['RealAdaBoost'] # shrinkage=['0.1'] out_file_name = lambda signal_name: os.path.expandvars( os.path.join('$CMSSW_BASE', 'src', 'HNL', 'TMVA', 'data', 'training', args. year, args.region + '-' + args.selection, signal_name, signal_name + '.root')) if not args.isChild and not args.plots: if args.batchSystem != 'foreground': jobs = [] for signal in ih.signal_names: jobs += [(signal, '0')] submitJobs(__file__, ('signalname', 'dummy'), jobs, argParser, jobLabel='TMVAtrainer') else: args.isChild = True submitArgs = getSubmitArgs(argParser, args) for signal in ih.signal_names: if args.signalname is not None and signal != args.signalname: continue command = 'python trainer.py' for arg, value in submitArgs.iteritems(): if value == False: continue elif isinstance(value, list): command += ' ' + '--' + arg + '=' + ' '.join( [str(x) for x in sorted(value)]) else:
return ['total'] from HNL.BackgroundEstimation.fakerateArray import FakeRateCollection, SingleFlavorFakeRateCollection from HNL.BackgroundEstimation.getTauFakeContributions import getWeights # # Code to process events # if not args.makePlots: # # Submit subjobs # if not args.isChild: from HNL.Tools.jobSubmitter import submitJobs submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel='closureTest') exit(0) # # Load in sample and chain # sample = sample_manager.getSample(args.sample) chain = sample.initTree(needhcount=False) # # Import and create cutter to provide cut flow # from HNL.EventSelection.cutter import Cutter cutter = Cutter(chain=chain)
for njob in xrange(sample.returnSplitJobs()): jobs[year] += [(sample.name, str(njob), None)] else: for njob in xrange(sample.returnSplitJobs()): for include_data in args.includeData: jobs[year] += [(sample.name, str(njob), include_data)] # # Submit subjobs # if not args.isChild and not args.makePlots and not args.makeDataCards: from HNL.Tools.jobSubmitter import submitJobs for year in jobs.keys(): submitJobs(__file__, ('sample', 'subJob', 'includeData'), jobs[year], argParser, jobLabel='plotVar', additionalArgs=[('year', year)]) exit(0) # # Some constants to make referring to signal leptons more readable # l1 = 0 l2 = 1 l3 = 2 l4 = 3 nl = 3 if args.region != 'ZZCR' else 4 #
print sample_name, "not found. Will skip this sample" continue for njob in xrange(sample.returnSplitJobs()): jobs += [(sample.name, str(njob))] if not args.checkLogs: from HNL.Tools.logger import getLogger, closeLogger log = getLogger(args.logLevel) # # Submit subjobs # if not args.isChild and not args.isTest: from HNL.Tools.jobSubmitter import submitJobs print 'submitjobs' submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel = 'skim') print 'endsubmit' if args.summaryFile: gen_name = 'Gen' if args.genSkim else 'Reco' f = open(os.path.expandvars(os.path.join('/user/$USER/public/ntuples/HNL', args.skimSelection, str(args.year), gen_name, 'summary.txt')), 'w') for arg in vars(args): if not getattr(args, arg): continue f.write(arg + ' ' + str(getattr(args, arg)) + '\n') f.close() print "Submitted "+str(len(jobs))+" jobs to cream" exit(0) # #Get specific sample for this subjob