def merge(paths,
script,
subjob_list,
subjobargs,
argparser=None,
istest=False,
groups_to_merge=None,
additionalArgs=None):
try:
if not istest and not checkShouldMerge(
script, argparser, additionalArgs=additionalArgs):
print "Nothing to merge"
return
except:
should_abort = raw_input(
"The log files you are asking for do not exist. Would you like to skip merging? (y/n) \n"
)
if should_abort in ['y', 'Y']: return
if not istest:
if argparser is None:
pass
else:
failed_jobs = checkCompletedJobs(script,
subjob_list,
argparser,
additionalArgs=additionalArgs)
print failed_jobs
if failed_jobs is not None and len(failed_jobs) != 0:
should_resubmit = raw_input(
"Would you like to resubmit the failed jobs? (y/n) \n")
if should_resubmit == 'y' or should_resubmit == 'Y':
print 'resubmitting:'
submitJobs(script,
subjobargs,
failed_jobs,
argparser,
resubmission=True)
else:
pass
if should_resubmit != 'skip': exit(0)
cleanJobFiles(argparser, script)
for f in paths:
if '.txt' in f or '.root' in f: continue
mergeSinglePath(f, groups_to_merge=groups_to_merge)
if not istest:
disableShouldMerge(script, argparser, additionalArgs=additionalArgs)
output_name += '/' + sample.name + '_events_' + subjobAppendix + '.root'
makeDirIfNeeded(output_name)
return output_name
#
# Code to process events if we are not making plots
#
if not args.makePlots:
#
# Submit subjobs
#
if not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
submitJobs(__file__, ('sample', 'subJob'),
jobs,
argParser,
jobLabel='tightToLoose')
exit(0)
#
# Load in sample and chain
#
sample = sample_manager.getSample(args.sample)
if not args.inData and 'Data' in sample.name: exit(0)
chain = sample.initTree(needhcount=False)
chain.HNLmass = sample.getMass()
chain.year = int(args.year)
#
# Initialize reweighter
#
sample_manager = SampleManager(args.year, 'noskim',
'compareTauIdList_' + str(args.year))
#
# Submit subjobs
#
if args.runOnCream and not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
jobs = []
for sample_name in sample_manager.sample_names:
sample = sample_manager.getSample(sample_name)
for njob in xrange(sample.split_jobs):
jobs += [(sample.name, str(njob))]
submitJobs(__file__, ('sample', 'subJob'),
jobs,
argParser,
jobLabel='plotTau')
exit(0)
#Start a loop over samples
sample_names = []
for sample in sample_manager.sample_list:
if sample.name not in sample_manager.sample_names: continue
if args.runOnCream and sample.name != args.sample: continue
if args.sample and sample.name != args.sample: continue
list_of_hist[sample.name] = {}
#
# Load in sample and chain
#
# Submit Jobs
# TODO: Rewrite so that it checks all ID's in a single event loop instead of using more resources than needed by having jobs for every algorithm
#
if not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
jobs = []
for sample_name in sample_manager.sample_names:
sample = sample_manager.getSample(sample_name)
for njob in xrange(sample.split_jobs):
jobs += [(sample.name, str(njob))]
submitJobs(__file__, ('sample', 'subJob'),
jobs,
argParser,
jobLabel='compareTauID')
exit(0)
#
#Initialize chain
#
sample = sample_manager.getSample(args.sample)
chain = sample.initTree(False)
chain.year = int(args.year)
isBkgr = not 'HNL' in sample.name
#
#Set output dir
#
from HNL.Tools.helpers import makeDirIfNeeded
log = getLogger('INFO')
args.isChild = True
if args.sample is None: args.sample = 'HNL-tau-m20'
if args.subJob is None: args.subJob = '0'
if not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
jobs = []
for sample_name in sample_manager.sample_names:
print sample_name
sample = sample_manager.getSample(sample_name)
for njob in xrange(sample.returnSplitJobs()):
jobs += [(sample.name, str(njob))]
submitJobs(__file__, ('sample', 'subJob'),
jobs,
argParser,
jobLabel='compression')
exit(0)
#Initialize chain
print 'Initializing chain'
sample = sample_manager.getSample(args.sample)
chain = sample.initTree()
chain.HNLmass = sample.getMass()
chain.year = int(args.year)
print 'Chain initialized'
#Set output dir
#Since number of subjobs was set to be 1 (HNL samples are small), this name was chosen since no overlap possible
#If this changes, this needs to be changed as well
from HNL.Tools.helpers import makeDirIfNeeded
sample = sample_manager.getSample(sample_name)
for njob in xrange(sample.returnSplitJobs()):
jobs += [(sample.name, str(njob))]
from HNL.EventSelection.eventCategorization import SUPER_CATEGORIES
if not args.merge:
#
# Submit subjobs
#
if not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
submitJobs(__file__, ('sample', 'subJob'),
jobs,
argParser,
jobLabel='prepareTMVA')
exit(0)
#
#Get specific sample for this subjob
#
sample = sample_manager.getSample(args.sample)
chain = sample.initTree(needhcount=False)
chain.year = int(args.year)
chain.is_signal = 'HNL' in sample.name
chain.selection = args.selection
chain.strategy = args.strategy
reweighter = Reweighter(sample, sample_manager)
from HNL.Samples.sampleManager import SampleManager
sample_manager = SampleManager(args.year, 'noskim', 'Triggers/triggerlist_'+str(args.year))
jobs = []
for sample_name in sample_manager.sample_names:
sample = sample_manager.getSample(sample_name)
for njob in xrange(sample.returnSplitJobs()):
jobs += [(sample.name, str(njob))]
#
# Submit subjobs
#
if not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel = 'trigger')
exit(0)
#
# Load in sample and chain
#
sample = sample_manager.getSample(args.sample)
chain = sample.initTree()
chain.year = int(args.year)
chain.is_signal = 'HNL' in sample.name
chain.HNLmass = sample.getMass()
#
# Does an event pass the triggers
#
# Submit subjobs
#
if not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
jobs = []
for sample in sample_list:
if args.sample and args.sample not in sample.name: continue
if not 'HNLtau' in sample.name: continue
if args.masses is not None and 'HNL' in sample.name and not any(
[str(m) in sample.name for m in args.masses]):
continue
for njob in xrange(sample.split_jobs):
jobs += [(sample.name, str(njob))]
submitJobs(__file__, ('sample', 'subJob'),
jobs,
argParser,
jobLabel='calcSignalEfficiency')
exit(0)
#
# Load in sample and chain
#
sample = getSampleFromList(sample_list, args.sample)
chain = sample.initTree(needhcount=False)
#
# Import and create cutter to provide cut flow
#
from HNL.EventSelection.cutter import Cutter
cutter = Cutter(chain=chain)
help='Choose the selection region',
choices=[
'baseline', 'highMassSR', 'lowMassSR',
'ZZCR', 'WZCR', 'ConversionCR'
])
args = argParser.parse_args()
if not args.isChild:
abort_script = raw_input("Did you check if all log files are ok? (y/n) \n")
if abort_script == 'n' or abort_script == 'N':
exit(0)
if args.batchSystem == 'HTCondor' and not args.isChild and not args.isTest and args.region is None:
jobs = [[0]]
submitJobs(__file__, ('subJob'), jobs, argParser, jobLabel='mergeSkim')
exit(0)
from HNL.Tools.helpers import fileSize, makeDirIfNeeded, isValidRootFile
def mergeSmallFile(merge_file):
path, name = merge_file.rsplit('/', 1)
if not args.isTest:
os.system('hadd -f -v ' + path + '/' + name.split('_', 1)[1] +
'.root ' + merge_file + '/*root')
os.system('rm -r -f ' + merge_file)
else:
makeDirIfNeeded(path + '/testArea/' + name.split('_', 1)[1] + '.root')
os.system('hadd -f -v ' + path + '/testArea/' + name.split('_', 1)[1] +
'.root ' + merge_file + '/*root')
efficiency[c][ptcuts] = Efficiency('efficiency_'+str(c)+'_l1_'+str(ptcuts[0])+'_l2_'+str(ptcuts[1])+'_l3_'+str(ptcuts[2]),
var_for_eff['HNLmass'][0], var_for_eff['HNLmass'][2], output_name, bins=bins_for_eff, subdirs=['efficiency_'+str(c),
'l1_'+str(ptcuts[0])+'_l2_'+str(ptcuts[1])+'_l3_'+str(ptcuts[2])])
return efficiency
from HNL.EventSelection.eventCategorization import CATEGORIES, SUPER_CATEGORIES, CATEGORY_NAMES
if not args.makePlots:
#
# Submit subjobs
#
if not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
flavors = ['tau', 'e', 'mu', '2l'] if args.flavor is None else [args.flavor]
for flavor in flavors:
submitJobs(__file__, ('sample', 'subJob'), jobs[flavor], argParser, jobLabel = 'calcSignalEfficiency', additionalArgs= [('flavor', flavor)])
exit(0)
if args.isChild and args.flavor is None:
raise RuntimeError("Flavor should be set at this point")
#
# Load in sample and chain
#
sample = sample_manager.getSample(args.sample)
chain = sample.initTree()
output_name = getOutputBase(args.flavor)
if args.isChild:
output_name += '/tmp_'+args.flavor
for sample_name in sample_manager.sample_names:
if args.sample and args.sample not in sample_name: continue
if args.signalOnly and not 'HNL' in sample_name: continue
if args.backgroundOnly and 'HNL' in sample_name: continue
if 'HNL' in sample_name and not 'HNL-' + args.flavor in sample_name:
continue
if not args.includeData and sample_name == 'Data': continue
sample = sample_manager.getSample(sample_name)
if args.masses is not None and 'HNL' in sample.name and not any(
[str(m) in sample.name for m in args.masses]):
continue
for njob in xrange(sample.split_jobs):
jobs += [(sample.name, str(njob))]
submitJobs(__file__, ('sample', 'subJob'),
jobs,
argParser,
jobLabel='calcYields')
exit(0)
#
# Load in sample and chain
#
if not args.includeData and args.sample == 'Data':
raise RuntimeError(
'Trying to run data while it is not allowed. Stopping the program')
sample = sample_manager.getSample(args.sample)
chain = sample.initTree(needhcount=False)
#
# Import and create cutter to provide cut flow
#
# maxdepth = ['4']
# boosttypes=['RealAdaBoost']
# shrinkage=['0.1']
out_file_name = lambda signal_name: os.path.expandvars(
os.path.join('$CMSSW_BASE', 'src', 'HNL', 'TMVA', 'data', 'training', args.
year, args.region + '-' + args.selection, signal_name,
signal_name + '.root'))
if not args.isChild and not args.plots:
if args.batchSystem != 'foreground':
jobs = []
for signal in ih.signal_names:
jobs += [(signal, '0')]
submitJobs(__file__, ('signalname', 'dummy'),
jobs,
argParser,
jobLabel='TMVAtrainer')
else:
args.isChild = True
submitArgs = getSubmitArgs(argParser, args)
for signal in ih.signal_names:
if args.signalname is not None and signal != args.signalname:
continue
command = 'python trainer.py'
for arg, value in submitArgs.iteritems():
if value == False:
continue
elif isinstance(value, list):
command += ' ' + '--' + arg + '=' + ' '.join(
[str(x) for x in sorted(value)])
else:
return ['total']
from HNL.BackgroundEstimation.fakerateArray import FakeRateCollection, SingleFlavorFakeRateCollection
from HNL.BackgroundEstimation.getTauFakeContributions import getWeights
#
# Code to process events
#
if not args.makePlots:
#
# Submit subjobs
#
if not args.isChild:
from HNL.Tools.jobSubmitter import submitJobs
submitJobs(__file__, ('sample', 'subJob'),
jobs,
argParser,
jobLabel='closureTest')
exit(0)
#
# Load in sample and chain
#
sample = sample_manager.getSample(args.sample)
chain = sample.initTree(needhcount=False)
#
# Import and create cutter to provide cut flow
#
from HNL.EventSelection.cutter import Cutter
cutter = Cutter(chain=chain)
for njob in xrange(sample.returnSplitJobs()):
jobs[year] += [(sample.name, str(njob), None)]
else:
for njob in xrange(sample.returnSplitJobs()):
for include_data in args.includeData:
jobs[year] += [(sample.name, str(njob), include_data)]
#
# Submit subjobs
#
if not args.isChild and not args.makePlots and not args.makeDataCards:
from HNL.Tools.jobSubmitter import submitJobs
for year in jobs.keys():
submitJobs(__file__, ('sample', 'subJob', 'includeData'),
jobs[year],
argParser,
jobLabel='plotVar',
additionalArgs=[('year', year)])
exit(0)
#
# Some constants to make referring to signal leptons more readable
#
l1 = 0
l2 = 1
l3 = 2
l4 = 3
nl = 3 if args.region != 'ZZCR' else 4
#
print sample_name, "not found. Will skip this sample"
continue
for njob in xrange(sample.returnSplitJobs()):
jobs += [(sample.name, str(njob))]
if not args.checkLogs:
from HNL.Tools.logger import getLogger, closeLogger
log = getLogger(args.logLevel)
#
# Submit subjobs
#
if not args.isChild and not args.isTest:
from HNL.Tools.jobSubmitter import submitJobs
print 'submitjobs'
submitJobs(__file__, ('sample', 'subJob'), jobs, argParser, jobLabel = 'skim')
print 'endsubmit'
if args.summaryFile:
gen_name = 'Gen' if args.genSkim else 'Reco'
f = open(os.path.expandvars(os.path.join('/user/$USER/public/ntuples/HNL', args.skimSelection, str(args.year), gen_name, 'summary.txt')), 'w')
for arg in vars(args):
if not getattr(args, arg): continue
f.write(arg + ' ' + str(getattr(args, arg)) + '\n')
f.close()
print "Submitted "+str(len(jobs))+" jobs to cream"
exit(0)
#
#Get specific sample for this subjob
