def runMT_pool(args=('', '', '')):
print "START %s" % (sh)
block, param, sh, analysisDir, MT = args
producePlots(param.selections[sh], block, param.colors, param.variables,
param.uncertainties, sh, param.intLumi, param.delphesVersion,
param.runFull, analysisDir, MT)
print "END %s" % (sh)
def main():
ops, args = options()
args = split_comma_args(args)
# give this analysis a name
analysisName = ops.analysis_name
# heppy analysis configuration
heppyCfg = ops.heppy_cfg
# process dictionary
processDict = ops.proc_file_json
# heppy trees location directory
treeDir = ops.heppy_tree_dir
# analysis dir
analysisDir = ops.analysis_output
# param file
paramFile = ops.param_file
sys.path.append(os.path.dirname(os.path.expanduser(paramFile)))
param = importlib.import_module(
os.path.splitext(ntpath.basename(paramFile))[0])
# tree location
treePath = 'heppy.{}.TreeProducer.TreeProducer_1/tree.root'.format(
analysisName)
# retrieve list of processes from heppy cfg
processes = []
with open(heppyCfg) as f:
lines = f.readlines()
for l in lines:
if 'splitFactor' in l:
processes.append(l.rsplit('.', 1)[0])
with open(processDict) as f:
procDict = json.load(f)
# prepare analysis dir
os.system('mkdir -p {}'.format(analysisDir))
### produce process dictionnaries
for sh in param.selections.keys():
block = collections.OrderedDict()
formBlock(processes, procDict, param.signal_groups,
param.background_groups, sh, treeDir, treePath, block)
### run analysis
producePlots(param.selections[sh], block, param.colors,
param.variables, param.uncertainties, sh, param.intLumi,
param.delphesVersion, param.runFull, analysisDir)
def runMT_pool(args=('', '', '', '', '', '')):
print "START %s" % (sh)
block, param, sh, analysisDir, MT, ops = args
producePlots(param.selections[sh],
block,
param.colors,
param.variables,
param.variables2D,
param.uncertainties,
sh,
param.intLumi,
param.delphesVersion,
param.runFull,
analysisDir,
MT,
latex_table=ops.latex_table,
no_plots=ops.no_plots,
nevents=ops.nevents)
print "END %s" % (sh)
def main():
ops, args = options()
args = split_comma_args(args)
# give this analysis a name
analysisName = ops.analysis_name
# heppy analysis configuration
heppyCfg = ops.heppy_cfg
# process dictionary
processDict = ops.proc_file_json
# heppy trees location directory
treeDir = ops.heppy_tree_dir
# analysis dir
analysisDir = ops.analysis_output
# param file
paramFile = ops.param_file
module_path = os.path.abspath(paramFile)
module_dir = os.path.dirname(module_path)
base_name = os.path.splitext(ntpath.basename(paramFile))[0]
sys.path.insert(0, module_dir)
param = importlib.import_module(base_name)
# tree location
treePath = '/heppy.FCChhAnalyses.{}.TreeProducer.TreeProducer_1/tree.root'.format(
analysisName)
#multi-threading
MT = ops.MT
#lsf
lsf = ops.lsf
chunks = ops.chunks
queue = ops.queue
sh = ops.sh
# retrieve list of processes from heppy cfg
processes = []
with open(heppyCfg) as f:
lines = f.readlines()
for l in lines:
if 'splitFactor' in l:
processes.append(l.rsplit('.', 1)[0])
#processes = [c.name for c in heppyCfg.selectedComponents]
with open(processDict) as f:
procDict = json.load(f)
# prepare analysis dir
os.system('mkdir -p {}'.format(analysisDir))
block = collections.OrderedDict()
formBlock(processes, procDict, param.signal_groups,
param.background_groups, sh, treeDir, treePath, block,
ops.nevents)
print sh
### run analysis
producePlots(param.selections[sh],
block,
param.colors,
param.variables,
param.variables2D,
param.uncertainties,
sh,
param.intLumi,
param.delphesVersion,
param.runFull,
analysisDir,
MT,
latex_table=ops.latex_table,
no_plots=ops.no_plots,
nevents=ops.nevents)
def main():
ops, args = options()
args = split_comma_args(args)
# give this analysis a name
analysisName = ops.analysis_name
# heppy analysis configuration
heppyCfg = ops.heppy_cfg
# process dictionary
processDict = ops.proc_file_json
# heppy trees location directory
treeDir = ops.heppy_tree_dir
# analysis dir
analysisDir = ops.analysis_output
# param file
paramFile = ops.param_file
module_path = os.path.abspath(paramFile)
module_dir = os.path.dirname(module_path)
base_name = os.path.splitext(ntpath.basename(paramFile))[0]
sys.path.insert(0, module_dir)
param = importlib.import_module(base_name)
# tree location
treePath = '/heppy.FCChhAnalyses.{}.TreeProducer.TreeProducer_1/tree.root'.format(
analysisName)
#multi-threading
MT = ops.MT
#lsf
lsf = ops.lsf
queue = ops.queue
# selection
sel = ops.sel
# check if output dir exists already
'''if ops.force and not ops.clean:
print 'removing {}'.format(analysisDir)
processCmd('rm -rf {}'.format(analysisDir))
os.makedirs(analysisDir)
elif os.path.exists(analysisDir) and not ops.clean and not ops.sel:
print ''
sys.exit('Output dir: "'+analysisDir+'" exists. To overwrite existing dir run with --force option')
'''
# retrieve list of processes from heppy cfg
processes = []
with open(heppyCfg) as f:
lines = f.readlines()
for l in lines:
if 'splitFactor' in l:
processes.append(l.rsplit('.', 1)[0])
#processes = [c.name for c in heppyCfg.selectedComponents]
with open(processDict) as f:
procDict = json.load(f)
# run on single signal hypothesis, also used in lsf btach submission
if sel:
block = collections.OrderedDict()
formBlock(processes, procDict, param.signal_groups,
param.background_groups, sel, treeDir, treePath, block,
ops.nevents)
producePlots(param, block, sel, ops)
elif not lsf:
if MT:
runMT(processes, procDict, param, treeDir, treePath, analysisDir,
MT, ops)
else:
for sh in param.selections.keys():
block = collections.OrderedDict()
formBlock(processes, procDict, param.signal_groups,
param.background_groups, sh, treeDir, treePath,
block, ops.nevents)
### run analysis
producePlots(param, block, sh, ops)
else:
runLSF(processes, procDict, param, treeDir, treePath, analysisDir, ops)
def runLSF(processes, procDict, param, treeDir, treePath, analysisDir, ops):
# clean if asked for it
if ops.clean:
print 'Cleaning LSF for {} jobs...'.format(ops.analysis_output)
processCmd('rm -rf BatchOutput/{} LSF*'.format(ops.analysis_output))
sys.exit('cleaned up everything.')
# first thing is to check whether previous submission (if any) succeeded
selection_list_4sub = []
nbad = 0
for sh in param.selections.keys():
selname = formatted(sh)
outseldir = 'sel_' + selname
name_batch_dir = 'BatchOutput/{}/{}'.format(ops.analysis_output,
outseldir)
rootfile = name_batch_dir + '/root_' + selname + '/histos.root'
if not os.path.isfile(rootfile) or not isValidROOTfile(
rootfile) or not getsize(rootfile):
selection_list_4sub.append(sh)
nbad += 1
# keep submitting until nbad = 0
if nbad > 0:
print ' '
print ' ========= Submitting {} jobs on {} queue ========='.format(
nbad, ops.queue)
jobCount = 0
for sh in selection_list_4sub:
block = collections.OrderedDict()
formBlock(processes, procDict, param.signal_groups,
param.background_groups, sh, treeDir, treePath, block,
ops.nevents)
selname = formatted(sh)
dummyscript = """
unset LD_LIBRARY_PATH
unset PYTHONHOME
unset PYTHONPATH
mkdir job
cd job
cp -r DUMMYHOMEDIR/init.sh .
cp -r DUMMYHOMEDIR/bin .
cp -r DUMMYHOMEDIR/templates .
source ./init.sh
python bin/analyze.py -n DUMMYANALYSISNAME -c DUMMYHEPPYCFG -t DUMMYTREELOCATION -p DUMMYTEMPLATEFILE -j DUMMYJSONFILE -o DUMMYOUTSELDIR --sel 'DUMMYSEL' -m --nev DUMMYNEVTS --no_plots
mkdir -p DUMMYHOMEDIR/BatchOutput/DUMMYOUTDIR/DUMMYOUTSELDIR
cp -r DUMMYOUTSELDIR DUMMYHOMEDIR/BatchOutput/DUMMYOUTDIR
"""
outseldir = 'sel_' + selname
# replace relevant parts in script and dump into file
dummyscript = dummyscript.replace('DUMMYHOMEDIR', os.getcwd())
dummyscript = dummyscript.replace('DUMMYANALYSISNAME',
ops.analysis_name)
dummyscript = dummyscript.replace('DUMMYHEPPYCFG',
os.path.abspath(ops.heppy_cfg))
dummyscript = dummyscript.replace(
'DUMMYTREELOCATION', os.path.abspath(ops.heppy_tree_dir))
dummyscript = dummyscript.replace('DUMMYTEMPLATEFILE',
os.path.abspath(ops.param_file))
dummyscript = dummyscript.replace(
'DUMMYJSONFILE', os.path.abspath(ops.proc_file_json))
dummyscript = dummyscript.replace('DUMMYOUTSELDIR', outseldir)
dummyscript = dummyscript.replace('DUMMYSEL', sh)
dummyscript = dummyscript.replace('DUMMYNEVTS', str(ops.nevents))
dummyscript = dummyscript.replace('DUMMYOUTDIR',
ops.analysis_output)
script = dummyscript
name_batch_dir = 'BatchOutput/{}/{}'.format(
ops.analysis_output, outseldir)
if not os.path.exists(name_batch_dir):
os.makedirs(name_batch_dir)
scriptdir = name_batch_dir + '/cfg/'
if not os.path.exists(scriptdir):
os.makedirs(scriptdir)
with open('script.sh', "w") as f:
f.write(script)
processCmd('chmod u+x script.sh')
processCmd('mv script.sh {}'.format(scriptdir))
script = scriptdir + 'script.sh'
print 'Submitting job ' + str(jobCount + 1) + ' out of ' + str(
len(selection_list_4sub))
cmd = 'bsub -o ' + name_batch_dir + '/std/STDOUT -e ' + name_batch_dir + '/std/STDERR -q ' + ops.queue
cmd += ' -J ' + outseldir + ' "' + os.path.abspath(script) + '" '
# submitting jobs
output = processCmd(cmd)
while ('error' in output):
time.sleep(1.0)
output = processCmd(cmd)
if ('error' not in output):
print 'Submitted after retry - job ' + str(jobCount + 1)
jobCount += 1
# no bad job is found, can thus collect output
else:
print '================================================================'
print 'Submission was successful, now collecting output ...'
print ''
# 1 root file per signal hypothesis, simply copy in properly name dir
# and run producePlots with runFull = false
for sh in param.selections.keys():
selname = formatted(sh)
outseldir = 'sel_' + selname
name_batch_dir = 'BatchOutput/{}/{}'.format(
ops.analysis_output, outseldir)
root_dir = '{}/root_{}'.format(name_batch_dir, selname)
cmd = 'cp -r {} {}'.format(root_dir, ops.analysis_output)
local_root_dir = '{}/root_{}'.format(ops.analysis_output, selname)
print local_root_dir
# collecting files
if not os.path.exists(local_root_dir):
processCmd(cmd)
# run analysis on histos
block = collections.OrderedDict()
formBlock(processes, procDict, param.signal_groups,
param.background_groups, sh, treeDir, treePath, block,
ops.nevents)
param.runFull = False
producePlots(param, block, sh, ops)
def runMT_join(block, param, sh, analysisDir, MT, ops):
print "START %s" % (sh)
producePlots(param, block, sh, ops)
print "END %s" % (sh)
def runCondor(processes, procDict, param, treeDir, treePath, analysisDir, ops):
# clean if asked for it
if ops.clean:
print('Cleaning LSF for {} jobs...'.format(ops.analysis_output))
processCmd('rm -rf BatchOutput/{} LSF*'.format(ops.analysis_output))
sys.exit('cleaned up everything.')
# first thing is to check whether previous submission (if any) succeeded
selection_list_4sub = []
nbad = 0
for sh in param.selections.keys():
selname = formatted(sh)
outseldir = 'sel_' + selname
name_batch_dir = 'BatchOutput/{}/{}'.format(ops.analysis_output,
outseldir)
rootfile = name_batch_dir + '/root_' + selname + '/histos.root'
if not os.path.isfile(rootfile) or not isValidROOTfile(
rootfile) or not getsize(rootfile):
selection_list_4sub.append(sh)
nbad += 1
# keep submitting until nbad = 0
if nbad > 0 and not ops.collect:
print(' ')
print(' ========= Submitting {} jobs on {} queue ========='.format(
nbad, ops.queue))
condor_filename = 'configs/condor_{}.sub'.format(ops.analysis_name)
cmdfile = """# here goes your shell script
executable = condorScript.sh
# here you specify where to put .log, .out and .err files
output = configs/std/condor.$(ClusterId).$(ProcId).out
error = configs/std/condor.$(ClusterId).$(ProcId).err
log = configs/std/condor.$(ClusterId).log
+AccountingGroup = "group_u_CMST3.all"
+JobFlavour = "{}"\n
""".format(ops.queue)
jobCount = 0
for sh in selection_list_4sub:
block = collections.OrderedDict()
formBlock(processes, procDict, param.signal_groups,
param.background_groups, sh, treeDir, treePath, block,
ops.nevents)
selname = formatted(sh)
outseldir = 'sel_' + selname
cmd = 'arguments="DUMMYHOMEDIR DUMMYANALYSISNAME DUMMYHEPPYCFG DUMMYTREELOCATION DUMMYTEMPLATEFILE DUMMYJSONFILE DUMMYOUTSELDIR DUMMYOUTDIR DUMMYSEL DUMMYNEVTS"\n'
print(outseldir)
# replace relevant parts in script and dump into file
cmd = cmd.replace('DUMMYHOMEDIR', os.getcwd())
cmd = cmd.replace('DUMMYANALYSISNAME', ops.analysis_name)
cmd = cmd.replace('DUMMYHEPPYCFG', os.path.abspath(ops.heppy_cfg))
cmd = cmd.replace('DUMMYTREELOCATION',
os.path.abspath(ops.heppy_tree_dir))
cmd = cmd.replace('DUMMYTEMPLATEFILE',
os.path.abspath(ops.param_file))
cmd = cmd.replace('DUMMYJSONFILE',
os.path.abspath(ops.proc_file_json))
cmd = cmd.replace('DUMMYOUTSELDIR', outseldir)
cmd = cmd.replace('DUMMYSEL', "'{}'".format(sh))
cmd = cmd.replace('DUMMYNEVTS', str(ops.nevents))
cmd = cmd.replace('DUMMYOUTDIR', ops.analysis_output)
#print cmd
cmdfile += cmd
cmdfile += 'queue\n'
with open(condor_filename, 'w') as dest_file:
dest_file.write(cmdfile)
print('-- launching --')
submit_cmd = "condor_submit {}".format(condor_filename)
print(condor_filename)
if (ops.DRYRUN):
print('Dry-run:')
#os.system('cat {}'.format(condor_filename))
else:
os.system(submit_cmd)
# no bad job is found, can thus collect output
else:
print(
'================================================================')
print('Submission was successful, now collecting output ...')
print('')
# 1 root file per signal hypothesis, simply copy in properly name dir
# and run producePlots with runFull = false
for sh in param.selections.keys():
selname = formatted(sh)
outseldir = 'sel_' + selname
name_batch_dir = 'BatchOutput/{}/{}'.format(
ops.analysis_output, outseldir)
root_dir = '{}/root_{}'.format(name_batch_dir, selname)
cmd = 'cp -r {} {}'.format(root_dir, ops.analysis_output)
local_root_dir = '{}/root_{}'.format(ops.analysis_output, selname)
print(local_root_dir)
# collecting files
if not os.path.exists(local_root_dir):
processCmd(cmd)
# run analysis on histos
block = collections.OrderedDict()
formBlock(processes, procDict, param.signal_groups,
param.background_groups, sh, treeDir, treePath, block,
ops.nevents)
producePlots(param, block, sh, ops)