def test2():
dataset_names = [
# "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM",
"/Dummy_test_StopBabyMaker_v25/CMS4",
]
# Make a base directory
basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format(
os.getenv("USER"))
MutableFile(basedir).touch()
# Make a directory sample, giving it the location and a dataset name for bookkeeping purposes
# The globber must be customized (by default, it is *.root) in order to pick up the text files
ds = DirectorySample(location=basedir,
dataset="/TEST/Examplev1/TEST",
globber="*.txt")
# Make a CondorTask (3 in total, one for each input)
task = CondorTask(
sample=ds,
files_per_output=1,
tag="v0",
output_name="ttbar_powheg_pythia8_92X.root",
executable="condor_executable.sh",
cmssw_version="CMSSW_9_3_1",
scram_arch="slc6_amd64_gcc700",
arguments="testarg1",
tarfile="input.tar.gz",
condor_submit_params={"sites": "UAF,T2_US_UCSD,UCSB"},
no_load_from_backup=
True, # for the purpose of the example, don't use a backup
)
# do_cmd("rm -rf {0}".format(task.get_outputdir()))
# Process and sleep until complete
is_complete = False
for t in [5.0, 5.0, 10.0, 15.0, 20.0]:
task.process()
print("Sleeping for {0} seconds".format(int(t)))
time.sleep(t)
is_complete = task.complete()
if is_complete: break
# If it's complete, make a dummy sample out of the output directory
# in order to pick up the files. Then cat out the contents and sum
# them up. This should be 3*2*10 = 100
if is_complete:
print("Job completed! Checking outputs...")
outsamp = DirectorySample(location=task.get_outputdir(),
dataset="/Blah/blah/BLAH",
globber="*.txt")
tot = 0
for f in outsamp.get_files():
mf = MutableFile(f.get_name())
tot += int(mf.cat())
print("It looks like we found 3*2*10 = {0}".format(tot))
def test2():
dataset_names = [
# "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM",
"/Dummy_test_StopBabyMaker_v25/CMS4",
]
# Make a base directory
basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format(os.getenv("USER"))
MutableFile(basedir).touch()
# Make a directory sample, giving it the location and a dataset name for bookkeeping purposes
# The globber must be customized (by default, it is *.root) in order to pick up the text files
ds = DirectorySample(location=basedir, dataset="/TEST/Examplev1/TEST", globber="*.txt")
# Make a CondorTask (3 in total, one for each input)
task = CondorTask(
sample = ds,
files_per_output = 1,
tag = "v0",
output_name = "ttbar_powheg_pythia8_92X.root",
executable = "condor_executable.sh",
cmssw_version = "CMSSW_9_3_1",
scram_arch = "slc6_amd64_gcc700",
arguments = "testarg1",
tarfile = "input.tar.gz",
condor_submit_params = {"sites": "UAF,T2_US_UCSD,UCSB"},
no_load_from_backup = True, # for the purpose of the example, don't use a backup
)
# do_cmd("rm -rf {0}".format(task.get_outputdir()))
# Process and sleep until complete
is_complete = False
for t in [5.0, 5.0, 10.0, 15.0, 20.0]:
task.process()
print("Sleeping for {0} seconds".format(int(t)))
time.sleep(t)
is_complete = task.complete()
if is_complete: break
# If it's complete, make a dummy sample out of the output directory
# in order to pick up the files. Then cat out the contents and sum
# them up. This should be 3*2*10 = 100
if is_complete:
print("Job completed! Checking outputs...")
outsamp = DirectorySample(location=task.get_outputdir(), dataset="/Blah/blah/BLAH", globber="*.txt")
tot = 0
for f in outsamp.get_files():
mf = MutableFile(f.get_name())
tot += int(mf.cat())
print("It looks like we found 3*2*10 = {0}".format(tot))
def test_full(self):
"""
Touch a root file ("input")
Submit condor jobs to touch output files for each input file
and copy them to hadoop
Jobs get submitted to local universe for speed reasons
Check output to make sure job completed
"""
njobs = 2
cmssw = "CMSSW_8_0_21"
basedir = "/tmp/{0}/metis/condortask_testfull/".format(
os.getenv("USER"))
Utils.do_cmd("mkdir -p {0}".format(basedir))
tag = "vfull"
for i in range(1, njobs + 1):
Utils.do_cmd("touch {0}/input_{1}.root".format(basedir, i))
logging.getLogger("logger_metis").disabled = True
dummy = CondorTask(
sample=DirectorySample(
location=basedir,
globber="*.root",
dataset="/test/test/TEST",
),
open_dataset=False,
files_per_output=1,
cmssw_version=cmssw,
executable=Utils.metis_base() +
"metis/executables/condor_test_exe.sh",
tag=tag,
condor_submit_params={"universe": "local"},
no_load_from_backup=True,
)
# clean up previous directory
Utils.do_cmd("rm -rf {0}".format(dummy.get_outputdir()))
is_complete = False
for t in [1.0, 1.0, 2.0, 3.0, 5.0, 10.0, 20.0]:
dummy.process()
time.sleep(t)
is_complete = dummy.complete()
if is_complete: break
self.assertEquals(is_complete, True)
self.assertEqual(njobs, len(glob.glob(dummy.get_outputdir() + "/*")))
def test_completion_fraction(self):
# Make dummy task with no inputs
# and require min completion fraction to be 0
logging.getLogger("logger_metis").disabled = True
dummy = CondorTask(
sample=DirectorySample(
location=".",
globber="*.fake",
dataset="/testprocess/testprocess/TEST",
),
open_dataset=False,
files_per_output=1,
cmssw_version="CMSSW_8_0_20",
tag="vtest",
no_load_from_backup=True,
min_completion_fraction=0.,
)
dummy.process()
self.assertEqual(dummy.complete(), True)
condorpath), # your tarfile with assorted goodies here
special_dir="VVVAnalysis/{}/{}".format(
tag, args.year
), # output files into /hadoop/cms/store/<user>/<special_dir>
min_completion_fraction=0.50 if skip_tail else 1.0,
# max_jobs = 10,
)
# When babymaking task finishes, fire off a task that takes outputs and merges them locally (hadd)
# into a file that ends up on nfs (specified by `merged_dir` above)
merge_task = LocalMergeTask(
input_filenames=task.get_outputs(),
output_filename="{}/{}.root".format(merged_dir, shortname),
ignore_bad=skip_tail,
)
# Straightforward logic
if not task.complete():
task.process()
else:
if not merge_task.complete():
merge_task.process()
# Aggregate whether all tasks are complete
all_tasks_complete = all_tasks_complete and task.complete()
# Set task summary
task_summary[
task.get_sample().get_datasetname()] = task.get_task_summary()
# Parse the summary and make a summary.txt that will be used to pretty status of the jobs
StatsParser(data=task_summary,
webdir="~/public_html/VVVNanoLooperDashboard").do()
# check_expectedevents = True,
tag=tag,
cmssw_version="CMSSW_9_2_8",
scram_arch="slc6_amd64_gcc530",
no_load_from_backup=True,
)
maker_tasks.append(maker_task)
merge_tasks.append(merge_task)
for i in range(100):
total_summary = {}
for maker_task, merge_task in zip(maker_tasks, merge_tasks):
maker_task.process()
if maker_task.complete():
do_cmd("mkdir -p {}/merged".format(maker_task.get_outputdir()))
do_cmd("mkdir -p {}/skimmed".format(
maker_task.get_outputdir()))
merge_task.reset_io_mapping()
merge_task.update_mapping()
merge_task.process()
total_summary[maker_task.get_sample().get_datasetname(
)] = maker_task.get_task_summary()
total_summary[merge_task.get_sample().get_datasetname(
)] = merge_task.get_task_summary()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary,
webdir="~/public_html/dump/metis_stopbaby/").do()
def submit_metis(job_tag, samples_map, sample_list=[], arguments_map="", exec_script="metis.sh", tar_files=[], hadoop_dirname="testjobs", files_per_output=1, globber="*.root", sites="T2_US_UCSD"):
import time
import json
import metis
from time import sleep
from metis.Sample import DirectorySample
from metis.CondorTask import CondorTask
from metis.StatsParser import StatsParser
import os
import glob
import subprocess
# file/dir paths
main_dir = os.getcwd()
metis_path = os.path.dirname(os.path.dirname(metis.__file__))
tar_path = os.path.join(metis_path, "package.tar")
tar_gz_path = tar_path + ".gz"
metis_dashboard_path = os.path.join(metis_path, "dashboard")
exec_path = os.path.join(main_dir, exec_script)
hadoop_path = "metis/{}/{}".format(hadoop_dirname, job_tag) # The output goes to /hadoop/cms/store/user/$USER/"hadoop_path"
# Create tarball
os.chdir(main_dir)
print os.getcwd()
print "tar -chzf {} {}".format(tar_gz_path, " ".join(tar_files))
os.system("tar -chzf {} {}".format(tar_gz_path, " ".join(tar_files)))
# Change directory to metis
os.chdir(metis_path)
total_summary = {}
# if no sample_list is provided then we form it via the keys of the samples_map
if len(sample_list) == 0:
for key in samples_map:
sample_list.append(key)
samples_to_run = []
for key in sample_list:
samples_to_run.append(
DirectorySample(
dataset=key,
location=samples_map[key],
globber=globber,
)
)
files_per_output_config_list = []
if isinstance(files_per_output, dict):
for key in sample_list:
files_per_output_config_list.append(files_per_output[key])
else:
for key in sample_list:
files_per_output_config_list.append(files_per_output)
# Loop over datasets to submit
while True:
all_tasks_complete = True
#for sample in sorted(samples):
for index, sample in enumerate(samples_to_run):
# define the task
maker_task = CondorTask(
sample = sample,
tag = job_tag,
arguments = arguments_map[sample.get_datasetname()] if arguments_map else "",
executable = exec_path,
tarfile = tar_gz_path,
special_dir = hadoop_path,
output_name = "output.root",
files_per_output = files_per_output_config_list[index],
condor_submit_params = {"sites" : sites},
open_dataset = False,
flush = True,
#no_load_from_backup = True,
)
# process the job (either submits, checks for resubmit, or finishes etc.)
maker_task.process()
# save some information for the dashboard
total_summary["["+job_tag+"] "+maker_task.get_sample().get_datasetname()] = maker_task.get_task_summary()
# Aggregate whether all tasks are complete
all_tasks_complete = all_tasks_complete and maker_task.complete()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary, webdir=metis_dashboard_path).do()
# Print msummary table so I don't have to load up website
os.system("msummary -r -p {} | tee summary.txt".format(job_tag))
os.system("chmod -R 755 {}".format(metis_dashboard_path))
os.system("chmod 644 {}/images/*".format(metis_dashboard_path))
# If all done exit the loop
if all_tasks_complete:
print ""
print "Job={} finished".format(job_tag)
print ""
break
# Neat trick to not exit the script for force updating
print 'Press Ctrl-C to force update, otherwise will sleep for 300 seconds'
try:
for i in range(0,60):
sleep(1) # could use a backward counter to be preeety :)
except KeyboardInterrupt:
raw_input("Press Enter to force update, or Ctrl-C to quit.")
print "Force updating..."
os.chdir(main_dir)
scram_arch="slc6_amd64_gcc700",
min_completion_fraction=1.00,
)
tasks.append(task)
if submit:
for i in range(100):
total_summary = {}
fracs = []
for task in tasks:
#for maker_task in maker_tasks:
task.process()
frac = task.complete(return_fraction=True)
total_summary[
task.get_sample().get_datasetname()] = task.get_task_summary()
fracs.append(frac)
# parse the total summary and write out the dashboard
StatsParser(data=total_summary,
webdir="~/public_html/dump/metis_%s/" % model).do()
print "%s/%s jobs are completed." % (sum(fracs), len(fracs))
if sum(fracs) / len(fracs) == 1:
print "Done."
break
files_per_output=6,
output_name="merged.root",
tag=job_tag,
#cmssw_version = "CMSSW_9_2_1", # doesn't do anything
arguments=args,
executable=exec_path,
tarfile=tar_path,
special_dir=hadoop_path,
#condor_submit_params = {"universe" : "local"}
condor_submit_params={"sites": "UAF"})
task.process()
# save some information for the dashboard
total_summary["WWW_v0_1_%s_%s" %
(baby_version, job_tag)] = task.get_task_summary()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary,
webdir="~/public_html/tasutil/Metis_WWW/").do()
if task.complete():
print ""
print "Job=%s finished" % job_tag
print ""
break
time.sleep(2 * 60)
#eof
total_summary = {}
if not os.path.isdir(maker_task.get_outputdir()):
os.makedirs(maker_task.get_outputdir())
if not os.path.isdir(maker_task.get_outputdir() + '/merged'):
os.makedirs(maker_task.get_outputdir() + '/merged')
os.makedirs(maker_task.get_outputdir() + '/skimmed')
#do_cmd("mkdir -p {}".format(maker_task.get_outputdir()))
#do_cmd("mkdir -p {}/merged".format(maker_task.get_outputdir()))
#do_cmd("mkdir -p {}/skimmed".format(maker_task.get_outputdir()))
for maker_task, merge_task in zip(maker_tasks, merge_tasks):
#for maker_task in maker_tasks:
maker_task.process()
frac = maker_task.complete(return_fraction=True)
if frac >= maker_task.min_completion_fraction:
# if maker_task.complete():
# do_cmd("mkdir -p {}/merged".format(maker_task.get_outputdir()))
# do_cmd("mkdir -p {}/skimmed".format(maker_task.get_outputdir()))
merge_task.reset_io_mapping()
merge_task.update_mapping()
merge_task.process()
total_summary[maker_task.get_sample().get_datasetname(
)] = maker_task.get_task_summary()
total_summary[merge_task.get_sample().get_datasetname(
)] = merge_task.get_task_summary()
print(frac)
def get_tasks(samples_dictionary, year, baby_type, baby_version_tag, dotestrun=False, files_per_output_func=UNITY):
job_tag = "{}{}_{}".format(baby_type, year, baby_version_tag)
# file/dir paths
main_dir = os.path.dirname(os.path.abspath(__file__))
metis_path = os.path.dirname(os.path.dirname(metis.__file__))
tar_path = os.path.join(metis_path, "package_{}.tar".format(job_tag))
tar_gz_path = tar_path + ".gz"
exec_path = os.path.join(main_dir, "metis.sh")
merge_exec_path = os.path.join(main_dir, "merge.sh")
hadoop_path = "metis/{}/{}".format(hadoop_dirname, job_tag) # The output goes to /hadoop/cms/store/user/$USER/"hadoop_path"
# Extra arguments that will be passed on to ./processBaby to specific which baby to create from the babymaker binary executable
args = ""
if job_tag.find("WVZ") != -1:
args = "1"
if job_tag.find("Dilep") != -1:
args = "2"
if job_tag.find("Trilep") != -1:
args = "3"
if job_tag.find("WVZMVA") != -1:
args = "4"
if job_tag.find("Truth") != -1:
args = "5"
if job_tag.find("WVZAll") != -1:
args = "6"
# Change directory to metis
os.chdir(metis_path)
tasks = []
# BEGIN Sample Loop -------->
# loop over the samples
for sample in sorted(samples_dictionary.iterkeys()):
#
# Job 1 : Creating baby
#
# define the task
maker_task = CondorTask(
sample = SNTSample(dataset=sample,
# exclude_tag_pattern="CMS4_V08-*", # ignore new samples by sicheng for 2016
exclude_tag_pattern="*v516*", # ignore new samples by sicheng for 2016
),
tag = job_tag,
arguments = args,
executable = exec_path,
tarfile = tar_gz_path,
special_dir = hadoop_path,
output_name = "output.root",
files_per_output = files_per_output_func(sample),
# files_per_output = 1,
condor_submit_params = {"sites" : "T2_US_UCSD"},
# condor_submit_params = {"sites" : "UAF,T2_US_Wisconsin,T2_US_Florida,T2_US_Nebraska,T2_US_Caltech,T2_US_MIT,T2_US_Purdue"},
# condor_submit_params = {"sites" : "UAF,T2_US_Wisconsin,T2_US_Florida,T2_US_Nebraska,T2_US_Caltech,T2_US_MIT"},
# condor_submit_params = {"sites" : "UAF"},
open_dataset = False,
flush = True,
max_jobs = 5 if dotestrun else 0,
# min_completion_fraction = 1.0 if "Run201" in sample else 0.9,
# min_completion_fraction = 0.9,
#no_load_from_backup = True,
)
print sample, job_tag
tasks.append(maker_task)
#
# Job 2 : Merging baby outputs
#
if maker_task.complete() and not dotestrun:
merge_sample_name = "/MERGE_"+sample[1:]
#merge_task = CondorTask(
# sample = DirectorySample(dataset=merge_sample_name, location=maker_task.get_outputdir()),
# # open_dataset = True, flush = True,
# executable = merge_exec_path,
# tarfile = tar_gz_path,
# files_per_output = 1,
# output_dir = maker_task.get_outputdir() + "/merged",
# output_name = samples_dictionary[sample] + ".root",
# condor_submit_params = {"sites":"T2_US_UCSD"},
# output_is_tree = True,
# # check_expectedevents = True,
# tag = job_tag,
# cmssw_version = "CMSSW_9_2_0",
# scram_arch = "slc6_amd64_gcc530",
# #no_load_from_backup = True,
# max_jobs = 1,
# )
# merge_task.reset_io_mapping()
# merge_task.update_mapping()
# tasks.append(merge_task)
# merge_task = LocalMergeTask(
# input_filenames=maker_task.get_outputs(),
# output_filename="{}/{}.root".format(maker_task.get_outputdir() + "/merged", samples_dictionary[sample]),
# ignore_bad = False,
# )
input_arg = maker_task.get_outputs()[0].name.replace("_1.root", "")
hadd_command = "sh ../rooutil/addHistos.sh /tmp/{}_1 {}".format(samples_dictionary[sample], input_arg)
hadoop_output = "{}/{}_1.root".format(maker_task.get_outputdir() + "/merged", samples_dictionary[sample])
cp_command = "cp /tmp/{}_1.root {}".format(samples_dictionary[sample], hadoop_output)
print ""
print ""
print ""
print ""
print ""
print hadoop_output
if not os.path.exists(hadoop_output):
print hadd_command
# os.system(hadd_command)
print cp_command
# os.system(cp_command)
print ""
print ""
print ""
print ""
# if not merge_task.complete():
# merge_task.process()
# <------ END Sample Loop
return tasks
output_name="test_rawreco.root",
tag=job_tag,
cmssw_version=cmssw_ver,
executable=exec_path,
tarfile="./package.tar.gz",
condor_submit_params={
"sites":
"T2_US_UCSD",
"container":
"/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel7-m20201010"
},
#condor_submit_params = {"sites" : "T2_US_UCSD,T2_US_CALTECH,T2_US_MIT,T2_US_WISCONSIN,T2_US_Nebraska,T2_US_Purdue,T2_US_Vanderbilt,T2_US_Florida"},
special_dir=hadoop_path,
arguments=args.replace(" ", "|"))
task.process()
allcomplete = allcomplete and task.complete()
# save some information for the dashboard
total_summary[ds] = task.get_task_summary()
with open("summary.json", "w") as f_out:
json.dump(total_summary, f_out, indent=4, sort_keys=True)
# Loop through local samples
#for ds,loc,fpo,args in local_sets[:]:
# sample = DirectorySample( dataset = ds, location = loc )
# files = [f.name for f in sample.get_files()]
# print "For sample %s in directory %s, there are %d input files" % (ds, loc, len(files))
# #for file in files:
# # print file
# task = CondorTask(
# sample = sample,
)
# print(merge_task.get_sample().get_datasetname())
# print(merge_task.get_sample().info["location"])
maker_tasks.append(child_tasks)
merge_tasks.append(merge_task)
for i in range(100):
total_summary = {}
for child_tasks, merge_task in zip(maker_tasks,merge_tasks):
all_child_finishes = True
for maker_task in child_tasks:
maker_task.process()
if not maker_task.complete():
all_child_finishes = False
if all_child_finishes:
do_cmd("mkdir -p {}/merged".format(maker_task.get_outputdir()))
merge_task.reset_io_mapping()
merge_task.update_mapping()
merge_task.process()
if merge_task.complete():
outfile = '{}/{}_1.root'.format(merge_task.get_outputdir(), merge_task.output_name.split('.')[0])
target = outfile.split('resub')[0]
tmp = merge_task.get_sample().get_datasetname().split('_')
target += 'stopBaby_{}/{}'.format('_'.join(tmp[1:-2]), merge_task.output_name)
# print('cp {} {}'.format(outfile, target))
print('cp {} {}'.format(outfile, target))
def get_tasks(samples_dictionary, year, baby_type, baby_version_tag, dotestrun=False):
job_tag = "{}{}_{}".format(baby_type, year, baby_version_tag)
# file/dir paths
main_dir = os.path.dirname(os.path.abspath(__file__))
metis_path = os.path.dirname(os.path.dirname(metis.__file__))
tar_path = os.path.join(metis_path, "package.tar")
tar_gz_path = tar_path + ".gz"
exec_path = os.path.join(main_dir, "metis.sh")
merge_exec_path = os.path.join(main_dir, "merge.sh")
hadoop_path = "metis/wwwbaby/{}".format(job_tag) # The output goes to /hadoop/cms/store/user/$USER/"hadoop_path"
# Extra arguments that will be passed on to ./processBaby to specific which baby to create from the babymaker binary executable
if job_tag.find("WWW") != -1:
args = "0" # WWWBaby
elif job_tag.find("FR") != -1:
args = "1" # FRBaby
elif job_tag.find("OS") != -1:
args = "2" # OSBaby
elif job_tag.find("TnP") != -1:
args = "3" # TnPBaby
elif job_tag.find("All") != -1:
args = "4" # AllBaby
elif job_tag.find("POG") != -1:
args = "5" # POGBaby
elif job_tag.find("Loose") != -1:
args = "6" # LooseBaby
# Change directory to metis
os.chdir(metis_path)
tasks = []
# BEGIN Sample Loop -------->
# loop over the samples
for sample in sorted(samples_dictionary.iterkeys()):
#
# Job 1 : Creating baby
#
# define the task
maker_task = CondorTask(
sample = SNTSample(dataset=sample,
exclude_tag_pattern="CMS4_V08-*", # ignore new samples by sicheng for 2016
),
tag = job_tag,
arguments = args,
executable = exec_path,
tarfile = tar_gz_path,
special_dir = hadoop_path,
output_name = "output.root",
files_per_output = 4,
condor_submit_params = {"sites" : "T2_US_UCSD"},
open_dataset = False,
flush = True,
max_jobs = 5 if dotestrun else 0
#no_load_from_backup = True,
)
print sample, job_tag
tasks.append(maker_task)
#
# Job 2 : Merging baby outputs
#
if maker_task.complete() and not dotestrun:
merge_sample_name = "/MERGE_"+sample[1:]
merge_task = CondorTask(
sample = DirectorySample(dataset=merge_sample_name, location=maker_task.get_outputdir()),
# open_dataset = True, flush = True,
executable = merge_exec_path,
tarfile = tar_gz_path,
files_per_output = 100000,
output_dir = maker_task.get_outputdir() + "/merged",
output_name = samples_dictionary[sample] + ".root",
condor_submit_params = {"sites":"T2_US_UCSD"},
output_is_tree = True,
# check_expectedevents = True,
tag = job_tag,
cmssw_version = "CMSSW_9_2_0",
scram_arch = "slc6_amd64_gcc530",
#no_load_from_backup = True,
)
merge_task.reset_io_mapping()
merge_task.update_mapping()
tasks.append(merge_task)
# <------ END Sample Loop
return tasks
def main():
# file/dir paths
main_dir = os.path.dirname(os.path.abspath(__file__))
metis_path = os.path.dirname(os.path.dirname(metis.__file__))
tar_path = os.path.join(metis_path, "package.tar")
tar_gz_path = tar_path + ".gz"
metis_dashboard_path = os.path.join(metis_path, "dashboard")
exec_path = os.path.join(main_dir, "metis.sh")
hadoop_path = "metis/baby/{}".format(
job_tag
) # The output goes to /hadoop/cms/store/user/$USER/"hadoop_path"
# Create tarball
os.chdir(main_dir)
os.system("tar -chzf {} a.out setup.sh *.xml".format(tar_gz_path))
# Change directory to metis
os.chdir(metis_path)
total_summary = {}
samples_to_run = [
DirectorySample(
dataset=
"/GluGluHToWWToLNuQQ_M125_13TeV_powheg_JHUGenV628_pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber="GluGluHToWWToLNuQQ_M125_13TeV_powheg_JHUGenV628*.root",
),
DirectorySample(
dataset=
"/VBFHToWWToLNuQQ_M125_13TeV_powheg_JHUGenV628_pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber="VBFHToWWToLNuQQ_M125_13TeV_powheg_JHUGenV628*.root",
),
DirectorySample(
dataset=
"/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber="TT_TuneCUETP8M2T4_13TeV-powheg*.root",
),
DirectorySample(
dataset=
"/WJetsToLNu_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber="WJetsToLNu_TuneCUETP8M1_13TeV-madgraphMLM*.root",
),
DirectorySample(
dataset=
"/WJetsToLNu_HT-400To600_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6_ext1-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber=
"WJetsToLNu_HT-400To600_TuneCUETP8M1_13TeV-madgraphMLM_ext1*.root",
),
DirectorySample(
dataset=
"/WJetsToLNu_HT-600To800_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6_ext1-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber=
"WJetsToLNu_HT-600To800_TuneCUETP8M1_13TeV-madgraphMLM_ext1*.root",
),
DirectorySample(
dataset=
"/WJetsToLNu_HT-800To1200_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6_ext1-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber=
"WJetsToLNu_HT-800To1200_TuneCUETP8M1_13TeV-madgraphMLM_ext1*.root",
),
DirectorySample(
dataset=
"/WJetsToLNu_HT-1200To2500_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber=
"WJetsToLNu_HT-1200To2500_TuneCUETP8M1_13TeV-madgraphMLM*.root",
),
DirectorySample(
dataset=
"/WJetsToLNu_HT-2500ToInf_TuneCUETP8M1_13TeV-madgraphMLM-pythia8/RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6_ext1-v1/MINIAODSIM",
location=
"/hadoop/cms/store/user/phchang/metis/baby/HWW2016_v1.15.1/merged4",
globber=
"WJetsToLNu_HT-2500ToInf_TuneCUETP8M1_13TeV-madgraphMLM_ext1*.root",
),
]
# Loop over datasets to submit
while True:
all_tasks_complete = True
#for sample in sorted(samples):
for sample in samples_to_run:
# define the task
maker_task = CondorTask(
sample=sample,
tag=job_tag,
arguments="",
executable=exec_path,
tarfile=tar_gz_path,
special_dir=hadoop_path,
output_name="output.root",
files_per_output=1,
condor_submit_params={"sites": "T2_US_UCSD"},
open_dataset=False,
flush=True,
#no_load_from_backup = True,
)
# process the job (either submits, checks for resubmit, or finishes etc.)
maker_task.process()
# save some information for the dashboard
total_summary[maker_task.get_sample().get_datasetname(
)] = maker_task.get_task_summary()
# Aggregate whether all tasks are complete
all_tasks_complete = all_tasks_complete and maker_task.complete()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary, webdir=metis_dashboard_path).do()
# Print msummary table so I don't have to load up website
os.system("msummary -r | tee summary.txt")
os.system("chmod -R 755 {}".format(metis_dashboard_path))
# If all done exit the loop
if all_tasks_complete:
print ""
print "Job={} finished".format(job_tag)
print ""
break
# Neat trick to not exit the script for force updating
print 'Press Ctrl-C to force update, otherwise will sleep for 300 seconds'
try:
for i in range(0, 300):
sleep(1) # could use a backward counter to be preeety :)
except KeyboardInterrupt:
raw_input("Press Enter to force update, or Ctrl-C to quit.")
print "Force updating..."
tag="v0",
output_name="output.txt",
executable=exefile.get_name(),
condor_submit_params={"sites": "UAF,T2_US_UCSD,UCSB"},
no_load_from_backup=
True, # for the purpose of the example, don't use a backup
)
do_cmd("rm -rf {0}".format(task.get_outputdir()))
# Process and sleep until complete
is_complete = False
for t in [5.0, 5.0, 10.0, 15.0, 20.0]:
task.process()
print("Sleeping for {0} seconds".format(int(t)))
time.sleep(t)
is_complete = task.complete()
if is_complete: break
# If it's complete, make a dummy sample out of the output directory
# in order to pick up the files. Then cat out the contents and sum
# them up. This should be 3*2*10 = 100
if is_complete:
print("Job completed! Checking outputs...")
outsamp = DirectorySample(location=task.get_outputdir(),
dataset="/Blah/blah/BLAH",
globber="*.txt")
tot = 0
for f in outsamp.get_files():
mf = MutableFile(f.get_name())
tot += int(mf.cat())
print("It looks like we found 3*2*10 = {0}".format(tot))
def main():
if data_year == "2016":
samples_to_run = dataset.hww_samples_to_run_2016
samples_short_name = dataset.samples_short_name_2016
dslocs = dataset.dslocscms4_2016_allpf
# file/dir paths
main_dir = os.path.dirname(os.path.abspath(__file__))
metis_path = os.path.dirname(os.path.dirname(metis.__file__))
tar_path = os.path.join(metis_path, "package.tar")
tar_gz_path = tar_path + ".gz"
metis_dashboard_path = os.path.join(metis_path, "dashboard")
exec_path = os.path.join(main_dir, "metis.sh")
merge_exec_path = os.path.join(main_dir, "merge.sh")
hadoop_path = "metis/baby/{}".format(
job_tag
) # The output goes to /hadoop/cms/store/user/$USER/"hadoop_path"
if job_tag.find("HWW") != -1:
args = "0" # HWWBaby
# Create tarball
os.chdir(main_dir)
os.system(
"tar -chzf {} localsetup.sh processBaby *.so *.pcm rooutil/lib*.so coreutil/data coreutil/lib*.so *.txt btagsf MVAinput jetCorrections leptonSFs puWeight2016.root pileup_jul21_nominalUpDown.root ../CORE/Tools/ mergeHadoopFiles.C rooutil/hadd.py fastjet/fastjet-install/lib"
.format(tar_gz_path))
# Change directory to metis
os.chdir(metis_path)
total_summary = {}
# Loop over datasets to submit
while True:
all_tasks_complete = True
for sample in samples_to_run:
loc = dslocs[sample]
# define the task
maker_sample_name = "/MAKER_" + sample[1:]
maker_task = CondorTask(
sample=DirectorySample(dataset=maker_sample_name,
location=loc),
tag=job_tag,
arguments=args,
executable=exec_path,
tarfile=tar_gz_path,
special_dir=hadoop_path,
output_name="output.root",
files_per_output=1,
condor_submit_params={"sites": "T2_US_UCSD"},
open_dataset=False,
flush=True,
#min_completion_fraction = 0.5,
#no_load_from_backup = True,
)
# process the job (either submits, checks for resubmit, or finishes etc.)
maker_task.process()
# save some information for the dashboard
total_summary[maker_task.get_sample().get_datasetname(
)] = maker_task.get_task_summary()
# define the task
merge_sample_name = "/MERGE_" + sample[1:]
merge_task = CondorTask(
sample=DirectorySample(dataset=merge_sample_name,
location=maker_task.get_outputdir()),
tag=job_tag,
executable=merge_exec_path,
tarfile=tar_gz_path,
files_per_output=100000,
output_dir=maker_task.get_outputdir() + "/merged",
output_name=samples_short_name[sample] + ".root",
condor_submit_params={"sites": "T2_US_UCSD"},
open_dataset=False,
flush=True,
output_is_tree=True,
cmssw_version="CMSSW_9_2_0",
scram_arch="slc6_amd64_gcc530",
#no_load_from_backup = True,
)
if maker_task.complete():
# process the job (either submits, checks for resubmit, or finishes etc.)
merge_task.process()
# save some information for the dashboard
total_summary[merge_task.get_sample().get_datasetname(
)] = merge_task.get_task_summary()
# Aggregate whether all tasks are complete
all_tasks_complete = all_tasks_complete and maker_task.complete()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary, webdir=metis_dashboard_path).do()
# Print msummary table so I don't have to load up website
os.system("msummary -r | tee summary.txt")
os.system("chmod -R 755 {}".format(metis_dashboard_path))
# If all done exit the loop
if all_tasks_complete:
print ""
print "Job={} finished".format(job_tag)
print ""
break
# Neat trick to not exit the script for force updating
print 'Press Ctrl-C to force update, otherwise will sleep for 300 seconds'
try:
for i in range(0, 300):
sleep(1) # could use a backward counter to be preeety :)
except KeyboardInterrupt:
raw_input("Press Enter to force update, or Ctrl-C to quit.")
print "Force updating..."
tag = tag,
cmssw_version = "CMSSW_9_2_8",
scram_arch = "slc6_amd64_gcc530",
no_load_from_backup = True,
)
maker_tasks.append(maker_task)
merge_tasks.append(merge_task)
for i in range(100):
total_summary = {}
for maker_task, merge_task in zip(maker_tasks,merge_tasks):
maker_task.process()
if maker_task.complete():
do_cmd("mkdir -p {}/merged".format(maker_task.get_outputdir()))
do_cmd("mkdir -p {}/skimmed".format(maker_task.get_outputdir()))
merge_task.reset_io_mapping()
merge_task.update_mapping()
merge_task.process()
total_summary[maker_task.get_sample().get_datasetname()] = maker_task.get_task_summary()
total_summary[merge_task.get_sample().get_datasetname()] = merge_task.get_task_summary()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary, webdir="~/public_html/dump/metis_stopbaby/").do()
# 1 hr power nap
time.sleep(15.*60)
files_per_output=100000,
output_dir=maker_task.get_outputdir() + "/merged",
output_name=samples[sample] + "_skim_1.root",
condor_submit_params={"sites": "T2_US_UCSD"},
output_is_tree=True,
# check_expectedevents = True,
tag=job_tag,
cmssw_version="CMSSW_9_2_0",
scram_arch="slc6_amd64_gcc530",
#no_load_from_backup = True,
)
# process the job (either submits, checks for resubmit, or finishes etc.)
maker_task.process()
if maker_task.complete():
merge_task.reset_io_mapping()
merge_task.update_mapping()
merge_task.process()
# save some information for the dashboard
total_summary[maker_task.get_sample().get_datasetname(
)] = maker_task.get_task_summary()
total_summary[merge_task.get_sample().get_datasetname(
)] = merge_task.get_task_summary()
# Aggregate whether all tasks are complete
all_tasks_complete = all_tasks_complete and maker_task.complete(
) and merge_task.complete()
# parse the total summary and write out the dashboard
files = [f.name for f in sample.get_files()]
print(len(files))
task = CondorTask(sample=sample,
open_dataset=False,
flush=True,
files_per_output=info["fpo"],
output_name="merged_ntuple.root",
tag=job_tag,
cmssw_version=cmssw_ver,
executable=exec_path,
tarfile=tar_path,
condor_submit_params={"sites": "T2_US_UCSD"},
special_dir=hadoop_path,
arguments=info["meta_conditions"])
task.process()
if not task.complete():
allcomplete = False
# save some information for the dashboard
total_summary[dataset] = task.get_task_summary()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary,
webdir="~/public_html/dump/ttH_BabyMaker/").do()
os.system("chmod -R 755 ~/public_html/dump/ttH_BabyMaker")
if allcomplete:
print("")
print("Job={} finished".format(job_tag))
print("")
break
print("Sleeping 300 seconds ...")
time.sleep(300)
# into a file that ends up on nfs (specified by `merged_dir` above)
for dsname,shortname in merge_map.items():
merge_task = LocalMergeTask(
input_filenames=task.get_outputs(),
output_filename="{}/{}.root".format(merged_dir,shortname)
)
merge_tasks[dsname] = merge_task
first_iteration = True
done = False
while not done:
# summary_info = {}
done = True # Assume all tasks are complete until we see otherwise
for dsname,shortname in sample_map.items():
task = tasks[dsname]
# Straightforward logic
if not task.complete():
task.process()
done = False
for dsname,shortname in sample_map_merge.items():
task = tasks[dsname]
merge_task = merge_tasks[dsname]
if task.complete() and not merge_task.complete():
merge_task.process()
done = False
# summary_info[shortname+"_babymaking"] = task.get_task_summary()
# StatsParser(data=summary_info, webdir="~/public_html/dump/metis/", make_plots=False).do()
# sleep for 10 min unless it's the very first run of the loop. In that case sleep for 1 min, to permit a quick check that things are working
if first_iteration:
time.sleep(60)
first_iteration = False
else: