def submit(which):
total_summary = {}
tag = "ZMuMuSkim_v0"
task = CMSSWTask(
sample = DBSSample(dataset="/SingleMuon/Run2017F-ZMu-17Nov2017-v1/RAW-RECO"),
events_per_output = 10000,
pset = "/home/users/mcitron/CMSSW_9_4_17/src/cms_lpc_llp/llp_ntupler/python/displacedJetMuon_ntupler_Data_2017_RAWRECO.py",
cmssw_version = "CMSSW_9_4_17",
scram_arch = "slc6_amd64_gcc700",
tag = tag,
# dont_check_tree = True,
executable = "/home/users/mcitron/exeForMetis/condor_genprod_exe.sh",
condor_submit_params = {
"container": "/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel6-m202006",
},
recopy_inputs=True
)
task.process()
total_summary[task.get_sample().get_datasetname()] = task.get_task_summary()
StatsParser(data=total_summary, webdir="~/public_html/dump/metis_test/").do()
location="/hadoop/cms/store/user/namin/yukawa_lhe/Events/run_{}/"
.format(runstr),
globber="*.lhe",
dataset="/tttt-LOytscan/kt-{}_v2/LHE".format(ktstr),
),
events_per_output=100,
total_nevents=10000,
pset="pset_gensim.py",
cmssw_version="CMSSW_7_1_25_patch2",
split_within_files=True,
)
raw = CMSSWTask(
sample=DirectorySample(
location=lhe.get_outputdir(),
dataset=lhe.get_sample().get_datasetname().replace(
"LHE", "RAW"),
),
open_dataset=True,
files_per_output=1,
pset="pset_raw.py",
cmssw_version="CMSSW_8_0_21",
)
aod = CMSSWTask(
sample=DirectorySample(
location=raw.get_outputdir(),
dataset=raw.get_sample().get_datasetname().replace(
"RAW", "AOD"),
),
open_dataset=True,
files_per_output=5,
def runall(special_dir, tag, total_nevents, events_per_output, config):
for _ in range(25):
proc_tag = "v1"
#special_dir = "workflowtest/ProjectMetis"
#special_dir = "miniaod_runII/JHUSample_ttH"
cmssw_v_gensim = config["cmssw_v_gensim"]
pset_gensim = config["pset_gensim"]
scram_arch_gensim = config["scram_arch_gensim"]
cmssw_v_aodsim = config["cmssw_v_aodsim"]
pset_aodsim = config["pset_aodsim"]
scram_arch_aodsim = config["scram_arch_aodsim"]
pset_aodsim2 = config["pset_aodsim2"]
cmssw_v_aodsim2 = cmssw_v_aodsim #config["pset_aodsim2"]
scram_arch_aodsim2 = scram_arch_aodsim #config["scram_arch_aodsim2"]
cmssw_v_miniaodsim = config["cmssw_v_miniaodsim"]
pset_miniaodsim = config["pset_miniaodsim"]
scram_arch_miniaodsim = config["scram_arch_miniaodsim"]
step1 = CMSSWTask(
# Change dataset to something more meaningful (but keep STEP1, as we use this
# for string replacement later); keep N=1
sample=DummySample(N=1, dataset="/" + tag + "_STEP1"),
# A unique identifier
tag=proc_tag,
special_dir=special_dir,
# Probably want to beef up the below two numbers to control splitting,
# but note that step2 is the bottleneck, so don't put too many events
# in one output file here
events_per_output=events_per_output,
total_nevents=total_nevents,
#events_per_output = 50,
#total_nevents = 1000,
# We have one input dummy file, so this must be True
split_within_files=True,
pset="psets/" + pset_gensim,
cmssw_version=cmssw_v_gensim,
scram_arch=scram_arch_gensim,
)
step2 = CMSSWTask(
sample=DirectorySample(
location=step1.get_outputdir(),
dataset=step1.get_sample().get_datasetname().replace(
"STEP1", "STEP2"),
),
tag=proc_tag,
special_dir=special_dir,
open_dataset=True,
files_per_output=1,
pset="psets/" + pset_aodsim,
cmssw_version=cmssw_v_aodsim,
scram_arch=scram_arch_aodsim,
)
step3 = CMSSWTask(
sample=DirectorySample(
location=step2.get_outputdir(),
dataset=step2.get_sample().get_datasetname().replace(
"STEP2", "STEP3"),
),
tag=proc_tag,
special_dir=special_dir,
open_dataset=True,
files_per_output=1,
pset="psets/" + pset_aodsim2,
cmssw_version=cmssw_v_aodsim2,
scram_arch=scram_arch_aodsim2,
)
step4 = CMSSWTask(
sample=DirectorySample(
location=step3.get_outputdir(),
dataset=step3.get_sample().get_datasetname().replace(
"STEP3", "STEP4"),
),
tag=proc_tag,
special_dir=special_dir,
open_dataset=True,
files_per_output=1,
output_name="step4.root",
pset="psets/" + pset_miniaodsim,
cmssw_version=cmssw_v_miniaodsim,
scram_arch=scram_arch_miniaodsim,
# condor_submit_params = {"sites":"UAF,UCSD"},
)
'''
step5 = CMSSWTask(
sample = DirectorySample(
location = step4.get_outputdir(),
dataset = step4.get_sample().get_datasetname().replace("STEP4","STEP5"),
),
tag = proc_tag,
special_dir = special_dir,
open_dataset = True,
files_per_output = 1,
pset = "psets/TOP-RunIIFall17NanoAODv7-00001_1_cfg.py",
# The below two lines should match output file names in the pset
output_name = "step5.root",
#other_outputs = ["step3_inMINIAODSIM.root","step3_inDQM.root"],
cmssw_version = "CMSSW_10_2_22",
scram_arch = "slc6_amd64_gcc700",
# condor_submit_params = {"sites":"UAF,UCSD"},
)
'''
#for _ in range(25):
total_summary = {}
for task in [step1, step2, step3, step4]:
task.process()
summary = task.get_task_summary()
total_summary[task.get_sample().get_datasetname()] = summary
StatsParser(data=total_summary,
webdir="~/public_html/dump/metis/").do()
time.sleep(600)
# in one output file here
events_per_output=200,
total_nevents=500000,
# We have one input dummy file, so this must be True
split_within_files=True,
pset="psets/HIG-RunIIFall17wmLHEGS-01343_1_cfg_jhu.py",
#cmssw_version = "CMSSW_9_3_6_patch2",
cmssw_version="CMSSW_9_4_6_patch1",
condor_submit_params={"sites": "T2_US_UCSD"},
scram_arch="slc6_amd64_gcc630",
)
step2 = CMSSWTask(
sample=DirectorySample(
location=step1.get_outputdir(),
dataset=step1.get_sample().get_datasetname().replace(
"STEP1", "STEP2"),
),
tag=proc_tag,
special_dir=special_dir,
open_dataset=True,
files_per_output=1,
pset="psets/HIG-RunIIFall17DRPremix-01240_1_cfg_jhu.py",
cmssw_version="CMSSW_9_4_4",
scram_arch="slc6_amd64_gcc630",
)
step3 = CMSSWTask(
sample=DirectorySample(
location=step2.get_outputdir(),
dataset=step2.get_sample().get_datasetname().replace(
"STEP2", "STEP3"),
from metis.CMSSWTask import CMSSWTask
from metis.Sample import DBSSample
from metis.StatsParser import StatsParser
import time
if __name__ == "__main__":
total_summary = {}
for _ in range(10000):
nano = CMSSWTask(
sample=DBSSample(
dataset=
"/DYJetsToLL_M-50_TuneCP5_13TeV-madgraphMLM-pythia8/RunIIAutumn18MiniAOD-102X_upgrade2018_realistic_v15-v1/MINIAODSIM",
),
events_per_output=150e3,
output_name="output.root",
tag="v1",
pset="../scratch/pset_nano.py",
cmssw_version="CMSSW_10_2_11",
scram_arch="slc6_amd64_gcc700",
)
nano.process()
total_summary[
nano.get_sample().get_datasetname()] = nano.get_task_summary()
StatsParser(data=total_summary,
webdir="~/public_html/dump/metis_nano/").do()
time.sleep(30 * 60)
gen = CMSSWTask(
sample=DummySample(
N=1, dataset="/WH_HtoRhoGammaPhiGamma/privateMC_102x/GENSIM"),
events_per_output=1000,
total_nevents=1000000,
pset="gensim_cfg.py",
cmssw_version="CMSSW_10_2_5",
scram_arch="slc6_amd64_gcc700",
tag=tag,
split_within_files=True,
)
raw = CMSSWTask(
sample=DirectorySample(
location=gen.get_outputdir(),
dataset=gen.get_sample().get_datasetname().replace(
"GENSIM", "RAWSIM"),
),
open_dataset=True,
files_per_output=1,
pset="rawsim_cfg.py",
cmssw_version="CMSSW_10_2_5",
scram_arch="slc6_amd64_gcc700",
tag=tag,
)
aod = CMSSWTask(
sample=DirectorySample(
location=raw.get_outputdir(),
dataset=raw.get_sample().get_datasetname().replace(
"RAWSIM", "AODSIM"),
),
for i in range(100):
total_summary = {}
# for dsname in dataset_names:
for task in ntuple_tasks:
try:
task.process()
except:
traceback_string = traceback.format_exc()
print "Runtime error:\n{0}".format(traceback_string)
# send_email(subject="metis error", body=traceback_string)
# if not task.complete():
# f = open("unfinished.txt","a+")
# print ""
# for output in task.get_uncompleted_outputs():
# f.write("for ouput file <"+output.get_name()+">:\n")
# for inp in task.get_inputs_for_output(output):
# f.write(inp.get_name()+'\n')
# f.write("\n\n")
# f.close()
total_summary[task.get_sample().get_datasetname()] = task.get_task_summary()
StatsParser(data=total_summary, webdir="~/public_html/dump/metis_80XTopTag/").do()
time.sleep(60.*30)
special_dir = "elemva/ProjectMetis",
open_dataset = False,
events_per_output = 700000,
output_name = "output.root",
output_is_tree = False,
check_expectedevents = True,
tag = tag,
pset = pset,
cmssw_version = cmsswver,
tarfile = tarfile,
pset_args = pset_args,
publish_to_dis = False,
)
cmssw_tasks.append(cmssw_task)
for i in range(100):
total_summary = {}
for cmssw_task in cmssw_tasks:
cmssw_task.process()
total_summary[cmssw_task.get_sample().get_datasetname()] = cmssw_task.get_task_summary()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary, webdir="~/public_html/dump/metis/").do()
# 1 hr power nap
time.sleep(10.*60)
lhe = CMSSWTask(
sample=DummySample(
N=1, dataset="/WH_HtoRhoGammaPhiGamma/privateMC_102x/step1"),
events_per_output=1000,
total_nevents=1000000,
pset="gensim_cfg.py",
cmssw_version="CMSSW_10_2_5",
scram_arch="slc6_amd64_gcc700",
tag=tag,
split_within_files=True,
)
raw = CMSSWTask(
sample=DirectorySample(
location=lhe.get_outputdir(),
dataset=lhe.get_sample().get_datasetname().replace(
"step1", "GENSIM"),
),
open_dataset=True,
files_per_output=1,
pset="rawsim_cfg.py",
cmssw_version="CMSSW_10_2_5",
scram_arch="slc6_amd64_gcc700",
tag=tag,
)
aod = CMSSWTask(
sample=DirectorySample(
location=raw.get_outputdir(),
dataset=raw.get_sample().get_datasetname().replace(
"GENSIM", "RAWSIM"),
),