def test2():
dataset_names = [
# "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM",
"/Dummy_test_StopBabyMaker_v25/CMS4",
]
# Make a base directory
basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format(os.getenv("USER"))
MutableFile(basedir).touch()
# Make a directory sample, giving it the location and a dataset name for bookkeeping purposes
# The globber must be customized (by default, it is *.root) in order to pick up the text files
ds = DirectorySample(location=basedir, dataset="/TEST/Examplev1/TEST", globber="*.txt")
# Make a CondorTask (3 in total, one for each input)
task = CondorTask(
sample = ds,
files_per_output = 1,
tag = "v0",
output_name = "ttbar_powheg_pythia8_92X.root",
executable = "condor_executable.sh",
cmssw_version = "CMSSW_9_3_1",
scram_arch = "slc6_amd64_gcc700",
arguments = "testarg1",
tarfile = "input.tar.gz",
condor_submit_params = {"sites": "UAF,T2_US_UCSD,UCSB"},
no_load_from_backup = True, # for the purpose of the example, don't use a backup
)
# do_cmd("rm -rf {0}".format(task.get_outputdir()))
# Process and sleep until complete
is_complete = False
for t in [5.0, 5.0, 10.0, 15.0, 20.0]:
task.process()
print("Sleeping for {0} seconds".format(int(t)))
time.sleep(t)
is_complete = task.complete()
if is_complete: break
# If it's complete, make a dummy sample out of the output directory
# in order to pick up the files. Then cat out the contents and sum
# them up. This should be 3*2*10 = 100
if is_complete:
print("Job completed! Checking outputs...")
outsamp = DirectorySample(location=task.get_outputdir(), dataset="/Blah/blah/BLAH", globber="*.txt")
tot = 0
for f in outsamp.get_files():
mf = MutableFile(f.get_name())
tot += int(mf.cat())
print("It looks like we found 3*2*10 = {0}".format(tot))
def setUpClass(cls):
super(CondorTaskTest, cls).setUpClass()
# make a test directory and touch some root files and executable there
basedir = "/tmp/{0}/metis/condortask_test/".format(os.getenv("USER"))
Utils.do_cmd("mkdir -p {0}".format(basedir))
for i in range(1, cls.nfiles + 1):
Utils.do_cmd("touch {0}/input_{1}.root".format(basedir, i))
Utils.do_cmd("echo hello > {0}/executable.sh".format(basedir))
# make dummy CondorTask with the files we
# touched in the basedir, and chunk
# the outputs
logging.getLogger("logger_metis").disabled = True
cls.dummy = CondorTask(sample=DirectorySample(
location=basedir,
globber="*.root",
dataset="/test/test/TEST",
),
open_dataset=False,
files_per_output=cls.files_per_job,
cmssw_version=cls.cmssw,
tag=cls.tag,
executable="{0}/executable.sh".format(basedir))
# prepare inputs and run,
# but pretend like outputs exist and don't submit
cls.dummy.prepare_inputs()
# run once to "submit to condor" and "create outputs" (set_fake)
cls.dummy.run(fake=True)
# run again to recognize that all outputs are there and
# we can then declare completion
cls.dummy.run(fake=True)
def get_data_tasks(info):
tasks = []
for reqname, tag, extra_args in info:
task = CondorTask(
sample=DirectorySample(
location=
"/hadoop/cms/store/user/namin/ScoutingCaloMuon/crab_{}/*/*/".
format(reqname),
dataset="/ScoutingCaloMuon/Run2018{}/RAW".format(reqname)),
output_name="output.root",
executable="executables/scouting_exe.sh",
tarfile="package.tar.gz",
MB_per_output=4000,
condor_submit_params={
"sites":
"T2_US_UCSD",
"classads": [
["metis_extraargs", extra_args],
["JobBatchName", reqname],
],
"requirements_line":
'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})'
.format(extra_requirements=extra_requirements),
},
cmssw_version="CMSSW_10_2_5",
scram_arch="slc6_amd64_gcc700",
tag=tag,
)
tasks.append(task)
return tasks
def test_completion_fraction(self):
# Make dummy task with no inputs
# and require min completion fraction to be 0
logging.getLogger("logger_metis").disabled = True
dummy = CondorTask(
sample=DirectorySample(
location=".",
globber="*.fake",
dataset="/testprocess/testprocess/TEST",
),
open_dataset=False,
files_per_output=1,
cmssw_version="CMSSW_8_0_20",
tag="vtest",
no_load_from_backup=True,
min_completion_fraction=0.,
)
dummy.process()
self.assertEqual(dummy.complete(), True)
def test1():
dataset_names = [
# "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM",
"/Dummy_test_StopBabyMaker_v25/CMS4",
]
basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format(
os.getenv("USER"))
for i in range(10000):
total_summary = {}
for dsname in dataset_names:
try:
task = CondorTask(
sample=DirectorySample(
location=basedir,
dataset=dsname,
),
files_per_output=1,
output_name="merged_ntuple.root",
tag="v25",
scram_arch="slc6_amd64_gcc700",
executable="condor_executable.sh",
arguments="testarg1",
tarfile="input.tar.gz",
condor_submit_params={"sites": "UAF,T2_US_UCSD,UCSB"},
no_load_from_backup=
True, # for the purpose of the example, don't use a backup
)
task.process()
except:
traceback_string = traceback.format_exc()
print("Runtime error:\n{0}".format(traceback_string))
# send_email(subject="metis error", body=traceback_string)
total_summary[dsname] = task.get_task_summary()
StatsParser(data=total_summary,
webdir="~/public_html/dump/metis/").do()
time.sleep(5)
def test1():
dataset_names = [
# "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM",
"/Dummy_test_StopBabyMaker_v25/CMS4",
]
basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format(os.getenv("USER"))
for i in range(10000):
total_summary = {}
for dsname in dataset_names:
try:
task = CondorTask(
sample = DirectorySample(
location=basedir,
dataset=dsname,
),
files_per_output = 1,
output_name = "merged_ntuple.root",
tag = "v25",
scram_arch = "slc6_amd64_gcc700",
executable = "condor_executable.sh",
arguments = "testarg1",
tarfile = "input.tar.gz",
condor_submit_params = {"sites": "UAF,T2_US_UCSD,UCSB"},
no_load_from_backup = True, # for the purpose of the example, don't use a backup
)
task.process()
except:
traceback_string = traceback.format_exc()
print("Runtime error:\n{0}".format(traceback_string))
# send_email(subject="metis error", body=traceback_string)
total_summary[dsname] = task.get_task_summary()
StatsParser(data=total_summary, webdir="~/public_html/dump/metis/").do()
time.sleep(5)
def submit_jobs(output_directory, tag, samples, events_total,
events_per_output, tarfile):
# Long loop to monitor jobs and resubmit failed jobs
for i in range(10000):
total_summary = {}
# Loop through samples
for sample in samples:
config_info = samples_config[sample]
executable = create_executable(tag, sample, config_info)
task = CondorTask(sample=DummySample(
N=int(float(events_total) / float(events_per_output)),
nevents=events_total,
dataset="/" + sample + "_NANO"),
tag=tag,
special_dir=output_directory,
events_per_output=events_per_output,
total_nevents=events_total,
split_within_files=True,
executable=executable,
open_dataset=False,
tarfile=tarfile,
condor_submit_params=CONDOR_SUBMIT_PARAMS)
task.process()
total_summary[
task.get_sample().get_datasetname()] = task.get_task_summary()
StatsParser(data=total_summary,
webdir="~/public_html/dump/Hgg-MC-Production/").do()
time.sleep(300) # power nap
def test_flush(self):
basedir = "/tmp/{0}/metis/condortask_testflush/".format(
os.getenv("USER"))
Utils.do_cmd("mkdir -p {0}".format(basedir))
tag = "vflush"
for i in range(1, self.nfiles + 1):
Utils.do_cmd("touch {0}/input_{1}.root".format(basedir, i))
dummy = CondorTask(
sample=DirectorySample(
location=basedir,
globber="*.root",
dataset="/test/test/TEST",
),
open_dataset=True,
files_per_output=self.files_per_job,
cmssw_version=self.cmssw,
tag=tag,
)
self.assertEqual(len(dummy.get_outputs()),
(self.nfiles // self.files_per_job))
dummy.flush()
self.assertEqual(len(dummy.get_outputs()),
(self.nfiles // self.files_per_job + 1))
def submit(which):
total_summary = {}
extra_requirements = "true"
tag = "v0_noFilter"
pdname = "HVNoFilter"
events_per_point = int(1E4)
events_per_job = int(100)
cfgsDir = "psets_gensim_noFilter/"
modelsFile = cfgsDir + "/models.txt"
df = pd.read_csv(modelsFile)
for year in ["2016", "2017", "2018"]:
for iterr, row in df.iterrows():
# fmass = float(mass)
# mass = str(mass).replace(".","p")
epp = int(events_per_point)
reqname = "noFilter_m{}{}_ctau{}_xi_{}_{}_{}".format(
row.portal, row.mass, row.ctau, row.xi, tag, year)
njobs = epp // events_per_job
sample = DummySample(
dataset="/{}/params_{}_m_{}_ctau_{}mm_xi_{}_{}/LLPNTUPLE".
format(pdname, row.portal, row.mass, row.ctau * 10, row.xi,
year),
N=njobs,
nevents=epp)
task = CondorTask(
sample=sample,
output_name="output.root",
executable="executables/condor_executable_{}.sh".format(which),
tarfile="package_{}.tar.xz".format(year),
open_dataset=True,
files_per_output=1,
condor_submit_params={
"classads": [
["param_mass", row.mass],
["param_ctau", row.ctau],
["param_xi", str(row.xi).replace(".", "p")],
["param_portal", row.portal],
["param_year", year],
["param_nevents", events_per_job],
["metis_extraargs", ""],
["JobBatchName", reqname],
],
"requirements_line":
'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})'
.format(extra_requirements=extra_requirements),
},
tag=tag,
recopy_inputs=True)
task.process()
total_summary[
task.get_sample().get_datasetname()] = task.get_task_summary()
StatsParser(data=total_summary,
webdir="~/public_html/dump/metis_test/").do()
def test2():
dataset_names = [
# "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM",
"/Dummy_test_StopBabyMaker_v25/CMS4",
]
# Make a base directory
basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format(
os.getenv("USER"))
MutableFile(basedir).touch()
# Make a directory sample, giving it the location and a dataset name for bookkeeping purposes
# The globber must be customized (by default, it is *.root) in order to pick up the text files
ds = DirectorySample(location=basedir,
dataset="/TEST/Examplev1/TEST",
globber="*.txt")
# Make a CondorTask (3 in total, one for each input)
task = CondorTask(
sample=ds,
files_per_output=1,
tag="v0",
output_name="ttbar_powheg_pythia8_92X.root",
executable="condor_executable.sh",
cmssw_version="CMSSW_9_3_1",
scram_arch="slc6_amd64_gcc700",
arguments="testarg1",
tarfile="input.tar.gz",
condor_submit_params={"sites": "UAF,T2_US_UCSD,UCSB"},
no_load_from_backup=
True, # for the purpose of the example, don't use a backup
)
# do_cmd("rm -rf {0}".format(task.get_outputdir()))
# Process and sleep until complete
is_complete = False
for t in [5.0, 5.0, 10.0, 15.0, 20.0]:
task.process()
print("Sleeping for {0} seconds".format(int(t)))
time.sleep(t)
is_complete = task.complete()
if is_complete: break
# If it's complete, make a dummy sample out of the output directory
# in order to pick up the files. Then cat out the contents and sum
# them up. This should be 3*2*10 = 100
if is_complete:
print("Job completed! Checking outputs...")
outsamp = DirectorySample(location=task.get_outputdir(),
dataset="/Blah/blah/BLAH",
globber="*.txt")
tot = 0
for f in outsamp.get_files():
mf = MutableFile(f.get_name())
tot += int(mf.cat())
print("It looks like we found 3*2*10 = {0}".format(tot))
def submit():
requests = {
#'TTWJetsToLNuEWK_5f_NLO': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_NLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball_retired.tar.xz', # that's the SM point, but using the SMEFT model. No lepton filtering, so name is actually confusing
#'TTWJetsToLNuEWK_5f_NLO_v2': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_NLO_slc7_amd64_gcc730_CMSSW_9_3_16_tarball.tar.xz', # that's the actual SM
#'TTWplusJetsToLNuEWK_5f_NLO_v2': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWplusJetsToLNuEWK_5f_NLO_slc7_amd64_gcc730_CMSSW_9_3_16_tarball.tar.xz', # that's the actual SM
#'TTWminusJetsToLNuEWK_5f_NLO_v2': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWminusJetsToLNuEWK_5f_NLO_slc7_amd64_gcc730_CMSSW_9_3_16_tarball.tar.xz', # that's the actual SM
#'TTWJetsToLNuEWK_5f_EFT_myNLO_full': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_cpt8_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # one of the BSM points
#'TTWJetsToLNuEWK_5f_EFT_mix_myNLO_full': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # EFT mix
'TTWJetsToLNuEWK_5f_EFT_cpq3_4_myNLO_full': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks//TTWJetsToLNuEWK_5f_EFT_myNLO_cpq3_4_slc7_amd64_gcc730_CMSSW_9_3_16_tarball.tar.xz', # C_pq3 = 4
}
total_summary = {}
extra_requirements = "true"
tag = "v4"
events_per_point = 500000 # produced 500k events before
events_per_job = 2000 # up to 2000 works
#events_per_point = 50
#events_per_job = 10
njobs = int(events_per_point)//events_per_job
for reqname in requests:
gridpack = requests[reqname]
#reqname = "TTWJetsToLNuEWK_5f_EFT_myNLO"
#gridpack = '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz'
task = CondorTask(
sample = DummySample(dataset="/%s/RunIIAutumn18/NANO"%reqname,N=njobs,nevents=int(events_per_point)),
output_name = "nanoAOD.root",
executable = "executables/condor_executable_Autumn18.sh",
tarfile = "package.tar.gz",
#scram_arch = "slc7_amd64_gcc630",
open_dataset = False,
files_per_output = 1,
arguments = gridpack,
condor_submit_params = {
"sites":"T2_US_UCSD", #
"classads": [
["param_nevents",events_per_job],
["metis_extraargs",""],
["JobBatchName",reqname],
#["SingularityImage", "/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel6-m202006"],
],
"requirements_line": 'Requirements = (HAS_SINGULARITY=?=True)' # && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})'.format(extra_requirements=extra_requirements),
},
tag = tag,
min_completion_fraction = 0.90,
)
task.process()
total_summary[task.get_sample().get_datasetname()] = task.get_task_summary()
StatsParser(data=total_summary, webdir="~/public_html/dump/tW_gen/").do()
def get_tasks(infos):
tasks = []
for info in infos:
location = info["location"]
dataset = info["dataset"]
isdata = info["isdata"]
open_dataset = info.get("open_dataset", False)
tag = info["tag"]
extra_args = info.get("extra_args", "")
kwargs = {}
if isdata:
kwargs["MB_per_output"] = (4000
if "skim1cm" in extra_args else 1000)
batchname = dataset.split("_", 1)[-1].split("/")[0] + "_" + tag
else:
kwargs["files_per_output"] = 200
batchname = "_".join(
dataset.split("params_", 1)[1].split("/", 1)[0].split("_")[:2]
+ [tag])
task = CondorTask(
sample=DirectorySample(location=location, dataset=dataset),
open_dataset=open_dataset,
flush=True,
output_name="output.root",
executable="executables/scouting_exe.sh",
tarfile="package.tar.gz",
condor_submit_params={
"container":
"/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel6-m202006",
"sites":
"T2_US_UCSD",
"classads": [
["metis_extraargs", extra_args],
["JobBatchName", batchname],
],
"requirements_line":
'Requirements = ((HAS_SINGULARITY=?=True) && {extra_requirements})'
.format(extra_requirements=extra_requirements),
},
cmssw_version="CMSSW_10_2_5",
scram_arch="slc6_amd64_gcc700",
tag=tag,
**kwargs)
tasks.append(task)
return tasks
def test_full(self):
"""
Touch a root file ("input")
Submit condor jobs to touch output files for each input file
and copy them to hadoop
Jobs get submitted to local universe for speed reasons
Check output to make sure job completed
"""
njobs = 2
cmssw = "CMSSW_8_0_21"
basedir = "/tmp/{0}/metis/condortask_testfull/".format(
os.getenv("USER"))
Utils.do_cmd("mkdir -p {0}".format(basedir))
tag = "vfull"
for i in range(1, njobs + 1):
Utils.do_cmd("touch {0}/input_{1}.root".format(basedir, i))
logging.getLogger("logger_metis").disabled = True
dummy = CondorTask(
sample=DirectorySample(
location=basedir,
globber="*.root",
dataset="/test/test/TEST",
),
open_dataset=False,
files_per_output=1,
cmssw_version=cmssw,
executable=Utils.metis_base() +
"metis/executables/condor_test_exe.sh",
tag=tag,
condor_submit_params={"universe": "local"},
no_load_from_backup=True,
)
# clean up previous directory
Utils.do_cmd("rm -rf {0}".format(dummy.get_outputdir()))
is_complete = False
for t in [1.0, 1.0, 2.0, 3.0, 5.0, 10.0, 20.0]:
dummy.process()
time.sleep(t)
is_complete = dummy.complete()
if is_complete: break
self.assertEquals(is_complete, True)
self.assertEqual(njobs, len(glob.glob(dummy.get_outputdir() + "/*")))
def submit():
requests = {
'TTWJetsToLNuEWK_5f_EFT_myNLO': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # that's EFT
'TTWJetsToLNuEWK_5f_EFT_myNLO_cpt8': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_cpt8_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # that's EFT
'TTWJetsToLNuEWK_5f_NLO': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_NLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # that's the SM
}
total_summary = {}
extra_requirements = "true"
tag = "v1"
events_per_point = 500000
events_per_job = 10000
#events_per_point = 500
#events_per_job = 100
njobs = int(events_per_point)//events_per_job
for reqname in requests:
gridpack = requests[reqname]
#reqname = "TTWJetsToLNuEWK_5f_EFT_myNLO"
#gridpack = '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz'
task = CondorTask(
sample = DummySample(dataset="/%s/RunIIAutumn18/NANOGEN"%reqname,N=njobs,nevents=int(events_per_point)),
output_name = "output.root",
executable = "executables/condor_executable.sh",
tarfile = "package.tar.gz",
open_dataset = False,
files_per_output = 1,
arguments = gridpack,
condor_submit_params = {
"sites":"T2_US_UCSD", #
"classads": [
["param_nevents",events_per_job],
["metis_extraargs",""],
["JobBatchName",reqname],
],
"requirements_line": 'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})'.format(extra_requirements=extra_requirements),
},
tag = tag,
)
task.process()
total_summary[task.get_sample().get_datasetname()] = task.get_task_summary()
StatsParser(data=total_summary, webdir="~/public_html/dump/tW_gen/").do()
def get_signal_tasks(pattern, tag, extra_args="", func_dataset=None):
samples = []
for location in glob.glob(pattern):
if func_dataset is not None:
dataset = func_dataset(location)
else:
taskname = location.rstrip("/").rsplit("/")[-1]
dataset = "/{}/{}/BABY".format(
taskname.split("_", 1)[0],
taskname.split("_", 1)[1].split("_RAWSIM")[0],
)
samples.append(DirectorySample(location=location, dataset=dataset))
tasks = []
for sample in samples:
task = CondorTask(
sample=sample,
output_name="output.root",
executable="executables/scouting_exe.sh",
tarfile="package.tar.gz",
files_per_output=int(1e5),
condor_submit_params={
"sites":
"T2_US_UCSD",
"classads": [
["metis_extraargs", extra_args],
[
"JobBatchName",
sample.get_datasetname().split("/")[2].replace(
"params_", "")
],
],
"requirements_line":
'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})'
.format(extra_requirements=extra_requirements),
},
cmssw_version="CMSSW_10_2_5",
scram_arch="slc6_amd64_gcc700",
tag=tag,
)
tasks.append(task)
return tasks
def get_bg_tasks(tag, extra_args=""):
samples = [
# DBSSample(dataset="/JpsiToMuMu_JpsiPt8_TuneCP5_13TeV-pythia8/RunIIAutumn18DRPremix-102X_upgrade2018_realistic_v15-v1/AODSIM")
DBSSample(
dataset=
"/BuToKJpsi_ToMuMu_MuFilter_SoftQCDnonD_TuneCP5_13TeV-pythia8-evtgen/RunIIAutumn18DR-PUPoissonAve20_102X_upgrade2018_realistic_v15-v2/AODSIM"
)
]
tasks = []
for sample in samples:
task = CondorTask(
sample=sample,
output_name="output.root",
executable="executables/scouting_exe.sh",
tarfile="package.tar.gz",
events_per_output=500e3,
condor_submit_params={
"sites":
"T2_US_UCSD",
"classads": [
["metis_extraargs", extra_args],
[
"JobBatchName",
sample.get_datasetname().split("/")[1].split("_")[0] +
"_" + tag
],
],
"requirements_line":
'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})'
.format(extra_requirements=extra_requirements),
},
cmssw_version="CMSSW_10_2_5",
scram_arch="slc6_amd64_gcc700",
tag=tag,
)
tasks.append(task)
return tasks
"/TTWJetsToLNu_TuneCP5_PSweights_13TeV-amcatnloFXFX-madspin-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "ttwlv",
"/TTZToLLNuNu_M-10_TuneCP5_PSweights_13TeV-amcatnlo-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "ttzllvv",
"/ttHTobb_ttToSemiLep_M125_TuneCP5_13TeV-powheg-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "tthbb",
"/WpWpJJ_EWK-QCD_TuneCUETP8M1_13TeV-madgraph-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "ssww",
"/WZTo3LNu_TuneCUETP8M1_13TeV-powheg-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "wz3lv",
}
# submission tag
tag = "v8"
for dsname, shortname in sample_map.items():
task = CondorTask(
sample = DBSSample(
dataset=dsname,
# tag="CMS4_V09-04-13", # if not specified, get latest CMS4 tag
),
files_per_output = 1,
output_name = "output.root",
tag = tag,
condor_submit_params = {"sites": "T2_US_UCSD", "use_xrootd":True},
cmssw_version = "CMSSW_10_5_0",
input_executable = "condor_executable_metis.sh", # your condor executable here
tarfile = "/nfs-7/userdata/phchang/VBSHWWNanoSkimmer_v1.package.tar.gz", # your tarfile with assorted goodies here
special_dir = "VBSHWWNanoSkim/{}".format(tag), # output files into /hadoop/cms/store/<user>/<special_dir>
)
# Straightforward logic
if not task.complete():
task.process()
break
# submission tag
tag = "fcnc_v10_SRonly_11june2021"
task_list = []
for sample in samples:
task = CondorTask(
sample=sample,
files_per_output=1,
output_name="output.root",
tag=tag,
condor_submit_params={
"sites": "T2_US_UCSD",
"use_xrootd": True
},
cmssw_version="CMSSW_10_2_9",
input_executable=
"condor_executable_metis.sh", # your condor executable here
# tarfile = "/hadoop/cms/store/user/ksalyer/FCNC_NanoSkimmer_v2.package.tar.gz", # your tarfile with assorted goodies here
tarfile=
"/hadoop/cms/store/user/dspitzba/FCNC/FCNC_NanoSkimmer_v10_SROnly.package.tar.gz",
# tarfile = "/nfs-7/userdata/phchang/VBSHWWNanoSkimmer_v1.package.tar.gz",
special_dir="FCNC_NanoSkim/{}".format(
tag
), # output files into /hadoop/cms/store/<user>/<special_dir>
# special_dir = "VBSHWWNanoSkim/{}".format(tag), # output files into /hadoop/cms/store/<user>/<special_dir>
)
task_list.append(task)
# Straightforward logic
# if not task.complete():
# task.process()
for i in range(100):
# Loop through central samples
for ds, fpo, args in dsdefs[:]:
if (job_filter != "") and (args not in job_filter): continue
sample = DBSSample(dataset=ds)
print(ds, args)
task = CondorTask(
sample=sample,
open_dataset=False,
files_per_output=fpo,
output_name="test_rawreco.root",
tag=job_tag,
cmssw_version=cmssw_ver,
executable=exec_path,
tarfile="./package.tar.gz",
condor_submit_params={
"sites":
"T2_US_UCSD",
"container":
"/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel7-m20201010"
},
#condor_submit_params = {"sites" : "T2_US_UCSD,T2_US_CALTECH,T2_US_MIT,T2_US_WISCONSIN,T2_US_Nebraska,T2_US_Purdue,T2_US_Vanderbilt,T2_US_Florida"},
special_dir=hadoop_path,
arguments=args.replace(" ", "|"))
task.process()
allcomplete = allcomplete and task.complete()
# save some information for the dashboard
total_summary[ds] = task.get_task_summary()
with open("summary.json", "w") as f_out:
json.dump(total_summary, f_out, indent=4, sort_keys=True)
total_summary = {}
while True:
allcomplete = True
for ds,loc,fpo in dslocs:
print(loc)
sample = DirectorySample( dataset=ds, location=loc )
files = [f.name for f in sample.get_files()]
sample.set_files([f.name for f in sample.get_files() if "/cms/store/user/bemarsh/flashgg/MicroAOD/forHualin_2017/GJet_Pt-40toInf_DoubleEMEnriched_MGG-80toInf_TuneCP5_13TeV_Pythia8_RunIIFall17MiniAODv2-PU2017_12Apr2018_94X_mc2017_realistic_v14-v2_MINIAODSIM_forHualin_2017/test_skim_630.root" not in f.name])
#print [f.name for f in sample.get_files()]
task = CondorTask(
sample = sample,
open_dataset = False,
flush = True,
files_per_output = fpo,
output_name = "merged_ntuple.root",
tag = job_tag,
cmssw_version = cmssw_ver, # doesn't do anything
executable = exec_path,
tarfile = tar_path,
condor_submit_params = {"sites" : "T2_US_UCSD"},
special_dir = hadoop_path
)
task.process()
if not task.complete():
allcomplete = False
# save some information for the dashboard
total_summary[ds] = task.get_task_summary()
# parse the total summary and write out the dashboard
#StatsParser(data=total_summary, webdir="~/public_html/dump/ttH_BabyMaker/").do()
os.system("chmod -R 755 ~/public_html/dump/ttH_BabyMaker")
if allcomplete:
fpo = jobs["processes"][pid]["filesPerOutput"]
for ds in jobs["processes"][pid]["datasets"]:
args = getArgs(pid, ds)
dsdefs.append((ds, fpo, args))
total_summary = {}
while True:
allcomplete = True
for ds, fpo, args in dsdefs[:]:
sample = DBSSample(dataset=ds)
task = CondorTask(sample=sample,
open_dataset=False,
files_per_output=fpo,
output_name="test_skim.root"
if DOSKIM else "myMicroAODOutputFile.root",
tag=job_tag,
executable=exec_path,
tarfile=tar_path,
condor_submit_params={"sites": "T2_US_UCSD"},
special_dir=hadoop_path,
arguments=args.replace(" ", "|"))
task.process()
allcomplete = allcomplete and task.complete()
# save some information for the dashboard
total_summary[ds] = task.get_task_summary()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary,
webdir="~/public_html/dump/metis_microaod_80x/").do()
os.system("chmod -R 755 ~/public_html/dump/metis_microaod_80x")
if allcomplete:
print ""
tag = "v15"
tarfile = "/nfs-7/userdata/phchang/VBSHWWNanoSkimmer_v8_CMSSW_10_2_13_slc7_amd64_gcc700.package.tar.gz" # >= 2 leptons of any charge (35 GeV each) + njets_20 >= 4
task_summary = {}
for sample in samples:
task = CondorTask(
sample=sample,
files_per_output=1,
output_name="output.root",
tag=tag,
# condor_submit_params = {"sites": "T2_US_UCSD", "use_xrootd":True, "classads": [ ["metis_extraargs", "fetch_nano"] ]},
condor_submit_params={
"sites": "T2_US_UCSD",
"use_xrootd": True
},
cmssw_version="CMSSW_10_2_13",
scram_arch="slc7_amd64_gcc700",
input_executable=
"condor_executable_metis.sh", # your condor executable here
tarfile=tarfile, # your tarfile with assorted goodies here
special_dir="VBSHWWNanoSkim/{}".format(
tag
), # output files into /hadoop/cms/store/<user>/<special_dir>
)
# Straightforward logic
if not task.complete():
task.process()
# Set task summary
task_summary[
# submission tag
tag = "v1_PMtest"
merged_dir = "/nfs-7/userdata/{}/tupler_babies/merged/FT/{}/output/".format(
os.getenv("USER"), tag)
for dsname, shortname in sample_map.items():
task = CondorTask(
sample=SNTSample(
dataset=dsname,
# tag="CMS4_V09-04-13", # if not specified, get latest CMS4 tag
),
files_per_output=split_func(dsname),
output_name="output.root",
tag=tag,
condor_submit_params={"use_xrootd": True},
cmssw_version="CMSSW_9_2_8",
input_executable=
"inputs/condor_executable_metis.sh", # your condor executable here
tarfile=
"inputs/package.tar.xz", # your tarfile with assorted goodies here
special_dir=
"FTbabies/", # output files into /hadoop/cms/store/<user>/<special_dir>
)
# When babymaking task finishes, fire off a task that takes outputs and merges them locally (hadd)
# into a file that ends up on nfs (specified by `merged_dir` above)
merge_task = LocalMergeTask(input_filenames=task.get_outputs(),
output_filename="{}/{}.root".format(
merged_dir, shortname))
# Straightforward logic
if not task.complete():
total_summary = {}
first_run = False
while True:
allcomplete = True
for sample, info in local_samples.items():
dataset = sample
year = info["year"]
metis_sample = DirectorySample(dataset = dataset, location = info["location"])
task = CondorTask(
sample = metis_sample,
open_dataset = False,
flush = True,
files_per_output = info["fpo"],
output_name = "merged_ntuple.root",
tag = job_tag,
cmssw_version = cmssw_ver,
executable = "condor_exe_%s.sh" % job_tag if (args.use_xrdcp or args.use_gridftp or args.use_wget) else "condor_exe.sh",
tarfile = "empty" if (args.use_xrdcp or args.use_gridftp or args.use_wget) else tar_path,
condor_submit_params = {"sites" : "T2_US_UCSD"},
special_dir = hadoop_path,
arguments = conds_dict[year]
)
task.process()
if not task.complete():
allcomplete = False
total_summary[dataset] = task.get_task_summary()
if not args.local_only:
for sample in samples.keys():
if skip(sample):
continue
skip_tail = "/SMS" in dsname
# skip_tail = False
task = CondorTask(
sample=sample,
files_per_output=split_func(dsname),
output_name="output.root",
tag=tag,
min_completion_fraction=0.90 if skip_tail else 1.0,
condor_submit_params={
# "sites":"T2_US_UCSD,UCSB", # I/O is hella faster
"sites":
"T2_US_UCSD", # I/O is hella faster
# "sites":"UAF", # I/O is hella faster
"classads": [
["metis_extraargs", extra_args],
[
"JobBatchName",
"FT_{}_{}".format(year, shortname)
],
],
# "classads": [ ["metis_extraargs","--ignorebadfiles"], ],
},
cmssw_version="CMSSW_9_4_9",
scram_arch="slc6_amd64_gcc630",
input_executable="inputs/condor_executable_metis.sh",
tarfile="inputs/package.tar.xz",
special_dir="FTbabies/{}/".format(tag),
recopy_inputs=False,
)
merge_task = LocalMergeTask(
input_filenames=task.get_outputs(),
excltag = "CMS4_V09*"
samptyp = "1" if "SMS" in babyname else "2" if (
"GJets" in dsname or "Photon" in dsname
or "Gamma" in dsname) else "0"
extrarg = ""
# extrarg = " topcandTree=topcands" if "TTJets" in dsname else ""
maker_task = CondorTask(
sample=SNTSample(dataset=dsname,
tag=cms3tag,
exclude_tag_pattern=excltag),
files_per_output=20
if "data" in babyname else 1 if "SMS" in babyname else 2,
tag=tag,
outdir_name="stopBaby_" + babyname,
output_name="stopbaby.root",
executable="condor_executable.sh",
cmssw_version=cmsswver,
scram_arch=scramarch,
arguments=samptyp + extrarg,
tarfile=tarfile,
# condor_submit_params = {"sites": "T2_US_UCSD"},
# condor_submit_params = {"sites": "UAF"},
condor_submit_params={"use_xrootd": True},
# max_jobs = 1, # temporary for submission test
no_load_from_backup=True,
)
merge_task = CondorTask(
sample=DirectorySample(
dataset="merge_" + babyname,
location=maker_task.get_outputdir(),
),
# open_dataset = True, flush = True,
lumiWeightString = 1 if (
isData or samples[s]['name'].count('TChiWH')
) else 1000 * samples[s]['xsec'] / samples[s]['sumWeight']
print("- found sumWeight %s and x-sec %s" %
(samples[s]['sumWeight'], samples[s]['xsec']))
maker_task = CondorTask(
sample=sample,
executable="executable.sh",
arguments=" ".join([
str(x) for x in [
tag, lumiWeightString, 1 if isData else 0, year, era,
1 if isFastSim else 0, args.skim, args.user
]
]),
files_per_output=int(mergeFactor),
output_dir=os.path.join(outDir, samples[s]['name']),
output_name="nanoSkim.root",
output_is_tree=True,
tag=tag,
condor_submit_params={"sites": "T2_US_UCSD,UAF"},
cmssw_version="CMSSW_10_2_9",
scram_arch="slc6_amd64_gcc700",
min_completion_fraction=1.00,
)
maker_tasks.append(maker_task)
if not args.dryRun:
for i in range(100):
total_summary = {}
mt2home = os.path.dirname(os.path.realpath(__file__))
# submission tag
tag = "V00-10-05_2016fullYear_17July2018"
merged_dir = "/nfs-6/userdata/mt2/{}/".format(tag)
tasks = {}
merge_tasks = {}
for dsname,shortname in sample_map.items():
task = CondorTask(
sample = SNTSample(
dataset=dsname,
# tag="CMS4_V09-04-13", # uncomment this and set it to the desired tag, otherwise Metis will run on the most recent version
),
files_per_output = 1,
output_name = "output.root",
tag = tag,
condor_submit_params = {"use_xrootd":True},
cmssw_version = "CMSSW_9_4_9",
input_executable = mt2home+"/metis_executable.sh", # your condor executable here
tarfile = mt2home+"/job_input/input.tar.xz", # your tarfile with assorted goodies here
special_dir = "mt2babies/{}".format(tag), # output files into /hadoop/cms/store/<user>/<special_dir>
)
tasks[dsname] = task
# When babymaking task finishes, fire off a task that takes outputs and merges them locally (hadd)
# into a file that ends up on nfs (specified by `merged_dir` above)
for dsname,shortname in merge_map.items():
merge_task = LocalMergeTask(
input_filenames=task.get_outputs(),
output_filename="{}/{}.root".format(merged_dir,shortname)
)
merge_tasks[dsname] = merge_task
for dsname, samploc in cms4_samples.items():
cmsswver = "CMSSW_10_1_0"
scramarch = "slc6_amd64_gcc700"
tarfile = "tarfiles/input_v29_10.tar.gz"
tag = "v29_10"
maker_task = CondorTask(
sample = DirectorySample(
dataset=dsname,
location=samploc,
),
# open_dataset = True,
files_per_output = 1,
tag = tag,
cmssw_version = cmsswver,
scram_arch = scramarch,
tarfile = tarfile,
executable = "condor_executable.sh",
outdir_name = "stopBaby_" + dsname,
output_name = "stopbaby.root",
arguments = "1" if "SMS" in dsname else "0", # isFastsim
# condor_submit_params = {"sites": "UAF,T2_US_UCSD,UCSB"},
condor_submit_params = {"sites": "T2_US_UCSD,UCSB"},
# condor_submit_params = {"use_xrootd": True},
# no_load_from_backup = True,
)
merge_task = CondorTask(
sample = DirectorySample(
dataset="merge_"+dsname,
location=maker_task.get_outputdir(),
),
# open_dataset = True, flush = True,
# Make a CondorTask (3 in total, one for each input)
maker_tasks = []
merge_tasks = []
for dsname in dataset_names:
cmsswver = "CMSSW_9_4_0_pre2"
scramarch = "slc6_amd64_gcc700"
tarfile = "input.tar.gz"
tag = "v24_3"
maker_task = CondorTask(
sample = SNTSample(dataset=dsname),
files_per_output = 1,
tag = tag,
outdir_name = "stopBaby_" + dsname[5:34].strip("_"),
output_name = "stopbaby.root",
executable = "condor_executable.sh",
cmssw_version = cmsswver,
scram_arch = scramarch,
arguments = "1", # isFastsim
tarfile = tarfile,
condor_submit_params = {"sites": "UAF,T2_US_UCSD"},
)
merge_task = CondorTask(
sample = DirectorySample(
dataset=dsname.replace("MINIAODSIM","MERGE"),
location=maker_task.get_outputdir(),
),
executable = "ProjectMetis/metis/executables/condor_hadd.sh",
open_dataset = False,
files_per_output = 12,
outdir_name = "merged_stopBaby_" + dsname[5:34].strip("_"),
def main():
main_dir = os.path.dirname(os.path.abspath(__file__))
metis_path = os.path.dirname(os.path.dirname(metis.__file__))
exec_path = main_dir + "/metis.sh"
hadoop_path = "metis/"
metis_dashboard_path = os.path.join(metis_path, "dashboard")
job_tag = ""
total_summary = {}
os.chdir(metis_path)
while True:
masspoints = [125]
tasks = []
for mass in masspoints:
miniaod = CondorTask(
## Dummy sample as no input is needed in generating the events
#sample = DummySample(
# N=3000,
# dataset="/VHToNonbb_M125_13TeV_amcatnloFXFX_madspin_pythia8/PRIVATE-RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6_ext1-v1/MINIAODSIM"
# ),
# Dummy sample as no input is needed in generating the events
sample = DummySample(
N=3500,
dataset="/WWW_4F_TuneCP5_13TeV-amcatnlo-pythia8/PRIVATE-RunIIFall17MiniAODv2-PU2017_12Apr2018_94X_mc2017_realistic_v14-v1/MINIAODSIM"
),
tag = job_tag,
executable = exec_path,
special_dir = hadoop_path + "/private_miniaod",
output_name = "output.root",
files_per_output = 1,
condor_submit_params = {"sites" : "T2_US_UCSD"},
open_dataset = True,
flush = True,
#no_load_from_backup = True,
)
tasks.extend([miniaod])
all_tasks_complete = True
for task in tasks:
task.process()
summary = task.get_task_summary()
total_summary[task.get_sample().get_datasetname()] = summary
all_tasks_complete = all_tasks_complete and task.complete()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary, webdir=metis_dashboard_path).do()
# Print msummary table so I don't have to load up website
os.system("msummary -v0 -p ext2 | tee summary.txt")
os.system("chmod -R 755 {}".format(metis_dashboard_path))
# If all done exit the loop
if all_tasks_complete:
print ""
print "Job={} finished".format(job_tag)
print ""
break
# Neat trick to not exit the script for force updating
print 'Press Ctrl-C to force update, otherwise will sleep for 300 seconds'
try:
for i in range(0,300):
sleep(1) # could use a backward counter to be preeety :)
except KeyboardInterrupt:
raw_input("Press Enter to force update, or Ctrl-C to quit.")
print "Force updating..."
child_tasks = []
for srf in srlst:
fname = srf.split('/')[-1]
fdir = '/'.join(srf.split('/')[:-1])
print(fdir, fname)
ifile = fname.split('.')[0].split('_')[-1]
maker_task = CondorTask(
sample = DirectorySample(
dataset=dsname+'_'+ifile,
location=fdir,
globber=fname,
),
# open_dataset = True,
files_per_output = 1,
tag = tag,
cmssw_version = cmsswver,
scram_arch = scramarch,
tarfile = tarfile,
executable = "condor_executable.sh",
outdir_name = "resub_"+dsname,
output_name = "stopbaby_"+ifile+".root",
arguments = "1" if "SMS" in dsname else "0", # isFastsim
condor_submit_params = {"sites": "UAF,T2_US_UCSD"},
# no_load_from_backup = True,
)
# print(maker_task.get_sample().get_datasetname())
# print(maker_task.get_sample().get_files())
# print(maker_task.get_outputdir())
# print(maker_task.get_io_mapping())
child_tasks.append(maker_task)
#raise NotImplementedError
isData = 1 if isData else 0
isFast = 1 if isFast else 0
if not dryRun:
maker_task = CondorTask(
sample=sample,
#'/hadoop/cms/store/user/dspitzba/nanoAOD/TTWJetsToLNu_TuneCP5_13TeV-amcatnloFXFX-madspin-pythia8__RunIIAutumn18NanoAODv6-Nano25Oct2019_102X_upgrade2018_realistic_v20_ext1-v1/',
# open_dataset = True, flush = True,
executable="executable.sh",
arguments="WHv1p2 %s %s %s" % (year, isData, isFast),
#tarfile = "merge_scripts.tar.gz",
files_per_output=1,
#output_dir = os.path.join(outDir, sample.get_datasetname()),
outdir_name="stopNano_" + s,
output_name="stopNano.root",
output_is_tree=True,
# check_expectedevents = True,
tag=tag,
condor_submit_params={"sites": "T2_US_UCSD,UAF"},
cmssw_version="CMSSW_10_2_9",
scram_arch="slc6_amd64_gcc700",
# recopy_inputs = True,
# no_load_from_backup = True,
min_completion_fraction=0.99,
)
maker_tasks.append(maker_task)
if not dryRun:
merge_task = CondorTask(
def submit_metis(job_tag, samples_map, sample_list=[], arguments_map="", exec_script="metis.sh", tar_files=[], hadoop_dirname="testjobs", files_per_output=1, globber="*.root", sites="T2_US_UCSD"):
import time
import json
import metis
from time import sleep
from metis.Sample import DirectorySample
from metis.CondorTask import CondorTask
from metis.StatsParser import StatsParser
import os
import glob
import subprocess
# file/dir paths
main_dir = os.getcwd()
metis_path = os.path.dirname(os.path.dirname(metis.__file__))
tar_path = os.path.join(metis_path, "package.tar")
tar_gz_path = tar_path + ".gz"
metis_dashboard_path = os.path.join(metis_path, "dashboard")
exec_path = os.path.join(main_dir, exec_script)
hadoop_path = "metis/{}/{}".format(hadoop_dirname, job_tag) # The output goes to /hadoop/cms/store/user/$USER/"hadoop_path"
# Create tarball
os.chdir(main_dir)
print os.getcwd()
print "tar -chzf {} {}".format(tar_gz_path, " ".join(tar_files))
os.system("tar -chzf {} {}".format(tar_gz_path, " ".join(tar_files)))
# Change directory to metis
os.chdir(metis_path)
total_summary = {}
# if no sample_list is provided then we form it via the keys of the samples_map
if len(sample_list) == 0:
for key in samples_map:
sample_list.append(key)
samples_to_run = []
for key in sample_list:
samples_to_run.append(
DirectorySample(
dataset=key,
location=samples_map[key],
globber=globber,
)
)
files_per_output_config_list = []
if isinstance(files_per_output, dict):
for key in sample_list:
files_per_output_config_list.append(files_per_output[key])
else:
for key in sample_list:
files_per_output_config_list.append(files_per_output)
# Loop over datasets to submit
while True:
all_tasks_complete = True
#for sample in sorted(samples):
for index, sample in enumerate(samples_to_run):
# define the task
maker_task = CondorTask(
sample = sample,
tag = job_tag,
arguments = arguments_map[sample.get_datasetname()] if arguments_map else "",
executable = exec_path,
tarfile = tar_gz_path,
special_dir = hadoop_path,
output_name = "output.root",
files_per_output = files_per_output_config_list[index],
condor_submit_params = {"sites" : sites},
open_dataset = False,
flush = True,
#no_load_from_backup = True,
)
# process the job (either submits, checks for resubmit, or finishes etc.)
maker_task.process()
# save some information for the dashboard
total_summary["["+job_tag+"] "+maker_task.get_sample().get_datasetname()] = maker_task.get_task_summary()
# Aggregate whether all tasks are complete
all_tasks_complete = all_tasks_complete and maker_task.complete()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary, webdir=metis_dashboard_path).do()
# Print msummary table so I don't have to load up website
os.system("msummary -r -p {} | tee summary.txt".format(job_tag))
os.system("chmod -R 755 {}".format(metis_dashboard_path))
os.system("chmod 644 {}/images/*".format(metis_dashboard_path))
# If all done exit the loop
if all_tasks_complete:
print ""
print "Job={} finished".format(job_tag)
print ""
break
# Neat trick to not exit the script for force updating
print 'Press Ctrl-C to force update, otherwise will sleep for 300 seconds'
try:
for i in range(0,60):
sleep(1) # could use a backward counter to be preeety :)
except KeyboardInterrupt:
raw_input("Press Enter to force update, or Ctrl-C to quit.")
print "Force updating..."
os.chdir(main_dir)
if "VHToGG" in dataset:
print "Skipping VH for now"
continue
#if not ("Run2016" in dataset or "Run2017" in dataset or "Run2018" in dataset):
# continue
if "ttHiggs" in dataset:
print("Submitting jobs for: ", dataset)
sample = DirectorySample(dataset=dataset, location=info["input_loc"])
files = [f.name for f in sample.get_files()]
print(len(files))
task = CondorTask(sample=sample,
open_dataset=False,
flush=True,
files_per_output=info["fpo"],
output_name="merged_ntuple.root",
tag=job_tag,
cmssw_version=cmssw_ver,
executable=exec_path,
tarfile=tar_path,
condor_submit_params={"sites": "T2_US_UCSD"},
special_dir=hadoop_path,
arguments=info["meta_conditions"])
task.process()
if not task.complete():
allcomplete = False
# save some information for the dashboard
total_summary[dataset] = task.get_task_summary()
# parse the total summary and write out the dashboard
StatsParser(data=total_summary,
webdir="~/public_html/dump/ttH_BabyMaker/").do()
os.system("chmod -R 755 ~/public_html/dump/ttH_BabyMaker")
if allcomplete:
import os
import time
from metis.CondorTask import CondorTask
from metis.Sample import DBSSample
from metis.StatsParser import StatsParser
if not os.path.exists("inputs.tar.gz"):
os.system("tar cvzf inputs.tar.gz looper.py")
for i in range(100):
total_summary = {}
for dataset in [
"/DYJetsToLL_M-4to50_HT-100to200_TuneCP5_PSweights_13TeV-madgraphMLM-pythia8/RunIIAutumn18NanoAODv7-Nano02Apr2020_102X_upgrade2018_realistic_v21-v1/NANOAODSIM",
]:
task = CondorTask(
sample = DBSSample(dataset=dataset),
events_per_output = 1e6,
output_name = "output.root",
tag = "nanotestv1",
cmssw_version = "CMSSW_10_2_5",
scram_arch = "slc6_amd64_gcc700",
tarfile = "inputs.tar.gz",
executable = "condor_nano_exe.sh",
)
task.process()
total_summary[task.get_sample().get_datasetname()] = task.get_task_summary()
StatsParser(data=total_summary, webdir="~/public_html/dump/metis_nanotest/").do()
time.sleep(30*60)