def test2(): dataset_names = [ # "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM", "/Dummy_test_StopBabyMaker_v25/CMS4", ] # Make a base directory basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format(os.getenv("USER")) MutableFile(basedir).touch() # Make a directory sample, giving it the location and a dataset name for bookkeeping purposes # The globber must be customized (by default, it is *.root) in order to pick up the text files ds = DirectorySample(location=basedir, dataset="/TEST/Examplev1/TEST", globber="*.txt") # Make a CondorTask (3 in total, one for each input) task = CondorTask( sample = ds, files_per_output = 1, tag = "v0", output_name = "ttbar_powheg_pythia8_92X.root", executable = "condor_executable.sh", cmssw_version = "CMSSW_9_3_1", scram_arch = "slc6_amd64_gcc700", arguments = "testarg1", tarfile = "input.tar.gz", condor_submit_params = {"sites": "UAF,T2_US_UCSD,UCSB"}, no_load_from_backup = True, # for the purpose of the example, don't use a backup ) # do_cmd("rm -rf {0}".format(task.get_outputdir())) # Process and sleep until complete is_complete = False for t in [5.0, 5.0, 10.0, 15.0, 20.0]: task.process() print("Sleeping for {0} seconds".format(int(t))) time.sleep(t) is_complete = task.complete() if is_complete: break # If it's complete, make a dummy sample out of the output directory # in order to pick up the files. Then cat out the contents and sum # them up. This should be 3*2*10 = 100 if is_complete: print("Job completed! Checking outputs...") outsamp = DirectorySample(location=task.get_outputdir(), dataset="/Blah/blah/BLAH", globber="*.txt") tot = 0 for f in outsamp.get_files(): mf = MutableFile(f.get_name()) tot += int(mf.cat()) print("It looks like we found 3*2*10 = {0}".format(tot))
def setUpClass(cls): super(CondorTaskTest, cls).setUpClass() # make a test directory and touch some root files and executable there basedir = "/tmp/{0}/metis/condortask_test/".format(os.getenv("USER")) Utils.do_cmd("mkdir -p {0}".format(basedir)) for i in range(1, cls.nfiles + 1): Utils.do_cmd("touch {0}/input_{1}.root".format(basedir, i)) Utils.do_cmd("echo hello > {0}/executable.sh".format(basedir)) # make dummy CondorTask with the files we # touched in the basedir, and chunk # the outputs logging.getLogger("logger_metis").disabled = True cls.dummy = CondorTask(sample=DirectorySample( location=basedir, globber="*.root", dataset="/test/test/TEST", ), open_dataset=False, files_per_output=cls.files_per_job, cmssw_version=cls.cmssw, tag=cls.tag, executable="{0}/executable.sh".format(basedir)) # prepare inputs and run, # but pretend like outputs exist and don't submit cls.dummy.prepare_inputs() # run once to "submit to condor" and "create outputs" (set_fake) cls.dummy.run(fake=True) # run again to recognize that all outputs are there and # we can then declare completion cls.dummy.run(fake=True)
def get_data_tasks(info): tasks = [] for reqname, tag, extra_args in info: task = CondorTask( sample=DirectorySample( location= "/hadoop/cms/store/user/namin/ScoutingCaloMuon/crab_{}/*/*/". format(reqname), dataset="/ScoutingCaloMuon/Run2018{}/RAW".format(reqname)), output_name="output.root", executable="executables/scouting_exe.sh", tarfile="package.tar.gz", MB_per_output=4000, condor_submit_params={ "sites": "T2_US_UCSD", "classads": [ ["metis_extraargs", extra_args], ["JobBatchName", reqname], ], "requirements_line": 'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})' .format(extra_requirements=extra_requirements), }, cmssw_version="CMSSW_10_2_5", scram_arch="slc6_amd64_gcc700", tag=tag, ) tasks.append(task) return tasks
def test_completion_fraction(self): # Make dummy task with no inputs # and require min completion fraction to be 0 logging.getLogger("logger_metis").disabled = True dummy = CondorTask( sample=DirectorySample( location=".", globber="*.fake", dataset="/testprocess/testprocess/TEST", ), open_dataset=False, files_per_output=1, cmssw_version="CMSSW_8_0_20", tag="vtest", no_load_from_backup=True, min_completion_fraction=0., ) dummy.process() self.assertEqual(dummy.complete(), True)
def test1(): dataset_names = [ # "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM", "/Dummy_test_StopBabyMaker_v25/CMS4", ] basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format( os.getenv("USER")) for i in range(10000): total_summary = {} for dsname in dataset_names: try: task = CondorTask( sample=DirectorySample( location=basedir, dataset=dsname, ), files_per_output=1, output_name="merged_ntuple.root", tag="v25", scram_arch="slc6_amd64_gcc700", executable="condor_executable.sh", arguments="testarg1", tarfile="input.tar.gz", condor_submit_params={"sites": "UAF,T2_US_UCSD,UCSB"}, no_load_from_backup= True, # for the purpose of the example, don't use a backup ) task.process() except: traceback_string = traceback.format_exc() print("Runtime error:\n{0}".format(traceback_string)) # send_email(subject="metis error", body=traceback_string) total_summary[dsname] = task.get_task_summary() StatsParser(data=total_summary, webdir="~/public_html/dump/metis/").do() time.sleep(5)
def test1(): dataset_names = [ # "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM", "/Dummy_test_StopBabyMaker_v25/CMS4", ] basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format(os.getenv("USER")) for i in range(10000): total_summary = {} for dsname in dataset_names: try: task = CondorTask( sample = DirectorySample( location=basedir, dataset=dsname, ), files_per_output = 1, output_name = "merged_ntuple.root", tag = "v25", scram_arch = "slc6_amd64_gcc700", executable = "condor_executable.sh", arguments = "testarg1", tarfile = "input.tar.gz", condor_submit_params = {"sites": "UAF,T2_US_UCSD,UCSB"}, no_load_from_backup = True, # for the purpose of the example, don't use a backup ) task.process() except: traceback_string = traceback.format_exc() print("Runtime error:\n{0}".format(traceback_string)) # send_email(subject="metis error", body=traceback_string) total_summary[dsname] = task.get_task_summary() StatsParser(data=total_summary, webdir="~/public_html/dump/metis/").do() time.sleep(5)
def submit_jobs(output_directory, tag, samples, events_total, events_per_output, tarfile): # Long loop to monitor jobs and resubmit failed jobs for i in range(10000): total_summary = {} # Loop through samples for sample in samples: config_info = samples_config[sample] executable = create_executable(tag, sample, config_info) task = CondorTask(sample=DummySample( N=int(float(events_total) / float(events_per_output)), nevents=events_total, dataset="/" + sample + "_NANO"), tag=tag, special_dir=output_directory, events_per_output=events_per_output, total_nevents=events_total, split_within_files=True, executable=executable, open_dataset=False, tarfile=tarfile, condor_submit_params=CONDOR_SUBMIT_PARAMS) task.process() total_summary[ task.get_sample().get_datasetname()] = task.get_task_summary() StatsParser(data=total_summary, webdir="~/public_html/dump/Hgg-MC-Production/").do() time.sleep(300) # power nap
def test_flush(self): basedir = "/tmp/{0}/metis/condortask_testflush/".format( os.getenv("USER")) Utils.do_cmd("mkdir -p {0}".format(basedir)) tag = "vflush" for i in range(1, self.nfiles + 1): Utils.do_cmd("touch {0}/input_{1}.root".format(basedir, i)) dummy = CondorTask( sample=DirectorySample( location=basedir, globber="*.root", dataset="/test/test/TEST", ), open_dataset=True, files_per_output=self.files_per_job, cmssw_version=self.cmssw, tag=tag, ) self.assertEqual(len(dummy.get_outputs()), (self.nfiles // self.files_per_job)) dummy.flush() self.assertEqual(len(dummy.get_outputs()), (self.nfiles // self.files_per_job + 1))
def submit(which): total_summary = {} extra_requirements = "true" tag = "v0_noFilter" pdname = "HVNoFilter" events_per_point = int(1E4) events_per_job = int(100) cfgsDir = "psets_gensim_noFilter/" modelsFile = cfgsDir + "/models.txt" df = pd.read_csv(modelsFile) for year in ["2016", "2017", "2018"]: for iterr, row in df.iterrows(): # fmass = float(mass) # mass = str(mass).replace(".","p") epp = int(events_per_point) reqname = "noFilter_m{}{}_ctau{}_xi_{}_{}_{}".format( row.portal, row.mass, row.ctau, row.xi, tag, year) njobs = epp // events_per_job sample = DummySample( dataset="/{}/params_{}_m_{}_ctau_{}mm_xi_{}_{}/LLPNTUPLE". format(pdname, row.portal, row.mass, row.ctau * 10, row.xi, year), N=njobs, nevents=epp) task = CondorTask( sample=sample, output_name="output.root", executable="executables/condor_executable_{}.sh".format(which), tarfile="package_{}.tar.xz".format(year), open_dataset=True, files_per_output=1, condor_submit_params={ "classads": [ ["param_mass", row.mass], ["param_ctau", row.ctau], ["param_xi", str(row.xi).replace(".", "p")], ["param_portal", row.portal], ["param_year", year], ["param_nevents", events_per_job], ["metis_extraargs", ""], ["JobBatchName", reqname], ], "requirements_line": 'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})' .format(extra_requirements=extra_requirements), }, tag=tag, recopy_inputs=True) task.process() total_summary[ task.get_sample().get_datasetname()] = task.get_task_summary() StatsParser(data=total_summary, webdir="~/public_html/dump/metis_test/").do()
def test2(): dataset_names = [ # "/TT_TuneCUETP8M2T4_13TeV-powheg-pythia8/RunIISummer17MiniAOD-92X_upgrade2017_realistic_v10_ext1-v1/MINIAODSIM", "/Dummy_test_StopBabyMaker_v25/CMS4", ] # Make a base directory basedir = "/hadoop/cms/store/user/{0}/metis_test/example/".format( os.getenv("USER")) MutableFile(basedir).touch() # Make a directory sample, giving it the location and a dataset name for bookkeeping purposes # The globber must be customized (by default, it is *.root) in order to pick up the text files ds = DirectorySample(location=basedir, dataset="/TEST/Examplev1/TEST", globber="*.txt") # Make a CondorTask (3 in total, one for each input) task = CondorTask( sample=ds, files_per_output=1, tag="v0", output_name="ttbar_powheg_pythia8_92X.root", executable="condor_executable.sh", cmssw_version="CMSSW_9_3_1", scram_arch="slc6_amd64_gcc700", arguments="testarg1", tarfile="input.tar.gz", condor_submit_params={"sites": "UAF,T2_US_UCSD,UCSB"}, no_load_from_backup= True, # for the purpose of the example, don't use a backup ) # do_cmd("rm -rf {0}".format(task.get_outputdir())) # Process and sleep until complete is_complete = False for t in [5.0, 5.0, 10.0, 15.0, 20.0]: task.process() print("Sleeping for {0} seconds".format(int(t))) time.sleep(t) is_complete = task.complete() if is_complete: break # If it's complete, make a dummy sample out of the output directory # in order to pick up the files. Then cat out the contents and sum # them up. This should be 3*2*10 = 100 if is_complete: print("Job completed! Checking outputs...") outsamp = DirectorySample(location=task.get_outputdir(), dataset="/Blah/blah/BLAH", globber="*.txt") tot = 0 for f in outsamp.get_files(): mf = MutableFile(f.get_name()) tot += int(mf.cat()) print("It looks like we found 3*2*10 = {0}".format(tot))
def submit(): requests = { #'TTWJetsToLNuEWK_5f_NLO': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_NLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball_retired.tar.xz', # that's the SM point, but using the SMEFT model. No lepton filtering, so name is actually confusing #'TTWJetsToLNuEWK_5f_NLO_v2': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_NLO_slc7_amd64_gcc730_CMSSW_9_3_16_tarball.tar.xz', # that's the actual SM #'TTWplusJetsToLNuEWK_5f_NLO_v2': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWplusJetsToLNuEWK_5f_NLO_slc7_amd64_gcc730_CMSSW_9_3_16_tarball.tar.xz', # that's the actual SM #'TTWminusJetsToLNuEWK_5f_NLO_v2': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWminusJetsToLNuEWK_5f_NLO_slc7_amd64_gcc730_CMSSW_9_3_16_tarball.tar.xz', # that's the actual SM #'TTWJetsToLNuEWK_5f_EFT_myNLO_full': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_cpt8_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # one of the BSM points #'TTWJetsToLNuEWK_5f_EFT_mix_myNLO_full': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # EFT mix 'TTWJetsToLNuEWK_5f_EFT_cpq3_4_myNLO_full': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks//TTWJetsToLNuEWK_5f_EFT_myNLO_cpq3_4_slc7_amd64_gcc730_CMSSW_9_3_16_tarball.tar.xz', # C_pq3 = 4 } total_summary = {} extra_requirements = "true" tag = "v4" events_per_point = 500000 # produced 500k events before events_per_job = 2000 # up to 2000 works #events_per_point = 50 #events_per_job = 10 njobs = int(events_per_point)//events_per_job for reqname in requests: gridpack = requests[reqname] #reqname = "TTWJetsToLNuEWK_5f_EFT_myNLO" #gridpack = '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz' task = CondorTask( sample = DummySample(dataset="/%s/RunIIAutumn18/NANO"%reqname,N=njobs,nevents=int(events_per_point)), output_name = "nanoAOD.root", executable = "executables/condor_executable_Autumn18.sh", tarfile = "package.tar.gz", #scram_arch = "slc7_amd64_gcc630", open_dataset = False, files_per_output = 1, arguments = gridpack, condor_submit_params = { "sites":"T2_US_UCSD", # "classads": [ ["param_nevents",events_per_job], ["metis_extraargs",""], ["JobBatchName",reqname], #["SingularityImage", "/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel6-m202006"], ], "requirements_line": 'Requirements = (HAS_SINGULARITY=?=True)' # && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})'.format(extra_requirements=extra_requirements), }, tag = tag, min_completion_fraction = 0.90, ) task.process() total_summary[task.get_sample().get_datasetname()] = task.get_task_summary() StatsParser(data=total_summary, webdir="~/public_html/dump/tW_gen/").do()
def get_tasks(infos): tasks = [] for info in infos: location = info["location"] dataset = info["dataset"] isdata = info["isdata"] open_dataset = info.get("open_dataset", False) tag = info["tag"] extra_args = info.get("extra_args", "") kwargs = {} if isdata: kwargs["MB_per_output"] = (4000 if "skim1cm" in extra_args else 1000) batchname = dataset.split("_", 1)[-1].split("/")[0] + "_" + tag else: kwargs["files_per_output"] = 200 batchname = "_".join( dataset.split("params_", 1)[1].split("/", 1)[0].split("_")[:2] + [tag]) task = CondorTask( sample=DirectorySample(location=location, dataset=dataset), open_dataset=open_dataset, flush=True, output_name="output.root", executable="executables/scouting_exe.sh", tarfile="package.tar.gz", condor_submit_params={ "container": "/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel6-m202006", "sites": "T2_US_UCSD", "classads": [ ["metis_extraargs", extra_args], ["JobBatchName", batchname], ], "requirements_line": 'Requirements = ((HAS_SINGULARITY=?=True) && {extra_requirements})' .format(extra_requirements=extra_requirements), }, cmssw_version="CMSSW_10_2_5", scram_arch="slc6_amd64_gcc700", tag=tag, **kwargs) tasks.append(task) return tasks
def test_full(self): """ Touch a root file ("input") Submit condor jobs to touch output files for each input file and copy them to hadoop Jobs get submitted to local universe for speed reasons Check output to make sure job completed """ njobs = 2 cmssw = "CMSSW_8_0_21" basedir = "/tmp/{0}/metis/condortask_testfull/".format( os.getenv("USER")) Utils.do_cmd("mkdir -p {0}".format(basedir)) tag = "vfull" for i in range(1, njobs + 1): Utils.do_cmd("touch {0}/input_{1}.root".format(basedir, i)) logging.getLogger("logger_metis").disabled = True dummy = CondorTask( sample=DirectorySample( location=basedir, globber="*.root", dataset="/test/test/TEST", ), open_dataset=False, files_per_output=1, cmssw_version=cmssw, executable=Utils.metis_base() + "metis/executables/condor_test_exe.sh", tag=tag, condor_submit_params={"universe": "local"}, no_load_from_backup=True, ) # clean up previous directory Utils.do_cmd("rm -rf {0}".format(dummy.get_outputdir())) is_complete = False for t in [1.0, 1.0, 2.0, 3.0, 5.0, 10.0, 20.0]: dummy.process() time.sleep(t) is_complete = dummy.complete() if is_complete: break self.assertEquals(is_complete, True) self.assertEqual(njobs, len(glob.glob(dummy.get_outputdir() + "/*")))
def submit(): requests = { 'TTWJetsToLNuEWK_5f_EFT_myNLO': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # that's EFT 'TTWJetsToLNuEWK_5f_EFT_myNLO_cpt8': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_cpt8_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # that's EFT 'TTWJetsToLNuEWK_5f_NLO': '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_NLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz', # that's the SM } total_summary = {} extra_requirements = "true" tag = "v1" events_per_point = 500000 events_per_job = 10000 #events_per_point = 500 #events_per_job = 100 njobs = int(events_per_point)//events_per_job for reqname in requests: gridpack = requests[reqname] #reqname = "TTWJetsToLNuEWK_5f_EFT_myNLO" #gridpack = '/hadoop/cms/store/user/dspitzba/tW_scattering/gridpacks/TTWJetsToLNuEWK_5f_EFT_myNLO_slc6_amd64_gcc630_CMSSW_9_3_16_tarball.tar.xz' task = CondorTask( sample = DummySample(dataset="/%s/RunIIAutumn18/NANOGEN"%reqname,N=njobs,nevents=int(events_per_point)), output_name = "output.root", executable = "executables/condor_executable.sh", tarfile = "package.tar.gz", open_dataset = False, files_per_output = 1, arguments = gridpack, condor_submit_params = { "sites":"T2_US_UCSD", # "classads": [ ["param_nevents",events_per_job], ["metis_extraargs",""], ["JobBatchName",reqname], ], "requirements_line": 'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})'.format(extra_requirements=extra_requirements), }, tag = tag, ) task.process() total_summary[task.get_sample().get_datasetname()] = task.get_task_summary() StatsParser(data=total_summary, webdir="~/public_html/dump/tW_gen/").do()
def get_signal_tasks(pattern, tag, extra_args="", func_dataset=None): samples = [] for location in glob.glob(pattern): if func_dataset is not None: dataset = func_dataset(location) else: taskname = location.rstrip("/").rsplit("/")[-1] dataset = "/{}/{}/BABY".format( taskname.split("_", 1)[0], taskname.split("_", 1)[1].split("_RAWSIM")[0], ) samples.append(DirectorySample(location=location, dataset=dataset)) tasks = [] for sample in samples: task = CondorTask( sample=sample, output_name="output.root", executable="executables/scouting_exe.sh", tarfile="package.tar.gz", files_per_output=int(1e5), condor_submit_params={ "sites": "T2_US_UCSD", "classads": [ ["metis_extraargs", extra_args], [ "JobBatchName", sample.get_datasetname().split("/")[2].replace( "params_", "") ], ], "requirements_line": 'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})' .format(extra_requirements=extra_requirements), }, cmssw_version="CMSSW_10_2_5", scram_arch="slc6_amd64_gcc700", tag=tag, ) tasks.append(task) return tasks
def get_bg_tasks(tag, extra_args=""): samples = [ # DBSSample(dataset="/JpsiToMuMu_JpsiPt8_TuneCP5_13TeV-pythia8/RunIIAutumn18DRPremix-102X_upgrade2018_realistic_v15-v1/AODSIM") DBSSample( dataset= "/BuToKJpsi_ToMuMu_MuFilter_SoftQCDnonD_TuneCP5_13TeV-pythia8-evtgen/RunIIAutumn18DR-PUPoissonAve20_102X_upgrade2018_realistic_v15-v2/AODSIM" ) ] tasks = [] for sample in samples: task = CondorTask( sample=sample, output_name="output.root", executable="executables/scouting_exe.sh", tarfile="package.tar.gz", events_per_output=500e3, condor_submit_params={ "sites": "T2_US_UCSD", "classads": [ ["metis_extraargs", extra_args], [ "JobBatchName", sample.get_datasetname().split("/")[1].split("_")[0] + "_" + tag ], ], "requirements_line": 'Requirements = ((HAS_SINGULARITY=?=True) && (HAS_CVMFS_cms_cern_ch =?= true) && {extra_requirements})' .format(extra_requirements=extra_requirements), }, cmssw_version="CMSSW_10_2_5", scram_arch="slc6_amd64_gcc700", tag=tag, ) tasks.append(task) return tasks
"/TTWJetsToLNu_TuneCP5_PSweights_13TeV-amcatnloFXFX-madspin-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "ttwlv", "/TTZToLLNuNu_M-10_TuneCP5_PSweights_13TeV-amcatnlo-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "ttzllvv", "/ttHTobb_ttToSemiLep_M125_TuneCP5_13TeV-powheg-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "tthbb", "/WpWpJJ_EWK-QCD_TuneCUETP8M1_13TeV-madgraph-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "ssww", "/WZTo3LNu_TuneCUETP8M1_13TeV-powheg-pythia8/RunIISummer16NanoAODv6-PUMoriond17_Nano25Oct2019_102X_mcRun2_asymptotic_v7-v1/NANOAODSIM" : "wz3lv", } # submission tag tag = "v8" for dsname, shortname in sample_map.items(): task = CondorTask( sample = DBSSample( dataset=dsname, # tag="CMS4_V09-04-13", # if not specified, get latest CMS4 tag ), files_per_output = 1, output_name = "output.root", tag = tag, condor_submit_params = {"sites": "T2_US_UCSD", "use_xrootd":True}, cmssw_version = "CMSSW_10_5_0", input_executable = "condor_executable_metis.sh", # your condor executable here tarfile = "/nfs-7/userdata/phchang/VBSHWWNanoSkimmer_v1.package.tar.gz", # your tarfile with assorted goodies here special_dir = "VBSHWWNanoSkim/{}".format(tag), # output files into /hadoop/cms/store/<user>/<special_dir> ) # Straightforward logic if not task.complete(): task.process() break
# submission tag tag = "fcnc_v10_SRonly_11june2021" task_list = [] for sample in samples: task = CondorTask( sample=sample, files_per_output=1, output_name="output.root", tag=tag, condor_submit_params={ "sites": "T2_US_UCSD", "use_xrootd": True }, cmssw_version="CMSSW_10_2_9", input_executable= "condor_executable_metis.sh", # your condor executable here # tarfile = "/hadoop/cms/store/user/ksalyer/FCNC_NanoSkimmer_v2.package.tar.gz", # your tarfile with assorted goodies here tarfile= "/hadoop/cms/store/user/dspitzba/FCNC/FCNC_NanoSkimmer_v10_SROnly.package.tar.gz", # tarfile = "/nfs-7/userdata/phchang/VBSHWWNanoSkimmer_v1.package.tar.gz", special_dir="FCNC_NanoSkim/{}".format( tag ), # output files into /hadoop/cms/store/<user>/<special_dir> # special_dir = "VBSHWWNanoSkim/{}".format(tag), # output files into /hadoop/cms/store/<user>/<special_dir> ) task_list.append(task) # Straightforward logic # if not task.complete(): # task.process() for i in range(100):
# Loop through central samples for ds, fpo, args in dsdefs[:]: if (job_filter != "") and (args not in job_filter): continue sample = DBSSample(dataset=ds) print(ds, args) task = CondorTask( sample=sample, open_dataset=False, files_per_output=fpo, output_name="test_rawreco.root", tag=job_tag, cmssw_version=cmssw_ver, executable=exec_path, tarfile="./package.tar.gz", condor_submit_params={ "sites": "T2_US_UCSD", "container": "/cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel7-m20201010" }, #condor_submit_params = {"sites" : "T2_US_UCSD,T2_US_CALTECH,T2_US_MIT,T2_US_WISCONSIN,T2_US_Nebraska,T2_US_Purdue,T2_US_Vanderbilt,T2_US_Florida"}, special_dir=hadoop_path, arguments=args.replace(" ", "|")) task.process() allcomplete = allcomplete and task.complete() # save some information for the dashboard total_summary[ds] = task.get_task_summary() with open("summary.json", "w") as f_out: json.dump(total_summary, f_out, indent=4, sort_keys=True)
total_summary = {} while True: allcomplete = True for ds,loc,fpo in dslocs: print(loc) sample = DirectorySample( dataset=ds, location=loc ) files = [f.name for f in sample.get_files()] sample.set_files([f.name for f in sample.get_files() if "/cms/store/user/bemarsh/flashgg/MicroAOD/forHualin_2017/GJet_Pt-40toInf_DoubleEMEnriched_MGG-80toInf_TuneCP5_13TeV_Pythia8_RunIIFall17MiniAODv2-PU2017_12Apr2018_94X_mc2017_realistic_v14-v2_MINIAODSIM_forHualin_2017/test_skim_630.root" not in f.name]) #print [f.name for f in sample.get_files()] task = CondorTask( sample = sample, open_dataset = False, flush = True, files_per_output = fpo, output_name = "merged_ntuple.root", tag = job_tag, cmssw_version = cmssw_ver, # doesn't do anything executable = exec_path, tarfile = tar_path, condor_submit_params = {"sites" : "T2_US_UCSD"}, special_dir = hadoop_path ) task.process() if not task.complete(): allcomplete = False # save some information for the dashboard total_summary[ds] = task.get_task_summary() # parse the total summary and write out the dashboard #StatsParser(data=total_summary, webdir="~/public_html/dump/ttH_BabyMaker/").do() os.system("chmod -R 755 ~/public_html/dump/ttH_BabyMaker") if allcomplete:
fpo = jobs["processes"][pid]["filesPerOutput"] for ds in jobs["processes"][pid]["datasets"]: args = getArgs(pid, ds) dsdefs.append((ds, fpo, args)) total_summary = {} while True: allcomplete = True for ds, fpo, args in dsdefs[:]: sample = DBSSample(dataset=ds) task = CondorTask(sample=sample, open_dataset=False, files_per_output=fpo, output_name="test_skim.root" if DOSKIM else "myMicroAODOutputFile.root", tag=job_tag, executable=exec_path, tarfile=tar_path, condor_submit_params={"sites": "T2_US_UCSD"}, special_dir=hadoop_path, arguments=args.replace(" ", "|")) task.process() allcomplete = allcomplete and task.complete() # save some information for the dashboard total_summary[ds] = task.get_task_summary() # parse the total summary and write out the dashboard StatsParser(data=total_summary, webdir="~/public_html/dump/metis_microaod_80x/").do() os.system("chmod -R 755 ~/public_html/dump/metis_microaod_80x") if allcomplete: print ""
tag = "v15" tarfile = "/nfs-7/userdata/phchang/VBSHWWNanoSkimmer_v8_CMSSW_10_2_13_slc7_amd64_gcc700.package.tar.gz" # >= 2 leptons of any charge (35 GeV each) + njets_20 >= 4 task_summary = {} for sample in samples: task = CondorTask( sample=sample, files_per_output=1, output_name="output.root", tag=tag, # condor_submit_params = {"sites": "T2_US_UCSD", "use_xrootd":True, "classads": [ ["metis_extraargs", "fetch_nano"] ]}, condor_submit_params={ "sites": "T2_US_UCSD", "use_xrootd": True }, cmssw_version="CMSSW_10_2_13", scram_arch="slc7_amd64_gcc700", input_executable= "condor_executable_metis.sh", # your condor executable here tarfile=tarfile, # your tarfile with assorted goodies here special_dir="VBSHWWNanoSkim/{}".format( tag ), # output files into /hadoop/cms/store/<user>/<special_dir> ) # Straightforward logic if not task.complete(): task.process() # Set task summary task_summary[
# submission tag tag = "v1_PMtest" merged_dir = "/nfs-7/userdata/{}/tupler_babies/merged/FT/{}/output/".format( os.getenv("USER"), tag) for dsname, shortname in sample_map.items(): task = CondorTask( sample=SNTSample( dataset=dsname, # tag="CMS4_V09-04-13", # if not specified, get latest CMS4 tag ), files_per_output=split_func(dsname), output_name="output.root", tag=tag, condor_submit_params={"use_xrootd": True}, cmssw_version="CMSSW_9_2_8", input_executable= "inputs/condor_executable_metis.sh", # your condor executable here tarfile= "inputs/package.tar.xz", # your tarfile with assorted goodies here special_dir= "FTbabies/", # output files into /hadoop/cms/store/<user>/<special_dir> ) # When babymaking task finishes, fire off a task that takes outputs and merges them locally (hadd) # into a file that ends up on nfs (specified by `merged_dir` above) merge_task = LocalMergeTask(input_filenames=task.get_outputs(), output_filename="{}/{}.root".format( merged_dir, shortname)) # Straightforward logic if not task.complete():
total_summary = {} first_run = False while True: allcomplete = True for sample, info in local_samples.items(): dataset = sample year = info["year"] metis_sample = DirectorySample(dataset = dataset, location = info["location"]) task = CondorTask( sample = metis_sample, open_dataset = False, flush = True, files_per_output = info["fpo"], output_name = "merged_ntuple.root", tag = job_tag, cmssw_version = cmssw_ver, executable = "condor_exe_%s.sh" % job_tag if (args.use_xrdcp or args.use_gridftp or args.use_wget) else "condor_exe.sh", tarfile = "empty" if (args.use_xrdcp or args.use_gridftp or args.use_wget) else tar_path, condor_submit_params = {"sites" : "T2_US_UCSD"}, special_dir = hadoop_path, arguments = conds_dict[year] ) task.process() if not task.complete(): allcomplete = False total_summary[dataset] = task.get_task_summary() if not args.local_only: for sample in samples.keys(): if skip(sample): continue
skip_tail = "/SMS" in dsname # skip_tail = False task = CondorTask( sample=sample, files_per_output=split_func(dsname), output_name="output.root", tag=tag, min_completion_fraction=0.90 if skip_tail else 1.0, condor_submit_params={ # "sites":"T2_US_UCSD,UCSB", # I/O is hella faster "sites": "T2_US_UCSD", # I/O is hella faster # "sites":"UAF", # I/O is hella faster "classads": [ ["metis_extraargs", extra_args], [ "JobBatchName", "FT_{}_{}".format(year, shortname) ], ], # "classads": [ ["metis_extraargs","--ignorebadfiles"], ], }, cmssw_version="CMSSW_9_4_9", scram_arch="slc6_amd64_gcc630", input_executable="inputs/condor_executable_metis.sh", tarfile="inputs/package.tar.xz", special_dir="FTbabies/{}/".format(tag), recopy_inputs=False, ) merge_task = LocalMergeTask( input_filenames=task.get_outputs(),
excltag = "CMS4_V09*" samptyp = "1" if "SMS" in babyname else "2" if ( "GJets" in dsname or "Photon" in dsname or "Gamma" in dsname) else "0" extrarg = "" # extrarg = " topcandTree=topcands" if "TTJets" in dsname else "" maker_task = CondorTask( sample=SNTSample(dataset=dsname, tag=cms3tag, exclude_tag_pattern=excltag), files_per_output=20 if "data" in babyname else 1 if "SMS" in babyname else 2, tag=tag, outdir_name="stopBaby_" + babyname, output_name="stopbaby.root", executable="condor_executable.sh", cmssw_version=cmsswver, scram_arch=scramarch, arguments=samptyp + extrarg, tarfile=tarfile, # condor_submit_params = {"sites": "T2_US_UCSD"}, # condor_submit_params = {"sites": "UAF"}, condor_submit_params={"use_xrootd": True}, # max_jobs = 1, # temporary for submission test no_load_from_backup=True, ) merge_task = CondorTask( sample=DirectorySample( dataset="merge_" + babyname, location=maker_task.get_outputdir(), ), # open_dataset = True, flush = True,
lumiWeightString = 1 if ( isData or samples[s]['name'].count('TChiWH') ) else 1000 * samples[s]['xsec'] / samples[s]['sumWeight'] print("- found sumWeight %s and x-sec %s" % (samples[s]['sumWeight'], samples[s]['xsec'])) maker_task = CondorTask( sample=sample, executable="executable.sh", arguments=" ".join([ str(x) for x in [ tag, lumiWeightString, 1 if isData else 0, year, era, 1 if isFastSim else 0, args.skim, args.user ] ]), files_per_output=int(mergeFactor), output_dir=os.path.join(outDir, samples[s]['name']), output_name="nanoSkim.root", output_is_tree=True, tag=tag, condor_submit_params={"sites": "T2_US_UCSD,UAF"}, cmssw_version="CMSSW_10_2_9", scram_arch="slc6_amd64_gcc700", min_completion_fraction=1.00, ) maker_tasks.append(maker_task) if not args.dryRun: for i in range(100): total_summary = {}
mt2home = os.path.dirname(os.path.realpath(__file__)) # submission tag tag = "V00-10-05_2016fullYear_17July2018" merged_dir = "/nfs-6/userdata/mt2/{}/".format(tag) tasks = {} merge_tasks = {} for dsname,shortname in sample_map.items(): task = CondorTask( sample = SNTSample( dataset=dsname, # tag="CMS4_V09-04-13", # uncomment this and set it to the desired tag, otherwise Metis will run on the most recent version ), files_per_output = 1, output_name = "output.root", tag = tag, condor_submit_params = {"use_xrootd":True}, cmssw_version = "CMSSW_9_4_9", input_executable = mt2home+"/metis_executable.sh", # your condor executable here tarfile = mt2home+"/job_input/input.tar.xz", # your tarfile with assorted goodies here special_dir = "mt2babies/{}".format(tag), # output files into /hadoop/cms/store/<user>/<special_dir> ) tasks[dsname] = task # When babymaking task finishes, fire off a task that takes outputs and merges them locally (hadd) # into a file that ends up on nfs (specified by `merged_dir` above) for dsname,shortname in merge_map.items(): merge_task = LocalMergeTask( input_filenames=task.get_outputs(), output_filename="{}/{}.root".format(merged_dir,shortname) ) merge_tasks[dsname] = merge_task
for dsname, samploc in cms4_samples.items(): cmsswver = "CMSSW_10_1_0" scramarch = "slc6_amd64_gcc700" tarfile = "tarfiles/input_v29_10.tar.gz" tag = "v29_10" maker_task = CondorTask( sample = DirectorySample( dataset=dsname, location=samploc, ), # open_dataset = True, files_per_output = 1, tag = tag, cmssw_version = cmsswver, scram_arch = scramarch, tarfile = tarfile, executable = "condor_executable.sh", outdir_name = "stopBaby_" + dsname, output_name = "stopbaby.root", arguments = "1" if "SMS" in dsname else "0", # isFastsim # condor_submit_params = {"sites": "UAF,T2_US_UCSD,UCSB"}, condor_submit_params = {"sites": "T2_US_UCSD,UCSB"}, # condor_submit_params = {"use_xrootd": True}, # no_load_from_backup = True, ) merge_task = CondorTask( sample = DirectorySample( dataset="merge_"+dsname, location=maker_task.get_outputdir(), ), # open_dataset = True, flush = True,
# Make a CondorTask (3 in total, one for each input) maker_tasks = [] merge_tasks = [] for dsname in dataset_names: cmsswver = "CMSSW_9_4_0_pre2" scramarch = "slc6_amd64_gcc700" tarfile = "input.tar.gz" tag = "v24_3" maker_task = CondorTask( sample = SNTSample(dataset=dsname), files_per_output = 1, tag = tag, outdir_name = "stopBaby_" + dsname[5:34].strip("_"), output_name = "stopbaby.root", executable = "condor_executable.sh", cmssw_version = cmsswver, scram_arch = scramarch, arguments = "1", # isFastsim tarfile = tarfile, condor_submit_params = {"sites": "UAF,T2_US_UCSD"}, ) merge_task = CondorTask( sample = DirectorySample( dataset=dsname.replace("MINIAODSIM","MERGE"), location=maker_task.get_outputdir(), ), executable = "ProjectMetis/metis/executables/condor_hadd.sh", open_dataset = False, files_per_output = 12, outdir_name = "merged_stopBaby_" + dsname[5:34].strip("_"),
def main(): main_dir = os.path.dirname(os.path.abspath(__file__)) metis_path = os.path.dirname(os.path.dirname(metis.__file__)) exec_path = main_dir + "/metis.sh" hadoop_path = "metis/" metis_dashboard_path = os.path.join(metis_path, "dashboard") job_tag = "" total_summary = {} os.chdir(metis_path) while True: masspoints = [125] tasks = [] for mass in masspoints: miniaod = CondorTask( ## Dummy sample as no input is needed in generating the events #sample = DummySample( # N=3000, # dataset="/VHToNonbb_M125_13TeV_amcatnloFXFX_madspin_pythia8/PRIVATE-RunIISummer16MiniAODv2-PUMoriond17_80X_mcRun2_asymptotic_2016_TrancheIV_v6_ext1-v1/MINIAODSIM" # ), # Dummy sample as no input is needed in generating the events sample = DummySample( N=3500, dataset="/WWW_4F_TuneCP5_13TeV-amcatnlo-pythia8/PRIVATE-RunIIFall17MiniAODv2-PU2017_12Apr2018_94X_mc2017_realistic_v14-v1/MINIAODSIM" ), tag = job_tag, executable = exec_path, special_dir = hadoop_path + "/private_miniaod", output_name = "output.root", files_per_output = 1, condor_submit_params = {"sites" : "T2_US_UCSD"}, open_dataset = True, flush = True, #no_load_from_backup = True, ) tasks.extend([miniaod]) all_tasks_complete = True for task in tasks: task.process() summary = task.get_task_summary() total_summary[task.get_sample().get_datasetname()] = summary all_tasks_complete = all_tasks_complete and task.complete() # parse the total summary and write out the dashboard StatsParser(data=total_summary, webdir=metis_dashboard_path).do() # Print msummary table so I don't have to load up website os.system("msummary -v0 -p ext2 | tee summary.txt") os.system("chmod -R 755 {}".format(metis_dashboard_path)) # If all done exit the loop if all_tasks_complete: print "" print "Job={} finished".format(job_tag) print "" break # Neat trick to not exit the script for force updating print 'Press Ctrl-C to force update, otherwise will sleep for 300 seconds' try: for i in range(0,300): sleep(1) # could use a backward counter to be preeety :) except KeyboardInterrupt: raw_input("Press Enter to force update, or Ctrl-C to quit.") print "Force updating..."
child_tasks = [] for srf in srlst: fname = srf.split('/')[-1] fdir = '/'.join(srf.split('/')[:-1]) print(fdir, fname) ifile = fname.split('.')[0].split('_')[-1] maker_task = CondorTask( sample = DirectorySample( dataset=dsname+'_'+ifile, location=fdir, globber=fname, ), # open_dataset = True, files_per_output = 1, tag = tag, cmssw_version = cmsswver, scram_arch = scramarch, tarfile = tarfile, executable = "condor_executable.sh", outdir_name = "resub_"+dsname, output_name = "stopbaby_"+ifile+".root", arguments = "1" if "SMS" in dsname else "0", # isFastsim condor_submit_params = {"sites": "UAF,T2_US_UCSD"}, # no_load_from_backup = True, ) # print(maker_task.get_sample().get_datasetname()) # print(maker_task.get_sample().get_files()) # print(maker_task.get_outputdir()) # print(maker_task.get_io_mapping()) child_tasks.append(maker_task)
#raise NotImplementedError isData = 1 if isData else 0 isFast = 1 if isFast else 0 if not dryRun: maker_task = CondorTask( sample=sample, #'/hadoop/cms/store/user/dspitzba/nanoAOD/TTWJetsToLNu_TuneCP5_13TeV-amcatnloFXFX-madspin-pythia8__RunIIAutumn18NanoAODv6-Nano25Oct2019_102X_upgrade2018_realistic_v20_ext1-v1/', # open_dataset = True, flush = True, executable="executable.sh", arguments="WHv1p2 %s %s %s" % (year, isData, isFast), #tarfile = "merge_scripts.tar.gz", files_per_output=1, #output_dir = os.path.join(outDir, sample.get_datasetname()), outdir_name="stopNano_" + s, output_name="stopNano.root", output_is_tree=True, # check_expectedevents = True, tag=tag, condor_submit_params={"sites": "T2_US_UCSD,UAF"}, cmssw_version="CMSSW_10_2_9", scram_arch="slc6_amd64_gcc700", # recopy_inputs = True, # no_load_from_backup = True, min_completion_fraction=0.99, ) maker_tasks.append(maker_task) if not dryRun: merge_task = CondorTask(
def submit_metis(job_tag, samples_map, sample_list=[], arguments_map="", exec_script="metis.sh", tar_files=[], hadoop_dirname="testjobs", files_per_output=1, globber="*.root", sites="T2_US_UCSD"): import time import json import metis from time import sleep from metis.Sample import DirectorySample from metis.CondorTask import CondorTask from metis.StatsParser import StatsParser import os import glob import subprocess # file/dir paths main_dir = os.getcwd() metis_path = os.path.dirname(os.path.dirname(metis.__file__)) tar_path = os.path.join(metis_path, "package.tar") tar_gz_path = tar_path + ".gz" metis_dashboard_path = os.path.join(metis_path, "dashboard") exec_path = os.path.join(main_dir, exec_script) hadoop_path = "metis/{}/{}".format(hadoop_dirname, job_tag) # The output goes to /hadoop/cms/store/user/$USER/"hadoop_path" # Create tarball os.chdir(main_dir) print os.getcwd() print "tar -chzf {} {}".format(tar_gz_path, " ".join(tar_files)) os.system("tar -chzf {} {}".format(tar_gz_path, " ".join(tar_files))) # Change directory to metis os.chdir(metis_path) total_summary = {} # if no sample_list is provided then we form it via the keys of the samples_map if len(sample_list) == 0: for key in samples_map: sample_list.append(key) samples_to_run = [] for key in sample_list: samples_to_run.append( DirectorySample( dataset=key, location=samples_map[key], globber=globber, ) ) files_per_output_config_list = [] if isinstance(files_per_output, dict): for key in sample_list: files_per_output_config_list.append(files_per_output[key]) else: for key in sample_list: files_per_output_config_list.append(files_per_output) # Loop over datasets to submit while True: all_tasks_complete = True #for sample in sorted(samples): for index, sample in enumerate(samples_to_run): # define the task maker_task = CondorTask( sample = sample, tag = job_tag, arguments = arguments_map[sample.get_datasetname()] if arguments_map else "", executable = exec_path, tarfile = tar_gz_path, special_dir = hadoop_path, output_name = "output.root", files_per_output = files_per_output_config_list[index], condor_submit_params = {"sites" : sites}, open_dataset = False, flush = True, #no_load_from_backup = True, ) # process the job (either submits, checks for resubmit, or finishes etc.) maker_task.process() # save some information for the dashboard total_summary["["+job_tag+"] "+maker_task.get_sample().get_datasetname()] = maker_task.get_task_summary() # Aggregate whether all tasks are complete all_tasks_complete = all_tasks_complete and maker_task.complete() # parse the total summary and write out the dashboard StatsParser(data=total_summary, webdir=metis_dashboard_path).do() # Print msummary table so I don't have to load up website os.system("msummary -r -p {} | tee summary.txt".format(job_tag)) os.system("chmod -R 755 {}".format(metis_dashboard_path)) os.system("chmod 644 {}/images/*".format(metis_dashboard_path)) # If all done exit the loop if all_tasks_complete: print "" print "Job={} finished".format(job_tag) print "" break # Neat trick to not exit the script for force updating print 'Press Ctrl-C to force update, otherwise will sleep for 300 seconds' try: for i in range(0,60): sleep(1) # could use a backward counter to be preeety :) except KeyboardInterrupt: raw_input("Press Enter to force update, or Ctrl-C to quit.") print "Force updating..." os.chdir(main_dir)
if "VHToGG" in dataset: print "Skipping VH for now" continue #if not ("Run2016" in dataset or "Run2017" in dataset or "Run2018" in dataset): # continue if "ttHiggs" in dataset: print("Submitting jobs for: ", dataset) sample = DirectorySample(dataset=dataset, location=info["input_loc"]) files = [f.name for f in sample.get_files()] print(len(files)) task = CondorTask(sample=sample, open_dataset=False, flush=True, files_per_output=info["fpo"], output_name="merged_ntuple.root", tag=job_tag, cmssw_version=cmssw_ver, executable=exec_path, tarfile=tar_path, condor_submit_params={"sites": "T2_US_UCSD"}, special_dir=hadoop_path, arguments=info["meta_conditions"]) task.process() if not task.complete(): allcomplete = False # save some information for the dashboard total_summary[dataset] = task.get_task_summary() # parse the total summary and write out the dashboard StatsParser(data=total_summary, webdir="~/public_html/dump/ttH_BabyMaker/").do() os.system("chmod -R 755 ~/public_html/dump/ttH_BabyMaker") if allcomplete:
import os import time from metis.CondorTask import CondorTask from metis.Sample import DBSSample from metis.StatsParser import StatsParser if not os.path.exists("inputs.tar.gz"): os.system("tar cvzf inputs.tar.gz looper.py") for i in range(100): total_summary = {} for dataset in [ "/DYJetsToLL_M-4to50_HT-100to200_TuneCP5_PSweights_13TeV-madgraphMLM-pythia8/RunIIAutumn18NanoAODv7-Nano02Apr2020_102X_upgrade2018_realistic_v21-v1/NANOAODSIM", ]: task = CondorTask( sample = DBSSample(dataset=dataset), events_per_output = 1e6, output_name = "output.root", tag = "nanotestv1", cmssw_version = "CMSSW_10_2_5", scram_arch = "slc6_amd64_gcc700", tarfile = "inputs.tar.gz", executable = "condor_nano_exe.sh", ) task.process() total_summary[task.get_sample().get_datasetname()] = task.get_task_summary() StatsParser(data=total_summary, webdir="~/public_html/dump/metis_nanotest/").do() time.sleep(30*60)