def test_trigger_success_listener(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_trigger_success_listener') jh.add_job(run_script=self.run_script_trigger_success, name='test') jh.run_jobs() self.assertIs(jh.jobs.test.status, Status.SUCCESS)
def test_run_max(self): ## This effectively only tests if run_jobs finishes or not from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_run_max', run_max=1) for i in range(3): jh.add_job(run_script=self.run_script, name='test_{}'.format(i)) jh.run_jobs() for i in range(3): self.assertIs(jh['test_{}'.format(i)].status, Status.SUCCESS)
def test_post_process(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_post_process') output_file = os.path.join(jh.config.output_dir, 'test') post_func = TestPostFunction(output_file) jh.add_job(run_script=self.run_script, name='test', post_func=post_func) jh.run_jobs() time.sleep(1) self.assertIs(jh.jobs.test.status, Status.SUCCESS) self.assertTrue(os.path.isfile(output_file))
def test_chain_fail(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_chain', listens=False) jh.add_job(run_script=self.run_script_fail, name='test_parent', tags='parent') jh.add_job(run_script=self.run_script_success, name='test_child', parent_tags='parent') jh.run_jobs() self.assertIs(jh.jobs.test_parent.status, Status.FAILED) self.assertIs(jh.jobs.test_child.status, Status.CANCELLED)
def test_output_listener(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_output_listener') jh.add_job(run_script=self.run_script_touch_file, name='test', output=self.output_file) jh.run_jobs() self.assertIs(jh.jobs.test.status, Status.SUCCESS) jh.reset() jh.jobs.test.config.backend.run_script = self.run_script_success jh.jobs.test.config.output = 'jwoigjwoijegoijwoijegoiwoeg' jh.run_jobs() self.assertIs(jh.jobs.test.status, Status.FAILED)
def test_chain(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_chain', listens=False) jh.add_job(run_script=self.run_script_touch_file, name='test_parent', tags='parent') jh.add_job(run_script=self.run_script_ls_file, name='test_child', parent_tags='parent') jh.run_jobs() self.assertIs(jh.jobs.test_parent.status, Status.SUCCESS) self.assertIs(jh.jobs.test_child.status, Status.SUCCESS)
def test_batch_listener(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_batch_listener') jh.add_job(run_script=self.run_script_fail, name='test') jh.run_jobs() status_fail = jh.jobs.test.status id_first = jh.jobs.test.id jh.jobs.test.config.backend.run_script = self.run_script_success jh.run_jobs(retry=True) status_success = jh.jobs.test.status id_second = jh.jobs.test.id self.assertIsNot(id_first, id_second) self.assertIs(status_fail, Status.FAILED) self.assertIs(status_success, Status.SUCCESS)
def main(): """Batch submission using slurmy.""" ## Set up the JobHandler jh = JobHandler( local_max=8, local_dynamic=True, work_dir=mkdir('batch'), printer_bar_mode=True, wrapper=SingularityWrapper('docker://philippgadow/checkmate')) ## Define the run script content ## Add a job jh.add_job(run_script='/work/run_example_in_docker.sh', job_type=Type.LOCAL) ## Run all jobs jh.run_jobs()
def test_reset(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_reset', listens=False) jh.add_job(run_script=self.run_script, name='test') jh.run_jobs() self.assertIs(jh.jobs.test.status, Status.SUCCESS) id_first = jh.jobs.test.id jh.reset() self.assertIs(jh.jobs.test.status, Status.CONFIGURED) jh.run_jobs() self.assertIs(jh.jobs.test.status, Status.SUCCESS) id_second = jh.jobs.test.id self.assertIsNot(id_first, id_second)
def test_type_local(self): from slurmy import JobHandler, Type jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_jobconfig_type_local', do_snapshot=False, local_max=1) job = jh.add_job(run_script=self.run_script, job_type=Type.LOCAL) self.assertIs(job.type, Type.LOCAL)
def test_mix_batch_local_listener(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_mix_batch_local_listener', local_max=1, local_dynamic=True) jh.add_job(run_script=self.run_script_fail, name='test_1') jh.add_job(run_script=self.run_script_fail, name='test_2') jh.run_jobs() self.assertIsNot(jh.jobs.test_1.type, jh.jobs.test_2.type) self.assertIs(jh.jobs.test_1.status, Status.FAILED) self.assertIs(jh.jobs.test_2.status, Status.FAILED) jh.jobs.test_1.config.backend.run_script = self.run_script_success jh.jobs.test_2.config.backend.run_script = self.run_script_success jh.run_jobs(retry=True) self.assertIs(jh.jobs.test_1.status, Status.SUCCESS) self.assertIs(jh.jobs.test_2.status, Status.SUCCESS)
def test_local_listener(self): from slurmy import JobHandler, Status, Type, test_mode test_mode(True) jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_local_listener', local_max=1) jh.add_job(run_script=self.run_script_fail, name='test', job_type=Type.LOCAL) jh.run_jobs() status_fail = jh.jobs.test.status jh.jobs.test.config.backend.run_script = self.run_script_success jh.run_jobs(retry=True) status_success = jh.jobs.test.status test_mode(False) self.assertIs(status_fail, Status.FAILED) self.assertIs(status_success, Status.SUCCESS)
def test_chain_multiparent(self): from slurmy import JobHandler, Status jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_chain_multiparent', listens=False) output1 = '@SLURMY.output_dir/parent1' run_script1 = '#!/bin/bash\ntouch {}; sleep 2;'.format(output1) jh.add_job(run_script=run_script1, name='test_parent1', tags='parent1', output=output1) output2 = '@SLURMY.output_dir/parent2' run_script2 = '#!/bin/bash\ntouch {}; sleep 2;'.format(output2) jh.add_job(run_script=run_script2, name='test_parent2', tags='parent2', output=output2) run_script3 = '#!/bin/bash\nls {} {};'.format(output1, output2) jh.add_job(run_script=run_script3, name='test_child', parent_tags=['parent1', 'parent2']) jh.run_jobs() self.assertIs(jh.jobs.test_parent1.status, Status.SUCCESS) self.assertIs(jh.jobs.test_parent2.status, Status.SUCCESS) self.assertIs(jh.jobs.test_child.status, Status.SUCCESS)
"MET_SoftTrk_ScaleUp", "MUON_ID", "MUON_MS", "MUON_SAGITTA_RESBIAS", "MUON_SAGITTA_RHO", "MUON_SCALE", # "NoSys", ] for process in processes: print "Starting with process {}".format(process) for sys in systematics: outputfile= os.path.join(output_path, "{}_merged_{}.root".format(process,sys)) name = "{}_merged_{}".format(process,sys) to_merge = [] for campaign in ["mc16e","mc16d","mc16a"]: # for f in glob.glob("/project/etp4/eschanet/ntuples/common/{}/{}/merged/{}_*tree_{}*done_noLHE.root".format(campaign,tag,process,sys)): for f in glob.glob("/project/etp4/eschanet/ntuples/common/{}/{}/merged/{}_*merged_processed_tree_{}*.root*".format(campaign,tag,process,sys)): if process == "ttbar" and "ttbar_allhad" in f: continue print f to_merge.append(f) # if len(to_merge)%3!=0: # print("{} probably incomplete!".format(process)) print(len(to_merge)) ft = FinishedTrigger(outputfile) jh.add_job(name=name+"_hadd", run_script=merge_script.format(outputfile=outputfile,finished_func=ft, inputfiles=" ".join(to_merge))) jh.run_jobs()
# print(inputfile) for skipEvents in range(0, totalEvents - maxEvents + 1, maxEvents): # print("Adding job for inputfile={}, skipEvents={}".format(inputfile, skipEvents)) dirname = os.path.basename(os.path.normpath(indirectory)) dirname = dirname.replace("EVNT", "DAOD_TRUTH3").replace("evgen", "deriv") outputfile = os.path.join(outdir, dirname, inputfile).replace( "EVNT", "DAOD_TRUTH3") myoutdir = os.path.join(outdir, dirname) outputfile = outputfile + "_{}".format(skipEvents) st = SuccessTrigger(outputfile, 20) test_finished = os.path.join(outdir, dirname, inputfile + "_testfile") # print(test_finished) jh.add_job( run_script=run_script.format(inputfile=inputfile, indir=indirectory, outdir=myoutdir, outputfile=outputfile, finished_file=test_finished, skipEvents=skipEvents, maxEvents=maxEvents), name="pantagruel_{}_{}".format(inputfile.replace(".", "_"), skipEvents), output=outputfile, success_func=st, backend=Slurm(mem="3500mb")) jh.run_jobs()
class Test(unittest.TestCase): def setUp(self): from slurmy import JobHandler, test_mode test_mode(True) self.test_dir = 'slurmy_unittest/jobconfig' self.jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_jobconfig', do_snapshot=False) self.run_script = 'echo "test"' self.run_script_trigger = '@SLURMY.FINISHED; @SLURMY.SUCCESS;' def tearDown(self): from slurmy import test_mode test_mode(False) ##TODO: run_script test --> with direct string and path to file def test_run_args(self): job = self.jh.add_job(run_script=self.run_script, run_args='test') self.assertIs(job.config.backend.run_args, 'test') def test_name(self): job = self.jh.add_job(run_script=self.run_script, name='test') self.assertIs(job.name, 'test') self.assertIn('test', self.jh.jobs) self.assertIs(self.jh.jobs.test.name, 'test') def test_type_local(self): from slurmy import JobHandler, Type jh = JobHandler(work_dir=self.test_dir, verbosity=0, name='test_jobconfig_type_local', do_snapshot=False, local_max=1) job = jh.add_job(run_script=self.run_script, job_type=Type.LOCAL) self.assertIs(job.type, Type.LOCAL) def test_finished_func(self): from slurmy import Status, Mode job = self.jh.add_job(run_script=self.run_script, finished_func=lambda x: x) self.assertIs(job.get_mode(Status.RUNNING), Mode.ACTIVE) self.assertTrue(job.config.finished_func(True)) def test_success_func(self): from slurmy import Status, Mode job = self.jh.add_job(run_script=self.run_script, success_func=lambda x: x) self.assertIs(job.get_mode(Status.FINISHED), Mode.ACTIVE) self.assertTrue(job.config.success_func(True)) def test_post_func(self): from slurmy import Status job = self.jh.add_job(run_script=self.run_script, post_func=lambda x: x) self.assertTrue(job.config.post_func(True)) def test_output(self): from slurmy import Status, Mode job = self.jh.add_job(run_script=self.run_script, output='test') self.assertIs(job.get_mode(Status.FINISHED), Mode.PASSIVE) self.assertIsNotNone(job.output) def test_tags(self): job = self.jh.add_job(run_script=self.run_script, tags='hans') self.assertIn('hans', job.tags) job = self.jh.add_job(run_script=self.run_script, tags=['hans', 'horst']) self.assertIn('hans', job.tags) self.assertIn('horst', job.tags) def test_parent_tags(self): job = self.jh.add_job(run_script=self.run_script, parent_tags='hans') self.assertIn('hans', job.parent_tags) job = self.jh.add_job(run_script=self.run_script, parent_tags=['hans', 'horst']) self.assertIn('hans', job.parent_tags) self.assertIn('horst', job.parent_tags) def test_variable_substitution(self): from slurmy import Status job = self.jh.add_job(run_script=self.run_script, output='@SLURMY.output_dir/test') output = os.path.join(self.jh.config.output_dir, 'test') self.assertTrue(job.output == output)
print point #TODO: make nicer if "onestep" in point.lower(): #onestep points have different formatting at tree- and AOD-level point = point.replace("onestepCC", "oneStep") elif "LQ" in point: point = "leptoquark_" + point.replace('ld_0p3_beta_0p5_hnd_1p0_','') + "_" searcher = "{}*merged_processed*.root".format(point) print(searcher) files_mc16a = glob.glob("/project/etp4/eschanet/ntuples/common/mc16a/{}/merged/{}".format(args.production,searcher)) files_mc16d = glob.glob("/project/etp4/eschanet/ntuples/common/mc16d/{}/merged/{}".format(args.production,searcher)) files_mc16e = glob.glob("/project/etp4/eschanet/ntuples/common/mc16e/{}/merged/{}".format(args.production,searcher)) if len(files_mc16a)==0 or len(files_mc16d)==0 or len(files_mc16e)==0: print "mc16a: %i"%len(files_mc16a) print "mc16d: %i"%len(files_mc16d) print "mc16e: %i"%len(files_mc16e) print "WARNING - {} is not complete".format(point) if not args.ignore_incomplete: continue print "WARNING - Ignoring incomplete!".format(point) inputfiles = " ".join(sorted(files_mc16a)) + " " + " ".join(sorted(files_mc16d)) + " " + " ".join(sorted(files_mc16e)) #pprint.pprint(inputfiles) outputfile = output_path + "{}_merged_processed.root".format(point) jh.add_job(name=point+"_campaignHadder", run_script=merge_script.format(outputfile=outputfile, inputfiles=inputfiles), output=outputfile) jh.run_jobs()
def run(): parser = argparse.ArgumentParser(description='Run stuff locally on etp') parser.add_argument('files', type=argparse.FileType('r'), nargs='+') parser.add_argument("-s",help="input sample",default=None) parser.add_argument("-selector",help="selector",default="OneLep") parser.add_argument("-writeTrees",help="sys or nominal",default="1") parser.add_argument("-deepConfig",help="input sample",default="SusySkim1LInclusive_Rel21.config") parser.add_argument("-outputPath",help="output path",default=None) parser.add_argument("-process",help="process tag to find your output",default=None) args = parser.parse_args() jobscript = """ echo Running on host `hostname` echo Time is `date` echo Directory is `pwd` shopt -s expand_aliases export ATLAS_LOCAL_ROOT_BASE=/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase source ${ATLAS_LOCAL_ROOT_BASE}/user/atlasLocalSetup.sh pushd $testarea set -- "" acmSetup popd echo "run_xAODNtMaker" echo " -s $sample" echo " -selector $selector" echo " -writeTrees $writeTrees" echo " -deepConfig $deepConfig" echo " -MaxEvents $maxEvents" echo " -SkipEvents $skipEvents" run_xAODNtMaker -s $sample -selector $selector -writeTrees $writeTrees -deepConfig $deepConfig -MaxEvents $maxEvents -SkipEvents $skipEvents [[ "$?" = "0" ]] && mv ${outputPath}/submitDir/data-tree/ ${outputPath}/${groupset}/${mytag}/merged/${process}_${minEvent}_${maxEvent}_merged_processed_${writeTrees}.root """ sw = SingularityWrapper('/cvmfs/atlas.cern.ch/repo/containers/images/singularity/x86_64-centos7.img') jh = JobHandler(wrapper = sw,work_dir="/project/etp2/eschanet/collect", name="run_ntuple")#, run_max=50) # sf = SuccessOutputFile() for exportopts, optDict in params: slurm = Slurm(export = exportopts) outputfile = os.path.abspath("{path}/{groupset}/{mytag}/merged/{process}_{minEvent}_{maxEvent}_merged_processed_{sys}.root.done".format(**optDict)) jobname = "run_xAODNtMaker_{groupset}_{process}_{sys}_{minEvent}".format(**optDict) jobname = clean_jobname(jobname) print(jobname) ft = FinishedTrigger(outputfile) jh.add_job(backend = slurm, run_script = jobscript, output = outputfile, success_func = ft, name = jobname) jh.run_jobs()
['"{}"'.format(b) for b in lhe_weights]) # if there are more than 50 trees, split into several jobs and merge later if len(treenames) > 50: i = 1 to_merge = [] for chunk in chunks(treenames, 25): print("creating chunk #{:02d}".format(i)) trees = " ,".join(['"{}"'.format(t) for t in chunk]) basename = "{:}_part{:03d}.root".format( os.path.splitext(os.path.basename(f))[0], i) outputfile = os.path.join(output_path, basename) jh.add_job(name=name + "_part{:03d}".format(i), run_script=skim_script.format(inputfile=f, outputfile=outputfile, treenames=trees, branches=branchnames, selection=selection), tags=name + "_skim") to_merge.append(outputfile) i += 1 outputfile = os.path.join(output_path, os.path.basename(f)) ft = FinishedTrigger(outputfile) jh.add_job(name=name + "_merge", run_script=merge_script.format(outputfile=outputfile, inputfiles=" ".join(to_merge)), finished_func=ft, parent_tags=name + '_skim') else: treenames = " ,".join(['"{}"'.format(t) for t in treenames]) outputfile = os.path.join(output_path, os.path.basename(f))