def defGridJob(jobName, jobGroup, inputFile): '''Defines Grid job''' from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob job = UserJob() job.setName(jobName) job.setJobGroup(jobGroup) job.setCPUTime(86400) job.dontPromptMe() job.setBannedSites([ 'LCG.IN2P3-CC.fr', 'OSG.UConn.us', 'LCG.Cracow.pl', 'OSG.MIT.us', 'LCG.Glasgow.uk', 'OSG.CIT.us', 'OSG.BNL.us', 'LCG.Brunel.uk' ]) job.setInputData(inputFile) job.setInputSandbox([ 'LFN:/ilc/user/l/lstroem/topasymmetry/lib_ilcsoft_2017-05-30_gcc62.tar.gz' ]) #only need LFN: for InputSandbox job.setOutputSandbox(["*.out", "*.log"]) outputFile = jobName + ".slcio" rootFile = jobName + ".root" #job.setOutputData([outputFile, rootFile], jobGroup, "CERN-DST-EOS") #keep some files on the GRID for further processing job.setOutputData([rootFile], jobGroup, "CERN-DST-EOS") return job, outputFile, rootFile
def getJob(): """ produce a job: it's always the same, so we don't need to put it in the main """ j = UserJob() ma = Marlin() ma.setVersion("v0111Prod") ma.setSteeringFile("clic_ild_cdr_steering.xml") ma.setGearFile("clic_ild_cdr.gear") result = j.append(ma) if not result['OK']: gLogger.error(result["Message"]) dexit(1) j.setCPUTime(10000) j.setOutputSandbox("*.log") return j
def getJob(jobid, jobpara): iser=jobid+1 outdst = "toto-ovl-%5.5i.dst.slcio"%iser outrec = "toto-ovl-%5.5i.rec.slcio"%iser ###In case one wants a loop: comment the folowing. #for i in range(2): j = UserJob() j.setJobGroup("Tutorial") j.setName("MarlinOverlayParametric%i"%iser) j.setInputSandbox(jobpara["setting_file"]) ## Define the overlay ov = OverlayInput() ov.setMachine("ilc_dbd") ov.setEnergy(energy) ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"])) ov.setBXOverlay(int(jobpara["BXOverlay"])) ov.setGGToHadInt(float(jobpara["GGToHadInt500"])) ov.setBkgEvtType("aa_lowpt") # ov.setBackgroundType("aa_lowpt") ov.setDetectorModel("ILD_o1_v05") res = j.append(ov) if not res['OK']: print res['Message'] exit(1) ## Define Marlin job ma = Marlin() ma.setDebug() ma.setVersion("ILCSoft-01-17-09") ma.setSteeringFile("marlin_ovl_stdreco.xml") ma.setGearFile("GearOutput.xml") # ma.setInputFile(simfile) ma.setInputFile(simlists[jobid]) ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma) if not res['OK']: print res['Message'] exit(1) j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM") j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"]) j.setCPUTime(10000) j.dontPromptMe() return j
def getJob(self): """ Define a generic job, it should be always the same """ from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob myjob = UserJob() myjob.setName("Testing") myjob.setJobGroup("Tests") myjob.setCPUTime(30000) myjob.dontPromptMe() myjob.setLogLevel("VERBOSE") myjob.setPlatform("x86_64-slc5-gcc43-opt") myjob.setOutputSandbox(["*.log","*.xml", "*.sh"]) myjob._addParameter( myjob.workflow, 'TestFailover', 'String', True, 'Test failoverRequest') myjob._addParameter( myjob.workflow, 'Platform', 'JDL', "x86_64-slc5-gcc43-opt", 'OS Platform') if self.ildConfig: myjob.setILDConfig(self.ildConfig) return myjob
def testAndProbeSites(): """submits jobs to test sites""" clip = Params() clip.registerSwitches() Script.parseCommandLine() from DIRAC import gLogger, exit as dexit from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import CheckWNs from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueues res = getQueues(siteList=clip.site, ceList=clip.ce) if not res['OK']: gLogger.error("Failed getting the queues", res['Message']) dexit(1) sitedict = res['Value'] CEs = [] for ces in sitedict.values(): CEs.extend(ces.keys()) gLogger.notice("Found %s CEs to look at." % len(CEs)) d = DiracILC(True, "SiteProbe.rep") for CE in CEs: j = UserJob() j.setDestinationCE(CE) c = CheckWNs() res = j.append(c) if not res['OK']: gLogger.error(res['Message']) continue j.setOutputSandbox("*.log") j.setCPUTime(30000) j.dontPromptMe() res = j.submit(d) if not res['OK']: gLogger.error("Failed to submit job, aborting") dexit(1) dexit(0)
def getJob(self): """ Define a generic job, it should be always the same """ from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob myjob = UserJob() myjob.setName("Testing") myjob.setJobGroup("Tests") myjob.setCPUTime(30000) myjob.dontPromptMe() myjob.setLogLevel("VERBOSE") myjob.setPlatform("x86_64-slc5-gcc43-opt") myjob.setOutputSandbox(["*.log", "*.xml", "*.sh"]) myjob._addParameter(myjob.workflow, 'TestFailover', 'String', True, 'Test failoverRequest') myjob._addParameter(myjob.workflow, 'Platform', 'JDL', "x86_64-slc5-gcc43-opt", 'OS Platform') if self.ildConfig: myjob.setILDConfig(self.ildConfig) return myjob
if ov: steeringf = "clic_ild_cdr_steering_overlay.xml" res = j.append(ovi) if not res['OK']: print res['Message'] continue ma = Marlin() ma.setVersion("v0111Prod") ma.setGearFile("clic_ild_cdr.gear") ma.setSteeringFile(steeringf) ma.setInputFile("LFN:" + lfn) ma.setNbEvts(10) ma.setEnergy(500.) ma.setOutputRecFile("myrec_overlay_%s.slcio" % ov) ma.setOutputDstFile("mydst_overlay_%s.slcio" % ov) res = j.append(ma) if not res['OK']: print res['Message'] exit() j.setCPUTime(86400) j.setOutputData("myrec_overlay_%s.slcio" % ov, "some/path") j.setName("SomeName") j.setJobGroup("SomeGroup") res = d.checkparams(j) if not res['OK']: print res['Message'] exit() j.submit(d)
ma.setLogFile("marlin.log") #ma.getInputFromApp(sim) ma.setInputFile([lcoutputSIM]) ma.setEnergy(energy) ma.setNumberOfEvents(evtsPerRun) ma.setOutputDstFile(lcoutputDST) ma.setOutputRecFile(lcoutputREC) RECoutput.append(lcoutputDST) RECoutput.append(lcoutputREC) job = UserJob() job.setName(jobname) job.setJobGroup(jobGrName) job.setILDConfig(ILDConfigVer) job.setCPUTime(86400) job.setInputData([lcinputREC]) job.setInputSandbox(["runRecoSplit_all_Tmp.py"]) job.setOutputSandbox(["*.log","*.sh","MarlinStdRecoParsed.xml","marlin*.xml","*.py "]) #job.setOutputSandbox(["*.log","*.sh","MarlinStdRecoParsed.xml","marlin*.xml","*.py ","*.root"]) #job.setDestinationCE('lyogrid07.in2p3.fr') job.dontPromptMe() job.setBannedSites(['LCG.QMUL.uk']) #job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.DESYZN.de','LCG.DESY-HH.de','LCG.KEK.jp','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us']) # run Malrin reco jobs mares = job.append(ma) if not mares['OK']: print mares['Not ok appending Marlin to job'] quit()
def getJob(dirac, jobid, jobpara): iser=jobid+100 outdir = "/ilc/user/a/amiyamoto/myprod2/test/" outdst = "toto-ovl-%5.5i.dst.slcio"%iser outrec = "toto-ovl-%5.5i.rec.slcio"%iser dstlfn = outdir+"dst/"+outdst reclfn = outdir+"rec/"+outrec outsrm = "CERN-SRM" ###In case one wants a loop: comment the folowing. #for i in range(2): j = UserJob() j.setJobGroup("Tutorial") j.setName("MarlinOverlayParametric%i"%iser) j.setInputSandbox(jobpara["setting_file"]) ## Define the overlay ov = OverlayInput() ov.setMachine("ilc_dbd") ov.setEnergy(energy) ov.setNumberOfSignalEventsPerJob(int(jobpara["n_events_per_job"])) ov.setBXOverlay(int(jobpara["BXOverlay"])) ov.setGGToHadInt(float(jobpara["GGToHadInt500"])) ov.setBkgEvtType("aa_lowpt") # ov.setBackgroundType("aa_lowpt") ov.setDetectorModel("ILD_o1_v05") res = j.append(ov) if not res['OK']: print res['Message'] exit(1) ## Define Marlin job ma = Marlin() ma.setDebug() ma.setVersion("ILCSoft-01-17-09") ma.setSteeringFile("marlin_stdreco.xml") ma.setGearFile("GearOutput.xml") # ma.setInputFile(simfile) ma.setInputFile(simlists[jobid]) ma.setOutputDstFile(outdst) ma.setOutputRecFile(outrec) res = j.append(ma) if not res['OK']: print res['Message'] exit(1) # Upload files to different directories upload_script="upload%i.sh"%iser upload = GenericApplication() # Create a script to upload files. shfile = open(upload_script,"w") shfile.write("#!/bin/bash\n") shfile.write("/bin/ls -l \n") shfile.write("dirac-dms-add-file -ddd "+dstlfn+" "+outdst+" "+outsrm+" \n") shfile.write("dirac-dms-add-file -ddd "+reclfn+" "+outrec+" "+outsrm+" \n") shfile.close() os.chmod(upload_script,0755) upload.setScript(upload_script) res = j.append(upload) if not res['OK'] : print res['Message'] exit(1) # j.setOutputData([outdst,outrec],"myprod2/test","PNNL-SRM") j.setInputSandbox([ setting_file, upload_script ] ) j.setOutputSandbox(["*.log","*.xml","*.sh","TaggingEfficiency.root","PfoAnalysis.root"]) j.setCPUTime(10000) j.dontPromptMe() res = j.submit(dirac) if not res["OK"] : print "Failed submit job, jobid=%s" %jobid print res os.remove(upload_script) return j
jobGroup = idin + "_" + detModelSim dirac = DiracILC(True,jobGroup+".rep") MRGoutput = [] # outputs to be saved onto grid SE outpath="MyProd_" + ILDConfigVer + "/E250-TDR_ws/" + chann + "/" +ireq+ "/mrg" jobname="m" + idin + "_" + str(mrgix1) print jobname job = UserJob() job.setName(jobname) job.setJobGroup(jobGrName) job.setILDConfig(ILDConfigVer) job.setCPUTime(6400) job.setInputSandbox(["runDSTmerge_Tmp.py"]) job.setOutputSandbox( ["*.log","*.sh","*.py"] ) job.setInputData(mergeList) #job.setInputData(mergeList[0]) #job.setOutputData( lcoutputMRG, outpath, "CERN-SRM" ) job.setOutputData( lcoutputMRG, outpath, "IN2P3-SRM" ) job.dontPromptMe() slcioconcat = SLCIOConcatenate() slcioconcat.setInputFile(mergeList) #slcioconcat.setInputFile(mergeList[0]) slcioconcat.setNumberOfEvents(evtsPerMrg) slcioconcat.setOutputFile( lcoutputMRG, outpath) slcioconcat.setLogFile("merge.log")
if ov: steeringf = "clic_ild_cdr_steering_overlay.xml" res = j.append(ovi) if not res['OK']: print(res['Message']) continue ma = Marlin() ma.setVersion("v0111Prod") ma.setGearFile("clic_ild_cdr.gear") ma.setSteeringFile(steeringf) ma.setInputFile("LFN:"+lfn) ma.setNbEvts(10) ma.setEnergy(500.) ma.setOutputRecFile("myrec_overlay_%s.slcio"%ov) ma.setOutputDstFile("mydst_overlay_%s.slcio"%ov) res =j.append(ma) if not res['OK']: print(res['Message']) exit() j.setCPUTime(86400) j.setOutputData("myrec_overlay_%s.slcio"%ov,"some/path") j.setName("SomeName") j.setJobGroup("SomeGroup") res = d.checkparams(j) if not res['OK']: print(res['Message']) exit() j.submit(d)
def main(): # Take the input arguments from the argument parser, and check they exist... args = parse_args() if not args: print 'Invalid Arguments' sys.exit(1) #### Software Versions #### softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended) # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working # Check the --runs and --split arguments to make sure they are compatible, if not exit... if not check_events_arguments(args.events, args.split): sys.exit(1) # Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning... lfn_check, lfn = check_input_LFN(args.stdhepInput) if not lfn_check: sys.exit(1) # Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included... dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector)) # Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging, # into the input sandbox inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag) # Prepares values for the job loop... if args.split < 0: nInputEvents = int(args.events) nOutputEvents = int(args.events) if args.split > 0: nInputEvents = int(args.events) nOutputEvents = int(args.split) # Loop that runs through the required number of jobs to be executed... for startEvent in range(0, nInputEvents, nOutputEvents): ################## Job Initialise ######################################## job = UserJob() job.setName(path.basename(args.stdhepInput)) job.setJobGroup('JobGroup') job.setInputSandbox(inputSandbox) fileNumber = startEvent/nOutputEvents print "Job ", fileNumber outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions) slicOutput=outputFiles['slicOutput'] prePandoraOutput=outputFiles['prePandoraOutput'] pandoraOutput=outputFiles['pandoraOutput'] vertexingOutput=outputFiles['vertexingOutput'] lcsimRecOutput=outputFiles['lcsimRecOutput'] lcsimDstOutput=outputFiles['lcsimDstOutput'] flavortagOutput=outputFiles['flavortagOutput'] diracOutput=outputFiles['diracOutput'] ################## SLIC ################################################## slic = SLIC() slic.setVersion(softVersions[0]) slic.setSteeringFile(args.macFile) # slic.setInputFile(lfn) slic.setOutputFile(slicOutput) slic.setDetectorModel(args.detector) slic.setNumberOfEvents(nOutputEvents) slic.setStartFrom(startEvent) #print slic.listAttributes() result = job.append(slic) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (digitization and tracking) ##################### lcsim = LCSIM() lcsim.setVersion(softVersions[1]) lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles lcsim.getInputFromApp(slic) lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml') # lcsim.setAliasProperties('alias.properties') lcsim.setDetectorModel('geometryFiles/sidloi3.zip') lcsim.setOutputFile(prePandoraOutput) lcsim.setNumberOfEvents(nOutputEvents) #print lcsim.listAttributes() result = job.append(lcsim) if not result['OK']: print result['Message'] sys.exit(2) ################## slicPandora ########################################### slicPandora = SLICPandora() slicPandora.setVersion(softVersions[2]) slicPandora.setDetectorModel(args.detector) slicPandora.getInputFromApp(lcsim) slicPandora.setOutputFile(pandoraOutput) slicPandora.setPandoraSettings('pandoraSettings.xml') slicPandora.setNumberOfEvents(nOutputEvents) #print slicPandora.listAttributes() result = job.append(slicPandora) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus Vertexing ############################ vertexing = Marlin() vertexing.setVersion(softVersions[3]) vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml') vertexing.setGearFile('steeringFiles/sidloi3.gear') vertexing.getInputFromApp(slicPandora) vertexing.setOutputFile(vertexingOutput) vertexing.setNumberOfEvents(nOutputEvents) #print vertexing.listAttributes() result = job.append(vertexing) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (DST production) ################################ lcsimDst = LCSIM() lcsimDst.setVersion(softVersions[1]) lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml') lcsimDst.getInputFromApp(vertexing) lcsimDst.setNumberOfEvents(nOutputEvents) # lcsimDst.setAliasProperties('alias.properties') lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip') lcsimDst.setOutputRecFile(lcsimRecOutput) lcsimDst.setOutputDstFile(lcsimDstOutput) #print lcsimDst.listAttributes() result = job.append(lcsimDst) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus flavortag ############################ if args.flavortag: flavortag = Marlin() flavortag.setVersion(softVersions[3]) flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml') flavortag.setGearFile('steeringFiles/sidloi3.gear') flavortag.setInputFile(lcsimDstOutput) flavortag.setOutputFile(flavortagOutput) flavortag.setNumberOfEvents(nOutputEvents) #print flavortag.listAttributes() result = job.append(flavortag) if not result['OK']: print result['Message'] sys.exit(2) ################## Job Finalise ########################################## # List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on, # This list is likely to change. job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp', 'OSG.PNNL.us',]) job.setCPUTime(50000) job.setPlatform('x86_64-slc5-gcc43-opt') # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/ # directory on the grid. if args.flavortag: job.setOutputData(flavortagOutput, diracOutput, args.SE) else: job.setOutputData(lcsimDstOutput, diracOutput, args.SE) job.setOutputSandbox(outputSandbox) job.setInputData(lfn) if args.dontPromptMe: job.dontPromptMe() # Submits Job!!! job.submit() return 0;
def main(): # Take the input arguments from the argument parser, and check they exist... args = parse_args() if not args: print 'Invalid Arguments' sys.exit(1) #### Software Versions #### softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended) # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working # Check the --runs and --split arguments to make sure they are compatible, if not exit... if not check_events_arguments(args.events, args.split): sys.exit(1) # Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning... lfn_check, lfn = check_input_LFN(args.stdhepInput) if not lfn_check: sys.exit(1) # Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included... dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector)) # Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging, # into the input sandbox inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag) # Prepares values for the job loop... if args.split < 0: nInputEvents = int(args.events) nOutputEvents = int(args.events) if args.split > 0: nInputEvents = int(args.events) nOutputEvents = int(args.split) # Loop that runs through the required number of jobs to be executed... for startEvent in range(0, nInputEvents, nOutputEvents): ################## Job Initialise ######################################## job = UserJob() job.setName(path.basename(args.stdhepInput)) job.setJobGroup('JobGroup') job.setInputSandbox(inputSandbox) fileNumber = startEvent / nOutputEvents print "Job ", fileNumber outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions) slicOutput = outputFiles['slicOutput'] prePandoraOutput = outputFiles['prePandoraOutput'] pandoraOutput = outputFiles['pandoraOutput'] vertexingOutput = outputFiles['vertexingOutput'] lcsimRecOutput = outputFiles['lcsimRecOutput'] lcsimDstOutput = outputFiles['lcsimDstOutput'] flavortagOutput = outputFiles['flavortagOutput'] diracOutput = outputFiles['diracOutput'] ################## SLIC ################################################## slic = SLIC() slic.setVersion(softVersions[0]) slic.setSteeringFile(args.macFile) # slic.setInputFile(lfn) slic.setOutputFile(slicOutput) slic.setDetectorModel(args.detector) slic.setNumberOfEvents(nOutputEvents) slic.setStartFrom(startEvent) #print slic.listAttributes() result = job.append(slic) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (digitization and tracking) ##################### lcsim = LCSIM() lcsim.setVersion(softVersions[1]) lcsim.setSteeringFile( 'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml' ) # Another version is included in /steeringFiles lcsim.getInputFromApp(slic) lcsim.setTrackingStrategy( 'steeringFiles/sidloi3_trackingStrategies_default.xml') # lcsim.setAliasProperties('alias.properties') lcsim.setDetectorModel('geometryFiles/sidloi3.zip') lcsim.setOutputFile(prePandoraOutput) lcsim.setNumberOfEvents(nOutputEvents) #print lcsim.listAttributes() result = job.append(lcsim) if not result['OK']: print result['Message'] sys.exit(2) ################## slicPandora ########################################### slicPandora = SLICPandora() slicPandora.setVersion(softVersions[2]) slicPandora.setDetectorModel(args.detector) slicPandora.getInputFromApp(lcsim) slicPandora.setOutputFile(pandoraOutput) slicPandora.setPandoraSettings('pandoraSettings.xml') slicPandora.setNumberOfEvents(nOutputEvents) #print slicPandora.listAttributes() result = job.append(slicPandora) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus Vertexing ############################ vertexing = Marlin() vertexing.setVersion(softVersions[3]) vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml') vertexing.setGearFile('steeringFiles/sidloi3.gear') vertexing.getInputFromApp(slicPandora) vertexing.setOutputFile(vertexingOutput) vertexing.setNumberOfEvents(nOutputEvents) #print vertexing.listAttributes() result = job.append(vertexing) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (DST production) ################################ lcsimDst = LCSIM() lcsimDst.setVersion(softVersions[1]) lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml') lcsimDst.getInputFromApp(vertexing) lcsimDst.setNumberOfEvents(nOutputEvents) # lcsimDst.setAliasProperties('alias.properties') lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip') lcsimDst.setOutputRecFile(lcsimRecOutput) lcsimDst.setOutputDstFile(lcsimDstOutput) #print lcsimDst.listAttributes() result = job.append(lcsimDst) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus flavortag ############################ if args.flavortag: flavortag = Marlin() flavortag.setVersion(softVersions[3]) flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml') flavortag.setGearFile('steeringFiles/sidloi3.gear') flavortag.setInputFile(lcsimDstOutput) flavortag.setOutputFile(flavortagOutput) flavortag.setNumberOfEvents(nOutputEvents) #print flavortag.listAttributes() result = job.append(flavortag) if not result['OK']: print result['Message'] sys.exit(2) ################## Job Finalise ########################################## # List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on, # This list is likely to change. job.setBannedSites([ 'LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp', 'OSG.PNNL.us', ]) job.setCPUTime(50000) job.setPlatform('x86_64-slc5-gcc43-opt') # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/ # directory on the grid. if args.flavortag: job.setOutputData(flavortagOutput, diracOutput, args.SE) else: job.setOutputData(lcsimDstOutput, diracOutput, args.SE) job.setOutputSandbox(outputSandbox) job.setInputData(lfn) if args.dontPromptMe: job.dontPromptMe() # Submits Job!!! job.submit() return 0
def main(): # Take the input arguments from the argument parser, and check they exist... args = parse_args() if not args: print 'Invalid Arguments' sys.exit(1) print args.chain[0] # softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"] softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended) # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working # softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"] check_events_arguments(args.events, args.split) detector = args.detector alias_properties(detector) outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering) inputSandbox, outputSandbox = setup_sandboxes(args.macFile) dirac = DiracILC(True, repoName) # Prepares values for the job loop... if args.split < 0: nInputEvents = int(args.events) nOutputEvents = int(args.events) if args.split > 0: nInputEvents = int(args.events) nOutputEvents = int(args.split) # Loop that runs through the required number of jobs to be executed... for startEvent in range(0, nInputEvents, nOutputEvents): ################## Job Initialise ######################################## job = UserJob() job.setName(outputBase) job.setJobGroup('JobGroup') job.setInputSandbox(inputSandbox) fileNumber = startEvent/nOutputEvents print "Job ---> ", fileNumber ################## SLIC ################################################## if 1 in args.chain: slic = SLIC() slic.setVersion(softVersions[0]) slic.setSteeringFile(args.macFile) # slic.setInputFile(lfn) slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio')) slic.setDetectorModel(detector) slic.setNumberOfEvents(nOutputEvents) slic.setStartFrom(startEvent) #print slic.listAttributes() result = job.append(slic) if not result['OK']: print result['Message'] sys.exit(2) ################## Overlay ############################################### if 2 in args.chain: ''' #Add the gghad background overlay. gghad = OverlayInput() #gghad.setProdID(1767) gghad.setEnergy(500.0) gghad.setBXOverlay('args.bunches') gghad.setGGToHadInt( 4.1 ) gghad.setNbSigEvtsPerJob(nOutputEvents) gghad.setMachine('ilc_dbd') gghad.setDetectorModel('sidloi3') gghad.setBkgEvtType('aa_lowpt') result = job.append( gghad ) if not result['OK']: print result['Message'] sys.exit(2) #Add the pair background overlay. pairs = OverlayInput() pairs.setProdID(2) pairs.setEnergy(500.0) pairs.setBXOverlay('args.bunches') pairs.setGGToHadInt(1.) pairs.setNbSigEvtsPerJob(nOutputEvents) pairs.setMachine('ilc_dbd') pairs.setDetectorModel('sidloi3') pairs.setBkgEvtType('eepairs') result = job.append( pairs ) if not result['OK']: print result['Message'] sys.exit(2) ''' gghad = OverlayInput() gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/') gghad.setBXOverlay(int(args.bunches)) gghad.setGGToHadInt( 4.1 ) gghad.setNbSigEvtsPerJob(nOutputEvents) gghad.setBkgEvtType('aa_lowpt') result = job.append( gghad ) if not result['OK']: print result['Message'] sys.exit(2) ''' pairs = OverlayInput() pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/') pairs.setBXOverlay(int(args.bunches)) pairs.setGGToHadInt(1.) pairs.setBkgEvtType('eepairs') pairs.setNbSigEvtsPerJob(nOutputEvents) result = job.append( pairs ) if not result['OK']: print result['Message'] sys.exit(2) ''' ################## lcsim (digitization and tracking) ##################### if 3 in args.chain: lcsim = LCSIM() lcsim.setVersion(softVersions[1]) lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles if 1 in args.chain: lcsim.getInputFromApp(slic) lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml') lcsim.setAliasProperties('steeringFiles/alias.properties') lcsim.setDetectorModel(detector+".zip") #lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio')) lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!! lcsim.setNumberOfEvents(nOutputEvents) #print lcsim.listAttributes() result = job.append(lcsim) if not result['OK']: print result['Message'] sys.exit(2) ################## slicPandora ########################################### if 4 in args.chain: slicPandora = SLICPandora() slicPandora.setVersion(softVersions[2]) slicPandora.setDetectorModel(detector) slicPandora.getInputFromApp(lcsim) slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio')) slicPandora.setPandoraSettings('pandoraSettings.xml') slicPandora.setNumberOfEvents(nOutputEvents) #print slicPandora.listAttributes() result = job.append(slicPandora) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus Vertexing ############################ if 5 in args.chain: vertexing = Marlin() vertexing.setVersion(softVersions[3]) vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml') vertexing.setGearFile('steeringFiles/' + detector + '.gear') vertexing.getInputFromApp(slicPandora) vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio')) vertexing.setNumberOfEvents(nOutputEvents) #print vertexing.listAttributes() result = job.append(vertexing) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (DST production) ################################ lcsimDst = LCSIM() lcsimDst.setVersion(softVersions[1]) lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml') lcsimDst.getInputFromApp(vertexing) lcsimDst.setNumberOfEvents(nOutputEvents) lcsimDst.setAliasProperties('steeringFiles/alias.properties') lcsimDst.setDetectorModel(detector+".zip") lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio')) lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #print lcsimDst.listAttributes() result = job.append(lcsimDst) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus flavortag ############################ if 6 in args.chain: flavortag = Marlin() flavortag.setVersion(softVersions[3]) flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml') flavortag.setGearFile('steeringFiles/' + detector + '.gear') flavortag.setInputFile(lcsimDstOutput) flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio')) flavortag.setNumberOfEvents(nOutputEvents) #print flavortag.listAttributes() result = job.append(flavortag) if not result['OK']: print result['Message'] sys.exit(2) ################## Job Finalise ########################################## # List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on, # This list is likely to change. job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp', 'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk', 'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk']) job.setCPUTime(50000) job.setPlatform('x86_64-slc5-gcc43-opt') # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/ # directory on the grid. outputLevel = max(args.chain) if outputLevel == 1: job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM') if outputLevel == 3: #job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM') job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM') if outputLevel == 4: job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM') if outputLevel == 5: job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM') if outputLevel == 6: job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM') job.setOutputSandbox(outputSandbox) job.setInputData(args.Input) if args.dontPromptMe: job.dontPromptMe() # Submits Job!!! job.submit() return 0;
def main(argv): # Input arguments ildconfig_version = "$ILDCONFIGVER" ilcsoft_version = "$ILCSOFTVER" evts_per_run = $EVTSPERRUN detector_model = "$DETECTOR" sim_input = "$SIMINPUT" process_name = "$PROCESS" index = $IND sim_input = diracpath_from_pnfspath( sim_input ) sim_detector_model = detector_model_wo_option( detector_model ) job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model dirac = DiracILC(True,job_group+".rep") # outputs to be saved onto grid SE RECoutput = [] # DDSim evtStart = (index-1)*evts_per_run evtEnd = index*evts_per_run - 1 RandSeed = random.randrange(11623, 99999) lcinputSIM = "LFN:" + sim_input lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd)) sim = DDSim() sim.setVersion(ilcsoft_version) sim.setDetectorModel(sim_detector_model) sim.setInputFile(lcinputSIM) sim.setSteeringFile("ddsim_steer.py") sim.setNumberOfEvents(evts_per_run) sim.setRandomSeed(RandSeed) sim.setEnergy(1000) sim.setStartFrom(evtStart) sim.setOutputFile(lcoutputSIM) # Marlin lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd)) ma = Marlin() ma.setVersion(ilcsoft_version) ma.setDetectorModel(detector_model) ma.setSteeringFile("MarlinStdReco.xml") ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) ) ma.setLogFile("marlin.log") ma.getInputFromApp(sim) ma.setEnergy(1000) ma.setOutputDstFile(lcoutputDST) RECoutput.append(lcoutputDST) # ILCDirac user job job = UserJob() job.setName("user_sim_reco") job.setJobGroup(job_group) job.setILDConfig(ildconfig_version) job.setCPUTime(86400) tmp_file_name = process_name + "_sim_reco_job_tmp.py" job.setInputSandbox([tmp_file_name]) job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "]) job.dontPromptMe() job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us']) # run simulation job simres = job.append(sim) if not simres['OK']: print simres['Not ok appending ddsim to job'] quit() # run Malrin reco jobs mares = job.append(ma) if not mares['OK']: print mares['Not ok appending Marlin to job'] quit() job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM") print RECoutput submit_output = job.submit(dirac) print submit_output