def subDDSim(): # Decide parameters for a job outputSE = "KEK-SRM" outputSE = "KEK-DISK" isLocal = _clip.isLocal nbevts = 10 if _clip.numberOfEvents == 0 else _clip.numberOfEvents outputFile = "ddsim_example.slcio" if _clip.outputFile == "" else _clip.outputFile outputDir = _clip.outputDir inputFile = _clip.inputFile if inputFile == "": gLogger.error("Input file for ddsim does not given.") exit(-1) # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setJobGroup("myddsimjob") job.setName("myddsim") job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setILDConfig("v02-00-02") # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"]) # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk" ]) # a list of sites not to submit job # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units ) ddsim = DDSim() ddsim.setVersion("ILCSoft-02-00-02_gcc49") ddsim.setDetectorModel("ILD_l5_v02") ddsim.setInputFile(inputFile) ddsim.setNumberOfEvents(nbevts) extraCLIArguments = " --steeringFile ddsim_steer.py " extraCLIArguments += " --outputFile %s " % outputFile extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 " ddsim.setExtraCLIArguments(extraCLIArguments) # ddsim.setRandomSeed(1234565) # ddsim.setStartFrom(20) # Number of events to skip before starting ddsim job.append(ddsim) if outputDir != "": job.setOutputData([outputFile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
def __init__(self, clip, params): self.clip = clip self.ildConfig = params.get("ildConfig", None) self.alwaysOverlay = params.get("alwaysOverlay", False) self.runOverlay = self.clip.testOverlay or self.alwaysOverlay self.mokkaVersion = params["mokkaVersion"] self.mokkaSteeringFile = params.get("mokkaSteeringFile") self.detectorModel = params.get("detectorModel") self.marlinVersion = params.get("marlinVersion") self.marlinSteeringFile = params.get("marlinSteeringFile") self.ddsimVersion = params.get("ddsimVersion") self.ddsimDetectorModel = params.get("ddsimDetectorModel") self.ddsimInputFile = params.get("ddsimInputFile") self.marlinInputdata = params.get("marlinInputdata") self.gearFile = params.get("gearFile") self.lcsimVersion = params.get("lcsimVersion") self.steeringFileVersion = params.get("steeringFileVersion", None) self.rootVersion = params["rootVersion"] self.whizard2Version = params.get("whizard2Version") self.whizard2SinFile = params.get("whizard2SinFile") self.energy = params.get("energy") self.backgroundType = params.get("backgroundType") self.machine = params.get("machine") self.gearFile = params.get("gearFile") self.marlinSteeringFile = params.get("marlinSteeringFile") self.marlinVersion = params.get("marlinVersion") self.lcsimPreSteeringFile = params.get("lcsimPreSteeringFile") self.lcsimPostSteeringFile = params.get("lcsimPostSteeringFile") self.fccSwPath = params.get("fccSwPath") self.fccSwSteeringFile = params.get("fccSwSteeringFile") self.fccAnalysisSteeringFile = params.get("fccAnalysisSteeringFile") ### other things needed to run tests self.log = gLogger.getSubLogger("JobCreater") from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid if self.clip.submitMode == "local": self.log.notice("") self.log.notice(" DIRAC RCSID:", drcsid) self.log.notice(" Job RCSID:", jrcsid) self.log.notice("Applications RCSID:", apprcsid) self.log.notice("") self.diracInstance = DiracILC(False, 'tests.rep') self.jobList = {}
def create_job(inputData, saveName, outputDir, dontPromptMe): slcioFile = saveName + '.slcio' rootFile = saveName + '.root' if check_file_existence(outputDir, slcioFile, dontPromptMe): remove_file(outputDir, slcioFile, dontPromptMe) if check_file_existence(outputDir, rootFile, dontPromptMe): remove_file(outputDir, rootFile, dontPromptMe) dIlc = DiracILC() job = UserJob() job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml']) if SAVE_SLCIO: job.setOutputData([slcioFile, rootFile], OutputPath=outputDir, OutputSE=STORAGE_SE) else: job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE) job.setJobGroup('myMarlinRun1') job.setName('MyMarlinJob1') # job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk']) job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz']) marl = Marlin() marl.setVersion('ILCSoft-2017-11-15_gcc62') marl.setInputFile(inputData) job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData))) # marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio']) marl.setSteeringFile('marlin/FullChain.xml') # marl.setSteeringFile('marlin/FullChainNewDetModel.xml') marl.setOutputFile(slcioFile) gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear' if not os.path.isfile(gearFile): print('Error: gear file', gearFile, 'does not exist! Abort submission.') return marl.setGearFile(gearFile) marl.setExtraCLIArguments( "--MyNtupleMaker.OutputFileName={rootOutfile}".format( rootOutfile=rootFile)) # marl.setNumberOfEvents(1000) job.append(marl) if dontPromptMe: job.dontPromptMe() job.submit(dIlc) return False
def submit(self, diracinstance=None, mode="wms"): """ Submit call: when your job is defined, and all applications are set, you need to call this to add the job to DIRAC. :param diracinstance: DiracILC instance :type diracinstance: ~ILCDIRAC.Interfaces.API.DiracILC.DiracILC :param str mode: "wms" (default), "agent", or "local" .. note :: The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts """ if self.splittingOption: result = self._split() if 'OK' in result and not result['OK']: return result #Check the credentials. If no proxy or not user proxy, return an error if not self.proxyinfo['OK']: self.log.error( "Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError( "Not allowed to submit a job, you need a %s proxy." % self.usergroup, self.__class__.__name__) if 'group' in self.proxyinfo['Value']: group = self.proxyinfo['Value']['group'] if group not in self.usergroup: self.log.error( "Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError( "Not allowed to submit job, you need a %s proxy." % self.usergroup, self.__class__.__name__) else: self.log.error( "Could not determine group, you do not have the right proxy.") return self._reportError( "Could not determine group, you do not have the right proxy.") res = self._addToWorkflow() if not res['OK']: return res self.oktosubmit = True if not diracinstance: self.diracinstance = DiracILC() else: self.diracinstance = diracinstance return self.diracinstance.submit(self, mode)
def testAndProbeSites(): """submits jobs to test sites""" clip = Params() clip.registerSwitches() Script.parseCommandLine() from DIRAC import gLogger, exit as dexit from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import CheckWNs from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueues res = getQueues(siteList=clip.site, ceList=clip.ce) if not res['OK']: gLogger.error("Failed getting the queues", res['Message']) dexit(1) sitedict = res['Value'] CEs = [] for ces in sitedict.values(): CEs.extend(ces.keys()) gLogger.notice("Found %s CEs to look at." % len(CEs)) d = DiracILC(True, "SiteProbe.rep") for CE in CEs: j = UserJob() j.setDestinationCE(CE) c = CheckWNs() res = j.append(c) if not res['OK']: gLogger.error(res['Message']) continue j.setOutputSandbox("*.log") j.setCPUTime(30000) j.dontPromptMe() res = j.submit(d) if not res['OK']: gLogger.error("Failed to submit job, aborting") dexit(1) dexit(0)
def subWhizard2(): from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin, Whizard2 # Decide parameters for a job outputSE = "KEK-SRM" isLocal = _clip.isLocal nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents outputFile = "E500.P2f_bB.GWhizard2.I100000.e0.p0.n001.slcio" if _clip.outputFile == "" else _clip.outputFile outputDir = _clip.outputDir # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setInputSandbox(["pythia6-parameters.sin", "P2f_qqbar.sin"]) job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setOutputData([outputFile]) job.setJobGroup("mywhiz2") job.setName("mywhizard2") whiz = Whizard2() whiz.setVersion("2.7.0") whiz.setNumberOfEvents(nbevts) whiz.setEvtType("P2f_bB") whiz.setProcessVariables("P2f_bB") # whiz.setRandomSeed(15) whiz.setSinFile("P2f_qqbar.sin") whiz.setOutputFile(outputFile) job.append(whiz) if outputDir != "": print " outputDir = " + outputDir print " outputSE = " + outputSE job.setOutputData([outputFile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
def subDDSim(clip1): # Decide parameters for a job outputSE = "KEK-SRM" isLocal = clip1.isLocal nbevts = 0 if clip1.numberOfEvents == 0 else clip1.numberOfEvents #print('inside subddsim(): nbevts ', nbevts) outputFile = "" if clip1.outputFile == "" else clip1.outputFile #print('inside subddsim outfile ', outputFile) outputDir = clip1.outputDir #print('inside subddsim outdir ', outputDir) inputFile = clip1.inputFile #print('inside subddsim inputFile ', inputFile) if inputFile == "": gLogger.error("Input file for ddsim is not given.") exit(-1) # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setJobGroup("myddsimjob") job.setName("myddsim") job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setILDConfig("v02-00-02") # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"]) # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk" ]) # a list of sites not to submit job # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units ) ddsim = DDSim() ddsim.setVersion("ILCSoft-02-00-02_gcc49") ddsim.setDetectorModel("ILD_l5_v05") ddsim.setInputFile(inputFile) ddsim.setNumberOfEvents(nbevts) extraCLIArguments = " --steeringFile ddsim_steer_July26.py" extraCLIArguments += " --outputFile %s " % outputFile extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 " ddsim.setExtraCLIArguments(extraCLIArguments) return ddsim
def _createLFNList(): """create the LFnList""" cliparams = _Params() cliparams.registerSwitches() Script.parseCommandLine(ignoreErrors=False) from DIRAC import gLogger repoLocation = cliparams.repo if not repoLocation: Script.showHelp() dexit(2) from ILCDIRAC.Interfaces.API.DiracILC import DiracILC dirac = DiracILC(True, repoLocation) dirac.monitorRepository(False) lfns = [] lfns = dirac.retrieveRepositoryOutputDataLFNs() gLogger.notice("lfnlist=[") for lfn in lfns: gLogger.notice('"LFN:%s",' % lfn) gLogger.notice("]") dexit(0)
def __init__(self): self.dIlc = DiracILC(False) self.job = UserJob() self.job.setJobGroup("FCC") self.job.setName("FCC APP") self.job.setOutputSandbox(["*.log", '*.root']) #self.job.setDestination('LCG.DESY-HH.de') #EOS public location self.EOS_MGM_URL = 'root://eospublic.cern.ch' #EOS environment self.setEOS = 'export EOS_MGM_URL=' + self.EOS_MGM_URL self.myclient = client.FileSystem(self.EOS_MGM_URL + ':1094') #sandbox self.InputSandbox = [] self.folders_to_upload = [] self.filtered_extensions = [] self.excludes_or_includes = [] self.temp_cwd = os.path.join(os.getcwd(), 'fcc_temp_dirac')
nameDir = nameJobGroup+'_files'+nameTag nameRepositoryFile = nameJobGroup+nameTag+'_repository.rep' ##################################################################### ##################################################################### #set enviroment import os import sys from DIRAC.Core.Base import Script #dirac enviroment Script.parseCommandLine() #dirac enviroment from ILCDIRAC.Interfaces.API.DiracILC import DiracILC #job receiver class dirac = DiracILC(True,nameRepositoryFile) #important to retrive jobs ##################################################################### ##################################################################### #job definition from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob ##################################################################### ##################################################################### #filecatalog from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient ##################################################################### ##################################################################### #ddsim from ILCDIRAC.Interfaces.API.NewInterface.Applications import DDSim
def all_jobs(name): d = DiracILC(True, "repo.rep") ################################################ j = UserJob() j.setJobGroup("PM1") j.setName("Exec1") banned_sites = [ "OSG.BNL.us", "LCG.UKI-NORTHGRID-LIV-HEP.uk", "OSG.UCSDT2.us", "LCG.SCOTGRIDDURHAM.uk", "LCG.NIKHEF.nl", "LCG.UKI-SOUTHGRID-RALPP.uk", "LCG.GRIF.fr", "LCG.Manchester.uk", "LCG.UKI-LT2-IC-HEP.uk", "LCG.Weizmann.il" ] j.setBannedSites(banned_sites) caindir = name #print('Cain directory is ',caindir) indata = [ 'LFN:/ilc/user/a/amustahid/cain.exe', str(caindir), 'LFN:/ilc/user/a/amustahid/runcain.sh', 'LFN:/ilc/user/a/amustahid/convert_pairs_lcio.py', 'LFN:/ilc/user/a/amustahid/pyLCIO.tar.gz', '/home/belle2/mustahid/useful/my.sh', './splitInput.py', './subddsim.py', './ddsim_steer_July26.py', './ILD_l5_v05.xml', './my2.sh', './dbd_500GeV.nung_1.xml', 'LFN:/ilc/user/a/amustahid/myProcessors.tar.gz', './create_dir.py', './conf.py', './util.py', './testcain.sh', './beam_250.i' ] j.setInputSandbox(indata) ################################################ #app = GenericApplication() #app.setScript("create_dir.py") #app.setInputFile("testcain.sh") #logf = 'create_dir.log' #app.setLogFile(logf) #app.setDebug(debug=True) #create_dirname = 'create_dir' #app.setName(create_dirname) #res=j.append(app) #if not res['OK']: # print res['Message'] # exit(1) ################################################ appre = GenericApplication() name = name.split('/') #print(name) cain_name = name[-1] subdir = name[-2] dirname = name[-3] #print('Cain file name ', cain_name) appre.setScript("LFN:/ilc/user/a/amustahid/runcain.sh") #appre.setScript("testcain.sh") ifile = cain_name.split('.') ifile = ifile[0] + '.' + ifile[1] + '.' + ifile[2] #print('ifile ',ifile) appre.setArguments(ifile) #direc = 'LFN:/ilc/user/a/amustahid/' #appre.setInputFile(ifile+".i") #appre.setArguments("This is input arguments") logf = ifile + '_' + subdir + '.log' appre.setLogFile(logf) appre.setDebug(debug=True) name = 'CAIN' appre.setName(name) res = j.append(appre) if not res['OK']: print res['Message'] exit(1) ################################################ ################################################ #appost = GenericApplication() #appost.setScript("myanal.sh") #appost.setArguments("This is my analysis step") #res=j.append(appost) #if not res['OK']: # print res['Message'] # exit(1) ap = GenericApplication() ap.setScript('my.sh') logf = 'my.log' ap.setLogFile(logf) ap.setDebug(debug=True) name = 'my' ap.setName(name) res = j.append(ap) if not res['OK']: print res['Message'] exit(1) outfile = 'incoherent_pair.dat' appre.setOutputFile(outfile) ################################################ direc = 'incoherent_pair' inputFile = direc + '/' + 'inco_pair_split.slcio' # global variables to hold command line parameters # ###################################### base = '.' #outdir=base+'/'+dirname+'/slcio_test_2ndrun' outdir = base + '/' + dirname + '/Run_7' #print('outdir'+' '+str(outdir)) geant_name = ifile outputFile = geant_name + '_' + subdir + '.slcio' #_clip = _Params(False,1,inputFile,outputFile,outdir) nbevents = 100 clip = _Params(nbevents, inputFile, outputFile, outdir) ddsim = subDDSim(clip) ################################################ res = j.append(ddsim) if not res['OK']: print res['Message'] exit(1) j.setOutputData(outputFile, outdir, "KEK-SRM") j.setOutputSandbox(["*.log", "*.dat", "*.slcio"]) j.dontPromptMe() res = j.submit(d) #res = j.submit(d, mode='local') if res['OK']: print str(res["Value"]) #print "Dirac job, "+str(res["Value"])+", was submitted." else: print "Failed to submit Dirac job. return message was as follows." pprint.pprint(res)
elif opt in ('v', 'version'): slicVersion = arg elif opt in ('w', 'workflow'): workflowName = arg elif opt in ('n', 'nocheck'): checkMeta = False if (detectorName == None) or (prodID == None): Script.showHelp() sys.exit(2) from ILCDIRAC.Interfaces.API.NewInterface.ProductionJob import ProductionJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC, LCSIM, SLICPandora, OverlayInput from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient dirac = DiracILC() meta = {} meta['ProdID'] = prodID if eventType: meta['EvtType'] = eventType if energy: meta['Energy'] = energy if dataType: meta['Datatype'] = dataType fc = FileCatalogClient() res = fc.getCompatibleMetadata(meta) if not res['OK']: print "Error looking up the catalog for metadata" exit(2)
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka, Marlin, OverlayInput from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC import exit as dexit d = DiracILC(True, "repo.rep") n_evts = 500 n_evts_per_job = 100 n_jobs = n_evts / n_evts_per_job for i in range(n_jobs): j = UserJob() mo = Mokka() mo.setEnergy(3000) mo.setVersion("0706P08") mo.setSteeringFile("clic_ild_cdr.steer") mo.setMacFile("particlegun_electron.mac") mo.setOutputFile("MyFile.slcio") mo.setNbEvts(n_evts_per_job) res = j.append(mo) if not res['OK']: print res['Message'] break ma = Marlin() ma.setVersion("v0111Prod")
def __init__(self, *args, **kwargs): AgentModule.__init__(self, *args, **kwargs) self.name = 'DataRecoveryAgent' self.enabled = False self.productionsToIgnore = self.am_getOption("ProductionsToIgnore", []) self.transformationTypes = self.am_getOption("TransformationTypes", [ 'MCReconstruction', 'MCSimulation', 'MCReconstruction_Overlay', 'MCGeneration' ]) self.transformationStatus = self.am_getOption("TransformationStatus", ['Active', 'Completing']) self.shifterProxy = self.am_setOption('shifterProxy', 'DataManager') self.jobStatus = [ 'Failed', 'Done' ] ##This needs to be both otherwise we cannot account for all cases self.jobMon = JobMonitoringClient() self.fcClient = FileCatalogClient() self.tClient = TransformationClient() self.reqClient = ReqClient() self.diracILC = DiracILC() self.inputFilesProcessed = set() self.todo = {'MCGeneration': [ dict( Message="MCGeneration: OutputExists: Job 'Done'", ShortMessage="MCGeneration: job 'Done' ", Counter=0, Check=lambda job: job.allFilesExist() and job.status=='Failed', Actions=lambda job,tInfo: [ job.setJobDone(tInfo) ] ), dict( Message="MCGeneration: OutputMissing: Job 'Failed'", ShortMessage="MCGeneration: job 'Failed' ", Counter=0, Check=lambda job: job.allFilesMissing() and job.status=='Done', Actions=lambda job,tInfo: [ job.setJobFailed(tInfo) ] ), # dict( Message="MCGeneration, job 'Done': OutputExists: Task 'Done'", # ShortMessage="MCGeneration: job already 'Done' ", # Counter=0, # Check=lambda job: job.allFilesExist() and job.status=='Done', # Actions=lambda job,tInfo: [ tInfo._TransformationInfo__setTaskStatus(job, 'Done') ] # ), ], 'OtherProductions': [ \ ## should always be first! dict( Message="One of many Successful: clean others", ShortMessage="Other Tasks --> Keep", Counter=0, Check=lambda job: job.allFilesExist() and job.otherTasks and job.inputFile not in self.inputFilesProcessed, Actions=lambda job,tInfo: [ self.inputFilesProcessed.add(job.inputFile), job.setJobDone(tInfo), job.setInputProcessed(tInfo) ] ), dict( Message="Other Task processed Input, no Output: Fail", ShortMessage="Other Tasks --> Fail", Counter=0, Check=lambda job: job.inputFile in self.inputFilesProcessed and job.allFilesMissing() and job.status!='Failed', Actions=lambda job,tInfo: [ job.setJobFailed(tInfo) ] ), dict( Message="Other Task processed Input: Fail and clean", ShortMessage="Other Tasks --> Cleanup", Counter=0, Check=lambda job: job.inputFile in self.inputFilesProcessed and not job.allFilesMissing(), Actions=lambda job,tInfo: [ job.setJobFailed(tInfo), job.cleanOutputs(tInfo) ] ), dict( Message="InputFile missing: mark job 'Failed', mark input 'Deleted', clean", ShortMessage="Input Missing --> Job 'Failed, Input 'Deleted', Cleanup", Counter=0, Check=lambda job: job.inputFile and not job.inputFileExists and job.fileStatus != "Deleted", Actions=lambda job,tInfo: [ job.cleanOutputs(tInfo), job.setJobFailed(tInfo), job.setInputDeleted(tInfo) ] ), dict( Message="InputFile Deleted, output Exists: mark job 'Failed', clean", ShortMessage="Input Deleted --> Job 'Failed, Cleanup", Counter=0, Check=lambda job: job.inputFile and not job.inputFileExists and job.fileStatus == "Deleted" and not job.allFilesMissing(), Actions=lambda job,tInfo: [ job.cleanOutputs(tInfo), job.setJobFailed(tInfo) ] ), ## All Output Exists dict( Message="Output Exists, job Failed, input not Processed --> Job Done, Input Processed", ShortMessage="Output Exists --> Job Done, Input Processed", Counter=0, Check=lambda job: job.allFilesExist() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus!="Processed" and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setJobDone(tInfo), job.setInputProcessed(tInfo) ] ), dict( Message="Output Exists, job Failed, input Processed --> Job Done", ShortMessage="Output Exists --> Job Done", Counter=0, Check=lambda job: job.allFilesExist() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus=="Processed" and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setJobDone(tInfo) ] ), dict( Message="Output Exists, job Done, input not Processed --> Input Processed", ShortMessage="Output Exists --> Input Processed", Counter=0, Check=lambda job: job.allFilesExist() and \ not job.otherTasks and \ job.status=='Done' and \ job.fileStatus!="Processed" and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setInputProcessed(tInfo) ] ), ## outputmissing dict( Message="Output Missing, job Failed, input Assigned, MaxError --> Input MaxReset", ShortMessage="Max ErrorCount --> Input MaxReset", Counter=0, Check=lambda job: job.allFilesMissing() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFile not in self.inputFilesProcessed and \ job.inputFileExists and \ job.errorCount > MAXRESET, Actions=lambda job,tInfo: [ job.setInputMaxReset(tInfo) ] ), dict( Message="Output Missing, job Failed, input Assigned --> Input Unused", ShortMessage="Output Missing --> Input Unused", Counter=0, Check=lambda job: job.allFilesMissing() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFile not in self.inputFilesProcessed and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setInputUnused(tInfo) ] ), dict( Message="Output Missing, job Done, input Assigned --> Job Failed, Input Unused", ShortMessage="Output Missing --> Job Failed, Input Unused", Counter=0, Check=lambda job: job.allFilesMissing() and \ not job.otherTasks and \ job.status=='Done' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFile not in self.inputFilesProcessed and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setInputUnused(tInfo), job.setJobFailed(tInfo) ] ), ## some files missing, needing cleanup. Only checking for ## assigned, because processed could mean an earlier job was ## succesful and this one is just the duplicate that needed ## to be removed! But we check for other tasks earlier, so ## this should not happen dict( Message="Some missing, job Failed, input Assigned --> cleanup, Input 'Unused'", ShortMessage="Output Missing --> Cleanup, Input Unused", Counter=0, Check=lambda job: job.someFilesMissing() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFileExists, Actions=lambda job,tInfo: [job.cleanOutputs(tInfo),job.setInputUnused(tInfo)] #Actions=lambda job,tInfo: [] ), dict( Message="Some missing, job Done, input Assigned --> cleanup, job Failed, Input 'Unused'", ShortMessage="Output Missing --> Cleanup, Job Failed, Input Unused", Counter=0, Check=lambda job: job.someFilesMissing() and \ not job.otherTasks and \ job.status=='Done' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFileExists, Actions=lambda job,tInfo: [job.cleanOutputs(tInfo),job.setInputUnused(tInfo),job.setJobFailed(tInfo)] #Actions=lambda job,tInfo: [] ), dict( Message="Some missing, job Done --> job Failed", ShortMessage="Output Missing, Done --> Job Failed", Counter=0, Check=lambda job: not job.allFilesExist() and job.status=='Done', Actions=lambda job,tInfo: [job.setJobFailed(tInfo)] ), dict ( Message="Something Strange", ShortMessage="Strange", Counter=0, Check=lambda job: job.status not in ("Failed","Done"), Actions=lambda job,tInfo: [] ), ##should always be the last one! dict ( Message="Failed Hard", ShortMessage="Failed Hard", Counter=0, Check=lambda job: False, ## never Actions=lambda job,tInfo: [] ), ] } self.jobCache = defaultdict(lambda: (0, 0)) self.printEveryNJobs = self.am_getOption('PrintEvery', 200) ##Notification self.notesToSend = "" self.addressTo = self.am_getOption('MailTo', ["*****@*****.**"]) self.addressFrom = self.am_getOption('MailFrom', "*****@*****.**") self.subject = "DataRecoveryAgent"
ma.setSteeringFile("clic_ild_cdr_steering.xml") ma.setGearFile("clic_ild_cdr.gear") result = j.append(ma) if not result['OK']: gLogger.error(result["Message"]) dexit(1) j.setCPUTime(10000) j.setOutputSandbox("*.log") return j if __name__ == '__main__': lfns = getFiles() #get a list of files d = DiracILC(True, "paramjobtest.rep") #get your dirac instance job = getJob() #get a job, any can do #here is where the interesting stuff happen from DIRAC.Core.Utilities.List import breakListIntoChunks for flist in breakListIntoChunks(lfns, 200): #200 is the number of files per chunk, and the max number of jobs produced in one go #This is the magical line job.setParametricInputData(flist) #The rest of the sumission is the same res = job.submit(d) if not res["OK"]: gLogger.error("Failed to submit the job: ", res["Message"])
#every single script has these four lines from DIRAC.Core.Base import Script Script.parseCommandLine() #create ilcdirac instance from ILCDIRAC.Interfaces.API.DiracILC import DiracILC dirac = DiracILC(True, "some_job_repository.rep") #job definition from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob #job = UserJob() from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM #jobParams = [('slicTest8_mu+_theta90.mac','diracTest8_mu+_theta90.slcio',50),('slicTest7_mu+_theta_5-175.mac','diracTest_mu+_theta_5-175.slcio',50),('slicTest3_e+.mac','diracTest3_e+.slcio',10),('slicTest2_pi+.mac','diractTest2_pi+.slcio',10)] #jobParams = [('slicTest10_mu+_100gev_theta70_testNewGeom.mac','diracTest10_mu+_100gev_theta70_testNewGeom.slcio',10),('slicTest10_mu+_100gev_theta90_testNewGeom.mac','diracTest10_mu+_100gev_theta90_testNewGeom.slcio',10)] jobParams = [ ('/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/slicmacros/slicTest8_mu+_100gev_theta60.mac', 'diracTest_100gev_theta60_vtx_matbudghalf_nonsensitivelayer.slcio', 100), ('/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/slicmacros/slicTest8_mu+_10gev_theta60.mac', 'diracTest_10gev_theta60_vtx_matbudghalf_nonsensitivelayer.slcio', 100) ] #slicMacros = ['slicTest8_mu+_theta90.mac','slicTest7_mu+_theta_5-175.mac','slicTest3_e+.mac','slicTest2_pi+.mac'] #fileOutputs = ['diracTest2Loop1.slcio','diracTest2Loop2.slcio','diracTest2Loop3.slcio','diractTest2Loop4.slcio'] #slicNumEvents = [100,100,10,10] for macro, output, nEvts in jobParams: job = UserJob() job.setName("ssetru_dirac_test1") job.setJobGroup("tests")
def subOverlay(): # Decide parameters for a job outputSE = "KEK-SRM" isLocal = _clip.isLocal nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents nbevts = 0 # To analize all input events outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix outputDir = _clip.outputDir inputFile = _clip.inputFile if inputFile == "": gLogger.error("Input file for ddsim does not given.") exit(-1) recfile = outputFilePrefix + ".rec.slcio" dstfile = outputFilePrefix + ".dst.slcio" detector_model = "ILD_l5_o1_v02" key = detector_model.split('_') sim_detectorModel = "_".join([key[0], key[1], key[3]]) # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setJobGroup("myoverlayjob") job.setName("myoverlay") job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setILDConfig("v02-00-02") # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"]) # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination # job.setBannedSites([]) # a list of sites not to submit job # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units ) # Create Overlay application ovldata = [{ "ProcessorName": "BgOverlayWW", "evttype": "aa_lowpt_WW", "ProdID": 10237, "expBG": 0.211, "subdir": "000" }, { "ProcessorName": "BgOverlayWB", "evttype": "aa_lowpt_WB", "ProdID": 10241, "expBG": 0.24605, "subdir": "000" }, { "ProcessorName": "BgOverlayBW", "evttype": "aa_lowpt_BW", "ProdID": 10239, "expBG": 0.243873, "subdir": "000" }, { "ProcessorName": "BgOverlayBB", "evttype": "aa_lowpt_BB", "ProdID": 10235, "expBG": 0.35063, "subdir": "000" }, { "ProcessorName": "PairBgOverlay", "evttype": "seeablepairs", "ProdID": 10233, "expBG": 1.0, "subdir": "100" }] BXOverlay = 1 NbSigEvtsPerJob = 100 numberOfSignalEvents = NbSigEvtsPerJob basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws" energy = "500" for ovl in ovldata: print "### OverlayInput ... " + ovl["ProcessorName"] ovlapp = OverlayInput() ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \ ( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] ) print " OverlayPath ... " + ovlpath ovlapp.setMachine("ilc_dbd") # ovlapp.setEnergy(energy) # ovlapp.setDetectorModel(sim_detectorModel) ovlapp.setProcessorName(ovl["ProcessorName"]) ovlapp.setBkgEvtType(ovl["evttype"]) ovlapp.setPathToFiles(ovlpath) ovlapp.setGGToHadInt(ovl["expBG"]) ovlapp.setBXOverlay(BXOverlay) ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob) ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents) res = job.append(ovlapp) if not res['OK']: print res['Message'] exit(1) # Create Marlin application marlin = Marlin() marlin.setVersion("ILCSoft-02-00-02_gcc49") marlin.setDetectorModel(detector_model) marlin.setSteeringFile("MarlinStdReco.xml") marlin.setInputFile(inputFile) marlin.setNumberOfEvents(nbevts) marlin.setOutputDstFile(dstfile) marlin.setOutputRecFile(recfile) extraCLIArguments = " --constant.DetectorModel=%s " % detector_model extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str( energy) extraCLIArguments += " --global.Verbosity=MESSAGE " marlin.setExtraCLIArguments(extraCLIArguments) job.append(marlin) if outputDir != "": job.setOutputData([dstfile, recfile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.UserJob import * from ILCDIRAC.Interfaces.API.NewInterface.Applications import * dirac = DiracILC(True, "CLICdet_repo.cfg") job = UserJob() job.setName("example") job.setInputData([ "/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep" ]) job.setCLICConfig("ILCSoft-2017-12-21") job.setOutputSandbox(["*.root", "*.slcio", "*.log"]) ddsim = DDSim() ddsim.setSteeringFile("CLICPerformance/examples/clic_steer.py") ddsim.setVersion("ILCSoft-2017-12-21_gcc62") ddsim.setDetectorModel("CLIC_o3_v14") ddsim.setInputFile( "/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep" ) ddsim.setOutputFile("ddsim_output.slcio") ddsim.setStartFrom(0) ddsim.setNumberOfEvents(10) res1 = job.append(ddsim) if not res1['OK']: print res1['Message'] sys.exit(2) # -------------------- Comment if gg_had overlay not wanted -----------------------------------------------------#
def main(): # Take the input arguments from the argument parser, and check they exist... args = parse_args() if not args: print 'Invalid Arguments' sys.exit(1) #### Software Versions #### softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended) # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working # Check the --runs and --split arguments to make sure they are compatible, if not exit... if not check_events_arguments(args.events, args.split): sys.exit(1) # Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning... lfn_check, lfn = check_input_LFN(args.stdhepInput) if not lfn_check: sys.exit(1) # Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included... dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector)) # Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging, # into the input sandbox inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag) # Prepares values for the job loop... if args.split < 0: nInputEvents = int(args.events) nOutputEvents = int(args.events) if args.split > 0: nInputEvents = int(args.events) nOutputEvents = int(args.split) # Loop that runs through the required number of jobs to be executed... for startEvent in range(0, nInputEvents, nOutputEvents): ################## Job Initialise ######################################## job = UserJob() job.setName(path.basename(args.stdhepInput)) job.setJobGroup('JobGroup') job.setInputSandbox(inputSandbox) fileNumber = startEvent / nOutputEvents print "Job ", fileNumber outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions) slicOutput = outputFiles['slicOutput'] prePandoraOutput = outputFiles['prePandoraOutput'] pandoraOutput = outputFiles['pandoraOutput'] vertexingOutput = outputFiles['vertexingOutput'] lcsimRecOutput = outputFiles['lcsimRecOutput'] lcsimDstOutput = outputFiles['lcsimDstOutput'] flavortagOutput = outputFiles['flavortagOutput'] diracOutput = outputFiles['diracOutput'] ################## SLIC ################################################## slic = SLIC() slic.setVersion(softVersions[0]) slic.setSteeringFile(args.macFile) # slic.setInputFile(lfn) slic.setOutputFile(slicOutput) slic.setDetectorModel(args.detector) slic.setNumberOfEvents(nOutputEvents) slic.setStartFrom(startEvent) #print slic.listAttributes() result = job.append(slic) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (digitization and tracking) ##################### lcsim = LCSIM() lcsim.setVersion(softVersions[1]) lcsim.setSteeringFile( 'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml' ) # Another version is included in /steeringFiles lcsim.getInputFromApp(slic) lcsim.setTrackingStrategy( 'steeringFiles/sidloi3_trackingStrategies_default.xml') # lcsim.setAliasProperties('alias.properties') lcsim.setDetectorModel('geometryFiles/sidloi3.zip') lcsim.setOutputFile(prePandoraOutput) lcsim.setNumberOfEvents(nOutputEvents) #print lcsim.listAttributes() result = job.append(lcsim) if not result['OK']: print result['Message'] sys.exit(2) ################## slicPandora ########################################### slicPandora = SLICPandora() slicPandora.setVersion(softVersions[2]) slicPandora.setDetectorModel(args.detector) slicPandora.getInputFromApp(lcsim) slicPandora.setOutputFile(pandoraOutput) slicPandora.setPandoraSettings('pandoraSettings.xml') slicPandora.setNumberOfEvents(nOutputEvents) #print slicPandora.listAttributes() result = job.append(slicPandora) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus Vertexing ############################ vertexing = Marlin() vertexing.setVersion(softVersions[3]) vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml') vertexing.setGearFile('steeringFiles/sidloi3.gear') vertexing.getInputFromApp(slicPandora) vertexing.setOutputFile(vertexingOutput) vertexing.setNumberOfEvents(nOutputEvents) #print vertexing.listAttributes() result = job.append(vertexing) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (DST production) ################################ lcsimDst = LCSIM() lcsimDst.setVersion(softVersions[1]) lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml') lcsimDst.getInputFromApp(vertexing) lcsimDst.setNumberOfEvents(nOutputEvents) # lcsimDst.setAliasProperties('alias.properties') lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip') lcsimDst.setOutputRecFile(lcsimRecOutput) lcsimDst.setOutputDstFile(lcsimDstOutput) #print lcsimDst.listAttributes() result = job.append(lcsimDst) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus flavortag ############################ if args.flavortag: flavortag = Marlin() flavortag.setVersion(softVersions[3]) flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml') flavortag.setGearFile('steeringFiles/sidloi3.gear') flavortag.setInputFile(lcsimDstOutput) flavortag.setOutputFile(flavortagOutput) flavortag.setNumberOfEvents(nOutputEvents) #print flavortag.listAttributes() result = job.append(flavortag) if not result['OK']: print result['Message'] sys.exit(2) ################## Job Finalise ########################################## # List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on, # This list is likely to change. job.setBannedSites([ 'LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp', 'OSG.PNNL.us', ]) job.setCPUTime(50000) job.setPlatform('x86_64-slc5-gcc43-opt') # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/ # directory on the grid. if args.flavortag: job.setOutputData(flavortagOutput, diracOutput, args.SE) else: job.setOutputData(lcsimDstOutput, diracOutput, args.SE) job.setOutputSandbox(outputSandbox) job.setInputData(lfn) if args.dontPromptMe: job.dontPromptMe() # Submits Job!!! job.submit() return 0
base = open(steeringTemplateFile,'r') steeringTemplateContent = base.read() base.close() for eventSelection in eventsToSimulate: eventType = eventSelection['EventType'] detectorModel = eventSelection['DetectorModel'] reconstructionVariant = eventSelection['ReconstructionVariant'] energy = eventSelection['Energy'] analysisTag = eventSelection['AnalysisTag'] # Make local gear file os.system('cp ' + gearFile + ' .') gearFileLocal = os.path.basename(gearFile) diracInstance = DiracILC(withRepo=False) slcioFormat = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_(.*?)_(.*?)_(.*?)_DST.slcio' slcioFilesToProcess = getDstSlcioFiles(jobDescription,detectorModel,reconstructionVariant,energy,eventType) if not slcioFilesToProcess: print 'No slcio files found. Exiting job submission.' sys.exit() for slcioFile in slcioFilesToProcess: print 'Checking ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. Slcio file ' + slcioFile + '.' slcioFileNoPath = os.path.basename(slcioFile) inputSandbox = ['LFN:/ilc/user/s/sgreen/SelectionProcessorTarBall/MarlinSelectionProcessor.tar.gz']
def main(): ''' This is the script main method, which will hold all the logic. ''' cliParams = Params() cliParams.registerSwitches() Script.parseCommandLine(ignoreErrors=True) consistent = cliParams.checkConsistency() if not consistent['OK']: gLogger.error("Error checking consistency:", consistent['Message']) Script.showHelp() dexit(2) ##Get prodID prodID = float(cliParams.prodID) ##Get all possible input files inputFiles = getInputFiles(prodID) ##Get suffix suffix = cliParams.suffix if suffix: suffix = "_" + suffix ##Jet clustering algorithm = "ValenciaPlugin {radius:.1f} {beta:.1f} {gamma:.1f}" jetRadius = float(cliParams.jetRadius) jetBeta = float(cliParams.jetBeta) jetGamma = float(cliParams.jetGamma) jetCluster = "ExclusiveNJets 2" jetRecomb = "E_scheme" ##Top tagger deltaR = float(cliParams.deltaR) deltaP = float(cliParams.deltaP) cos_theta_W_max = float(cliParams.cos_theta_W_max) ##Sixfermion sample sixFermionSample = cliParams.sixFermionSample ##Number of files per job nrFilesPerJob = int(cliParams.nrFilesPerJob) from ILCDIRAC.Interfaces.API.DiracILC import DiracILC repDir = "/afs/cern.ch/user/l/lstroem/clicdp/analysis/steering/chain/topasymmetry_wflavourtag/submit/{prodID:04.0f}/rep/".format( prodID=prodID) subprocess.call("mkdir -p " + repDir, shell=True) dirac = DiracILC( False ) #, repDir+"topasymmetry_vlc{suffix}.rep".format(suffix = suffix)) jetAlgo = algorithm.format(radius=jetRadius, beta=jetBeta, gamma=jetGamma) inputFileList = [] i = 0 j = 1 for inputFile in inputFiles: inputFileList.append(inputFile) i += 1 if (i >= nrFilesPerJob * j) or (i == len(inputFiles)): jobName = "topasymmetry_chain_{jetSettings}_dR{deltaR:.2f}_dP{deltaP:.2f}_cthetaWmax{cos_theta_W_max:.2f}_1Jun2017_part{index}_{prodID:04.0f}".format( jetSettings=getJetSettings(jetAlgo, jetCluster, jetRecomb), deltaR=deltaR, deltaP=deltaP, cos_theta_W_max=cos_theta_W_max, index=j, prodID=prodID) jobGroup = "topasymmetry_chain_{prodID:04.0f}".format( prodID=prodID) job, outputFile, rootFile = defGridJob(jobName, jobGroup, inputFileList) ##Check if outputfile already exists pOutCheck = subprocess.Popen( "dirac-dms-lfn-replicas /ilc/user/r/rstrom/" + jobGroup + "/" + rootFile, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) outCheck, errCheck = pOutCheck.communicate() #gLogger.notice(jobName) if "no such file" not in outCheck.lower(): gLogger.notice('File exists! Skipping!') inputFileList = [] j += 1 continue #continue #use break if only part1, use continue of run over all parts #gLogger.notice(jetAlgo) #doLogger(jobName, inputFileList, outputFile, rootFile) res = job.append( defMarlin(outputFile, rootFile, jetAlgo=jetAlgo, jetCluster=jetCluster, jetRecomb=jetRecomb, deltaR=deltaR, deltaP=deltaP, cos_theta_W_max=cos_theta_W_max, sixFermionSample=sixFermionSample)) if not res['OK']: #Catch if there is an error print res['Message'] #Print the error message dexit ##Job submission print job.submit(dirac) inputFileList = [] j += 1 #break #add break of only part1 gLogger.notice("All done!") dexit(0)
lfns = res['Value'] print "found %s files" % len(lfns) ovi = OverlayInput() ovi.setEnergy(500.) ovi.setBXOverlay(300) ovi.setGGToHadInt(0.3) ovi.setNbSigEvtsPerJob(10) ovi.setBkgEvtType("gghad") ovi.setDetectorModel("CLIC_ILD_CDR") overlay = [True, False] for ov in overlay: d = DiracILC(True, "repo_overlay_%s.rep" % ov) for lfn in lfns: j = UserJob() steeringf = "clic_ild_cdr_steering.xml" if ov: steeringf = "clic_ild_cdr_steering_overlay.xml" res = j.append(ovi) if not res['OK']: print res['Message'] continue ma = Marlin() ma.setVersion("v0111Prod") ma.setGearFile("clic_ild_cdr.gear") ma.setSteeringFile(steeringf) ma.setInputFile("LFN:" + lfn) ma.setNbEvts(10)
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient import time dIlc = DiracILC() lcoutput = [] for i in range(1, 301): lcoutput = "aa_%d.root" % i job = UserJob() job.setDestination('LCG.CERN.ch') job.setInputSandbox([ "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraLikelihoodData9EBin_CLIC_ILD.xml", "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraSettingsFast.xml", "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/steering.xml", "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/clic_ild_cdr.gear", "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/lib.tar.gz" ]) job.setInputData( "LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" % i) job.setOutputSandbox( ["*.log", "*.sh", "*.py", "*.out", "*.xml", "*.steer "]) job.setJobGroup("myRoot") job.setName("root_aa_%d" % i) marl = Marlin() marl.setVersion('ILCSoft-2016-09-27_gcc48')
res = fc.findFilesByMetadata(meta) if not res['OK']: print res['Message'] lfns = res['Value'] #print "Found %s files" % len(lfns) filelist=[] for lfn in lfns: filelist.append(lfn) #print filelist #filelist2=filelist[0] #check if big radius in fastjetanalyzer indeed 0.7 or 1.0 jobGroup = "HZAnalyzer_190417_ee_qqqq_m_qqqq_2TeV_13696_VLC7PFOs" dirac = DiracILC(True,jobGroup+".rep") job = UserJob() #job.setExecutionEnv({'ROOT_INCLUDE_PATH':'./'}) job.setJobGroup(jobGroup) job.setOutputSandbox ( [ "*.log","*.out","*.py"] ) job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk']) #pay attention that the Zuds200 here is NOT changed job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190606/HZAnalyzerlib.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] ) job.setBannedSites(['LCG.INP3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk','LCG.Oxford.uk']) job.setSplitInputData(filelist, numberOfFilesPerJob=50) ma = Marlin() ma.setVersion('ILCSoft-2019-04-17_gcc62') #ma.setInputFile("LFN:/ilc/user/w/webermat/ddsimstdheptautau/ILC18-10-11_gcc62_CLIC_o3_v14/tautau200/ddsim_ILC181011_gcc62_tautau_200_CLIC_o3_v14_0%s.slcio"%(str(input_ind))) ma.setSteeringFile("/eos/user/w/weberma2/steeringFiles/testHZAnalyzer.xml") ma.setDetectorModel("CLIC_o3_v14") HZrootfilename2="HZStudy_ee_qqqq_m_qqqq_2TeV_13696_polm80_3TeV_wO_CLIC_o3_v14_DR7.root"
def main(argv): # Input arguments ildconfig_version = "$ILDCONFIGVER" ilcsoft_version = "$ILCSOFTVER" evts_per_run = $EVTSPERRUN detector_model = "$DETECTOR" sim_input = "$SIMINPUT" process_name = "$PROCESS" index = $IND sim_input = diracpath_from_pnfspath( sim_input ) sim_detector_model = detector_model_wo_option( detector_model ) job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model dirac = DiracILC(True,job_group+".rep") # outputs to be saved onto grid SE RECoutput = [] # DDSim evtStart = (index-1)*evts_per_run evtEnd = index*evts_per_run - 1 RandSeed = random.randrange(11623, 99999) lcinputSIM = "LFN:" + sim_input lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd)) sim = DDSim() sim.setVersion(ilcsoft_version) sim.setDetectorModel(sim_detector_model) sim.setInputFile(lcinputSIM) sim.setSteeringFile("ddsim_steer.py") sim.setNumberOfEvents(evts_per_run) sim.setRandomSeed(RandSeed) sim.setEnergy(1000) sim.setStartFrom(evtStart) sim.setOutputFile(lcoutputSIM) # Marlin lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd)) ma = Marlin() ma.setVersion(ilcsoft_version) ma.setDetectorModel(detector_model) ma.setSteeringFile("MarlinStdReco.xml") ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) ) ma.setLogFile("marlin.log") ma.getInputFromApp(sim) ma.setEnergy(1000) ma.setOutputDstFile(lcoutputDST) RECoutput.append(lcoutputDST) # ILCDirac user job job = UserJob() job.setName("user_sim_reco") job.setJobGroup(job_group) job.setILDConfig(ildconfig_version) job.setCPUTime(86400) tmp_file_name = process_name + "_sim_reco_job_tmp.py" job.setInputSandbox([tmp_file_name]) job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "]) job.dontPromptMe() job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us']) # run simulation job simres = job.append(sim) if not simres['OK']: print simres['Not ok appending ddsim to job'] quit() # run Malrin reco jobs mares = job.append(ma) if not mares['OK']: print mares['Not ok appending Marlin to job'] quit() job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM") print RECoutput submit_output = job.submit(dirac) print submit_output