def __init__(self, clip, params): self.clip = clip self.ildConfig = params.get("ildConfig", None) self.alwaysOverlay = params.get("alwaysOverlay", False) self.runOverlay = self.clip.testOverlay or self.alwaysOverlay self.mokkaVersion = params["mokkaVersion"] self.mokkaSteeringFile = params.get("mokkaSteeringFile") self.detectorModel = params.get("detectorModel") self.marlinVersion = params.get("marlinVersion") self.marlinSteeringFile = params.get("marlinSteeringFile") self.ddsimVersion = params.get("ddsimVersion") self.ddsimDetectorModel = params.get("ddsimDetectorModel") self.ddsimInputFile = params.get("ddsimInputFile") self.marlinInputdata = params.get("marlinInputdata") self.gearFile = params.get("gearFile") self.lcsimVersion = params.get("lcsimVersion") self.steeringFileVersion = params.get("steeringFileVersion", None) self.rootVersion = params["rootVersion"] self.whizard2Version = params.get("whizard2Version") self.whizard2SinFile = params.get("whizard2SinFile") self.energy = params.get("energy") self.backgroundType = params.get("backgroundType") self.machine = params.get("machine") self.gearFile = params.get("gearFile") self.marlinSteeringFile = params.get("marlinSteeringFile") self.marlinVersion = params.get("marlinVersion") self.lcsimPreSteeringFile = params.get("lcsimPreSteeringFile") self.lcsimPostSteeringFile = params.get("lcsimPostSteeringFile") self.fccSwPath = params.get("fccSwPath") self.fccSwSteeringFile = params.get("fccSwSteeringFile") self.fccAnalysisSteeringFile = params.get("fccAnalysisSteeringFile") ### other things needed to run tests self.log = gLogger.getSubLogger("JobCreater") from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid if self.clip.submitMode == "local": self.log.notice("") self.log.notice(" DIRAC RCSID:", drcsid) self.log.notice(" Job RCSID:", jrcsid) self.log.notice("Applications RCSID:", apprcsid) self.log.notice("") self.diracInstance = DiracILC(False, 'tests.rep') self.jobList = {}
def submit(self, diracinstance=None, mode="wms"): """ Submit call: when your job is defined, and all applications are set, you need to call this to add the job to DIRAC. :param diracinstance: DiracILC instance :type diracinstance: ~ILCDIRAC.Interfaces.API.DiracILC.DiracILC :param str mode: "wms" (default), "agent", or "local" .. note :: The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts """ if self.splittingOption: result = self._split() if 'OK' in result and not result['OK']: return result #Check the credentials. If no proxy or not user proxy, return an error if not self.proxyinfo['OK']: self.log.error( "Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError( "Not allowed to submit a job, you need a %s proxy." % self.usergroup, self.__class__.__name__) if 'group' in self.proxyinfo['Value']: group = self.proxyinfo['Value']['group'] if group not in self.usergroup: self.log.error( "Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError( "Not allowed to submit job, you need a %s proxy." % self.usergroup, self.__class__.__name__) else: self.log.error( "Could not determine group, you do not have the right proxy.") return self._reportError( "Could not determine group, you do not have the right proxy.") res = self._addToWorkflow() if not res['OK']: return res self.oktosubmit = True if not diracinstance: self.diracinstance = DiracILC() else: self.diracinstance = diracinstance return self.diracinstance.submit(self, mode)
def submit(self, diracinstance = None, mode = "wms"): """ Submit call: when your job is defined, and all applications are set, you need to call this to add the job to DIRAC. :param diracinstance: :any:`DiracILC <ILCDIRAC.Interfaces.API.DiracILC.DiracILC>` instance :param string mode: "wms" (default), "agent", or "local" .. note :: The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts """ #Check the credentials. If no proxy or not user proxy, return an error if not self.proxyinfo['OK']: self.log.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError("Not allowed to submit a job, you need a %s proxy." % self.usergroup, self.__class__.__name__) if self.proxyinfo['Value'].has_key('group'): group = self.proxyinfo['Value']['group'] if not group in self.usergroup: self.log.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError("Not allowed to submit job, you need a %s proxy." % self.usergroup, self.__class__.__name__) else: self.log.error("Could not determine group, you do not have the right proxy.") return self._reportError("Could not determine group, you do not have the right proxy.") res = self._addToWorkflow() if not res['OK']: return res self.oktosubmit = True if not diracinstance: self.diracinstance = DiracILC() else: self.diracinstance = diracinstance return self.diracinstance.submit(self, mode)
def __init__( self, clip, params ): self.clip = clip self.ildConfig = params.get( "ildConfig", None ) self.alwaysOverlay = params.get( "alwaysOverlay", False ) self.runOverlay = self.clip.testOverlay or self.alwaysOverlay self.mokkaVersion = params["mokkaVersion"] self.mokkaSteeringFile = params.get( "mokkaSteeringFile" ) self.detectorModel = params.get( "detectorModel" ) self.marlinVersion = params.get( "marlinVersion" ) self.marlinSteeringFile = params.get( "marlinSteeringFile" ) self.ddsimVersion = params.get( "ddsimVersion" ) self.ddsimDetectorModel = params.get( "ddsimDetectorModel") self.ddsimInputFile = params.get( "ddsimInputFile" ) self.marlinInputdata = params.get ( "marlinInputdata" ) self.gearFile = params.get( "gearFile" ) self.lcsimVersion = params.get( "lcsimVersion" ) self.steeringFileVersion = params.get( "steeringFileVersion", None ) self.rootVersion = params["rootVersion"] self.whizard2Version = params.get( "whizard2Version" ) self.whizard2SinFile = params.get( "whizard2SinFile" ) self.energy = params.get("energy") self.backgroundType = params.get("backgroundType") self.machine = params.get("machine") self.gearFile = params.get( "gearFile" ) self.marlinSteeringFile = params.get( "marlinSteeringFile" ) self.marlinVersion = params.get( "marlinVersion" ) self.lcsimPreSteeringFile = params.get( "lcsimPreSteeringFile" ) self.lcsimPostSteeringFile = params.get( "lcsimPostSteeringFile" ) self.fccSwPath = params.get( "fccSwPath" ) self.fccSwSteeringFile = params.get( "fccSwSteeringFile" ) self.fccAnalysisSteeringFile = params.get( "fccAnalysisSteeringFile" ) ### other things needed to run tests self.log = gLogger.getSubLogger("JobCreater") from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid if self.clip.submitMode == "local": self.log.notice("") self.log.notice(" DIRAC RCSID:", drcsid ) self.log.notice(" Job RCSID:", jrcsid ) self.log.notice("Applications RCSID:", apprcsid ) self.log.notice("") self.diracInstance = DiracILC(False, 'tests.rep') self.jobList = {}
def initialize(self): self.pollingTime = self.am_getOption('PollingTime', 86400) gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM) self.ppc = ProcessProdClient() self.dirac = DiracILC() self.diracadmin = DiracAdmin() self.am_setOption( 'shifterProxy', 'Admin' ) return S_OK()
def subDDSim(): # Decide parameters for a job outputSE = "KEK-SRM" outputSE = "KEK-DISK" isLocal = _clip.isLocal nbevts = 10 if _clip.numberOfEvents == 0 else _clip.numberOfEvents outputFile = "ddsim_example.slcio" if _clip.outputFile == "" else _clip.outputFile outputDir = _clip.outputDir inputFile = _clip.inputFile if inputFile == "": gLogger.error("Input file for ddsim does not given.") exit(-1) # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setJobGroup("myddsimjob") job.setName("myddsim") job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setILDConfig("v02-00-02") # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"]) # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk" ]) # a list of sites not to submit job # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units ) ddsim = DDSim() ddsim.setVersion("ILCSoft-02-00-02_gcc49") ddsim.setDetectorModel("ILD_l5_v02") ddsim.setInputFile(inputFile) ddsim.setNumberOfEvents(nbevts) extraCLIArguments = " --steeringFile ddsim_steer.py " extraCLIArguments += " --outputFile %s " % outputFile extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 " ddsim.setExtraCLIArguments(extraCLIArguments) # ddsim.setRandomSeed(1234565) # ddsim.setStartFrom(20) # Number of events to skip before starting ddsim job.append(ddsim) if outputDir != "": job.setOutputData([outputFile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
def _createLFNList(): """create the LFnList""" cliparams = _Params() cliparams.registerSwitches() Script.parseCommandLine( ignoreErrors = False ) repoLocation = cliparams.repo if not repoLocation: Script.showHelp() dexit(2) from ILCDIRAC.Interfaces.API.DiracILC import DiracILC dirac = DiracILC(True, repoLocation) dirac.monitorRepository(False) lfns = [] lfns = dirac.retrieveRepositoryOutputDataLFNs() LOG.notice("lfnlist=[") for lfn in lfns : LOG.notice('"LFN:%s",' % lfn) LOG.notice("]") dexit(0)
def _createLFNList(): """create the LFnList""" cliparams = _Params() cliparams.registerSwitches() Script.parseCommandLine(ignoreErrors=False) from DIRAC import gLogger repoLocation = cliparams.repo if not repoLocation: Script.showHelp() dexit(2) from ILCDIRAC.Interfaces.API.DiracILC import DiracILC dirac = DiracILC(True, repoLocation) dirac.monitorRepository(False) lfns = [] lfns = dirac.retrieveRepositoryOutputDataLFNs() gLogger.notice("lfnlist=[") for lfn in lfns: gLogger.notice('"LFN:%s",' % lfn) gLogger.notice("]") dexit(0)
def submit(self, diracinstance = None, mode = "wms"): """ Submit call: when your job is defined, and all applications are set, you need to call this to add the job to DIRAC. """ res = self._addToWorkflow() if not res['OK']: return res self.oktosubmit = True if not diracinstance: self.diracinstance = DiracILC() else: self.diracinstance = diracinstance return self.diracinstance.submit(self, mode)
def create_job(inputData, saveName, outputDir, dontPromptMe): slcioFile = saveName + '.slcio' rootFile = saveName + '.root' if check_file_existence(outputDir, slcioFile, dontPromptMe): remove_file(outputDir, slcioFile, dontPromptMe) if check_file_existence(outputDir, rootFile, dontPromptMe): remove_file(outputDir, rootFile, dontPromptMe) dIlc = DiracILC() job = UserJob() job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml']) if SAVE_SLCIO: job.setOutputData([slcioFile, rootFile], OutputPath=outputDir, OutputSE=STORAGE_SE) else: job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE) job.setJobGroup('myMarlinRun1') job.setName('MyMarlinJob1') # job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk']) job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz']) marl = Marlin() marl.setVersion('ILCSoft-2017-11-15_gcc62') marl.setInputFile(inputData) job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData))) # marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio']) marl.setSteeringFile('marlin/FullChain.xml') # marl.setSteeringFile('marlin/FullChainNewDetModel.xml') marl.setOutputFile(slcioFile) gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear' if not os.path.isfile(gearFile): print('Error: gear file', gearFile, 'does not exist! Abort submission.') return marl.setGearFile(gearFile) marl.setExtraCLIArguments( "--MyNtupleMaker.OutputFileName={rootOutfile}".format( rootOutfile=rootFile)) # marl.setNumberOfEvents(1000) job.append(marl) if dontPromptMe: job.dontPromptMe() job.submit(dIlc) return False
def testAndProbeSites(): """submits jobs to test sites""" clip = Params() clip.registerSwitches() Script.parseCommandLine() from DIRAC import gLogger, exit as dexit from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import CheckWNs from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueues res = getQueues(siteList=clip.site, ceList=clip.ce) if not res['OK']: gLogger.error("Failed getting the queues", res['Message']) dexit(1) sitedict = res['Value'] CEs = [] for ces in sitedict.values(): CEs.extend(ces.keys()) gLogger.notice("Found %s CEs to look at." % len(CEs)) d = DiracILC(True, "SiteProbe.rep") for CE in CEs: j = UserJob() j.setDestinationCE(CE) c = CheckWNs() res = j.append(c) if not res['OK']: gLogger.error(res['Message']) continue j.setOutputSandbox("*.log") j.setCPUTime(30000) j.dontPromptMe() res = j.submit(d) if not res['OK']: gLogger.error("Failed to submit job, aborting") dexit(1) dexit(0)
def subWhizard2(): from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin, Whizard2 # Decide parameters for a job outputSE = "KEK-SRM" isLocal = _clip.isLocal nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents outputFile = "E500.P2f_bB.GWhizard2.I100000.e0.p0.n001.slcio" if _clip.outputFile == "" else _clip.outputFile outputDir = _clip.outputDir # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setInputSandbox(["pythia6-parameters.sin", "P2f_qqbar.sin"]) job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setOutputData([outputFile]) job.setJobGroup("mywhiz2") job.setName("mywhizard2") whiz = Whizard2() whiz.setVersion("2.7.0") whiz.setNumberOfEvents(nbevts) whiz.setEvtType("P2f_bB") whiz.setProcessVariables("P2f_bB") # whiz.setRandomSeed(15) whiz.setSinFile("P2f_qqbar.sin") whiz.setOutputFile(outputFile) job.append(whiz) if outputDir != "": print " outputDir = " + outputDir print " outputSE = " + outputSE job.setOutputData([outputFile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
def subDDSim(clip1): # Decide parameters for a job outputSE = "KEK-SRM" isLocal = clip1.isLocal nbevts = 0 if clip1.numberOfEvents == 0 else clip1.numberOfEvents #print('inside subddsim(): nbevts ', nbevts) outputFile = "" if clip1.outputFile == "" else clip1.outputFile #print('inside subddsim outfile ', outputFile) outputDir = clip1.outputDir #print('inside subddsim outdir ', outputDir) inputFile = clip1.inputFile #print('inside subddsim inputFile ', inputFile) if inputFile == "": gLogger.error("Input file for ddsim is not given.") exit(-1) # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setJobGroup("myddsimjob") job.setName("myddsim") job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setILDConfig("v02-00-02") # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"]) # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk" ]) # a list of sites not to submit job # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units ) ddsim = DDSim() ddsim.setVersion("ILCSoft-02-00-02_gcc49") ddsim.setDetectorModel("ILD_l5_v05") ddsim.setInputFile(inputFile) ddsim.setNumberOfEvents(nbevts) extraCLIArguments = " --steeringFile ddsim_steer_July26.py" extraCLIArguments += " --outputFile %s " % outputFile extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 " ddsim.setExtraCLIArguments(extraCLIArguments) return ddsim
def setUp( self ): """set up the objects""" ops_mock = Mock() mocked_modules = { 'DIRAC.ConfigurationSystem.Client.Helpers.Operations' : ops_mock } self.module_patcher = patch.dict( sys.modules, mocked_modules ) self.module_patcher.start() from ILCDIRAC.Interfaces.API.DiracILC import DiracILC self.dilc = DiracILC() def setOptions(*args): if 'SingleReplicaSEs' in args[0]: return ['SE'] if 'Minimum' in args[0]: return 1 if args[0].endswith('PreferredSEs'): return ['Awesome-Tape-SE'] ops_mock = Mock() ops_mock.getValue = Mock() ops_mock.getValue.side_effect = setOptions self.dilc.ops = ops_mock
def __init__(self): self.dIlc = DiracILC(False) self.job = UserJob() self.job.setJobGroup("FCC") self.job.setName("FCC APP") self.job.setOutputSandbox(["*.log", '*.root']) #self.job.setDestination('LCG.DESY-HH.de') #EOS public location self.EOS_MGM_URL = 'root://eospublic.cern.ch' #EOS environment self.setEOS = 'export EOS_MGM_URL=' + self.EOS_MGM_URL self.myclient = client.FileSystem(self.EOS_MGM_URL + ':1094') #sandbox self.InputSandbox = [] self.folders_to_upload = [] self.filtered_extensions = [] self.excludes_or_includes = [] self.temp_cwd = os.path.join(os.getcwd(), 'fcc_temp_dirac')
ma.setSteeringFile("clic_ild_cdr_steering.xml") ma.setGearFile("clic_ild_cdr.gear") result = j.append(ma) if not result['OK']: gLogger.error(result["Message"]) dexit(1) j.setCPUTime(10000) j.setOutputSandbox("*.log") return j if __name__ == '__main__': lfns = getFiles() #get a list of files d = DiracILC(True, "paramjobtest.rep") #get your dirac instance job = getJob() #get a job, any can do #here is where the interesting stuff happen from DIRAC.Core.Utilities.List import breakListIntoChunks for flist in breakListIntoChunks(lfns, 200): #200 is the number of files per chunk, and the max number of jobs produced in one go #This is the magical line job.setParametricInputData(flist) #The rest of the sumission is the same res = job.submit(d) if not res["OK"]: gLogger.error("Failed to submit the job: ", res["Message"])
lfns = res['Value'] print("found %s files" % len(lfns)) ovi = OverlayInput() ovi.setEnergy(500.) ovi.setBXOverlay(300) ovi.setGGToHadInt(0.3) ovi.setNbSigEvtsPerJob(10) ovi.setBkgEvtType("gghad") ovi.setDetectorModel("CLIC_ILD_CDR") overlay = [True,False] for ov in overlay: d = DiracILC(True,"repo_overlay_%s.rep"%ov) for lfn in lfns: j = UserJob() steeringf = "clic_ild_cdr_steering.xml" if ov: steeringf = "clic_ild_cdr_steering_overlay.xml" res = j.append(ovi) if not res['OK']: print(res['Message']) continue ma = Marlin() ma.setVersion("v0111Prod") ma.setGearFile("clic_ild_cdr.gear") ma.setSteeringFile(steeringf) ma.setInputFile("LFN:"+lfn) ma.setNbEvts(10)
nameDir = nameJobGroup+'_files'+nameTag nameRepositoryFile = nameJobGroup+nameTag+'_repository.rep' ##################################################################### ##################################################################### #set enviroment import os import sys from DIRAC.Core.Base import Script #dirac enviroment Script.parseCommandLine() #dirac enviroment from ILCDIRAC.Interfaces.API.DiracILC import DiracILC #job receiver class dirac = DiracILC(True,nameRepositoryFile) #important to retrive jobs ##################################################################### ##################################################################### #job definition from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob ##################################################################### ##################################################################### #filecatalog from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient ##################################################################### ##################################################################### #ddsim from ILCDIRAC.Interfaces.API.NewInterface.Applications import DDSim
elif opt in ('v', 'version'): slicVersion = arg elif opt in ('w', 'workflow'): workflowName = arg elif opt in ('n', 'nocheck'): checkMeta = False if (detectorName == None) or (prodID == None): Script.showHelp() sys.exit(2) from ILCDIRAC.Interfaces.API.NewInterface.ProductionJob import ProductionJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC, LCSIM, SLICPandora, OverlayInput from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient dirac = DiracILC() meta = {} meta['ProdID'] = prodID if eventType: meta['EvtType'] = eventType if energy: meta['Energy'] = energy if dataType: meta['Datatype'] = dataType fc = FileCatalogClient() res = fc.getCompatibleMetadata(meta) if not res['OK']: print "Error looking up the catalog for metadata" exit(2)
def all_jobs(name): d = DiracILC(True, "repo.rep") ################################################ j = UserJob() j.setJobGroup("PM1") j.setName("Exec1") banned_sites = [ "OSG.BNL.us", "LCG.UKI-NORTHGRID-LIV-HEP.uk", "OSG.UCSDT2.us", "LCG.SCOTGRIDDURHAM.uk", "LCG.NIKHEF.nl", "LCG.UKI-SOUTHGRID-RALPP.uk", "LCG.GRIF.fr", "LCG.Manchester.uk", "LCG.UKI-LT2-IC-HEP.uk", "LCG.Weizmann.il" ] j.setBannedSites(banned_sites) caindir = name #print('Cain directory is ',caindir) indata = [ 'LFN:/ilc/user/a/amustahid/cain.exe', str(caindir), 'LFN:/ilc/user/a/amustahid/runcain.sh', 'LFN:/ilc/user/a/amustahid/convert_pairs_lcio.py', 'LFN:/ilc/user/a/amustahid/pyLCIO.tar.gz', '/home/belle2/mustahid/useful/my.sh', './splitInput.py', './subddsim.py', './ddsim_steer_July26.py', './ILD_l5_v05.xml', './my2.sh', './dbd_500GeV.nung_1.xml', 'LFN:/ilc/user/a/amustahid/myProcessors.tar.gz', './create_dir.py', './conf.py', './util.py', './testcain.sh', './beam_250.i' ] j.setInputSandbox(indata) ################################################ #app = GenericApplication() #app.setScript("create_dir.py") #app.setInputFile("testcain.sh") #logf = 'create_dir.log' #app.setLogFile(logf) #app.setDebug(debug=True) #create_dirname = 'create_dir' #app.setName(create_dirname) #res=j.append(app) #if not res['OK']: # print res['Message'] # exit(1) ################################################ appre = GenericApplication() name = name.split('/') #print(name) cain_name = name[-1] subdir = name[-2] dirname = name[-3] #print('Cain file name ', cain_name) appre.setScript("LFN:/ilc/user/a/amustahid/runcain.sh") #appre.setScript("testcain.sh") ifile = cain_name.split('.') ifile = ifile[0] + '.' + ifile[1] + '.' + ifile[2] #print('ifile ',ifile) appre.setArguments(ifile) #direc = 'LFN:/ilc/user/a/amustahid/' #appre.setInputFile(ifile+".i") #appre.setArguments("This is input arguments") logf = ifile + '_' + subdir + '.log' appre.setLogFile(logf) appre.setDebug(debug=True) name = 'CAIN' appre.setName(name) res = j.append(appre) if not res['OK']: print res['Message'] exit(1) ################################################ ################################################ #appost = GenericApplication() #appost.setScript("myanal.sh") #appost.setArguments("This is my analysis step") #res=j.append(appost) #if not res['OK']: # print res['Message'] # exit(1) ap = GenericApplication() ap.setScript('my.sh') logf = 'my.log' ap.setLogFile(logf) ap.setDebug(debug=True) name = 'my' ap.setName(name) res = j.append(ap) if not res['OK']: print res['Message'] exit(1) outfile = 'incoherent_pair.dat' appre.setOutputFile(outfile) ################################################ direc = 'incoherent_pair' inputFile = direc + '/' + 'inco_pair_split.slcio' # global variables to hold command line parameters # ###################################### base = '.' #outdir=base+'/'+dirname+'/slcio_test_2ndrun' outdir = base + '/' + dirname + '/Run_7' #print('outdir'+' '+str(outdir)) geant_name = ifile outputFile = geant_name + '_' + subdir + '.slcio' #_clip = _Params(False,1,inputFile,outputFile,outdir) nbevents = 100 clip = _Params(nbevents, inputFile, outputFile, outdir) ddsim = subDDSim(clip) ################################################ res = j.append(ddsim) if not res['OK']: print res['Message'] exit(1) j.setOutputData(outputFile, outdir, "KEK-SRM") j.setOutputSandbox(["*.log", "*.dat", "*.slcio"]) j.dontPromptMe() res = j.submit(d) #res = j.submit(d, mode='local') if res['OK']: print str(res["Value"]) #print "Dirac job, "+str(res["Value"])+", was submitted." else: print "Failed to submit Dirac job. return message was as follows." pprint.pprint(res)
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard, Mokka, Marlin, OverlayInput from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from DIRAC import exit as dexit dirac = DiracILC() # wh.setOutputFile("myfile.stdhep") j = UserJob() wh = Whizard(processlist=dirac.getProcessList()) wh.setEnergy(3000) wh.setEvtType("ee_h_mumu") wh.setNbEvts(1) wh.setEnergy(3000) params = {} params["USERB1"] = "F" wh.setParameterDict(params) wh.setModel("sm") res = j.append(wh) if not res["OK"]: print res["Message"] dexit(1)
Created on Feb 8, 2012 :author: Stephane Poss ''' __RCSID__ = "$Id$" #pylint: skip-file #pylint: disable=C0103 from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.NewInterface.ProductionJob import ProductionJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard, Mokka, Marlin, OverlayInput, StdhepCut, StdhepCutJava from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC, LCSIM, SLICPandora, SLCIOSplit, StdHepSplit from ILCDIRAC.Interfaces.API.DiracILC import DiracILC dirac = DiracILC() ###As it's a full chain, we start at generation ##so we need to define the process and the energy ## The rest will be set later. We could also set the process ##and the energy directly in the whizard def, but for clarity ## it's better to do it before, that way we know the very ##essential def getdicts(process): """ Create the proper structures to build all the prodcutions for the samples with ee_, ea_ aa_. """ plist = [] if process.count("ee_"): plist.append({'process':process,'pname1':'e1', 'pname2':'E1', "epa_b1":'F', "epa_b2":'F'})
#every single script has these four lines from DIRAC.Core.Base import Script Script.parseCommandLine() #create ilcdirac instance from ILCDIRAC.Interfaces.API.DiracILC import DiracILC dirac = DiracILC(True, "some_job_repository.rep") #job definition from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob #job = UserJob() from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM #jobParams = [('slicTest8_mu+_theta90.mac','diracTest8_mu+_theta90.slcio',50),('slicTest7_mu+_theta_5-175.mac','diracTest_mu+_theta_5-175.slcio',50),('slicTest3_e+.mac','diracTest3_e+.slcio',10),('slicTest2_pi+.mac','diractTest2_pi+.slcio',10)] #jobParams = [('slicTest10_mu+_100gev_theta70_testNewGeom.mac','diracTest10_mu+_100gev_theta70_testNewGeom.slcio',10),('slicTest10_mu+_100gev_theta90_testNewGeom.mac','diracTest10_mu+_100gev_theta90_testNewGeom.slcio',10)] jobParams = [ ('/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/slicmacros/slicTest8_mu+_100gev_theta60.mac', 'diracTest_100gev_theta60_vtx_matbudghalf_nonsensitivelayer.slcio', 100), ('/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/slicmacros/slicTest8_mu+_10gev_theta60.mac', 'diracTest_10gev_theta60_vtx_matbudghalf_nonsensitivelayer.slcio', 100) ] #slicMacros = ['slicTest8_mu+_theta90.mac','slicTest7_mu+_theta_5-175.mac','slicTest3_e+.mac','slicTest2_pi+.mac'] #fileOutputs = ['diracTest2Loop1.slcio','diracTest2Loop2.slcio','diracTest2Loop3.slcio','diractTest2Loop4.slcio'] #slicNumEvents = [100,100,10,10] for macro, output, nEvts in jobParams: job = UserJob() job.setName("ssetru_dirac_test1") job.setJobGroup("tests")
def main(): # Take the input arguments from the argument parser, and check they exist... args = parse_args() if not args: print 'Invalid Arguments' sys.exit(1) #### Software Versions #### softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended) # softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working # softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working # Check the --runs and --split arguments to make sure they are compatible, if not exit... if not check_events_arguments(args.events, args.split): sys.exit(1) # Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning... lfn_check, lfn = check_input_LFN(args.stdhepInput) if not lfn_check: sys.exit(1) # Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included... dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector)) # Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging, # into the input sandbox inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag) # Prepares values for the job loop... if args.split < 0: nInputEvents = int(args.events) nOutputEvents = int(args.events) if args.split > 0: nInputEvents = int(args.events) nOutputEvents = int(args.split) # Loop that runs through the required number of jobs to be executed... for startEvent in range(0, nInputEvents, nOutputEvents): ################## Job Initialise ######################################## job = UserJob() job.setName(path.basename(args.stdhepInput)) job.setJobGroup('JobGroup') job.setInputSandbox(inputSandbox) fileNumber = startEvent / nOutputEvents print "Job ", fileNumber outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions) slicOutput = outputFiles['slicOutput'] prePandoraOutput = outputFiles['prePandoraOutput'] pandoraOutput = outputFiles['pandoraOutput'] vertexingOutput = outputFiles['vertexingOutput'] lcsimRecOutput = outputFiles['lcsimRecOutput'] lcsimDstOutput = outputFiles['lcsimDstOutput'] flavortagOutput = outputFiles['flavortagOutput'] diracOutput = outputFiles['diracOutput'] ################## SLIC ################################################## slic = SLIC() slic.setVersion(softVersions[0]) slic.setSteeringFile(args.macFile) # slic.setInputFile(lfn) slic.setOutputFile(slicOutput) slic.setDetectorModel(args.detector) slic.setNumberOfEvents(nOutputEvents) slic.setStartFrom(startEvent) #print slic.listAttributes() result = job.append(slic) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (digitization and tracking) ##################### lcsim = LCSIM() lcsim.setVersion(softVersions[1]) lcsim.setSteeringFile( 'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml' ) # Another version is included in /steeringFiles lcsim.getInputFromApp(slic) lcsim.setTrackingStrategy( 'steeringFiles/sidloi3_trackingStrategies_default.xml') # lcsim.setAliasProperties('alias.properties') lcsim.setDetectorModel('geometryFiles/sidloi3.zip') lcsim.setOutputFile(prePandoraOutput) lcsim.setNumberOfEvents(nOutputEvents) #print lcsim.listAttributes() result = job.append(lcsim) if not result['OK']: print result['Message'] sys.exit(2) ################## slicPandora ########################################### slicPandora = SLICPandora() slicPandora.setVersion(softVersions[2]) slicPandora.setDetectorModel(args.detector) slicPandora.getInputFromApp(lcsim) slicPandora.setOutputFile(pandoraOutput) slicPandora.setPandoraSettings('pandoraSettings.xml') slicPandora.setNumberOfEvents(nOutputEvents) #print slicPandora.listAttributes() result = job.append(slicPandora) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus Vertexing ############################ vertexing = Marlin() vertexing.setVersion(softVersions[3]) vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml') vertexing.setGearFile('steeringFiles/sidloi3.gear') vertexing.getInputFromApp(slicPandora) vertexing.setOutputFile(vertexingOutput) vertexing.setNumberOfEvents(nOutputEvents) #print vertexing.listAttributes() result = job.append(vertexing) if not result['OK']: print result['Message'] sys.exit(2) ################## lcsim (DST production) ################################ lcsimDst = LCSIM() lcsimDst.setVersion(softVersions[1]) lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml') lcsimDst.getInputFromApp(vertexing) lcsimDst.setNumberOfEvents(nOutputEvents) # lcsimDst.setAliasProperties('alias.properties') lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip') lcsimDst.setOutputRecFile(lcsimRecOutput) lcsimDst.setOutputDstFile(lcsimDstOutput) #print lcsimDst.listAttributes() result = job.append(lcsimDst) if not result['OK']: print result['Message'] sys.exit(2) ################## Marlin, LCFIPlus flavortag ############################ if args.flavortag: flavortag = Marlin() flavortag.setVersion(softVersions[3]) flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml') flavortag.setGearFile('steeringFiles/sidloi3.gear') flavortag.setInputFile(lcsimDstOutput) flavortag.setOutputFile(flavortagOutput) flavortag.setNumberOfEvents(nOutputEvents) #print flavortag.listAttributes() result = job.append(flavortag) if not result['OK']: print result['Message'] sys.exit(2) ################## Job Finalise ########################################## # List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on, # This list is likely to change. job.setBannedSites([ 'LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp', 'OSG.PNNL.us', ]) job.setCPUTime(50000) job.setPlatform('x86_64-slc5-gcc43-opt') # Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/ # directory on the grid. if args.flavortag: job.setOutputData(flavortagOutput, diracOutput, args.SE) else: job.setOutputData(lcsimDstOutput, diracOutput, args.SE) job.setOutputSandbox(outputSandbox) job.setInputData(lfn) if args.dontPromptMe: job.dontPromptMe() # Submits Job!!! job.submit() return 0
class SoftwareManagementAgent( AgentModule ): """ Agent to run software management things """ def initialize(self): self.pollingTime = self.am_getOption('PollingTime', 86400) gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM) self.ppc = ProcessProdClient() self.dirac = DiracILC() self.diracadmin = DiracAdmin() self.am_setOption( 'shifterProxy', 'Admin' ) return S_OK() ############################################################################## def execute(self): """ First we update the site list and banned site """ res = getProxyInfo(False, False) if not res['OK']: self.log.error("submitTasks: Failed to determine credentials for submission", res['Message']) return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] self.log.info("submitTasks: Jobs will be submitted with the credentials %s:%s" % (owner, ownerGroup)) sites = self.diracadmin.getSiteMask()['Value'] for site in sites: res = self.ppc.changeSiteStatus( {'SiteName' : site, 'Status' : 'OK'} ) if not res['OK']: self.log.error('Cannot add or update site %s' % site) banned_sites = self.diracadmin.getBannedSites()['Value'] for banned_site in banned_sites: self.ppc.changeSiteStatus( {'SiteName' : banned_site, 'Status' : 'Banned'} ) if not res['OK']: self.log.error('Cannot mark as banned site %s' % banned_site) ##Then we need to get new installation tasks res = self.ppc.getInstallSoftwareTask() if not res['OK']: self.log.error('Failed to obtain task') task_dict = res['Value'] for softdict in task_dict.values(): self.log.info('Will install %s %s at %s' % (softdict['AppName'], softdict['AppVersion'], softdict['Sites'])) for site in softdict['Sites']: j = UserJob() j.setPlatform(softdict['Platform']) j.dontPromptMe() j.setDestination(site) j.setJobGroup("Installation") j.setName('install_%s' % site) j._addSoftware(softdict['AppName'], softdict['AppVersion']) #Add the application here somehow. res = j.append(SoftwareInstall()) if not res['OK']: self.log.error(res['Message']) continue res = j.submit(self.dirac) #res = self.dirac.submit(j) if not res['OK']: self.log.error('Could not create the job') continue jobdict = {} jobdict['AppName'] = softdict['AppName'] jobdict['AppVersion'] = softdict['AppVersion'] jobdict['Platform'] = softdict['Platform'] jobdict['JobID'] = res['Value'] jobdict['Status'] = 'Waiting' jobdict['Site'] = site res = self.ppc.addOrUpdateJob(jobdict) if not res['OK']: self.log.error('Could not add job %s: %s' % (jobdict['JobID'], res['Message'])) ##Monitor jobs jobs = {} res = self.ppc.getJobs() if not res['OK']: self.log.error('Could not retrieve jobs') else: jobs = res['Value'] for job in jobs: res = self.dirac.status(job['JobID']) if res['OK']: jobstatuses = res['Value'] job['Status'] = jobstatuses['JobID']['Status'] res = self.ppc.addOrUpdateJob(job) if not res['OK']: self.log.error("Failed to updated job %s: %s" % (job['JobID'], res['Message'])) else: self.log.error("Failed to update job %s status" % job['JobID']) return S_OK()
base = open(steeringTemplateFile,'r') steeringTemplateContent = base.read() base.close() for eventSelection in eventsToSimulate: eventType = eventSelection['EventType'] detectorModel = eventSelection['DetectorModel'] reconstructionVariant = eventSelection['ReconstructionVariant'] energy = eventSelection['Energy'] analysisTag = eventSelection['AnalysisTag'] # Make local gear file os.system('cp ' + gearFile + ' .') gearFileLocal = os.path.basename(gearFile) diracInstance = DiracILC(withRepo=False) slcioFormat = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_(.*?)_(.*?)_(.*?)_DST.slcio' slcioFilesToProcess = getDstSlcioFiles(jobDescription,detectorModel,reconstructionVariant,energy,eventType) if not slcioFilesToProcess: print 'No slcio files found. Exiting job submission.' sys.exit() for slcioFile in slcioFilesToProcess: print 'Checking ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. Slcio file ' + slcioFile + '.' slcioFileNoPath = os.path.basename(slcioFile) inputSandbox = ['LFN:/ilc/user/s/sgreen/SelectionProcessorTarBall/MarlinSelectionProcessor.tar.gz']
lfns = res['Value'] print "found %s files" % len(lfns) ovi = OverlayInput() ovi.setEnergy(500.) ovi.setBXOverlay(300) ovi.setGGToHadInt(0.3) ovi.setNbSigEvtsPerJob(10) ovi.setBkgEvtType("gghad") ovi.setDetectorModel("CLIC_ILD_CDR") overlay = [True, False] for ov in overlay: d = DiracILC(True, "repo_overlay_%s.rep" % ov) for lfn in lfns: j = UserJob() steeringf = "clic_ild_cdr_steering.xml" if ov: steeringf = "clic_ild_cdr_steering_overlay.xml" res = j.append(ovi) if not res['OK']: print res['Message'] continue ma = Marlin() ma.setVersion("v0111Prod") ma.setGearFile("clic_ild_cdr.gear") ma.setSteeringFile(steeringf) ma.setInputFile("LFN:" + lfn) ma.setNbEvts(10)
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.UserJob import * from ILCDIRAC.Interfaces.API.NewInterface.Applications import * dirac = DiracILC(True, "CLICdet_repo.cfg") job = UserJob() job.setName("example") job.setInputData([ "/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep" ]) job.setCLICConfig("ILCSoft-2017-12-21") job.setOutputSandbox(["*.root", "*.slcio", "*.log"]) ddsim = DDSim() ddsim.setSteeringFile("CLICPerformance/examples/clic_steer.py") ddsim.setVersion("ILCSoft-2017-12-21_gcc62") ddsim.setDetectorModel("CLIC_o3_v14") ddsim.setInputFile( "/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep" ) ddsim.setOutputFile("ddsim_output.slcio") ddsim.setStartFrom(0) ddsim.setNumberOfEvents(10) res1 = job.append(ddsim) if not res1['OK']: print res1['Message'] sys.exit(2) # -------------------- Comment if gg_had overlay not wanted -----------------------------------------------------#
class JobCreater(object): """contains all the versions and parameters to create all theses tests""" # pylint: disable=too-many-instance-attributes # Test parameters, necessary due to amount of tests in this class. def __init__(self, clip, params): self.clip = clip self.ildConfig = params.get("ildConfig", None) self.alwaysOverlay = params.get("alwaysOverlay", False) self.runOverlay = self.clip.testOverlay or self.alwaysOverlay self.mokkaVersion = params["mokkaVersion"] self.mokkaSteeringFile = params.get("mokkaSteeringFile") self.detectorModel = params.get("detectorModel") self.marlinVersion = params.get("marlinVersion") self.marlinSteeringFile = params.get("marlinSteeringFile") self.ddsimVersion = params.get("ddsimVersion") self.ddsimDetectorModel = params.get("ddsimDetectorModel") self.ddsimInputFile = params.get("ddsimInputFile") self.marlinInputdata = params.get("marlinInputdata") self.gearFile = params.get("gearFile") self.lcsimVersion = params.get("lcsimVersion") self.steeringFileVersion = params.get("steeringFileVersion", None) self.rootVersion = params["rootVersion"] self.whizard2Version = params.get("whizard2Version") self.whizard2SinFile = params.get("whizard2SinFile") self.energy = params.get("energy") self.backgroundType = params.get("backgroundType") self.machine = params.get("machine") self.gearFile = params.get("gearFile") self.marlinSteeringFile = params.get("marlinSteeringFile") self.marlinVersion = params.get("marlinVersion") self.lcsimPreSteeringFile = params.get("lcsimPreSteeringFile") self.lcsimPostSteeringFile = params.get("lcsimPostSteeringFile") self.fccSwPath = params.get("fccSwPath") self.fccSwSteeringFile = params.get("fccSwSteeringFile") self.fccAnalysisSteeringFile = params.get("fccAnalysisSteeringFile") ### other things needed to run tests self.log = gLogger.getSubLogger("JobCreater") from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid if self.clip.submitMode == "local": self.log.notice("") self.log.notice(" DIRAC RCSID:", drcsid) self.log.notice(" Job RCSID:", jrcsid) self.log.notice("Applications RCSID:", apprcsid) self.log.notice("") self.diracInstance = DiracILC(False, 'tests.rep') self.jobList = {} def createDDSimTest(self, inputfile=None, detectorModel=None): """Create a job running ddsim""" if inputfile is None: inputfile = self.ddsimInputFile sandbox = [inputfile] if detectorModel is None: detectorModel = self.ddsimDetectorModel else: sandbox.append(detectorModel) jobdd = self.getJob() from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim ddsim = DDSim() ddsim.setVersion(self.ddsimVersion) ddsim.setDetectorModel(detectorModel) ddsim.setNumberOfEvents(1) ddsim.setInputFile(inputfile) jobdd.setInputSandbox(sandbox) res = jobdd.append(ddsim) if not res['OK']: self.log.error("Failed adding DDSim:", res['Message']) return S_ERROR("Failed adding DDSim to Job") return S_OK(jobdd) def createWhizard2Test(self): """Create a job running Whizard2""" jobdd = self.getJob() from ILCDIRAC.Interfaces.API.NewInterface.Applications.Whizard2 import Whizard2 whiz = Whizard2() whiz.setVersion(self.whizard2Version) whiz.setNumberOfEvents(1) whiz.setSinFile(self.whizard2SinFile) whiz.setOutputFile("test.stdhep") res = jobdd.append(whiz) if not res['OK']: self.log.error("Failed adding Whizard2:", res['Message']) return S_ERROR("Failed adding Whizard2 to Job") return S_OK(jobdd) def createMokkaTest(self): """create a job running mokka, and maybe whizard before""" self.log.notice("Creating jobs for Mokka") #(Whizard + )Mokka jobmo = self.getJob() if self.clip.testChain: whmo = self.getWhizard(2) res = jobmo.append(whmo) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR("Failed adding Whizard") elif self.clip.testInputData: jobmo.setInputData( "/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep" ) else: self.log.error("Mokka does not know where to get its input from") return S_ERROR("Mokka does not know where to gets its input from") mo = self.getMokka() if self.clip.testChain: mo.getInputFromApp(whmo) else: mo.setNumberOfEvents(1) res = jobmo.append(mo) if not res['OK']: self.log.error("Failed adding Mokka:", res['Message']) return S_ERROR("Failed adding Mokka to Job") jobmo.setOutputData("testsim.slcio", OutputSE="CERN-DIP-4") self.jobList['Mokka1'] = jobmo return S_OK(jobmo) def createRootScriptTest(self): """create a job running root""" self.log.notice("Creating jobs for Root") jobRoot = self.getJob() jobRoot.setInputSandbox(["root.sh", "input.root", "input2.root"]) root = self.getRoot() res = jobRoot.append(root) if not res['OK']: self.log.error("Failed adding Root:", res['Message']) return S_ERROR("Failed adding Root to Job") self.jobList['Root'] = jobRoot return S_OK(jobRoot) def createRootHaddTest(self): """create a job running root""" self.log.notice("Creating jobs for Root") jobRoot = self.getJob() jobRoot.setInputSandbox(["input.root", "input2.root"]) root = self.getRoot() root.setScript("hadd") res = jobRoot.append(root) if not res['OK']: self.log.error("Failed adding Root:", res['Message']) return S_ERROR("Failed adding Root to Job") self.jobList['Root'] = jobRoot return S_OK(jobRoot) def createRootMacroTest(self): """create a job running root""" self.log.notice("Creating jobs for Root") jobRoot = self.getJob() jobRoot.setInputSandbox(["func.C", "input.root", "input2.root"]) root = self.getRootMacro() root.setScript("func.C") res = jobRoot.append(root) if not res['OK']: self.log.error("Failed adding Root:", res['Message']) return S_ERROR("Failed adding Root to Job") self.jobList['Root'] = jobRoot return S_OK(jobRoot) def getOverlay(self, nbevts): """ Create an overlay step """ pathToFiles = None from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput overlay = OverlayInput() if self.energy == 350: if self.detectorModel == "ILD_o1_v05": pathToFiles = "/ilc/user/s/sailer/testFiles/overlay/ild_350/" if pathToFiles: overlay.setPathToFiles(pathToFiles) else: self.log.warn( "better define pathToFiles for this overlay: %s, %s, %s" % (self.energy, self.machine, self.backgroundType)) overlay.setMachine(self.machine) overlay.setEnergy(self.energy) overlay.setDetectorModel(self.detectorModel) overlay.setBkgEvtType(self.backgroundType) overlay.setBXOverlay(60) overlay.setGGToHadInt(0.3) overlay.setNumberOfSignalEventsPerJob(nbevts) return overlay def getMokka(self): """ Define a mokka app """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka mokka = Mokka() mokka.setVersion(self.mokkaVersion) mokka.setSteeringFile(self.mokkaSteeringFile) mokka.setOutputFile("testsim.slcio") mokka.setDetectorModel(self.detectorModel) if self.steeringFileVersion: mokka.setSteeringFileVersion(self.steeringFileVersion) return mokka def getRoot(self): """ Define a root app """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootScript root = RootScript() root.setScript("root.sh") root.setArguments("output.root input.root input2.root") root.setVersion(self.rootVersion) root.setOutputFile("output.root") return root def getRootMacro(self): """ Define a root app """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootMacro root = RootMacro() root.setMacro("func.C") root.setArguments(r"\"input.root\"") root.setVersion(self.rootVersion) return root @staticmethod def getSLIC(): """ Get a SLIC instance """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC slic = SLIC() slic.setVersion('v2r9p8') slic.setSteeringFile('defaultClicCrossingAngle.mac') slic.setDetectorModel('clic_sid_cdr') slic.setOutputFile('testsim.slcio') return slic @staticmethod def getSLICPandora(): """ Get some SLICPAndora app """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLICPandora slicp = SLICPandora() slicp.setVersion('CLIC_CDR') slicp.setDetectorModel('clic_sid_cdr') slicp.setPandoraSettings("PandoraSettingsSlic.xml") slicp.setOutputFile('testpandora.slcio') return slicp def getMarlin(self): """ Define a marlin step """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin marlin = Marlin() # marlin.setVersion("v0111Prod") marlin.setVersion(self.marlinVersion) marlin.setSteeringFile(self.marlinSteeringFile) marlin.setGearFile(self.gearFile) marlin.setOutputDstFile("testmarlinDST.slcio") marlin.setOutputRecFile("testmarlinREC.slcio") marlin.setNumberOfEvents(1) return marlin def getDD(self): """ Create a DDSim object """ from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim ddsim = DDSim() ddsim.setVersion(self.ddsimVersion) ddsim.setDetectorModel(self.ddsimDetectorModel) ddsim.setInputFile(self.ddsimInputFile) ddsim.setNumberOfEvents(2) return ddsim def getLCSIM(self, prepandora=True): """ Get some LCSIM """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM lcsim = LCSIM() lcsim.setVersion('CLIC_CDR') lcsim.setDetectorModel('clic_sid_cdr.zip') if prepandora: lcsim.setSteeringFile(self.lcsimPreSteeringFile) lcsim.setOutputFile("testlcsim.slcio") else: lcsim.setSteeringFile(self.lcsimPostSteeringFile) #lcsim.setOutputFile("testlcsimfinal.slcio") lcsim.setOutputDstFile("testlcsimDST.slcio") lcsim.setOutputRecFile("testlcsimREC.slcio") lcsim.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml") return lcsim def getFccSw(self): """ Define a fccsw step """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccSw fccsw = FccSw() fccsw.fccSwPath = self.fccSwPath fccsw.setSteeringFile(self.fccSwSteeringFile) return fccsw def getFccAnalysis(self): """ Define a fccanalysis step """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccAnalysis fccanalysis = FccAnalysis() fccanalysis.setSteeringFile(self.fccAnalysisSteeringFile) return fccanalysis @staticmethod def getStdhepcut(generated): """ Get some cuts in """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdhepCutJava stdhepcut = StdhepCutJava() stdhepcut.setVersion('1.0') stdhepcut.setSelectionEfficiency(1.) #stdhepcut.setSteeringFile("cuts_testing_1400.txt") stdhepcut.setInlineCuts("leptonInvMass_R 13 100 200") stdhepcut.setSteeringFileVersion("V18") stdhepcut.setMaxNbEvts(1) stdhepcut.setNumberOfEvents(generated) return stdhepcut @staticmethod def getStdhepSplit(): """ Get some stdhep split """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit stdhepsplit = StdHepSplit() stdhepsplit.setVersion("V2") stdhepsplit.setNumberOfEventsPerFile(5) stdhepsplit.setOutputFile("teststdhepsplit.stdhep") stdhepsplit.setMaxRead(10) return stdhepsplit @staticmethod def getLCIOSplit(events_per_file): """ Get a LCIO split """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOSplit lciosplit = SLCIOSplit() lciosplit.setNumberOfEventsPerFile(events_per_file) lciosplit.setOutputFile("testlciosplit.slcio") return lciosplit @staticmethod def getLCIOConcat(): """ Get a LCIO Concat """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOConcatenate lcioconcat = SLCIOConcatenate() lcioconcat.setOutputFile("testlcioconcat.slcio") return lcioconcat def getJob(self): """ Define a generic job, it should be always the same """ from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob myjob = UserJob() myjob.setName("Testing") myjob.setJobGroup("Tests") myjob.setCPUTime(30000) myjob.dontPromptMe() myjob.setLogLevel("VERBOSE") myjob.setPlatform("x86_64-slc5-gcc43-opt") myjob.setOutputSandbox(["*.log", "*.xml", "*.sh"]) myjob._addParameter(myjob.workflow, 'TestFailover', 'String', True, 'Test failoverRequest') myjob._addParameter(myjob.workflow, 'Platform', 'JDL', "x86_64-slc5-gcc43-opt", 'OS Platform') if self.ildConfig: myjob.setILDConfig(self.ildConfig) return myjob def getWhizardModel(self, nbevts, energy, model): """ Create a default whizard """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard proddict = "e2e2_o" if model != "sm": proddict = "se2se2_r" whiz = Whizard(processlist=self.diracInstance.getProcessList()) whiz.setModel(model) pdict = {} pdict['process_input'] = {} pdict['process_input']['process_id'] = proddict pdict['process_input']['sqrts'] = energy if model != 'sm': pdict['process_input']['input_file'] = "LesHouches.msugra_1" pdict['process_input']['input_slha_format'] = 'T' pdict['process_input']['beam_recoil'] = 'T' pdict['integration_input'] = {} pdict['integration_input']['calls'] = '1 50000 10 5000 1 15000' pdict['simulation_input'] = {} pdict['simulation_input']['normalize_weight'] = 'F' pdict['simulation_input']['n_events'] = nbevts pdict['simulation_input']['keep_initials'] = 'T' pdict['simulation_input']['events_per_file'] = 500000 if model != 'sm': pdict['simulation_input'][ 'pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1;IMSS(1)=11; IMSS(21)=71; IMSS(22)=71" else: pdict['simulation_input'][ 'pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ; MSTJ(28)=2 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1" pdict['parameter_input'] = {} # pdict['parameter_input']['mmu']=mmu # pdict['parameter_input']['mtau']=mtau # pdict['parameter_input']['mb']=mb # pdict['parameter_input']['mc']=mc pdict['parameter_input']['mH'] = 125 pdict['beam_input_1'] = {} pdict['beam_input_1']['particle_name'] = "e1" pdict['beam_input_1']['polarization'] = "0.0 0.0" pdict['beam_input_1']['USER_spectrum_on'] = 'T' pdict['beam_input_1'][ 'USER_spectrum_mode'] = 19 if energy == 1400 else 11 pdict['beam_input_1']['ISR_on'] = 'T' pdict['beam_input_1']['EPA_on'] = "F" pdict['beam_input_2'] = {} pdict['beam_input_2']['particle_name'] = "E1" pdict['beam_input_2']['polarization'] = "0.0 0.0" pdict['beam_input_2']['USER_spectrum_on'] = 'T' pdict['beam_input_2']['ISR_on'] = 'T' pdict['beam_input_2'][ 'USER_spectrum_mode'] = 19 if energy == 1400 else 11 pdict['beam_input_2']['EPA_on'] = 'F' whiz.setFullParameterDict(pdict) whiz.setOutputFile("testgen.stdhep") return whiz def getWhizard(self, nbevts): """ Get some defualt SM whizard """ return self.getWhizardModel(nbevts, 1400, "sm") def getWhizardSUSY(self, nbevts): """ Get a susy whizard """ return self.getWhizardModel(nbevts, 3000, "slsqhh") def createWhizardTest(self): """create a test for whizard""" self.log.notice("Creating jobs for Whizard") ##### WhizardJob jobw = self.getJob() wh = self.getWhizard(2) res = jobw.append(wh) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() self.jobList['Whizard1'] = jobw ##### WhizardJob jobwsusy = self.getJob() whsusy = self.getWhizardSUSY(2) res = jobwsusy.append(whsusy) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() self.jobList['WhizSusy'] = jobwsusy return S_OK((jobw, jobwsusy)) def createSlicTest(self): """create tests for slic""" self.log.notice("Creating jobs for SLIC") #run (Whizard +)SLIC jobslic = self.getJob() if self.clip.testChain: whslic = self.getWhizard(2) res = jobslic.append(whslic) if not res["OK"]: self.log.error("Failed adding Whizard:", res['Value']) return S_ERROR() elif self.clip.testInputData: jobslic.setInputData( "/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep" ) else: self.log.error("SLIC does not know where to get its input from") return S_ERROR() myslic = self.getSLIC() if self.clip.testChain: myslic.getInputFromApp(whslic) else: myslic.setNumberOfEvents(2) res = jobslic.append(myslic) if not res['OK']: self.log.error("Failed adding slic: ", res["Message"]) return S_ERROR() self.jobList['Slic1'] = jobslic return S_OK(jobslic) def createMarlinTest(self, setInputData=False): """create tests for marlin""" self.log.notice("Creating test for Marlin") #((Whizard + Mokka +)Overlay+) Marlin jobma = self.getJob() if self.clip.testChain: moma = self.getMokka() if not self.clip.testInputData: whma = self.getWhizard(2) res = jobma.append(whma) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() moma.getInputFromApp(whma) else: jobma.setInputData( "/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep" ) moma.setNumberOfEvents(1) res = jobma.append(moma) if not res['OK']: self.log.error("Failed adding Mokka:", res['Message']) return S_ERROR() elif self.clip.testInputData: jobma.setInputData(self.marlinInputdata) else: self.log.error("Marlin does not know where to get its input from") return S_ERROR() if self.runOverlay: ov = self.getOverlay(2) res = jobma.append(ov) if not res["OK"]: self.log.error("Failed adding Overlay:", res['Message']) return S_ERROR ma = self.getMarlin() if self.clip.testChain: ma.getInputFromApp(moma) else: ma.setNumberOfEvents(2) res = jobma.append(ma) if not res['OK']: self.log.error("Failed adding Marlin:", res['Message']) return S_ERROR() self.jobList['Marlin1'] = jobma return S_OK(jobma) def createLCSimTest(self): """create tests for LCSIM""" self.log.notice("Creating test for LCSIM") #run ((whiz+SLIC+)+Overlay+)LCSIM joblcsim = self.getJob() if self.clip.testChain: mysliclcsim = self.getSLIC() if not self.clip.testInputData: whlcsim = self.getWhizard(2) res = joblcsim.append(whlcsim) if not res["OK"]: self.log.error("Failed adding Whizard:", res['Value']) return S_ERROR() mysliclcsim.getInputFromApp(whlcsim) else: joblcsim.setInputData( "/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep" ) mysliclcsim.setNumberOfEvents(2) res = joblcsim.append(mysliclcsim) if not res['OK']: self.log.error("Failed adding slic: ", res["Message"]) return S_ERROR() elif self.clip.testInputData: #joblcsim.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio") joblcsim.setInputData( "/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio") else: self.log.error("LCSIM does not know where to get its input from") return S_ERROR() if self.runOverlay: ovlcsim = self.getOverlay(2) res = joblcsim.append(ovlcsim) if not res["OK"]: self.log.error("Failed adding Overlay:", res['Message']) return S_ERROR() mylcsim = self.getLCSIM(True) if self.clip.testChain: mylcsim.getInputFromApp(mysliclcsim) else: mylcsim.setNumberOfEvents(2) res = joblcsim.append(mylcsim) if not res['OK']: self.log.error("Failed adding LCSIM: ", res["Message"]) return S_ERROR() self.jobList['lcsim1'] = joblcsim return S_OK(joblcsim) def createSlicPandoraTest(self): """create tests for slicPandora""" self.log.notice("Creating tests for SLICPandora") #run ((whiz+SLIC) + (Overlay +) LCSIM +) SLICPandora + LCSIM joblcsimov = self.getJob() if not self.clip.testChain: self.log.error( "SLICPandora does not know where to get its input from") return S_ERROR() mylcsimov = self.getLCSIM(True) if not self.clip.testInputData: whlcsimov = self.getWhizard(2) res = joblcsimov.append(whlcsimov) if not res["OK"]: self.log.error("Failed adding Whizard:", res['Value']) return S_ERROR() mysliclcsimov = self.getSLIC() mysliclcsimov.getInputFromApp(whlcsimov) res = joblcsimov.append(mysliclcsimov) if not res['OK']: self.log.error("Failed adding slic: ", res["Message"]) return S_ERROR() mylcsimov.getInputFromApp(mysliclcsimov) else: #joblcsimov.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio") joblcsimov.setInputData( "/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio") mylcsimov.setNumberOfEvents(2) if self.runOverlay: ovslicp = self.getOverlay(2) res = joblcsimov.append(ovslicp) if not res["OK"]: self.log.error("Failed adding Overlay:", res['Message']) return S_ERROR() res = joblcsimov.append(mylcsimov) if not res['OK']: self.log.error("Failed adding LCSIM: ", res["Message"]) return S_ERROR() myslicpov = self.getSLICPandora() myslicpov.getInputFromApp(mylcsimov) res = joblcsimov.append(myslicpov) if not res['OK']: self.log.error("Failed adding SLICPandora: ", res["Message"]) return S_ERROR() mylcsimovp = self.getLCSIM(False) mylcsimovp.getInputFromApp(myslicpov) res = joblcsimov.append(mylcsimovp) if not res['OK']: self.log.error("Failed adding LCSIM: ", res["Message"]) return S_ERROR() self.jobList['lcsimov1'] = joblcsimov return S_OK(joblcsimov) def createUtilityTests(self): """Create tests for utility applications""" self.log.notice("Creating tests for utility applications") jobwsplit = self.getJob() whsplit = self.getWhizard(10) res = jobwsplit.append(whsplit) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() mystdsplit = JobCreater.getStdhepSplit() mystdsplit.getInputFromApp(whsplit) res = jobwsplit.append(mystdsplit) if not res['OK']: self.log.error("Failed adding StdHepSplit:", res['Message']) return S_ERROR() self.jobList['whizSplit'] = jobwsplit ##### WhizardJob + split jobwcut = self.getJob() whcut = self.getWhizard(100) res = jobwcut.append(whcut) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() mystdcut = self.getStdhepcut(100) mystdcut.getInputFromApp(whcut) res = jobwcut.append(mystdcut) if not res['OK']: self.log.error("Failed adding StdHepCut:", res['Message']) return S_ERROR() self.jobList['whizCut'] = jobwcut #LCIO split joblciosplit = self.getJob() # joblciosplit.setInputData("/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio") joblciosplit.setInputData( "/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio" ) mylciosplit = self.getLCIOSplit(100) res = joblciosplit.append(mylciosplit) if not res['OK']: self.log.error("Failed adding SLCIOSplit:", res['Message']) return S_ERROR() self.jobList['lcioSplit'] = joblciosplit #LCIO concat jobconcat = self.getJob() # jobconcat.setInputData(["/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_27.slcio", # "/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio"]) jobconcat.setInputData([ "/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2104.slcio", "/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2105.slcio" ]) myconcat = self.getLCIOConcat() res = jobconcat.append(myconcat) if not res['OK']: self.log.error("Failed adding SLCIOConcatenate:", res['Message']) return S_ERROR() self.jobList['concat'] = jobconcat return S_OK((jobconcat, joblciosplit, jobwcut, jobwsplit)) def createFccSwTest(self): """create tests for fccsw""" self.log.notice("Creating jobs for FccSW") #### FccSwJob jobfccsw = self.getJob() fccsw = self.getFccSw() res = jobfccsw.append(fccsw) if not res['OK']: self.log.error("Failed adding FccSw:", res['Message']) return S_ERROR() self.jobList['FccSw1'] = jobfccsw return S_OK(jobfccsw) def createFccAnalysisTest(self): """create tests for fccanalysis""" self.log.notice("Creating jobs for FccAnalysis") #### FccAnalysisJob jobfccanalysis = self.getJob() fccanalysis = self.getFccAnalysis() res = jobfccanalysis.append(fccanalysis) if not res['OK']: self.log.error("Failed adding FccAnalysis:", res['Message']) return S_ERROR() self.jobList['FccAnalysis1'] = jobfccanalysis return S_OK(jobfccanalysis) def runJobLocally(self, job, jobName="unknown"): """run a job locally""" self.log.notice("I will run the tests locally.") from DIRAC import gConfig localarea = gConfig.getValue("/LocalSite/LocalArea", "") if not localarea: self.log.error( "You need to have /LocalSite/LocalArea defined in your dirac.cfg" ) return S_ERROR() if localarea.find("/afs") == 0: self.log.error( "Don't set /LocalSite/LocalArea set to /afs/... as you'll get to install there" ) self.log.error( "check ${HOME}/.dirac.cfg and ${DIRAC}/etc/dirac.cfg") return S_ERROR() self.log.notice("To run locally, I will create a temp directory here.") curdir = os.getcwd() tmpdir = tempfile.mkdtemp("", dir="./") os.chdir(tmpdir) # Jobs that need separate input files specialJobs = ['root', 'ddsim'] filesForJob = { 'root': ['input2.root', 'input.root'], 'ddsim': ['FCalTB.tar.gz', 'Muon_50GeV_Fixed_cosTheta0.7.stdhep'] } for specialName in specialJobs: if "root" in jobName.lower() and specialName == "root": with open("root.sh", "w") as rScript: rScript.write("echo $ROOTSYS") with open("func.C", "w") as rMacro: rMacro.write(''' void func( TString string ) { std::cout << string << std::endl; TFile* file = TFile::Open(string); file->ls(); } ''') testfiledir = 'Testfiles' for fileName in ['input.root', 'input2.root']: shutil.copy(os.path.join(curdir, testfiledir, fileName), os.getcwd()) print os.path.join(curdir, "input2.root"), os.getcwd() if specialName in jobName.lower(): for fileName in filesForJob[specialName]: shutil.copy(os.path.join(curdir, testfiledir, fileName), os.getcwd()) resJob = self.runJob(job, jobName) os.chdir(curdir) if not resJob['OK']: return resJob os.chdir(curdir) if not self.clip.nocleanup: cleanup(tmpdir) return S_OK() def run(self): """submit and run all the tests in jobList""" res = S_ERROR() for name, finjob in self.jobList.iteritems(): if self.clip.submitMode == 'local': res = self.runJobLocally(finjob, name) else: res = self.runJob(finjob, name) return res def runJob(self, finjob, name): """runs or submits the job""" self.log.notice( "############################################################") self.log.notice(" Running or submitting job: %s " % name) self.log.notice("\n\n") res = finjob.submit(self.diracInstance, mode=self.clip.submitMode) if not res["OK"]: self.log.error("Failed job:", res['Message']) return S_ERROR() return S_OK() def checkForTests(self): """check which tests to run""" if self.clip.testMokka: resMokka = self.createMokkaTest() if not resMokka['OK']: return S_ERROR() if self.clip.testWhizard: resWhiz = self.createWhizardTest() if not resWhiz['OK']: return S_ERROR() if self.clip.testSlic: resSlic = self.createSlicTest() if not resSlic['OK']: return S_ERROR() if self.clip.testMarlin: resMarlin = self.createMarlinTest() if not resMarlin['OK']: return S_ERROR() if self.clip.testLCSIM: resLCSim = self.createLCSimTest() if not resLCSim['OK']: return S_ERROR() if self.clip.testSlicPandora: resSP = self.createSlicPandoraTest() if not resSP['OK']: return S_ERROR() if self.clip.testUtilities: resUtil = self.createUtilityTests() if not resUtil['OK']: return S_ERROR() if self.clip.testRoot: resRoot = self.createRootScriptTest() if not resRoot['OK']: return S_ERROR() resRoot = self.createRootHaddTest() if not resRoot['OK']: return S_ERROR() resRoot = self.createRootMacroTest() if not resRoot['OK']: return S_ERROR() if self.clip.testFccSw: resFccSw = self.createFccSwTest() if not resFccSw['OK']: return S_ERROR() if self.clip.testFccAnalysis: resFccAnalysis = self.createFccAnalysisTest() if not resFccAnalysis['OK']: return S_ERROR() return S_OK()
def subOverlay(): # Decide parameters for a job outputSE = "KEK-SRM" isLocal = _clip.isLocal nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents nbevts = 0 # To analize all input events outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix outputDir = _clip.outputDir inputFile = _clip.inputFile if inputFile == "": gLogger.error("Input file for ddsim does not given.") exit(-1) recfile = outputFilePrefix + ".rec.slcio" dstfile = outputFilePrefix + ".dst.slcio" detector_model = "ILD_l5_o1_v02" key = detector_model.split('_') sim_detectorModel = "_".join([key[0], key[1], key[3]]) # Create DIRAC objects for job submission dIlc = DiracILC() job = UserJob() job.setJobGroup("myoverlayjob") job.setName("myoverlay") job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml']) job.setILDConfig("v02-00-02") # job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"]) # job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination # job.setBannedSites([]) # a list of sites not to submit job # job.setCPUTime( cputime_limit_in_seconds_by_dirac_units ) # Create Overlay application ovldata = [{ "ProcessorName": "BgOverlayWW", "evttype": "aa_lowpt_WW", "ProdID": 10237, "expBG": 0.211, "subdir": "000" }, { "ProcessorName": "BgOverlayWB", "evttype": "aa_lowpt_WB", "ProdID": 10241, "expBG": 0.24605, "subdir": "000" }, { "ProcessorName": "BgOverlayBW", "evttype": "aa_lowpt_BW", "ProdID": 10239, "expBG": 0.243873, "subdir": "000" }, { "ProcessorName": "BgOverlayBB", "evttype": "aa_lowpt_BB", "ProdID": 10235, "expBG": 0.35063, "subdir": "000" }, { "ProcessorName": "PairBgOverlay", "evttype": "seeablepairs", "ProdID": 10233, "expBG": 1.0, "subdir": "100" }] BXOverlay = 1 NbSigEvtsPerJob = 100 numberOfSignalEvents = NbSigEvtsPerJob basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws" energy = "500" for ovl in ovldata: print "### OverlayInput ... " + ovl["ProcessorName"] ovlapp = OverlayInput() ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \ ( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] ) print " OverlayPath ... " + ovlpath ovlapp.setMachine("ilc_dbd") # ovlapp.setEnergy(energy) # ovlapp.setDetectorModel(sim_detectorModel) ovlapp.setProcessorName(ovl["ProcessorName"]) ovlapp.setBkgEvtType(ovl["evttype"]) ovlapp.setPathToFiles(ovlpath) ovlapp.setGGToHadInt(ovl["expBG"]) ovlapp.setBXOverlay(BXOverlay) ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob) ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents) res = job.append(ovlapp) if not res['OK']: print res['Message'] exit(1) # Create Marlin application marlin = Marlin() marlin.setVersion("ILCSoft-02-00-02_gcc49") marlin.setDetectorModel(detector_model) marlin.setSteeringFile("MarlinStdReco.xml") marlin.setInputFile(inputFile) marlin.setNumberOfEvents(nbevts) marlin.setOutputDstFile(dstfile) marlin.setOutputRecFile(recfile) extraCLIArguments = " --constant.DetectorModel=%s " % detector_model extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str( energy) extraCLIArguments += " --global.Verbosity=MESSAGE " marlin.setExtraCLIArguments(extraCLIArguments) job.append(marlin) if outputDir != "": job.setOutputData([dstfile, recfile], OutputPath=outputDir, OutputSE=outputSE) if isLocal: job.submit(dIlc, mode="local") else: job.submit(dIlc)
class UserJob(SplitMixin, Job): """User job class. To be used by users, not for production.""" def __init__(self, script=None): """Initialize UserJob, including proxy and splitmixin.""" super(UserJob, self).__init__(script) self.type = 'User' self.diracinstance = None self.usergroup = ['ilc_user', 'calice_user'] self.proxyinfo = getProxyInfo() SplitMixin._initialize(self) def submit(self, diracinstance = None, mode = "wms"): """ Submit call: when your job is defined, and all applications are set, you need to call this to add the job to DIRAC. :param diracinstance: DiracILC instance :type diracinstance: ~ILCDIRAC.Interfaces.API.DiracILC.DiracILC :param str mode: "wms" (default), "agent", or "local" .. note :: The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts """ if self._splittingOption: result = self._split() if 'OK' in result and not result['OK']: return result # Check the credentials. If no proxy or not user proxy, return an error if not self.proxyinfo['OK']: LOG.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError("Not allowed to submit a job, you need a %s proxy." % self.usergroup, self.__class__.__name__) if 'group' in self.proxyinfo['Value']: group = self.proxyinfo['Value']['group'] if group not in self.usergroup: LOG.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError("Not allowed to submit job, you need a %s proxy." % self.usergroup, self.__class__.__name__) else: LOG.error("Could not determine group, you do not have the right proxy.") return self._reportError("Could not determine group, you do not have the right proxy.") res = self._addToWorkflow() if not res['OK']: return res self.oktosubmit = True if not diracinstance: self.diracinstance = DiracILC() else: self.diracinstance = diracinstance return self.diracinstance.submitJob(self, mode) ############################################################################# def setInputData( self, lfns ): """Specify input data by Logical File Name (LFN). Input files specified via this function will be automatically staged if necessary. Example usage: >>> job = UserJob() >>> job.setInputData(['/ilc/prod/whizard/processlist.whiz']) :param lfns: Logical File Names :type lfns: Single LFN string or list of LFNs """ if isinstance( lfns, list ) and lfns: for i, lfn in enumerate( lfns ): lfns[i] = lfn.replace( 'LFN:', '' ) #inputData = map( lambda x: 'LFN:' + x, lfns ) inputData = lfns #because we don't need the LFN: for inputData, and it breaks the #resolution of the metadata in the InputFilesUtilities inputDataStr = ';'.join( inputData ) description = 'List of input data specified by LFNs' self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description ) elif isinstance( lfns, basestring ): #single LFN description = 'Input data specified by LFN' self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description ) else: kwargs = {'lfns':lfns} return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs ) return S_OK() def setInputSandbox(self, flist): """ Add files to the input sandbox, can be on the local machine or on the grid >>> job = UserJob() >>> job.setInputSandbox( ['LFN:/ilc/user/u/username/libraries.tar.gz', >>> 'mySteeringFile.xml'] ) :param flist: Files for the inputsandbox :type flist: `python:list` or `str` """ if isinstance( flist, basestring ): flist = [flist] if not isinstance( flist, list ): return self._reportError("File passed must be either single file or list of files.") self.inputsandbox.extend(flist) return S_OK() ############################################################################# def setOutputData(self, lfns, OutputPath = '', OutputSE = ''): """For specifying output data to be registered in Grid storage. If a list of OutputSEs are specified the job wrapper will try each in turn until successful. Example usage: >>> job = UserJob() >>> job.setOutputData(['Ntuple.root']) :param lfns: Output data file or files :type lfns: Single `str` or `python:list` of strings ['',''] :param str OutputPath: Optional parameter to specify the Path in the Storage, postpended to /ilc/user/u/username/ :param OutputSE: Optional parameter to specify the Storage Element to store data or files, e.g. CERN-SRM :type OutputSE: `python:list` or `str` """ kwargs = {'lfns' : lfns, 'OutputSE' : OutputSE, 'OutputPath' : OutputPath} if isinstance( lfns, list ) and lfns: outputDataStr = ';'.join(lfns) description = 'List of output data files' self._addParameter(self.workflow, 'UserOutputData', 'JDL', outputDataStr, description) elif isinstance( lfns, basestring ): description = 'Output data file' self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns, description) else: return self._reportError('Expected file name string or list of file names for output data', **kwargs) if OutputSE: description = 'User specified Output SE' if isinstance( OutputSE, basestring ): OutputSE = [OutputSE] elif not isinstance( OutputSE, list ): return self._reportError('Expected string or list for OutputSE', **kwargs) OutputSE = ';'.join(OutputSE) self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE, description) if OutputPath: description = 'User specified Output Path' if not isinstance( OutputPath, basestring ): return self._reportError('Expected string for OutputPath', **kwargs) # Remove leading "/" that might cause problems with os.path.join while OutputPath[0] == '/': OutputPath = OutputPath[1:] if OutputPath.count("ilc/user"): return self._reportError('Output path contains /ilc/user/ which is not what you want', **kwargs) self._addParameter(self.workflow, 'UserOutputPath', 'JDL', OutputPath, description) return S_OK() ############################################################################# def setOutputSandbox( self, files ): """Specify output sandbox files. If specified files are over 10MB, these may be uploaded to Grid storage with a notification returned in the output sandbox. .. Note :: Sandbox files are removed after 2 weeks. Example usage: >>> job = UserJob() >>> job.setOutputSandbox(['*.log','*.sh', 'myfile.txt']) Use the output sandbox only for small files. Larger files should be stored on the grid and downloaded later if necessary. See :func:`setOutputData` :param files: Output sandbox files :type files: Single `str` or `python:list` of strings ['',''] """ if isinstance( files, list ) and files: fileList = ";".join( files ) description = 'Output sandbox file list' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description ) elif isinstance( files, basestring ): description = 'Output sandbox file' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description ) else: kwargs = {'files' : files} return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs ) return S_OK() def setILDConfig(self,version): """ Define the Configuration package to obtain """ appName = 'ILDConfig' self._addSoftware(appName.lower(), version) self._addParameter( self.workflow, 'ILDConfigPackage', 'JDL', appName+version, 'ILDConfig package' ) return S_OK() def setCLICConfig(self, version): """Define the CLIC Configuration package to obtain, copies steering files from CLIC Configuration folder to working directory :param str version: version string, e.g.: 'ILCSoft-2017-07-27' """ appName = 'ClicConfig' self._addSoftware(appName.lower(), version) self._addParameter( self.workflow, 'ClicConfigPackage', 'JDL', appName+version, 'CLIC Config package' ) return S_OK()
def main(): ''' This is the script main method, which will hold all the logic. ''' cliParams = Params() cliParams.registerSwitches() Script.parseCommandLine(ignoreErrors=True) consistent = cliParams.checkConsistency() if not consistent['OK']: gLogger.error("Error checking consistency:", consistent['Message']) Script.showHelp() dexit(2) ##Get prodID prodID = float(cliParams.prodID) ##Get all possible input files inputFiles = getInputFiles(prodID) ##Get suffix suffix = cliParams.suffix if suffix: suffix = "_" + suffix ##Jet clustering algorithm = "ValenciaPlugin {radius:.1f} {beta:.1f} {gamma:.1f}" jetRadius = float(cliParams.jetRadius) jetBeta = float(cliParams.jetBeta) jetGamma = float(cliParams.jetGamma) jetCluster = "ExclusiveNJets 2" jetRecomb = "E_scheme" ##Top tagger deltaR = float(cliParams.deltaR) deltaP = float(cliParams.deltaP) cos_theta_W_max = float(cliParams.cos_theta_W_max) ##Sixfermion sample sixFermionSample = cliParams.sixFermionSample ##Number of files per job nrFilesPerJob = int(cliParams.nrFilesPerJob) from ILCDIRAC.Interfaces.API.DiracILC import DiracILC repDir = "/afs/cern.ch/user/l/lstroem/clicdp/analysis/steering/chain/topasymmetry_wflavourtag/submit/{prodID:04.0f}/rep/".format( prodID=prodID) subprocess.call("mkdir -p " + repDir, shell=True) dirac = DiracILC( False ) #, repDir+"topasymmetry_vlc{suffix}.rep".format(suffix = suffix)) jetAlgo = algorithm.format(radius=jetRadius, beta=jetBeta, gamma=jetGamma) inputFileList = [] i = 0 j = 1 for inputFile in inputFiles: inputFileList.append(inputFile) i += 1 if (i >= nrFilesPerJob * j) or (i == len(inputFiles)): jobName = "topasymmetry_chain_{jetSettings}_dR{deltaR:.2f}_dP{deltaP:.2f}_cthetaWmax{cos_theta_W_max:.2f}_1Jun2017_part{index}_{prodID:04.0f}".format( jetSettings=getJetSettings(jetAlgo, jetCluster, jetRecomb), deltaR=deltaR, deltaP=deltaP, cos_theta_W_max=cos_theta_W_max, index=j, prodID=prodID) jobGroup = "topasymmetry_chain_{prodID:04.0f}".format( prodID=prodID) job, outputFile, rootFile = defGridJob(jobName, jobGroup, inputFileList) ##Check if outputfile already exists pOutCheck = subprocess.Popen( "dirac-dms-lfn-replicas /ilc/user/r/rstrom/" + jobGroup + "/" + rootFile, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) outCheck, errCheck = pOutCheck.communicate() #gLogger.notice(jobName) if "no such file" not in outCheck.lower(): gLogger.notice('File exists! Skipping!') inputFileList = [] j += 1 continue #continue #use break if only part1, use continue of run over all parts #gLogger.notice(jetAlgo) #doLogger(jobName, inputFileList, outputFile, rootFile) res = job.append( defMarlin(outputFile, rootFile, jetAlgo=jetAlgo, jetCluster=jetCluster, jetRecomb=jetRecomb, deltaR=deltaR, deltaP=deltaP, cos_theta_W_max=cos_theta_W_max, sixFermionSample=sixFermionSample)) if not res['OK']: #Catch if there is an error print res['Message'] #Print the error message dexit ##Job submission print job.submit(dirac) inputFileList = [] j += 1 #break #add break of only part1 gLogger.notice("All done!") dexit(0)
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard, Mokka, Marlin, OverlayInput from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from DIRAC import exit as dexit dirac =DiracILC() #wh.setOutputFile("myfile.stdhep") j = UserJob() wh = Whizard(processlist=dirac.getProcessList()) wh.setEnergy(3000) wh.setEvtType("ee_h_mumu") wh.setNbEvts(1) wh.setEnergy(3000) params = {} params['USERB1']='F' wh.setParameterDict(params) wh.setModel("sm") res = j.append(wh) if not res['OK']: print res['Message'] dexit(1) mo = Mokka()
class DiracILCTestCase( unittest.TestCase ): """ Base class for the DiracILC test cases """ def setUp( self ): """set up the objects""" ops_mock = Mock() mocked_modules = { 'DIRAC.ConfigurationSystem.Client.Helpers.Operations' : ops_mock } self.module_patcher = patch.dict( sys.modules, mocked_modules ) self.module_patcher.start() from ILCDIRAC.Interfaces.API.DiracILC import DiracILC self.dilc = DiracILC() def setOptions(*args): if 'SingleReplicaSEs' in args[0]: return ['SE'] if 'Minimum' in args[0]: return 1 if args[0].endswith('PreferredSEs'): return ['Awesome-Tape-SE'] ops_mock = Mock() ops_mock.getValue = Mock() ops_mock.getValue.side_effect = setOptions self.dilc.ops = ops_mock def tearDown( self ): self.module_patcher.stop() def test_getprocesslist( self ): with patch('%s.gConfig.getValue' % MODULE_NAME, new=Mock(return_value='some_gconf_testval')) as conf_mock, \ patch('%s.ProcessList' % MODULE_NAME, new=Mock()) as pl_mock: res = self.dilc.getProcessList() pl_mock.assert_called_once_with( 'some_gconf_testval' ) assertEqualsImproved( res, pl_mock(), self ) conf_mock.assert_called_once_with( '/LocalSite/ProcessListPath', '' ) def test_getprocesslist_nopath( self ): ops_mock = Mock() ops_mock.getValue.return_value = '' self.dilc.ops = ops_mock with patch('%s.gConfig.getValue' % MODULE_NAME, new=Mock(return_value='')) as conf_mock, \ patch('%s.ProcessList' % MODULE_NAME, new=Mock()) as pl_mock: res = self.dilc.getProcessList() pl_mock.assert_called_once_with( '' ) assertEqualsImproved( res, pl_mock(), self ) conf_mock.assert_called_once_with( '/LocalSite/ProcessListPath', '' ) ops_mock.getValue.assert_called_once_with( '/ProcessList/Location', '' ) def test_presubmissionchecks_notoktosubmit( self ): job_mock = Mock() job_mock.oktosubmit = False assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ), 'you should use job.submit(dirac)', self ) def test_presubmissionchecks_checkfails( self ): job_mock = Mock() job_mock.oktosubmit = True with patch('%s.DiracILC._do_check' % MODULE_NAME, new=Mock(return_value=S_ERROR('mytest_check_failed'))): assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ), 'mytest_check_failed', self ) def test_presubmissionchecks_askuser_fails( self ): job_mock = Mock() job_mock.oktosubmit = True job_mock._askUser.return_value = S_ERROR( 'user says no' ) self.dilc.checked = False with patch('%s.DiracILC._do_check' % MODULE_NAME, new=Mock(return_value=S_OK())): assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ), 'user says no', self ) def test_checkparams( self ): job_mock = Mock() job_mock.errorDict = {} with patch('%s.DiracILC.preSubmissionChecks' % MODULE_NAME, new=Mock(return_value=S_OK('mytest'))) as check_mock: assertDiracSucceedsWith( self.dilc.checkparams( job_mock ), 'mytest', self ) check_mock.assert_called_once_with( job_mock, mode = '' ) def test_checkparams_fails( self ): job_mock = Mock() job_mock.errorDict = { 'myerror1' : [ 'Terrible failure' ], 'last_error' : [ 'True' ] } assertDiracFailsWith_equals( self.dilc.checkparams( job_mock ), { 'myerror1' : [ 'Terrible failure' ], 'last_error' : [ 'True' ] }, self ) def test_giveprocesslist( self ): self.dilc.processList = '13985u185r9135r' assertEqualsImproved( self.dilc.giveProcessList(), '13985u185r9135r', self ) def test_giveprocesslist_empty( self ): self.dilc.processList = '' assertEqualsImproved( self.dilc.giveProcessList(), '', self ) def test_giveprocesslist_false( self ): self.dilc.processList = False assertEqualsImproved( self.dilc.giveProcessList(), False, self ) def test_giveprocesslist_none( self ): self.dilc.processList = None assertEqualsImproved( self.dilc.giveProcessList(), None, self ) def test_retrievelfns_norepo( self ): self.dilc.jobRepo = None assertDiracSucceeds( self.dilc.retrieveRepositoryOutputDataLFNs(), self ) def test_retrievelfns( self ): repo_mock = Mock() ret_dict = { '1' : { 'State' : 'Done', 'UserOutputData' : '1389' }, '2' : {}, '3' : { 'State' : 'secret_teststate' }, '4' : { 'State' : 'invalid_state' }, '5' : { 'State' : 'Done', 'UserOutputData' : 0 }, '6' : { 'ignore_me' : True }, '7' : { 'State' : 'secret_teststate', 'UserOutputData' : 0 }, '148' : { 'State' : 'Done', 1 : False, True : 941, 'values_' : 'keys' } } repo_mock.readRepository.return_value = S_OK( ret_dict ) self.dilc.jobRepo = repo_mock with patch('%s.DiracILC.parameters' % MODULE_NAME, new=Mock(side_effect=[S_OK({'UploadedOutputData':'/my/test/lfn1'}),S_ERROR(),S_OK({}),S_OK({'some_entries':'some_values',1:True,'UploadedOutputData':'/more_lfns/append/testlfn.log'})])) as param_mock: assertEqualsImproved( self.dilc.retrieveRepositoryOutputDataLFNs( [ 'Done', 'secret_teststate' ] ), [ '/my/test/lfn1', '/more_lfns/append/testlfn.log' ], self ) assertMockCalls( param_mock, [ 3, 5, 7, 148 ], self ) def test_docheck_checksandbox_fails( self ): job_mock = Mock() job_mock.inputsandbox = [ 'mysandbox', 'other_value' ] with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_ERROR('test_err_sandbox'))) as check_mock: assertDiracFailsWith( self.dilc._do_check( job_mock ), 'test_err_sandbox', self ) check_mock.assert_called_once_with( job_mock ) def test_docheck_too_many_lists( self ): job_mock = Mock() job_mock.inputsandbox = [ 'mysandbox', [ 'im_a_sandbox_file.stdhep', [ 'evil_list', 'deletethis'] ] ] assertDiracFailsWith( self.dilc._do_check( job_mock ), 'too many lists of lists in the input sandbox', self ) def test_docheck_checkapps_fails( self ): platform_mock = Mock() platform_mock.getValue.return_value = 'pf14081' apps_mock = Mock() apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0' param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock } job_mock = Mock() wf_mock = Mock() wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ] job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ] job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ] job_mock.workflow = wf_mock with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \ patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(side_effect=[S_OK()] * 3 + [S_ERROR('checkapp_failed_testme')])) as checkapp_mock: assertDiracFailsWith( self.dilc._do_check( job_mock ), 'checkapp_failed_testme', self ) checksb_mock.assert_called_once_with( job_mock ) job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value', 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ) job_mock._addParameter.assert_called_once_with( wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' ) assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ), ( 'pf14081', 'other_appv', '91.3' ), ( 'pf14081', 'more_dependencies', '1' ), ( 'pf14081', 'last_app', '0' ) ], self ) def test_docheck_checkoutputpath_fails( self ): platform_mock = Mock() platform_mock.getValue.return_value = 'pf14081' apps_mock = Mock() apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0' path_mock = Mock() path_mock.getValue.return_value = 'path1948512895' param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock, 'UserOutputPath' : path_mock } job_mock = Mock() wf_mock = Mock() wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ] job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ] job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ] job_mock.workflow = wf_mock with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \ patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \ patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_ERROR('outputpath_check_testerr'))) as checkpath_mock: assertDiracFailsWith( self.dilc._do_check( job_mock ), 'outputpath_check_testerr', self ) checksb_mock.assert_called_once_with( job_mock ) job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value', 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ) job_mock._addParameter.assert_called_once_with( wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' ) assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ), ( 'pf14081', 'other_appv', '91.3' ), ( 'pf14081', 'more_dependencies', '1' ), ( 'pf14081', 'last_app', '0' ) ], self ) checkpath_mock.assert_called_once_with( 'path1948512895' ) def test_docheck_checkconsistency_fails( self ): platform_mock = Mock() platform_mock.getValue.return_value = 'pf14081' apps_mock = Mock() apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0' path_mock = Mock() path_mock.getValue.return_value = 'path1948512895' data_mock = Mock() data_mock.getValue.return_value = 'data1389518' param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock, 'UserOutputPath' : path_mock, 'UserOutputData' : data_mock } job_mock = Mock() job_mock.addToOutputSandbox = 'job_sandbox13895' wf_mock = Mock() wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ] job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ] job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ] job_mock.workflow = wf_mock with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \ patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \ patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkpath_mock, \ patch('%s.DiracILC._checkdataconsistency' % MODULE_NAME, new=Mock(return_value=S_ERROR('consistency_testerr'))) as checkconsistency_mock: assertDiracFailsWith( self.dilc._do_check( job_mock ), 'consistency_testerr', self ) checksb_mock.assert_called_once_with( job_mock ) job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value', 'sandbox_file1.txt', 'sandbox_file2.log', 'last.file' ] ) job_mock._addParameter.assert_called_once_with( wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' ) assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ), ( 'pf14081', 'other_appv', '91.3' ), ( 'pf14081', 'more_dependencies', '1' ), ( 'pf14081', 'last_app', '0' ) ], self ) checkpath_mock.assert_called_once_with( 'path1948512895' ) checkconsistency_mock.assert_called_once_with( 'data1389518', 'job_sandbox13895' ) def test_checkapp( self ): ops_mock = Mock() ops_mock.getValue.return_value = '' self.dilc.ops = ops_mock assertDiracFailsWith( self.dilc._checkapp( 'test_platform_341', 'testapp', 'v13.2' ), 'could not find the specified software testapp_v13.2 for test_platform_341, check in CS', self ) assertMockCalls( ops_mock.getValue, [ ( '/AvailableTarBalls/test_platform_341/testapp/v13.2/TarBall', '' ), ( '/AvailableTarBalls/test_platform_341/testapp/v13.2/CVMFSPath', '' ) ], self ) def test_checkoutputpath_invalidchar_1( self ): assertDiracFailsWith( self.dilc._checkoutputpath( 'http://www.mysitedoesnotexist3h3.abc/some/file.txt' ), 'invalid path', self ) def test_checkoutputpath_invalidchar_2( self ): assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/./some/file.log' ), 'invalid path', self ) def test_checkoutputpath_invalidchar_3( self ): assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/../dir2/somefile.txt' ), 'invalid path', self ) def test_checkoutputpath_trailing_slash( self ): assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/myfile.txt/ ' ), 'invalid path', self ) def test_checkdataconsistency_outputdata_sandbox_equal( self ): assertDiracFailsWith( self.dilc._checkdataconsistency( 'same_item;something_else', [ 'distinct_item1', 'same_item' ] ), 'output data and sandbox should not contain the same thing', self ) def test_checkdataconsistency_wildcardchar( self ): assertDiracFailsWith( self.dilc._checkdataconsistency( '/test/dir/file.txt;/mydir/something/*;/file/dir/log.log', [ '/input/sandbox.pdf', '/other/sb/file.stdhep' ] ), 'wildcard character in outputdata definition', self ) def test_checkinputsb_getreplicas_notok( self ): job_mock = Mock() job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf' with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_err'))) as replica_mock: assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'could not get replicas', self ) replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] ) def test_checkinputsb_getreplicas_fails( self ): job_mock = Mock() job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf' ret_dict = { 'Failed' : [ '/failed/replica1', '/other/inval/replica' ], 'Successful' : {} } with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock: assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'failed to find replicas', self ) replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] ) def test_checkinputsb( self ): job_mock = Mock() job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf' ret_dict = {'Failed': [], 'Successful': {'/one/replica': {'SE': 'surl'}}} with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock: assertDiracSucceeds( self.dilc.checkInputSandboxLFNs( job_mock ), self ) replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] ) def test_checkinputsb_notInputSB(self): job_mock = Mock() job_mock.workflow.findParameter.return_value = None assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self) def test_checkinputsb_notInputSB_Value(self): job_mock = Mock() job_mock.workflow.findParameter.return_value.getValue.return_value = '' assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self) def test_checkinputsb_noLFNs(self): job_mock = Mock() job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep' assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self) def test_checkinputsb_noRepl(self): job_mock = Mock() job_mock.workflow.findParameter.return_value.getValue.return_value = 'LFN:/some/file.txt' ret_dict = {'Failed': [], 'Successful': {'/some/file.txt': {'Bad-SE': 'surl'}}} def setOptions(*args): if 'SingleReplicaSEs' in args[0]: return ['Awesome-Disk-SE'] if 'Minimum' in args[0]: return 2 if args[0].endswith('PreferredSEs'): return ['Awesome-Tape-SE'] ops_mock = Mock() ops_mock.getValue = setOptions self.dilc.ops = ops_mock with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock: assertDiracFailsWith(self.dilc.checkInputSandboxLFNs(job_mock), 'Not enough replicas', self) replica_mock.assert_called_once_with(['/some/file.txt']) def test_checkinputsb_goodRepl(self): job_mock = Mock() job_mock.workflow.findParameter.return_value.getValue.return_value = 'LFN:/some/file.txt' ret_dict = {'Failed': [], 'Successful': {'/some/file.txt': {'Awesome-Disk-SE': 'surl'}}} def setOptions(*args): if 'SingleReplicaSEs' in args[0]: return ['Awesome-Disk-SE'] if 'Minimum' in args[0]: return 2 if args[0].endswith('PreferredSEs'): return ['Awesome-Tape-SE'] ops_mock = Mock() ops_mock.getValue = setOptions self.dilc.ops = ops_mock with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock: assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self) replica_mock.assert_called_once_with(['/some/file.txt'])
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka, Marlin, OverlayInput from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from DIRAC import exit as dexit d = DiracILC(True, "repo.rep") n_evts = 500 n_evts_per_job = 100 n_jobs = n_evts / n_evts_per_job for i in range(n_jobs): j = UserJob() mo = Mokka() mo.setEnergy(3000) mo.setVersion("0706P08") mo.setSteeringFile("clic_ild_cdr.steer") mo.setMacFile("particlegun_electron.mac") mo.setOutputFile("MyFile.slcio") mo.setNbEvts(n_evts_per_job) res = j.append(mo) if not res['OK']: print res['Message'] break ma = Marlin() ma.setVersion("v0111Prod")
class JobCreater(object): """contains all the versions and parameters to create all theses tests""" # pylint: disable=too-many-instance-attributes # Test parameters, necessary due to amount of tests in this class. def __init__( self, clip, params ): self.clip = clip self.ildConfig = params.get( "ildConfig", None ) self.alwaysOverlay = params.get( "alwaysOverlay", False ) self.runOverlay = self.clip.testOverlay or self.alwaysOverlay self.mokkaVersion = params["mokkaVersion"] self.mokkaSteeringFile = params.get( "mokkaSteeringFile" ) self.detectorModel = params.get( "detectorModel" ) self.marlinVersion = params.get( "marlinVersion" ) self.marlinSteeringFile = params.get( "marlinSteeringFile" ) self.ddsimVersion = params.get( "ddsimVersion" ) self.ddsimDetectorModel = params.get( "ddsimDetectorModel") self.ddsimInputFile = params.get( "ddsimInputFile" ) self.marlinInputdata = params.get ( "marlinInputdata" ) self.gearFile = params.get( "gearFile" ) self.lcsimVersion = params.get( "lcsimVersion" ) self.steeringFileVersion = params.get( "steeringFileVersion", None ) self.rootVersion = params["rootVersion"] self.whizard2Version = params.get( "whizard2Version" ) self.whizard2SinFile = params.get( "whizard2SinFile" ) self.energy = params.get("energy") self.backgroundType = params.get("backgroundType") self.machine = params.get("machine") self.gearFile = params.get( "gearFile" ) self.marlinSteeringFile = params.get( "marlinSteeringFile" ) self.marlinVersion = params.get( "marlinVersion" ) self.lcsimPreSteeringFile = params.get( "lcsimPreSteeringFile" ) self.lcsimPostSteeringFile = params.get( "lcsimPostSteeringFile" ) self.fccSwPath = params.get( "fccSwPath" ) self.fccSwSteeringFile = params.get( "fccSwSteeringFile" ) self.fccAnalysisSteeringFile = params.get( "fccAnalysisSteeringFile" ) ### other things needed to run tests self.log = gLogger.getSubLogger("JobCreater") from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid if self.clip.submitMode == "local": self.log.notice("") self.log.notice(" DIRAC RCSID:", drcsid ) self.log.notice(" Job RCSID:", jrcsid ) self.log.notice("Applications RCSID:", apprcsid ) self.log.notice("") self.diracInstance = DiracILC(False, 'tests.rep') self.jobList = {} def createDDSimTest( self, inputfile = None, detectorModel = None): """Create a job running ddsim""" if inputfile is None: inputfile = self.ddsimInputFile sandbox = [ inputfile ] if detectorModel is None: detectorModel = self.ddsimDetectorModel else: sandbox.append(detectorModel) jobdd = self.getJob() from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim ddsim = DDSim() ddsim.setVersion(self.ddsimVersion) ddsim.setDetectorModel(detectorModel) ddsim.setNumberOfEvents(1) ddsim.setInputFile(inputfile) jobdd.setInputSandbox(sandbox) res = jobdd.append(ddsim) if not res['OK']: self.log.error("Failed adding DDSim:", res['Message']) return S_ERROR("Failed adding DDSim to Job") return S_OK(jobdd) def createWhizard2Test( self ): """Create a job running Whizard2""" jobdd = self.getJob() from ILCDIRAC.Interfaces.API.NewInterface.Applications.Whizard2 import Whizard2 whiz = Whizard2() whiz.setVersion(self.whizard2Version) whiz.setNumberOfEvents(1) whiz.setSinFile(self.whizard2SinFile) whiz.setOutputFile("test.stdhep") res = jobdd.append(whiz) if not res['OK']: self.log.error("Failed adding Whizard2:", res['Message']) return S_ERROR("Failed adding Whizard2 to Job") return S_OK(jobdd) def createMokkaTest(self): """create a job running mokka, and maybe whizard before""" self.log.notice("Creating jobs for Mokka") #(Whizard + )Mokka jobmo = self.getJob() if self.clip.testChain: whmo = self.getWhizard(2) res = jobmo.append(whmo) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR("Failed adding Whizard") elif self.clip.testInputData: jobmo.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep") else: self.log.error("Mokka does not know where to get its input from") return S_ERROR("Mokka does not know where to gets its input from") mo = self.getMokka() if self.clip.testChain: mo.getInputFromApp(whmo) else: mo.setNumberOfEvents(1) res = jobmo.append(mo) if not res['OK']: self.log.error("Failed adding Mokka:", res['Message']) return S_ERROR("Failed adding Mokka to Job") jobmo.setOutputData("testsim.slcio", OutputSE="CERN-DIP-4") self.jobList['Mokka1'] = jobmo return S_OK(jobmo) def createRootScriptTest(self): """create a job running root""" self.log.notice("Creating jobs for Root") jobRoot = self.getJob() jobRoot.setInputSandbox(["root.sh", "input.root","input2.root"]) root = self.getRoot() res = jobRoot.append(root) if not res['OK']: self.log.error("Failed adding Root:", res['Message']) return S_ERROR("Failed adding Root to Job") self.jobList['Root'] = jobRoot return S_OK(jobRoot) def createRootHaddTest(self): """create a job running root""" self.log.notice("Creating jobs for Root") jobRoot = self.getJob() jobRoot.setInputSandbox(["input.root","input2.root"]) root = self.getRoot() root.setScript("hadd") res = jobRoot.append(root) if not res['OK']: self.log.error("Failed adding Root:", res['Message']) return S_ERROR("Failed adding Root to Job") self.jobList['Root'] = jobRoot return S_OK(jobRoot) def createRootMacroTest(self): """create a job running root""" self.log.notice("Creating jobs for Root") jobRoot = self.getJob() jobRoot.setInputSandbox(["func.C", "input.root","input2.root"]) root = self.getRootMacro() root.setScript("func.C") res = jobRoot.append(root) if not res['OK']: self.log.error("Failed adding Root:", res['Message']) return S_ERROR("Failed adding Root to Job") self.jobList['Root'] = jobRoot return S_OK(jobRoot) def getOverlay(self, nbevts): """ Create an overlay step """ pathToFiles = None from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput overlay = OverlayInput() if self.energy==350: if self.detectorModel=="ILD_o1_v05": pathToFiles="/ilc/user/s/sailer/testFiles/overlay/ild_350/" if pathToFiles: overlay.setPathToFiles(pathToFiles) else: self.log.warn("better define pathToFiles for this overlay: %s, %s, %s" % (self.energy, self.machine, self.backgroundType) ) overlay.setMachine(self.machine) overlay.setEnergy(self.energy) overlay.setDetectorModel(self.detectorModel) overlay.setBkgEvtType(self.backgroundType) overlay.setBXOverlay(60) overlay.setGGToHadInt(0.3) overlay.setNumberOfSignalEventsPerJob(nbevts) return overlay def getMokka(self): """ Define a mokka app """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka mokka = Mokka() mokka.setVersion(self.mokkaVersion) mokka.setSteeringFile(self.mokkaSteeringFile) mokka.setOutputFile("testsim.slcio") mokka.setDetectorModel(self.detectorModel) if self.steeringFileVersion: mokka.setSteeringFileVersion(self.steeringFileVersion) return mokka def getRoot(self): """ Define a root app """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootScript root = RootScript() root.setScript("root.sh") root.setArguments("output.root input.root input2.root") root.setVersion(self.rootVersion) root.setOutputFile("output.root") return root def getRootMacro(self): """ Define a root app """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootMacro root = RootMacro() root.setMacro("func.C") root.setArguments(r"\"input.root\"") root.setVersion(self.rootVersion) return root @staticmethod def getSLIC(): """ Get a SLIC instance """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC slic = SLIC() slic.setVersion('v2r9p8') slic.setSteeringFile('defaultClicCrossingAngle.mac') slic.setDetectorModel('clic_sid_cdr') slic.setOutputFile('testsim.slcio') return slic @staticmethod def getSLICPandora(): """ Get some SLICPAndora app """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLICPandora slicp = SLICPandora() slicp.setVersion('CLIC_CDR') slicp.setDetectorModel('clic_sid_cdr') slicp.setPandoraSettings("PandoraSettingsSlic.xml") slicp.setOutputFile('testpandora.slcio') return slicp def getMarlin( self ): """ Define a marlin step """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin marlin = Marlin() # marlin.setVersion("v0111Prod") marlin.setVersion(self.marlinVersion) marlin.setSteeringFile(self.marlinSteeringFile) marlin.setGearFile(self.gearFile) marlin.setOutputDstFile("testmarlinDST.slcio") marlin.setOutputRecFile("testmarlinREC.slcio") marlin.setNumberOfEvents(1) return marlin def getDD( self ): """ Create a DDSim object """ from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim ddsim = DDSim() ddsim.setVersion(self.ddsimVersion) ddsim.setDetectorModel(self.ddsimDetectorModel) ddsim.setInputFile(self.ddsimInputFile) ddsim.setNumberOfEvents(2) return ddsim def getLCSIM(self, prepandora = True): """ Get some LCSIM """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM lcsim = LCSIM() lcsim.setVersion('CLIC_CDR') lcsim.setDetectorModel('clic_sid_cdr.zip') if prepandora: lcsim.setSteeringFile(self.lcsimPreSteeringFile) lcsim.setOutputFile("testlcsim.slcio") else: lcsim.setSteeringFile(self.lcsimPostSteeringFile) #lcsim.setOutputFile("testlcsimfinal.slcio") lcsim.setOutputDstFile("testlcsimDST.slcio") lcsim.setOutputRecFile("testlcsimREC.slcio") lcsim.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml") return lcsim def getFccSw( self ): """ Define a fccsw step """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccSw fccsw = FccSw() fccsw.fccSwPath = self.fccSwPath fccsw.setSteeringFile(self.fccSwSteeringFile) return fccsw def getFccAnalysis( self ): """ Define a fccanalysis step """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccAnalysis fccanalysis = FccAnalysis() fccanalysis.setSteeringFile(self.fccAnalysisSteeringFile) return fccanalysis @staticmethod def getStdhepcut(generated): """ Get some cuts in """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdhepCutJava stdhepcut = StdhepCutJava() stdhepcut.setVersion('1.0') stdhepcut.setSelectionEfficiency(1.) #stdhepcut.setSteeringFile("cuts_testing_1400.txt") stdhepcut.setInlineCuts("leptonInvMass_R 13 100 200") stdhepcut.setSteeringFileVersion("V18") stdhepcut.setMaxNbEvts(1) stdhepcut.setNumberOfEvents(generated) return stdhepcut @staticmethod def getStdhepSplit(): """ Get some stdhep split """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit stdhepsplit = StdHepSplit() stdhepsplit.setVersion("V2") stdhepsplit.setNumberOfEventsPerFile(5) stdhepsplit.setOutputFile("teststdhepsplit.stdhep") stdhepsplit.setMaxRead(10) return stdhepsplit @staticmethod def getLCIOSplit(events_per_file): """ Get a LCIO split """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOSplit lciosplit = SLCIOSplit() lciosplit.setNumberOfEventsPerFile(events_per_file) lciosplit.setOutputFile("testlciosplit.slcio") return lciosplit @staticmethod def getLCIOConcat(): """ Get a LCIO Concat """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOConcatenate lcioconcat = SLCIOConcatenate() lcioconcat.setOutputFile("testlcioconcat.slcio") return lcioconcat def getJob(self): """ Define a generic job, it should be always the same """ from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob myjob = UserJob() myjob.setName("Testing") myjob.setJobGroup("Tests") myjob.setCPUTime(30000) myjob.dontPromptMe() myjob.setLogLevel("VERBOSE") myjob.setPlatform("x86_64-slc5-gcc43-opt") myjob.setOutputSandbox(["*.log","*.xml", "*.sh"]) myjob._addParameter( myjob.workflow, 'TestFailover', 'String', True, 'Test failoverRequest') myjob._addParameter( myjob.workflow, 'Platform', 'JDL', "x86_64-slc5-gcc43-opt", 'OS Platform') if self.ildConfig: myjob.setILDConfig(self.ildConfig) return myjob def getWhizardModel(self, nbevts, energy, model): """ Create a default whizard """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard proddict = "e2e2_o" if model != "sm": proddict = "se2se2_r" whiz = Whizard(processlist = self.diracInstance.getProcessList()) whiz.setModel(model) pdict = {} pdict['process_input'] = {} pdict['process_input']['process_id'] = proddict pdict['process_input']['sqrts'] = energy if model != 'sm': pdict['process_input']['input_file'] = "LesHouches.msugra_1" pdict['process_input']['input_slha_format'] = 'T' pdict['process_input']['beam_recoil'] = 'T' pdict['integration_input'] = {} pdict['integration_input']['calls'] = '1 50000 10 5000 1 15000' pdict['simulation_input'] = {} pdict['simulation_input']['normalize_weight'] = 'F' pdict['simulation_input']['n_events'] = nbevts pdict['simulation_input']['keep_initials'] = 'T' pdict['simulation_input']['events_per_file'] = 500000 if model != 'sm': pdict['simulation_input']['pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1;IMSS(1)=11; IMSS(21)=71; IMSS(22)=71" else: pdict['simulation_input']['pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ; MSTJ(28)=2 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1" pdict['parameter_input'] = {} # pdict['parameter_input']['mmu']=mmu # pdict['parameter_input']['mtau']=mtau # pdict['parameter_input']['mb']=mb # pdict['parameter_input']['mc']=mc pdict['parameter_input']['mH'] = 125 pdict['beam_input_1'] = {} pdict['beam_input_1']['particle_name'] = "e1" pdict['beam_input_1']['polarization'] = "0.0 0.0" pdict['beam_input_1']['USER_spectrum_on'] = 'T' pdict['beam_input_1']['USER_spectrum_mode'] = 19 if energy == 1400 else 11 pdict['beam_input_1']['ISR_on'] = 'T' pdict['beam_input_1']['EPA_on'] = "F" pdict['beam_input_2'] = {} pdict['beam_input_2']['particle_name'] = "E1" pdict['beam_input_2']['polarization'] = "0.0 0.0" pdict['beam_input_2']['USER_spectrum_on'] = 'T' pdict['beam_input_2']['ISR_on'] = 'T' pdict['beam_input_2']['USER_spectrum_mode'] = 19 if energy == 1400 else 11 pdict['beam_input_2']['EPA_on'] = 'F' whiz.setFullParameterDict(pdict) whiz.setOutputFile("testgen.stdhep") return whiz def getWhizard(self, nbevts): """ Get some defualt SM whizard """ return self.getWhizardModel(nbevts, 1400, "sm") def getWhizardSUSY(self, nbevts): """ Get a susy whizard """ return self.getWhizardModel(nbevts, 3000, "slsqhh") def createWhizardTest(self): """create a test for whizard""" self.log.notice("Creating jobs for Whizard") ##### WhizardJob jobw = self.getJob() wh = self.getWhizard(2) res = jobw.append(wh) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() self.jobList['Whizard1'] = jobw ##### WhizardJob jobwsusy = self.getJob() whsusy = self.getWhizardSUSY(2) res = jobwsusy.append(whsusy) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() self.jobList['WhizSusy'] = jobwsusy return S_OK((jobw, jobwsusy)) def createSlicTest(self): """create tests for slic""" self.log.notice("Creating jobs for SLIC") #run (Whizard +)SLIC jobslic = self.getJob() if self.clip.testChain: whslic = self.getWhizard(2) res = jobslic.append(whslic) if not res["OK"]: self.log.error("Failed adding Whizard:", res['Value']) return S_ERROR() elif self.clip.testInputData: jobslic.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep") else: self.log.error("SLIC does not know where to get its input from") return S_ERROR() myslic = self.getSLIC() if self.clip.testChain: myslic.getInputFromApp(whslic) else: myslic.setNumberOfEvents(2) res = jobslic.append(myslic) if not res['OK']: self.log.error("Failed adding slic: ", res["Message"]) return S_ERROR() self.jobList['Slic1'] = jobslic return S_OK(jobslic) def createMarlinTest(self , setInputData = False): """create tests for marlin""" self.log.notice( "Creating test for Marlin" ) #((Whizard + Mokka +)Overlay+) Marlin jobma = self.getJob() if self.clip.testChain: moma = self.getMokka() if not self.clip.testInputData: whma = self.getWhizard(2) res = jobma.append(whma) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() moma.getInputFromApp(whma) else: jobma.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep") moma.setNumberOfEvents(1) res = jobma.append(moma) if not res['OK']: self.log.error("Failed adding Mokka:", res['Message']) return S_ERROR() elif self.clip.testInputData: jobma.setInputData(self.marlinInputdata) else: self.log.error("Marlin does not know where to get its input from") return S_ERROR() if self.runOverlay: ov = self.getOverlay(2) res = jobma.append(ov) if not res["OK"]: self.log.error("Failed adding Overlay:", res['Message']) return S_ERROR ma = self.getMarlin() if self.clip.testChain: ma.getInputFromApp(moma) else: ma.setNumberOfEvents(2) res = jobma.append(ma) if not res['OK']: self.log.error("Failed adding Marlin:", res['Message']) return S_ERROR() self.jobList['Marlin1'] =jobma return S_OK(jobma) def createLCSimTest(self): """create tests for LCSIM""" self.log.notice( "Creating test for LCSIM" ) #run ((whiz+SLIC+)+Overlay+)LCSIM joblcsim = self.getJob() if self.clip.testChain: mysliclcsim = self.getSLIC() if not self.clip.testInputData: whlcsim = self.getWhizard(2) res = joblcsim.append(whlcsim) if not res["OK"]: self.log.error("Failed adding Whizard:", res['Value']) return S_ERROR() mysliclcsim.getInputFromApp(whlcsim) else: joblcsim.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep") mysliclcsim.setNumberOfEvents(2) res = joblcsim.append(mysliclcsim) if not res['OK']: self.log.error("Failed adding slic: ", res["Message"]) return S_ERROR() elif self.clip.testInputData: #joblcsim.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio") joblcsim.setInputData("/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio") else: self.log.error("LCSIM does not know where to get its input from") return S_ERROR() if self.runOverlay: ovlcsim = self.getOverlay(2) res = joblcsim.append(ovlcsim) if not res["OK"]: self.log.error("Failed adding Overlay:", res['Message']) return S_ERROR() mylcsim = self.getLCSIM(True) if self.clip.testChain: mylcsim.getInputFromApp(mysliclcsim) else: mylcsim.setNumberOfEvents(2) res = joblcsim.append(mylcsim) if not res['OK']: self.log.error("Failed adding LCSIM: ", res["Message"]) return S_ERROR() self.jobList['lcsim1'] = joblcsim return S_OK(joblcsim) def createSlicPandoraTest(self): """create tests for slicPandora""" self.log.notice("Creating tests for SLICPandora") #run ((whiz+SLIC) + (Overlay +) LCSIM +) SLICPandora + LCSIM joblcsimov = self.getJob() if not self.clip.testChain: self.log.error("SLICPandora does not know where to get its input from") return S_ERROR() mylcsimov = self.getLCSIM(True) if not self.clip.testInputData: whlcsimov = self.getWhizard(2) res = joblcsimov.append(whlcsimov) if not res["OK"]: self.log.error("Failed adding Whizard:", res['Value']) return S_ERROR() mysliclcsimov = self.getSLIC() mysliclcsimov.getInputFromApp(whlcsimov) res = joblcsimov.append(mysliclcsimov) if not res['OK']: self.log.error("Failed adding slic: ", res["Message"]) return S_ERROR() mylcsimov.getInputFromApp(mysliclcsimov) else: #joblcsimov.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio") joblcsimov.setInputData("/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio") mylcsimov.setNumberOfEvents(2) if self.runOverlay: ovslicp = self.getOverlay(2) res = joblcsimov.append(ovslicp) if not res["OK"]: self.log.error("Failed adding Overlay:", res['Message']) return S_ERROR() res = joblcsimov.append(mylcsimov) if not res['OK']: self.log.error("Failed adding LCSIM: ", res["Message"]) return S_ERROR() myslicpov = self.getSLICPandora() myslicpov.getInputFromApp(mylcsimov) res = joblcsimov.append(myslicpov) if not res['OK']: self.log.error("Failed adding SLICPandora: ", res["Message"]) return S_ERROR() mylcsimovp = self.getLCSIM(False) mylcsimovp.getInputFromApp(myslicpov) res = joblcsimov.append(mylcsimovp) if not res['OK']: self.log.error("Failed adding LCSIM: ", res["Message"]) return S_ERROR() self.jobList['lcsimov1'] = joblcsimov return S_OK(joblcsimov) def createUtilityTests(self): """Create tests for utility applications""" self.log.notice("Creating tests for utility applications") jobwsplit = self.getJob() whsplit = self.getWhizard(10) res = jobwsplit.append(whsplit) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() mystdsplit = JobCreater.getStdhepSplit() mystdsplit.getInputFromApp(whsplit) res = jobwsplit.append(mystdsplit) if not res['OK']: self.log.error("Failed adding StdHepSplit:", res['Message']) return S_ERROR() self.jobList['whizSplit'] = jobwsplit ##### WhizardJob + split jobwcut = self.getJob() whcut = self.getWhizard(100) res = jobwcut.append(whcut) if not res['OK']: self.log.error("Failed adding Whizard:", res['Message']) return S_ERROR() mystdcut = self.getStdhepcut( 100 ) mystdcut.getInputFromApp(whcut) res = jobwcut.append(mystdcut) if not res['OK']: self.log.error("Failed adding StdHepCut:", res['Message']) return S_ERROR() self.jobList['whizCut'] = jobwcut #LCIO split joblciosplit = self.getJob() # joblciosplit.setInputData("/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio") joblciosplit.setInputData("/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio") mylciosplit = self.getLCIOSplit(100) res = joblciosplit.append(mylciosplit) if not res['OK']: self.log.error("Failed adding SLCIOSplit:", res['Message']) return S_ERROR() self.jobList['lcioSplit'] = joblciosplit #LCIO concat jobconcat = self.getJob() # jobconcat.setInputData(["/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_27.slcio", # "/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio"]) jobconcat.setInputData(["/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2104.slcio", "/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2105.slcio"]) myconcat = self.getLCIOConcat() res = jobconcat.append(myconcat) if not res['OK']: self.log.error("Failed adding SLCIOConcatenate:", res['Message']) return S_ERROR() self.jobList['concat'] = jobconcat return S_OK((jobconcat, joblciosplit,jobwcut,jobwsplit)) def createFccSwTest(self): """create tests for fccsw""" self.log.notice("Creating jobs for FccSW") #### FccSwJob jobfccsw = self.getJob() fccsw = self.getFccSw() res = jobfccsw.append(fccsw) if not res['OK']: self.log.error("Failed adding FccSw:", res['Message']) return S_ERROR() self.jobList['FccSw1'] = jobfccsw return S_OK(jobfccsw) def createFccAnalysisTest(self): """create tests for fccanalysis""" self.log.notice("Creating jobs for FccAnalysis") #### FccAnalysisJob jobfccanalysis = self.getJob() fccanalysis = self.getFccAnalysis() res = jobfccanalysis.append(fccanalysis) if not res['OK']: self.log.error("Failed adding FccAnalysis:", res['Message']) return S_ERROR() self.jobList['FccAnalysis1'] = jobfccanalysis return S_OK(jobfccanalysis) def runJobLocally(self, job, jobName="unknown"): """run a job locally""" self.log.notice("I will run the tests locally.") from DIRAC import gConfig localarea = gConfig.getValue("/LocalSite/LocalArea", "") if not localarea: self.log.error("You need to have /LocalSite/LocalArea defined in your dirac.cfg") return S_ERROR() if localarea.find("/afs") == 0: self.log.error("Don't set /LocalSite/LocalArea set to /afs/... as you'll get to install there") self.log.error("check ${HOME}/.dirac.cfg and ${DIRAC}/etc/dirac.cfg") return S_ERROR() self.log.notice("To run locally, I will create a temp directory here.") curdir = os.getcwd() tmpdir = tempfile.mkdtemp("", dir = "./") os.chdir(tmpdir) # Jobs that need separate input files specialJobs = ['root', 'ddsim'] filesForJob = { 'root' : [ 'input2.root', 'input.root' ], 'ddsim' : [ 'FCalTB.tar.gz', 'Muon_50GeV_Fixed_cosTheta0.7.stdhep' ] } for specialName in specialJobs: if "root" in jobName.lower() and specialName == "root": with open("root.sh", "w") as rScript: rScript.write( "echo $ROOTSYS" ) with open("func.C", "w") as rMacro: rMacro.write( ''' void func( TString string ) { std::cout << string << std::endl; TFile* file = TFile::Open(string); file->ls(); } ''' ) testfiledir = 'Testfiles' for fileName in ['input.root', 'input2.root']: shutil.copy( os.path.join( curdir, testfiledir, fileName ), os.getcwd() ) print(os.path.join(curdir, "input2.root"), os.getcwd()) if specialName in jobName.lower(): for fileName in filesForJob[specialName]: shutil.copy( os.path.join( curdir, testfiledir, fileName ), os.getcwd() ) resJob = self.runJob(job, jobName) os.chdir(curdir) if not resJob['OK']: return resJob os.chdir(curdir) if not self.clip.nocleanup: cleanup(tmpdir) return S_OK() def run(self): """submit and run all the tests in jobList""" res = S_ERROR() for name, finjob in self.jobList.iteritems(): if self.clip.submitMode == 'local': res = self.runJobLocally(finjob, name) else: res = self.runJob(finjob, name) return res def runJob(self, finjob, name): """runs or submits the job""" self.log.notice("############################################################") self.log.notice(" Running or submitting job: %s " % name) self.log.notice("\n\n") res = finjob.submit(self.diracInstance, mode = self.clip.submitMode) if not res["OK"]: self.log.error("Failed job:", res['Message']) return S_ERROR() return S_OK() def checkForTests(self): """check which tests to run""" if self.clip.testMokka: resMokka = self.createMokkaTest() if not resMokka['OK']: return S_ERROR() if self.clip.testWhizard: resWhiz = self.createWhizardTest() if not resWhiz['OK']: return S_ERROR() if self.clip.testSlic: resSlic = self.createSlicTest() if not resSlic['OK']: return S_ERROR() if self.clip.testMarlin: resMarlin = self.createMarlinTest() if not resMarlin['OK']: return S_ERROR() if self.clip.testLCSIM: resLCSim = self.createLCSimTest() if not resLCSim['OK']: return S_ERROR() if self.clip.testSlicPandora: resSP = self.createSlicPandoraTest() if not resSP['OK']: return S_ERROR() if self.clip.testUtilities: resUtil = self.createUtilityTests() if not resUtil['OK']: return S_ERROR() if self.clip.testRoot: resRoot = self.createRootScriptTest() if not resRoot['OK']: return S_ERROR() resRoot = self.createRootHaddTest() if not resRoot['OK']: return S_ERROR() resRoot = self.createRootMacroTest() if not resRoot['OK']: return S_ERROR() if self.clip.testFccSw: resFccSw = self.createFccSwTest() if not resFccSw['OK']: return S_ERROR() if self.clip.testFccAnalysis: resFccAnalysis = self.createFccAnalysisTest() if not resFccAnalysis['OK']: return S_ERROR() return S_OK()
class UserJob(Job): """ User job class. To be used by users, not for production. """ def __init__(self, script = None): super(UserJob, self).__init__( script ) self.type = 'User' self.diracinstance = None def submit(self, diracinstance = None, mode = "wms"): """ Submit call: when your job is defined, and all applications are set, you need to call this to add the job to DIRAC. """ res = self._addToWorkflow() if not res['OK']: return res self.oktosubmit = True if not diracinstance: self.diracinstance = DiracILC() else: self.diracinstance = diracinstance return self.diracinstance.submit(self, mode) ############################################################################# def setInputData( self, lfns ): """Helper function. Specify input data by Logical File Name (LFN). Example usage: >>> job = Job() >>> job.setInputData(['/ilc/prod/whizard/processlist.whiz']) @param lfns: Logical File Names @type lfns: Single LFN string or list of LFNs """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): lfns[i] = lfns[i].replace( 'LFN:', '' ) inputData = map( lambda x: 'LFN:' + x, lfns ) inputDataStr = string.join( inputData, ';' ) description = 'List of input data specified by LFNs' self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description ) elif type( lfns ) == type( ' ' ): #single LFN description = 'Input data specified by LFN' self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description ) else: kwargs = {'lfns':lfns} return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs ) return S_OK() def setInputSandbox(self, flist): """ Mostly inherited from DIRAC.Job """ if type(flist) == type(""): flist = [flist] if not type(flist) == type([]) : return self._reportError("File passed must be either single file or list of files.") self.inputsandbox.extend(flist) return S_OK() ############################################################################# def setOutputData(self, lfns, OutputPath = '', OutputSE = ['']): """Helper function, used in preference to Job.setOutputData() for ILC. For specifying output data to be registered in Grid storage. If a list of OutputSEs are specified the job wrapper will try each in turn until successful. Example usage: >>> job = Job() >>> job.setOutputData(['Ntuple.root']) @param lfns: Output data file or files @type lfns: Single string or list of strings ['',''] @param OutputSE: Optional parameter to specify the Storage @param OutputPath: Optional parameter to specify the Path in the Storage, postpented to /ilc/user/u/username/ Element to store data or files, e.g. CERN-tape @type OutputSE: string or list @type OutputPath: string """ kwargs = {'lfns' : lfns, 'OutputSE' : OutputSE, 'OutputPath' : OutputPath} if type(lfns) == list and len(lfns): outputDataStr = string.join(lfns, ';') description = 'List of output data files' self._addParameter(self.workflow, 'UserOutputData', 'JDL', outputDataStr, description) elif type(lfns) == type(" "): description = 'Output data file' self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns, description) else: return self._reportError('Expected file name string or list of file names for output data', **kwargs) if OutputSE: description = 'User specified Output SE' if type(OutputSE) in types.StringTypes: OutputSE = [OutputSE] elif type(OutputSE) != types.ListType: return self._reportError('Expected string or list for OutputSE', **kwargs) OutputSE = ';'.join(OutputSE) self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE, description) if OutputPath: description = 'User specified Output Path' if not type(OutputPath) in types.StringTypes: return self._reportError('Expected string for OutputPath', **kwargs) # Remove leading "/" that might cause problems with os.path.join while OutputPath[0] == '/': OutputPath = OutputPath[1:] if OutputPath.count("ilc/user"): return self._reportError('Output path contains /ilc/user/ which is not what you want', **kwargs) self._addParameter(self.workflow, 'UserOutputPath', 'JDL', OutputPath, description) return S_OK() ############################################################################# def setOutputSandbox( self, files ): """Helper function. Specify output sandbox files. If specified files are over 10MB, these may be uploaded to Grid storage with a notification returned in the output sandbox. Example usage: >>> job = Job() >>> job.setOutputSandbox(['*.log','myfile.slcio']) @param files: Output sandbox files @type files: Single string or list of strings ['',''] """ if type( files ) == list and len( files ): fileList = string.join( files, ";" ) description = 'Output sandbox file list' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description ) elif type( files ) == type( " " ): description = 'Output sandbox file' self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description ) else: kwargs = {'files' : files} return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs ) return S_OK() def setILDConfig(self,Version): """ Define the Configuration package to obtain """ appName = 'ILDConfig' self._addSoftware(appName.lower(), Version) self._addParameter( self.workflow, 'ILDConfigPackage', 'JDL', appName+Version, 'ILDConfig package' ) return S_OK()
def setRepo(self, optionVal): self.repo = optionVal return S_OK() def registerSwitches(self): Script.registerSwitch('r:', 'repository=', 'Path to repository file', self.setRepo) Script.setUsageMessage('\n'.join( [ __doc__.split( '\n' )[1], '\nUsage:', ' %s [option|cfgfile] ...\n' % Script.scriptName ] ) ) if __name__=="__main": cliparams = Params() cliparams.registerSwitches() Script.parseCommandLine( ignoreErrors = False ) from DIRAC import gLogger repoLocation = cliparams.repo if not repoLocation: Script.showHelp() dexit(2) from ILCDIRAC.Interfaces.API.DiracILC import DiracILC dirac = DiracILC(True, repoLocation) dirac.monitorRepository(False) lfns = [] lfns = dirac.retrieveRepositoryOutputDataLFNs() gLogger.notice("lfnlist=[") for lfn in lfns : gLogger.notice('"LFN:%s",' % lfn) gLogger.notice("]") dexit(0)
from DIRAC.Core.Base import Script Script.parseCommandLine() from ILCDIRAC.Interfaces.API.DiracILC import DiracILC from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient import time dIlc = DiracILC() lcoutput = [] for i in range(1, 301): lcoutput = "aa_%d.root" % i job = UserJob() job.setDestination('LCG.CERN.ch') job.setInputSandbox([ "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraLikelihoodData9EBin_CLIC_ILD.xml", "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraSettingsFast.xml", "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/steering.xml", "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/clic_ild_cdr.gear", "LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/lib.tar.gz" ]) job.setInputData( "LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" % i) job.setOutputSandbox( ["*.log", "*.sh", "*.py", "*.out", "*.xml", "*.steer "]) job.setJobGroup("myRoot") job.setName("root_aa_%d" % i) marl = Marlin() marl.setVersion('ILCSoft-2016-09-27_gcc48')
class UserJob(Job): """ User job class. To be used by users, not for production. """ def __init__(self, script=None): super(UserJob, self).__init__(script) self.type = 'User' self.diracinstance = None self.usergroup = ['ilc_user', 'calice_user'] self.proxyinfo = getProxyInfo() ########## SPLITTING STUFF: ATTRIBUTES ########## self._data = [] self.splittingOption = None self._switch = {} self.numberOfJobs = None self.totalNumberOfEvents = None self.eventsPerJob = None self.numberOfFilesPerJob = 1 def submit(self, diracinstance=None, mode="wms"): """ Submit call: when your job is defined, and all applications are set, you need to call this to add the job to DIRAC. :param diracinstance: DiracILC instance :type diracinstance: ~ILCDIRAC.Interfaces.API.DiracILC.DiracILC :param str mode: "wms" (default), "agent", or "local" .. note :: The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts """ if self.splittingOption: result = self._split() if 'OK' in result and not result['OK']: return result #Check the credentials. If no proxy or not user proxy, return an error if not self.proxyinfo['OK']: self.log.error( "Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError( "Not allowed to submit a job, you need a %s proxy." % self.usergroup, self.__class__.__name__) if 'group' in self.proxyinfo['Value']: group = self.proxyinfo['Value']['group'] if group not in self.usergroup: self.log.error( "Not allowed to submit a job, you need a %s proxy." % self.usergroup) return self._reportError( "Not allowed to submit job, you need a %s proxy." % self.usergroup, self.__class__.__name__) else: self.log.error( "Could not determine group, you do not have the right proxy.") return self._reportError( "Could not determine group, you do not have the right proxy.") res = self._addToWorkflow() if not res['OK']: return res self.oktosubmit = True if not diracinstance: self.diracinstance = DiracILC() else: self.diracinstance = diracinstance return self.diracinstance.submit(self, mode) ############################################################################# def setInputData(self, lfns): """Specify input data by Logical File Name (LFN). Input files specified via this function will be automatically staged if necessary. Example usage: >>> job = UserJob() >>> job.setInputData(['/ilc/prod/whizard/processlist.whiz']) :param lfns: Logical File Names :type lfns: Single LFN string or list of LFNs """ if isinstance(lfns, list) and lfns: for i, lfn in enumerate(lfns): lfns[i] = lfn.replace('LFN:', '') #inputData = map( lambda x: 'LFN:' + x, lfns ) inputData = lfns #because we don't need the LFN: for inputData, and it breaks the #resolution of the metadata in the InputFilesUtilities inputDataStr = ';'.join(inputData) description = 'List of input data specified by LFNs' self._addParameter(self.workflow, 'InputData', 'JDL', inputDataStr, description) elif isinstance(lfns, basestring): #single LFN description = 'Input data specified by LFN' self._addParameter(self.workflow, 'InputData', 'JDL', lfns, description) else: kwargs = {'lfns': lfns} return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs) return S_OK() def setInputSandbox(self, flist): """ Add files to the input sandbox, can be on the local machine or on the grid >>> job = UserJob() >>> job.setInputSandbox( ['LFN:/ilc/user/u/username/libraries.tar.gz', >>> 'mySteeringFile.xml'] ) :param flist: Files for the inputsandbox :type flist: `python:list` or `str` """ if isinstance(flist, basestring): flist = [flist] if not isinstance(flist, list): return self._reportError( "File passed must be either single file or list of files.") self.inputsandbox.extend(flist) return S_OK() ############################################################################# def setOutputData(self, lfns, OutputPath='', OutputSE=''): """For specifying output data to be registered in Grid storage. If a list of OutputSEs are specified the job wrapper will try each in turn until successful. Example usage: >>> job = UserJob() >>> job.setOutputData(['Ntuple.root']) :param lfns: Output data file or files :type lfns: Single `str` or `python:list` of strings ['',''] :param str OutputPath: Optional parameter to specify the Path in the Storage, postpended to /ilc/user/u/username/ :param OutputSE: Optional parameter to specify the Storage Element to store data or files, e.g. CERN-SRM :type OutputSE: `python:list` or `str` """ kwargs = {'lfns': lfns, 'OutputSE': OutputSE, 'OutputPath': OutputPath} if isinstance(lfns, list) and lfns: outputDataStr = ';'.join(lfns) description = 'List of output data files' self._addParameter(self.workflow, 'UserOutputData', 'JDL', outputDataStr, description) elif isinstance(lfns, basestring): description = 'Output data file' self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns, description) else: return self._reportError( 'Expected file name string or list of file names for output data', **kwargs) if OutputSE: description = 'User specified Output SE' if isinstance(OutputSE, basestring): OutputSE = [OutputSE] elif not isinstance(OutputSE, list): return self._reportError( 'Expected string or list for OutputSE', **kwargs) OutputSE = ';'.join(OutputSE) self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE, description) if OutputPath: description = 'User specified Output Path' if not isinstance(OutputPath, basestring): return self._reportError('Expected string for OutputPath', **kwargs) # Remove leading "/" that might cause problems with os.path.join while OutputPath[0] == '/': OutputPath = OutputPath[1:] if OutputPath.count("ilc/user"): return self._reportError( 'Output path contains /ilc/user/ which is not what you want', **kwargs) self._addParameter(self.workflow, 'UserOutputPath', 'JDL', OutputPath, description) return S_OK() ############################################################################# def setOutputSandbox(self, files): """Specify output sandbox files. If specified files are over 10MB, these may be uploaded to Grid storage with a notification returned in the output sandbox. .. Note :: Sandbox files are removed after 2 weeks. Example usage: >>> job = UserJob() >>> job.setOutputSandbox(['*.log','*.sh', 'myfile.txt']) Use the output sandbox only for small files. Larger files should be stored on the grid and downloaded later if necessary. See :func:`setOutputData` :param files: Output sandbox files :type files: Single `str` or `python:list` of strings ['',''] """ if isinstance(files, list) and files: fileList = ";".join(files) description = 'Output sandbox file list' self._addParameter(self.workflow, 'OutputSandbox', 'JDL', fileList, description) elif isinstance(files, basestring): description = 'Output sandbox file' self._addParameter(self.workflow, 'OutputSandbox', 'JDL', files, description) else: kwargs = {'files': files} return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs) return S_OK() def setILDConfig(self, version): """ Define the Configuration package to obtain """ appName = 'ILDConfig' self._addSoftware(appName.lower(), version) self._addParameter(self.workflow, 'ILDConfigPackage', 'JDL', appName + version, 'ILDConfig package') return S_OK() def setCLICConfig(self, version): """Define the CLIC Configuration package to obtain, copies steering files from CLIC Configuration folder to working directory :param str version: version string, e.g.: 'ILCSoft-2017-07-27' """ appName = 'ClicConfig' self._addSoftware(appName.lower(), version) self._addParameter(self.workflow, 'ClicConfigPackage', 'JDL', appName + version, 'CLIC Config package') return S_OK() ############################## SPLITTING STUFF: METHODS ################################ # Some methods have been added : # # * _atomicSubmission # * _checkJobConsistency # * setSplitEvents # * setSplitInputData # * setSplitDoNotAlterOutputFilename # * _split # * _splitByData # * _splitByEvents # * _toInt # # Given the type of splitting (Events or Data), these methods compute # the right parameters of the method 'Job.setParameterSequence()' ########################################################################################## def setSplitEvents(self, eventsPerJob=None, numberOfJobs=None, totalNumberOfEvents=None): """This function sets split parameters for doing splitting over events Example usage: >>> job = UserJob() >>> job.setSplitEvents( numberOfJobs=42, totalNumberOfEvents=126 ) Exactly two of the parmeters should be set :param int eventsPerJob: The events processed by a single job :param int numberOfJobs: The number of jobs :param int totalNumberOfEvents: The total number of events processed by all jobs """ self.totalNumberOfEvents = totalNumberOfEvents self.eventsPerJob = eventsPerJob self.numberOfJobs = numberOfJobs self._addParameter(self.workflow, 'NbOfEvts', 'JDL', -1, 'Number of Events') self.splittingOption = "byEvents" def setSplitInputData(self, lfns, numberOfFilesPerJob=1): """sets split parameters for doing splitting over input data Example usage: >>> job = UserJob() >>> job.setSplitInputData( listOfLFNs ) :param lfns: Logical File Names :type lfns: list of LFNs :param int numberOfFilesPerJob: The number of input data processed by a single job """ self._data = lfns if isinstance(lfns, list) else [lfns] self.numberOfFilesPerJob = numberOfFilesPerJob self.splittingOption = "byData" def setSplitDoNotAlterOutputFilename(self, value=True): """if this option is set the output data lfns will _not_ include the JobIndex :param bool value: if *True* disable the changing of the output data filenames. If *False* the JobIndex will be added at the end of OutputData LFNs before the extension. Or replace '%n' with the jobIndex in the fileName. See :func:`Core.Utilities.Splitting.addJobIndexToFilename` """ self._addParameter(self.workflow, 'DoNotAlterOutputData', 'JDL', value, 'Do Not Change Output Data') def _split(self): """checks the consistency of the job and call the right split method. :return: The success or the failure of the consistency checking :rtype: DIRAC.S_OK, DIRAC.S_ERROR """ self.eventsPerJob = self._toInt(self.eventsPerJob) self.numberOfJobs = self._toInt(self.numberOfJobs) if self.numberOfJobs is False or self.eventsPerJob is False: return self._reportError("Splitting: Invalid values for splitting") # FIXME: move somewhere more prominent self._switch = { "byEvents": self._splitByEvents, "byData": self._splitByData, None: self._atomicSubmission, } self.log.info("Job splitting...") if not self._checkJobConsistency(): errorMessage = "Job._checkJobConsistency() failed" self.log.error(errorMessage) return self._reportError(errorMessage) sequence = self._switch[self.splittingOption]() if not sequence: errorMessage = "Job._splitBySomething() failed" self.log.error(errorMessage) return self._reportError(errorMessage) sequenceType, sequenceList, addToWorkflow = sequence[0], sequence[ 1], sequence[2] if sequenceType != "Atomic": self.setParameterSequence(sequenceType, sequenceList, addToWorkflow) self.setParameterSequence('JobIndexList', range(len(sequenceList)), addToWorkflow='JobIndex') self._addParameter(self.workflow, 'JobIndex', 'int', 0, 'JobIndex') self.log.info("Job splitting successful") return S_OK() ############################################################################# def _atomicSubmission(self): """called when no splitting is necessary, do not return valid parameters fot setParameterSequence(). :return: parameter name and parameter values for setParameterSequence(), addToWorkflow flag :rtype: tuple of (str, list, bool/str) """ self.log.verbose( "Job splitting: No splitting to apply, 'atomic submission' will be used" ) return "Atomic", [], False ############################################################################# def _checkJobConsistency(self): """checks if Job parameters are valid. :return: The success or the failure of the consistency checking :rtype: bool :Example: >>> self._checkJobConsistency() """ self.log.info("Job consistency: _checkJobConsistency()...") if self.splittingOption not in self._switch: splitOptions = ",".join(self._switch.keys()) errorMessage = "checkJobConsistency failed: Bad split value: possible values are %s" % splitOptions self.log.error(errorMessage) return False # All applications should have the same number of events # We can get this number from the first application for example sameNumberOfEvents = next(iter(self.applicationlist)).numberOfEvents if not all(app.numberOfEvents == sameNumberOfEvents for app in self.applicationlist): self.log.warn( "Job: Applications should all have the same number of events") if (self.totalNumberOfEvents == -1 or sameNumberOfEvents == -1) and not self._data: self.log.warn( "Job: Number of events is -1 without input data. Was that intentional?" ) self.log.info("job._checkJobConsistency successful") return True ############################################################################# def _splitByData(self): """a job is submitted per input data. :return: parameter name and parameter values for setParameterSequence() :rtype: tuple of (str, list, bool/str) """ # reset split attribute to avoid infinite loop self.splittingOption = None self.log.info("Job splitting: Splitting 'byData' method...") # Ensure that data have been specified by setInputData() method if not self._data: errorMessage = "Job splitting: missing input data" self.log.error(errorMessage) return False if self.numberOfFilesPerJob > len(self._data): errorMessage = "Job splitting: 'numberOfFilesPerJob' must be less/equal than the number of input data" self.log.error(errorMessage) return False self._data = breakListIntoChunks(self._data, self.numberOfFilesPerJob) self.log.info("Job splitting: submission consists of %d job(s)" % len(self._data)) return ["InputData", self._data, 'ParametricInputData'] ############################################################################# def _splitByEvents(self): """a job is submitted per subset of events. :return: parameter name and parameter values for setParameterSequence() :rtype: tuple of (str, list, bool/str) """ # reset split attribute to avoid infinite loop self.splittingOption = None self.log.info("Job splitting: splitting 'byEvents' method...") if self.eventsPerJob and self.numberOfJobs: # 1st case: (numberOfJobs=3, eventsPerJob=10) # trivial case => each job (total of 3) run applications of 10 events each self.log.debug("Job splitting: events per job and number of jobs") mapEventJob = [self.eventsPerJob] * self.numberOfJobs elif self.eventsPerJob and self.totalNumberOfEvents: # 2nd case: (split="byEvents", eventsPerJob=10, totalNumberOfEvents=10) # Given the number of events per job and total of number of event we want, # we can compute the unknown which is the number of jobs. self.log.debug( "Job splitting: Events per job and total number of events") if self.eventsPerJob > self.totalNumberOfEvents: self.log.error( "Job splitting: The number of events per job has to be lower than or equal to the total number of events" ) return False numberOfJobsIntDiv = self.totalNumberOfEvents / self.eventsPerJob numberOfJobsRest = self.totalNumberOfEvents % self.eventsPerJob mapEventJob = [self.eventsPerJob] * numberOfJobsIntDiv mapEventJob += [numberOfJobsRest] if numberOfJobsRest != 0 else [] else: # 3rd case: (split='byEvents', njobs=10, totalNumberOfEvents=10) # Then compute the right number of events per job self.log.debug( "Job splitting: The number of jobs and the total number of events" ) if (not self.totalNumberOfEvents) or (self.totalNumberOfEvents < self.numberOfJobs): self.log.error( "Job splitting: The number of events has to be greater than or equal to the number of jobs" ) return False eventPerJobIntDiv = self.totalNumberOfEvents / self.numberOfJobs eventPerJobRest = self.totalNumberOfEvents % self.numberOfJobs mapEventJob = [eventPerJobIntDiv] * self.numberOfJobs if eventPerJobRest != 0: for suplement in xrange(eventPerJobRest): mapEventJob[suplement] += 1 self.log.debug("Job splitting: events over the jobs: %s" % mapEventJob) self.log.info("Job splitting: submission consists of %d job(s)" % len(mapEventJob)) return ['NumberOfEvents', mapEventJob, 'NbOfEvts'] ############################################################################# def _toInt(self, number): """casts number parameter to an integer. It also accepts 'string integer' parameter. :param number: the number to cast (number of events, number of jobs) :type number: str or int :return: The success or the failure of the casting :rtype: bool, int or None :Example: >>> number = self._toInt("1000") """ if number is None: return number try: number = int(number) if number <= 0: raise ValueError except ValueError: self.log.error( "Job splitting: arguments must be positive integers") return False return number
def __init__(self, *args, **kwargs): AgentModule.__init__(self, *args, **kwargs) self.name = 'DataRecoveryAgent' self.enabled = False self.productionsToIgnore = self.am_getOption("ProductionsToIgnore", []) self.transformationTypes = self.am_getOption("TransformationTypes", [ 'MCReconstruction', 'MCSimulation', 'MCReconstruction_Overlay', 'MCGeneration' ]) self.transformationStatus = self.am_getOption("TransformationStatus", ['Active', 'Completing']) self.shifterProxy = self.am_setOption('shifterProxy', 'DataManager') self.jobStatus = [ 'Failed', 'Done' ] ##This needs to be both otherwise we cannot account for all cases self.jobMon = JobMonitoringClient() self.fcClient = FileCatalogClient() self.tClient = TransformationClient() self.reqClient = ReqClient() self.diracILC = DiracILC() self.inputFilesProcessed = set() self.todo = {'MCGeneration': [ dict( Message="MCGeneration: OutputExists: Job 'Done'", ShortMessage="MCGeneration: job 'Done' ", Counter=0, Check=lambda job: job.allFilesExist() and job.status=='Failed', Actions=lambda job,tInfo: [ job.setJobDone(tInfo) ] ), dict( Message="MCGeneration: OutputMissing: Job 'Failed'", ShortMessage="MCGeneration: job 'Failed' ", Counter=0, Check=lambda job: job.allFilesMissing() and job.status=='Done', Actions=lambda job,tInfo: [ job.setJobFailed(tInfo) ] ), # dict( Message="MCGeneration, job 'Done': OutputExists: Task 'Done'", # ShortMessage="MCGeneration: job already 'Done' ", # Counter=0, # Check=lambda job: job.allFilesExist() and job.status=='Done', # Actions=lambda job,tInfo: [ tInfo._TransformationInfo__setTaskStatus(job, 'Done') ] # ), ], 'OtherProductions': [ \ ## should always be first! dict( Message="One of many Successful: clean others", ShortMessage="Other Tasks --> Keep", Counter=0, Check=lambda job: job.allFilesExist() and job.otherTasks and job.inputFile not in self.inputFilesProcessed, Actions=lambda job,tInfo: [ self.inputFilesProcessed.add(job.inputFile), job.setJobDone(tInfo), job.setInputProcessed(tInfo) ] ), dict( Message="Other Task processed Input, no Output: Fail", ShortMessage="Other Tasks --> Fail", Counter=0, Check=lambda job: job.inputFile in self.inputFilesProcessed and job.allFilesMissing() and job.status!='Failed', Actions=lambda job,tInfo: [ job.setJobFailed(tInfo) ] ), dict( Message="Other Task processed Input: Fail and clean", ShortMessage="Other Tasks --> Cleanup", Counter=0, Check=lambda job: job.inputFile in self.inputFilesProcessed and not job.allFilesMissing(), Actions=lambda job,tInfo: [ job.setJobFailed(tInfo), job.cleanOutputs(tInfo) ] ), dict( Message="InputFile missing: mark job 'Failed', mark input 'Deleted', clean", ShortMessage="Input Missing --> Job 'Failed, Input 'Deleted', Cleanup", Counter=0, Check=lambda job: job.inputFile and not job.inputFileExists and job.fileStatus != "Deleted", Actions=lambda job,tInfo: [ job.cleanOutputs(tInfo), job.setJobFailed(tInfo), job.setInputDeleted(tInfo) ] ), dict( Message="InputFile Deleted, output Exists: mark job 'Failed', clean", ShortMessage="Input Deleted --> Job 'Failed, Cleanup", Counter=0, Check=lambda job: job.inputFile and not job.inputFileExists and job.fileStatus == "Deleted" and not job.allFilesMissing(), Actions=lambda job,tInfo: [ job.cleanOutputs(tInfo), job.setJobFailed(tInfo) ] ), ## All Output Exists dict( Message="Output Exists, job Failed, input not Processed --> Job Done, Input Processed", ShortMessage="Output Exists --> Job Done, Input Processed", Counter=0, Check=lambda job: job.allFilesExist() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus!="Processed" and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setJobDone(tInfo), job.setInputProcessed(tInfo) ] ), dict( Message="Output Exists, job Failed, input Processed --> Job Done", ShortMessage="Output Exists --> Job Done", Counter=0, Check=lambda job: job.allFilesExist() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus=="Processed" and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setJobDone(tInfo) ] ), dict( Message="Output Exists, job Done, input not Processed --> Input Processed", ShortMessage="Output Exists --> Input Processed", Counter=0, Check=lambda job: job.allFilesExist() and \ not job.otherTasks and \ job.status=='Done' and \ job.fileStatus!="Processed" and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setInputProcessed(tInfo) ] ), ## outputmissing dict( Message="Output Missing, job Failed, input Assigned, MaxError --> Input MaxReset", ShortMessage="Max ErrorCount --> Input MaxReset", Counter=0, Check=lambda job: job.allFilesMissing() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFile not in self.inputFilesProcessed and \ job.inputFileExists and \ job.errorCount > MAXRESET, Actions=lambda job,tInfo: [ job.setInputMaxReset(tInfo) ] ), dict( Message="Output Missing, job Failed, input Assigned --> Input Unused", ShortMessage="Output Missing --> Input Unused", Counter=0, Check=lambda job: job.allFilesMissing() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFile not in self.inputFilesProcessed and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setInputUnused(tInfo) ] ), dict( Message="Output Missing, job Done, input Assigned --> Job Failed, Input Unused", ShortMessage="Output Missing --> Job Failed, Input Unused", Counter=0, Check=lambda job: job.allFilesMissing() and \ not job.otherTasks and \ job.status=='Done' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFile not in self.inputFilesProcessed and \ job.inputFileExists, Actions=lambda job,tInfo: [ job.setInputUnused(tInfo), job.setJobFailed(tInfo) ] ), ## some files missing, needing cleanup. Only checking for ## assigned, because processed could mean an earlier job was ## succesful and this one is just the duplicate that needed ## to be removed! But we check for other tasks earlier, so ## this should not happen dict( Message="Some missing, job Failed, input Assigned --> cleanup, Input 'Unused'", ShortMessage="Output Missing --> Cleanup, Input Unused", Counter=0, Check=lambda job: job.someFilesMissing() and \ not job.otherTasks and \ job.status=='Failed' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFileExists, Actions=lambda job,tInfo: [job.cleanOutputs(tInfo),job.setInputUnused(tInfo)] #Actions=lambda job,tInfo: [] ), dict( Message="Some missing, job Done, input Assigned --> cleanup, job Failed, Input 'Unused'", ShortMessage="Output Missing --> Cleanup, Job Failed, Input Unused", Counter=0, Check=lambda job: job.someFilesMissing() and \ not job.otherTasks and \ job.status=='Done' and \ job.fileStatus in ASSIGNEDSTATES and \ job.inputFileExists, Actions=lambda job,tInfo: [job.cleanOutputs(tInfo),job.setInputUnused(tInfo),job.setJobFailed(tInfo)] #Actions=lambda job,tInfo: [] ), dict( Message="Some missing, job Done --> job Failed", ShortMessage="Output Missing, Done --> Job Failed", Counter=0, Check=lambda job: not job.allFilesExist() and job.status=='Done', Actions=lambda job,tInfo: [job.setJobFailed(tInfo)] ), dict ( Message="Something Strange", ShortMessage="Strange", Counter=0, Check=lambda job: job.status not in ("Failed","Done"), Actions=lambda job,tInfo: [] ), ##should always be the last one! dict ( Message="Failed Hard", ShortMessage="Failed Hard", Counter=0, Check=lambda job: False, ## never Actions=lambda job,tInfo: [] ), ] } self.jobCache = defaultdict(lambda: (0, 0)) self.printEveryNJobs = self.am_getOption('PrintEvery', 200) ##Notification self.notesToSend = "" self.addressTo = self.am_getOption('MailTo', ["*****@*****.**"]) self.addressFrom = self.am_getOption('MailFrom', "*****@*****.**") self.subject = "DataRecoveryAgent"
res = fc.findFilesByMetadata(meta) if not res['OK']: print res['Message'] lfns = res['Value'] #print "Found %s files" % len(lfns) filelist=[] for lfn in lfns: filelist.append(lfn) #print filelist #filelist2=filelist[0] #check if big radius in fastjetanalyzer indeed 0.7 or 1.0 jobGroup = "HZAnalyzer_190417_ee_qqqq_m_qqqq_2TeV_13696_VLC7PFOs" dirac = DiracILC(True,jobGroup+".rep") job = UserJob() #job.setExecutionEnv({'ROOT_INCLUDE_PATH':'./'}) job.setJobGroup(jobGroup) job.setOutputSandbox ( [ "*.log","*.out","*.py"] ) job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk']) #pay attention that the Zuds200 here is NOT changed job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190606/HZAnalyzerlib.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] ) job.setBannedSites(['LCG.INP3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk','LCG.Oxford.uk']) job.setSplitInputData(filelist, numberOfFilesPerJob=50) ma = Marlin() ma.setVersion('ILCSoft-2019-04-17_gcc62') #ma.setInputFile("LFN:/ilc/user/w/webermat/ddsimstdheptautau/ILC18-10-11_gcc62_CLIC_o3_v14/tautau200/ddsim_ILC181011_gcc62_tautau_200_CLIC_o3_v14_0%s.slcio"%(str(input_ind))) ma.setSteeringFile("/eos/user/w/weberma2/steeringFiles/testHZAnalyzer.xml") ma.setDetectorModel("CLIC_o3_v14") HZrootfilename2="HZStudy_ee_qqqq_m_qqqq_2TeV_13696_polm80_3TeV_wO_CLIC_o3_v14_DR7.root"
def main(argv): # Input arguments ildconfig_version = "$ILDCONFIGVER" ilcsoft_version = "$ILCSOFTVER" evts_per_run = $EVTSPERRUN detector_model = "$DETECTOR" sim_input = "$SIMINPUT" process_name = "$PROCESS" index = $IND sim_input = diracpath_from_pnfspath( sim_input ) sim_detector_model = detector_model_wo_option( detector_model ) job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model dirac = DiracILC(True,job_group+".rep") # outputs to be saved onto grid SE RECoutput = [] # DDSim evtStart = (index-1)*evts_per_run evtEnd = index*evts_per_run - 1 RandSeed = random.randrange(11623, 99999) lcinputSIM = "LFN:" + sim_input lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd)) sim = DDSim() sim.setVersion(ilcsoft_version) sim.setDetectorModel(sim_detector_model) sim.setInputFile(lcinputSIM) sim.setSteeringFile("ddsim_steer.py") sim.setNumberOfEvents(evts_per_run) sim.setRandomSeed(RandSeed) sim.setEnergy(1000) sim.setStartFrom(evtStart) sim.setOutputFile(lcoutputSIM) # Marlin lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd)) ma = Marlin() ma.setVersion(ilcsoft_version) ma.setDetectorModel(detector_model) ma.setSteeringFile("MarlinStdReco.xml") ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) ) ma.setLogFile("marlin.log") ma.getInputFromApp(sim) ma.setEnergy(1000) ma.setOutputDstFile(lcoutputDST) RECoutput.append(lcoutputDST) # ILCDirac user job job = UserJob() job.setName("user_sim_reco") job.setJobGroup(job_group) job.setILDConfig(ildconfig_version) job.setCPUTime(86400) tmp_file_name = process_name + "_sim_reco_job_tmp.py" job.setInputSandbox([tmp_file_name]) job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "]) job.dontPromptMe() job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us']) # run simulation job simres = job.append(sim) if not simres['OK']: print simres['Not ok appending ddsim to job'] quit() # run Malrin reco jobs mares = job.append(ma) if not mares['OK']: print mares['Not ok appending Marlin to job'] quit() job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM") print RECoutput submit_output = job.submit(dirac) print submit_output