def __init__(self, clip, params):
self.clip = clip
self.ildConfig = params.get("ildConfig", None)
self.alwaysOverlay = params.get("alwaysOverlay", False)
self.runOverlay = self.clip.testOverlay or self.alwaysOverlay
self.mokkaVersion = params["mokkaVersion"]
self.mokkaSteeringFile = params.get("mokkaSteeringFile")
self.detectorModel = params.get("detectorModel")
self.marlinVersion = params.get("marlinVersion")
self.marlinSteeringFile = params.get("marlinSteeringFile")
self.ddsimVersion = params.get("ddsimVersion")
self.ddsimDetectorModel = params.get("ddsimDetectorModel")
self.ddsimInputFile = params.get("ddsimInputFile")
self.marlinInputdata = params.get("marlinInputdata")
self.gearFile = params.get("gearFile")
self.lcsimVersion = params.get("lcsimVersion")
self.steeringFileVersion = params.get("steeringFileVersion", None)
self.rootVersion = params["rootVersion"]
self.whizard2Version = params.get("whizard2Version")
self.whizard2SinFile = params.get("whizard2SinFile")
self.energy = params.get("energy")
self.backgroundType = params.get("backgroundType")
self.machine = params.get("machine")
self.gearFile = params.get("gearFile")
self.marlinSteeringFile = params.get("marlinSteeringFile")
self.marlinVersion = params.get("marlinVersion")
self.lcsimPreSteeringFile = params.get("lcsimPreSteeringFile")
self.lcsimPostSteeringFile = params.get("lcsimPostSteeringFile")
self.fccSwPath = params.get("fccSwPath")
self.fccSwSteeringFile = params.get("fccSwSteeringFile")
self.fccAnalysisSteeringFile = params.get("fccAnalysisSteeringFile")
### other things needed to run tests
self.log = gLogger.getSubLogger("JobCreater")
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid
from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid
if self.clip.submitMode == "local":
self.log.notice("")
self.log.notice(" DIRAC RCSID:", drcsid)
self.log.notice(" Job RCSID:", jrcsid)
self.log.notice("Applications RCSID:", apprcsid)
self.log.notice("")
self.diracInstance = DiracILC(False, 'tests.rep')
self.jobList = {}
def submit(self, diracinstance=None, mode="wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
:param diracinstance: DiracILC instance
:type diracinstance: ~ILCDIRAC.Interfaces.API.DiracILC.DiracILC
:param str mode: "wms" (default), "agent", or "local"
.. note ::
The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts
"""
if self.splittingOption:
result = self._split()
if 'OK' in result and not result['OK']:
return result
#Check the credentials. If no proxy or not user proxy, return an error
if not self.proxyinfo['OK']:
self.log.error(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup)
return self._reportError(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup, self.__class__.__name__)
if 'group' in self.proxyinfo['Value']:
group = self.proxyinfo['Value']['group']
if group not in self.usergroup:
self.log.error(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup)
return self._reportError(
"Not allowed to submit job, you need a %s proxy." %
self.usergroup, self.__class__.__name__)
else:
self.log.error(
"Could not determine group, you do not have the right proxy.")
return self._reportError(
"Could not determine group, you do not have the right proxy.")
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submit(self, mode)
def submit(self, diracinstance = None, mode = "wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
:param diracinstance: :any:`DiracILC <ILCDIRAC.Interfaces.API.DiracILC.DiracILC>` instance
:param string mode: "wms" (default), "agent", or "local"
.. note ::
The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts
"""
#Check the credentials. If no proxy or not user proxy, return an error
if not self.proxyinfo['OK']:
self.log.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup)
return self._reportError("Not allowed to submit a job, you need a %s proxy." % self.usergroup,
self.__class__.__name__)
if self.proxyinfo['Value'].has_key('group'):
group = self.proxyinfo['Value']['group']
if not group in self.usergroup:
self.log.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup)
return self._reportError("Not allowed to submit job, you need a %s proxy." % self.usergroup,
self.__class__.__name__)
else:
self.log.error("Could not determine group, you do not have the right proxy.")
return self._reportError("Could not determine group, you do not have the right proxy.")
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submit(self, mode)
def __init__( self,
clip,
params
):
self.clip = clip
self.ildConfig = params.get( "ildConfig", None )
self.alwaysOverlay = params.get( "alwaysOverlay", False )
self.runOverlay = self.clip.testOverlay or self.alwaysOverlay
self.mokkaVersion = params["mokkaVersion"]
self.mokkaSteeringFile = params.get( "mokkaSteeringFile" )
self.detectorModel = params.get( "detectorModel" )
self.marlinVersion = params.get( "marlinVersion" )
self.marlinSteeringFile = params.get( "marlinSteeringFile" )
self.ddsimVersion = params.get( "ddsimVersion" )
self.ddsimDetectorModel = params.get( "ddsimDetectorModel")
self.ddsimInputFile = params.get( "ddsimInputFile" )
self.marlinInputdata = params.get ( "marlinInputdata" )
self.gearFile = params.get( "gearFile" )
self.lcsimVersion = params.get( "lcsimVersion" )
self.steeringFileVersion = params.get( "steeringFileVersion", None )
self.rootVersion = params["rootVersion"]
self.whizard2Version = params.get( "whizard2Version" )
self.whizard2SinFile = params.get( "whizard2SinFile" )
self.energy = params.get("energy")
self.backgroundType = params.get("backgroundType")
self.machine = params.get("machine")
self.gearFile = params.get( "gearFile" )
self.marlinSteeringFile = params.get( "marlinSteeringFile" )
self.marlinVersion = params.get( "marlinVersion" )
self.lcsimPreSteeringFile = params.get( "lcsimPreSteeringFile" )
self.lcsimPostSteeringFile = params.get( "lcsimPostSteeringFile" )
self.fccSwPath = params.get( "fccSwPath" )
self.fccSwSteeringFile = params.get( "fccSwSteeringFile" )
self.fccAnalysisSteeringFile = params.get( "fccAnalysisSteeringFile" )
### other things needed to run tests
self.log = gLogger.getSubLogger("JobCreater")
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid
from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid
if self.clip.submitMode == "local":
self.log.notice("")
self.log.notice(" DIRAC RCSID:", drcsid )
self.log.notice(" Job RCSID:", jrcsid )
self.log.notice("Applications RCSID:", apprcsid )
self.log.notice("")
self.diracInstance = DiracILC(False, 'tests.rep')
self.jobList = {}
def initialize(self):
self.pollingTime = self.am_getOption('PollingTime', 86400)
gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM)
self.ppc = ProcessProdClient()
self.dirac = DiracILC()
self.diracadmin = DiracAdmin()
self.am_setOption( 'shifterProxy', 'Admin' )
return S_OK()
def subDDSim():
# Decide parameters for a job
outputSE = "KEK-SRM"
outputSE = "KEK-DISK"
isLocal = _clip.isLocal
nbevts = 10 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
outputFile = "ddsim_example.slcio" if _clip.outputFile == "" else _clip.outputFile
outputDir = _clip.outputDir
inputFile = _clip.inputFile
if inputFile == "":
gLogger.error("Input file for ddsim does not given.")
exit(-1)
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myddsimjob")
job.setName("myddsim")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk"
]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
ddsim = DDSim()
ddsim.setVersion("ILCSoft-02-00-02_gcc49")
ddsim.setDetectorModel("ILD_l5_v02")
ddsim.setInputFile(inputFile)
ddsim.setNumberOfEvents(nbevts)
extraCLIArguments = " --steeringFile ddsim_steer.py "
extraCLIArguments += " --outputFile %s " % outputFile
extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 "
ddsim.setExtraCLIArguments(extraCLIArguments)
# ddsim.setRandomSeed(1234565)
# ddsim.setStartFrom(20) # Number of events to skip before starting ddsim
job.append(ddsim)
if outputDir != "":
job.setOutputData([outputFile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
def _createLFNList():
"""create the LFnList"""
cliparams = _Params()
cliparams.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
repoLocation = cliparams.repo
if not repoLocation:
Script.showHelp()
dexit(2)
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
dirac = DiracILC(True, repoLocation)
dirac.monitorRepository(False)
lfns = []
lfns = dirac.retrieveRepositoryOutputDataLFNs()
LOG.notice("lfnlist=[")
for lfn in lfns :
LOG.notice('"LFN:%s",' % lfn)
LOG.notice("]")
dexit(0)
def _createLFNList():
"""create the LFnList"""
cliparams = _Params()
cliparams.registerSwitches()
Script.parseCommandLine(ignoreErrors=False)
from DIRAC import gLogger
repoLocation = cliparams.repo
if not repoLocation:
Script.showHelp()
dexit(2)
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
dirac = DiracILC(True, repoLocation)
dirac.monitorRepository(False)
lfns = []
lfns = dirac.retrieveRepositoryOutputDataLFNs()
gLogger.notice("lfnlist=[")
for lfn in lfns:
gLogger.notice('"LFN:%s",' % lfn)
gLogger.notice("]")
dexit(0)
def submit(self, diracinstance = None, mode = "wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
"""
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submit(self, mode)
def create_job(inputData, saveName, outputDir, dontPromptMe):
slcioFile = saveName + '.slcio'
rootFile = saveName + '.root'
if check_file_existence(outputDir, slcioFile, dontPromptMe):
remove_file(outputDir, slcioFile, dontPromptMe)
if check_file_existence(outputDir, rootFile, dontPromptMe):
remove_file(outputDir, rootFile, dontPromptMe)
dIlc = DiracILC()
job = UserJob()
job.setOutputSandbox(['*.out', '*.log', '*.sh', '*.py', '*.xml'])
if SAVE_SLCIO:
job.setOutputData([slcioFile, rootFile],
OutputPath=outputDir,
OutputSE=STORAGE_SE)
else:
job.setOutputData(rootFile, OutputPath=outputDir, OutputSE=STORAGE_SE)
job.setJobGroup('myMarlinRun1')
job.setName('MyMarlinJob1')
# job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk'])
job.setInputSandbox(['LFN:/ilc/user/a/amaier/mylibs.tar.gz'])
marl = Marlin()
marl.setVersion('ILCSoft-2017-11-15_gcc62')
marl.setInputFile(inputData)
job.setInputData(list(map(lambda x: x.lstrip('LFN:'), inputData)))
# marl.setInputFile(['LFN:/ilc/prod/clic/1.4tev/qq_ln/ILD/DST/00003249/010/qq_ln_dst_3249_10000.slcio'])
marl.setSteeringFile('marlin/FullChain.xml')
# marl.setSteeringFile('marlin/FullChainNewDetModel.xml')
marl.setOutputFile(slcioFile)
gearFile = '/afs/cern.ch/user/a/amaier/projects/CLIC_analysis/grid/marlin/clic_ild_cdr.gear'
if not os.path.isfile(gearFile):
print('Error: gear file', gearFile,
'does not exist! Abort submission.')
return
marl.setGearFile(gearFile)
marl.setExtraCLIArguments(
"--MyNtupleMaker.OutputFileName={rootOutfile}".format(
rootOutfile=rootFile))
# marl.setNumberOfEvents(1000)
job.append(marl)
if dontPromptMe:
job.dontPromptMe()
job.submit(dIlc)
return False
def testAndProbeSites():
"""submits jobs to test sites"""
clip = Params()
clip.registerSwitches()
Script.parseCommandLine()
from DIRAC import gLogger, exit as dexit
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import CheckWNs
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getQueues
res = getQueues(siteList=clip.site, ceList=clip.ce)
if not res['OK']:
gLogger.error("Failed getting the queues", res['Message'])
dexit(1)
sitedict = res['Value']
CEs = []
for ces in sitedict.values():
CEs.extend(ces.keys())
gLogger.notice("Found %s CEs to look at." % len(CEs))
d = DiracILC(True, "SiteProbe.rep")
for CE in CEs:
j = UserJob()
j.setDestinationCE(CE)
c = CheckWNs()
res = j.append(c)
if not res['OK']:
gLogger.error(res['Message'])
continue
j.setOutputSandbox("*.log")
j.setCPUTime(30000)
j.dontPromptMe()
res = j.submit(d)
if not res['OK']:
gLogger.error("Failed to submit job, aborting")
dexit(1)
dexit(0)
def subWhizard2():
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin, Whizard2
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = _clip.isLocal
nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
outputFile = "E500.P2f_bB.GWhizard2.I100000.e0.p0.n001.slcio" if _clip.outputFile == "" else _clip.outputFile
outputDir = _clip.outputDir
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setInputSandbox(["pythia6-parameters.sin", "P2f_qqbar.sin"])
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setOutputData([outputFile])
job.setJobGroup("mywhiz2")
job.setName("mywhizard2")
whiz = Whizard2()
whiz.setVersion("2.7.0")
whiz.setNumberOfEvents(nbevts)
whiz.setEvtType("P2f_bB")
whiz.setProcessVariables("P2f_bB")
# whiz.setRandomSeed(15)
whiz.setSinFile("P2f_qqbar.sin")
whiz.setOutputFile(outputFile)
job.append(whiz)
if outputDir != "":
print " outputDir = " + outputDir
print " outputSE = " + outputSE
job.setOutputData([outputFile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
def subDDSim(clip1):
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = clip1.isLocal
nbevts = 0 if clip1.numberOfEvents == 0 else clip1.numberOfEvents
#print('inside subddsim(): nbevts ', nbevts)
outputFile = "" if clip1.outputFile == "" else clip1.outputFile
#print('inside subddsim outfile ', outputFile)
outputDir = clip1.outputDir
#print('inside subddsim outdir ', outputDir)
inputFile = clip1.inputFile
#print('inside subddsim inputFile ', inputFile)
if inputFile == "":
gLogger.error("Input file for ddsim is not given.")
exit(-1)
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myddsimjob")
job.setName("myddsim")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
job.setBannedSites(["LCG.UKI-SOUTHGRID-RALPP.uk"
]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
ddsim = DDSim()
ddsim.setVersion("ILCSoft-02-00-02_gcc49")
ddsim.setDetectorModel("ILD_l5_v05")
ddsim.setInputFile(inputFile)
ddsim.setNumberOfEvents(nbevts)
extraCLIArguments = " --steeringFile ddsim_steer_July26.py"
extraCLIArguments += " --outputFile %s " % outputFile
extraCLIArguments += " --vertexSigma 0.0 0.0 0.1968 0.0 --vertexOffset 0.0 0.0 0.0 0.0 "
ddsim.setExtraCLIArguments(extraCLIArguments)
return ddsim
def setUp( self ):
"""set up the objects"""
ops_mock = Mock()
mocked_modules = { 'DIRAC.ConfigurationSystem.Client.Helpers.Operations' : ops_mock }
self.module_patcher = patch.dict( sys.modules, mocked_modules )
self.module_patcher.start()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
self.dilc = DiracILC()
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['SE']
if 'Minimum' in args[0]:
return 1
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = Mock()
ops_mock.getValue.side_effect = setOptions
self.dilc.ops = ops_mock
def __init__(self):
self.dIlc = DiracILC(False)
self.job = UserJob()
self.job.setJobGroup("FCC")
self.job.setName("FCC APP")
self.job.setOutputSandbox(["*.log", '*.root'])
#self.job.setDestination('LCG.DESY-HH.de')
#EOS public location
self.EOS_MGM_URL = 'root://eospublic.cern.ch'
#EOS environment
self.setEOS = 'export EOS_MGM_URL=' + self.EOS_MGM_URL
self.myclient = client.FileSystem(self.EOS_MGM_URL + ':1094')
#sandbox
self.InputSandbox = []
self.folders_to_upload = []
self.filtered_extensions = []
self.excludes_or_includes = []
self.temp_cwd = os.path.join(os.getcwd(), 'fcc_temp_dirac')
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.setGearFile("clic_ild_cdr.gear")
result = j.append(ma)
if not result['OK']:
gLogger.error(result["Message"])
dexit(1)
j.setCPUTime(10000)
j.setOutputSandbox("*.log")
return j
if __name__ == '__main__':
lfns = getFiles() #get a list of files
d = DiracILC(True, "paramjobtest.rep") #get your dirac instance
job = getJob() #get a job, any can do
#here is where the interesting stuff happen
from DIRAC.Core.Utilities.List import breakListIntoChunks
for flist in breakListIntoChunks(lfns, 200):
#200 is the number of files per chunk, and the max number of jobs produced in one go
#This is the magical line
job.setParametricInputData(flist)
#The rest of the sumission is the same
res = job.submit(d)
if not res["OK"]:
gLogger.error("Failed to submit the job: ", res["Message"])
lfns = res['Value']
print("found %s files" % len(lfns))
ovi = OverlayInput()
ovi.setEnergy(500.)
ovi.setBXOverlay(300)
ovi.setGGToHadInt(0.3)
ovi.setNbSigEvtsPerJob(10)
ovi.setBkgEvtType("gghad")
ovi.setDetectorModel("CLIC_ILD_CDR")
overlay = [True,False]
for ov in overlay:
d = DiracILC(True,"repo_overlay_%s.rep"%ov)
for lfn in lfns:
j = UserJob()
steeringf = "clic_ild_cdr_steering.xml"
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print(res['Message'])
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:"+lfn)
ma.setNbEvts(10)
nameDir = nameJobGroup+'_files'+nameTag nameRepositoryFile = nameJobGroup+nameTag+'_repository.rep' ##################################################################### ##################################################################### #set enviroment import os import sys from DIRAC.Core.Base import Script #dirac enviroment Script.parseCommandLine() #dirac enviroment from ILCDIRAC.Interfaces.API.DiracILC import DiracILC #job receiver class dirac = DiracILC(True,nameRepositoryFile) #important to retrive jobs ##################################################################### ##################################################################### #job definition from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob ##################################################################### ##################################################################### #filecatalog from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient ##################################################################### ##################################################################### #ddsim from ILCDIRAC.Interfaces.API.NewInterface.Applications import DDSim
elif opt in ('v', 'version'):
slicVersion = arg
elif opt in ('w', 'workflow'):
workflowName = arg
elif opt in ('n', 'nocheck'):
checkMeta = False
if (detectorName == None) or (prodID == None):
Script.showHelp()
sys.exit(2)
from ILCDIRAC.Interfaces.API.NewInterface.ProductionJob import ProductionJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC, LCSIM, SLICPandora, OverlayInput
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
dirac = DiracILC()
meta = {}
meta['ProdID'] = prodID
if eventType:
meta['EvtType'] = eventType
if energy:
meta['Energy'] = energy
if dataType:
meta['Datatype'] = dataType
fc = FileCatalogClient()
res = fc.getCompatibleMetadata(meta)
if not res['OK']:
print "Error looking up the catalog for metadata"
exit(2)
def all_jobs(name):
d = DiracILC(True, "repo.rep")
################################################
j = UserJob()
j.setJobGroup("PM1")
j.setName("Exec1")
banned_sites = [
"OSG.BNL.us", "LCG.UKI-NORTHGRID-LIV-HEP.uk", "OSG.UCSDT2.us",
"LCG.SCOTGRIDDURHAM.uk", "LCG.NIKHEF.nl", "LCG.UKI-SOUTHGRID-RALPP.uk",
"LCG.GRIF.fr", "LCG.Manchester.uk", "LCG.UKI-LT2-IC-HEP.uk",
"LCG.Weizmann.il"
]
j.setBannedSites(banned_sites)
caindir = name
#print('Cain directory is ',caindir)
indata = [
'LFN:/ilc/user/a/amustahid/cain.exe',
str(caindir), 'LFN:/ilc/user/a/amustahid/runcain.sh',
'LFN:/ilc/user/a/amustahid/convert_pairs_lcio.py',
'LFN:/ilc/user/a/amustahid/pyLCIO.tar.gz',
'/home/belle2/mustahid/useful/my.sh', './splitInput.py',
'./subddsim.py', './ddsim_steer_July26.py', './ILD_l5_v05.xml',
'./my2.sh', './dbd_500GeV.nung_1.xml',
'LFN:/ilc/user/a/amustahid/myProcessors.tar.gz', './create_dir.py',
'./conf.py', './util.py', './testcain.sh', './beam_250.i'
]
j.setInputSandbox(indata)
################################################
#app = GenericApplication()
#app.setScript("create_dir.py")
#app.setInputFile("testcain.sh")
#logf = 'create_dir.log'
#app.setLogFile(logf)
#app.setDebug(debug=True)
#create_dirname = 'create_dir'
#app.setName(create_dirname)
#res=j.append(app)
#if not res['OK']:
# print res['Message']
# exit(1)
################################################
appre = GenericApplication()
name = name.split('/')
#print(name)
cain_name = name[-1]
subdir = name[-2]
dirname = name[-3]
#print('Cain file name ', cain_name)
appre.setScript("LFN:/ilc/user/a/amustahid/runcain.sh")
#appre.setScript("testcain.sh")
ifile = cain_name.split('.')
ifile = ifile[0] + '.' + ifile[1] + '.' + ifile[2]
#print('ifile ',ifile)
appre.setArguments(ifile)
#direc = 'LFN:/ilc/user/a/amustahid/'
#appre.setInputFile(ifile+".i")
#appre.setArguments("This is input arguments")
logf = ifile + '_' + subdir + '.log'
appre.setLogFile(logf)
appre.setDebug(debug=True)
name = 'CAIN'
appre.setName(name)
res = j.append(appre)
if not res['OK']:
print res['Message']
exit(1)
################################################
################################################
#appost = GenericApplication()
#appost.setScript("myanal.sh")
#appost.setArguments("This is my analysis step")
#res=j.append(appost)
#if not res['OK']:
# print res['Message']
# exit(1)
ap = GenericApplication()
ap.setScript('my.sh')
logf = 'my.log'
ap.setLogFile(logf)
ap.setDebug(debug=True)
name = 'my'
ap.setName(name)
res = j.append(ap)
if not res['OK']:
print res['Message']
exit(1)
outfile = 'incoherent_pair.dat'
appre.setOutputFile(outfile)
################################################
direc = 'incoherent_pair'
inputFile = direc + '/' + 'inco_pair_split.slcio'
# global variables to hold command line parameters
# ######################################
base = '.'
#outdir=base+'/'+dirname+'/slcio_test_2ndrun'
outdir = base + '/' + dirname + '/Run_7'
#print('outdir'+' '+str(outdir))
geant_name = ifile
outputFile = geant_name + '_' + subdir + '.slcio'
#_clip = _Params(False,1,inputFile,outputFile,outdir)
nbevents = 100
clip = _Params(nbevents, inputFile, outputFile, outdir)
ddsim = subDDSim(clip)
################################################
res = j.append(ddsim)
if not res['OK']:
print res['Message']
exit(1)
j.setOutputData(outputFile, outdir, "KEK-SRM")
j.setOutputSandbox(["*.log", "*.dat", "*.slcio"])
j.dontPromptMe()
res = j.submit(d)
#res = j.submit(d, mode='local')
if res['OK']:
print str(res["Value"])
#print "Dirac job, "+str(res["Value"])+", was submitted."
else:
print "Failed to submit Dirac job. return message was as follows."
pprint.pprint(res)
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard, Mokka, Marlin, OverlayInput
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from DIRAC import exit as dexit
dirac = DiracILC()
# wh.setOutputFile("myfile.stdhep")
j = UserJob()
wh = Whizard(processlist=dirac.getProcessList())
wh.setEnergy(3000)
wh.setEvtType("ee_h_mumu")
wh.setNbEvts(1)
wh.setEnergy(3000)
params = {}
params["USERB1"] = "F"
wh.setParameterDict(params)
wh.setModel("sm")
res = j.append(wh)
if not res["OK"]:
print res["Message"]
dexit(1)
Created on Feb 8, 2012
:author: Stephane Poss
'''
__RCSID__ = "$Id$"
#pylint: skip-file
#pylint: disable=C0103
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.NewInterface.ProductionJob import ProductionJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard, Mokka, Marlin, OverlayInput, StdhepCut, StdhepCutJava
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC, LCSIM, SLICPandora, SLCIOSplit, StdHepSplit
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
dirac = DiracILC()
###As it's a full chain, we start at generation
##so we need to define the process and the energy
## The rest will be set later. We could also set the process
##and the energy directly in the whizard def, but for clarity
## it's better to do it before, that way we know the very
##essential
def getdicts(process):
""" Create the proper structures to build all the prodcutions for the samples with ee_, ea_ aa_.
"""
plist = []
if process.count("ee_"):
plist.append({'process':process,'pname1':'e1', 'pname2':'E1', "epa_b1":'F', "epa_b2":'F'})
#every single script has these four lines
from DIRAC.Core.Base import Script
Script.parseCommandLine()
#create ilcdirac instance
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
dirac = DiracILC(True, "some_job_repository.rep")
#job definition
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
#job = UserJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC
from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM
#jobParams = [('slicTest8_mu+_theta90.mac','diracTest8_mu+_theta90.slcio',50),('slicTest7_mu+_theta_5-175.mac','diracTest_mu+_theta_5-175.slcio',50),('slicTest3_e+.mac','diracTest3_e+.slcio',10),('slicTest2_pi+.mac','diractTest2_pi+.slcio',10)]
#jobParams = [('slicTest10_mu+_100gev_theta70_testNewGeom.mac','diracTest10_mu+_100gev_theta70_testNewGeom.slcio',10),('slicTest10_mu+_100gev_theta90_testNewGeom.mac','diracTest10_mu+_100gev_theta90_testNewGeom.slcio',10)]
jobParams = [
('/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/slicmacros/slicTest8_mu+_100gev_theta60.mac',
'diracTest_100gev_theta60_vtx_matbudghalf_nonsensitivelayer.slcio', 100),
('/users/detector/ssetru/SiDSim/detectors/detector_vtx_matbudghalf_nonsensitivelayer/slicmacros/slicTest8_mu+_10gev_theta60.mac',
'diracTest_10gev_theta60_vtx_matbudghalf_nonsensitivelayer.slcio', 100)
]
#slicMacros = ['slicTest8_mu+_theta90.mac','slicTest7_mu+_theta_5-175.mac','slicTest3_e+.mac','slicTest2_pi+.mac']
#fileOutputs = ['diracTest2Loop1.slcio','diracTest2Loop2.slcio','diracTest2Loop3.slcio','diractTest2Loop4.slcio']
#slicNumEvents = [100,100,10,10]
for macro, output, nEvts in jobParams:
job = UserJob()
job.setName("ssetru_dirac_test1")
job.setJobGroup("tests")
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD",
"0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True,
setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent / nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector,
fileNumber, args.outputPath,
softVersions)
slicOutput = outputFiles['slicOutput']
prePandoraOutput = outputFiles['prePandoraOutput']
pandoraOutput = outputFiles['pandoraOutput']
vertexingOutput = outputFiles['vertexingOutput']
lcsimRecOutput = outputFiles['lcsimRecOutput']
lcsimDstOutput = outputFiles['lcsimDstOutput']
flavortagOutput = outputFiles['flavortagOutput']
diracOutput = outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(
'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml'
) # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy(
'steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites([
'LCG.IN2P3-CC.fr',
'LCG.RAL-LCG2.uk',
'LCG.DESY-HH.de',
'LCG.DESYZN.de',
'LCG.KEK.jp',
'OSG.PNNL.us',
])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0
class SoftwareManagementAgent( AgentModule ):
""" Agent to run software management things
"""
def initialize(self):
self.pollingTime = self.am_getOption('PollingTime', 86400)
gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM)
self.ppc = ProcessProdClient()
self.dirac = DiracILC()
self.diracadmin = DiracAdmin()
self.am_setOption( 'shifterProxy', 'Admin' )
return S_OK()
##############################################################################
def execute(self):
""" First we update the site list and banned site
"""
res = getProxyInfo(False, False)
if not res['OK']:
self.log.error("submitTasks: Failed to determine credentials for submission", res['Message'])
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
self.log.info("submitTasks: Jobs will be submitted with the credentials %s:%s" % (owner, ownerGroup))
sites = self.diracadmin.getSiteMask()['Value']
for site in sites:
res = self.ppc.changeSiteStatus( {'SiteName' : site, 'Status' : 'OK'} )
if not res['OK']:
self.log.error('Cannot add or update site %s' % site)
banned_sites = self.diracadmin.getBannedSites()['Value']
for banned_site in banned_sites:
self.ppc.changeSiteStatus( {'SiteName' : banned_site, 'Status' : 'Banned'} )
if not res['OK']:
self.log.error('Cannot mark as banned site %s' % banned_site)
##Then we need to get new installation tasks
res = self.ppc.getInstallSoftwareTask()
if not res['OK']:
self.log.error('Failed to obtain task')
task_dict = res['Value']
for softdict in task_dict.values():
self.log.info('Will install %s %s at %s' % (softdict['AppName'], softdict['AppVersion'], softdict['Sites']))
for site in softdict['Sites']:
j = UserJob()
j.setPlatform(softdict['Platform'])
j.dontPromptMe()
j.setDestination(site)
j.setJobGroup("Installation")
j.setName('install_%s' % site)
j._addSoftware(softdict['AppName'], softdict['AppVersion'])
#Add the application here somehow.
res = j.append(SoftwareInstall())
if not res['OK']:
self.log.error(res['Message'])
continue
res = j.submit(self.dirac)
#res = self.dirac.submit(j)
if not res['OK']:
self.log.error('Could not create the job')
continue
jobdict = {}
jobdict['AppName'] = softdict['AppName']
jobdict['AppVersion'] = softdict['AppVersion']
jobdict['Platform'] = softdict['Platform']
jobdict['JobID'] = res['Value']
jobdict['Status'] = 'Waiting'
jobdict['Site'] = site
res = self.ppc.addOrUpdateJob(jobdict)
if not res['OK']:
self.log.error('Could not add job %s: %s' % (jobdict['JobID'], res['Message']))
##Monitor jobs
jobs = {}
res = self.ppc.getJobs()
if not res['OK']:
self.log.error('Could not retrieve jobs')
else:
jobs = res['Value']
for job in jobs:
res = self.dirac.status(job['JobID'])
if res['OK']:
jobstatuses = res['Value']
job['Status'] = jobstatuses['JobID']['Status']
res = self.ppc.addOrUpdateJob(job)
if not res['OK']:
self.log.error("Failed to updated job %s: %s" % (job['JobID'], res['Message']))
else:
self.log.error("Failed to update job %s status" % job['JobID'])
return S_OK()
base = open(steeringTemplateFile,'r')
steeringTemplateContent = base.read()
base.close()
for eventSelection in eventsToSimulate:
eventType = eventSelection['EventType']
detectorModel = eventSelection['DetectorModel']
reconstructionVariant = eventSelection['ReconstructionVariant']
energy = eventSelection['Energy']
analysisTag = eventSelection['AnalysisTag']
# Make local gear file
os.system('cp ' + gearFile + ' .')
gearFileLocal = os.path.basename(gearFile)
diracInstance = DiracILC(withRepo=False)
slcioFormat = 'DetModel_' + detectorModel + '_RecoVar_' + reconstructionVariant + '_' + eventType + '_' + str(energy) + 'GeV_GenNumber_(.*?)_(.*?)_(.*?)_DST.slcio'
slcioFilesToProcess = getDstSlcioFiles(jobDescription,detectorModel,reconstructionVariant,energy,eventType)
if not slcioFilesToProcess:
print 'No slcio files found. Exiting job submission.'
sys.exit()
for slcioFile in slcioFilesToProcess:
print 'Checking ' + eventType + ' ' + str(energy) + 'GeV jobs. Detector model ' + detectorModel + '. Reconstruction stage ' + reconstructionVariant + '. Slcio file ' + slcioFile + '.'
slcioFileNoPath = os.path.basename(slcioFile)
inputSandbox = ['LFN:/ilc/user/s/sgreen/SelectionProcessorTarBall/MarlinSelectionProcessor.tar.gz']
lfns = res['Value']
print "found %s files" % len(lfns)
ovi = OverlayInput()
ovi.setEnergy(500.)
ovi.setBXOverlay(300)
ovi.setGGToHadInt(0.3)
ovi.setNbSigEvtsPerJob(10)
ovi.setBkgEvtType("gghad")
ovi.setDetectorModel("CLIC_ILD_CDR")
overlay = [True, False]
for ov in overlay:
d = DiracILC(True, "repo_overlay_%s.rep" % ov)
for lfn in lfns:
j = UserJob()
steeringf = "clic_ild_cdr_steering.xml"
if ov:
steeringf = "clic_ild_cdr_steering_overlay.xml"
res = j.append(ovi)
if not res['OK']:
print res['Message']
continue
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setGearFile("clic_ild_cdr.gear")
ma.setSteeringFile(steeringf)
ma.setInputFile("LFN:" + lfn)
ma.setNbEvts(10)
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import *
from ILCDIRAC.Interfaces.API.NewInterface.Applications import *
dirac = DiracILC(True, "CLICdet_repo.cfg")
job = UserJob()
job.setName("example")
job.setInputData([
"/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep"
])
job.setCLICConfig("ILCSoft-2017-12-21")
job.setOutputSandbox(["*.root", "*.slcio", "*.log"])
ddsim = DDSim()
ddsim.setSteeringFile("CLICPerformance/examples/clic_steer.py")
ddsim.setVersion("ILCSoft-2017-12-21_gcc62")
ddsim.setDetectorModel("CLIC_o3_v14")
ddsim.setInputFile(
"/ilc/user/p/proloff/stdhep/2f_samples_23_04_2013/whizard_SM_bb_500_90deg.stdhep"
)
ddsim.setOutputFile("ddsim_output.slcio")
ddsim.setStartFrom(0)
ddsim.setNumberOfEvents(10)
res1 = job.append(ddsim)
if not res1['OK']:
print res1['Message']
sys.exit(2)
# -------------------- Comment if gg_had overlay not wanted -----------------------------------------------------#
class JobCreater(object):
"""contains all the versions and parameters to create all theses tests"""
# pylint: disable=too-many-instance-attributes
# Test parameters, necessary due to amount of tests in this class.
def __init__(self, clip, params):
self.clip = clip
self.ildConfig = params.get("ildConfig", None)
self.alwaysOverlay = params.get("alwaysOverlay", False)
self.runOverlay = self.clip.testOverlay or self.alwaysOverlay
self.mokkaVersion = params["mokkaVersion"]
self.mokkaSteeringFile = params.get("mokkaSteeringFile")
self.detectorModel = params.get("detectorModel")
self.marlinVersion = params.get("marlinVersion")
self.marlinSteeringFile = params.get("marlinSteeringFile")
self.ddsimVersion = params.get("ddsimVersion")
self.ddsimDetectorModel = params.get("ddsimDetectorModel")
self.ddsimInputFile = params.get("ddsimInputFile")
self.marlinInputdata = params.get("marlinInputdata")
self.gearFile = params.get("gearFile")
self.lcsimVersion = params.get("lcsimVersion")
self.steeringFileVersion = params.get("steeringFileVersion", None)
self.rootVersion = params["rootVersion"]
self.whizard2Version = params.get("whizard2Version")
self.whizard2SinFile = params.get("whizard2SinFile")
self.energy = params.get("energy")
self.backgroundType = params.get("backgroundType")
self.machine = params.get("machine")
self.gearFile = params.get("gearFile")
self.marlinSteeringFile = params.get("marlinSteeringFile")
self.marlinVersion = params.get("marlinVersion")
self.lcsimPreSteeringFile = params.get("lcsimPreSteeringFile")
self.lcsimPostSteeringFile = params.get("lcsimPostSteeringFile")
self.fccSwPath = params.get("fccSwPath")
self.fccSwSteeringFile = params.get("fccSwSteeringFile")
self.fccAnalysisSteeringFile = params.get("fccAnalysisSteeringFile")
### other things needed to run tests
self.log = gLogger.getSubLogger("JobCreater")
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid
from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid
if self.clip.submitMode == "local":
self.log.notice("")
self.log.notice(" DIRAC RCSID:", drcsid)
self.log.notice(" Job RCSID:", jrcsid)
self.log.notice("Applications RCSID:", apprcsid)
self.log.notice("")
self.diracInstance = DiracILC(False, 'tests.rep')
self.jobList = {}
def createDDSimTest(self, inputfile=None, detectorModel=None):
"""Create a job running ddsim"""
if inputfile is None:
inputfile = self.ddsimInputFile
sandbox = [inputfile]
if detectorModel is None:
detectorModel = self.ddsimDetectorModel
else:
sandbox.append(detectorModel)
jobdd = self.getJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim
ddsim = DDSim()
ddsim.setVersion(self.ddsimVersion)
ddsim.setDetectorModel(detectorModel)
ddsim.setNumberOfEvents(1)
ddsim.setInputFile(inputfile)
jobdd.setInputSandbox(sandbox)
res = jobdd.append(ddsim)
if not res['OK']:
self.log.error("Failed adding DDSim:", res['Message'])
return S_ERROR("Failed adding DDSim to Job")
return S_OK(jobdd)
def createWhizard2Test(self):
"""Create a job running Whizard2"""
jobdd = self.getJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications.Whizard2 import Whizard2
whiz = Whizard2()
whiz.setVersion(self.whizard2Version)
whiz.setNumberOfEvents(1)
whiz.setSinFile(self.whizard2SinFile)
whiz.setOutputFile("test.stdhep")
res = jobdd.append(whiz)
if not res['OK']:
self.log.error("Failed adding Whizard2:", res['Message'])
return S_ERROR("Failed adding Whizard2 to Job")
return S_OK(jobdd)
def createMokkaTest(self):
"""create a job running mokka, and maybe whizard before"""
self.log.notice("Creating jobs for Mokka")
#(Whizard + )Mokka
jobmo = self.getJob()
if self.clip.testChain:
whmo = self.getWhizard(2)
res = jobmo.append(whmo)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR("Failed adding Whizard")
elif self.clip.testInputData:
jobmo.setInputData(
"/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep"
)
else:
self.log.error("Mokka does not know where to get its input from")
return S_ERROR("Mokka does not know where to gets its input from")
mo = self.getMokka()
if self.clip.testChain:
mo.getInputFromApp(whmo)
else:
mo.setNumberOfEvents(1)
res = jobmo.append(mo)
if not res['OK']:
self.log.error("Failed adding Mokka:", res['Message'])
return S_ERROR("Failed adding Mokka to Job")
jobmo.setOutputData("testsim.slcio", OutputSE="CERN-DIP-4")
self.jobList['Mokka1'] = jobmo
return S_OK(jobmo)
def createRootScriptTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["root.sh", "input.root", "input2.root"])
root = self.getRoot()
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def createRootHaddTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["input.root", "input2.root"])
root = self.getRoot()
root.setScript("hadd")
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def createRootMacroTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["func.C", "input.root", "input2.root"])
root = self.getRootMacro()
root.setScript("func.C")
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def getOverlay(self, nbevts):
""" Create an overlay step
"""
pathToFiles = None
from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput
overlay = OverlayInput()
if self.energy == 350:
if self.detectorModel == "ILD_o1_v05":
pathToFiles = "/ilc/user/s/sailer/testFiles/overlay/ild_350/"
if pathToFiles:
overlay.setPathToFiles(pathToFiles)
else:
self.log.warn(
"better define pathToFiles for this overlay: %s, %s, %s" %
(self.energy, self.machine, self.backgroundType))
overlay.setMachine(self.machine)
overlay.setEnergy(self.energy)
overlay.setDetectorModel(self.detectorModel)
overlay.setBkgEvtType(self.backgroundType)
overlay.setBXOverlay(60)
overlay.setGGToHadInt(0.3)
overlay.setNumberOfSignalEventsPerJob(nbevts)
return overlay
def getMokka(self):
""" Define a mokka app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka
mokka = Mokka()
mokka.setVersion(self.mokkaVersion)
mokka.setSteeringFile(self.mokkaSteeringFile)
mokka.setOutputFile("testsim.slcio")
mokka.setDetectorModel(self.detectorModel)
if self.steeringFileVersion:
mokka.setSteeringFileVersion(self.steeringFileVersion)
return mokka
def getRoot(self):
""" Define a root app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootScript
root = RootScript()
root.setScript("root.sh")
root.setArguments("output.root input.root input2.root")
root.setVersion(self.rootVersion)
root.setOutputFile("output.root")
return root
def getRootMacro(self):
""" Define a root app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootMacro
root = RootMacro()
root.setMacro("func.C")
root.setArguments(r"\"input.root\"")
root.setVersion(self.rootVersion)
return root
@staticmethod
def getSLIC():
""" Get a SLIC instance
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
slic.setOutputFile('testsim.slcio')
return slic
@staticmethod
def getSLICPandora():
""" Get some SLICPAndora app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLICPandora
slicp = SLICPandora()
slicp.setVersion('CLIC_CDR')
slicp.setDetectorModel('clic_sid_cdr')
slicp.setPandoraSettings("PandoraSettingsSlic.xml")
slicp.setOutputFile('testpandora.slcio')
return slicp
def getMarlin(self):
""" Define a marlin step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
# marlin.setVersion("v0111Prod")
marlin.setVersion(self.marlinVersion)
marlin.setSteeringFile(self.marlinSteeringFile)
marlin.setGearFile(self.gearFile)
marlin.setOutputDstFile("testmarlinDST.slcio")
marlin.setOutputRecFile("testmarlinREC.slcio")
marlin.setNumberOfEvents(1)
return marlin
def getDD(self):
""" Create a DDSim object
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim
ddsim = DDSim()
ddsim.setVersion(self.ddsimVersion)
ddsim.setDetectorModel(self.ddsimDetectorModel)
ddsim.setInputFile(self.ddsimInputFile)
ddsim.setNumberOfEvents(2)
return ddsim
def getLCSIM(self, prepandora=True):
""" Get some LCSIM
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM
lcsim = LCSIM()
lcsim.setVersion('CLIC_CDR')
lcsim.setDetectorModel('clic_sid_cdr.zip')
if prepandora:
lcsim.setSteeringFile(self.lcsimPreSteeringFile)
lcsim.setOutputFile("testlcsim.slcio")
else:
lcsim.setSteeringFile(self.lcsimPostSteeringFile)
#lcsim.setOutputFile("testlcsimfinal.slcio")
lcsim.setOutputDstFile("testlcsimDST.slcio")
lcsim.setOutputRecFile("testlcsimREC.slcio")
lcsim.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml")
return lcsim
def getFccSw(self):
""" Define a fccsw step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccSw
fccsw = FccSw()
fccsw.fccSwPath = self.fccSwPath
fccsw.setSteeringFile(self.fccSwSteeringFile)
return fccsw
def getFccAnalysis(self):
""" Define a fccanalysis step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccAnalysis
fccanalysis = FccAnalysis()
fccanalysis.setSteeringFile(self.fccAnalysisSteeringFile)
return fccanalysis
@staticmethod
def getStdhepcut(generated):
""" Get some cuts in
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdhepCutJava
stdhepcut = StdhepCutJava()
stdhepcut.setVersion('1.0')
stdhepcut.setSelectionEfficiency(1.)
#stdhepcut.setSteeringFile("cuts_testing_1400.txt")
stdhepcut.setInlineCuts("leptonInvMass_R 13 100 200")
stdhepcut.setSteeringFileVersion("V18")
stdhepcut.setMaxNbEvts(1)
stdhepcut.setNumberOfEvents(generated)
return stdhepcut
@staticmethod
def getStdhepSplit():
""" Get some stdhep split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(5)
stdhepsplit.setOutputFile("teststdhepsplit.stdhep")
stdhepsplit.setMaxRead(10)
return stdhepsplit
@staticmethod
def getLCIOSplit(events_per_file):
""" Get a LCIO split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOSplit
lciosplit = SLCIOSplit()
lciosplit.setNumberOfEventsPerFile(events_per_file)
lciosplit.setOutputFile("testlciosplit.slcio")
return lciosplit
@staticmethod
def getLCIOConcat():
""" Get a LCIO Concat
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOConcatenate
lcioconcat = SLCIOConcatenate()
lcioconcat.setOutputFile("testlcioconcat.slcio")
return lcioconcat
def getJob(self):
""" Define a generic job, it should be always the same
"""
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
myjob = UserJob()
myjob.setName("Testing")
myjob.setJobGroup("Tests")
myjob.setCPUTime(30000)
myjob.dontPromptMe()
myjob.setLogLevel("VERBOSE")
myjob.setPlatform("x86_64-slc5-gcc43-opt")
myjob.setOutputSandbox(["*.log", "*.xml", "*.sh"])
myjob._addParameter(myjob.workflow, 'TestFailover', 'String', True,
'Test failoverRequest')
myjob._addParameter(myjob.workflow, 'Platform', 'JDL',
"x86_64-slc5-gcc43-opt", 'OS Platform')
if self.ildConfig:
myjob.setILDConfig(self.ildConfig)
return myjob
def getWhizardModel(self, nbevts, energy, model):
""" Create a default whizard
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard
proddict = "e2e2_o"
if model != "sm":
proddict = "se2se2_r"
whiz = Whizard(processlist=self.diracInstance.getProcessList())
whiz.setModel(model)
pdict = {}
pdict['process_input'] = {}
pdict['process_input']['process_id'] = proddict
pdict['process_input']['sqrts'] = energy
if model != 'sm':
pdict['process_input']['input_file'] = "LesHouches.msugra_1"
pdict['process_input']['input_slha_format'] = 'T'
pdict['process_input']['beam_recoil'] = 'T'
pdict['integration_input'] = {}
pdict['integration_input']['calls'] = '1 50000 10 5000 1 15000'
pdict['simulation_input'] = {}
pdict['simulation_input']['normalize_weight'] = 'F'
pdict['simulation_input']['n_events'] = nbevts
pdict['simulation_input']['keep_initials'] = 'T'
pdict['simulation_input']['events_per_file'] = 500000
if model != 'sm':
pdict['simulation_input'][
'pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1;IMSS(1)=11; IMSS(21)=71; IMSS(22)=71"
else:
pdict['simulation_input'][
'pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ; MSTJ(28)=2 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1"
pdict['parameter_input'] = {}
# pdict['parameter_input']['mmu']=mmu
# pdict['parameter_input']['mtau']=mtau
# pdict['parameter_input']['mb']=mb
# pdict['parameter_input']['mc']=mc
pdict['parameter_input']['mH'] = 125
pdict['beam_input_1'] = {}
pdict['beam_input_1']['particle_name'] = "e1"
pdict['beam_input_1']['polarization'] = "0.0 0.0"
pdict['beam_input_1']['USER_spectrum_on'] = 'T'
pdict['beam_input_1'][
'USER_spectrum_mode'] = 19 if energy == 1400 else 11
pdict['beam_input_1']['ISR_on'] = 'T'
pdict['beam_input_1']['EPA_on'] = "F"
pdict['beam_input_2'] = {}
pdict['beam_input_2']['particle_name'] = "E1"
pdict['beam_input_2']['polarization'] = "0.0 0.0"
pdict['beam_input_2']['USER_spectrum_on'] = 'T'
pdict['beam_input_2']['ISR_on'] = 'T'
pdict['beam_input_2'][
'USER_spectrum_mode'] = 19 if energy == 1400 else 11
pdict['beam_input_2']['EPA_on'] = 'F'
whiz.setFullParameterDict(pdict)
whiz.setOutputFile("testgen.stdhep")
return whiz
def getWhizard(self, nbevts):
""" Get some defualt SM whizard
"""
return self.getWhizardModel(nbevts, 1400, "sm")
def getWhizardSUSY(self, nbevts):
""" Get a susy whizard
"""
return self.getWhizardModel(nbevts, 3000, "slsqhh")
def createWhizardTest(self):
"""create a test for whizard"""
self.log.notice("Creating jobs for Whizard")
##### WhizardJob
jobw = self.getJob()
wh = self.getWhizard(2)
res = jobw.append(wh)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
self.jobList['Whizard1'] = jobw
##### WhizardJob
jobwsusy = self.getJob()
whsusy = self.getWhizardSUSY(2)
res = jobwsusy.append(whsusy)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
self.jobList['WhizSusy'] = jobwsusy
return S_OK((jobw, jobwsusy))
def createSlicTest(self):
"""create tests for slic"""
self.log.notice("Creating jobs for SLIC")
#run (Whizard +)SLIC
jobslic = self.getJob()
if self.clip.testChain:
whslic = self.getWhizard(2)
res = jobslic.append(whslic)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
elif self.clip.testInputData:
jobslic.setInputData(
"/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep"
)
else:
self.log.error("SLIC does not know where to get its input from")
return S_ERROR()
myslic = self.getSLIC()
if self.clip.testChain:
myslic.getInputFromApp(whslic)
else:
myslic.setNumberOfEvents(2)
res = jobslic.append(myslic)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
self.jobList['Slic1'] = jobslic
return S_OK(jobslic)
def createMarlinTest(self, setInputData=False):
"""create tests for marlin"""
self.log.notice("Creating test for Marlin")
#((Whizard + Mokka +)Overlay+) Marlin
jobma = self.getJob()
if self.clip.testChain:
moma = self.getMokka()
if not self.clip.testInputData:
whma = self.getWhizard(2)
res = jobma.append(whma)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
moma.getInputFromApp(whma)
else:
jobma.setInputData(
"/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep"
)
moma.setNumberOfEvents(1)
res = jobma.append(moma)
if not res['OK']:
self.log.error("Failed adding Mokka:", res['Message'])
return S_ERROR()
elif self.clip.testInputData:
jobma.setInputData(self.marlinInputdata)
else:
self.log.error("Marlin does not know where to get its input from")
return S_ERROR()
if self.runOverlay:
ov = self.getOverlay(2)
res = jobma.append(ov)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR
ma = self.getMarlin()
if self.clip.testChain:
ma.getInputFromApp(moma)
else:
ma.setNumberOfEvents(2)
res = jobma.append(ma)
if not res['OK']:
self.log.error("Failed adding Marlin:", res['Message'])
return S_ERROR()
self.jobList['Marlin1'] = jobma
return S_OK(jobma)
def createLCSimTest(self):
"""create tests for LCSIM"""
self.log.notice("Creating test for LCSIM")
#run ((whiz+SLIC+)+Overlay+)LCSIM
joblcsim = self.getJob()
if self.clip.testChain:
mysliclcsim = self.getSLIC()
if not self.clip.testInputData:
whlcsim = self.getWhizard(2)
res = joblcsim.append(whlcsim)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
mysliclcsim.getInputFromApp(whlcsim)
else:
joblcsim.setInputData(
"/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep"
)
mysliclcsim.setNumberOfEvents(2)
res = joblcsim.append(mysliclcsim)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
elif self.clip.testInputData:
#joblcsim.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio")
joblcsim.setInputData(
"/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio")
else:
self.log.error("LCSIM does not know where to get its input from")
return S_ERROR()
if self.runOverlay:
ovlcsim = self.getOverlay(2)
res = joblcsim.append(ovlcsim)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR()
mylcsim = self.getLCSIM(True)
if self.clip.testChain:
mylcsim.getInputFromApp(mysliclcsim)
else:
mylcsim.setNumberOfEvents(2)
res = joblcsim.append(mylcsim)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
self.jobList['lcsim1'] = joblcsim
return S_OK(joblcsim)
def createSlicPandoraTest(self):
"""create tests for slicPandora"""
self.log.notice("Creating tests for SLICPandora")
#run ((whiz+SLIC) + (Overlay +) LCSIM +) SLICPandora + LCSIM
joblcsimov = self.getJob()
if not self.clip.testChain:
self.log.error(
"SLICPandora does not know where to get its input from")
return S_ERROR()
mylcsimov = self.getLCSIM(True)
if not self.clip.testInputData:
whlcsimov = self.getWhizard(2)
res = joblcsimov.append(whlcsimov)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
mysliclcsimov = self.getSLIC()
mysliclcsimov.getInputFromApp(whlcsimov)
res = joblcsimov.append(mysliclcsimov)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
mylcsimov.getInputFromApp(mysliclcsimov)
else:
#joblcsimov.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio")
joblcsimov.setInputData(
"/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio")
mylcsimov.setNumberOfEvents(2)
if self.runOverlay:
ovslicp = self.getOverlay(2)
res = joblcsimov.append(ovslicp)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR()
res = joblcsimov.append(mylcsimov)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
myslicpov = self.getSLICPandora()
myslicpov.getInputFromApp(mylcsimov)
res = joblcsimov.append(myslicpov)
if not res['OK']:
self.log.error("Failed adding SLICPandora: ", res["Message"])
return S_ERROR()
mylcsimovp = self.getLCSIM(False)
mylcsimovp.getInputFromApp(myslicpov)
res = joblcsimov.append(mylcsimovp)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
self.jobList['lcsimov1'] = joblcsimov
return S_OK(joblcsimov)
def createUtilityTests(self):
"""Create tests for utility applications"""
self.log.notice("Creating tests for utility applications")
jobwsplit = self.getJob()
whsplit = self.getWhizard(10)
res = jobwsplit.append(whsplit)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
mystdsplit = JobCreater.getStdhepSplit()
mystdsplit.getInputFromApp(whsplit)
res = jobwsplit.append(mystdsplit)
if not res['OK']:
self.log.error("Failed adding StdHepSplit:", res['Message'])
return S_ERROR()
self.jobList['whizSplit'] = jobwsplit
##### WhizardJob + split
jobwcut = self.getJob()
whcut = self.getWhizard(100)
res = jobwcut.append(whcut)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
mystdcut = self.getStdhepcut(100)
mystdcut.getInputFromApp(whcut)
res = jobwcut.append(mystdcut)
if not res['OK']:
self.log.error("Failed adding StdHepCut:", res['Message'])
return S_ERROR()
self.jobList['whizCut'] = jobwcut
#LCIO split
joblciosplit = self.getJob()
# joblciosplit.setInputData("/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio")
joblciosplit.setInputData(
"/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio"
)
mylciosplit = self.getLCIOSplit(100)
res = joblciosplit.append(mylciosplit)
if not res['OK']:
self.log.error("Failed adding SLCIOSplit:", res['Message'])
return S_ERROR()
self.jobList['lcioSplit'] = joblciosplit
#LCIO concat
jobconcat = self.getJob()
# jobconcat.setInputData(["/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_27.slcio",
# "/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio"])
jobconcat.setInputData([
"/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2104.slcio",
"/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2105.slcio"
])
myconcat = self.getLCIOConcat()
res = jobconcat.append(myconcat)
if not res['OK']:
self.log.error("Failed adding SLCIOConcatenate:", res['Message'])
return S_ERROR()
self.jobList['concat'] = jobconcat
return S_OK((jobconcat, joblciosplit, jobwcut, jobwsplit))
def createFccSwTest(self):
"""create tests for fccsw"""
self.log.notice("Creating jobs for FccSW")
#### FccSwJob
jobfccsw = self.getJob()
fccsw = self.getFccSw()
res = jobfccsw.append(fccsw)
if not res['OK']:
self.log.error("Failed adding FccSw:", res['Message'])
return S_ERROR()
self.jobList['FccSw1'] = jobfccsw
return S_OK(jobfccsw)
def createFccAnalysisTest(self):
"""create tests for fccanalysis"""
self.log.notice("Creating jobs for FccAnalysis")
#### FccAnalysisJob
jobfccanalysis = self.getJob()
fccanalysis = self.getFccAnalysis()
res = jobfccanalysis.append(fccanalysis)
if not res['OK']:
self.log.error("Failed adding FccAnalysis:", res['Message'])
return S_ERROR()
self.jobList['FccAnalysis1'] = jobfccanalysis
return S_OK(jobfccanalysis)
def runJobLocally(self, job, jobName="unknown"):
"""run a job locally"""
self.log.notice("I will run the tests locally.")
from DIRAC import gConfig
localarea = gConfig.getValue("/LocalSite/LocalArea", "")
if not localarea:
self.log.error(
"You need to have /LocalSite/LocalArea defined in your dirac.cfg"
)
return S_ERROR()
if localarea.find("/afs") == 0:
self.log.error(
"Don't set /LocalSite/LocalArea set to /afs/... as you'll get to install there"
)
self.log.error(
"check ${HOME}/.dirac.cfg and ${DIRAC}/etc/dirac.cfg")
return S_ERROR()
self.log.notice("To run locally, I will create a temp directory here.")
curdir = os.getcwd()
tmpdir = tempfile.mkdtemp("", dir="./")
os.chdir(tmpdir)
# Jobs that need separate input files
specialJobs = ['root', 'ddsim']
filesForJob = {
'root': ['input2.root', 'input.root'],
'ddsim': ['FCalTB.tar.gz', 'Muon_50GeV_Fixed_cosTheta0.7.stdhep']
}
for specialName in specialJobs:
if "root" in jobName.lower() and specialName == "root":
with open("root.sh", "w") as rScript:
rScript.write("echo $ROOTSYS")
with open("func.C", "w") as rMacro:
rMacro.write('''
void func( TString string ) {
std::cout << string << std::endl;
TFile* file = TFile::Open(string);
file->ls();
}
''')
testfiledir = 'Testfiles'
for fileName in ['input.root', 'input2.root']:
shutil.copy(os.path.join(curdir, testfiledir, fileName),
os.getcwd())
print os.path.join(curdir, "input2.root"), os.getcwd()
if specialName in jobName.lower():
for fileName in filesForJob[specialName]:
shutil.copy(os.path.join(curdir, testfiledir, fileName),
os.getcwd())
resJob = self.runJob(job, jobName)
os.chdir(curdir)
if not resJob['OK']:
return resJob
os.chdir(curdir)
if not self.clip.nocleanup:
cleanup(tmpdir)
return S_OK()
def run(self):
"""submit and run all the tests in jobList"""
res = S_ERROR()
for name, finjob in self.jobList.iteritems():
if self.clip.submitMode == 'local':
res = self.runJobLocally(finjob, name)
else:
res = self.runJob(finjob, name)
return res
def runJob(self, finjob, name):
"""runs or submits the job"""
self.log.notice(
"############################################################")
self.log.notice(" Running or submitting job: %s " % name)
self.log.notice("\n\n")
res = finjob.submit(self.diracInstance, mode=self.clip.submitMode)
if not res["OK"]:
self.log.error("Failed job:", res['Message'])
return S_ERROR()
return S_OK()
def checkForTests(self):
"""check which tests to run"""
if self.clip.testMokka:
resMokka = self.createMokkaTest()
if not resMokka['OK']:
return S_ERROR()
if self.clip.testWhizard:
resWhiz = self.createWhizardTest()
if not resWhiz['OK']:
return S_ERROR()
if self.clip.testSlic:
resSlic = self.createSlicTest()
if not resSlic['OK']:
return S_ERROR()
if self.clip.testMarlin:
resMarlin = self.createMarlinTest()
if not resMarlin['OK']:
return S_ERROR()
if self.clip.testLCSIM:
resLCSim = self.createLCSimTest()
if not resLCSim['OK']:
return S_ERROR()
if self.clip.testSlicPandora:
resSP = self.createSlicPandoraTest()
if not resSP['OK']:
return S_ERROR()
if self.clip.testUtilities:
resUtil = self.createUtilityTests()
if not resUtil['OK']:
return S_ERROR()
if self.clip.testRoot:
resRoot = self.createRootScriptTest()
if not resRoot['OK']:
return S_ERROR()
resRoot = self.createRootHaddTest()
if not resRoot['OK']:
return S_ERROR()
resRoot = self.createRootMacroTest()
if not resRoot['OK']:
return S_ERROR()
if self.clip.testFccSw:
resFccSw = self.createFccSwTest()
if not resFccSw['OK']:
return S_ERROR()
if self.clip.testFccAnalysis:
resFccAnalysis = self.createFccAnalysisTest()
if not resFccAnalysis['OK']:
return S_ERROR()
return S_OK()
def subOverlay():
# Decide parameters for a job
outputSE = "KEK-SRM"
isLocal = _clip.isLocal
nbevts = 50 if _clip.numberOfEvents == 0 else _clip.numberOfEvents
nbevts = 0 # To analize all input events
outputFilePrefix = "overlay_example" if _clip.outputFilePrefix == "" else _clip.outputFilePrefix
outputDir = _clip.outputDir
inputFile = _clip.inputFile
if inputFile == "":
gLogger.error("Input file for ddsim does not given.")
exit(-1)
recfile = outputFilePrefix + ".rec.slcio"
dstfile = outputFilePrefix + ".dst.slcio"
detector_model = "ILD_l5_o1_v02"
key = detector_model.split('_')
sim_detectorModel = "_".join([key[0], key[1], key[3]])
# Create DIRAC objects for job submission
dIlc = DiracILC()
job = UserJob()
job.setJobGroup("myoverlayjob")
job.setName("myoverlay")
job.setOutputSandbox(['*.log', '*.sh', '*.py', '*.xml'])
job.setILDConfig("v02-00-02")
# job.setInputSandbox(["a6-parameters.sin", "P2f_qqbar.sin"])
# job.setDestination(["LCG.KEK.jp", "LCG.DESY-HH.de"]) # job submission destination
# job.setBannedSites([]) # a list of sites not to submit job
# job.setCPUTime( cputime_limit_in_seconds_by_dirac_units )
# Create Overlay application
ovldata = [{
"ProcessorName": "BgOverlayWW",
"evttype": "aa_lowpt_WW",
"ProdID": 10237,
"expBG": 0.211,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayWB",
"evttype": "aa_lowpt_WB",
"ProdID": 10241,
"expBG": 0.24605,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBW",
"evttype": "aa_lowpt_BW",
"ProdID": 10239,
"expBG": 0.243873,
"subdir": "000"
}, {
"ProcessorName": "BgOverlayBB",
"evttype": "aa_lowpt_BB",
"ProdID": 10235,
"expBG": 0.35063,
"subdir": "000"
}, {
"ProcessorName": "PairBgOverlay",
"evttype": "seeablepairs",
"ProdID": 10233,
"expBG": 1.0,
"subdir": "100"
}]
BXOverlay = 1
NbSigEvtsPerJob = 100
numberOfSignalEvents = NbSigEvtsPerJob
basebkgpath = "/ilc/prod/ilc/mc-opt-3/ild/sim/500-TDR_ws"
energy = "500"
for ovl in ovldata:
print "### OverlayInput ... " + ovl["ProcessorName"]
ovlapp = OverlayInput()
ovlpath = "%s/%s/%s/v02-00-01/%8.8d/%s" % \
( basebkgpath, ovl["evttype"], sim_detectorModel, ovl["ProdID"] , ovl["subdir"] )
print " OverlayPath ... " + ovlpath
ovlapp.setMachine("ilc_dbd")
# ovlapp.setEnergy(energy)
# ovlapp.setDetectorModel(sim_detectorModel)
ovlapp.setProcessorName(ovl["ProcessorName"])
ovlapp.setBkgEvtType(ovl["evttype"])
ovlapp.setPathToFiles(ovlpath)
ovlapp.setGGToHadInt(ovl["expBG"])
ovlapp.setBXOverlay(BXOverlay)
ovlapp.setNbSigEvtsPerJob(NbSigEvtsPerJob)
ovlapp.setNumberOfSignalEventsPerJob(numberOfSignalEvents)
res = job.append(ovlapp)
if not res['OK']:
print res['Message']
exit(1)
# Create Marlin application
marlin = Marlin()
marlin.setVersion("ILCSoft-02-00-02_gcc49")
marlin.setDetectorModel(detector_model)
marlin.setSteeringFile("MarlinStdReco.xml")
marlin.setInputFile(inputFile)
marlin.setNumberOfEvents(nbevts)
marlin.setOutputDstFile(dstfile)
marlin.setOutputRecFile(recfile)
extraCLIArguments = " --constant.DetectorModel=%s " % detector_model
extraCLIArguments += " --constant.RunOverlay=true --constant.CMSEnergy=%s " % str(
energy)
extraCLIArguments += " --global.Verbosity=MESSAGE "
marlin.setExtraCLIArguments(extraCLIArguments)
job.append(marlin)
if outputDir != "":
job.setOutputData([dstfile, recfile],
OutputPath=outputDir,
OutputSE=outputSE)
if isLocal:
job.submit(dIlc, mode="local")
else:
job.submit(dIlc)
class UserJob(SplitMixin, Job):
"""User job class. To be used by users, not for production."""
def __init__(self, script=None):
"""Initialize UserJob, including proxy and splitmixin."""
super(UserJob, self).__init__(script)
self.type = 'User'
self.diracinstance = None
self.usergroup = ['ilc_user', 'calice_user']
self.proxyinfo = getProxyInfo()
SplitMixin._initialize(self)
def submit(self, diracinstance = None, mode = "wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
:param diracinstance: DiracILC instance
:type diracinstance: ~ILCDIRAC.Interfaces.API.DiracILC.DiracILC
:param str mode: "wms" (default), "agent", or "local"
.. note ::
The *local* mode means that the job will be run on the submission machine. Use this mode for testing of
submission scripts
"""
if self._splittingOption:
result = self._split()
if 'OK' in result and not result['OK']:
return result
# Check the credentials. If no proxy or not user proxy, return an error
if not self.proxyinfo['OK']:
LOG.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup)
return self._reportError("Not allowed to submit a job, you need a %s proxy." % self.usergroup,
self.__class__.__name__)
if 'group' in self.proxyinfo['Value']:
group = self.proxyinfo['Value']['group']
if group not in self.usergroup:
LOG.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup)
return self._reportError("Not allowed to submit job, you need a %s proxy." % self.usergroup,
self.__class__.__name__)
else:
LOG.error("Could not determine group, you do not have the right proxy.")
return self._reportError("Could not determine group, you do not have the right proxy.")
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submitJob(self, mode)
#############################################################################
def setInputData( self, lfns ):
"""Specify input data by Logical File Name (LFN).
Input files specified via this function will be automatically staged if necessary.
Example usage:
>>> job = UserJob()
>>> job.setInputData(['/ilc/prod/whizard/processlist.whiz'])
:param lfns: Logical File Names
:type lfns: Single LFN string or list of LFNs
"""
if isinstance( lfns, list ) and lfns:
for i, lfn in enumerate( lfns ):
lfns[i] = lfn.replace( 'LFN:', '' )
#inputData = map( lambda x: 'LFN:' + x, lfns )
inputData = lfns #because we don't need the LFN: for inputData, and it breaks the
#resolution of the metadata in the InputFilesUtilities
inputDataStr = ';'.join( inputData )
description = 'List of input data specified by LFNs'
self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description )
elif isinstance( lfns, basestring ): #single LFN
description = 'Input data specified by LFN'
self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description )
else:
kwargs = {'lfns':lfns}
return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs )
return S_OK()
def setInputSandbox(self, flist):
""" Add files to the input sandbox, can be on the local machine or on the grid
>>> job = UserJob()
>>> job.setInputSandbox( ['LFN:/ilc/user/u/username/libraries.tar.gz',
>>> 'mySteeringFile.xml'] )
:param flist: Files for the inputsandbox
:type flist: `python:list` or `str`
"""
if isinstance( flist, basestring ):
flist = [flist]
if not isinstance( flist, list ):
return self._reportError("File passed must be either single file or list of files.")
self.inputsandbox.extend(flist)
return S_OK()
#############################################################################
def setOutputData(self, lfns, OutputPath = '', OutputSE = ''):
"""For specifying output data to be registered in Grid storage. If a list
of OutputSEs are specified the job wrapper will try each in turn until
successful.
Example usage:
>>> job = UserJob()
>>> job.setOutputData(['Ntuple.root'])
:param lfns: Output data file or files
:type lfns: Single `str` or `python:list` of strings ['','']
:param str OutputPath: Optional parameter to specify the Path in the Storage, postpended to /ilc/user/u/username/
:param OutputSE: Optional parameter to specify the Storage Element to store data or files, e.g. CERN-SRM
:type OutputSE: `python:list` or `str`
"""
kwargs = {'lfns' : lfns, 'OutputSE' : OutputSE, 'OutputPath' : OutputPath}
if isinstance( lfns, list ) and lfns:
outputDataStr = ';'.join(lfns)
description = 'List of output data files'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', outputDataStr, description)
elif isinstance( lfns, basestring ):
description = 'Output data file'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns, description)
else:
return self._reportError('Expected file name string or list of file names for output data', **kwargs)
if OutputSE:
description = 'User specified Output SE'
if isinstance( OutputSE, basestring ):
OutputSE = [OutputSE]
elif not isinstance( OutputSE, list ):
return self._reportError('Expected string or list for OutputSE', **kwargs)
OutputSE = ';'.join(OutputSE)
self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE, description)
if OutputPath:
description = 'User specified Output Path'
if not isinstance( OutputPath, basestring ):
return self._reportError('Expected string for OutputPath', **kwargs)
# Remove leading "/" that might cause problems with os.path.join
while OutputPath[0] == '/':
OutputPath = OutputPath[1:]
if OutputPath.count("ilc/user"):
return self._reportError('Output path contains /ilc/user/ which is not what you want', **kwargs)
self._addParameter(self.workflow, 'UserOutputPath', 'JDL', OutputPath, description)
return S_OK()
#############################################################################
def setOutputSandbox( self, files ):
"""Specify output sandbox files. If specified files are over 10MB, these
may be uploaded to Grid storage with a notification returned in the
output sandbox.
.. Note ::
Sandbox files are removed after 2 weeks.
Example usage:
>>> job = UserJob()
>>> job.setOutputSandbox(['*.log','*.sh', 'myfile.txt'])
Use the output sandbox only for small files. Larger files should be stored
on the grid and downloaded later if necessary. See :func:`setOutputData`
:param files: Output sandbox files
:type files: Single `str` or `python:list` of strings ['','']
"""
if isinstance( files, list ) and files:
fileList = ";".join( files )
description = 'Output sandbox file list'
self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description )
elif isinstance( files, basestring ):
description = 'Output sandbox file'
self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description )
else:
kwargs = {'files' : files}
return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs )
return S_OK()
def setILDConfig(self,version):
""" Define the Configuration package to obtain
"""
appName = 'ILDConfig'
self._addSoftware(appName.lower(), version)
self._addParameter( self.workflow, 'ILDConfigPackage', 'JDL', appName+version, 'ILDConfig package' )
return S_OK()
def setCLICConfig(self, version):
"""Define the CLIC Configuration package to obtain, copies steering files
from CLIC Configuration folder to working directory
:param str version: version string, e.g.: 'ILCSoft-2017-07-27'
"""
appName = 'ClicConfig'
self._addSoftware(appName.lower(), version)
self._addParameter( self.workflow, 'ClicConfigPackage', 'JDL', appName+version, 'CLIC Config package' )
return S_OK()
def main():
'''
This is the script main method, which will hold all the logic.
'''
cliParams = Params()
cliParams.registerSwitches()
Script.parseCommandLine(ignoreErrors=True)
consistent = cliParams.checkConsistency()
if not consistent['OK']:
gLogger.error("Error checking consistency:", consistent['Message'])
Script.showHelp()
dexit(2)
##Get prodID
prodID = float(cliParams.prodID)
##Get all possible input files
inputFiles = getInputFiles(prodID)
##Get suffix
suffix = cliParams.suffix
if suffix: suffix = "_" + suffix
##Jet clustering
algorithm = "ValenciaPlugin {radius:.1f} {beta:.1f} {gamma:.1f}"
jetRadius = float(cliParams.jetRadius)
jetBeta = float(cliParams.jetBeta)
jetGamma = float(cliParams.jetGamma)
jetCluster = "ExclusiveNJets 2"
jetRecomb = "E_scheme"
##Top tagger
deltaR = float(cliParams.deltaR)
deltaP = float(cliParams.deltaP)
cos_theta_W_max = float(cliParams.cos_theta_W_max)
##Sixfermion sample
sixFermionSample = cliParams.sixFermionSample
##Number of files per job
nrFilesPerJob = int(cliParams.nrFilesPerJob)
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
repDir = "/afs/cern.ch/user/l/lstroem/clicdp/analysis/steering/chain/topasymmetry_wflavourtag/submit/{prodID:04.0f}/rep/".format(
prodID=prodID)
subprocess.call("mkdir -p " + repDir, shell=True)
dirac = DiracILC(
False
) #, repDir+"topasymmetry_vlc{suffix}.rep".format(suffix = suffix))
jetAlgo = algorithm.format(radius=jetRadius, beta=jetBeta, gamma=jetGamma)
inputFileList = []
i = 0
j = 1
for inputFile in inputFiles:
inputFileList.append(inputFile)
i += 1
if (i >= nrFilesPerJob * j) or (i == len(inputFiles)):
jobName = "topasymmetry_chain_{jetSettings}_dR{deltaR:.2f}_dP{deltaP:.2f}_cthetaWmax{cos_theta_W_max:.2f}_1Jun2017_part{index}_{prodID:04.0f}".format(
jetSettings=getJetSettings(jetAlgo, jetCluster, jetRecomb),
deltaR=deltaR,
deltaP=deltaP,
cos_theta_W_max=cos_theta_W_max,
index=j,
prodID=prodID)
jobGroup = "topasymmetry_chain_{prodID:04.0f}".format(
prodID=prodID)
job, outputFile, rootFile = defGridJob(jobName, jobGroup,
inputFileList)
##Check if outputfile already exists
pOutCheck = subprocess.Popen(
"dirac-dms-lfn-replicas /ilc/user/r/rstrom/" + jobGroup + "/" +
rootFile,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
outCheck, errCheck = pOutCheck.communicate()
#gLogger.notice(jobName)
if "no such file" not in outCheck.lower():
gLogger.notice('File exists! Skipping!')
inputFileList = []
j += 1
continue #continue #use break if only part1, use continue of run over all parts
#gLogger.notice(jetAlgo)
#doLogger(jobName, inputFileList, outputFile, rootFile)
res = job.append(
defMarlin(outputFile,
rootFile,
jetAlgo=jetAlgo,
jetCluster=jetCluster,
jetRecomb=jetRecomb,
deltaR=deltaR,
deltaP=deltaP,
cos_theta_W_max=cos_theta_W_max,
sixFermionSample=sixFermionSample))
if not res['OK']: #Catch if there is an error
print res['Message'] #Print the error message
dexit
##Job submission
print job.submit(dirac)
inputFileList = []
j += 1
#break #add break of only part1
gLogger.notice("All done!")
dexit(0)
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard, Mokka, Marlin, OverlayInput
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from DIRAC import exit as dexit
dirac =DiracILC()
#wh.setOutputFile("myfile.stdhep")
j = UserJob()
wh = Whizard(processlist=dirac.getProcessList())
wh.setEnergy(3000)
wh.setEvtType("ee_h_mumu")
wh.setNbEvts(1)
wh.setEnergy(3000)
params = {}
params['USERB1']='F'
wh.setParameterDict(params)
wh.setModel("sm")
res = j.append(wh)
if not res['OK']:
print res['Message']
dexit(1)
mo = Mokka()
class DiracILCTestCase( unittest.TestCase ):
""" Base class for the DiracILC test cases
"""
def setUp( self ):
"""set up the objects"""
ops_mock = Mock()
mocked_modules = { 'DIRAC.ConfigurationSystem.Client.Helpers.Operations' : ops_mock }
self.module_patcher = patch.dict( sys.modules, mocked_modules )
self.module_patcher.start()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
self.dilc = DiracILC()
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['SE']
if 'Minimum' in args[0]:
return 1
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = Mock()
ops_mock.getValue.side_effect = setOptions
self.dilc.ops = ops_mock
def tearDown( self ):
self.module_patcher.stop()
def test_getprocesslist( self ):
with patch('%s.gConfig.getValue' % MODULE_NAME, new=Mock(return_value='some_gconf_testval')) as conf_mock, \
patch('%s.ProcessList' % MODULE_NAME, new=Mock()) as pl_mock:
res = self.dilc.getProcessList()
pl_mock.assert_called_once_with( 'some_gconf_testval' )
assertEqualsImproved( res, pl_mock(), self )
conf_mock.assert_called_once_with( '/LocalSite/ProcessListPath', '' )
def test_getprocesslist_nopath( self ):
ops_mock = Mock()
ops_mock.getValue.return_value = ''
self.dilc.ops = ops_mock
with patch('%s.gConfig.getValue' % MODULE_NAME, new=Mock(return_value='')) as conf_mock, \
patch('%s.ProcessList' % MODULE_NAME, new=Mock()) as pl_mock:
res = self.dilc.getProcessList()
pl_mock.assert_called_once_with( '' )
assertEqualsImproved( res, pl_mock(), self )
conf_mock.assert_called_once_with( '/LocalSite/ProcessListPath', '' )
ops_mock.getValue.assert_called_once_with( '/ProcessList/Location', '' )
def test_presubmissionchecks_notoktosubmit( self ):
job_mock = Mock()
job_mock.oktosubmit = False
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'you should use job.submit(dirac)', self )
def test_presubmissionchecks_checkfails( self ):
job_mock = Mock()
job_mock.oktosubmit = True
with patch('%s.DiracILC._do_check' % MODULE_NAME, new=Mock(return_value=S_ERROR('mytest_check_failed'))):
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'mytest_check_failed', self )
def test_presubmissionchecks_askuser_fails( self ):
job_mock = Mock()
job_mock.oktosubmit = True
job_mock._askUser.return_value = S_ERROR( 'user says no' )
self.dilc.checked = False
with patch('%s.DiracILC._do_check' % MODULE_NAME, new=Mock(return_value=S_OK())):
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'user says no', self )
def test_checkparams( self ):
job_mock = Mock()
job_mock.errorDict = {}
with patch('%s.DiracILC.preSubmissionChecks' % MODULE_NAME, new=Mock(return_value=S_OK('mytest'))) as check_mock:
assertDiracSucceedsWith( self.dilc.checkparams( job_mock ), 'mytest', self )
check_mock.assert_called_once_with( job_mock, mode = '' )
def test_checkparams_fails( self ):
job_mock = Mock()
job_mock.errorDict = { 'myerror1' : [ 'Terrible failure' ], 'last_error' : [ 'True' ] }
assertDiracFailsWith_equals( self.dilc.checkparams( job_mock ),
{ 'myerror1' : [ 'Terrible failure' ], 'last_error' : [ 'True' ] }, self )
def test_giveprocesslist( self ):
self.dilc.processList = '13985u185r9135r'
assertEqualsImproved( self.dilc.giveProcessList(), '13985u185r9135r', self )
def test_giveprocesslist_empty( self ):
self.dilc.processList = ''
assertEqualsImproved( self.dilc.giveProcessList(), '', self )
def test_giveprocesslist_false( self ):
self.dilc.processList = False
assertEqualsImproved( self.dilc.giveProcessList(), False, self )
def test_giveprocesslist_none( self ):
self.dilc.processList = None
assertEqualsImproved( self.dilc.giveProcessList(), None, self )
def test_retrievelfns_norepo( self ):
self.dilc.jobRepo = None
assertDiracSucceeds( self.dilc.retrieveRepositoryOutputDataLFNs(), self )
def test_retrievelfns( self ):
repo_mock = Mock()
ret_dict = { '1' : { 'State' : 'Done', 'UserOutputData' : '1389' }, '2' : {},
'3' : { 'State' : 'secret_teststate' }, '4' : { 'State' : 'invalid_state' },
'5' : { 'State' : 'Done', 'UserOutputData' : 0 }, '6' : { 'ignore_me' : True },
'7' : { 'State' : 'secret_teststate', 'UserOutputData' : 0 },
'148' : { 'State' : 'Done', 1 : False, True : 941, 'values_' : 'keys' } }
repo_mock.readRepository.return_value = S_OK( ret_dict )
self.dilc.jobRepo = repo_mock
with patch('%s.DiracILC.parameters' % MODULE_NAME, new=Mock(side_effect=[S_OK({'UploadedOutputData':'/my/test/lfn1'}),S_ERROR(),S_OK({}),S_OK({'some_entries':'some_values',1:True,'UploadedOutputData':'/more_lfns/append/testlfn.log'})])) as param_mock:
assertEqualsImproved( self.dilc.retrieveRepositoryOutputDataLFNs( [ 'Done', 'secret_teststate' ] ),
[ '/my/test/lfn1', '/more_lfns/append/testlfn.log' ], self )
assertMockCalls( param_mock, [ 3, 5, 7, 148 ], self )
def test_docheck_checksandbox_fails( self ):
job_mock = Mock()
job_mock.inputsandbox = [ 'mysandbox', 'other_value' ]
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_ERROR('test_err_sandbox'))) as check_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'test_err_sandbox', self )
check_mock.assert_called_once_with( job_mock )
def test_docheck_too_many_lists( self ):
job_mock = Mock()
job_mock.inputsandbox = [ 'mysandbox', [ 'im_a_sandbox_file.stdhep', [ 'evil_list', 'deletethis'] ] ]
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'too many lists of lists in the input sandbox', self )
def test_docheck_checkapps_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock }
job_mock = Mock()
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(side_effect=[S_OK()] * 3 + [S_ERROR('checkapp_failed_testme')])) as checkapp_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'checkapp_failed_testme', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
def test_docheck_checkoutputpath_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
path_mock = Mock()
path_mock.getValue.return_value = 'path1948512895'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock, 'UserOutputPath' : path_mock }
job_mock = Mock()
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \
patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_ERROR('outputpath_check_testerr'))) as checkpath_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'outputpath_check_testerr', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
checkpath_mock.assert_called_once_with( 'path1948512895' )
def test_docheck_checkconsistency_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
path_mock = Mock()
path_mock.getValue.return_value = 'path1948512895'
data_mock = Mock()
data_mock.getValue.return_value = 'data1389518'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock,
'UserOutputPath' : path_mock, 'UserOutputData' : data_mock }
job_mock = Mock()
job_mock.addToOutputSandbox = 'job_sandbox13895'
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \
patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkpath_mock, \
patch('%s.DiracILC._checkdataconsistency' % MODULE_NAME, new=Mock(return_value=S_ERROR('consistency_testerr'))) as checkconsistency_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'consistency_testerr', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
checkpath_mock.assert_called_once_with( 'path1948512895' )
checkconsistency_mock.assert_called_once_with( 'data1389518', 'job_sandbox13895' )
def test_checkapp( self ):
ops_mock = Mock()
ops_mock.getValue.return_value = ''
self.dilc.ops = ops_mock
assertDiracFailsWith( self.dilc._checkapp( 'test_platform_341', 'testapp', 'v13.2' ),
'could not find the specified software testapp_v13.2 for test_platform_341, check in CS',
self )
assertMockCalls( ops_mock.getValue, [
( '/AvailableTarBalls/test_platform_341/testapp/v13.2/TarBall', '' ),
( '/AvailableTarBalls/test_platform_341/testapp/v13.2/CVMFSPath', '' ) ], self )
def test_checkoutputpath_invalidchar_1( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( 'http://www.mysitedoesnotexist3h3.abc/some/file.txt' ),
'invalid path', self )
def test_checkoutputpath_invalidchar_2( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/./some/file.log' ),
'invalid path', self )
def test_checkoutputpath_invalidchar_3( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/../dir2/somefile.txt' ),
'invalid path', self )
def test_checkoutputpath_trailing_slash( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/myfile.txt/ ' ),
'invalid path', self )
def test_checkdataconsistency_outputdata_sandbox_equal( self ):
assertDiracFailsWith( self.dilc._checkdataconsistency( 'same_item;something_else',
[ 'distinct_item1', 'same_item' ] ),
'output data and sandbox should not contain the same thing', self )
def test_checkdataconsistency_wildcardchar( self ):
assertDiracFailsWith( self.dilc._checkdataconsistency(
'/test/dir/file.txt;/mydir/something/*;/file/dir/log.log',
[ '/input/sandbox.pdf', '/other/sb/file.stdhep' ] ),
'wildcard character in outputdata definition', self )
def test_checkinputsb_getreplicas_notok( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_err'))) as replica_mock:
assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'could not get replicas', self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb_getreplicas_fails( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
ret_dict = { 'Failed' : [ '/failed/replica1', '/other/inval/replica' ], 'Successful' : {} }
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'failed to find replicas', self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
ret_dict = {'Failed': [], 'Successful': {'/one/replica': {'SE': 'surl'}}}
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracSucceeds( self.dilc.checkInputSandboxLFNs( job_mock ), self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb_notInputSB(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value = None
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_notInputSB_Value(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = ''
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_noLFNs(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep'
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_noRepl(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = 'LFN:/some/file.txt'
ret_dict = {'Failed': [], 'Successful': {'/some/file.txt': {'Bad-SE': 'surl'}}}
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['Awesome-Disk-SE']
if 'Minimum' in args[0]:
return 2
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = setOptions
self.dilc.ops = ops_mock
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracFailsWith(self.dilc.checkInputSandboxLFNs(job_mock), 'Not enough replicas', self)
replica_mock.assert_called_once_with(['/some/file.txt'])
def test_checkinputsb_goodRepl(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = 'LFN:/some/file.txt'
ret_dict = {'Failed': [], 'Successful': {'/some/file.txt': {'Awesome-Disk-SE': 'surl'}}}
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['Awesome-Disk-SE']
if 'Minimum' in args[0]:
return 2
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = setOptions
self.dilc.ops = ops_mock
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
replica_mock.assert_called_once_with(['/some/file.txt'])
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka, Marlin, OverlayInput
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from DIRAC import exit as dexit
d = DiracILC(True, "repo.rep")
n_evts = 500
n_evts_per_job = 100
n_jobs = n_evts / n_evts_per_job
for i in range(n_jobs):
j = UserJob()
mo = Mokka()
mo.setEnergy(3000)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setMacFile("particlegun_electron.mac")
mo.setOutputFile("MyFile.slcio")
mo.setNbEvts(n_evts_per_job)
res = j.append(mo)
if not res['OK']:
print res['Message']
break
ma = Marlin()
ma.setVersion("v0111Prod")
class JobCreater(object):
"""contains all the versions and parameters to create all theses tests"""
# pylint: disable=too-many-instance-attributes
# Test parameters, necessary due to amount of tests in this class.
def __init__( self,
clip,
params
):
self.clip = clip
self.ildConfig = params.get( "ildConfig", None )
self.alwaysOverlay = params.get( "alwaysOverlay", False )
self.runOverlay = self.clip.testOverlay or self.alwaysOverlay
self.mokkaVersion = params["mokkaVersion"]
self.mokkaSteeringFile = params.get( "mokkaSteeringFile" )
self.detectorModel = params.get( "detectorModel" )
self.marlinVersion = params.get( "marlinVersion" )
self.marlinSteeringFile = params.get( "marlinSteeringFile" )
self.ddsimVersion = params.get( "ddsimVersion" )
self.ddsimDetectorModel = params.get( "ddsimDetectorModel")
self.ddsimInputFile = params.get( "ddsimInputFile" )
self.marlinInputdata = params.get ( "marlinInputdata" )
self.gearFile = params.get( "gearFile" )
self.lcsimVersion = params.get( "lcsimVersion" )
self.steeringFileVersion = params.get( "steeringFileVersion", None )
self.rootVersion = params["rootVersion"]
self.whizard2Version = params.get( "whizard2Version" )
self.whizard2SinFile = params.get( "whizard2SinFile" )
self.energy = params.get("energy")
self.backgroundType = params.get("backgroundType")
self.machine = params.get("machine")
self.gearFile = params.get( "gearFile" )
self.marlinSteeringFile = params.get( "marlinSteeringFile" )
self.marlinVersion = params.get( "marlinVersion" )
self.lcsimPreSteeringFile = params.get( "lcsimPreSteeringFile" )
self.lcsimPostSteeringFile = params.get( "lcsimPostSteeringFile" )
self.fccSwPath = params.get( "fccSwPath" )
self.fccSwSteeringFile = params.get( "fccSwSteeringFile" )
self.fccAnalysisSteeringFile = params.get( "fccAnalysisSteeringFile" )
### other things needed to run tests
self.log = gLogger.getSubLogger("JobCreater")
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid
from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid
if self.clip.submitMode == "local":
self.log.notice("")
self.log.notice(" DIRAC RCSID:", drcsid )
self.log.notice(" Job RCSID:", jrcsid )
self.log.notice("Applications RCSID:", apprcsid )
self.log.notice("")
self.diracInstance = DiracILC(False, 'tests.rep')
self.jobList = {}
def createDDSimTest( self, inputfile = None, detectorModel = None):
"""Create a job running ddsim"""
if inputfile is None:
inputfile = self.ddsimInputFile
sandbox = [ inputfile ]
if detectorModel is None:
detectorModel = self.ddsimDetectorModel
else:
sandbox.append(detectorModel)
jobdd = self.getJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim
ddsim = DDSim()
ddsim.setVersion(self.ddsimVersion)
ddsim.setDetectorModel(detectorModel)
ddsim.setNumberOfEvents(1)
ddsim.setInputFile(inputfile)
jobdd.setInputSandbox(sandbox)
res = jobdd.append(ddsim)
if not res['OK']:
self.log.error("Failed adding DDSim:", res['Message'])
return S_ERROR("Failed adding DDSim to Job")
return S_OK(jobdd)
def createWhizard2Test( self ):
"""Create a job running Whizard2"""
jobdd = self.getJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications.Whizard2 import Whizard2
whiz = Whizard2()
whiz.setVersion(self.whizard2Version)
whiz.setNumberOfEvents(1)
whiz.setSinFile(self.whizard2SinFile)
whiz.setOutputFile("test.stdhep")
res = jobdd.append(whiz)
if not res['OK']:
self.log.error("Failed adding Whizard2:", res['Message'])
return S_ERROR("Failed adding Whizard2 to Job")
return S_OK(jobdd)
def createMokkaTest(self):
"""create a job running mokka, and maybe whizard before"""
self.log.notice("Creating jobs for Mokka")
#(Whizard + )Mokka
jobmo = self.getJob()
if self.clip.testChain:
whmo = self.getWhizard(2)
res = jobmo.append(whmo)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR("Failed adding Whizard")
elif self.clip.testInputData:
jobmo.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep")
else:
self.log.error("Mokka does not know where to get its input from")
return S_ERROR("Mokka does not know where to gets its input from")
mo = self.getMokka()
if self.clip.testChain:
mo.getInputFromApp(whmo)
else:
mo.setNumberOfEvents(1)
res = jobmo.append(mo)
if not res['OK']:
self.log.error("Failed adding Mokka:", res['Message'])
return S_ERROR("Failed adding Mokka to Job")
jobmo.setOutputData("testsim.slcio", OutputSE="CERN-DIP-4")
self.jobList['Mokka1'] = jobmo
return S_OK(jobmo)
def createRootScriptTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["root.sh", "input.root","input2.root"])
root = self.getRoot()
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def createRootHaddTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["input.root","input2.root"])
root = self.getRoot()
root.setScript("hadd")
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def createRootMacroTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["func.C", "input.root","input2.root"])
root = self.getRootMacro()
root.setScript("func.C")
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def getOverlay(self, nbevts):
""" Create an overlay step
"""
pathToFiles = None
from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput
overlay = OverlayInput()
if self.energy==350:
if self.detectorModel=="ILD_o1_v05":
pathToFiles="/ilc/user/s/sailer/testFiles/overlay/ild_350/"
if pathToFiles:
overlay.setPathToFiles(pathToFiles)
else:
self.log.warn("better define pathToFiles for this overlay: %s, %s, %s" %
(self.energy, self.machine, self.backgroundType) )
overlay.setMachine(self.machine)
overlay.setEnergy(self.energy)
overlay.setDetectorModel(self.detectorModel)
overlay.setBkgEvtType(self.backgroundType)
overlay.setBXOverlay(60)
overlay.setGGToHadInt(0.3)
overlay.setNumberOfSignalEventsPerJob(nbevts)
return overlay
def getMokka(self):
""" Define a mokka app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka
mokka = Mokka()
mokka.setVersion(self.mokkaVersion)
mokka.setSteeringFile(self.mokkaSteeringFile)
mokka.setOutputFile("testsim.slcio")
mokka.setDetectorModel(self.detectorModel)
if self.steeringFileVersion:
mokka.setSteeringFileVersion(self.steeringFileVersion)
return mokka
def getRoot(self):
""" Define a root app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootScript
root = RootScript()
root.setScript("root.sh")
root.setArguments("output.root input.root input2.root")
root.setVersion(self.rootVersion)
root.setOutputFile("output.root")
return root
def getRootMacro(self):
""" Define a root app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootMacro
root = RootMacro()
root.setMacro("func.C")
root.setArguments(r"\"input.root\"")
root.setVersion(self.rootVersion)
return root
@staticmethod
def getSLIC():
""" Get a SLIC instance
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
slic.setOutputFile('testsim.slcio')
return slic
@staticmethod
def getSLICPandora():
""" Get some SLICPAndora app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLICPandora
slicp = SLICPandora()
slicp.setVersion('CLIC_CDR')
slicp.setDetectorModel('clic_sid_cdr')
slicp.setPandoraSettings("PandoraSettingsSlic.xml")
slicp.setOutputFile('testpandora.slcio')
return slicp
def getMarlin( self ):
""" Define a marlin step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
# marlin.setVersion("v0111Prod")
marlin.setVersion(self.marlinVersion)
marlin.setSteeringFile(self.marlinSteeringFile)
marlin.setGearFile(self.gearFile)
marlin.setOutputDstFile("testmarlinDST.slcio")
marlin.setOutputRecFile("testmarlinREC.slcio")
marlin.setNumberOfEvents(1)
return marlin
def getDD( self ):
""" Create a DDSim object
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim
ddsim = DDSim()
ddsim.setVersion(self.ddsimVersion)
ddsim.setDetectorModel(self.ddsimDetectorModel)
ddsim.setInputFile(self.ddsimInputFile)
ddsim.setNumberOfEvents(2)
return ddsim
def getLCSIM(self, prepandora = True):
""" Get some LCSIM
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM
lcsim = LCSIM()
lcsim.setVersion('CLIC_CDR')
lcsim.setDetectorModel('clic_sid_cdr.zip')
if prepandora:
lcsim.setSteeringFile(self.lcsimPreSteeringFile)
lcsim.setOutputFile("testlcsim.slcio")
else:
lcsim.setSteeringFile(self.lcsimPostSteeringFile)
#lcsim.setOutputFile("testlcsimfinal.slcio")
lcsim.setOutputDstFile("testlcsimDST.slcio")
lcsim.setOutputRecFile("testlcsimREC.slcio")
lcsim.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml")
return lcsim
def getFccSw( self ):
""" Define a fccsw step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccSw
fccsw = FccSw()
fccsw.fccSwPath = self.fccSwPath
fccsw.setSteeringFile(self.fccSwSteeringFile)
return fccsw
def getFccAnalysis( self ):
""" Define a fccanalysis step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccAnalysis
fccanalysis = FccAnalysis()
fccanalysis.setSteeringFile(self.fccAnalysisSteeringFile)
return fccanalysis
@staticmethod
def getStdhepcut(generated):
""" Get some cuts in
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdhepCutJava
stdhepcut = StdhepCutJava()
stdhepcut.setVersion('1.0')
stdhepcut.setSelectionEfficiency(1.)
#stdhepcut.setSteeringFile("cuts_testing_1400.txt")
stdhepcut.setInlineCuts("leptonInvMass_R 13 100 200")
stdhepcut.setSteeringFileVersion("V18")
stdhepcut.setMaxNbEvts(1)
stdhepcut.setNumberOfEvents(generated)
return stdhepcut
@staticmethod
def getStdhepSplit():
""" Get some stdhep split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(5)
stdhepsplit.setOutputFile("teststdhepsplit.stdhep")
stdhepsplit.setMaxRead(10)
return stdhepsplit
@staticmethod
def getLCIOSplit(events_per_file):
""" Get a LCIO split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOSplit
lciosplit = SLCIOSplit()
lciosplit.setNumberOfEventsPerFile(events_per_file)
lciosplit.setOutputFile("testlciosplit.slcio")
return lciosplit
@staticmethod
def getLCIOConcat():
""" Get a LCIO Concat
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOConcatenate
lcioconcat = SLCIOConcatenate()
lcioconcat.setOutputFile("testlcioconcat.slcio")
return lcioconcat
def getJob(self):
""" Define a generic job, it should be always the same
"""
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
myjob = UserJob()
myjob.setName("Testing")
myjob.setJobGroup("Tests")
myjob.setCPUTime(30000)
myjob.dontPromptMe()
myjob.setLogLevel("VERBOSE")
myjob.setPlatform("x86_64-slc5-gcc43-opt")
myjob.setOutputSandbox(["*.log","*.xml", "*.sh"])
myjob._addParameter( myjob.workflow, 'TestFailover', 'String', True, 'Test failoverRequest')
myjob._addParameter( myjob.workflow, 'Platform', 'JDL', "x86_64-slc5-gcc43-opt", 'OS Platform')
if self.ildConfig:
myjob.setILDConfig(self.ildConfig)
return myjob
def getWhizardModel(self, nbevts, energy, model):
""" Create a default whizard
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard
proddict = "e2e2_o"
if model != "sm":
proddict = "se2se2_r"
whiz = Whizard(processlist = self.diracInstance.getProcessList())
whiz.setModel(model)
pdict = {}
pdict['process_input'] = {}
pdict['process_input']['process_id'] = proddict
pdict['process_input']['sqrts'] = energy
if model != 'sm':
pdict['process_input']['input_file'] = "LesHouches.msugra_1"
pdict['process_input']['input_slha_format'] = 'T'
pdict['process_input']['beam_recoil'] = 'T'
pdict['integration_input'] = {}
pdict['integration_input']['calls'] = '1 50000 10 5000 1 15000'
pdict['simulation_input'] = {}
pdict['simulation_input']['normalize_weight'] = 'F'
pdict['simulation_input']['n_events'] = nbevts
pdict['simulation_input']['keep_initials'] = 'T'
pdict['simulation_input']['events_per_file'] = 500000
if model != 'sm':
pdict['simulation_input']['pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1;IMSS(1)=11; IMSS(21)=71; IMSS(22)=71"
else:
pdict['simulation_input']['pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ; MSTJ(28)=2 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1"
pdict['parameter_input'] = {}
# pdict['parameter_input']['mmu']=mmu
# pdict['parameter_input']['mtau']=mtau
# pdict['parameter_input']['mb']=mb
# pdict['parameter_input']['mc']=mc
pdict['parameter_input']['mH'] = 125
pdict['beam_input_1'] = {}
pdict['beam_input_1']['particle_name'] = "e1"
pdict['beam_input_1']['polarization'] = "0.0 0.0"
pdict['beam_input_1']['USER_spectrum_on'] = 'T'
pdict['beam_input_1']['USER_spectrum_mode'] = 19 if energy == 1400 else 11
pdict['beam_input_1']['ISR_on'] = 'T'
pdict['beam_input_1']['EPA_on'] = "F"
pdict['beam_input_2'] = {}
pdict['beam_input_2']['particle_name'] = "E1"
pdict['beam_input_2']['polarization'] = "0.0 0.0"
pdict['beam_input_2']['USER_spectrum_on'] = 'T'
pdict['beam_input_2']['ISR_on'] = 'T'
pdict['beam_input_2']['USER_spectrum_mode'] = 19 if energy == 1400 else 11
pdict['beam_input_2']['EPA_on'] = 'F'
whiz.setFullParameterDict(pdict)
whiz.setOutputFile("testgen.stdhep")
return whiz
def getWhizard(self, nbevts):
""" Get some defualt SM whizard
"""
return self.getWhizardModel(nbevts, 1400, "sm")
def getWhizardSUSY(self, nbevts):
""" Get a susy whizard
"""
return self.getWhizardModel(nbevts, 3000, "slsqhh")
def createWhizardTest(self):
"""create a test for whizard"""
self.log.notice("Creating jobs for Whizard")
##### WhizardJob
jobw = self.getJob()
wh = self.getWhizard(2)
res = jobw.append(wh)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
self.jobList['Whizard1'] = jobw
##### WhizardJob
jobwsusy = self.getJob()
whsusy = self.getWhizardSUSY(2)
res = jobwsusy.append(whsusy)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
self.jobList['WhizSusy'] = jobwsusy
return S_OK((jobw, jobwsusy))
def createSlicTest(self):
"""create tests for slic"""
self.log.notice("Creating jobs for SLIC")
#run (Whizard +)SLIC
jobslic = self.getJob()
if self.clip.testChain:
whslic = self.getWhizard(2)
res = jobslic.append(whslic)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
elif self.clip.testInputData:
jobslic.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep")
else:
self.log.error("SLIC does not know where to get its input from")
return S_ERROR()
myslic = self.getSLIC()
if self.clip.testChain:
myslic.getInputFromApp(whslic)
else:
myslic.setNumberOfEvents(2)
res = jobslic.append(myslic)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
self.jobList['Slic1'] = jobslic
return S_OK(jobslic)
def createMarlinTest(self , setInputData = False):
"""create tests for marlin"""
self.log.notice( "Creating test for Marlin" )
#((Whizard + Mokka +)Overlay+) Marlin
jobma = self.getJob()
if self.clip.testChain:
moma = self.getMokka()
if not self.clip.testInputData:
whma = self.getWhizard(2)
res = jobma.append(whma)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
moma.getInputFromApp(whma)
else:
jobma.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep")
moma.setNumberOfEvents(1)
res = jobma.append(moma)
if not res['OK']:
self.log.error("Failed adding Mokka:", res['Message'])
return S_ERROR()
elif self.clip.testInputData:
jobma.setInputData(self.marlinInputdata)
else:
self.log.error("Marlin does not know where to get its input from")
return S_ERROR()
if self.runOverlay:
ov = self.getOverlay(2)
res = jobma.append(ov)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR
ma = self.getMarlin()
if self.clip.testChain:
ma.getInputFromApp(moma)
else:
ma.setNumberOfEvents(2)
res = jobma.append(ma)
if not res['OK']:
self.log.error("Failed adding Marlin:", res['Message'])
return S_ERROR()
self.jobList['Marlin1'] =jobma
return S_OK(jobma)
def createLCSimTest(self):
"""create tests for LCSIM"""
self.log.notice( "Creating test for LCSIM" )
#run ((whiz+SLIC+)+Overlay+)LCSIM
joblcsim = self.getJob()
if self.clip.testChain:
mysliclcsim = self.getSLIC()
if not self.clip.testInputData:
whlcsim = self.getWhizard(2)
res = joblcsim.append(whlcsim)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
mysliclcsim.getInputFromApp(whlcsim)
else:
joblcsim.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep")
mysliclcsim.setNumberOfEvents(2)
res = joblcsim.append(mysliclcsim)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
elif self.clip.testInputData:
#joblcsim.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio")
joblcsim.setInputData("/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio")
else:
self.log.error("LCSIM does not know where to get its input from")
return S_ERROR()
if self.runOverlay:
ovlcsim = self.getOverlay(2)
res = joblcsim.append(ovlcsim)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR()
mylcsim = self.getLCSIM(True)
if self.clip.testChain:
mylcsim.getInputFromApp(mysliclcsim)
else:
mylcsim.setNumberOfEvents(2)
res = joblcsim.append(mylcsim)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
self.jobList['lcsim1'] = joblcsim
return S_OK(joblcsim)
def createSlicPandoraTest(self):
"""create tests for slicPandora"""
self.log.notice("Creating tests for SLICPandora")
#run ((whiz+SLIC) + (Overlay +) LCSIM +) SLICPandora + LCSIM
joblcsimov = self.getJob()
if not self.clip.testChain:
self.log.error("SLICPandora does not know where to get its input from")
return S_ERROR()
mylcsimov = self.getLCSIM(True)
if not self.clip.testInputData:
whlcsimov = self.getWhizard(2)
res = joblcsimov.append(whlcsimov)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
mysliclcsimov = self.getSLIC()
mysliclcsimov.getInputFromApp(whlcsimov)
res = joblcsimov.append(mysliclcsimov)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
mylcsimov.getInputFromApp(mysliclcsimov)
else:
#joblcsimov.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio")
joblcsimov.setInputData("/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio")
mylcsimov.setNumberOfEvents(2)
if self.runOverlay:
ovslicp = self.getOverlay(2)
res = joblcsimov.append(ovslicp)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR()
res = joblcsimov.append(mylcsimov)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
myslicpov = self.getSLICPandora()
myslicpov.getInputFromApp(mylcsimov)
res = joblcsimov.append(myslicpov)
if not res['OK']:
self.log.error("Failed adding SLICPandora: ", res["Message"])
return S_ERROR()
mylcsimovp = self.getLCSIM(False)
mylcsimovp.getInputFromApp(myslicpov)
res = joblcsimov.append(mylcsimovp)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
self.jobList['lcsimov1'] = joblcsimov
return S_OK(joblcsimov)
def createUtilityTests(self):
"""Create tests for utility applications"""
self.log.notice("Creating tests for utility applications")
jobwsplit = self.getJob()
whsplit = self.getWhizard(10)
res = jobwsplit.append(whsplit)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
mystdsplit = JobCreater.getStdhepSplit()
mystdsplit.getInputFromApp(whsplit)
res = jobwsplit.append(mystdsplit)
if not res['OK']:
self.log.error("Failed adding StdHepSplit:", res['Message'])
return S_ERROR()
self.jobList['whizSplit'] = jobwsplit
##### WhizardJob + split
jobwcut = self.getJob()
whcut = self.getWhizard(100)
res = jobwcut.append(whcut)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
mystdcut = self.getStdhepcut( 100 )
mystdcut.getInputFromApp(whcut)
res = jobwcut.append(mystdcut)
if not res['OK']:
self.log.error("Failed adding StdHepCut:", res['Message'])
return S_ERROR()
self.jobList['whizCut'] = jobwcut
#LCIO split
joblciosplit = self.getJob()
# joblciosplit.setInputData("/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio")
joblciosplit.setInputData("/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio")
mylciosplit = self.getLCIOSplit(100)
res = joblciosplit.append(mylciosplit)
if not res['OK']:
self.log.error("Failed adding SLCIOSplit:", res['Message'])
return S_ERROR()
self.jobList['lcioSplit'] = joblciosplit
#LCIO concat
jobconcat = self.getJob()
# jobconcat.setInputData(["/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_27.slcio",
# "/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio"])
jobconcat.setInputData(["/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2104.slcio",
"/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2105.slcio"])
myconcat = self.getLCIOConcat()
res = jobconcat.append(myconcat)
if not res['OK']:
self.log.error("Failed adding SLCIOConcatenate:", res['Message'])
return S_ERROR()
self.jobList['concat'] = jobconcat
return S_OK((jobconcat, joblciosplit,jobwcut,jobwsplit))
def createFccSwTest(self):
"""create tests for fccsw"""
self.log.notice("Creating jobs for FccSW")
#### FccSwJob
jobfccsw = self.getJob()
fccsw = self.getFccSw()
res = jobfccsw.append(fccsw)
if not res['OK']:
self.log.error("Failed adding FccSw:", res['Message'])
return S_ERROR()
self.jobList['FccSw1'] = jobfccsw
return S_OK(jobfccsw)
def createFccAnalysisTest(self):
"""create tests for fccanalysis"""
self.log.notice("Creating jobs for FccAnalysis")
#### FccAnalysisJob
jobfccanalysis = self.getJob()
fccanalysis = self.getFccAnalysis()
res = jobfccanalysis.append(fccanalysis)
if not res['OK']:
self.log.error("Failed adding FccAnalysis:", res['Message'])
return S_ERROR()
self.jobList['FccAnalysis1'] = jobfccanalysis
return S_OK(jobfccanalysis)
def runJobLocally(self, job, jobName="unknown"):
"""run a job locally"""
self.log.notice("I will run the tests locally.")
from DIRAC import gConfig
localarea = gConfig.getValue("/LocalSite/LocalArea", "")
if not localarea:
self.log.error("You need to have /LocalSite/LocalArea defined in your dirac.cfg")
return S_ERROR()
if localarea.find("/afs") == 0:
self.log.error("Don't set /LocalSite/LocalArea set to /afs/... as you'll get to install there")
self.log.error("check ${HOME}/.dirac.cfg and ${DIRAC}/etc/dirac.cfg")
return S_ERROR()
self.log.notice("To run locally, I will create a temp directory here.")
curdir = os.getcwd()
tmpdir = tempfile.mkdtemp("", dir = "./")
os.chdir(tmpdir)
# Jobs that need separate input files
specialJobs = ['root', 'ddsim']
filesForJob = {
'root' : [ 'input2.root', 'input.root' ],
'ddsim' : [ 'FCalTB.tar.gz', 'Muon_50GeV_Fixed_cosTheta0.7.stdhep' ]
}
for specialName in specialJobs:
if "root" in jobName.lower() and specialName == "root":
with open("root.sh", "w") as rScript:
rScript.write( "echo $ROOTSYS" )
with open("func.C", "w") as rMacro:
rMacro.write( '''
void func( TString string ) {
std::cout << string << std::endl;
TFile* file = TFile::Open(string);
file->ls();
}
''' )
testfiledir = 'Testfiles'
for fileName in ['input.root', 'input2.root']:
shutil.copy( os.path.join( curdir, testfiledir, fileName ), os.getcwd() )
print(os.path.join(curdir, "input2.root"), os.getcwd())
if specialName in jobName.lower():
for fileName in filesForJob[specialName]:
shutil.copy( os.path.join( curdir, testfiledir, fileName ), os.getcwd() )
resJob = self.runJob(job, jobName)
os.chdir(curdir)
if not resJob['OK']:
return resJob
os.chdir(curdir)
if not self.clip.nocleanup:
cleanup(tmpdir)
return S_OK()
def run(self):
"""submit and run all the tests in jobList"""
res = S_ERROR()
for name, finjob in self.jobList.iteritems():
if self.clip.submitMode == 'local':
res = self.runJobLocally(finjob, name)
else:
res = self.runJob(finjob, name)
return res
def runJob(self, finjob, name):
"""runs or submits the job"""
self.log.notice("############################################################")
self.log.notice(" Running or submitting job: %s " % name)
self.log.notice("\n\n")
res = finjob.submit(self.diracInstance, mode = self.clip.submitMode)
if not res["OK"]:
self.log.error("Failed job:", res['Message'])
return S_ERROR()
return S_OK()
def checkForTests(self):
"""check which tests to run"""
if self.clip.testMokka:
resMokka = self.createMokkaTest()
if not resMokka['OK']:
return S_ERROR()
if self.clip.testWhizard:
resWhiz = self.createWhizardTest()
if not resWhiz['OK']:
return S_ERROR()
if self.clip.testSlic:
resSlic = self.createSlicTest()
if not resSlic['OK']:
return S_ERROR()
if self.clip.testMarlin:
resMarlin = self.createMarlinTest()
if not resMarlin['OK']:
return S_ERROR()
if self.clip.testLCSIM:
resLCSim = self.createLCSimTest()
if not resLCSim['OK']:
return S_ERROR()
if self.clip.testSlicPandora:
resSP = self.createSlicPandoraTest()
if not resSP['OK']:
return S_ERROR()
if self.clip.testUtilities:
resUtil = self.createUtilityTests()
if not resUtil['OK']:
return S_ERROR()
if self.clip.testRoot:
resRoot = self.createRootScriptTest()
if not resRoot['OK']:
return S_ERROR()
resRoot = self.createRootHaddTest()
if not resRoot['OK']:
return S_ERROR()
resRoot = self.createRootMacroTest()
if not resRoot['OK']:
return S_ERROR()
if self.clip.testFccSw:
resFccSw = self.createFccSwTest()
if not resFccSw['OK']:
return S_ERROR()
if self.clip.testFccAnalysis:
resFccAnalysis = self.createFccAnalysisTest()
if not resFccAnalysis['OK']:
return S_ERROR()
return S_OK()
class UserJob(Job):
""" User job class. To be used by users, not for production.
"""
def __init__(self, script = None):
super(UserJob, self).__init__( script )
self.type = 'User'
self.diracinstance = None
def submit(self, diracinstance = None, mode = "wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
"""
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submit(self, mode)
#############################################################################
def setInputData( self, lfns ):
"""Helper function.
Specify input data by Logical File Name (LFN).
Example usage:
>>> job = Job()
>>> job.setInputData(['/ilc/prod/whizard/processlist.whiz'])
@param lfns: Logical File Names
@type lfns: Single LFN string or list of LFNs
"""
if type( lfns ) == list and len( lfns ):
for i in xrange( len( lfns ) ):
lfns[i] = lfns[i].replace( 'LFN:', '' )
inputData = map( lambda x: 'LFN:' + x, lfns )
inputDataStr = string.join( inputData, ';' )
description = 'List of input data specified by LFNs'
self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description )
elif type( lfns ) == type( ' ' ): #single LFN
description = 'Input data specified by LFN'
self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description )
else:
kwargs = {'lfns':lfns}
return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs )
return S_OK()
def setInputSandbox(self, flist):
""" Mostly inherited from DIRAC.Job
"""
if type(flist) == type(""):
flist = [flist]
if not type(flist) == type([]) :
return self._reportError("File passed must be either single file or list of files.")
self.inputsandbox.extend(flist)
return S_OK()
#############################################################################
def setOutputData(self, lfns, OutputPath = '', OutputSE = ['']):
"""Helper function, used in preference to Job.setOutputData() for ILC.
For specifying output data to be registered in Grid storage. If a list
of OutputSEs are specified the job wrapper will try each in turn until
successful.
Example usage:
>>> job = Job()
>>> job.setOutputData(['Ntuple.root'])
@param lfns: Output data file or files
@type lfns: Single string or list of strings ['','']
@param OutputSE: Optional parameter to specify the Storage
@param OutputPath: Optional parameter to specify the Path in the Storage, postpented to /ilc/user/u/username/
Element to store data or files, e.g. CERN-tape
@type OutputSE: string or list
@type OutputPath: string
"""
kwargs = {'lfns' : lfns, 'OutputSE' : OutputSE, 'OutputPath' : OutputPath}
if type(lfns) == list and len(lfns):
outputDataStr = string.join(lfns, ';')
description = 'List of output data files'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', outputDataStr, description)
elif type(lfns) == type(" "):
description = 'Output data file'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns, description)
else:
return self._reportError('Expected file name string or list of file names for output data', **kwargs)
if OutputSE:
description = 'User specified Output SE'
if type(OutputSE) in types.StringTypes:
OutputSE = [OutputSE]
elif type(OutputSE) != types.ListType:
return self._reportError('Expected string or list for OutputSE', **kwargs)
OutputSE = ';'.join(OutputSE)
self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE, description)
if OutputPath:
description = 'User specified Output Path'
if not type(OutputPath) in types.StringTypes:
return self._reportError('Expected string for OutputPath', **kwargs)
# Remove leading "/" that might cause problems with os.path.join
while OutputPath[0] == '/':
OutputPath = OutputPath[1:]
if OutputPath.count("ilc/user"):
return self._reportError('Output path contains /ilc/user/ which is not what you want', **kwargs)
self._addParameter(self.workflow, 'UserOutputPath', 'JDL', OutputPath, description)
return S_OK()
#############################################################################
def setOutputSandbox( self, files ):
"""Helper function.
Specify output sandbox files. If specified files are over 10MB, these
may be uploaded to Grid storage with a notification returned in the
output sandbox.
Example usage:
>>> job = Job()
>>> job.setOutputSandbox(['*.log','myfile.slcio'])
@param files: Output sandbox files
@type files: Single string or list of strings ['','']
"""
if type( files ) == list and len( files ):
fileList = string.join( files, ";" )
description = 'Output sandbox file list'
self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description )
elif type( files ) == type( " " ):
description = 'Output sandbox file'
self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description )
else:
kwargs = {'files' : files}
return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs )
return S_OK()
def setILDConfig(self,Version):
""" Define the Configuration package to obtain
"""
appName = 'ILDConfig'
self._addSoftware(appName.lower(), Version)
self._addParameter( self.workflow, 'ILDConfigPackage', 'JDL', appName+Version, 'ILDConfig package' )
return S_OK()
def setRepo(self, optionVal):
self.repo = optionVal
return S_OK()
def registerSwitches(self):
Script.registerSwitch('r:', 'repository=', 'Path to repository file', self.setRepo)
Script.setUsageMessage('\n'.join( [ __doc__.split( '\n' )[1],
'\nUsage:',
' %s [option|cfgfile] ...\n' % Script.scriptName ] ) )
if __name__=="__main":
cliparams = Params()
cliparams.registerSwitches()
Script.parseCommandLine( ignoreErrors = False )
from DIRAC import gLogger
repoLocation = cliparams.repo
if not repoLocation:
Script.showHelp()
dexit(2)
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
dirac = DiracILC(True, repoLocation)
dirac.monitorRepository(False)
lfns = []
lfns = dirac.retrieveRepositoryOutputDataLFNs()
gLogger.notice("lfnlist=[")
for lfn in lfns :
gLogger.notice('"LFN:%s",' % lfn)
gLogger.notice("]")
dexit(0)
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
import time
dIlc = DiracILC()
lcoutput = []
for i in range(1, 301):
lcoutput = "aa_%d.root" % i
job = UserJob()
job.setDestination('LCG.CERN.ch')
job.setInputSandbox([
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraLikelihoodData9EBin_CLIC_ILD.xml",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/PandoraSettingsFast.xml",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/steering.xml",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/clic_ild_cdr.gear",
"LFN:/ilc/user/k/kacarevic/hgamgam/PandoraPFA/MarlinRecoRootFiles/lib.tar.gz"
])
job.setInputData(
"LFN:/ilc/user/k/kacarevic/hgamgam/Marlin/newPandora/aa/aa_%d.slcio" %
i)
job.setOutputSandbox(
["*.log", "*.sh", "*.py", "*.out", "*.xml", "*.steer "])
job.setJobGroup("myRoot")
job.setName("root_aa_%d" % i)
marl = Marlin()
marl.setVersion('ILCSoft-2016-09-27_gcc48')
class UserJob(Job):
""" User job class. To be used by users, not for production.
"""
def __init__(self, script=None):
super(UserJob, self).__init__(script)
self.type = 'User'
self.diracinstance = None
self.usergroup = ['ilc_user', 'calice_user']
self.proxyinfo = getProxyInfo()
########## SPLITTING STUFF: ATTRIBUTES ##########
self._data = []
self.splittingOption = None
self._switch = {}
self.numberOfJobs = None
self.totalNumberOfEvents = None
self.eventsPerJob = None
self.numberOfFilesPerJob = 1
def submit(self, diracinstance=None, mode="wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
:param diracinstance: DiracILC instance
:type diracinstance: ~ILCDIRAC.Interfaces.API.DiracILC.DiracILC
:param str mode: "wms" (default), "agent", or "local"
.. note ::
The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts
"""
if self.splittingOption:
result = self._split()
if 'OK' in result and not result['OK']:
return result
#Check the credentials. If no proxy or not user proxy, return an error
if not self.proxyinfo['OK']:
self.log.error(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup)
return self._reportError(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup, self.__class__.__name__)
if 'group' in self.proxyinfo['Value']:
group = self.proxyinfo['Value']['group']
if group not in self.usergroup:
self.log.error(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup)
return self._reportError(
"Not allowed to submit job, you need a %s proxy." %
self.usergroup, self.__class__.__name__)
else:
self.log.error(
"Could not determine group, you do not have the right proxy.")
return self._reportError(
"Could not determine group, you do not have the right proxy.")
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submit(self, mode)
#############################################################################
def setInputData(self, lfns):
"""Specify input data by Logical File Name (LFN).
Input files specified via this function will be automatically staged if necessary.
Example usage:
>>> job = UserJob()
>>> job.setInputData(['/ilc/prod/whizard/processlist.whiz'])
:param lfns: Logical File Names
:type lfns: Single LFN string or list of LFNs
"""
if isinstance(lfns, list) and lfns:
for i, lfn in enumerate(lfns):
lfns[i] = lfn.replace('LFN:', '')
#inputData = map( lambda x: 'LFN:' + x, lfns )
inputData = lfns #because we don't need the LFN: for inputData, and it breaks the
#resolution of the metadata in the InputFilesUtilities
inputDataStr = ';'.join(inputData)
description = 'List of input data specified by LFNs'
self._addParameter(self.workflow, 'InputData', 'JDL', inputDataStr,
description)
elif isinstance(lfns, basestring): #single LFN
description = 'Input data specified by LFN'
self._addParameter(self.workflow, 'InputData', 'JDL', lfns,
description)
else:
kwargs = {'lfns': lfns}
return self._reportError(
'Expected lfn string or list of lfns for input data', **kwargs)
return S_OK()
def setInputSandbox(self, flist):
""" Add files to the input sandbox, can be on the local machine or on the grid
>>> job = UserJob()
>>> job.setInputSandbox( ['LFN:/ilc/user/u/username/libraries.tar.gz',
>>> 'mySteeringFile.xml'] )
:param flist: Files for the inputsandbox
:type flist: `python:list` or `str`
"""
if isinstance(flist, basestring):
flist = [flist]
if not isinstance(flist, list):
return self._reportError(
"File passed must be either single file or list of files.")
self.inputsandbox.extend(flist)
return S_OK()
#############################################################################
def setOutputData(self, lfns, OutputPath='', OutputSE=''):
"""For specifying output data to be registered in Grid storage. If a list
of OutputSEs are specified the job wrapper will try each in turn until
successful.
Example usage:
>>> job = UserJob()
>>> job.setOutputData(['Ntuple.root'])
:param lfns: Output data file or files
:type lfns: Single `str` or `python:list` of strings ['','']
:param str OutputPath: Optional parameter to specify the Path in the Storage, postpended to /ilc/user/u/username/
:param OutputSE: Optional parameter to specify the Storage Element to store data or files, e.g. CERN-SRM
:type OutputSE: `python:list` or `str`
"""
kwargs = {'lfns': lfns, 'OutputSE': OutputSE, 'OutputPath': OutputPath}
if isinstance(lfns, list) and lfns:
outputDataStr = ';'.join(lfns)
description = 'List of output data files'
self._addParameter(self.workflow, 'UserOutputData', 'JDL',
outputDataStr, description)
elif isinstance(lfns, basestring):
description = 'Output data file'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns,
description)
else:
return self._reportError(
'Expected file name string or list of file names for output data',
**kwargs)
if OutputSE:
description = 'User specified Output SE'
if isinstance(OutputSE, basestring):
OutputSE = [OutputSE]
elif not isinstance(OutputSE, list):
return self._reportError(
'Expected string or list for OutputSE', **kwargs)
OutputSE = ';'.join(OutputSE)
self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE,
description)
if OutputPath:
description = 'User specified Output Path'
if not isinstance(OutputPath, basestring):
return self._reportError('Expected string for OutputPath',
**kwargs)
# Remove leading "/" that might cause problems with os.path.join
while OutputPath[0] == '/':
OutputPath = OutputPath[1:]
if OutputPath.count("ilc/user"):
return self._reportError(
'Output path contains /ilc/user/ which is not what you want',
**kwargs)
self._addParameter(self.workflow, 'UserOutputPath', 'JDL',
OutputPath, description)
return S_OK()
#############################################################################
def setOutputSandbox(self, files):
"""Specify output sandbox files. If specified files are over 10MB, these
may be uploaded to Grid storage with a notification returned in the
output sandbox.
.. Note ::
Sandbox files are removed after 2 weeks.
Example usage:
>>> job = UserJob()
>>> job.setOutputSandbox(['*.log','*.sh', 'myfile.txt'])
Use the output sandbox only for small files. Larger files should be stored
on the grid and downloaded later if necessary. See :func:`setOutputData`
:param files: Output sandbox files
:type files: Single `str` or `python:list` of strings ['','']
"""
if isinstance(files, list) and files:
fileList = ";".join(files)
description = 'Output sandbox file list'
self._addParameter(self.workflow, 'OutputSandbox', 'JDL', fileList,
description)
elif isinstance(files, basestring):
description = 'Output sandbox file'
self._addParameter(self.workflow, 'OutputSandbox', 'JDL', files,
description)
else:
kwargs = {'files': files}
return self._reportError(
'Expected file string or list of files for output sandbox contents',
**kwargs)
return S_OK()
def setILDConfig(self, version):
""" Define the Configuration package to obtain
"""
appName = 'ILDConfig'
self._addSoftware(appName.lower(), version)
self._addParameter(self.workflow, 'ILDConfigPackage', 'JDL',
appName + version, 'ILDConfig package')
return S_OK()
def setCLICConfig(self, version):
"""Define the CLIC Configuration package to obtain, copies steering files
from CLIC Configuration folder to working directory
:param str version: version string, e.g.: 'ILCSoft-2017-07-27'
"""
appName = 'ClicConfig'
self._addSoftware(appName.lower(), version)
self._addParameter(self.workflow, 'ClicConfigPackage', 'JDL',
appName + version, 'CLIC Config package')
return S_OK()
############################## SPLITTING STUFF: METHODS ################################
# Some methods have been added :
#
# * _atomicSubmission
# * _checkJobConsistency
# * setSplitEvents
# * setSplitInputData
# * setSplitDoNotAlterOutputFilename
# * _split
# * _splitByData
# * _splitByEvents
# * _toInt
#
# Given the type of splitting (Events or Data), these methods compute
# the right parameters of the method 'Job.setParameterSequence()'
##########################################################################################
def setSplitEvents(self,
eventsPerJob=None,
numberOfJobs=None,
totalNumberOfEvents=None):
"""This function sets split parameters for doing splitting over events
Example usage:
>>> job = UserJob()
>>> job.setSplitEvents( numberOfJobs=42, totalNumberOfEvents=126 )
Exactly two of the parmeters should be set
:param int eventsPerJob: The events processed by a single job
:param int numberOfJobs: The number of jobs
:param int totalNumberOfEvents: The total number of events processed by all jobs
"""
self.totalNumberOfEvents = totalNumberOfEvents
self.eventsPerJob = eventsPerJob
self.numberOfJobs = numberOfJobs
self._addParameter(self.workflow, 'NbOfEvts', 'JDL', -1,
'Number of Events')
self.splittingOption = "byEvents"
def setSplitInputData(self, lfns, numberOfFilesPerJob=1):
"""sets split parameters for doing splitting over input data
Example usage:
>>> job = UserJob()
>>> job.setSplitInputData( listOfLFNs )
:param lfns: Logical File Names
:type lfns: list of LFNs
:param int numberOfFilesPerJob: The number of input data processed by a single job
"""
self._data = lfns if isinstance(lfns, list) else [lfns]
self.numberOfFilesPerJob = numberOfFilesPerJob
self.splittingOption = "byData"
def setSplitDoNotAlterOutputFilename(self, value=True):
"""if this option is set the output data lfns will _not_ include the JobIndex
:param bool value: if *True* disable the changing of the output data
filenames. If *False* the JobIndex will be added at the end of
OutputData LFNs before the extension. Or replace '%n' with the jobIndex
in the fileName. See :func:`Core.Utilities.Splitting.addJobIndexToFilename`
"""
self._addParameter(self.workflow, 'DoNotAlterOutputData', 'JDL', value,
'Do Not Change Output Data')
def _split(self):
"""checks the consistency of the job and call the right split method.
:return: The success or the failure of the consistency checking
:rtype: DIRAC.S_OK, DIRAC.S_ERROR
"""
self.eventsPerJob = self._toInt(self.eventsPerJob)
self.numberOfJobs = self._toInt(self.numberOfJobs)
if self.numberOfJobs is False or self.eventsPerJob is False:
return self._reportError("Splitting: Invalid values for splitting")
# FIXME: move somewhere more prominent
self._switch = {
"byEvents": self._splitByEvents,
"byData": self._splitByData,
None: self._atomicSubmission,
}
self.log.info("Job splitting...")
if not self._checkJobConsistency():
errorMessage = "Job._checkJobConsistency() failed"
self.log.error(errorMessage)
return self._reportError(errorMessage)
sequence = self._switch[self.splittingOption]()
if not sequence:
errorMessage = "Job._splitBySomething() failed"
self.log.error(errorMessage)
return self._reportError(errorMessage)
sequenceType, sequenceList, addToWorkflow = sequence[0], sequence[
1], sequence[2]
if sequenceType != "Atomic":
self.setParameterSequence(sequenceType, sequenceList,
addToWorkflow)
self.setParameterSequence('JobIndexList',
range(len(sequenceList)),
addToWorkflow='JobIndex')
self._addParameter(self.workflow, 'JobIndex', 'int', 0, 'JobIndex')
self.log.info("Job splitting successful")
return S_OK()
#############################################################################
def _atomicSubmission(self):
"""called when no splitting is necessary, do not return valid parameters fot setParameterSequence().
:return: parameter name and parameter values for setParameterSequence(), addToWorkflow flag
:rtype: tuple of (str, list, bool/str)
"""
self.log.verbose(
"Job splitting: No splitting to apply, 'atomic submission' will be used"
)
return "Atomic", [], False
#############################################################################
def _checkJobConsistency(self):
"""checks if Job parameters are valid.
:return: The success or the failure of the consistency checking
:rtype: bool
:Example:
>>> self._checkJobConsistency()
"""
self.log.info("Job consistency: _checkJobConsistency()...")
if self.splittingOption not in self._switch:
splitOptions = ",".join(self._switch.keys())
errorMessage = "checkJobConsistency failed: Bad split value: possible values are %s" % splitOptions
self.log.error(errorMessage)
return False
# All applications should have the same number of events
# We can get this number from the first application for example
sameNumberOfEvents = next(iter(self.applicationlist)).numberOfEvents
if not all(app.numberOfEvents == sameNumberOfEvents
for app in self.applicationlist):
self.log.warn(
"Job: Applications should all have the same number of events")
if (self.totalNumberOfEvents == -1
or sameNumberOfEvents == -1) and not self._data:
self.log.warn(
"Job: Number of events is -1 without input data. Was that intentional?"
)
self.log.info("job._checkJobConsistency successful")
return True
#############################################################################
def _splitByData(self):
"""a job is submitted per input data.
:return: parameter name and parameter values for setParameterSequence()
:rtype: tuple of (str, list, bool/str)
"""
# reset split attribute to avoid infinite loop
self.splittingOption = None
self.log.info("Job splitting: Splitting 'byData' method...")
# Ensure that data have been specified by setInputData() method
if not self._data:
errorMessage = "Job splitting: missing input data"
self.log.error(errorMessage)
return False
if self.numberOfFilesPerJob > len(self._data):
errorMessage = "Job splitting: 'numberOfFilesPerJob' must be less/equal than the number of input data"
self.log.error(errorMessage)
return False
self._data = breakListIntoChunks(self._data, self.numberOfFilesPerJob)
self.log.info("Job splitting: submission consists of %d job(s)" %
len(self._data))
return ["InputData", self._data, 'ParametricInputData']
#############################################################################
def _splitByEvents(self):
"""a job is submitted per subset of events.
:return: parameter name and parameter values for setParameterSequence()
:rtype: tuple of (str, list, bool/str)
"""
# reset split attribute to avoid infinite loop
self.splittingOption = None
self.log.info("Job splitting: splitting 'byEvents' method...")
if self.eventsPerJob and self.numberOfJobs:
# 1st case: (numberOfJobs=3, eventsPerJob=10)
# trivial case => each job (total of 3) run applications of 10 events each
self.log.debug("Job splitting: events per job and number of jobs")
mapEventJob = [self.eventsPerJob] * self.numberOfJobs
elif self.eventsPerJob and self.totalNumberOfEvents:
# 2nd case: (split="byEvents", eventsPerJob=10, totalNumberOfEvents=10)
# Given the number of events per job and total of number of event we want,
# we can compute the unknown which is the number of jobs.
self.log.debug(
"Job splitting: Events per job and total number of events")
if self.eventsPerJob > self.totalNumberOfEvents:
self.log.error(
"Job splitting: The number of events per job has to be lower than or equal to the total number of events"
)
return False
numberOfJobsIntDiv = self.totalNumberOfEvents / self.eventsPerJob
numberOfJobsRest = self.totalNumberOfEvents % self.eventsPerJob
mapEventJob = [self.eventsPerJob] * numberOfJobsIntDiv
mapEventJob += [numberOfJobsRest] if numberOfJobsRest != 0 else []
else:
# 3rd case: (split='byEvents', njobs=10, totalNumberOfEvents=10)
# Then compute the right number of events per job
self.log.debug(
"Job splitting: The number of jobs and the total number of events"
)
if (not self.totalNumberOfEvents) or (self.totalNumberOfEvents <
self.numberOfJobs):
self.log.error(
"Job splitting: The number of events has to be greater than or equal to the number of jobs"
)
return False
eventPerJobIntDiv = self.totalNumberOfEvents / self.numberOfJobs
eventPerJobRest = self.totalNumberOfEvents % self.numberOfJobs
mapEventJob = [eventPerJobIntDiv] * self.numberOfJobs
if eventPerJobRest != 0:
for suplement in xrange(eventPerJobRest):
mapEventJob[suplement] += 1
self.log.debug("Job splitting: events over the jobs: %s" % mapEventJob)
self.log.info("Job splitting: submission consists of %d job(s)" %
len(mapEventJob))
return ['NumberOfEvents', mapEventJob, 'NbOfEvts']
#############################################################################
def _toInt(self, number):
"""casts number parameter to an integer.
It also accepts 'string integer' parameter.
:param number: the number to cast (number of events, number of jobs)
:type number: str or int
:return: The success or the failure of the casting
:rtype: bool, int or None
:Example:
>>> number = self._toInt("1000")
"""
if number is None:
return number
try:
number = int(number)
if number <= 0:
raise ValueError
except ValueError:
self.log.error(
"Job splitting: arguments must be positive integers")
return False
return number
def __init__(self, *args, **kwargs):
AgentModule.__init__(self, *args, **kwargs)
self.name = 'DataRecoveryAgent'
self.enabled = False
self.productionsToIgnore = self.am_getOption("ProductionsToIgnore", [])
self.transformationTypes = self.am_getOption("TransformationTypes", [
'MCReconstruction', 'MCSimulation', 'MCReconstruction_Overlay',
'MCGeneration'
])
self.transformationStatus = self.am_getOption("TransformationStatus",
['Active', 'Completing'])
self.shifterProxy = self.am_setOption('shifterProxy', 'DataManager')
self.jobStatus = [
'Failed', 'Done'
] ##This needs to be both otherwise we cannot account for all cases
self.jobMon = JobMonitoringClient()
self.fcClient = FileCatalogClient()
self.tClient = TransformationClient()
self.reqClient = ReqClient()
self.diracILC = DiracILC()
self.inputFilesProcessed = set()
self.todo = {'MCGeneration':
[ dict( Message="MCGeneration: OutputExists: Job 'Done'",
ShortMessage="MCGeneration: job 'Done' ",
Counter=0,
Check=lambda job: job.allFilesExist() and job.status=='Failed',
Actions=lambda job,tInfo: [ job.setJobDone(tInfo) ]
),
dict( Message="MCGeneration: OutputMissing: Job 'Failed'",
ShortMessage="MCGeneration: job 'Failed' ",
Counter=0,
Check=lambda job: job.allFilesMissing() and job.status=='Done',
Actions=lambda job,tInfo: [ job.setJobFailed(tInfo) ]
),
# dict( Message="MCGeneration, job 'Done': OutputExists: Task 'Done'",
# ShortMessage="MCGeneration: job already 'Done' ",
# Counter=0,
# Check=lambda job: job.allFilesExist() and job.status=='Done',
# Actions=lambda job,tInfo: [ tInfo._TransformationInfo__setTaskStatus(job, 'Done') ]
# ),
],
'OtherProductions':
[ \
## should always be first!
dict( Message="One of many Successful: clean others",
ShortMessage="Other Tasks --> Keep",
Counter=0,
Check=lambda job: job.allFilesExist() and job.otherTasks and job.inputFile not in self.inputFilesProcessed,
Actions=lambda job,tInfo: [ self.inputFilesProcessed.add(job.inputFile), job.setJobDone(tInfo), job.setInputProcessed(tInfo) ]
),
dict( Message="Other Task processed Input, no Output: Fail",
ShortMessage="Other Tasks --> Fail",
Counter=0,
Check=lambda job: job.inputFile in self.inputFilesProcessed and job.allFilesMissing() and job.status!='Failed',
Actions=lambda job,tInfo: [ job.setJobFailed(tInfo) ]
),
dict( Message="Other Task processed Input: Fail and clean",
ShortMessage="Other Tasks --> Cleanup",
Counter=0,
Check=lambda job: job.inputFile in self.inputFilesProcessed and not job.allFilesMissing(),
Actions=lambda job,tInfo: [ job.setJobFailed(tInfo), job.cleanOutputs(tInfo) ]
),
dict( Message="InputFile missing: mark job 'Failed', mark input 'Deleted', clean",
ShortMessage="Input Missing --> Job 'Failed, Input 'Deleted', Cleanup",
Counter=0,
Check=lambda job: job.inputFile and not job.inputFileExists and job.fileStatus != "Deleted",
Actions=lambda job,tInfo: [ job.cleanOutputs(tInfo), job.setJobFailed(tInfo), job.setInputDeleted(tInfo) ]
),
dict( Message="InputFile Deleted, output Exists: mark job 'Failed', clean",
ShortMessage="Input Deleted --> Job 'Failed, Cleanup",
Counter=0,
Check=lambda job: job.inputFile and not job.inputFileExists and job.fileStatus == "Deleted" and not job.allFilesMissing(),
Actions=lambda job,tInfo: [ job.cleanOutputs(tInfo), job.setJobFailed(tInfo) ]
),
## All Output Exists
dict( Message="Output Exists, job Failed, input not Processed --> Job Done, Input Processed",
ShortMessage="Output Exists --> Job Done, Input Processed",
Counter=0,
Check=lambda job: job.allFilesExist() and \
not job.otherTasks and \
job.status=='Failed' and \
job.fileStatus!="Processed" and \
job.inputFileExists,
Actions=lambda job,tInfo: [ job.setJobDone(tInfo), job.setInputProcessed(tInfo) ]
),
dict( Message="Output Exists, job Failed, input Processed --> Job Done",
ShortMessage="Output Exists --> Job Done",
Counter=0,
Check=lambda job: job.allFilesExist() and \
not job.otherTasks and \
job.status=='Failed' and \
job.fileStatus=="Processed" and \
job.inputFileExists,
Actions=lambda job,tInfo: [ job.setJobDone(tInfo) ]
),
dict( Message="Output Exists, job Done, input not Processed --> Input Processed",
ShortMessage="Output Exists --> Input Processed",
Counter=0,
Check=lambda job: job.allFilesExist() and \
not job.otherTasks and \
job.status=='Done' and \
job.fileStatus!="Processed" and \
job.inputFileExists,
Actions=lambda job,tInfo: [ job.setInputProcessed(tInfo) ]
),
## outputmissing
dict( Message="Output Missing, job Failed, input Assigned, MaxError --> Input MaxReset",
ShortMessage="Max ErrorCount --> Input MaxReset",
Counter=0,
Check=lambda job: job.allFilesMissing() and \
not job.otherTasks and \
job.status=='Failed' and \
job.fileStatus in ASSIGNEDSTATES and \
job.inputFile not in self.inputFilesProcessed and \
job.inputFileExists and \
job.errorCount > MAXRESET,
Actions=lambda job,tInfo: [ job.setInputMaxReset(tInfo) ]
),
dict( Message="Output Missing, job Failed, input Assigned --> Input Unused",
ShortMessage="Output Missing --> Input Unused",
Counter=0,
Check=lambda job: job.allFilesMissing() and \
not job.otherTasks and \
job.status=='Failed' and \
job.fileStatus in ASSIGNEDSTATES and \
job.inputFile not in self.inputFilesProcessed and \
job.inputFileExists,
Actions=lambda job,tInfo: [ job.setInputUnused(tInfo) ]
),
dict( Message="Output Missing, job Done, input Assigned --> Job Failed, Input Unused",
ShortMessage="Output Missing --> Job Failed, Input Unused",
Counter=0,
Check=lambda job: job.allFilesMissing() and \
not job.otherTasks and \
job.status=='Done' and \
job.fileStatus in ASSIGNEDSTATES and \
job.inputFile not in self.inputFilesProcessed and \
job.inputFileExists,
Actions=lambda job,tInfo: [ job.setInputUnused(tInfo), job.setJobFailed(tInfo) ]
),
## some files missing, needing cleanup. Only checking for
## assigned, because processed could mean an earlier job was
## succesful and this one is just the duplicate that needed
## to be removed! But we check for other tasks earlier, so
## this should not happen
dict( Message="Some missing, job Failed, input Assigned --> cleanup, Input 'Unused'",
ShortMessage="Output Missing --> Cleanup, Input Unused",
Counter=0,
Check=lambda job: job.someFilesMissing() and \
not job.otherTasks and \
job.status=='Failed' and \
job.fileStatus in ASSIGNEDSTATES and \
job.inputFileExists,
Actions=lambda job,tInfo: [job.cleanOutputs(tInfo),job.setInputUnused(tInfo)]
#Actions=lambda job,tInfo: []
),
dict( Message="Some missing, job Done, input Assigned --> cleanup, job Failed, Input 'Unused'",
ShortMessage="Output Missing --> Cleanup, Job Failed, Input Unused",
Counter=0,
Check=lambda job: job.someFilesMissing() and \
not job.otherTasks and \
job.status=='Done' and \
job.fileStatus in ASSIGNEDSTATES and \
job.inputFileExists,
Actions=lambda job,tInfo: [job.cleanOutputs(tInfo),job.setInputUnused(tInfo),job.setJobFailed(tInfo)]
#Actions=lambda job,tInfo: []
),
dict( Message="Some missing, job Done --> job Failed",
ShortMessage="Output Missing, Done --> Job Failed",
Counter=0,
Check=lambda job: not job.allFilesExist() and job.status=='Done',
Actions=lambda job,tInfo: [job.setJobFailed(tInfo)]
),
dict ( Message="Something Strange",
ShortMessage="Strange",
Counter=0,
Check=lambda job: job.status not in ("Failed","Done"),
Actions=lambda job,tInfo: []
),
##should always be the last one!
dict ( Message="Failed Hard",
ShortMessage="Failed Hard",
Counter=0,
Check=lambda job: False, ## never
Actions=lambda job,tInfo: []
),
]
}
self.jobCache = defaultdict(lambda: (0, 0))
self.printEveryNJobs = self.am_getOption('PrintEvery', 200)
##Notification
self.notesToSend = ""
self.addressTo = self.am_getOption('MailTo',
["*****@*****.**"])
self.addressFrom = self.am_getOption('MailFrom',
"*****@*****.**")
self.subject = "DataRecoveryAgent"
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print res['Message']
lfns = res['Value']
#print "Found %s files" % len(lfns)
filelist=[]
for lfn in lfns:
filelist.append(lfn)
#print filelist
#filelist2=filelist[0]
#check if big radius in fastjetanalyzer indeed 0.7 or 1.0
jobGroup = "HZAnalyzer_190417_ee_qqqq_m_qqqq_2TeV_13696_VLC7PFOs"
dirac = DiracILC(True,jobGroup+".rep")
job = UserJob()
#job.setExecutionEnv({'ROOT_INCLUDE_PATH':'./'})
job.setJobGroup(jobGroup)
job.setOutputSandbox ( [ "*.log","*.out","*.py"] )
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk'])
#pay attention that the Zuds200 here is NOT changed
job.setInputSandbox( ["LFN:/ilc/user/w/webermat/190606/HZAnalyzerlib.tar.gz", "LFN:/ilc/user/w/webermat/190412/vtxprob.tar.gz","LFN:/ilc/user/w/webermat/190412/flavourTagging04-01_ct_90deg/lcfiweights.tar.gz"] )
job.setBannedSites(['LCG.INP3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.QMUL.uk','LCG.Oxford.uk'])
job.setSplitInputData(filelist, numberOfFilesPerJob=50)
ma = Marlin()
ma.setVersion('ILCSoft-2019-04-17_gcc62')
#ma.setInputFile("LFN:/ilc/user/w/webermat/ddsimstdheptautau/ILC18-10-11_gcc62_CLIC_o3_v14/tautau200/ddsim_ILC181011_gcc62_tautau_200_CLIC_o3_v14_0%s.slcio"%(str(input_ind)))
ma.setSteeringFile("/eos/user/w/weberma2/steeringFiles/testHZAnalyzer.xml")
ma.setDetectorModel("CLIC_o3_v14")
HZrootfilename2="HZStudy_ee_qqqq_m_qqqq_2TeV_13696_polm80_3TeV_wO_CLIC_o3_v14_DR7.root"
def main(argv):
# Input arguments
ildconfig_version = "$ILDCONFIGVER"
ilcsoft_version = "$ILCSOFTVER"
evts_per_run = $EVTSPERRUN
detector_model = "$DETECTOR"
sim_input = "$SIMINPUT"
process_name = "$PROCESS"
index = $IND
sim_input = diracpath_from_pnfspath( sim_input )
sim_detector_model = detector_model_wo_option( detector_model )
job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model
dirac = DiracILC(True,job_group+".rep")
# outputs to be saved onto grid SE
RECoutput = []
# DDSim
evtStart = (index-1)*evts_per_run
evtEnd = index*evts_per_run - 1
RandSeed = random.randrange(11623, 99999)
lcinputSIM = "LFN:" + sim_input
lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd))
sim = DDSim()
sim.setVersion(ilcsoft_version)
sim.setDetectorModel(sim_detector_model)
sim.setInputFile(lcinputSIM)
sim.setSteeringFile("ddsim_steer.py")
sim.setNumberOfEvents(evts_per_run)
sim.setRandomSeed(RandSeed)
sim.setEnergy(1000)
sim.setStartFrom(evtStart)
sim.setOutputFile(lcoutputSIM)
# Marlin
lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd))
ma = Marlin()
ma.setVersion(ilcsoft_version)
ma.setDetectorModel(detector_model)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) )
ma.setLogFile("marlin.log")
ma.getInputFromApp(sim)
ma.setEnergy(1000)
ma.setOutputDstFile(lcoutputDST)
RECoutput.append(lcoutputDST)
# ILCDirac user job
job = UserJob()
job.setName("user_sim_reco")
job.setJobGroup(job_group)
job.setILDConfig(ildconfig_version)
job.setCPUTime(86400)
tmp_file_name = process_name + "_sim_reco_job_tmp.py"
job.setInputSandbox([tmp_file_name])
job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])
job.dontPromptMe()
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])
# run simulation job
simres = job.append(sim)
if not simres['OK']:
print simres['Not ok appending ddsim to job']
quit()
# run Malrin reco jobs
mares = job.append(ma)
if not mares['OK']:
print mares['Not ok appending Marlin to job']
quit()
job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM")
print RECoutput
submit_output = job.submit(dirac)
print submit_output