class JobCreater(object):
"""contains all the versions and parameters to create all theses tests"""
# pylint: disable=too-many-instance-attributes
# Test parameters, necessary due to amount of tests in this class.
def __init__( self,
clip,
params
):
self.clip = clip
self.ildConfig = params.get( "ildConfig", None )
self.alwaysOverlay = params.get( "alwaysOverlay", False )
self.runOverlay = self.clip.testOverlay or self.alwaysOverlay
self.mokkaVersion = params["mokkaVersion"]
self.mokkaSteeringFile = params.get( "mokkaSteeringFile" )
self.detectorModel = params.get( "detectorModel" )
self.marlinVersion = params.get( "marlinVersion" )
self.marlinSteeringFile = params.get( "marlinSteeringFile" )
self.ddsimVersion = params.get( "ddsimVersion" )
self.ddsimDetectorModel = params.get( "ddsimDetectorModel")
self.ddsimInputFile = params.get( "ddsimInputFile" )
self.marlinInputdata = params.get ( "marlinInputdata" )
self.gearFile = params.get( "gearFile" )
self.lcsimVersion = params.get( "lcsimVersion" )
self.steeringFileVersion = params.get( "steeringFileVersion", None )
self.rootVersion = params["rootVersion"]
self.whizard2Version = params.get( "whizard2Version" )
self.whizard2SinFile = params.get( "whizard2SinFile" )
self.energy = params.get("energy")
self.backgroundType = params.get("backgroundType")
self.machine = params.get("machine")
self.gearFile = params.get( "gearFile" )
self.marlinSteeringFile = params.get( "marlinSteeringFile" )
self.marlinVersion = params.get( "marlinVersion" )
self.lcsimPreSteeringFile = params.get( "lcsimPreSteeringFile" )
self.lcsimPostSteeringFile = params.get( "lcsimPostSteeringFile" )
self.fccSwPath = params.get( "fccSwPath" )
self.fccSwSteeringFile = params.get( "fccSwSteeringFile" )
self.fccAnalysisSteeringFile = params.get( "fccAnalysisSteeringFile" )
### other things needed to run tests
self.log = gLogger.getSubLogger("JobCreater")
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid
from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid
if self.clip.submitMode == "local":
self.log.notice("")
self.log.notice(" DIRAC RCSID:", drcsid )
self.log.notice(" Job RCSID:", jrcsid )
self.log.notice("Applications RCSID:", apprcsid )
self.log.notice("")
self.diracInstance = DiracILC(False, 'tests.rep')
self.jobList = {}
def createDDSimTest( self, inputfile = None, detectorModel = None):
"""Create a job running ddsim"""
if inputfile is None:
inputfile = self.ddsimInputFile
sandbox = [ inputfile ]
if detectorModel is None:
detectorModel = self.ddsimDetectorModel
else:
sandbox.append(detectorModel)
jobdd = self.getJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim
ddsim = DDSim()
ddsim.setVersion(self.ddsimVersion)
ddsim.setDetectorModel(detectorModel)
ddsim.setNumberOfEvents(1)
ddsim.setInputFile(inputfile)
jobdd.setInputSandbox(sandbox)
res = jobdd.append(ddsim)
if not res['OK']:
self.log.error("Failed adding DDSim:", res['Message'])
return S_ERROR("Failed adding DDSim to Job")
return S_OK(jobdd)
def createWhizard2Test( self ):
"""Create a job running Whizard2"""
jobdd = self.getJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications.Whizard2 import Whizard2
whiz = Whizard2()
whiz.setVersion(self.whizard2Version)
whiz.setNumberOfEvents(1)
whiz.setSinFile(self.whizard2SinFile)
whiz.setOutputFile("test.stdhep")
res = jobdd.append(whiz)
if not res['OK']:
self.log.error("Failed adding Whizard2:", res['Message'])
return S_ERROR("Failed adding Whizard2 to Job")
return S_OK(jobdd)
def createMokkaTest(self):
"""create a job running mokka, and maybe whizard before"""
self.log.notice("Creating jobs for Mokka")
#(Whizard + )Mokka
jobmo = self.getJob()
if self.clip.testChain:
whmo = self.getWhizard(2)
res = jobmo.append(whmo)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR("Failed adding Whizard")
elif self.clip.testInputData:
jobmo.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep")
else:
self.log.error("Mokka does not know where to get its input from")
return S_ERROR("Mokka does not know where to gets its input from")
mo = self.getMokka()
if self.clip.testChain:
mo.getInputFromApp(whmo)
else:
mo.setNumberOfEvents(1)
res = jobmo.append(mo)
if not res['OK']:
self.log.error("Failed adding Mokka:", res['Message'])
return S_ERROR("Failed adding Mokka to Job")
jobmo.setOutputData("testsim.slcio", OutputSE="CERN-DIP-4")
self.jobList['Mokka1'] = jobmo
return S_OK(jobmo)
def createRootScriptTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["root.sh", "input.root","input2.root"])
root = self.getRoot()
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def createRootHaddTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["input.root","input2.root"])
root = self.getRoot()
root.setScript("hadd")
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def createRootMacroTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["func.C", "input.root","input2.root"])
root = self.getRootMacro()
root.setScript("func.C")
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def getOverlay(self, nbevts):
""" Create an overlay step
"""
pathToFiles = None
from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput
overlay = OverlayInput()
if self.energy==350:
if self.detectorModel=="ILD_o1_v05":
pathToFiles="/ilc/user/s/sailer/testFiles/overlay/ild_350/"
if pathToFiles:
overlay.setPathToFiles(pathToFiles)
else:
self.log.warn("better define pathToFiles for this overlay: %s, %s, %s" %
(self.energy, self.machine, self.backgroundType) )
overlay.setMachine(self.machine)
overlay.setEnergy(self.energy)
overlay.setDetectorModel(self.detectorModel)
overlay.setBkgEvtType(self.backgroundType)
overlay.setBXOverlay(60)
overlay.setGGToHadInt(0.3)
overlay.setNumberOfSignalEventsPerJob(nbevts)
return overlay
def getMokka(self):
""" Define a mokka app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka
mokka = Mokka()
mokka.setVersion(self.mokkaVersion)
mokka.setSteeringFile(self.mokkaSteeringFile)
mokka.setOutputFile("testsim.slcio")
mokka.setDetectorModel(self.detectorModel)
if self.steeringFileVersion:
mokka.setSteeringFileVersion(self.steeringFileVersion)
return mokka
def getRoot(self):
""" Define a root app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootScript
root = RootScript()
root.setScript("root.sh")
root.setArguments("output.root input.root input2.root")
root.setVersion(self.rootVersion)
root.setOutputFile("output.root")
return root
def getRootMacro(self):
""" Define a root app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootMacro
root = RootMacro()
root.setMacro("func.C")
root.setArguments(r"\"input.root\"")
root.setVersion(self.rootVersion)
return root
@staticmethod
def getSLIC():
""" Get a SLIC instance
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
slic.setOutputFile('testsim.slcio')
return slic
@staticmethod
def getSLICPandora():
""" Get some SLICPAndora app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLICPandora
slicp = SLICPandora()
slicp.setVersion('CLIC_CDR')
slicp.setDetectorModel('clic_sid_cdr')
slicp.setPandoraSettings("PandoraSettingsSlic.xml")
slicp.setOutputFile('testpandora.slcio')
return slicp
def getMarlin( self ):
""" Define a marlin step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
# marlin.setVersion("v0111Prod")
marlin.setVersion(self.marlinVersion)
marlin.setSteeringFile(self.marlinSteeringFile)
marlin.setGearFile(self.gearFile)
marlin.setOutputDstFile("testmarlinDST.slcio")
marlin.setOutputRecFile("testmarlinREC.slcio")
marlin.setNumberOfEvents(1)
return marlin
def getDD( self ):
""" Create a DDSim object
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim
ddsim = DDSim()
ddsim.setVersion(self.ddsimVersion)
ddsim.setDetectorModel(self.ddsimDetectorModel)
ddsim.setInputFile(self.ddsimInputFile)
ddsim.setNumberOfEvents(2)
return ddsim
def getLCSIM(self, prepandora = True):
""" Get some LCSIM
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM
lcsim = LCSIM()
lcsim.setVersion('CLIC_CDR')
lcsim.setDetectorModel('clic_sid_cdr.zip')
if prepandora:
lcsim.setSteeringFile(self.lcsimPreSteeringFile)
lcsim.setOutputFile("testlcsim.slcio")
else:
lcsim.setSteeringFile(self.lcsimPostSteeringFile)
#lcsim.setOutputFile("testlcsimfinal.slcio")
lcsim.setOutputDstFile("testlcsimDST.slcio")
lcsim.setOutputRecFile("testlcsimREC.slcio")
lcsim.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml")
return lcsim
def getFccSw( self ):
""" Define a fccsw step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccSw
fccsw = FccSw()
fccsw.fccSwPath = self.fccSwPath
fccsw.setSteeringFile(self.fccSwSteeringFile)
return fccsw
def getFccAnalysis( self ):
""" Define a fccanalysis step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccAnalysis
fccanalysis = FccAnalysis()
fccanalysis.setSteeringFile(self.fccAnalysisSteeringFile)
return fccanalysis
@staticmethod
def getStdhepcut(generated):
""" Get some cuts in
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdhepCutJava
stdhepcut = StdhepCutJava()
stdhepcut.setVersion('1.0')
stdhepcut.setSelectionEfficiency(1.)
#stdhepcut.setSteeringFile("cuts_testing_1400.txt")
stdhepcut.setInlineCuts("leptonInvMass_R 13 100 200")
stdhepcut.setSteeringFileVersion("V18")
stdhepcut.setMaxNbEvts(1)
stdhepcut.setNumberOfEvents(generated)
return stdhepcut
@staticmethod
def getStdhepSplit():
""" Get some stdhep split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(5)
stdhepsplit.setOutputFile("teststdhepsplit.stdhep")
stdhepsplit.setMaxRead(10)
return stdhepsplit
@staticmethod
def getLCIOSplit(events_per_file):
""" Get a LCIO split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOSplit
lciosplit = SLCIOSplit()
lciosplit.setNumberOfEventsPerFile(events_per_file)
lciosplit.setOutputFile("testlciosplit.slcio")
return lciosplit
@staticmethod
def getLCIOConcat():
""" Get a LCIO Concat
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOConcatenate
lcioconcat = SLCIOConcatenate()
lcioconcat.setOutputFile("testlcioconcat.slcio")
return lcioconcat
def getJob(self):
""" Define a generic job, it should be always the same
"""
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
myjob = UserJob()
myjob.setName("Testing")
myjob.setJobGroup("Tests")
myjob.setCPUTime(30000)
myjob.dontPromptMe()
myjob.setLogLevel("VERBOSE")
myjob.setPlatform("x86_64-slc5-gcc43-opt")
myjob.setOutputSandbox(["*.log","*.xml", "*.sh"])
myjob._addParameter( myjob.workflow, 'TestFailover', 'String', True, 'Test failoverRequest')
myjob._addParameter( myjob.workflow, 'Platform', 'JDL', "x86_64-slc5-gcc43-opt", 'OS Platform')
if self.ildConfig:
myjob.setILDConfig(self.ildConfig)
return myjob
def getWhizardModel(self, nbevts, energy, model):
""" Create a default whizard
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard
proddict = "e2e2_o"
if model != "sm":
proddict = "se2se2_r"
whiz = Whizard(processlist = self.diracInstance.getProcessList())
whiz.setModel(model)
pdict = {}
pdict['process_input'] = {}
pdict['process_input']['process_id'] = proddict
pdict['process_input']['sqrts'] = energy
if model != 'sm':
pdict['process_input']['input_file'] = "LesHouches.msugra_1"
pdict['process_input']['input_slha_format'] = 'T'
pdict['process_input']['beam_recoil'] = 'T'
pdict['integration_input'] = {}
pdict['integration_input']['calls'] = '1 50000 10 5000 1 15000'
pdict['simulation_input'] = {}
pdict['simulation_input']['normalize_weight'] = 'F'
pdict['simulation_input']['n_events'] = nbevts
pdict['simulation_input']['keep_initials'] = 'T'
pdict['simulation_input']['events_per_file'] = 500000
if model != 'sm':
pdict['simulation_input']['pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1;IMSS(1)=11; IMSS(21)=71; IMSS(22)=71"
else:
pdict['simulation_input']['pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ; MSTJ(28)=2 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1"
pdict['parameter_input'] = {}
# pdict['parameter_input']['mmu']=mmu
# pdict['parameter_input']['mtau']=mtau
# pdict['parameter_input']['mb']=mb
# pdict['parameter_input']['mc']=mc
pdict['parameter_input']['mH'] = 125
pdict['beam_input_1'] = {}
pdict['beam_input_1']['particle_name'] = "e1"
pdict['beam_input_1']['polarization'] = "0.0 0.0"
pdict['beam_input_1']['USER_spectrum_on'] = 'T'
pdict['beam_input_1']['USER_spectrum_mode'] = 19 if energy == 1400 else 11
pdict['beam_input_1']['ISR_on'] = 'T'
pdict['beam_input_1']['EPA_on'] = "F"
pdict['beam_input_2'] = {}
pdict['beam_input_2']['particle_name'] = "E1"
pdict['beam_input_2']['polarization'] = "0.0 0.0"
pdict['beam_input_2']['USER_spectrum_on'] = 'T'
pdict['beam_input_2']['ISR_on'] = 'T'
pdict['beam_input_2']['USER_spectrum_mode'] = 19 if energy == 1400 else 11
pdict['beam_input_2']['EPA_on'] = 'F'
whiz.setFullParameterDict(pdict)
whiz.setOutputFile("testgen.stdhep")
return whiz
def getWhizard(self, nbevts):
""" Get some defualt SM whizard
"""
return self.getWhizardModel(nbevts, 1400, "sm")
def getWhizardSUSY(self, nbevts):
""" Get a susy whizard
"""
return self.getWhizardModel(nbevts, 3000, "slsqhh")
def createWhizardTest(self):
"""create a test for whizard"""
self.log.notice("Creating jobs for Whizard")
##### WhizardJob
jobw = self.getJob()
wh = self.getWhizard(2)
res = jobw.append(wh)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
self.jobList['Whizard1'] = jobw
##### WhizardJob
jobwsusy = self.getJob()
whsusy = self.getWhizardSUSY(2)
res = jobwsusy.append(whsusy)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
self.jobList['WhizSusy'] = jobwsusy
return S_OK((jobw, jobwsusy))
def createSlicTest(self):
"""create tests for slic"""
self.log.notice("Creating jobs for SLIC")
#run (Whizard +)SLIC
jobslic = self.getJob()
if self.clip.testChain:
whslic = self.getWhizard(2)
res = jobslic.append(whslic)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
elif self.clip.testInputData:
jobslic.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep")
else:
self.log.error("SLIC does not know where to get its input from")
return S_ERROR()
myslic = self.getSLIC()
if self.clip.testChain:
myslic.getInputFromApp(whslic)
else:
myslic.setNumberOfEvents(2)
res = jobslic.append(myslic)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
self.jobList['Slic1'] = jobslic
return S_OK(jobslic)
def createMarlinTest(self , setInputData = False):
"""create tests for marlin"""
self.log.notice( "Creating test for Marlin" )
#((Whizard + Mokka +)Overlay+) Marlin
jobma = self.getJob()
if self.clip.testChain:
moma = self.getMokka()
if not self.clip.testInputData:
whma = self.getWhizard(2)
res = jobma.append(whma)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
moma.getInputFromApp(whma)
else:
jobma.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep")
moma.setNumberOfEvents(1)
res = jobma.append(moma)
if not res['OK']:
self.log.error("Failed adding Mokka:", res['Message'])
return S_ERROR()
elif self.clip.testInputData:
jobma.setInputData(self.marlinInputdata)
else:
self.log.error("Marlin does not know where to get its input from")
return S_ERROR()
if self.runOverlay:
ov = self.getOverlay(2)
res = jobma.append(ov)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR
ma = self.getMarlin()
if self.clip.testChain:
ma.getInputFromApp(moma)
else:
ma.setNumberOfEvents(2)
res = jobma.append(ma)
if not res['OK']:
self.log.error("Failed adding Marlin:", res['Message'])
return S_ERROR()
self.jobList['Marlin1'] =jobma
return S_OK(jobma)
def createLCSimTest(self):
"""create tests for LCSIM"""
self.log.notice( "Creating test for LCSIM" )
#run ((whiz+SLIC+)+Overlay+)LCSIM
joblcsim = self.getJob()
if self.clip.testChain:
mysliclcsim = self.getSLIC()
if not self.clip.testInputData:
whlcsim = self.getWhizard(2)
res = joblcsim.append(whlcsim)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
mysliclcsim.getInputFromApp(whlcsim)
else:
joblcsim.setInputData("/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep")
mysliclcsim.setNumberOfEvents(2)
res = joblcsim.append(mysliclcsim)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
elif self.clip.testInputData:
#joblcsim.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio")
joblcsim.setInputData("/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio")
else:
self.log.error("LCSIM does not know where to get its input from")
return S_ERROR()
if self.runOverlay:
ovlcsim = self.getOverlay(2)
res = joblcsim.append(ovlcsim)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR()
mylcsim = self.getLCSIM(True)
if self.clip.testChain:
mylcsim.getInputFromApp(mysliclcsim)
else:
mylcsim.setNumberOfEvents(2)
res = joblcsim.append(mylcsim)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
self.jobList['lcsim1'] = joblcsim
return S_OK(joblcsim)
def createSlicPandoraTest(self):
"""create tests for slicPandora"""
self.log.notice("Creating tests for SLICPandora")
#run ((whiz+SLIC) + (Overlay +) LCSIM +) SLICPandora + LCSIM
joblcsimov = self.getJob()
if not self.clip.testChain:
self.log.error("SLICPandora does not know where to get its input from")
return S_ERROR()
mylcsimov = self.getLCSIM(True)
if not self.clip.testInputData:
whlcsimov = self.getWhizard(2)
res = joblcsimov.append(whlcsimov)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
mysliclcsimov = self.getSLIC()
mysliclcsimov.getInputFromApp(whlcsimov)
res = joblcsimov.append(mysliclcsimov)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
mylcsimov.getInputFromApp(mysliclcsimov)
else:
#joblcsimov.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio")
joblcsimov.setInputData("/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio")
mylcsimov.setNumberOfEvents(2)
if self.runOverlay:
ovslicp = self.getOverlay(2)
res = joblcsimov.append(ovslicp)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR()
res = joblcsimov.append(mylcsimov)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
myslicpov = self.getSLICPandora()
myslicpov.getInputFromApp(mylcsimov)
res = joblcsimov.append(myslicpov)
if not res['OK']:
self.log.error("Failed adding SLICPandora: ", res["Message"])
return S_ERROR()
mylcsimovp = self.getLCSIM(False)
mylcsimovp.getInputFromApp(myslicpov)
res = joblcsimov.append(mylcsimovp)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
self.jobList['lcsimov1'] = joblcsimov
return S_OK(joblcsimov)
def createUtilityTests(self):
"""Create tests for utility applications"""
self.log.notice("Creating tests for utility applications")
jobwsplit = self.getJob()
whsplit = self.getWhizard(10)
res = jobwsplit.append(whsplit)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
mystdsplit = JobCreater.getStdhepSplit()
mystdsplit.getInputFromApp(whsplit)
res = jobwsplit.append(mystdsplit)
if not res['OK']:
self.log.error("Failed adding StdHepSplit:", res['Message'])
return S_ERROR()
self.jobList['whizSplit'] = jobwsplit
##### WhizardJob + split
jobwcut = self.getJob()
whcut = self.getWhizard(100)
res = jobwcut.append(whcut)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
mystdcut = self.getStdhepcut( 100 )
mystdcut.getInputFromApp(whcut)
res = jobwcut.append(mystdcut)
if not res['OK']:
self.log.error("Failed adding StdHepCut:", res['Message'])
return S_ERROR()
self.jobList['whizCut'] = jobwcut
#LCIO split
joblciosplit = self.getJob()
# joblciosplit.setInputData("/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio")
joblciosplit.setInputData("/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio")
mylciosplit = self.getLCIOSplit(100)
res = joblciosplit.append(mylciosplit)
if not res['OK']:
self.log.error("Failed adding SLCIOSplit:", res['Message'])
return S_ERROR()
self.jobList['lcioSplit'] = joblciosplit
#LCIO concat
jobconcat = self.getJob()
# jobconcat.setInputData(["/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_27.slcio",
# "/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio"])
jobconcat.setInputData(["/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2104.slcio",
"/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2105.slcio"])
myconcat = self.getLCIOConcat()
res = jobconcat.append(myconcat)
if not res['OK']:
self.log.error("Failed adding SLCIOConcatenate:", res['Message'])
return S_ERROR()
self.jobList['concat'] = jobconcat
return S_OK((jobconcat, joblciosplit,jobwcut,jobwsplit))
def createFccSwTest(self):
"""create tests for fccsw"""
self.log.notice("Creating jobs for FccSW")
#### FccSwJob
jobfccsw = self.getJob()
fccsw = self.getFccSw()
res = jobfccsw.append(fccsw)
if not res['OK']:
self.log.error("Failed adding FccSw:", res['Message'])
return S_ERROR()
self.jobList['FccSw1'] = jobfccsw
return S_OK(jobfccsw)
def createFccAnalysisTest(self):
"""create tests for fccanalysis"""
self.log.notice("Creating jobs for FccAnalysis")
#### FccAnalysisJob
jobfccanalysis = self.getJob()
fccanalysis = self.getFccAnalysis()
res = jobfccanalysis.append(fccanalysis)
if not res['OK']:
self.log.error("Failed adding FccAnalysis:", res['Message'])
return S_ERROR()
self.jobList['FccAnalysis1'] = jobfccanalysis
return S_OK(jobfccanalysis)
def runJobLocally(self, job, jobName="unknown"):
"""run a job locally"""
self.log.notice("I will run the tests locally.")
from DIRAC import gConfig
localarea = gConfig.getValue("/LocalSite/LocalArea", "")
if not localarea:
self.log.error("You need to have /LocalSite/LocalArea defined in your dirac.cfg")
return S_ERROR()
if localarea.find("/afs") == 0:
self.log.error("Don't set /LocalSite/LocalArea set to /afs/... as you'll get to install there")
self.log.error("check ${HOME}/.dirac.cfg and ${DIRAC}/etc/dirac.cfg")
return S_ERROR()
self.log.notice("To run locally, I will create a temp directory here.")
curdir = os.getcwd()
tmpdir = tempfile.mkdtemp("", dir = "./")
os.chdir(tmpdir)
# Jobs that need separate input files
specialJobs = ['root', 'ddsim']
filesForJob = {
'root' : [ 'input2.root', 'input.root' ],
'ddsim' : [ 'FCalTB.tar.gz', 'Muon_50GeV_Fixed_cosTheta0.7.stdhep' ]
}
for specialName in specialJobs:
if "root" in jobName.lower() and specialName == "root":
with open("root.sh", "w") as rScript:
rScript.write( "echo $ROOTSYS" )
with open("func.C", "w") as rMacro:
rMacro.write( '''
void func( TString string ) {
std::cout << string << std::endl;
TFile* file = TFile::Open(string);
file->ls();
}
''' )
testfiledir = 'Testfiles'
for fileName in ['input.root', 'input2.root']:
shutil.copy( os.path.join( curdir, testfiledir, fileName ), os.getcwd() )
print(os.path.join(curdir, "input2.root"), os.getcwd())
if specialName in jobName.lower():
for fileName in filesForJob[specialName]:
shutil.copy( os.path.join( curdir, testfiledir, fileName ), os.getcwd() )
resJob = self.runJob(job, jobName)
os.chdir(curdir)
if not resJob['OK']:
return resJob
os.chdir(curdir)
if not self.clip.nocleanup:
cleanup(tmpdir)
return S_OK()
def run(self):
"""submit and run all the tests in jobList"""
res = S_ERROR()
for name, finjob in self.jobList.iteritems():
if self.clip.submitMode == 'local':
res = self.runJobLocally(finjob, name)
else:
res = self.runJob(finjob, name)
return res
def runJob(self, finjob, name):
"""runs or submits the job"""
self.log.notice("############################################################")
self.log.notice(" Running or submitting job: %s " % name)
self.log.notice("\n\n")
res = finjob.submit(self.diracInstance, mode = self.clip.submitMode)
if not res["OK"]:
self.log.error("Failed job:", res['Message'])
return S_ERROR()
return S_OK()
def checkForTests(self):
"""check which tests to run"""
if self.clip.testMokka:
resMokka = self.createMokkaTest()
if not resMokka['OK']:
return S_ERROR()
if self.clip.testWhizard:
resWhiz = self.createWhizardTest()
if not resWhiz['OK']:
return S_ERROR()
if self.clip.testSlic:
resSlic = self.createSlicTest()
if not resSlic['OK']:
return S_ERROR()
if self.clip.testMarlin:
resMarlin = self.createMarlinTest()
if not resMarlin['OK']:
return S_ERROR()
if self.clip.testLCSIM:
resLCSim = self.createLCSimTest()
if not resLCSim['OK']:
return S_ERROR()
if self.clip.testSlicPandora:
resSP = self.createSlicPandoraTest()
if not resSP['OK']:
return S_ERROR()
if self.clip.testUtilities:
resUtil = self.createUtilityTests()
if not resUtil['OK']:
return S_ERROR()
if self.clip.testRoot:
resRoot = self.createRootScriptTest()
if not resRoot['OK']:
return S_ERROR()
resRoot = self.createRootHaddTest()
if not resRoot['OK']:
return S_ERROR()
resRoot = self.createRootMacroTest()
if not resRoot['OK']:
return S_ERROR()
if self.clip.testFccSw:
resFccSw = self.createFccSwTest()
if not resFccSw['OK']:
return S_ERROR()
if self.clip.testFccAnalysis:
resFccAnalysis = self.createFccAnalysisTest()
if not resFccAnalysis['OK']:
return S_ERROR()
return S_OK()
class JobCreater(object):
"""contains all the versions and parameters to create all theses tests"""
# pylint: disable=too-many-instance-attributes
# Test parameters, necessary due to amount of tests in this class.
def __init__(self, clip, params):
self.clip = clip
self.ildConfig = params.get("ildConfig", None)
self.alwaysOverlay = params.get("alwaysOverlay", False)
self.runOverlay = self.clip.testOverlay or self.alwaysOverlay
self.mokkaVersion = params["mokkaVersion"]
self.mokkaSteeringFile = params.get("mokkaSteeringFile")
self.detectorModel = params.get("detectorModel")
self.marlinVersion = params.get("marlinVersion")
self.marlinSteeringFile = params.get("marlinSteeringFile")
self.ddsimVersion = params.get("ddsimVersion")
self.ddsimDetectorModel = params.get("ddsimDetectorModel")
self.ddsimInputFile = params.get("ddsimInputFile")
self.marlinInputdata = params.get("marlinInputdata")
self.gearFile = params.get("gearFile")
self.lcsimVersion = params.get("lcsimVersion")
self.steeringFileVersion = params.get("steeringFileVersion", None)
self.rootVersion = params["rootVersion"]
self.whizard2Version = params.get("whizard2Version")
self.whizard2SinFile = params.get("whizard2SinFile")
self.energy = params.get("energy")
self.backgroundType = params.get("backgroundType")
self.machine = params.get("machine")
self.gearFile = params.get("gearFile")
self.marlinSteeringFile = params.get("marlinSteeringFile")
self.marlinVersion = params.get("marlinVersion")
self.lcsimPreSteeringFile = params.get("lcsimPreSteeringFile")
self.lcsimPostSteeringFile = params.get("lcsimPostSteeringFile")
self.fccSwPath = params.get("fccSwPath")
self.fccSwSteeringFile = params.get("fccSwSteeringFile")
self.fccAnalysisSteeringFile = params.get("fccAnalysisSteeringFile")
### other things needed to run tests
self.log = gLogger.getSubLogger("JobCreater")
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC, __RCSID__ as drcsid
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import __RCSID__ as jrcsid
from ILCDIRAC.Interfaces.API.NewInterface.Applications import __RCSID__ as apprcsid
if self.clip.submitMode == "local":
self.log.notice("")
self.log.notice(" DIRAC RCSID:", drcsid)
self.log.notice(" Job RCSID:", jrcsid)
self.log.notice("Applications RCSID:", apprcsid)
self.log.notice("")
self.diracInstance = DiracILC(False, 'tests.rep')
self.jobList = {}
def createDDSimTest(self, inputfile=None, detectorModel=None):
"""Create a job running ddsim"""
if inputfile is None:
inputfile = self.ddsimInputFile
sandbox = [inputfile]
if detectorModel is None:
detectorModel = self.ddsimDetectorModel
else:
sandbox.append(detectorModel)
jobdd = self.getJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim
ddsim = DDSim()
ddsim.setVersion(self.ddsimVersion)
ddsim.setDetectorModel(detectorModel)
ddsim.setNumberOfEvents(1)
ddsim.setInputFile(inputfile)
jobdd.setInputSandbox(sandbox)
res = jobdd.append(ddsim)
if not res['OK']:
self.log.error("Failed adding DDSim:", res['Message'])
return S_ERROR("Failed adding DDSim to Job")
return S_OK(jobdd)
def createWhizard2Test(self):
"""Create a job running Whizard2"""
jobdd = self.getJob()
from ILCDIRAC.Interfaces.API.NewInterface.Applications.Whizard2 import Whizard2
whiz = Whizard2()
whiz.setVersion(self.whizard2Version)
whiz.setNumberOfEvents(1)
whiz.setSinFile(self.whizard2SinFile)
whiz.setOutputFile("test.stdhep")
res = jobdd.append(whiz)
if not res['OK']:
self.log.error("Failed adding Whizard2:", res['Message'])
return S_ERROR("Failed adding Whizard2 to Job")
return S_OK(jobdd)
def createMokkaTest(self):
"""create a job running mokka, and maybe whizard before"""
self.log.notice("Creating jobs for Mokka")
#(Whizard + )Mokka
jobmo = self.getJob()
if self.clip.testChain:
whmo = self.getWhizard(2)
res = jobmo.append(whmo)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR("Failed adding Whizard")
elif self.clip.testInputData:
jobmo.setInputData(
"/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep"
)
else:
self.log.error("Mokka does not know where to get its input from")
return S_ERROR("Mokka does not know where to gets its input from")
mo = self.getMokka()
if self.clip.testChain:
mo.getInputFromApp(whmo)
else:
mo.setNumberOfEvents(1)
res = jobmo.append(mo)
if not res['OK']:
self.log.error("Failed adding Mokka:", res['Message'])
return S_ERROR("Failed adding Mokka to Job")
jobmo.setOutputData("testsim.slcio", OutputSE="CERN-DIP-4")
self.jobList['Mokka1'] = jobmo
return S_OK(jobmo)
def createRootScriptTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["root.sh", "input.root", "input2.root"])
root = self.getRoot()
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def createRootHaddTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["input.root", "input2.root"])
root = self.getRoot()
root.setScript("hadd")
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def createRootMacroTest(self):
"""create a job running root"""
self.log.notice("Creating jobs for Root")
jobRoot = self.getJob()
jobRoot.setInputSandbox(["func.C", "input.root", "input2.root"])
root = self.getRootMacro()
root.setScript("func.C")
res = jobRoot.append(root)
if not res['OK']:
self.log.error("Failed adding Root:", res['Message'])
return S_ERROR("Failed adding Root to Job")
self.jobList['Root'] = jobRoot
return S_OK(jobRoot)
def getOverlay(self, nbevts):
""" Create an overlay step
"""
pathToFiles = None
from ILCDIRAC.Interfaces.API.NewInterface.Applications import OverlayInput
overlay = OverlayInput()
if self.energy == 350:
if self.detectorModel == "ILD_o1_v05":
pathToFiles = "/ilc/user/s/sailer/testFiles/overlay/ild_350/"
if pathToFiles:
overlay.setPathToFiles(pathToFiles)
else:
self.log.warn(
"better define pathToFiles for this overlay: %s, %s, %s" %
(self.energy, self.machine, self.backgroundType))
overlay.setMachine(self.machine)
overlay.setEnergy(self.energy)
overlay.setDetectorModel(self.detectorModel)
overlay.setBkgEvtType(self.backgroundType)
overlay.setBXOverlay(60)
overlay.setGGToHadInt(0.3)
overlay.setNumberOfSignalEventsPerJob(nbevts)
return overlay
def getMokka(self):
""" Define a mokka app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka
mokka = Mokka()
mokka.setVersion(self.mokkaVersion)
mokka.setSteeringFile(self.mokkaSteeringFile)
mokka.setOutputFile("testsim.slcio")
mokka.setDetectorModel(self.detectorModel)
if self.steeringFileVersion:
mokka.setSteeringFileVersion(self.steeringFileVersion)
return mokka
def getRoot(self):
""" Define a root app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootScript
root = RootScript()
root.setScript("root.sh")
root.setArguments("output.root input.root input2.root")
root.setVersion(self.rootVersion)
root.setOutputFile("output.root")
return root
def getRootMacro(self):
""" Define a root app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import RootMacro
root = RootMacro()
root.setMacro("func.C")
root.setArguments(r"\"input.root\"")
root.setVersion(self.rootVersion)
return root
@staticmethod
def getSLIC():
""" Get a SLIC instance
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
slic.setOutputFile('testsim.slcio')
return slic
@staticmethod
def getSLICPandora():
""" Get some SLICPAndora app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLICPandora
slicp = SLICPandora()
slicp.setVersion('CLIC_CDR')
slicp.setDetectorModel('clic_sid_cdr')
slicp.setPandoraSettings("PandoraSettingsSlic.xml")
slicp.setOutputFile('testpandora.slcio')
return slicp
def getMarlin(self):
""" Define a marlin step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Marlin
marlin = Marlin()
# marlin.setVersion("v0111Prod")
marlin.setVersion(self.marlinVersion)
marlin.setSteeringFile(self.marlinSteeringFile)
marlin.setGearFile(self.gearFile)
marlin.setOutputDstFile("testmarlinDST.slcio")
marlin.setOutputRecFile("testmarlinREC.slcio")
marlin.setNumberOfEvents(1)
return marlin
def getDD(self):
""" Create a DDSim object
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications.DDSim import DDSim
ddsim = DDSim()
ddsim.setVersion(self.ddsimVersion)
ddsim.setDetectorModel(self.ddsimDetectorModel)
ddsim.setInputFile(self.ddsimInputFile)
ddsim.setNumberOfEvents(2)
return ddsim
def getLCSIM(self, prepandora=True):
""" Get some LCSIM
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import LCSIM
lcsim = LCSIM()
lcsim.setVersion('CLIC_CDR')
lcsim.setDetectorModel('clic_sid_cdr.zip')
if prepandora:
lcsim.setSteeringFile(self.lcsimPreSteeringFile)
lcsim.setOutputFile("testlcsim.slcio")
else:
lcsim.setSteeringFile(self.lcsimPostSteeringFile)
#lcsim.setOutputFile("testlcsimfinal.slcio")
lcsim.setOutputDstFile("testlcsimDST.slcio")
lcsim.setOutputRecFile("testlcsimREC.slcio")
lcsim.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml")
return lcsim
def getFccSw(self):
""" Define a fccsw step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccSw
fccsw = FccSw()
fccsw.fccSwPath = self.fccSwPath
fccsw.setSteeringFile(self.fccSwSteeringFile)
return fccsw
def getFccAnalysis(self):
""" Define a fccanalysis step
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import FccAnalysis
fccanalysis = FccAnalysis()
fccanalysis.setSteeringFile(self.fccAnalysisSteeringFile)
return fccanalysis
@staticmethod
def getStdhepcut(generated):
""" Get some cuts in
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdhepCutJava
stdhepcut = StdhepCutJava()
stdhepcut.setVersion('1.0')
stdhepcut.setSelectionEfficiency(1.)
#stdhepcut.setSteeringFile("cuts_testing_1400.txt")
stdhepcut.setInlineCuts("leptonInvMass_R 13 100 200")
stdhepcut.setSteeringFileVersion("V18")
stdhepcut.setMaxNbEvts(1)
stdhepcut.setNumberOfEvents(generated)
return stdhepcut
@staticmethod
def getStdhepSplit():
""" Get some stdhep split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(5)
stdhepsplit.setOutputFile("teststdhepsplit.stdhep")
stdhepsplit.setMaxRead(10)
return stdhepsplit
@staticmethod
def getLCIOSplit(events_per_file):
""" Get a LCIO split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOSplit
lciosplit = SLCIOSplit()
lciosplit.setNumberOfEventsPerFile(events_per_file)
lciosplit.setOutputFile("testlciosplit.slcio")
return lciosplit
@staticmethod
def getLCIOConcat():
""" Get a LCIO Concat
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLCIOConcatenate
lcioconcat = SLCIOConcatenate()
lcioconcat.setOutputFile("testlcioconcat.slcio")
return lcioconcat
def getJob(self):
""" Define a generic job, it should be always the same
"""
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
myjob = UserJob()
myjob.setName("Testing")
myjob.setJobGroup("Tests")
myjob.setCPUTime(30000)
myjob.dontPromptMe()
myjob.setLogLevel("VERBOSE")
myjob.setPlatform("x86_64-slc5-gcc43-opt")
myjob.setOutputSandbox(["*.log", "*.xml", "*.sh"])
myjob._addParameter(myjob.workflow, 'TestFailover', 'String', True,
'Test failoverRequest')
myjob._addParameter(myjob.workflow, 'Platform', 'JDL',
"x86_64-slc5-gcc43-opt", 'OS Platform')
if self.ildConfig:
myjob.setILDConfig(self.ildConfig)
return myjob
def getWhizardModel(self, nbevts, energy, model):
""" Create a default whizard
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Whizard
proddict = "e2e2_o"
if model != "sm":
proddict = "se2se2_r"
whiz = Whizard(processlist=self.diracInstance.getProcessList())
whiz.setModel(model)
pdict = {}
pdict['process_input'] = {}
pdict['process_input']['process_id'] = proddict
pdict['process_input']['sqrts'] = energy
if model != 'sm':
pdict['process_input']['input_file'] = "LesHouches.msugra_1"
pdict['process_input']['input_slha_format'] = 'T'
pdict['process_input']['beam_recoil'] = 'T'
pdict['integration_input'] = {}
pdict['integration_input']['calls'] = '1 50000 10 5000 1 15000'
pdict['simulation_input'] = {}
pdict['simulation_input']['normalize_weight'] = 'F'
pdict['simulation_input']['n_events'] = nbevts
pdict['simulation_input']['keep_initials'] = 'T'
pdict['simulation_input']['events_per_file'] = 500000
if model != 'sm':
pdict['simulation_input'][
'pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1;IMSS(1)=11; IMSS(21)=71; IMSS(22)=71"
else:
pdict['simulation_input'][
'pythia_parameters'] = "PMAS(25,1)=125; PMAS(25,2)=0.3605E-02; MSTU(22)=20 ; MSTJ(28)=2 ;PARJ(21)=0.40000;PARJ(41)=0.11000; PARJ(42)=0.52000; PARJ(81)=0.25000; PARJ(82)=1.90000; MSTJ(11)=3; PARJ(54)=-0.03100; PARJ(55)=-0.00200;PARJ(1)=0.08500; PARJ(3)=0.45000; PARJ(4)=0.02500; PARJ(2)=0.31000; PARJ(11)=0.60000; PARJ(12)=0.40000; PARJ(13)=0.72000;PARJ(14)=0.43000; PARJ(15)=0.08000; PARJ(16)=0.08000; PARJ(17)=0.17000; MSTP(3)=1"
pdict['parameter_input'] = {}
# pdict['parameter_input']['mmu']=mmu
# pdict['parameter_input']['mtau']=mtau
# pdict['parameter_input']['mb']=mb
# pdict['parameter_input']['mc']=mc
pdict['parameter_input']['mH'] = 125
pdict['beam_input_1'] = {}
pdict['beam_input_1']['particle_name'] = "e1"
pdict['beam_input_1']['polarization'] = "0.0 0.0"
pdict['beam_input_1']['USER_spectrum_on'] = 'T'
pdict['beam_input_1'][
'USER_spectrum_mode'] = 19 if energy == 1400 else 11
pdict['beam_input_1']['ISR_on'] = 'T'
pdict['beam_input_1']['EPA_on'] = "F"
pdict['beam_input_2'] = {}
pdict['beam_input_2']['particle_name'] = "E1"
pdict['beam_input_2']['polarization'] = "0.0 0.0"
pdict['beam_input_2']['USER_spectrum_on'] = 'T'
pdict['beam_input_2']['ISR_on'] = 'T'
pdict['beam_input_2'][
'USER_spectrum_mode'] = 19 if energy == 1400 else 11
pdict['beam_input_2']['EPA_on'] = 'F'
whiz.setFullParameterDict(pdict)
whiz.setOutputFile("testgen.stdhep")
return whiz
def getWhizard(self, nbevts):
""" Get some defualt SM whizard
"""
return self.getWhizardModel(nbevts, 1400, "sm")
def getWhizardSUSY(self, nbevts):
""" Get a susy whizard
"""
return self.getWhizardModel(nbevts, 3000, "slsqhh")
def createWhizardTest(self):
"""create a test for whizard"""
self.log.notice("Creating jobs for Whizard")
##### WhizardJob
jobw = self.getJob()
wh = self.getWhizard(2)
res = jobw.append(wh)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
self.jobList['Whizard1'] = jobw
##### WhizardJob
jobwsusy = self.getJob()
whsusy = self.getWhizardSUSY(2)
res = jobwsusy.append(whsusy)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
self.jobList['WhizSusy'] = jobwsusy
return S_OK((jobw, jobwsusy))
def createSlicTest(self):
"""create tests for slic"""
self.log.notice("Creating jobs for SLIC")
#run (Whizard +)SLIC
jobslic = self.getJob()
if self.clip.testChain:
whslic = self.getWhizard(2)
res = jobslic.append(whslic)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
elif self.clip.testInputData:
jobslic.setInputData(
"/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep"
)
else:
self.log.error("SLIC does not know where to get its input from")
return S_ERROR()
myslic = self.getSLIC()
if self.clip.testChain:
myslic.getInputFromApp(whslic)
else:
myslic.setNumberOfEvents(2)
res = jobslic.append(myslic)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
self.jobList['Slic1'] = jobslic
return S_OK(jobslic)
def createMarlinTest(self, setInputData=False):
"""create tests for marlin"""
self.log.notice("Creating test for Marlin")
#((Whizard + Mokka +)Overlay+) Marlin
jobma = self.getJob()
if self.clip.testChain:
moma = self.getMokka()
if not self.clip.testInputData:
whma = self.getWhizard(2)
res = jobma.append(whma)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
moma.getInputFromApp(whma)
else:
jobma.setInputData(
"/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep"
)
moma.setNumberOfEvents(1)
res = jobma.append(moma)
if not res['OK']:
self.log.error("Failed adding Mokka:", res['Message'])
return S_ERROR()
elif self.clip.testInputData:
jobma.setInputData(self.marlinInputdata)
else:
self.log.error("Marlin does not know where to get its input from")
return S_ERROR()
if self.runOverlay:
ov = self.getOverlay(2)
res = jobma.append(ov)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR
ma = self.getMarlin()
if self.clip.testChain:
ma.getInputFromApp(moma)
else:
ma.setNumberOfEvents(2)
res = jobma.append(ma)
if not res['OK']:
self.log.error("Failed adding Marlin:", res['Message'])
return S_ERROR()
self.jobList['Marlin1'] = jobma
return S_OK(jobma)
def createLCSimTest(self):
"""create tests for LCSIM"""
self.log.notice("Creating test for LCSIM")
#run ((whiz+SLIC+)+Overlay+)LCSIM
joblcsim = self.getJob()
if self.clip.testChain:
mysliclcsim = self.getSLIC()
if not self.clip.testInputData:
whlcsim = self.getWhizard(2)
res = joblcsim.append(whlcsim)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
mysliclcsim.getInputFromApp(whlcsim)
else:
joblcsim.setInputData(
"/ilc/prod/clic/SingleParticles/Muon/50GeV/Muon_50GeV_Fixed_cosTheta0.7.stdhep"
)
mysliclcsim.setNumberOfEvents(2)
res = joblcsim.append(mysliclcsim)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
elif self.clip.testInputData:
#joblcsim.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio")
joblcsim.setInputData(
"/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio")
else:
self.log.error("LCSIM does not know where to get its input from")
return S_ERROR()
if self.runOverlay:
ovlcsim = self.getOverlay(2)
res = joblcsim.append(ovlcsim)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR()
mylcsim = self.getLCSIM(True)
if self.clip.testChain:
mylcsim.getInputFromApp(mysliclcsim)
else:
mylcsim.setNumberOfEvents(2)
res = joblcsim.append(mylcsim)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
self.jobList['lcsim1'] = joblcsim
return S_OK(joblcsim)
def createSlicPandoraTest(self):
"""create tests for slicPandora"""
self.log.notice("Creating tests for SLICPandora")
#run ((whiz+SLIC) + (Overlay +) LCSIM +) SLICPandora + LCSIM
joblcsimov = self.getJob()
if not self.clip.testChain:
self.log.error(
"SLICPandora does not know where to get its input from")
return S_ERROR()
mylcsimov = self.getLCSIM(True)
if not self.clip.testInputData:
whlcsimov = self.getWhizard(2)
res = joblcsimov.append(whlcsimov)
if not res["OK"]:
self.log.error("Failed adding Whizard:", res['Value'])
return S_ERROR()
mysliclcsimov = self.getSLIC()
mysliclcsimov.getInputFromApp(whlcsimov)
res = joblcsimov.append(mysliclcsimov)
if not res['OK']:
self.log.error("Failed adding slic: ", res["Message"])
return S_ERROR()
mylcsimov.getInputFromApp(mysliclcsimov)
else:
#joblcsimov.setInputData("/ilc/prod/clic/1.4tev/ee_qqaa/SID/SIM/00002308/000/ee_qqaa_sim_2308_222.slcio")
joblcsimov.setInputData(
"/ilc/user/s/sailer/testFiles/clic_prod_sid_h_nunu_sim.slcio")
mylcsimov.setNumberOfEvents(2)
if self.runOverlay:
ovslicp = self.getOverlay(2)
res = joblcsimov.append(ovslicp)
if not res["OK"]:
self.log.error("Failed adding Overlay:", res['Message'])
return S_ERROR()
res = joblcsimov.append(mylcsimov)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
myslicpov = self.getSLICPandora()
myslicpov.getInputFromApp(mylcsimov)
res = joblcsimov.append(myslicpov)
if not res['OK']:
self.log.error("Failed adding SLICPandora: ", res["Message"])
return S_ERROR()
mylcsimovp = self.getLCSIM(False)
mylcsimovp.getInputFromApp(myslicpov)
res = joblcsimov.append(mylcsimovp)
if not res['OK']:
self.log.error("Failed adding LCSIM: ", res["Message"])
return S_ERROR()
self.jobList['lcsimov1'] = joblcsimov
return S_OK(joblcsimov)
def createUtilityTests(self):
"""Create tests for utility applications"""
self.log.notice("Creating tests for utility applications")
jobwsplit = self.getJob()
whsplit = self.getWhizard(10)
res = jobwsplit.append(whsplit)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
mystdsplit = JobCreater.getStdhepSplit()
mystdsplit.getInputFromApp(whsplit)
res = jobwsplit.append(mystdsplit)
if not res['OK']:
self.log.error("Failed adding StdHepSplit:", res['Message'])
return S_ERROR()
self.jobList['whizSplit'] = jobwsplit
##### WhizardJob + split
jobwcut = self.getJob()
whcut = self.getWhizard(100)
res = jobwcut.append(whcut)
if not res['OK']:
self.log.error("Failed adding Whizard:", res['Message'])
return S_ERROR()
mystdcut = self.getStdhepcut(100)
mystdcut.getInputFromApp(whcut)
res = jobwcut.append(mystdcut)
if not res['OK']:
self.log.error("Failed adding StdHepCut:", res['Message'])
return S_ERROR()
self.jobList['whizCut'] = jobwcut
#LCIO split
joblciosplit = self.getJob()
# joblciosplit.setInputData("/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio")
joblciosplit.setInputData(
"/ilc/user/s/sailer/testFiles/prod_clic_ild_e2e2_o_sim_2214_26.slcio"
)
mylciosplit = self.getLCIOSplit(100)
res = joblciosplit.append(mylciosplit)
if not res['OK']:
self.log.error("Failed adding SLCIOSplit:", res['Message'])
return S_ERROR()
self.jobList['lcioSplit'] = joblciosplit
#LCIO concat
jobconcat = self.getJob()
# jobconcat.setInputData(["/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_27.slcio",
# "/ilc/prod/clic/1.4tev/e2e2_o/ILD/DST/00002215/000/e2e2_o_dst_2215_46.slcio"])
jobconcat.setInputData([
"/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2104.slcio",
"/ilc/prod/clic/1.4tev/aa_qqll_all/ILD/DST/00004275/002/aa_qqll_all_dst_4275_2105.slcio"
])
myconcat = self.getLCIOConcat()
res = jobconcat.append(myconcat)
if not res['OK']:
self.log.error("Failed adding SLCIOConcatenate:", res['Message'])
return S_ERROR()
self.jobList['concat'] = jobconcat
return S_OK((jobconcat, joblciosplit, jobwcut, jobwsplit))
def createFccSwTest(self):
"""create tests for fccsw"""
self.log.notice("Creating jobs for FccSW")
#### FccSwJob
jobfccsw = self.getJob()
fccsw = self.getFccSw()
res = jobfccsw.append(fccsw)
if not res['OK']:
self.log.error("Failed adding FccSw:", res['Message'])
return S_ERROR()
self.jobList['FccSw1'] = jobfccsw
return S_OK(jobfccsw)
def createFccAnalysisTest(self):
"""create tests for fccanalysis"""
self.log.notice("Creating jobs for FccAnalysis")
#### FccAnalysisJob
jobfccanalysis = self.getJob()
fccanalysis = self.getFccAnalysis()
res = jobfccanalysis.append(fccanalysis)
if not res['OK']:
self.log.error("Failed adding FccAnalysis:", res['Message'])
return S_ERROR()
self.jobList['FccAnalysis1'] = jobfccanalysis
return S_OK(jobfccanalysis)
def runJobLocally(self, job, jobName="unknown"):
"""run a job locally"""
self.log.notice("I will run the tests locally.")
from DIRAC import gConfig
localarea = gConfig.getValue("/LocalSite/LocalArea", "")
if not localarea:
self.log.error(
"You need to have /LocalSite/LocalArea defined in your dirac.cfg"
)
return S_ERROR()
if localarea.find("/afs") == 0:
self.log.error(
"Don't set /LocalSite/LocalArea set to /afs/... as you'll get to install there"
)
self.log.error(
"check ${HOME}/.dirac.cfg and ${DIRAC}/etc/dirac.cfg")
return S_ERROR()
self.log.notice("To run locally, I will create a temp directory here.")
curdir = os.getcwd()
tmpdir = tempfile.mkdtemp("", dir="./")
os.chdir(tmpdir)
# Jobs that need separate input files
specialJobs = ['root', 'ddsim']
filesForJob = {
'root': ['input2.root', 'input.root'],
'ddsim': ['FCalTB.tar.gz', 'Muon_50GeV_Fixed_cosTheta0.7.stdhep']
}
for specialName in specialJobs:
if "root" in jobName.lower() and specialName == "root":
with open("root.sh", "w") as rScript:
rScript.write("echo $ROOTSYS")
with open("func.C", "w") as rMacro:
rMacro.write('''
void func( TString string ) {
std::cout << string << std::endl;
TFile* file = TFile::Open(string);
file->ls();
}
''')
testfiledir = 'Testfiles'
for fileName in ['input.root', 'input2.root']:
shutil.copy(os.path.join(curdir, testfiledir, fileName),
os.getcwd())
print os.path.join(curdir, "input2.root"), os.getcwd()
if specialName in jobName.lower():
for fileName in filesForJob[specialName]:
shutil.copy(os.path.join(curdir, testfiledir, fileName),
os.getcwd())
resJob = self.runJob(job, jobName)
os.chdir(curdir)
if not resJob['OK']:
return resJob
os.chdir(curdir)
if not self.clip.nocleanup:
cleanup(tmpdir)
return S_OK()
def run(self):
"""submit and run all the tests in jobList"""
res = S_ERROR()
for name, finjob in self.jobList.iteritems():
if self.clip.submitMode == 'local':
res = self.runJobLocally(finjob, name)
else:
res = self.runJob(finjob, name)
return res
def runJob(self, finjob, name):
"""runs or submits the job"""
self.log.notice(
"############################################################")
self.log.notice(" Running or submitting job: %s " % name)
self.log.notice("\n\n")
res = finjob.submit(self.diracInstance, mode=self.clip.submitMode)
if not res["OK"]:
self.log.error("Failed job:", res['Message'])
return S_ERROR()
return S_OK()
def checkForTests(self):
"""check which tests to run"""
if self.clip.testMokka:
resMokka = self.createMokkaTest()
if not resMokka['OK']:
return S_ERROR()
if self.clip.testWhizard:
resWhiz = self.createWhizardTest()
if not resWhiz['OK']:
return S_ERROR()
if self.clip.testSlic:
resSlic = self.createSlicTest()
if not resSlic['OK']:
return S_ERROR()
if self.clip.testMarlin:
resMarlin = self.createMarlinTest()
if not resMarlin['OK']:
return S_ERROR()
if self.clip.testLCSIM:
resLCSim = self.createLCSimTest()
if not resLCSim['OK']:
return S_ERROR()
if self.clip.testSlicPandora:
resSP = self.createSlicPandoraTest()
if not resSP['OK']:
return S_ERROR()
if self.clip.testUtilities:
resUtil = self.createUtilityTests()
if not resUtil['OK']:
return S_ERROR()
if self.clip.testRoot:
resRoot = self.createRootScriptTest()
if not resRoot['OK']:
return S_ERROR()
resRoot = self.createRootHaddTest()
if not resRoot['OK']:
return S_ERROR()
resRoot = self.createRootMacroTest()
if not resRoot['OK']:
return S_ERROR()
if self.clip.testFccSw:
resFccSw = self.createFccSwTest()
if not resFccSw['OK']:
return S_ERROR()
if self.clip.testFccAnalysis:
resFccAnalysis = self.createFccAnalysisTest()
if not resFccAnalysis['OK']:
return S_ERROR()
return S_OK()
if energy==1400.:
spectrum = 19
elif energy == 3000.:
spectrum = 11
elif energy == 500.:
spectrum = 13
elif energy == 350.:
spectrum = 20
else:
print "No spectrum defined, cannot proceed"
exit(1)
##Start by defining the whizard application
wh = Whizard(processlist=dirac.getProcessList())
wh.setModel(model)
pdict = {}
pdict['process_input'] = {}
pdict['process_input']['process_id']=proddict['process']
pdict['process_input']['sqrts'] = energy
if not model=='sm':
pdict['process_input']['input_file'] = "LesHouches.msugra_1"
pdict['process_input']['input_slha_format'] = 'T'
if beamrecoil and gen:
prod_name += "_beamrecoil"
pdict['process_input']['beam_recoil']='T'
pdict['integration_input'] = {}
pdict['integration_input']['calls'] = '1 500000 10 500000 1 1500000'
if gridfiles:
class DiracILCTestCase( unittest.TestCase ):
""" Base class for the DiracILC test cases
"""
def setUp( self ):
"""set up the objects"""
ops_mock = Mock()
mocked_modules = { 'DIRAC.ConfigurationSystem.Client.Helpers.Operations' : ops_mock }
self.module_patcher = patch.dict( sys.modules, mocked_modules )
self.module_patcher.start()
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
self.dilc = DiracILC()
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['SE']
if 'Minimum' in args[0]:
return 1
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = Mock()
ops_mock.getValue.side_effect = setOptions
self.dilc.ops = ops_mock
def tearDown( self ):
self.module_patcher.stop()
def test_getprocesslist( self ):
with patch('%s.gConfig.getValue' % MODULE_NAME, new=Mock(return_value='some_gconf_testval')) as conf_mock, \
patch('%s.ProcessList' % MODULE_NAME, new=Mock()) as pl_mock:
res = self.dilc.getProcessList()
pl_mock.assert_called_once_with( 'some_gconf_testval' )
assertEqualsImproved( res, pl_mock(), self )
conf_mock.assert_called_once_with( '/LocalSite/ProcessListPath', '' )
def test_getprocesslist_nopath( self ):
ops_mock = Mock()
ops_mock.getValue.return_value = ''
self.dilc.ops = ops_mock
with patch('%s.gConfig.getValue' % MODULE_NAME, new=Mock(return_value='')) as conf_mock, \
patch('%s.ProcessList' % MODULE_NAME, new=Mock()) as pl_mock:
res = self.dilc.getProcessList()
pl_mock.assert_called_once_with( '' )
assertEqualsImproved( res, pl_mock(), self )
conf_mock.assert_called_once_with( '/LocalSite/ProcessListPath', '' )
ops_mock.getValue.assert_called_once_with( '/ProcessList/Location', '' )
def test_presubmissionchecks_notoktosubmit( self ):
job_mock = Mock()
job_mock.oktosubmit = False
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'you should use job.submit(dirac)', self )
def test_presubmissionchecks_checkfails( self ):
job_mock = Mock()
job_mock.oktosubmit = True
with patch('%s.DiracILC._do_check' % MODULE_NAME, new=Mock(return_value=S_ERROR('mytest_check_failed'))):
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'mytest_check_failed', self )
def test_presubmissionchecks_askuser_fails( self ):
job_mock = Mock()
job_mock.oktosubmit = True
job_mock._askUser.return_value = S_ERROR( 'user says no' )
self.dilc.checked = False
with patch('%s.DiracILC._do_check' % MODULE_NAME, new=Mock(return_value=S_OK())):
assertDiracFailsWith( self.dilc.preSubmissionChecks( job_mock, None ),
'user says no', self )
def test_checkparams( self ):
job_mock = Mock()
job_mock.errorDict = {}
with patch('%s.DiracILC.preSubmissionChecks' % MODULE_NAME, new=Mock(return_value=S_OK('mytest'))) as check_mock:
assertDiracSucceedsWith( self.dilc.checkparams( job_mock ), 'mytest', self )
check_mock.assert_called_once_with( job_mock, mode = '' )
def test_checkparams_fails( self ):
job_mock = Mock()
job_mock.errorDict = { 'myerror1' : [ 'Terrible failure' ], 'last_error' : [ 'True' ] }
assertDiracFailsWith_equals( self.dilc.checkparams( job_mock ),
{ 'myerror1' : [ 'Terrible failure' ], 'last_error' : [ 'True' ] }, self )
def test_giveprocesslist( self ):
self.dilc.processList = '13985u185r9135r'
assertEqualsImproved( self.dilc.giveProcessList(), '13985u185r9135r', self )
def test_giveprocesslist_empty( self ):
self.dilc.processList = ''
assertEqualsImproved( self.dilc.giveProcessList(), '', self )
def test_giveprocesslist_false( self ):
self.dilc.processList = False
assertEqualsImproved( self.dilc.giveProcessList(), False, self )
def test_giveprocesslist_none( self ):
self.dilc.processList = None
assertEqualsImproved( self.dilc.giveProcessList(), None, self )
def test_retrievelfns_norepo( self ):
self.dilc.jobRepo = None
assertDiracSucceeds( self.dilc.retrieveRepositoryOutputDataLFNs(), self )
def test_retrievelfns( self ):
repo_mock = Mock()
ret_dict = { '1' : { 'State' : 'Done', 'UserOutputData' : '1389' }, '2' : {},
'3' : { 'State' : 'secret_teststate' }, '4' : { 'State' : 'invalid_state' },
'5' : { 'State' : 'Done', 'UserOutputData' : 0 }, '6' : { 'ignore_me' : True },
'7' : { 'State' : 'secret_teststate', 'UserOutputData' : 0 },
'148' : { 'State' : 'Done', 1 : False, True : 941, 'values_' : 'keys' } }
repo_mock.readRepository.return_value = S_OK( ret_dict )
self.dilc.jobRepo = repo_mock
with patch('%s.DiracILC.parameters' % MODULE_NAME, new=Mock(side_effect=[S_OK({'UploadedOutputData':'/my/test/lfn1'}),S_ERROR(),S_OK({}),S_OK({'some_entries':'some_values',1:True,'UploadedOutputData':'/more_lfns/append/testlfn.log'})])) as param_mock:
assertEqualsImproved( self.dilc.retrieveRepositoryOutputDataLFNs( [ 'Done', 'secret_teststate' ] ),
[ '/my/test/lfn1', '/more_lfns/append/testlfn.log' ], self )
assertMockCalls( param_mock, [ 3, 5, 7, 148 ], self )
def test_docheck_checksandbox_fails( self ):
job_mock = Mock()
job_mock.inputsandbox = [ 'mysandbox', 'other_value' ]
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_ERROR('test_err_sandbox'))) as check_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'test_err_sandbox', self )
check_mock.assert_called_once_with( job_mock )
def test_docheck_too_many_lists( self ):
job_mock = Mock()
job_mock.inputsandbox = [ 'mysandbox', [ 'im_a_sandbox_file.stdhep', [ 'evil_list', 'deletethis'] ] ]
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'too many lists of lists in the input sandbox', self )
def test_docheck_checkapps_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock }
job_mock = Mock()
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(side_effect=[S_OK()] * 3 + [S_ERROR('checkapp_failed_testme')])) as checkapp_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'checkapp_failed_testme', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
def test_docheck_checkoutputpath_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
path_mock = Mock()
path_mock.getValue.return_value = 'path1948512895'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock, 'UserOutputPath' : path_mock }
job_mock = Mock()
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \
patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_ERROR('outputpath_check_testerr'))) as checkpath_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'outputpath_check_testerr', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
checkpath_mock.assert_called_once_with( 'path1948512895' )
def test_docheck_checkconsistency_fails( self ):
platform_mock = Mock()
platform_mock.getValue.return_value = 'pf14081'
apps_mock = Mock()
apps_mock.getValue.return_value = 'Myapp1v.9.2.1;other_Appv.91.3;more_Dependencies.1;LasT_APP.0'
path_mock = Mock()
path_mock.getValue.return_value = 'path1948512895'
data_mock = Mock()
data_mock.getValue.return_value = 'data1389518'
param_dict = { 'Platform' : platform_mock, 'SoftwarePackages' : apps_mock,
'UserOutputPath' : path_mock, 'UserOutputData' : data_mock }
job_mock = Mock()
job_mock.addToOutputSandbox = 'job_sandbox13895'
wf_mock = Mock()
wf_mock.findParameter.side_effect = lambda param_name : param_dict[ param_name ]
job_mock.inputsandbox = [ 'mysandbox', 'other_value', [ 'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] ]
job_mock._resolveInputSandbox.return_value = [ 'resolved_file.1.txt', 'other_resolved_file.txt' ]
job_mock.workflow = wf_mock
with patch('%s.DiracILC.checkInputSandboxLFNs' % MODULE_NAME, new=Mock(return_value=S_OK())) as checksb_mock, \
patch('%s.DiracILC._checkapp' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkapp_mock, \
patch('%s.DiracILC._checkoutputpath' % MODULE_NAME, new=Mock(return_value=S_OK())) as checkpath_mock, \
patch('%s.DiracILC._checkdataconsistency' % MODULE_NAME, new=Mock(return_value=S_ERROR('consistency_testerr'))) as checkconsistency_mock:
assertDiracFailsWith( self.dilc._do_check( job_mock ), 'consistency_testerr', self )
checksb_mock.assert_called_once_with( job_mock )
job_mock._resolveInputSandbox.assert_called_once_with( [ 'mysandbox', 'other_value',
'sandbox_file1.txt', 'sandbox_file2.log',
'last.file' ] )
job_mock._addParameter.assert_called_once_with(
wf_mock, 'InputSandbox', 'JDL', 'resolved_file.1.txt;other_resolved_file.txt', 'Input sandbox file list' )
assertMockCalls( checkapp_mock, [ ( 'pf14081', 'myapp1v', '9.2.1' ),
( 'pf14081', 'other_appv', '91.3' ),
( 'pf14081', 'more_dependencies', '1' ),
( 'pf14081', 'last_app', '0' ) ], self )
checkpath_mock.assert_called_once_with( 'path1948512895' )
checkconsistency_mock.assert_called_once_with( 'data1389518', 'job_sandbox13895' )
def test_checkapp( self ):
ops_mock = Mock()
ops_mock.getValue.return_value = ''
self.dilc.ops = ops_mock
assertDiracFailsWith( self.dilc._checkapp( 'test_platform_341', 'testapp', 'v13.2' ),
'could not find the specified software testapp_v13.2 for test_platform_341, check in CS',
self )
assertMockCalls( ops_mock.getValue, [
( '/AvailableTarBalls/test_platform_341/testapp/v13.2/TarBall', '' ),
( '/AvailableTarBalls/test_platform_341/testapp/v13.2/CVMFSPath', '' ) ], self )
def test_checkoutputpath_invalidchar_1( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( 'http://www.mysitedoesnotexist3h3.abc/some/file.txt' ),
'invalid path', self )
def test_checkoutputpath_invalidchar_2( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/./some/file.log' ),
'invalid path', self )
def test_checkoutputpath_invalidchar_3( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/../dir2/somefile.txt' ),
'invalid path', self )
def test_checkoutputpath_trailing_slash( self ):
assertDiracFailsWith( self.dilc._checkoutputpath( '/my/dir/myfile.txt/ ' ),
'invalid path', self )
def test_checkdataconsistency_outputdata_sandbox_equal( self ):
assertDiracFailsWith( self.dilc._checkdataconsistency( 'same_item;something_else',
[ 'distinct_item1', 'same_item' ] ),
'output data and sandbox should not contain the same thing', self )
def test_checkdataconsistency_wildcardchar( self ):
assertDiracFailsWith( self.dilc._checkdataconsistency(
'/test/dir/file.txt;/mydir/something/*;/file/dir/log.log',
[ '/input/sandbox.pdf', '/other/sb/file.stdhep' ] ),
'wildcard character in outputdata definition', self )
def test_checkinputsb_getreplicas_notok( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_err'))) as replica_mock:
assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'could not get replicas', self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb_getreplicas_fails( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
ret_dict = { 'Failed' : [ '/failed/replica1', '/other/inval/replica' ], 'Successful' : {} }
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracFailsWith( self.dilc.checkInputSandboxLFNs( job_mock ), 'failed to find replicas', self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb( self ):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep;LFN:/my/dir/inputsandbox/in1.stdio;lfn:/my/dir/inputsandbox/in2.pdf'
ret_dict = {'Failed': [], 'Successful': {'/one/replica': {'SE': 'surl'}}}
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracSucceeds( self.dilc.checkInputSandboxLFNs( job_mock ), self )
replica_mock.assert_called_once_with( [ '/my/dir/inputsandbox/in1.stdio', '/my/dir/inputsandbox/in2.pdf' ] )
def test_checkinputsb_notInputSB(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value = None
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_notInputSB_Value(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = ''
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_noLFNs(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = '/some/file.txt;/other/some/file.stdhep'
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
def test_checkinputsb_noRepl(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = 'LFN:/some/file.txt'
ret_dict = {'Failed': [], 'Successful': {'/some/file.txt': {'Bad-SE': 'surl'}}}
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['Awesome-Disk-SE']
if 'Minimum' in args[0]:
return 2
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = setOptions
self.dilc.ops = ops_mock
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracFailsWith(self.dilc.checkInputSandboxLFNs(job_mock), 'Not enough replicas', self)
replica_mock.assert_called_once_with(['/some/file.txt'])
def test_checkinputsb_goodRepl(self):
job_mock = Mock()
job_mock.workflow.findParameter.return_value.getValue.return_value = 'LFN:/some/file.txt'
ret_dict = {'Failed': [], 'Successful': {'/some/file.txt': {'Awesome-Disk-SE': 'surl'}}}
def setOptions(*args):
if 'SingleReplicaSEs' in args[0]:
return ['Awesome-Disk-SE']
if 'Minimum' in args[0]:
return 2
if args[0].endswith('PreferredSEs'):
return ['Awesome-Tape-SE']
ops_mock = Mock()
ops_mock.getValue = setOptions
self.dilc.ops = ops_mock
with patch('%s.DiracILC.getReplicas' % MODULE_NAME, new=Mock(return_value=S_OK(ret_dict))) as replica_mock:
assertDiracSucceeds(self.dilc.checkInputSandboxLFNs(job_mock), self)
replica_mock.assert_called_once_with(['/some/file.txt'])