def test_setStartFrom2(self):
"""test DDSIm setStartFrom 2...................................................................."""
ddsim = DDSim()
ddsim.setStartFrom(42)
self.assertEqual(ddsim.startFrom, 42)
job.setOutputSandbox(["*.log","*.sh","*.py "])
#job.setOutputData(lcoutputSIM,OutputPath="MyTest/sim1",OutputSE="IN2P3-SRM")
job.setOutputData( SIMoutput,"MyProd_" + ILDConfigVer + "/E250-TDR_ws/" + chann + "/" +ireq+ "/sim","IN2P3-SRM")
#job.setDestinationCE('lyogrid07.in2p3.fr')
job.dontPromptMe()
job.setBannedSites(['LCG.Tau.il'])
#job.setBannedSites(['LCG.IN2P3-CC.fr','LCG.DESYZN.de','LCG.DESY-HH.de','LCG.KEK.jp','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])
sim = DDSim()
sim.setVersion(ILCSoftVer)
sim.setDetectorModel(detModelSim)
sim.setInputFile(lcinputSIM)
sim.setSteeringFile("ddsim_steer.py")
sim.setNumberOfEvents(evtsPrun)
sim.setEnergy(energy)
sim.setOutputFile(lcoutputSIM)
#sim.setRandomSeed(RandSeed)
sim.setStartFrom(evtStart)
simres = job.append(sim)
if not simres['OK']:
print simres['Not ok appending ddsim to job']
quit()
print SIMoutput
print job.submit(dirac)
def test_setStartFrom1(self):
"""test DDSIm setStartFrom 1...................................................................."""
ddsim = DDSim()
ddsim.setStartFrom("Arg")
self.assertTrue(ddsim._errorDict)
class DDSimTestCase(unittest.TestCase):
""" Base class for the DDSim test cases
"""
def setUp(self):
"""set up the objects"""
self.dds = DDSim({})
def test_setrandomseed(self):
self.assertFalse(self.dds._errorDict)
self.dds.setRandomSeed(89421)
self.assertFalse(self.dds._errorDict)
assertEqualsImproved(self.dds.randomSeed, 89421, self)
def test_setrandomseed_fails(self):
self.assertFalse(self.dds._errorDict)
self.dds.setRandomSeed(['abc'])
self.assertIn('_checkArgs', self.dds._errorDict)
def test_setstartfrom(self):
self.assertFalse(self.dds._errorDict)
self.dds.setStartFrom(89421)
self.assertFalse(self.dds._errorDict)
assertEqualsImproved(self.dds.startFrom, 89421, self)
def test_setstartfrom_fails(self):
self.assertFalse(self.dds._errorDict)
self.dds.setStartFrom('adgiuj')
self.assertIn('_checkArgs', self.dds._errorDict)
def test_resolvelinkedparams(self):
step_mock = Mock()
input_mock = Mock()
input_mock.getType.return_value = {'abc': False}
self.dds._linkedidx = 3
self.dds._jobsteps = [None, None, None, input_mock]
assertDiracSucceeds(self.dds._resolveLinkedStepParameters(step_mock),
self)
step_mock.setLink.assert_called_once_with('InputFile', {'abc': False},
'OutputFile')
def test_resolvelinkedparams_noinputstep(self):
self.dds._linkedidx = None
self.dds._inputappstep = []
assertDiracSucceeds(self.dds._resolveLinkedStepParameters(None), self)
def test_checkworkflow_app_missing(self):
self.dds._inputapp = [
'some_depdency', 'unavailable_dependency_fail_on_this'
]
self.dds._jobapps = ['myjobapp_1', 'some_dependency']
assertDiracFailsWith(self.dds._checkWorkflowConsistency(),
'job order not correct', self)
def test_checkworkflow_empty(self):
self.dds._inputapp = []
self.dds._jobapps = []
assertDiracSucceeds(self.dds._checkWorkflowConsistency(), self)
def test_checkworkflow_success(self):
self.dds._inputapp = [
'some_dependency', 'other_dependencies', 'many_more'
]
self.dds._jobapps = [
'ignore_me', 'many_more', 'some_dependency', 'other_dependencies'
]
assertDiracSucceeds(self.dds._checkWorkflowConsistency(), self)
def test_userjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.dds._userjobmodules(module_mock), self)
def test_prodjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.dds._prodjobmodules(module_mock), self)
def test_userjobmodules_fails(self):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith(self.dds._userjobmodules(None),
'userjobmodules failed', self)
def test_prodjobmodules_fails(self):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith(self.dds._prodjobmodules(None),
'prodjobmodules failed', self)
def test_checkconsistency(self):
self.dds.version = '134'
self.dds.detectorModel = 'mymodel.det'
self.dds.outputFile = 'myoutput.file'
self.dds._jobtype = 'User'
assertDiracSucceeds(self.dds._checkConsistency(Mock()), self)
self.assertNotIn(
{
'outputFile': '@{OutputFile}',
'outputPath': '@{OutputPath}',
'outputDataSE': '@{OutputSE}'
}, self.dds._listofoutput)
self.assertNotIn('nbevts', self.dds.prodparameters)
self.assertNotIn('Process', self.dds.prodparameters)
def test_checkconsistency_nodetectormodel(self):
self.dds.version = 123
self.dds.steeringFile = None
self.dds.detectorModel = None
assertDiracFailsWith(self.dds._checkConsistency(Mock()),
'no detectormodel set', self)
def test_checkconsistency_noversion(self):
self.dds.version = None
assertDiracFailsWith(self.dds._checkConsistency(Mock()),
'no version found', self)
def test_checkconsistency_existsfails(self):
self.dds.version = '134'
self.dds.steeringFile = 'mysteer.file'
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(DDSim), 'Exists', new=Mock(return_value=S_ERROR('testerr_exists_mock'))):
assertDiracFailsWith(self.dds._checkConsistency(Mock()),
'testerr_exists_mock', self)
def test_checkconsistency_userjob(self):
self.dds.version = '134'
self.dds.steeringFile = 'mysteer.file'
self.dds._jobtype = 'notUser'
self.dds.detectorModel = 'myDetectorv200'
with patch('os.path.exists', new=Mock(return_value=True)), \
patch.object(inspect.getmodule(DDSim), 'Exists', new=Mock(return_value=S_ERROR('testerr_exists_mock'))):
assertDiracSucceeds(self.dds._checkConsistency(Mock()), self)
self.assertIn(
{
'outputFile': '@{OutputFile}',
'outputPath': '@{OutputPath}',
'outputDataSE': '@{OutputSE}'
}, self.dds._listofoutput)
for keyword in ['detectorType', 'slic_detectormodel']:
self.assertIn(keyword, self.dds.prodparameters)
def test_checkconsistency_userjob_notdetmodel(self):
self.dds.version = '134'
self.dds.steeringFile = 'mysteer.file'
self.dds._jobtype = 'notUser'
self.dds.detectorModel = True
self.dds.setStartFrom(148)
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(DDSim), 'Exists', new=Mock(return_value=S_OK())):
assertDiracSucceeds(self.dds._checkConsistency(Mock()), self)
self.assertIn(
{
'outputFile': '@{OutputFile}',
'outputPath': '@{OutputPath}',
'outputDataSE': '@{OutputSE}'
}, self.dds._listofoutput)
for keyword in ['detectorType', 'slic_detectormodel']:
self.assertIn(keyword, self.dds.prodparameters)
def test_setStartFrom2( self ): """test DDSIm setStartFrom 2....................................................................""" ddsim = DDSim() ddsim.setStartFrom( 42 ) self.assertEqual( ddsim.startFrom, 42 )
def test_setStartFrom1( self ): """test DDSIm setStartFrom 1....................................................................""" ddsim = DDSim() ddsim.setStartFrom( "Arg") self.assertTrue( ddsim._errorDict )
def main(argv):
# Input arguments
ildconfig_version = "$ILDCONFIGVER"
ilcsoft_version = "$ILCSOFTVER"
evts_per_run = $EVTSPERRUN
detector_model = "$DETECTOR"
sim_input = "$SIMINPUT"
process_name = "$PROCESS"
index = $IND
sim_input = diracpath_from_pnfspath( sim_input )
sim_detector_model = detector_model_wo_option( detector_model )
job_group = ilcsoft_version + "_" + ildconfig_version + "_" + process_name + "_" + detector_model
dirac = DiracILC(True,job_group+".rep")
# outputs to be saved onto grid SE
RECoutput = []
# DDSim
evtStart = (index-1)*evts_per_run
evtEnd = index*evts_per_run - 1
RandSeed = random.randrange(11623, 99999)
lcinputSIM = "LFN:" + sim_input
lcoutputSIM = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_SIM.slcio"%(str(evtStart),(str)(evtEnd))
sim = DDSim()
sim.setVersion(ilcsoft_version)
sim.setDetectorModel(sim_detector_model)
sim.setInputFile(lcinputSIM)
sim.setSteeringFile("ddsim_steer.py")
sim.setNumberOfEvents(evts_per_run)
sim.setRandomSeed(RandSeed)
sim.setEnergy(1000)
sim.setStartFrom(evtStart)
sim.setOutputFile(lcoutputSIM)
# Marlin
lcoutputDST = ilcsoft_version + ".ILDConfig_" + ildconfig_version + ".E1000." + process_name + ".eLpR.evt%s-%s_DST.slcio"%(str(evtStart),(str)(evtEnd))
ma = Marlin()
ma.setVersion(ilcsoft_version)
ma.setDetectorModel(detector_model)
ma.setSteeringFile("MarlinStdReco.xml")
ma.setExtraCLIArguments( "--constant.lcgeo_DIR=$lcgeo_DIR --constant.DetectorModel={} --global.MaxRecordNumber=0".format(detector_model) )
ma.setLogFile("marlin.log")
ma.getInputFromApp(sim)
ma.setEnergy(1000)
ma.setOutputDstFile(lcoutputDST)
RECoutput.append(lcoutputDST)
# ILCDirac user job
job = UserJob()
job.setName("user_sim_reco")
job.setJobGroup(job_group)
job.setILDConfig(ildconfig_version)
job.setCPUTime(86400)
tmp_file_name = process_name + "_sim_reco_job_tmp.py"
job.setInputSandbox([tmp_file_name])
job.setOutputSandbox(["*.log","MarlinStdRecoParsed.xml","marlin*.xml","*.py "])
job.dontPromptMe()
job.setBannedSites(['LCG.IN2P3-CC.fr','OSG.UConn.us','LCG.Cracow.pl','OSG.MIT.us','LCG.Glasgow.uk','OSG.CIT.us','OSG.BNL.us','LCG.Brunel.uk','LCG.RAL-LCG2.uk','LCG.Oxford.uk','OSG.UCSDT2.us'])
# run simulation job
simres = job.append(sim)
if not simres['OK']:
print simres['Not ok appending ddsim to job']
quit()
# run Malrin reco jobs
mares = job.append(ma)
if not mares['OK']:
print mares['Not ok appending Marlin to job']
quit()
job.setOutputData(RECoutput,"ILDPerformance/WWZZSeparation/{}_ILDConfig_{}_{}".format(ilcsoft_version,ildconfig_version,detector_model),"DESY-SRM")
print RECoutput
submit_output = job.submit(dirac)
print submit_output