def getMokka(self):
""" Define a mokka app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka
mokka = Mokka()
mokka.setVersion(self.mokkaVersion)
mokka.setSteeringFile(self.mokkaSteeringFile)
mokka.setOutputFile("testsim.slcio")
mokka.setDetectorModel(self.detectorModel)
if self.steeringFileVersion:
mokka.setSteeringFileVersion(self.steeringFileVersion)
return mokka
#Do Reco with Overlay
ild_rec_ov = True
nbtasks_rec_ov = 5 # See comment on nbtasks_sim
#Do Reco
ild_rec = False # please, use WITH OVERLAY
nbtasks_rec = -1 # See comment on nbtasks_sim
###### Whatever is below is not to be touched... Or at least only when something changes
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion(MokkaVer) ###SET HERE YOUR MOKKA VERSION, the software will come from the ILDConfig
mo.setDetectorModel(detectorModel)
mo.setSteeringFile("bbudsc_3evt.steer")
### Do not include '.tgz'
mo.setDbSlice(dbslice)
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
##Define the overlay
overlay = OverlayInput()
overlay.setMachine("ilc_dbd") #Don't touch, this is how the system knows what files to get
overlay.setEnergy(energy) #Don't touch, this is how the system knows what files to get
overlay.setDetectorModel(detectorModel) #Don't touch, this is how the system knows what files to get
print "Basepath for sim&rec (output) = " + basepath
print "Diskpath for stdhepsplit&dst (output) = " + diskpath
print "Outtput SE : stdhepsplit(%s), SUM(%s), psplit(%s), REC&DST(%s)" % (SE_stdhepsplit, SE_sim, SE_psplit, SE_rec)
print "####################################################################################"
##############################################################################################
## [PART2] Below is not to be touched
##############################################################################################
###### Whatever is below is not to be touched... Or at least only when something changes
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion(MokkaVer) ###SET HERE YOUR MOKKA VERSION, the software will come from the ILDConfig
mo.setDetectorModel(detectorModel)
mo.setSteeringFile("bbudsc_3evt.steer")
### Do not include '.tgz'
mo.setDbSlice(dbslice)
##Simulation ILD
ddsim = None
if UseDD4hepGeometry:
ddsim = DDSim()
ddsim.setVersion(DDSimVer) ###SET HERE YOUR MOKKA VERSION, the software will come from the ILDConfig
ddsim.setDetectorModel(detectorModel)
ddsim.setSteeringFile("ddsim_steer.py")
class MokkaTestCase( unittest.TestCase ):
""" Base class for the Mokka test cases
"""
def setUp(self):
"""set up the objects"""
self.mok = Mokka( {} )
def test_setters( self ):
self.assertFalse( self.mok._errorDict )
self.assertFalse( self.mok.inputSB )
self.mok.setRandomSeed( 'invalid_seed' )
self.mok.setmcRunNumber( [ 'something', False, [] ] )
self.mok.setDetectorModel( { 'bla' : True } )
self.mok.setMacFile( 'lfn:/inval/dir/somefile.mac' )
self.mok.setStartFrom( { 'myset', False } )
self.mok.setProcessID( 129843 )
self.mok.setDbSlice( 'lfn:/inval/dir/myDB.slice' )
print(self.mok._errorDict)
assertEqualsImproved( len( self.mok._errorDict['_checkArgs'] ), 5, self )
assertEqualsImproved( self.mok.inputSB, [
'lfn:/inval/dir/somefile.mac', 'lfn:/inval/dir/myDB.slice' ], self )
assertEqualsImproved( ( self.mok.macFile, self.mok.dbSlice ), (
'lfn:/inval/dir/somefile.mac', 'lfn:/inval/dir/myDB.slice' ), self )
def test_setfiles_othercase( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.assertFalse( self.mok._errorDict )
self.mok.setMacFile( '/invalid/dir/myMac.file' )
self.mok.setDbSlice( '/invalid/dir/someDb.sql' )
assertEqualsImproved( ( self.mok.macFile, self.mok.dbSlice ),
( '/invalid/dir/myMac.file', '/invalid/dir/someDb.sql' ), self )
self.assertFalse( self.mok._errorDict )
def test_userjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.mok._userjobmodules( module_mock ), self )
def test_prodjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.mok._prodjobmodules( module_mock ), self )
def test_userjobmodules_fails( self ):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith( self.mok._userjobmodules( None ),
'userjobmodules failed', self )
def test_prodjobmodules_fails( self ):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith( self.mok._prodjobmodules( None ),
'prodjobmodules failed', self )
def test_checkconsistency( self ):
self.mok.version = 8431
self.mok.steeringFile = 'lfn:/nonvalid/dir/mysteer.stdhep'
self.mok._jobtype = 'User'
assertDiracSucceeds( self.mok._checkConsistency(), self )
def test_checkconsistency_nouserjob( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.mok.version = 8431
self.mok.steeringFile = '/nonvalid/dir/mysteer.stdhep'
self.mok._jobtype = 'notUser'
self.mok.detectorModel = 'myTestDetv100'
assertDiracSucceeds( self.mok._checkConsistency(), self )
assertEqualsImproved( self.mok._listofoutput, [
{ "outputFile":"@{OutputFile}", "outputPath":"@{OutputPath}", "outputDataSE":'@{OutputSE}' } ], self )
assertEqualsImproved( self.mok.prodparameters, {
'mokka_steeringfile' : '/nonvalid/dir/mysteer.stdhep', 'mokka_detectormodel' : 'myTestDetv100',
'detectorType' : 'ILD' }, self )
def test_checkconsistency_nouserjob_2( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.mok.version = 8431
self.mok.steeringFile = '/nonvalid/dir/mysteer.stdhep'
self.mok._jobtype = 'notUser'
self.mok.detectorModel = 0
assertDiracSucceeds( self.mok._checkConsistency(), self )
assertEqualsImproved( self.mok._listofoutput, [ {
"outputFile":"@{OutputFile}", "outputPath":"@{OutputPath}", "outputDataSE":'@{OutputSE}'
} ], self )
assertEqualsImproved( self.mok.prodparameters, {
'mokka_steeringfile' : '/nonvalid/dir/mysteer.stdhep', 'detectorType' : 'ILD' }, self )
def test_checkconsistency_noversion( self ):
self.mok.version = None
assertDiracFailsWith( self.mok._checkConsistency(), 'no version found', self )
def test_checkconsistency_nosteeringfile( self ):
self.mok.version = True
self.mok.steeringFile = None
assertDiracFailsWith( self.mok._checkConsistency(), 'no steering file', self )
def test_resolvelinkedparams( self ):
step_mock = Mock()
input_mock = Mock()
input_mock.getType.return_value = { 'abc' : False }
self.mok._linkedidx = 3
self.mok._jobsteps = [ None, None, None, input_mock ]
assertDiracSucceeds( self.mok._resolveLinkedStepParameters( step_mock ), self )
step_mock.setLink.assert_called_once_with( 'InputFile', { 'abc' : False }, 'OutputFile' )
def test_resolvelinkedparams_noinputstep( self ):
self.mok._linkedidx = None
self.mok._inputappstep = []
assertDiracSucceeds( self.mok._resolveLinkedStepParameters( None ), self )
def setUp(self):
"""set up the objects"""
self.mok = Mokka( {} )
wh = Whizard(processlist=dirac.getProcessList())
wh.setEnergy(3000)
wh.setEvtType("ee_h_mumu")
wh.setNbEvts(1)
wh.setEnergy(3000)
params = {}
params["USERB1"] = "F"
wh.setParameterDict(params)
wh.setModel("sm")
res = j.append(wh)
if not res["OK"]:
print res["Message"]
dexit(1)
mo = Mokka()
mo.getInputFromApp(wh)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setNbEvts(1)
mo.setOutputFile("somefile.slcio")
res = j.append(mo)
if not res["OK"]:
print res["Message"]
dexit(1)
ov = OverlayInput()
ov.setDetectorType("ILD")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
stdhepc = StdhepCut()
stdhepc.setVersion("V7")
if cut and not cutfile:
print "No cut file defined, cannot proceed"
exit(1)
stdhepc.setSteeringFile(cutfile)
stdhepc.setMaxNbEvts(n_keep)
stdhepc.setSelectionEfficiency(seleff)
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion('0706P08')
#mo.setNbEvts(1000)
if energy in [500., 375., 350.]:
mo.setSteeringFile("clic_ild_cdr500.steer")
elif energy in [3000., 1400.]:
mo.setSteeringFile("clic_ild_cdr.steer")
else:
print 'Detector Model for Mokka undefined for this energy'
if detectormodel=='ild_00':
mo.setSteeringFile("ild_00.steer")
##Simulation SID
slic = SLIC()
slic.setVersion('v2r9p8')
wh = Whizard(processlist=dirac.getProcessList())
wh.setEnergy(3000)
wh.setEvtType("ee_h_mumu")
wh.setNbEvts(1)
wh.setEnergy(3000)
params = {}
params['USERB1']='F'
wh.setParameterDict(params)
wh.setModel("sm")
res = j.append(wh)
if not res['OK']:
print res['Message']
dexit(1)
mo = Mokka()
mo.getInputFromApp(wh)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setNbEvts(1)
mo.setOutputFile("somefile.slcio")
res = j.append(mo)
if not res['OK']:
print res['Message']
dexit(1)
ov = OverlayInput()
ov.setDetectorModel("CLIC_ILD_CDR")
ov.setBXOverlay(60)
ov.setGGToHadInt(3.2)
from ILCDIRAC.Interfaces.API.NewInterface.UserJob import UserJob
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka, Marlin, OverlayInput
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from DIRAC import exit as dexit
d = DiracILC(True, "repo.rep")
n_evts = 500
n_evts_per_job = 100
n_jobs = n_evts / n_evts_per_job
for i in range(n_jobs):
j = UserJob()
mo = Mokka()
mo.setEnergy(3000)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setMacFile("particlegun_electron.mac")
mo.setOutputFile("MyFile.slcio")
mo.setNbEvts(n_evts_per_job)
res = j.append(mo)
if not res['OK']:
print res['Message']
break
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.getInputFromApp(mo)
ma.setOutputDstFile("mydst_no_ov_%s.slcio" % i)
stdhepc = StdhepCut()
stdhepc.setVersion("V5")
if cut and not cutfile:
print "No cut file defined, cannot proceed"
exit(1)
stdhepc.setSteeringFile(cutfile)
stdhepc.setMaxNbEvts(n_keep)
stdhepc.setSelectionEfficiency(seleff)
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion("0706P08")
# mo.setNbEvts(1000)
if energy in [500.0, 350.0]:
mo.setSteeringFile("clic_ild_cdr500.steer")
else:
mo.setSteeringFile("clic_ild_cdr.steer")
##Simulation SID
slic = SLIC()
slic.setVersion("v2r9p8")
slic.setSteeringFile("defaultClicCrossingAngle.mac")
slic.setDetectorModel("clic_sid_cdr")
ild_rec = False
#Do Reco with Overlay
ild_rec_ov = False
###### Whatever is below is not to be touched... Or at least only when something changes
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion('0706P08') ###SET HERE YOUR MOKKA VERSION
mo.setDetectorModel(detectorModel)
if energy in [500.]: ##YOU COULD HAVE THE SAME STEERING FILE FOR DIFFERENT ENERGIES
mo.setSteeringFile("clic_ild_cdr500.steer") ## define the prod steering file
else:
mo.setSteeringFile("clic_ild_cdr.steer")## define the prod steering file
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
##Define the overlay
overlay = OverlayInput()
overlay.setMachine("ilc_dbd") #Don't touch, this is how the system knows what files to get
def getMokka(self):
""" Define a mokka app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka
mokka = Mokka()
mokka.setVersion(self.mokkaVersion)
mokka.setSteeringFile(self.mokkaSteeringFile)
mokka.setOutputFile("testsim.slcio")
mokka.setDetectorModel(self.detectorModel)
if self.steeringFileVersion:
mokka.setSteeringFileVersion(self.steeringFileVersion)
return mokka
stdhepc = StdhepCut()
stdhepc.setVersion("V7")
if cut and not cutfile:
print "No cut file defined, cannot proceed"
raise RuntimeError("1")
stdhepc.setSteeringFile(cutfile)
stdhepc.setMaxNbEvts(n_keep)
stdhepc.setSelectionEfficiency(seleff)
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion(SOFTWAREVERSION)
#mo.setNbEvts(10)
if energy in [500., 420., 375., 350., 250.]:
mo.setSteeringFile("clic_ild_cdr500.steer")
elif energy in [3000., 1400.]:
mo.setSteeringFile("clic_ild_cdr.steer")
else:
print 'Detector Model for Mokka undefined for this energy'
if detectormodel == 'ild_00':
mo.setSteeringFile("ild_00.steer")
if detectormodel == 'ILD_o1_v05':
mo.setSteeringFile("bbudsc_3evt.steer")
##Simulation DDSim
dd = DDSim()
from ILCDIRAC.Interfaces.API.NewInterface.Applications import Mokka, Marlin, OverlayInput
from ILCDIRAC.Interfaces.API.DiracILC import DiracILC
from DIRAC import exit as dexit
d = DiracILC(True,"repo.rep")
n_evts = 500
n_evts_per_job= 100
n_jobs = n_evts/n_evts_per_job
for i in range(n_jobs):
j = UserJob()
mo = Mokka()
mo.setEnergy(3000)
mo.setVersion("0706P08")
mo.setSteeringFile("clic_ild_cdr.steer")
mo.setMacFile("particlegun_electron.mac")
mo.setOutputFile("MyFile.slcio")
mo.setNbEvts(n_evts_per_job)
res = j.append(mo)
if not res['OK']:
print res['Message']
break
ma = Marlin()
ma.setVersion("v0111Prod")
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.getInputFromApp(mo)
ma.setOutputDstFile("mydst_no_ov_%s.slcio"%i)
def setUp(self):
"""set up the objects"""
self.mok = Mokka({})
class MokkaTestCase(unittest.TestCase):
""" Base class for the Mokka test cases
"""
def setUp(self):
"""set up the objects"""
self.mok = Mokka({})
def test_setters(self):
self.assertFalse(self.mok._errorDict)
self.assertFalse(self.mok.inputSB)
self.mok.setRandomSeed('invalid_seed')
self.mok.setmcRunNumber(['something', False, []])
self.mok.setDetectorModel({'bla': True})
self.mok.setMacFile('lfn:/inval/dir/somefile.mac')
self.mok.setStartFrom({'myset', False})
self.mok.setProcessID(129843)
self.mok.setDbSlice('lfn:/inval/dir/myDB.slice')
print self.mok._errorDict
assertEqualsImproved(len(self.mok._errorDict['_checkArgs']), 5, self)
assertEqualsImproved(
self.mok.inputSB,
['lfn:/inval/dir/somefile.mac', 'lfn:/inval/dir/myDB.slice'], self)
assertEqualsImproved(
(self.mok.macFile, self.mok.dbSlice),
('lfn:/inval/dir/somefile.mac', 'lfn:/inval/dir/myDB.slice'), self)
def test_setfiles_othercase(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.assertFalse(self.mok._errorDict)
self.mok.setMacFile('/invalid/dir/myMac.file')
self.mok.setDbSlice('/invalid/dir/someDb.sql')
assertEqualsImproved(
(self.mok.macFile, self.mok.dbSlice),
('/invalid/dir/myMac.file', '/invalid/dir/someDb.sql'), self)
self.assertFalse(self.mok._errorDict)
def test_userjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.mok._userjobmodules(module_mock), self)
def test_prodjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.mok._prodjobmodules(module_mock), self)
def test_userjobmodules_fails(self):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith(self.mok._userjobmodules(None),
'userjobmodules failed', self)
def test_prodjobmodules_fails(self):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith(self.mok._prodjobmodules(None),
'prodjobmodules failed', self)
def test_checkconsistency(self):
self.mok.version = 8431
self.mok.steeringFile = 'lfn:/nonvalid/dir/mysteer.stdhep'
self.mok._jobtype = 'User'
assertDiracSucceeds(self.mok._checkConsistency(), self)
def test_checkconsistency_nouserjob(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.mok.version = 8431
self.mok.steeringFile = '/nonvalid/dir/mysteer.stdhep'
self.mok._jobtype = 'notUser'
self.mok.detectorModel = 'myTestDetv100'
assertDiracSucceeds(self.mok._checkConsistency(), self)
assertEqualsImproved(self.mok._listofoutput,
[{
"outputFile": "@{OutputFile}",
"outputPath": "@{OutputPath}",
"outputDataSE": '@{OutputSE}'
}], self)
assertEqualsImproved(
self.mok.prodparameters, {
'mokka_steeringfile': '/nonvalid/dir/mysteer.stdhep',
'mokka_detectormodel': 'myTestDetv100',
'detectorType': 'ILD'
}, self)
def test_checkconsistency_nouserjob_2(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.mok.version = 8431
self.mok.steeringFile = '/nonvalid/dir/mysteer.stdhep'
self.mok._jobtype = 'notUser'
self.mok.detectorModel = 0
assertDiracSucceeds(self.mok._checkConsistency(), self)
assertEqualsImproved(self.mok._listofoutput,
[{
"outputFile": "@{OutputFile}",
"outputPath": "@{OutputPath}",
"outputDataSE": '@{OutputSE}'
}], self)
assertEqualsImproved(
self.mok.prodparameters, {
'mokka_steeringfile': '/nonvalid/dir/mysteer.stdhep',
'detectorType': 'ILD'
}, self)
def test_checkconsistency_noversion(self):
self.mok.version = None
assertDiracFailsWith(self.mok._checkConsistency(), 'no version found',
self)
def test_checkconsistency_nosteeringfile(self):
self.mok.version = True
self.mok.steeringFile = None
assertDiracFailsWith(self.mok._checkConsistency(), 'no steering file',
self)
def test_resolvelinkedparams(self):
step_mock = Mock()
input_mock = Mock()
input_mock.getType.return_value = {'abc': False}
self.mok._linkedidx = 3
self.mok._jobsteps = [None, None, None, input_mock]
assertDiracSucceeds(self.mok._resolveLinkedStepParameters(step_mock),
self)
step_mock.setLink.assert_called_once_with('InputFile', {'abc': False},
'OutputFile')
def test_resolvelinkedparams_noinputstep(self):
self.mok._linkedidx = None
self.mok._inputappstep = []
assertDiracSucceeds(self.mok._resolveLinkedStepParameters(None), self)