def getStdhepSplit():
""" Get some stdhep split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(5)
stdhepsplit.setOutputFile("teststdhepsplit.stdhep")
stdhepsplit.setMaxRead(10)
return stdhepsplit
def getStdhepSplit():
""" Get some stdhep split
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(5)
stdhepsplit.setOutputFile("teststdhepsplit.stdhep")
stdhepsplit.setMaxRead(10)
return stdhepsplit
def setUp( self ):
"""set up the objects"""
# Mock out modules that spawn other threads
mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : Mock() }
self.module_patcher = patch.dict( sys.modules, mocked_modules )
self.module_patcher.start()
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit
self.shs = StdHepSplit( {} )
def createSplitApplication( eventsPerJob, eventsPerBaseFile, splitType='stdhep' ):
""" create Split application """
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit, SLCIOSplit
if splitType.lower() == 'stdhep':
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V3")
stdhepsplit.setNumberOfEventsPerFile( eventsPerJob )
stdhepsplit.datatype = 'gen'
stdhepsplit.setMaxRead( eventsPerBaseFile )
return stdhepsplit
if splitType.lower() == 'lcio':
split = SLCIOSplit()
split.setNumberOfEventsPerFile( eventsPerJob )
return stdhepsplit
raise NotImplementedError( 'unknown splitType: %s ' % splitType )
def createSplitApplication( eventsPerJob, eventsPerBaseFile, splitType='stdhep' ):
""" create Split application """
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit, SLCIOSplit
if splitType.lower() == 'stdhep':
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V3")
stdhepsplit.setNumberOfEventsPerFile( eventsPerJob )
stdhepsplit.datatype = 'gen'
stdhepsplit.setMaxRead( eventsPerBaseFile )
return stdhepsplit
if splitType.lower() == 'lcio':
split = SLCIOSplit()
split.setNumberOfEventsPerFile( eventsPerJob )
return stdhepsplit
raise NotImplementedError( 'unknown splitType: %s ' % splitType )
sid_sim = True
#DoSplit
activesplit = False
nbevtsperfile = 200
#Do Reco
sid_rec = True
#Do Reco with Overlay
sid_rec_ov = False
n_events = 100 #How many events per job. (Is also obtained from the FC in principle)
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation SID
slic = SLIC()
slic.setVersion('v2r9p8')#This will change only once
slic.setSteeringFile('defaultClicCrossingAngle.mac')#This will change only once
slic.setDetectorModel(detector_model)
slic.setNumberOfEvents(n_events)
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
class StdHepSplitTestCase( unittest.TestCase ):
""" Base class for the StdHepSplit test cases
"""
def setUp( self ):
"""set up the objects"""
# Mock out modules that spawn other threads
mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : Mock() }
self.module_patcher = patch.dict( sys.modules, mocked_modules )
self.module_patcher.start()
from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit
self.shs = StdHepSplit( {} )
def tearDown( self ):
self.module_patcher.stop()
def test_userjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.shs._userjobmodules( module_mock ), self )
def test_prodjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.shs._prodjobmodules( module_mock ), self )
def test_userjobmodules_fails( self ):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith( self.shs._userjobmodules( None ),
'userjobmodules failed', self )
def test_prodjobmodules_fails( self ):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith( self.shs._prodjobmodules( None ),
'prodjobmodules failed', self )
def test_checkproductionmeta( self ):
self.shs.numberOfEventsPerFile = 12348
meta_dict = { 'NumberOfEvents' : True }
assertDiracSucceeds( self.shs.checkProductionMetaData( meta_dict ), self )
assertEqualsImproved( { 'NumberOfEvents' : 12348 }, meta_dict, self )
def test_checkproductionmeta_changenothing( self ):
meta_dict = { 'myentry' : True, 'other_entry' : 81943, 'other' : 'ae8fj', False : 1 }
assertDiracSucceeds( self.shs.checkProductionMetaData( meta_dict ), self )
assertEqualsImproved( { 'myentry' : True, 'other_entry' : 81943, 'other' : 'ae8fj', False : 1 },
meta_dict, self )
def test_resolvelinkedstepparams( self ):
instance_mock = Mock()
step_mock = Mock()
step_mock.getType.return_value = 'abc'
self.shs._inputappstep = None
self.shs._jobsteps = [ '', '', step_mock ]
self.shs._linkedidx = 2
assertDiracSucceeds( self.shs._resolveLinkedStepParameters( instance_mock ), self )
instance_mock.setLink.assert_called_once_with( 'InputFile', 'abc', 'OutputFile' )
def test_resolvelinkedstepparams_nothing_happens( self ):
instance_mock = Mock()
self.shs._inputappstep = None
self.shs._jobsteps = None
self.shs._linkedidx = [ 'abc' ]
assertDiracSucceeds( self.shs._resolveLinkedStepParameters( instance_mock ), self )
self.assertFalse( instance_mock.setLink.called )
def test_checkconsistency( self ):
self.shs._jobtype = 'notUser'
self.shs.OutputFile = None
assertDiracSucceeds( self.shs._checkConsistency(), self )
assertInImproved( { 'outputFile' : '@{OutputFile}', 'outputPath' : '@{OutputPath}',
'outputDataSE' : '@{OutputSE}' }, self.shs._listofoutput, self )
def test_checkconsistency_userjob( self ):
job_mock = Mock()
job_mock.datatype = 'testDatatype'
self.shs._job = job_mock
self.shs._jobtype = 'User'
self.shs.OutputFile = None
assertDiracSucceeds( self.shs._checkConsistency(), self )
self.assertFalse( self.shs.outputFile )
assertEqualsImproved( self.shs.datatype, 'testDatatype', self )
while res not in [ 'y', 'Y' ]:
res = raw_input('\nContinue? (y/n)\t')
if res in [ 'n', 'N' ]:
sys.exit(0)
# Do Split
activesplitstdhep = True
# Do Sim
sid_sim = True
# Do Replication
replicateFiles = True
## Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion( stdhepSplitVersion )
stdhepsplit.setNumberOfEventsPerFile( nEvtsPerJob )
## Simulation
slic = SLIC()
slic.setVersion( slicVersion )
slic.setSteeringFile( slicMacro )
slic.setDetectorModel( detectorModel )
slic.setNumberOfEvents( nEvtsPerJob )
############################################
#What is below WILL NEVER NEED TO BE TOUCHED
#(I'm not kidding, if you touch and break, not my problem)
#
# Define production step splitting the stdhep files
print "matchToInput_stdhepsplit (input) = " + matchToInput_stdhepsplit + ' (Valid when stdhep production is included)'
print "stdhepsplit_basepath (output) = " + stdhepsplit_basepath + ' (Valid when stdhep production is included)'
print "matchToInput_mokka (input) = " + matchToInput_mokka
print "matchToInput_marlin (input) = " + matchToInput_marlin
print "Basepath for sim&rec (output) = " + basepath
print "Diskpath for stdhepsplit&dst (output) = " + diskpath
print "Outtput SE : stdhepsplit(%s), SUM(%s), psplit(%s), REC&DST(%s)" % (SE_stdhepsplit, SE_sim, SE_psplit, SE_rec)
print "####################################################################################"
##############################################################################################
## [PART2] Below is not to be touched
##############################################################################################
###### Whatever is below is not to be touched... Or at least only when something changes
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion(MokkaVer) ###SET HERE YOUR MOKKA VERSION, the software will come from the ILDConfig
mo.setDetectorModel(detectorModel)
mo.setSteeringFile("bbudsc_3evt.steer")
### Do not include '.tgz'
mo.setDbSlice(dbslice)
##Simulation ILD
ddsim = None
if UseDD4hepGeometry:
ddsim = DDSim()
while res not in ['y', 'Y']:
res = raw_input('\nContinue? (y/n)\t')
if res in ['n', 'N']:
sys.exit(0)
# Do Split
activesplitstdhep = True
# Do Sim
sid_sim = True
# Do Replication
replicateFiles = True
## Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion(stdhepSplitVersion)
stdhepsplit.setNumberOfEventsPerFile(nEvtsPerJob)
## Simulation
slic = SLIC()
slic.setVersion(slicVersion)
slic.setSteeringFile(slicMacro)
slic.setDetectorModel(detectorModel)
slic.setNumberOfEvents(nEvtsPerJob)
############################################
#What is below WILL NEVER NEED TO BE TOUCHED
#(I'm not kidding, if you touch and break, not my problem)
#
# Define production step splitting the stdhep files