def getStdhepSplit(): """ Get some stdhep split """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit stdhepsplit = StdHepSplit() stdhepsplit.setVersion("V2") stdhepsplit.setNumberOfEventsPerFile(5) stdhepsplit.setOutputFile("teststdhepsplit.stdhep") stdhepsplit.setMaxRead(10) return stdhepsplit
def setUp( self ): """set up the objects""" # Mock out modules that spawn other threads mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : Mock() } self.module_patcher = patch.dict( sys.modules, mocked_modules ) self.module_patcher.start() from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit self.shs = StdHepSplit( {} )
def createSplitApplication( eventsPerJob, eventsPerBaseFile, splitType='stdhep' ): """ create Split application """ from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit, SLCIOSplit if splitType.lower() == 'stdhep': stdhepsplit = StdHepSplit() stdhepsplit.setVersion("V3") stdhepsplit.setNumberOfEventsPerFile( eventsPerJob ) stdhepsplit.datatype = 'gen' stdhepsplit.setMaxRead( eventsPerBaseFile ) return stdhepsplit if splitType.lower() == 'lcio': split = SLCIOSplit() split.setNumberOfEventsPerFile( eventsPerJob ) return stdhepsplit raise NotImplementedError( 'unknown splitType: %s ' % splitType )
sid_sim = True #DoSplit activesplit = False nbevtsperfile = 200 #Do Reco sid_rec = True #Do Reco with Overlay sid_rec_ov = False n_events = 100 #How many events per job. (Is also obtained from the FC in principle) ##Split stdhepsplit = StdHepSplit() stdhepsplit.setVersion("V2") stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep) ##Simulation SID slic = SLIC() slic.setVersion('v2r9p8')#This will change only once slic.setSteeringFile('defaultClicCrossingAngle.mac')#This will change only once slic.setDetectorModel(detector_model) slic.setNumberOfEvents(n_events) ##Split split = SLCIOSplit() split.setNumberOfEventsPerFile(nbevtsperfile)
class StdHepSplitTestCase( unittest.TestCase ): """ Base class for the StdHepSplit test cases """ def setUp( self ): """set up the objects""" # Mock out modules that spawn other threads mocked_modules = { 'DIRAC.DataManagementSystem.Client.DataManager' : Mock() } self.module_patcher = patch.dict( sys.modules, mocked_modules ) self.module_patcher.start() from ILCDIRAC.Interfaces.API.NewInterface.Applications import StdHepSplit self.shs = StdHepSplit( {} ) def tearDown( self ): self.module_patcher.stop() def test_userjobmodules( self ): module_mock = Mock() assertDiracSucceeds( self.shs._userjobmodules( module_mock ), self ) def test_prodjobmodules( self ): module_mock = Mock() assertDiracSucceeds( self.shs._prodjobmodules( module_mock ), self ) def test_userjobmodules_fails( self ): with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\ patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))): assertDiracFailsWith( self.shs._userjobmodules( None ), 'userjobmodules failed', self ) def test_prodjobmodules_fails( self ): with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \ patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))): assertDiracFailsWith( self.shs._prodjobmodules( None ), 'prodjobmodules failed', self ) def test_checkproductionmeta( self ): self.shs.numberOfEventsPerFile = 12348 meta_dict = { 'NumberOfEvents' : True } assertDiracSucceeds( self.shs.checkProductionMetaData( meta_dict ), self ) assertEqualsImproved( { 'NumberOfEvents' : 12348 }, meta_dict, self ) def test_checkproductionmeta_changenothing( self ): meta_dict = { 'myentry' : True, 'other_entry' : 81943, 'other' : 'ae8fj', False : 1 } assertDiracSucceeds( self.shs.checkProductionMetaData( meta_dict ), self ) assertEqualsImproved( { 'myentry' : True, 'other_entry' : 81943, 'other' : 'ae8fj', False : 1 }, meta_dict, self ) def test_resolvelinkedstepparams( self ): instance_mock = Mock() step_mock = Mock() step_mock.getType.return_value = 'abc' self.shs._inputappstep = None self.shs._jobsteps = [ '', '', step_mock ] self.shs._linkedidx = 2 assertDiracSucceeds( self.shs._resolveLinkedStepParameters( instance_mock ), self ) instance_mock.setLink.assert_called_once_with( 'InputFile', 'abc', 'OutputFile' ) def test_resolvelinkedstepparams_nothing_happens( self ): instance_mock = Mock() self.shs._inputappstep = None self.shs._jobsteps = None self.shs._linkedidx = [ 'abc' ] assertDiracSucceeds( self.shs._resolveLinkedStepParameters( instance_mock ), self ) self.assertFalse( instance_mock.setLink.called ) def test_checkconsistency( self ): self.shs._jobtype = 'notUser' self.shs.OutputFile = None assertDiracSucceeds( self.shs._checkConsistency(), self ) assertInImproved( { 'outputFile' : '@{OutputFile}', 'outputPath' : '@{OutputPath}', 'outputDataSE' : '@{OutputSE}' }, self.shs._listofoutput, self ) def test_checkconsistency_userjob( self ): job_mock = Mock() job_mock.datatype = 'testDatatype' self.shs._job = job_mock self.shs._jobtype = 'User' self.shs.OutputFile = None assertDiracSucceeds( self.shs._checkConsistency(), self ) self.assertFalse( self.shs.outputFile ) assertEqualsImproved( self.shs.datatype, 'testDatatype', self )
while res not in [ 'y', 'Y' ]: res = raw_input('\nContinue? (y/n)\t') if res in [ 'n', 'N' ]: sys.exit(0) # Do Split activesplitstdhep = True # Do Sim sid_sim = True # Do Replication replicateFiles = True ## Split stdhepsplit = StdHepSplit() stdhepsplit.setVersion( stdhepSplitVersion ) stdhepsplit.setNumberOfEventsPerFile( nEvtsPerJob ) ## Simulation slic = SLIC() slic.setVersion( slicVersion ) slic.setSteeringFile( slicMacro ) slic.setDetectorModel( detectorModel ) slic.setNumberOfEvents( nEvtsPerJob ) ############################################ #What is below WILL NEVER NEED TO BE TOUCHED #(I'm not kidding, if you touch and break, not my problem) # # Define production step splitting the stdhep files
print "matchToInput_stdhepsplit (input) = " + matchToInput_stdhepsplit + ' (Valid when stdhep production is included)' print "stdhepsplit_basepath (output) = " + stdhepsplit_basepath + ' (Valid when stdhep production is included)' print "matchToInput_mokka (input) = " + matchToInput_mokka print "matchToInput_marlin (input) = " + matchToInput_marlin print "Basepath for sim&rec (output) = " + basepath print "Diskpath for stdhepsplit&dst (output) = " + diskpath print "Outtput SE : stdhepsplit(%s), SUM(%s), psplit(%s), REC&DST(%s)" % (SE_stdhepsplit, SE_sim, SE_psplit, SE_rec) print "####################################################################################" ############################################################################################## ## [PART2] Below is not to be touched ############################################################################################## ###### Whatever is below is not to be touched... Or at least only when something changes ##Split stdhepsplit = StdHepSplit() stdhepsplit.setVersion("V2") stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep) ##Simulation ILD mo = Mokka() mo.setVersion(MokkaVer) ###SET HERE YOUR MOKKA VERSION, the software will come from the ILDConfig mo.setDetectorModel(detectorModel) mo.setSteeringFile("bbudsc_3evt.steer") ### Do not include '.tgz' mo.setDbSlice(dbslice) ##Simulation ILD ddsim = None if UseDD4hepGeometry: ddsim = DDSim()
while res not in ['y', 'Y']: res = raw_input('\nContinue? (y/n)\t') if res in ['n', 'N']: sys.exit(0) # Do Split activesplitstdhep = True # Do Sim sid_sim = True # Do Replication replicateFiles = True ## Split stdhepsplit = StdHepSplit() stdhepsplit.setVersion(stdhepSplitVersion) stdhepsplit.setNumberOfEventsPerFile(nEvtsPerJob) ## Simulation slic = SLIC() slic.setVersion(slicVersion) slic.setSteeringFile(slicMacro) slic.setDetectorModel(detectorModel) slic.setNumberOfEvents(nEvtsPerJob) ############################################ #What is below WILL NEVER NEED TO BE TOUCHED #(I'm not kidding, if you touch and break, not my problem) # # Define production step splitting the stdhep files