def getSLICPandora():
""" Get some SLICPAndora app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLICPandora
slicp = SLICPandora()
slicp.setVersion('CLIC_CDR')
slicp.setDetectorModel('clic_sid_cdr')
slicp.setPandoraSettings("PandoraSettingsSlic.xml")
slicp.setOutputFile('testpandora.slcio')
return slicp
def getSLICPandora():
""" Get some SLICPAndora app
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLICPandora
slicp = SLICPandora()
slicp.setVersion('CLIC_CDR')
slicp.setDetectorModel('clic_sid_cdr')
slicp.setPandoraSettings("PandoraSettingsSlic.xml")
slicp.setOutputFile('testpandora.slcio')
return slicp
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
## SID Reco w/o overlay
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion('CLIC_CDR')#This will change only once
lcsim_prepandora.setSteeringFile("clic_cdr_prePandora.lcsim")#The steering files should NOT CHANGE
lcsim_prepandora.setTrackingStrategy(tracking_strategy)
#lcsim_prepandora.setDetectorModel(detector_model)
lcsim_prepandora.setOutputFile("prePandora.slcio")#NEVER CHANGE THIS, this file is not stored in any case
lcsim_prepandora.setNumberOfEvents(n_events)
lcsim_prepandora.willRunSLICPandora()
slicpandora = SLICPandora()
slicpandora.setVersion('CLIC_CDR')#This will change only once
slicpandora.setDetectorModel(detector_model)
slicpandora.setPandoraSettings("PandoraSettingsSlic.xml")
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile('pandora.slcio')#NEVER CHANGE THIS, this file is not stored in any case
lcfivertex = Marlin()
lcfivertex.setVersion()
lcfivertex.getInputFromApp(slicpandora)
lcfivertex.setSteeringFile()
lcfivertex.setOutputFile("lcfivertex.slcio")#NEVER CHANGE THIS, this file is not stored in any case
#Final step. Outpufile is resolved automatically
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion('CLIC_CDR')#This will change only once
## Simulation
slic = SLIC()
slic.setVersion(slicVersion)
slic.setSteeringFile(slicMacro)
slic.setDetectorModel(detectorName)
## Reco without overlay: Do not take the input from SLIC as those are ran in different prods
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion(lcsimVers)
lcsim_prepandora.setSteeringFile(lcsimSteering1)
lcsim_prepandora.setTrackingStrategy(strategies)
lcsim_prepandora.setDetectorModel(detectorName)
lcsim_prepandora.setOutputFile("prePandora.slcio")
slicpandora = SLICPandora()
slicpandora.setVersion(pandoraVers)
slicpandora.setDetectorModel(detectorName)
slicpandora.setPandoraSettings(pandoraSettings)
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile('pandora.slcio')
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion(lcsimVers)
lcsim_postpandora.getInputFromApp(slicpandora)
lcsim_postpandora.setSteeringFile(lcsimSteering2)
lcsim_postpandora.setTrackingStrategy(strategies)
lcsim_postpandora.setDetectorModel(detectorName)
### Now with overlay
overlay = OverlayInput()
def setUp(self):
"""set up the objects"""
self.slic = SLICPandora( {} )
self.slic.version = 941
class SLICPandoraTestCase( unittest.TestCase ):
""" Base class for the SLICPandora test cases
"""
def setUp(self):
"""set up the objects"""
self.slic = SLICPandora( {} )
self.slic.version = 941
def test_setters( self ):
with patch('os.path.exists', new=Mock(return_value=True)):
self.slic.setDetectorModel( 123 )
self.slic.setPandoraSettings( 94 )
self.slic.setStartFrom( 'soirmgf' )
assertInImproved( '_checkArgs', self.slic._errorDict, self )
( det_found, start_found, pand_found ) = ( False, False, False )
for err in self.slic._errorDict[ '_checkArgs' ]:
if 'detectorModel' in err:
det_found = True
elif 'startfrom' in err:
start_found = True
elif 'pandoraSettings' in err:
pand_found = True
assertEqualsImproved( ( True, True, True ), ( det_found, start_found, pand_found ), self )
assertEqualsImproved( len(self.slic.inputSB), 2, self )
def test_setdetectormodel( self ):
self.slic.setDetectorModel( 'lfn:/some/path/testDetectorv2' )
assertEqualsImproved( ( self.slic.detectorModel, self.slic.inputSB ),
( 'lfn:/some/path/testDetectorv2', [ 'lfn:/some/path/testDetectorv2' ] ), self )
self.assertFalse( self.slic._errorDict )
def test_setdectormodel_notfound( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.slic.setDetectorModel( 'invalid_detector' )
assertEqualsImproved( ( self.slic.detectorModel, self.slic.inputSB ),
( 'invalid_detector', [] ), self )
self.assertFalse( self.slic._errorDict )
def test_setpandorasettings( self ):
self.slic.setPandoraSettings( 'lfn:/some/path/my_pand_settings.txt' )
assertEqualsImproved( ( self.slic.pandoraSettings, self.slic.inputSB ),
( 'lfn:/some/path/my_pand_settings.txt', [ 'lfn:/some/path/my_pand_settings.txt' ] ),
self )
self.assertFalse( self.slic._errorDict )
def test_setpandorasettings_notfound( self ):
with patch('os.path.exists', new=Mock(return_value=False)):
self.slic.setPandoraSettings( 'invalid_settings.xml' )
assertEqualsImproved( ( self.slic.pandoraSettings, self.slic.inputSB ),
( 'invalid_settings.xml', [] ), self )
self.assertFalse( self.slic._errorDict )
def test_applicationmodule( self ):
result = self.slic._applicationModule()
self.assertIsNotNone( result )
def test_applicationmodulevalues( self ):
module_mock = Mock()
self.slic._applicationModuleValues( module_mock )
self.assertTrue( module_mock.setValue.called )
assertEqualsImproved( len(module_mock.mock_calls), 4, self )
def test_userjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.slic._userjobmodules( module_mock ), self )
def test_prodjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.slic._prodjobmodules( module_mock ), self )
def test_userjobmodules_fails( self ):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith( self.slic._userjobmodules( None ),
'userjobmodules failed', self )
def test_prodjobmodules_fails( self ):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith( self.slic._prodjobmodules( None ),
'prodjobmodules failed', self )
def test_resolveparameters( self ):
step_mock = Mock()
assertDiracSucceeds( self.slic._resolveLinkedStepParameters( step_mock() ), self )
assertDiracSucceeds( self.slic._checkWorkflowConsistency(), self )
def test_resolveparameters_setlink( self ):
step_mock = Mock()
self.slic._linkedidx = 1
self.slic._jobsteps = [ None, Mock() ]
assertDiracSucceeds( self.slic._resolveLinkedStepParameters( step_mock() ), self )
def test_checkconsistency( self ):
import inspect
self.slic.version = 'v2.4'
self.slic.steeringFile = 'myTestSteerFile.txt'
self.slic.pandoraSettings = 'mysettings.xml'
self.slic.startFrom = False
self.slic._jobtype = 'notUser'
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(SLICPandora), 'Exists', new=Mock(return_value=S_OK('success!'))):
assertDiracSucceeds( self.slic._checkConsistency( 'myTestJob' ), self )
def test_checkconsistency_noversion( self ):
self.slic.version = None
assertDiracFailsWith( self.slic._checkConsistency(), 'no version found', self )
def test_checkconsistency_steering_Exists_fails( self ):
import inspect
self.slic.version = 'v2.4'
self.slic.steeringFile = 'myTestSteerFile.txt'
self.slic.trackingStrategy = 'myTestStrat'
self.slic.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(SLICPandora), 'Exists', new=Mock(return_value=S_ERROR('Exists_test_err'))):
assertDiracFailsWith( self.slic._checkConsistency( 'myTestJob' ), 'exists_test_err', self )
def test_checkconsistency_nosettings( self ):
self.slic.pandoraSettings = None
self.slic.steeringFile = None
assertDiracFailsWith( self.slic._checkConsistency(), 'pandorasettings not set', self )
def test_checkconsistency_othercase( self ):
import inspect
self.slic.version = 'v2.4'
self.slic.steeringFile = 'myTestSteerFile.txt'
self.slic.pandoraSettings = 'mysettings.xml'
self.slic.startFrom = True
self.slic._jobtype = 'User'
with patch('os.path.exists', new=Mock(side_effect=[ True ])), \
patch.object(inspect.getmodule(SLICPandora), 'Exists', new=Mock(side_effect=IOError('dont call me'))):
assertDiracSucceeds( self.slic._checkConsistency( 'myTestJob' ), self )
## Simulation
slic = SLIC()
slic.setVersion(slicVersion)
slic.setSteeringFile(slicMacro)
slic.setDetectorModel(detectorName)
## Reco without overlay: Do not take the input from SLIC as those are ran in different prods
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion(lcsimVers)
lcsim_prepandora.setSteeringFile(lcsimSteering1)
lcsim_prepandora.setTrackingStrategy(strategies)
lcsim_prepandora.setDetectorModel(detectorName)
lcsim_prepandora.setOutputFile("prePandora.slcio")
slicpandora = SLICPandora()
slicpandora.setVersion(pandoraVers)
slicpandora.setDetectorModel(detectorName)
slicpandora.setPandoraSettings(pandoraSettings)
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile('pandora.slcio')
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion(lcsimVers)
lcsim_postpandora.getInputFromApp(slicpandora)
lcsim_postpandora.setSteeringFile(lcsimSteering2)
lcsim_postpandora.setTrackingStrategy(strategies)
lcsim_postpandora.setDetectorModel(detectorName)
### Now with overlay
overlay = OverlayInput()
elif energy in [3000., 1400.]:
ma.setSteeringFile("clic_ild_cdr_steering.xml")
ma.setGearFile('clic_ild_cdr.gear')
else:
print "Marlin: No reconstruction suitable for this energy"
## SID Reco w/o overlay
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion('CLIC_CDR')
lcsim_prepandora.setSteeringFile("clic_cdr_prePandora.lcsim")
lcsim_prepandora.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml")
#lcsim_prepandora.setDetectorModel('clic_sid_cdr')
lcsim_prepandora.setOutputFile("prePandora.slcio")
lcsim_prepandora.willRunSLICPandora()
slicpandora = SLICPandora()
slicpandora.setVersion('CLIC_CDR')
slicpandora.setDetectorModel('clic_sid_cdr')
slicpandora.setPandoraSettings("PandoraSettingsSlic.xml")
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile('pandora.slcio')
lcsim_postpandora = LCSIM()
lcsim_postpandora.setVersion('CLIC_CDR')
lcsim_postpandora.getInputFromApp(slicpandora)
lcsim_postpandora.setSteeringFile("clic_cdr_postPandoraOverlay.lcsim")
lcsim_postpandora.setTrackingStrategy("defaultStrategies_clic_sid_cdr.xml")
#lcsim_postpandora.setDetectorModel('clic_sid_cdr')
## SID Reco w/o overlay
overlay_sid = OverlayInput()
split.setNumberOfEventsPerFile(nbevtsperfile)
## SID Reco w/o overlay
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion('CLIC_CDR') #This will change only once
lcsim_prepandora.setSteeringFile(
"clic_cdr_prePandora.lcsim") #The steering files should NOT CHANGE
lcsim_prepandora.setTrackingStrategy(tracking_strategy)
#lcsim_prepandora.setDetectorModel(detector_model)
lcsim_prepandora.setOutputFile(
"prePandora.slcio"
) #NEVER CHANGE THIS, this file is not stored in any case
lcsim_prepandora.setNumberOfEvents(n_events)
lcsim_prepandora.willRunSLICPandora()
slicpandora = SLICPandora()
slicpandora.setVersion('CLIC_CDR') #This will change only once
slicpandora.setDetectorModel(detector_model)
slicpandora.setPandoraSettings("PandoraSettingsSlic.xml")
slicpandora.getInputFromApp(lcsim_prepandora)
slicpandora.setOutputFile(
'pandora.slcio') #NEVER CHANGE THIS, this file is not stored in any case
lcfivertex = Marlin()
lcfivertex.setVersion()
lcfivertex.getInputFromApp(slicpandora)
lcfivertex.setSteeringFile()
lcfivertex.setOutputFile(
"lcfivertex.slcio"
) #NEVER CHANGE THIS, this file is not stored in any case
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions)
slicOutput=outputFiles['slicOutput']
prePandoraOutput=outputFiles['prePandoraOutput']
pandoraOutput=outputFiles['pandoraOutput']
vertexingOutput=outputFiles['vertexingOutput']
lcsimRecOutput=outputFiles['lcsimRecOutput']
lcsimDstOutput=outputFiles['lcsimDstOutput']
flavortagOutput=outputFiles['flavortagOutput']
diracOutput=outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us',])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD",
"0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True,
setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent / nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector,
fileNumber, args.outputPath,
softVersions)
slicOutput = outputFiles['slicOutput']
prePandoraOutput = outputFiles['prePandoraOutput']
pandoraOutput = outputFiles['pandoraOutput']
vertexingOutput = outputFiles['vertexingOutput']
lcsimRecOutput = outputFiles['lcsimRecOutput']
lcsimDstOutput = outputFiles['lcsimDstOutput']
flavortagOutput = outputFiles['flavortagOutput']
diracOutput = outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(
'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml'
) # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy(
'steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites([
'LCG.IN2P3-CC.fr',
'LCG.RAL-LCG2.uk',
'LCG.DESY-HH.de',
'LCG.DESYZN.de',
'LCG.KEK.jp',
'OSG.PNNL.us',
])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
print args.chain[0]
# softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"]
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"]
check_events_arguments(args.events, args.split)
detector = args.detector
alias_properties(detector)
outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering)
inputSandbox, outputSandbox = setup_sandboxes(args.macFile)
dirac = DiracILC(True, repoName)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(outputBase)
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ---> ", fileNumber
################## SLIC ##################################################
if 1 in args.chain:
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'))
slic.setDetectorModel(detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Overlay ###############################################
if 2 in args.chain:
'''
#Add the gghad background overlay.
gghad = OverlayInput()
#gghad.setProdID(1767)
gghad.setEnergy(500.0)
gghad.setBXOverlay('args.bunches')
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setMachine('ilc_dbd')
gghad.setDetectorModel('sidloi3')
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
#Add the pair background overlay.
pairs = OverlayInput()
pairs.setProdID(2)
pairs.setEnergy(500.0)
pairs.setBXOverlay('args.bunches')
pairs.setGGToHadInt(1.)
pairs.setNbSigEvtsPerJob(nOutputEvents)
pairs.setMachine('ilc_dbd')
pairs.setDetectorModel('sidloi3')
pairs.setBkgEvtType('eepairs')
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
gghad = OverlayInput()
gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/')
gghad.setBXOverlay(int(args.bunches))
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
pairs = OverlayInput()
pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/')
pairs.setBXOverlay(int(args.bunches))
pairs.setGGToHadInt(1.)
pairs.setBkgEvtType('eepairs')
pairs.setNbSigEvtsPerJob(nOutputEvents)
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
################## lcsim (digitization and tracking) #####################
if 3 in args.chain:
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles
if 1 in args.chain:
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
lcsim.setAliasProperties('steeringFiles/alias.properties')
lcsim.setDetectorModel(detector+".zip")
#lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'))
lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!!
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
if 4 in args.chain:
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'))
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
if 5 in args.chain:
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/' + detector + '.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio'))
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
lcsimDst.setAliasProperties('steeringFiles/alias.properties')
lcsimDst.setDetectorModel(detector+".zip")
lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio'))
lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'))
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if 6 in args.chain:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/' + detector + '.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio'))
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk',
'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk'])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
outputLevel = max(args.chain)
if outputLevel == 1:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 3:
#job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM')
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 4:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 5:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 6:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM')
job.setOutputSandbox(outputSandbox)
job.setInputData(args.Input)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
def setUp(self):
"""set up the objects"""
self.slic = SLICPandora({})
self.slic.version = 941
class SLICPandoraTestCase(unittest.TestCase):
""" Base class for the SLICPandora test cases
"""
def setUp(self):
"""set up the objects"""
self.slic = SLICPandora({})
self.slic.version = 941
def test_setters(self):
with patch('os.path.exists', new=Mock(return_value=True)):
self.slic.setDetectorModel(123)
self.slic.setPandoraSettings(94)
self.slic.setStartFrom('soirmgf')
assertInImproved('_checkArgs', self.slic._errorDict, self)
(det_found, start_found, pand_found) = (False, False, False)
for err in self.slic._errorDict['_checkArgs']:
if 'detectorModel' in err:
det_found = True
elif 'startfrom' in err:
start_found = True
elif 'pandoraSettings' in err:
pand_found = True
assertEqualsImproved((True, True, True),
(det_found, start_found, pand_found), self)
assertEqualsImproved(len(self.slic.inputSB), 2, self)
def test_setdetectormodel(self):
self.slic.setDetectorModel('lfn:/some/path/testDetectorv2')
assertEqualsImproved((self.slic.detectorModel, self.slic.inputSB),
('lfn:/some/path/testDetectorv2',
['lfn:/some/path/testDetectorv2']), self)
self.assertFalse(self.slic._errorDict)
def test_setdectormodel_notfound(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.slic.setDetectorModel('invalid_detector')
assertEqualsImproved((self.slic.detectorModel, self.slic.inputSB),
('invalid_detector', []), self)
self.assertFalse(self.slic._errorDict)
def test_setpandorasettings(self):
self.slic.setPandoraSettings('lfn:/some/path/my_pand_settings.txt')
assertEqualsImproved((self.slic.pandoraSettings, self.slic.inputSB),
('lfn:/some/path/my_pand_settings.txt',
['lfn:/some/path/my_pand_settings.txt']), self)
self.assertFalse(self.slic._errorDict)
def test_setpandorasettings_notfound(self):
with patch('os.path.exists', new=Mock(return_value=False)):
self.slic.setPandoraSettings('invalid_settings.xml')
assertEqualsImproved(
(self.slic.pandoraSettings, self.slic.inputSB),
('invalid_settings.xml', []), self)
self.assertFalse(self.slic._errorDict)
def test_applicationmodule(self):
result = self.slic._applicationModule()
self.assertIsNotNone(result)
def test_applicationmodulevalues(self):
module_mock = Mock()
self.slic._applicationModuleValues(module_mock)
self.assertTrue(module_mock.setValue.called)
assertEqualsImproved(len(module_mock.mock_calls), 4, self)
def test_userjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.slic._userjobmodules(module_mock), self)
def test_prodjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.slic._prodjobmodules(module_mock), self)
def test_userjobmodules_fails(self):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith(self.slic._userjobmodules(None),
'userjobmodules failed', self)
def test_prodjobmodules_fails(self):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith(self.slic._prodjobmodules(None),
'prodjobmodules failed', self)
def test_resolveparameters(self):
step_mock = Mock()
assertDiracSucceeds(
self.slic._resolveLinkedStepParameters(step_mock()), self)
assertDiracSucceeds(self.slic._checkWorkflowConsistency(), self)
def test_resolveparameters_setlink(self):
step_mock = Mock()
self.slic._linkedidx = 1
self.slic._jobsteps = [None, Mock()]
assertDiracSucceeds(
self.slic._resolveLinkedStepParameters(step_mock()), self)
def test_checkconsistency(self):
import inspect
self.slic.version = 'v2.4'
self.slic.steeringFile = 'myTestSteerFile.txt'
self.slic.pandoraSettings = 'mysettings.xml'
self.slic.startFrom = False
self.slic._jobtype = 'notUser'
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(SLICPandora), 'Exists', new=Mock(return_value=S_OK('success!'))):
assertDiracSucceeds(self.slic._checkConsistency('myTestJob'), self)
def test_checkconsistency_noversion(self):
self.slic.version = None
assertDiracFailsWith(self.slic._checkConsistency(), 'no version found',
self)
def test_checkconsistency_steering_Exists_fails(self):
import inspect
self.slic.version = 'v2.4'
self.slic.steeringFile = 'myTestSteerFile.txt'
self.slic.trackingStrategy = 'myTestStrat'
self.slic.detectorModel = 'myDetModel.wrongFormat'
with patch('os.path.exists', new=Mock(side_effect=[ False ])), \
patch.object(inspect.getmodule(SLICPandora), 'Exists', new=Mock(return_value=S_ERROR('Exists_test_err'))):
assertDiracFailsWith(self.slic._checkConsistency('myTestJob'),
'exists_test_err', self)
def test_checkconsistency_nosettings(self):
self.slic.pandoraSettings = None
self.slic.steeringFile = None
assertDiracFailsWith(self.slic._checkConsistency(),
'pandorasettings not set', self)
def test_checkconsistency_othercase(self):
import inspect
self.slic.version = 'v2.4'
self.slic.steeringFile = 'myTestSteerFile.txt'
self.slic.pandoraSettings = 'mysettings.xml'
self.slic.startFrom = True
self.slic._jobtype = 'User'
with patch('os.path.exists', new=Mock(side_effect=[ True ])), \
patch.object(inspect.getmodule(SLICPandora), 'Exists', new=Mock(side_effect=IOError('dont call me'))):
assertDiracSucceeds(self.slic._checkConsistency('myTestJob'), self)