def getSLIC():
""" Get a SLIC instance
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
slic.setOutputFile('testsim.slcio')
return slic
def getSLIC():
""" Get a SLIC instance
"""
from ILCDIRAC.Interfaces.API.NewInterface.Applications import SLIC
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
slic.setOutputFile('testsim.slcio')
return slic
def test_setdetectormodel( self ):
self.slic.setDetectorModel( 'lfn:/my/det.model' )
assertInImproved( 'lfn:/my/det.model', self.slic.inputSB, self )
assertEqualsImproved( self.slic.detectorModel, 'det.model', self )
self.slic = SLIC( {} )
with patch('os.path.exists', new=Mock(return_value=True)):
self.slic.setDetectorModel( '/my/local/dir/detectorv212.stdhep.zip' )
assertEqualsImproved( ( self.slic.detectorModel, self.slic.inputSB ),
( 'detectorv212.stdhep',
[ '/my/local/dir/detectorv212.stdhep.zip' ] ), self )
self.slic = SLIC( {} )
with patch('os.path.exists', new=Mock(return_value=False)):
self.slic.setDetectorModel( '/my/local/dir/detectorv212.stdhep.zip' )
assertEqualsImproved( ( self.slic.detectorModel, self.slic.inputSB ),
( 'detectorv212.stdhep', [] ), self )
def test_setdetectormodel(self):
self.slic.setDetectorModel('lfn:/my/det.model')
assertInImproved('lfn:/my/det.model', self.slic.inputSB, self)
assertEqualsImproved(self.slic.detectorModel, 'det.model', self)
self.slic = SLIC({})
with patch('os.path.exists', new=Mock(return_value=True)):
self.slic.setDetectorModel('/my/local/dir/detectorv212.stdhep.zip')
assertEqualsImproved((self.slic.detectorModel, self.slic.inputSB),
('detectorv212.stdhep',
['/my/local/dir/detectorv212.stdhep.zip']),
self)
self.slic = SLIC({})
with patch('os.path.exists', new=Mock(return_value=False)):
self.slic.setDetectorModel('/my/local/dir/detectorv212.stdhep.zip')
assertEqualsImproved((self.slic.detectorModel, self.slic.inputSB),
('detectorv212.stdhep', []), self)
# Do Split activesplitstdhep = True # Do Sim sid_sim = True # Do Replication replicateFiles = True ## Split stdhepsplit = StdHepSplit() stdhepsplit.setVersion( stdhepSplitVersion ) stdhepsplit.setNumberOfEventsPerFile( nEvtsPerJob ) ## Simulation slic = SLIC() slic.setVersion( slicVersion ) slic.setSteeringFile( slicMacro ) slic.setDetectorModel( detectorModel ) slic.setNumberOfEvents( nEvtsPerJob ) ############################################ #What is below WILL NEVER NEED TO BE TOUCHED #(I'm not kidding, if you touch and break, not my problem) # # Define production step splitting the stdhep files if activesplitstdhep and meta: pstdhepsplit = SIDProductionJob() pstdhepsplit.setLogLevel( 'verbose' ) pstdhepsplit.setProdType( 'Split' ) res = pstdhepsplit.setInputDataQuery(meta)
def setUp(self):
"""set up the objects"""
self.slic = SLIC( {} )
self.slic.version = 941
class SLICTestCase( unittest.TestCase ):
""" Base class for the SLIC test cases
"""
def setUp(self):
"""set up the objects"""
self.slic = SLIC( {} )
self.slic.version = 941
def test_setters( self ):
self.slic.setRandomSeed( 'someInvalidSeed' )
self.slic.setDetectorModel( 'myDetModelTestme' )
self.slic.setStartFrom( 'soirmgf' )
assertInImproved( '_checkArgs', self.slic._errorDict, self )
( rand_found, start_found ) = ( False, False )
for err in self.slic._errorDict[ '_checkArgs' ]:
if 'randomSeed' in err:
rand_found = True
elif 'startfrom' in err:
start_found = True
assertEqualsImproved( ( True, True, 'myDetModelTestme' ), ( rand_found, start_found,
self.slic.detectorModel ), self )
def test_setdetectormodel( self ):
self.slic.setDetectorModel( 'lfn:/my/det.model' )
assertInImproved( 'lfn:/my/det.model', self.slic.inputSB, self )
assertEqualsImproved( self.slic.detectorModel, 'det.model', self )
self.slic = SLIC( {} )
with patch('os.path.exists', new=Mock(return_value=True)):
self.slic.setDetectorModel( '/my/local/dir/detectorv212.stdhep.zip' )
assertEqualsImproved( ( self.slic.detectorModel, self.slic.inputSB ),
( 'detectorv212.stdhep',
[ '/my/local/dir/detectorv212.stdhep.zip' ] ), self )
self.slic = SLIC( {} )
with patch('os.path.exists', new=Mock(return_value=False)):
self.slic.setDetectorModel( '/my/local/dir/detectorv212.stdhep.zip' )
assertEqualsImproved( ( self.slic.detectorModel, self.slic.inputSB ),
( 'detectorv212.stdhep', [] ), self )
def test_userjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.slic._userjobmodules( module_mock ), self )
def test_prodjobmodules( self ):
module_mock = Mock()
assertDiracSucceeds( self.slic._prodjobmodules( module_mock ), self )
def test_userjobmodules_fails( self ):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith( self.slic._userjobmodules( None ), 'userjobmodules failed', self )
def test_prodjobmodules_fails( self ):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith( self.slic._prodjobmodules( None ), 'prodjobmodules failed', self )
def test_checkconsistency( self ):
self.slic._jobtype = 'User'
self.slic.startFrom = True
with patch('os.path.exists', new=Mock(return_value=True)):
assertDiracSucceeds( self.slic._checkConsistency(), self )
def test_checkconsistency_othercase( self ):
self.slic.detectorModel = 'someModel'
self.slic.steeringFile = 'LFN:/my/consistent/steeringfile.st'
self.slic._jobtype = 'notUser'
self.slic.startFrom = False
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds( self.slic._checkConsistency(), self )
def test_checkconsistency_download_succeeds( self ):
import inspect
self.slic.steeringFile = '/my/remote/path.st'
self.slic._jobtype = 'notUser'
self.slic.startFrom = False
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(SLIC), 'Exists', new=Mock(return_value=S_OK('ok'))):
assertDiracSucceeds( self.slic._checkConsistency(), self )
def test_checkconsistency_noversion( self ):
self.slic.version = None
assertDiracFailsWith( self.slic._checkConsistency(), 'no version found', self )
def test_checkconsistency_nosteeringfile( self ):
import inspect
self.slic.steeringFile = 'mysteer.file'
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(SLIC), 'Exists', new=Mock(return_value=S_ERROR('failed downloading, testerr'))):
assertDiracFailsWith( self.slic._checkConsistency(), 'failed downloading, testerr', self )
def test_resolvestepparams( self ):
step_mock = Mock()
inputstep_mock = Mock()
self.slic._linkedidx = 2
self.slic._jobsteps = [ None, None, inputstep_mock ]
assertDiracSucceeds( self.slic._resolveLinkedStepParameters( step_mock ), self )
assertDiracSucceeds( self.slic._checkWorkflowConsistency(), self )
def test_resolvestepparams_noinputstep( self ):
self.slic._linkedidx = '139'
self.slic._jobsteps = []
assertDiracSucceeds( self.slic._resolveLinkedStepParameters( '' ), self )
workflowName = eventType + '_' + energy
print 'Workflow name:', workflowName
if not workflowDescription:
workflowDescription = eventType + ' at ' + energy
print 'Workflow description:', workflowDescription
if checkMeta:
answer = raw_input('Submit production? (Y/N): ')
if not answer.lower() in ('y', 'yes'):
sys.exit(2)
####Define the applications
## Simulation
slic = SLIC()
slic.setVersion(slicVersion)
slic.setSteeringFile(slicMacro)
slic.setDetectorModel(detectorName)
## Reco without overlay: Do not take the input from SLIC as those are ran in different prods
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion(lcsimVers)
lcsim_prepandora.setSteeringFile(lcsimSteering1)
lcsim_prepandora.setTrackingStrategy(strategies)
lcsim_prepandora.setDetectorModel(detectorName)
lcsim_prepandora.setOutputFile("prePandora.slcio")
slicpandora = SLICPandora()
slicpandora.setVersion(pandoraVers)
slicpandora.setDetectorModel(detectorName)
def setUp(self):
"""set up the objects"""
self.slic = SLIC({})
self.slic.version = 941
print 'Workflow name:', workflowName
if not workflowDescription:
workflowDescription = eventType+' at '+energy
print 'Workflow description:', workflowDescription
if checkMeta:
answer = raw_input('Submit production? (Y/N): ')
if not answer.lower() in ('y', 'yes'):
sys.exit(2)
####Define the applications
## Simulation
slic = SLIC()
slic.setVersion(slicVersion)
slic.setSteeringFile(slicMacro)
slic.setDetectorModel(detectorName)
## Reco without overlay: Do not take the input from SLIC as those are ran in different prods
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion(lcsimVers)
lcsim_prepandora.setSteeringFile(lcsimSteering1)
lcsim_prepandora.setTrackingStrategy(strategies)
lcsim_prepandora.setDetectorModel(detectorName)
lcsim_prepandora.setOutputFile("prePandora.slcio")
slicpandora = SLICPandora()
slicpandora.setVersion(pandoraVers)
slicpandora.setDetectorModel(detectorName)
nbevtsperfile = 200
#Do Reco
sid_rec = True
#Do Reco with Overlay
sid_rec_ov = False
n_events = 100 #How many events per job. (Is also obtained from the FC in principle)
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation SID
slic = SLIC()
slic.setVersion('v2r9p8') #This will change only once
slic.setSteeringFile(
'defaultClicCrossingAngle.mac') #This will change only once
slic.setDetectorModel(detector_model)
slic.setNumberOfEvents(n_events)
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
## SID Reco w/o overlay
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion('CLIC_CDR') #This will change only once
lcsim_prepandora.setSteeringFile(
"clic_cdr_prePandora.lcsim") #The steering files should NOT CHANGE
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation ILD
mo = Mokka()
mo.setVersion("0706P08")
# mo.setNbEvts(1000)
if energy in [500.0, 350.0]:
mo.setSteeringFile("clic_ild_cdr500.steer")
else:
mo.setSteeringFile("clic_ild_cdr.steer")
##Simulation SID
slic = SLIC()
slic.setVersion("v2r9p8")
slic.setSteeringFile("defaultClicCrossingAngle.mac")
slic.setDetectorModel("clic_sid_cdr")
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
overlay = OverlayInput()
overlay.setMachine("clic_cdr")
overlay.setEnergy(energy)
if energy == 500.0:
overlay.setBXOverlay(300)
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True, setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector, fileNumber, args.outputPath, softVersions)
slicOutput=outputFiles['slicOutput']
prePandoraOutput=outputFiles['prePandoraOutput']
pandoraOutput=outputFiles['pandoraOutput']
vertexingOutput=outputFiles['vertexingOutput']
lcsimRecOutput=outputFiles['lcsimRecOutput']
lcsimDstOutput=outputFiles['lcsimDstOutput']
flavortagOutput=outputFiles['flavortagOutput']
diracOutput=outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile('steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml') # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us',])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
#### Software Versions ####
softVersions = ["v3r0p3", "HEAD", "ILC_DBD",
"0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# Check the --runs and --split arguments to make sure they are compatible, if not exit...
if not check_events_arguments(args.events, args.split):
sys.exit(1)
# Check the input LFN given by user, it needs to have .stdhep extension and should not have LFN: at the beginning...
lfn_check, lfn = check_input_LFN(args.stdhepInput)
if not lfn_check:
sys.exit(1)
# Call when you begin ILC-DIRAC jobs, the true indicates a repository file is included...
dirac = DiracILC(True,
setup_repository_name(args.stdhepInput, args.detector))
# Prepares the input and output sandboxes, if -f, then adds the files required for flavortagging,
# into the input sandbox
inputSandbox, outputSandbox = setup_sandboxes(args.macFile, args.flavortag)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(path.basename(args.stdhepInput))
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent / nOutputEvents
print "Job ", fileNumber
outputFiles = setup_output_dict(args.stdhepInput, args.detector,
fileNumber, args.outputPath,
softVersions)
slicOutput = outputFiles['slicOutput']
prePandoraOutput = outputFiles['prePandoraOutput']
pandoraOutput = outputFiles['pandoraOutput']
vertexingOutput = outputFiles['vertexingOutput']
lcsimRecOutput = outputFiles['lcsimRecOutput']
lcsimDstOutput = outputFiles['lcsimDstOutput']
flavortagOutput = outputFiles['flavortagOutput']
diracOutput = outputFiles['diracOutput']
################## SLIC ##################################################
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(slicOutput)
slic.setDetectorModel(args.detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (digitization and tracking) #####################
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(
'steeringFiles/sid_dbd_prePandora_noOverlay_v22.xml'
) # Another version is included in /steeringFiles
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy(
'steeringFiles/sidloi3_trackingStrategies_default.xml')
# lcsim.setAliasProperties('alias.properties')
lcsim.setDetectorModel('geometryFiles/sidloi3.zip')
lcsim.setOutputFile(prePandoraOutput)
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(args.detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(pandoraOutput)
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/sidloi3.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(vertexingOutput)
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
# lcsimDst.setAliasProperties('alias.properties')
lcsimDst.setDetectorModel('geometryFiles/sidloi3.zip')
lcsimDst.setOutputRecFile(lcsimRecOutput)
lcsimDst.setOutputDstFile(lcsimDstOutput)
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if args.flavortag:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/sidloi3.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(flavortagOutput)
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites([
'LCG.IN2P3-CC.fr',
'LCG.RAL-LCG2.uk',
'LCG.DESY-HH.de',
'LCG.DESYZN.de',
'LCG.KEK.jp',
'OSG.PNNL.us',
])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
if args.flavortag:
job.setOutputData(flavortagOutput, diracOutput, args.SE)
else:
job.setOutputData(lcsimDstOutput, diracOutput, args.SE)
job.setOutputSandbox(outputSandbox)
job.setInputData(lfn)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0
def main():
# Take the input arguments from the argument parser, and check they exist...
args = parse_args()
if not args:
print 'Invalid Arguments'
sys.exit(1)
print args.chain[0]
# softVersions = ["v3r0p3", "3.0-SNAPSHOT", "ILC_DBD", "0116"]
softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "0116"] # Working (recommended)
# softVersions = ["v3r0p3", "2.5", "ILC_DBD", "0116"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILC_DBD", "ILCSoft-01-17-07"] # Working
# softVersions = ["v3r0p3", "HEAD", "ILCSoft-01-17-08", "0116"]
check_events_arguments(args.events, args.split)
detector = args.detector
alias_properties(detector)
outputPath, outputBase, repoName = input_output(args.Input, detector, args.chain, args.digiSteering)
inputSandbox, outputSandbox = setup_sandboxes(args.macFile)
dirac = DiracILC(True, repoName)
# Prepares values for the job loop...
if args.split < 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.events)
if args.split > 0:
nInputEvents = int(args.events)
nOutputEvents = int(args.split)
# Loop that runs through the required number of jobs to be executed...
for startEvent in range(0, nInputEvents, nOutputEvents):
################## Job Initialise ########################################
job = UserJob()
job.setName(outputBase)
job.setJobGroup('JobGroup')
job.setInputSandbox(inputSandbox)
fileNumber = startEvent/nOutputEvents
print "Job ---> ", fileNumber
################## SLIC ##################################################
if 1 in args.chain:
slic = SLIC()
slic.setVersion(softVersions[0])
slic.setSteeringFile(args.macFile)
# slic.setInputFile(lfn)
slic.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'))
slic.setDetectorModel(detector)
slic.setNumberOfEvents(nOutputEvents)
slic.setStartFrom(startEvent)
#print slic.listAttributes()
result = job.append(slic)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Overlay ###############################################
if 2 in args.chain:
'''
#Add the gghad background overlay.
gghad = OverlayInput()
#gghad.setProdID(1767)
gghad.setEnergy(500.0)
gghad.setBXOverlay('args.bunches')
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setMachine('ilc_dbd')
gghad.setDetectorModel('sidloi3')
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
#Add the pair background overlay.
pairs = OverlayInput()
pairs.setProdID(2)
pairs.setEnergy(500.0)
pairs.setBXOverlay('args.bunches')
pairs.setGGToHadInt(1.)
pairs.setNbSigEvtsPerJob(nOutputEvents)
pairs.setMachine('ilc_dbd')
pairs.setDetectorModel('sidloi3')
pairs.setBkgEvtType('eepairs')
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
gghad = OverlayInput()
gghad.setPathToFiles('/ilc/user/j/jstrube/gghadron_lowpt/sidloi3/')
gghad.setBXOverlay(int(args.bunches))
gghad.setGGToHadInt( 4.1 )
gghad.setNbSigEvtsPerJob(nOutputEvents)
gghad.setBkgEvtType('aa_lowpt')
result = job.append( gghad )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
pairs = OverlayInput()
pairs.setPathToFiles('/ilc/user/j/jstrube/GuineaPig/sidloi3/')
pairs.setBXOverlay(int(args.bunches))
pairs.setGGToHadInt(1.)
pairs.setBkgEvtType('eepairs')
pairs.setNbSigEvtsPerJob(nOutputEvents)
result = job.append( pairs )
if not result['OK']:
print result['Message']
sys.exit(2)
'''
################## lcsim (digitization and tracking) #####################
if 3 in args.chain:
lcsim = LCSIM()
lcsim.setVersion(softVersions[1])
lcsim.setSteeringFile(args.digiSteering) # Another version is included in /steeringFiles
if 1 in args.chain:
lcsim.getInputFromApp(slic)
lcsim.setTrackingStrategy('steeringFiles/sidloi3_trackingStrategies_default.xml')
lcsim.setAliasProperties('steeringFiles/alias.properties')
lcsim.setDetectorModel(detector+".zip")
#lcsim.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'))
lcsim.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio')) #NEED TO CHANGE!!!
lcsim.setNumberOfEvents(nOutputEvents)
#print lcsim.listAttributes()
result = job.append(lcsim)
if not result['OK']:
print result['Message']
sys.exit(2)
################## slicPandora ###########################################
if 4 in args.chain:
slicPandora = SLICPandora()
slicPandora.setVersion(softVersions[2])
slicPandora.setDetectorModel(detector)
slicPandora.getInputFromApp(lcsim)
slicPandora.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'))
slicPandora.setPandoraSettings('pandoraSettings.xml')
slicPandora.setNumberOfEvents(nOutputEvents)
#print slicPandora.listAttributes()
result = job.append(slicPandora)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus Vertexing ############################
if 5 in args.chain:
vertexing = Marlin()
vertexing.setVersion(softVersions[3])
vertexing.setSteeringFile('steeringFiles/sid_dbd_vertexing.xml')
vertexing.setGearFile('steeringFiles/' + detector + '.gear')
vertexing.getInputFromApp(slicPandora)
vertexing.setOutputFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_vertexing.slcio'))
vertexing.setNumberOfEvents(nOutputEvents)
#print vertexing.listAttributes()
result = job.append(vertexing)
if not result['OK']:
print result['Message']
sys.exit(2)
################## lcsim (DST production) ################################
lcsimDst = LCSIM()
lcsimDst.setVersion(softVersions[1])
lcsimDst.setSteeringFile('steeringFiles/sid_dbd_postPandora.xml')
lcsimDst.getInputFromApp(vertexing)
lcsimDst.setNumberOfEvents(nOutputEvents)
lcsimDst.setAliasProperties('steeringFiles/alias.properties')
lcsimDst.setDetectorModel(detector+".zip")
lcsimDst.setOutputRecFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_Rec.slcio'))
lcsimDst.setOutputDstFile(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'))
#print lcsimDst.listAttributes()
result = job.append(lcsimDst)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Marlin, LCFIPlus flavortag ############################
if 6 in args.chain:
flavortag = Marlin()
flavortag.setVersion(softVersions[3])
flavortag.setSteeringFile('steeringFiles/sid_dbd_flavortag.xml')
flavortag.setGearFile('steeringFiles/' + detector + '.gear')
flavortag.setInputFile(lcsimDstOutput)
flavortag.setOutputFile(outputBase.replace('.slcio', '_' + '_flavortag.slcio'))
flavortag.setNumberOfEvents(nOutputEvents)
#print flavortag.listAttributes()
result = job.append(flavortag)
if not result['OK']:
print result['Message']
sys.exit(2)
################## Job Finalise ##########################################
# List of banned sites that the job shall not be sent too. These are sites that jobs tend to fail on,
# This list is likely to change.
job.setBannedSites(['LCG.IN2P3-CC.fr', 'LCG.RAL-LCG2.uk', 'LCG.DESY-HH.de', 'LCG.DESYZN.de', 'LCG.KEK.jp',
'OSG.PNNL.us','OSG.UConn.us','OSG.GridUNESP_CENTRAL.br','LCG.SCOTGRIDDURHAM.uk',
'LCG.TECHNIONself.il','LCG.UKI-SOUTHGRID-RALPP.uk','OSG.FNAL_FERMIGRID.us','LCG.UKI-LT2-IC-HEP.uk'])
job.setCPUTime(50000)
job.setPlatform('x86_64-slc5-gcc43-opt')
# Sets the output data file according to if -f is selcted, ships ouput to your /ilc/user/a/aPerson/
# directory on the grid.
outputLevel = max(args.chain)
if outputLevel == 1:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_sim.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 3:
#job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_digiTracking.slcio'), outputPath, 'CERN-SRM')
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 4:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_pandora.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 5:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_DST.slcio'), outputPath, 'CERN-SRM')
if outputLevel == 6:
job.setOutputData(outputBase.replace('.slcio', '_' + str(fileNumber) + '_flavortag.slcio'), outputPath, 'CERN-SRM')
job.setOutputSandbox(outputSandbox)
job.setInputData(args.Input)
if args.dontPromptMe:
job.dontPromptMe()
# Submits Job!!!
job.submit()
return 0;
job.setCPUTime(86400)
#below ten mb, specified local path
#larger input files are put into a grid storage unit, specified with grid path
#job.setInputSandbox(["newDetector.zip"])
job.setInputSandbox(["alias.properties"])
#'/afs/cern.ch/user/s/ssetru/www/newDetector.zip'
#job.setInputSandbox.append('/afs/cern.ch/user/s/ssetru/www/newDetector.zip')
#has log files, also may want to specify *.xml, generally short term data
job.setOutputSandbox(["*.log", "*.mac", "*.xml"])
#stored forever, in grid storage until you delete, path specified goes after your user directory in dirac
job.setOutputData(output, "test_vtx_matbudghalf_nonsensitivelayer",
"CERN-SRM")
#for index in xrange(0,len(slicNumEvents)):
slic = SLIC()
slic.setVersion('v3r0p3')
#can loop over stdhep files, takes precedence over what is in macro
#slic.setInputFile("some_file.stdhep")
#need not loop over macro
#slic.setInputFile("newDetector.zip")
slic.setSteeringFile(macro)
slic.setNumberOfEvents(nEvts)
#also overwrites macro
#two options
#1) specify lcdd filename (newDetector.lcdd) local or grid
#2) specify detector name (like below). looks up detector in org.lcsim detector page and downloads the tarball
slic.setDetectorModel('detector_vtx_matbudghalf_nonsensitivelayer.zip')
#same rules as for InputFile EXCEPT NOT automatically added to output data (sandbox too small usually)
slic.setOutputFile('slicOut.slcio')
res = job.append(slic)
# Do Split
activesplitstdhep = True
# Do Sim
sid_sim = True
# Do Replication
replicateFiles = True
## Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion(stdhepSplitVersion)
stdhepsplit.setNumberOfEventsPerFile(nEvtsPerJob)
## Simulation
slic = SLIC()
slic.setVersion(slicVersion)
slic.setSteeringFile(slicMacro)
slic.setDetectorModel(detectorModel)
slic.setNumberOfEvents(nEvtsPerJob)
############################################
#What is below WILL NEVER NEED TO BE TOUCHED
#(I'm not kidding, if you touch and break, not my problem)
#
# Define production step splitting the stdhep files
if activesplitstdhep and meta:
pstdhepsplit = SIDProductionJob()
pstdhepsplit.setLogLevel('verbose')
pstdhepsplit.setProdType('Split')
res = pstdhepsplit.setInputDataQuery(meta)
##Simulation ILD
mo = Mokka()
mo.setVersion('0706P08')
#mo.setNbEvts(1000)
if energy in [500., 375., 350.]:
mo.setSteeringFile("clic_ild_cdr500.steer")
elif energy in [3000., 1400.]:
mo.setSteeringFile("clic_ild_cdr.steer")
else:
print 'Detector Model for Mokka undefined for this energy'
if detectormodel=='ild_00':
mo.setSteeringFile("ild_00.steer")
##Simulation SID
slic = SLIC()
slic.setVersion('v2r9p8')
slic.setSteeringFile('defaultClicCrossingAngle.mac')
slic.setDetectorModel('clic_sid_cdr')
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
overlay = OverlayInput()
overlay.setMachine("clic_cdr")
overlay.setEnergy(energy)
overlay.setBkgEvtType("gghad")
if energy==500.:
#Do Reco
sid_rec = True
#Do Reco with Overlay
sid_rec_ov = False
n_events = 100 #How many events per job. (Is also obtained from the FC in principle)
##Split
stdhepsplit = StdHepSplit()
stdhepsplit.setVersion("V2")
stdhepsplit.setNumberOfEventsPerFile(nbevtsperfilestdhep)
##Simulation SID
slic = SLIC()
slic.setVersion('v2r9p8')#This will change only once
slic.setSteeringFile('defaultClicCrossingAngle.mac')#This will change only once
slic.setDetectorModel(detector_model)
slic.setNumberOfEvents(n_events)
##Split
split = SLCIOSplit()
split.setNumberOfEventsPerFile(nbevtsperfile)
## SID Reco w/o overlay
lcsim_prepandora = LCSIM()
lcsim_prepandora.setVersion('CLIC_CDR')#This will change only once
lcsim_prepandora.setSteeringFile("clic_cdr_prePandora.lcsim")#The steering files should NOT CHANGE
lcsim_prepandora.setTrackingStrategy(tracking_strategy)
#lcsim_prepandora.setDetectorModel(detector_model)
class SLICTestCase(unittest.TestCase):
""" Base class for the SLIC test cases
"""
def setUp(self):
"""set up the objects"""
self.slic = SLIC({})
self.slic.version = 941
def test_setters(self):
self.slic.setRandomSeed('someInvalidSeed')
self.slic.setDetectorModel('myDetModelTestme')
self.slic.setStartFrom('soirmgf')
assertInImproved('_checkArgs', self.slic._errorDict, self)
(rand_found, start_found) = (False, False)
for err in self.slic._errorDict['_checkArgs']:
if 'randomSeed' in err:
rand_found = True
elif 'startfrom' in err:
start_found = True
assertEqualsImproved(
(True, True, 'myDetModelTestme'),
(rand_found, start_found, self.slic.detectorModel), self)
def test_setdetectormodel(self):
self.slic.setDetectorModel('lfn:/my/det.model')
assertInImproved('lfn:/my/det.model', self.slic.inputSB, self)
assertEqualsImproved(self.slic.detectorModel, 'det.model', self)
self.slic = SLIC({})
with patch('os.path.exists', new=Mock(return_value=True)):
self.slic.setDetectorModel('/my/local/dir/detectorv212.stdhep.zip')
assertEqualsImproved((self.slic.detectorModel, self.slic.inputSB),
('detectorv212.stdhep',
['/my/local/dir/detectorv212.stdhep.zip']),
self)
self.slic = SLIC({})
with patch('os.path.exists', new=Mock(return_value=False)):
self.slic.setDetectorModel('/my/local/dir/detectorv212.stdhep.zip')
assertEqualsImproved((self.slic.detectorModel, self.slic.inputSB),
('detectorv212.stdhep', []), self)
def test_userjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.slic._userjobmodules(module_mock), self)
def test_prodjobmodules(self):
module_mock = Mock()
assertDiracSucceeds(self.slic._prodjobmodules(module_mock), self)
def test_userjobmodules_fails(self):
with patch('%s._setUserJobFinalization' % MODULE_NAME, new=Mock(return_value=S_OK('something'))),\
patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_test_err'))):
assertDiracFailsWith(self.slic._userjobmodules(None),
'userjobmodules failed', self)
def test_prodjobmodules_fails(self):
with patch('%s._setApplicationModuleAndParameters' % MODULE_NAME, new=Mock(return_value=S_OK('something'))), \
patch('%s._setOutputComputeDataList' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_other_test_err'))):
assertDiracFailsWith(self.slic._prodjobmodules(None),
'prodjobmodules failed', self)
def test_checkconsistency(self):
self.slic._jobtype = 'User'
self.slic.startFrom = True
with patch('os.path.exists', new=Mock(return_value=True)):
assertDiracSucceeds(self.slic._checkConsistency(), self)
def test_checkconsistency_othercase(self):
self.slic.detectorModel = 'someModel'
self.slic.steeringFile = 'LFN:/my/consistent/steeringfile.st'
self.slic._jobtype = 'notUser'
self.slic.startFrom = False
with patch('os.path.exists', new=Mock(return_value=False)):
assertDiracSucceeds(self.slic._checkConsistency(), self)
def test_checkconsistency_download_succeeds(self):
import inspect
self.slic.steeringFile = '/my/remote/path.st'
self.slic._jobtype = 'notUser'
self.slic.startFrom = False
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(SLIC), 'Exists', new=Mock(return_value=S_OK('ok'))):
assertDiracSucceeds(self.slic._checkConsistency(), self)
def test_checkconsistency_noversion(self):
self.slic.version = None
assertDiracFailsWith(self.slic._checkConsistency(), 'no version found',
self)
def test_checkconsistency_nosteeringfile(self):
import inspect
self.slic.steeringFile = 'mysteer.file'
with patch('os.path.exists', new=Mock(return_value=False)), \
patch.object(inspect.getmodule(SLIC), 'Exists', new=Mock(return_value=S_ERROR('failed downloading, testerr'))):
assertDiracFailsWith(self.slic._checkConsistency(),
'failed downloading, testerr', self)
def test_resolvestepparams(self):
step_mock = Mock()
inputstep_mock = Mock()
self.slic._linkedidx = 2
self.slic._jobsteps = [None, None, inputstep_mock]
assertDiracSucceeds(self.slic._resolveLinkedStepParameters(step_mock),
self)
assertDiracSucceeds(self.slic._checkWorkflowConsistency(), self)
def test_resolvestepparams_noinputstep(self):
self.slic._linkedidx = '139'
self.slic._jobsteps = []
assertDiracSucceeds(self.slic._resolveLinkedStepParameters(''), self)