def addSimulationSubstep(executorSet, overlayTransform=False):
TRExe = athenaExecutor(
name='TRtoHITS',
skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_ISF.py',
substep='simTRIn',
tryDropAndReload=False,
perfMonFile='ntuple.pmon.gz',
inData=['EVNT_TR'],
outData=['HITS', 'NULL'])
executorSet.add(TRExe)
SimExe = athenaExecutor(
name='EVNTtoHITS',
skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_ISF.py',
substep='sim',
tryDropAndReload=False,
perfMonFile='ntuple.pmon.gz',
inData=['NULL', 'EVNT'],
outData=['EVNT_TR', 'HITS', 'NULL'])
if overlayTransform:
from PyJobTransforms.trfUtils import releaseIsOlderThan
if releaseIsOlderThan(20, 3):
SimExe.inData = [('EVNT', 'BS_SKIM')]
else:
SimExe.inData = [('EVNT', 'TXT_EVENTID')]
SimExe.outData = ['HITS']
SimExe.inputDataTypeCountCheck = ['EVNT']
executorSet.add(SimExe)
def getTransform():
executorSet = set()
for subregion in range(subregions):
executorSet.add(athenaExecutor(name = 'FTKFullSimulationBank{0}'.format(subregion),
skeletonFile = 'TrigFTKSim/skeleton.FTKStandaloneSim.py',
inData = ['RDO','NTUP_FTKIP','TXT_FTKIP'], outData = ['NTUP_FTKTMP_{0}'.format(subregion)],
extraRunargs = {'banksubregion': [subregion]},
# Need to ensure that the correct subregion is used
runtimeRunargs = {'patternbankpath': 'runArgs.patternbank{0}path'.format(subregion),
'fitconstantspath': 'runArgs.fitconstants{0}path'.format(subregion),
'fit711constantspath': 'runArgs.fit711constants{0}path'.format(subregion),
'sectorpath': 'runArgs.sector{0}path'.format(subregion),
'outputNTUP_FTKTMPFile': 'runArgs.outputNTUP_FTKTMP_{0}File'.format(subregion)}))
executorSet.add(athenaExecutor(name = 'FTKSimulationMerge',
skeletonFile = 'TrigFTKSim/skeleton.FTKStandaloneMerge.py',
inData = [tuple([ 'NTUP_FTKTMP_{0}'.format(subregion) for subregion in range(subregions) ])],
outData = ['NTUP_FTKTMP'],
extraRunargs = {'inputNTUP_FTKTMPFile': [ 'tmp.NTUP_FTKTMP_{0}'.format(subregion) for subregion in range(subregions)]},
runtimeRunargs = {'MergeRegion': 'runArgs.bankregion[0]',
'FirstRegion': 'runArgs.bankregion[0]'},
))
trf = transform(executor = executorSet, description = 'FTK Subregion simulate x {0} and merge.'.format(subregions))
addAthenaArguments(trf.parser,maxEventsDefaultSubstep='all')
addFTKSimulationArgs(trf.parser)
return trf
def getTransform():
executorSet = set()
addRecoSubsteps(executorSet)
addDigitizationSubstep(executorSet)
# Sim + Digi - factor these out into an importable function in time
executorSet.add(
athenaExecutor(
name='TRtoHITS',
skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_ISF.py',
substep='simTRIn',
tryDropAndReload=False,
perfMonFile='ntuple.pmon.gz',
inData=['EVNT_TR'],
outData=['HITS', 'NULL']))
executorSet.add(
athenaExecutor(
name='EVNTtoHITS',
skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_ISF.py',
substep='sim',
tryDropAndReload=False,
perfMonFile='ntuple.pmon.gz',
inData=['NULL', 'EVNT'],
outData=['EVNT_TR', 'HITS', 'NULL']))
trf = transform(
executor=executorSet,
description=
'Full chain ATLAS transform with ISF simulation, digitisation'
' and reconstruction. Inputs can be EVNT, EVNT_TR, HITS, RDO, BS, ESD or AOD, with outputs of RDO, ESD, AOD or DPDs.'
' See https://twiki.cern.ch/twiki/bin/viewauth/Atlas/FullChainTf for more details.'
)
# Common arguments
addAthenaArguments(trf.parser)
addDetectorArguments(trf.parser)
addTriggerArguments(trf.parser)
# Reconstruction arguments and outputs (use the factorised 'do it all' function)
addAllRecoArgs(trf)
# Simulation and digitisation options
addCommonSimTrfArgs(trf.parser)
addCommonSimDigTrfArgs(trf.parser)
addCosmicsTrfArgs(trf.parser)
addBasicDigiArgs(trf.parser)
addSim_tfArgs(trf.parser)
addForwardDetTrfArgs(trf.parser)
addPileUpTrfArgs(trf.parser)
addCommonSimDigTrfArgs(trf.parser)
addTrackRecordArgs(trf.parser)
return trf
def getTransform():
executorSet = set()
executorSet.add(
athenaExecutor(
name='FTKSimulationMerge',
skeletonFile='TrigFTKSim/skeleton.FTKStandaloneMerge.py',
inData=['NTUP_FTKTMP'],
disableMP=True,
outData=['NTUP_FTK', 'RDO_FTK'],
))
executorSet.add(
athenaExecutor(
name='FTKSimulationRDOMerge',
skeletonFile='TrigFTKSim/skeleton.FTKStandaloneMerge.py',
disableMP=True,
inData=[('NTUP_FTKTMP', 'RDO')],
outData=['RDO_FTK'],
))
executorSet.add(
athenaExecutor(
name='FTKRecoRDOtoESD',
skeletonFile='RecJobTransforms/skeleton.RAWtoESD.py',
substep='r2e',
inData=[('RDO', 'NTUP_FTK')],
outData=['DESD_FTK'],
perfMonFile='ntuple_RAWtoESD.pmon.gz',
))
# extraRunargs = {'preInclude': ['TrigFTKSim/FTKReco_jobOptions.py']}))
executorSet.add(
athenaExecutor(
name='FTKRecoESDtoNTUP',
skeletonFile='PyJobTransforms/skeleton.ESDtoDPD.py',
substep='e2d',
inData=['DESD_FTK'],
outData=['NTUP_TRIG'],
perfMonFile='ntuple_ESDtoDPD.pmon.gz',
))
# extraRunargs = {'preInclude': ['TrigFTKSim/FTKReco_jobOptions.py']}))
trf = transform(executor=executorSet,
description='FTK full region merge and reco.')
addAthenaArguments(trf.parser, maxEventsDefaultSubstep='all')
addDetectorArguments(trf.parser)
addCommonRecTrfArgs(trf.parser)
addFTKSimulationArgs(trf.parser)
addD3PDArguments(trf.parser, pick=['NTUP_TRIG'], transform=trf)
addBasicDigiArgs(trf.parser)
return trf
def getTransform():
trf = transform(executor = athenaExecutor(name = 'FilterHitTf', substep="filthits", skeletonFile = 'SimuJobTransforms/skeleton.FilterHit.py',
tryDropAndReload = False))
addAthenaArguments(trf.parser)
addDetectorArguments(trf.parser)
addMyArgs(trf.parser)
return trf
def addStandardHITSMergeSubstep(executorSet):
executorSet.add(
athenaExecutor(name='HITSMerge',
substep="hitsmerge",
skeletonFile='SimuJobTransforms/skeleton.HITSMerge.py',
tryDropAndReload=False,
inputDataTypeCountCheck=['HITS']))
def getTransform():
trf = transform(executor = athenaExecutor(name = 'RAWtoESD', skeletonFile = 'RecJobTransforms/skeleton.RAWtoESD_tf.py',
substep = 'r2e',
))
addAthenaArguments(trf.parser)
addMyArgs(trf.parser)
return trf
def getTransform():
# Get the base transform with all arguments added
trf = transform(executor = athenaExecutor(name = 'FTKPattGenRoot',
skeletonFile = 'TrigFTKBankGen/skeleton.FTKPattBankGenRoot.py'))
addAthenaArguments(trf.parser)
addFTKPattGenArgs(trf.parser)
return trf
def getTransform():
executorSet = set()
for subregion in range(subregions):
executorSet.add(
athenaExecutor(
name='FTKFullSimulationBank{0}'.format(subregion),
skeletonFile='TrigFTKSim/skeleton.FTKStandaloneSim.py',
inData=['NTUP_FTKIP', 'TXT_FTKIP'],
outData=['NTUP_FTKTMP'],
extraRunargs={'banksubregion': [subregion]},
# Need to ensure that the correct subregion is used
runtimeRunargs={
'patternbankpath':
'runArgs.patternbank{0}path'.format(subregion),
'outputNTUP_FTKTMPFile':
'runArgs.outputNTUP_FTKFile',
'cachedbankpath':
'runArgs.cachedbank{0}path'.format(subregion),
'CachePath':
'runArgs.CachePath{0}'.format(subregion)
}))
trf = transform(
executor=executorSet,
description='FTK Subregion simulate x {0} .'.format(subregions))
addFTKSimulationArgs(trf.parser)
addTrigFTKSimOptions(trf.parser, nsubregions=subregions)
addTrigFTKSimMergeOptions(trf.parser)
addTrigFTKSimTFOptions(trf.parser)
addTrigFTKSimRFOptions(trf.parser)
return trf
def getTransform():
executorSet = set()
from EventIndexProducer.EITransformUtils import addEI_MRG_Substep, addEI_MRG_arguments
executorSet.add(
hybridPOOLMergeExecutor(
name='AODMerge',
skeletonFile='RecJobTransforms/skeleton.MergePool_tf.py',
inData=['AOD'],
outData=['AOD_MRG'],
perfMonFile='ntuple_POOLMerge.pmon.gz'))
executorSet.add(
athenaExecutor(
name='AODtoTAG',
skeletonFile='RecJobTransforms/skeleton.AODtoTAG_tf.py',
inData=['AOD_MRG'],
outData=['TAG'],
))
addEI_MRG_Substep(executorSet)
trf = transform(executor=executorSet)
addAthenaArguments(trf.parser)
addDetectorArguments(trf.parser)
addCommonRecTrfArgs(trf.parser)
addMyArgs(trf.parser)
addEI_MRG_arguments(trf.parser)
return trf
def addDigiValidationSubstep(executorSet):
executorSet.add(
athenaExecutor(
name='DigiValidation',
skeletonFile='SimuJobTransforms/skeleton.RDOtoHIST_DIGI.py',
inData=['RDO'],
outData=['HIST_DIGI'],
))
def addAtlasG4Substep(executorSet):
executorSet.add(
athenaExecutor(
name='AtlasG4TfTRIn',
skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_MC12.py',
substep='simTRIn',
tryDropAndReload=False,
inData=['EVNT_TR'],
outData=['HITS', 'NULL']))
executorSet.add(
athenaExecutor(
name='AtlasG4Tf',
skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_MC12.py',
substep='sim',
tryDropAndReload=False,
inData=['NULL', 'EVNT'],
outData=['EVNT_TR', 'HITS', 'NULL']))
def getTransform():
trf = transform(executor=athenaExecutor(
name='FTKSimulationMerge',
disableMP=True,
skeletonFile='TrigFTKSim/skeleton.FTKStandaloneMerge.py'))
addAthenaArguments(trf.parser)
addFTKMergerArgs(trf.parser)
return trf
def addPhysValidationSubstep(executorSet):
executorSet.add(
athenaExecutor(
name='PhysicsValidation',
skeletonFile='PATJobTransforms/skeleton.PhysicsValidation_tf.py',
inData=['AOD'],
outData=['NTUP_PHYSVAL'],
substep='pval'))
def addSimValidationSubstep(executorSet):
executorSet.add(
athenaExecutor(
name='SimValidation',
skeletonFile='SimuJobTransforms/skeleton.HITStoHIST_SIM.py',
inData=['HITS'],
outData=['HIST_SIM'],
))
def getTransform():
trf = transform(executor=athenaExecutor(
name='athena',
skeletonFile=None,
skeletonCA="PyJobTransforms.HelloWorldSkeleton"))
addAthenaArguments(trf.parser)
addDetectorArguments(trf.parser)
return trf
def addEI_MRG_Substep(executorSet):
executorSet.add(
athenaExecutor(
name='EventIndexTf',
skeletonFile='EventIndexProducer/skeleton.POOLtoEI_tf.py',
substep='EI',
tryDropAndReload=False,
inData=['AOD_MRG'],
outData=['NULL', 'EI']))
def addOverlaySubstep(executorSet, inRecoChain = False):
executor = athenaExecutor(name = 'Overlay', skeletonFile = 'EventOverlayJobTransforms/skeleton.Overlay_tf.py',
substep = 'overlay', tryDropAndReload = False, perfMonFile = 'ntuple.pmon.gz',
inData = [('HITS', 'RDO_BKG')], outData = ['RDO', 'RDO_SGNL'])
if inRecoChain:
executor.inData = []
executor.outData = []
executorSet.add(executor)
def getTransform():
trf = transform(executor=athenaExecutor(
name='ESDtoAOD',
skeletonFile='RecJobTransforms/skeleton.ESDtoAOD_tf.py',
substep='e2a'))
addAthenaArguments(trf.parser)
addDetectorArguments(trf.parser)
addCommonRecTrfArgs(trf.parser)
addMyArgs(trf.parser)
return trf
def getTransform():
executorSet = set()
executorSet.add(hybridPOOLMergeExecutor(name = 'ESDMerge', skeletonFile = 'RecJobTransforms/skeleton.MergePool_tf.py',
inData = ['ESD'], outData = ['ESD_MRG']))
executorSet.add(hybridPOOLMergeExecutor(name = 'AODMerge', skeletonFile = 'RecJobTransforms/skeleton.MergePool_tf.py',
inData = ['AOD'], outData = ['AOD_MRG']))
executorSet.add(athenaExecutor(name = 'AODtoTAG', skeletonFile = 'RecJobTransforms/skeleton.AODtoTAG_tf.py',
inData = ['AOD_MRG'], outData = ['TAG'],))
executorSet.add(tagMergeExecutor(name = 'TAGFileMerge', exe = 'CollAppend', inData = set(['TAG']), outData = set(['TAG_MRG'])))
executorSet.add(DQMergeExecutor(name = 'DQHistogramMerge', inData = [('HIST_ESD', 'HIST_AOD'), 'HIST'], outData = ['HIST_MRG']))
executorSet.add(athenaExecutor(name = 'RDOMerge', skeletonFile = 'RecJobTransforms/skeleton.MergeRDO_tf.py',
inData = ['RDO'], outData = ['RDO_MRG']))
executorSet.add(bsMergeExecutor(name = 'RAWFileMerge', exe = 'file_merging', inData = set(['BS']), outData = set(['BS_MRG'])))
executorSet.add(athenaExecutor(name = 'EVNTMerge', skeletonFile = 'PyJobTransforms/skeleton.EVNTMerge.py',inData = ['EVNT'], outData = ['EVNT_MRG']))
addDAODMergerSubsteps(executorSet)
addNTUPMergeSubsteps(executorSet)
trf = transform(executor = executorSet)
addAthenaArguments(trf.parser)
addDetectorArguments(trf.parser)
addCommonRecTrfArgs(trf.parser)
addMyArgs(trf.parser)
addDAODArguments(trf.parser)
addPhysValidationMergeFiles(trf.parser)
addD3PDArguments(trf.parser, transform=trf, addD3PDMRGtypes=True)
addExtraDPDTypes(trf.parser, transform=trf, NTUPMergerArgs = True)
# Add HITSMerge only if SimuJobTransforms is available
try:
from SimuJobTransforms.SimTransformUtils import addHITSMergeArguments
addHITSMergeArguments(trf.parser)
simStepSet = set()
simStepSet.add(athenaExecutor(name = 'HITSMerge', substep="hitsmerge", skeletonFile = 'SimuJobTransforms/skeleton.HITSMerge.py',
tryDropAndReload = False, inData = ['HITS'], outData = ['HITS_MRG']))
trf.appendToExecutorSet(list(simStepSet)[0])
except ImportError as e:
msg.warning('Failed to import simulation arguments ({0}). HITSMerge will not be available.'.format(e))
return trf
def execute(self):
if 'maxFilesPerSubjob' in self.conf.argdict:
self._hasExecuted = True
msg.info('Activating job splitting with {0} files per subjob'.format(self.conf.argdict['maxFilesPerSubjob'].value))
tmpFiles = self.conf.argdict['inputZeroBiasBSFile'].value
self._subJobInputs = []
while len(tmpFiles) > 0:
self._subJobInputs.append(tmpFiles[0:self.conf.argdict['maxFilesPerSubjob'].value-1])
del tmpFiles[0:self.conf.argdict['maxFilesPerSubjob'].value-1]
#### TESTING TESTING!
# self._subJobInputs = self._subJobInputs[0:2]
####
msg.info('Job will be split into {0} pieces: {1}'.format(len(self._subJobInputs), self._subJobInputs))
# Now loop over each subjob
from PyJobTransforms.trfExe import athenaExecutor, executorConfig, bsMergeExecutor
jobCounter = 0
outputBSFileList = []
for sjInputs in self._subJobInputs:
sjobArgdict = {'triggerBit': self.conf.argdict['triggerBit'],
'InputLbnMapFile': self.conf.argdict['InputLbnMapFile'],
'checkEventCount': trfArgClasses.argSubstepBool('False', runarg=False)}
sjOutput = self.conf.argdict['outputBS_SKIMFile'].value[0] + '_sub{0:03d}.tmp'.format(jobCounter)
outputBSFileList.append(sjOutput)
sjDataDictionary = {'BS': trfArgClasses.argBSFile(sjInputs, io='input', type='BS', subtype='BS'),
'BS_SKIM': trfArgClasses.argBSFile(sjOutput, io='output', type='BS', subtype='BS_SKIM')}
sjconf = executorConfig(sjobArgdict, sjDataDictionary)
sjexe = athenaExecutor(name='BSOverlayFilter_{0:03d}'.format(jobCounter), skeletonFile = 'EventOverlayJobTransforms/skeleton.BSOverlayFilter_tf.py',
conf=sjconf, inData=set(['BS']), outData=set(['BS_SKIM']))
sjexe.doAll(input=set(['BS']), output=set(['BS_SKIM']))
jobCounter += 1
# Now do the merging
mrgArgdict = {'maskEmptyInputs': trfArgClasses.argBool(True, runarg=False),
'allowRename': trfArgClasses.argBool(True, runarg=False)}
mrgDataDictionary = {'BS': trfArgClasses.argBSFile(outputBSFileList, io='input', type='BS', subtype='BS_SKIM'),
'BS_MRG': self.conf.argdict['outputBS_SKIMFile']}
mrgConf = executorConfig(mrgArgdict, mrgDataDictionary)
mrgExe = bsMergeExecutor(name = 'RAWFileMerge', exe = 'file_merging', conf=mrgConf,
inData = set(['BS']), outData = set(['BS_MRG']))
mrgExe.doAll(input=set(['BS']), output=set(['BS_MRG']))
# If we got here then all must be well, so we have executed successfully
self._rc = 0
else:
# Ordinary run
super(BSJobSplitterExecutor, self).execute()
def addConfigurableSimSubstep(executorSet, confName, extraSkeleton,
confSubStep, confInData, confOutData,
confExtraRunargs, confRuntimeRunargs):
executorSet.add(
athenaExecutor(name=confName,
skeletonFile=extraSkeleton +
['SimuJobTransforms/skeleton.EVGENtoHIT_MC12.py'],
substep=confSubStep,
tryDropAndReload=False,
inData=confInData,
outData=confOutData,
extraRunargs=confExtraRunargs,
runtimeRunargs=confRuntimeRunargs))
def getTransform():
executorSet = set()
executorSet.add(
athenaExecutor(
name='EvtGenFromEVNT',
skeletonFile='EvgenJobTransforms/skeleton.EvtGenFromEVNT.py',
inData=['EVNT'],
outData=['EVNT_MRG']))
trf = transform(executor=executorSet)
addAthenaArguments(trf.parser)
addMyArgs(trf.parser)
return trf
def getTransform():
executorSet = set()
print type(executorSet)
executorSet.add(
athenaExecutor(name='EVNTMerge',
skeletonFile='PyJobTransforms/skeleton.EVNTMerge.py',
inData=['EVNT'],
outData=['EVNT_MRG']))
trf = transform(executor=executorSet)
addAthenaArguments(trf.parser)
addMyArgs(trf.parser)
return trf
def getTransform():
exeSet = set()
exeSet.add(EvgenExecutor("EvgenJobTransforms/skeleton.ABtoEVGEN.py"))
exeSet.add(
athenaExecutor(name="AODtoDPD",
skeletonFile="PATJobTransforms/skeleton.AODtoDPD_tf.py",
substep="a2d",
inData=["EVNT"],
outData=["NTUP_TRUTH"],
perfMonFile="ntuple_AODtoDPD.pmon.gz"))
trf = transform(executor=exeSet)
addAthenaArguments(trf.parser)
addStdEvgenArgs(trf.parser)
return trf
def get_transform():
from PyJobTransforms.transform import transform
from PyJobTransforms.trfExe import athenaExecutor
trf = transform(executor=athenaExecutor(
name='RDOtoBS',
skeletonFile='TrigSimTransforms/skeleton.RDOtoBS.py',
substep='r2b',
))
from PyJobTransforms.trfArgs import addAthenaArguments
addAthenaArguments(trf.parser)
add_RDOtoBS_args(trf.parser)
return trf
def addTrigSimSubsteps(executorSet):
executorSet.add(
athenaExecutor(name='RDOtoBSOLD',
skeletonFile='TrigSimTransforms/skeleton.RDOtoBS.py',
substep='r2bo',
inData=['RDO'],
outData=['BS_OLD']))
executorSet.add(
athenaExecutorJO(
name='BStoTRIGBS',
skeletonFile='TrigSimTransforms/skeleton.BStoTRIGBS.py',
substep='b2tb',
inData=['BS_OLD'],
outData=['BS_TRIG_OLD']))
executorSet.add(
athenaExecutor(
name='TRIGBStoRDO',
skeletonFile='TrigSimTransforms/skeleton.TRIGBStoRDO.py',
substep='tb2tr',
inData=[('BS_TRIG_OLD', 'RDO')],
outData=['RDO_TRIG_OLD']))
def getTransform():
executorSet = set()
executorSet.add(athenaExecutor(name = 'FTKSimulationMergeDigi',
skeletonFile = 'TrigFTKSim/skeleton.FTKStandaloneMergeDigi.py',
inData = ['NTUP_FTKTMP'],disableMP=True,
outData = ['RDO_FTK'],))
trf = transform(executor = executorSet, description = 'FTK full region merge and reco.')
addAthenaArguments(trf.parser, maxEventsDefaultSubstep='all')
addDetectorArguments(trf.parser)
addCommonRecTrfArgs(trf.parser)
addFTKSimulationArgs(trf.parser)
addD3PDArguments(trf.parser, pick=['NTUP_TRIG'], transform=trf)
addBasicDigiArgs(trf.parser)
return trf
def addOverlaySubstep(executor_set, in_reco_chain=False):
executor = athenaExecutor(
name='Overlay',
skeletonFile='OverlayConfiguration/skeleton_LegacyOverlay.py',
skeletonCA='OverlayConfiguration.OverlaySkeleton',
substep='overlay',
tryDropAndReload=False,
perfMonFile='ntuple.pmon.gz',
inData=['RDO_BKG', 'BS_SKIM', 'HITS'],
outData=['RDO', 'RDO_SGNL'])
if in_reco_chain:
executor.inData = []
executor.outData = []
executor_set.add(executor)
def getTransform():
executorSet = set()
executorSet.add(
athenaExecutor(name='RDOMerge',
skeletonFile='RecJobTransforms/skeleton.MergeRDO_tf.py',
inData=['RDO'],
outData=['RDO_MRG']))
trf = transform(executor=executorSet)
addAthenaArguments(trf.parser)
addDetectorArguments(trf.parser)
addForwardDetTrfArgs(trf.parser)
addCommonRecTrfArgs(trf.parser)
addMyArgs(trf.parser)
return trf