def addSimulationSubstep(executorSet, overlayTransform=False): TRExe = athenaExecutor( name='TRtoHITS', skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_ISF.py', substep='simTRIn', tryDropAndReload=False, perfMonFile='ntuple.pmon.gz', inData=['EVNT_TR'], outData=['HITS', 'NULL']) executorSet.add(TRExe) SimExe = athenaExecutor( name='EVNTtoHITS', skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_ISF.py', substep='sim', tryDropAndReload=False, perfMonFile='ntuple.pmon.gz', inData=['NULL', 'EVNT'], outData=['EVNT_TR', 'HITS', 'NULL']) if overlayTransform: from PyJobTransforms.trfUtils import releaseIsOlderThan if releaseIsOlderThan(20, 3): SimExe.inData = [('EVNT', 'BS_SKIM')] else: SimExe.inData = [('EVNT', 'TXT_EVENTID')] SimExe.outData = ['HITS'] SimExe.inputDataTypeCountCheck = ['EVNT'] executorSet.add(SimExe)
def getTransform(): executorSet = set() for subregion in range(subregions): executorSet.add(athenaExecutor(name = 'FTKFullSimulationBank{0}'.format(subregion), skeletonFile = 'TrigFTKSim/skeleton.FTKStandaloneSim.py', inData = ['RDO','NTUP_FTKIP','TXT_FTKIP'], outData = ['NTUP_FTKTMP_{0}'.format(subregion)], extraRunargs = {'banksubregion': [subregion]}, # Need to ensure that the correct subregion is used runtimeRunargs = {'patternbankpath': 'runArgs.patternbank{0}path'.format(subregion), 'fitconstantspath': 'runArgs.fitconstants{0}path'.format(subregion), 'fit711constantspath': 'runArgs.fit711constants{0}path'.format(subregion), 'sectorpath': 'runArgs.sector{0}path'.format(subregion), 'outputNTUP_FTKTMPFile': 'runArgs.outputNTUP_FTKTMP_{0}File'.format(subregion)})) executorSet.add(athenaExecutor(name = 'FTKSimulationMerge', skeletonFile = 'TrigFTKSim/skeleton.FTKStandaloneMerge.py', inData = [tuple([ 'NTUP_FTKTMP_{0}'.format(subregion) for subregion in range(subregions) ])], outData = ['NTUP_FTKTMP'], extraRunargs = {'inputNTUP_FTKTMPFile': [ 'tmp.NTUP_FTKTMP_{0}'.format(subregion) for subregion in range(subregions)]}, runtimeRunargs = {'MergeRegion': 'runArgs.bankregion[0]', 'FirstRegion': 'runArgs.bankregion[0]'}, )) trf = transform(executor = executorSet, description = 'FTK Subregion simulate x {0} and merge.'.format(subregions)) addAthenaArguments(trf.parser,maxEventsDefaultSubstep='all') addFTKSimulationArgs(trf.parser) return trf
def getTransform(): executorSet = set() addRecoSubsteps(executorSet) addDigitizationSubstep(executorSet) # Sim + Digi - factor these out into an importable function in time executorSet.add( athenaExecutor( name='TRtoHITS', skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_ISF.py', substep='simTRIn', tryDropAndReload=False, perfMonFile='ntuple.pmon.gz', inData=['EVNT_TR'], outData=['HITS', 'NULL'])) executorSet.add( athenaExecutor( name='EVNTtoHITS', skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_ISF.py', substep='sim', tryDropAndReload=False, perfMonFile='ntuple.pmon.gz', inData=['NULL', 'EVNT'], outData=['EVNT_TR', 'HITS', 'NULL'])) trf = transform( executor=executorSet, description= 'Full chain ATLAS transform with ISF simulation, digitisation' ' and reconstruction. Inputs can be EVNT, EVNT_TR, HITS, RDO, BS, ESD or AOD, with outputs of RDO, ESD, AOD or DPDs.' ' See https://twiki.cern.ch/twiki/bin/viewauth/Atlas/FullChainTf for more details.' ) # Common arguments addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) addTriggerArguments(trf.parser) # Reconstruction arguments and outputs (use the factorised 'do it all' function) addAllRecoArgs(trf) # Simulation and digitisation options addCommonSimTrfArgs(trf.parser) addCommonSimDigTrfArgs(trf.parser) addCosmicsTrfArgs(trf.parser) addBasicDigiArgs(trf.parser) addSim_tfArgs(trf.parser) addForwardDetTrfArgs(trf.parser) addPileUpTrfArgs(trf.parser) addCommonSimDigTrfArgs(trf.parser) addTrackRecordArgs(trf.parser) return trf
def getTransform(): executorSet = set() executorSet.add( athenaExecutor( name='FTKSimulationMerge', skeletonFile='TrigFTKSim/skeleton.FTKStandaloneMerge.py', inData=['NTUP_FTKTMP'], disableMP=True, outData=['NTUP_FTK', 'RDO_FTK'], )) executorSet.add( athenaExecutor( name='FTKSimulationRDOMerge', skeletonFile='TrigFTKSim/skeleton.FTKStandaloneMerge.py', disableMP=True, inData=[('NTUP_FTKTMP', 'RDO')], outData=['RDO_FTK'], )) executorSet.add( athenaExecutor( name='FTKRecoRDOtoESD', skeletonFile='RecJobTransforms/skeleton.RAWtoESD.py', substep='r2e', inData=[('RDO', 'NTUP_FTK')], outData=['DESD_FTK'], perfMonFile='ntuple_RAWtoESD.pmon.gz', )) # extraRunargs = {'preInclude': ['TrigFTKSim/FTKReco_jobOptions.py']})) executorSet.add( athenaExecutor( name='FTKRecoESDtoNTUP', skeletonFile='PyJobTransforms/skeleton.ESDtoDPD.py', substep='e2d', inData=['DESD_FTK'], outData=['NTUP_TRIG'], perfMonFile='ntuple_ESDtoDPD.pmon.gz', )) # extraRunargs = {'preInclude': ['TrigFTKSim/FTKReco_jobOptions.py']})) trf = transform(executor=executorSet, description='FTK full region merge and reco.') addAthenaArguments(trf.parser, maxEventsDefaultSubstep='all') addDetectorArguments(trf.parser) addCommonRecTrfArgs(trf.parser) addFTKSimulationArgs(trf.parser) addD3PDArguments(trf.parser, pick=['NTUP_TRIG'], transform=trf) addBasicDigiArgs(trf.parser) return trf
def getTransform(): trf = transform(executor = athenaExecutor(name = 'FilterHitTf', substep="filthits", skeletonFile = 'SimuJobTransforms/skeleton.FilterHit.py', tryDropAndReload = False)) addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) addMyArgs(trf.parser) return trf
def addStandardHITSMergeSubstep(executorSet): executorSet.add( athenaExecutor(name='HITSMerge', substep="hitsmerge", skeletonFile='SimuJobTransforms/skeleton.HITSMerge.py', tryDropAndReload=False, inputDataTypeCountCheck=['HITS']))
def getTransform(): trf = transform(executor = athenaExecutor(name = 'RAWtoESD', skeletonFile = 'RecJobTransforms/skeleton.RAWtoESD_tf.py', substep = 'r2e', )) addAthenaArguments(trf.parser) addMyArgs(trf.parser) return trf
def getTransform(): # Get the base transform with all arguments added trf = transform(executor = athenaExecutor(name = 'FTKPattGenRoot', skeletonFile = 'TrigFTKBankGen/skeleton.FTKPattBankGenRoot.py')) addAthenaArguments(trf.parser) addFTKPattGenArgs(trf.parser) return trf
def getTransform(): executorSet = set() for subregion in range(subregions): executorSet.add( athenaExecutor( name='FTKFullSimulationBank{0}'.format(subregion), skeletonFile='TrigFTKSim/skeleton.FTKStandaloneSim.py', inData=['NTUP_FTKIP', 'TXT_FTKIP'], outData=['NTUP_FTKTMP'], extraRunargs={'banksubregion': [subregion]}, # Need to ensure that the correct subregion is used runtimeRunargs={ 'patternbankpath': 'runArgs.patternbank{0}path'.format(subregion), 'outputNTUP_FTKTMPFile': 'runArgs.outputNTUP_FTKFile', 'cachedbankpath': 'runArgs.cachedbank{0}path'.format(subregion), 'CachePath': 'runArgs.CachePath{0}'.format(subregion) })) trf = transform( executor=executorSet, description='FTK Subregion simulate x {0} .'.format(subregions)) addFTKSimulationArgs(trf.parser) addTrigFTKSimOptions(trf.parser, nsubregions=subregions) addTrigFTKSimMergeOptions(trf.parser) addTrigFTKSimTFOptions(trf.parser) addTrigFTKSimRFOptions(trf.parser) return trf
def getTransform(): executorSet = set() from EventIndexProducer.EITransformUtils import addEI_MRG_Substep, addEI_MRG_arguments executorSet.add( hybridPOOLMergeExecutor( name='AODMerge', skeletonFile='RecJobTransforms/skeleton.MergePool_tf.py', inData=['AOD'], outData=['AOD_MRG'], perfMonFile='ntuple_POOLMerge.pmon.gz')) executorSet.add( athenaExecutor( name='AODtoTAG', skeletonFile='RecJobTransforms/skeleton.AODtoTAG_tf.py', inData=['AOD_MRG'], outData=['TAG'], )) addEI_MRG_Substep(executorSet) trf = transform(executor=executorSet) addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) addCommonRecTrfArgs(trf.parser) addMyArgs(trf.parser) addEI_MRG_arguments(trf.parser) return trf
def addDigiValidationSubstep(executorSet): executorSet.add( athenaExecutor( name='DigiValidation', skeletonFile='SimuJobTransforms/skeleton.RDOtoHIST_DIGI.py', inData=['RDO'], outData=['HIST_DIGI'], ))
def addAtlasG4Substep(executorSet): executorSet.add( athenaExecutor( name='AtlasG4TfTRIn', skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_MC12.py', substep='simTRIn', tryDropAndReload=False, inData=['EVNT_TR'], outData=['HITS', 'NULL'])) executorSet.add( athenaExecutor( name='AtlasG4Tf', skeletonFile='SimuJobTransforms/skeleton.EVGENtoHIT_MC12.py', substep='sim', tryDropAndReload=False, inData=['NULL', 'EVNT'], outData=['EVNT_TR', 'HITS', 'NULL']))
def getTransform(): trf = transform(executor=athenaExecutor( name='FTKSimulationMerge', disableMP=True, skeletonFile='TrigFTKSim/skeleton.FTKStandaloneMerge.py')) addAthenaArguments(trf.parser) addFTKMergerArgs(trf.parser) return trf
def addPhysValidationSubstep(executorSet): executorSet.add( athenaExecutor( name='PhysicsValidation', skeletonFile='PATJobTransforms/skeleton.PhysicsValidation_tf.py', inData=['AOD'], outData=['NTUP_PHYSVAL'], substep='pval'))
def addSimValidationSubstep(executorSet): executorSet.add( athenaExecutor( name='SimValidation', skeletonFile='SimuJobTransforms/skeleton.HITStoHIST_SIM.py', inData=['HITS'], outData=['HIST_SIM'], ))
def getTransform(): trf = transform(executor=athenaExecutor( name='athena', skeletonFile=None, skeletonCA="PyJobTransforms.HelloWorldSkeleton")) addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) return trf
def addEI_MRG_Substep(executorSet): executorSet.add( athenaExecutor( name='EventIndexTf', skeletonFile='EventIndexProducer/skeleton.POOLtoEI_tf.py', substep='EI', tryDropAndReload=False, inData=['AOD_MRG'], outData=['NULL', 'EI']))
def addOverlaySubstep(executorSet, inRecoChain = False): executor = athenaExecutor(name = 'Overlay', skeletonFile = 'EventOverlayJobTransforms/skeleton.Overlay_tf.py', substep = 'overlay', tryDropAndReload = False, perfMonFile = 'ntuple.pmon.gz', inData = [('HITS', 'RDO_BKG')], outData = ['RDO', 'RDO_SGNL']) if inRecoChain: executor.inData = [] executor.outData = [] executorSet.add(executor)
def getTransform(): trf = transform(executor=athenaExecutor( name='ESDtoAOD', skeletonFile='RecJobTransforms/skeleton.ESDtoAOD_tf.py', substep='e2a')) addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) addCommonRecTrfArgs(trf.parser) addMyArgs(trf.parser) return trf
def getTransform(): executorSet = set() executorSet.add(hybridPOOLMergeExecutor(name = 'ESDMerge', skeletonFile = 'RecJobTransforms/skeleton.MergePool_tf.py', inData = ['ESD'], outData = ['ESD_MRG'])) executorSet.add(hybridPOOLMergeExecutor(name = 'AODMerge', skeletonFile = 'RecJobTransforms/skeleton.MergePool_tf.py', inData = ['AOD'], outData = ['AOD_MRG'])) executorSet.add(athenaExecutor(name = 'AODtoTAG', skeletonFile = 'RecJobTransforms/skeleton.AODtoTAG_tf.py', inData = ['AOD_MRG'], outData = ['TAG'],)) executorSet.add(tagMergeExecutor(name = 'TAGFileMerge', exe = 'CollAppend', inData = set(['TAG']), outData = set(['TAG_MRG']))) executorSet.add(DQMergeExecutor(name = 'DQHistogramMerge', inData = [('HIST_ESD', 'HIST_AOD'), 'HIST'], outData = ['HIST_MRG'])) executorSet.add(athenaExecutor(name = 'RDOMerge', skeletonFile = 'RecJobTransforms/skeleton.MergeRDO_tf.py', inData = ['RDO'], outData = ['RDO_MRG'])) executorSet.add(bsMergeExecutor(name = 'RAWFileMerge', exe = 'file_merging', inData = set(['BS']), outData = set(['BS_MRG']))) executorSet.add(athenaExecutor(name = 'EVNTMerge', skeletonFile = 'PyJobTransforms/skeleton.EVNTMerge.py',inData = ['EVNT'], outData = ['EVNT_MRG'])) addDAODMergerSubsteps(executorSet) addNTUPMergeSubsteps(executorSet) trf = transform(executor = executorSet) addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) addCommonRecTrfArgs(trf.parser) addMyArgs(trf.parser) addDAODArguments(trf.parser) addPhysValidationMergeFiles(trf.parser) addD3PDArguments(trf.parser, transform=trf, addD3PDMRGtypes=True) addExtraDPDTypes(trf.parser, transform=trf, NTUPMergerArgs = True) # Add HITSMerge only if SimuJobTransforms is available try: from SimuJobTransforms.SimTransformUtils import addHITSMergeArguments addHITSMergeArguments(trf.parser) simStepSet = set() simStepSet.add(athenaExecutor(name = 'HITSMerge', substep="hitsmerge", skeletonFile = 'SimuJobTransforms/skeleton.HITSMerge.py', tryDropAndReload = False, inData = ['HITS'], outData = ['HITS_MRG'])) trf.appendToExecutorSet(list(simStepSet)[0]) except ImportError as e: msg.warning('Failed to import simulation arguments ({0}). HITSMerge will not be available.'.format(e)) return trf
def execute(self): if 'maxFilesPerSubjob' in self.conf.argdict: self._hasExecuted = True msg.info('Activating job splitting with {0} files per subjob'.format(self.conf.argdict['maxFilesPerSubjob'].value)) tmpFiles = self.conf.argdict['inputZeroBiasBSFile'].value self._subJobInputs = [] while len(tmpFiles) > 0: self._subJobInputs.append(tmpFiles[0:self.conf.argdict['maxFilesPerSubjob'].value-1]) del tmpFiles[0:self.conf.argdict['maxFilesPerSubjob'].value-1] #### TESTING TESTING! # self._subJobInputs = self._subJobInputs[0:2] #### msg.info('Job will be split into {0} pieces: {1}'.format(len(self._subJobInputs), self._subJobInputs)) # Now loop over each subjob from PyJobTransforms.trfExe import athenaExecutor, executorConfig, bsMergeExecutor jobCounter = 0 outputBSFileList = [] for sjInputs in self._subJobInputs: sjobArgdict = {'triggerBit': self.conf.argdict['triggerBit'], 'InputLbnMapFile': self.conf.argdict['InputLbnMapFile'], 'checkEventCount': trfArgClasses.argSubstepBool('False', runarg=False)} sjOutput = self.conf.argdict['outputBS_SKIMFile'].value[0] + '_sub{0:03d}.tmp'.format(jobCounter) outputBSFileList.append(sjOutput) sjDataDictionary = {'BS': trfArgClasses.argBSFile(sjInputs, io='input', type='BS', subtype='BS'), 'BS_SKIM': trfArgClasses.argBSFile(sjOutput, io='output', type='BS', subtype='BS_SKIM')} sjconf = executorConfig(sjobArgdict, sjDataDictionary) sjexe = athenaExecutor(name='BSOverlayFilter_{0:03d}'.format(jobCounter), skeletonFile = 'EventOverlayJobTransforms/skeleton.BSOverlayFilter_tf.py', conf=sjconf, inData=set(['BS']), outData=set(['BS_SKIM'])) sjexe.doAll(input=set(['BS']), output=set(['BS_SKIM'])) jobCounter += 1 # Now do the merging mrgArgdict = {'maskEmptyInputs': trfArgClasses.argBool(True, runarg=False), 'allowRename': trfArgClasses.argBool(True, runarg=False)} mrgDataDictionary = {'BS': trfArgClasses.argBSFile(outputBSFileList, io='input', type='BS', subtype='BS_SKIM'), 'BS_MRG': self.conf.argdict['outputBS_SKIMFile']} mrgConf = executorConfig(mrgArgdict, mrgDataDictionary) mrgExe = bsMergeExecutor(name = 'RAWFileMerge', exe = 'file_merging', conf=mrgConf, inData = set(['BS']), outData = set(['BS_MRG'])) mrgExe.doAll(input=set(['BS']), output=set(['BS_MRG'])) # If we got here then all must be well, so we have executed successfully self._rc = 0 else: # Ordinary run super(BSJobSplitterExecutor, self).execute()
def addConfigurableSimSubstep(executorSet, confName, extraSkeleton, confSubStep, confInData, confOutData, confExtraRunargs, confRuntimeRunargs): executorSet.add( athenaExecutor(name=confName, skeletonFile=extraSkeleton + ['SimuJobTransforms/skeleton.EVGENtoHIT_MC12.py'], substep=confSubStep, tryDropAndReload=False, inData=confInData, outData=confOutData, extraRunargs=confExtraRunargs, runtimeRunargs=confRuntimeRunargs))
def getTransform(): executorSet = set() executorSet.add( athenaExecutor( name='EvtGenFromEVNT', skeletonFile='EvgenJobTransforms/skeleton.EvtGenFromEVNT.py', inData=['EVNT'], outData=['EVNT_MRG'])) trf = transform(executor=executorSet) addAthenaArguments(trf.parser) addMyArgs(trf.parser) return trf
def getTransform(): executorSet = set() print type(executorSet) executorSet.add( athenaExecutor(name='EVNTMerge', skeletonFile='PyJobTransforms/skeleton.EVNTMerge.py', inData=['EVNT'], outData=['EVNT_MRG'])) trf = transform(executor=executorSet) addAthenaArguments(trf.parser) addMyArgs(trf.parser) return trf
def getTransform(): exeSet = set() exeSet.add(EvgenExecutor("EvgenJobTransforms/skeleton.ABtoEVGEN.py")) exeSet.add( athenaExecutor(name="AODtoDPD", skeletonFile="PATJobTransforms/skeleton.AODtoDPD_tf.py", substep="a2d", inData=["EVNT"], outData=["NTUP_TRUTH"], perfMonFile="ntuple_AODtoDPD.pmon.gz")) trf = transform(executor=exeSet) addAthenaArguments(trf.parser) addStdEvgenArgs(trf.parser) return trf
def get_transform(): from PyJobTransforms.transform import transform from PyJobTransforms.trfExe import athenaExecutor trf = transform(executor=athenaExecutor( name='RDOtoBS', skeletonFile='TrigSimTransforms/skeleton.RDOtoBS.py', substep='r2b', )) from PyJobTransforms.trfArgs import addAthenaArguments addAthenaArguments(trf.parser) add_RDOtoBS_args(trf.parser) return trf
def addTrigSimSubsteps(executorSet): executorSet.add( athenaExecutor(name='RDOtoBSOLD', skeletonFile='TrigSimTransforms/skeleton.RDOtoBS.py', substep='r2bo', inData=['RDO'], outData=['BS_OLD'])) executorSet.add( athenaExecutorJO( name='BStoTRIGBS', skeletonFile='TrigSimTransforms/skeleton.BStoTRIGBS.py', substep='b2tb', inData=['BS_OLD'], outData=['BS_TRIG_OLD'])) executorSet.add( athenaExecutor( name='TRIGBStoRDO', skeletonFile='TrigSimTransforms/skeleton.TRIGBStoRDO.py', substep='tb2tr', inData=[('BS_TRIG_OLD', 'RDO')], outData=['RDO_TRIG_OLD']))
def getTransform(): executorSet = set() executorSet.add(athenaExecutor(name = 'FTKSimulationMergeDigi', skeletonFile = 'TrigFTKSim/skeleton.FTKStandaloneMergeDigi.py', inData = ['NTUP_FTKTMP'],disableMP=True, outData = ['RDO_FTK'],)) trf = transform(executor = executorSet, description = 'FTK full region merge and reco.') addAthenaArguments(trf.parser, maxEventsDefaultSubstep='all') addDetectorArguments(trf.parser) addCommonRecTrfArgs(trf.parser) addFTKSimulationArgs(trf.parser) addD3PDArguments(trf.parser, pick=['NTUP_TRIG'], transform=trf) addBasicDigiArgs(trf.parser) return trf
def addOverlaySubstep(executor_set, in_reco_chain=False): executor = athenaExecutor( name='Overlay', skeletonFile='OverlayConfiguration/skeleton_LegacyOverlay.py', skeletonCA='OverlayConfiguration.OverlaySkeleton', substep='overlay', tryDropAndReload=False, perfMonFile='ntuple.pmon.gz', inData=['RDO_BKG', 'BS_SKIM', 'HITS'], outData=['RDO', 'RDO_SGNL']) if in_reco_chain: executor.inData = [] executor.outData = [] executor_set.add(executor)
def getTransform(): executorSet = set() executorSet.add( athenaExecutor(name='RDOMerge', skeletonFile='RecJobTransforms/skeleton.MergeRDO_tf.py', inData=['RDO'], outData=['RDO_MRG'])) trf = transform(executor=executorSet) addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) addForwardDetTrfArgs(trf.parser) addCommonRecTrfArgs(trf.parser) addMyArgs(trf.parser) return trf