def getTransform(): executorSet = set() addNTUPMergeSubsteps(executorSet) trf = transform(executor=executorSet) addPhysValidationMergeFiles(trf.parser) addD3PDArguments(trf.parser, transform=trf, addD3PDMRGtypes=True) addExtraDPDTypes(trf.parser, transform=trf, NTUPMergerArgs=True) return trf
def getTransform(): executorSet = set() executorSet.add(hybridPOOLMergeExecutor(name = 'ESDMerge', skeletonFile = 'RecJobTransforms/skeleton.MergePool_tf.py', inData = ['ESD'], outData = ['ESD_MRG'])) executorSet.add(hybridPOOLMergeExecutor(name = 'AODMerge', skeletonFile = 'RecJobTransforms/skeleton.MergePool_tf.py', inData = ['AOD'], outData = ['AOD_MRG'])) executorSet.add(athenaExecutor(name = 'AODtoTAG', skeletonFile = 'RecJobTransforms/skeleton.AODtoTAG_tf.py', inData = ['AOD_MRG'], outData = ['TAG'],)) executorSet.add(tagMergeExecutor(name = 'TAGFileMerge', exe = 'CollAppend', inData = set(['TAG']), outData = set(['TAG_MRG']))) executorSet.add(DQMergeExecutor(name = 'DQHistogramMerge', inData = [('HIST_ESD', 'HIST_AOD'), 'HIST'], outData = ['HIST_MRG'])) executorSet.add(athenaExecutor(name = 'RDOMerge', skeletonFile = 'RecJobTransforms/skeleton.MergeRDO_tf.py', inData = ['RDO'], outData = ['RDO_MRG'])) executorSet.add(bsMergeExecutor(name = 'RAWFileMerge', exe = 'file_merging', inData = set(['BS']), outData = set(['BS_MRG']))) executorSet.add(athenaExecutor(name = 'EVNTMerge', skeletonFile = 'PyJobTransforms/skeleton.EVNTMerge.py',inData = ['EVNT'], outData = ['EVNT_MRG'])) addDAODMergerSubsteps(executorSet) addNTUPMergeSubsteps(executorSet) trf = transform(executor = executorSet) addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) addCommonRecTrfArgs(trf.parser) addMyArgs(trf.parser) addDAODArguments(trf.parser) addPhysValidationMergeFiles(trf.parser) addD3PDArguments(trf.parser, transform=trf, addD3PDMRGtypes=True) addExtraDPDTypes(trf.parser, transform=trf, NTUPMergerArgs = True) # Add HITSMerge only if SimuJobTransforms is available try: from SimuJobTransforms.SimTransformUtils import addHITSMergeArguments addHITSMergeArguments(trf.parser) simStepSet = set() simStepSet.add(athenaExecutor(name = 'HITSMerge', substep="hitsmerge", skeletonFile = 'SimuJobTransforms/skeleton.HITSMerge.py', tryDropAndReload = False, inData = ['HITS'], outData = ['HITS_MRG'])) trf.appendToExecutorSet(list(simStepSet)[0]) except ImportError as e: msg.warning('Failed to import simulation arguments ({0}). HITSMerge will not be available.'.format(e)) return trf
def getTransform(): executorSet = set() #BSRDOtoRAW is new option for trigger transform #runs primarily using athenaHLT #literalRunargs used for when running with athena executorSet.add(trigRecoExecutor(name = 'BSRDOtoRAW', skeletonFile = 'TriggerJobOpts/runHLT_standalone.py', exe = 'setsid athenaHLT.py', # setsid is needed to fix the process-group id of child processes to be the same as mother process; discussed in https://its.cern.ch/jira/browse/ATR-20513 substep = 'b2r', tryDropAndReload = False, inData = ['BS_RDO', 'RDO'], outData = ['BS', 'BS_TRIGCOST', 'HIST_HLTMON','HIST_DEBUGSTREAMMON'], perfMonFile = 'ntuple_BSRDOtoRAW.pmon.gz', literalRunargs = ['writeBS = runArgs.writeBS', 'from AthenaCommon.AthenaCommonFlags import jobproperties as jps', 'jps.AthenaCommonFlags.FilesInput.set_Value_and_Lock(runArgs.inputBS_RDOFile)', 'jps.AthenaCommonFlags.EvtMax.set_Value_and_Lock(runArgs.maxEvents)', 'if hasattr(runArgs,\'skipEvents\'):', ' jps.AthenaCommonFlags.SkipEvents.set_Value_and_Lock(runArgs.skipEvents)', 'if hasattr(runArgs,\'outputBSFile\'):', ' jps.AthenaCommonFlags.BSRDOOutput.set_Value_and_Lock( runArgs.outputBSFile )'])) #RAWtoCOST is new option for trigger transform #runs in athena and will succeed if input BS file has costmon enabled executorSet.add(trigCostExecutor(name = 'RAWtoCOST', skeletonFile = 'TrigCostMonitor/readTrigCost.py', substep = 'r2c', inData = ['BS_TRIGCOST'], outData = ['NTUP_TRIGCOST','NTUP_TRIGRATE','NTUP_TRIGEBWGHT'], perfMonFile = 'ntuple_RAWtoCOST.pmon.gz', literalRunargs = ['BSRDOInput = runArgs.inputBSFile', 'EvtMax = runArgs.maxEvents'])) #add default reconstruction steps # eventually to be replaced by: #from RecJobTransforms.recTransformUtils import addRecoSubsteps #addRecoSubsteps(executorSet) # one difference to reco_tf is that NTUP_TRIG has to be added - needs pyjobtf update before can use above #NTUP_TRIG is added as is not available in ATLASP1HLT, but is available in the reco release #hence can be used later in a ATLASP1HLT job if switch releases executorSet.add(athenaExecutor(name = 'RAWtoESD', skeletonFile = 'RecJobTransforms/skeleton.RAWtoESD_tf.py', substep = 'r2e', inData = ['BS', 'RDO'], outData = ['ESD', 'HIST_ESD_INT'], perfMonFile = 'ntuple_RAWtoESD.pmon.gz')) executorSet.add(athenaExecutor(name = 'ESDtoAOD', skeletonFile = 'RecJobTransforms/skeleton.ESDtoAOD_tf.py', substep = 'e2a', inData = ['ESD'], outData = ['AOD', 'HIST_AOD_INT'], perfMonFile = 'ntuple_ESDtoAOD.pmon.gz')) executorSet.add(DQMergeExecutor(name = 'DQHistogramMerge', inData = [('HIST_ESD_INT', 'HIST_AOD_INT')], outData = ['HIST'])) executorSet.add(athenaExecutor(name = 'ESDtoDPD', skeletonFile = 'PATJobTransforms/skeleton.ESDtoDPD_tf.py', substep = 'e2d', inData = ['ESD'], outData = ['NTUP_TRIG'], perfMonFile = 'ntuple_ESDtoDPD.pmon.gz')) executorSet.add(athenaExecutor(name = 'AODtoDPD', skeletonFile = 'PATJobTransforms/skeleton.AODtoDPD_tf.py', substep = 'a2d', inData = ['AOD', 'EVNT'], outData = ['NTUP_TRIG'], perfMonFile = 'ntuple_AODtoDPD.pmon.gz')) executorSet.add(athenaExecutor(name = 'AODtoTAG', skeletonFile = 'RecJobTransforms/skeleton.AODtoTAG_tf.py', inData = ['AOD'], outData = ['TAG'],)) executorSet.add(reductionFrameworkExecutor(name = 'AODtoRED', skeletonFile = 'PATJobTransforms/skeleton.AODtoRED_tf.py', substep = 'a2r', inData = ['AOD'], outData = ['DAOD_RED'])) executorSet.add(reductionFrameworkExecutorNTUP(name = 'NTUPtoRED', skeletonFile = 'PATJobTransforms/skeleton.NTUPtoRED_tf.py', substep = 'n2n', inData = ['NTUP_COMMON'], outData = ['NTUP_RED'])) trf = transform(executor = executorSet, description = 'Trigger transform to run HLT_standalone, followed by' ' general purpose ATLAS reconstruction transform. Input to HLT_Standalone is inputBS_RDOFile' ' with outputs of RDO, ESD, AOD or DPDs. For more details see:' ' https://twiki.cern.ch/twiki/bin/viewauth/Atlas/TriggerTransform or for reco_tf, see:' ' https://twiki.cern.ch/twiki/bin/viewauth/Atlas/RecoTf') #add arguments as donw in reco_tf addAthenaArguments(trf.parser) addDetectorArguments(trf.parser) addCommonRecTrfArgs(trf.parser) addStandardRecoFiles(trf.parser) addPrimaryDPDArguments(trf.parser, transform = trf) addD3PDArguments(trf.parser, transform = trf) addExtraDPDTypes(trf.parser, transform = trf) addReductionArguments(trf.parser, transform = trf) #currently ignoring simulation arguments as can't find the package when have below in cmt/requirements: #use SimuJobTransforms SimuJobTransforms-* Simulation # For digi step - make sure we can add the digitisation/simulation arguments # before we add this substep # try: # from SimuJobTransforms.simTrfArgs import addForwardDetTrfArgs, addBasicDigiArgs, addPileUpTrfArgs, addCommonSimDigTrfArgs # addBasicDigiArgs(trf.parser) # addForwardDetTrfArgs(trf.parser) # addPileUpTrfArgs(trf.parser) # #addCommonSimDigTrfArgs(trf.parser) # executorSet.add(athenaExecutor(name = 'HITtoRDO', skeletonFile = 'SimuJobTransforms/skeleton.HITtoRDO.py', # substep = 'h2r', tryDropAndReload = False, perfMonFile = 'ntuple.pmon.gz', # inData = ['Hits'], outData = ['RDO'], # runtimeRunargs = {'LowPtMinbiasHitsFile' : 'runArgs.inputLowPtMinbiasHitsFile', # 'HighPtMinbiasHitsFile' : 'runArgs.inputHighPtMinbiasHitsFile',})) # # except ImportError: # msg.warning('Failed to import simulation/digitisation arguments. These substeps will not be available.') #now add specific trigger arguments # putting this last makes them appear last in the help so easier to find addTriggerArgs(trf.parser) addTrigCostArgs(trf.parser) addTriggerDBArgs(trf.parser) addDebugArgs(trf.parser) return trf
def addAllRecoArgs(trf, RAWtoALL=False): addCommonRecTrfArgs(trf.parser) addStandardRecoFiles(trf.parser) addPrimaryDPDArguments(trf.parser, transform=trf, RAWtoALL=RAWtoALL) addExtraDPDTypes(trf.parser, transform=trf) addReductionArguments(trf.parser, transform=trf)