def filterBS(stream_name):
'''Extract ByteStream data for a given stream from a file with multiple streams'''
filterStep = ExecStep.ExecStep('FilterBS_' + stream_name)
filterStep.type = 'other'
filterStep.executable = 'trigbs_extractStream.py'
filterStep.input = ''
filterStep.args = '-s ' + stream_name + ' ' + findFile(
'*_HLTMPPy_output.*.data')
return filterStep
def decodeBS(stream_name):
'''Deserialise HLT data from ByteStream and save to an ESD file'''
decodeStep = ExecStep.ExecStep('DecodeBS_' + stream_name)
decodeStep.type = 'athena'
decodeStep.job_options = 'TriggerJobOpts/decodeBS.py'
decodeStep.input = ''
decodeStep.explicit_input = True
decodeStep.args = '--filesInput=' + findFile('*' + stream_name +
'*._athenaHLT*.data')
decodeStep.perfmon = False # no need to run PerfMon for this step
return decodeStep
def make_exec_steps(nforks, nthreads, nslots):
id_str = '{:d}f_{:d}s_{:d}t'.format(nforks, nthreads, nslots)
# Step dumping athenaHLT config
ex = ExecStep.ExecStep('Config.' + id_str)
ex.type = 'athenaHLT'
ex.job_options = 'TriggerJobOpts/runHLT_standalone.py'
ex.input = 'data'
ex.forks = nforks
ex.threads = nthreads
ex.concurrent_events = nslots
ex.perfmon = False
ex.prmon = False
ex.args = '--dump-config-exit'
ex.args += ' -c "setMenu=\'LS2_v1\';"' # LS2_v1 soon to be renamed to Dev_pp_run3_v1
# Step renaming the config dump
rename = ExecStep.ExecStep('RenameConfigDump.' + id_str)
rename.type = 'other'
rename.input = ''
rename.executable = 'mv'
rename.args = 'HLTJobOptions.json HLTJobOptions.' + id_str + '.json'
rename.output_stream = Step.Step.OutputStream.STDOUT_ONLY # no log file for this step
rename.auto_report_result = False # no art-result for this step
# Step comparing the config to reference (hard-coded name with 1 fork, 1 thread, 1 slot)
refname = 'HLTJobOptions.1f_1s_1t.json'
testname = 'HLTJobOptions.' + id_str + '.json'
compare = ExecStep.ExecStep('CompareConfig.' + id_str)
compare.type = 'other'
compare.input = ''
compare.executable = '(diff'
compare.args = testname + ' ' + refname
msg_base = 'Configuration with forks/threads/slots = {:d}/{:d}/{:d}'.format(
nforks, nthreads, nslots)
msg_error = 'ERROR ' + msg_base + ' differs from configuration with 1/1/1'
msg_success = 'SUCCESS ' + msg_base + ' is the same as with 1/1/1'
compare.args += '; ret=$?; if [ $ret -ne 0 ]; then echo "{:s}"; else echo "{:s}"; fi; exit $ret)'.format(
msg_error, msg_success)
return [ex, rename, compare]
def generate_steps(slice_name=None):
name = slice_name or 'FullMenu'
# athena
ex = ExecStep.ExecStep(name)
ex.type = 'athena'
ex.job_options = 'TriggerJobOpts/runHLT_standalone.py'
ex.input = 'ttbar'
ex.threads = 1
hist_file_name = 'onlinemon_{:s}.root'.format(name)
pre_exec = hist_rename_pre_exec(hist_file_name)
if slice_name:
pre_exec += 'doEmptyMenu=True;do{:s}Slice=True;'.format(slice_name)
ex.args = '-c "setMenu=\'LS2_v1\';doWriteBS=False;doWriteRDOTrigger=False;{:s}"'.format(
pre_exec)
# chainDump
cd = ExecStep.ExecStep('ChainDump' + name)
cd.type = 'other'
cd.executable = 'chainDump.py'
cd.input = ''
cd.args = '-f {:s} --json ChainDump.{:s}.json'.format(hist_file_name, name)
cd.auto_report_result = False
return [ex, cd]
# art-output: *.json
# art-output: *.root
# art-output: *.pmon.gz
# art-output: *perfmon*
# art-output: prmon*
# art-output: *.check*
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
from TrigAnalysisTest.TrigAnalysisSteps import add_analysis_steps
import os
# To run single-process transform on MCORE sites
if 'ATHENA_NPROC_NUM' in os.environ:
del os.environ['ATHENA_NPROC_NUM']
rdo2aod = ExecStep.ExecStep()
rdo2aod.type = 'Reco_tf'
rdo2aod.input = 'ttbar'
rdo2aod.max_events = 500
rdo2aod.threads = 1
rdo2aod.args = '--outputAODFile=AOD.pool.root --steering="doRDO_TRIG"'
rdo2aod.args += ' --preExec="all:from TriggerJobOpts.TriggerFlags import TriggerFlags; TriggerFlags.AODEDMSet.set_Value_and_Lock(\\\"AODFULL\\\");"'
test = Test.Test()
test.art_type = 'grid'
test.exec_steps = [rdo2aod]
test.check_steps = CheckSteps.default_check_steps(test)
add_analysis_steps(test)
import sys
sys.exit(test.run())
#!/usr/bin/env python
# art-description: Test of transform HITS->RDO with serial athena followed by RDO->RDO_TRIG with threads=1
# art-type: build
# art-include: master/Athena
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps, Input
from TrigAnalysisTest.TrigAnalysisSteps import add_analysis_steps
hit2rdo = ExecStep.ExecStep('HITtoRDO')
hit2rdo.type = 'Reco_tf'
hit2rdo.input = 'ttbar_HITS'
hit2rdo.args = '--outputRDOFile=RDO.pool.root'
pu_low = Input.get_input('pileup_low')
pu_high = Input.get_input('pileup_high')
hit2rdo.args += ' --inputLowPtMinbiasHitsFile=' + pu_low.paths[0]
hit2rdo.args += ' --inputHighPtMinbiasHitsFile=' + pu_high.paths[0]
hit2rdo.args += ' --numberOfCavernBkg="0"'
hit2rdo.args += ' --numberOfHighPtMinBias="0.15520183"'
hit2rdo.args += ' --numberOfLowPtMinBias="59.3447981771"'
hit2rdo.args += ' --pileupFinalBunch="6"'
hit2rdo.args += ' --jobNumber="1"'
hit2rdo.args += ' --preExec "HITtoRDO:userRunLumiOverride={\'run\':300000, \'startmu\':40.0, \'endmu\':70.0, \'stepmu\':1.0, \'startlb\':1, \'timestamp\': 1500000000};ScaleTaskLength=0.1"'
hit2rdo.args += ' --preInclude "HITtoRDO:Digitization/ForceUseOfPileUpTools.py,SimulationJobOptions/preInlcude.PileUpBunchTrainsMC16c_2017_Config1.py,RunDependentSimData/configLumi_muRange.py"'
rdo2rdotrig = ExecStep.ExecStep('RDOtoRDOTrigger')
rdo2rdotrig.type = 'Reco_tf'
# art-output: *.check*
# art-output: HLTconfig*.xml
# art-output: L1Topoconfig*.xml
# art-output: LVL1config*.xml
# art-output: PHYSVAL_WEB
# art-html: PHYSVAL_WEB
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
from TrigAnalysisTest.TrigAnalysisSteps import add_physvalweb_steps
import os
# To run single-process transform on MCORE sites
if 'ATHENA_NPROC_NUM' in os.environ:
del os.environ['ATHENA_NPROC_NUM']
rdo2aod = ExecStep.ExecStep('RDOtoAOD')
rdo2aod.type = 'Reco_tf'
rdo2aod.input = 'ttbar'
rdo2aod.threads = 1
rdo2aod.max_events = 500
rdo2aod.args = '--outputAODFile=AOD.pool.root --steering="doRDO_TRIG" --valid=True'
rdo2aod.args += ' --preExec="all:from TriggerJobOpts.TriggerFlags import TriggerFlags; TriggerFlags.AODEDMSet.set_Value_and_Lock(\\\"AODFULL\\\");"'
physval = ExecStep.ExecStep('PhysVal')
physval.type = 'Reco_tf'
physval.input = ''
physval.explicit_input = True
physval.args = '--inputAODFile=AOD.pool.root --outputNTUP_PHYSVALFile=NTUP_PHYSVAL.pool.root --valid=True'
validationFlags = 'doTrigEgamma,doTrigBphys,doTrigMET,doTrigJet,doTrigMuon,doTrigHLTResult,doTrigCalo,doTrigMinBias,doTrigTau,doTrigIDtrk,doTrigBjet'
physval.args += ' --validationFlags="{:s}"'.format(validationFlags)
#!/usr/bin/env python
# art-description: Test running HITS->RDO in master, then RDO->RDO_TRIG in 21.0, then RDO_TRIG->AOD in master
# art-type: build
# art-include: master/Athena
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps, Input
from TrigAnalysisTest.TrigAnalysisSteps import add_analysis_steps
# HITS -> RDO step in master
hit2rdo = ExecStep.ExecStep('HITtoRDO')
hit2rdo.type = 'Reco_tf'
hit2rdo.input = 'ttbar_HITS'
hit2rdo.args = '--outputRDOFile=RDO.pool.root'
pu_low = Input.get_input('pileup_low')
pu_high = Input.get_input('pileup_high')
hit2rdo.args += ' --inputLowPtMinbiasHitsFile=' + pu_low.paths[0]
hit2rdo.args += ' --inputHighPtMinbiasHitsFile=' + pu_high.paths[0]
hit2rdo.args += ' --numberOfCavernBkg="0"'
hit2rdo.args += ' --numberOfHighPtMinBias="0.15520183"'
hit2rdo.args += ' --numberOfLowPtMinBias="59.3447981771"'
hit2rdo.args += ' --pileupFinalBunch="6"'
hit2rdo.args += ' --jobNumber="1"'
hit2rdo.args += ' --preExec "HITtoRDO:userRunLumiOverride={\'run\':300000, \'startmu\':40.0, \'endmu\':70.0, \'stepmu\':1.0, \'startlb\':1, \'timestamp\': 1500000000};ScaleTaskLength=0.1" '
hit2rdo.args += ' --preInclude "HITtoRDO:Digitization/ForceUseOfPileUpTools.py,SimulationJobOptions/preInlcude.PileUpBunchTrainsMC16c_2017_Config1.py,RunDependentSimData/configLumi_muRange.py"'
#!/usr/bin/env python
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
#
# art-description: athenaHLT test of the Dev_pp_run3_v1 menu imitating partition with preloaded data at P1
# art-type: build
# art-include: master/Athena
from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep
from TrigP1Test import TrigP1TestSteps
# Input file
f = str(Input.get_input('data').paths[0])
# Delete any previous bytestream file
ex_rm = ExecStep.ExecStep('cleanup')
ex_rm.type = 'other'
ex_rm.input = ''
ex_rm.executable = 'rm'
ex_rm.args = '-f raw._0001.data'
ex_rm.auto_report_result = False # Do not set art-result for this step
ex_rm.output_stream = Step.Step.OutputStream.STDOUT_ONLY # Do not create a log file for this step
# Create new bytestream file with future run number and renumber LBs
ex_bs = ExecStep.ExecStep('create_bs')
ex_bs.type = 'other'
ex_bs.input = ''
ex_bs.executable = 'trigbs_modifyEvent.py'
ex_bs.args = '-n 50 --runNumber 999999 --incLB 6 --firstLB 4 --eventsPerLB=3 -o raw %s' % f
# Run athenaHLT forcing run and timestamp of original data file
import eformat
# art-output: *.json
# art-output: *.root
# art-output: *.pmon.gz
# art-output: *perfmon*
# art-output: prmon*
# art-output: *.check*
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
def findFile(pattern):
return '`find . -name \'{:s}\' | tail -n 1`'.format(pattern)
# HLT step (BS->BS)
hlt = ExecStep.ExecStep()
hlt.type = 'athenaHLT'
hlt.job_options = 'TriggerJobOpts/runHLT_standalone.py'
hlt.input = 'data'
hlt.args = '-c "setMenu=\'PhysicsP1_pp_run3_v1\';"'
hlt.args += ' -o output'
# Extract the physics_Main stream out of the BS file with many streams
filter_bs = ExecStep.ExecStep('FilterBS')
filter_bs.type = 'other'
filter_bs.executable = 'trigbs_extractStream.py'
filter_bs.input = ''
filter_bs.args = '-s Main ' + findFile('*_HLTMPPy_output.*.data')
# Tier-0 reco step (BS->ESD->AOD)
tzrecoPreExec = ' '.join([
#!/usr/bin/env python
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# art-description: Trigger AthenaMT test running new-style job options
# art-type: build
# art-include: master/Athena
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
from TrigValTools.TrigValSteering import Test, Step, ExecStep, CheckSteps
# Copy the job options to the working directory
copy_jo = ExecStep.ExecStep('CopyJO')
copy_jo.type = 'other'
copy_jo.input = ''
copy_jo.executable = 'get_files'
copy_jo.args = '-remove -jo TriggerJobOpts/runHLT_standalone_newJO.py'
copy_jo.prmon = False
copy_jo.auto_report_result = False # Do not set art-result for this step
copy_jo.output_stream = Step.Step.OutputStream.STDOUT_ONLY # Do not create a log file for this step
# Generate configuration pickle file
pickle = ExecStep.ExecStep('GeneratePickle')
pickle.type = 'other'
pickle.input = ''
pickle.executable = 'python'
pickle.args = 'runHLT_standalone_newJO.py'
pickle.prmon = False
# The main job running athena from pickle
run_athena = ExecStep.ExecStep('newJO')
#!/usr/bin/env python
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# art-description: transform test of BSRDOtoRAW + T0Reco + T0Mon, using v1PhysP1 menu
# art-type: build
# art-include: master/Athena
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
#====================================================================================================
# HLT BS_RDO->RAW
hlt = ExecStep.ExecStep('BSRDOtoRAW')
hlt.type = 'Trig_reco_tf'
hlt.forks = 1
hlt.threads = 1
hlt.concurrent_events = 1
hlt.max_events = 50
hlt.args = '--precommand=\\\"setMenu=\\\'PhysicsP1_pp_run3_v1\\\'\\\"'
hlt.args += ' --prodSysBSRDO True'
hlt.args += ' --outputBSFile=RAW.pool.root'
hlt.args += ' --outputHIST_HLTMONFile=hltmon.root'
hlt.input = 'data'
#====================================================================================================
# Tier-0 reco step BS->ESD->AOD
tzrecoPreExec = ' '.join([
"from AthenaConfiguration.AllConfigFlags import ConfigFlags;",
"ConfigFlags.Trigger.triggerMenuSetup=\'PhysicsP1_pp_run3_v1\';",
"from TriggerJobOpts.TriggerFlags import TriggerFlags;",
"TriggerFlags.configForStartup=\'HLToffline\';",
"TriggerFlags.inputHLTconfigFile.set_Value_and_Lock(\'NONE\');",
# art-type: build
# art-include: master/Athena
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
from TrigAnalysisTest.TrigAnalysisSteps import add_analysis_steps
def findFile(pattern):
'''Bash inline file name finder'''
return '`find . -name \'{:s}\' | tail -n 1`'.format(pattern)
menu_name = 'LS2_v1'
# Run athena BS->BS job
BStoBS = ExecStep.ExecStep("BStoBS")
BStoBS.type = 'athena'
BStoBS.job_options = 'TriggerJobOpts/runHLT_standalone.py'
BStoBS.input = 'data'
BStoBS.threads = 1
BStoBSPreExec = ''.join([
'setMenu=\'{:s}\';'.format(menu_name),
'rewriteLVL1=True;', # Write L1 result to BS
'doWriteBS=True;', # Write HLT result to BS
'forceEnableAllChains=True;', # Ignore L1 decisions in seeding chains
])
BStoBS.args = ' -c "{:s}"'.format(BStoBSPreExec)
# Extract the Main stream data
filterBS = ExecStep.ExecStep('FilterBS')
filterBS.type = 'other'
decodeStep.type = 'athena'
decodeStep.job_options = 'TriggerJobOpts/decodeBS.py'
decodeStep.input = ''
decodeStep.explicit_input = True
decodeStep.args = '--filesInput=' + findFile('*' + stream_name +
'*._athenaHLT*.data')
decodeStep.perfmon = False # no need to run PerfMon for this step
return decodeStep
##################################################
# Test definition
##################################################
# Run standard athenaHLT BS->BS job
writeBS = ExecStep.ExecStep("WriteBS")
writeBS.type = 'athenaHLT'
writeBS.job_options = 'TriggerJobOpts/runHLT_standalone.py'
writeBS.input = 'data'
writeBS.args = '-o output'
writeBS.args += ' -c "setMenu=\'LS2_v1\';"' # LS2_v1 to be renamed to Dev_pp_run3_v1
# Extract and decode physics_Main
filterMain = filterBS("Main")
decodeMain = decodeBS("Main")
# Extract and decode calibration_CostMonitoring
filterCost = filterBS("CostMonitoring")
decodeCost = decodeBS("CostMonitoring")
decodeCost.args += ' -c "ModuleID=1"'
# art-output: *.new
# art-output: *.json
# art-output: *.root
# art-output: *.pmon.gz
# art-output: *perfmon*
# art-output: prmon*
# art-output: *.check*
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
import os
# To run single-process transform on MCORE sites
if 'ATHENA_NPROC_NUM' in os.environ:
del os.environ['ATHENA_NPROC_NUM']
rdo2aod = ExecStep.ExecStep('RDOtoAOD')
rdo2aod.type = 'Reco_tf'
rdo2aod.input = 'ttbar'
rdo2aod.max_events = 500
rdo2aod.threads = 1
rdo2aod.args = '--outputAODFile=AOD.pool.root --steering="doRDO_TRIG" --valid=True'
rdo2aod.args += ' --preExec="all:from TriggerJobOpts.TriggerFlags import TriggerFlags; TriggerFlags.AODEDMSet.set_Value_and_Lock(\\\"AODFULL\\\");"'
rdo2aod.args += ' --postInclude="TriggerTest/disableChronoStatSvcPrintout.py"'
tzmon = ExecStep.ExecStep('Tier0Mon')
tzmon.type = 'other'
tzmon.executable = 'Run3DQTestingDriver.py'
tzmon.input = ''
tzmon.explicit_input = True
tzmon.args = '--dqOffByDefault Input.Files="[\'AOD.pool.root\']" DQ.Steering.doHLTMon=True'
#!/usr/bin/env python
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# art-description: Trigger BS->RDO_TRIG athena test of the Dev_pp_run3_v1 menu
# art-type: build
# art-include: master/Athena
# Skipping art-output which has no effect for build tests.
# If you create a grid version, check art-output in existing grid tests.
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps
ex = ExecStep.ExecStep()
ex.type = 'athena'
ex.job_options = 'TriggerJobOpts/runHLT_standalone.py'
ex.input = 'data'
ex.threads = 1
precommand = ''.join([
"setMenu='LS2_v1';", # LS2_v1 soon to be renamed to Dev_pp_run3_v1
"doWriteBS=False;",
"doWriteRDOTrigger=True;",
"forceEnableAllChains=True;",
"fpeAuditor=True;",
"failIfNoProxy=True;"
])
ex.args = '-c "{:s}"'.format(precommand)
test = Test.Test()
test.art_type = 'build'
test.exec_steps = [ex]
test.check_steps = CheckSteps.default_check_steps(test)
#!/usr/bin/env python
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
# art-description: PhysicsP1_pp_run3_v1 menu test only dumping options for SMK generation and running RuleBook to create prescales
# art-type: build
# art-include: master/Athena
from TrigValTools.TrigValSteering import Step, Test, ExecStep, CheckSteps
#====================================================================================================
# Run Physics menu and produce files to create SMK
menu = ExecStep.ExecStep('RunPhysMenu')
menu.type = 'athenaHLT'
menu.job_options = 'TriggerJobOpts/runHLT_standalone.py'
menu.input = 'data'
menu.args = '-c "setMenu=\'PhysicsP1_pp_run3_v1\';"'
menu.args += ' -M --dump-config-exit'
menu.perfmon = False # Cannot use PerfMon with -M
#====================================================================================================
# Download and setup RuleBook and create prescales
rbsetup = ExecStep.ExecStep('SetupRulebook')
rbsetup.type = 'other'
rbsetup.executable = 'setupTrigMenuRulebook.sh'
rbsetup.input = ''
#====================================================================================================
# Run RuleBook and create prescales
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps, Common
import logging
import os
if not os.path.exists('test_unit_trigvalsteering'):
os.mkdir('test_unit_trigvalsteering')
os.chdir('test_unit_trigvalsteering')
Common.trigvalsteering_logging_level = logging.DEBUG
Common.package_prefix_dict['TrigValTools'] = 'unit_'
test = Test.Test()
for test_type in ['athena', 'athenaHLT', 'Reco_tf', 'Trig_reco_tf']:
ex = ExecStep.ExecStep('Test_' + test_type)
ex.type = test_type
if '_tf' not in test_type:
ex.job_options = 'AthExJobOptions/AthExJobOptions_MinimalJobOptionsNoEvt.py'
ex.input = ''
ex.args = '--help'
test.exec_steps.append(ex)
test.art_type = 'build'
test.check_steps = CheckSteps.default_check_steps(test)
regtest_ref_text = [
'### athena.log ###',
'### athena.Test_athena.log ###',
'### athenaHLT.Test_athenaHLT.log ###',
'### Reco_tf.Test_Reco_tf.log ###',
