def filterBS(stream_name): '''Extract ByteStream data for a given stream from a file with multiple streams''' filterStep = ExecStep.ExecStep('FilterBS_' + stream_name) filterStep.type = 'other' filterStep.executable = 'trigbs_extractStream.py' filterStep.input = '' filterStep.args = '-s ' + stream_name + ' ' + findFile( '*_HLTMPPy_output.*.data') return filterStep
def decodeBS(stream_name): '''Deserialise HLT data from ByteStream and save to an ESD file''' decodeStep = ExecStep.ExecStep('DecodeBS_' + stream_name) decodeStep.type = 'athena' decodeStep.job_options = 'TriggerJobOpts/decodeBS.py' decodeStep.input = '' decodeStep.explicit_input = True decodeStep.args = '--filesInput=' + findFile('*' + stream_name + '*._athenaHLT*.data') decodeStep.perfmon = False # no need to run PerfMon for this step return decodeStep
def make_exec_steps(nforks, nthreads, nslots): id_str = '{:d}f_{:d}s_{:d}t'.format(nforks, nthreads, nslots) # Step dumping athenaHLT config ex = ExecStep.ExecStep('Config.' + id_str) ex.type = 'athenaHLT' ex.job_options = 'TriggerJobOpts/runHLT_standalone.py' ex.input = 'data' ex.forks = nforks ex.threads = nthreads ex.concurrent_events = nslots ex.perfmon = False ex.prmon = False ex.args = '--dump-config-exit' ex.args += ' -c "setMenu=\'LS2_v1\';"' # LS2_v1 soon to be renamed to Dev_pp_run3_v1 # Step renaming the config dump rename = ExecStep.ExecStep('RenameConfigDump.' + id_str) rename.type = 'other' rename.input = '' rename.executable = 'mv' rename.args = 'HLTJobOptions.json HLTJobOptions.' + id_str + '.json' rename.output_stream = Step.Step.OutputStream.STDOUT_ONLY # no log file for this step rename.auto_report_result = False # no art-result for this step # Step comparing the config to reference (hard-coded name with 1 fork, 1 thread, 1 slot) refname = 'HLTJobOptions.1f_1s_1t.json' testname = 'HLTJobOptions.' + id_str + '.json' compare = ExecStep.ExecStep('CompareConfig.' + id_str) compare.type = 'other' compare.input = '' compare.executable = '(diff' compare.args = testname + ' ' + refname msg_base = 'Configuration with forks/threads/slots = {:d}/{:d}/{:d}'.format( nforks, nthreads, nslots) msg_error = 'ERROR ' + msg_base + ' differs from configuration with 1/1/1' msg_success = 'SUCCESS ' + msg_base + ' is the same as with 1/1/1' compare.args += '; ret=$?; if [ $ret -ne 0 ]; then echo "{:s}"; else echo "{:s}"; fi; exit $ret)'.format( msg_error, msg_success) return [ex, rename, compare]
def generate_steps(slice_name=None): name = slice_name or 'FullMenu' # athena ex = ExecStep.ExecStep(name) ex.type = 'athena' ex.job_options = 'TriggerJobOpts/runHLT_standalone.py' ex.input = 'ttbar' ex.threads = 1 hist_file_name = 'onlinemon_{:s}.root'.format(name) pre_exec = hist_rename_pre_exec(hist_file_name) if slice_name: pre_exec += 'doEmptyMenu=True;do{:s}Slice=True;'.format(slice_name) ex.args = '-c "setMenu=\'LS2_v1\';doWriteBS=False;doWriteRDOTrigger=False;{:s}"'.format( pre_exec) # chainDump cd = ExecStep.ExecStep('ChainDump' + name) cd.type = 'other' cd.executable = 'chainDump.py' cd.input = '' cd.args = '-f {:s} --json ChainDump.{:s}.json'.format(hist_file_name, name) cd.auto_report_result = False return [ex, cd]
# art-output: *.json # art-output: *.root # art-output: *.pmon.gz # art-output: *perfmon* # art-output: prmon* # art-output: *.check* from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps from TrigAnalysisTest.TrigAnalysisSteps import add_analysis_steps import os # To run single-process transform on MCORE sites if 'ATHENA_NPROC_NUM' in os.environ: del os.environ['ATHENA_NPROC_NUM'] rdo2aod = ExecStep.ExecStep() rdo2aod.type = 'Reco_tf' rdo2aod.input = 'ttbar' rdo2aod.max_events = 500 rdo2aod.threads = 1 rdo2aod.args = '--outputAODFile=AOD.pool.root --steering="doRDO_TRIG"' rdo2aod.args += ' --preExec="all:from TriggerJobOpts.TriggerFlags import TriggerFlags; TriggerFlags.AODEDMSet.set_Value_and_Lock(\\\"AODFULL\\\");"' test = Test.Test() test.art_type = 'grid' test.exec_steps = [rdo2aod] test.check_steps = CheckSteps.default_check_steps(test) add_analysis_steps(test) import sys sys.exit(test.run())
#!/usr/bin/env python # art-description: Test of transform HITS->RDO with serial athena followed by RDO->RDO_TRIG with threads=1 # art-type: build # art-include: master/Athena # Skipping art-output which has no effect for build tests. # If you create a grid version, check art-output in existing grid tests. from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps, Input from TrigAnalysisTest.TrigAnalysisSteps import add_analysis_steps hit2rdo = ExecStep.ExecStep('HITtoRDO') hit2rdo.type = 'Reco_tf' hit2rdo.input = 'ttbar_HITS' hit2rdo.args = '--outputRDOFile=RDO.pool.root' pu_low = Input.get_input('pileup_low') pu_high = Input.get_input('pileup_high') hit2rdo.args += ' --inputLowPtMinbiasHitsFile=' + pu_low.paths[0] hit2rdo.args += ' --inputHighPtMinbiasHitsFile=' + pu_high.paths[0] hit2rdo.args += ' --numberOfCavernBkg="0"' hit2rdo.args += ' --numberOfHighPtMinBias="0.15520183"' hit2rdo.args += ' --numberOfLowPtMinBias="59.3447981771"' hit2rdo.args += ' --pileupFinalBunch="6"' hit2rdo.args += ' --jobNumber="1"' hit2rdo.args += ' --preExec "HITtoRDO:userRunLumiOverride={\'run\':300000, \'startmu\':40.0, \'endmu\':70.0, \'stepmu\':1.0, \'startlb\':1, \'timestamp\': 1500000000};ScaleTaskLength=0.1"' hit2rdo.args += ' --preInclude "HITtoRDO:Digitization/ForceUseOfPileUpTools.py,SimulationJobOptions/preInlcude.PileUpBunchTrainsMC16c_2017_Config1.py,RunDependentSimData/configLumi_muRange.py"' rdo2rdotrig = ExecStep.ExecStep('RDOtoRDOTrigger') rdo2rdotrig.type = 'Reco_tf'
# art-output: *.check* # art-output: HLTconfig*.xml # art-output: L1Topoconfig*.xml # art-output: LVL1config*.xml # art-output: PHYSVAL_WEB # art-html: PHYSVAL_WEB from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps from TrigAnalysisTest.TrigAnalysisSteps import add_physvalweb_steps import os # To run single-process transform on MCORE sites if 'ATHENA_NPROC_NUM' in os.environ: del os.environ['ATHENA_NPROC_NUM'] rdo2aod = ExecStep.ExecStep('RDOtoAOD') rdo2aod.type = 'Reco_tf' rdo2aod.input = 'ttbar' rdo2aod.threads = 1 rdo2aod.max_events = 500 rdo2aod.args = '--outputAODFile=AOD.pool.root --steering="doRDO_TRIG" --valid=True' rdo2aod.args += ' --preExec="all:from TriggerJobOpts.TriggerFlags import TriggerFlags; TriggerFlags.AODEDMSet.set_Value_and_Lock(\\\"AODFULL\\\");"' physval = ExecStep.ExecStep('PhysVal') physval.type = 'Reco_tf' physval.input = '' physval.explicit_input = True physval.args = '--inputAODFile=AOD.pool.root --outputNTUP_PHYSVALFile=NTUP_PHYSVAL.pool.root --valid=True' validationFlags = 'doTrigEgamma,doTrigBphys,doTrigMET,doTrigJet,doTrigMuon,doTrigHLTResult,doTrigCalo,doTrigMinBias,doTrigTau,doTrigIDtrk,doTrigBjet' physval.args += ' --validationFlags="{:s}"'.format(validationFlags)
#!/usr/bin/env python # art-description: Test running HITS->RDO in master, then RDO->RDO_TRIG in 21.0, then RDO_TRIG->AOD in master # art-type: build # art-include: master/Athena # Skipping art-output which has no effect for build tests. # If you create a grid version, check art-output in existing grid tests. from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps, Input from TrigAnalysisTest.TrigAnalysisSteps import add_analysis_steps # HITS -> RDO step in master hit2rdo = ExecStep.ExecStep('HITtoRDO') hit2rdo.type = 'Reco_tf' hit2rdo.input = 'ttbar_HITS' hit2rdo.args = '--outputRDOFile=RDO.pool.root' pu_low = Input.get_input('pileup_low') pu_high = Input.get_input('pileup_high') hit2rdo.args += ' --inputLowPtMinbiasHitsFile=' + pu_low.paths[0] hit2rdo.args += ' --inputHighPtMinbiasHitsFile=' + pu_high.paths[0] hit2rdo.args += ' --numberOfCavernBkg="0"' hit2rdo.args += ' --numberOfHighPtMinBias="0.15520183"' hit2rdo.args += ' --numberOfLowPtMinBias="59.3447981771"' hit2rdo.args += ' --pileupFinalBunch="6"' hit2rdo.args += ' --jobNumber="1"' hit2rdo.args += ' --preExec "HITtoRDO:userRunLumiOverride={\'run\':300000, \'startmu\':40.0, \'endmu\':70.0, \'stepmu\':1.0, \'startlb\':1, \'timestamp\': 1500000000};ScaleTaskLength=0.1" ' hit2rdo.args += ' --preInclude "HITtoRDO:Digitization/ForceUseOfPileUpTools.py,SimulationJobOptions/preInlcude.PileUpBunchTrainsMC16c_2017_Config1.py,RunDependentSimData/configLumi_muRange.py"'
#!/usr/bin/env python # Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration # # art-description: athenaHLT test of the Dev_pp_run3_v1 menu imitating partition with preloaded data at P1 # art-type: build # art-include: master/Athena from TrigValTools.TrigValSteering import Input, Test, Step, ExecStep from TrigP1Test import TrigP1TestSteps # Input file f = str(Input.get_input('data').paths[0]) # Delete any previous bytestream file ex_rm = ExecStep.ExecStep('cleanup') ex_rm.type = 'other' ex_rm.input = '' ex_rm.executable = 'rm' ex_rm.args = '-f raw._0001.data' ex_rm.auto_report_result = False # Do not set art-result for this step ex_rm.output_stream = Step.Step.OutputStream.STDOUT_ONLY # Do not create a log file for this step # Create new bytestream file with future run number and renumber LBs ex_bs = ExecStep.ExecStep('create_bs') ex_bs.type = 'other' ex_bs.input = '' ex_bs.executable = 'trigbs_modifyEvent.py' ex_bs.args = '-n 50 --runNumber 999999 --incLB 6 --firstLB 4 --eventsPerLB=3 -o raw %s' % f # Run athenaHLT forcing run and timestamp of original data file import eformat
# art-output: *.json # art-output: *.root # art-output: *.pmon.gz # art-output: *perfmon* # art-output: prmon* # art-output: *.check* from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps def findFile(pattern): return '`find . -name \'{:s}\' | tail -n 1`'.format(pattern) # HLT step (BS->BS) hlt = ExecStep.ExecStep() hlt.type = 'athenaHLT' hlt.job_options = 'TriggerJobOpts/runHLT_standalone.py' hlt.input = 'data' hlt.args = '-c "setMenu=\'PhysicsP1_pp_run3_v1\';"' hlt.args += ' -o output' # Extract the physics_Main stream out of the BS file with many streams filter_bs = ExecStep.ExecStep('FilterBS') filter_bs.type = 'other' filter_bs.executable = 'trigbs_extractStream.py' filter_bs.input = '' filter_bs.args = '-s Main ' + findFile('*_HLTMPPy_output.*.data') # Tier-0 reco step (BS->ESD->AOD) tzrecoPreExec = ' '.join([
#!/usr/bin/env python # Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration # art-description: Trigger AthenaMT test running new-style job options # art-type: build # art-include: master/Athena # Skipping art-output which has no effect for build tests. # If you create a grid version, check art-output in existing grid tests. from TrigValTools.TrigValSteering import Test, Step, ExecStep, CheckSteps # Copy the job options to the working directory copy_jo = ExecStep.ExecStep('CopyJO') copy_jo.type = 'other' copy_jo.input = '' copy_jo.executable = 'get_files' copy_jo.args = '-remove -jo TriggerJobOpts/runHLT_standalone_newJO.py' copy_jo.prmon = False copy_jo.auto_report_result = False # Do not set art-result for this step copy_jo.output_stream = Step.Step.OutputStream.STDOUT_ONLY # Do not create a log file for this step # Generate configuration pickle file pickle = ExecStep.ExecStep('GeneratePickle') pickle.type = 'other' pickle.input = '' pickle.executable = 'python' pickle.args = 'runHLT_standalone_newJO.py' pickle.prmon = False # The main job running athena from pickle run_athena = ExecStep.ExecStep('newJO')
#!/usr/bin/env python # Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration # art-description: transform test of BSRDOtoRAW + T0Reco + T0Mon, using v1PhysP1 menu # art-type: build # art-include: master/Athena from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps #==================================================================================================== # HLT BS_RDO->RAW hlt = ExecStep.ExecStep('BSRDOtoRAW') hlt.type = 'Trig_reco_tf' hlt.forks = 1 hlt.threads = 1 hlt.concurrent_events = 1 hlt.max_events = 50 hlt.args = '--precommand=\\\"setMenu=\\\'PhysicsP1_pp_run3_v1\\\'\\\"' hlt.args += ' --prodSysBSRDO True' hlt.args += ' --outputBSFile=RAW.pool.root' hlt.args += ' --outputHIST_HLTMONFile=hltmon.root' hlt.input = 'data' #==================================================================================================== # Tier-0 reco step BS->ESD->AOD tzrecoPreExec = ' '.join([ "from AthenaConfiguration.AllConfigFlags import ConfigFlags;", "ConfigFlags.Trigger.triggerMenuSetup=\'PhysicsP1_pp_run3_v1\';", "from TriggerJobOpts.TriggerFlags import TriggerFlags;", "TriggerFlags.configForStartup=\'HLToffline\';", "TriggerFlags.inputHLTconfigFile.set_Value_and_Lock(\'NONE\');",
# art-type: build # art-include: master/Athena from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps from TrigAnalysisTest.TrigAnalysisSteps import add_analysis_steps def findFile(pattern): '''Bash inline file name finder''' return '`find . -name \'{:s}\' | tail -n 1`'.format(pattern) menu_name = 'LS2_v1' # Run athena BS->BS job BStoBS = ExecStep.ExecStep("BStoBS") BStoBS.type = 'athena' BStoBS.job_options = 'TriggerJobOpts/runHLT_standalone.py' BStoBS.input = 'data' BStoBS.threads = 1 BStoBSPreExec = ''.join([ 'setMenu=\'{:s}\';'.format(menu_name), 'rewriteLVL1=True;', # Write L1 result to BS 'doWriteBS=True;', # Write HLT result to BS 'forceEnableAllChains=True;', # Ignore L1 decisions in seeding chains ]) BStoBS.args = ' -c "{:s}"'.format(BStoBSPreExec) # Extract the Main stream data filterBS = ExecStep.ExecStep('FilterBS') filterBS.type = 'other'
decodeStep.type = 'athena' decodeStep.job_options = 'TriggerJobOpts/decodeBS.py' decodeStep.input = '' decodeStep.explicit_input = True decodeStep.args = '--filesInput=' + findFile('*' + stream_name + '*._athenaHLT*.data') decodeStep.perfmon = False # no need to run PerfMon for this step return decodeStep ################################################## # Test definition ################################################## # Run standard athenaHLT BS->BS job writeBS = ExecStep.ExecStep("WriteBS") writeBS.type = 'athenaHLT' writeBS.job_options = 'TriggerJobOpts/runHLT_standalone.py' writeBS.input = 'data' writeBS.args = '-o output' writeBS.args += ' -c "setMenu=\'LS2_v1\';"' # LS2_v1 to be renamed to Dev_pp_run3_v1 # Extract and decode physics_Main filterMain = filterBS("Main") decodeMain = decodeBS("Main") # Extract and decode calibration_CostMonitoring filterCost = filterBS("CostMonitoring") decodeCost = decodeBS("CostMonitoring") decodeCost.args += ' -c "ModuleID=1"'
# art-output: *.new # art-output: *.json # art-output: *.root # art-output: *.pmon.gz # art-output: *perfmon* # art-output: prmon* # art-output: *.check* from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps import os # To run single-process transform on MCORE sites if 'ATHENA_NPROC_NUM' in os.environ: del os.environ['ATHENA_NPROC_NUM'] rdo2aod = ExecStep.ExecStep('RDOtoAOD') rdo2aod.type = 'Reco_tf' rdo2aod.input = 'ttbar' rdo2aod.max_events = 500 rdo2aod.threads = 1 rdo2aod.args = '--outputAODFile=AOD.pool.root --steering="doRDO_TRIG" --valid=True' rdo2aod.args += ' --preExec="all:from TriggerJobOpts.TriggerFlags import TriggerFlags; TriggerFlags.AODEDMSet.set_Value_and_Lock(\\\"AODFULL\\\");"' rdo2aod.args += ' --postInclude="TriggerTest/disableChronoStatSvcPrintout.py"' tzmon = ExecStep.ExecStep('Tier0Mon') tzmon.type = 'other' tzmon.executable = 'Run3DQTestingDriver.py' tzmon.input = '' tzmon.explicit_input = True tzmon.args = '--dqOffByDefault Input.Files="[\'AOD.pool.root\']" DQ.Steering.doHLTMon=True'
#!/usr/bin/env python # Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration # art-description: Trigger BS->RDO_TRIG athena test of the Dev_pp_run3_v1 menu # art-type: build # art-include: master/Athena # Skipping art-output which has no effect for build tests. # If you create a grid version, check art-output in existing grid tests. from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps ex = ExecStep.ExecStep() ex.type = 'athena' ex.job_options = 'TriggerJobOpts/runHLT_standalone.py' ex.input = 'data' ex.threads = 1 precommand = ''.join([ "setMenu='LS2_v1';", # LS2_v1 soon to be renamed to Dev_pp_run3_v1 "doWriteBS=False;", "doWriteRDOTrigger=True;", "forceEnableAllChains=True;", "fpeAuditor=True;", "failIfNoProxy=True;" ]) ex.args = '-c "{:s}"'.format(precommand) test = Test.Test() test.art_type = 'build' test.exec_steps = [ex] test.check_steps = CheckSteps.default_check_steps(test)
#!/usr/bin/env python # Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration # art-description: PhysicsP1_pp_run3_v1 menu test only dumping options for SMK generation and running RuleBook to create prescales # art-type: build # art-include: master/Athena from TrigValTools.TrigValSteering import Step, Test, ExecStep, CheckSteps #==================================================================================================== # Run Physics menu and produce files to create SMK menu = ExecStep.ExecStep('RunPhysMenu') menu.type = 'athenaHLT' menu.job_options = 'TriggerJobOpts/runHLT_standalone.py' menu.input = 'data' menu.args = '-c "setMenu=\'PhysicsP1_pp_run3_v1\';"' menu.args += ' -M --dump-config-exit' menu.perfmon = False # Cannot use PerfMon with -M #==================================================================================================== # Download and setup RuleBook and create prescales rbsetup = ExecStep.ExecStep('SetupRulebook') rbsetup.type = 'other' rbsetup.executable = 'setupTrigMenuRulebook.sh' rbsetup.input = '' #==================================================================================================== # Run RuleBook and create prescales
from TrigValTools.TrigValSteering import Test, ExecStep, CheckSteps, Common import logging import os if not os.path.exists('test_unit_trigvalsteering'): os.mkdir('test_unit_trigvalsteering') os.chdir('test_unit_trigvalsteering') Common.trigvalsteering_logging_level = logging.DEBUG Common.package_prefix_dict['TrigValTools'] = 'unit_' test = Test.Test() for test_type in ['athena', 'athenaHLT', 'Reco_tf', 'Trig_reco_tf']: ex = ExecStep.ExecStep('Test_' + test_type) ex.type = test_type if '_tf' not in test_type: ex.job_options = 'AthExJobOptions/AthExJobOptions_MinimalJobOptionsNoEvt.py' ex.input = '' ex.args = '--help' test.exec_steps.append(ex) test.art_type = 'build' test.check_steps = CheckSteps.default_check_steps(test) regtest_ref_text = [ '### athena.log ###', '### athena.Test_athena.log ###', '### athenaHLT.Test_athenaHLT.log ###', '### Reco_tf.Test_Reco_tf.log ###',