def main(): parser = argparse.ArgumentParser( argument_default=argparse.SUPPRESS, description= 'GetTfCommand.py - prints the job transform commands accociated with an AMI tag.' ) parser.add_argument('--AMI', '--AMIConfig', help='Production tag to be interpreted', required=True) parser.add_argument('--verbose', '--debug', action='store_true', help='set logging level to DEBUG') parser.add_argument('--doNotSuppressNonJobOptions', action='store_true', help='get full output from AMI') parser.add_argument( '--printOnlyCmdLine', action='store_true', help='simply put out the TRF command line, nothing else') args = vars(parser.parse_args(sys.argv[1:])) if 'verbose' in args: msg.setLevel(stdLogLevels['DEBUG']) suppressNonJobOptions = True if 'doNotSuppressNonJobOptions' in args: suppressNonJobOptions = False try: tag = TagInfo(args['AMI'], suppressNonJobOptions) except TransformAMIException as e: print( 'An AMI exception was raised when trying to resolve the tag {0}.'. format(args['AMI'])) print('Exception message was: {0}'.format(e.errMsg)) print( 'Note that you need both suitable credentials to access AMI and access to the panda database (only works from inside CERN) for GetTfCommand.py to work.' ) sys.exit(1) if 'printOnlyCmdLine' not in args: print(tag) if 'argdict' in args: tag.dump(args['argdict']) else: # only print the command line, allows stuff like # pathena --trf "`GetTfCommand --AMI q1234 --printOnlyCmdLine` --inputFile bla.input --maxEvents 42" trfCmdLine = tag.trfs[0].name + " " + tag.trfs[0]._argsToString( tag.trfs[0].physics) print(trfCmdLine.replace('"', '\\' + '"'))
def test_info_x306(self): self.maxDiff = None physics = { 'conditionsTag': { 'all': 'CONDBR2-ES1PA-2014-03' }, 'beamType': 'cosmics', 'ignoreErrors': True, 'autoConfiguration': ['everything'], 'maxEvents': '-1', 'AMITag': 'x306', 'preExec': { 'all': [ 'from CaloRec.CaloCellFlags import jobproperties;jobproperties.CaloCellFlags.doLArHVCorr=False;jobproperties.CaloCellFlags.doPileupOffsetBCIDCorr.set_Value_and_Lock(False);from InDetRecExample.InDetJobProperties import InDetFlags;InDetFlags.doInnerDetectorCommissioning.set_Value_and_Lock(True);InDetFlags.useBroadClusterErrors.set_Value_and_Lock(False);DQMonFlags.doStreamAwareMon=False;DQMonFlags.enableLumiAccess=False;from JetRec.JetRecFlags import jetFlags;jetFlags.useTracks=False;DQMonFlags.doCTPMon=False;' ] }, 'geometryVersion': { 'all': 'ATLAS-R2-2015-02-00-00' } } tag = TagInfo("x306") self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertEqual(tag.isProdSys, False) self.assertEqual(tag.trfs[0].name, 'Reco_tf.py') self.assertEqual(tag.trfs[0].release, 'AtlasProduction,20.1.3.3') self.assertEqual(tag.trfs[0].physics, physics)
def test_info_r6390(self): self.maxDiff = None physics = { 'postInclude': 'all:RecJobTransforms/UseFrontier.py', 'conditionsTag': 'all:OFLCOND-RUN12-SDR-14', 'numberOfHighPtMinBias': '0.12311', 'postExec': [ 'all:CfgMgr.MessageSvc().setError+=["HepMcParticleLink"] HITtoRDO:job.StandardPileUpToolsAlg pTools["MergeMcEventCollTool"].OnlySaveSignalTruth=True RAWtoESD:ToolSvc.egammaConversionFinderTool.VertexFitterTo lSvc.InDetAdaptiveVxFitterTool;ToolSvc.egammaAmbiguityTool.minNoPixHits=3 -> --postinclude = RecJobTransforms/UseFrontier.py' ], 'preExec': [ 'all:rec.Commissioning.set_Value_and_Lock(True);from AthenaCommon.BeamFlags import jobproperties; perties.Beam.numberOfCollisions.set_Value_and_Lock(20.0);jobproperties.Beam.bunchSpacing.set_Value_and_Lock(25);from LAr rRODFlags import larRODFlags;larRODFlags.doOFCPileupOptimization.set_Value_and_Lock(True);larRODFlags.NumberOfCollisio _Value_and_Lock(20);larRODFlags.nSamples.set_Value_and_Lock(4) RAWtoESD:from TriggerJobOpts.TriggerFlags import% gerFlags;TriggerFlags.triggerMenuSetup="DC14_loose_mc_prescale";from CaloRec.CaloCellFlags import jobproperties% roperties.CaloCellFlags.doLArCellEmMisCalib=False ESDtoAOD:TriggerFlags.AODEDMSet="AODFULL"' ], 'geometryVersion': 'all:ATLAS-R2-2015-01-01-00', 'numberOfLowPtMinBias': '39.87688', } tag = TagInfo("r6390") self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertEqual(tag.isProdSys, False) self.assertEqual(tag.trfs[0].name, 'Reco_tf.py') self.assertEqual(tag.trfs[0].release, 'AtlasProduction,19.1.1.5') self.assertEqual(tag.trfs[0].newTransform, True) self.assertEqual(tag.trfs[0].physics, physics) self.assertEqual(tag.trfs[0].inFiles, {}) self.assertEqual(tag.trfs[0].outFiles, {}) self.assertEqual(tag.trfs[0].outputs, {}) self.assertEqual(tag.trfs[0].inDS, None) self.assertEqual(tag.trfs[0].outfmts, [])
def test_info_f559(self): self.maxDiff = None physics = { 'AMITag': 'f', 'autoConfiguration': 'everything', 'conditionsTag': 'CONDBR2-BLKPA-2014-03', 'ignoreErrors': 'True', 'maxEvents': '-1', 'postExec': 'CfgMgr.MessageSvc().setError+=["HepMcParticleLink"]', 'preExec': 'rec.doTrigger.set_Value_and_Lock(False);' } tag = TagInfo("f559") self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertEqual(tag.isProdSys, False) self.assertEqual(tag.trfs[0].name, 'Reco_tf.py') self.assertEqual(tag.trfs[0].release, 'AtlasProduction,19.3.0.4') self.assertEqual(tag.trfs[0].newTransform, True) self.assertEqual(tag.trfs[0].physics, physics) self.assertEqual( tag.trfs[0].inFiles, { 'inputRDOFile': '/afs/cern.ch/atlas/offline/test/mc11_valid.105200.T1_McAtNlo_Jimmy.digit.RDO.e835_s1310_s1300_d622.RDO.10evts.pool.root' }) self.assertEqual(tag.trfs[0].outFiles, { 'outputAODFile': 'myAOD.pool.root', 'outputESDFile': 'myESD.pool.root' }) self.assertEqual( tag.trfs[0].outputs, "{'outputESDFile': {'dstype': 'ESD', 'ifMatch': '(?!.*DRAW.*)'}, 'outputAODFile': {'dstype': 'AOD', 'ifMatch': 'data[0-9][0-9]_(cos|1beam|.*eV|comm)(?!.(.*DRAW.*|.*ZeroBias.*x[0-9].*))'}}" ) self.assertEqual(tag.trfs[0].inDS, None) self.assertEqual(tag.trfs[0].outfmts, ['AOD', 'ESD'])
def test_info_m1380(self): self.maxDiff = None physics = {} tag = TagInfo("m1380") self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertEqual(tag.isProdSys, False) self.assertEqual(tag.trfs[0].name, 'NTUPMerge_tf.py') self.assertEqual(tag.trfs[0].release, 'AtlasProduction,20.1.0.4,gcc48') self.assertEqual(tag.trfs[0].physics, physics)
def test_info_r6594(self): self.maxDiff = None physics = { 'autoConfiguration': 'everything', 'conditionsTag': 'default:OFLCOND-RUN12-SDR-28', 'geometryVersion': 'default:ATLAS-R2-2015-03-01-00', 'numberOfCavernBkg': '0', 'numberOfHighPtMinBias': '0.12268057', 'numberOfLowPtMinBias': '39.8773194', 'postExec': [ 'all:CfgMgr.MessageSvc().setError+=["HepMcParticleLink"]', 'HITtoRDO:job.StandardPileUpToolsAlg.PileUpTools["MergeMcEventCollTool"].OnlySaveSignalTruth=True', 'RAWtoESD:ToolSvc.LArAutoCorrTotalToolDefault.deltaBunch=1', 'HITtoRDO:job.StandardPileUpToolsAlg.PileUpTools["MdtDigitizationTool"].LastXing=150;ToolSvc.LArAutoCorrTotalToolDefault.deltaBunch=1' ], 'postInclude': 'default:RecJobTransforms/UseFrontier.py', 'preExec': [ 'all:rec.Commissioning.set_Value_and_Lock(True);rec.doTrigger.set_Value_and_Lock(True);from AthenaCommon.BeamFlags import jobproperties;jobproperties.Beam.numberOfCollisions.set_Value_and_Lock(20.0);from LArROD.LArRODFlags import larRODFlags;larRODFlags.NumberOfCollisions.set_Value_and_Lock(20);larRODFlags.nSamples.set_Value_and_Lock(4);larRODFlags.doOFCPileupOptimization.set_Value_and_Lock(True);larRODFlags.firstSample.set_Value_and_Lock(0);larRODFlags.useHighestGainAutoCorr.set_Value_and_Lock(True)', 'RAWtoESD:from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.triggerMenuSetup="MC_pp_v5_loose_mc_prescale";from CaloRec.CaloCellFlags import jobproperties;jobproperties.CaloCellFlags.doLArCellEmMisCalib=False', 'ESDtoAOD:TriggerFlags.AODEDMSet="AODFULL"', 'RDOtoRDOTrigger:from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.triggerMenuSetup="MC_pp_v5_loose_mc_prescale";', 'HITtoRDO:from AthenaCommon.BeamFlags import jobproperties;jobproperties.Beam.bunchSpacing.set_Value_and_Lock(50);' ], 'preInclude': [ 'HITtoRDO:Digitization/ForceUseOfPileUpTools.py,SimulationJobOptions/preInclude.PileUpBunchTrainsMC15_2015_50ns_Config1.py,RunDependentSimData/configLumi_run222510.py', 'RDOtoRDOTrigger:RecExPers/RecoOutputMetadataList_jobOptions.py' ], 'steering': 'RAWtoESD:in-RDO,in+RDO_TRIG,in-BS' } tag = TagInfo("r6594") self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertEqual(tag.isProdSys, False) self.assertEqual(tag.trfs[0].name, 'Reco_tf.py') self.assertEqual(tag.trfs[0].release, 'AtlasProduction,20.1.4.5') self.assertEqual(tag.trfs[0].newTransform, True) self.assertEqual(tag.trfs[0].physics, physics) self.assertEqual( tag.trfs[0].inFiles, { 'inputHighPtMinbiasHitsFile': 'myHighPtMinbiasHits', 'inputLowPtMinbiasHitsFile': 'myLowPtMinbiasHits' }) self.assertEqual(tag.trfs[0].outFiles, {}) self.assertEqual(tag.trfs[0].outputs, {}) self.assertEqual(tag.trfs[0].inDS, None) self.assertEqual(tag.trfs[0].outfmts, [])
def test_info_r6411(self): self.maxDiff = None physics = { 'steering': 'RAWtoESD:in-RDO,in+RDO_TRIG,in-BS', 'conditionsTag': 'default:OFLCOND-RUN12-SDR-14', 'numberOfHighPtMinBias': '0.12311', 'autoConfiguration': 'everything', 'numberOfLowPtMinBias': '39.87688', 'DBRelease': 'default:current', 'geometryVersion': 'default:ATLAS-R2-2015-01-01-00', 'preInclude': 'HITtoRDO:Digitization/ForceUseOfPileUpTools.py,SimulationJobOptions/preInclude.PileUpBunchTrainsUpgradeConfig1_25ns.py,RunDependentSimData/configLumi_run222222.py', 'postExec': [ 'default:CfgMgr.MessageSvc().setError+=["HepMcParticleLink"]', 'HITtoRDO:job.StandardPileUpToolsAlg.PileUpTools["MergeMcEventCollTool"].OnlySaveSignalTruth=True' ], 'preExec': [ 'default:rec.Commissioning.set_Value_and_Lock(True);from AthenaCommon.BeamFlags import jobproperties;jobproperties.Beam.numberOfCollisions.set_Value_and_Lock(20.0);jobproperties.Beam.bunchSpacing.set_Value_and_Lock(25);from LArROD.LArRODFlags import larRODFlags;larRODFlags.doOFCPileupOptimization.set_Value_and_Lock(True);larRODFlags.NumberOfCollisions.set_Value_and_Lock(20);larRODFlags.nSamples.set_Value_and_Lock(4)', 'RAWtoESD:from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.triggerMenuSetup="DC14_loose_mc_prescale";from CaloRec.CaloCellFlags import jobproperties;jobproperties.CaloCellFlags.doLArCellEmMisCalib=False', 'ESDtoAOD:TriggerFlags.AODEDMSet="AODFULL"' ], 'postInclude': 'default:RecJobTransforms/UseFrontier.py' } tag = TagInfo("r6411") self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertEqual(tag.isProdSys, False) self.assertEqual(tag.trfs[0].name, 'Reco_tf.py') self.assertEqual(tag.trfs[0].release, 'AtlasProduction,19.1.1.9') self.assertEqual(tag.trfs[0].newTransform, True) self.assertEqual(tag.trfs[0].physics, physics) self.assertEqual( tag.trfs[0].inFiles, { 'inputHighPtMinbiasHitsFile': 'myHighPtMinbiasHits', 'inputLowPtMinbiasHitsFile': 'myLowPtMinbiasHits' }) self.assertEqual(tag.trfs[0].outFiles, {}) self.assertEqual(tag.trfs[0].outputs, {}) self.assertEqual(tag.trfs[0].inDS, None) self.assertEqual(tag.trfs[0].outfmts, [])
def test_info_q120(self): physics = { 'AMITag': 'q120', 'maxEvents': '25', 'autoConfiguration': ['everything'], 'preExec': { 'all': [ 'rec.doTrigger=False;rec.doFloatingPointException.set_Value_and_Lock(True)' ] } } tag = TagInfo('q120') self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertEqual(tag.isProdSys, False) self.assertEqual(tag.trfs[0].name, 'Reco_trf.py') self.assertEqual(tag.trfs[0].release, 'AtlasProduction,17.2.0.3') self.assertEqual(tag.trfs[0].physics, physics)
def test_info_r6540(self): self.maxDiff = None physics = { 'DataRunNumber': '222525', 'autoConfiguration': 'everything', 'conditionsTag': 'default:OFLCOND-RUN12-SDR-25', 'geometryVersion': 'default:ATLAS-R2-2015-03-01-00', 'numberOfCavernBkg': '0', 'postExec': [ 'RAWtoESD:ToolSvc.LArAutoCorrTotalToolDefault.deltaBunch=1', 'HITtoRDO:job.StandardPileUpToolsAlg.PileUpTools["MdtDigitizationTool"].LastXing=150;ToolSvc.LArAutoCorrTotalToolDefault.deltaBunch=1', 'RDOtoRDOTrigger:from AthenaCommon.AlgSequence import AlgSequence;AlgSequence().LVL1TGCTrigger.TILEMU=True' ], 'postInclude': 'default:RecJobTransforms/UseFrontier.py', 'preExec': [ 'all:rec.Commissioning.set_Value_and_Lock(True);rec.doTrigger.set_Value_and_Lock(True);from AthenaCommon.BeamFlags import jobproperties;jobproperties.Beam.numberOfCollisions.set_Value_and_Lock(0.);from LArROD.LArRODFlags import larRODFlags;larRODFlags.NumberOfCollisions.set_Value_and_Lock(20);larRODFlags.nSamples.set_Value_and_Lock(4);larRODFlags.doOFCPileupOptimization.set_Value_and_Lock(True);larRODFlags.firstSample.set_Value_and_Lock(0);larRODFlags.useHighestGainAutoCorr.set_Value_and_Lock(True)', 'RAWtoESD:from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.triggerMenuSetup="MC_pp_v5_loose_mc_prescale";from CaloRec.CaloCellFlags import jobproperties;jobproperties.CaloCellFlags.doLArCellEmMisCalib=False', 'ESDtoAOD:TriggerFlags.AODEDMSet="AODFULL"', 'RDOtoRDOTrigger:from TriggerJobOpts.TriggerFlags import TriggerFlags;TriggerFlags.triggerMenuSetup="MC_pp_v5_loose_mc_prescale";' ], 'steering': 'RAWtoESD:in-RDO,in+RDO_TRIG,in-BS' } tag = TagInfo("r6540") self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertEqual(tag.isProdSys, False) self.assertEqual(tag.trfs[0].name, 'Reco_tf.py') self.assertEqual(tag.trfs[0].release, 'AtlasProduction,20.1.4.3') self.assertEqual(tag.trfs[0].newTransform, True) self.assertEqual(tag.trfs[0].physics, physics) self.assertEqual(tag.trfs[0].inFiles, {}) self.assertEqual(tag.trfs[0].outFiles, {}) self.assertEqual(tag.trfs[0].outputs, {}) self.assertEqual(tag.trfs[0].inDS, None) self.assertEqual(tag.trfs[0].outfmts, [])
def test_info_a180(self): physics = { 'conditionsTag': 'OFLCOND-MC12-SIM-00', 'postInclude': 'FastCaloSimHit/postInclude.AF2FilterHitItems.py FastSimulationJobTransforms/jobConfig.FastCaloSim_ID_cuts.py FastSimulationJobTransforms/jobConfig.egamma_lateral_shape_tuning.config20.py', 'DBRelease': '20.2.2', 'preInclude': 'FastSimulationJobTransforms/jobConfig.v14_Parametrisation.py FastCaloSimHit/preInclude.AF2Hit.py', 'geometryVersion': 'ATLAS-GEO-20-00-01' } tag = TagInfo('a180') self.assertEqual(len(tag.trfs), 2) self.assertTrue(isinstance(tag.trfs[0], TrfConfig)) self.assertTrue(isinstance(tag.trfs[1], TrfConfig)) self.assertEqual(tag.isProdSys, True) self.assertEqual(tag.trfs[0].name, 'Merging_trf.py') self.assertEqual(tag.trfs[0].release, '17.2.1.4.2,TrigMC') self.assertEqual(tag.trfs[0].physics, physics) self.assertEqual(tag.trfs[1].name, 'DigiMReco_trf.py') self.assertEqual(tag.trfs[1].release, '17.2.1.4.2,TrigMC')
def detectRAWtoALL(argv): limitedParser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) limitedParser.add_argument("--steering", nargs="+") limitedParser.add_argument("--argJSON") limitedParser.add_argument("--AMIConfig", "--AMI") args, unknown = limitedParser.parse_known_args(argv) args = vars(args) # Note if we have steering and it does _not_ contain doRAWtoALL we # return false, as CLI overrides JSON and AMI if "steering" in args: if "doRAWtoALL" in args["steering"]: return True else: return False # JSON overrides AMI if "argJSON" in args: import json from PyJobTransforms.trfUtils import convertToStr argfile = open(args['argJSON'], 'r') jsonParams = json.load(argfile) jsonParams = convertToStr(jsonParams) if "steering" in jsonParams: if "doRAWtoALL" in jsonParams["steering"]: return True else: return False if "AMIConfig" in args: from PyJobTransforms.trfAMI import TagInfo tag = dict(TagInfo(args['AMIConfig']).trfs[0]) if "steering" in tag and "doRAWtoALL" in tag["steering"]: return True return False
def parseCmdLineArgs(self, args): msg.info('Transform command line was: %s' % ' '.join(shQuoteStrings(sys.argv))) try: # Use the argparse infrastructure to get the actual command line arguments self._argdict = vars(self.parser.parse_args(args)) # Need to know if any input or output files were set - if so then we suppress the # corresponding parameters from AMI inputFiles = outputFiles = False for k, v in self._argdict.iteritems(): if k.startswith('input') and isinstance(v, argFile): inputFiles = True elif k.startswith('output') and isinstance(v, argFile): outputFiles = True msg.debug("CLI Input files: {0}; Output files {1}".format( inputFiles, outputFiles)) # Now look for special arguments, which expand out to other parameters # Note that the pickled argdict beats AMIConfig because dict.update() will overwrite # (However, we defend the real command line against updates from either source) extraParameters = {} # AMI configuration? if 'AMIConfig' in self._argdict: msg.debug('Given AMI tag configuration {0}'.format( self._argdict['AMIConfig'])) from PyJobTransforms.trfAMI import TagInfo tag = TagInfo(self._argdict['AMIConfig'].value) updateDict = {} for k, v in dict(tag.trfs[0]).iteritems(): # Convert to correct internal key form k = cliToKey(k) if inputFiles and k.startswith('input'): msg.debug( 'Suppressing argument {0} from AMI' ' because input files have been specified on the command line' .format(k)) continue if outputFiles and k.startswith('output'): msg.debug( 'Suppressing argument {0} from AMI' ' because output files have been specified on the command line' .format(k)) continue updateDict[k] = v extraParameters.update(updateDict) # JSON arguments? if 'argJSON' in self._argdict: try: import json msg.debug('Given JSON encoded arguments in {0}'.format( self._argdict['argJSON'])) argfile = open(self._argdict['argJSON'], 'r') jsonParams = json.load(argfile) msg.debug('Read: {0}'.format(jsonParams)) extraParameters.update(convertToStr(jsonParams)) argfile.close() except Exception, e: raise trfExceptions.TransformArgException( trfExit.nameToCode('TRF_ARG_ERROR'), 'Error when deserialising JSON file {0} ({1})'.format( self._argdict['argJSON'], e)) # Event Service if 'eventService' in self._argdict and self._argdict[ 'eventService'].value: updateDict = {} updateDict['athenaMPMergeTargetSize'] = '*:0' updateDict['checkEventCount'] = False updateDict['outputFileValidation'] = False extraParameters.update(updateDict) # Process anything we found for k, v in extraParameters.iteritems(): msg.debug( 'Found this extra argument: {0} with value: {1} ({2})'. format(k, v, type(v))) if k not in self.parser._argClass: raise trfExceptions.TransformArgException( trfExit.nameToCode('TRF_ARG_ERROR'), 'Argument "{0}" not known (try "--help")'.format(k)) if k in self._argdict: msg.debug( 'Ignored {0}={1} as extra parameter because this argument was given on the command line.' .format(k, v)) continue # For callable classes we instantiate properly, otherwise we set the value for simple arguments if '__call__' in dir(self.parser._argClass[k]): self._argdict[k] = self.parser._argClass[k](v) else: self._argdict[k] = v msg.debug('Argument {0} set to {1}'.format( k, self._argdict[k])) # Set the key name as an argument property - useful to be able to look bask at where this # argument came from for k, v in self._argdict.iteritems(): if isinstance(v, argument): v.name = k # Now we parsed all arguments, if a pickle/json dump is requested do it here and exit if 'dumpPickle' in self._argdict: msg.info('Now dumping pickled version of command line to {0}'. format(self._argdict['dumpPickle'])) pickledDump(self._argdict) sys.exit(0) # Now we parsed all arguments, if a pickle/json dump is requested do it here and exit if 'dumpJSON' in self._argdict: msg.info( 'Now dumping JSON version of command line to {0}'.format( self._argdict['dumpJSON'])) JSONDump(self._argdict) sys.exit(0)