def detectRAWtoALL(argv):
limitedParser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS)
limitedParser.add_argument("--steering", nargs="+")
limitedParser.add_argument("--argJSON")
limitedParser.add_argument("--AMIConfig", "--AMI")
args, unknown = limitedParser.parse_known_args(argv)
args = vars(args)
# Note if we have steering and it does _not_ contain doRAWtoALL we
# return false, as CLI overrides JSON and AMI
if "steering" in args:
if "doRAWtoALL" in args["steering"]:
return True
else:
return False
# JSON overrides AMI
if "argJSON" in args:
import json
from PyJobTransforms.trfUtils import convertToStr
argfile = open(args['argJSON'], 'r')
jsonParams = json.load(argfile)
jsonParams = convertToStr(jsonParams)
if "steering" in jsonParams:
if "doRAWtoALL" in jsonParams["steering"]:
return True
else:
return False
if "AMIConfig" in args:
from PyJobTransforms.trfAMI import TagInfo
tag = dict(TagInfo(args['AMIConfig']).trfs[0])
if "steering" in tag and "doRAWtoALL" in tag["steering"]:
return True
return False
try:
trf = TrfConfig()
trf.name = result[0]['transformation']
trf.inputs = result[0].get('inputs', {})
trf.outputs = result[0].get('outputs', {})
trf.release = result[0]['SWReleaseCache'].replace('_', ',')
if 'phconfig' in result[0].keys():
trf.physics = deserialiseFromAMIString(result[0]['phconfig'])
else:
physics = {}
for k, v in result[0].iteritems():
if 'Exec' in k:
execStrList = [
execStr for execStr in convertToStr(v).replace(
'" "', '"" ""').split('" "')
]
physics[convertToStr(k)] = [
remove_enclosing_quotes(execStr).replace('\\"', '"')
for execStr in execStrList
]
elif '" "' in v:
msg.info(
'found a quoted space (" ") in parameter value for %s, converting to list'
% k)
subStrList = [
subStr for subStr in convertToStr(v).replace(
'" "', '"" ""').split('" "')
]
physics[convertToStr(k)] = [
remove_enclosing_quotes(subStr).replace('\\"', '"')
def parseCmdLineArgs(self, args):
msg.info('Transform command line was: %s' %
' '.join(shQuoteStrings(sys.argv)))
try:
# Use the argparse infrastructure to get the actual command line arguments
self._argdict = vars(self.parser.parse_args(args))
# Need to know if any input or output files were set - if so then we suppress the
# corresponding parameters from AMI
inputFiles = outputFiles = False
for k, v in self._argdict.iteritems():
if k.startswith('input') and isinstance(v, argFile):
inputFiles = True
elif k.startswith('output') and isinstance(v, argFile):
outputFiles = True
msg.debug("CLI Input files: {0}; Output files {1}".format(
inputFiles, outputFiles))
# Now look for special arguments, which expand out to other parameters
# Note that the pickled argdict beats AMIConfig because dict.update() will overwrite
# (However, we defend the real command line against updates from either source)
extraParameters = {}
# AMI configuration?
if 'AMIConfig' in self._argdict:
msg.debug('Given AMI tag configuration {0}'.format(
self._argdict['AMIConfig']))
from PyJobTransforms.trfAMI import TagInfo
tag = TagInfo(self._argdict['AMIConfig'].value)
updateDict = {}
for k, v in dict(tag.trfs[0]).iteritems():
# Convert to correct internal key form
k = cliToKey(k)
if inputFiles and k.startswith('input'):
msg.debug(
'Suppressing argument {0} from AMI'
' because input files have been specified on the command line'
.format(k))
continue
if outputFiles and k.startswith('output'):
msg.debug(
'Suppressing argument {0} from AMI'
' because output files have been specified on the command line'
.format(k))
continue
updateDict[k] = v
extraParameters.update(updateDict)
# JSON arguments?
if 'argJSON' in self._argdict:
try:
import json
msg.debug('Given JSON encoded arguments in {0}'.format(
self._argdict['argJSON']))
argfile = open(self._argdict['argJSON'], 'r')
jsonParams = json.load(argfile)
msg.debug('Read: {0}'.format(jsonParams))
extraParameters.update(convertToStr(jsonParams))
argfile.close()
except Exception, e:
raise trfExceptions.TransformArgException(
trfExit.nameToCode('TRF_ARG_ERROR'),
'Error when deserialising JSON file {0} ({1})'.format(
self._argdict['argJSON'], e))
# Event Service
if 'eventService' in self._argdict and self._argdict[
'eventService'].value:
updateDict = {}
updateDict['athenaMPMergeTargetSize'] = '*:0'
updateDict['checkEventCount'] = False
updateDict['outputFileValidation'] = False
extraParameters.update(updateDict)
# Process anything we found
for k, v in extraParameters.iteritems():
msg.debug(
'Found this extra argument: {0} with value: {1} ({2})'.
format(k, v, type(v)))
if k not in self.parser._argClass:
raise trfExceptions.TransformArgException(
trfExit.nameToCode('TRF_ARG_ERROR'),
'Argument "{0}" not known (try "--help")'.format(k))
if k in self._argdict:
msg.debug(
'Ignored {0}={1} as extra parameter because this argument was given on the command line.'
.format(k, v))
continue
# For callable classes we instantiate properly, otherwise we set the value for simple arguments
if '__call__' in dir(self.parser._argClass[k]):
self._argdict[k] = self.parser._argClass[k](v)
else:
self._argdict[k] = v
msg.debug('Argument {0} set to {1}'.format(
k, self._argdict[k]))
# Set the key name as an argument property - useful to be able to look bask at where this
# argument came from
for k, v in self._argdict.iteritems():
if isinstance(v, argument):
v.name = k
# Now we parsed all arguments, if a pickle/json dump is requested do it here and exit
if 'dumpPickle' in self._argdict:
msg.info('Now dumping pickled version of command line to {0}'.
format(self._argdict['dumpPickle']))
pickledDump(self._argdict)
sys.exit(0)
# Now we parsed all arguments, if a pickle/json dump is requested do it here and exit
if 'dumpJSON' in self._argdict:
msg.info(
'Now dumping JSON version of command line to {0}'.format(
self._argdict['dumpJSON']))
JSONDump(self._argdict)
sys.exit(0)
def getTrfConfigFromAMI(tag, suppressNonJobOptions = True):
msg.debug('Using AMI to get info about tag %s', tag)
try:
# import pyAMI.atlas.api
import pyAMI.exception
except ImportError as e:
raise TransformAMIException(AMIerrorCode, 'Import of pyAMI modules failed ({0})'.format(e))
try:
amiclient=getAMIClient()
# result = pyAMI.atlas.api.get_ami_tag(amiclient, tag)
result = get_ami_tag(amiclient, tag, suppressNonJobOptions)
except pyAMI.exception.Error as e:
msg.warning('An exception occured when connecting to primary AMI: {0}'.format(e))
msg.debug('Exception: {0}'.format(e))
if 'please login' in e.message or 'certificate expired' in e.message:
raise TransformAMIException(AMIerrorCode, 'Getting tag info from AMI failed with credential problem. '
'Please check your AMI account status.')
if 'Invalid amiTag' in e.message:
raise TransformAMIException(AMIerrorCode, 'Invalid AMI tag ({0}).'.format(tag))
msg.debug("Error may not be fatal - will try AMI replica catalog")
try:
trf = TrfConfig()
trf.name = result[0]['transformation']
trf.inputs=result[0].get('inputs', {})
trf.outputs=result[0].get('outputs', {})
trf.release = result[0]['SWReleaseCache'].replace('_', ',')
if 'phconfig' in result[0]:
trf.physics=deserialiseFromAMIString(result[0]['phconfig'])
else:
physics = {}
for k, v in iteritems(result[0]):
if 'Exec' in k:
execStrList = [execStr for execStr in convertToStr(v).replace('" "', '"" ""').split('" "')]
physics[convertToStr(k)] = [remove_enclosing_quotes(execStr).replace('\\"', '"') for execStr in execStrList]
elif '" "' in v:
msg.info('found a quoted space (" ") in parameter value for %s, converting to list', k)
subStrList = [subStr for subStr in convertToStr(v).replace('" "', '"" ""').split('" "')]
physics[convertToStr(k)] = [remove_enclosing_quotes(subStr).replace('\\"', '"') for subStr in subStrList]
else:
physics[convertToStr(k)] = convertToStr(remove_enclosing_quotes(v))
msg.debug('Result from AMI after string cleaning:')
msg.debug('%s', dumps(physics, indent = 4))
if suppressNonJobOptions:
for k in list(physics):
if k in ['productionStep', 'transformation', 'SWReleaseCache']:
physics.pop(k)
for k, v in iteritems(physics):
if 'triggerConfig' in k or 'triggerConfigByRun' in k:
if ' ' in v:
physics[k] = v.replace(' ', ',')
msg.warning('Attempted to correct illegal trigger configuration string: {0} -> {1}'.format(v, physics[k]))
msg.debug("Checking for pseudo-argument internal to ProdSys...")
if 'extraParameter' in physics:
val = physics.pop('extraParameter')
msg.debug("Removed extraParamater=%s from arguments.", val)
msg.debug("Checking for input/output file arguments...")
for arg in list(physics):
if arg.lstrip('-').startswith('input') and arg.endswith('File'):
value = physics.pop(arg)
msg.debug("Found input file argument %s=%s.", arg, value)
fmt = arg.lstrip('-').replace('input', '').replace('File', '')
trf.inFiles[arg] = getInputFileName(arg)
elif arg.lstrip('-').startswith('output') and arg.endswith('File'):
value = physics.pop(arg)
msg.debug("Found output file argument %s=%s.", arg, value)
fmt = arg.lstrip('-').replace('output', '').replace('File', '')
trf.outFiles[arg] = getOutputFileName(fmt)
msg.debug("Checking for not set arguments...")
for arg, value in listitems(physics):
if value == "NONE" or value == "none" or value == ["NONE"]:
val = physics.pop(arg)
msg.debug("Removed %s=%s from arguments.", arg, val)
trf.physics = physics
if not isinstance(trf.physics, dict):
raise TransformAMIException(AMIerrorCode, "Bad result for tag's phconfig: {0}".format(trf.physics))
if trf.inFiles == {}:
if 'inputs' in result[0]:
trf.inFiles=deserialiseFromAMIString(result[0]['inputs'])
for inFileType, inFileName in iteritems(trf.inFiles):
# Not all AMI tags actually have a working filename, so fallback to trfDefaultFiles
# if necessary
if inFileName == '' or inFileName =={} or inFileName == [] or inFileName == '{}':
trf.inFiles[inFileType] = getInputFileName(inFileType, tag)
if 'outputs' in result[0]:
outputs=deserialiseFromAMIString(result[0]['outputs'])
trf.outFiles=dict( (k, getOutputFileName(k.lstrip('output').rstrip('File')) ) for k in outputs )
trf.outfmts=[ outputs[k]['dstype'] for k in outputs ]
except KeyError as e:
raise TransformAMIException(AMIerrorCode, "Missing key in AMI data: {0}".format(e))
except Exception as e:
raise TransformAMIException(AMIerrorCode, "Got a very unexpected exception while parsing AMI outputs!"
" Please report.\nParsing:\n{0}\nRaised:\n{1}".format(result, e))
# Now fix up for command line in the case of a new transform:
if '_tf.py' in trf.name:
trf.newTransform=True
else:
trf.newTransform=False
return [ trf ]