def test_transform(self):
from PyJobTransforms.transform import transform
from PyJobTransforms.trfArgClasses import argFactory, argString
from PyJobTransforms.trfDefaultFiles import getInputFileName
tf = transform()
addAthenaArguments(tf.parser)
tf.parser.add_argument('--inputBSFile', type=argFactory(argString))
tf.parser.add_argument('--outputBSFile', type=argFactory(argString))
tf.parser.add_argument('--extraArg', type=argFactory(argString))
tf.parseCmdLineArgs(
'--AMIConfig=p1346 --extraArg=special --outputBSFile=myOutput'.
split())
# not part of tag
self.assertEqual(tf.argdict['extraArg'].value, 'special')
# part of tag
self.assertEqual(tf.argdict['inputBSFile'].value,
getInputFileName('inputBSFile'))
# part of tag but overwritten at command line
self.assertEqual(tf.argdict['outputBSFile'].value, 'myOutput')
def getTrfConfigFromPANDA(tag):
msg.debug('Using PANDA to get info about tag %s' % tag)
try:
pandaclient = getPANDAClient()
pandaclient.execute(
"select trf,trfv,lparams,vparams,formats,cache from t_trf_config where tag='%s' and cid=%d"
% (tag[:1], int(tag[1:])))
result = pandaclient.fetchone()
except:
msg.info('An exception occurred: %s' % traceback.format_exc())
raise TransformAMIException(AMIerrorCode,
'Getting tag info from PANDA failed.')
if result is None:
raise TransformAMIException(AMIerrorCode,
'Tag %s not found in PANDA database' % tag)
msg.debug('Raw data returned from panda DB is:' + os.linesep + str(result))
trfn = result[0].split(',')
msg.debug('List of transforms: %s' % trfn)
trfv = result[1].split(',')
msg.debug('List of releases: %s' % trfv)
lparams = result[2].split(';')
msg.debug('List of arguments: %s' % lparams)
vparams = result[3].split(';')
msg.debug('List of argument values: %s' % vparams)
formats = result[4].split('.')
msg.debug('List of formats: %s' % formats)
cache = result[5].split(',')
msg.debug('List of caches: %s' % formats)
if not (len(trfn) == len(trfv) == len(lparams) == len(vparams)):
raise TransformAMIException(AMIerrorCode,
'Inconsistency in number of trfs.')
# Cache can be a single value, in which case it needs replicated for other
# transform steps, or it can be multivalued - great schema design guys :-(
if len(cache) != len(trfv):
if len(cache) == 1:
cache = cache * len(trfv)
else:
raise TransformAMIException(
AMIerrorCode,
'Inconsistency in number of caches entries vs. release numbers ({0}; {1}).'
.format(cache, trfv))
listOfTrfs = []
for iTrf in range(len(trfn)):
trf = TrfConfig()
trf.name = trfn[iTrf]
if '_tf.py' in trf.name:
trf.newTransform = True
else:
trf.newTransform = False
trf.release = trfv[iTrf] + "," + cache[iTrf]
keys = lparams[iTrf].split(',')
values = vparams[iTrf].split(',')
if (len(keys) != len(values)):
raise TransformAMIException(
AMIerrorCode, 'Inconsistency in number of arguments.')
physics = dict((k, ReadablePANDA(v)) for (k, v) in zip(keys, values))
# Hack to correct trigger keys being stored with spaces in panda
for k, v in physics.iteritems():
if 'triggerConfig' in k or 'triggerConfigByRun' in k:
if ' ' in v:
physics[k] = v.replace(' ', ',')
msg.warning(
'Attempted to correct illegal trigger configuration string: {0} -> {1}'
.format(v, physics[k]))
if 'Exec' in k:
# Mash up to a list, where %8C is used as the quote delimitation character
physics[k] = [
execElement.replace("%8C", "")
for execElement in v.split("%8C %8C")
]
msg.debug("Checking for pseudo-argument internal to ProdSys...")
if 'extraParameter' in physics:
val = physics.pop('extraParameter')
msg.debug("Removed extraParamater=%s from arguments." % val)
msg.debug("Checking for input/output file arguments...")
for arg in physics.keys():
if arg.lstrip('-').startswith('input') and arg.endswith('File'):
value = physics.pop(arg)
msg.debug("Found input file argument %s=%s." % (arg, value))
fmt = arg.lstrip('-').replace('input', '').replace('File', '')
trf.inFiles[arg] = getInputFileName(arg)
elif arg.lstrip('-').startswith('output') and arg.endswith('File'):
value = physics.pop(arg)
msg.debug("Found output file argument %s=%s." % (arg, value))
fmt = arg.lstrip('-').replace('output', '').replace('File', '')
trf.outFiles[arg] = getOutputFileName(fmt)
msg.debug("Checking for not set arguments...")
for arg, value in physics.items():
if value == "NONE" or value == "none" or value == ["NONE"]:
val = physics.pop(arg)
msg.debug("Removed %s=%s from arguments." % (arg, val))
trf.physics = physics
listOfTrfs.append(trf)
listOfTrfs[0].inDS = None # not yet implemented
listOfTrfs[-1].outfmts = formats
return listOfTrfs
msg.debug("Checking for pseudo-argument internal to ProdSys...")
if 'extraParameter' in physics:
val = physics.pop('extraParameter')
msg.debug("Removed extraParamater=%s from arguments." % val)
msg.debug("Checking for input/output file arguments...")
for arg in physics.keys():
if arg.lstrip('-').startswith('input') and arg.endswith(
'File'):
value = physics.pop(arg)
msg.debug("Found input file argument %s=%s." %
(arg, value))
fmt = arg.lstrip('-').replace('input',
'').replace('File', '')
trf.inFiles[arg] = getInputFileName(arg)
elif arg.lstrip('-').startswith('output') and arg.endswith(
'File'):
value = physics.pop(arg)
msg.debug("Found output file argument %s=%s." %
(arg, value))
fmt = arg.lstrip('-').replace('output',
'').replace('File', '')
trf.outFiles[arg] = getOutputFileName(fmt)
msg.debug("Checking for not set arguments...")
for arg, value in physics.items():
if value == "NONE" or value == "none" or value == ["NONE"]:
val = physics.pop(arg)
msg.debug("Removed %s=%s from arguments." % (arg, val))
def getTrfConfigFromAMI(tag, suppressNonJobOptions = True):
msg.debug('Using AMI to get info about tag %s', tag)
try:
# import pyAMI.atlas.api
import pyAMI.exception
except ImportError as e:
raise TransformAMIException(AMIerrorCode, 'Import of pyAMI modules failed ({0})'.format(e))
try:
amiclient=getAMIClient()
# result = pyAMI.atlas.api.get_ami_tag(amiclient, tag)
result = get_ami_tag(amiclient, tag, suppressNonJobOptions)
except pyAMI.exception.Error as e:
msg.warning('An exception occured when connecting to primary AMI: {0}'.format(e))
msg.debug('Exception: {0}'.format(e))
if 'please login' in e.message or 'certificate expired' in e.message:
raise TransformAMIException(AMIerrorCode, 'Getting tag info from AMI failed with credential problem. '
'Please check your AMI account status.')
if 'Invalid amiTag' in e.message:
raise TransformAMIException(AMIerrorCode, 'Invalid AMI tag ({0}).'.format(tag))
msg.debug("Error may not be fatal - will try AMI replica catalog")
try:
trf = TrfConfig()
trf.name = result[0]['transformation']
trf.inputs=result[0].get('inputs', {})
trf.outputs=result[0].get('outputs', {})
trf.release = result[0]['SWReleaseCache'].replace('_', ',')
if 'phconfig' in result[0]:
trf.physics=deserialiseFromAMIString(result[0]['phconfig'])
else:
physics = {}
for k, v in iteritems(result[0]):
if 'Exec' in k:
execStrList = [execStr for execStr in convertToStr(v).replace('" "', '"" ""').split('" "')]
physics[convertToStr(k)] = [remove_enclosing_quotes(execStr).replace('\\"', '"') for execStr in execStrList]
elif '" "' in v:
msg.info('found a quoted space (" ") in parameter value for %s, converting to list', k)
subStrList = [subStr for subStr in convertToStr(v).replace('" "', '"" ""').split('" "')]
physics[convertToStr(k)] = [remove_enclosing_quotes(subStr).replace('\\"', '"') for subStr in subStrList]
else:
physics[convertToStr(k)] = convertToStr(remove_enclosing_quotes(v))
msg.debug('Result from AMI after string cleaning:')
msg.debug('%s', dumps(physics, indent = 4))
if suppressNonJobOptions:
for k in list(physics):
if k in ['productionStep', 'transformation', 'SWReleaseCache']:
physics.pop(k)
for k, v in iteritems(physics):
if 'triggerConfig' in k or 'triggerConfigByRun' in k:
if ' ' in v:
physics[k] = v.replace(' ', ',')
msg.warning('Attempted to correct illegal trigger configuration string: {0} -> {1}'.format(v, physics[k]))
msg.debug("Checking for pseudo-argument internal to ProdSys...")
if 'extraParameter' in physics:
val = physics.pop('extraParameter')
msg.debug("Removed extraParamater=%s from arguments.", val)
msg.debug("Checking for input/output file arguments...")
for arg in list(physics):
if arg.lstrip('-').startswith('input') and arg.endswith('File'):
value = physics.pop(arg)
msg.debug("Found input file argument %s=%s.", arg, value)
fmt = arg.lstrip('-').replace('input', '').replace('File', '')
trf.inFiles[arg] = getInputFileName(arg)
elif arg.lstrip('-').startswith('output') and arg.endswith('File'):
value = physics.pop(arg)
msg.debug("Found output file argument %s=%s.", arg, value)
fmt = arg.lstrip('-').replace('output', '').replace('File', '')
trf.outFiles[arg] = getOutputFileName(fmt)
msg.debug("Checking for not set arguments...")
for arg, value in listitems(physics):
if value == "NONE" or value == "none" or value == ["NONE"]:
val = physics.pop(arg)
msg.debug("Removed %s=%s from arguments.", arg, val)
trf.physics = physics
if not isinstance(trf.physics, dict):
raise TransformAMIException(AMIerrorCode, "Bad result for tag's phconfig: {0}".format(trf.physics))
if trf.inFiles == {}:
if 'inputs' in result[0]:
trf.inFiles=deserialiseFromAMIString(result[0]['inputs'])
for inFileType, inFileName in iteritems(trf.inFiles):
# Not all AMI tags actually have a working filename, so fallback to trfDefaultFiles
# if necessary
if inFileName == '' or inFileName =={} or inFileName == [] or inFileName == '{}':
trf.inFiles[inFileType] = getInputFileName(inFileType, tag)
if 'outputs' in result[0]:
outputs=deserialiseFromAMIString(result[0]['outputs'])
trf.outFiles=dict( (k, getOutputFileName(k.lstrip('output').rstrip('File')) ) for k in outputs )
trf.outfmts=[ outputs[k]['dstype'] for k in outputs ]
except KeyError as e:
raise TransformAMIException(AMIerrorCode, "Missing key in AMI data: {0}".format(e))
except Exception as e:
raise TransformAMIException(AMIerrorCode, "Got a very unexpected exception while parsing AMI outputs!"
" Please report.\nParsing:\n{0}\nRaised:\n{1}".format(result, e))
# Now fix up for command line in the case of a new transform:
if '_tf.py' in trf.name:
trf.newTransform=True
else:
trf.newTransform=False
return [ trf ]