def test_transform(self): from PyJobTransforms.transform import transform from PyJobTransforms.trfArgClasses import argFactory, argString from PyJobTransforms.trfDefaultFiles import getInputFileName tf = transform() addAthenaArguments(tf.parser) tf.parser.add_argument('--inputBSFile', type=argFactory(argString)) tf.parser.add_argument('--outputBSFile', type=argFactory(argString)) tf.parser.add_argument('--extraArg', type=argFactory(argString)) tf.parseCmdLineArgs( '--AMIConfig=p1346 --extraArg=special --outputBSFile=myOutput'. split()) # not part of tag self.assertEqual(tf.argdict['extraArg'].value, 'special') # part of tag self.assertEqual(tf.argdict['inputBSFile'].value, getInputFileName('inputBSFile')) # part of tag but overwritten at command line self.assertEqual(tf.argdict['outputBSFile'].value, 'myOutput')
def getTrfConfigFromPANDA(tag): msg.debug('Using PANDA to get info about tag %s' % tag) try: pandaclient = getPANDAClient() pandaclient.execute( "select trf,trfv,lparams,vparams,formats,cache from t_trf_config where tag='%s' and cid=%d" % (tag[:1], int(tag[1:]))) result = pandaclient.fetchone() except: msg.info('An exception occurred: %s' % traceback.format_exc()) raise TransformAMIException(AMIerrorCode, 'Getting tag info from PANDA failed.') if result is None: raise TransformAMIException(AMIerrorCode, 'Tag %s not found in PANDA database' % tag) msg.debug('Raw data returned from panda DB is:' + os.linesep + str(result)) trfn = result[0].split(',') msg.debug('List of transforms: %s' % trfn) trfv = result[1].split(',') msg.debug('List of releases: %s' % trfv) lparams = result[2].split(';') msg.debug('List of arguments: %s' % lparams) vparams = result[3].split(';') msg.debug('List of argument values: %s' % vparams) formats = result[4].split('.') msg.debug('List of formats: %s' % formats) cache = result[5].split(',') msg.debug('List of caches: %s' % formats) if not (len(trfn) == len(trfv) == len(lparams) == len(vparams)): raise TransformAMIException(AMIerrorCode, 'Inconsistency in number of trfs.') # Cache can be a single value, in which case it needs replicated for other # transform steps, or it can be multivalued - great schema design guys :-( if len(cache) != len(trfv): if len(cache) == 1: cache = cache * len(trfv) else: raise TransformAMIException( AMIerrorCode, 'Inconsistency in number of caches entries vs. release numbers ({0}; {1}).' .format(cache, trfv)) listOfTrfs = [] for iTrf in range(len(trfn)): trf = TrfConfig() trf.name = trfn[iTrf] if '_tf.py' in trf.name: trf.newTransform = True else: trf.newTransform = False trf.release = trfv[iTrf] + "," + cache[iTrf] keys = lparams[iTrf].split(',') values = vparams[iTrf].split(',') if (len(keys) != len(values)): raise TransformAMIException( AMIerrorCode, 'Inconsistency in number of arguments.') physics = dict((k, ReadablePANDA(v)) for (k, v) in zip(keys, values)) # Hack to correct trigger keys being stored with spaces in panda for k, v in physics.iteritems(): if 'triggerConfig' in k or 'triggerConfigByRun' in k: if ' ' in v: physics[k] = v.replace(' ', ',') msg.warning( 'Attempted to correct illegal trigger configuration string: {0} -> {1}' .format(v, physics[k])) if 'Exec' in k: # Mash up to a list, where %8C is used as the quote delimitation character physics[k] = [ execElement.replace("%8C", "") for execElement in v.split("%8C %8C") ] msg.debug("Checking for pseudo-argument internal to ProdSys...") if 'extraParameter' in physics: val = physics.pop('extraParameter') msg.debug("Removed extraParamater=%s from arguments." % val) msg.debug("Checking for input/output file arguments...") for arg in physics.keys(): if arg.lstrip('-').startswith('input') and arg.endswith('File'): value = physics.pop(arg) msg.debug("Found input file argument %s=%s." % (arg, value)) fmt = arg.lstrip('-').replace('input', '').replace('File', '') trf.inFiles[arg] = getInputFileName(arg) elif arg.lstrip('-').startswith('output') and arg.endswith('File'): value = physics.pop(arg) msg.debug("Found output file argument %s=%s." % (arg, value)) fmt = arg.lstrip('-').replace('output', '').replace('File', '') trf.outFiles[arg] = getOutputFileName(fmt) msg.debug("Checking for not set arguments...") for arg, value in physics.items(): if value == "NONE" or value == "none" or value == ["NONE"]: val = physics.pop(arg) msg.debug("Removed %s=%s from arguments." % (arg, val)) trf.physics = physics listOfTrfs.append(trf) listOfTrfs[0].inDS = None # not yet implemented listOfTrfs[-1].outfmts = formats return listOfTrfs
msg.debug("Checking for pseudo-argument internal to ProdSys...") if 'extraParameter' in physics: val = physics.pop('extraParameter') msg.debug("Removed extraParamater=%s from arguments." % val) msg.debug("Checking for input/output file arguments...") for arg in physics.keys(): if arg.lstrip('-').startswith('input') and arg.endswith( 'File'): value = physics.pop(arg) msg.debug("Found input file argument %s=%s." % (arg, value)) fmt = arg.lstrip('-').replace('input', '').replace('File', '') trf.inFiles[arg] = getInputFileName(arg) elif arg.lstrip('-').startswith('output') and arg.endswith( 'File'): value = physics.pop(arg) msg.debug("Found output file argument %s=%s." % (arg, value)) fmt = arg.lstrip('-').replace('output', '').replace('File', '') trf.outFiles[arg] = getOutputFileName(fmt) msg.debug("Checking for not set arguments...") for arg, value in physics.items(): if value == "NONE" or value == "none" or value == ["NONE"]: val = physics.pop(arg) msg.debug("Removed %s=%s from arguments." % (arg, val))
def getTrfConfigFromAMI(tag, suppressNonJobOptions = True): msg.debug('Using AMI to get info about tag %s', tag) try: # import pyAMI.atlas.api import pyAMI.exception except ImportError as e: raise TransformAMIException(AMIerrorCode, 'Import of pyAMI modules failed ({0})'.format(e)) try: amiclient=getAMIClient() # result = pyAMI.atlas.api.get_ami_tag(amiclient, tag) result = get_ami_tag(amiclient, tag, suppressNonJobOptions) except pyAMI.exception.Error as e: msg.warning('An exception occured when connecting to primary AMI: {0}'.format(e)) msg.debug('Exception: {0}'.format(e)) if 'please login' in e.message or 'certificate expired' in e.message: raise TransformAMIException(AMIerrorCode, 'Getting tag info from AMI failed with credential problem. ' 'Please check your AMI account status.') if 'Invalid amiTag' in e.message: raise TransformAMIException(AMIerrorCode, 'Invalid AMI tag ({0}).'.format(tag)) msg.debug("Error may not be fatal - will try AMI replica catalog") try: trf = TrfConfig() trf.name = result[0]['transformation'] trf.inputs=result[0].get('inputs', {}) trf.outputs=result[0].get('outputs', {}) trf.release = result[0]['SWReleaseCache'].replace('_', ',') if 'phconfig' in result[0]: trf.physics=deserialiseFromAMIString(result[0]['phconfig']) else: physics = {} for k, v in iteritems(result[0]): if 'Exec' in k: execStrList = [execStr for execStr in convertToStr(v).replace('" "', '"" ""').split('" "')] physics[convertToStr(k)] = [remove_enclosing_quotes(execStr).replace('\\"', '"') for execStr in execStrList] elif '" "' in v: msg.info('found a quoted space (" ") in parameter value for %s, converting to list', k) subStrList = [subStr for subStr in convertToStr(v).replace('" "', '"" ""').split('" "')] physics[convertToStr(k)] = [remove_enclosing_quotes(subStr).replace('\\"', '"') for subStr in subStrList] else: physics[convertToStr(k)] = convertToStr(remove_enclosing_quotes(v)) msg.debug('Result from AMI after string cleaning:') msg.debug('%s', dumps(physics, indent = 4)) if suppressNonJobOptions: for k in list(physics): if k in ['productionStep', 'transformation', 'SWReleaseCache']: physics.pop(k) for k, v in iteritems(physics): if 'triggerConfig' in k or 'triggerConfigByRun' in k: if ' ' in v: physics[k] = v.replace(' ', ',') msg.warning('Attempted to correct illegal trigger configuration string: {0} -> {1}'.format(v, physics[k])) msg.debug("Checking for pseudo-argument internal to ProdSys...") if 'extraParameter' in physics: val = physics.pop('extraParameter') msg.debug("Removed extraParamater=%s from arguments.", val) msg.debug("Checking for input/output file arguments...") for arg in list(physics): if arg.lstrip('-').startswith('input') and arg.endswith('File'): value = physics.pop(arg) msg.debug("Found input file argument %s=%s.", arg, value) fmt = arg.lstrip('-').replace('input', '').replace('File', '') trf.inFiles[arg] = getInputFileName(arg) elif arg.lstrip('-').startswith('output') and arg.endswith('File'): value = physics.pop(arg) msg.debug("Found output file argument %s=%s.", arg, value) fmt = arg.lstrip('-').replace('output', '').replace('File', '') trf.outFiles[arg] = getOutputFileName(fmt) msg.debug("Checking for not set arguments...") for arg, value in listitems(physics): if value == "NONE" or value == "none" or value == ["NONE"]: val = physics.pop(arg) msg.debug("Removed %s=%s from arguments.", arg, val) trf.physics = physics if not isinstance(trf.physics, dict): raise TransformAMIException(AMIerrorCode, "Bad result for tag's phconfig: {0}".format(trf.physics)) if trf.inFiles == {}: if 'inputs' in result[0]: trf.inFiles=deserialiseFromAMIString(result[0]['inputs']) for inFileType, inFileName in iteritems(trf.inFiles): # Not all AMI tags actually have a working filename, so fallback to trfDefaultFiles # if necessary if inFileName == '' or inFileName =={} or inFileName == [] or inFileName == '{}': trf.inFiles[inFileType] = getInputFileName(inFileType, tag) if 'outputs' in result[0]: outputs=deserialiseFromAMIString(result[0]['outputs']) trf.outFiles=dict( (k, getOutputFileName(k.lstrip('output').rstrip('File')) ) for k in outputs ) trf.outfmts=[ outputs[k]['dstype'] for k in outputs ] except KeyError as e: raise TransformAMIException(AMIerrorCode, "Missing key in AMI data: {0}".format(e)) except Exception as e: raise TransformAMIException(AMIerrorCode, "Got a very unexpected exception while parsing AMI outputs!" " Please report.\nParsing:\n{0}\nRaised:\n{1}".format(result, e)) # Now fix up for command line in the case of a new transform: if '_tf.py' in trf.name: trf.newTransform=True else: trf.newTransform=False return [ trf ]