def _expandMacrosInValues(self, data, maps): # arrays if isinstance(data, types.ListType): result = [] for e in range(len(data)): enew = self._expandMacrosInValues(data[e], maps) if enew != data[e]: console.debug("expanding: %r ==> %r" % (data[e], enew)) #data[e] = enew result.append(enew) # dicts elif isinstance(data, types.DictType): result = {} for e in data.keys( ): # have to use keys() explicitly since i modify data in place # expand in values enew = self._expandMacrosInValues(data[e], maps) if enew != data[e]: console.debug("expanding: %r ==> %r" % (data[e], enew)) #data[e] = enew result[e] = enew # expand in keys if (isinstance(e, types.StringTypes) and Key.hasMacro(e)): enew = self._expandString(e, maps['str'], {}) # no bin expand here! if enew == e: #self._console.warn("! Empty expansion for macro in config key: \"%s\"" % e) pass # TODO: the above warning produces too many false positives else: result[enew] = result[e] del result[e] console.debug("expanding key: %s ==> %s" % (e, enew)) # JobMergeValues elif isinstance(data, JobMergeValue): # macro-expand and merge further source = self._expandMacrosInValues(data.val1, maps) target = self._expandMacrosInValues(data.val2, maps) result = self.mergeValues(source, target) # strings elif isinstance(data, types.StringTypes): if Key.hasMacro(data): result = self._expandString(data, maps['str'], maps['bin']) if result == data: #self._console.warn("! Empty expansion for macro in config value: \"%s\"" % data) pass # TODO: see other Empty expansion warning else: result = data # leave everything else alone else: result = data #print "OUT: %r" % result return result
def _expandMacrosInValues(self, data, maps): # arrays if isinstance(data, types.ListType): result = [] for e in range(len(data)): enew = self._expandMacrosInValues(data[e], maps) if enew != data[e]: console.debug("expanding: %r ==> %r" % (data[e], enew)) #data[e] = enew result.append(enew) # dicts elif isinstance(data, types.DictType): result = {} for e in data.keys(): # have to use keys() explicitly since i modify data in place # expand in values enew = self._expandMacrosInValues(data[e], maps) if enew != data[e]: console.debug("expanding: %r ==> %r" % (data[e], enew)) #data[e] = enew result[e] = enew # expand in keys if (isinstance(e, types.StringTypes) and Key.hasMacro(e)): enew = self._expandString(e, maps['str'], {}) # no bin expand here! if enew == e: #self._console.warn("! Empty expansion for macro in config key: \"%s\"" % e) pass # TODO: the above warning produces too many false positives else: result[enew] = result[e] del result[e] console.debug("expanding key: %s ==> %s" % (e, enew)) # JobMergeValues elif isinstance(data, JobMergeValue): # macro-expand and merge further source = self._expandMacrosInValues(data.val1, maps) target = self._expandMacrosInValues(data.val2, maps) result = self.mergeValues(source, target) # strings elif isinstance(data, types.StringTypes): if Key.hasMacro(data): result = self._expandString(data, maps['str'], maps['bin']) if result == data: #self._console.warn("! Empty expansion for macro in config value: \"%s\"" % data) pass # TODO: see other Empty expansion warning else: result = data # leave everything else alone else: result = data #print "OUT: %r" % result return result
def _expandString(self, s, mapstr, mapbin): assert isinstance(s, types.StringTypes) if not Key.hasMacro(s): # optimization: no macro -> return return s macro = "" sub = "" possiblyBin = re.match(r'^\${(.*)}$', s) # look for '${...}' as a bin replacement if possiblyBin: macro = possiblyBin.group(1) if macro and (macro in mapbin.keys()): replval = mapbin[macro] if isinstance(replval, types.DictType): sub = copy.deepcopy( replval ) # make sure macro values are not affected during value merges later else: sub = replval # array references are ok for now else: templ = string.Template(s) #sub = templ.safe_substitute(mapstr) try: sub = templ.substitute(mapstr) except KeyError, e: raise ValueError( "Macro left undefined in job (%s): '%s'\n(might be from an included config)" % (self.name, e.args[0]))
def _expandString(self, s, mapstr, mapbin): assert isinstance(s, types.StringTypes) if not Key.hasMacro(s): # optimization: no macro -> return return s macro = "" sub = "" possiblyBin = re.match(r'^\${(.*)}$', s) # look for '${...}' as a bin replacement if possiblyBin: macro = possiblyBin.group(1) if macro and (macro in mapbin.keys()): replval = mapbin[macro] if isinstance(replval, types.DictType): sub = copy.deepcopy(replval) # make sure macro values are not affected during value merges later else: sub = replval # array references are ok for now else: templ = string.Template(s) # allow stringyfied value of bin macros to be spliced into result value mapall = mapstr.copy() mapall.update(dict((k,json.dumps(v)) for (k,v) in mapbin.items())) try: sub = templ.substitute(mapall) except KeyError, e: raise ValueError("Macro left undefined in job (%s): '%s'\n(might be from an included config)" % (self.name, e.args[0]))
def patchJobReferences(job, key, renamedJobs): newlist = [] oldlist = job.getFeature(key) for jobentry in oldlist: if Key.hasMacro(jobentry) and renamedJobs: console.warn("Potential pitfall: Cannot rename job reference containing macros (%s#%s[\"%s\"]:%s)" \ % (extConfig._fname, extJob.name, key, oldlist)) if (isinstance(jobentry, types.StringTypes) and jobentry in renamedJobs): newlist.append(renamedJobs[jobentry]) else: newlist.append(jobentry) job.setFeature(key, newlist)
def resolveRun(self, cfg=None): config = cfg or self._config subJobs = [] job = self if not job.hasFeature("run"): return [job] else: # prepare a Let object for potential macro expansion letObj = Let(self.get(Key.LET_KEY, {})) letObj.expandMacrosInLet() # do self-expansion of macros for subjob in job.getFeature("run"): # make best effort on macro expansion if isinstance(subjob, types.StringTypes): if Key.hasMacro(subjob): subjob = letObj.expandMacros(subjob) # get job object subjobObj = self._getJob(subjob, config) if not subjobObj: raise RuntimeError, "No such job: \"%s\"" % subjob # make new job map job::subjob as copy of job, but extend[subjob] newjobname = self.name + self._config.COMPOSED_NAME_SEP + \ subjobObj.name.replace(self._config.NS_SEP, self._config.COMPOSED_NAME_SEP) newjob = job.clone() newjob.name = newjobname newjob.removeFeature('run') # remove 'run' key # we assume the initial 'run' job has already been resolved, so # we reset it here and set the 'extend' to the subjob if newjob.hasFeature(Key.RESOLVED_KEY): newjob.removeFeature(Key.RESOLVED_KEY) else: raise RuntimeError, "Cannot resolve 'run' key before 'extend' key" newjob.setFeature('extend', [subjobObj]) # extend subjob # add to config self._config.addJob( newjobname, newjob) # TODO: why not config.addJob(...) ?! # add to job list subJobs.append(newjob) job.setFeature( 'run', subJobs) # overwrite with list of Jobs (instead of Strings) return subJobs
def resolveRun(self, cfg=None): config = cfg or self._config subJobs = [] job = self if not job.hasFeature("run"): return [job] else: # prepare a Let object for potential macro expansion letObj = Let(self.get(Key.LET_KEY, {})) letObj.expandMacrosInLet() # do self-expansion of macros for subjob in job.getFeature("run"): # make best effort on macro expansion if isinstance(subjob, types.StringTypes): if Key.hasMacro(subjob): subjob = letObj.expandMacros(subjob) # get job object subjobObj = self._getJob(subjob, config) if not subjobObj: raise RuntimeError, 'No such job: "%s"' % subjob # make new job map job::subjob as copy of job, but extend[subjob] newjobname = ( self.name + self._config.COMPOSED_NAME_SEP + subjobObj.name.replace(self._config.NS_SEP, self._config.COMPOSED_NAME_SEP) ) newjob = job.clone() newjob.name = newjobname newjob.removeFeature("run") # remove 'run' key # we assume the initial 'run' job has already been resolved, so # we reset it here and set the 'extend' to the subjob if newjob.hasFeature(Key.RESOLVED_KEY): newjob.removeFeature(Key.RESOLVED_KEY) else: raise RuntimeError, "Cannot resolve 'run' key before 'extend' key" newjob.setFeature("extend", [subjobObj]) # extend subjob # add to config self._config.addJob(newjobname, newjob) # TODO: why not config.addJob(...) ?! # add to job list subJobs.append(newjob) job.setFeature("run", subJobs) # overwrite with list of Jobs (instead of Strings) return subJobs
def patchJobReferences(job, key, renamedJobs): newlist = [] oldlist = job.getFeature(key) for jobentry in oldlist: # it's a string reference if isinstance(jobentry, types.StringTypes): if Key.hasMacro(jobentry) and renamedJobs: console.warn("Potential pitfall: Cannot rename job reference containing macros (%s#%s[\"%s\"]:%s)" \ % (extConfig._fname, extJob.name, key, oldlist)) if jobentry in renamedJobs: newlist.append(renamedJobs[jobentry]) else: newlist.append(jobentry) # it's a Job() object else: newlist.append(jobentry) job.setFeature(key, newlist)
def _expandString(self, s, mapstr, mapbin): assert isinstance(s, types.StringTypes) if not Key.hasMacro(s): # optimization: no macro -> return return s macro = "" sub = "" possiblyBin = re.match(r'^\${(.*)}$', s) # look for '${...}' as a bin replacement if possiblyBin: macro = possiblyBin.group(1) if macro and (macro in mapbin.keys()): replval = mapbin[macro] if isinstance(replval, types.DictType): sub = copy.deepcopy(replval) # make sure macro values are not affected during value merges later else: sub = replval # array references are ok for now else: templ = string.Template(s) sub = templ.safe_substitute(mapstr) return sub
def resolveExtend(self, entryTrace=[], cfg=None): # resolve the 'extend' entry of a job config = cfg or self._config if self.hasFeature(Key.RESOLVED_KEY): return #self.includeGlobalLet() # make sure potential global let is included first self.includeGlobalDefaults( ) # make sure potential global let is included first if self.hasFeature("extend"): # prepare a Let object for potential macro expansion letObj = Let(self.get(Key.LET_KEY, {})) letObj.expandMacrosInLet() # do self-expansion of macros # loop through 'extend' entries extends = self.getFeature("extend") self._console.indent() for entry in extends: # make best effort on macro expansion if isinstance(entry, types.StringTypes): if Key.hasMacro(entry): entry = letObj.expandMacros(entry) entryJob = self._getJob(entry, config) if not entryJob: raise RuntimeError, "No such job: \"%s\" (trace: %s)" % ( entry, entryTrace + [self.name]) if entryJob.name in entryTrace: # cycle check raise RuntimeError, "Extend entry already seen: %s" % str( entryTrace + [self.name, entryJob.name]) self._console.debug('Including "%s" into "%s"' % (entryJob.name, self.name)) # make sure this entry job is fully resolved in its context entryJob.resolveExtend(entryTrace + [self.name], config) # now merge the fully expanded job into the current job self.mergeJob(entryJob) self._console.outdent() self.setFeature(Key.RESOLVED_KEY, True)
def _expandString(self, s, mapstr, mapbin): assert isinstance(s, types.StringTypes) if not Key.hasMacro(s): # optimization: no macro -> return return s macro = "" sub = "" possiblyBin = re.match(r'^\${(.*)}$', s) # look for '${...}' as a bin replacement if possiblyBin: macro = possiblyBin.group(1) if macro and (macro in mapbin.keys()): replval = mapbin[macro] if isinstance(replval, types.DictType): sub = copy.deepcopy( replval ) # make sure macro values are not affected during value merges later else: sub = replval # array references are ok for now else: templ = string.Template(s) sub = templ.safe_substitute(mapstr) return sub
def resolveExtend(self, entryTrace=[], cfg=None): # resolve the 'extend' entry of a job config = cfg or self._config if self.hasFeature(Key.RESOLVED_KEY): return #self.includeGlobalLet() # make sure potential global let is included first self.includeGlobalDefaults() # make sure potential global let is included first if self.hasFeature("extend"): # prepare a Let object for potential macro expansion letObj = Let(self.get(Key.LET_KEY, {})) letObj.expandMacrosInLet() # do self-expansion of macros # loop through 'extend' entries extends = self.getFeature("extend") self._console.indent() for entry in extends: # make best effort on macro expansion if isinstance(entry, types.StringTypes): if Key.hasMacro(entry): entry = letObj.expandMacros(entry) entryJob = self._getJob(entry, config) if not entryJob: raise RuntimeError, "No such job: \"%s\" (trace: %s)" % (entry, entryTrace+[self.name]) if entryJob.name in entryTrace: # cycle check raise RuntimeError, "Extend entry already seen: %s" % str(entryTrace+[self.name, entryJob.name]) self._console.debug('Including "%s" into "%s"' % (entryJob.name, self.name)) # make sure this entry job is fully resolved in its context entryJob.resolveExtend(entryTrace + [self.name], config) # now merge the fully expanded job into the current job self.mergeJob(entryJob) self._console.outdent() self.setFeature(Key.RESOLVED_KEY, True)