def getJob(self, job, withIncludes=False, default=None):
''' takes jobname or job object ref, and returns job object ref or default;
searches recursively through imported configs'''
if isinstance(job, Job): # you already found it :)
return job
assert isinstance(job, types.StringTypes)
# local job?
# this also finds imported jobs, namespaced or not
if Key.JOBS_KEY in self._data and job in self._data[Key.JOBS_KEY]:
jobEntry = self._data[Key.JOBS_KEY][job]
if isinstance(jobEntry, Job):
# make sure it has link to this config
if not jobEntry.getConfig():
jobEntry.setConfig(self)
return jobEntry
else:
# create Job object
jobObj = Job(job, jobEntry, self._console, self)
self._data[
Key.JOBS_KEY][job] = jobObj # overwrite map with obj
return jobObj
# job from included config? (to find required, but blocked jobs (e.g. through 'block' key)
elif withIncludes:
if not 'include' in self._data:
return default
else:
for econfig in self._includedConfigs:
jobObj = econfig.getJob(job)
if jobObj:
return jobObj
return default
def getJob(self, job, default=None):
''' takes jobname or job object ref, and returns job object ref or default '''
if isinstance(job, Job): # you already found it :)
return job
assert isinstance(job, types.StringTypes)
if ~job.find(self.NS_SEP): # nested job?
part, rest = job.split(self.NS_SEP, 1)
if part == "." or part == "":
return default
else:
if not 'include' in self._data:
return default
for incSpec in self._data['include']:
if (incSpec.has_key('as') and incSpec['as'] == part):
return incSpec['config'].getJob(rest)
else:
return default
# local job?
elif self._data.has_key(
self.JOBS_KEY) and self._data[self.JOBS_KEY].has_key(job):
jobEntry = self._data[self.JOBS_KEY][job]
if isinstance(jobEntry, Job):
# make sure it has link to this config
if not jobEntry.getConfig():
jobEntry.setConfig(self)
return jobEntry
else:
# create Job object
jobObj = Job(job, jobEntry, self._console, self)
self._data[
self.JOBS_KEY][job] = jobObj # overwrite map with obj
return jobObj
return default
def _integrateExternalConfig(self,
extConfig,
namespace,
impJobsList=None,
blockJobsList=None):
# Some helper functions
##
# Construct new job name for the imported job
def createNewJobName(extJobEntry):
if (importJobsList
and extJobEntry in importJobsList and isinstance(
importJobsList[extJobEntry], types.DictType)):
newjobname = namepfx + importJobsList[extJobEntry]['as']
else:
newjobname = namepfx + extJobEntry # create a job name
return newjobname
##
# In case of a name collision, do some householding and return an
# alternate job name.
def clashPrepare(jobname):
# import external job under different name
jobs_ignored_shadowing = self.get("config-warnings/job-shadowing",
[])
if jobname not in jobs_ignored_shadowing:
console.warn("! Shadowing job \"%s\" with local one" % jobname)
# construct a name prefix
extConfigName = extConfig._fname or self.SHADOW_PREFIX
extConfigName = os.path.splitext(
os.path.basename(extConfigName))[0]
# TODO: this might not be unique enough! (user could use extConfigName in 'as' param for other include)
newjobname = extConfigName + self.COMPOSED_NAME_SEP + jobname
return newjobname
def clashProcess(clashCase):
# check whether the local job is protected
jobMap = self.getJobsMap()
if ((Key.OVERRIDE_KEY not in jobMap)
or (clashCase.name not in jobMap[Key.OVERRIDE_KEY])):
# put shaddowed job in the local 'extend'
if not newJob:
raise Error, "unsuitable new job"
localjob = self.getJob(clashCase.name)
extList = localjob.getFeature('extend', [])
extList.append(newJob)
localjob.setFeature('extend', extList)
# add to config's shadowed list
self._shadowedJobs[newJob] = localjob
return
##
# Fix job references, but only for the jobs from the just imported config
def patchJobReferences(job, key, renamedJobs):
newlist = []
oldlist = job.getFeature(key)
for jobentry in oldlist:
# it's a string reference
if isinstance(jobentry, types.StringTypes):
if Key.hasMacro(jobentry) and renamedJobs:
console.warn("Potential pitfall: Cannot rename job reference containing macros (%s#%s[\"%s\"]:%s)" \
% (extConfig._fname, extJob.name, key, oldlist))
if jobentry in renamedJobs:
newlist.append(renamedJobs[jobentry])
else:
newlist.append(jobentry)
# it's a Job() object
else:
newlist.append(jobentry)
job.setFeature(key, newlist)
# -- Main --------------------------------------------------------------
if namespace:
namepfx = namespace + self.COMPOSED_NAME_SEP # job names will be namespace'd
else:
namepfx = "" # job names will not be namespace'd
renamedJobs = {
} # map for job renamings - done after all jobs have been imported
clashCase = NameSpace() # to record a single clashing job name
# Construct a map of import symbols (better lookup, esp. when aliased)
importJobsList = {}
if impJobsList:
for e in impJobsList:
if isinstance(e, types.StringTypes):
importJobsList[e] = None
elif isinstance(e,
types.DictType): # {name: <name>, as: <alias>}
importJobsList[e['name']] = {'as': e['as']}
else:
raise TypeError, "Illegal import entry: %s (Config: %s)" % (
str(e), self._fname)
# Merge global "let" -- currently disabled, see Bug#4126
#extLet = extConfig.get(Key.LET_KEY, False)
#if extLet:
# tmp = extLet.copy()
# tmp.update(self.get(Key.LET_KEY, {})) # this should probably be deepMerge
# self.set(Key.LET_KEY, tmp)
# self.expandTopLevelKeys() # we're making macro expansion in selected top-level keys eager
# Go through the list of jobs to import
newList = []
extJobsList = extConfig.getExportedJobsList()
for extJobEntry in extJobsList:
# Checks and preparations
if importJobsList and extJobEntry not in importJobsList:
continue
if blockJobsList and extJobEntry in blockJobsList:
continue
newjobname = createNewJobName(extJobEntry)
# Check for name clashes
if self.hasJob(newjobname):
clashCase.name_clashed = True
clashCase.name = newjobname
newjobname = clashPrepare(newjobname)
else:
clashCase.name_clashed = False # reset
# Now process the external job
# take essentially the external job into the local joblist
extJob = extConfig.getJob(extJobEntry) # fetch this job
if not extJob:
raise RuntimeError, "No such job: \"%s\" while including config: \"%s\")" % (
extJobEntry, extConfig._fname)
newJob = Job(
newjobname, {}, self._console, self
) # fake as local job, for _includeGlobalLet to run locally
#newJob.includeGlobalLet() # have to draw in local let before all the external let's are processed
newJob.includeGlobalDefaults(
) # have to draw in local let before all the external let's are processed
newJob.mergeJob(extJob) # now merge in the external guy
newJob.setConfig(
extJob.getConfig()) # retain link to original config
if (newjobname !=
extJobEntry # adapt modified names; otherwise, delay name resolution until resolveExtendsAndRun()
and not clashCase.name_clashed
): # keep job references if there is shadowing
renamedJobs[
extJobEntry] = newjobname # keep string reference for later binding
self.addJob(newjobname, newJob) # and add it
newList.append(
newJob) # memorize jobs added from extConfig for later
# Now process a possible name clash
if clashCase.name_clashed:
clashProcess(clashCase)
# Fix job references, but only for the jobs from the just imported config
# go through the list of just added jobs again
for job in newList: # there is no easy way to get newList from other data
# patch job references in 'run', 'extend', ... keys
for key in Key.KEYS_WITH_JOB_REFS:
if job.hasFeature(key):
patchJobReferences(job, key, renamedJobs)
return
def _integrateExternalConfig(self, extConfig, namespace, impJobsList=None, blockJobsList=None):
'''jobs of external config are spliced into current job list'''
if namespace:
namepfx = namespace + self.COMPOSED_NAME_SEP # job names will be namespace'd
else:
namepfx = "" # job names will not be namespace'd
renamedJobs = {} # map for job renamings - done after all jobs have been imported
l = NameSpace() # for out-params of nested functions
# Construct a map of import symbols (better lookup, esp. when aliased)
importJobsList = {}
if impJobsList:
for e in impJobsList:
if isinstance(e, types.StringTypes):
importJobsList[e]=None
elif isinstance(e, types.DictType): # {name: <name>, as: <alias>}
importJobsList[e['name']] = {'as': e['as']}
else:
raise TypeError, "Illegal import entry: %s (Config: %s)" % (str(e), self._fname)
# Some helper functions
def createNewJobName(extJobEntry):
# Construct new job name for the imported job
if (importJobsList and extJobEntry in importJobsList
and isinstance(importJobsList[extJobEntry], types.DictType)):
newjobname = namepfx + importJobsList[extJobEntry]['as']
else:
newjobname = namepfx + extJobEntry # create a job name
return newjobname
def clashPrepare(newjobname):
'''do some householding and return a new job name'''
l.hasClash = True
l.clashname = newjobname
# import external job under different name
console.warn("! Shadowing job \"%s\" with local one" % newjobname)
# construct a name prefix
extConfigName = extConfig._fname or self.SHADOW_PREFIX
extConfigName = os.path.splitext(os.path.basename(extConfigName))[0]
# TODO: this might not be unique enough! (user could use extConfigName in 'as' param for other include)
newjobname = extConfigName + self.COMPOSED_NAME_SEP + newjobname
return newjobname
def clashProcess():
# check whether the local job is protected
jobMap = self.getJobsMap()
if ((Lang.OVERRIDE_KEY not in jobMap) or
(l.clashname not in jobMap[Lang.OVERRIDE_KEY])):
# put shaddowed job in the local 'extend'
if not newJob:
raise Error, "unsuitable new job"
localjob = self.getJob(l.clashname)
extList = localjob.getFeature('extend', [])
extList.append(newJob)
localjob.setFeature('extend', extList)
# add to config's shadowed list
self._shadowedJobs[newJob] = localjob
# Go through the list of jobs to import
newList = []
extJobsList = extConfig.getExportedJobsList()
for extJobEntry in extJobsList:
# Checks and preparations
if importJobsList and extJobEntry not in importJobsList:
continue
if blockJobsList and extJobEntry in blockJobsList:
continue
newjobname = createNewJobName(extJobEntry)
# Check for name clashes
l.hasClash = False
if self.hasJob(newjobname):
newjobname = clashPrepare(newjobname)
# Now process the external job
# take essentially the external job into the local joblist
extJob = extConfig.getJob(extJobEntry) # fetch this job
if not extJob:
raise RuntimeError, "No such job: \"%s\" while including config: \"%s\")" % (extJobEntry, extConfig._fname)
newJob = Job(newjobname, {}, self._console, self) # fake as local job, for _includeGlobalLet to run locally
newJob.includeGlobalLet() # have to draw in local let before all the external let's are processed
newJob.mergeJob(extJob) # now merge in the external guy
newJob.setConfig(extJob.getConfig()) # retain link to original config
if (newjobname != extJobEntry # adapt modified names; otherwise, delay name resolution until resolveExtendsAndRun()
and not l.hasClash): # keep job references if there is shadowing
renamedJobs[extJobEntry] = newjobname # keep string reference for later binding
self.addJob(newjobname, newJob) # and add it
newList.append(newJob) # memorize jobs added from extConfig for later
# Now process a possible name clash
if l.hasClash:
clashProcess()
# Fix job references, but only for the jobs from the just imported config
# helper function
def patchFeature(job, key, renamedJobs):
newlist = []
oldlist = job.getFeature(key)
for jobentry in oldlist:
if (isinstance(jobentry, types.StringTypes)
and jobentry in renamedJobs):
newlist.append(renamedJobs[jobentry])
else:
newlist.append(jobentry)
job.setFeature(key, newlist)
# go through the list of just added jobs again
for job in newList: # there is no easy way to get newList from other data
# patch job references in 'run', 'extend', ... keys
for key in Lang.KEYS_WITH_JOB_REFS:
if job.hasFeature(key):
patchFeature(job, key, renamedJobs)
return
def _integrateExternalConfig(self,
extConfig,
namespace,
importJobsList=None,
blockJobsList=None):
# jobs of external config are spliced into current job list
if namespace:
namepfx = namespace + self.COMPOSED_NAME_SEP # job names will be namespace'd
else:
namepfx = "" # job names will not be namespace'd
# get the list of jobs to import
extJobsList = extConfig.getExportedJobsList()
for extJobEntry in extJobsList:
if importJobsList and extJobEntry not in importJobsList:
continue
if blockJobsList and extJobEntry in blockJobsList:
continue
newjobname = namepfx + extJobEntry # create a job name
if self.hasJob(newjobname):
raise KeyError, "Job already exists: \"%s\"" % newjobname
else:
# take essentially the external job into the local joblist
extJob = extConfig.getJob(extJobEntry) # fetch this job
if not extJob:
raise RuntimeError, "No such job: \"%s\" while including config: \"%s\")" % (
extJobEntry, extConfig._fname)
newJob = Job(
newjobname, {}, self._console, self
) # fake as local job, for _includeGlobalLet to run locally
newJob.includeGlobalLet(
) # have to draw in local let before all the external let's are processed
newJob.mergeJob(extJob) # now merge in the external guy
newJob.setConfig(
extJob.getConfig()) # retain link to external config
# patch job references in 'run', 'extend', ... keys
for key in Job.KEYS_WITH_JOB_REFS:
if newJob.hasFeature(key):
newlist = []
oldlist = newJob.getFeature(key)
for jobname in oldlist:
newlist.append(extConfig.getJob(jobname))
newJob.setFeature(key, newlist)
self.addJob(newjobname, newJob) # and add it
return
def _integrateExternalConfig(self, extConfig, namespace, importJobsList=None, blockJobsList=None):
# jobs of external config are spliced into current job list
if namespace:
namepfx = namespace + self.COMPOSED_NAME_SEP # job names will be namespace'd
else:
namepfx = "" # job names will not be namespace'd
# get the list of jobs to import
extJobsList = extConfig.getExportedJobsList()
for extJobEntry in extJobsList:
if importJobsList and extJobEntry not in importJobsList:
continue
if blockJobsList and extJobEntry in blockJobsList:
continue
newjobname = namepfx + extJobEntry # create a job name
if self.hasJob(newjobname):
raise KeyError, "Job already exists: \"%s\"" % newjobname
else:
# take essentially the external job into the local joblist
extJob = extConfig.getJob(extJobEntry) # fetch this job
if not extJob:
raise RuntimeError, "No such job: \"%s\" while including config: \"%s\")" % (extJobEntry, extConfig._fname)
newJob = Job(newjobname, {}, self._console, self) # fake as local job, for _includeGlobalLet to run locally
newJob.includeGlobalLet() # have to draw in local let before all the external let's are processed
newJob.mergeJob(extJob) # now merge in the external guy
newJob.setConfig(extJob.getConfig()) # retain link to external config
# patch job references in 'run', 'extend', ... keys
for key in Job.KEYS_WITH_JOB_REFS:
if newJob.hasFeature(key):
newlist = []
oldlist = newJob.getFeature(key)
for jobname in oldlist:
newlist.append(extConfig.getJob(jobname))
newJob.setFeature(key, newlist)
self.addJob(newjobname, newJob) # and add it
return
def _integrateExternalConfig(self, extConfig, namespace, impJobsList=None, blockJobsList=None):
# Some helper functions
##
# Construct new job name for the imported job
def createNewJobName(extJobEntry):
if (importJobsList and extJobEntry in importJobsList
and isinstance(importJobsList[extJobEntry], types.DictType)):
newjobname = namepfx + importJobsList[extJobEntry]['as']
else:
newjobname = namepfx + extJobEntry # create a job name
return newjobname
##
# In case of a name collision, do some householding and return an
# alternate job name.
def clashPrepare(jobname):
# import external job under different name
console.warn("! Shadowing job \"%s\" with local one" % jobname)
# construct a name prefix
extConfigName = extConfig._fname or self.SHADOW_PREFIX
extConfigName = os.path.splitext(os.path.basename(extConfigName))[0]
# TODO: this might not be unique enough! (user could use extConfigName in 'as' param for other include)
newjobname = extConfigName + self.COMPOSED_NAME_SEP + jobname
return newjobname
def clashProcess(clashCase):
# check whether the local job is protected
jobMap = self.getJobsMap()
if ((Key.OVERRIDE_KEY not in jobMap) or
(clashCase.name not in jobMap[Key.OVERRIDE_KEY])):
# put shaddowed job in the local 'extend'
if not newJob:
raise Error, "unsuitable new job"
localjob = self.getJob(clashCase.name)
extList = localjob.getFeature('extend', [])
extList.append(newJob)
localjob.setFeature('extend', extList)
# add to config's shadowed list
self._shadowedJobs[newJob] = localjob
return
##
# Fix job references, but only for the jobs from the just imported config
def patchJobReferences(job, key, renamedJobs):
newlist = []
oldlist = job.getFeature(key)
for jobentry in oldlist:
# it's a string reference
if isinstance(jobentry, types.StringTypes):
if Key.hasMacro(jobentry) and renamedJobs:
console.warn("Potential pitfall: Cannot rename job reference containing macros (%s#%s[\"%s\"]:%s)" \
% (extConfig._fname, extJob.name, key, oldlist))
if jobentry in renamedJobs:
newlist.append(renamedJobs[jobentry])
else:
newlist.append(jobentry)
# it's a Job() object
else:
newlist.append(jobentry)
job.setFeature(key, newlist)
# -- Main --------------------------------------------------------------
if namespace:
namepfx = namespace + self.COMPOSED_NAME_SEP # job names will be namespace'd
else:
namepfx = "" # job names will not be namespace'd
renamedJobs = {} # map for job renamings - done after all jobs have been imported
clashCase = NameSpace() # to record a single clashing job name
# Construct a map of import symbols (better lookup, esp. when aliased)
importJobsList = {}
if impJobsList:
for e in impJobsList:
if isinstance(e, types.StringTypes):
importJobsList[e]=None
elif isinstance(e, types.DictType): # {name: <name>, as: <alias>}
importJobsList[e['name']] = {'as': e['as']}
else:
raise TypeError, "Illegal import entry: %s (Config: %s)" % (str(e), self._fname)
# Merge global "let" -- currently disabled, see Bug#4126
#extLet = extConfig.get(Key.LET_KEY, False)
#if extLet:
# tmp = extLet.copy()
# tmp.update(self.get(Key.LET_KEY, {})) # this should probably be deepMerge
# self.set(Key.LET_KEY, tmp)
# self.expandTopLevelKeys() # we're making macro expansion in selected top-level keys eager
# Go through the list of jobs to import
newList = []
extJobsList = extConfig.getExportedJobsList()
for extJobEntry in extJobsList:
# Checks and preparations
if importJobsList and extJobEntry not in importJobsList:
continue
if blockJobsList and extJobEntry in blockJobsList:
continue
newjobname = createNewJobName(extJobEntry)
# Check for name clashes
if self.hasJob(newjobname):
clashCase.name_clashed = True
clashCase.name = newjobname
newjobname = clashPrepare(newjobname)
else:
clashCase.name_clashed = False # reset
# Now process the external job
# take essentially the external job into the local joblist
extJob = extConfig.getJob(extJobEntry) # fetch this job
if not extJob:
raise RuntimeError, "No such job: \"%s\" while including config: \"%s\")" % (extJobEntry, extConfig._fname)
newJob = Job(newjobname, {}, self._console, self) # fake as local job, for _includeGlobalLet to run locally
newJob.includeGlobalLet() # have to draw in local let before all the external let's are processed
newJob.mergeJob(extJob) # now merge in the external guy
newJob.setConfig(extJob.getConfig()) # retain link to original config
if (newjobname != extJobEntry # adapt modified names; otherwise, delay name resolution until resolveExtendsAndRun()
and not clashCase.name_clashed): # keep job references if there is shadowing
renamedJobs[extJobEntry] = newjobname # keep string reference for later binding
self.addJob(newjobname, newJob) # and add it
newList.append(newJob) # memorize jobs added from extConfig for later
# Now process a possible name clash
if clashCase.name_clashed:
clashProcess(clashCase)
# Fix job references, but only for the jobs from the just imported config
# go through the list of just added jobs again
for job in newList: # there is no easy way to get newList from other data
# patch job references in 'run', 'extend', ... keys
for key in Key.KEYS_WITH_JOB_REFS:
if job.hasFeature(key):
patchJobReferences(job, key, renamedJobs)
return