def _createAggregatedSource(self, psource_old, psource_new, missingInfos):
currentInfoKeys = psource_new.getJobKeys()
missingInfoKeys = lfilter(lambda key: key not in currentInfoKeys,
psource_old.getJobKeys())
ps_miss = ParameterSource.createInstance('InternalParameterSource',
missingInfos, missingInfoKeys)
return ParameterSource.createInstance('ChainParameterSource',
self._rawSource, ps_miss)
def getSource(self):
source_list = self._constSources + [self._pfactory.getSource()] + self._lookupSources
source = ParameterSource.createInstance('ZipLongParameterSource', *source_list)
for (PSourceClass, args) in self._nestedSources:
source = PSourceClass(source, *args)
if self._req:
req_source = ParameterSource.createInstance('RequirementParameterSource')
source = ParameterSource.createInstance('ZipLongParameterSource', source, req_source)
source = self._useAvailableDataSource(source)
return ParameterSource.createInstance('RepeatParameterSource', source, self._repeat)
def _combineSources(self, clsName, args):
repeat = reduce(lambda a, b: a * b, ifilter(lambda expr: isinstance(expr, int), args), 1)
args = lfilter(lambda expr: not isinstance(expr, int), args)
if args:
result = ParameterSource.createInstance(clsName, *args)
if repeat > 1:
return ParameterSource.createInstance('RepeatParameterSource', result, repeat)
return result
elif repeat > 1:
return repeat
return NullParameterSource()
def _addConstantPSource(self, config, cName, varName):
lookupVar = config.get('%s lookup' % cName, '', onChange = None)
if lookupVar:
matcher = Matcher.createInstance(config.get('%s matcher' % cName, 'start', onChange = None), config, cName)
content = config.getDict(cName, {}, onChange = None)
content_fixed = {}
content_order = lmap(lambda x: (x,), content[1])
for key in content[0]:
content_fixed[(key,)] = (content[0][key],)
ps = ParameterSource.createInstance('SimpleLookupParameterSource', varName, [lookupVar], [matcher], (content_fixed, content_order))
self.lookupSources.append(ps)
else:
ps = ParameterSource.createInstance('ConstParameterSource', varName, config.get(cName).strip())
self.constSources.append(ps)
def _resyncInternal(self): # This function is _VERY_ time critical!
tmp = self._rawSource.resync() # First ask about psource changes
(redoNewPNum, disableNewPNum, sizeChange) = (set(tmp[0]), set(tmp[1]), tmp[2])
hashNew = self._rawSource.getHash()
hashChange = self._storedHash != hashNew
self._storedHash = hashNew
if not (redoNewPNum or disableNewPNum or sizeChange or hashChange):
self._resyncState = None
return
psource_old = ParameterAdapter(None, ParameterSource.createInstance('GCDumpParameterSource', self._pathParams))
psource_new = ParameterAdapter(None, self._rawSource)
mapJob2PID = {}
(pAdded, pMissing, _) = self._diffParams(psource_old, psource_new, mapJob2PID, redoNewPNum, disableNewPNum)
self._source = self._getResyncSource(psource_old, psource_new, mapJob2PID, pAdded, pMissing, disableNewPNum)
self._mapJob2PID = mapJob2PID # Update Job2PID map
redoNewPNum = redoNewPNum.difference(disableNewPNum)
if redoNewPNum or disableNewPNum:
mapPID2Job = dict(ismap(utils.swap, self._mapJob2PID.items()))
translate = lambda pNum: mapPID2Job.get(pNum, pNum)
self._resyncState = (set(imap(translate, redoNewPNum)), set(imap(translate, disableNewPNum)), sizeChange)
elif sizeChange:
self._resyncState = (set(), set(), sizeChange)
# Write resynced state
self._writeJob2PID(self._pathJob2PID + '.tmp')
ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams + '.tmp', self)
os.rename(self._pathJob2PID + '.tmp', self._pathJob2PID)
os.rename(self._pathParams + '.tmp', self._pathParams)
def _useAvailableDataSource(self, source):
DataParameterSource = Plugin.getClass('DataParameterSource')
if DataParameterSource.datasetsAvailable and not DataParameterSource.datasetsUsed:
if source is not None:
return ParameterSource.createInstance('CrossParameterSource', DataParameterSource.create(), source)
return DataParameterSource.create()
return source
def _useAvailableDataSource(self, source):
usedSources = source.getUsedSources()
for (srcName, dataSource) in self._repository.items():
if srcName.startswith('dataset:') and (dataSource
not in usedSources):
source = ParameterSource.createInstance(
'CrossParameterSource', dataSource, source)
return source
def _createVarSource(self, var_list, lookup_list): # create variable source
psource_list = []
for (doElevate, PSourceClass, args) in createLookupHelper(self._paramConfig, var_list, lookup_list):
if doElevate: # switch needs elevation beyond local scope
self._nestedSources.append((PSourceClass, args))
else:
psource_list.append(PSourceClass(*args))
# Optimize away unnecessary cross operations
return ParameterSource.createInstance('CrossParameterSource', *psource_list)
def __init__(self, config):
ParameterFactory.__init__(self, config)
(self._constSources, self._lookupSources, self._nestedSources) = ([], [], [])
# Random number variables
jobs_config = config.changeView(addSections = ['jobs'])
for name in jobs_config.getList('random variables', ['JOB_RANDOM'], onChange = None):
self._constSources.append(ParameterSource.createInstance('RNGParameterSource', name))
nseeds = jobs_config.getInt('nseeds', 10)
newSeeds = lmap(lambda x: str(random.randint(0, 10000000)), irange(nseeds))
for (idx, seed) in enumerate(jobs_config.getList('seeds', newSeeds, persistent = True)):
ps = ParameterSource.createInstance('CounterParameterSource', 'SEED_%d' % idx, int(seed))
self._constSources.append(ps)
# Get constants from [constants <tags...>]
constants_config = config.changeView(viewClass = 'TaggedConfigView',
setClasses = None, setSections = ['constants'], setNames = None)
constants_pconfig = ParameterConfig(constants_config)
for cName in ifilter(lambda o: not o.endswith(' lookup'), constants_config.getOptions()):
constants_config.set('%s type' % cName, 'verbatim', '?=')
self._registerPSource(constants_pconfig, cName.upper())
param_config = config.changeView(viewClass = 'TaggedConfigView',
setClasses = None, addSections = ['parameters'], inheritSections = True)
# Get constants from [<Module>] constants
task_pconfig = ParameterConfig(param_config)
for cName in param_config.getList('constants', []):
config.set('%s type' % cName, 'verbatim', '?=')
self._registerPSource(task_pconfig, cName)
# Get global repeat value from 'parameters' section
self._repeat = param_config.getInt('repeat', 1, onChange = None)
self._req = param_config.getBool('translate requirements', True, onChange = None)
self._pfactory = param_config.getPlugin('parameter factory', 'SimpleParameterFactory',
cls = ParameterFactory)
def __init__(self, config, name):
(self.constSources, self.lookupSources) = ([], [])
ParameterFactory.__init__(self, config, name)
# Get constants from [constants <tags...>]
configConstants = config.changeView(viewClass = 'TaggedConfigView',
setClasses = None, setSections = ['constants'], addTags = [self])
for cName in ifilter(lambda o: not o.endswith(' lookup'), configConstants.getOptions()):
self._addConstantPSource(configConstants, cName, cName.upper())
# Get constants from [<Module>] constants
for cName in config.getList('constants', []):
self._addConstantPSource(config, cName, cName)
# Random number variables
configJobs = config.changeView(addSections = ['jobs'])
nseeds = configJobs.getInt('nseeds', 10)
newSeeds = lmap(lambda x: str(random.randint(0, 10000000)), irange(nseeds))
for (idx, seed) in enumerate(configJobs.getList('seeds', newSeeds, persistent = True)):
ps = ParameterSource.createInstance('CounterParameterSource', 'SEED_%d' % idx, int(seed))
self.constSources.append(ps)
self.repeat = config.getInt('repeat', 1, onChange = None) # ALL config.x -> paramconfig.x !
def getSource(self):
source_list = []
for name in self._randomVariables:
source_list.append(
ParameterSource.createInstance('RNGParameterSource', name))
for (idx, seed) in enumerate(self._randomSeeds):
source_list.append(
ParameterSource.createInstance('CounterParameterSource',
'SEED_%d' % idx, int(seed)))
source_list += self._constSources + [self._pfactory.getSource()
] + self._lookupSources
source = ParameterSource.createInstance('ZipLongParameterSource',
*source_list)
for (PSourceClass, args) in self._nestedSources:
source = PSourceClass(source, *args)
if self._req:
req_source = ParameterSource.createInstance(
'RequirementParameterSource')
source = ParameterSource.createInstance('ZipLongParameterSource',
source, req_source)
source = self._useAvailableDataSource(source)
return ParameterSource.createInstance('RepeatParameterSource', source,
self._repeat)
def _getRawSource(self, parent):
source_list = self.constSources + [parent, ParameterSource.createInstance('RequirementParameterSource')] + self.lookupSources
source = ParameterSource.createInstance('ZipLongParameterSource', *source_list)
if self.repeat > 1:
source = ParameterSource.createInstance('RepeatParameterSource', source, self.repeat)
return ParameterFactory._getRawSource(self, source)
def getSource(self, config):
DataParameterSource = Plugin.getClass('DataParameterSource')
source = self._getRawSource(ParameterSource.createInstance('RNGParameterSource'))
if DataParameterSource.datasetsAvailable and not DataParameterSource.datasetsUsed:
source = ParameterSource.createInstance('CrossParameterSource', DataParameterSource.create(), source)
return ParameterAdapter.createInstance(self.adapter, config, source)
def _createAggregatedSource(self, psource_old, psource_new, missingInfos):
currentInfoKeys = psource_new.getJobKeys()
missingInfoKeys = lfilter(lambda key: key not in currentInfoKeys, psource_old.getJobKeys())
ps_miss = ParameterSource.createInstance('InternalParameterSource', missingInfos, missingInfoKeys)
return ParameterSource.createInstance('ChainParameterSource', self._rawSource, ps_miss)
