def createLookupHelper(pconfig, var_list, lookup_list):
# Return list of (doElevate, PSourceClass, arguments) entries
if len(var_list) != 1: # multi-lookup handling
result = []
for var_name in var_list:
result.extend(createLookupHelper(pconfig, [var_name], lookup_list))
return result
var_name = var_list[0]
pvalue = pconfig.getParameter(var_name.lstrip('!'))
if isinstance(pvalue, list): # simple parameter source
if len(pvalue) == 1:
return [(False, ParameterSource.getClass('ConstParameterSource'), [var_name, pvalue[0]])]
else:
return [(False, ParameterSource.getClass('SimpleParameterSource'), [var_name, pvalue])]
elif isinstance(pvalue, tuple) and pvalue[0] == 'format':
return [(False, ParameterSource.getClass('FormatterParameterSource'), pvalue[1:])]
lookup_key = None
if lookup_list: # default lookup key
lookup_key = KeyParameterSource(*lookup_list)
# Determine kind of lookup, [3] == lookupDictConfig, [0] == lookupContent
tmp = lookupConfigParser(pconfig, KeyParameterSource(var_name), lookup_key)
lookupContent = tmp[3][0]
lookupLen = lmap(len, lookupContent.values())
if (min(lookupLen) == 1) and (max(lookupLen) == 1): # simple lookup sufficient for this setup
return [(False, SimpleLookupParameterSource, list(tmp))]
# switch needs elevation beyond local scope
return [(True, SwitchingLookupParameterSource, list(tmp))]
def _resyncInternal(self): # This function is _VERY_ time critical!
tmp = self._rawSource.resync() # First ask about psource changes
(redoNewPNum, disableNewPNum, sizeChange) = (set(tmp[0]), set(tmp[1]), tmp[2])
hashNew = self._rawSource.getHash()
hashChange = self._storedHash != hashNew
self._storedHash = hashNew
if not (redoNewPNum or disableNewPNum or sizeChange or hashChange):
self._resyncState = None
return
psource_old = ParameterAdapter(None, ParameterSource.createInstance('GCDumpParameterSource', self._pathParams))
psource_new = ParameterAdapter(None, self._rawSource)
mapJob2PID = {}
(pAdded, pMissing, _) = self._diffParams(psource_old, psource_new, mapJob2PID, redoNewPNum, disableNewPNum)
self._source = self._getResyncSource(psource_old, psource_new, mapJob2PID, pAdded, pMissing, disableNewPNum)
self._mapJob2PID = mapJob2PID # Update Job2PID map
redoNewPNum = redoNewPNum.difference(disableNewPNum)
if redoNewPNum or disableNewPNum:
mapPID2Job = dict(ismap(utils.swap, self._mapJob2PID.items()))
translate = lambda pNum: mapPID2Job.get(pNum, pNum)
self._resyncState = (set(imap(translate, redoNewPNum)), set(imap(translate, disableNewPNum)), sizeChange)
elif sizeChange:
self._resyncState = (set(), set(), sizeChange)
# Write resynced state
self._writeJob2PID(self._pathJob2PID + '.tmp')
ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams + '.tmp', self)
os.rename(self._pathJob2PID + '.tmp', self._pathJob2PID)
os.rename(self._pathParams + '.tmp', self._pathParams)
def __init__(self, config, source):
self._rawSource = source
BasicParameterAdapter.__init__(self, config, source)
self._mapJob2PID = {}
if not os.path.isdir(config.getWorkPath()):
os.makedirs(config.getWorkPath())
self._pathJob2PID = config.getWorkPath('params.map.gz')
self._pathParams = config.getWorkPath('params.dat.gz')
# Find out if init should be performed - overrides userResync!
userInit = config.getState('init', detail='parameters')
needInit = False
if not (os.path.exists(self._pathParams)
and os.path.exists(self._pathJob2PID)):
needInit = True # Init needed if no parameter log exists
if userInit and not needInit and (source.getMaxParameters()
is not None):
utils.eprint(
'Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!'
)
if utils.getUserBool(
'Do you want to perform a syncronization between the current mapping and the new one to avoid this?',
True):
userInit = False
doInit = userInit or needInit
# Find out if resync should be performed
userResync = config.getState('resync', detail='parameters')
config.setState(False, 'resync', detail='parameters')
needResync = False
pHash = self._rawSource.getHash()
self.storedHash = config.get('parameter hash', pHash, persistent=True)
if self.storedHash != pHash:
needResync = True # Resync needed if parameters have changed
self._log.info('Parameter hash has changed')
self._log.debug('\told hash: %s', self.storedHash)
self._log.debug('\tnew hash: %s', pHash)
config.setState(True, 'init', detail='config')
doResync = (userResync or needResync) and not doInit
if not doResync and not doInit: # Reuse old mapping
activity = utils.ActivityLog(
'Loading cached parameter information')
self.readJob2PID()
activity.finish()
return
elif doResync: # Perform sync
activity = utils.ActivityLog('Syncronizing parameter information')
self.storedHash = None
self._resyncState = self.resync()
activity.finish()
elif doInit: # Write current state
self.writeJob2PID(self._pathJob2PID)
ParameterSource.getClass('GCDumpParameterSource').write(
self._pathParams, self)
config.set('parameter hash', self._rawSource.getHash())
def _createRef(self, arg):
refTypeDefault = 'dataset'
DataParameterSource = ParameterSource.getClass('DataParameterSource')
if arg not in DataParameterSource.datasetsAvailable:
refTypeDefault = 'csv'
refType = self._paramConfig.get(arg, 'type', refTypeDefault)
if refType == 'dataset':
return DataParameterSource.create(self._paramConfig, arg)
elif refType == 'csv':
return ParameterSource.getClass('CSVParameterSource').create(self._paramConfig, arg)
raise APIError('Unknown reference type: "%s"' % refType)
def __init__(self, config, source):
self._rawSource = source
BasicParameterAdapter.__init__(self, config, source)
self._mapJob2PID = {}
if not os.path.isdir(config.getWorkPath()):
os.makedirs(config.getWorkPath())
self._pathJob2PID = config.getWorkPath('params.map.gz')
self._pathParams = config.getWorkPath('params.dat.gz')
# Find out if init should be performed - overrides userResync!
userInit = config.getState('init', detail = 'parameters')
needInit = False
if not (os.path.exists(self._pathParams) and os.path.exists(self._pathJob2PID)):
needInit = True # Init needed if no parameter log exists
if userInit and not needInit and (source.getMaxParameters() is not None):
utils.eprint('Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!')
if utils.getUserBool('Do you want to perform a syncronization between the current mapping and the new one to avoid this?', True):
userInit = False
doInit = userInit or needInit
# Find out if resync should be performed
userResync = config.getState('resync', detail = 'parameters')
config.setState(False, 'resync', detail = 'parameters')
needResync = False
pHash = self._rawSource.getHash()
self._storedHash = config.get('parameter hash', pHash, persistent = True)
if self._storedHash != pHash:
needResync = True # Resync needed if parameters have changed
self._log.info('Parameter hash has changed')
self._log.debug('\told hash: %s', self._storedHash)
self._log.debug('\tnew hash: %s', pHash)
config.setState(True, 'init', detail = 'config')
doResync = (userResync or needResync) and not doInit
if not doResync and not doInit: # Reuse old mapping
activity = utils.ActivityLog('Loading cached parameter information')
self._readJob2PID()
activity.finish()
return
elif doResync: # Perform sync
activity = utils.ActivityLog('Syncronizing parameter information')
self._storedHash = None
self._resyncState = self.resync()
activity.finish()
elif doInit: # Write current state
self._writeJob2PID(self._pathJob2PID)
ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams, self)
config.set('parameter hash', self._rawSource.getHash())
def _createPSpace(self, args):
SubSpaceParameterSource = ParameterSource.getClass('SubSpaceParameterSource')
if len(args) == 1:
return SubSpaceParameterSource.create(self._paramConfig, args[0])
elif len(args) == 3:
return SubSpaceParameterSource.create(self._paramConfig, args[2], args[0])
else:
raise APIError('Invalid subspace reference!: %r' % args)
def wrapper(*args):
parameterClass = ParameterSource.getClass(clsName)
try:
return parameterClass.create(self._paramConfig,
self._repository, *args)
except Exception:
raise ParameterError(
'Error while creating %r with arguments %r' %
(parameterClass.__name__, args))
def wrapper(*args):
try:
parameterClass = ParameterSource.getClass(clsName)
except Exception:
raise ParameterError('Unable to create parameter source "%r"!' % clsName)
try:
return parameterClass.create(self._paramConfig, *args)
except Exception:
raise ParameterError('Error while creating "%r" with arguments "%r"' % (parameterClass.__name__, args))
def createLookupHelper(pconfig, var_list, lookup_list):
# Return list of (doElevate, PSourceClass, arguments) entries
if len(var_list) != 1: # multi-lookup handling
result = []
for var_name in var_list:
result.extend(createLookupHelper(pconfig, [var_name], lookup_list))
return result
var_name = var_list[0]
pvalue = pconfig.getParameter(var_name.lstrip('!'))
if isinstance(pvalue, list): # simple parameter source
if len(pvalue) == 1:
return [(False, ParameterSource.getClass('ConstParameterSource'),
[var_name, pvalue[0]])]
else:
return [(False, ParameterSource.getClass('SimpleParameterSource'),
[var_name, pvalue])]
elif isinstance(pvalue, tuple) and pvalue[0] == 'format':
return [(False, ParameterSource.getClass('FormatterParameterSource'),
pvalue[1:])]
lookup_key = None
if lookup_list: # default lookup key
lookup_key = KeyParameterSource(*lookup_list)
# Determine kind of lookup, [3] == lookupDictConfig, [0] == lookupContent
tmp = lookupConfigParser(pconfig, KeyParameterSource(var_name), lookup_key)
lookupContent = tmp[3][0]
lookupLen = lmap(len, lookupContent.values())
if (min(lookupLen)
== 1) and (max(lookupLen)
== 1): # simple lookup sufficient for this setup
return [(False, SimpleLookupParameterSource, list(tmp))]
# switch needs elevation beyond local scope
return [(True, SwitchingLookupParameterSource, list(tmp))]
