def createLookupHelper(pconfig, var_list, lookup_list): # Return list of (doElevate, PSourceClass, arguments) entries if len(var_list) != 1: # multi-lookup handling result = [] for var_name in var_list: result.extend(createLookupHelper(pconfig, [var_name], lookup_list)) return result var_name = var_list[0] pvalue = pconfig.getParameter(var_name.lstrip('!')) if isinstance(pvalue, list): # simple parameter source if len(pvalue) == 1: return [(False, ParameterSource.getClass('ConstParameterSource'), [var_name, pvalue[0]])] else: return [(False, ParameterSource.getClass('SimpleParameterSource'), [var_name, pvalue])] elif isinstance(pvalue, tuple) and pvalue[0] == 'format': return [(False, ParameterSource.getClass('FormatterParameterSource'), pvalue[1:])] lookup_key = None if lookup_list: # default lookup key lookup_key = KeyParameterSource(*lookup_list) # Determine kind of lookup, [3] == lookupDictConfig, [0] == lookupContent tmp = lookupConfigParser(pconfig, KeyParameterSource(var_name), lookup_key) lookupContent = tmp[3][0] lookupLen = lmap(len, lookupContent.values()) if (min(lookupLen) == 1) and (max(lookupLen) == 1): # simple lookup sufficient for this setup return [(False, SimpleLookupParameterSource, list(tmp))] # switch needs elevation beyond local scope return [(True, SwitchingLookupParameterSource, list(tmp))]
def _resyncInternal(self): # This function is _VERY_ time critical! tmp = self._rawSource.resync() # First ask about psource changes (redoNewPNum, disableNewPNum, sizeChange) = (set(tmp[0]), set(tmp[1]), tmp[2]) hashNew = self._rawSource.getHash() hashChange = self._storedHash != hashNew self._storedHash = hashNew if not (redoNewPNum or disableNewPNum or sizeChange or hashChange): self._resyncState = None return psource_old = ParameterAdapter(None, ParameterSource.createInstance('GCDumpParameterSource', self._pathParams)) psource_new = ParameterAdapter(None, self._rawSource) mapJob2PID = {} (pAdded, pMissing, _) = self._diffParams(psource_old, psource_new, mapJob2PID, redoNewPNum, disableNewPNum) self._source = self._getResyncSource(psource_old, psource_new, mapJob2PID, pAdded, pMissing, disableNewPNum) self._mapJob2PID = mapJob2PID # Update Job2PID map redoNewPNum = redoNewPNum.difference(disableNewPNum) if redoNewPNum or disableNewPNum: mapPID2Job = dict(ismap(utils.swap, self._mapJob2PID.items())) translate = lambda pNum: mapPID2Job.get(pNum, pNum) self._resyncState = (set(imap(translate, redoNewPNum)), set(imap(translate, disableNewPNum)), sizeChange) elif sizeChange: self._resyncState = (set(), set(), sizeChange) # Write resynced state self._writeJob2PID(self._pathJob2PID + '.tmp') ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams + '.tmp', self) os.rename(self._pathJob2PID + '.tmp', self._pathJob2PID) os.rename(self._pathParams + '.tmp', self._pathParams)
def __init__(self, config, source): self._rawSource = source BasicParameterAdapter.__init__(self, config, source) self._mapJob2PID = {} if not os.path.isdir(config.getWorkPath()): os.makedirs(config.getWorkPath()) self._pathJob2PID = config.getWorkPath('params.map.gz') self._pathParams = config.getWorkPath('params.dat.gz') # Find out if init should be performed - overrides userResync! userInit = config.getState('init', detail='parameters') needInit = False if not (os.path.exists(self._pathParams) and os.path.exists(self._pathJob2PID)): needInit = True # Init needed if no parameter log exists if userInit and not needInit and (source.getMaxParameters() is not None): utils.eprint( 'Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!' ) if utils.getUserBool( 'Do you want to perform a syncronization between the current mapping and the new one to avoid this?', True): userInit = False doInit = userInit or needInit # Find out if resync should be performed userResync = config.getState('resync', detail='parameters') config.setState(False, 'resync', detail='parameters') needResync = False pHash = self._rawSource.getHash() self.storedHash = config.get('parameter hash', pHash, persistent=True) if self.storedHash != pHash: needResync = True # Resync needed if parameters have changed self._log.info('Parameter hash has changed') self._log.debug('\told hash: %s', self.storedHash) self._log.debug('\tnew hash: %s', pHash) config.setState(True, 'init', detail='config') doResync = (userResync or needResync) and not doInit if not doResync and not doInit: # Reuse old mapping activity = utils.ActivityLog( 'Loading cached parameter information') self.readJob2PID() activity.finish() return elif doResync: # Perform sync activity = utils.ActivityLog('Syncronizing parameter information') self.storedHash = None self._resyncState = self.resync() activity.finish() elif doInit: # Write current state self.writeJob2PID(self._pathJob2PID) ParameterSource.getClass('GCDumpParameterSource').write( self._pathParams, self) config.set('parameter hash', self._rawSource.getHash())
def _createRef(self, arg): refTypeDefault = 'dataset' DataParameterSource = ParameterSource.getClass('DataParameterSource') if arg not in DataParameterSource.datasetsAvailable: refTypeDefault = 'csv' refType = self._paramConfig.get(arg, 'type', refTypeDefault) if refType == 'dataset': return DataParameterSource.create(self._paramConfig, arg) elif refType == 'csv': return ParameterSource.getClass('CSVParameterSource').create(self._paramConfig, arg) raise APIError('Unknown reference type: "%s"' % refType)
def __init__(self, config, source): self._rawSource = source BasicParameterAdapter.__init__(self, config, source) self._mapJob2PID = {} if not os.path.isdir(config.getWorkPath()): os.makedirs(config.getWorkPath()) self._pathJob2PID = config.getWorkPath('params.map.gz') self._pathParams = config.getWorkPath('params.dat.gz') # Find out if init should be performed - overrides userResync! userInit = config.getState('init', detail = 'parameters') needInit = False if not (os.path.exists(self._pathParams) and os.path.exists(self._pathJob2PID)): needInit = True # Init needed if no parameter log exists if userInit and not needInit and (source.getMaxParameters() is not None): utils.eprint('Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!') if utils.getUserBool('Do you want to perform a syncronization between the current mapping and the new one to avoid this?', True): userInit = False doInit = userInit or needInit # Find out if resync should be performed userResync = config.getState('resync', detail = 'parameters') config.setState(False, 'resync', detail = 'parameters') needResync = False pHash = self._rawSource.getHash() self._storedHash = config.get('parameter hash', pHash, persistent = True) if self._storedHash != pHash: needResync = True # Resync needed if parameters have changed self._log.info('Parameter hash has changed') self._log.debug('\told hash: %s', self._storedHash) self._log.debug('\tnew hash: %s', pHash) config.setState(True, 'init', detail = 'config') doResync = (userResync or needResync) and not doInit if not doResync and not doInit: # Reuse old mapping activity = utils.ActivityLog('Loading cached parameter information') self._readJob2PID() activity.finish() return elif doResync: # Perform sync activity = utils.ActivityLog('Syncronizing parameter information') self._storedHash = None self._resyncState = self.resync() activity.finish() elif doInit: # Write current state self._writeJob2PID(self._pathJob2PID) ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams, self) config.set('parameter hash', self._rawSource.getHash())
def _createPSpace(self, args): SubSpaceParameterSource = ParameterSource.getClass('SubSpaceParameterSource') if len(args) == 1: return SubSpaceParameterSource.create(self._paramConfig, args[0]) elif len(args) == 3: return SubSpaceParameterSource.create(self._paramConfig, args[2], args[0]) else: raise APIError('Invalid subspace reference!: %r' % args)
def wrapper(*args): parameterClass = ParameterSource.getClass(clsName) try: return parameterClass.create(self._paramConfig, self._repository, *args) except Exception: raise ParameterError( 'Error while creating %r with arguments %r' % (parameterClass.__name__, args))
def wrapper(*args): try: parameterClass = ParameterSource.getClass(clsName) except Exception: raise ParameterError('Unable to create parameter source "%r"!' % clsName) try: return parameterClass.create(self._paramConfig, *args) except Exception: raise ParameterError('Error while creating "%r" with arguments "%r"' % (parameterClass.__name__, args))