def _resyncInternal(self): # This function is _VERY_ time critical! tmp = self._rawSource.resync() # First ask about psource changes (redoNewPNum, disableNewPNum, sizeChange) = (set(tmp[0]), set(tmp[1]), tmp[2]) hashNew = self._rawSource.getHash() hashChange = self._storedHash != hashNew self._storedHash = hashNew if not (redoNewPNum or disableNewPNum or sizeChange or hashChange): self._resyncState = None return psource_old = ParameterAdapter(None, ParameterSource.createInstance('GCDumpParameterSource', self._pathParams)) psource_new = ParameterAdapter(None, self._rawSource) mapJob2PID = {} (pAdded, pMissing, _) = self._diffParams(psource_old, psource_new, mapJob2PID, redoNewPNum, disableNewPNum) self._source = self._getResyncSource(psource_old, psource_new, mapJob2PID, pAdded, pMissing, disableNewPNum) self._mapJob2PID = mapJob2PID # Update Job2PID map redoNewPNum = redoNewPNum.difference(disableNewPNum) if redoNewPNum or disableNewPNum: mapPID2Job = dict(ismap(utils.swap, self._mapJob2PID.items())) translate = lambda pNum: mapPID2Job.get(pNum, pNum) self._resyncState = (set(imap(translate, redoNewPNum)), set(imap(translate, disableNewPNum)), sizeChange) elif sizeChange: self._resyncState = (set(), set(), sizeChange) # Write resynced state self._writeJob2PID(self._pathJob2PID + '.tmp') ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams + '.tmp', self) os.rename(self._pathJob2PID + '.tmp', self._pathJob2PID) os.rename(self._pathParams + '.tmp', self._pathParams)
def _resync_adapter(self, pa_old, pa_new, result_redo, result_disable, size_change): (map_jobnum2pnum, pspi_list_added, pspi_list_missing) = _diff_pspi_list(pa_old, pa_new, result_redo, result_disable) # Reorder and reconstruct parameter space with the following layout: # NNNNNNNNNNNNN OOOOOOOOO | source: NEW (==self) and OLD (==from file) # <same><added> <missing> | same: both in NEW and OLD, added: only in NEW, missing: only in OLD if pspi_list_added: _extend_map_jobnum2pnum(map_jobnum2pnum, pa_old.get_job_len(), pspi_list_added) if pspi_list_missing: # extend the parameter source by placeholders for the missing parameter space points psrc_missing = _create_placeholder_psrc(pa_old, pa_new, map_jobnum2pnum, pspi_list_missing, result_disable) self._psrc = ParameterSource.create_instance('ChainParameterSource', self._psrc_raw, psrc_missing) self._map_jobnum2pnum = map_jobnum2pnum # Update Job2PID map # Write resynced state self._write_jobnum2pnum(self._path_jobnum2pnum + '.tmp') ParameterSource.get_class('GCDumpParameterSource').write(self._path_params + '.tmp', self.get_job_len(), self.get_job_metadata(), self.iter_jobs()) os.rename(self._path_jobnum2pnum + '.tmp', self._path_jobnum2pnum) os.rename(self._path_params + '.tmp', self._path_params) result_redo = result_redo.difference(result_disable) if result_redo or result_disable: map_pnum2jobnum = reverse_dict(self._map_jobnum2pnum) def _translate_pnum(pnum): return map_pnum2jobnum.get(pnum, pnum) result_redo = set(imap(_translate_pnum, result_redo)) result_disable = set(imap(_translate_pnum, result_disable)) return (result_redo, result_disable, size_change) return (set(), set(), size_change)
def createLookupHelper(pconfig, var_list, lookup_list): # Return list of (doElevate, PSourceClass, arguments) entries if len(var_list) != 1: # multi-lookup handling result = [] for var_name in var_list: result.extend(createLookupHelper(pconfig, [var_name], lookup_list)) return result var_name = var_list[0] pvalue = pconfig.getParameter(var_name.lstrip('!')) if isinstance(pvalue, list): # simple parameter source if len(pvalue) == 1: return [(False, ParameterSource.getClass('ConstParameterSource'), [var_name, pvalue[0]])] else: return [(False, ParameterSource.getClass('SimpleParameterSource'), [var_name, pvalue])] elif isinstance(pvalue, tuple) and pvalue[0] == 'format': return [(False, ParameterSource.getClass('FormatterParameterSource'), pvalue[1:])] lookup_key = None if lookup_list: # default lookup key lookup_key = KeyParameterSource(*lookup_list) # Determine kind of lookup, [3] == lookupDictConfig, [0] == lookupContent tmp = lookupConfigParser(pconfig, KeyParameterSource(var_name), lookup_key) lookupContent = tmp[3][0] lookupLen = lmap(len, lookupContent.values()) if (min(lookupLen) == 1) and (max(lookupLen) == 1): # simple lookup sufficient for this setup return [(False, SimpleLookupParameterSource, list(tmp))] # switch needs elevation beyond local scope return [(True, SwitchingLookupParameterSource, list(tmp))]
def _tree2expr(self, node, repository): if isinstance(node, int): return node elif isinstance(node, tuple): (operator, args) = node if operator == '[]': psrc_list = [] for output_vn in _tree2names(args[0]): psrc_list.append( ParameterSource.create_instance( 'InternalAutoParameterSource', self._parameter_config, repository, output_vn, _tree2names(args[1]))) return ParameterSource.create_psrc_safe( 'CrossParameterSource', self._parameter_config, repository, *psrc_list) elif operator in self._operator_map_raw: return ParameterSource.create_psrc_safe( self._operator_map_raw[operator], self._parameter_config, repository, *args) elif operator in self._operator_map_eval: evaluated_args = lmap( lambda node: self._tree2expr(node, repository), args) return ParameterSource.create_psrc_safe( self._operator_map_eval[operator], self._parameter_config, repository, *evaluated_args) else: return ParameterSource.create_instance( 'InternalAutoParameterSource', self._parameter_config, repository, node) raise APIError('Unable to parse node %s!' % repr(node))
def _createAggregatedSource(self, psource_old, psource_new, missingInfos): currentInfoKeys = psource_new.getJobKeys() missingInfoKeys = lfilter(lambda key: key not in currentInfoKeys, psource_old.getJobKeys()) ps_miss = ParameterSource.createInstance('InternalParameterSource', missingInfos, missingInfoKeys) return ParameterSource.createInstance('ChainParameterSource', self._rawSource, ps_miss)
def __init__(self, config, source): self._rawSource = source BasicParameterAdapter.__init__(self, config, source) self._mapJob2PID = {} if not os.path.isdir(config.getWorkPath()): os.makedirs(config.getWorkPath()) self._pathJob2PID = config.getWorkPath('params.map.gz') self._pathParams = config.getWorkPath('params.dat.gz') # Find out if init should be performed - overrides userResync! userInit = config.getState('init', detail='parameters') needInit = False if not (os.path.exists(self._pathParams) and os.path.exists(self._pathJob2PID)): needInit = True # Init needed if no parameter log exists if userInit and not needInit and (source.getMaxParameters() is not None): utils.eprint( 'Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!' ) if utils.getUserBool( 'Do you want to perform a syncronization between the current mapping and the new one to avoid this?', True): userInit = False doInit = userInit or needInit # Find out if resync should be performed userResync = config.getState('resync', detail='parameters') config.setState(False, 'resync', detail='parameters') needResync = False pHash = self._rawSource.getHash() self.storedHash = config.get('parameter hash', pHash, persistent=True) if self.storedHash != pHash: needResync = True # Resync needed if parameters have changed self._log.info('Parameter hash has changed') self._log.debug('\told hash: %s', self.storedHash) self._log.debug('\tnew hash: %s', pHash) config.setState(True, 'init', detail='config') doResync = (userResync or needResync) and not doInit if not doResync and not doInit: # Reuse old mapping activity = utils.ActivityLog( 'Loading cached parameter information') self.readJob2PID() activity.finish() return elif doResync: # Perform sync activity = utils.ActivityLog('Syncronizing parameter information') self.storedHash = None self._resyncState = self.resync() activity.finish() elif doInit: # Write current state self.writeJob2PID(self._pathJob2PID) ParameterSource.getClass('GCDumpParameterSource').write( self._pathParams, self) config.set('parameter hash', self._rawSource.getHash())
def __new__(cls, pconfig, repository, output_vn, lookup_vn_list=None): parameter_value = pconfig.get_parameter(output_vn.lstrip('!')) if isinstance(parameter_value, list): if len(parameter_value) != 1: # Simplify single value parameters to const parameters return ParameterSource.create_instance('SimpleParameterSource', output_vn, parameter_value) return ParameterSource.create_instance('ConstParameterSource', output_vn, parameter_value[0]) elif isinstance(parameter_value, tuple) and isinstance(parameter_value[0], str): return ParameterSource.create_instance(*parameter_value) return ParameterSource.create_instance('InternalAutoLookupParameterSource', pconfig, output_vn, lookup_vn_list)
def getSource(self): source_list = self._constSources + [self._pfactory.getSource()] + self._lookupSources source = ParameterSource.createInstance('ZipLongParameterSource', *source_list) for (PSourceClass, args) in self._nestedSources: source = PSourceClass(source, *args) if self._req: req_source = ParameterSource.createInstance('RequirementParameterSource') source = ParameterSource.createInstance('ZipLongParameterSource', source, req_source) source = self._useAvailableDataSource(source) return ParameterSource.createInstance('RepeatParameterSource', source, self._repeat)
def __init__(self, config, source): self._psrc_raw = source BasicParameterAdapter.__init__(self, config, source) self._map_jobnum2pnum = {} ensure_dir_exists(config.get_work_path(), 'parameter storage directory', ParameterError) self._path_jobnum2pnum = config.get_work_path('params.map.gz') self._path_params = config.get_work_path('params.dat.gz') # Find out if init should be performed - overrides resync_requested! init_requested = config.get_state('init', detail='parameters') init_needed = False if not (os.path.exists(self._path_params) and os.path.exists(self._path_jobnum2pnum)): init_needed = True # Init needed if no parameter log exists if init_requested and not init_needed and (source.get_parameter_len() is not None): self._log.warning('Re-Initialization will overwrite the current mapping ' + 'between jobs and parameter/dataset content! This can lead to invalid results!') user_msg = ('Do you want to perform a syncronization between ' + 'the current mapping and the new one to avoid this?') if UserInputInterface().prompt_bool(user_msg, True): init_requested = False do_init = init_requested or init_needed # Find out if resync should be performed resync_by_user = config.get_state('resync', detail='parameters') config.set_state(False, 'resync', detail='parameters') psrc_hash = self._psrc_raw.get_psrc_hash() self._psrc_hash_stored = config.get('parameter hash', psrc_hash, persistent=True) psrc_hash_changed = self._psrc_hash_stored != psrc_hash # Resync if parameters have changed resync_by_psrc = self._psrc_raw.get_resync_request() if do_init: # Write current state self._write_jobnum2pnum(self._path_jobnum2pnum) ParameterSource.get_class('GCDumpParameterSource').write(self._path_params, self.get_job_len(), self.get_job_metadata(), self.iter_jobs()) elif resync_by_user or resync_by_psrc or psrc_hash_changed: # Perform sync if psrc_hash_changed: self._log.info('Parameter hash has changed') self._log.debug('\told hash: %s', self._psrc_hash_stored) self._log.debug('\tnew hash: %s', psrc_hash) self._log.log(logging.DEBUG1, '\tnew src: %s', self._psrc_raw) config.set_state(True, 'init', detail='config') elif resync_by_psrc: self._log.info('Parameter source requested resync') self._log.debug('\t%r', str.join(', ', imap(repr, resync_by_psrc))) elif resync_by_user: self._log.info('User requested resync') self._psrc_hash_stored = None self._resync_state = self.resync(force=True) else: # Reuse old mapping activity = Activity('Loading cached parameter information') self._read_jobnum2pnum() activity.finish() return # do not set parameter hash in config config.set('parameter hash', self._psrc_raw.get_psrc_hash())
def _createRef(self, arg): refTypeDefault = 'dataset' DataParameterSource = ParameterSource.getClass('DataParameterSource') if arg not in DataParameterSource.datasetsAvailable: refTypeDefault = 'csv' refType = self._paramConfig.get(arg, 'type', refTypeDefault) if refType == 'dataset': return DataParameterSource.create(self._paramConfig, arg) elif refType == 'csv': return ParameterSource.getClass('CSVParameterSource').create(self._paramConfig, arg) raise APIError('Unknown reference type: "%s"' % refType)
def _combineSources(self, clsName, args): repeat = reduce(lambda a, b: a * b, ifilter(lambda expr: isinstance(expr, int), args), 1) args = lfilter(lambda expr: not isinstance(expr, int), args) if args: result = ParameterSource.createInstance(clsName, *args) if repeat > 1: return ParameterSource.createInstance('RepeatParameterSource', result, repeat) return result elif repeat > 1: return repeat return NullParameterSource()
def __init__(self, fn): ParameterSource.__init__(self) fp = gzip.open(fn, 'rb') keyline = fp.readline().lstrip('#').strip() self.keys = [] if keyline: self.keys = eval(keyline) def parseLine(line): if not line.startswith('#'): pNumStr, stored = map(str.strip, line.split('\t', 1)) return ('!' in pNumStr, int(pNumStr.rstrip('!')), map(eval, stored.split('\t'))) self.values = map(parseLine, fp.readlines())
def _resync(self): # This function is _VERY_ time critical! tmp = self._psrc_raw.resync_psrc() # First ask about psrc changes (result_redo, result_disable, size_change) = (set(tmp[0]), set(tmp[1]), tmp[2]) psrc_hash_new = self._psrc_raw.get_psrc_hash() psrc_hash_changed = self._psrc_hash_stored != psrc_hash_new self._psrc_hash_stored = psrc_hash_new if not (result_redo or result_disable or size_change or psrc_hash_changed): return ParameterSource.get_empty_resync_result() ps_old = ParameterSource.create_instance('GCDumpParameterSource', self._path_params) pa_old = ParameterAdapter(None, ps_old) pa_new = ParameterAdapter(None, self._psrc_raw) return self._resync_adapter(pa_old, pa_new, result_redo, result_disable, size_change)
def _addConstantPSource(self, config, cName, varName): lookupVar = config.get('%s lookup' % cName, '', onChange = None) if lookupVar: matcher = Matcher.createInstance(config.get('%s matcher' % cName, 'start', onChange = None), config, cName) content = config.getDict(cName, {}, onChange = None) content_fixed = {} content_order = lmap(lambda x: (x,), content[1]) for key in content[0]: content_fixed[(key,)] = (content[0][key],) ps = ParameterSource.createInstance('SimpleLookupParameterSource', varName, [lookupVar], [matcher], (content_fixed, content_order)) self.lookupSources.append(ps) else: ps = ParameterSource.createInstance('ConstParameterSource', varName, config.get(cName).strip()) self.constSources.append(ps)
def resync(self, force = False): # Do not overwrite resync results - eg. from external or init trigger source_hash = self._source.getHash() if (self._resync_state == ParameterSource.EmptyResyncResult()) and ((source_hash != self._source_hash) or force): activity = Activity('Syncronizing parameter information') t_start = time.time() try: self._resync_state = self._resync() except Exception: raise ParameterError('Unable to resync parameters!') self._source_hash = self._source.getHash() activity.finish() self._log.log(logging.INFO, 'Finished resync of parameter source (%s)', strTimeShort(time.time() - t_start)) result = self._resync_state self._resync_state = ParameterSource.EmptyResyncResult() return result
def __init__(self, dataDir, srcName, dataProvider, dataSplitter, dataProc): ParameterSource.__init__(self) (self.dataDir, self.srcName, self.dataProvider, self.dataSplitter, self.dataProc) = \ (dataDir, srcName, dataProvider, dataSplitter, dataProc) if not dataProvider: pass # debug mode - used by scripts - disables resync elif os.path.exists(self.getDataPath('cache.dat') and self.getDataPath('map.tar')): self.dataSplitter.importState(self.getDataPath('map.tar')) else: self.dataProvider.saveState(self.getDataPath('cache.dat')) self.dataSplitter.splitDataset(self.getDataPath('map.tar'), self.dataProvider.getBlocks()) self.maxN = self.dataSplitter.getMaxJobs() self.keepOld = True
def __init__(self, dataDir, srcName, dataProvider, dataSplitter, dataProc, keepOld = True): ParameterSource.__init__(self) (self._dataDir, self._srcName, self._dataProvider, self._dataSplitter, self._part_proc) = \ (dataDir, srcName, dataProvider, dataSplitter, dataProc) if not dataProvider: pass # debug mode - used by scripts - disables resync elif os.path.exists(self.getDataPath('cache.dat') and self.getDataPath('map.tar')): self._dataSplitter.importPartitions(self.getDataPath('map.tar')) else: DataProvider.saveToFile(self.getDataPath('cache.dat'), self._dataProvider.getBlocks(silent = False)) self._dataSplitter.splitDataset(self.getDataPath('map.tar'), self._dataProvider.getBlocks()) self._maxN = self._dataSplitter.getMaxJobs() self._keepOld = keepOld
def __init__(self, fn): ParameterSource.__init__(self) fp = ZipFile(fn, 'r') try: keyline = fp.readline().lstrip('#').strip() self._keys = [] if keyline: self._keys = parseJSON(keyline) def parseLine(line): if not line.startswith('#'): pNumStr, stored = lmap(str.strip, line.split('\t', 1)) return ('!' in pNumStr, int(pNumStr.rstrip('!')), lmap(parseJSON, stored.split('\t'))) self._values = lmap(parseLine, fp.readlines()) finally: fp.close()
def __init__(self, config, source): self._rawSource = source BasicParameterAdapter.__init__(self, config, source) self._mapJob2PID = {} if not os.path.isdir(config.getWorkPath()): os.makedirs(config.getWorkPath()) self._pathJob2PID = config.getWorkPath('params.map.gz') self._pathParams = config.getWorkPath('params.dat.gz') # Find out if init should be performed - overrides userResync! userInit = config.getState('init', detail = 'parameters') needInit = False if not (os.path.exists(self._pathParams) and os.path.exists(self._pathJob2PID)): needInit = True # Init needed if no parameter log exists if userInit and not needInit and (source.getMaxParameters() is not None): utils.eprint('Re-Initialization will overwrite the current mapping between jobs and parameter/dataset content! This can lead to invalid results!') if utils.getUserBool('Do you want to perform a syncronization between the current mapping and the new one to avoid this?', True): userInit = False doInit = userInit or needInit # Find out if resync should be performed userResync = config.getState('resync', detail = 'parameters') config.setState(False, 'resync', detail = 'parameters') needResync = False pHash = self._rawSource.getHash() self._storedHash = config.get('parameter hash', pHash, persistent = True) if self._storedHash != pHash: needResync = True # Resync needed if parameters have changed self._log.info('Parameter hash has changed') self._log.debug('\told hash: %s', self._storedHash) self._log.debug('\tnew hash: %s', pHash) config.setState(True, 'init', detail = 'config') doResync = (userResync or needResync) and not doInit if not doResync and not doInit: # Reuse old mapping activity = utils.ActivityLog('Loading cached parameter information') self._readJob2PID() activity.finish() return elif doResync: # Perform sync activity = utils.ActivityLog('Syncronizing parameter information') self._storedHash = None self._resyncState = self.resync() activity.finish() elif doInit: # Write current state self._writeJob2PID(self._pathJob2PID) ParameterSource.getClass('GCDumpParameterSource').write(self._pathParams, self) config.set('parameter hash', self._rawSource.getHash())
def create_psrc(cls, pconfig, repository, ref_name, *args): # pylint:disable=arguments-differ ref_type_default = 'data' if 'dataset:' + ref_name not in repository: ref_type_default = 'csv' ref_type = pconfig.get(ref_name, 'type', ref_type_default) return ParameterSource.create_psrc_safe(ref_type, pconfig, repository, ref_name, *args)
def __new__(cls, *psources): psources = strip_null_sources(psources) if len(psources) == 1: return psources[0] elif not psources: return NullParameterSource() return ParameterSource.__new__(cls)
def _useAvailableDataSource(self, source): DataParameterSource = Plugin.getClass('DataParameterSource') if DataParameterSource.datasetsAvailable and not DataParameterSource.datasetsUsed: if source is not None: return ParameterSource.createInstance('CrossParameterSource', DataParameterSource.create(), source) return DataParameterSource.create() return source
def _getUserSource(self, pExpr): # Wrap psource factory functions def createWrapper(clsName): def wrapper(*args): parameterClass = ParameterSource.getClass(clsName) try: return parameterClass.create(self._paramConfig, self._repository, *args) except Exception: raise ParameterError( 'Error while creating %r with arguments %r' % (parameterClass.__name__, args)) return wrapper userFun = {} for clsInfo in ParameterSource.getClassList(): for clsName in ifilter(lambda name: name != 'depth', clsInfo.keys()): userFun[clsName] = createWrapper(clsName) try: return eval(pExpr, dict(userFun)) # pylint:disable=eval-used except Exception: self._log.warning('Available functions: %s', sorted(userFun.keys())) raise
def resync(self): (result_redo, result_disable, result_sizeChange) = ParameterSource.resync(self) if self.resyncEnabled() and self._dataProvider: # Get old and new dataset information old = DataProvider.loadFromFile(self.getDataPath('cache.dat')).getBlocks() self._dataProvider.clearCache() new = self._dataProvider.getBlocks() self._dataProvider.saveToFile(self.getDataPath('cache-new.dat'), new) # Use old splitting information to synchronize with new dataset infos jobChanges = self._dataSplitter.resyncMapping(self.getDataPath('map-new.tar'), old, new) if jobChanges: # Move current splitting to backup and use the new splitting from now on def backupRename(old, cur, new): if self._keepOld: os.rename(self.getDataPath(cur), self.getDataPath(old)) os.rename(self.getDataPath(new), self.getDataPath(cur)) backupRename( 'map-old-%d.tar' % time.time(), 'map.tar', 'map-new.tar') backupRename('cache-old-%d.dat' % time.time(), 'cache.dat', 'cache-new.dat') old_maxN = self._dataSplitter.getMaxJobs() self._dataSplitter.importPartitions(self.getDataPath('map.tar')) self._maxN = self._dataSplitter.getMaxJobs() result_redo.update(jobChanges[0]) result_disable.update(jobChanges[1]) result_sizeChange = result_sizeChange or (old_maxN != self._maxN) self.resyncFinished() return (result_redo, result_disable, result_sizeChange)
def resync(self): # Quicker version than the general purpose implementation result = ParameterSource.resync(self) for psource in self._psourceList: result = combineSyncResult(result, psource.resync()) oldMaxParameters = self._maxParameters self._maxParameters = self.initMaxParameters() return (result[0], result[1], oldMaxParameters != self._maxParameters)
def resync_psrc(self): # Quicker version than the general purpose implementation result = ParameterSource.get_empty_resync_result() for psrc in self._psrc_list: result = _combine_resync_result(result, psrc.resync_psrc()) psrc_max_old = self._psrc_max self._psrc_max_list = lmap(lambda p: p.get_parameter_len(), self._psrc_list) self._psrc_max = self._init_psrc_max() return (result[0], result[1], psrc_max_old != self._psrc_max)
def _useAvailableDataSource(self, source): usedSources = source.getUsedSources() for (srcName, dataSource) in self._repository.items(): if srcName.startswith('dataset:') and (dataSource not in usedSources): source = ParameterSource.createInstance( 'CrossParameterSource', dataSource, source) return source
def _createPSpace(self, args): SubSpaceParameterSource = ParameterSource.getClass('SubSpaceParameterSource') if len(args) == 1: return SubSpaceParameterSource.create(self._paramConfig, args[0]) elif len(args) == 3: return SubSpaceParameterSource.create(self._paramConfig, args[2], args[0]) else: raise APIError('Invalid subspace reference!: %r' % args)
def wrapper(*args): try: parameterClass = ParameterSource.getClass(clsName) except Exception: raise ParameterError('Unable to create parameter source "%r"!' % clsName) try: return parameterClass.create(self._paramConfig, *args) except Exception: raise ParameterError('Error while creating "%r" with arguments "%r"' % (parameterClass.__name__, args))
def _createVarSource(self, var_list, lookup_list): # create variable source psource_list = [] for (doElevate, PSourceClass, args) in createLookupHelper(self._paramConfig, var_list, lookup_list): if doElevate: # switch needs elevation beyond local scope self._nestedSources.append((PSourceClass, args)) else: psource_list.append(PSourceClass(*args)) # Optimize away unnecessary cross operations return ParameterSource.createInstance('CrossParameterSource', *psource_list)
def __new__(cls, *psrc_list): (repeat, psrc_list) = _separate_repeat(_strip_null_sources(psrc_list)) if repeat != 1: return RepeatParameterSource(cls(*psrc_list), repeat) elif not psrc_list: return NullParameterSource() elif len(psrc_list) == 1: return psrc_list[0] return ParameterSource.__new__(cls)
def resync(self, force=False): source_hash = self._psrc.get_psrc_hash() do_resync = (source_hash != self._psrc_hash) or self._psrc.get_resync_request() or force # Do not overwrite resync results - eg. from external or init trigger if (self._resync_state == ParameterSource.get_empty_resync_result()) and do_resync: activity = Activity('Syncronizing parameter information') t_start = time.time() try: self._resync_state = self._resync() except Exception: raise ParameterError('Unable to resync parameters!') self._psrc_hash = self._psrc.get_psrc_hash() activity.finish() self._log.log(logging.INFO, 'Finished resync of parameter source (%s)', str_time_short(time.time() - t_start)) result = self._resync_state self._resync_state = ParameterSource.get_empty_resync_result() return result
def __init__(self, fn): ParameterSource.__init__(self) fp = GZipTextFile(fn, 'r') try: header = fp.readline().lstrip('#').strip() self._output_vn_list = [] if header: self._output_vn_list = parse_json(header) def _parse_line(line): if not line.startswith('#'): pnum_str, stored_json = line.split('\t', 1) is_invalid = '!' in pnum_str pnum = int(pnum_str.replace('!', ' ')) return (is_invalid, pnum, lmap(parse_json, stored_json.strip().split('\t'))) self._values = lmap(_parse_line, fp.readlines()) finally: fp.close()
def create_psrc(cls, pconfig, repository, output_vn, lookup_vn_list=None): # pylint:disable=arguments-differ parameter_value = pconfig.get_parameter(output_vn.lstrip('!')) if isinstance(parameter_value, list): if len( parameter_value ) != 1: # Simplify single value parameters to const parameters return ParameterSource.create_instance('SimpleParameterSource', output_vn, parameter_value) return ParameterSource.create_instance('ConstParameterSource', output_vn, parameter_value[0]) elif isinstance(parameter_value, tuple) and not isinstance(parameter_value[0], dict): return ParameterSource.create_instance(*parameter_value) return ParameterSource.create_instance( 'InternalAutoLookupParameterSource', pconfig, output_vn, lookup_vn_list)
def _register_psrc(self, pconfig, output_vn): def _replace_nonalnum(value): if str.isalnum(value): return value return ' ' lookup_str = pconfig.get(output_vn, 'lookup', '') lookup_vn_list = lidfilter(str.join('', imap(_replace_nonalnum, lookup_str)).split()) self._psrc_list.append(ParameterSource.create_psrc_safe('InternalAutoParameterSource', pconfig, {}, output_vn, lookup_vn_list))
def wrapper(*args): parameterClass = ParameterSource.getClass(clsName) try: return parameterClass.create(self._paramConfig, self._repository, *args) except Exception: raise ParameterError( 'Error while creating %r with arguments %r' % (parameterClass.__name__, args))
def resync_psrc( self): # Quicker version than the general purpose implementation result = ParameterSource.get_empty_resync_result() for psrc in self._psrc_list: result = _combine_resync_result(result, psrc.resync_psrc()) psrc_max_old = self._psrc_max self._psrc_max_list = lmap(lambda p: p.get_parameter_len(), self._psrc_list) self._psrc_max = self._init_psrc_max() return (result[0], result[1], psrc_max_old != self._psrc_max)
def __new__(cls, psrc, times): # pylint:disable=arguments-differ if times < 0: return psrc elif psrc.get_parameter_len() is None: return TruncateParameterSource(psrc, times) elif times == 0: return NullParameterSource() elif times == 1: return psrc return ParameterSource.__new__(cls)
def resync_psrc(self): (result_redo, result_disable, _) = ParameterSource.get_empty_resync_result() (psrc_redo, psrc_disable, psrc_size_change) = self._psrc.resync_psrc() self._psp_field = self._init_psp_field() for pnum, psp_info in enumerate(self._psp_field): psrc_pnum, _ = psp_info # ignore output_idx if psrc_pnum in psrc_redo: result_redo.add(pnum) if psrc_pnum in psrc_disable: result_disable.add(pnum) return (result_redo, result_disable, psrc_size_change)
def resync(self): (result_redo, result_disable, result_sizeChange) = ParameterSource.resync(self) (psource_redo, psource_disable, _) = self._psource.resync() # size change is irrelevant if outside of range for pNum in psource_redo: if (pNum >= self._posStart) and (pNum <= self._posEnd): result_redo.add(pNum - self._posStart) for pNum in psource_disable: if (pNum >= self._posStart) and (pNum <= self._posEnd): result_disable.add(pNum - self._posStart) oldPosEnd = self._posEnd self._posEnd = utils.QM(self._posEndUser is None, self._psource.getMaxParameters() - 1, self._posEndUser) return (result_redo, result_disable, result_sizeChange or (oldPosEnd != self._posEnd))
def resync(self): (result_redo, result_disable, result_sizeChange) = ParameterSource.resync(self) if self.resyncEnabled(): (psource_redo, psource_disable, psource_sizeChange) = self._psource.resync() self._pSpace = self.initPSpace() for pNum, pInfo in enumerate(self._pSpace): subNum, _ = pInfo # ignore lookupIndex if subNum in psource_redo: result_redo.add(pNum) if subNum in psource_disable: result_disable.add(pNum) self.resyncFinished() return (result_redo, result_disable, result_sizeChange or psource_sizeChange)