def _resync_adapter(self, pa_old, pa_new, result_redo, result_disable, size_change): (map_jobnum2pnum, pspi_list_added, pspi_list_missing) = _diff_pspi_list(pa_old, pa_new, result_redo, result_disable) # Reorder and reconstruct parameter space with the following layout: # NNNNNNNNNNNNN OOOOOOOOO | source: NEW (==self) and OLD (==from file) # <same><added> <missing> | same: both in NEW and OLD, added: only in NEW, missing: only in OLD if pspi_list_added: _extend_map_jobnum2pnum(map_jobnum2pnum, pa_old.get_job_len(), pspi_list_added) if pspi_list_missing: # extend the parameter source by placeholders for the missing parameter space points psrc_missing = _create_placeholder_psrc(pa_old, pa_new, map_jobnum2pnum, pspi_list_missing, result_disable) self._psrc = ParameterSource.create_instance('ChainParameterSource', self._psrc_raw, psrc_missing) self._map_jobnum2pnum = map_jobnum2pnum # Update Job2PID map # Write resynced state self._write_jobnum2pnum(self._path_jobnum2pnum + '.tmp') ParameterSource.get_class('GCDumpParameterSource').write(self._path_params + '.tmp', self.get_job_len(), self.get_job_metadata(), self.iter_jobs()) os.rename(self._path_jobnum2pnum + '.tmp', self._path_jobnum2pnum) os.rename(self._path_params + '.tmp', self._path_params) result_redo = result_redo.difference(result_disable) if result_redo or result_disable: map_pnum2jobnum = reverse_dict(self._map_jobnum2pnum) def _translate_pnum(pnum): return map_pnum2jobnum.get(pnum, pnum) result_redo = set(imap(_translate_pnum, result_redo)) result_disable = set(imap(_translate_pnum, result_disable)) return (result_redo, result_disable, size_change) return (set(), set(), size_change)
def __init__(self, config, source): self._psrc_raw = source BasicParameterAdapter.__init__(self, config, source) self._map_jobnum2pnum = {} ensure_dir_exists(config.get_work_path(), 'parameter storage directory', ParameterError) self._path_jobnum2pnum = config.get_work_path('params.map.gz') self._path_params = config.get_work_path('params.dat.gz') # Find out if init should be performed - overrides resync_requested! init_requested = config.get_state('init', detail='parameters') init_needed = False if not (os.path.exists(self._path_params) and os.path.exists(self._path_jobnum2pnum)): init_needed = True # Init needed if no parameter log exists if init_requested and not init_needed and (source.get_parameter_len() is not None): self._log.warning('Re-Initialization will overwrite the current mapping ' + 'between jobs and parameter/dataset content! This can lead to invalid results!') user_msg = ('Do you want to perform a syncronization between ' + 'the current mapping and the new one to avoid this?') if UserInputInterface().prompt_bool(user_msg, True): init_requested = False do_init = init_requested or init_needed # Find out if resync should be performed resync_by_user = config.get_state('resync', detail='parameters') config.set_state(False, 'resync', detail='parameters') psrc_hash = self._psrc_raw.get_psrc_hash() self._psrc_hash_stored = config.get('parameter hash', psrc_hash, persistent=True) psrc_hash_changed = self._psrc_hash_stored != psrc_hash # Resync if parameters have changed resync_by_psrc = self._psrc_raw.get_resync_request() if do_init: # Write current state self._write_jobnum2pnum(self._path_jobnum2pnum) ParameterSource.get_class('GCDumpParameterSource').write(self._path_params, self.get_job_len(), self.get_job_metadata(), self.iter_jobs()) elif resync_by_user or resync_by_psrc or psrc_hash_changed: # Perform sync if psrc_hash_changed: self._log.info('Parameter hash has changed') self._log.debug('\told hash: %s', self._psrc_hash_stored) self._log.debug('\tnew hash: %s', psrc_hash) self._log.log(logging.DEBUG1, '\tnew src: %s', self._psrc_raw) config.set_state(True, 'init', detail='config') elif resync_by_psrc: self._log.info('Parameter source requested resync') self._log.debug('\t%r', str.join(', ', imap(repr, resync_by_psrc))) elif resync_by_user: self._log.info('User requested resync') self._psrc_hash_stored = None self._resync_state = self.resync(force=True) else: # Reuse old mapping activity = Activity('Loading cached parameter information') self._read_jobnum2pnum() activity.finish() return # do not set parameter hash in config config.set('parameter hash', self._psrc_raw.get_psrc_hash())