def _resync_adapter(self, pa_old, pa_new, result_redo, result_disable, size_change):
(map_jobnum2pnum, pspi_list_added, pspi_list_missing) = _diff_pspi_list(pa_old, pa_new,
result_redo, result_disable)
# Reorder and reconstruct parameter space with the following layout:
# NNNNNNNNNNNNN OOOOOOOOO | source: NEW (==self) and OLD (==from file)
# <same><added> <missing> | same: both in NEW and OLD, added: only in NEW, missing: only in OLD
if pspi_list_added:
_extend_map_jobnum2pnum(map_jobnum2pnum, pa_old.get_job_len(), pspi_list_added)
if pspi_list_missing:
# extend the parameter source by placeholders for the missing parameter space points
psrc_missing = _create_placeholder_psrc(pa_old, pa_new,
map_jobnum2pnum, pspi_list_missing, result_disable)
self._psrc = ParameterSource.create_instance('ChainParameterSource',
self._psrc_raw, psrc_missing)
self._map_jobnum2pnum = map_jobnum2pnum # Update Job2PID map
# Write resynced state
self._write_jobnum2pnum(self._path_jobnum2pnum + '.tmp')
ParameterSource.get_class('GCDumpParameterSource').write(self._path_params + '.tmp',
self.get_job_len(), self.get_job_metadata(), self.iter_jobs())
os.rename(self._path_jobnum2pnum + '.tmp', self._path_jobnum2pnum)
os.rename(self._path_params + '.tmp', self._path_params)
result_redo = result_redo.difference(result_disable)
if result_redo or result_disable:
map_pnum2jobnum = reverse_dict(self._map_jobnum2pnum)
def _translate_pnum(pnum):
return map_pnum2jobnum.get(pnum, pnum)
result_redo = set(imap(_translate_pnum, result_redo))
result_disable = set(imap(_translate_pnum, result_disable))
return (result_redo, result_disable, size_change)
return (set(), set(), size_change)
def __init__(self, config, source):
self._psrc_raw = source
BasicParameterAdapter.__init__(self, config, source)
self._map_jobnum2pnum = {}
ensure_dir_exists(config.get_work_path(), 'parameter storage directory', ParameterError)
self._path_jobnum2pnum = config.get_work_path('params.map.gz')
self._path_params = config.get_work_path('params.dat.gz')
# Find out if init should be performed - overrides resync_requested!
init_requested = config.get_state('init', detail='parameters')
init_needed = False
if not (os.path.exists(self._path_params) and os.path.exists(self._path_jobnum2pnum)):
init_needed = True # Init needed if no parameter log exists
if init_requested and not init_needed and (source.get_parameter_len() is not None):
self._log.warning('Re-Initialization will overwrite the current mapping ' +
'between jobs and parameter/dataset content! This can lead to invalid results!')
user_msg = ('Do you want to perform a syncronization between ' +
'the current mapping and the new one to avoid this?')
if UserInputInterface().prompt_bool(user_msg, True):
init_requested = False
do_init = init_requested or init_needed
# Find out if resync should be performed
resync_by_user = config.get_state('resync', detail='parameters')
config.set_state(False, 'resync', detail='parameters')
psrc_hash = self._psrc_raw.get_psrc_hash()
self._psrc_hash_stored = config.get('parameter hash', psrc_hash, persistent=True)
psrc_hash_changed = self._psrc_hash_stored != psrc_hash # Resync if parameters have changed
resync_by_psrc = self._psrc_raw.get_resync_request()
if do_init: # Write current state
self._write_jobnum2pnum(self._path_jobnum2pnum)
ParameterSource.get_class('GCDumpParameterSource').write(self._path_params,
self.get_job_len(), self.get_job_metadata(), self.iter_jobs())
elif resync_by_user or resync_by_psrc or psrc_hash_changed: # Perform sync
if psrc_hash_changed:
self._log.info('Parameter hash has changed')
self._log.debug('\told hash: %s', self._psrc_hash_stored)
self._log.debug('\tnew hash: %s', psrc_hash)
self._log.log(logging.DEBUG1, '\tnew src: %s', self._psrc_raw)
config.set_state(True, 'init', detail='config')
elif resync_by_psrc:
self._log.info('Parameter source requested resync')
self._log.debug('\t%r', str.join(', ', imap(repr, resync_by_psrc)))
elif resync_by_user:
self._log.info('User requested resync')
self._psrc_hash_stored = None
self._resync_state = self.resync(force=True)
else: # Reuse old mapping
activity = Activity('Loading cached parameter information')
self._read_jobnum2pnum()
activity.finish()
return # do not set parameter hash in config
config.set('parameter hash', self._psrc_raw.get_psrc_hash())