def _tree2expr(self, node, repository):
if isinstance(node, int):
return node
elif isinstance(node, tuple):
(operator, args) = node
if operator == '[]':
psrc_list = []
for output_vn in _tree2names(args[0]):
psrc_list.append(
ParameterSource.create_instance(
'InternalAutoParameterSource',
self._parameter_config, repository, output_vn,
_tree2names(args[1])))
return ParameterSource.create_psrc_safe(
'CrossParameterSource', self._parameter_config, repository,
*psrc_list)
elif operator in self._operator_map_raw:
return ParameterSource.create_psrc_safe(
self._operator_map_raw[operator], self._parameter_config,
repository, *args)
elif operator in self._operator_map_eval:
evaluated_args = lmap(
lambda node: self._tree2expr(node, repository), args)
return ParameterSource.create_psrc_safe(
self._operator_map_eval[operator], self._parameter_config,
repository, *evaluated_args)
else:
return ParameterSource.create_instance(
'InternalAutoParameterSource', self._parameter_config,
repository, node)
raise APIError('Unable to parse node %s!' % repr(node))
def __new__(cls, pconfig, repository, output_vn, lookup_vn_list=None):
parameter_value = pconfig.get_parameter(output_vn.lstrip('!'))
if isinstance(parameter_value, list):
if len(parameter_value) != 1: # Simplify single value parameters to const parameters
return ParameterSource.create_instance('SimpleParameterSource', output_vn, parameter_value)
return ParameterSource.create_instance('ConstParameterSource', output_vn, parameter_value[0])
elif isinstance(parameter_value, tuple) and isinstance(parameter_value[0], str):
return ParameterSource.create_instance(*parameter_value)
return ParameterSource.create_instance('InternalAutoLookupParameterSource',
pconfig, output_vn, lookup_vn_list)
def _resync_adapter(self, pa_old, pa_new, result_redo, result_disable, size_change):
(map_jobnum2pnum, pspi_list_added, pspi_list_missing) = _diff_pspi_list(pa_old, pa_new,
result_redo, result_disable)
# Reorder and reconstruct parameter space with the following layout:
# NNNNNNNNNNNNN OOOOOOOOO | source: NEW (==self) and OLD (==from file)
# <same><added> <missing> | same: both in NEW and OLD, added: only in NEW, missing: only in OLD
if pspi_list_added:
_extend_map_jobnum2pnum(map_jobnum2pnum, pa_old.get_job_len(), pspi_list_added)
if pspi_list_missing:
# extend the parameter source by placeholders for the missing parameter space points
psrc_missing = _create_placeholder_psrc(pa_old, pa_new,
map_jobnum2pnum, pspi_list_missing, result_disable)
self._psrc = ParameterSource.create_instance('ChainParameterSource',
self._psrc_raw, psrc_missing)
self._map_jobnum2pnum = map_jobnum2pnum # Update Job2PID map
# Write resynced state
self._write_jobnum2pnum(self._path_jobnum2pnum + '.tmp')
ParameterSource.get_class('GCDumpParameterSource').write(self._path_params + '.tmp',
self.get_job_len(), self.get_job_metadata(), self.iter_jobs())
os.rename(self._path_jobnum2pnum + '.tmp', self._path_jobnum2pnum)
os.rename(self._path_params + '.tmp', self._path_params)
result_redo = result_redo.difference(result_disable)
if result_redo or result_disable:
map_pnum2jobnum = reverse_dict(self._map_jobnum2pnum)
def _translate_pnum(pnum):
return map_pnum2jobnum.get(pnum, pnum)
result_redo = set(imap(_translate_pnum, result_redo))
result_disable = set(imap(_translate_pnum, result_disable))
return (result_redo, result_disable, size_change)
return (set(), set(), size_change)
def create_psrc(cls, pconfig, repository, output_vn, lookup_vn_list=None): # pylint:disable=arguments-differ
parameter_value = pconfig.get_parameter(output_vn.lstrip('!'))
if isinstance(parameter_value, list):
if len(
parameter_value
) != 1: # Simplify single value parameters to const parameters
return ParameterSource.create_instance('SimpleParameterSource',
output_vn,
parameter_value)
return ParameterSource.create_instance('ConstParameterSource',
output_vn,
parameter_value[0])
elif isinstance(parameter_value,
tuple) and not isinstance(parameter_value[0], dict):
return ParameterSource.create_instance(*parameter_value)
return ParameterSource.create_instance(
'InternalAutoLookupParameterSource', pconfig, output_vn,
lookup_vn_list)
def _resync(self): # This function is _VERY_ time critical!
tmp = self._psrc_raw.resync_psrc() # First ask about psrc changes
(result_redo, result_disable, size_change) = (set(tmp[0]), set(tmp[1]), tmp[2])
psrc_hash_new = self._psrc_raw.get_psrc_hash()
psrc_hash_changed = self._psrc_hash_stored != psrc_hash_new
self._psrc_hash_stored = psrc_hash_new
if not (result_redo or result_disable or size_change or psrc_hash_changed):
return ParameterSource.get_empty_resync_result()
ps_old = ParameterSource.create_instance('GCDumpParameterSource', self._path_params)
pa_old = ParameterAdapter(None, ps_old)
pa_new = ParameterAdapter(None, self._psrc_raw)
return self._resync_adapter(pa_old, pa_new, result_redo, result_disable, size_change)
def _tree2expr(self, node, repository):
if isinstance(node, int):
return node
elif isinstance(node, tuple):
(operator, args) = node
if operator == '[]':
psrc_list = []
for output_vn in _tree2names(args[0]):
psrc_list.append(ParameterSource.create_instance('InternalAutoParameterSource',
self._parameter_config, repository, output_vn, _tree2names(args[1])))
return ParameterSource.create_psrc_safe('CrossParameterSource',
self._parameter_config, repository, *psrc_list)
elif operator in self._operator_map_raw:
return ParameterSource.create_psrc_safe(self._operator_map_raw[operator],
self._parameter_config, repository, *args)
elif operator in self._operator_map_eval:
evaluated_args = lmap(lambda node: self._tree2expr(node, repository), args)
return ParameterSource.create_psrc_safe(self._operator_map_eval[operator],
self._parameter_config, repository, *evaluated_args)
else:
return ParameterSource.create_instance('InternalAutoParameterSource',
self._parameter_config, repository, node)
raise APIError('Unable to parse node %s!' % repr(node))
def _create_placeholder_psrc(pa_old, pa_new, map_jobnum2pnum, pspi_list_missing, result_disable):
# Construct placeholder parameter source with missing parameter entries and intervention state
psp_list_missing = []
missing_pnum_start = pa_new.get_job_len()
sort_inplace(pspi_list_missing, key=itemgetter(TrackingInfo.pnum))
for (idx, pspi_missing) in enumerate(pspi_list_missing):
map_jobnum2pnum[pspi_missing[TrackingInfo.pnum]] = missing_pnum_start + idx
psp_missing = pa_old.get_job_content(missing_pnum_start + idx, pspi_missing[TrackingInfo.pnum])
psp_missing.pop('GC_PARAM')
if psp_missing[ParameterInfo.ACTIVE]:
psp_missing[ParameterInfo.ACTIVE] = False
result_disable.add(missing_pnum_start + idx)
psp_list_missing.append(psp_missing)
meta_list_new = pa_new.get_job_metadata()
meta_name_list_new = lmap(lambda key: key.value, meta_list_new)
meta_list_old = pa_old.get_job_metadata()
meta_list_missing = lfilter(lambda key: key.value not in meta_name_list_new, meta_list_old)
return ParameterSource.create_instance('InternalParameterSource',
psp_list_missing, meta_list_missing)
def _create_psrc(psrc_name, *args): return ParameterSource.create_instance(psrc_name, *args)
def _create_psrc(psrc_name, *args, **kwargs):
return ParameterSource.create_instance(psrc_name, *args, **kwargs)
