def _tree2expr(self, node, repository): if isinstance(node, int): return node elif isinstance(node, tuple): (operator, args) = node if operator == '[]': psrc_list = [] for output_vn in _tree2names(args[0]): psrc_list.append( ParameterSource.create_instance( 'InternalAutoParameterSource', self._parameter_config, repository, output_vn, _tree2names(args[1]))) return ParameterSource.create_psrc_safe( 'CrossParameterSource', self._parameter_config, repository, *psrc_list) elif operator in self._operator_map_raw: return ParameterSource.create_psrc_safe( self._operator_map_raw[operator], self._parameter_config, repository, *args) elif operator in self._operator_map_eval: evaluated_args = lmap( lambda node: self._tree2expr(node, repository), args) return ParameterSource.create_psrc_safe( self._operator_map_eval[operator], self._parameter_config, repository, *evaluated_args) else: return ParameterSource.create_instance( 'InternalAutoParameterSource', self._parameter_config, repository, node) raise APIError('Unable to parse node %s!' % repr(node))
def __new__(cls, pconfig, repository, output_vn, lookup_vn_list=None): parameter_value = pconfig.get_parameter(output_vn.lstrip('!')) if isinstance(parameter_value, list): if len(parameter_value) != 1: # Simplify single value parameters to const parameters return ParameterSource.create_instance('SimpleParameterSource', output_vn, parameter_value) return ParameterSource.create_instance('ConstParameterSource', output_vn, parameter_value[0]) elif isinstance(parameter_value, tuple) and isinstance(parameter_value[0], str): return ParameterSource.create_instance(*parameter_value) return ParameterSource.create_instance('InternalAutoLookupParameterSource', pconfig, output_vn, lookup_vn_list)
def _resync_adapter(self, pa_old, pa_new, result_redo, result_disable, size_change): (map_jobnum2pnum, pspi_list_added, pspi_list_missing) = _diff_pspi_list(pa_old, pa_new, result_redo, result_disable) # Reorder and reconstruct parameter space with the following layout: # NNNNNNNNNNNNN OOOOOOOOO | source: NEW (==self) and OLD (==from file) # <same><added> <missing> | same: both in NEW and OLD, added: only in NEW, missing: only in OLD if pspi_list_added: _extend_map_jobnum2pnum(map_jobnum2pnum, pa_old.get_job_len(), pspi_list_added) if pspi_list_missing: # extend the parameter source by placeholders for the missing parameter space points psrc_missing = _create_placeholder_psrc(pa_old, pa_new, map_jobnum2pnum, pspi_list_missing, result_disable) self._psrc = ParameterSource.create_instance('ChainParameterSource', self._psrc_raw, psrc_missing) self._map_jobnum2pnum = map_jobnum2pnum # Update Job2PID map # Write resynced state self._write_jobnum2pnum(self._path_jobnum2pnum + '.tmp') ParameterSource.get_class('GCDumpParameterSource').write(self._path_params + '.tmp', self.get_job_len(), self.get_job_metadata(), self.iter_jobs()) os.rename(self._path_jobnum2pnum + '.tmp', self._path_jobnum2pnum) os.rename(self._path_params + '.tmp', self._path_params) result_redo = result_redo.difference(result_disable) if result_redo or result_disable: map_pnum2jobnum = reverse_dict(self._map_jobnum2pnum) def _translate_pnum(pnum): return map_pnum2jobnum.get(pnum, pnum) result_redo = set(imap(_translate_pnum, result_redo)) result_disable = set(imap(_translate_pnum, result_disable)) return (result_redo, result_disable, size_change) return (set(), set(), size_change)
def create_psrc(cls, pconfig, repository, output_vn, lookup_vn_list=None): # pylint:disable=arguments-differ parameter_value = pconfig.get_parameter(output_vn.lstrip('!')) if isinstance(parameter_value, list): if len( parameter_value ) != 1: # Simplify single value parameters to const parameters return ParameterSource.create_instance('SimpleParameterSource', output_vn, parameter_value) return ParameterSource.create_instance('ConstParameterSource', output_vn, parameter_value[0]) elif isinstance(parameter_value, tuple) and not isinstance(parameter_value[0], dict): return ParameterSource.create_instance(*parameter_value) return ParameterSource.create_instance( 'InternalAutoLookupParameterSource', pconfig, output_vn, lookup_vn_list)
def _resync(self): # This function is _VERY_ time critical! tmp = self._psrc_raw.resync_psrc() # First ask about psrc changes (result_redo, result_disable, size_change) = (set(tmp[0]), set(tmp[1]), tmp[2]) psrc_hash_new = self._psrc_raw.get_psrc_hash() psrc_hash_changed = self._psrc_hash_stored != psrc_hash_new self._psrc_hash_stored = psrc_hash_new if not (result_redo or result_disable or size_change or psrc_hash_changed): return ParameterSource.get_empty_resync_result() ps_old = ParameterSource.create_instance('GCDumpParameterSource', self._path_params) pa_old = ParameterAdapter(None, ps_old) pa_new = ParameterAdapter(None, self._psrc_raw) return self._resync_adapter(pa_old, pa_new, result_redo, result_disable, size_change)
def _tree2expr(self, node, repository): if isinstance(node, int): return node elif isinstance(node, tuple): (operator, args) = node if operator == '[]': psrc_list = [] for output_vn in _tree2names(args[0]): psrc_list.append(ParameterSource.create_instance('InternalAutoParameterSource', self._parameter_config, repository, output_vn, _tree2names(args[1]))) return ParameterSource.create_psrc_safe('CrossParameterSource', self._parameter_config, repository, *psrc_list) elif operator in self._operator_map_raw: return ParameterSource.create_psrc_safe(self._operator_map_raw[operator], self._parameter_config, repository, *args) elif operator in self._operator_map_eval: evaluated_args = lmap(lambda node: self._tree2expr(node, repository), args) return ParameterSource.create_psrc_safe(self._operator_map_eval[operator], self._parameter_config, repository, *evaluated_args) else: return ParameterSource.create_instance('InternalAutoParameterSource', self._parameter_config, repository, node) raise APIError('Unable to parse node %s!' % repr(node))
def _create_placeholder_psrc(pa_old, pa_new, map_jobnum2pnum, pspi_list_missing, result_disable): # Construct placeholder parameter source with missing parameter entries and intervention state psp_list_missing = [] missing_pnum_start = pa_new.get_job_len() sort_inplace(pspi_list_missing, key=itemgetter(TrackingInfo.pnum)) for (idx, pspi_missing) in enumerate(pspi_list_missing): map_jobnum2pnum[pspi_missing[TrackingInfo.pnum]] = missing_pnum_start + idx psp_missing = pa_old.get_job_content(missing_pnum_start + idx, pspi_missing[TrackingInfo.pnum]) psp_missing.pop('GC_PARAM') if psp_missing[ParameterInfo.ACTIVE]: psp_missing[ParameterInfo.ACTIVE] = False result_disable.add(missing_pnum_start + idx) psp_list_missing.append(psp_missing) meta_list_new = pa_new.get_job_metadata() meta_name_list_new = lmap(lambda key: key.value, meta_list_new) meta_list_old = pa_old.get_job_metadata() meta_list_missing = lfilter(lambda key: key.value not in meta_name_list_new, meta_list_old) return ParameterSource.create_instance('InternalParameterSource', psp_list_missing, meta_list_missing)
def _create_psrc(psrc_name, *args): return ParameterSource.create_instance(psrc_name, *args)
def _create_psrc(psrc_name, *args, **kwargs): return ParameterSource.create_instance(psrc_name, *args, **kwargs)