def _run_section(self, section_path_str: str) -> Results: section = self._get_func_or_collection(section_path_str) section = cast(PipelineCollection, section) results = [] for section_or_object_view in section: # Get section path by which to call this item subsection_name = _get_public_name_or_special_name(section_or_object_view, accept_output_names=False) subsection_path_str = SectionPath.join(section_path_str, subsection_name).path_str # Get from object view if necessary if isinstance(section_or_object_view, ObjectView): section_or_callable = section_or_object_view.item else: section_or_callable = section_or_object_view if isinstance(section_or_callable, PipelineCollection): # got another section within this section. recursively call run section results.append(self._run_section(subsection_path_str)) elif self._is_specific_class(section_or_callable): results.append(self._run_one_specific_class(section_path_str)) elif inspect.isclass(section_or_callable): results.append(self._run_one_class(section_path_str)) elif callable(section_or_callable): # run function results.append(self._run_one_func(subsection_path_str)) else: raise ValueError(f'could not run section {subsection_path_str}. expected PipelineCollection or ' f'function or class,' f'got {section_or_callable} of type {type(section_or_callable)}') return results
def _get_section(self, section_path_str: str): section = self._get_func_or_collection(section_path_str) section = cast(Collection, section) # Need to handle definition structure which can be lists inside dicts or more dicts inside dicts # This method will be called recursively on each section. Check to see if there are any sections # inside this section. If so, then that section would be defined by a dict key in the definition, # and therefore results should be put in a dict. If there are no sections within this section, then # a list was used to store the items in this section and so results will be put in a list. results: Union[Dict[str, Union[Collection, Any]], List[Any]] if any(isinstance(item, Collection) for item in section): results = {} else: results = [] for section_or_object_view in section: # Get from object view if necessary if isinstance(section_or_object_view, ObjectView): section_or_callable = section_or_object_view.item else: section_or_callable = section_or_object_view # Get section path by which to call this item if self._is_specific_class(section_or_callable): # If specific class, need to look up which key holds the name subsection_name = section.name_for_obj(section_or_object_view) else: # If in the main dict, or is a collection, the name attribute or function/class name holds the name subsection_name = _get_public_name_or_special_name(section_or_object_view, accept_output_names=False) subsection_path_str = SectionPath.join(section_path_str, subsection_name).path_str if isinstance(section_or_callable, Collection): # not expected to hit these, for mypy assert isinstance(results, dict) assert section_or_callable.name is not None # got another section within this section. recursively call get section results[section_or_callable.name] = self._get_section(subsection_path_str) elif isinstance(results, dict): # got a non-collection, but results were defined as a dict. Should only have collections in dict raise ValueError( f'section {section_or_object_view.name} has both collections and items, must only ' f'have collections if there is at least one collection. ' f'Got {section_or_callable} as non-collection.' ) elif self._is_specific_class(section_or_callable): results.append(self._get_one_obj_with_config(subsection_path_str)) elif callable(section_or_callable): # get function results.append(self._get_one_func_with_config(subsection_path_str)) else: raise ValueError(f'could not get section {subsection_path_str}. expected Collection or ' f'function or specific class,' f'got {section_or_callable} of type {type(section_or_callable)}') return results
def from_file_path(cls, file_path: str, action: PyfileconfActions): from pyfileconf import PipelineManager from pyfileconf.sectionpath.sectionpath import SectionPath dependent_manager = PipelineManager.get_manager_by_filepath(file_path) dependent_sp = SectionPath.from_filepath( dependent_manager.default_config_path, file_path) full_sp = SectionPath.join(dependent_manager.name, dependent_sp) return cls(full_sp, action, file_path=file_path)
def pyfileconf_post_config_changed( manager: 'ConfigManager', new_config: 'ConfigBase', updates: Dict[str, Any], section_path_str: str, ): full_sp = SectionPath.join(manager.pipeline_manager_name, section_path_str) reset_roots([full_sp.path_str]) return None
def __init__( self, section_path_str: Optional[str] = None, action: PyfileconfActions = PyfileconfActions.RUN, file_path: Optional[str] = None, base_section_path_str: Optional[str] = None, ): if section_path_str is None and file_path is None: raise ValueError( "must provide one of section_path_str or file_path") if base_section_path_str is not None and section_path_str is not None: new_sp = SectionPath.join(base_section_path_str, section_path_str) section_path_str = new_sp.path_str self.section_path_str = section_path_str self.action = action self.file_path = file_path
def refresh_dependent_configs(self, section_path_str: str): from pyfileconf import context full_sp = SectionPath.join(self.pipeline_manager_name, section_path_str) update_deps = {*context.force_update_dependencies[full_sp.path_str]} all_updated_deps = set() while update_deps: _refresh_configs(update_deps) all_updated_deps.update(update_deps) # Get any newly updated dependencies caused by process of updating dependencies new_update_deps = context.force_update_dependencies[ full_sp.path_str].difference(all_updated_deps) if update_deps == new_update_deps: # Not expected, but somehow got stuck in an infinite loop where it is # always trying to update the same dependency raise CannotResolveConfigDependenciesException(update_deps) update_deps = new_update_deps
def _update(self, d_: dict = None, section_path_str: str = None, pyfileconf_persist: bool = True, **kwargs): """ Update the configuration for an item by section path :param d_: dictionary of updates :param section_path_str: section path of item to be updated :param pyfileconf_persist: whether to make changes last through refreshing config :param kwargs: kwarg updates :return: """ from pyfileconf.selector.models.itemview import is_item_view all_updates = {} if d_ is not None: all_updates.update(d_) all_updates.update(kwargs) logger.info(f'Updating {section_path_str} with config: {all_updates}') if section_path_str: # If any of the updates are putting an ItemView as the value, then # record that this config is dependent on the config referenced # by the ItemView all_updates = {**kwargs} if d_ is not None: all_updates.update(d_) full_sp = SectionPath.join(self.name, section_path_str) for value in all_updates.values(): if is_item_view(value): context.add_config_dependency(full_sp, value) self.runner.update(d_, section_path_str, pyfileconf_persist=pyfileconf_persist, **kwargs) logger.debug( f'Finished updating {section_path_str} with config: {all_updates}')
def _add_full_section_path_str_to_obj(self, section_path_str: str, obj: Any): full_sp_str = SectionPath.join(self._manager_name, section_path_str).path_str obj._section_path_str = full_sp_str
def _add_to_config_dependencies_if_necessary(self, section_path_str: str): from pyfileconf import context full_sp = SectionPath.join(self._manager_name, section_path_str) context.add_config_dependency_for_currently_running_item_if_exists(full_sp, force_update=True)
def create(self, section_path_str: str, func_or_class: Optional[Union[Callable, Type]] = None): """ Create a new configuration entry dynamically rather than manually modifying dict file :param func_or_class: function or class to use to generate config :param section_path_str: section path at which the config should be stored :return: """ logger.info(f'Creating config for {section_path_str}') section_path = SectionPath(section_path_str) if section_path[0] in self.specific_class_names: # Got path for a specific class dict, add to specific class dict if len(section_path) < 3: raise ValueError( 'when targeting a specific class dict, section path must have minimum length of ' '3, e.g. example_class.thing.stuff') if func_or_class is not None: raise ValueError( 'only pass func_or_class when targeting main pipeline dict, not specific class dict' ) self._create_specific_class_dict_entry(section_path) full_sp = section_path registrar = self._registrar_dict[section_path[0]] klass = registrar.collection.klass if klass is None: raise ValueError( f'must have specific class set, got None for class in {registrar}' ) key_attr = registrar.collection.key_attr registrar_obj = convert_to_empty_obj_if_necessary( section_path[-1], klass, key_attr=key_attr) set_section_path_str = SectionPath.from_section_str_list( section_path[1:-1]).path_str scaffold_path_str = SectionPath.from_section_str_list( section_path[1:]).path_str else: # Got path for main pipeline dict, add to main pipeline dict if func_or_class is None: raise ValueError( 'when adding creating item in main pipeline dict, must pass function or class' ) self._create_pipeline_dict_entry(section_path, func_or_class) name = func_or_class.__name__ full_sp = SectionPath.join(section_path, name) mod, import_base = get_module_and_name_imported_from(func_or_class) obj_import = ObjectImportStatement([name], import_base) item = ast.Name(id=name) imports = ImportStatementContainer([obj_import]) registrar_obj = ObjectView.from_ast_and_imports(item, imports) if self._general_registrar is None: raise ValueError('general registrar must be defined') registrar = self._general_registrar set_section_path_str = section_path_str scaffold_path_str = full_sp.path_str registrar.set(set_section_path_str, registrar_obj) registrar.scaffold_config_for(scaffold_path_str) self.reset(full_sp.path_str, allow_create=True) logger.debug(f'Finished creating config for {section_path_str}')
def _get_full_section_path_str(self, section_path_str: str) -> str: if not self.base_section_path_str: return section_path_str return SectionPath.join(self.base_section_path_str, section_path_str).path_str