def _run_section(self, section_path_str: str) -> Results:
section = self._get_func_or_collection(section_path_str)
section = cast(PipelineCollection, section)
results = []
for section_or_object_view in section:
# Get section path by which to call this item
subsection_name = _get_public_name_or_special_name(section_or_object_view, accept_output_names=False)
subsection_path_str = SectionPath.join(section_path_str, subsection_name).path_str
# Get from object view if necessary
if isinstance(section_or_object_view, ObjectView):
section_or_callable = section_or_object_view.item
else:
section_or_callable = section_or_object_view
if isinstance(section_or_callable, PipelineCollection):
# got another section within this section. recursively call run section
results.append(self._run_section(subsection_path_str))
elif self._is_specific_class(section_or_callable):
results.append(self._run_one_specific_class(section_path_str))
elif inspect.isclass(section_or_callable):
results.append(self._run_one_class(section_path_str))
elif callable(section_or_callable):
# run function
results.append(self._run_one_func(subsection_path_str))
else:
raise ValueError(f'could not run section {subsection_path_str}. expected PipelineCollection or '
f'function or class,'
f'got {section_or_callable} of type {type(section_or_callable)}')
return results
def _get_section(self, section_path_str: str):
section = self._get_func_or_collection(section_path_str)
section = cast(Collection, section)
# Need to handle definition structure which can be lists inside dicts or more dicts inside dicts
# This method will be called recursively on each section. Check to see if there are any sections
# inside this section. If so, then that section would be defined by a dict key in the definition,
# and therefore results should be put in a dict. If there are no sections within this section, then
# a list was used to store the items in this section and so results will be put in a list.
results: Union[Dict[str, Union[Collection, Any]], List[Any]]
if any(isinstance(item, Collection) for item in section):
results = {}
else:
results = []
for section_or_object_view in section:
# Get from object view if necessary
if isinstance(section_or_object_view, ObjectView):
section_or_callable = section_or_object_view.item
else:
section_or_callable = section_or_object_view
# Get section path by which to call this item
if self._is_specific_class(section_or_callable):
# If specific class, need to look up which key holds the name
subsection_name = section.name_for_obj(section_or_object_view)
else:
# If in the main dict, or is a collection, the name attribute or function/class name holds the name
subsection_name = _get_public_name_or_special_name(section_or_object_view, accept_output_names=False)
subsection_path_str = SectionPath.join(section_path_str, subsection_name).path_str
if isinstance(section_or_callable, Collection):
# not expected to hit these, for mypy
assert isinstance(results, dict)
assert section_or_callable.name is not None
# got another section within this section. recursively call get section
results[section_or_callable.name] = self._get_section(subsection_path_str)
elif isinstance(results, dict):
# got a non-collection, but results were defined as a dict. Should only have collections in dict
raise ValueError(
f'section {section_or_object_view.name} has both collections and items, must only '
f'have collections if there is at least one collection. '
f'Got {section_or_callable} as non-collection.'
)
elif self._is_specific_class(section_or_callable):
results.append(self._get_one_obj_with_config(subsection_path_str))
elif callable(section_or_callable):
# get function
results.append(self._get_one_func_with_config(subsection_path_str))
else:
raise ValueError(f'could not get section {subsection_path_str}. expected Collection or '
f'function or specific class,'
f'got {section_or_callable} of type {type(section_or_callable)}')
return results
def from_file_path(cls, file_path: str, action: PyfileconfActions):
from pyfileconf import PipelineManager
from pyfileconf.sectionpath.sectionpath import SectionPath
dependent_manager = PipelineManager.get_manager_by_filepath(file_path)
dependent_sp = SectionPath.from_filepath(
dependent_manager.default_config_path, file_path)
full_sp = SectionPath.join(dependent_manager.name, dependent_sp)
return cls(full_sp, action, file_path=file_path)
def pyfileconf_post_config_changed(
manager: 'ConfigManager',
new_config: 'ConfigBase',
updates: Dict[str, Any],
section_path_str: str,
):
full_sp = SectionPath.join(manager.pipeline_manager_name, section_path_str)
reset_roots([full_sp.path_str])
return None
def __init__(
self,
section_path_str: Optional[str] = None,
action: PyfileconfActions = PyfileconfActions.RUN,
file_path: Optional[str] = None,
base_section_path_str: Optional[str] = None,
):
if section_path_str is None and file_path is None:
raise ValueError(
"must provide one of section_path_str or file_path")
if base_section_path_str is not None and section_path_str is not None:
new_sp = SectionPath.join(base_section_path_str, section_path_str)
section_path_str = new_sp.path_str
self.section_path_str = section_path_str
self.action = action
self.file_path = file_path
def refresh_dependent_configs(self, section_path_str: str):
from pyfileconf import context
full_sp = SectionPath.join(self.pipeline_manager_name,
section_path_str)
update_deps = {*context.force_update_dependencies[full_sp.path_str]}
all_updated_deps = set()
while update_deps:
_refresh_configs(update_deps)
all_updated_deps.update(update_deps)
# Get any newly updated dependencies caused by process of updating dependencies
new_update_deps = context.force_update_dependencies[
full_sp.path_str].difference(all_updated_deps)
if update_deps == new_update_deps:
# Not expected, but somehow got stuck in an infinite loop where it is
# always trying to update the same dependency
raise CannotResolveConfigDependenciesException(update_deps)
update_deps = new_update_deps
def _update(self,
d_: dict = None,
section_path_str: str = None,
pyfileconf_persist: bool = True,
**kwargs):
"""
Update the configuration for an item by section path
:param d_: dictionary of updates
:param section_path_str: section path of item to be updated
:param pyfileconf_persist: whether to make changes last through refreshing config
:param kwargs: kwarg updates
:return:
"""
from pyfileconf.selector.models.itemview import is_item_view
all_updates = {}
if d_ is not None:
all_updates.update(d_)
all_updates.update(kwargs)
logger.info(f'Updating {section_path_str} with config: {all_updates}')
if section_path_str:
# If any of the updates are putting an ItemView as the value, then
# record that this config is dependent on the config referenced
# by the ItemView
all_updates = {**kwargs}
if d_ is not None:
all_updates.update(d_)
full_sp = SectionPath.join(self.name, section_path_str)
for value in all_updates.values():
if is_item_view(value):
context.add_config_dependency(full_sp, value)
self.runner.update(d_,
section_path_str,
pyfileconf_persist=pyfileconf_persist,
**kwargs)
logger.debug(
f'Finished updating {section_path_str} with config: {all_updates}')
def _add_full_section_path_str_to_obj(self, section_path_str: str, obj: Any):
full_sp_str = SectionPath.join(self._manager_name, section_path_str).path_str
obj._section_path_str = full_sp_str
def _add_to_config_dependencies_if_necessary(self, section_path_str: str):
from pyfileconf import context
full_sp = SectionPath.join(self._manager_name, section_path_str)
context.add_config_dependency_for_currently_running_item_if_exists(full_sp, force_update=True)
def create(self,
section_path_str: str,
func_or_class: Optional[Union[Callable, Type]] = None):
"""
Create a new configuration entry dynamically rather than manually modifying dict file
:param func_or_class: function or class to use to generate config
:param section_path_str: section path at which the config should be stored
:return:
"""
logger.info(f'Creating config for {section_path_str}')
section_path = SectionPath(section_path_str)
if section_path[0] in self.specific_class_names:
# Got path for a specific class dict, add to specific class dict
if len(section_path) < 3:
raise ValueError(
'when targeting a specific class dict, section path must have minimum length of '
'3, e.g. example_class.thing.stuff')
if func_or_class is not None:
raise ValueError(
'only pass func_or_class when targeting main pipeline dict, not specific class dict'
)
self._create_specific_class_dict_entry(section_path)
full_sp = section_path
registrar = self._registrar_dict[section_path[0]]
klass = registrar.collection.klass
if klass is None:
raise ValueError(
f'must have specific class set, got None for class in {registrar}'
)
key_attr = registrar.collection.key_attr
registrar_obj = convert_to_empty_obj_if_necessary(
section_path[-1], klass, key_attr=key_attr)
set_section_path_str = SectionPath.from_section_str_list(
section_path[1:-1]).path_str
scaffold_path_str = SectionPath.from_section_str_list(
section_path[1:]).path_str
else:
# Got path for main pipeline dict, add to main pipeline dict
if func_or_class is None:
raise ValueError(
'when adding creating item in main pipeline dict, must pass function or class'
)
self._create_pipeline_dict_entry(section_path, func_or_class)
name = func_or_class.__name__
full_sp = SectionPath.join(section_path, name)
mod, import_base = get_module_and_name_imported_from(func_or_class)
obj_import = ObjectImportStatement([name], import_base)
item = ast.Name(id=name)
imports = ImportStatementContainer([obj_import])
registrar_obj = ObjectView.from_ast_and_imports(item, imports)
if self._general_registrar is None:
raise ValueError('general registrar must be defined')
registrar = self._general_registrar
set_section_path_str = section_path_str
scaffold_path_str = full_sp.path_str
registrar.set(set_section_path_str, registrar_obj)
registrar.scaffold_config_for(scaffold_path_str)
self.reset(full_sp.path_str, allow_create=True)
logger.debug(f'Finished creating config for {section_path_str}')
def _get_full_section_path_str(self, section_path_str: str) -> str:
if not self.base_section_path_str:
return section_path_str
return SectionPath.join(self.base_section_path_str,
section_path_str).path_str