def get_cases(self) -> List[Tuple[Dict[str, Any], ...]]:
logger.debug('Determining cases for IterativeRunner')
cases_lol: List[List[Tuple[Dict[
str, Any], ...]]] = manager.plm.hook.pyfileconf_iter_get_cases(
config_updates=self.config_updates, runner=self)
cases = list(itertools.chain(*cases_lol))
manager.plm.hook.pyfileconf_iter_modify_cases(cases=cases, runner=self)
logger.debug(f'Got {cases} for IterativeRunner.cases')
return cases
def reset_option(self, attr: str):
"""
Reset a particular option by name
:param attr: Name of option
:return:
"""
logger.debug(f'Resetting option {attr}')
opt: PyfileconfOption = getattr(self, attr)
opt.reset()
def reset(self):
"""
Undo all changes made through the options interface
:return:
"""
logger.debug(f"Resetting pyfileconf options")
for attr in options.option_attrs:
opt = getattr(self, attr)
opt.reset()
def refresh(self, section_path_str: str):
"""
Reloads from the existing file, then reapplies any config updates. Useful for when
this config depends on the attribute of some other config which was updated.
:param section_path_str: section path of item to be refreshed
:return:
"""
logger.info(f'Refreshing config for {section_path_str}')
self.runner.refresh(section_path_str)
logger.debug(f'Finished refreshing config for {section_path_str}')
def set_option(self, attr: str, value: Any):
"""
Set a particular option by its name
:param attr: Option name
:param value: New option value
:return:
"""
logger.debug(f"Setting option {attr} to {value}")
opt: PyfileconfOption = getattr(self, attr)
opt.value = value
return self
def get(self, section_path_str: str):
logger.debug(f'Getting {section_path_str} in runner')
func_or_collection = self._get_func_or_collection(section_path_str)
if isinstance(func_or_collection, Collection):
return self._get_section(section_path_str)
elif self._is_specific_class(func_or_collection):
return self._get_one_obj_with_config(section_path_str)
elif inspect.isclass(func_or_collection):
return self._get_one_obj_with_config(section_path_str)
elif callable(func_or_collection):
return self._get_one_func_with_config(section_path_str)
else:
raise ValueError(f'could not get section {section_path_str}. expected PipelineCollection or function,'
f'got {func_or_collection} of type {type(func_or_collection)}')
def reload(self) -> None:
"""
Useful for getting file system changes without having to start a new Python session. Also resets
any locally defined configuration.
Reloads functions from pipeline dict, scaffolds config files for any new functions,
and updates configuration from files.
Returns: None
"""
logger.info(f'Reloading {self.name}')
self._wipe_loaded_modules()
self.load()
logger.debug(f'Finished reloading {self.name}')
def set_options(self, updates: Iterable[Tuple[str, Any]]):
"""
Set multiple options at once by their name
:param updates: Multiple option updates
:return:
:Examples:
>>> import pyfileconf
>>> pyfileconf.options.set_options([('log_stdout', True), ('log_folder', 'MyLogs')])
"""
for attr, value in updates:
logger.debug(f"Setting option {attr} to {value}")
opt: PyfileconfOption = getattr(self, attr)
opt.value = value
return self
def reset(self,
section_path_str_or_view: 'StrOrView',
allow_create: bool = False):
"""
Resets a function or section config to default.
To reset all configs, use .reload() instead.
:param section_path_str_or_view:
:return:
"""
logger.info(f'Resetting config for {section_path_str_or_view}')
section_path_str = self._get_section_path_str_from_section_path_str_or_view(
section_path_str_or_view)
self.runner.reset(section_path_str, allow_create=allow_create)
logger.debug(
f'Finished resetting config for {section_path_str_or_view}')
def load(self) -> None:
"""
Wrapper to track imported modules so that can reimport them upon reloading
"""
logger.debug(f'Running load for {self.name}')
self._import_tracker = ImportTracker()
try:
self._load_pipeline_config_and_runner()
except Exception as e:
# Reset loaded modules from import, completely canceling load so it can be tried again
self._loaded_modules = self._import_tracker.imported_modules
self._wipe_loaded_modules()
self._loaded_modules = [
] # modules have been wiped, must be sure they won't be wiped again
raise e
self._loaded_modules = self._import_tracker.imported_modules
logger.debug(f'Finished running load for {self.name}')
def get_defaults(self) -> Dict[str, Dict[str, Any]]:
logger.debug('Determining defaults for IterativeRunner')
from pyfileconf import PipelineManager
if not hasattr(self, 'cases'):
raise ValueError('must set cases before calling get_defaults')
case = self.cases[0]
section_path_strs = [
self._get_full_section_path_str(conf['section_path_str'])
for conf in case
]
defaults: Dict[str, Dict[str, Any]] = {}
for sp_str in section_path_strs:
pm = PipelineManager.get_manager_by_section_path_str(sp_str)
sp = SectionPath(sp_str)
relative_section_path_str = SectionPath(".".join(sp[1:])).path_str
config = pm.config.get(relative_section_path_str)
if config is not None:
defaults[sp_str] = {**config}
logger.debug(f'Got {defaults} for IterativeRunner.defaults')
return defaults
def _update(self,
d_: dict = None,
section_path_str: str = None,
pyfileconf_persist: bool = True,
**kwargs):
"""
Update the configuration for an item by section path
:param d_: dictionary of updates
:param section_path_str: section path of item to be updated
:param pyfileconf_persist: whether to make changes last through refreshing config
:param kwargs: kwarg updates
:return:
"""
from pyfileconf.selector.models.itemview import is_item_view
all_updates = {}
if d_ is not None:
all_updates.update(d_)
all_updates.update(kwargs)
logger.info(f'Updating {section_path_str} with config: {all_updates}')
if section_path_str:
# If any of the updates are putting an ItemView as the value, then
# record that this config is dependent on the config referenced
# by the ItemView
all_updates = {**kwargs}
if d_ is not None:
all_updates.update(d_)
full_sp = SectionPath.join(self.name, section_path_str)
for value in all_updates.values():
if is_item_view(value):
context.add_config_dependency(full_sp, value)
self.runner.update(d_,
section_path_str,
pyfileconf_persist=pyfileconf_persist,
**kwargs)
logger.debug(
f'Finished updating {section_path_str} with config: {all_updates}')
def reset(self):
logger.debug(f"Resetting option {self.name}")
self.value = self.default_value
def create(self,
section_path_str: str,
func_or_class: Optional[Union[Callable, Type]] = None):
"""
Create a new configuration entry dynamically rather than manually modifying dict file
:param func_or_class: function or class to use to generate config
:param section_path_str: section path at which the config should be stored
:return:
"""
logger.info(f'Creating config for {section_path_str}')
section_path = SectionPath(section_path_str)
if section_path[0] in self.specific_class_names:
# Got path for a specific class dict, add to specific class dict
if len(section_path) < 3:
raise ValueError(
'when targeting a specific class dict, section path must have minimum length of '
'3, e.g. example_class.thing.stuff')
if func_or_class is not None:
raise ValueError(
'only pass func_or_class when targeting main pipeline dict, not specific class dict'
)
self._create_specific_class_dict_entry(section_path)
full_sp = section_path
registrar = self._registrar_dict[section_path[0]]
klass = registrar.collection.klass
if klass is None:
raise ValueError(
f'must have specific class set, got None for class in {registrar}'
)
key_attr = registrar.collection.key_attr
registrar_obj = convert_to_empty_obj_if_necessary(
section_path[-1], klass, key_attr=key_attr)
set_section_path_str = SectionPath.from_section_str_list(
section_path[1:-1]).path_str
scaffold_path_str = SectionPath.from_section_str_list(
section_path[1:]).path_str
else:
# Got path for main pipeline dict, add to main pipeline dict
if func_or_class is None:
raise ValueError(
'when adding creating item in main pipeline dict, must pass function or class'
)
self._create_pipeline_dict_entry(section_path, func_or_class)
name = func_or_class.__name__
full_sp = SectionPath.join(section_path, name)
mod, import_base = get_module_and_name_imported_from(func_or_class)
obj_import = ObjectImportStatement([name], import_base)
item = ast.Name(id=name)
imports = ImportStatementContainer([obj_import])
registrar_obj = ObjectView.from_ast_and_imports(item, imports)
if self._general_registrar is None:
raise ValueError('general registrar must be defined')
registrar = self._general_registrar
set_section_path_str = section_path_str
scaffold_path_str = full_sp.path_str
registrar.set(set_section_path_str, registrar_obj)
registrar.scaffold_config_for(scaffold_path_str)
self.reset(full_sp.path_str, allow_create=True)
logger.debug(f'Finished creating config for {section_path_str}')
"""
:param collect_results: Whether to aggregate and return results, set to False to save memory
if results are stored in some other way
:return:
"""
from pyfileconf.main import PipelineManager
logger.info(f'Running {self.run_items} with cases')
all_results = []
for case in self.cases:
if not self.should_run(case):
continue
result = self._run_case(case)
if collect_results:
in_out_tup = (case, result)
all_results.append(in_out_tup)
logger.debug(
f'Finished running {self.run_items} with cases {self.cases}')
return all_results
def run_gen(self) -> Iterator[IterativeResult]:
logger.info(f'Running {self.run_items} with cases')
for case in self.cases:
if not self.should_run(case):
continue
result = self._run_case(case)
in_out_tup = (case, result)
yield in_out_tup
def _run_case(self, case: Tuple[Dict[str, Any], ...]) -> Any:
case_with_defaults = self._fill_case_with_defaults(case)
manager.plm.hook.pyfileconf_iter_update_for_case(
case=case_with_defaults, runner=self)