def run_gen(self) -> Iterator[IterativeResult]:
logger.info(f'Running {self.run_items} with cases')
for case in self.cases:
if not self.should_run(case):
continue
result = self._run_case(case)
in_out_tup = (case, result)
yield in_out_tup
def _run_one_class(self, section_path_str: str) -> Result:
with StackTracker(section_path_str, base_section_path_str=self._manager_name):
klass, config_dict = self._get_func_and_config(section_path_str)
obj = self._get_one_obj_with_config(section_path_str)
logger.info(f'Running class {section_path_str}({dict_as_function_kwarg_str(config_dict)})')
result = obj()
logger.info(f'Result:\n{result}\n')
self._add_to_config_dependencies_if_necessary(section_path_str)
return result
def report_runner_exceptions(runner_exceptions: List[RunnerException]) -> None:
if len(runner_exceptions) == 0:
logger.info(
'\n\nEverything ran successfully, no exceptions to report.\n\n')
return
sp_strs = ', '.join(
[str(exc.section_path_str) for exc in runner_exceptions])
full_exc_str = f'Exception summary for running {sp_strs} (exceptions were also shown when raised):\n'
full_exc_str += '\n\n'.join([str(exc) for exc in runner_exceptions])
logger.error(full_exc_str)
def refresh(self, section_path_str: str):
"""
Reloads from the existing file, then reapplies any config updates. Useful for when
this config depends on the attribute of some other config which was updated.
:param section_path_str: section path of item to be refreshed
:return:
"""
logger.info(f'Refreshing config for {section_path_str}')
self.runner.refresh(section_path_str)
logger.debug(f'Finished refreshing config for {section_path_str}')
def _run_one_specific_class(self, section_path_str: str) -> Result:
with StackTracker(section_path_str, base_section_path_str=self._manager_name):
klass, config_dict = self._get_class_and_config(section_path_str)
obj = self._get_one_obj_with_config(section_path_str)
registrar = self._specific_class_registrar_map[klass]
execute_attr = registrar.execute_attr
func = getattr(obj, execute_attr)
logger.info(f'Running class {section_path_str}({dict_as_function_kwarg_str(config_dict)})')
result = func()
logger.info(f'Result:\n{result}\n')
self._add_to_config_dependencies_if_necessary(section_path_str)
return result
def reload(self) -> None:
"""
Useful for getting file system changes without having to start a new Python session. Also resets
any locally defined configuration.
Reloads functions from pipeline dict, scaffolds config files for any new functions,
and updates configuration from files.
Returns: None
"""
logger.info(f'Reloading {self.name}')
self._wipe_loaded_modules()
self.load()
logger.debug(f'Finished reloading {self.name}')
def run(self, collect_results: bool = True) -> IterativeResults:
"""
:param collect_results: Whether to aggregate and return results, set to False to save memory
if results are stored in some other way
:return:
"""
from pyfileconf.main import PipelineManager
logger.info(f'Running {self.run_items} with cases')
all_results = []
for case in self.cases:
if not self.should_run(case):
continue
result = self._run_case(case)
if collect_results:
in_out_tup = (case, result)
all_results.append(in_out_tup)
def reset(self,
section_path_str_or_view: 'StrOrView',
allow_create: bool = False):
"""
Resets a function or section config to default.
To reset all configs, use .reload() instead.
:param section_path_str_or_view:
:return:
"""
logger.info(f'Resetting config for {section_path_str_or_view}')
section_path_str = self._get_section_path_str_from_section_path_str_or_view(
section_path_str_or_view)
self.runner.reset(section_path_str, allow_create=allow_create)
logger.debug(
f'Finished resetting config for {section_path_str_or_view}')
def _update(self,
d_: dict = None,
section_path_str: str = None,
pyfileconf_persist: bool = True,
**kwargs):
"""
Update the configuration for an item by section path
:param d_: dictionary of updates
:param section_path_str: section path of item to be updated
:param pyfileconf_persist: whether to make changes last through refreshing config
:param kwargs: kwarg updates
:return:
"""
from pyfileconf.selector.models.itemview import is_item_view
all_updates = {}
if d_ is not None:
all_updates.update(d_)
all_updates.update(kwargs)
logger.info(f'Updating {section_path_str} with config: {all_updates}')
if section_path_str:
# If any of the updates are putting an ItemView as the value, then
# record that this config is dependent on the config referenced
# by the ItemView
all_updates = {**kwargs}
if d_ is not None:
all_updates.update(d_)
full_sp = SectionPath.join(self.name, section_path_str)
for value in all_updates.values():
if is_item_view(value):
context.add_config_dependency(full_sp, value)
self.runner.update(d_,
section_path_str,
pyfileconf_persist=pyfileconf_persist,
**kwargs)
logger.debug(
f'Finished updating {section_path_str} with config: {all_updates}')
def test_log_file(self):
logger.info('woo')
assert not os.path.exists(self.logs_path)
pyfileconf.options.set_option('log_folder', self.logs_folder)
logger.info('woo2')
pyfileconf.options.reset()
logger.info('woo3')
assert os.path.exists(self.logs_path)
with open(self.logs_path, 'r') as f:
contents = f.read()
assert contents == '[pyfileconf INFO]: woo2\n'
def create(self,
section_path_str: str,
func_or_class: Optional[Union[Callable, Type]] = None):
"""
Create a new configuration entry dynamically rather than manually modifying dict file
:param func_or_class: function or class to use to generate config
:param section_path_str: section path at which the config should be stored
:return:
"""
logger.info(f'Creating config for {section_path_str}')
section_path = SectionPath(section_path_str)
if section_path[0] in self.specific_class_names:
# Got path for a specific class dict, add to specific class dict
if len(section_path) < 3:
raise ValueError(
'when targeting a specific class dict, section path must have minimum length of '
'3, e.g. example_class.thing.stuff')
if func_or_class is not None:
raise ValueError(
'only pass func_or_class when targeting main pipeline dict, not specific class dict'
)
self._create_specific_class_dict_entry(section_path)
full_sp = section_path
registrar = self._registrar_dict[section_path[0]]
klass = registrar.collection.klass
if klass is None:
raise ValueError(
f'must have specific class set, got None for class in {registrar}'
)
key_attr = registrar.collection.key_attr
registrar_obj = convert_to_empty_obj_if_necessary(
section_path[-1], klass, key_attr=key_attr)
set_section_path_str = SectionPath.from_section_str_list(
section_path[1:-1]).path_str
scaffold_path_str = SectionPath.from_section_str_list(
section_path[1:]).path_str
else:
# Got path for main pipeline dict, add to main pipeline dict
if func_or_class is None:
raise ValueError(
'when adding creating item in main pipeline dict, must pass function or class'
)
self._create_pipeline_dict_entry(section_path, func_or_class)
name = func_or_class.__name__
full_sp = SectionPath.join(section_path, name)
mod, import_base = get_module_and_name_imported_from(func_or_class)
obj_import = ObjectImportStatement([name], import_base)
item = ast.Name(id=name)
imports = ImportStatementContainer([obj_import])
registrar_obj = ObjectView.from_ast_and_imports(item, imports)
if self._general_registrar is None:
raise ValueError('general registrar must be defined')
registrar = self._general_registrar
set_section_path_str = section_path_str
scaffold_path_str = full_sp.path_str
registrar.set(set_section_path_str, registrar_obj)
registrar.scaffold_config_for(scaffold_path_str)
self.reset(full_sp.path_str, allow_create=True)
logger.debug(f'Finished creating config for {section_path_str}')
def write(self, message):
if message != '\n':
logger.info(message)