def test_update_batch_with_plugins(self):
self.add_plugin()
self.write_a_function_to_pipeline_dict_file()
pipeline_manager = self.create_pm()
pipeline_manager.load()
pipeline_manager.create('stuff2', a_function)
sel = Selector()
ivs = [
sel.test_pipeline_manager.stuff.a_function,
sel.test_pipeline_manager.stuff2.a_function,
]
expected_b_result = ["a", "b"]
updates = []
for iv in ivs:
section_path = SectionPath.from_section_str_list(
SectionPath(iv.section_path_str)[1:])
updates.append(
dict(b=expected_b_result,
section_path_str=section_path.path_str))
pipeline_manager.update_batch(updates)
for iv in ivs:
result = pipeline_manager.run(iv)
assert result == [
(None, OVERRIDDEN_B_RESULT),
(None, OVERRIDDEN_B_RESULT),
"abc",
]
assert PRE_UPDATE_BATCH_COUNTER == 1
assert POST_UPDATE_BATCH_COUNTER == 1
def test_create_then_update_entry_for_specific_class_dict(self):
self.write_example_class_dict_to_file()
pipeline_manager = self.create_pm(
specific_class_config_dicts=CLASS_CONFIG_DICT_LIST)
pipeline_manager.load()
sel = Selector()
pipeline_manager.create('example_class.thing.data')
iv = sel.test_pipeline_manager.example_class.thing.data
expected_a_result = (1, 2)
section_path = SectionPath.from_section_str_list(
SectionPath(iv.section_path_str)[1:])
pipeline_manager.update(a=expected_a_result,
section_path_str=section_path.path_str)
class_folder = os.path.join(self.defaults_path, 'example_class')
module_folder = os.path.join(class_folder, 'thing')
class_path = os.path.join(module_folder, 'data.py')
with open(class_path, 'r') as f:
contents = f.read()
self.assert_example_class_dict_config_file_contents(contents)
ec = sel.test_pipeline_manager.example_class.thing.data
result = pipeline_manager.run(ec)
assert result == 'woo'
expect_ec = ExampleClass(name='data', a=expected_a_result)
got_ec = ec.item
assert ec.name == expect_ec.name == got_ec.name
assert ec.a == expect_ec.a == got_ec.a
def test_update_no_plugins(self):
self.write_a_function_to_pipeline_dict_file()
pipeline_manager = self.create_pm()
pipeline_manager.load()
sel = Selector()
iv = sel.test_pipeline_manager.stuff.a_function
expected_b_result = ["a", "b"]
section_path = SectionPath.from_section_str_list(
SectionPath(iv.section_path_str)[1:])
pipeline_manager.update(b=expected_b_result,
section_path_str=section_path.path_str)
result = pipeline_manager.run(iv)
assert result == (None, expected_b_result)
assert PRE_UPDATE_COUNTER == 0
assert POST_UPDATE_COUNTER == 0
def _get_func_or_collection(self, section_path_str: str) -> FunctionOrCollection:
section_path = SectionPath(section_path_str)
registrar_name = section_path[0]
# Check for specific class dict matching name
for registrar in self._registrars:
if registrar.name == registrar_name:
lookup_in_registrar_section_path = SectionPath.from_section_str_list(section_path[1:]).path_str
if not lookup_in_registrar_section_path:
# Was looking up registrar collection itself
return registrar.collection
# Looking up within registrar
return registrar.get(lookup_in_registrar_section_path)
# Try to return from general registrar
return self._general_registrar.get(section_path_str)
def _get_section_path_str_from_section_path_str_or_view(
self, section_path_str_or_view: 'StrOrView') -> str:
from pyfileconf.selector.models.itemview import is_item_view
if is_item_view(section_path_str_or_view):
# ItemView will have PipelineManager.name as first section, must strip
section_path = SectionPath(
section_path_str_or_view.section_path_str) # type: ignore
relative_section_path = SectionPath.from_section_str_list(
section_path[1:])
return relative_section_path.path_str
elif isinstance(section_path_str_or_view, str):
return section_path_str_or_view
else:
raise ValueError(
f'expected str or ItemView. Got {section_path_str_or_view} of '
f'type {type(section_path_str_or_view)}')
def test_create_then_update_entry_for_function(self):
self.write_a_function_to_pipeline_dict_file()
pipeline_manager = self.create_pm()
pipeline_manager.load()
sel = Selector()
pipeline_manager.create('thing', a_function)
iv = sel.test_pipeline_manager.thing.a_function
expected_b_result = ['a', 'b']
section_path = SectionPath.from_section_str_list(
SectionPath(iv.section_path_str)[1:])
pipeline_manager.update(b=expected_b_result,
section_path_str=section_path.path_str)
module_folder = os.path.join(self.defaults_path, 'thing')
function_path = os.path.join(module_folder, 'a_function.py')
with open(function_path, 'r') as f:
contents = f.read()
self.assert_a_function_config_file_contents(contents)
result = pipeline_manager.run(iv)
assert result == (None, expected_b_result)
def create(self,
section_path_str: str,
func_or_class: Optional[Union[Callable, Type]] = None):
"""
Create a new configuration entry dynamically rather than manually modifying dict file
:param func_or_class: function or class to use to generate config
:param section_path_str: section path at which the config should be stored
:return:
"""
logger.info(f'Creating config for {section_path_str}')
section_path = SectionPath(section_path_str)
if section_path[0] in self.specific_class_names:
# Got path for a specific class dict, add to specific class dict
if len(section_path) < 3:
raise ValueError(
'when targeting a specific class dict, section path must have minimum length of '
'3, e.g. example_class.thing.stuff')
if func_or_class is not None:
raise ValueError(
'only pass func_or_class when targeting main pipeline dict, not specific class dict'
)
self._create_specific_class_dict_entry(section_path)
full_sp = section_path
registrar = self._registrar_dict[section_path[0]]
klass = registrar.collection.klass
if klass is None:
raise ValueError(
f'must have specific class set, got None for class in {registrar}'
)
key_attr = registrar.collection.key_attr
registrar_obj = convert_to_empty_obj_if_necessary(
section_path[-1], klass, key_attr=key_attr)
set_section_path_str = SectionPath.from_section_str_list(
section_path[1:-1]).path_str
scaffold_path_str = SectionPath.from_section_str_list(
section_path[1:]).path_str
else:
# Got path for main pipeline dict, add to main pipeline dict
if func_or_class is None:
raise ValueError(
'when adding creating item in main pipeline dict, must pass function or class'
)
self._create_pipeline_dict_entry(section_path, func_or_class)
name = func_or_class.__name__
full_sp = SectionPath.join(section_path, name)
mod, import_base = get_module_and_name_imported_from(func_or_class)
obj_import = ObjectImportStatement([name], import_base)
item = ast.Name(id=name)
imports = ImportStatementContainer([obj_import])
registrar_obj = ObjectView.from_ast_and_imports(item, imports)
if self._general_registrar is None:
raise ValueError('general registrar must be defined')
registrar = self._general_registrar
set_section_path_str = section_path_str
scaffold_path_str = full_sp.path_str
registrar.set(set_section_path_str, registrar_obj)
registrar.scaffold_config_for(scaffold_path_str)
self.reset(full_sp.path_str, allow_create=True)
logger.debug(f'Finished creating config for {section_path_str}')
def __call__(self, *args, **kwargs):
from pyfileconf import PipelineManager
# When calling, assume user always wants the real item
actual_item = self.selector._get_real_item(self.section_path_str)
# If this happened while running another item, add to dependencies
self._add_to_config_dependencies_if_necessary()
# Determine whether this object is from a specific class collection
manager = PipelineManager.get_manager_by_section_path_str(
self.section_path_str)
collection_name = SectionPath(self.section_path_str)[1]
try:
manager._registrar_dict[collection_name]
specific_class = True
except KeyError:
specific_class = False
# Handle depending on the type of item
if isinstance(actual_item, partial):
# Got a function in the general registrar
func = actual_item
elif specific_class and isinstance(actual_item,
self._specific_classes):
# Got specific registrar class
# Need to look up the execute attribute and apply section path str
actual_item._section_path_str = self.section_path_str
collection = self._specific_class_collection_map[type(actual_item)]
execute_attr = collection.execute_attr
func = getattr(actual_item, execute_attr)
else:
cannot_parse_error = ValueError(
f'could not parse actual item, expected partial, '
f'specific class, or method of specific class. '
f'Got {actual_item} of type {type(actual_item)}')
orig_item: Any = None
try:
orig_item = actual_item.__self__
is_bound_method = True
except AttributeError:
is_bound_method = False
if is_bound_method:
if not isinstance(orig_item, self._specific_classes):
# Is bound method, but not for one of defined specific classes
raise cannot_parse_error
# Got specific class method
# Add section path to original item and then set method to be called
orig_item_sp = SectionPath.from_section_str_list(
SectionPath(self.section_path_str)[:-1])
orig_item_sp_str = orig_item_sp.path_str
orig_item._section_path_str = orig_item_sp_str
func = actual_item
else:
if inspect.isclass(actual_item):
raise cannot_parse_error
# Got a class object in the general registrar
actual_item._section_path_str = self.section_path_str
# Simply return it
return actual_item
result = func(*args, **kwargs)
return result