def _create_pipeline_dict_entry(self, section_path: SectionPath,
func_or_class: Union[Callable, Type]):
pipeline_dict_file = PipelineDictFile(self.pipeline_dict_path,
name='pipeline_dict')
pipeline_dict = pipeline_dict_file.load()
mod, import_base = get_module_and_name_imported_from(func_or_class)
obj_import = ObjectImportStatement([func_or_class.__name__],
import_base)
imports = ImportStatementContainer([obj_import])
# Modify pipeline dict to add entry
add_item_into_nested_dict_at_section_path(pipeline_dict, section_path,
func_or_class.__name__)
# Convert pipeline dict to string for output
pipeline_dict_str = create_dict_assignment_str_from_nested_dict_with_ast_names(
pipeline_dict)
pipeline_dict_config = PipelineDictConfig(
dict(pipeline_dict=pipeline_dict_str),
name='pipeline_dict',
_file=pipeline_dict_file,
imports=imports)
pipeline_dict_file.save(pipeline_dict_config)
def __init__(self,
filepath: str,
name: str = None,
klass: Optional[Type] = None,
always_import_strs: Optional[Sequence[str]] = None,
always_assign_strs: Optional[Sequence[str]] = None):
super().__init__(filepath,
name=name,
klass=klass,
always_import_strs=always_import_strs,
always_assign_strs=always_assign_strs)
# Override class definitions with object specific definitions, if specifics are passed
if self.always_import_strs:
imports = []
for import_str in self.always_import_strs:
try:
imports.append(ObjectImportStatement.from_str(import_str))
except ExtractedIncorrectTypeOfImportException:
imports.append(ModuleImportStatement.from_str(import_str))
self.always_imports = imports
elif self.always_import_strs == []:
# None passed, remove default imports
self.always_imports = []
if self.always_assign_strs:
self.always_assigns = [
AssignmentStatement.from_str(assign_str)
for assign_str in self.always_assign_strs
]
elif self.always_assign_strs == []:
# None passed, remove default assignments
self.always_assigns = []
class FunctionConfigFile(ConfigFileBase):
"""
Represents config file on filesystem. Handles low-level functions for writing and reading config file
"""
# lines to always import. pass import objects
always_imports = [
ObjectImportStatement.from_str('from pyfileconf import Selector', preferred_position='begin')
]
# assignment lines to always include at beginning. pass assign objects
always_assigns = [
AssignmentStatement.from_str('s = Selector()', preferred_position='begin'),
]
# always assign dict, where assigns will get added if item name matches dict key
always_assign_with_names_dict = {
'DataPipeline': [
AssignmentStatement.from_str('cleanup_kwargs = dict(\n \n)', preferred_position='end')
]
}
# class to use for interfacing with file
# no need to override default
# interface_class = ConfigFileInterface
def save(self, config: 'ConfigBase'):
# If empty cleanup kwargs on DataPipeline, then delete so can be replaced with dict() constructor
# from always assigns
if _should_replace_cleanup_kwargs(self, config):
config.pop('cleanup_kwargs')
super().save(config)
def get_import_for_module_or_obj_name(self, name: str) -> ImportOrNone:
found_import = False
for imp in self:
if isinstance(imp, ModuleImportStatement):
if name in imp.modules:
# match on original module name
# set up for creating a new import
found_import = True
renames = None
modules = [name]
elif name in imp.renames.new_names:
# match on renamed module name
# set up for creating a new import
found_import = True
renames = RenameStatementCollection(
# Pull the rename matching this name
[
rename for rename in imp.renames
if rename.new_name == name
])
# grab original module matching this rename
modules = [renames.reverse_name_map[name]]
if found_import:
# May be multiple modules imported in this one statement. Create a new statement with just this module
return ModuleImportStatement(modules=modules,
renames=renames)
elif isinstance(imp, ObjectImportStatement):
if name in imp.objs:
# match on original object name
# set up for creating a new import
found_import = True
renames = None
objs = [name]
elif name in imp.renames.new_names:
# match on renamed object name
# set up for creating a new import
found_import = True
renames = RenameStatementCollection(
# Pull the rename matching this name
[
rename for rename in imp.renames
if rename.new_name == name
])
# grab original object matching this rename
objs = [renames.reverse_name_map[name]]
if found_import:
# May be multiple objects imported in this one statement. Create a new statement with just this object
return ObjectImportStatement(objs,
module=imp.module,
renames=renames)
raise NoImportMatchingNameException(
f'could not find name {name} in {self}')
def _create_item(self, n_clicks: int, section_path: str, function_class_import: Optional[str]):
if not section_path:
return dash.no_update
if function_class_import:
imp = ObjectImportStatement.from_str(function_class_import)
if len(imp.objs) != 1:
raise ValueError(f'must have exactly one object import, got {imp.objs}')
mod = importlib.import_module(imp.module)
func_or_class = getattr(mod, imp.objs[0])
else:
func_or_class = None
output = self.gui.runner.create(section_path, func_or_class)
return f'Created {section_path}'
def _output_config_file(self, item: ObjOrCollection) -> None:
if isinstance(item, SpecificClassCollection):
# if collection, recursively call creating config files
return item._output_config_files()
# Dealing with object itself
item_name = getattr(item, self.key_attr)
item_filepath = os.path.join(self.basepath, item_name + '.py')
class_config = dict(
klass=self.klass,
always_import_strs=self.always_import_strs,
always_assign_strs=self.always_assign_strs
)
if os.path.exists(item_filepath):
# if config file already exists, load confguration from file, use to update file defaults
existing_config = SpecificClassConfig.from_file(
item_filepath,
name=item_name,
**class_config, # type: ignore
)
existing_imports = existing_config._file.interface.imports
else:
existing_imports = None
item_config = SpecificClassConfig(imports=existing_imports,
file_path=item_filepath,
**class_config) # type: ignore
# Get config by extracting from class __init__
# First need to create dummy import for compatibility
mod, import_base = get_module_and_name_imported_from(self.klass)
obj_import = ObjectImportStatement([item_name], import_base)
# Now get the arguments and the imports for any type annotations
args, func_arg_imports = extract_function_args_and_arg_imports_from_import(
self.klass.__name__,
obj_import
)
# Convert into a usable formats:
# defaults_dict: a dictionary where keys are variable names and values are ast defaults
# annotation_dict: a dicrionary where keys are variable names and values are ast type annotations
defaults_dict, annotation_dict = function_args_as_arg_and_annotation_dict(args)
defaults_dict[self.key_attr] = ast_str(item_name) # set name attribute as item name by default
# Apply all the new extracted defaults to the created config
item_config.update(defaults_dict)
item_config.annotations.update(annotation_dict)
item_config.imports.extend(func_arg_imports)
item_config.to_file(item_filepath)
def create(self,
section_path_str: str,
func_or_class: Optional[Union[Callable, Type]] = None):
"""
Create a new configuration entry dynamically rather than manually modifying dict file
:param func_or_class: function or class to use to generate config
:param section_path_str: section path at which the config should be stored
:return:
"""
logger.info(f'Creating config for {section_path_str}')
section_path = SectionPath(section_path_str)
if section_path[0] in self.specific_class_names:
# Got path for a specific class dict, add to specific class dict
if len(section_path) < 3:
raise ValueError(
'when targeting a specific class dict, section path must have minimum length of '
'3, e.g. example_class.thing.stuff')
if func_or_class is not None:
raise ValueError(
'only pass func_or_class when targeting main pipeline dict, not specific class dict'
)
self._create_specific_class_dict_entry(section_path)
full_sp = section_path
registrar = self._registrar_dict[section_path[0]]
klass = registrar.collection.klass
if klass is None:
raise ValueError(
f'must have specific class set, got None for class in {registrar}'
)
key_attr = registrar.collection.key_attr
registrar_obj = convert_to_empty_obj_if_necessary(
section_path[-1], klass, key_attr=key_attr)
set_section_path_str = SectionPath.from_section_str_list(
section_path[1:-1]).path_str
scaffold_path_str = SectionPath.from_section_str_list(
section_path[1:]).path_str
else:
# Got path for main pipeline dict, add to main pipeline dict
if func_or_class is None:
raise ValueError(
'when adding creating item in main pipeline dict, must pass function or class'
)
self._create_pipeline_dict_entry(section_path, func_or_class)
name = func_or_class.__name__
full_sp = SectionPath.join(section_path, name)
mod, import_base = get_module_and_name_imported_from(func_or_class)
obj_import = ObjectImportStatement([name], import_base)
item = ast.Name(id=name)
imports = ImportStatementContainer([obj_import])
registrar_obj = ObjectView.from_ast_and_imports(item, imports)
if self._general_registrar is None:
raise ValueError('general registrar must be defined')
registrar = self._general_registrar
set_section_path_str = section_path_str
scaffold_path_str = full_sp.path_str
registrar.set(set_section_path_str, registrar_obj)
registrar.scaffold_config_for(scaffold_path_str)
self.reset(full_sp.path_str, allow_create=True)
logger.debug(f'Finished creating config for {section_path_str}')
class SpecificClassConfigFile(ConfigFileBase):
# lines to always import. pass import objects
always_imports = [
ObjectImportStatement.from_str('from pyfileconf import Selector',
preferred_position='begin')
]
# assignment lines to always include at beginning. pass assign objects
always_assigns = [
AssignmentStatement.from_str('s = Selector()',
preferred_position='begin'),
]
def __init__(self,
filepath: str,
name: str = None,
klass: Optional[Type] = None,
always_import_strs: Optional[Sequence[str]] = None,
always_assign_strs: Optional[Sequence[str]] = None):
super().__init__(filepath,
name=name,
klass=klass,
always_import_strs=always_import_strs,
always_assign_strs=always_assign_strs)
# Override class definitions with object specific definitions, if specifics are passed
if self.always_import_strs:
imports = []
for import_str in self.always_import_strs:
try:
imports.append(ObjectImportStatement.from_str(import_str))
except ExtractedIncorrectTypeOfImportException:
imports.append(ModuleImportStatement.from_str(import_str))
self.always_imports = imports
elif self.always_import_strs == []:
# None passed, remove default imports
self.always_imports = []
if self.always_assign_strs:
self.always_assigns = [
AssignmentStatement.from_str(assign_str)
for assign_str in self.always_assign_strs
]
elif self.always_assign_strs == []:
# None passed, remove default assignments
self.always_assigns = []
def __call__(self, *args, **kwargs):
"""
For compatibility with BaseConfig which expects to call class, while here an object will be used
"""
# Create new object
# Use defaults from this object
obj_kwargs = dict(name=self.name,
klass=self.klass,
always_import_strs=self.always_import_strs,
always_assign_strs=self.always_assign_strs)
obj_kwargs.update(kwargs)
obj = self.__class__(*args, **obj_kwargs)
return obj
def load(self, config_class: type = None) -> 'SpecificClassConfig':
# Override base class method to pull a single dict, and not pass annotations
from pyfileconf.data.models.config import SpecificClassConfig
config_dict, annotation_dict = self.interface.load()
if config_class is None:
config_class = SpecificClassConfig
return config_class(
d=config_dict,
annotations=annotation_dict,
imports=self.interface.imports,
_file=self,
name=self.name,
klass=self.klass,
always_assign_strs=self.always_assign_strs,
always_import_strs=self.always_import_strs,
)