def _create_pipeline_dict_entry(self, section_path: SectionPath, func_or_class: Union[Callable, Type]): pipeline_dict_file = PipelineDictFile(self.pipeline_dict_path, name='pipeline_dict') pipeline_dict = pipeline_dict_file.load() mod, import_base = get_module_and_name_imported_from(func_or_class) obj_import = ObjectImportStatement([func_or_class.__name__], import_base) imports = ImportStatementContainer([obj_import]) # Modify pipeline dict to add entry add_item_into_nested_dict_at_section_path(pipeline_dict, section_path, func_or_class.__name__) # Convert pipeline dict to string for output pipeline_dict_str = create_dict_assignment_str_from_nested_dict_with_ast_names( pipeline_dict) pipeline_dict_config = PipelineDictConfig( dict(pipeline_dict=pipeline_dict_str), name='pipeline_dict', _file=pipeline_dict_file, imports=imports) pipeline_dict_file.save(pipeline_dict_config)
def __init__(self, filepath: str, name: str = None, klass: Optional[Type] = None, always_import_strs: Optional[Sequence[str]] = None, always_assign_strs: Optional[Sequence[str]] = None): super().__init__(filepath, name=name, klass=klass, always_import_strs=always_import_strs, always_assign_strs=always_assign_strs) # Override class definitions with object specific definitions, if specifics are passed if self.always_import_strs: imports = [] for import_str in self.always_import_strs: try: imports.append(ObjectImportStatement.from_str(import_str)) except ExtractedIncorrectTypeOfImportException: imports.append(ModuleImportStatement.from_str(import_str)) self.always_imports = imports elif self.always_import_strs == []: # None passed, remove default imports self.always_imports = [] if self.always_assign_strs: self.always_assigns = [ AssignmentStatement.from_str(assign_str) for assign_str in self.always_assign_strs ] elif self.always_assign_strs == []: # None passed, remove default assignments self.always_assigns = []
class FunctionConfigFile(ConfigFileBase): """ Represents config file on filesystem. Handles low-level functions for writing and reading config file """ # lines to always import. pass import objects always_imports = [ ObjectImportStatement.from_str('from pyfileconf import Selector', preferred_position='begin') ] # assignment lines to always include at beginning. pass assign objects always_assigns = [ AssignmentStatement.from_str('s = Selector()', preferred_position='begin'), ] # always assign dict, where assigns will get added if item name matches dict key always_assign_with_names_dict = { 'DataPipeline': [ AssignmentStatement.from_str('cleanup_kwargs = dict(\n \n)', preferred_position='end') ] } # class to use for interfacing with file # no need to override default # interface_class = ConfigFileInterface def save(self, config: 'ConfigBase'): # If empty cleanup kwargs on DataPipeline, then delete so can be replaced with dict() constructor # from always assigns if _should_replace_cleanup_kwargs(self, config): config.pop('cleanup_kwargs') super().save(config)
def get_import_for_module_or_obj_name(self, name: str) -> ImportOrNone: found_import = False for imp in self: if isinstance(imp, ModuleImportStatement): if name in imp.modules: # match on original module name # set up for creating a new import found_import = True renames = None modules = [name] elif name in imp.renames.new_names: # match on renamed module name # set up for creating a new import found_import = True renames = RenameStatementCollection( # Pull the rename matching this name [ rename for rename in imp.renames if rename.new_name == name ]) # grab original module matching this rename modules = [renames.reverse_name_map[name]] if found_import: # May be multiple modules imported in this one statement. Create a new statement with just this module return ModuleImportStatement(modules=modules, renames=renames) elif isinstance(imp, ObjectImportStatement): if name in imp.objs: # match on original object name # set up for creating a new import found_import = True renames = None objs = [name] elif name in imp.renames.new_names: # match on renamed object name # set up for creating a new import found_import = True renames = RenameStatementCollection( # Pull the rename matching this name [ rename for rename in imp.renames if rename.new_name == name ]) # grab original object matching this rename objs = [renames.reverse_name_map[name]] if found_import: # May be multiple objects imported in this one statement. Create a new statement with just this object return ObjectImportStatement(objs, module=imp.module, renames=renames) raise NoImportMatchingNameException( f'could not find name {name} in {self}')
def _create_item(self, n_clicks: int, section_path: str, function_class_import: Optional[str]): if not section_path: return dash.no_update if function_class_import: imp = ObjectImportStatement.from_str(function_class_import) if len(imp.objs) != 1: raise ValueError(f'must have exactly one object import, got {imp.objs}') mod = importlib.import_module(imp.module) func_or_class = getattr(mod, imp.objs[0]) else: func_or_class = None output = self.gui.runner.create(section_path, func_or_class) return f'Created {section_path}'
def _output_config_file(self, item: ObjOrCollection) -> None: if isinstance(item, SpecificClassCollection): # if collection, recursively call creating config files return item._output_config_files() # Dealing with object itself item_name = getattr(item, self.key_attr) item_filepath = os.path.join(self.basepath, item_name + '.py') class_config = dict( klass=self.klass, always_import_strs=self.always_import_strs, always_assign_strs=self.always_assign_strs ) if os.path.exists(item_filepath): # if config file already exists, load confguration from file, use to update file defaults existing_config = SpecificClassConfig.from_file( item_filepath, name=item_name, **class_config, # type: ignore ) existing_imports = existing_config._file.interface.imports else: existing_imports = None item_config = SpecificClassConfig(imports=existing_imports, file_path=item_filepath, **class_config) # type: ignore # Get config by extracting from class __init__ # First need to create dummy import for compatibility mod, import_base = get_module_and_name_imported_from(self.klass) obj_import = ObjectImportStatement([item_name], import_base) # Now get the arguments and the imports for any type annotations args, func_arg_imports = extract_function_args_and_arg_imports_from_import( self.klass.__name__, obj_import ) # Convert into a usable formats: # defaults_dict: a dictionary where keys are variable names and values are ast defaults # annotation_dict: a dicrionary where keys are variable names and values are ast type annotations defaults_dict, annotation_dict = function_args_as_arg_and_annotation_dict(args) defaults_dict[self.key_attr] = ast_str(item_name) # set name attribute as item name by default # Apply all the new extracted defaults to the created config item_config.update(defaults_dict) item_config.annotations.update(annotation_dict) item_config.imports.extend(func_arg_imports) item_config.to_file(item_filepath)
def create(self, section_path_str: str, func_or_class: Optional[Union[Callable, Type]] = None): """ Create a new configuration entry dynamically rather than manually modifying dict file :param func_or_class: function or class to use to generate config :param section_path_str: section path at which the config should be stored :return: """ logger.info(f'Creating config for {section_path_str}') section_path = SectionPath(section_path_str) if section_path[0] in self.specific_class_names: # Got path for a specific class dict, add to specific class dict if len(section_path) < 3: raise ValueError( 'when targeting a specific class dict, section path must have minimum length of ' '3, e.g. example_class.thing.stuff') if func_or_class is not None: raise ValueError( 'only pass func_or_class when targeting main pipeline dict, not specific class dict' ) self._create_specific_class_dict_entry(section_path) full_sp = section_path registrar = self._registrar_dict[section_path[0]] klass = registrar.collection.klass if klass is None: raise ValueError( f'must have specific class set, got None for class in {registrar}' ) key_attr = registrar.collection.key_attr registrar_obj = convert_to_empty_obj_if_necessary( section_path[-1], klass, key_attr=key_attr) set_section_path_str = SectionPath.from_section_str_list( section_path[1:-1]).path_str scaffold_path_str = SectionPath.from_section_str_list( section_path[1:]).path_str else: # Got path for main pipeline dict, add to main pipeline dict if func_or_class is None: raise ValueError( 'when adding creating item in main pipeline dict, must pass function or class' ) self._create_pipeline_dict_entry(section_path, func_or_class) name = func_or_class.__name__ full_sp = SectionPath.join(section_path, name) mod, import_base = get_module_and_name_imported_from(func_or_class) obj_import = ObjectImportStatement([name], import_base) item = ast.Name(id=name) imports = ImportStatementContainer([obj_import]) registrar_obj = ObjectView.from_ast_and_imports(item, imports) if self._general_registrar is None: raise ValueError('general registrar must be defined') registrar = self._general_registrar set_section_path_str = section_path_str scaffold_path_str = full_sp.path_str registrar.set(set_section_path_str, registrar_obj) registrar.scaffold_config_for(scaffold_path_str) self.reset(full_sp.path_str, allow_create=True) logger.debug(f'Finished creating config for {section_path_str}')
class SpecificClassConfigFile(ConfigFileBase): # lines to always import. pass import objects always_imports = [ ObjectImportStatement.from_str('from pyfileconf import Selector', preferred_position='begin') ] # assignment lines to always include at beginning. pass assign objects always_assigns = [ AssignmentStatement.from_str('s = Selector()', preferred_position='begin'), ] def __init__(self, filepath: str, name: str = None, klass: Optional[Type] = None, always_import_strs: Optional[Sequence[str]] = None, always_assign_strs: Optional[Sequence[str]] = None): super().__init__(filepath, name=name, klass=klass, always_import_strs=always_import_strs, always_assign_strs=always_assign_strs) # Override class definitions with object specific definitions, if specifics are passed if self.always_import_strs: imports = [] for import_str in self.always_import_strs: try: imports.append(ObjectImportStatement.from_str(import_str)) except ExtractedIncorrectTypeOfImportException: imports.append(ModuleImportStatement.from_str(import_str)) self.always_imports = imports elif self.always_import_strs == []: # None passed, remove default imports self.always_imports = [] if self.always_assign_strs: self.always_assigns = [ AssignmentStatement.from_str(assign_str) for assign_str in self.always_assign_strs ] elif self.always_assign_strs == []: # None passed, remove default assignments self.always_assigns = [] def __call__(self, *args, **kwargs): """ For compatibility with BaseConfig which expects to call class, while here an object will be used """ # Create new object # Use defaults from this object obj_kwargs = dict(name=self.name, klass=self.klass, always_import_strs=self.always_import_strs, always_assign_strs=self.always_assign_strs) obj_kwargs.update(kwargs) obj = self.__class__(*args, **obj_kwargs) return obj def load(self, config_class: type = None) -> 'SpecificClassConfig': # Override base class method to pull a single dict, and not pass annotations from pyfileconf.data.models.config import SpecificClassConfig config_dict, annotation_dict = self.interface.load() if config_class is None: config_class = SpecificClassConfig return config_class( d=config_dict, annotations=annotation_dict, imports=self.interface.imports, _file=self, name=self.name, klass=self.klass, always_assign_strs=self.always_assign_strs, always_import_strs=self.always_import_strs, )