def preprocess(self, secrets: Optional[str] = None, download_ext: bool = True, install_ext: bool = False, import_ext: bool = True, check_tags: bool = True, **kwargs) -> Tuple[Runnable, Dict[str, str]]: """Preprocess the runnable file. Looks for syntax errors, import errors, etc. Also injects the secrets into the runnables. If this method runs and ends without exceptions, then the experiment is ok to be run. If this method raises an Error and the SafeExecutionContext is used as context manager, then the __exit__ method will be executed. Parameters ---------- secrets: Optional[str] Optional path to the secrets file install_ext: bool Whether to install the extensions or not. This process also downloads the remote extensions. Defaults to False install_ext: bool Whether to import the extensions or not. Defaults to True. check_tags: bool Whether to check that all tags are valid. Defaults to True. Returns ------- Tuple[Runnable, Dict[str, str]] A tuple containing the compiled Runnable and a dict containing the extensions the Runnable uses. Raises ------ Exception Depending on the error. """ content, extensions = self.first_parse() config = configparser.ConfigParser() if secrets: config.read(secrets) if install_ext: t = os.path.join(FLAMBE_GLOBAL_FOLDER, "extensions") extensions = download_extensions(extensions, t) install_extensions(extensions, user_flag=False) if import_ext: import_modules(extensions.keys()) # Check that all tags are valid if check_tags: self.check_tags(content) # Compile the runnable now that the extensions were imported. runnable = self.compile_runnable(content) if secrets: runnable.inject_secrets(secrets) if extensions: runnable.inject_extensions(extensions) runnable.parse() return runnable, extensions
def load(path: str, map_location=None, auto_install=False, pickle_module=dill, **pickle_load_args): """Load object with state from the given path Loads a flambe object by using the saved config files, and then loads the saved state into said object. See `load_state_from_file` for details regarding how the state is loaded from the save file or directory. Parameters ---------- path : str Path to the save file or directory map_location : type Location (device) where items will be moved. ONLY used when the directory save format is used. See torch.load documentation for more details (the default is None). auto_install : bool If True, automatically installs extensions as needed. pickle_module : type Pickle module that has load and dump methods; dump should accept a pickle_protocol parameter (the default is dill). **pickle_load_args : type Additional args that `pickle_module` should use to load; see torch.load documentation for more details Returns ------- Component object with both the architecture (config) and state that was saved to path Raises ------ LoadError If a Component object is not loadable from the given path because extensions are not installed, or the config is empty, nonexistent, or otherwise invalid. """ state = load_state_from_file(path, map_location, pickle_module, **pickle_load_args) yaml_config = state._metadata[''][FLAMBE_CONFIG_KEY] stash = state._metadata[''][FLAMBE_STASH_KEY] \ if FLAMBE_STASH_KEY in state._metadata[''] else None setup_default_modules() yamls = list(yaml.load_all(yaml_config)) if yamls is None: raise LoadError( "Cannot load schema from empty config. This object may not have been saved" " for any of the following reasons:\n - The object was not created from a" "config or with compile method\n - The object originally linked to other" "objects that cannot be represented in YAML") if len(yamls) > 2: raise LoadError( f"{os.path.join(path, CONFIG_FILE_NAME)} should contain an (optional) " "extensions section and the main object.") if len(yamls) == 2: if yamls[0] is not None: extensions = dict(yamls[0]) custom_modules = extensions.keys() for x in custom_modules: if not is_installed_module(x): if auto_install: logger.warn( f"auto_install==True, installing missing Module " f"{x}: {extensions[x]}") install_extensions({x: extensions[x]}) logger.debug( f"Installed module {x} from {extensions[x]}") else: raise ImportError( f"Module {x} is required and not installed. Please 'pip install'" "the package containing the module or set auto_install flag" " to True.") import_modules([x]) logger.debug(f"Automatically imported {x}") # Reload with extensions' module imported (and registered) schema = list(yaml.load_all(yaml_config))[1] # Set the extensions to the schema so that they are # passed when compiling the component. schema.add_extensions_metadata(extensions) else: schema = yamls[1] elif len(yamls) == 1: schema = yamls[0] else: raise LoadError( "No config found at location; cannot load. Try just loading state with " "the function 'load_state_from_file'") if schema is None: raise LoadError( "Cannot load schema from empty config. This object may not have been saved" " for any of the following reasons:\n - The object was not created from a" "config or with compile method\n - The object originally linked to other" "objects that cannot be represented in YAML") _update_link_refs(schema) # TODO: maybe replace with instance check if solution to circular # dependency with component is found try: instance = schema(stash) except TypeError: raise LoadError( f"Loaded object is not callable - likely because an extension is not " f"installed. Check if {os.path.join(path, CONFIG_FILE_NAME)} has an " f"extensions section at the top and install as necessary. Alternatively " f"set auto_install=True") instance.load_state(state) return instance