def _load_configuration_impl( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, from_shell: bool = True, ) -> DictConfig: from hydra import __version__ self.ensure_main_config_source_available() caching_repo = CachingConfigRepository(self.repository) parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) self._process_config_searchpath(config_name, parsed_overrides, caching_repo) self.validate_sweep_overrides_legal(overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell) defaults_list = create_defaults_list( repo=caching_repo, config_name=config_name, overrides_list=parsed_overrides, prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) config_overrides = defaults_list.config_overrides cfg = self._compose_config_from_defaults_list( defaults=defaults_list.defaults, repo=caching_repo) # Set config root to struct mode. # Note that this will close any dictionaries (including dicts annotated as Dict[K, V]. # One must use + to add new fields to them. OmegaConf.set_struct(cfg, True) # The Hydra node should not be read-only even if the root config is read-only. OmegaConf.set_readonly(cfg.hydra, False) # Apply command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg) app_overrides = [] for override in parsed_overrides: if override.is_hydra_override(): cfg.hydra.overrides.hydra.append(override.input_line) else: cfg.hydra.overrides.task.append(override.input_line) app_overrides.append(override) with open_dict(cfg.hydra): cfg.hydra.runtime.choices.update( defaults_list.overrides.known_choices) for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() cfg.hydra.runtime.config_sources = [ ConfigSourceInfo(path=x.path, schema=x.scheme(), provider=x.provider) for x in caching_repo.get_sources() ] if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( overrides=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys, ) cfg.hydra.job.config_name = config_name return cfg
def _process_config_searchpath( self, config_name: Optional[str], parsed_overrides: List[Override], repo: CachingConfigRepository, ) -> None: if config_name is not None: loaded = repo.load_config(config_path=config_name) primary_config: Container if loaded is None: primary_config = OmegaConf.create() else: primary_config = loaded.config else: primary_config = OmegaConf.create() if not OmegaConf.is_dict(primary_config): raise ConfigCompositionException( f"primary config '{config_name}' must be a DictConfig, got {type(primary_config).__name__}" ) def is_searchpath_override(v: Override) -> bool: return v.get_key_element() == "hydra.searchpath" override = None for v in parsed_overrides: if is_searchpath_override(v): override = v.value() break searchpath = OmegaConf.select(primary_config, "hydra.searchpath") if override is not None: provider = "hydra.searchpath in command-line" searchpath = override else: provider = "hydra.searchpath in main" def _err() -> None: raise ConfigCompositionException( f"hydra.searchpath must be a list of strings. Got: {searchpath}" ) if searchpath is None: return # validate hydra.searchpath. # Note that we cannot rely on OmegaConf validation here because we did not yet merge with the Hydra schema node if not isinstance(searchpath, MutableSequence): _err() for v in searchpath: if not isinstance(v, str): _err() new_csp = copy.deepcopy(self.config_search_path) schema = new_csp.get_path().pop(-1) assert schema.provider == "schema" for sp in searchpath: new_csp.append(provider=provider, path=sp) new_csp.append("schema", "structured://") repo.initialize_sources(new_csp) for source in repo.get_sources(): if not source.available(): warnings.warn( category=UserWarning, message= f"provider={source.provider}, path={source.path} is not available.", )