def _process_config_searchpath( self, config_name: Optional[str], parsed_overrides: List[Override], repo: CachingConfigRepository, ) -> None: if config_name is not None: loaded = repo.load_config(config_path=config_name) primary_config: Container if loaded is None: primary_config = OmegaConf.create() else: primary_config = loaded.config else: primary_config = OmegaConf.create() def is_searchpath_override(v: Override) -> bool: return v.get_key_element() == "hydra.searchpath" override = None for v in parsed_overrides: if is_searchpath_override(v): override = v.value() break searchpath = OmegaConf.select(primary_config, "hydra.searchpath") if override is not None: provider = "hydra.searchpath in command-line" searchpath = override else: provider = "hydra.searchpath in main" def _err() -> None: raise ConfigCompositionException( f"hydra.searchpath must be a list of strings. Got: {searchpath}" ) if searchpath is None: return # validate hydra.searchpath. # Note that we cannot rely on OmegaConf validation here because we did not yet merge with the Hydra schema node if not isinstance(searchpath, MutableSequence): _err() for v in searchpath: if not isinstance(v, str): _err() new_csp = copy.deepcopy(self.config_search_path) schema = new_csp.get_path().pop(-1) assert schema.provider == "schema" for sp in searchpath: new_csp.append(provider=provider, path=sp) new_csp.append("schema", "structured://") repo.initialize_sources(new_csp)
def compute_defaults_list( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, ) -> DefaultsList: parser = OverridesParser.create() repo = CachingConfigRepository(self.repository) defaults_list = create_defaults_list( repo=repo, config_name=config_name, overrides_list=parser.parse_overrides(overrides=overrides), prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) return defaults_list
def _load_configuration_impl( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, from_shell: bool = True, ) -> DictConfig: from hydra import __version__ self.ensure_main_config_source_available() caching_repo = CachingConfigRepository(self.repository) parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) self._process_config_searchpath(config_name, parsed_overrides, caching_repo) self.validate_sweep_overrides_legal(overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell) defaults_list = create_defaults_list( repo=caching_repo, config_name=config_name, overrides_list=parsed_overrides, prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) config_overrides = defaults_list.config_overrides cfg = self._compose_config_from_defaults_list( defaults=defaults_list.defaults, repo=caching_repo) # Set config root to struct mode. # Note that this will close any dictionaries (including dicts annotated as Dict[K, V]. # One must use + to add new fields to them. OmegaConf.set_struct(cfg, True) # The Hydra node should not be read-only even if the root config is read-only. OmegaConf.set_readonly(cfg.hydra, False) # Apply command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg) app_overrides = [] for override in parsed_overrides: if override.is_hydra_override(): cfg.hydra.overrides.hydra.append(override.input_line) else: cfg.hydra.overrides.task.append(override.input_line) app_overrides.append(override) with open_dict(cfg.hydra): cfg.hydra.runtime.choices.update( defaults_list.overrides.known_choices) for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() cfg.hydra.runtime.config_sources = [ ConfigSourceInfo(path=x.path, schema=x.scheme(), provider=x.provider) for x in caching_repo.get_sources() ] if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( overrides=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys, ) cfg.hydra.job.config_name = config_name return cfg
def _load_configuration_impl( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, from_shell: bool = True, ) -> DictConfig: self.ensure_main_config_source_available() caching_repo = CachingConfigRepository(self.repository) parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) self.validate_sweep_overrides_legal(overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell) defaults_list = create_defaults_list( repo=caching_repo, config_name=config_name, overrides_list=parser.parse_overrides(overrides=overrides), prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) config_overrides = defaults_list.config_overrides cfg = self._compose_config_from_defaults_list( defaults=defaults_list.defaults, repo=caching_repo) # set struct mode on user config if the root node is not typed. if OmegaConf.get_type(cfg) is dict: OmegaConf.set_struct(cfg, True) # Turn off struct mode on the Hydra node. It gets its type safety from it being a Structured Config. # This enables adding fields to nested dicts like hydra.job.env_set without having to using + to append. OmegaConf.set_struct(cfg.hydra, False) # Make the Hydra node writeable (The user may have made the primary node read-only). OmegaConf.set_readonly(cfg.hydra, False) # Apply command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg) app_overrides = [] for override in parsed_overrides: if override.is_hydra_override(): cfg.hydra.overrides.hydra.append(override.input_line) else: cfg.hydra.overrides.task.append(override.input_line) app_overrides.append(override) cfg.hydra.choices.update(defaults_list.overrides.known_choices) with open_dict(cfg.hydra): from hydra import __version__ cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( overrides=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname. exclude_keys, ) cfg.hydra.job.config_name = config_name for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] return cfg
def _load_configuration_impl( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, strict: Optional[bool] = None, from_shell: bool = True, ) -> DictConfig: self.ensure_main_config_source_available() caching_repo = CachingConfigRepository(self.repository) if config_name is not None and not caching_repo.config_exists( config_name): self._missing_config_error( config_name=config_name, msg= f"Cannot find primary config : {config_name}, check that it's in your config search path", with_search_path=True, ) if strict is None: strict = self.default_strict parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) config_overrides = ConfigLoaderImpl.parse_overrides( overrides=overrides, run_mode=run_mode, from_shell=from_shell) split_res = self.split_by_override_type(config_overrides) config_group_overrides = split_res.config_group_overrides config_overrides = split_res.config_overrides input_defaults = [DefaultElement(config_name="hydra_config")] if config_name is not None: input_defaults.append( DefaultElement(config_name=config_name, primary=True)) for default in convert_overrides_to_defaults(config_group_overrides): input_defaults.append(default) skip_missing = run_mode == RunMode.MULTIRUN defaults = expand_defaults_list( defaults=input_defaults, skip_missing=skip_missing, repo=caching_repo, ) cfg, composition_trace = self._compose_config_from_defaults_list( defaults=defaults, repo=caching_repo) OmegaConf.set_struct(cfg, strict) OmegaConf.set_readonly(cfg.hydra, False) # Apply command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg) app_overrides = [] for override in parsed_overrides: if override.is_hydra_override(): cfg.hydra.overrides.hydra.append(override.input_line) else: cfg.hydra.overrides.task.append(override.input_line) app_overrides.append(override) with open_dict(cfg.hydra): from hydra import __version__ cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() cfg.hydra.composition_trace = composition_trace if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( overrides=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname. exclude_keys, ) cfg.hydra.job.config_name = config_name for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] return cfg
def _load_configuration_impl( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, strict: Optional[bool] = None, from_shell: bool = True, ) -> DictConfig: self.ensure_main_config_source_available() caching_repo = CachingConfigRepository(self.repository) if strict is None: strict = self.default_strict parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) self.validate_sweep_overrides_legal( overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell ) defaults_list = create_defaults_list( repo=caching_repo, config_name=config_name, overrides_list=parser.parse_overrides(overrides=overrides), prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) config_overrides = defaults_list.config_overrides cfg, composition_trace = self._compose_config_from_defaults_list( defaults=defaults_list.defaults, repo=caching_repo ) OmegaConf.set_struct(cfg, strict) OmegaConf.set_readonly(cfg.hydra, False) # Apply command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg) app_overrides = [] for override in parsed_overrides: if override.is_hydra_override(): cfg.hydra.overrides.hydra.append(override.input_line) else: cfg.hydra.overrides.task.append(override.input_line) app_overrides.append(override) # TODO: should this open_dict be required given that choices is a Dict? with open_dict(cfg.hydra.choices): cfg.hydra.choices.update(defaults_list.overrides.known_choices) with open_dict(cfg.hydra): from hydra import __version__ cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() cfg.hydra.composition_trace = composition_trace if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( overrides=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys, ) cfg.hydra.job.config_name = config_name for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] return cfg