def compute_defaults_list( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, ) -> DefaultsList: parser = OverridesParser.create() repo = CachingConfigRepository(self.repository) defaults_list = create_defaults_list( repo=repo, config_name=config_name, overrides_list=parser.parse_overrides(overrides=overrides), prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) return defaults_list
def _load_configuration_impl( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, from_shell: bool = True, ) -> DictConfig: from hydra import __version__ self.ensure_main_config_source_available() caching_repo = CachingConfigRepository(self.repository) parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) self._process_config_searchpath(config_name, parsed_overrides, caching_repo) self.validate_sweep_overrides_legal(overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell) defaults_list = create_defaults_list( repo=caching_repo, config_name=config_name, overrides_list=parsed_overrides, prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) config_overrides = defaults_list.config_overrides cfg = self._compose_config_from_defaults_list( defaults=defaults_list.defaults, repo=caching_repo) # Set config root to struct mode. # Note that this will close any dictionaries (including dicts annotated as Dict[K, V]. # One must use + to add new fields to them. OmegaConf.set_struct(cfg, True) # The Hydra node should not be read-only even if the root config is read-only. OmegaConf.set_readonly(cfg.hydra, False) # Apply command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg) app_overrides = [] for override in parsed_overrides: if override.is_hydra_override(): cfg.hydra.overrides.hydra.append(override.input_line) else: cfg.hydra.overrides.task.append(override.input_line) app_overrides.append(override) with open_dict(cfg.hydra): cfg.hydra.runtime.choices.update( defaults_list.overrides.known_choices) for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() cfg.hydra.runtime.config_sources = [ ConfigSourceInfo(path=x.path, schema=x.scheme(), provider=x.provider) for x in caching_repo.get_sources() ] if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( overrides=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys, ) cfg.hydra.job.config_name = config_name return cfg
def _load_configuration_impl( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, from_shell: bool = True, ) -> DictConfig: self.ensure_main_config_source_available() caching_repo = CachingConfigRepository(self.repository) parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) self.validate_sweep_overrides_legal(overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell) defaults_list = create_defaults_list( repo=caching_repo, config_name=config_name, overrides_list=parser.parse_overrides(overrides=overrides), prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) config_overrides = defaults_list.config_overrides cfg = self._compose_config_from_defaults_list( defaults=defaults_list.defaults, repo=caching_repo) # set struct mode on user config if the root node is not typed. if OmegaConf.get_type(cfg) is dict: OmegaConf.set_struct(cfg, True) # Turn off struct mode on the Hydra node. It gets its type safety from it being a Structured Config. # This enables adding fields to nested dicts like hydra.job.env_set without having to using + to append. OmegaConf.set_struct(cfg.hydra, False) # Make the Hydra node writeable (The user may have made the primary node read-only). OmegaConf.set_readonly(cfg.hydra, False) # Apply command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg) app_overrides = [] for override in parsed_overrides: if override.is_hydra_override(): cfg.hydra.overrides.hydra.append(override.input_line) else: cfg.hydra.overrides.task.append(override.input_line) app_overrides.append(override) cfg.hydra.choices.update(defaults_list.overrides.known_choices) with open_dict(cfg.hydra): from hydra import __version__ cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( overrides=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname. exclude_keys, ) cfg.hydra.job.config_name = config_name for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] return cfg
def _load_configuration_impl( self, config_name: Optional[str], overrides: List[str], run_mode: RunMode, strict: Optional[bool] = None, from_shell: bool = True, ) -> DictConfig: self.ensure_main_config_source_available() caching_repo = CachingConfigRepository(self.repository) if strict is None: strict = self.default_strict parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) self.validate_sweep_overrides_legal( overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell ) defaults_list = create_defaults_list( repo=caching_repo, config_name=config_name, overrides_list=parser.parse_overrides(overrides=overrides), prepend_hydra=True, skip_missing=run_mode == RunMode.MULTIRUN, ) config_overrides = defaults_list.config_overrides cfg, composition_trace = self._compose_config_from_defaults_list( defaults=defaults_list.defaults, repo=caching_repo ) OmegaConf.set_struct(cfg, strict) OmegaConf.set_readonly(cfg.hydra, False) # Apply command line overrides after enabling strict flag ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg) app_overrides = [] for override in parsed_overrides: if override.is_hydra_override(): cfg.hydra.overrides.hydra.append(override.input_line) else: cfg.hydra.overrides.task.append(override.input_line) app_overrides.append(override) # TODO: should this open_dict be required given that choices is a Dict? with open_dict(cfg.hydra.choices): cfg.hydra.choices.update(defaults_list.overrides.known_choices) with open_dict(cfg.hydra): from hydra import __version__ cfg.hydra.runtime.version = __version__ cfg.hydra.runtime.cwd = os.getcwd() cfg.hydra.composition_trace = composition_trace if "name" not in cfg.hydra.job: cfg.hydra.job.name = JobRuntime().get("name") cfg.hydra.job.override_dirname = get_overrides_dirname( overrides=app_overrides, kv_sep=cfg.hydra.job.config.override_dirname.kv_sep, item_sep=cfg.hydra.job.config.override_dirname.item_sep, exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys, ) cfg.hydra.job.config_name = config_name for key in cfg.hydra.job.env_copy: cfg.hydra.job.env_set[key] = os.environ[key] return cfg