def _load_configuration_impl(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
from_shell: bool = True,
) -> DictConfig:
from hydra import __version__
self.ensure_main_config_source_available()
caching_repo = CachingConfigRepository(self.repository)
parser = OverridesParser.create()
parsed_overrides = parser.parse_overrides(overrides=overrides)
self._process_config_searchpath(config_name, parsed_overrides,
caching_repo)
self.validate_sweep_overrides_legal(overrides=parsed_overrides,
run_mode=run_mode,
from_shell=from_shell)
defaults_list = create_defaults_list(
repo=caching_repo,
config_name=config_name,
overrides_list=parsed_overrides,
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
config_overrides = defaults_list.config_overrides
cfg = self._compose_config_from_defaults_list(
defaults=defaults_list.defaults, repo=caching_repo)
# Set config root to struct mode.
# Note that this will close any dictionaries (including dicts annotated as Dict[K, V].
# One must use + to add new fields to them.
OmegaConf.set_struct(cfg, True)
# The Hydra node should not be read-only even if the root config is read-only.
OmegaConf.set_readonly(cfg.hydra, False)
# Apply command line overrides after enabling strict flag
ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
app_overrides = []
for override in parsed_overrides:
if override.is_hydra_override():
cfg.hydra.overrides.hydra.append(override.input_line)
else:
cfg.hydra.overrides.task.append(override.input_line)
app_overrides.append(override)
with open_dict(cfg.hydra):
cfg.hydra.runtime.choices.update(
defaults_list.overrides.known_choices)
for key in cfg.hydra.job.env_copy:
cfg.hydra.job.env_set[key] = os.environ[key]
cfg.hydra.runtime.version = __version__
cfg.hydra.runtime.cwd = os.getcwd()
cfg.hydra.runtime.config_sources = [
ConfigSourceInfo(path=x.path,
schema=x.scheme(),
provider=x.provider)
for x in caching_repo.get_sources()
]
if "name" not in cfg.hydra.job:
cfg.hydra.job.name = JobRuntime().get("name")
cfg.hydra.job.override_dirname = get_overrides_dirname(
overrides=app_overrides,
kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
item_sep=cfg.hydra.job.config.override_dirname.item_sep,
exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys,
)
cfg.hydra.job.config_name = config_name
return cfg
def _process_config_searchpath(
self,
config_name: Optional[str],
parsed_overrides: List[Override],
repo: CachingConfigRepository,
) -> None:
if config_name is not None:
loaded = repo.load_config(config_path=config_name)
primary_config: Container
if loaded is None:
primary_config = OmegaConf.create()
else:
primary_config = loaded.config
else:
primary_config = OmegaConf.create()
if not OmegaConf.is_dict(primary_config):
raise ConfigCompositionException(
f"primary config '{config_name}' must be a DictConfig, got {type(primary_config).__name__}"
)
def is_searchpath_override(v: Override) -> bool:
return v.get_key_element() == "hydra.searchpath"
override = None
for v in parsed_overrides:
if is_searchpath_override(v):
override = v.value()
break
searchpath = OmegaConf.select(primary_config, "hydra.searchpath")
if override is not None:
provider = "hydra.searchpath in command-line"
searchpath = override
else:
provider = "hydra.searchpath in main"
def _err() -> None:
raise ConfigCompositionException(
f"hydra.searchpath must be a list of strings. Got: {searchpath}"
)
if searchpath is None:
return
# validate hydra.searchpath.
# Note that we cannot rely on OmegaConf validation here because we did not yet merge with the Hydra schema node
if not isinstance(searchpath, MutableSequence):
_err()
for v in searchpath:
if not isinstance(v, str):
_err()
new_csp = copy.deepcopy(self.config_search_path)
schema = new_csp.get_path().pop(-1)
assert schema.provider == "schema"
for sp in searchpath:
new_csp.append(provider=provider, path=sp)
new_csp.append("schema", "structured://")
repo.initialize_sources(new_csp)
for source in repo.get_sources():
if not source.available():
warnings.warn(
category=UserWarning,
message=
f"provider={source.provider}, path={source.path} is not available.",
)