def _process_config_searchpath(
self,
config_name: Optional[str],
parsed_overrides: List[Override],
repo: CachingConfigRepository,
) -> None:
if config_name is not None:
loaded = repo.load_config(config_path=config_name)
primary_config: Container
if loaded is None:
primary_config = OmegaConf.create()
else:
primary_config = loaded.config
else:
primary_config = OmegaConf.create()
def is_searchpath_override(v: Override) -> bool:
return v.get_key_element() == "hydra.searchpath"
override = None
for v in parsed_overrides:
if is_searchpath_override(v):
override = v.value()
break
searchpath = OmegaConf.select(primary_config, "hydra.searchpath")
if override is not None:
provider = "hydra.searchpath in command-line"
searchpath = override
else:
provider = "hydra.searchpath in main"
def _err() -> None:
raise ConfigCompositionException(
f"hydra.searchpath must be a list of strings. Got: {searchpath}"
)
if searchpath is None:
return
# validate hydra.searchpath.
# Note that we cannot rely on OmegaConf validation here because we did not yet merge with the Hydra schema node
if not isinstance(searchpath, MutableSequence):
_err()
for v in searchpath:
if not isinstance(v, str):
_err()
new_csp = copy.deepcopy(self.config_search_path)
schema = new_csp.get_path().pop(-1)
assert schema.provider == "schema"
for sp in searchpath:
new_csp.append(provider=provider, path=sp)
new_csp.append("schema", "structured://")
repo.initialize_sources(new_csp)
def compute_defaults_list(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
) -> DefaultsList:
parser = OverridesParser.create()
repo = CachingConfigRepository(self.repository)
defaults_list = create_defaults_list(
repo=repo,
config_name=config_name,
overrides_list=parser.parse_overrides(overrides=overrides),
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
return defaults_list
def _load_configuration_impl(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
from_shell: bool = True,
) -> DictConfig:
from hydra import __version__
self.ensure_main_config_source_available()
caching_repo = CachingConfigRepository(self.repository)
parser = OverridesParser.create()
parsed_overrides = parser.parse_overrides(overrides=overrides)
self._process_config_searchpath(config_name, parsed_overrides,
caching_repo)
self.validate_sweep_overrides_legal(overrides=parsed_overrides,
run_mode=run_mode,
from_shell=from_shell)
defaults_list = create_defaults_list(
repo=caching_repo,
config_name=config_name,
overrides_list=parsed_overrides,
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
config_overrides = defaults_list.config_overrides
cfg = self._compose_config_from_defaults_list(
defaults=defaults_list.defaults, repo=caching_repo)
# Set config root to struct mode.
# Note that this will close any dictionaries (including dicts annotated as Dict[K, V].
# One must use + to add new fields to them.
OmegaConf.set_struct(cfg, True)
# The Hydra node should not be read-only even if the root config is read-only.
OmegaConf.set_readonly(cfg.hydra, False)
# Apply command line overrides after enabling strict flag
ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
app_overrides = []
for override in parsed_overrides:
if override.is_hydra_override():
cfg.hydra.overrides.hydra.append(override.input_line)
else:
cfg.hydra.overrides.task.append(override.input_line)
app_overrides.append(override)
with open_dict(cfg.hydra):
cfg.hydra.runtime.choices.update(
defaults_list.overrides.known_choices)
for key in cfg.hydra.job.env_copy:
cfg.hydra.job.env_set[key] = os.environ[key]
cfg.hydra.runtime.version = __version__
cfg.hydra.runtime.cwd = os.getcwd()
cfg.hydra.runtime.config_sources = [
ConfigSourceInfo(path=x.path,
schema=x.scheme(),
provider=x.provider)
for x in caching_repo.get_sources()
]
if "name" not in cfg.hydra.job:
cfg.hydra.job.name = JobRuntime().get("name")
cfg.hydra.job.override_dirname = get_overrides_dirname(
overrides=app_overrides,
kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
item_sep=cfg.hydra.job.config.override_dirname.item_sep,
exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys,
)
cfg.hydra.job.config_name = config_name
return cfg
def _load_configuration_impl(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
from_shell: bool = True,
) -> DictConfig:
self.ensure_main_config_source_available()
caching_repo = CachingConfigRepository(self.repository)
parser = OverridesParser.create()
parsed_overrides = parser.parse_overrides(overrides=overrides)
self.validate_sweep_overrides_legal(overrides=parsed_overrides,
run_mode=run_mode,
from_shell=from_shell)
defaults_list = create_defaults_list(
repo=caching_repo,
config_name=config_name,
overrides_list=parser.parse_overrides(overrides=overrides),
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
config_overrides = defaults_list.config_overrides
cfg = self._compose_config_from_defaults_list(
defaults=defaults_list.defaults, repo=caching_repo)
# set struct mode on user config if the root node is not typed.
if OmegaConf.get_type(cfg) is dict:
OmegaConf.set_struct(cfg, True)
# Turn off struct mode on the Hydra node. It gets its type safety from it being a Structured Config.
# This enables adding fields to nested dicts like hydra.job.env_set without having to using + to append.
OmegaConf.set_struct(cfg.hydra, False)
# Make the Hydra node writeable (The user may have made the primary node read-only).
OmegaConf.set_readonly(cfg.hydra, False)
# Apply command line overrides after enabling strict flag
ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
app_overrides = []
for override in parsed_overrides:
if override.is_hydra_override():
cfg.hydra.overrides.hydra.append(override.input_line)
else:
cfg.hydra.overrides.task.append(override.input_line)
app_overrides.append(override)
cfg.hydra.choices.update(defaults_list.overrides.known_choices)
with open_dict(cfg.hydra):
from hydra import __version__
cfg.hydra.runtime.version = __version__
cfg.hydra.runtime.cwd = os.getcwd()
if "name" not in cfg.hydra.job:
cfg.hydra.job.name = JobRuntime().get("name")
cfg.hydra.job.override_dirname = get_overrides_dirname(
overrides=app_overrides,
kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
item_sep=cfg.hydra.job.config.override_dirname.item_sep,
exclude_keys=cfg.hydra.job.config.override_dirname.
exclude_keys,
)
cfg.hydra.job.config_name = config_name
for key in cfg.hydra.job.env_copy:
cfg.hydra.job.env_set[key] = os.environ[key]
return cfg
def _load_configuration_impl(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
strict: Optional[bool] = None,
from_shell: bool = True,
) -> DictConfig:
self.ensure_main_config_source_available()
caching_repo = CachingConfigRepository(self.repository)
if config_name is not None and not caching_repo.config_exists(
config_name):
self._missing_config_error(
config_name=config_name,
msg=
f"Cannot find primary config : {config_name}, check that it's in your config search path",
with_search_path=True,
)
if strict is None:
strict = self.default_strict
parser = OverridesParser.create()
parsed_overrides = parser.parse_overrides(overrides=overrides)
config_overrides = ConfigLoaderImpl.parse_overrides(
overrides=overrides, run_mode=run_mode, from_shell=from_shell)
split_res = self.split_by_override_type(config_overrides)
config_group_overrides = split_res.config_group_overrides
config_overrides = split_res.config_overrides
input_defaults = [DefaultElement(config_name="hydra_config")]
if config_name is not None:
input_defaults.append(
DefaultElement(config_name=config_name, primary=True))
for default in convert_overrides_to_defaults(config_group_overrides):
input_defaults.append(default)
skip_missing = run_mode == RunMode.MULTIRUN
defaults = expand_defaults_list(
defaults=input_defaults,
skip_missing=skip_missing,
repo=caching_repo,
)
cfg, composition_trace = self._compose_config_from_defaults_list(
defaults=defaults, repo=caching_repo)
OmegaConf.set_struct(cfg, strict)
OmegaConf.set_readonly(cfg.hydra, False)
# Apply command line overrides after enabling strict flag
ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
app_overrides = []
for override in parsed_overrides:
if override.is_hydra_override():
cfg.hydra.overrides.hydra.append(override.input_line)
else:
cfg.hydra.overrides.task.append(override.input_line)
app_overrides.append(override)
with open_dict(cfg.hydra):
from hydra import __version__
cfg.hydra.runtime.version = __version__
cfg.hydra.runtime.cwd = os.getcwd()
cfg.hydra.composition_trace = composition_trace
if "name" not in cfg.hydra.job:
cfg.hydra.job.name = JobRuntime().get("name")
cfg.hydra.job.override_dirname = get_overrides_dirname(
overrides=app_overrides,
kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
item_sep=cfg.hydra.job.config.override_dirname.item_sep,
exclude_keys=cfg.hydra.job.config.override_dirname.
exclude_keys,
)
cfg.hydra.job.config_name = config_name
for key in cfg.hydra.job.env_copy:
cfg.hydra.job.env_set[key] = os.environ[key]
return cfg
def _load_configuration_impl(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
strict: Optional[bool] = None,
from_shell: bool = True,
) -> DictConfig:
self.ensure_main_config_source_available()
caching_repo = CachingConfigRepository(self.repository)
if strict is None:
strict = self.default_strict
parser = OverridesParser.create()
parsed_overrides = parser.parse_overrides(overrides=overrides)
self.validate_sweep_overrides_legal(
overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell
)
defaults_list = create_defaults_list(
repo=caching_repo,
config_name=config_name,
overrides_list=parser.parse_overrides(overrides=overrides),
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
config_overrides = defaults_list.config_overrides
cfg, composition_trace = self._compose_config_from_defaults_list(
defaults=defaults_list.defaults, repo=caching_repo
)
OmegaConf.set_struct(cfg, strict)
OmegaConf.set_readonly(cfg.hydra, False)
# Apply command line overrides after enabling strict flag
ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
app_overrides = []
for override in parsed_overrides:
if override.is_hydra_override():
cfg.hydra.overrides.hydra.append(override.input_line)
else:
cfg.hydra.overrides.task.append(override.input_line)
app_overrides.append(override)
# TODO: should this open_dict be required given that choices is a Dict?
with open_dict(cfg.hydra.choices):
cfg.hydra.choices.update(defaults_list.overrides.known_choices)
with open_dict(cfg.hydra):
from hydra import __version__
cfg.hydra.runtime.version = __version__
cfg.hydra.runtime.cwd = os.getcwd()
cfg.hydra.composition_trace = composition_trace
if "name" not in cfg.hydra.job:
cfg.hydra.job.name = JobRuntime().get("name")
cfg.hydra.job.override_dirname = get_overrides_dirname(
overrides=app_overrides,
kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
item_sep=cfg.hydra.job.config.override_dirname.item_sep,
exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys,
)
cfg.hydra.job.config_name = config_name
for key in cfg.hydra.job.env_copy:
cfg.hydra.job.env_set[key] = os.environ[key]
return cfg