def compute_defaults_list(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
) -> DefaultsList:
parser = OverridesParser.create()
repo = CachingConfigRepository(self.repository)
defaults_list = create_defaults_list(
repo=repo,
config_name=config_name,
overrides_list=parser.parse_overrides(overrides=overrides),
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
return defaults_list
def _load_configuration_impl(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
from_shell: bool = True,
) -> DictConfig:
from hydra import __version__
self.ensure_main_config_source_available()
caching_repo = CachingConfigRepository(self.repository)
parser = OverridesParser.create()
parsed_overrides = parser.parse_overrides(overrides=overrides)
self._process_config_searchpath(config_name, parsed_overrides,
caching_repo)
self.validate_sweep_overrides_legal(overrides=parsed_overrides,
run_mode=run_mode,
from_shell=from_shell)
defaults_list = create_defaults_list(
repo=caching_repo,
config_name=config_name,
overrides_list=parsed_overrides,
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
config_overrides = defaults_list.config_overrides
cfg = self._compose_config_from_defaults_list(
defaults=defaults_list.defaults, repo=caching_repo)
# Set config root to struct mode.
# Note that this will close any dictionaries (including dicts annotated as Dict[K, V].
# One must use + to add new fields to them.
OmegaConf.set_struct(cfg, True)
# The Hydra node should not be read-only even if the root config is read-only.
OmegaConf.set_readonly(cfg.hydra, False)
# Apply command line overrides after enabling strict flag
ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
app_overrides = []
for override in parsed_overrides:
if override.is_hydra_override():
cfg.hydra.overrides.hydra.append(override.input_line)
else:
cfg.hydra.overrides.task.append(override.input_line)
app_overrides.append(override)
with open_dict(cfg.hydra):
cfg.hydra.runtime.choices.update(
defaults_list.overrides.known_choices)
for key in cfg.hydra.job.env_copy:
cfg.hydra.job.env_set[key] = os.environ[key]
cfg.hydra.runtime.version = __version__
cfg.hydra.runtime.cwd = os.getcwd()
cfg.hydra.runtime.config_sources = [
ConfigSourceInfo(path=x.path,
schema=x.scheme(),
provider=x.provider)
for x in caching_repo.get_sources()
]
if "name" not in cfg.hydra.job:
cfg.hydra.job.name = JobRuntime().get("name")
cfg.hydra.job.override_dirname = get_overrides_dirname(
overrides=app_overrides,
kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
item_sep=cfg.hydra.job.config.override_dirname.item_sep,
exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys,
)
cfg.hydra.job.config_name = config_name
return cfg
def _load_configuration_impl(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
from_shell: bool = True,
) -> DictConfig:
self.ensure_main_config_source_available()
caching_repo = CachingConfigRepository(self.repository)
parser = OverridesParser.create()
parsed_overrides = parser.parse_overrides(overrides=overrides)
self.validate_sweep_overrides_legal(overrides=parsed_overrides,
run_mode=run_mode,
from_shell=from_shell)
defaults_list = create_defaults_list(
repo=caching_repo,
config_name=config_name,
overrides_list=parser.parse_overrides(overrides=overrides),
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
config_overrides = defaults_list.config_overrides
cfg = self._compose_config_from_defaults_list(
defaults=defaults_list.defaults, repo=caching_repo)
# set struct mode on user config if the root node is not typed.
if OmegaConf.get_type(cfg) is dict:
OmegaConf.set_struct(cfg, True)
# Turn off struct mode on the Hydra node. It gets its type safety from it being a Structured Config.
# This enables adding fields to nested dicts like hydra.job.env_set without having to using + to append.
OmegaConf.set_struct(cfg.hydra, False)
# Make the Hydra node writeable (The user may have made the primary node read-only).
OmegaConf.set_readonly(cfg.hydra, False)
# Apply command line overrides after enabling strict flag
ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
app_overrides = []
for override in parsed_overrides:
if override.is_hydra_override():
cfg.hydra.overrides.hydra.append(override.input_line)
else:
cfg.hydra.overrides.task.append(override.input_line)
app_overrides.append(override)
cfg.hydra.choices.update(defaults_list.overrides.known_choices)
with open_dict(cfg.hydra):
from hydra import __version__
cfg.hydra.runtime.version = __version__
cfg.hydra.runtime.cwd = os.getcwd()
if "name" not in cfg.hydra.job:
cfg.hydra.job.name = JobRuntime().get("name")
cfg.hydra.job.override_dirname = get_overrides_dirname(
overrides=app_overrides,
kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
item_sep=cfg.hydra.job.config.override_dirname.item_sep,
exclude_keys=cfg.hydra.job.config.override_dirname.
exclude_keys,
)
cfg.hydra.job.config_name = config_name
for key in cfg.hydra.job.env_copy:
cfg.hydra.job.env_set[key] = os.environ[key]
return cfg
def _load_configuration_impl(
self,
config_name: Optional[str],
overrides: List[str],
run_mode: RunMode,
strict: Optional[bool] = None,
from_shell: bool = True,
) -> DictConfig:
self.ensure_main_config_source_available()
caching_repo = CachingConfigRepository(self.repository)
if strict is None:
strict = self.default_strict
parser = OverridesParser.create()
parsed_overrides = parser.parse_overrides(overrides=overrides)
self.validate_sweep_overrides_legal(
overrides=parsed_overrides, run_mode=run_mode, from_shell=from_shell
)
defaults_list = create_defaults_list(
repo=caching_repo,
config_name=config_name,
overrides_list=parser.parse_overrides(overrides=overrides),
prepend_hydra=True,
skip_missing=run_mode == RunMode.MULTIRUN,
)
config_overrides = defaults_list.config_overrides
cfg, composition_trace = self._compose_config_from_defaults_list(
defaults=defaults_list.defaults, repo=caching_repo
)
OmegaConf.set_struct(cfg, strict)
OmegaConf.set_readonly(cfg.hydra, False)
# Apply command line overrides after enabling strict flag
ConfigLoaderImpl._apply_overrides_to_config(config_overrides, cfg)
app_overrides = []
for override in parsed_overrides:
if override.is_hydra_override():
cfg.hydra.overrides.hydra.append(override.input_line)
else:
cfg.hydra.overrides.task.append(override.input_line)
app_overrides.append(override)
# TODO: should this open_dict be required given that choices is a Dict?
with open_dict(cfg.hydra.choices):
cfg.hydra.choices.update(defaults_list.overrides.known_choices)
with open_dict(cfg.hydra):
from hydra import __version__
cfg.hydra.runtime.version = __version__
cfg.hydra.runtime.cwd = os.getcwd()
cfg.hydra.composition_trace = composition_trace
if "name" not in cfg.hydra.job:
cfg.hydra.job.name = JobRuntime().get("name")
cfg.hydra.job.override_dirname = get_overrides_dirname(
overrides=app_overrides,
kv_sep=cfg.hydra.job.config.override_dirname.kv_sep,
item_sep=cfg.hydra.job.config.override_dirname.item_sep,
exclude_keys=cfg.hydra.job.config.override_dirname.exclude_keys,
)
cfg.hydra.job.config_name = config_name
for key in cfg.hydra.job.env_copy:
cfg.hydra.job.env_set[key] = os.environ[key]
return cfg
