def _load_single_config(self, default: ResultDefault,
repo: IConfigRepository) -> ConfigResult:
config_path = default.config_path
assert config_path is not None
ret = repo.load_config(config_path=config_path)
assert ret is not None
if not OmegaConf.is_config(ret.config):
raise ValueError(
f"Config {config_path} must be an OmegaConf config, got {type(ret.config).__name__}"
)
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
cname = ConfigSource._normalize_file_name(filename=config_path)
schema = schema_source.load_config(cname)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
url = "https://hydra.cc/docs/next/upgrades/1.0_to_1.1/automatic_schema_matching"
if "defaults" in schema.config:
raise ConfigCompositionException(
dedent(f"""\
'{config_path}' is validated against ConfigStore schema with the same name.
This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2.
In addition, the automatically matched schema contains a defaults list.
This combination is no longer supported.
See {url} for migration instructions."""))
else:
warnings.warn(
dedent(f"""\
'{config_path}' is validated against ConfigStore schema with the same name.
This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2.
See {url} for migration instructions."""),
category=UserWarning,
stacklevel=11,
)
# if primary config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring it's node
# in the schema of every primary config
hydra = None
hydra_config_group = (
default.config_path is not None
and default.config_path.startswith("hydra/"))
config = ret.config
if (default.primary and isinstance(config, DictConfig)
and "hydra" in config and not hydra_config_group):
hydra = config.pop("hydra")
merged = OmegaConf.merge(schema.config, config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema") from e
assert isinstance(merged, DictConfig)
res = self._embed_result_config(ret, default.package)
if (not default.primary and config_path != "hydra/config"
and isinstance(res.config, DictConfig) and OmegaConf.select(
res.config, "hydra.searchpath") is not None):
raise ConfigCompositionException(
f"In '{config_path}': Overriding hydra.searchpath is only supported from the primary config"
)
return res
def _load_config_impl(
self,
input_file: str,
package_override: Optional[str],
is_primary_config: bool,
record_load: bool = True,
) -> Tuple[Optional[DictConfig], Optional[LoadTrace]]:
"""
:param input_file:
:param record_load:
:return: the loaded config or None if it was not found
"""
ret = self.repository.load_config(
config_path=input_file,
is_primary_config=is_primary_config,
package_override=package_override,
)
if ret is not None:
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {input_file} must be a Dictionary, got {type(ret).__name__}"
)
if not ret.is_schema_source:
try:
schema_source = self.repository.get_schema_source()
config_path = ConfigSource._normalize_file_name(filename=input_file)
schema = schema_source.load_config(
config_path,
is_primary_config=is_primary_config,
package_override=package_override,
)
try:
if is_primary_config:
# Add as placeholders for hydra and defaults to allow
# overriding them from the config even if not in schema
schema.config = OmegaConf.merge(
{"hydra": None, "defaults": []}, schema.config
)
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
# remove placeholders if unused
with open_dict(merged):
if "hydra" in merged and merged.hydra is None:
del merged["hydra"]
if "defaults" in merged and merged["defaults"] == []:
del merged["defaults"]
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{input_file}' with schema"
) from e
assert isinstance(merged, DictConfig)
return (
merged,
self._record_loading(
name=input_file,
path=ret.path,
provider=ret.provider,
schema_provider=schema.provider,
record_load=record_load,
),
)
except ConfigLoadError:
# schema not found, ignore
pass
return (
ret.config,
self._record_loading(
name=input_file,
path=ret.path,
provider=ret.provider,
schema_provider=None,
record_load=record_load,
),
)
else:
return (
None,
self._record_loading(
name=input_file,
path=None,
provider=None,
schema_provider=None,
record_load=record_load,
),
)
def _load_config_impl(
self,
input_file: str,
package_override: Optional[str],
is_primary_config: bool,
record_load: bool = True,
) -> Tuple[Optional[DictConfig], Optional[LoadTrace]]:
"""
:param input_file:
:param record_load:
:return: the loaded config or None if it was not found
"""
def record_loading(
name: str,
path: Optional[str],
provider: Optional[str],
schema_provider: Optional[str],
) -> Optional[LoadTrace]:
trace = LoadTrace(
filename=name,
path=path,
provider=provider,
schema_provider=schema_provider,
)
if record_load:
self.all_config_checked.append(trace)
return trace
ret = self.repository.load_config(
config_path=input_file,
is_primary_config=is_primary_config,
package_override=package_override,
)
if ret is not None:
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {input_file} must be a Dictionary, got {type(ret).__name__}"
)
if not ret.is_schema_source:
try:
schema_source = self.repository.get_schema_source()
config_path = ConfigSource._normalize_file_name(filename=input_file)
schema = schema_source.load_config(
config_path,
is_primary_config=is_primary_config,
package_override=package_override,
)
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
return (
merged,
record_loading(
name=input_file,
path=ret.path,
provider=ret.provider,
schema_provider=schema.provider,
),
)
except ConfigLoadError:
# schema not found, ignore
pass
return (
ret.config,
record_loading(
name=input_file,
path=ret.path,
provider=ret.provider,
schema_provider=None,
),
)
else:
return (
None,
record_loading(
name=input_file, path=None, provider=None, schema_provider=None
),
)
def _load_single_config(
self,
default: DefaultElement,
repo: IConfigRepository,
) -> Tuple[ConfigResult, LoadTrace]:
config_path = default.config_path()
package_override = default.package
ret = repo.load_config(
config_path=config_path,
is_primary_config=default.primary,
package_override=package_override,
)
assert ret is not None
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {config_path} must be a Dictionary, got {type(ret).__name__}"
)
default.search_path = ret.path
schema_provider = None
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
schema = schema_source.load_config(
ConfigSource._normalize_file_name(filename=config_path),
is_primary_config=default.primary,
package_override=package_override,
)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
# if config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring this node
# in every program
hydra = None
hydra_config_group = (
default.config_group is not None
and default.config_group.startswith("hydra/"))
if "hydra" in ret.config and not hydra_config_group:
hydra = ret.config.pop("hydra")
schema_provider = schema.provider
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema") from e
assert isinstance(merged, DictConfig)
trace = LoadTrace(
config_group=default.config_group,
config_name=default.config_name,
package=default.get_subject_package(),
search_path=ret.path,
parent=default.parent,
provider=ret.provider,
schema_provider=schema_provider,
)
return ret, trace
def _load_single_config(
self, default: ResultDefault, repo: IConfigRepository
) -> Tuple[ConfigResult, LoadTrace]:
config_path = default.config_path
assert config_path is not None
ret = repo.load_config(config_path=config_path)
assert ret is not None
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {config_path} must be a Dictionary, got {type(ret).__name__}"
)
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
cname = ConfigSource._normalize_file_name(filename=config_path)
schema = schema_source.load_config(cname)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
# TODO: deprecate schema matching in favor of extension via Defaults List
# if primary config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring it's node
# in the schema of every primary config
hydra = None
hydra_config_group = (
default.config_path is not None
and default.config_path.startswith("hydra/")
)
if (
default.primary
and "hydra" in ret.config
and not hydra_config_group
):
hydra = ret.config.pop("hydra")
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema"
) from e
assert isinstance(merged, DictConfig)
trace = LoadTrace(
config_path=default.config_path,
package=default.package,
parent=default.parent,
is_self=default.is_self,
search_path=ret.path,
provider=ret.provider,
)
ret = self._embed_result_config(ret, default.package)
return ret, trace
