def test_get_config_header(cfg_text: str, expected: Any, sep: str) -> None:
cfg_text = cfg_text.format(sep=sep)
if isinstance(expected, dict):
header = ConfigSource._get_header_dict(cfg_text)
assert header == expected
else:
with expected:
ConfigSource._get_header_dict(cfg_text)
def load_config(
self,
config_path: str,
is_primary_config: bool,
package_override: Optional[str] = None,
) -> ConfigResult:
normalized_config_path = self._normalize_file_name(config_path)
res = importlib_resources.files(self.path).joinpath(normalized_config_path)
if not res.exists():
raise ConfigLoadError(f"Config not found : {normalized_config_path}")
with open(res, encoding="utf-8") as f:
header_text = f.read(512)
header = ConfigSource._get_header_dict(header_text)
self._update_package_in_header(
header=header,
normalized_config_path=normalized_config_path,
is_primary_config=is_primary_config,
package_override=package_override,
)
f.seek(0)
cfg = OmegaConf.load(f)
defaults_list = self._extract_defaults_list(
config_path=config_path, cfg=cfg
)
return ConfigResult(
config=self._embed_config(cfg, header["package"]),
path=f"{self.scheme()}://{self.path}",
provider=self.provider,
header=header,
defaults_list=defaults_list,
)
def load_config(
self,
config_path: str,
is_primary_config: bool,
package_override: Optional[str] = None,
) -> ConfigResult:
normalized_config_path = self._normalize_file_name(config_path)
full_path = os.path.realpath(
os.path.join(self.path, normalized_config_path))
if not os.path.exists(full_path):
raise ConfigLoadError(f"Config not found : {full_path}")
with open(full_path) as f:
header_text = f.read(512)
header = ConfigSource._get_header_dict(header_text)
self._update_package_in_header(
header=header,
normalized_config_path=normalized_config_path,
is_primary_config=is_primary_config,
package_override=package_override,
)
f.seek(0)
cfg = OmegaConf.load(f)
return ConfigResult(
config=self._embed_config(cfg, header["package"]),
path=f"{self.scheme()}://{self.path}",
provider=self.provider,
header=header,
)
def load_config(
self,
config_path: str,
is_primary_config: bool,
package_override: Optional[str] = None,
) -> ConfigResult:
normalized_config_path = self._normalize_file_name(
filename=config_path)
module_name, resource_name = PackageConfigSource._split_module_and_resource(
self.concat(self.path, normalized_config_path))
try:
with resource_stream(module_name, resource_name) as stream:
header_text = stream.read(512)
header = ConfigSource._get_header_dict(header_text.decode())
self._update_package_in_header(
header=header,
normalized_config_path=normalized_config_path,
is_primary_config=is_primary_config,
package_override=package_override,
)
stream.seek(0)
cfg = OmegaConf.load(stream)
return ConfigResult(
config=self._embed_config(cfg, header["package"]),
path=f"{self.scheme()}://{self.path}",
provider=self.provider,
header=header,
)
except FileNotFoundError:
raise ConfigLoadError(
f"Config not found: module={module_name}, resource_name={resource_name}"
)
def load_config(self, config_path: str) -> ConfigResult:
normalized_config_path = self._normalize_file_name(config_path)
res = resources.files(self.path).joinpath(normalized_config_path) # type:ignore
if not res.exists():
raise ConfigLoadError(f"Config not found : {normalized_config_path}")
with res.open(encoding="utf-8") as f:
header_text = f.read(512)
header = ConfigSource._get_header_dict(header_text)
f.seek(0)
cfg = OmegaConf.load(f)
return ConfigResult(
config=cfg,
path=f"{self.scheme()}://{self.path}",
provider=self.provider,
header=header,
)
def load_config(self, config_path: str) -> ConfigResult:
normalized_config_path = self._normalize_file_name(config_path)
full_path = os.path.realpath(os.path.join(self.path, normalized_config_path))
if not os.path.exists(full_path):
raise ConfigLoadError(f"Config not found : {full_path}")
with open(full_path, encoding="utf-8") as f:
header_text = f.read(512)
header = ConfigSource._get_header_dict(header_text)
f.seek(0)
cfg = OmegaConf.load(f)
return ConfigResult(
config=cfg,
path=f"{self.scheme()}://{self.path}",
provider=self.provider,
header=header,
)
def _read_config(self, res: Any) -> ConfigResult:
try:
if sys.version_info[0:2] >= (3, 8) and isinstance(
res, zipfile.Path):
# zipfile does not support encoding, read() calls returns bytes.
f = res.open()
else:
f = res.open(encoding="utf-8")
header_text = f.read(512)
if isinstance(header_text, bytes):
# if header is bytes, utf-8 decode (zipfile path)
header_text = header_text.decode("utf-8")
header = ConfigSource._get_header_dict(header_text)
f.seek(0)
cfg = OmegaConf.load(f)
return ConfigResult(
config=cfg,
path=f"{self.scheme()}://{self.path}",
provider=self.provider,
header=header,
)
finally:
f.close()
def _load_single_config(self, default: ResultDefault,
repo: IConfigRepository) -> ConfigResult:
config_path = default.config_path
assert config_path is not None
ret = repo.load_config(config_path=config_path)
assert ret is not None
if not OmegaConf.is_config(ret.config):
raise ValueError(
f"Config {config_path} must be an OmegaConf config, got {type(ret.config).__name__}"
)
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
cname = ConfigSource._normalize_file_name(filename=config_path)
schema = schema_source.load_config(cname)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
url = "https://hydra.cc/docs/next/upgrades/1.0_to_1.1/automatic_schema_matching"
if "defaults" in schema.config:
raise ConfigCompositionException(
dedent(f"""\
'{config_path}' is validated against ConfigStore schema with the same name.
This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2.
In addition, the automatically matched schema contains a defaults list.
This combination is no longer supported.
See {url} for migration instructions."""))
else:
warnings.warn(
dedent(f"""\
'{config_path}' is validated against ConfigStore schema with the same name.
This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2.
See {url} for migration instructions."""),
category=UserWarning,
stacklevel=11,
)
# if primary config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring it's node
# in the schema of every primary config
hydra = None
hydra_config_group = (
default.config_path is not None
and default.config_path.startswith("hydra/"))
config = ret.config
if (default.primary and isinstance(config, DictConfig)
and "hydra" in config and not hydra_config_group):
hydra = config.pop("hydra")
merged = OmegaConf.merge(schema.config, config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema") from e
assert isinstance(merged, DictConfig)
res = self._embed_result_config(ret, default.package)
if (not default.primary and config_path != "hydra/config"
and isinstance(res.config, DictConfig) and OmegaConf.select(
res.config, "hydra.searchpath") is not None):
raise ConfigCompositionException(
f"In '{config_path}': Overriding hydra.searchpath is only supported from the primary config"
)
return res
def _load_config_impl(
self,
input_file: str,
package_override: Optional[str],
is_primary_config: bool,
record_load: bool = True,
) -> Tuple[Optional[DictConfig], Optional[LoadTrace]]:
"""
:param input_file:
:param record_load:
:return: the loaded config or None if it was not found
"""
ret = self.repository.load_config(
config_path=input_file,
is_primary_config=is_primary_config,
package_override=package_override,
)
if ret is not None:
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {input_file} must be a Dictionary, got {type(ret).__name__}"
)
if not ret.is_schema_source:
try:
schema_source = self.repository.get_schema_source()
config_path = ConfigSource._normalize_file_name(filename=input_file)
schema = schema_source.load_config(
config_path,
is_primary_config=is_primary_config,
package_override=package_override,
)
try:
if is_primary_config:
# Add as placeholders for hydra and defaults to allow
# overriding them from the config even if not in schema
schema.config = OmegaConf.merge(
{"hydra": None, "defaults": []}, schema.config
)
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
# remove placeholders if unused
with open_dict(merged):
if "hydra" in merged and merged.hydra is None:
del merged["hydra"]
if "defaults" in merged and merged["defaults"] == []:
del merged["defaults"]
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{input_file}' with schema"
) from e
assert isinstance(merged, DictConfig)
return (
merged,
self._record_loading(
name=input_file,
path=ret.path,
provider=ret.provider,
schema_provider=schema.provider,
record_load=record_load,
),
)
except ConfigLoadError:
# schema not found, ignore
pass
return (
ret.config,
self._record_loading(
name=input_file,
path=ret.path,
provider=ret.provider,
schema_provider=None,
record_load=record_load,
),
)
else:
return (
None,
self._record_loading(
name=input_file,
path=None,
provider=None,
schema_provider=None,
record_load=record_load,
),
)
def _load_single_config(
self,
default: DefaultElement,
repo: IConfigRepository,
) -> Tuple[ConfigResult, LoadTrace]:
config_path = default.config_path()
package_override = default.package
ret = repo.load_config(
config_path=config_path,
is_primary_config=default.primary,
package_override=package_override,
)
assert ret is not None
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {config_path} must be a Dictionary, got {type(ret).__name__}"
)
default.search_path = ret.path
schema_provider = None
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
schema = schema_source.load_config(
ConfigSource._normalize_file_name(filename=config_path),
is_primary_config=default.primary,
package_override=package_override,
)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
# if config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring this node
# in every program
hydra = None
hydra_config_group = (
default.config_group is not None
and default.config_group.startswith("hydra/"))
if "hydra" in ret.config and not hydra_config_group:
hydra = ret.config.pop("hydra")
schema_provider = schema.provider
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema") from e
assert isinstance(merged, DictConfig)
trace = LoadTrace(
config_group=default.config_group,
config_name=default.config_name,
package=default.get_subject_package(),
search_path=ret.path,
parent=default.parent,
provider=ret.provider,
schema_provider=schema_provider,
)
return ret, trace
def _load_config_impl(
self,
input_file: str,
package_override: Optional[str],
is_primary_config: bool,
record_load: bool = True,
) -> Tuple[Optional[DictConfig], Optional[LoadTrace]]:
"""
:param input_file:
:param record_load:
:return: the loaded config or None if it was not found
"""
def record_loading(
name: str,
path: Optional[str],
provider: Optional[str],
schema_provider: Optional[str],
) -> Optional[LoadTrace]:
trace = LoadTrace(
filename=name,
path=path,
provider=provider,
schema_provider=schema_provider,
)
if record_load:
self.all_config_checked.append(trace)
return trace
ret = self.repository.load_config(
config_path=input_file,
is_primary_config=is_primary_config,
package_override=package_override,
)
if ret is not None:
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {input_file} must be a Dictionary, got {type(ret).__name__}"
)
if not ret.is_schema_source:
try:
schema_source = self.repository.get_schema_source()
config_path = ConfigSource._normalize_file_name(filename=input_file)
schema = schema_source.load_config(
config_path,
is_primary_config=is_primary_config,
package_override=package_override,
)
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
return (
merged,
record_loading(
name=input_file,
path=ret.path,
provider=ret.provider,
schema_provider=schema.provider,
),
)
except ConfigLoadError:
# schema not found, ignore
pass
return (
ret.config,
record_loading(
name=input_file,
path=ret.path,
provider=ret.provider,
schema_provider=None,
),
)
else:
return (
None,
record_loading(
name=input_file, path=None, provider=None, schema_provider=None
),
)
def _load_single_config(
self, default: ResultDefault, repo: IConfigRepository
) -> Tuple[ConfigResult, LoadTrace]:
config_path = default.config_path
assert config_path is not None
ret = repo.load_config(config_path=config_path)
assert ret is not None
if not isinstance(ret.config, DictConfig):
raise ValueError(
f"Config {config_path} must be a Dictionary, got {type(ret).__name__}"
)
if not ret.is_schema_source:
schema = None
try:
schema_source = repo.get_schema_source()
cname = ConfigSource._normalize_file_name(filename=config_path)
schema = schema_source.load_config(cname)
except ConfigLoadError:
# schema not found, ignore
pass
if schema is not None:
try:
# TODO: deprecate schema matching in favor of extension via Defaults List
# if primary config has a hydra node, remove it during validation and add it back.
# This allows overriding Hydra's configuration without declaring it's node
# in the schema of every primary config
hydra = None
hydra_config_group = (
default.config_path is not None
and default.config_path.startswith("hydra/")
)
if (
default.primary
and "hydra" in ret.config
and not hydra_config_group
):
hydra = ret.config.pop("hydra")
merged = OmegaConf.merge(schema.config, ret.config)
assert isinstance(merged, DictConfig)
if hydra is not None:
with open_dict(merged):
merged.hydra = hydra
ret.config = merged
except OmegaConfBaseException as e:
raise ConfigCompositionException(
f"Error merging '{config_path}' with schema"
) from e
assert isinstance(merged, DictConfig)
trace = LoadTrace(
config_path=default.config_path,
package=default.package,
parent=default.parent,
is_self=default.is_self,
search_path=ret.path,
provider=ret.provider,
)
ret = self._embed_result_config(ret, default.package)
return ret, trace
