def test_get_config_header(cfg_text: str, expected: Any, sep: str) -> None: cfg_text = cfg_text.format(sep=sep) if isinstance(expected, dict): header = ConfigSource._get_header_dict(cfg_text) assert header == expected else: with expected: ConfigSource._get_header_dict(cfg_text)
def load_config( self, config_path: str, is_primary_config: bool, package_override: Optional[str] = None, ) -> ConfigResult: normalized_config_path = self._normalize_file_name(config_path) res = importlib_resources.files(self.path).joinpath(normalized_config_path) if not res.exists(): raise ConfigLoadError(f"Config not found : {normalized_config_path}") with open(res, encoding="utf-8") as f: header_text = f.read(512) header = ConfigSource._get_header_dict(header_text) self._update_package_in_header( header=header, normalized_config_path=normalized_config_path, is_primary_config=is_primary_config, package_override=package_override, ) f.seek(0) cfg = OmegaConf.load(f) defaults_list = self._extract_defaults_list( config_path=config_path, cfg=cfg ) return ConfigResult( config=self._embed_config(cfg, header["package"]), path=f"{self.scheme()}://{self.path}", provider=self.provider, header=header, defaults_list=defaults_list, )
def load_config( self, config_path: str, is_primary_config: bool, package_override: Optional[str] = None, ) -> ConfigResult: normalized_config_path = self._normalize_file_name(config_path) full_path = os.path.realpath( os.path.join(self.path, normalized_config_path)) if not os.path.exists(full_path): raise ConfigLoadError(f"Config not found : {full_path}") with open(full_path) as f: header_text = f.read(512) header = ConfigSource._get_header_dict(header_text) self._update_package_in_header( header=header, normalized_config_path=normalized_config_path, is_primary_config=is_primary_config, package_override=package_override, ) f.seek(0) cfg = OmegaConf.load(f) return ConfigResult( config=self._embed_config(cfg, header["package"]), path=f"{self.scheme()}://{self.path}", provider=self.provider, header=header, )
def load_config( self, config_path: str, is_primary_config: bool, package_override: Optional[str] = None, ) -> ConfigResult: normalized_config_path = self._normalize_file_name( filename=config_path) module_name, resource_name = PackageConfigSource._split_module_and_resource( self.concat(self.path, normalized_config_path)) try: with resource_stream(module_name, resource_name) as stream: header_text = stream.read(512) header = ConfigSource._get_header_dict(header_text.decode()) self._update_package_in_header( header=header, normalized_config_path=normalized_config_path, is_primary_config=is_primary_config, package_override=package_override, ) stream.seek(0) cfg = OmegaConf.load(stream) return ConfigResult( config=self._embed_config(cfg, header["package"]), path=f"{self.scheme()}://{self.path}", provider=self.provider, header=header, ) except FileNotFoundError: raise ConfigLoadError( f"Config not found: module={module_name}, resource_name={resource_name}" )
def load_config(self, config_path: str) -> ConfigResult: normalized_config_path = self._normalize_file_name(config_path) res = resources.files(self.path).joinpath(normalized_config_path) # type:ignore if not res.exists(): raise ConfigLoadError(f"Config not found : {normalized_config_path}") with res.open(encoding="utf-8") as f: header_text = f.read(512) header = ConfigSource._get_header_dict(header_text) f.seek(0) cfg = OmegaConf.load(f) return ConfigResult( config=cfg, path=f"{self.scheme()}://{self.path}", provider=self.provider, header=header, )
def load_config(self, config_path: str) -> ConfigResult: normalized_config_path = self._normalize_file_name(config_path) full_path = os.path.realpath(os.path.join(self.path, normalized_config_path)) if not os.path.exists(full_path): raise ConfigLoadError(f"Config not found : {full_path}") with open(full_path, encoding="utf-8") as f: header_text = f.read(512) header = ConfigSource._get_header_dict(header_text) f.seek(0) cfg = OmegaConf.load(f) return ConfigResult( config=cfg, path=f"{self.scheme()}://{self.path}", provider=self.provider, header=header, )
def _read_config(self, res: Any) -> ConfigResult: try: if sys.version_info[0:2] >= (3, 8) and isinstance( res, zipfile.Path): # zipfile does not support encoding, read() calls returns bytes. f = res.open() else: f = res.open(encoding="utf-8") header_text = f.read(512) if isinstance(header_text, bytes): # if header is bytes, utf-8 decode (zipfile path) header_text = header_text.decode("utf-8") header = ConfigSource._get_header_dict(header_text) f.seek(0) cfg = OmegaConf.load(f) return ConfigResult( config=cfg, path=f"{self.scheme()}://{self.path}", provider=self.provider, header=header, ) finally: f.close()
def _load_single_config(self, default: ResultDefault, repo: IConfigRepository) -> ConfigResult: config_path = default.config_path assert config_path is not None ret = repo.load_config(config_path=config_path) assert ret is not None if not OmegaConf.is_config(ret.config): raise ValueError( f"Config {config_path} must be an OmegaConf config, got {type(ret.config).__name__}" ) if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() cname = ConfigSource._normalize_file_name(filename=config_path) schema = schema_source.load_config(cname) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: url = "https://hydra.cc/docs/next/upgrades/1.0_to_1.1/automatic_schema_matching" if "defaults" in schema.config: raise ConfigCompositionException( dedent(f"""\ '{config_path}' is validated against ConfigStore schema with the same name. This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2. In addition, the automatically matched schema contains a defaults list. This combination is no longer supported. See {url} for migration instructions.""")) else: warnings.warn( dedent(f"""\ '{config_path}' is validated against ConfigStore schema with the same name. This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2. See {url} for migration instructions."""), category=UserWarning, stacklevel=11, ) # if primary config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring it's node # in the schema of every primary config hydra = None hydra_config_group = ( default.config_path is not None and default.config_path.startswith("hydra/")) config = ret.config if (default.primary and isinstance(config, DictConfig) and "hydra" in config and not hydra_config_group): hydra = config.pop("hydra") merged = OmegaConf.merge(schema.config, config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema") from e assert isinstance(merged, DictConfig) res = self._embed_result_config(ret, default.package) if (not default.primary and config_path != "hydra/config" and isinstance(res.config, DictConfig) and OmegaConf.select( res.config, "hydra.searchpath") is not None): raise ConfigCompositionException( f"In '{config_path}': Overriding hydra.searchpath is only supported from the primary config" ) return res
def _load_config_impl( self, input_file: str, package_override: Optional[str], is_primary_config: bool, record_load: bool = True, ) -> Tuple[Optional[DictConfig], Optional[LoadTrace]]: """ :param input_file: :param record_load: :return: the loaded config or None if it was not found """ ret = self.repository.load_config( config_path=input_file, is_primary_config=is_primary_config, package_override=package_override, ) if ret is not None: if not isinstance(ret.config, DictConfig): raise ValueError( f"Config {input_file} must be a Dictionary, got {type(ret).__name__}" ) if not ret.is_schema_source: try: schema_source = self.repository.get_schema_source() config_path = ConfigSource._normalize_file_name(filename=input_file) schema = schema_source.load_config( config_path, is_primary_config=is_primary_config, package_override=package_override, ) try: if is_primary_config: # Add as placeholders for hydra and defaults to allow # overriding them from the config even if not in schema schema.config = OmegaConf.merge( {"hydra": None, "defaults": []}, schema.config ) merged = OmegaConf.merge(schema.config, ret.config) assert isinstance(merged, DictConfig) # remove placeholders if unused with open_dict(merged): if "hydra" in merged and merged.hydra is None: del merged["hydra"] if "defaults" in merged and merged["defaults"] == []: del merged["defaults"] except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{input_file}' with schema" ) from e assert isinstance(merged, DictConfig) return ( merged, self._record_loading( name=input_file, path=ret.path, provider=ret.provider, schema_provider=schema.provider, record_load=record_load, ), ) except ConfigLoadError: # schema not found, ignore pass return ( ret.config, self._record_loading( name=input_file, path=ret.path, provider=ret.provider, schema_provider=None, record_load=record_load, ), ) else: return ( None, self._record_loading( name=input_file, path=None, provider=None, schema_provider=None, record_load=record_load, ), )
def _load_single_config( self, default: DefaultElement, repo: IConfigRepository, ) -> Tuple[ConfigResult, LoadTrace]: config_path = default.config_path() package_override = default.package ret = repo.load_config( config_path=config_path, is_primary_config=default.primary, package_override=package_override, ) assert ret is not None if not isinstance(ret.config, DictConfig): raise ValueError( f"Config {config_path} must be a Dictionary, got {type(ret).__name__}" ) default.search_path = ret.path schema_provider = None if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() schema = schema_source.load_config( ConfigSource._normalize_file_name(filename=config_path), is_primary_config=default.primary, package_override=package_override, ) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: # if config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring this node # in every program hydra = None hydra_config_group = ( default.config_group is not None and default.config_group.startswith("hydra/")) if "hydra" in ret.config and not hydra_config_group: hydra = ret.config.pop("hydra") schema_provider = schema.provider merged = OmegaConf.merge(schema.config, ret.config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema") from e assert isinstance(merged, DictConfig) trace = LoadTrace( config_group=default.config_group, config_name=default.config_name, package=default.get_subject_package(), search_path=ret.path, parent=default.parent, provider=ret.provider, schema_provider=schema_provider, ) return ret, trace
def _load_config_impl( self, input_file: str, package_override: Optional[str], is_primary_config: bool, record_load: bool = True, ) -> Tuple[Optional[DictConfig], Optional[LoadTrace]]: """ :param input_file: :param record_load: :return: the loaded config or None if it was not found """ def record_loading( name: str, path: Optional[str], provider: Optional[str], schema_provider: Optional[str], ) -> Optional[LoadTrace]: trace = LoadTrace( filename=name, path=path, provider=provider, schema_provider=schema_provider, ) if record_load: self.all_config_checked.append(trace) return trace ret = self.repository.load_config( config_path=input_file, is_primary_config=is_primary_config, package_override=package_override, ) if ret is not None: if not isinstance(ret.config, DictConfig): raise ValueError( f"Config {input_file} must be a Dictionary, got {type(ret).__name__}" ) if not ret.is_schema_source: try: schema_source = self.repository.get_schema_source() config_path = ConfigSource._normalize_file_name(filename=input_file) schema = schema_source.load_config( config_path, is_primary_config=is_primary_config, package_override=package_override, ) merged = OmegaConf.merge(schema.config, ret.config) assert isinstance(merged, DictConfig) return ( merged, record_loading( name=input_file, path=ret.path, provider=ret.provider, schema_provider=schema.provider, ), ) except ConfigLoadError: # schema not found, ignore pass return ( ret.config, record_loading( name=input_file, path=ret.path, provider=ret.provider, schema_provider=None, ), ) else: return ( None, record_loading( name=input_file, path=None, provider=None, schema_provider=None ), )
def _load_single_config( self, default: ResultDefault, repo: IConfigRepository ) -> Tuple[ConfigResult, LoadTrace]: config_path = default.config_path assert config_path is not None ret = repo.load_config(config_path=config_path) assert ret is not None if not isinstance(ret.config, DictConfig): raise ValueError( f"Config {config_path} must be a Dictionary, got {type(ret).__name__}" ) if not ret.is_schema_source: schema = None try: schema_source = repo.get_schema_source() cname = ConfigSource._normalize_file_name(filename=config_path) schema = schema_source.load_config(cname) except ConfigLoadError: # schema not found, ignore pass if schema is not None: try: # TODO: deprecate schema matching in favor of extension via Defaults List # if primary config has a hydra node, remove it during validation and add it back. # This allows overriding Hydra's configuration without declaring it's node # in the schema of every primary config hydra = None hydra_config_group = ( default.config_path is not None and default.config_path.startswith("hydra/") ) if ( default.primary and "hydra" in ret.config and not hydra_config_group ): hydra = ret.config.pop("hydra") merged = OmegaConf.merge(schema.config, ret.config) assert isinstance(merged, DictConfig) if hydra is not None: with open_dict(merged): merged.hydra = hydra ret.config = merged except OmegaConfBaseException as e: raise ConfigCompositionException( f"Error merging '{config_path}' with schema" ) from e assert isinstance(merged, DictConfig) trace = LoadTrace( config_path=default.config_path, package=default.package, parent=default.parent, is_self=default.is_self, search_path=ret.path, provider=ret.provider, ) ret = self._embed_result_config(ret, default.package) return ret, trace