def test_list_of_objects_missing(self, module: Any) -> None: conf = OmegaConf.structured(module.ListOfObjectsMissing) assert OmegaConf.is_missing(conf.users, 0) conf.users.append(MISSING) assert OmegaConf.is_missing(conf.users, 1)
def test_merge_missing_structured_config_is_missing(self, class_type: str) -> None: # Test that the merged type is that of the last merged config module: Any = import_module(class_type) c1 = OmegaConf.structured(module.MissingStructuredConfigField) assert OmegaConf.is_missing(c1, "plugin") c2 = OmegaConf.merge(c1, module.MissingStructuredConfigField) assert OmegaConf.is_missing(c2, "plugin")
def test_dict_of_objects_missing(self, module: Any) -> None: conf = OmegaConf.structured(module.DictOfObjectsMissing) dct = conf.users assert OmegaConf.is_missing(dct, "moe") dct.miss = MISSING assert OmegaConf.is_missing(dct, "miss")
def test_interpolation_with_missing() -> None: cfg = OmegaConf.create({ "a": "${x.missing}.txt", "b": "${x.missing}", "x": { "missing": "???" } }) assert OmegaConf.is_missing(cfg, "a") assert OmegaConf.is_missing(cfg, "b")
def _map_merge(dest: "BaseContainer", src: "BaseContainer") -> None: """merge src into dest and return a new copy, does not modified input""" from omegaconf import OmegaConf from .dictconfig import DictConfig from .nodes import ValueNode assert isinstance(dest, DictConfig) assert isinstance(src, DictConfig) src_type = src._metadata.object_type dest._validate_set_merge_impl(key=None, value=src, is_assign=False) for key, src_value in src.items_ex(resolve=False): if OmegaConf.is_missing(dest, key): if isinstance(src_value, DictConfig): if OmegaConf.is_missing(dest, key): dest[key] = src_value dest_node = dest._get_node(key, validate_access=False) if dest_node is not None: if dest_node._is_interpolation(): target_node = dest_node._dereference_node( throw_on_resolution_failure=False) if isinstance(target_node, Container): dest[key] = target_node dest_node = dest._get_node(key) if is_structured_config(dest._metadata.element_type): dest[key] = DictConfig(content=dest._metadata.element_type, parent=dest) dest_node = dest._get_node(key) if dest_node is not None: if isinstance(dest_node, BaseContainer): if isinstance(src_value, BaseContainer): dest._validate_merge(key=key, value=src_value) dest_node._merge_with(src_value) else: dest.__setitem__(key, src_value) else: if isinstance(src_value, BaseContainer): dest.__setitem__(key, src_value) else: assert isinstance(dest_node, ValueNode) try: dest_node._set_value(src_value) except ValidationError as e: dest._format_and_raise(key=key, value=src_value, cause=e) else: dest[key] = src._get_node(key) if src_type is not None and not is_primitive_dict(src_type): dest._metadata.object_type = src_type
def test_is_missing_resets() -> None: cfg = OmegaConf.structured(StructuredWithMissing) assert OmegaConf.is_missing(cfg, "dict") cfg.dict = {} assert not OmegaConf.is_missing(cfg, "dict") assert OmegaConf.is_missing(cfg, "list") cfg.list = [1, 2, 3] assert not OmegaConf.is_missing(cfg, "list") cfg.list = "???" assert OmegaConf.is_missing(cfg, "list")
def test_get_type(self, class_type: str) -> None: module: Any = import_module(class_type) linked_list = module.LinkedList cfg1 = OmegaConf.create(linked_list) assert OmegaConf.get_type(cfg1) == linked_list assert cfg1.next is None assert OmegaConf.is_missing(cfg1, "value") cfg2 = OmegaConf.create(module.MissingTest.Missing1) assert OmegaConf.is_missing(cfg2, "head") assert OmegaConf.get_type(cfg2, "head") == module.LinkedList
def _map_merge(dest: "BaseContainer", src: "BaseContainer") -> None: """merge src into dest and return a new copy, does not modified input""" from omegaconf import OmegaConf from .dictconfig import DictConfig from .nodes import ValueNode assert isinstance(dest, DictConfig) assert isinstance(src, DictConfig) src = copy.deepcopy(src) src_type = src._metadata.object_type dest_type = dest._metadata.object_type if src_type is not None and src_type is not dest_type: prototype = DictConfig( annotated_type=src_type, content=src_type, ) dest.__dict__["_content"] = copy.deepcopy( prototype.__dict__["_content"]) dest.__dict__["_metadata"] = copy.deepcopy(prototype._metadata) for key, value in src.items_ex(resolve=False): dest_element_type = dest._metadata.element_type typed = dest_element_type not in (None, Any) if OmegaConf.is_missing(dest, key): if isinstance(value, DictConfig): if OmegaConf.is_missing(src, key): dest[key] = DictConfig(content="???") else: dest[key] = {} if (dest.get_node(key) is not None) or typed: dest_node = dest.get_node(key) if dest_node is None and typed: dest[key] = DictConfig(content=dest_element_type, parent=dest) dest_node = dest.get_node(key) if isinstance(dest_node, BaseContainer): if isinstance(value, BaseContainer): dest._validate_set(key=key, value=value) dest_node.merge_with(value) else: dest.__setitem__(key, value) else: if isinstance(value, BaseContainer): dest.__setitem__(key, value) else: assert isinstance(dest_node, ValueNode) dest_node._set_value(value) else: dest[key] = src.get_node(key)
def test_is_missing(cfg: Any, key: str, expected_is_missing: bool, expectation: Any) -> None: cfg = OmegaConf.create(cfg) with expectation: cfg.get(key) assert OmegaConf.is_missing(cfg, key) == expected_is_missing OmegaConf.set_struct(cfg, True) assert OmegaConf.is_missing(cfg, key) == expected_is_missing OmegaConf.set_readonly(cfg, True) assert OmegaConf.is_missing(cfg, key) == expected_is_missing
def test_container_inheritance(self, module: Any) -> None: parent = OmegaConf.structured( module.StructuredSubclass.ParentContainers) child = OmegaConf.structured(module.StructuredSubclass.ChildContainers) assert OmegaConf.is_missing(parent, "list1") assert child.list1 == [1, 2, 3] assert parent.list2 == [5, 6] assert child.list2 == [5, 6] assert OmegaConf.is_missing(parent, "dict") assert child.dict == {"a": 5, "b": 6}
def test_get_type(self, module: Any) -> None: cfg1 = OmegaConf.create(module.LinkedList) assert OmegaConf.get_type(cfg1) == module.LinkedList assert _utils.get_ref_type(cfg1, "next") == Optional[module.LinkedList] assert OmegaConf.get_type(cfg1, "next") is None assert cfg1.next is None assert OmegaConf.is_missing(cfg1, "value") cfg2 = OmegaConf.create(module.MissingTest.Missing1) assert OmegaConf.is_missing(cfg2, "head") assert _utils.get_ref_type(cfg2, "head") == module.LinkedList assert OmegaConf.get_type(cfg2, "head") is None
def test_leaf_node_inheritance(self, module: Any) -> None: parent = OmegaConf.structured(module.StructuredSubclass.ParentInts) child = OmegaConf.structured(module.StructuredSubclass.ChildInts) assert OmegaConf.is_missing(parent, "int1") assert OmegaConf.is_missing(child, "int1") assert OmegaConf.is_missing(parent, "int2") assert child.int2 == 5 assert OmegaConf.is_missing(parent, "int3") assert child.int3 == 10 assert OmegaConf.is_missing(parent, "int4") assert child.int4 == 15
def test_missing2(self, module: Any) -> None: cfg = OmegaConf.create(module.MissingTest.Missing2) assert cfg == {"head": {"next": "???", "value": 1}} assert OmegaConf.is_missing(cfg.head, "next") cfg.head.next = module.LinkedList(value=2) assert cfg == {"head": {"next": {"next": None, "value": 2}, "value": 1}}
def test_merge() -> None: @dataclass class Config: num: int = 10 user: User = User(name=MISSING, height=MISSING) domains: Dict[str, Domain] = field(default_factory=dict) yaml = """ user: name: Omry domains: blog_website: name: blog protocols: - HTTPS """ schema: Config = OmegaConf.structured(Config) cfg = OmegaConf.create(yaml) merged = OmegaConf.merge(schema, cfg) assert merged == { "num": 10, "user": { "name": "Omry", "height": "???" }, "domains": { "blog_website": { "name": "blog", "path": "???", "protocols": [Protocol.HTTPS], } }, } assert OmegaConf.is_missing(merged.domains.blog_website, "path")
def test_merge_missing_object_onto_typed_dictconfig( self, class_type: str) -> None: module: Any = import_module(class_type) c1 = OmegaConf.structured(module.DictOfObjects) c2 = OmegaConf.merge(c1, {"users": {"bob": "???"}}) assert isinstance(c2, DictConfig) assert OmegaConf.is_missing(c2.users, "bob")
def main(cfg: Config): if cfg.init_config_dir is not None: init_config(cfg.init_config_dir) return if OmegaConf.is_missing(cfg.configen, "modules"): # type: ignore log.error( dedent( """\ Use --config-dir DIR --config-name NAME e.g: \tconfigen --config-dir conf --config-name configen If you have no config dir yet use init_config_dir=DIR to create an initial config dir. e.g: \tconfigen init_config_dir=conf """ ) ) sys.exit(1) for module in cfg.configen.modules: code = generate_module(cfg=cfg.configen, module=module) save(cfg=cfg.configen, module=module.name, code=code)
def verify( cfg: Any, key: Any, none: bool, opt: bool, missing: bool, inter: bool, none_public: Optional[bool] = None, exp: Any = SKIP, ) -> None: if none_public is None: none_public = none target_node = cfg._get_node(key) assert target_node._key() == key assert target_node._is_none() == none assert target_node._is_optional() == opt assert target_node._is_missing() == missing assert target_node._is_interpolation() == inter if exp is not SKIP: assert cfg.get(key) == exp assert OmegaConf.is_missing(cfg, key) == missing with warns(UserWarning): assert OmegaConf.is_none(cfg, key) == none_public assert _is_optional(cfg, key) == opt assert OmegaConf.is_interpolation(cfg, key) == inter
def test_value_without_a_default(self, module: Any) -> None: cfg = OmegaConf.structured(module.NoDefaultValue) assert OmegaConf.is_missing(cfg, "no_default") OmegaConf.structured(module.NoDefaultValue(no_default=10)) == { "no_default": 10 }
def merge_defaults_list_into_config( merged_cfg: DictConfig, def_list: List[DefaultElement] ) -> DictConfig: # Reconstruct the defaults to make use of the interpolation capabilities of OmegaConf. dict_with_list = OmegaConf.create({"defaults": []}) for item in def_list: d: Any if item.config_group is not None: d = {item.config_group: item.config_name} else: d = item.config_name dict_with_list.defaults.append(d) for idx, default1 in enumerate(def_list): if default1.config_group is not None: if OmegaConf.is_missing( dict_with_list.defaults[idx], default1.config_group ): if run_mode == RunMode.RUN: raise UnspecifiedMandatoryDefault( config_group=default1.config_group ) else: config_name = "???" else: config_name = dict_with_list.defaults[idx][ default1.config_group ] else: config_name = dict_with_list.defaults[idx] if config_name == "__SELF__": if "defaults" in job_cfg: with open_dict(job_cfg): del job_cfg["defaults"] merged_cfg.merge_with(job_cfg) if job_cfg_load_trace is not None: self.all_config_checked.append(job_cfg_load_trace) elif default1.config_group is not None: if default1.config_name not in (None, "_SKIP_", "???"): merged_cfg = self._merge_config( cfg=merged_cfg, config_group=default1.config_group, name=config_name, required=not default1.optional, is_primary_config=False, package_override=default1.package, ) else: if default1.config_name != "_SKIP_": merged_cfg = self._merge_config( cfg=merged_cfg, config_group="", name=config_name, required=True, is_primary_config=False, package_override=default1.package, ) return merged_cfg
def _has_config_content(cfg: DictConfig) -> bool: if cfg._is_none() or cfg._is_missing(): return False for key in cfg.keys(): if not OmegaConf.is_missing(cfg, key): return True return False
def test_missing1(self, module: Any) -> None: cfg = OmegaConf.create(module.MissingTest.Missing1) assert OmegaConf.is_missing(cfg, "head") assert OmegaConf.get_type(cfg, "head") is None with raises(ValidationError): cfg.head = 10
def test_value_without_a_default(self, class_type: str) -> None: module: Any = import_module(class_type) cfg = OmegaConf.structured(module.NoDefaultValue) assert OmegaConf.is_missing(cfg, "no_default") OmegaConf.structured(module.NoDefaultValue(no_default=10)) == { "no_default": 10 }
def test_missing1(self, class_type: str) -> None: module: Any = import_module(class_type) cfg = OmegaConf.create(module.MissingTest.Missing1) assert OmegaConf.is_missing(cfg, "head") assert OmegaConf.get_type(cfg, "head") is None with pytest.raises(ValidationError): cfg.head = 10
def validate(cfg: DictConfig) -> None: assert cfg == { "list1": [1, 2, 3], "list2": [1, 2, 3], "missing": MISSING } with pytest.raises(ValidationError): cfg.list1[1] = "foo" assert OmegaConf.is_missing(cfg, "missing")
def test_promote_to_class(self, module: Any) -> None: conf = OmegaConf.create(module.AnyTypeConfig) assert OmegaConf.get_type(conf) == module.AnyTypeConfig conf._promote(module.BoolConfig) assert OmegaConf.get_type(conf) == module.BoolConfig assert conf.with_default is True assert conf.null_default is None assert OmegaConf.is_missing(conf, "mandatory_missing")
def _has_config_content(cfg: DictConfig) -> bool: if cfg._is_none() or cfg._is_missing(): return False for key in cfg.keys(): if not OmegaConf.is_missing(cfg, key) and key not in ( "defaults", "__HYDRA_REMOVE_TOP_LEVEL_DEFAULTS__", ): return True return False
def remove_missing_cfg(cfg): """ Remove config keys that are missing """ return_cfg = {} for k in cfg.keys(): if OmegaConf.is_missing(cfg, k): logging.warning(f"key {k} is missing and skipped") pass else: return_cfg[k] = cfg[k] return OmegaConf.create(return_cfg)
def _get_kwargs(config: Union[ObjectConf, DictConfig], **kwargs: Any) -> Any: if isinstance(config, ObjectConf): config = OmegaConf.structured(config) if config.params is not None: params = config.params else: params = OmegaConf.create() else: config = copy.deepcopy(config) if "params" in config: msg = ( "\nField 'params' is deprecated since Hydra 1.0 and will be removed in Hydra 1.1." "\nInline the content of params directly at the containing node." "\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/object_instantiation_changes" ) warnings.warn(category=UserWarning, message=msg) params = config.params else: params = config assert isinstance( params, DictConfig ), f"Input config params are expected to be a mapping, found {type(config.params).__name__}" config_overrides = {} passthrough = {} for k, v in kwargs.items(): if k in params and not ( get_ref_type(params, k) is Any and OmegaConf.is_missing(params, k) ): config_overrides[k] = v else: passthrough[k] = v final_kwargs = {} with read_write(params): params.merge_with(config_overrides) for k in params.keys(): if k == "_target_": continue if k not in passthrough: final_kwargs[k] = params[k] for k, v in passthrough.items(): final_kwargs[k] = v for k, v in passthrough.items(): final_kwargs[k] = v return final_kwargs
def main(cfg: Config): if cfg.init_config_dir is not None: init_config(cfg.init_config_dir) return if OmegaConf.is_missing(cfg.configen, "modules"): # type: ignore log.error( "Use --config-dir DIR." "\nIf you have no config dir yet use the following command to create an initial config in the `conf` dir:" "\n\tconfigen init_config_dir=conf") sys.exit(1) for module in cfg.configen.modules: code = generate_module(cfg=cfg.configen, module=module) save(cfg=cfg.configen, module=module.name, code=code)
def verify( cfg: Any, key: Any, none: bool, opt: bool, missing: bool, inter: bool, exp: Any = SKIP, ) -> None: target_node = cfg._get_node(key) assert target_node._key() == key assert target_node._is_none() == none assert target_node._is_optional() == opt assert target_node._is_missing() == missing assert target_node._is_interpolation() == inter if exp is not SKIP: assert cfg.get(key) == exp assert OmegaConf.is_missing(cfg, key) == missing assert OmegaConf.is_none(cfg, key) == none assert OmegaConf.is_optional(cfg, key) == opt assert OmegaConf.is_interpolation(cfg, key) == inter