def test_list_of_objects_missing(self, module: Any) -> None:
conf = OmegaConf.structured(module.ListOfObjectsMissing)
assert OmegaConf.is_missing(conf.users, 0)
conf.users.append(MISSING)
assert OmegaConf.is_missing(conf.users, 1)
def test_merge_missing_structured_config_is_missing(self, class_type: str) -> None:
# Test that the merged type is that of the last merged config
module: Any = import_module(class_type)
c1 = OmegaConf.structured(module.MissingStructuredConfigField)
assert OmegaConf.is_missing(c1, "plugin")
c2 = OmegaConf.merge(c1, module.MissingStructuredConfigField)
assert OmegaConf.is_missing(c2, "plugin")
def test_dict_of_objects_missing(self, module: Any) -> None:
conf = OmegaConf.structured(module.DictOfObjectsMissing)
dct = conf.users
assert OmegaConf.is_missing(dct, "moe")
dct.miss = MISSING
assert OmegaConf.is_missing(dct, "miss")
def test_interpolation_with_missing() -> None:
cfg = OmegaConf.create({
"a": "${x.missing}.txt",
"b": "${x.missing}",
"x": {
"missing": "???"
}
})
assert OmegaConf.is_missing(cfg, "a")
assert OmegaConf.is_missing(cfg, "b")
def _map_merge(dest: "BaseContainer", src: "BaseContainer") -> None:
"""merge src into dest and return a new copy, does not modified input"""
from omegaconf import OmegaConf
from .dictconfig import DictConfig
from .nodes import ValueNode
assert isinstance(dest, DictConfig)
assert isinstance(src, DictConfig)
src_type = src._metadata.object_type
dest._validate_set_merge_impl(key=None, value=src, is_assign=False)
for key, src_value in src.items_ex(resolve=False):
if OmegaConf.is_missing(dest, key):
if isinstance(src_value, DictConfig):
if OmegaConf.is_missing(dest, key):
dest[key] = src_value
dest_node = dest._get_node(key, validate_access=False)
if dest_node is not None:
if dest_node._is_interpolation():
target_node = dest_node._dereference_node(
throw_on_resolution_failure=False)
if isinstance(target_node, Container):
dest[key] = target_node
dest_node = dest._get_node(key)
if is_structured_config(dest._metadata.element_type):
dest[key] = DictConfig(content=dest._metadata.element_type,
parent=dest)
dest_node = dest._get_node(key)
if dest_node is not None:
if isinstance(dest_node, BaseContainer):
if isinstance(src_value, BaseContainer):
dest._validate_merge(key=key, value=src_value)
dest_node._merge_with(src_value)
else:
dest.__setitem__(key, src_value)
else:
if isinstance(src_value, BaseContainer):
dest.__setitem__(key, src_value)
else:
assert isinstance(dest_node, ValueNode)
try:
dest_node._set_value(src_value)
except ValidationError as e:
dest._format_and_raise(key=key,
value=src_value,
cause=e)
else:
dest[key] = src._get_node(key)
if src_type is not None and not is_primitive_dict(src_type):
dest._metadata.object_type = src_type
def test_is_missing_resets() -> None:
cfg = OmegaConf.structured(StructuredWithMissing)
assert OmegaConf.is_missing(cfg, "dict")
cfg.dict = {}
assert not OmegaConf.is_missing(cfg, "dict")
assert OmegaConf.is_missing(cfg, "list")
cfg.list = [1, 2, 3]
assert not OmegaConf.is_missing(cfg, "list")
cfg.list = "???"
assert OmegaConf.is_missing(cfg, "list")
def test_get_type(self, class_type: str) -> None:
module: Any = import_module(class_type)
linked_list = module.LinkedList
cfg1 = OmegaConf.create(linked_list)
assert OmegaConf.get_type(cfg1) == linked_list
assert cfg1.next is None
assert OmegaConf.is_missing(cfg1, "value")
cfg2 = OmegaConf.create(module.MissingTest.Missing1)
assert OmegaConf.is_missing(cfg2, "head")
assert OmegaConf.get_type(cfg2, "head") == module.LinkedList
def _map_merge(dest: "BaseContainer", src: "BaseContainer") -> None:
"""merge src into dest and return a new copy, does not modified input"""
from omegaconf import OmegaConf
from .dictconfig import DictConfig
from .nodes import ValueNode
assert isinstance(dest, DictConfig)
assert isinstance(src, DictConfig)
src = copy.deepcopy(src)
src_type = src._metadata.object_type
dest_type = dest._metadata.object_type
if src_type is not None and src_type is not dest_type:
prototype = DictConfig(
annotated_type=src_type,
content=src_type,
)
dest.__dict__["_content"] = copy.deepcopy(
prototype.__dict__["_content"])
dest.__dict__["_metadata"] = copy.deepcopy(prototype._metadata)
for key, value in src.items_ex(resolve=False):
dest_element_type = dest._metadata.element_type
typed = dest_element_type not in (None, Any)
if OmegaConf.is_missing(dest, key):
if isinstance(value, DictConfig):
if OmegaConf.is_missing(src, key):
dest[key] = DictConfig(content="???")
else:
dest[key] = {}
if (dest.get_node(key) is not None) or typed:
dest_node = dest.get_node(key)
if dest_node is None and typed:
dest[key] = DictConfig(content=dest_element_type,
parent=dest)
dest_node = dest.get_node(key)
if isinstance(dest_node, BaseContainer):
if isinstance(value, BaseContainer):
dest._validate_set(key=key, value=value)
dest_node.merge_with(value)
else:
dest.__setitem__(key, value)
else:
if isinstance(value, BaseContainer):
dest.__setitem__(key, value)
else:
assert isinstance(dest_node, ValueNode)
dest_node._set_value(value)
else:
dest[key] = src.get_node(key)
def test_is_missing(cfg: Any, key: str, expected_is_missing: bool,
expectation: Any) -> None:
cfg = OmegaConf.create(cfg)
with expectation:
cfg.get(key)
assert OmegaConf.is_missing(cfg, key) == expected_is_missing
OmegaConf.set_struct(cfg, True)
assert OmegaConf.is_missing(cfg, key) == expected_is_missing
OmegaConf.set_readonly(cfg, True)
assert OmegaConf.is_missing(cfg, key) == expected_is_missing
def test_container_inheritance(self, module: Any) -> None:
parent = OmegaConf.structured(
module.StructuredSubclass.ParentContainers)
child = OmegaConf.structured(module.StructuredSubclass.ChildContainers)
assert OmegaConf.is_missing(parent, "list1")
assert child.list1 == [1, 2, 3]
assert parent.list2 == [5, 6]
assert child.list2 == [5, 6]
assert OmegaConf.is_missing(parent, "dict")
assert child.dict == {"a": 5, "b": 6}
def test_get_type(self, module: Any) -> None:
cfg1 = OmegaConf.create(module.LinkedList)
assert OmegaConf.get_type(cfg1) == module.LinkedList
assert _utils.get_ref_type(cfg1, "next") == Optional[module.LinkedList]
assert OmegaConf.get_type(cfg1, "next") is None
assert cfg1.next is None
assert OmegaConf.is_missing(cfg1, "value")
cfg2 = OmegaConf.create(module.MissingTest.Missing1)
assert OmegaConf.is_missing(cfg2, "head")
assert _utils.get_ref_type(cfg2, "head") == module.LinkedList
assert OmegaConf.get_type(cfg2, "head") is None
def test_leaf_node_inheritance(self, module: Any) -> None:
parent = OmegaConf.structured(module.StructuredSubclass.ParentInts)
child = OmegaConf.structured(module.StructuredSubclass.ChildInts)
assert OmegaConf.is_missing(parent, "int1")
assert OmegaConf.is_missing(child, "int1")
assert OmegaConf.is_missing(parent, "int2")
assert child.int2 == 5
assert OmegaConf.is_missing(parent, "int3")
assert child.int3 == 10
assert OmegaConf.is_missing(parent, "int4")
assert child.int4 == 15
def test_missing2(self, module: Any) -> None:
cfg = OmegaConf.create(module.MissingTest.Missing2)
assert cfg == {"head": {"next": "???", "value": 1}}
assert OmegaConf.is_missing(cfg.head, "next")
cfg.head.next = module.LinkedList(value=2)
assert cfg == {"head": {"next": {"next": None, "value": 2}, "value": 1}}
def test_merge() -> None:
@dataclass
class Config:
num: int = 10
user: User = User(name=MISSING, height=MISSING)
domains: Dict[str, Domain] = field(default_factory=dict)
yaml = """
user:
name: Omry
domains:
blog_website:
name: blog
protocols:
- HTTPS
"""
schema: Config = OmegaConf.structured(Config)
cfg = OmegaConf.create(yaml)
merged = OmegaConf.merge(schema, cfg)
assert merged == {
"num": 10,
"user": {
"name": "Omry",
"height": "???"
},
"domains": {
"blog_website": {
"name": "blog",
"path": "???",
"protocols": [Protocol.HTTPS],
}
},
}
assert OmegaConf.is_missing(merged.domains.blog_website, "path")
def test_merge_missing_object_onto_typed_dictconfig(
self, class_type: str) -> None:
module: Any = import_module(class_type)
c1 = OmegaConf.structured(module.DictOfObjects)
c2 = OmegaConf.merge(c1, {"users": {"bob": "???"}})
assert isinstance(c2, DictConfig)
assert OmegaConf.is_missing(c2.users, "bob")
def main(cfg: Config):
if cfg.init_config_dir is not None:
init_config(cfg.init_config_dir)
return
if OmegaConf.is_missing(cfg.configen, "modules"): # type: ignore
log.error(
dedent(
"""\
Use --config-dir DIR --config-name NAME
e.g:
\tconfigen --config-dir conf --config-name configen
If you have no config dir yet use init_config_dir=DIR to create an initial config dir.
e.g:
\tconfigen init_config_dir=conf
"""
)
)
sys.exit(1)
for module in cfg.configen.modules:
code = generate_module(cfg=cfg.configen, module=module)
save(cfg=cfg.configen, module=module.name, code=code)
def verify(
cfg: Any,
key: Any,
none: bool,
opt: bool,
missing: bool,
inter: bool,
none_public: Optional[bool] = None,
exp: Any = SKIP,
) -> None:
if none_public is None:
none_public = none
target_node = cfg._get_node(key)
assert target_node._key() == key
assert target_node._is_none() == none
assert target_node._is_optional() == opt
assert target_node._is_missing() == missing
assert target_node._is_interpolation() == inter
if exp is not SKIP:
assert cfg.get(key) == exp
assert OmegaConf.is_missing(cfg, key) == missing
with warns(UserWarning):
assert OmegaConf.is_none(cfg, key) == none_public
assert _is_optional(cfg, key) == opt
assert OmegaConf.is_interpolation(cfg, key) == inter
def test_value_without_a_default(self, module: Any) -> None:
cfg = OmegaConf.structured(module.NoDefaultValue)
assert OmegaConf.is_missing(cfg, "no_default")
OmegaConf.structured(module.NoDefaultValue(no_default=10)) == {
"no_default": 10
}
def merge_defaults_list_into_config(
merged_cfg: DictConfig, def_list: List[DefaultElement]
) -> DictConfig:
# Reconstruct the defaults to make use of the interpolation capabilities of OmegaConf.
dict_with_list = OmegaConf.create({"defaults": []})
for item in def_list:
d: Any
if item.config_group is not None:
d = {item.config_group: item.config_name}
else:
d = item.config_name
dict_with_list.defaults.append(d)
for idx, default1 in enumerate(def_list):
if default1.config_group is not None:
if OmegaConf.is_missing(
dict_with_list.defaults[idx], default1.config_group
):
if run_mode == RunMode.RUN:
raise UnspecifiedMandatoryDefault(
config_group=default1.config_group
)
else:
config_name = "???"
else:
config_name = dict_with_list.defaults[idx][
default1.config_group
]
else:
config_name = dict_with_list.defaults[idx]
if config_name == "__SELF__":
if "defaults" in job_cfg:
with open_dict(job_cfg):
del job_cfg["defaults"]
merged_cfg.merge_with(job_cfg)
if job_cfg_load_trace is not None:
self.all_config_checked.append(job_cfg_load_trace)
elif default1.config_group is not None:
if default1.config_name not in (None, "_SKIP_", "???"):
merged_cfg = self._merge_config(
cfg=merged_cfg,
config_group=default1.config_group,
name=config_name,
required=not default1.optional,
is_primary_config=False,
package_override=default1.package,
)
else:
if default1.config_name != "_SKIP_":
merged_cfg = self._merge_config(
cfg=merged_cfg,
config_group="",
name=config_name,
required=True,
is_primary_config=False,
package_override=default1.package,
)
return merged_cfg
def _has_config_content(cfg: DictConfig) -> bool:
if cfg._is_none() or cfg._is_missing():
return False
for key in cfg.keys():
if not OmegaConf.is_missing(cfg, key):
return True
return False
def test_missing1(self, module: Any) -> None:
cfg = OmegaConf.create(module.MissingTest.Missing1)
assert OmegaConf.is_missing(cfg, "head")
assert OmegaConf.get_type(cfg, "head") is None
with raises(ValidationError):
cfg.head = 10
def test_value_without_a_default(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.NoDefaultValue)
assert OmegaConf.is_missing(cfg, "no_default")
OmegaConf.structured(module.NoDefaultValue(no_default=10)) == {
"no_default": 10
}
def test_missing1(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.create(module.MissingTest.Missing1)
assert OmegaConf.is_missing(cfg, "head")
assert OmegaConf.get_type(cfg, "head") is None
with pytest.raises(ValidationError):
cfg.head = 10
def validate(cfg: DictConfig) -> None:
assert cfg == {
"list1": [1, 2, 3],
"list2": [1, 2, 3],
"missing": MISSING
}
with pytest.raises(ValidationError):
cfg.list1[1] = "foo"
assert OmegaConf.is_missing(cfg, "missing")
def test_promote_to_class(self, module: Any) -> None:
conf = OmegaConf.create(module.AnyTypeConfig)
assert OmegaConf.get_type(conf) == module.AnyTypeConfig
conf._promote(module.BoolConfig)
assert OmegaConf.get_type(conf) == module.BoolConfig
assert conf.with_default is True
assert conf.null_default is None
assert OmegaConf.is_missing(conf, "mandatory_missing")
def _has_config_content(cfg: DictConfig) -> bool:
if cfg._is_none() or cfg._is_missing():
return False
for key in cfg.keys():
if not OmegaConf.is_missing(cfg, key) and key not in (
"defaults",
"__HYDRA_REMOVE_TOP_LEVEL_DEFAULTS__",
):
return True
return False
def remove_missing_cfg(cfg):
"""
Remove config keys that are missing
"""
return_cfg = {}
for k in cfg.keys():
if OmegaConf.is_missing(cfg, k):
logging.warning(f"key {k} is missing and skipped")
pass
else:
return_cfg[k] = cfg[k]
return OmegaConf.create(return_cfg)
def _get_kwargs(config: Union[ObjectConf, DictConfig], **kwargs: Any) -> Any:
if isinstance(config, ObjectConf):
config = OmegaConf.structured(config)
if config.params is not None:
params = config.params
else:
params = OmegaConf.create()
else:
config = copy.deepcopy(config)
if "params" in config:
msg = (
"\nField 'params' is deprecated since Hydra 1.0 and will be removed in Hydra 1.1."
"\nInline the content of params directly at the containing node."
"\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/object_instantiation_changes"
)
warnings.warn(category=UserWarning, message=msg)
params = config.params
else:
params = config
assert isinstance(
params, DictConfig
), f"Input config params are expected to be a mapping, found {type(config.params).__name__}"
config_overrides = {}
passthrough = {}
for k, v in kwargs.items():
if k in params and not (
get_ref_type(params, k) is Any and OmegaConf.is_missing(params, k)
):
config_overrides[k] = v
else:
passthrough[k] = v
final_kwargs = {}
with read_write(params):
params.merge_with(config_overrides)
for k in params.keys():
if k == "_target_":
continue
if k not in passthrough:
final_kwargs[k] = params[k]
for k, v in passthrough.items():
final_kwargs[k] = v
for k, v in passthrough.items():
final_kwargs[k] = v
return final_kwargs
def main(cfg: Config):
if cfg.init_config_dir is not None:
init_config(cfg.init_config_dir)
return
if OmegaConf.is_missing(cfg.configen, "modules"): # type: ignore
log.error(
"Use --config-dir DIR."
"\nIf you have no config dir yet use the following command to create an initial config in the `conf` dir:"
"\n\tconfigen init_config_dir=conf")
sys.exit(1)
for module in cfg.configen.modules:
code = generate_module(cfg=cfg.configen, module=module)
save(cfg=cfg.configen, module=module.name, code=code)
def verify(
cfg: Any,
key: Any,
none: bool,
opt: bool,
missing: bool,
inter: bool,
exp: Any = SKIP,
) -> None:
target_node = cfg._get_node(key)
assert target_node._key() == key
assert target_node._is_none() == none
assert target_node._is_optional() == opt
assert target_node._is_missing() == missing
assert target_node._is_interpolation() == inter
if exp is not SKIP:
assert cfg.get(key) == exp
assert OmegaConf.is_missing(cfg, key) == missing
assert OmegaConf.is_none(cfg, key) == none
assert OmegaConf.is_optional(cfg, key) == opt
assert OmegaConf.is_interpolation(cfg, key) == inter
