def test_merge_structured_interpolation_onto_dict(self,
class_type: str) -> None:
module: Any = import_module(class_type)
c1 = OmegaConf.create({
"user_1": {
"name": "bob"
},
"user_2": {
"name": "alice"
},
"user_3": {
"name": "joe"
},
})
src = OmegaConf.create({
"user_2": module.User(),
"user_3": module.User()
})
src.user_2 = "${user_1}"
src.user_3 = None
c2 = OmegaConf.merge(c1, src)
assert c2.user_2.name == "bob"
assert get_ref_type(c2, "user_2") == Any
assert c2.user_3 is None
assert get_ref_type(c2, "user_3") == Any
def test_create_untyped_list(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.UntypedList)
assert _utils.get_ref_type(cfg, "list") == List[Any]
assert _utils.get_ref_type(cfg, "opt_list") == Optional[List[Any]]
assert cfg.list == [1, 2]
assert cfg.opt_list is None
def _test_assign(self, ref_type: Any, value: Any, assign: Any) -> None:
cfg = OmegaConf.create(
{"foo": DictConfig(ref_type=ref_type, content=value)})
assert _utils.get_ref_type(cfg, "foo") == Optional[ref_type]
cfg.foo = assign
assert cfg.foo == assign
assert _utils.get_ref_type(cfg, "foo") == Optional[ref_type]
def test_create_untyped_dict(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.UntypedDict)
assert _utils.get_ref_type(cfg, "dict") == Dict[Any, Any]
assert _utils.get_ref_type(cfg, "opt_dict") == Optional[Dict[Any, Any]]
assert cfg.dict == {"foo": "var"}
assert cfg.opt_dict is None
def test_merge_with_subclass_into_missing(self, module: Any) -> None:
base = OmegaConf.structured(module.PluginHolder)
assert _utils.get_ref_type(base, "missing") == module.Plugin
assert OmegaConf.get_type(base, "missing") is None
res = OmegaConf.merge(base, {"missing": module.Plugin})
assert OmegaConf.get_type(res) == module.PluginHolder
assert _utils.get_ref_type(base, "missing") == module.Plugin
assert OmegaConf.get_type(res, "missing") == module.Plugin
def test_get_ref_type_with_conflict() -> None:
cfg = OmegaConf.create(
{"user": User, "inter": DictConfig(ref_type=Plugin, content="${user}")}
)
assert OmegaConf.get_type(cfg.user) == User
assert _utils.get_ref_type(cfg.user) == Optional[User]
# Interpolation inherits both type and ref type from the target
assert OmegaConf.get_type(cfg.inter) == User
assert _utils.get_ref_type(cfg.inter) == Optional[User]
def test_get_type(self, module: Any) -> None:
cfg1 = OmegaConf.create(module.LinkedList)
assert OmegaConf.get_type(cfg1) == module.LinkedList
assert _utils.get_ref_type(cfg1, "next") == Optional[module.LinkedList]
assert OmegaConf.get_type(cfg1, "next") is None
assert cfg1.next is None
assert OmegaConf.is_missing(cfg1, "value")
cfg2 = OmegaConf.create(module.MissingTest.Missing1)
assert OmegaConf.is_missing(cfg2, "head")
assert _utils.get_ref_type(cfg2, "head") == module.LinkedList
assert OmegaConf.get_type(cfg2, "head") is None
def _test_merge(self, ref_type: Any, value: Any, assign: Any) -> None:
cfg = OmegaConf.create(
{"foo": DictConfig(ref_type=ref_type, content=value)})
cfg2 = OmegaConf.merge(cfg, {"foo": assign})
assert isinstance(cfg2, DictConfig)
assert cfg2.foo == assign
assert _utils.get_ref_type(cfg2, "foo") == Optional[ref_type]
def test_pickle_untyped(
input_: Any,
node: str,
optional: bool,
element_type: Any,
key_type: Any,
ref_type: Any,
) -> None:
cfg = OmegaConf.structured(input_)
with tempfile.TemporaryFile() as fp:
import pickle
pickle.dump(cfg, fp)
fp.flush()
fp.seek(0)
cfg2 = pickle.load(fp)
def get_node(cfg: Any, key: str) -> Any:
if key is None:
return cfg
else:
return cfg._get_node(key)
assert cfg == cfg2
assert get_ref_type(get_node(cfg2, node)) == ref_type
assert get_node(cfg2, node)._metadata.element_type == element_type
assert get_node(cfg2, node)._metadata.optional == optional
if isinstance(input_, DictConfig):
assert get_node(cfg2, node)._metadata.key_type == key_type
def test_merge_structured_onto_dict_nested2(self, module: Any) -> None:
c1 = OmegaConf.create({"user": {"name": IntegerNode(value=7)}})
c2 = OmegaConf.merge(c1, module.MissingUserField)
assert c1 == {"user": {"name": 7}}
# type of name remains int
assert c2 == {"user": {"name": 7, "age": "???"}}
assert isinstance(c2, DictConfig)
assert get_ref_type(c2, "user") == module.User
def test_merge_structured_onto_dict_nested(self, module: Any) -> None:
c1 = OmegaConf.create({"user": {"name": 7}})
c2 = OmegaConf.merge(c1, module.MissingUserField)
assert c1 == {"user": {"name": 7}}
# type of name becomes str
assert c2 == {"user": {"name": "7", "age": "???"}}
assert isinstance(c2, DictConfig)
assert get_ref_type(c2, "user") == module.User
def test_merge_structured_onto_dict_nested3(self, module: Any) -> None:
c1 = OmegaConf.create({"user": {"name": "alice"}})
c2 = OmegaConf.merge(c1, module.MissingUserWithDefaultNameField)
assert c1 == {"user": {"name": "alice"}}
# name is not changed
assert c2 == {"user": {"name": "alice", "age": "???"}}
assert isinstance(c2, DictConfig)
assert get_ref_type(c2, "user") == module.UserWithDefaultName
def test_assign_to_reftype_plugin(ref_type: Any, values: List[Any],
assign: Any, expectation: Any) -> None:
for value in values:
cfg = OmegaConf.create(
{"foo": DictConfig(ref_type=ref_type, content=value)})
with expectation():
assert _utils.get_ref_type(cfg, "foo") == Optional[ref_type]
cfg.foo = assign
assert cfg.foo == assign
# validate assignment does not change ref type.
assert _utils.get_ref_type(cfg, "foo") == Optional[ref_type]
if value is not None:
cfg = OmegaConf.create(
{"foo": DictConfig(ref_type=ref_type, content=value)})
with expectation():
cfg2 = OmegaConf.merge(cfg, {"foo": assign})
assert isinstance(cfg2, DictConfig)
assert cfg2.foo == assign
def test_str2str_as_sub_node(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.create({"foo": module.DictSubclass.Str2Str})
assert OmegaConf.get_type(cfg.foo) == module.DictSubclass.Str2Str
assert _utils.get_ref_type(cfg.foo) == Optional[module.DictSubclass.Str2Str]
cfg.foo.hello = "world"
assert cfg.foo.hello == "world"
with pytest.raises(KeyValidationError):
cfg.foo[Color.RED] = "fail"
def test_merge_optional_structured_onto_dict(self, module: Any) -> None:
c1 = OmegaConf.create({"user": {"name": "bob"}})
c2 = OmegaConf.merge(c1, module.OptionalUser(module.User(name="alice")))
assert c2.user.name == "alice"
assert get_ref_type(c2, "user") == Optional[module.User]
assert isinstance(c2, DictConfig)
c2_user = c2._get_node("user")
assert isinstance(c2_user, Node)
# Compared to the previous assert, here we verify that the `ref_type` found
# in the metadata is *not* optional: instead, the `optional` flag must be set.
assert c2_user._metadata.ref_type == module.User
assert c2_user._metadata.optional
def test_str2str_as_sub_node(self, module: Any) -> None:
cfg = OmegaConf.create({"foo": module.DictSubclass.Str2Str})
assert OmegaConf.get_type(cfg.foo) == module.DictSubclass.Str2Str
assert _utils.get_ref_type(cfg.foo) == Any
cfg.foo.hello = "world"
assert cfg.foo.hello == "world"
with raises(KeyValidationError):
cfg.foo[Color.RED] = "fail"
with raises(KeyValidationError):
cfg.foo[123] = "fail"
def test_pickle(obj: Any) -> None:
with tempfile.TemporaryFile() as fp:
c = OmegaConf.create(obj)
pickle.dump(c, fp)
fp.flush()
fp.seek(0)
c1 = pickle.load(fp)
assert c == c1
assert get_ref_type(c1) == Any
assert c1._metadata.element_type is Any
assert c1._metadata.optional is True
if isinstance(c, DictConfig):
assert c1._metadata.key_type is Any
def _get_kwargs(config: Union[ObjectConf, DictConfig], **kwargs: Any) -> Any:
if isinstance(config, ObjectConf):
config = OmegaConf.structured(config)
if config.params is not None:
params = config.params
else:
params = OmegaConf.create()
else:
config = copy.deepcopy(config)
if "params" in config:
msg = (
"\nField 'params' is deprecated since Hydra 1.0 and will be removed in Hydra 1.1."
"\nInline the content of params directly at the containing node."
"\nSee https://hydra.cc/docs/next/upgrades/0.11_to_1.0/object_instantiation_changes"
)
warnings.warn(category=UserWarning, message=msg)
params = config.params
else:
params = config
assert isinstance(
params, DictConfig
), f"Input config params are expected to be a mapping, found {type(config.params).__name__}"
config_overrides = {}
passthrough = {}
for k, v in kwargs.items():
if k in params and not (
get_ref_type(params, k) is Any and OmegaConf.is_missing(params, k)
):
config_overrides[k] = v
else:
passthrough[k] = v
final_kwargs = {}
with read_write(params):
params.merge_with(config_overrides)
for k in params.keys():
if k == "_target_":
continue
if k not in passthrough:
final_kwargs[k] = params[k]
for k, v in passthrough.items():
final_kwargs[k] = v
for k, v in passthrough.items():
final_kwargs[k] = v
return final_kwargs
def test_plugin_holder(self, module: Any) -> None:
cfg = OmegaConf.create(module.PluginHolder)
assert _is_optional(cfg, "none")
assert _utils.get_ref_type(cfg, "none") == Optional[module.Plugin]
assert OmegaConf.get_type(cfg, "none") is None
assert not _is_optional(cfg, "missing")
assert _utils.get_ref_type(cfg, "missing") == module.Plugin
assert OmegaConf.get_type(cfg, "missing") is None
assert not _is_optional(cfg, "plugin")
assert _utils.get_ref_type(cfg, "plugin") == module.Plugin
assert OmegaConf.get_type(cfg, "plugin") == module.Plugin
cfg.plugin = module.ConcretePlugin()
assert not _is_optional(cfg, "plugin")
assert _utils.get_ref_type(cfg, "plugin") == module.Plugin
assert OmegaConf.get_type(cfg, "plugin") == module.ConcretePlugin
assert not _is_optional(cfg, "plugin2")
assert _utils.get_ref_type(cfg, "plugin2") == module.Plugin
assert OmegaConf.get_type(cfg, "plugin2") == module.ConcretePlugin
def test_int2str_as_sub_node(self, module: Any) -> None:
cfg = OmegaConf.create({"foo": module.DictSubclass.Int2Str})
assert OmegaConf.get_type(cfg.foo) == module.DictSubclass.Int2Str
assert _utils.get_ref_type(cfg.foo) == Any
cfg.foo[10] = "ten"
assert cfg.foo[10] == "ten"
with raises(KeyValidationError):
cfg.foo[10.0] = "float" # fail
with raises(KeyValidationError):
cfg.foo["10"] = "string" # fail
with raises(KeyValidationError):
cfg.foo.hello = "fail"
with raises(KeyValidationError):
cfg.foo[Color.RED] = "fail"
def test_get_ref_type_error() -> None:
with raises(ValueError):
_utils.get_ref_type(AnyNode(), "foo")
def test_get_node_ref_type(obj: Any, key: str, expected: Any) -> None:
cfg = OmegaConf.create(obj)
assert _utils.get_ref_type(cfg, key) == expected
def test_get_ref_type(obj: Any, expected: Any) -> None:
assert _utils.get_ref_type(obj) == expected
def test_create_untyped_dict() -> None:
from omegaconf._utils import get_ref_type
cfg = DictConfig(ref_type=Dict, content={})
assert get_ref_type(cfg) == Optional[Dict]
def test_create_untyped_list() -> None:
from omegaconf._utils import get_ref_type
cfg = ListConfig(ref_type=List, content=[])
assert get_ref_type(cfg) == Optional[List]
def test_get_ref_type(cfg: Any, expected_ref_type: Any) -> None:
assert _utils.get_ref_type(cfg.plugin) == expected_ref_type
def test_nested_config_is_none(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.NestedWithNone)
assert cfg == {"plugin": None}
assert OmegaConf.get_type(cfg, "plugin") is None
assert _utils.get_ref_type(cfg, "plugin") == Optional[module.Plugin]
