def test_validation():
rest_api = Role('rest_api')
rest_api.lockdown(Bicycle)\
.writeable_by(Bicycle.owner == ContextParam('user'))\
.validate(Bicycle.owner, Bicycle.owner == ContextParam('user'))\
.validate(Bicycle.serial, lambda b, f, v: v.startswith('a'))
with test_database(test_db, [User, Group, Bicycle]):
b = Bicycle.create()
u1 = User.create(username='******')
u2 = User.create(username='******')
lockdown_context.role = rest_api
try:
b.owner = u1
assert False, 'should have thrown exception'
except:
pass
lockdown_context.user = u1.id
try:
b.owner = u2
assert False, 'should have thrown exception'
except:
pass
lockdown_context.user = u1.id
b.owner = u1
b.serial = 'a'
try:
b.serial = 'b'
assert False, 'should have thrown exception'
except LockdownException:
pass
def test_lockdown_user():
lockdown_context.role = None
lockdown_context.user = None
lockdown_context.group = None
rest_api = Role('rest_api')
rest_api.lockdown(Bicycle) \
.field_writeable_by(Bicycle.owner, Bicycle.owner == ContextParam('user'))
with test_database(test_db, [User, Group, Bicycle]):
u = User.create(username='******')
u2 = User.create(username='******')
g = Group.create(name='test')
b = Bicycle.create(owner=u, group=g)
lockdown_context.role = rest_api
assert b.is_field_writeable(Bicycle.owner) is False
lockdown_context.user = 10
assert b.is_field_writeable(Bicycle.owner) is False
try:
b.owner = u2
assert False, 'should have thrown'
except:
pass
lockdown_context.user = u.id
assert b.is_field_writeable(Bicycle.owner) is True
def test_dict_values_are_typed() -> None:
cfg = OmegaConf.create({
"x":
"${oc.dict.values: y.name2user}",
"y":
Users(name2user={
"john": User(name="john", age=30),
"jane": User(name="jane", age=33),
}),
})
x = cfg.x
assert x._metadata.ref_type == List[User]
assert x._metadata.element_type == User
def test_instantiated_regular_class_container_types_partial(
instantiate_func: Any, ) -> None:
cfg = {"_target_": "tests.SimpleClass", "a": {}, "b": User()}
ret = instantiate_func(cfg, _convert_=ConvertMode.PARTIAL)
assert isinstance(ret.a, dict)
assert isinstance(ret.b, DictConfig)
assert OmegaConf.get_type(ret.b) is User
def test_interpolation_readonly_node() -> None:
cfg = OmegaConf.structured(User(name="7", age=II("name")))
resolved = dereference_node(cfg, "age")
assert resolved == 7
# The `resolved` node must be read-only because `age` is an integer, so the
# interpolation cannot return directly the `name` node.
assert resolved._get_flag("readonly")
def test_nested_dataclass_with_partial_convert() -> None:
# dict
cfg = OmegaConf.structured(NestedConf)
ret = utils.instantiate(cfg, _convert_="partial")
assert isinstance(ret.a, DictConfig) and OmegaConf.get_type(ret.a) == User
assert isinstance(ret.b, DictConfig) and OmegaConf.get_type(ret.b) == User
expected = SimpleClass(a=User(name="a", age=1), b=User(name="b", age=2))
assert ret == expected
# list
lst = [User(name="a", age=1)]
cfg = OmegaConf.structured(NestedConf(a=lst))
ret = utils.instantiate(cfg, _convert_="partial")
assert isinstance(ret.a, list) and OmegaConf.get_type(ret.a[0]) == User
assert isinstance(ret.b, DictConfig) and OmegaConf.get_type(ret.b) == User
expected = SimpleClass(a=lst, b=User(name="b", age=2))
assert ret == expected
def test_dict_structured_delitem() -> None:
c = OmegaConf.structured(User(name="Bond"))
with raises(ConfigTypeError):
del c["name"]
with open_dict(c):
del c["name"]
assert "name" not in c
def test_insert():
lockdown_context.role = None
lockdown_context.user = None
lockdown_context.group = None
rest_api = Role('rest_api')
rest_api.lockdown(BigWheel) \
.writeable_by(BigWheel.owner == ContextParam('user'))\
.validate(BigWheel.owner, BigWheel.owner == ContextParam('user')) \
.field_writeable_by(BigWheel.serial, BigWheel.owner == ContextParam('user'))
with test_database(test_db, [User, Group, BigWheel]):
u1 = User.create(username='******')
u2 = User.create(username='******')
g = Group.create(name='test')
# set the role/user before inserting
lockdown_context.role = rest_api
lockdown_context.user = u1.id
try:
bwheel = BigWheel()
bwheel.owner = u2
bwheel.group = g
bwheel.serial = '1'
assert False, 'should have failed'
except LockdownException:
pass
# insert should work fine since object validates
bwheel = BigWheel()
bwheel.owner = u1
bwheel.group = g
bwheel.serial = '1'
bwheel.save()
b = BigWheel.get()
assert b.modified is not None
assert b.created is not None
assert b.serial == '1'
assert b.owner == u1
assert b.group == g
def test_get_type() -> None:
cfg = OmegaConf.structured(User)
assert OmegaConf.get_type(cfg) == User
cfg = OmegaConf.structured(User(name="bond"))
assert OmegaConf.get_type(cfg) == User
cfg = OmegaConf.create({"user": User, "inter": "${user}"})
assert OmegaConf.get_type(cfg.user) == User
assert OmegaConf.get_type(cfg.inter) == User
def test_dict_nested_structured_delitem() -> None:
c = OmegaConf.create({"user": User(name="Bond")})
with raises(ConfigTypeError):
del c.user["name"]
# Unlocking the top level node is not enough.
with raises(ConfigTypeError):
with open_dict(c):
del c.user["name"]
# You need to unlock the specified structured node to delete a field from it.
with open_dict(c.user):
del c.user["name"]
assert "name" not in c.user
def test_is_readable_writable_field():
lockdown_context.role = None
lockdown_context.user = None
lockdown_context.group = None
rest_api = Role('rest_api')
rest_api.lockdown(BaseModel)\
.field_writeable_by(BaseModel.created, NO_ONE)\
.field_writeable_by(BaseModel.modified, NO_ONE)
rest_api.lockdown(Bicycle) \
.field_readable_by(Bicycle.serial, Bicycle.group == ContextParam('group')) \
.field_writeable_by(Bicycle.serial, Bicycle.owner == ContextParam('user'))
with test_database(test_db, [User, Group, Bicycle]):
u = User.create(username='******')
g = Group.create(name='test')
b = Bicycle.create(owner=u, group=g)
lockdown_context.role = rest_api
assert b.is_field_readable(Bicycle.serial) is False
assert b.is_field_writeable(Bicycle.serial) is False
assert b.is_field_writeable(Bicycle.modified) is False
assert b.is_field_writeable(Bicycle.created) is False
lockdown_context.group = 10
lockdown_context.user = 10
assert b.is_field_readable(Bicycle.serial) is False
assert b.is_field_writeable(Bicycle.serial) is False
lockdown_context.group = g.id
assert b.is_field_readable(Bicycle.serial) is True
assert b.is_field_writeable(Bicycle.serial) is False
lockdown_context.user = u.id
assert b.is_field_readable(Bicycle.serial) is True
assert b.is_field_writeable(Bicycle.serial) is True
def test_dict_structured_mode_pop() -> None:
cfg = OmegaConf.create({"user": User(name="Bond")})
with raises(ConfigTypeError):
cfg.user.pop("name")
with raises(ConfigTypeError):
cfg.user.pop("bar")
with raises(ConfigTypeError):
cfg.user.pop("bar", "not even with default")
# Unlocking the top level node is not enough.
with raises(ConfigTypeError):
with open_dict(cfg):
cfg.user.pop("name")
# You need to unlock the specified structured node to pop a field from it.
with open_dict(cfg.user):
cfg.user.pop("name")
assert "name" not in cfg.user
def test_save():
lockdown_context.role = None
lockdown_context.user = None
lockdown_context.group = None
rest_api = Role('rest_api')
rest_api.lockdown(Bicycle) \
.field_readable_by(Bicycle.serial, Bicycle.group == ContextParam('group')) \
.field_writeable_by(Bicycle.serial, Bicycle.owner == ContextParam('user'))
with test_database(test_db, [User, Group, Bicycle]):
u = User.create(username='******')
g = Group.create(name='test')
b = Bicycle.create(owner=u, group=g, serial='1')
lockdown_context.role = rest_api
try:
b.serial = '10'
assert False, 'should have thrown'
except:
pass
assert b.serial == '1'
b = Bicycle.get()
assert b.serial is None, 'cant read this field, no group'
lockdown_context.group = g.id
b = Bicycle.get()
assert b.serial == '1', 'now this field can be read, should be 1 still'
lockdown_context.user = u.id
b.serial = '10'
b.save()
b = Bicycle.get()
assert b.serial == '10'
def test_readable_writable():
lockdown_context.role = None
lockdown_context.user = None
lockdown_context.group = None
rest_api = Role('rest_api')
ldown = rest_api.lockdown(Bicycle) \
.readable_by(Bicycle.group == ContextParam('group')) \
.writeable_by(Bicycle.owner == ContextParam('user'))
with test_database(test_db, [User, Group, Bicycle]):
u = User.create(username='******')
g = Group.create(name='test')
b = Bicycle.create(owner=u, group=g)
assert b.is_readable() is True
assert b.is_writable() is True
lockdown_context.role = rest_api
assert b.is_readable() is False
assert b.is_writable() is False
lockdown_context.group = 10
lockdown_context.user = 10
assert b.is_readable() is False
assert b.is_writable() is False
lockdown_context.group = g.id
assert b.is_readable() is True
assert b.is_writable() is False
lockdown_context.user = u.id
assert b.is_readable() is True
assert b.is_writable() is True
def test_no_one():
lockdown_context.role = None
lockdown_context.user = None
lockdown_context.group = None
rest_api = Role('rest_api')
rest_api.lockdown(Bicycle).readable_by(NO_ONE).writeable_by(NO_ONE)
with test_database(test_db, [User, Group, Bicycle]):
u = User.create(username='******')
g = Group.create(name='test')
b = Bicycle.create(owner=u, group=g)
lockdown_context.role = rest_api
assert b.is_readable() is False
assert b.is_writable() is False
lockdown_context.group = g.id
lockdown_context.user = u.id
assert b.is_readable() is False
assert b.is_writable() is False
def test_getattr_dict() -> None:
c = OmegaConf.create("a: {b: 1}")
assert isinstance(c, DictConfig)
assert {"b": 1} == c.a
@mark.parametrize("struct", [False, True])
@mark.parametrize(
"cfg",
[
param({
"name": "alice",
"age": 1
}, id="dict"),
param(User(name="alice", age=1), id="structured_config"),
],
)
def test_delattr(cfg: Any, struct: bool) -> None:
cfg = OmegaConf.create(cfg)
OmegaConf.set_struct(cfg, struct)
delattr(cfg, "name")
assert cfg == {"age": 1}
with raises(ConfigAttributeError):
delattr(cfg, "c")
@mark.parametrize(
"key,match",
[
param("a", "a", id="str"),
class TestConfigs:
def test_nested_config_is_none(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.NestedWithNone)
assert cfg == {"plugin": None}
assert OmegaConf.get_type(cfg, "plugin") is None
assert _utils.get_ref_type(cfg, "plugin") == Optional[module.Plugin]
def test_nested_config(self, class_type: str) -> None:
module: Any = import_module(class_type)
def validate(cfg: DictConfig) -> None:
assert cfg == {
"default_value": {
"with_default": 10,
"null_default": None,
"mandatory_missing": "???",
"interpolation": "${value_at_root}",
},
"user_provided_default": {
"with_default": 42,
"null_default": None,
"mandatory_missing": "???",
"interpolation": "${value_at_root}",
},
"value_at_root": 1000,
}
with pytest.raises(ValidationError):
cfg.user_provided_default = 10
with pytest.raises(ValidationError):
cfg.default_value = 10
# assign subclass
cfg.default_value = module.NestedSubclass()
assert cfg.default_value == {
"additional": 20,
"with_default": 10,
"null_default": None,
"mandatory_missing": "???",
"interpolation": "${value_at_root}",
}
# assign original ref type back
cfg.default_value = module.Nested()
assert cfg.default_value == module.Nested()
conf1 = OmegaConf.structured(
module.NestedConfig(default_value=module.Nested()))
validate(conf1)
def test_nested_config2(self, class_type: str) -> None:
module: Any = import_module(class_type)
def validate(cfg: DictConfig) -> None:
assert cfg == {
"default_value": "???",
"user_provided_default": {
"with_default": 42,
"null_default": None,
"mandatory_missing": "???",
"interpolation": "${value_at_root}",
},
"value_at_root": 1000,
}
with pytest.raises(ValidationError):
cfg.user_provided_default = 10
with pytest.raises(ValidationError):
cfg.default_value = 10
# assign subclass
cfg.default_value = module.NestedSubclass()
assert cfg.default_value == {
"additional": 20,
"with_default": 10,
"null_default": None,
"mandatory_missing": "???",
"interpolation": "${value_at_root}",
}
# assign original ref type back
cfg.default_value = module.Nested()
assert cfg.default_value == module.Nested()
conf1 = OmegaConf.structured(module.NestedConfig)
validate(conf1)
def test_value_without_a_default(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.NoDefaultValue)
assert OmegaConf.is_missing(cfg, "no_default")
OmegaConf.structured(module.NoDefaultValue(no_default=10)) == {
"no_default": 10
}
def test_union_errors(self, class_type: str) -> None:
module: Any = import_module(class_type)
with pytest.raises(ValueError):
OmegaConf.structured(module.UnionError)
def test_config_with_list(self, class_type: str) -> None:
module: Any = import_module(class_type)
def validate(cfg: DictConfig) -> None:
assert cfg == {
"list1": [1, 2, 3],
"list2": [1, 2, 3],
"missing": MISSING
}
with pytest.raises(ValidationError):
cfg.list1[1] = "foo"
assert OmegaConf.is_missing(cfg, "missing")
conf1 = OmegaConf.structured(module.ConfigWithList)
validate(conf1)
conf1 = OmegaConf.structured(module.ConfigWithList())
validate(conf1)
def test_assignment_to_nested_structured_config(self,
class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.structured(module.NestedConfig)
with pytest.raises(ValidationError):
conf.default_value = 10
conf.default_value = module.Nested()
def test_assignment_to_structured_inside_dict_config(
self, class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.create({"val": module.Nested})
with pytest.raises(ValidationError):
conf.val = 10
def test_config_with_dict(self, class_type: str) -> None:
module: Any = import_module(class_type)
def validate(cfg: DictConfig) -> None:
assert cfg == {"dict1": {"foo": "bar"}, "missing": MISSING}
assert OmegaConf.is_missing(cfg, "missing")
conf1 = OmegaConf.structured(module.ConfigWithDict)
validate(conf1)
conf1 = OmegaConf.structured(module.ConfigWithDict())
validate(conf1)
def test_structured_config_struct_behavior(self, class_type: str) -> None:
module: Any = import_module(class_type)
def validate(cfg: DictConfig) -> None:
assert not OmegaConf.is_struct(cfg)
with pytest.raises(AttributeError):
# noinspection PyStatementEffect
cfg.foo
cfg.dict1.foo = 10
assert cfg.dict1.foo == 10
# setting struct False on a specific typed node opens it up even though it's
# still typed
OmegaConf.set_struct(cfg, False)
cfg.foo = 20
assert cfg.foo == 20
conf = OmegaConf.structured(module.ConfigWithDict)
validate(conf)
conf = OmegaConf.structured(module.ConfigWithDict())
validate(conf)
@pytest.mark.parametrize(
"tested_type,assignment_data, init_dict",
[
# Use class to build config
("BoolConfig", BoolConfigAssignments, {}),
("IntegersConfig", IntegersConfigAssignments, {}),
("FloatConfig", FloatConfigAssignments, {}),
("StringConfig", StringConfigAssignments, {}),
("EnumConfig", EnumConfigAssignments, {}),
# Use instance to build config
("BoolConfig", BoolConfigAssignments, {
"with_default": False
}),
("IntegersConfig", IntegersConfigAssignments, {
"with_default": 42
}),
("FloatConfig", FloatConfigAssignments, {
"with_default": 42.0
}),
("StringConfig", StringConfigAssignments, {
"with_default": "fooooooo"
}),
("EnumConfig", EnumConfigAssignments, {
"with_default": Color.BLUE
}),
("AnyTypeConfig", AnyTypeConfigAssignments, {}),
],
)
def test_field_with_default_value(
self,
class_type: str,
tested_type: str,
init_dict: Dict[str, Any],
assignment_data: Any,
) -> None:
module: Any = import_module(class_type)
input_class = getattr(module, tested_type)
def validate(input_: Any, expected: Any) -> None:
conf = OmegaConf.structured(input_)
# Test access
assert conf.with_default == expected.with_default
assert conf.null_default is None
# Test that accessing a variable without a default value
# results in a MissingMandatoryValue exception
with pytest.raises(MissingMandatoryValue):
# noinspection PyStatementEffect
conf.mandatory_missing
# Test interpolation preserves type and value
assert type(conf.with_default) == type(
conf.interpolation) # noqa E721
assert conf.with_default == conf.interpolation
# Test that assignment of illegal values
for illegal_value in assignment_data.illegal:
with pytest.raises(ValidationError):
conf.with_default = illegal_value
with pytest.raises(ValidationError):
conf.null_default = illegal_value
with pytest.raises(ValidationError):
conf.mandatory_missing = illegal_value
# Test assignment of legal values
for legal_value in assignment_data.legal:
expected_data = legal_value
if isinstance(legal_value, tuple):
expected_data = legal_value[1]
legal_value = legal_value[0]
conf.with_default = legal_value
conf.null_default = legal_value
conf.mandatory_missing = legal_value
msg = "Error: {} : {}".format(input_class.__name__,
legal_value)
assert conf.with_default == expected_data, msg
assert conf.null_default == expected_data, msg
assert conf.mandatory_missing == expected_data, msg
validate(input_class, input_class())
validate(input_class(**init_dict), input_class(**init_dict))
@pytest.mark.parametrize(
"input_init, expected_init",
[
# attr class as class
(None, {}),
# attr class object with custom values
({
"int_default": 30
}, {
"int_default": 30
}),
# dataclass as class
(None, {}),
# dataclass as object with custom values
({
"int_default": 30
}, {
"int_default": 30
}),
],
)
def test_untyped(self, class_type: str, input_init: Any,
expected_init: Any) -> None:
module: Any = import_module(class_type)
input_ = module.AnyTypeConfig
expected = input_(**expected_init)
if input_init is not None:
input_ = input_(**input_init)
conf = OmegaConf.structured(input_)
assert conf.null_default == expected.null_default
assert conf.int_default == expected.int_default
assert conf.float_default == expected.float_default
assert conf.str_default == expected.str_default
assert conf.bool_default == expected.bool_default
# yes, this is weird.
assert "mandatory_missing" in conf.keys(
) and "mandatory_missing" not in conf
with pytest.raises(MissingMandatoryValue):
# noinspection PyStatementEffect
conf.mandatory_missing
assert type(conf._get_node("null_default")) == AnyNode
assert type(conf._get_node("int_default")) == AnyNode
assert type(conf._get_node("float_default")) == AnyNode
assert type(conf._get_node("str_default")) == AnyNode
assert type(conf._get_node("bool_default")) == AnyNode
assert type(conf._get_node("mandatory_missing")) == AnyNode
assert conf.int_default == expected.int_default
with pytest.raises(ValidationError):
conf.typed_int_default = "foo"
values = [10, True, False, None, 1.0, -1.0, "10", float("inf")]
for val in values:
conf.null_default = val
conf.int_default = val
conf.float_default = val
conf.str_default = val
conf.bool_default = val
assert conf.null_default == val
assert conf.int_default == val
assert conf.float_default == val
assert conf.str_default == val
assert conf.bool_default == val
def test_interpolation(self, class_type: str) -> Any:
module: Any = import_module(class_type)
input_ = module.Interpolation()
conf = OmegaConf.structured(input_)
assert conf.x == input_.x
assert conf.z1 == conf.x
assert conf.z2 == f"{conf.x}_{conf.y}"
assert type(conf.x) == int
assert type(conf.y) == int
assert type(conf.z1) == int
assert type(conf.z2) == str
@pytest.mark.parametrize(
"tested_type",
[
"BoolOptional",
"IntegerOptional",
"FloatOptional",
"StringOptional",
"ListOptional",
"TupleOptional",
"EnumOptional",
"StructuredOptional",
"DictOptional",
],
)
def test_optional(self, class_type: str, tested_type: str) -> None:
module: Any = import_module(class_type)
input_ = getattr(module, tested_type)
obj = input_()
conf = OmegaConf.structured(input_)
# verify non-optional fields are rejecting None
with pytest.raises(ValidationError):
conf.not_optional = None
assert conf.as_none is None
assert conf.with_default == obj.with_default
# assign None to an optional field
conf.with_default = None
assert conf.with_default is None
def test_list_field(self, class_type: str) -> None:
module: Any = import_module(class_type)
input_ = module.WithListField
conf = OmegaConf.structured(input_)
with pytest.raises(ValidationError):
conf.list[0] = "fail"
with pytest.raises(ValidationError):
conf.list.append("fail")
with pytest.raises(ValidationError):
cfg2 = OmegaConf.create({"list": ["fail"]})
OmegaConf.merge(conf, cfg2)
def test_dict_field(self, class_type: str) -> None:
module: Any = import_module(class_type)
input_ = module.WithDictField
conf = OmegaConf.structured(input_)
with pytest.raises(ValidationError):
conf.dict["foo"] = "fail"
with pytest.raises(ValidationError):
OmegaConf.merge(conf, OmegaConf.create({"dict": {"foo": "fail"}}))
@pytest.mark.skipif(sys.version_info < (3, 8),
reason="requires Python 3.8 or newer")
def test_typed_dict_field(self, class_type: str) -> None:
module: Any = import_module(class_type)
input_ = module.WithTypedDictField
conf = OmegaConf.structured(input_(dict={"foo": 10}))
assert conf.dict["foo"] == 10
# typed dicts does not currently runtime type safety.
conf = OmegaConf.merge(conf, {"dict": {"foo": "not_failed"}})
assert conf.dict["foo"] == "not_failed"
def test_merged_type1(self, class_type: str) -> None:
# Test that the merged type is that of the last merged config
module: Any = import_module(class_type)
input_ = module.WithDictField
conf = OmegaConf.structured(input_)
res = OmegaConf.merge(OmegaConf.create(), conf)
assert OmegaConf.get_type(res) == input_
def test_merged_type2(self, class_type: str) -> None:
# Test that the merged type is that of the last merged config
module: Any = import_module(class_type)
input_ = module.WithDictField
conf = OmegaConf.structured(input_)
res = OmegaConf.merge(conf, {"dict": {"foo": 99}})
assert OmegaConf.get_type(res) == input_
def test_merged_with_subclass(self, class_type: str) -> None:
# Test that the merged type is that of the last merged config
module: Any = import_module(class_type)
c1 = OmegaConf.structured(module.Plugin)
c2 = OmegaConf.structured(module.ConcretePlugin)
res = OmegaConf.merge(c1, c2)
assert OmegaConf.get_type(res) == module.ConcretePlugin
def test_merge_missing_structured_on_self(self, class_type: str) -> None:
module: Any = import_module(class_type)
c1 = OmegaConf.structured(module.MissingStructuredConfigField)
assert OmegaConf.is_missing(c1, "plugin")
c2 = OmegaConf.merge(c1, module.MissingStructuredConfigField)
assert OmegaConf.is_missing(c2, "plugin")
def test_merge_missing_structured_config_is_missing(
self, class_type: str) -> None:
module: Any = import_module(class_type)
c1 = OmegaConf.structured(module.MissingStructuredConfigField)
assert OmegaConf.is_missing(c1, "plugin")
def test_merge_missing_structured(self, class_type: str) -> None:
# Test that the merged type is that of the last merged config
module: Any = import_module(class_type)
c1 = OmegaConf.create({"plugin": "???"})
c2 = OmegaConf.merge(c1, module.MissingStructuredConfigField)
assert OmegaConf.is_missing(c2, "plugin")
def test_merge_none_is_none(self, class_type: str) -> None:
# Test that the merged type is that of the last merged config
module: Any = import_module(class_type)
c1 = OmegaConf.structured(module.StructuredOptional)
assert c1.with_default == module.Nested()
c2 = OmegaConf.merge(c1, {"with_default": None})
assert OmegaConf.is_none(c2, "with_default")
def test_merge_with_subclass_into_missing(self, class_type: str) -> None:
module: Any = import_module(class_type)
base = OmegaConf.structured(module.PluginHolder)
assert _utils.get_ref_type(base, "missing") == module.Plugin
assert OmegaConf.get_type(base, "missing") is None
res = OmegaConf.merge(base, {"missing": module.Plugin})
assert OmegaConf.get_type(res) == module.PluginHolder
assert _utils.get_ref_type(base, "missing") == module.Plugin
assert OmegaConf.get_type(res, "missing") == module.Plugin
def test_merged_with_nons_subclass(self, class_type: str) -> None:
module: Any = import_module(class_type)
c1 = OmegaConf.structured(module.Plugin)
c2 = OmegaConf.structured(module.FaultyPlugin)
with pytest.raises(ValidationError):
OmegaConf.merge(c1, c2)
def test_merge_into_Dict(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.DictExamples)
res = OmegaConf.merge(cfg, {"strings": {"x": "abc"}})
assert res.strings == {"a": "foo", "b": "bar", "x": "abc"}
def test_merge_user_list_with_wrong_key(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.UserList)
with pytest.raises(ConfigKeyError):
OmegaConf.merge(cfg, {"list": [{"foo": "var"}]})
def test_merge_list_with_correct_type(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.UserList)
user = module.User(name="John", age=21)
res = OmegaConf.merge(cfg, {"list": [user]})
assert res.list == [user]
def test_merge_dict_with_wrong_type(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.UserDict)
with pytest.raises(ValidationError):
OmegaConf.merge(cfg, {"dict": {"foo": "var"}})
def test_merge_dict_with_correct_type(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.UserDict)
user = module.User(name="John", age=21)
res = OmegaConf.merge(cfg, {"dict": {"foo": user}})
assert res.dict == {"foo": user}
def test_dict_field_key_type_error(self, class_type: str) -> None:
module: Any = import_module(class_type)
input_ = module.ErrorDictObjectKey
with pytest.raises(KeyValidationError):
OmegaConf.structured(input_)
def test_dict_field_value_type_error(self, class_type: str) -> None:
module: Any = import_module(class_type)
input_ = module.ErrorDictUnsupportedValue
with pytest.raises(ValidationError):
OmegaConf.structured(input_)
def test_list_field_value_type_error(self, class_type: str) -> None:
module: Any = import_module(class_type)
input_ = module.ErrorListUnsupportedValue
with pytest.raises(ValidationError):
OmegaConf.structured(input_)
@pytest.mark.parametrize("example", ["ListExamples", "TupleExamples"])
def test_list_examples(self, class_type: str, example: str) -> None:
module: Any = import_module(class_type)
input_ = getattr(module, example)
conf = OmegaConf.structured(input_)
def test_any(name: str) -> None:
conf[name].append(True)
conf[name].extend([Color.RED, 3.1415])
conf[name][2] = False
assert conf[name] == [1, "foo", False, Color.RED, 3.1415]
# any and untyped
test_any("any")
# test ints
with pytest.raises(ValidationError):
conf.ints[0] = "foo"
conf.ints.append(10)
assert conf.ints == [1, 2, 10]
# test strings
conf.strings.append(Color.BLUE)
assert conf.strings == ["foo", "bar", "Color.BLUE"]
# test booleans
with pytest.raises(ValidationError):
conf.booleans[0] = "foo"
conf.booleans.append(True)
conf.booleans.append("off")
conf.booleans.append(1)
assert conf.booleans == [True, False, True, False, True]
# test colors
with pytest.raises(ValidationError):
conf.colors[0] = "foo"
conf.colors.append(Color.BLUE)
conf.colors.append("RED")
conf.colors.append("Color.GREEN")
conf.colors.append(3)
assert conf.colors == [
Color.RED,
Color.GREEN,
Color.BLUE,
Color.RED,
Color.GREEN,
Color.BLUE,
]
def test_dict_examples(self, class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.structured(module.DictExamples)
def test_any(name: str) -> None:
conf[name].c = True
conf[name].d = Color.RED
conf[name].e = 3.1415
assert conf[name] == {
"a": 1,
"b": "foo",
"c": True,
"d": Color.RED,
"e": 3.1415,
}
# any and untyped
test_any("any")
# test ints
with pytest.raises(ValidationError):
conf.ints.a = "foo"
conf.ints.c = 10
assert conf.ints == {"a": 10, "b": 20, "c": 10}
# test strings
conf.strings.c = Color.BLUE
assert conf.strings == {"a": "foo", "b": "bar", "c": "Color.BLUE"}
# tests booleans
with pytest.raises(ValidationError):
conf.booleans.a = "foo"
conf.booleans.c = True
conf.booleans.d = "off"
conf.booleans.e = 1
assert conf.booleans == {
"a": True,
"b": False,
"c": True,
"d": False,
"e": True,
}
# test colors
with pytest.raises(ValidationError):
conf.colors.foo = "foo"
conf.colors.c = Color.BLUE
conf.colors.d = "RED"
conf.colors.e = "Color.GREEN"
conf.colors.f = 3
assert conf.colors == {
"red": Color.RED,
"green": Color.GREEN,
"blue": Color.BLUE,
"c": Color.BLUE,
"d": Color.RED,
"e": Color.GREEN,
"f": Color.BLUE,
}
# test int_keys
with pytest.raises(KeyValidationError):
conf.int_keys.foo_key = "foo_value"
conf.int_keys[3] = "three"
assert conf.int_keys == {
1: "one",
2: "two",
3: "three",
}
def test_enum_key(self, class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.structured(module.DictWithEnumKeys)
# When an Enum is a dictionary key the name of the Enum is actually used
# as the key
assert conf.enum_key.RED == "red"
assert conf.enum_key["GREEN"] == "green"
assert conf.enum_key[Color.GREEN] == "green"
conf.enum_key["BLUE"] = "Blue too"
assert conf.enum_key[Color.BLUE] == "Blue too"
with pytest.raises(KeyValidationError):
conf.enum_key["error"] = "error"
def test_dict_of_objects(self, class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.structured(module.DictOfObjects)
assert conf.users.joe.age == 18
assert conf.users.joe.name == "Joe"
conf.users.bond = module.User(name="James Bond", age=7)
assert conf.users.bond.name == "James Bond"
assert conf.users.bond.age == 7
with pytest.raises(ValidationError):
conf.users.fail = "fail"
def test_list_of_objects(self, class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.structured(module.ListOfObjects)
assert conf.users[0].age == 18
assert conf.users[0].name == "Joe"
conf.users.append(module.User(name="James Bond", age=7))
assert conf.users[1].name == "James Bond"
assert conf.users[1].age == 7
with pytest.raises(ValidationError):
conf.users.append("fail")
def test_promote_api(self, class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.create(module.AnyTypeConfig)
conf._promote(None)
assert conf == OmegaConf.create(module.AnyTypeConfig)
with pytest.raises(ValueError):
conf._promote(42)
assert conf == OmegaConf.create(module.AnyTypeConfig)
def test_promote_to_class(self, class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.create(module.AnyTypeConfig)
assert OmegaConf.get_type(conf) == module.AnyTypeConfig
conf._promote(module.BoolConfig)
assert OmegaConf.get_type(conf) == module.BoolConfig
assert conf.with_default is True
assert conf.null_default is None
assert OmegaConf.is_missing(conf, "mandatory_missing")
def test_promote_to_object(self, class_type: str) -> None:
module: Any = import_module(class_type)
conf = OmegaConf.create(module.AnyTypeConfig)
assert OmegaConf.get_type(conf) == module.AnyTypeConfig
conf._promote(module.BoolConfig(with_default=False))
assert OmegaConf.get_type(conf) == module.BoolConfig
assert conf.with_default is False
def test_set_key_with_with_dataclass(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.create({"foo": [1, 2]})
cfg.foo = module.ListClass()
def test_set_list_correct_type(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.ListClass)
value = [1, 2, 3]
cfg.list = value
cfg.tuple = value
assert cfg.list == value
assert cfg.tuple == value
@pytest.mark.parametrize(
"value", [1, True, "str", 3.1415, ["foo", True, 1.2],
User()])
def test_assign_wrong_type_to_list(self, class_type: str,
value: Any) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.ListClass)
with pytest.raises(ValidationError):
cfg.list = value
with pytest.raises(ValidationError):
cfg.tuple = value
assert cfg == OmegaConf.structured(module.ListClass)
@pytest.mark.parametrize(
"value",
[
1,
True,
"str",
3.1415,
["foo", True, 1.2],
{
"foo": True
},
User(age=1, name="foo"),
{
"user": User(age=1, name="foo")
},
ListConfig(content=[1, 2], ref_type=List[int], element_type=int),
],
)
def test_assign_wrong_type_to_dict(self, class_type: str,
value: Any) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.ConfigWithDict2)
with pytest.raises(ValidationError):
cfg.dict1 = value
assert cfg == OmegaConf.structured(module.ConfigWithDict2)
def test_recursive_dict(self, class_type: str) -> None:
module: Any = import_module(class_type)
rd = module.RecursiveDict
o = rd(d={"a": rd(), "b": rd()})
cfg = OmegaConf.structured(o)
assert cfg == {
"d": {
"a": {
"d": "???"
},
"b": {
"d": "???"
},
}
}
def test_recursive_list(self, class_type: str) -> None:
module: Any = import_module(class_type)
rl = module.RecursiveList
o = rl(d=[rl(), rl()])
cfg = OmegaConf.structured(o)
assert cfg == {"d": [{"d": "???"}, {"d": "???"}]}
def test_create_untyped_dict(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.UntypedDict)
dt = Dict[Union[str, Enum], Any]
assert _utils.get_ref_type(cfg, "dict") == dt
assert _utils.get_ref_type(cfg, "opt_dict") == Optional[dt]
assert cfg.dict == {"foo": "var"}
assert cfg.opt_dict is None
def test_create_untyped_list(self, class_type: str) -> None:
module: Any = import_module(class_type)
cfg = OmegaConf.structured(module.UntypedList)
assert _utils.get_ref_type(cfg, "list") == List[Any]
assert _utils.get_ref_type(cfg, "opt_list") == Optional[List[Any]]
assert cfg.list == [1, 2]
assert cfg.opt_list is None
c = OmegaConf.create({"foo": "bar"})
with pytest.warns(
expected_warning=UserWarning,
match=re.escape(
dedent(
"""\
cfg.pretty() is deprecated and will be removed in a future version.
Use OmegaConf.to_yaml(cfg)
""", )),
):
assert c.pretty() == "foo: bar\n"
@pytest.mark.parametrize(
"user",
[
User(name="Bond", age=7),
OmegaConf.structured(User(name="Bond", age=7)),
{
"name": "Bond",
"age": 7
},
],
)
def test_structured_configs(user: User) -> None:
expected = dedent("""\
name: Bond
age: 7
""")
assert OmegaConf.to_yaml(user) == expected
elif isinstance(item, dict):
for _k, v in item.items():
assert_container_with_primitives(v)
else:
assert isinstance(item, (int, float, str, bool, type(None), Enum))
c = OmegaConf.create(input_)
res = OmegaConf.to_container(c, resolve=True)
assert_container_with_primitives(res)
@mark.parametrize(
"src,ex_dict,ex_dict_config,key",
[
param(
{"user": User(age=7, name="Bond")},
{"user": {"name": "Bond", "age": 7}},
{"user": User(age=7, name="Bond")},
"user",
id="structured-inside-dict",
),
param(
[1, User(age=7, name="Bond")],
[1, {"name": "Bond", "age": 7}],
[1, User(age=7, name="Bond")],
1,
id="structured-inside-list",
),
],
)
class TestSCMode:
),
pytest.param(
({"v": 10, "n": {"a": 20}}, {"v": "${n}"}),
{"v": {"a": 20}, "n": {"a": 20}},
id="inter:node_inter_over_data",
),
pytest.param(
(
{"n": {"a": 10}, "i": "${n}"},
{"i": {"b": 20}},
),
{"n": {"a": 10}, "i": {"a": 10, "b": 20}},
id="inter:node_over_node_interpolation",
),
# Structured configs
(({"user": User}, {}), {"user": User(name=MISSING, age=MISSING)}),
(({"user": User}, {"user": {}}), {"user": User(name=MISSING, age=MISSING)}),
(
({"user": User}, {"user": {"name": "Joe"}}),
{"user": User(name="Joe", age=MISSING)},
),
(
({"user": User}, {"user": {"name": "Joe", "age": 10}}),
{"user": User(name="Joe", age=10)},
),
([{"users": Users}], {"users": {"name2user": {}}}),
([Users], {"name2user": {}}),
([Users, {"name2user": {}}], {"name2user": {}}),
(
[Users, {"name2user": {"joe": User}}],
{"name2user": {"joe": {"name": MISSING, "age": MISSING}}},
# In python 3.6+ insert order changes iteration order. this ensures that equality is preserved.
(dict(a=1, b=2, c=3, d=4, e=5), dict(e=5, b=2, c=3, d=4, a=1)),
(DictConfig(content=None), DictConfig(content=None)),
pytest.param({"a": [1, 2]}, {"a": [1, 2]}, id="list_in_dict"),
# With interpolations
([10, "${0}"], [10, 10]),
(dict(a=12, b="${a}"), dict(a=12, b=12)),
# With missing interpolation
pytest.param([10, "${0}"], [10, 10], id="list_simple_interpolation"),
pytest.param(
{"a": "${ref_error}"}, {"a": "${ref_error}"}, id="dict==dict,ref_error"
),
pytest.param({"a": "???"}, {"a": "???"}, id="dict==dict,missing"),
pytest.param(User, User, id="User==User"),
pytest.param(
{"name": "poo", "age": 7}, User(name="poo", age=7), id="dict==User"
),
pytest.param(Group, Group, id="Group==Group"),
pytest.param({"group": {"admin": None}}, {"group": Group}, id="dict==Group"),
pytest.param(
{"i1": "${n1}", "n1": {"a": 10}},
{"i1": "${n1}", "n1": {"a": 10}},
id="node_interpolation",
),
# Inter containers
pytest.param(
{"foo": DictConfig(content="${bar}"), "bar": 10},
{"foo": 10, "bar": 10},
id="dictconfig_inter",
),
pytest.param(
pytest.param(
Expected(
create=lambda: OmegaConf.structured(SubscriptedDict),
op=lambda cfg: cfg.__setitem__("dict", 1),
exception_type=ValidationError,
msg="Cannot assign int to Dict[str, int]",
key="dict",
ref_type=Optional[Dict[str, int]],
low_level=True,
),
id="DictConfig[str,int]:assigned_primitive_type",
),
pytest.param(
Expected(
create=lambda: OmegaConf.structured(SubscriptedDict),
op=lambda cfg: cfg.__setitem__("dict", User(age=2, name="bar")),
exception_type=ValidationError,
msg="Cannot assign User to Dict[str, int]",
key="dict",
ref_type=Optional[Dict[str, int]],
low_level=True,
),
id="DictConfig[str,int]:assigned_structured_config",
),
# delete
pytest.param(
Expected(
create=lambda: create_readonly({"foo": "bar"}),
op=lambda cfg: cfg.__delitem__("foo"),
exception_type=ReadonlyConfigError,
msg="DictConfig in read-only mode does not support deletion",
x_node = cfg._get_node("x")
assert isinstance(x_node, Node)
assert x_node._dereference_node(throw_on_resolution_failure=False) is None
def test_none_value_in_quoted_string(restore_resolvers: Any) -> None:
OmegaConf.register_new_resolver("test", lambda x: x)
cfg = OmegaConf.create({"x": "${test:'${missing}'}", "missing": None})
assert cfg.x == "None"
@mark.parametrize(
("cfg", "key", "expected_value", "expected_node_type"),
[
param(
User(name="Bond", age=SI("${cast:int,'7'}")),
"age",
7,
IntegerNode,
id="expected_type",
),
param(
# This example specifically test the case where intermediate resolver results
# cannot be cast to the same type as the key.
User(name="Bond", age=SI("${cast:int,${drop_last:${drop_last:7xx}}}")),
"age",
7,
IntegerNode,
id="intermediate_type_mismatch_ok",
),
param(
(DictConfig(content=None), DictConfig(content=None)),
param({"a": [1, 2]}, {"a": [1, 2]}, id="list_in_dict"),
# With interpolations
([10, "${0}"], [10, 10]),
(dict(a=12, b="${a}"), dict(a=12, b=12)),
# With missing interpolation
param([10, "${0}"], [10, 10], id="list_simple_interpolation"),
param({"a": "${ref_error}"}, {"a": "${ref_error}"},
id="dict==dict,ref_error"),
param({"a": "???"}, {"a": "???"}, id="dict==dict,missing"),
param(User, User, id="User==User"),
param({
"name": "poo",
"age": 7
},
User(name="poo", age=7),
id="dict==User"),
param(Group, Group, id="Group==Group"),
param({"group": {
"admin": None
}}, {"group": Group}, id="dict==Group"),
param(
{
"i1": "${n1}",
"n1": {
"a": 10
}
},
{
"i1": "${n1}",
"n1": {
elif isinstance(item, dict):
for _k, v in item.items():
assert_container_with_primitives(v)
else:
assert isinstance(item, (int, float, str, bool, type(None), Enum))
c = OmegaConf.create(input_)
res = OmegaConf.to_container(c, resolve=True)
assert_container_with_primitives(res)
@pytest.mark.parametrize(
"cfg,ex_false,ex_true",
[
pytest.param(
{"user": User(age=7, name="Bond")},
{"user": {
"name": "Bond",
"age": 7
}},
{"user": User(age=7, name="Bond")},
),
pytest.param(
[1, User(age=7, name="Bond")],
[1, {
"name": "Bond",
"age": 7
}],
[1, User(age=7, name="Bond")],
),
pytest.param(
True,
id="opt_dict_elt"),
param(SubscriptedListOpt, "opt_list", None, False, id="opt_list"),
param(SubscriptedDictOpt, "opt_dict", None, False, id="opt_dict"),
param(SubscriptedListOpt, "list_opt", [None], False,
id="list_opt_elt"),
param(SubscriptedDictOpt,
"dict_opt", {"key": None},
False,
id="dict_opt_elt"),
param(SubscriptedListOpt, "list_opt", None, True, id="list_opt"),
param(SubscriptedDictOpt, "dict_opt", None, True, id="dict_opt"),
param(
ListConfig([None], element_type=Optional[User]),
0,
User("Bond", 7),
False,
id="set_optional_user",
),
param(
ListConfig([User], element_type=User),
0,
None,
True,
id="illegal_set_user_to_none",
),
],
)
def test_optional_assign(cls: Any, key: str, assignment: Any,
error: bool) -> None:
cfg = OmegaConf.structured(cls)
cfg = OmegaConf.create({"a": {"b": 10}})
OmegaConf.update(cfg, "a", {"c": 20})
assert cfg == {"a": {"b": 10, "c": 20}}
@mark.parametrize(
"cfg,key,value,expected",
[
param({}, "a", 10, {"a": 10}, id="add_value"),
param({}, "a.b", 10, {"a": {"b": 10}}, id="add_value"),
param({}, "a", {"b": 10}, {"a": {"b": 10}}, id="add_dict"),
param({}, "a.b", {"c": 10}, {"a": {"b": {"c": 10}}}, id="add_dict"),
param({}, "a", [1, 2], {"a": [1, 2]}, id="add_list"),
param({}, "a.b", [1, 2], {"a": {"b": [1, 2]}}, id="add_list"),
param(
{"user": User(name="Bond", age=7)},
"user.location",
"London",
{"user": {"name": "Bond", "age": 7, "location": "London"}},
id="inserting_into_nested_structured_config",
),
],
)
def test_update_force_add(cfg: Any, key: str, value: Any, expected: Any) -> None:
cfg = _ensure_container(cfg)
OmegaConf.set_struct(cfg, True)
with raises((ConfigAttributeError, ConfigKeyError)): # type: ignore
OmegaConf.update(cfg, key, value, force_add=False)
OmegaConf.update(cfg, key, value, force_add=True)
ListConfig(element_type=Optional[int], content=[]),
123,
[123],
int,
id="optional_typed_list",
),
param(
ListConfig(element_type=Optional[int], content=[]),
None,
[None],
int,
id="optional_typed_list_append_none",
),
param(
ListConfig(element_type=User, content=[]),
User(name="bond"),
[User(name="bond")],
User,
id="user_list",
),
param(
ListConfig(element_type=User, content=[]),
None,
ValidationError,
None,
id="user_list_append_none",
),
param(
ListConfig(element_type=Optional[User], content=[]),
User(name="bond"),
[User(name="bond")],
{
"n": {
"a": 10
},
"i": {
"a": 10,
"b": 20
}
},
id="inter:node_over_node_interpolation",
),
# Structured configs
(({
"user": User
}, {}), {
"user": User(name=MISSING, age=MISSING)
}),
(({
"user": User
}, {
"user": {}
}), {
"user": User(name=MISSING, age=MISSING)
}),
(
({
"user": User
}, {
"user": {
"name": "Joe"
}
def test_type_validation_error_no_throw() -> None:
cfg = OmegaConf.structured(User(name="Bond", age=SI("${name}")))
bad_node = cfg._get_node("age")
assert bad_node._dereference_node(throw_on_resolution_failure=False) is None
