def test_set():
test_map = WhitelistMap()
@_whitelist_for_serdes(whitelist_map=test_map)
class HasSets(namedtuple("_HasSets", "reg_set frozen_set")):
def __new__(cls, reg_set, frozen_set):
set_param(reg_set, "reg_set")
inst_param(frozen_set, "frozen_set", frozenset)
return super(HasSets, cls).__new__(cls, reg_set, frozen_set)
foo = HasSets({1, 2, 3, "3"}, frozenset([4, 5, 6, "6"]))
serialized = _serialize_dagster_namedtuple(foo, whitelist_map=test_map)
foo_2 = _deserialize_json_to_dagster_namedtuple(serialized,
whitelist_map=test_map)
assert foo == foo_2
# verify that set elements are serialized in a consistent order so that
# equal objects always have a consistent serialization / snapshot ID
big_foo = HasSets(set(string.ascii_lowercase),
frozenset(string.ascii_lowercase))
assert create_snapshot_id(big_foo) == create_snapshot_id(
_deserialize_json_to_dagster_namedtuple(_serialize_dagster_namedtuple(
big_foo, whitelist_map=test_map),
whitelist_map=test_map))
def test_forward_compat_serdes_new_field_with_default():
test_map = WhitelistMap()
@_whitelist_for_serdes(whitelist_map=test_map)
class Quux(namedtuple("_Quux", "foo bar")):
def __new__(cls, foo, bar):
return super(Quux, cls).__new__(cls, foo, bar) # pylint: disable=bad-super-call
assert test_map.has_tuple_entry("Quux")
klass, _ = test_map.get_tuple_entry("Quux")
assert klass is Quux
quux = Quux("zip", "zow")
serialized = _serialize_dagster_namedtuple(quux, whitelist_map=test_map)
# pylint: disable=function-redefined
@_whitelist_for_serdes(whitelist_map=test_map)
class Quux(namedtuple("_Quux", "foo bar baz")): # pylint: disable=bad-super-call
def __new__(cls, foo, bar, baz=None):
return super(Quux, cls).__new__(cls, foo, bar, baz=baz)
assert test_map.has_tuple_entry("Quux")
klass, _ = test_map.get_tuple_entry("Quux")
assert klass is Quux
deserialized = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map)
assert deserialized != quux
assert deserialized.foo == quux.foo
assert deserialized.bar == quux.bar
assert deserialized.baz is None
def test_to_storage_value():
test_map = WhitelistMap()
class MySerializer(DefaultNamedTupleSerializer):
@staticmethod
def value_to_storage_dict(value, whitelist_map):
return DefaultNamedTupleSerializer.value_to_storage_dict(
SubstituteAlphabet(value.a, value.b, value.c), test_map
)
@_whitelist_for_serdes(whitelist_map=test_map, serializer=MySerializer)
class DeprecatedAlphabet(namedtuple("_DeprecatedAlphabet", "a b c")):
def __new__(cls, a, b, c):
return super(DeprecatedAlphabet, cls).__new__(cls, a, b, c)
@_whitelist_for_serdes(whitelist_map=test_map)
class SubstituteAlphabet(namedtuple("_SubstituteAlphabet", "a b c")):
def __new__(cls, a, b, c):
return super(SubstituteAlphabet, cls).__new__(cls, a, b, c)
nested = DeprecatedAlphabet(None, None, "_C")
deprecated = DeprecatedAlphabet("A", "B", nested)
serialized = _serialize_dagster_namedtuple(deprecated, whitelist_map=test_map)
alphabet = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map)
assert not isinstance(alphabet, DeprecatedAlphabet)
assert isinstance(alphabet, SubstituteAlphabet)
assert not isinstance(alphabet.c, DeprecatedAlphabet)
assert isinstance(alphabet.c, SubstituteAlphabet)
def test_from_storage_dict():
test_map = WhitelistMap()
class CompatSerializer(DefaultNamedTupleSerializer):
@staticmethod
def value_from_storage_dict(storage_dict, klass):
return DeprecatedAlphabet.legacy_load(storage_dict)
@_whitelist_for_serdes(whitelist_map=test_map, serializer=CompatSerializer)
class DeprecatedAlphabet(namedtuple("_DeprecatedAlphabet", "a b c")):
def __new__(cls, a, b, c):
raise Exception("DeprecatedAlphabet is deprecated")
@classmethod
def legacy_load(cls, storage_dict):
# instead of the DeprecatedAlphabet, directly invoke the namedtuple constructor
return super().__new__(
cls,
storage_dict.get("a"),
storage_dict.get("b"),
storage_dict.get("c"),
)
serialized = '{"__class__": "DeprecatedAlphabet", "a": "A", "b": "B", "c": "C"}'
nt = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map)
assert isinstance(nt, DeprecatedAlphabet)
def test_backward_compat_serdes():
test_map = WhitelistMap()
@_whitelist_for_serdes(whitelist_map=test_map)
class Quux(namedtuple("_Quux", "foo bar baz")):
def __new__(cls, foo, bar, baz):
return super(Quux, cls).__new__(cls, foo, bar, baz) # pylint: disable=bad-super-call
quux = Quux("zip", "zow", "whoopie")
serialized = _serialize_dagster_namedtuple(quux, whitelist_map=test_map)
# pylint: disable=function-redefined
@_whitelist_for_serdes(whitelist_map=test_map)
class Quux(namedtuple("_Quux", "foo bar")): # pylint: disable=bad-super-call
def __new__(cls, foo, bar):
return super(Quux, cls).__new__(cls, foo, bar)
deserialized = _deserialize_json_to_dagster_namedtuple(
serialized, whitelist_map=test_map)
assert deserialized != quux
assert deserialized.foo == quux.foo
assert deserialized.bar == quux.bar
assert not hasattr(deserialized, "baz")
def test_long_int():
test_map = WhitelistMap()
@_whitelist_for_serdes(whitelist_map=test_map)
class NumHolder(NamedTuple):
num: int
x = NumHolder(98765432109876543210)
ser_x = _serialize_dagster_namedtuple(x, test_map)
roundtrip_x = _deserialize_json_to_dagster_namedtuple(ser_x, test_map)
assert x.num == roundtrip_x.num
def test_persistent_tuple():
test_map = WhitelistMap()
@_whitelist_for_serdes(whitelist_map=test_map)
class Alphabet(namedtuple("_Alphabet", "a b c")):
def __new__(cls, a, b, c):
return super(Alphabet, cls).__new__(cls, a, b, c)
foo = Alphabet(a="A", b="B", c="C")
serialized = _serialize_dagster_namedtuple(foo, whitelist_map=test_map)
foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map)
assert foo == foo_2
def test_set():
test_map = WhitelistMap()
@_whitelist_for_serdes(whitelist_map=test_map)
class HasSets(namedtuple("_HasSets", "reg_set frozen_set")):
def __new__(cls, reg_set, frozen_set):
set_param(reg_set, "reg_set")
inst_param(frozen_set, "frozen_set", frozenset)
return super(HasSets, cls).__new__(cls, reg_set, frozen_set)
foo = HasSets({1, 2, 3, "3"}, frozenset([4, 5, 6, "6"]))
serialized = _serialize_dagster_namedtuple(foo, whitelist_map=test_map)
foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map)
assert foo == foo_2
