def test_set(): test_map = WhitelistMap() @_whitelist_for_serdes(whitelist_map=test_map) class HasSets(namedtuple("_HasSets", "reg_set frozen_set")): def __new__(cls, reg_set, frozen_set): set_param(reg_set, "reg_set") inst_param(frozen_set, "frozen_set", frozenset) return super(HasSets, cls).__new__(cls, reg_set, frozen_set) foo = HasSets({1, 2, 3, "3"}, frozenset([4, 5, 6, "6"])) serialized = _serialize_dagster_namedtuple(foo, whitelist_map=test_map) foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map) assert foo == foo_2 # verify that set elements are serialized in a consistent order so that # equal objects always have a consistent serialization / snapshot ID big_foo = HasSets(set(string.ascii_lowercase), frozenset(string.ascii_lowercase)) assert create_snapshot_id(big_foo) == create_snapshot_id( _deserialize_json_to_dagster_namedtuple(_serialize_dagster_namedtuple( big_foo, whitelist_map=test_map), whitelist_map=test_map))
def test_forward_compat_serdes_new_field_with_default(): test_map = WhitelistMap() @_whitelist_for_serdes(whitelist_map=test_map) class Quux(namedtuple("_Quux", "foo bar")): def __new__(cls, foo, bar): return super(Quux, cls).__new__(cls, foo, bar) # pylint: disable=bad-super-call assert test_map.has_tuple_entry("Quux") klass, _ = test_map.get_tuple_entry("Quux") assert klass is Quux quux = Quux("zip", "zow") serialized = _serialize_dagster_namedtuple(quux, whitelist_map=test_map) # pylint: disable=function-redefined @_whitelist_for_serdes(whitelist_map=test_map) class Quux(namedtuple("_Quux", "foo bar baz")): # pylint: disable=bad-super-call def __new__(cls, foo, bar, baz=None): return super(Quux, cls).__new__(cls, foo, bar, baz=baz) assert test_map.has_tuple_entry("Quux") klass, _ = test_map.get_tuple_entry("Quux") assert klass is Quux deserialized = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map) assert deserialized != quux assert deserialized.foo == quux.foo assert deserialized.bar == quux.bar assert deserialized.baz is None
def test_to_storage_value(): test_map = WhitelistMap() class MySerializer(DefaultNamedTupleSerializer): @staticmethod def value_to_storage_dict(value, whitelist_map): return DefaultNamedTupleSerializer.value_to_storage_dict( SubstituteAlphabet(value.a, value.b, value.c), test_map ) @_whitelist_for_serdes(whitelist_map=test_map, serializer=MySerializer) class DeprecatedAlphabet(namedtuple("_DeprecatedAlphabet", "a b c")): def __new__(cls, a, b, c): return super(DeprecatedAlphabet, cls).__new__(cls, a, b, c) @_whitelist_for_serdes(whitelist_map=test_map) class SubstituteAlphabet(namedtuple("_SubstituteAlphabet", "a b c")): def __new__(cls, a, b, c): return super(SubstituteAlphabet, cls).__new__(cls, a, b, c) nested = DeprecatedAlphabet(None, None, "_C") deprecated = DeprecatedAlphabet("A", "B", nested) serialized = _serialize_dagster_namedtuple(deprecated, whitelist_map=test_map) alphabet = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map) assert not isinstance(alphabet, DeprecatedAlphabet) assert isinstance(alphabet, SubstituteAlphabet) assert not isinstance(alphabet.c, DeprecatedAlphabet) assert isinstance(alphabet.c, SubstituteAlphabet)
def test_from_storage_dict(): test_map = WhitelistMap() class CompatSerializer(DefaultNamedTupleSerializer): @staticmethod def value_from_storage_dict(storage_dict, klass): return DeprecatedAlphabet.legacy_load(storage_dict) @_whitelist_for_serdes(whitelist_map=test_map, serializer=CompatSerializer) class DeprecatedAlphabet(namedtuple("_DeprecatedAlphabet", "a b c")): def __new__(cls, a, b, c): raise Exception("DeprecatedAlphabet is deprecated") @classmethod def legacy_load(cls, storage_dict): # instead of the DeprecatedAlphabet, directly invoke the namedtuple constructor return super().__new__( cls, storage_dict.get("a"), storage_dict.get("b"), storage_dict.get("c"), ) serialized = '{"__class__": "DeprecatedAlphabet", "a": "A", "b": "B", "c": "C"}' nt = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map) assert isinstance(nt, DeprecatedAlphabet)
def test_backward_compat_serdes(): test_map = WhitelistMap() @_whitelist_for_serdes(whitelist_map=test_map) class Quux(namedtuple("_Quux", "foo bar baz")): def __new__(cls, foo, bar, baz): return super(Quux, cls).__new__(cls, foo, bar, baz) # pylint: disable=bad-super-call quux = Quux("zip", "zow", "whoopie") serialized = _serialize_dagster_namedtuple(quux, whitelist_map=test_map) # pylint: disable=function-redefined @_whitelist_for_serdes(whitelist_map=test_map) class Quux(namedtuple("_Quux", "foo bar")): # pylint: disable=bad-super-call def __new__(cls, foo, bar): return super(Quux, cls).__new__(cls, foo, bar) deserialized = _deserialize_json_to_dagster_namedtuple( serialized, whitelist_map=test_map) assert deserialized != quux assert deserialized.foo == quux.foo assert deserialized.bar == quux.bar assert not hasattr(deserialized, "baz")
def test_long_int(): test_map = WhitelistMap() @_whitelist_for_serdes(whitelist_map=test_map) class NumHolder(NamedTuple): num: int x = NumHolder(98765432109876543210) ser_x = _serialize_dagster_namedtuple(x, test_map) roundtrip_x = _deserialize_json_to_dagster_namedtuple(ser_x, test_map) assert x.num == roundtrip_x.num
def test_persistent_tuple(): test_map = WhitelistMap() @_whitelist_for_serdes(whitelist_map=test_map) class Alphabet(namedtuple("_Alphabet", "a b c")): def __new__(cls, a, b, c): return super(Alphabet, cls).__new__(cls, a, b, c) foo = Alphabet(a="A", b="B", c="C") serialized = _serialize_dagster_namedtuple(foo, whitelist_map=test_map) foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map) assert foo == foo_2
def test_set(): test_map = WhitelistMap() @_whitelist_for_serdes(whitelist_map=test_map) class HasSets(namedtuple("_HasSets", "reg_set frozen_set")): def __new__(cls, reg_set, frozen_set): set_param(reg_set, "reg_set") inst_param(frozen_set, "frozen_set", frozenset) return super(HasSets, cls).__new__(cls, reg_set, frozen_set) foo = HasSets({1, 2, 3, "3"}, frozenset([4, 5, 6, "6"])) serialized = _serialize_dagster_namedtuple(foo, whitelist_map=test_map) foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=test_map) assert foo == foo_2