def test_to_storage_value(): _TEST_WHITELIST_MAP = _initial_whitelist_map() def whitelist_for_persistence(klass): return compose( _whitelist_for_persistence(whitelist_map=_TEST_WHITELIST_MAP), _whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP), )(klass) @whitelist_for_persistence class DeprecatedAlphabet(namedtuple("_DeprecatedAlphabet", "a b c"), Persistable): def __new__(cls, a, b, c): return super(DeprecatedAlphabet, cls).__new__(cls, a, b, c) def to_storage_value(self): return default_to_storage_value( SubstituteAlphabet(self.a, self.b, self.c), _TEST_WHITELIST_MAP ) @whitelist_for_persistence class SubstituteAlphabet(namedtuple("_SubstituteAlphabet", "a b c"), Persistable): def __new__(cls, a, b, c): return super(SubstituteAlphabet, cls).__new__(cls, a, b, c) nested = DeprecatedAlphabet(None, None, "_C") deprecated = DeprecatedAlphabet("A", "B", nested) serialized = _serialize_dagster_namedtuple(deprecated, whitelist_map=_TEST_WHITELIST_MAP) alphabet = _deserialize_json_to_dagster_namedtuple( serialized, whitelist_map=_TEST_WHITELIST_MAP ) assert not isinstance(alphabet, DeprecatedAlphabet) assert isinstance(alphabet, SubstituteAlphabet) assert not isinstance(alphabet.c, DeprecatedAlphabet) assert isinstance(alphabet.c, SubstituteAlphabet)
def test_forward_compat_serdes_new_field_with_default(): _TEST_WHITELIST_MAP = _initial_whitelist_map() @_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP) class Quux(namedtuple("_Quux", "foo bar")): def __new__(cls, foo, bar): return super(Quux, cls).__new__(cls, foo, bar) # pylint: disable=bad-super-call assert "Quux" in _TEST_WHITELIST_MAP["types"]["tuple"] assert _TEST_WHITELIST_MAP["types"]["tuple"]["Quux"] == Quux quux = Quux("zip", "zow") serialized = _serialize_dagster_namedtuple(quux, whitelist_map=_TEST_WHITELIST_MAP) # pylint: disable=function-redefined @_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP) class Quux(namedtuple("_Quux", "foo bar baz")): # pylint: disable=bad-super-call def __new__(cls, foo, bar, baz=None): return super(Quux, cls).__new__(cls, foo, bar, baz=baz) assert "Quux" in _TEST_WHITELIST_MAP["types"]["tuple"] assert _TEST_WHITELIST_MAP["types"]["tuple"]["Quux"] == Quux deserialized = _deserialize_json_to_dagster_namedtuple( serialized, whitelist_map=_TEST_WHITELIST_MAP ) assert deserialized != quux assert deserialized.foo == quux.foo assert deserialized.bar == quux.bar assert deserialized.baz is None
def test_backward_compat_serdes(): _TEST_WHITELIST_MAP = _initial_whitelist_map() @_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP) class Quux(namedtuple("_Quux", "foo bar baz")): def __new__(cls, foo, bar, baz): return super(Quux, cls).__new__(cls, foo, bar, baz) # pylint: disable=bad-super-call quux = Quux("zip", "zow", "whoopie") serialized = _serialize_dagster_namedtuple(quux, whitelist_map=_TEST_WHITELIST_MAP) # pylint: disable=function-redefined @_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP) class Quux(namedtuple("_Quux", "foo bar")): # pylint: disable=bad-super-call def __new__(cls, foo, bar): return super(Quux, cls).__new__(cls, foo, bar) deserialized = _deserialize_json_to_dagster_namedtuple( serialized, whitelist_map=_TEST_WHITELIST_MAP ) assert deserialized != quux assert deserialized.foo == quux.foo assert deserialized.bar == quux.bar assert not hasattr(deserialized, "baz")
def test_from_storage_dict(): _TEST_WHITELIST_MAP = _initial_whitelist_map() def whitelist_for_persistence(klass): return compose( _whitelist_for_persistence(whitelist_map=_TEST_WHITELIST_MAP), _whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP), )(klass) @whitelist_for_persistence class DeprecatedAlphabet(namedtuple("_DeprecatedAlphabet", "a b c"), Persistable): def __new__(cls, a, b, c): raise Exception("DeprecatedAlphabet is deprecated") @classmethod def from_storage_dict(cls, storage_dict): # instead of the DeprecatedAlphabet, directly invoke the namedtuple constructor return super(DeprecatedAlphabet, cls).__new__( cls, storage_dict.get("a"), storage_dict.get("b"), storage_dict.get("c"), ) assert "DeprecatedAlphabet" in _TEST_WHITELIST_MAP["persistence"] serialized = '{"__class__": "DeprecatedAlphabet", "a": "A", "b": "B", "c": "C"}' nt = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=_TEST_WHITELIST_MAP) assert isinstance(nt, DeprecatedAlphabet)
def test_forward_compat_serdes_new_field_with_default(): _TEST_WHITELIST_MAP = _initial_whitelist_map() @_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP) class Quux(namedtuple('_Quux', 'foo bar')): def __new__(cls, foo, bar): return super(Quux, cls).__new__(cls, foo, bar) # pylint: disable=bad-super-call assert 'Quux' in _TEST_WHITELIST_MAP['types']['tuple'] assert _TEST_WHITELIST_MAP['types']['tuple']['Quux'] == Quux quux = Quux('zip', 'zow') serialized = _serialize_dagster_namedtuple( quux, whitelist_map=_TEST_WHITELIST_MAP) # pylint: disable=function-redefined @_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP) class Quux(namedtuple('_Quux', 'foo bar baz')): # pylint: disable=bad-super-call def __new__(cls, foo, bar, baz=None): return super(Quux, cls).__new__(cls, foo, bar, baz=baz) assert 'Quux' in _TEST_WHITELIST_MAP['types']['tuple'] assert _TEST_WHITELIST_MAP['types']['tuple']['Quux'] == Quux deserialized = _deserialize_json_to_dagster_namedtuple( serialized, whitelist_map=_TEST_WHITELIST_MAP) assert deserialized != quux assert deserialized.foo == quux.foo assert deserialized.bar == quux.bar assert deserialized.baz is None
def test_backward_compat_serdes(): _TEST_TUPLE_MAP = {} _TEST_ENUM_MAP = {} @_whitelist_for_serdes(tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP) class Quux(namedtuple('_Quux', 'foo bar baz')): def __new__(cls, foo, bar, baz): return super(Quux, cls).__new__(cls, foo, bar, baz) # pylint: disable=bad-super-call quux = Quux('zip', 'zow', 'whoopie') serialized = _serialize_dagster_namedtuple(quux, tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP) @_whitelist_for_serdes(tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP) # pylint: disable=function-redefined class Quux(namedtuple('_Quux', 'foo bar')): # pylint: disable=bad-super-call def __new__(cls, foo, bar): return super(Quux, cls).__new__(cls, foo, bar) deserialized = _deserialize_json_to_dagster_namedtuple( serialized, tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP) assert deserialized != quux assert deserialized.foo == quux.foo assert deserialized.bar == quux.bar assert not hasattr(deserialized, 'baz')
def test_set(): _TEST_WHITELIST_MAP = _initial_whitelist_map() @_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP) class HasSets(namedtuple("_HasSets", "reg_set frozen_set")): def __new__(cls, reg_set, frozen_set): set_param(reg_set, "reg_set") inst_param(frozen_set, "frozen_set", frozenset) return super(HasSets, cls).__new__(cls, reg_set, frozen_set) foo = HasSets({1, 2, 3, "3"}, frozenset([4, 5, 6, "6"])) serialized = _serialize_dagster_namedtuple(foo, whitelist_map=_TEST_WHITELIST_MAP) foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=_TEST_WHITELIST_MAP) assert foo == foo_2
def test_persistent_tuple(): _TEST_WHITELIST_MAP = _initial_whitelist_map() def whitelist_for_persistence(klass): return compose( _whitelist_for_persistence(whitelist_map=_TEST_WHITELIST_MAP), _whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP), )(klass) @whitelist_for_persistence class Alphabet(namedtuple("_Alphabet", "a b c"), Persistable): def __new__(cls, a, b, c): return super(Alphabet, cls).__new__(cls, a, b, c) foo = Alphabet(a="A", b="B", c="C") serialized = _serialize_dagster_namedtuple(foo, whitelist_map=_TEST_WHITELIST_MAP) foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=_TEST_WHITELIST_MAP) assert foo == foo_2
def test_set(): _TEST_TUPLE_MAP = {} _TEST_ENUM_MAP = {} @_whitelist_for_serdes(tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP) class HasSets(namedtuple('_HasSets', 'reg_set frozen_set')): def __new__(cls, reg_set, frozen_set): set_param(reg_set, 'reg_set') inst_param(frozen_set, 'frozen_set', frozenset) return super(HasSets, cls).__new__(cls, reg_set, frozen_set) foo = HasSets({1, 2, 3, '3'}, frozenset([4, 5, 6, '6'])) serialized = _serialize_dagster_namedtuple(foo, tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP) foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP) assert foo == foo_2