def test_to_storage_value():
_TEST_WHITELIST_MAP = _initial_whitelist_map()
def whitelist_for_persistence(klass):
return compose(
_whitelist_for_persistence(whitelist_map=_TEST_WHITELIST_MAP),
_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP),
)(klass)
@whitelist_for_persistence
class DeprecatedAlphabet(namedtuple("_DeprecatedAlphabet", "a b c"), Persistable):
def __new__(cls, a, b, c):
return super(DeprecatedAlphabet, cls).__new__(cls, a, b, c)
def to_storage_value(self):
return default_to_storage_value(
SubstituteAlphabet(self.a, self.b, self.c), _TEST_WHITELIST_MAP
)
@whitelist_for_persistence
class SubstituteAlphabet(namedtuple("_SubstituteAlphabet", "a b c"), Persistable):
def __new__(cls, a, b, c):
return super(SubstituteAlphabet, cls).__new__(cls, a, b, c)
nested = DeprecatedAlphabet(None, None, "_C")
deprecated = DeprecatedAlphabet("A", "B", nested)
serialized = _serialize_dagster_namedtuple(deprecated, whitelist_map=_TEST_WHITELIST_MAP)
alphabet = _deserialize_json_to_dagster_namedtuple(
serialized, whitelist_map=_TEST_WHITELIST_MAP
)
assert not isinstance(alphabet, DeprecatedAlphabet)
assert isinstance(alphabet, SubstituteAlphabet)
assert not isinstance(alphabet.c, DeprecatedAlphabet)
assert isinstance(alphabet.c, SubstituteAlphabet)
def test_forward_compat_serdes_new_field_with_default():
_TEST_WHITELIST_MAP = _initial_whitelist_map()
@_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP)
class Quux(namedtuple("_Quux", "foo bar")):
def __new__(cls, foo, bar):
return super(Quux, cls).__new__(cls, foo, bar) # pylint: disable=bad-super-call
assert "Quux" in _TEST_WHITELIST_MAP["types"]["tuple"]
assert _TEST_WHITELIST_MAP["types"]["tuple"]["Quux"] == Quux
quux = Quux("zip", "zow")
serialized = _serialize_dagster_namedtuple(quux, whitelist_map=_TEST_WHITELIST_MAP)
# pylint: disable=function-redefined
@_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP)
class Quux(namedtuple("_Quux", "foo bar baz")): # pylint: disable=bad-super-call
def __new__(cls, foo, bar, baz=None):
return super(Quux, cls).__new__(cls, foo, bar, baz=baz)
assert "Quux" in _TEST_WHITELIST_MAP["types"]["tuple"]
assert _TEST_WHITELIST_MAP["types"]["tuple"]["Quux"] == Quux
deserialized = _deserialize_json_to_dagster_namedtuple(
serialized, whitelist_map=_TEST_WHITELIST_MAP
)
assert deserialized != quux
assert deserialized.foo == quux.foo
assert deserialized.bar == quux.bar
assert deserialized.baz is None
def test_backward_compat_serdes():
_TEST_WHITELIST_MAP = _initial_whitelist_map()
@_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP)
class Quux(namedtuple("_Quux", "foo bar baz")):
def __new__(cls, foo, bar, baz):
return super(Quux, cls).__new__(cls, foo, bar, baz) # pylint: disable=bad-super-call
quux = Quux("zip", "zow", "whoopie")
serialized = _serialize_dagster_namedtuple(quux, whitelist_map=_TEST_WHITELIST_MAP)
# pylint: disable=function-redefined
@_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP)
class Quux(namedtuple("_Quux", "foo bar")): # pylint: disable=bad-super-call
def __new__(cls, foo, bar):
return super(Quux, cls).__new__(cls, foo, bar)
deserialized = _deserialize_json_to_dagster_namedtuple(
serialized, whitelist_map=_TEST_WHITELIST_MAP
)
assert deserialized != quux
assert deserialized.foo == quux.foo
assert deserialized.bar == quux.bar
assert not hasattr(deserialized, "baz")
def test_from_storage_dict():
_TEST_WHITELIST_MAP = _initial_whitelist_map()
def whitelist_for_persistence(klass):
return compose(
_whitelist_for_persistence(whitelist_map=_TEST_WHITELIST_MAP),
_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP),
)(klass)
@whitelist_for_persistence
class DeprecatedAlphabet(namedtuple("_DeprecatedAlphabet", "a b c"), Persistable):
def __new__(cls, a, b, c):
raise Exception("DeprecatedAlphabet is deprecated")
@classmethod
def from_storage_dict(cls, storage_dict):
# instead of the DeprecatedAlphabet, directly invoke the namedtuple constructor
return super(DeprecatedAlphabet, cls).__new__(
cls,
storage_dict.get("a"),
storage_dict.get("b"),
storage_dict.get("c"),
)
assert "DeprecatedAlphabet" in _TEST_WHITELIST_MAP["persistence"]
serialized = '{"__class__": "DeprecatedAlphabet", "a": "A", "b": "B", "c": "C"}'
nt = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=_TEST_WHITELIST_MAP)
assert isinstance(nt, DeprecatedAlphabet)
def test_forward_compat_serdes_new_field_with_default():
_TEST_WHITELIST_MAP = _initial_whitelist_map()
@_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP)
class Quux(namedtuple('_Quux', 'foo bar')):
def __new__(cls, foo, bar):
return super(Quux, cls).__new__(cls, foo, bar) # pylint: disable=bad-super-call
assert 'Quux' in _TEST_WHITELIST_MAP['types']['tuple']
assert _TEST_WHITELIST_MAP['types']['tuple']['Quux'] == Quux
quux = Quux('zip', 'zow')
serialized = _serialize_dagster_namedtuple(
quux, whitelist_map=_TEST_WHITELIST_MAP)
# pylint: disable=function-redefined
@_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP)
class Quux(namedtuple('_Quux', 'foo bar baz')): # pylint: disable=bad-super-call
def __new__(cls, foo, bar, baz=None):
return super(Quux, cls).__new__(cls, foo, bar, baz=baz)
assert 'Quux' in _TEST_WHITELIST_MAP['types']['tuple']
assert _TEST_WHITELIST_MAP['types']['tuple']['Quux'] == Quux
deserialized = _deserialize_json_to_dagster_namedtuple(
serialized, whitelist_map=_TEST_WHITELIST_MAP)
assert deserialized != quux
assert deserialized.foo == quux.foo
assert deserialized.bar == quux.bar
assert deserialized.baz is None
def test_backward_compat_serdes():
_TEST_TUPLE_MAP = {}
_TEST_ENUM_MAP = {}
@_whitelist_for_serdes(tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP)
class Quux(namedtuple('_Quux', 'foo bar baz')):
def __new__(cls, foo, bar, baz):
return super(Quux, cls).__new__(cls, foo, bar, baz) # pylint: disable=bad-super-call
quux = Quux('zip', 'zow', 'whoopie')
serialized = _serialize_dagster_namedtuple(quux,
tuple_map=_TEST_TUPLE_MAP,
enum_map=_TEST_ENUM_MAP)
@_whitelist_for_serdes(tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP) # pylint: disable=function-redefined
class Quux(namedtuple('_Quux', 'foo bar')): # pylint: disable=bad-super-call
def __new__(cls, foo, bar):
return super(Quux, cls).__new__(cls, foo, bar)
deserialized = _deserialize_json_to_dagster_namedtuple(
serialized, tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP)
assert deserialized != quux
assert deserialized.foo == quux.foo
assert deserialized.bar == quux.bar
assert not hasattr(deserialized, 'baz')
def test_set():
_TEST_WHITELIST_MAP = _initial_whitelist_map()
@_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP)
class HasSets(namedtuple("_HasSets", "reg_set frozen_set")):
def __new__(cls, reg_set, frozen_set):
set_param(reg_set, "reg_set")
inst_param(frozen_set, "frozen_set", frozenset)
return super(HasSets, cls).__new__(cls, reg_set, frozen_set)
foo = HasSets({1, 2, 3, "3"}, frozenset([4, 5, 6, "6"]))
serialized = _serialize_dagster_namedtuple(foo, whitelist_map=_TEST_WHITELIST_MAP)
foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=_TEST_WHITELIST_MAP)
assert foo == foo_2
def test_persistent_tuple():
_TEST_WHITELIST_MAP = _initial_whitelist_map()
def whitelist_for_persistence(klass):
return compose(
_whitelist_for_persistence(whitelist_map=_TEST_WHITELIST_MAP),
_whitelist_for_serdes(whitelist_map=_TEST_WHITELIST_MAP),
)(klass)
@whitelist_for_persistence
class Alphabet(namedtuple("_Alphabet", "a b c"), Persistable):
def __new__(cls, a, b, c):
return super(Alphabet, cls).__new__(cls, a, b, c)
foo = Alphabet(a="A", b="B", c="C")
serialized = _serialize_dagster_namedtuple(foo, whitelist_map=_TEST_WHITELIST_MAP)
foo_2 = _deserialize_json_to_dagster_namedtuple(serialized, whitelist_map=_TEST_WHITELIST_MAP)
assert foo == foo_2
def test_set():
_TEST_TUPLE_MAP = {}
_TEST_ENUM_MAP = {}
@_whitelist_for_serdes(tuple_map=_TEST_TUPLE_MAP, enum_map=_TEST_ENUM_MAP)
class HasSets(namedtuple('_HasSets', 'reg_set frozen_set')):
def __new__(cls, reg_set, frozen_set):
set_param(reg_set, 'reg_set')
inst_param(frozen_set, 'frozen_set', frozenset)
return super(HasSets, cls).__new__(cls, reg_set, frozen_set)
foo = HasSets({1, 2, 3, '3'}, frozenset([4, 5, 6, '6']))
serialized = _serialize_dagster_namedtuple(foo,
tuple_map=_TEST_TUPLE_MAP,
enum_map=_TEST_ENUM_MAP)
foo_2 = _deserialize_json_to_dagster_namedtuple(serialized,
tuple_map=_TEST_TUPLE_MAP,
enum_map=_TEST_ENUM_MAP)
assert foo == foo_2
