def test_set():
test_map = WhitelistMap()
@_whitelist_for_serdes(whitelist_map=test_map)
class HasSets(namedtuple("_HasSets", "reg_set frozen_set")):
def __new__(cls, reg_set, frozen_set):
set_param(reg_set, "reg_set")
inst_param(frozen_set, "frozen_set", frozenset)
return super(HasSets, cls).__new__(cls, reg_set, frozen_set)
foo = HasSets({1, 2, 3, "3"}, frozenset([4, 5, 6, "6"]))
serialized = _serialize_dagster_namedtuple(foo, whitelist_map=test_map)
foo_2 = _deserialize_json_to_dagster_namedtuple(serialized,
whitelist_map=test_map)
assert foo == foo_2
# verify that set elements are serialized in a consistent order so that
# equal objects always have a consistent serialization / snapshot ID
big_foo = HasSets(set(string.ascii_lowercase),
frozenset(string.ascii_lowercase))
assert create_snapshot_id(big_foo) == create_snapshot_id(
_deserialize_json_to_dagster_namedtuple(_serialize_dagster_namedtuple(
big_foo, whitelist_map=test_map),
whitelist_map=test_map))
def test_skip_when_empty():
test_map = WhitelistMap()
@_whitelist_for_serdes(whitelist_map=test_map)
class SameSnapshotTuple(namedtuple("_Tuple", "foo")):
def __new__(cls, foo):
return super(SameSnapshotTuple, cls).__new__(cls, foo) # pylint: disable=bad-super-call
old_tuple = SameSnapshotTuple(foo="A")
old_serialized = serialize_dagster_namedtuple(old_tuple)
old_snapshot = create_snapshot_id(old_tuple)
# Without setting skip_when_empty, the ID changes
@_whitelist_for_serdes(whitelist_map=test_map)
class SameSnapshotTuple(namedtuple("_Tuple", "foo bar")): # pylint: disable=function-redefined
def __new__(cls, foo, bar=None):
return super(SameSnapshotTuple, cls).__new__( # pylint: disable=bad-super-call
cls, foo, bar)
new_tuple_without_serializer = SameSnapshotTuple(foo="A")
new_snapshot_without_serializer = create_snapshot_id(
new_tuple_without_serializer)
assert new_snapshot_without_serializer != old_snapshot
# By setting a custom serializer and skip_when_empty, the snapshot stays the same
# as long as the new field is None
class SkipWhenEmptySerializer(DefaultNamedTupleSerializer):
@classmethod
def skip_when_empty(cls) -> Set[str]:
return {"bar"}
@_whitelist_for_serdes(whitelist_map=test_map,
serializer=SkipWhenEmptySerializer)
class SameSnapshotTuple(namedtuple("_Tuple", "foo bar")): # pylint: disable=function-redefined
def __new__(cls, foo, bar=None):
return super(SameSnapshotTuple, cls).__new__( # pylint: disable=bad-super-call
cls, foo, bar)
new_tuple = SameSnapshotTuple(foo="A")
new_snapshot = create_snapshot_id(new_tuple)
assert old_snapshot == new_snapshot
rehydrated_tuple = deserialize_json_to_dagster_namedtuple(old_serialized)
assert rehydrated_tuple.foo == "A"
assert rehydrated_tuple.bar is None
new_tuple_with_bar = SameSnapshotTuple(foo="A", bar="B")
assert new_tuple_with_bar.foo == "A"
assert new_tuple_with_bar.bar == "B"