def test_set():
test_map = WhitelistMap.create()
@_whitelist_for_serdes(whitelist_map=test_map)
class HasSets(namedtuple("_HasSets", "reg_set frozen_set")):
def __new__(cls, reg_set, frozen_set):
set_param(reg_set, "reg_set")
inst_param(frozen_set, "frozen_set", frozenset)
return super(HasSets, cls).__new__(cls, reg_set, frozen_set)
foo = HasSets({1, 2, 3, "3"}, frozenset([4, 5, 6, "6"]))
serialized = _serialize_dagster_namedtuple(foo, whitelist_map=test_map)
foo_2 = _deserialize_json(serialized, whitelist_map=test_map)
assert foo == foo_2
# verify that set elements are serialized in a consistent order so that
# equal objects always have a consistent serialization / snapshot ID
big_foo = HasSets(set(string.ascii_lowercase),
frozenset(string.ascii_lowercase))
snap_id = hash_str(
_serialize_dagster_namedtuple(big_foo, whitelist_map=test_map))
roundtrip_snap_id = hash_str(
_serialize_dagster_namedtuple(
_deserialize_json(
_serialize_dagster_namedtuple(big_foo, whitelist_map=test_map),
whitelist_map=test_map,
),
whitelist_map=test_map,
))
assert snap_id == roundtrip_snap_id
def test_skip_when_empty():
test_map = WhitelistMap.create()
@_whitelist_for_serdes(whitelist_map=test_map)
class SameSnapshotTuple(namedtuple("_Tuple", "foo")):
def __new__(cls, foo):
return super(SameSnapshotTuple, cls).__new__(cls, foo) # pylint: disable=bad-super-call
old_tuple = SameSnapshotTuple(foo="A")
old_serialized = _serialize_dagster_namedtuple(old_tuple,
whitelist_map=test_map)
old_snapshot = hash_str(old_serialized)
# Without setting skip_when_empty, the ID changes
@_whitelist_for_serdes(whitelist_map=test_map) # pylint: disable=function-redefined
class SameSnapshotTuple(namedtuple("_Tuple", "foo bar")):
def __new__(cls, foo, bar=None):
return super(SameSnapshotTuple, cls).__new__( # pylint: disable=bad-super-call
cls, foo, bar)
new_tuple_without_serializer = SameSnapshotTuple(foo="A")
new_snapshot_without_serializer = hash_str(
_serialize_dagster_namedtuple(new_tuple_without_serializer,
whitelist_map=test_map))
assert new_snapshot_without_serializer != old_snapshot
# By setting a custom serializer and skip_when_empty, the snapshot stays the same
# as long as the new field is None
class SkipWhenEmptySerializer(DefaultNamedTupleSerializer):
@classmethod
def skip_when_empty(cls) -> Set[str]:
return {"bar"}
@_whitelist_for_serdes(whitelist_map=test_map,
serializer=SkipWhenEmptySerializer) # pylint: disable=function-redefined
class SameSnapshotTuple(namedtuple("_Tuple", "foo bar")):
def __new__(cls, foo, bar=None):
return super(SameSnapshotTuple, cls).__new__( # pylint: disable=bad-super-call
cls, foo, bar)
for bar_val in [None, [], {}, set()]:
new_tuple = SameSnapshotTuple(foo="A", bar=bar_val)
new_snapshot = hash_str(
_serialize_dagster_namedtuple(new_tuple, whitelist_map=test_map))
assert old_snapshot == new_snapshot
rehydrated_tuple = _deserialize_json(old_serialized,
whitelist_map=test_map)
assert rehydrated_tuple.foo == "A"
assert rehydrated_tuple.bar is None
new_tuple_with_bar = SameSnapshotTuple(foo="A", bar="B")
assert new_tuple_with_bar.foo == "A"
assert new_tuple_with_bar.bar == "B"
def get_id(self):
json_rep = _serialize_dagster_namedtuple(self,
whitelist_map=new_map)
return hash_str(json_rep)