def test_descent_path(): class Foo(NamedTuple): bar: int with pytest.raises(SerializationError, match=re.escape("Descent path: <root:dict>.a.b[2].c")): serialize_value({"a": {"b": [{}, {}, {"c": Foo(1)}]}}) test_map = WhitelistMap.create() blank_map = WhitelistMap.create() @_whitelist_for_serdes(whitelist_map=test_map) class Fizz(NamedTuple): buzz: int ser = _serialize_dagster_namedtuple({"a": { "b": [{}, {}, { "c": Fizz(1) }] }}, whitelist_map=test_map) with pytest.raises(DeserializationError, match=re.escape("Descent path: <root:dict>.a.b[2].c")): _deserialize_json(ser, whitelist_map=blank_map)
def test_pipeline_run_status_dagster_run_status(): # serialize in current code test_status = DagsterRunStatus("QUEUED") test_str = serialize_value(test_status) # deserialize in "legacy" code legacy_env = WhitelistMap.create() @_whitelist_for_serdes(legacy_env) class PipelineRunStatus(Enum): QUEUED = "QUEUED" result = _deserialize_json(test_str, legacy_env) assert isinstance(result, PipelineRunStatus) assert result.value == test_status.value
def test_enum_backcompat(): test_env = WhitelistMap.create() class MyEnumSerializer(EnumSerializer): @classmethod def value_from_storage_str(cls, storage_str, klass): return getattr(klass, storage_str) @classmethod def value_to_storage_str(cls, value, whitelist_map, descent_path): val_as_str = str(value) actual_enum_val = val_as_str.split(".")[1:] backcompat_name = ( "OldEnum" # Simulate changing the storage name to some legacy backcompat name ) return ".".join([backcompat_name, *actual_enum_val]) @_whitelist_for_serdes(test_env, serializer=MyEnumSerializer) class MyEnum(Enum): RED = "color.red" BLUE = "color.red" # Ensure that serdes roundtrip preserves value register_serdes_enum_fallbacks({"OldEnum": MyEnum}, whitelist_map=test_env) my_enum = MyEnum("color.red") enum_json = serialize_value(my_enum, whitelist_map=test_env) result = _deserialize_json(enum_json, test_env) assert result == my_enum # ensure that "legacy" environment can correctly interpret enum stored under legacy name. legacy_env = WhitelistMap.create() @_whitelist_for_serdes(legacy_env) class OldEnum(Enum): RED = "color.red" BLUE = "color.blue" result = _deserialize_json(enum_json, legacy_env) old_enum = OldEnum("color.red") assert old_enum == result
def test_deserialize_empty_set(): assert set() == deserialize_value(serialize_value(set())) assert frozenset() == deserialize_value(serialize_value(frozenset()))
def test_external_job_origin_instigator_origin(): def build_legacy_whitelist_map(): legacy_env = WhitelistMap.create() @_whitelist_for_serdes(legacy_env) class ExternalJobOrigin( namedtuple("_ExternalJobOrigin", "external_repository_origin job_name")): def get_id(self): return create_snapshot_id(self) @_whitelist_for_serdes(legacy_env) class ExternalRepositoryOrigin( namedtuple("_ExternalRepositoryOrigin", "repository_location_origin repository_name")): def get_id(self): return create_snapshot_id(self) class GrpcServerOriginSerializer(DefaultNamedTupleSerializer): @classmethod def skip_when_empty(cls): return {"use_ssl"} @_whitelist_for_serdes(whitelist_map=legacy_env, serializer=GrpcServerOriginSerializer) class GrpcServerRepositoryLocationOrigin( namedtuple( "_GrpcServerRepositoryLocationOrigin", "host port socket location_name use_ssl", ), ): def __new__(cls, host, port=None, socket=None, location_name=None, use_ssl=None): return super(GrpcServerRepositoryLocationOrigin, cls).__new__(cls, host, port, socket, location_name, use_ssl) return ( legacy_env, ExternalJobOrigin, ExternalRepositoryOrigin, GrpcServerRepositoryLocationOrigin, ) legacy_env, klass, repo_klass, location_klass = build_legacy_whitelist_map( ) from dagster.core.host_representation.origin import ( ExternalInstigatorOrigin, ExternalRepositoryOrigin, GrpcServerRepositoryLocationOrigin, ) # serialize from current code, compare against old code instigator_origin = ExternalInstigatorOrigin( external_repository_origin=ExternalRepositoryOrigin( repository_location_origin=GrpcServerRepositoryLocationOrigin( host="localhost", port=1234, location_name="test_location"), repository_name="the_repo", ), instigator_name="simple_schedule", ) instigator_origin_str = serialize_dagster_namedtuple(instigator_origin) instigator_to_job = _deserialize_json(instigator_origin_str, legacy_env) assert isinstance(instigator_to_job, klass) # ensure that the origin id is stable assert instigator_to_job.get_id() == instigator_origin.get_id() # # serialize from old code, compare against current code job_origin = klass( external_repository_origin=repo_klass( repository_location_origin=location_klass( host="localhost", port=1234, location_name="test_location"), repository_name="the_repo", ), job_name="simple_schedule", ) job_origin_str = serialize_value(job_origin, legacy_env) from dagster.serdes.serdes import _WHITELIST_MAP job_to_instigator = deserialize_json_to_dagster_namedtuple(job_origin_str) assert isinstance(job_to_instigator, ExternalInstigatorOrigin) # ensure that the origin id is stable assert job_to_instigator.get_id() == job_origin.get_id()