def test_descent_path():
class Foo(NamedTuple):
bar: int
with pytest.raises(SerializationError,
match=re.escape("Descent path: <root:dict>.a.b[2].c")):
serialize_value({"a": {"b": [{}, {}, {"c": Foo(1)}]}})
test_map = WhitelistMap.create()
blank_map = WhitelistMap.create()
@_whitelist_for_serdes(whitelist_map=test_map)
class Fizz(NamedTuple):
buzz: int
ser = _serialize_dagster_namedtuple({"a": {
"b": [{}, {}, {
"c": Fizz(1)
}]
}},
whitelist_map=test_map)
with pytest.raises(DeserializationError,
match=re.escape("Descent path: <root:dict>.a.b[2].c")):
_deserialize_json(ser, whitelist_map=blank_map)
def test_pipeline_run_status_dagster_run_status():
# serialize in current code
test_status = DagsterRunStatus("QUEUED")
test_str = serialize_value(test_status)
# deserialize in "legacy" code
legacy_env = WhitelistMap.create()
@_whitelist_for_serdes(legacy_env)
class PipelineRunStatus(Enum):
QUEUED = "QUEUED"
result = _deserialize_json(test_str, legacy_env)
assert isinstance(result, PipelineRunStatus)
assert result.value == test_status.value
def test_enum_backcompat():
test_env = WhitelistMap.create()
class MyEnumSerializer(EnumSerializer):
@classmethod
def value_from_storage_str(cls, storage_str, klass):
return getattr(klass, storage_str)
@classmethod
def value_to_storage_str(cls, value, whitelist_map, descent_path):
val_as_str = str(value)
actual_enum_val = val_as_str.split(".")[1:]
backcompat_name = (
"OldEnum" # Simulate changing the storage name to some legacy backcompat name
)
return ".".join([backcompat_name, *actual_enum_val])
@_whitelist_for_serdes(test_env, serializer=MyEnumSerializer)
class MyEnum(Enum):
RED = "color.red"
BLUE = "color.red"
# Ensure that serdes roundtrip preserves value
register_serdes_enum_fallbacks({"OldEnum": MyEnum}, whitelist_map=test_env)
my_enum = MyEnum("color.red")
enum_json = serialize_value(my_enum, whitelist_map=test_env)
result = _deserialize_json(enum_json, test_env)
assert result == my_enum
# ensure that "legacy" environment can correctly interpret enum stored under legacy name.
legacy_env = WhitelistMap.create()
@_whitelist_for_serdes(legacy_env)
class OldEnum(Enum):
RED = "color.red"
BLUE = "color.blue"
result = _deserialize_json(enum_json, legacy_env)
old_enum = OldEnum("color.red")
assert old_enum == result
def test_deserialize_empty_set():
assert set() == deserialize_value(serialize_value(set()))
assert frozenset() == deserialize_value(serialize_value(frozenset()))
def test_external_job_origin_instigator_origin():
def build_legacy_whitelist_map():
legacy_env = WhitelistMap.create()
@_whitelist_for_serdes(legacy_env)
class ExternalJobOrigin(
namedtuple("_ExternalJobOrigin",
"external_repository_origin job_name")):
def get_id(self):
return create_snapshot_id(self)
@_whitelist_for_serdes(legacy_env)
class ExternalRepositoryOrigin(
namedtuple("_ExternalRepositoryOrigin",
"repository_location_origin repository_name")):
def get_id(self):
return create_snapshot_id(self)
class GrpcServerOriginSerializer(DefaultNamedTupleSerializer):
@classmethod
def skip_when_empty(cls):
return {"use_ssl"}
@_whitelist_for_serdes(whitelist_map=legacy_env,
serializer=GrpcServerOriginSerializer)
class GrpcServerRepositoryLocationOrigin(
namedtuple(
"_GrpcServerRepositoryLocationOrigin",
"host port socket location_name use_ssl",
), ):
def __new__(cls,
host,
port=None,
socket=None,
location_name=None,
use_ssl=None):
return super(GrpcServerRepositoryLocationOrigin,
cls).__new__(cls, host, port, socket,
location_name, use_ssl)
return (
legacy_env,
ExternalJobOrigin,
ExternalRepositoryOrigin,
GrpcServerRepositoryLocationOrigin,
)
legacy_env, klass, repo_klass, location_klass = build_legacy_whitelist_map(
)
from dagster.core.host_representation.origin import (
ExternalInstigatorOrigin,
ExternalRepositoryOrigin,
GrpcServerRepositoryLocationOrigin,
)
# serialize from current code, compare against old code
instigator_origin = ExternalInstigatorOrigin(
external_repository_origin=ExternalRepositoryOrigin(
repository_location_origin=GrpcServerRepositoryLocationOrigin(
host="localhost", port=1234, location_name="test_location"),
repository_name="the_repo",
),
instigator_name="simple_schedule",
)
instigator_origin_str = serialize_dagster_namedtuple(instigator_origin)
instigator_to_job = _deserialize_json(instigator_origin_str, legacy_env)
assert isinstance(instigator_to_job, klass)
# ensure that the origin id is stable
assert instigator_to_job.get_id() == instigator_origin.get_id()
# # serialize from old code, compare against current code
job_origin = klass(
external_repository_origin=repo_klass(
repository_location_origin=location_klass(
host="localhost", port=1234, location_name="test_location"),
repository_name="the_repo",
),
job_name="simple_schedule",
)
job_origin_str = serialize_value(job_origin, legacy_env)
from dagster.serdes.serdes import _WHITELIST_MAP
job_to_instigator = deserialize_json_to_dagster_namedtuple(job_origin_str)
assert isinstance(job_to_instigator, ExternalInstigatorOrigin)
# ensure that the origin id is stable
assert job_to_instigator.get_id() == job_origin.get_id()
