PARTITION_SET_TAG: partition_set.name
}
@whitelist_for_serdes(serializer=DagsterRunSerializer)
class DagsterRun(PipelineRun):
"""Serializable internal representation of a dagster run, as stored in a
:py:class:`~dagster.core.storage.runs.RunStorage`.
Subclasses PipelineRun for backcompat purposes. DagsterRun is the actual initialized class used throughout the system.
"""
# DagsterRun is serialized as PipelineRun so that it can be read by older (pre 0.13.x) version of
# Dagster, but is read back in as a DagsterRun.
register_serdes_tuple_fallbacks({"PipelineRun": DagsterRun})
@whitelist_for_serdes
class PipelineRunsFilter(
namedtuple(
"_PipelineRunsFilter",
"run_ids pipeline_name statuses tags snapshot_id updated_after mode"
)):
def __new__(
cls,
run_ids=None,
pipeline_name=None,
statuses=None,
tags=None,
snapshot_id=None,
if event_specific_data.op in ("GET_ASSET", AssetStoreOperationType.GET_ASSET):
return (
DagsterEventType.LOADED_INPUT.value,
LoadedInputData(
event_specific_data.output_name, event_specific_data.asset_store_key
),
)
if event_specific_data.op in ("SET_ASSET", AssetStoreOperationType.SET_ASSET):
return (
DagsterEventType.HANDLED_OUTPUT.value,
HandledOutputData(
event_specific_data.output_name, event_specific_data.asset_store_key, []
),
)
# previous name for ASSET_MATERIALIZATION was STEP_MATERIALIZATION
if event_type_value == "STEP_MATERIALIZATION":
return DagsterEventType.ASSET_MATERIALIZATION.value, event_specific_data
else:
return event_type_value, event_specific_data
register_serdes_tuple_fallbacks(
{
"PipelineProcessStartedData": None,
"PipelineProcessExitedData": None,
"PipelineProcessStartData": None,
}
)
def construct_json_event_logger(json_path):
"""Record a stream of event records to json"""
check.str_param(json_path, "json_path")
return construct_single_handler_logger(
"json-event-record-logger",
"debug",
JsonEventLoggerHandler(
json_path,
lambda record: construct_event_record(
StructuredLoggerMessage(
name=record.name,
message=record.msg,
level=record.levelno,
meta=record.dagster_meta,
record=record,
)),
),
)
register_serdes_tuple_fallbacks({
# These were originally distinguished from each other but ended up being empty subclasses
# of EventRecord -- instead of using the subclasses we were relying on
# EventRecord.is_dagster_event to distinguish events that originate in the logging
# machinery from events that are yielded by user code
"DagsterEventRecord": EventRecord,
"LogMessageRecord": EventRecord,
})
"external_repository_origin",
ExternalRepositoryOrigin,
),
check.str_param(instigator_name, "instigator_name"),
)
def get_id(self):
return create_snapshot_id(self)
# ExternalInstigatorOrigin used to be called ExternalJobOrigin, before the concept of "job" was
# introduced in 0.12.0. For clarity, we changed the name of the namedtuple with `0.14.0`, but we
# need to maintain the serialized format in order to avoid changing the origin id that is stored in
# our schedule storage. This registers the serialized ExternalJobOrigin named tuple class to be
# deserialized as an ExternalInstigatorOrigin, using its corresponding serializer for serdes.
register_serdes_tuple_fallbacks({"ExternalJobOrigin": ExternalInstigatorOrigin})
@whitelist_for_serdes
class ExternalPartitionSetOrigin(
NamedTuple(
"_PartitionSetOrigin",
[("external_repository_origin", ExternalRepositoryOrigin), ("partition_set_name", str)],
)
):
"""Serializable representation of an ExternalPartitionSet that can be used to
uniquely it or reload it in across process boundaries.
"""
def __new__(cls, external_repository_origin: ExternalRepositoryOrigin, partition_set_name: str):
return super(ExternalPartitionSetOrigin, cls).__new__(
return DagsterEventType.ENGINE_EVENT.value, EngineEventData([])
elif event_type_value == "ASSET_STORE_OPERATION":
if event_specific_data.op in ("GET_ASSET",
AssetStoreOperationType.GET_ASSET):
return (
DagsterEventType.LOADED_INPUT.value,
LoadedInputData(event_specific_data.output_name,
event_specific_data.asset_store_key),
)
if event_specific_data.op in ("SET_ASSET",
AssetStoreOperationType.SET_ASSET):
return (
DagsterEventType.HANDLED_OUTPUT.value,
HandledOutputData(event_specific_data.output_name,
event_specific_data.asset_store_key),
)
else:
return event_type_value, event_specific_data
register_serdes_tuple_fallbacks({
"PipelineProcessStartedData":
None,
"PipelineProcessExitedData":
None,
"PipelineProcessStartData":
None,
"AssetStoreOperationData":
AssetStoreOperationData,
})
