def test_construct_log_string_for_event():
step_output_event = DagsterEvent(
event_type_value="STEP_OUTPUT",
pipeline_name="my_pipeline",
step_key="solid2.compute",
solid_handle=SolidHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepOutputData(
step_output_handle=StepOutputHandle("solid2.compute", "result")),
message=
'Yielded output "result" of type "Any" for step "solid2.compute". (Type check passed).',
pid=54348,
)
message_props = {
"dagster_event": step_output_event,
"pipeline_name": "my_pipeline"
}
synth_props = {
"orig_message": step_output_event.message,
"run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d",
}
assert (
construct_log_string(message_props=message_props,
logging_tags={},
synth_props=synth_props) ==
'my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_OUTPUT - Yielded output "result" of type "Any" for step "solid2.compute". (Type check passed).'
)
def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, 'event_dict', key_type=str)
check.str_param(pipeline_name, 'pipeline_name')
materialization = event_dict.get('intermediateMaterialization') or {}
# Get event_type
event_type = _handled_events().get(event_dict['__typename'])
if not event_type:
raise Exception('unhandled event type %s' % event_dict['__typename'])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(event_dict['step']['key'],
event_dict['outputName']),
value_repr=event_dict['valueRepr'],
intermediate_materialization=Materialization(
path=materialization.get('path'),
description=materialization.get('description')),
)
elif event_type == DagsterEventType.STEP_SUCCESS:
event_specific_data = StepSuccessData(0.0)
elif event_type == DagsterEventType.STEP_MATERIALIZATION:
event_specific_data = StepMaterializationData(
materialization=Materialization(path=materialization.get('path'),
description=materialization.get(
'description')))
elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
result_metadata = event_dict['expectationResult'][
'resultMetadataJsonString']
expectation_result = ExpectationResult(
event_dict['expectationResult']['success'],
event_dict['expectationResult']['name'],
event_dict['expectationResult']['message'],
json.loads(result_metadata) if result_metadata else None,
)
event_specific_data = StepExpectationResultData(expectation_result)
elif event_type == DagsterEventType.STEP_FAILURE:
error_info = SerializableErrorInfo(event_dict['error']['message'],
stack=None,
cls_name=None)
event_specific_data = StepFailureData(error_info)
return DagsterEvent(
event_type_value=event_type.value,
pipeline_name=pipeline_name,
step_key=event_dict['step']['key'],
solid_handle=SolidHandle(event_dict['step']['solidHandleID'], None,
None),
step_kind_value=event_dict['step']['kind'],
logging_tags=None,
event_specific_data=event_specific_data,
)
def _create_step_output_event(step_context, output, type_check, success):
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=StepOutputHandle.from_step(
step=step_context.step, output_name=output.output_name),
type_check_data=TypeCheckData(
success=success,
label=output.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
),
)
def _create_step_output_event(step_context, result, intermediates_manager):
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.inst_param(result, 'result', Result)
check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager)
step = step_context.step
step_output = step.step_output_named(result.output_name)
try:
value = step_output.runtime_type.coerce_runtime_value(result.value)
step_output_handle = StepOutputHandle.from_step(step=step, output_name=result.output_name)
object_key = intermediates_manager.set_intermediate(
context=step_context,
runtime_type=step_output.runtime_type,
step_output_handle=step_output_handle,
value=value,
)
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
value_repr=repr(value),
intermediate_materialization=Materialization(path=object_key)
if object_key
else None,
),
)
except DagsterRuntimeCoercionError as e:
raise DagsterInvariantViolationError(
(
'In solid "{handle}" the output "{output_name}" returned '
'an invalid type: {error_msg}.'
).format(
handle=str(step.solid_handle),
error_msg=','.join(e.args),
output_name=result.output_name,
)
)
def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, 'event_dict', key_type=str)
check.str_param(pipeline_name, 'pipeline_name')
# Get event_type
event_type = HANDLED_EVENTS.get(event_dict['__typename'])
if not event_type:
raise Exception('unhandled event type %s' % event_dict['__typename'])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(event_dict['step']['key'],
event_dict['outputName']),
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
label=event_dict['typeCheck']['label'],
description=event_dict.get('description'),
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
),
)
elif event_type == DagsterEventType.STEP_INPUT:
event_specific_data = StepInputData(
input_name=event_dict['inputName'],
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
label=event_dict['typeCheck']['label'],
description=event_dict.get('description'),
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
),
)
elif event_type == DagsterEventType.STEP_SUCCESS:
event_specific_data = StepSuccessData(0.0)
elif event_type == DagsterEventType.STEP_UP_FOR_RETRY:
event_specific_data = StepRetryData(
error=error_from_data(event_dict['retryError']),
seconds_to_wait=event_dict['secondsToWait'],
)
elif event_type == DagsterEventType.STEP_MATERIALIZATION:
materialization = event_dict['materialization']
event_specific_data = StepMaterializationData(
materialization=materialization_from_data(materialization))
elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
expectation_result = expectation_result_from_data(
event_dict['expectationResult'])
event_specific_data = StepExpectationResultData(expectation_result)
elif event_type == DagsterEventType.STEP_FAILURE:
event_specific_data = StepFailureData(
error_from_data(event_dict['error']),
UserFailureData(
label=event_dict['failureMetadata']['label'],
description=event_dict['failureMetadata']['description'],
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
) if event_dict.get('failureMetadata') else None,
)
elif event_type == DagsterEventType.ENGINE_EVENT:
event_specific_data = EngineEventData(
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))),
marker_start=event_dict.get('markerStart'),
marker_end=event_dict.get('markerEnd'),
error=error_from_data(event_dict['engineError'])
if event_dict.get('engineError') else None,
)
# We should update the GraphQL response so that clients don't need to do this handle parsing.
# See: https://github.com/dagster-io/dagster/issues/1559
handle = None
step_key = None
step_kind_value = None
if 'step' in event_dict and event_dict['step']:
step_key = event_dict['step']['key']
step_kind_value = event_dict['step']['kind']
keys = event_dict['step']['solidHandleID'].split('.')
while keys:
handle = SolidHandle(keys.pop(0), parent=handle)
return DagsterEvent(
event_type_value=event_type.value,
pipeline_name=pipeline_name,
step_key=step_key,
solid_handle=handle,
step_kind_value=step_kind_value,
logging_tags=None,
event_specific_data=event_specific_data,
)
def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, "event_dict", key_type=str)
check.str_param(pipeline_name, "pipeline_name")
# Get event_type
event_type = HANDLED_EVENTS.get(event_dict["__typename"])
if not event_type:
raise Exception("unhandled event type %s" % event_dict["__typename"])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(event_dict["stepKey"],
event_dict["outputName"]),
type_check_data=TypeCheckData(
success=event_dict["typeCheck"]["success"],
label=event_dict["typeCheck"]["label"],
description=event_dict.get("description"),
metadata_entries=list(
event_metadata_entries(event_dict.get("metadataEntries"))
or []),
),
)
elif event_type == DagsterEventType.STEP_INPUT:
event_specific_data = StepInputData(
input_name=event_dict["inputName"],
type_check_data=TypeCheckData(
success=event_dict["typeCheck"]["success"],
label=event_dict["typeCheck"]["label"],
description=event_dict.get("description"),
metadata_entries=list(
event_metadata_entries(event_dict.get("metadataEntries"))
or []),
),
)
elif event_type == DagsterEventType.STEP_SUCCESS:
event_specific_data = StepSuccessData(0.0)
elif event_type == DagsterEventType.STEP_UP_FOR_RETRY:
event_specific_data = StepRetryData(
error=error_from_data(event_dict["retryError"]),
seconds_to_wait=event_dict["secondsToWait"],
)
elif event_type == DagsterEventType.STEP_MATERIALIZATION:
materialization = event_dict["materialization"]
event_specific_data = StepMaterializationData(
materialization=materialization_from_data(materialization))
elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
expectation_result = expectation_result_from_data(
event_dict["expectationResult"])
event_specific_data = StepExpectationResultData(expectation_result)
elif event_type == DagsterEventType.STEP_FAILURE:
event_specific_data = StepFailureData(
error_from_data(event_dict["error"]),
UserFailureData(
label=event_dict["failureMetadata"]["label"],
description=event_dict["failureMetadata"]["description"],
metadata_entries=list(
event_metadata_entries(event_dict.get("metadataEntries"))
or []),
) if event_dict.get("failureMetadata") else None,
)
elif event_type == DagsterEventType.ENGINE_EVENT:
event_specific_data = EngineEventData(
metadata_entries=list(
event_metadata_entries(event_dict.get("metadataEntries"))),
marker_start=event_dict.get("markerStart"),
marker_end=event_dict.get("markerEnd"),
error=error_from_data(event_dict["engineError"])
if event_dict.get("engineError") else None,
)
return DagsterEvent(
event_type_value=event_type.value,
pipeline_name=pipeline_name,
step_key=event_dict.get("stepKey"),
solid_handle=SolidHandle.from_string(event_dict["solidHandleID"])
if event_dict.get("solidHandleID") else None,
# at the time of writing this:
# * 'COMPUTE` is the only step kind
# * this code should get deleted in the near future as we move away from
# dagster-graphql CLI as what we invoke in dask/k8s/etc.
step_kind_value="COMPUTE" if event_dict.get("stepKey") else None,
logging_tags=None,
event_specific_data=event_specific_data,
)
def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, 'event_dict', key_type=str)
check.str_param(pipeline_name, 'pipeline_name')
# Get event_type
event_type = _handled_events().get(event_dict['__typename'])
if not event_type:
raise Exception('unhandled event type %s' % event_dict['__typename'])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(event_dict['step']['key'],
event_dict['outputName']),
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
label=event_dict['typeCheck']['label'],
description=event_dict.get('description'),
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
),
)
elif event_type == DagsterEventType.STEP_INPUT:
event_specific_data = StepInputData(
input_name=event_dict['inputName'],
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
label=event_dict['typeCheck']['label'],
description=event_dict.get('description'),
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
),
)
elif event_type == DagsterEventType.STEP_SUCCESS:
event_specific_data = StepSuccessData(0.0)
elif event_type == DagsterEventType.STEP_MATERIALIZATION:
materialization = event_dict['materialization']
event_specific_data = StepMaterializationData(
materialization=materialization_from_data(materialization))
elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
expectation_result = expectation_result_from_data(
event_dict['expectationResult'])
event_specific_data = StepExpectationResultData(expectation_result)
elif event_type == DagsterEventType.STEP_FAILURE:
error_info = SerializableErrorInfo(event_dict['error']['message'],
stack=None,
cls_name=None)
event_specific_data = StepFailureData(
error_info,
UserFailureData(
label=event_dict['failureMetadata']['label'],
description=event_dict['failureMetadata']['description'],
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
) if event_dict.get('failureMetadata') else None,
)
# We should update the GraphQL response so that clients don't need to do this handle parsing.
# See: https://github.com/dagster-io/dagster/issues/1559
keys = event_dict['step']['solidHandleID'].split('.')
handle = None
while keys:
handle = SolidHandle(keys.pop(0), definition_name=None, parent=handle)
return DagsterEvent(
event_type_value=event_type.value,
pipeline_name=pipeline_name,
step_key=event_dict['step']['key'],
solid_handle=handle,
step_kind_value=event_dict['step']['kind'],
logging_tags=None,
event_specific_data=event_specific_data,
)
def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, 'event_dict', key_type=str)
check.str_param(pipeline_name, 'pipeline_name')
# Get event_type
event_type = _handled_events().get(event_dict['__typename'])
if not event_type:
raise Exception('unhandled event type %s' % event_dict['__typename'])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(event_dict['step']['key'],
event_dict['outputName']),
value_repr=event_dict['valueRepr'],
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
label=event_dict['typeCheck']['label'],
description=event_dict.get('description'),
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
),
)
elif event_type == DagsterEventType.STEP_INPUT:
event_specific_data = StepInputData(
input_name=event_dict['inputName'],
value_repr=event_dict['valueRepr'],
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
label=event_dict['typeCheck']['label'],
description=event_dict.get('description'),
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
),
)
elif event_type == DagsterEventType.STEP_SUCCESS:
event_specific_data = StepSuccessData(0.0)
elif event_type == DagsterEventType.STEP_MATERIALIZATION:
materialization = event_dict['materialization']
event_specific_data = StepMaterializationData(
materialization=materialization_from_data(materialization))
elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
expectation_result = expectation_result_from_data(
event_dict['expectationResult'])
event_specific_data = StepExpectationResultData(expectation_result)
elif event_type == DagsterEventType.STEP_FAILURE:
error_info = SerializableErrorInfo(event_dict['error']['message'],
stack=None,
cls_name=None)
event_specific_data = StepFailureData(
error_info,
UserFailureData(
label=event_dict['failureMetadata']['label'],
description=event_dict['failureMetadata']['description'],
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries'))
or []),
) if event_dict.get('failureMetadata') else None,
)
return DagsterEvent(
event_type_value=event_type.value,
pipeline_name=pipeline_name,
step_key=event_dict['step']['key'],
solid_handle=SolidHandle(event_dict['step']['solidHandleID'], None,
None),
step_kind_value=event_dict['step']['kind'],
logging_tags=None,
event_specific_data=event_specific_data,
)
def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, 'event_dict', key_type=str)
check.str_param(pipeline_name, 'pipeline_name')
# Get event_type
event_type = HANDLED_EVENTS.get(event_dict['__typename'])
if not event_type:
raise Exception('unhandled event type %s' % event_dict['__typename'])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(event_dict['stepKey'], event_dict['outputName']),
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
label=event_dict['typeCheck']['label'],
description=event_dict.get('description'),
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries')) or []
),
),
)
elif event_type == DagsterEventType.STEP_INPUT:
event_specific_data = StepInputData(
input_name=event_dict['inputName'],
type_check_data=TypeCheckData(
success=event_dict['typeCheck']['success'],
label=event_dict['typeCheck']['label'],
description=event_dict.get('description'),
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries')) or []
),
),
)
elif event_type == DagsterEventType.STEP_SUCCESS:
event_specific_data = StepSuccessData(0.0)
elif event_type == DagsterEventType.STEP_UP_FOR_RETRY:
event_specific_data = StepRetryData(
error=error_from_data(event_dict['retryError']),
seconds_to_wait=event_dict['secondsToWait'],
)
elif event_type == DagsterEventType.STEP_MATERIALIZATION:
materialization = event_dict['materialization']
event_specific_data = StepMaterializationData(
materialization=materialization_from_data(materialization)
)
elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
expectation_result = expectation_result_from_data(event_dict['expectationResult'])
event_specific_data = StepExpectationResultData(expectation_result)
elif event_type == DagsterEventType.STEP_FAILURE:
event_specific_data = StepFailureData(
error_from_data(event_dict['error']),
UserFailureData(
label=event_dict['failureMetadata']['label'],
description=event_dict['failureMetadata']['description'],
metadata_entries=list(
event_metadata_entries(event_dict.get('metadataEntries')) or []
),
)
if event_dict.get('failureMetadata')
else None,
)
elif event_type == DagsterEventType.ENGINE_EVENT:
event_specific_data = EngineEventData(
metadata_entries=list(event_metadata_entries(event_dict.get('metadataEntries'))),
marker_start=event_dict.get('markerStart'),
marker_end=event_dict.get('markerEnd'),
error=error_from_data(event_dict['engineError'])
if event_dict.get('engineError')
else None,
)
return DagsterEvent(
event_type_value=event_type.value,
pipeline_name=pipeline_name,
step_key=event_dict.get('stepKey'),
solid_handle=SolidHandle.from_string(event_dict['solidHandleID'])
if event_dict.get('solidHandleID')
else None,
# at the time of writing this:
# * 'COMPUTE` is the only step kind
# * this code should get deleted in the near future as we move away from
# dagster-graphql CLI as what we invoke in dask/k8s/etc.
step_kind_value='COMPUTE' if event_dict.get('stepKey') else None,
logging_tags=None,
event_specific_data=event_specific_data,
)