def dagster_event_from_dict(event_dict, pipeline_name):
check.dict_param(event_dict, 'event_dict', key_type=str)
check.str_param(pipeline_name, 'pipeline_name')
materialization = event_dict.get('intermediateMaterialization') or {}
# Get event_type
event_type = _handled_events().get(event_dict['__typename'])
if not event_type:
raise Exception('unhandled event type %s' % event_dict['__typename'])
# Get event_specific_data
event_specific_data = None
if event_type == DagsterEventType.STEP_OUTPUT:
event_specific_data = StepOutputData(
step_output_handle=StepOutputHandle(event_dict['step']['key'],
event_dict['outputName']),
value_repr=event_dict['valueRepr'],
intermediate_materialization=Materialization(
path=materialization.get('path'),
description=materialization.get('description')),
)
elif event_type == DagsterEventType.STEP_SUCCESS:
event_specific_data = StepSuccessData(0.0)
elif event_type == DagsterEventType.STEP_MATERIALIZATION:
event_specific_data = StepMaterializationData(
materialization=Materialization(path=materialization.get('path'),
description=materialization.get(
'description')))
elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT:
result_metadata = event_dict['expectationResult'][
'resultMetadataJsonString']
expectation_result = ExpectationResult(
event_dict['expectationResult']['success'],
event_dict['expectationResult']['name'],
event_dict['expectationResult']['message'],
json.loads(result_metadata) if result_metadata else None,
)
event_specific_data = StepExpectationResultData(expectation_result)
elif event_type == DagsterEventType.STEP_FAILURE:
error_info = SerializableErrorInfo(event_dict['error']['message'],
stack=None,
cls_name=None)
event_specific_data = StepFailureData(error_info)
return DagsterEvent(
event_type_value=event_type.value,
pipeline_name=pipeline_name,
step_key=event_dict['step']['key'],
solid_handle=SolidHandle(event_dict['step']['solidHandleID'], None,
None),
step_kind_value=event_dict['step']['kind'],
logging_tags=None,
event_specific_data=event_specific_data,
)
def materialization_from_data(data):
return Materialization(
label=data['label'],
description=data.get('description'), # enforce?
metadata_entries=list(
event_metadata_entries(data.get('metadataEntries')) or []),
)
def _fn(step_context, inputs):
runtime_value = inputs[MATERIALIZATION_THUNK_INPUT]
path = output_def.runtime_type.output_schema.materialize_runtime_value(
step_context, config_spec, runtime_value
)
if not isinstance(path, six.string_types):
raise DagsterInvariantViolationError(
(
'materialize_runtime_value on type {type_name} has returned '
'value {value} of type {python_type}. You must return a '
'string (and ideally a valid file path).'
).format(
type_name=output_def.runtime_type.name,
value=repr(path),
python_type=type(path).__name__,
)
)
yield StepOutputValue(output_name=MATERIALIZATION_THUNK_OUTPUT, value=runtime_value)
yield Materialization(
path=path,
description=('Materialization of {solid_name}.{output_name}').format(
output_name=output_def.name, solid_name=str(step_context.solid_handle)
),
)
def materialization_from_data(data):
if data.get("asset_key"):
return AssetMaterialization(
asset_key=data["asset_key"],
description=data.get("description"),
metadata_entries=list(event_metadata_entries(data.get("metadataEntries")) or []),
)
return Materialization(
label=data["label"],
description=data.get("description"), # enforce?
metadata_entries=list(event_metadata_entries(data.get("metadataEntries")) or []),
)
def _create_step_output_event(step_context, result, intermediates_manager):
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.inst_param(result, 'result', Result)
check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager)
step = step_context.step
step_output = step.step_output_named(result.output_name)
try:
value = step_output.runtime_type.coerce_runtime_value(result.value)
step_output_handle = StepOutputHandle.from_step(step=step, output_name=result.output_name)
object_key = intermediates_manager.set_intermediate(
context=step_context,
runtime_type=step_output.runtime_type,
step_output_handle=step_output_handle,
value=value,
)
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
value_repr=repr(value),
intermediate_materialization=Materialization(path=object_key)
if object_key
else None,
),
)
except DagsterRuntimeCoercionError as e:
raise DagsterInvariantViolationError(
(
'In solid "{handle}" the output "{output_name}" returned '
'an invalid type: {error_msg}.'
).format(
handle=str(step.solid_handle),
error_msg=','.join(e.args),
output_name=result.output_name,
)
)
def _stats_records(run_id):
now = time.time()
return [
_event_record(run_id, 'A', now - 325, DagsterEventType.STEP_START),
_event_record(
run_id,
'A',
now - 225,
DagsterEventType.STEP_SUCCESS,
StepSuccessData(duration_ms=100000.0),
),
_event_record(run_id, 'B', now - 225, DagsterEventType.STEP_START),
_event_record(
run_id,
'B',
now - 175,
DagsterEventType.STEP_FAILURE,
StepFailureData(error=None, user_failure_data=None),
),
_event_record(run_id, 'C', now - 175, DagsterEventType.STEP_START),
_event_record(run_id, 'C', now - 150, DagsterEventType.STEP_SKIPPED),
_event_record(run_id, 'D', now - 150, DagsterEventType.STEP_START),
_event_record(
run_id,
'D',
now - 125,
DagsterEventType.STEP_MATERIALIZATION,
StepMaterializationData(Materialization(label='mat 1')),
),
_event_record(
run_id,
'D',
now - 100,
DagsterEventType.STEP_EXPECTATION_RESULT,
StepExpectationResultData(ExpectationResult(success=True, label='exp 1')),
),
_event_record(
run_id,
'D',
now - 75,
DagsterEventType.STEP_MATERIALIZATION,
StepMaterializationData(Materialization(label='mat 2')),
),
_event_record(
run_id,
'D',
now - 50,
DagsterEventType.STEP_EXPECTATION_RESULT,
StepExpectationResultData(ExpectationResult(success=False, label='exp 2')),
),
_event_record(
run_id,
'D',
now - 25,
DagsterEventType.STEP_MATERIALIZATION,
StepMaterializationData(Materialization(label='mat 3')),
),
_event_record(
run_id, 'D', now, DagsterEventType.STEP_SUCCESS, StepSuccessData(duration_ms=150000.0)
),
]
def test_materialization():
assert isinstance(Materialization('foo', 'foo.txt'), Materialization)
def test_materialization_assign_label_from_asset_key():
mat = Materialization(asset_key=AssetKey(['foo', 'bar']))
assert mat.label == 'foo.bar'
