def _create_step_events_for_output(step_context, output):
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", Output)
step = step_context.step
step_output = step.step_output_named(output.output_name)
version = resolve_step_output_versions(
step_context.execution_plan,
step_context.environment_config,
step_context.mode_def,
)[StepOutputHandle(step_context.step.key, output.output_name)]
for output_event in _type_checked_step_output_event_sequence(
step_context, output, version):
yield output_event
step_output_handle = StepOutputHandle.from_step(
step=step, output_name=output.output_name)
for evt in _set_intermediates(step_context, step_output,
step_output_handle, output, version):
yield evt
for evt in _create_output_materializations(step_context,
output.output_name,
output.value):
yield evt
def output_value(self, context, step_key, result, execution_step, kwargs):
if (result.event_type == DagsterEventType.STEP_OUTPUT
and result.step_key == step_key
and result.step_output_data.output_name
in execution_step.step_output_dict
and result.step_output_data.output_name in kwargs):
dagster_type = execution_step.step_output_named(
result.step_output_data.output_name).dagster_type
step_output_handle = StepOutputHandle.from_step(
step=execution_step,
output_name=result.step_output_data.output_name)
if not context.intermediate_storage.has_intermediate(
context, step_output_handle):
# Not working for InMemoryIntermediateManager ?
raise KeyError(
"No Intermediate Store record for StepOutput: {}".format(
step_output_handle))
output_value = context.intermediate_storage.get_intermediate(
context=context,
dagster_type=dagster_type,
step_output_handle=step_output_handle,
).obj
kwargs[result.step_output_data.output_name].set(output_value)
def _create_step_events_for_output(step_context, output):
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.inst_param(output, 'output', Output)
step = step_context.step
step_output = step.step_output_named(output.output_name)
for output_event in _type_checked_step_output_event_sequence(
step_context, output):
yield output_event
step_output_handle = StepOutputHandle.from_step(
step=step, output_name=output.output_name)
step_context.intermediates_manager.set_intermediate(
context=step_context,
runtime_type=step_output.runtime_type,
step_output_handle=step_output_handle,
value=output.value,
)
for evt in _create_output_materializations(step_context,
output.output_name,
output.value):
yield evt
def _create_step_output_event(step_context, output, type_check, success):
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=StepOutputHandle.from_step(
step=step_context.step, output_name=output.output_name),
type_check_data=TypeCheckData(
success=success,
label=output.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
),
)
def _create_step_output_event(step_context, result, intermediates_manager):
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.inst_param(result, 'result', Result)
check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager)
step = step_context.step
step_output = step.step_output_named(result.output_name)
try:
value = step_output.runtime_type.coerce_runtime_value(result.value)
step_output_handle = StepOutputHandle.from_step(step=step, output_name=result.output_name)
object_key = intermediates_manager.set_intermediate(
context=step_context,
runtime_type=step_output.runtime_type,
step_output_handle=step_output_handle,
value=value,
)
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
value_repr=repr(value),
intermediate_materialization=Materialization(path=object_key)
if object_key
else None,
),
)
except DagsterRuntimeCoercionError as e:
raise DagsterInvariantViolationError(
(
'In solid "{handle}" the output "{output_name}" returned '
'an invalid type: {error_msg}.'
).format(
handle=str(step.solid_handle),
error_msg=','.join(e.args),
output_name=result.output_name,
)
)
def _create_step_events_for_output(step_context, output):
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", Output)
step = step_context.step
step_output = step.step_output_named(output.output_name)
for output_event in _type_checked_step_output_event_sequence(
step_context, output):
yield output_event
step_output_handle = StepOutputHandle.from_step(
step=step, output_name=output.output_name)
for evt in _set_intermediates(step_context, step_output,
step_output_handle, output):
yield evt
for evt in _create_output_materializations(step_context,
output.output_name,
output.value):
yield evt
def test_basic_int_and_string_execution_plan():
pipeline = multiple_output_pipeline()
execution_plan = create_execution_plan(
pipeline,
{
'solids': {
'return_one_and_foo': {
'outputs': [
{
'string': {
'json': {
'path': 'dummy_string.json'
}
}
},
{
'number': {
'json': {
'path': 'dummy_number.json'
}
}
},
]
}
}
},
)
assert len(execution_plan.steps) == 5
steps = execution_plan.topological_steps()
assert steps[0].key == 'return_one_and_foo.compute'
assert_plan_topological_level(
steps,
[1, 2],
[
'return_one_and_foo.outputs.string.materialize.0',
'return_one_and_foo.outputs.number.materialize.0',
],
)
assert_plan_topological_level(
steps,
[3, 4],
[
'return_one_and_foo.outputs.string.materialize.join',
'return_one_and_foo.outputs.number.materialize.join',
],
)
transform_step = execution_plan.get_step_by_key(
'return_one_and_foo.compute')
string_mat_step = execution_plan.get_step_by_key(
'return_one_and_foo.outputs.string.materialize.0')
assert len(string_mat_step.step_inputs) == 1
assert string_mat_step.step_inputs[
0].prev_output_handle == StepOutputHandle.from_step(
step=transform_step, output_name='string')
string_mat_join_step = execution_plan.get_step_by_key(
'return_one_and_foo.outputs.string.materialize.join')
assert len(string_mat_join_step.step_inputs) == 1
assert string_mat_join_step.step_inputs[
0].prev_output_handle == StepOutputHandle.from_step(
step=string_mat_step, output_name=MATERIALIZATION_THUNK_OUTPUT)
