def test_construct_log_string_for_event(): step_output_event = DagsterEvent( event_type_value="STEP_OUTPUT", pipeline_name="my_pipeline", step_key="solid2", solid_handle=NodeHandle("solid2", None), step_kind_value="COMPUTE", logging_tags={}, event_specific_data=StepOutputData( step_output_handle=StepOutputHandle("solid2", "result")), message= 'Yielded output "result" of type "Any" for step "solid2". (Type check passed).', pid=54348, ) logging_metadata = DagsterLoggingMetadata( run_id="f79a8a93-27f1-41b5-b465-b35d0809b26d", pipeline_name="my_pipeline") dagster_message_props = DagsterMessageProps( orig_message=step_output_event.message, dagster_event=step_output_event, ) assert ( construct_log_string(logging_metadata=logging_metadata, message_props=dagster_message_props) == 'my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_OUTPUT - Yielded output "result" of type "Any" for step "solid2". (Type check passed).' )
def test_construct_log_string_for_event(): step_output_event = DagsterEvent( event_type_value="STEP_OUTPUT", pipeline_name="my_pipeline", step_key="solid2", solid_handle=SolidHandle("solid2", None), step_kind_value="COMPUTE", logging_tags={}, event_specific_data=StepOutputData( step_output_handle=StepOutputHandle("solid2", "result")), message= 'Yielded output "result" of type "Any" for step "solid2". (Type check passed).', pid=54348, ) message_props = { "dagster_event": step_output_event, "pipeline_name": "my_pipeline" } synth_props = { "orig_message": step_output_event.message, "run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d", } assert ( construct_log_string(message_props=message_props, logging_tags={}, synth_props=synth_props) == 'my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_OUTPUT - Yielded output "result" of type "Any" for step "solid2". (Type check passed).' )
def _type_check_output( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type with user_code_error_boundary( DagsterTypeCheckError, lambda: ('In solid "{handle}" the output "{output_name}" received ' "value {output_value} of Python type {output_type} which " "does not pass the typecheck for Dagster type " "{dagster_type_name}. Step {step_key}.").format( handle=str(step_context.step.solid_handle), output_name=output.output_name, output_value=output.value, output_type=type(output.value), dagster_type_name=dagster_type.display_name, step_key=step_context.step.key, ), ): type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description= 'Type check failed for step output "{output_name}" - expected type "{dagster_type}".' .format( output_name=output.output_name, dagster_type=dagster_type.display_name, ), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _type_check_output( step_context: StepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", StepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type type_check_context = step_context.for_type(dagster_type) op_label = step_context.describe_op() output_type = type(output.value) with user_code_error_boundary( DagsterTypeCheckError, lambda: (f'Error occurred while type-checking output "{output.output_name}" of {op_label}, with ' f"Python type {output_type} and Dagster type {dagster_type.display_name}" ), log_manager=type_check_context.log, ): type_check = do_type_check(type_check_context, dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, metadata_entries=[ entry for entry in output.metadata_entries if isinstance(entry, MetadataEntry) ], ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description=( f'Type check failed for step output "{output.output_name}" - ' f'expected type "{dagster_type.display_name}". ' f"Description: {type_check.description}"), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _create_step_output_event(step_context, step_output_handle, type_check, success, version): return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), )
def _type_check_output( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type with user_code_error_boundary( DagsterTypeCheckError, lambda: ( f'Error occurred while type-checking output "{output.output_name}" of solid ' f'"{str(step_context.step.solid_handle)}", with Python type {type(output.value)} and ' f"Dagster type {dagster_type.display_name}" ), ): type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description='Type check failed for step output "{output_name}" - expected type "{dagster_type}".'.format( output_name=output.output_name, dagster_type=dagster_type.display_name, ), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _create_step_output_event( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, type_check: TypeCheck, success: bool, version: Optional[str], ) -> DagsterEvent: return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), )
def test_recover_in_between_steps(): two_op_job = define_two_op_job() events = [ DagsterEvent( DagsterEventType.STEP_START.value, pipeline_name=two_op_job.name, step_key="foo_op", ), DagsterEvent( DagsterEventType.STEP_OUTPUT.value, pipeline_name=two_op_job.name, event_specific_data=StepOutputData( StepOutputHandle(step_key="foo_op", output_name="result") ), step_key="foo_op", ), DagsterEvent( DagsterEventType.STEP_SUCCESS.value, pipeline_name=two_op_job.name, event_specific_data=StepSuccessData(duration_ms=10.0), step_key="foo_op", ), ] with pytest.raises( DagsterInvariantViolationError, match="Execution finished without completing the execution plan", ): with create_execution_plan(two_op_job).start(RetryMode.DISABLED) as active_execution: steps = active_execution.get_steps_to_execute() assert len(steps) == 1 step_1 = steps[0] assert step_1.key == "foo_op" active_execution.handle_event(events[0]) active_execution.handle_event(events[1]) active_execution.handle_event(events[2]) # CRASH!- we've closed the active execution. Now we recover, spinning up a new one with create_execution_plan(two_op_job).start(RetryMode.DISABLED) as active_execution: possibly_in_flight_steps = active_execution.rebuild_from_events(events) assert len(possibly_in_flight_steps) == 1 step_2 = possibly_in_flight_steps[0] assert step_2.key == "bar_op" assert not active_execution.get_steps_to_execute() active_execution.handle_event( DagsterEvent( DagsterEventType.STEP_START.value, pipeline_name=two_op_job.name, step_key="bar_op", ) ) active_execution.handle_event( DagsterEvent( DagsterEventType.STEP_SUCCESS.value, pipeline_name=two_op_job.name, event_specific_data=StepSuccessData(duration_ms=10.0), step_key="bar_op", ) )