def test_construct_log_string_for_event(): step_output_event = DagsterEvent( event_type_value="STEP_OUTPUT", pipeline_name="my_pipeline", step_key="solid2.compute", solid_handle=SolidHandle("solid2", None), step_kind_value="COMPUTE", logging_tags={}, event_specific_data=StepOutputData( step_output_handle=StepOutputHandle("solid2.compute", "result")), message= 'Yielded output "result" of type "Any" for step "solid2.compute". (Type check passed).', pid=54348, ) message_props = { "dagster_event": step_output_event, "pipeline_name": "my_pipeline" } synth_props = { "orig_message": step_output_event.message, "run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d", } assert ( construct_log_string(message_props=message_props, logging_tags={}, synth_props=synth_props) == 'my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_OUTPUT - Yielded output "result" of type "Any" for step "solid2.compute". (Type check passed).' )
def dagster_event_from_dict(event_dict, pipeline_name): check.dict_param(event_dict, 'event_dict', key_type=str) check.str_param(pipeline_name, 'pipeline_name') materialization = event_dict.get('intermediateMaterialization') or {} # Get event_type event_type = _handled_events().get(event_dict['__typename']) if not event_type: raise Exception('unhandled event type %s' % event_dict['__typename']) # Get event_specific_data event_specific_data = None if event_type == DagsterEventType.STEP_OUTPUT: event_specific_data = StepOutputData( step_output_handle=StepOutputHandle(event_dict['step']['key'], event_dict['outputName']), value_repr=event_dict['valueRepr'], intermediate_materialization=Materialization( path=materialization.get('path'), description=materialization.get('description')), ) elif event_type == DagsterEventType.STEP_SUCCESS: event_specific_data = StepSuccessData(0.0) elif event_type == DagsterEventType.STEP_MATERIALIZATION: event_specific_data = StepMaterializationData( materialization=Materialization(path=materialization.get('path'), description=materialization.get( 'description'))) elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT: result_metadata = event_dict['expectationResult'][ 'resultMetadataJsonString'] expectation_result = ExpectationResult( event_dict['expectationResult']['success'], event_dict['expectationResult']['name'], event_dict['expectationResult']['message'], json.loads(result_metadata) if result_metadata else None, ) event_specific_data = StepExpectationResultData(expectation_result) elif event_type == DagsterEventType.STEP_FAILURE: error_info = SerializableErrorInfo(event_dict['error']['message'], stack=None, cls_name=None) event_specific_data = StepFailureData(error_info) return DagsterEvent( event_type_value=event_type.value, pipeline_name=pipeline_name, step_key=event_dict['step']['key'], solid_handle=SolidHandle(event_dict['step']['solidHandleID'], None, None), step_kind_value=event_dict['step']['kind'], logging_tags=None, event_specific_data=event_specific_data, )
def _create_step_output_event(step_context, output, type_check, success): return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=StepOutputHandle.from_step( step=step_context.step, output_name=output.output_name), type_check_data=TypeCheckData( success=success, label=output.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), ), )
def _create_step_output_event(step_context, result, intermediates_manager): check.inst_param(step_context, 'step_context', SystemStepExecutionContext) check.inst_param(result, 'result', Result) check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager) step = step_context.step step_output = step.step_output_named(result.output_name) try: value = step_output.runtime_type.coerce_runtime_value(result.value) step_output_handle = StepOutputHandle.from_step(step=step, output_name=result.output_name) object_key = intermediates_manager.set_intermediate( context=step_context, runtime_type=step_output.runtime_type, step_output_handle=step_output_handle, value=value, ) return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, value_repr=repr(value), intermediate_materialization=Materialization(path=object_key) if object_key else None, ), ) except DagsterRuntimeCoercionError as e: raise DagsterInvariantViolationError( ( 'In solid "{handle}" the output "{output_name}" returned ' 'an invalid type: {error_msg}.' ).format( handle=str(step.solid_handle), error_msg=','.join(e.args), output_name=result.output_name, ) )
def dagster_event_from_dict(event_dict, pipeline_name): check.dict_param(event_dict, 'event_dict', key_type=str) check.str_param(pipeline_name, 'pipeline_name') # Get event_type event_type = HANDLED_EVENTS.get(event_dict['__typename']) if not event_type: raise Exception('unhandled event type %s' % event_dict['__typename']) # Get event_specific_data event_specific_data = None if event_type == DagsterEventType.STEP_OUTPUT: event_specific_data = StepOutputData( step_output_handle=StepOutputHandle(event_dict['step']['key'], event_dict['outputName']), type_check_data=TypeCheckData( success=event_dict['typeCheck']['success'], label=event_dict['typeCheck']['label'], description=event_dict.get('description'), metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ), ) elif event_type == DagsterEventType.STEP_INPUT: event_specific_data = StepInputData( input_name=event_dict['inputName'], type_check_data=TypeCheckData( success=event_dict['typeCheck']['success'], label=event_dict['typeCheck']['label'], description=event_dict.get('description'), metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ), ) elif event_type == DagsterEventType.STEP_SUCCESS: event_specific_data = StepSuccessData(0.0) elif event_type == DagsterEventType.STEP_UP_FOR_RETRY: event_specific_data = StepRetryData( error=error_from_data(event_dict['retryError']), seconds_to_wait=event_dict['secondsToWait'], ) elif event_type == DagsterEventType.STEP_MATERIALIZATION: materialization = event_dict['materialization'] event_specific_data = StepMaterializationData( materialization=materialization_from_data(materialization)) elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT: expectation_result = expectation_result_from_data( event_dict['expectationResult']) event_specific_data = StepExpectationResultData(expectation_result) elif event_type == DagsterEventType.STEP_FAILURE: event_specific_data = StepFailureData( error_from_data(event_dict['error']), UserFailureData( label=event_dict['failureMetadata']['label'], description=event_dict['failureMetadata']['description'], metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ) if event_dict.get('failureMetadata') else None, ) elif event_type == DagsterEventType.ENGINE_EVENT: event_specific_data = EngineEventData( metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries'))), marker_start=event_dict.get('markerStart'), marker_end=event_dict.get('markerEnd'), error=error_from_data(event_dict['engineError']) if event_dict.get('engineError') else None, ) # We should update the GraphQL response so that clients don't need to do this handle parsing. # See: https://github.com/dagster-io/dagster/issues/1559 handle = None step_key = None step_kind_value = None if 'step' in event_dict and event_dict['step']: step_key = event_dict['step']['key'] step_kind_value = event_dict['step']['kind'] keys = event_dict['step']['solidHandleID'].split('.') while keys: handle = SolidHandle(keys.pop(0), parent=handle) return DagsterEvent( event_type_value=event_type.value, pipeline_name=pipeline_name, step_key=step_key, solid_handle=handle, step_kind_value=step_kind_value, logging_tags=None, event_specific_data=event_specific_data, )
def dagster_event_from_dict(event_dict, pipeline_name): check.dict_param(event_dict, "event_dict", key_type=str) check.str_param(pipeline_name, "pipeline_name") # Get event_type event_type = HANDLED_EVENTS.get(event_dict["__typename"]) if not event_type: raise Exception("unhandled event type %s" % event_dict["__typename"]) # Get event_specific_data event_specific_data = None if event_type == DagsterEventType.STEP_OUTPUT: event_specific_data = StepOutputData( step_output_handle=StepOutputHandle(event_dict["stepKey"], event_dict["outputName"]), type_check_data=TypeCheckData( success=event_dict["typeCheck"]["success"], label=event_dict["typeCheck"]["label"], description=event_dict.get("description"), metadata_entries=list( event_metadata_entries(event_dict.get("metadataEntries")) or []), ), ) elif event_type == DagsterEventType.STEP_INPUT: event_specific_data = StepInputData( input_name=event_dict["inputName"], type_check_data=TypeCheckData( success=event_dict["typeCheck"]["success"], label=event_dict["typeCheck"]["label"], description=event_dict.get("description"), metadata_entries=list( event_metadata_entries(event_dict.get("metadataEntries")) or []), ), ) elif event_type == DagsterEventType.STEP_SUCCESS: event_specific_data = StepSuccessData(0.0) elif event_type == DagsterEventType.STEP_UP_FOR_RETRY: event_specific_data = StepRetryData( error=error_from_data(event_dict["retryError"]), seconds_to_wait=event_dict["secondsToWait"], ) elif event_type == DagsterEventType.STEP_MATERIALIZATION: materialization = event_dict["materialization"] event_specific_data = StepMaterializationData( materialization=materialization_from_data(materialization)) elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT: expectation_result = expectation_result_from_data( event_dict["expectationResult"]) event_specific_data = StepExpectationResultData(expectation_result) elif event_type == DagsterEventType.STEP_FAILURE: event_specific_data = StepFailureData( error_from_data(event_dict["error"]), UserFailureData( label=event_dict["failureMetadata"]["label"], description=event_dict["failureMetadata"]["description"], metadata_entries=list( event_metadata_entries(event_dict.get("metadataEntries")) or []), ) if event_dict.get("failureMetadata") else None, ) elif event_type == DagsterEventType.ENGINE_EVENT: event_specific_data = EngineEventData( metadata_entries=list( event_metadata_entries(event_dict.get("metadataEntries"))), marker_start=event_dict.get("markerStart"), marker_end=event_dict.get("markerEnd"), error=error_from_data(event_dict["engineError"]) if event_dict.get("engineError") else None, ) return DagsterEvent( event_type_value=event_type.value, pipeline_name=pipeline_name, step_key=event_dict.get("stepKey"), solid_handle=SolidHandle.from_string(event_dict["solidHandleID"]) if event_dict.get("solidHandleID") else None, # at the time of writing this: # * 'COMPUTE` is the only step kind # * this code should get deleted in the near future as we move away from # dagster-graphql CLI as what we invoke in dask/k8s/etc. step_kind_value="COMPUTE" if event_dict.get("stepKey") else None, logging_tags=None, event_specific_data=event_specific_data, )
def dagster_event_from_dict(event_dict, pipeline_name): check.dict_param(event_dict, 'event_dict', key_type=str) check.str_param(pipeline_name, 'pipeline_name') # Get event_type event_type = _handled_events().get(event_dict['__typename']) if not event_type: raise Exception('unhandled event type %s' % event_dict['__typename']) # Get event_specific_data event_specific_data = None if event_type == DagsterEventType.STEP_OUTPUT: event_specific_data = StepOutputData( step_output_handle=StepOutputHandle(event_dict['step']['key'], event_dict['outputName']), type_check_data=TypeCheckData( success=event_dict['typeCheck']['success'], label=event_dict['typeCheck']['label'], description=event_dict.get('description'), metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ), ) elif event_type == DagsterEventType.STEP_INPUT: event_specific_data = StepInputData( input_name=event_dict['inputName'], type_check_data=TypeCheckData( success=event_dict['typeCheck']['success'], label=event_dict['typeCheck']['label'], description=event_dict.get('description'), metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ), ) elif event_type == DagsterEventType.STEP_SUCCESS: event_specific_data = StepSuccessData(0.0) elif event_type == DagsterEventType.STEP_MATERIALIZATION: materialization = event_dict['materialization'] event_specific_data = StepMaterializationData( materialization=materialization_from_data(materialization)) elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT: expectation_result = expectation_result_from_data( event_dict['expectationResult']) event_specific_data = StepExpectationResultData(expectation_result) elif event_type == DagsterEventType.STEP_FAILURE: error_info = SerializableErrorInfo(event_dict['error']['message'], stack=None, cls_name=None) event_specific_data = StepFailureData( error_info, UserFailureData( label=event_dict['failureMetadata']['label'], description=event_dict['failureMetadata']['description'], metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ) if event_dict.get('failureMetadata') else None, ) # We should update the GraphQL response so that clients don't need to do this handle parsing. # See: https://github.com/dagster-io/dagster/issues/1559 keys = event_dict['step']['solidHandleID'].split('.') handle = None while keys: handle = SolidHandle(keys.pop(0), definition_name=None, parent=handle) return DagsterEvent( event_type_value=event_type.value, pipeline_name=pipeline_name, step_key=event_dict['step']['key'], solid_handle=handle, step_kind_value=event_dict['step']['kind'], logging_tags=None, event_specific_data=event_specific_data, )
def dagster_event_from_dict(event_dict, pipeline_name): check.dict_param(event_dict, 'event_dict', key_type=str) check.str_param(pipeline_name, 'pipeline_name') # Get event_type event_type = _handled_events().get(event_dict['__typename']) if not event_type: raise Exception('unhandled event type %s' % event_dict['__typename']) # Get event_specific_data event_specific_data = None if event_type == DagsterEventType.STEP_OUTPUT: event_specific_data = StepOutputData( step_output_handle=StepOutputHandle(event_dict['step']['key'], event_dict['outputName']), value_repr=event_dict['valueRepr'], type_check_data=TypeCheckData( success=event_dict['typeCheck']['success'], label=event_dict['typeCheck']['label'], description=event_dict.get('description'), metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ), ) elif event_type == DagsterEventType.STEP_INPUT: event_specific_data = StepInputData( input_name=event_dict['inputName'], value_repr=event_dict['valueRepr'], type_check_data=TypeCheckData( success=event_dict['typeCheck']['success'], label=event_dict['typeCheck']['label'], description=event_dict.get('description'), metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ), ) elif event_type == DagsterEventType.STEP_SUCCESS: event_specific_data = StepSuccessData(0.0) elif event_type == DagsterEventType.STEP_MATERIALIZATION: materialization = event_dict['materialization'] event_specific_data = StepMaterializationData( materialization=materialization_from_data(materialization)) elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT: expectation_result = expectation_result_from_data( event_dict['expectationResult']) event_specific_data = StepExpectationResultData(expectation_result) elif event_type == DagsterEventType.STEP_FAILURE: error_info = SerializableErrorInfo(event_dict['error']['message'], stack=None, cls_name=None) event_specific_data = StepFailureData( error_info, UserFailureData( label=event_dict['failureMetadata']['label'], description=event_dict['failureMetadata']['description'], metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or []), ) if event_dict.get('failureMetadata') else None, ) return DagsterEvent( event_type_value=event_type.value, pipeline_name=pipeline_name, step_key=event_dict['step']['key'], solid_handle=SolidHandle(event_dict['step']['solidHandleID'], None, None), step_kind_value=event_dict['step']['kind'], logging_tags=None, event_specific_data=event_specific_data, )
def dagster_event_from_dict(event_dict, pipeline_name): check.dict_param(event_dict, 'event_dict', key_type=str) check.str_param(pipeline_name, 'pipeline_name') # Get event_type event_type = HANDLED_EVENTS.get(event_dict['__typename']) if not event_type: raise Exception('unhandled event type %s' % event_dict['__typename']) # Get event_specific_data event_specific_data = None if event_type == DagsterEventType.STEP_OUTPUT: event_specific_data = StepOutputData( step_output_handle=StepOutputHandle(event_dict['stepKey'], event_dict['outputName']), type_check_data=TypeCheckData( success=event_dict['typeCheck']['success'], label=event_dict['typeCheck']['label'], description=event_dict.get('description'), metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or [] ), ), ) elif event_type == DagsterEventType.STEP_INPUT: event_specific_data = StepInputData( input_name=event_dict['inputName'], type_check_data=TypeCheckData( success=event_dict['typeCheck']['success'], label=event_dict['typeCheck']['label'], description=event_dict.get('description'), metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or [] ), ), ) elif event_type == DagsterEventType.STEP_SUCCESS: event_specific_data = StepSuccessData(0.0) elif event_type == DagsterEventType.STEP_UP_FOR_RETRY: event_specific_data = StepRetryData( error=error_from_data(event_dict['retryError']), seconds_to_wait=event_dict['secondsToWait'], ) elif event_type == DagsterEventType.STEP_MATERIALIZATION: materialization = event_dict['materialization'] event_specific_data = StepMaterializationData( materialization=materialization_from_data(materialization) ) elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT: expectation_result = expectation_result_from_data(event_dict['expectationResult']) event_specific_data = StepExpectationResultData(expectation_result) elif event_type == DagsterEventType.STEP_FAILURE: event_specific_data = StepFailureData( error_from_data(event_dict['error']), UserFailureData( label=event_dict['failureMetadata']['label'], description=event_dict['failureMetadata']['description'], metadata_entries=list( event_metadata_entries(event_dict.get('metadataEntries')) or [] ), ) if event_dict.get('failureMetadata') else None, ) elif event_type == DagsterEventType.ENGINE_EVENT: event_specific_data = EngineEventData( metadata_entries=list(event_metadata_entries(event_dict.get('metadataEntries'))), marker_start=event_dict.get('markerStart'), marker_end=event_dict.get('markerEnd'), error=error_from_data(event_dict['engineError']) if event_dict.get('engineError') else None, ) return DagsterEvent( event_type_value=event_type.value, pipeline_name=pipeline_name, step_key=event_dict.get('stepKey'), solid_handle=SolidHandle.from_string(event_dict['solidHandleID']) if event_dict.get('solidHandleID') else None, # at the time of writing this: # * 'COMPUTE` is the only step kind # * this code should get deleted in the near future as we move away from # dagster-graphql CLI as what we invoke in dask/k8s/etc. step_kind_value='COMPUTE' if event_dict.get('stepKey') else None, logging_tags=None, event_specific_data=event_specific_data, )