def dagster_event_from_dict(event_dict, pipeline_name): check.dict_param(event_dict, 'event_dict', key_type=str) check.str_param(pipeline_name, 'pipeline_name') materialization = event_dict.get('intermediateMaterialization') or {} # Get event_type event_type = _handled_events().get(event_dict['__typename']) if not event_type: raise Exception('unhandled event type %s' % event_dict['__typename']) # Get event_specific_data event_specific_data = None if event_type == DagsterEventType.STEP_OUTPUT: event_specific_data = StepOutputData( step_output_handle=StepOutputHandle(event_dict['step']['key'], event_dict['outputName']), value_repr=event_dict['valueRepr'], intermediate_materialization=Materialization( path=materialization.get('path'), description=materialization.get('description')), ) elif event_type == DagsterEventType.STEP_SUCCESS: event_specific_data = StepSuccessData(0.0) elif event_type == DagsterEventType.STEP_MATERIALIZATION: event_specific_data = StepMaterializationData( materialization=Materialization(path=materialization.get('path'), description=materialization.get( 'description'))) elif event_type == DagsterEventType.STEP_EXPECTATION_RESULT: result_metadata = event_dict['expectationResult'][ 'resultMetadataJsonString'] expectation_result = ExpectationResult( event_dict['expectationResult']['success'], event_dict['expectationResult']['name'], event_dict['expectationResult']['message'], json.loads(result_metadata) if result_metadata else None, ) event_specific_data = StepExpectationResultData(expectation_result) elif event_type == DagsterEventType.STEP_FAILURE: error_info = SerializableErrorInfo(event_dict['error']['message'], stack=None, cls_name=None) event_specific_data = StepFailureData(error_info) return DagsterEvent( event_type_value=event_type.value, pipeline_name=pipeline_name, step_key=event_dict['step']['key'], solid_handle=SolidHandle(event_dict['step']['solidHandleID'], None, None), step_kind_value=event_dict['step']['kind'], logging_tags=None, event_specific_data=event_specific_data, )
def materialization_from_data(data): return Materialization( label=data['label'], description=data.get('description'), # enforce? metadata_entries=list( event_metadata_entries(data.get('metadataEntries')) or []), )
def _fn(step_context, inputs): runtime_value = inputs[MATERIALIZATION_THUNK_INPUT] path = output_def.runtime_type.output_schema.materialize_runtime_value( step_context, config_spec, runtime_value ) if not isinstance(path, six.string_types): raise DagsterInvariantViolationError( ( 'materialize_runtime_value on type {type_name} has returned ' 'value {value} of type {python_type}. You must return a ' 'string (and ideally a valid file path).' ).format( type_name=output_def.runtime_type.name, value=repr(path), python_type=type(path).__name__, ) ) yield StepOutputValue(output_name=MATERIALIZATION_THUNK_OUTPUT, value=runtime_value) yield Materialization( path=path, description=('Materialization of {solid_name}.{output_name}').format( output_name=output_def.name, solid_name=str(step_context.solid_handle) ), )
def materialization_from_data(data): if data.get("asset_key"): return AssetMaterialization( asset_key=data["asset_key"], description=data.get("description"), metadata_entries=list(event_metadata_entries(data.get("metadataEntries")) or []), ) return Materialization( label=data["label"], description=data.get("description"), # enforce? metadata_entries=list(event_metadata_entries(data.get("metadataEntries")) or []), )
def _create_step_output_event(step_context, result, intermediates_manager): check.inst_param(step_context, 'step_context', SystemStepExecutionContext) check.inst_param(result, 'result', Result) check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager) step = step_context.step step_output = step.step_output_named(result.output_name) try: value = step_output.runtime_type.coerce_runtime_value(result.value) step_output_handle = StepOutputHandle.from_step(step=step, output_name=result.output_name) object_key = intermediates_manager.set_intermediate( context=step_context, runtime_type=step_output.runtime_type, step_output_handle=step_output_handle, value=value, ) return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, value_repr=repr(value), intermediate_materialization=Materialization(path=object_key) if object_key else None, ), ) except DagsterRuntimeCoercionError as e: raise DagsterInvariantViolationError( ( 'In solid "{handle}" the output "{output_name}" returned ' 'an invalid type: {error_msg}.' ).format( handle=str(step.solid_handle), error_msg=','.join(e.args), output_name=result.output_name, ) )
def _stats_records(run_id): now = time.time() return [ _event_record(run_id, 'A', now - 325, DagsterEventType.STEP_START), _event_record( run_id, 'A', now - 225, DagsterEventType.STEP_SUCCESS, StepSuccessData(duration_ms=100000.0), ), _event_record(run_id, 'B', now - 225, DagsterEventType.STEP_START), _event_record( run_id, 'B', now - 175, DagsterEventType.STEP_FAILURE, StepFailureData(error=None, user_failure_data=None), ), _event_record(run_id, 'C', now - 175, DagsterEventType.STEP_START), _event_record(run_id, 'C', now - 150, DagsterEventType.STEP_SKIPPED), _event_record(run_id, 'D', now - 150, DagsterEventType.STEP_START), _event_record( run_id, 'D', now - 125, DagsterEventType.STEP_MATERIALIZATION, StepMaterializationData(Materialization(label='mat 1')), ), _event_record( run_id, 'D', now - 100, DagsterEventType.STEP_EXPECTATION_RESULT, StepExpectationResultData(ExpectationResult(success=True, label='exp 1')), ), _event_record( run_id, 'D', now - 75, DagsterEventType.STEP_MATERIALIZATION, StepMaterializationData(Materialization(label='mat 2')), ), _event_record( run_id, 'D', now - 50, DagsterEventType.STEP_EXPECTATION_RESULT, StepExpectationResultData(ExpectationResult(success=False, label='exp 2')), ), _event_record( run_id, 'D', now - 25, DagsterEventType.STEP_MATERIALIZATION, StepMaterializationData(Materialization(label='mat 3')), ), _event_record( run_id, 'D', now, DagsterEventType.STEP_SUCCESS, StepSuccessData(duration_ms=150000.0) ), ]
def test_materialization(): assert isinstance(Materialization('foo', 'foo.txt'), Materialization)
def test_materialization_assign_label_from_asset_key(): mat = Materialization(asset_key=AssetKey(['foo', 'bar'])) assert mat.label == 'foo.bar'