def _yield_transform_results(transform_context, inputs, compute_fn): check.inst_param(transform_context, 'transform_context', SystemTransformExecutionContext) step = transform_context.step gen = compute_fn(TransformExecutionContext(transform_context), inputs) if isinstance(gen, Result): raise DagsterInvariantViolationError(( 'Transform for solid {solid_name} returned a Result rather than ' 'yielding it. The compute_fn of the core SolidDefinition must yield ' 'its results').format(solid_name=str(step.solid_handle))) if gen is None: return for result in gen: if isinstance(result, Result): transform_context.log.info( 'Solid {solid} emitted output "{output}" value {value}'.format( solid=str(step.solid_handle), output=result.output_name, value=repr(result.value), )) yield Result(output_name=result.output_name, value=result.value) elif isinstance(result, (Materialization, ExpectationResult)): yield result else: raise DagsterInvariantViolationError( ('Transform for solid {solid_name} yielded {result} rather an ' 'an instance of the Result or Materialization class.').format( result=repr(result), solid_name=str(step.solid_handle)))
def _do_expectation(expectation_context, inputs): check.inst_param(expectation_context, 'step_context', SystemStepExecutionContext) value = inputs[EXPECTATION_INPUT] expectation_context = expectation_context.for_expectation( inout_def, expectation_def) expt_result = expectation_def.expectation_fn( ExpectationExecutionContext(expectation_context), value) if not isinstance(expt_result, ExpectationResult): raise DagsterInvariantViolationError(( 'Expectation for solid {solid_name} on {desc_key} {inout_name} ' 'did not return an ExpectationResult'.format( solid_name=solid.name, desc_key=inout_def.descriptive_key, inout_name=inout_def.name, ))) if expt_result.success: expectation_context.log.debug( 'Expectation {key} succeeded on {value}.'.format( key=expectation_context.step.key, value=value)) yield expt_result yield Result(output_name=internal_output_name, value=inputs[EXPECTATION_INPUT]) else: expectation_context.log.debug( 'Expectation {key} failed on {value}.'.format( key=expectation_context.step.key, value=value)) raise DagsterExpectationFailedError(expectation_context, value)
def _compute_fn(context, _step, _inputs): yield Result( context.persistence_policy.read_value( step_input.runtime_type.serialization_strategy, marshalling_key ), UNMARSHAL_INPUT_OUTPUT, )
def _step_output_error_checked_event_sequence(step_context, event_sequence): ''' Process the event sequence to check for invariant violations in the event sequence related to Result events emitted from the compute_fn. This consumes and emits an event sequence. ''' check.inst_param(step_context, 'step_context', SystemStepExecutionContext) check.generator_param(event_sequence, 'event_sequence') step = step_context.step output_names = list([output_def.name for output_def in step.step_outputs]) seen_outputs = set() for event in event_sequence: if not isinstance(event, Result): yield event continue # do additional processing on Results result = event if not step.has_step_output(result.output_name): raise DagsterInvariantViolationError( 'Core compute for solid "{handle}" returned an output ' '"{result.output_name}" that does not exist. The available ' 'outputs are {output_names}'.format( handle=str(step.solid_handle), result=result, output_names=output_names ) ) if result.output_name in seen_outputs: raise DagsterInvariantViolationError( 'Core compute for solid "{handle}" returned an output ' '"{result.output_name}" multiple times'.format( handle=str(step.solid_handle), result=result ) ) yield result seen_outputs.add(result.output_name) for step_output_def in step.step_outputs: if not step_output_def.name in seen_outputs and not step_output_def.optional: if step_output_def.runtime_type.is_nothing: step_context.log.info( 'Emitting implicit Nothing for output "{output}" on solid {solid}'.format( output=step_output_def.name, solid={str(step.solid_handle)} ) ) yield Result(output_name=step_output_def.name, value=None) else: raise DagsterStepOutputNotFoundError( 'Core compute for solid "{handle}" did not return an output ' 'for non-optional output "{step_output_def.name}"'.format( handle=str(step.solid_handle), step_output_def=step_output_def ), step_key=step.key, output_name=step_output_def.name, )
def _do_expectation(context, step, inputs): value = inputs[EXPECTATION_INPUT] info = ExpectationExecutionInfo(context, inout_def, solid, expectation_def) expt_result = expectation_def.expectation_fn(info, value) if expt_result.success: context.debug( 'Expectation {key} succeeded on {value}.'.format(key=step.key, value=value) ) yield Result(output_name=internal_output_name, value=inputs[EXPECTATION_INPUT]) else: context.debug('Expectation {key} failed on {value}.'.format(key=step.key, value=value)) raise DagsterExpectationFailedError(info, value)
def _fn(step_context, inputs): runtime_value = inputs[MATERIALIZATION_THUNK_INPUT] path = output_def.runtime_type.output_schema.materialize_runtime_value( step_context, config_spec, runtime_value) if not isinstance(path, six.string_types): raise DagsterInvariantViolationError( ('materialize_runtime_value on type {type_name} has returned ' 'value {value} of type {python_type}. You must return a ' 'string (and ideally a valid file path).').format( type_name=output_def.runtime_type.name, value=repr(path), python_type=type(path).__name__, )) yield Result(output_name=MATERIALIZATION_THUNK_OUTPUT, value=runtime_value) yield Materialization( path=path, description=( 'Materialization of {solid_name}.{output_name}').format( output_name=output_def.name, solid_name=str(step_context.solid_handle)), )
def _fn(_context, _step, _inputs): value = input_def.runtime_type.input_schema.construct_from_config_value( config_value) yield Result(value, INPUT_THUNK_OUTPUT)
def _fn(_info, _step, inputs): runtime_value = inputs[MATERIALIZATION_THUNK_INPUT] runtime_type.output_schema.materialize_runtime_value( config_spec, runtime_value) yield Result(runtime_value, MATERIALIZATION_THUNK_OUTPUT)
def _fn(_context, _step, _inputs): yield Result(value, VALUE_OUTPUT)
def __join_lambda(_context, _step, inputs): yield Result(output_name=JOIN_OUTPUT, value=list(inputs.values())[0])
def _fn(step_context, _inputs): value = input_def.runtime_type.input_schema.construct_from_config_value( step_context, input_spec) yield Result(output_name=INPUT_THUNK_OUTPUT, value=value)
def __empty_join(_context, _inputs): yield Result(output_name=JOIN_OUTPUT, value=None)
def __merge_join(_context, inputs): yield Result(output_name=JOIN_OUTPUT, value=list(inputs.values()))
def __select_first_join(_context, inputs): yield Result(output_name=JOIN_OUTPUT, value=list(inputs.values())[0])