def _yield_transform_results(transform_context, inputs, compute_fn):
check.inst_param(transform_context, 'transform_context',
SystemTransformExecutionContext)
step = transform_context.step
gen = compute_fn(TransformExecutionContext(transform_context), inputs)
if isinstance(gen, Result):
raise DagsterInvariantViolationError((
'Transform for solid {solid_name} returned a Result rather than '
'yielding it. The compute_fn of the core SolidDefinition must yield '
'its results').format(solid_name=str(step.solid_handle)))
if gen is None:
return
for result in gen:
if isinstance(result, Result):
transform_context.log.info(
'Solid {solid} emitted output "{output}" value {value}'.format(
solid=str(step.solid_handle),
output=result.output_name,
value=repr(result.value),
))
yield Result(output_name=result.output_name, value=result.value)
elif isinstance(result, (Materialization, ExpectationResult)):
yield result
else:
raise DagsterInvariantViolationError(
('Transform for solid {solid_name} yielded {result} rather an '
'an instance of the Result or Materialization class.').format(
result=repr(result), solid_name=str(step.solid_handle)))
def _do_expectation(expectation_context, inputs):
check.inst_param(expectation_context, 'step_context',
SystemStepExecutionContext)
value = inputs[EXPECTATION_INPUT]
expectation_context = expectation_context.for_expectation(
inout_def, expectation_def)
expt_result = expectation_def.expectation_fn(
ExpectationExecutionContext(expectation_context), value)
if not isinstance(expt_result, ExpectationResult):
raise DagsterInvariantViolationError((
'Expectation for solid {solid_name} on {desc_key} {inout_name} '
'did not return an ExpectationResult'.format(
solid_name=solid.name,
desc_key=inout_def.descriptive_key,
inout_name=inout_def.name,
)))
if expt_result.success:
expectation_context.log.debug(
'Expectation {key} succeeded on {value}.'.format(
key=expectation_context.step.key, value=value))
yield expt_result
yield Result(output_name=internal_output_name,
value=inputs[EXPECTATION_INPUT])
else:
expectation_context.log.debug(
'Expectation {key} failed on {value}.'.format(
key=expectation_context.step.key, value=value))
raise DagsterExpectationFailedError(expectation_context, value)
def _compute_fn(context, _step, _inputs):
yield Result(
context.persistence_policy.read_value(
step_input.runtime_type.serialization_strategy, marshalling_key
),
UNMARSHAL_INPUT_OUTPUT,
)
def _step_output_error_checked_event_sequence(step_context, event_sequence):
'''
Process the event sequence to check for invariant violations in the event
sequence related to Result events emitted from the compute_fn.
This consumes and emits an event sequence.
'''
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.generator_param(event_sequence, 'event_sequence')
step = step_context.step
output_names = list([output_def.name for output_def in step.step_outputs])
seen_outputs = set()
for event in event_sequence:
if not isinstance(event, Result):
yield event
continue
# do additional processing on Results
result = event
if not step.has_step_output(result.output_name):
raise DagsterInvariantViolationError(
'Core compute for solid "{handle}" returned an output '
'"{result.output_name}" that does not exist. The available '
'outputs are {output_names}'.format(
handle=str(step.solid_handle), result=result, output_names=output_names
)
)
if result.output_name in seen_outputs:
raise DagsterInvariantViolationError(
'Core compute for solid "{handle}" returned an output '
'"{result.output_name}" multiple times'.format(
handle=str(step.solid_handle), result=result
)
)
yield result
seen_outputs.add(result.output_name)
for step_output_def in step.step_outputs:
if not step_output_def.name in seen_outputs and not step_output_def.optional:
if step_output_def.runtime_type.is_nothing:
step_context.log.info(
'Emitting implicit Nothing for output "{output}" on solid {solid}'.format(
output=step_output_def.name, solid={str(step.solid_handle)}
)
)
yield Result(output_name=step_output_def.name, value=None)
else:
raise DagsterStepOutputNotFoundError(
'Core compute for solid "{handle}" did not return an output '
'for non-optional output "{step_output_def.name}"'.format(
handle=str(step.solid_handle), step_output_def=step_output_def
),
step_key=step.key,
output_name=step_output_def.name,
)
def _do_expectation(context, step, inputs):
value = inputs[EXPECTATION_INPUT]
info = ExpectationExecutionInfo(context, inout_def, solid, expectation_def)
expt_result = expectation_def.expectation_fn(info, value)
if expt_result.success:
context.debug(
'Expectation {key} succeeded on {value}.'.format(key=step.key, value=value)
)
yield Result(output_name=internal_output_name, value=inputs[EXPECTATION_INPUT])
else:
context.debug('Expectation {key} failed on {value}.'.format(key=step.key, value=value))
raise DagsterExpectationFailedError(info, value)
def _fn(step_context, inputs):
runtime_value = inputs[MATERIALIZATION_THUNK_INPUT]
path = output_def.runtime_type.output_schema.materialize_runtime_value(
step_context, config_spec, runtime_value)
if not isinstance(path, six.string_types):
raise DagsterInvariantViolationError(
('materialize_runtime_value on type {type_name} has returned '
'value {value} of type {python_type}. You must return a '
'string (and ideally a valid file path).').format(
type_name=output_def.runtime_type.name,
value=repr(path),
python_type=type(path).__name__,
))
yield Result(output_name=MATERIALIZATION_THUNK_OUTPUT,
value=runtime_value)
yield Materialization(
path=path,
description=(
'Materialization of {solid_name}.{output_name}').format(
output_name=output_def.name,
solid_name=str(step_context.solid_handle)),
)
def _fn(_context, _step, _inputs):
value = input_def.runtime_type.input_schema.construct_from_config_value(
config_value)
yield Result(value, INPUT_THUNK_OUTPUT)
def _fn(_info, _step, inputs):
runtime_value = inputs[MATERIALIZATION_THUNK_INPUT]
runtime_type.output_schema.materialize_runtime_value(
config_spec, runtime_value)
yield Result(runtime_value, MATERIALIZATION_THUNK_OUTPUT)
def _fn(_context, _step, _inputs):
yield Result(value, VALUE_OUTPUT)
def __join_lambda(_context, _step, inputs):
yield Result(output_name=JOIN_OUTPUT, value=list(inputs.values())[0])
def _fn(step_context, _inputs):
value = input_def.runtime_type.input_schema.construct_from_config_value(
step_context, input_spec)
yield Result(output_name=INPUT_THUNK_OUTPUT, value=value)
def __empty_join(_context, _inputs):
yield Result(output_name=JOIN_OUTPUT, value=None)
def __merge_join(_context, inputs):
yield Result(output_name=JOIN_OUTPUT, value=list(inputs.values()))
def __select_first_join(_context, inputs):
yield Result(output_name=JOIN_OUTPUT, value=list(inputs.values())[0])
