def _yield_transform_results(compute_context, inputs, compute_fn): check.inst_param(compute_context, 'compute_context', SystemComputeExecutionContext) step = compute_context.step gen = compute_fn(ComputeExecutionContext(compute_context), inputs) if isinstance(gen, Output): raise DagsterInvariantViolationError(( 'Transform for solid {solid_name} returned a Output rather than ' 'yielding it. The compute_fn of the core SolidDefinition must yield ' 'its results').format(solid_name=str(step.solid_handle))) if gen is None: return for result in gen: if isinstance(result, Output): value_repr = repr(result.value) compute_context.log.info( 'Solid {solid} emitted output "{output}" value {value}'.format( solid=str(step.solid_handle), output=result.output_name, value=value_repr[:200] + '...' if len(value_repr) > 200 else value_repr, )) yield Output(output_name=result.output_name, value=result.value) elif isinstance(result, (Materialization, ExpectationResult)): yield result else: raise DagsterInvariantViolationError( ('Transform for solid {solid_name} yielded {result} rather an ' 'an instance of the Output or Materialization class.').format( result=repr(result), solid_name=str(step.solid_handle)))
def _do_expectation(expectation_context, inputs): check.inst_param(expectation_context, 'step_context', SystemStepExecutionContext) value = inputs[EXPECTATION_INPUT] expectation_context = expectation_context.for_expectation(inout_def, expectation_def) expt_result = expectation_def.expectation_fn( ExpectationExecutionContext(expectation_context), value ) if not isinstance(expt_result, ExpectationResult): raise DagsterInvariantViolationError( ( 'Expectation for solid {solid_name} on {desc_key} {inout_name} ' 'did not return an ExpectationResult'.format( solid_name=solid.name, desc_key=inout_def.descriptive_key, inout_name=inout_def.name, ) ) ) if expt_result.success: yield expt_result yield Output(output_name=internal_output_name, value=inputs[EXPECTATION_INPUT]) else: expectation_context.log.debug( 'Expectation {key} failed on {value}.'.format( key=expectation_context.step.key, value=value ) ) raise DagsterIOExpectationFailedError(expectation_context, value)
def _step_output_error_checked_user_event_sequence(step_context, user_event_sequence): ''' Process the event sequence to check for invariant violations in the event sequence related to Output events emitted from the compute_fn. This consumes and emits an event sequence. ''' check.inst_param(step_context, 'step_context', SystemStepExecutionContext) check.generator_param(user_event_sequence, 'user_event_sequence') step = step_context.step output_names = list([output_def.name for output_def in step.step_outputs]) seen_outputs = set() for user_event in user_event_sequence: if not isinstance(user_event, Output): yield user_event continue # do additional processing on Outputs output = user_event if not step.has_step_output(output.output_name): raise DagsterInvariantViolationError( 'Core compute for solid "{handle}" returned an output ' '"{output.output_name}" that does not exist. The available ' 'outputs are {output_names}'.format( handle=str(step.solid_handle), output=output, output_names=output_names ) ) if output.output_name in seen_outputs: raise DagsterInvariantViolationError( 'Core compute for solid "{handle}" returned an output ' '"{output.output_name}" multiple times'.format( handle=str(step.solid_handle), output=output ) ) yield output seen_outputs.add(output.output_name) for step_output_def in step.step_outputs: if not step_output_def.name in seen_outputs and not step_output_def.optional: if step_output_def.runtime_type.is_nothing: step_context.log.info( 'Emitting implicit Nothing for output "{output}" on solid {solid}'.format( output=step_output_def.name, solid={str(step.solid_handle)} ) ) yield Output(output_name=step_output_def.name, value=None) else: raise DagsterStepOutputNotFoundError( 'Core compute for solid "{handle}" did not return an output ' 'for non-optional output "{step_output_def.name}"'.format( handle=str(step.solid_handle), step_output_def=step_output_def ), step_key=step.key, output_name=step_output_def.name, )
def _yield_compute_results(compute_context, inputs, compute_fn): check.inst_param(compute_context, 'compute_context', SystemComputeExecutionContext) step = compute_context.step gen = compute_fn(ComputeExecutionContext(compute_context), inputs) if isinstance(gen, Output): raise DagsterInvariantViolationError(( 'Compute function for solid {solid_name} returned a Output rather than ' 'yielding it. The compute_fn of the core SolidDefinition must yield ' 'its results').format(solid_name=str(step.solid_handle))) if gen is None: return for result in gen: if isinstance(result, Output): value_repr = repr(result.value) compute_context.log.info( "Solid '{solid}' emitted output '{output}' value {value}". format( solid=str(step.solid_handle), output=result.output_name, value=value_repr[:200] + '...' if len(value_repr) > 200 else value_repr, )) yield Output(output_name=result.output_name, value=result.value) elif isinstance(result, Materialization): compute_context.log.info( "Solid '{solid}' materialized '{label}'".format( solid=str(step.solid_handle), label=result.label)) yield result elif isinstance(result, ExpectationResult): compute_context.log.info( "Solid '{solid}' {status} expectation '{label}'".format( solid=str(step.solid_handle), status='passed' if result.success else 'failed', label=result.label, )) yield result else: raise DagsterInvariantViolationError(( 'Compute function for solid {solid_name} yielded {result} rather than ' 'an instance of the Output or Materialization class.').format( result=repr(result), solid_name=str(step.solid_handle)))
def __empty_join(_context, _inputs): yield Output(output_name=JOIN_OUTPUT, value=None)
def __merge_join(_context, inputs): yield Output(output_name=JOIN_OUTPUT, value=list(inputs.values()))
def _validate_and_coerce_solid_result_to_iterator(result, context, output_defs): if isinstance(result, (AssetMaterialization, Materialization, ExpectationResult)): raise DagsterInvariantViolationError(( "Error in {described_op}: If you are returning an AssetMaterialization " "or an ExpectationResult from {node_type} you must yield them to avoid " "ambiguity with an implied result from returning a value.".format( described_op=context.describe_op(), node_type=context.solid_def.node_type_str, ))) if inspect.isgenerator(result): # this happens when a user explicitly returns a generator in the solid for event in result: yield event elif isinstance(result, Output): yield result elif len(output_defs) == 1: if result is None and output_defs[0].is_required is False: context.log.warn( 'Value "None" returned for non-required output "{output_name}" of {described_op}. ' "This value will be passed to downstream {node_type}s. For conditional execution use\n" ' yield Output(value, "{output_name}")\n' "when you want the downstream {node_type}s to execute, " "and do not yield it when you want downstream solids to skip.". format( output_name=output_defs[0].name, described_op=context.describe_op(), node_type=context.solid_def.node_type_str, )) yield Output(value=result, output_name=output_defs[0].name) elif len(output_defs) > 1 and isinstance(result, tuple): if len(result) != len(output_defs): check.failed( f"Solid '{context.solid_name}' has {len(output_defs)} output definitions, but " f"returned a tuple with {len(result)} elements") for output_defs, element in zip(output_defs, result): yield Output(output_name=output_defs.name, value=element) elif result is not None: if not output_defs: raise DagsterInvariantViolationError(( "Error in {described_op}: Unexpectedly returned output {result} " "of type {type_}. {node_type} is explicitly defined to return no " "results.").format( described_op=context.describe_op(), result=result, type_=type(result), node_type=context.solid_def.node_type_str.capitalize(), )) raise DagsterInvariantViolationError( ("Error in {described_op}: {node_type} unexpectedly returned " "output {result} of type {type_}. Should " "be a generator, containing or yielding " "{n_results} results: {{{expected_results}}}.").format( described_op=context.describe_op(), node_type=context.solid_def.node_type_str, result=result, type_=type(result), n_results=len(output_defs), expected_results=", ".join([ "'{result_name}': {dagster_type}".format( result_name=output_def.name, dagster_type=output_def.dagster_type, ) for output_def in output_defs ]), ))
def _step_output_error_checked_user_event_sequence( step_context: SystemStepExecutionContext, user_event_sequence: Iterator[SolidOutputUnion] ) -> Iterator[SolidOutputUnion]: """ Process the event sequence to check for invariant violations in the event sequence related to Output events emitted from the compute_fn. This consumes and emits an event sequence. """ check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.generator_param(user_event_sequence, "user_event_sequence") step = step_context.step output_names = list([output_def.name for output_def in step.step_outputs]) seen_outputs: Set[str] = set() seen_mapping_keys: Dict[str, Set[str]] = defaultdict(set) for user_event in user_event_sequence: if not isinstance(user_event, (Output, DynamicOutput)): yield user_event continue # do additional processing on Outputs output = user_event if not step.has_step_output(output.output_name): raise DagsterInvariantViolationError( 'Core compute for solid "{handle}" returned an output ' '"{output.output_name}" that does not exist. The available ' "outputs are {output_names}".format(handle=str( step.solid_handle), output=output, output_names=output_names)) step_output = step.step_output_named(output.output_name) output_def = step_context.pipeline_def.get_solid( step_output.solid_handle).output_def_named(step_output.name) if isinstance(output, Output): if output.output_name in seen_outputs: raise DagsterInvariantViolationError( 'Compute for solid "{handle}" returned an output ' '"{output.output_name}" multiple times'.format( handle=str(step.solid_handle), output=output)) if output_def.is_dynamic: raise DagsterInvariantViolationError( f'Compute for solid "{step.solid_handle}" for output "{output.output_name}" ' "defined as dynamic must yield DynamicOutput, got Output.") else: if not output_def.is_dynamic: raise DagsterInvariantViolationError( f'Compute for solid "{step.solid_handle}" yielded a DynamicOutput, ' "but did not use DynamicOutputDefinition.") if output.mapping_key in seen_mapping_keys[output.output_name]: raise DagsterInvariantViolationError( f'Compute for solid "{step.solid_handle}" yielded a DynamicOutput with ' f'mapping_key "{output.mapping_key}" multiple times.') seen_mapping_keys[output.output_name].add(output.mapping_key) yield output seen_outputs.add(output.output_name) for step_output in step.step_outputs: step_output_def = step_context.solid_def.output_def_named( step_output.name) if not step_output_def.name in seen_outputs and not step_output_def.optional: if step_output_def.dagster_type.kind == DagsterTypeKind.NOTHING: step_context.log.info( 'Emitting implicit Nothing for output "{output}" on solid {solid}' .format(output=step_output_def.name, solid={str(step.solid_handle)})) yield Output(output_name=step_output_def.name, value=None) else: raise DagsterStepOutputNotFoundError( 'Core compute for solid "{handle}" did not return an output ' 'for non-optional output "{step_output_def.name}"'.format( handle=str(step.solid_handle), step_output_def=step_output_def), step_key=step.key, output_name=step_output_def.name, )
def _step_output_error_checked_user_event_sequence( step_context: StepExecutionContext, user_event_sequence: Iterator[SolidOutputUnion] ) -> Iterator[SolidOutputUnion]: """ Process the event sequence to check for invariant violations in the event sequence related to Output events emitted from the compute_fn. This consumes and emits an event sequence. """ check.inst_param(step_context, "step_context", StepExecutionContext) check.generator_param(user_event_sequence, "user_event_sequence") step = step_context.step op_label = step_context.describe_op() output_names = list([output_def.name for output_def in step.step_outputs]) for user_event in user_event_sequence: if not isinstance(user_event, (Output, DynamicOutput)): yield user_event continue # do additional processing on Outputs output = user_event if not step.has_step_output(cast(str, output.output_name)): raise DagsterInvariantViolationError( f'Core compute for {op_label} returned an output "{output.output_name}" that does ' f"not exist. The available outputs are {output_names}" ) step_output = step.step_output_named(cast(str, output.output_name)) output_def = step_context.pipeline_def.get_solid(step_output.solid_handle).output_def_named( step_output.name ) if isinstance(output, Output): if step_context.has_seen_output(output.output_name): raise DagsterInvariantViolationError( f'Compute for {op_label} returned an output "{output.output_name}" multiple ' "times" ) if output_def.is_dynamic: raise DagsterInvariantViolationError( f'Compute for {op_label} for output "{output.output_name}" defined as dynamic ' "must yield DynamicOutput, got Output." ) step_context.observe_output(output.output_name) metadata = step_context.get_output_metadata(output.output_name) output = Output( value=output.value, output_name=output.output_name, metadata_entries=output.metadata_entries + normalize_metadata(cast(Dict[str, Any], metadata), []), ) else: if not output_def.is_dynamic: raise DagsterInvariantViolationError( f"Compute for {op_label} yielded a DynamicOutput, but did not use " "DynamicOutputDefinition." ) if step_context.has_seen_output(output.output_name, output.mapping_key): raise DagsterInvariantViolationError( f"Compute for {op_label} yielded a DynamicOutput with mapping_key " f'"{output.mapping_key}" multiple times.' ) step_context.observe_output(output.output_name, output.mapping_key) metadata = step_context.get_output_metadata( output.output_name, mapping_key=output.mapping_key ) output = DynamicOutput( value=output.value, output_name=output.output_name, metadata_entries=output.metadata_entries + normalize_metadata(cast(Dict[str, Any], metadata), []), mapping_key=output.mapping_key, ) yield output for step_output in step.step_outputs: step_output_def = step_context.solid_def.output_def_named(step_output.name) if not step_context.has_seen_output(step_output_def.name) and not step_output_def.optional: if step_output_def.dagster_type.kind == DagsterTypeKind.NOTHING: step_context.log.info( f'Emitting implicit Nothing for output "{step_output_def.name}" on {op_label}' ) yield Output(output_name=step_output_def.name, value=None) elif not step_output_def.is_dynamic: raise DagsterStepOutputNotFoundError( ( f"Core compute for {op_label} did not return an output for non-optional " f'output "{step_output_def.name}"' ), step_key=step.key, output_name=step_output_def.name, )
def __select_first_join(_context, inputs): yield Output(output_name=JOIN_OUTPUT, value=list(inputs.values())[0])