def output_for_node(self, node_str: str, output_name: Optional[str] = DEFAULT_OUTPUT) -> Any: """Retrieves output value with a particular name from the in-process run of the job. Args: node_str (str): Name of the op/graph whose output should be retrieved. If the intended graph/op is nested within another graph, the syntax is `outer_graph.inner_node`. output_name (Optional[str]): Name of the output on the op/graph to retrieve. Defaults to `result`, the default output name in dagster. Returns: Any: The value of the retrieved output. """ # resolve handle of node that node_str is referring to target_handle = NodeHandle.from_string(node_str) target_node_def = self._node_def.ensure_graph_def().get_solid( target_handle).definition origin_output_def, origin_handle = target_node_def.resolve_output_to_origin( output_name, NodeHandle.from_string(node_str)) # retrieve output value from resolved handle return _filter_outputs_by_handle(self._output_capture, origin_handle, origin_output_def.name)
def test_hook_decorator(): called_hook_to_solids = defaultdict(list) @success_hook def a_success_hook(context): called_hook_to_solids[context.hook_def.name].append(context.solid.name) @solid def a_solid(_): pass @a_success_hook @pipeline( description="i am a pipeline", mode_defs=[ModeDefinition(name="my_mode")], solid_retry_policy=RetryPolicy(max_retries=3), preset_defs=[PresetDefinition("my_empty_preset", mode="my_mode")], tags={"foo": "FOO"}, ) def a_pipeline(): a_solid() assert isinstance(a_pipeline, PipelineDefinition) assert a_pipeline.tags assert a_pipeline.tags.get("foo") == "FOO" assert a_pipeline.tags.get("foo") == "FOO" assert a_pipeline.description == "i am a pipeline" assert a_pipeline.has_mode_definition("my_mode") assert a_pipeline.has_preset("my_empty_preset") retry_policy = a_pipeline.get_retry_policy_for_handle( NodeHandle("a_solid", parent=None)) assert isinstance(retry_policy, RetryPolicy) assert retry_policy.max_retries == 3
def __new__( cls, sources: Sequence[StepInputSource], # deprecated, preserved for back-compat solid_handle: Optional[NodeHandle] = None, input_name: Optional[str] = None, ): check.list_param(sources, "sources", StepInputSource) for source in sources: check.invariant( not isinstance(source, FromMultipleSources), "Can not have multiple levels of FromMultipleSources StepInputSource", ) return super().__new__( cls, sources=sources, # add placeholder values for back-compat solid_handle=check.opt_inst_param(solid_handle, "solid_handle", NodeHandle, default=NodeHandle("", None)), input_name=check.opt_str_param(input_name, "input_handle", default=""), )
def _result_for_handle(self, solid: Node, handle: NodeHandle) -> NodeExecutionResult: node_def = solid.definition events_for_handle = _filter_step_events_by_handle( self.event_list, self.handle, handle) outputs_for_handle = (_filter_outputs_by_handle( self._output_capture, self.handle, handle) if self._output_capture else None) if self.handle: handle_with_ancestor = handle.with_ancestor(self.handle) else: handle_with_ancestor = handle if not handle_with_ancestor: raise DagsterInvariantViolationError( f"No handle provided for solid {solid.name}") if isinstance(node_def, SolidDefinition): return InProcessSolidResult( solid_def=node_def, handle=handle_with_ancestor, all_events=events_for_handle, output_capture=outputs_for_handle, ) elif isinstance(node_def, GraphDefinition): return InProcessGraphResult( graph_def=node_def, handle=handle_with_ancestor, all_events=events_for_handle, output_capture=outputs_for_handle, ) check.failed("Unhandled node type {node_def}")
def output_value(self, output_name=DEFAULT_OUTPUT): check.str_param(output_name, "output_name") if not self.solid.definition.has_output(output_name): raise DagsterInvariantViolationError( "Output '{output_name}' not defined in composite solid '{solid}': " "{outputs_clause}. If you were expecting this output to be present, you may " "be missing an output_mapping from an inner solid to its enclosing composite " "solid.".format( output_name=output_name, solid=self.solid.name, outputs_clause="found outputs {output_names}".format( output_names=str( list(self.solid.definition.output_dict.keys()))) if self.solid.definition.output_dict else "no output mappings were defined", )) output_mapping = self.solid.definition.get_output_mapping(output_name) return self._result_for_handle( self.solid.definition.solid_named( output_mapping.maps_from.solid_name), NodeHandle(output_mapping.maps_from.solid_name, None), ).output_value(output_mapping.maps_from.output_name)
def __new__( cls, step_output_handle: StepOutputHandle, # deprecated, preserved for back-compat solid_handle: Optional[NodeHandle] = None, input_name: Optional[str] = None, ): # Model the unknown mapping key from known execution step # using a StepOutputHandle with None mapping_key. check.inst_param(step_output_handle, "step_output_handle", StepOutputHandle) check.invariant(step_output_handle.mapping_key is None) return super().__new__( cls, step_output_handle=step_output_handle, # add placeholder values for back-compat solid_handle=check.opt_inst_param(solid_handle, "solid_handle", NodeHandle, default=NodeHandle("", None)), input_name=check.opt_str_param(input_name, "input_handle", default=""), )
def all_node_events(self) -> List[DagsterEvent]: """List[DagsterEvent]: All dagster events from the in-process execution.""" step_events = [] for node_name in self._node_def.ensure_graph_def().node_dict.keys(): handle = NodeHandle(node_name, None) step_events += _filter_events_by_handle(self._event_list, handle) return step_events
def result_for_node(self, name: str) -> NodeExecutionResult: """ The inner result for a node within the graph. """ if not self._graph_def.has_solid_named(name): raise DagsterInvariantViolationError( "Tried to get result for node '{name}' in '{container}'. No such top level " "node.".format(name=name, container=self._graph_def.name)) handle = NodeHandle(name, None) solid = self._graph_def.get_solid(handle) return self._result_for_handle(solid, handle)
def events_for_node(self, node_name: str) -> List[DagsterEvent]: """Retrieves all dagster events for a specific node. Args: node_name (str): The name of the node for which outputs should be retrieved. Returns: List[DagsterEvent]: A list of all dagster events associated with provided node name. """ check.str_param(node_name, "node_name") return _filter_events_by_handle(self._event_list, NodeHandle.from_string(node_name))
def output_for_solid(self, handle_str, output_name=DEFAULT_OUTPUT): """Get the output of a solid by its solid handle string and output name. Args: handle_str (str): The string handle for the solid. output_name (str): Optional. The name of the output, default to DEFAULT_OUTPUT. Returns: The output value for the handle and output_name. """ check.str_param(handle_str, "handle_str") check.str_param(output_name, "output_name") return self.result_for_handle( NodeHandle.from_string(handle_str)).output_value(output_name)
def test_multiline_logging_complex(): msg = "DagsterEventType.STEP_FAILURE for step start.materialization.output.result.0" kwargs = { "pipeline": "example", "pipeline_name": "error_monster", "step_key": "start.materialization.output.result.0", "solid": "start", "solid_definition": "emit_num", "dagster_event": DagsterEvent( event_type_value="STEP_FAILURE", pipeline_name="error_monster", step_key="start.materialization.output.result.0", solid_handle=NodeHandle("start", None), step_kind_value="MATERIALIZATION_THUNK", logging_tags={ "pipeline": "error_monster", "step_key": "start.materialization.output.result.0", "solid": "start", "solid_definition": "emit_num", }, event_specific_data=StepFailureData( error=SerializableErrorInfo( message="FileNotFoundError: [Errno 2] No such file or directory: '/path/to/file'\n", stack=["a stack message"], cls_name="FileNotFoundError", ), user_failure_data=None, ), ), } with _setup_logger(DAGSTER_DEFAULT_LOGGER) as (captured_results, logger): dl = DagsterLogManager("123", {}, [logger]) dl.info(msg, **kwargs) expected_results = [ "error_monster - 123 - STEP_FAILURE - DagsterEventType.STEP_FAILURE for step " "start.materialization.output.result.0", "", "FileNotFoundError: [Errno 2] No such file or directory: '/path/to/file'", "", "Stack Trace:", "a stack message", ] assert captured_results[0].split("\n") == expected_results
def __init__( self, node_def: NodeDefinition, all_events: List[DagsterEvent], run_id: str, output_capture: Optional[Dict[StepOutputHandle, Any]], ): self._node_def = node_def # If top-level result, no handle will be provided self._handle = NodeHandle(node_def.name, parent=None) self._event_list = all_events self._run_id = run_id self._output_capture = check.opt_dict_param( output_capture, "output_capture", key_type=StepOutputHandle )
def result_for_solid(self, name): """Get the result of a top level solid. Args: name (str): The name of the top-level solid or aliased solid for which to retrieve the result. Returns: Union[CompositeSolidExecutionResult, SolidExecutionResult]: The result of the solid execution within the pipeline. """ if not self.container.has_solid_named(name): raise DagsterInvariantViolationError( "Tried to get result for solid '{name}' in '{container}'. No such top level " "solid.".format(name=name, container=self.container.name)) return self.result_for_handle(NodeHandle(name, None))
def _filter_outputs_by_handle( output_dict: Dict[StepOutputHandle, Any], ancestor_handle: Optional[NodeHandle], current_handle: NodeHandle, ): if ancestor_handle: handle_with_ancestor = current_handle.with_ancestor(ancestor_handle) else: handle_with_ancestor = current_handle outputs = {} for step_output_handle, value in output_dict.items(): handle = _get_solid_handle_from_output(step_output_handle) if handle and handle_with_ancestor and handle.is_or_descends_from( handle_with_ancestor): outputs[step_output_handle] = value return outputs
def output_values(self): values = {} for output_name in self.solid.definition.output_dict: output_mapping = self.solid.definition.get_output_mapping( output_name) inner_solid_values = self._result_for_handle( self.solid.definition.solid_named( output_mapping.maps_from.solid_name), NodeHandle(output_mapping.maps_from.solid_name, None), ).output_values if inner_solid_values is not None: # may be None if inner solid was skipped if output_mapping.maps_from.output_name in inner_solid_values: values[output_name] = inner_solid_values[ output_mapping.maps_from.output_name] return values
def _filter_step_events_by_handle( event_list: List[DagsterEvent], ancestor_handle: Optional[NodeHandle], current_handle: NodeHandle, ) -> List[DagsterEvent]: events = [] if ancestor_handle: handle_with_ancestor = cast( NodeHandle, current_handle.with_ancestor(ancestor_handle)) else: handle_with_ancestor = current_handle for event in event_list: if event.is_step_event: solid_handle = cast(NodeHandle, event.solid_handle) if solid_handle.is_or_descends_from(handle_with_ancestor): events.append(event) return events
def __new__( cls, source: Union[FromPendingDynamicStepOutput, FromUnresolvedStepOutput], # deprecated, preserved for back-compat solid_handle: Optional[NodeHandle] = None, input_name: Optional[str] = None, ): return super().__new__( cls, source=source, # add placeholder values for back-compat solid_handle=check.opt_inst_param(solid_handle, "solid_handle", NodeHandle, default=NodeHandle("", None)), input_name=check.opt_str_param(input_name, "input_handle", default=""), )
def result_for_handle(self, handle): """Get the result of a solid by its solid handle. This allows indexing into top-level solids to retrieve the results of children of composite solids. Args: handle (Union[str,NodeHandle]): The handle for the solid. Returns: Union[CompositeSolidExecutionResult, SolidExecutionResult]: The result of the given solid. """ if isinstance(handle, str): handle = NodeHandle.from_string(handle) else: check.inst_param(handle, "handle", NodeHandle) solid = self.container.get_solid(handle) return self._result_for_handle(solid, handle)
def output_values(self) -> Dict[str, Any]: """ The values for any outputs that this associated graph maps. """ values = {} for output_name in self._graph_def.output_dict: output_mapping = self._graph_def.get_output_mapping(output_name) inner_solid_values = self._result_for_handle( self._graph_def.solid_named( output_mapping.maps_from.solid_name), NodeHandle(output_mapping.maps_from.solid_name, None), ).output_values if inner_solid_values is not None: # may be None if inner solid was skipped if output_mapping.maps_from.output_name in inner_solid_values: values[output_name] = inner_solid_values[ output_mapping.maps_from.output_name] return values
def _build_solid_handles(represented_pipeline, current_dep_index, parent=None): check.inst_param(represented_pipeline, "represented_pipeline", RepresentedPipeline) check.opt_inst_param(parent, "parent", GrapheneSolidHandle) all_handle = [] for solid_invocation in current_dep_index.solid_invocations: solid_name, solid_def_name = solid_invocation.solid_name, solid_invocation.solid_def_name handle = GrapheneSolidHandle( solid=GrapheneSolid(represented_pipeline, solid_name, current_dep_index), handle=NodeHandle(solid_name, parent.handleID if parent else None), parent=parent if parent else None, ) solid_def_snap = represented_pipeline.get_node_def_snap(solid_def_name) if isinstance(solid_def_snap, CompositeSolidDefSnap): all_handle += _build_solid_handles( represented_pipeline, represented_pipeline.get_dep_structure_index(solid_def_name), handle, ) all_handle.append(handle) return all_handle
def output_value(self, output_name: str = DEFAULT_OUTPUT) -> Any: """Retrieves output of top-level job, if an output is returned. If the top-level job has no output, calling this method will result in a DagsterInvariantViolationError. Args: output_name (Optional[str]): The name of the output to retrieve. Defaults to `result`, the default output name in dagster. Returns: Any: The value of the retrieved output. """ check.str_param(output_name, "output_name") graph_def = self._node_def.ensure_graph_def() if not graph_def.has_output(output_name) and len( graph_def.output_mappings) == 0: raise DagsterInvariantViolationError( f"Attempted to retrieve top-level outputs for '{graph_def.name}', which has no outputs." ) elif not graph_def.has_output(output_name): raise DagsterInvariantViolationError( f"Could not find top-level output '{output_name}' in '{graph_def.name}'." ) # Resolve the first layer of mapping output_mapping = graph_def.get_output_mapping(output_name) mapped_node = graph_def.solid_named( output_mapping.maps_from.solid_name) origin_output_def, origin_handle = mapped_node.definition.resolve_output_to_origin( output_mapping.maps_from.output_name, NodeHandle(mapped_node.name, None), ) # Get output from origin node return _filter_outputs_by_handle(self._output_capture, origin_handle, origin_output_def.name)
def __new__( cls, unresolved_step_output_handle: UnresolvedStepOutputHandle, # deprecated, preserved for back-compat solid_handle: Optional[NodeHandle] = None, input_name: Optional[str] = None, ): return super().__new__( cls, unresolved_step_output_handle=check.inst_param( unresolved_step_output_handle, "unresolved_step_output_handle", UnresolvedStepOutputHandle, ), # add placeholder values for back-compat solid_handle=check.opt_inst_param(solid_handle, "solid_handle", NodeHandle, default=NodeHandle("", None)), input_name=check.opt_str_param(input_name, "input_handle", default=""), )
def __new__( cls, step_output_handle: StepOutputHandle, fan_in: bool, # deprecated, preserved for back-compat solid_handle: Optional[NodeHandle] = None, input_name: Optional[str] = None, ): return super().__new__( cls, step_output_handle=check.inst_param(step_output_handle, "step_output_handle", StepOutputHandle), fan_in=check.bool_param(fan_in, "fan_in"), # add placeholder values for back-compat solid_handle=check.opt_inst_param(solid_handle, "solid_handle", NodeHandle, default=NodeHandle("", None)), input_name=check.opt_str_param(input_name, "input_handle", default=""), )
def _get_solid_handle_from_output( step_output_handle: StepOutputHandle) -> Optional[NodeHandle]: return NodeHandle.from_string(step_output_handle.step_key)