def output_for_node(self,
node_str: str,
output_name: Optional[str] = DEFAULT_OUTPUT) -> Any:
"""Retrieves output value with a particular name from the in-process run of the job.
Args:
node_str (str): Name of the op/graph whose output should be retrieved. If the intended
graph/op is nested within another graph, the syntax is `outer_graph.inner_node`.
output_name (Optional[str]): Name of the output on the op/graph to retrieve. Defaults to
`result`, the default output name in dagster.
Returns:
Any: The value of the retrieved output.
"""
# resolve handle of node that node_str is referring to
target_handle = NodeHandle.from_string(node_str)
target_node_def = self._node_def.ensure_graph_def().get_solid(
target_handle).definition
origin_output_def, origin_handle = target_node_def.resolve_output_to_origin(
output_name, NodeHandle.from_string(node_str))
# retrieve output value from resolved handle
return _filter_outputs_by_handle(self._output_capture, origin_handle,
origin_output_def.name)
def test_hook_decorator():
called_hook_to_solids = defaultdict(list)
@success_hook
def a_success_hook(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
@solid
def a_solid(_):
pass
@a_success_hook
@pipeline(
description="i am a pipeline",
mode_defs=[ModeDefinition(name="my_mode")],
solid_retry_policy=RetryPolicy(max_retries=3),
preset_defs=[PresetDefinition("my_empty_preset", mode="my_mode")],
tags={"foo": "FOO"},
)
def a_pipeline():
a_solid()
assert isinstance(a_pipeline, PipelineDefinition)
assert a_pipeline.tags
assert a_pipeline.tags.get("foo") == "FOO"
assert a_pipeline.tags.get("foo") == "FOO"
assert a_pipeline.description == "i am a pipeline"
assert a_pipeline.has_mode_definition("my_mode")
assert a_pipeline.has_preset("my_empty_preset")
retry_policy = a_pipeline.get_retry_policy_for_handle(
NodeHandle("a_solid", parent=None))
assert isinstance(retry_policy, RetryPolicy)
assert retry_policy.max_retries == 3
def __new__(
cls,
sources: Sequence[StepInputSource],
# deprecated, preserved for back-compat
solid_handle: Optional[NodeHandle] = None,
input_name: Optional[str] = None,
):
check.list_param(sources, "sources", StepInputSource)
for source in sources:
check.invariant(
not isinstance(source, FromMultipleSources),
"Can not have multiple levels of FromMultipleSources StepInputSource",
)
return super().__new__(
cls,
sources=sources,
# add placeholder values for back-compat
solid_handle=check.opt_inst_param(solid_handle,
"solid_handle",
NodeHandle,
default=NodeHandle("", None)),
input_name=check.opt_str_param(input_name,
"input_handle",
default=""),
)
def _result_for_handle(self, solid: Node,
handle: NodeHandle) -> NodeExecutionResult:
node_def = solid.definition
events_for_handle = _filter_step_events_by_handle(
self.event_list, self.handle, handle)
outputs_for_handle = (_filter_outputs_by_handle(
self._output_capture, self.handle, handle)
if self._output_capture else None)
if self.handle:
handle_with_ancestor = handle.with_ancestor(self.handle)
else:
handle_with_ancestor = handle
if not handle_with_ancestor:
raise DagsterInvariantViolationError(
f"No handle provided for solid {solid.name}")
if isinstance(node_def, SolidDefinition):
return InProcessSolidResult(
solid_def=node_def,
handle=handle_with_ancestor,
all_events=events_for_handle,
output_capture=outputs_for_handle,
)
elif isinstance(node_def, GraphDefinition):
return InProcessGraphResult(
graph_def=node_def,
handle=handle_with_ancestor,
all_events=events_for_handle,
output_capture=outputs_for_handle,
)
check.failed("Unhandled node type {node_def}")
def output_value(self, output_name=DEFAULT_OUTPUT):
check.str_param(output_name, "output_name")
if not self.solid.definition.has_output(output_name):
raise DagsterInvariantViolationError(
"Output '{output_name}' not defined in composite solid '{solid}': "
"{outputs_clause}. If you were expecting this output to be present, you may "
"be missing an output_mapping from an inner solid to its enclosing composite "
"solid.".format(
output_name=output_name,
solid=self.solid.name,
outputs_clause="found outputs {output_names}".format(
output_names=str(
list(self.solid.definition.output_dict.keys())))
if self.solid.definition.output_dict else
"no output mappings were defined",
))
output_mapping = self.solid.definition.get_output_mapping(output_name)
return self._result_for_handle(
self.solid.definition.solid_named(
output_mapping.maps_from.solid_name),
NodeHandle(output_mapping.maps_from.solid_name, None),
).output_value(output_mapping.maps_from.output_name)
def __new__(
cls,
step_output_handle: StepOutputHandle,
# deprecated, preserved for back-compat
solid_handle: Optional[NodeHandle] = None,
input_name: Optional[str] = None,
):
# Model the unknown mapping key from known execution step
# using a StepOutputHandle with None mapping_key.
check.inst_param(step_output_handle, "step_output_handle",
StepOutputHandle)
check.invariant(step_output_handle.mapping_key is None)
return super().__new__(
cls,
step_output_handle=step_output_handle,
# add placeholder values for back-compat
solid_handle=check.opt_inst_param(solid_handle,
"solid_handle",
NodeHandle,
default=NodeHandle("", None)),
input_name=check.opt_str_param(input_name,
"input_handle",
default=""),
)
def all_node_events(self) -> List[DagsterEvent]:
"""List[DagsterEvent]: All dagster events from the in-process execution."""
step_events = []
for node_name in self._node_def.ensure_graph_def().node_dict.keys():
handle = NodeHandle(node_name, None)
step_events += _filter_events_by_handle(self._event_list, handle)
return step_events
def result_for_node(self, name: str) -> NodeExecutionResult:
"""
The inner result for a node within the graph.
"""
if not self._graph_def.has_solid_named(name):
raise DagsterInvariantViolationError(
"Tried to get result for node '{name}' in '{container}'. No such top level "
"node.".format(name=name, container=self._graph_def.name))
handle = NodeHandle(name, None)
solid = self._graph_def.get_solid(handle)
return self._result_for_handle(solid, handle)
def events_for_node(self, node_name: str) -> List[DagsterEvent]:
"""Retrieves all dagster events for a specific node.
Args:
node_name (str): The name of the node for which outputs should be retrieved.
Returns:
List[DagsterEvent]: A list of all dagster events associated with provided node name.
"""
check.str_param(node_name, "node_name")
return _filter_events_by_handle(self._event_list, NodeHandle.from_string(node_name))
def output_for_solid(self, handle_str, output_name=DEFAULT_OUTPUT):
"""Get the output of a solid by its solid handle string and output name.
Args:
handle_str (str): The string handle for the solid.
output_name (str): Optional. The name of the output, default to DEFAULT_OUTPUT.
Returns:
The output value for the handle and output_name.
"""
check.str_param(handle_str, "handle_str")
check.str_param(output_name, "output_name")
return self.result_for_handle(
NodeHandle.from_string(handle_str)).output_value(output_name)
def test_multiline_logging_complex():
msg = "DagsterEventType.STEP_FAILURE for step start.materialization.output.result.0"
kwargs = {
"pipeline": "example",
"pipeline_name": "error_monster",
"step_key": "start.materialization.output.result.0",
"solid": "start",
"solid_definition": "emit_num",
"dagster_event": DagsterEvent(
event_type_value="STEP_FAILURE",
pipeline_name="error_monster",
step_key="start.materialization.output.result.0",
solid_handle=NodeHandle("start", None),
step_kind_value="MATERIALIZATION_THUNK",
logging_tags={
"pipeline": "error_monster",
"step_key": "start.materialization.output.result.0",
"solid": "start",
"solid_definition": "emit_num",
},
event_specific_data=StepFailureData(
error=SerializableErrorInfo(
message="FileNotFoundError: [Errno 2] No such file or directory: '/path/to/file'\n",
stack=["a stack message"],
cls_name="FileNotFoundError",
),
user_failure_data=None,
),
),
}
with _setup_logger(DAGSTER_DEFAULT_LOGGER) as (captured_results, logger):
dl = DagsterLogManager("123", {}, [logger])
dl.info(msg, **kwargs)
expected_results = [
"error_monster - 123 - STEP_FAILURE - DagsterEventType.STEP_FAILURE for step "
"start.materialization.output.result.0",
"",
"FileNotFoundError: [Errno 2] No such file or directory: '/path/to/file'",
"",
"Stack Trace:",
"a stack message",
]
assert captured_results[0].split("\n") == expected_results
def __init__(
self,
node_def: NodeDefinition,
all_events: List[DagsterEvent],
run_id: str,
output_capture: Optional[Dict[StepOutputHandle, Any]],
):
self._node_def = node_def
# If top-level result, no handle will be provided
self._handle = NodeHandle(node_def.name, parent=None)
self._event_list = all_events
self._run_id = run_id
self._output_capture = check.opt_dict_param(
output_capture, "output_capture", key_type=StepOutputHandle
)
def result_for_solid(self, name):
"""Get the result of a top level solid.
Args:
name (str): The name of the top-level solid or aliased solid for which to retrieve the
result.
Returns:
Union[CompositeSolidExecutionResult, SolidExecutionResult]: The result of the solid
execution within the pipeline.
"""
if not self.container.has_solid_named(name):
raise DagsterInvariantViolationError(
"Tried to get result for solid '{name}' in '{container}'. No such top level "
"solid.".format(name=name, container=self.container.name))
return self.result_for_handle(NodeHandle(name, None))
def _filter_outputs_by_handle(
output_dict: Dict[StepOutputHandle, Any],
ancestor_handle: Optional[NodeHandle],
current_handle: NodeHandle,
):
if ancestor_handle:
handle_with_ancestor = current_handle.with_ancestor(ancestor_handle)
else:
handle_with_ancestor = current_handle
outputs = {}
for step_output_handle, value in output_dict.items():
handle = _get_solid_handle_from_output(step_output_handle)
if handle and handle_with_ancestor and handle.is_or_descends_from(
handle_with_ancestor):
outputs[step_output_handle] = value
return outputs
def output_values(self):
values = {}
for output_name in self.solid.definition.output_dict:
output_mapping = self.solid.definition.get_output_mapping(
output_name)
inner_solid_values = self._result_for_handle(
self.solid.definition.solid_named(
output_mapping.maps_from.solid_name),
NodeHandle(output_mapping.maps_from.solid_name, None),
).output_values
if inner_solid_values is not None: # may be None if inner solid was skipped
if output_mapping.maps_from.output_name in inner_solid_values:
values[output_name] = inner_solid_values[
output_mapping.maps_from.output_name]
return values
def _filter_step_events_by_handle(
event_list: List[DagsterEvent],
ancestor_handle: Optional[NodeHandle],
current_handle: NodeHandle,
) -> List[DagsterEvent]:
events = []
if ancestor_handle:
handle_with_ancestor = cast(
NodeHandle, current_handle.with_ancestor(ancestor_handle))
else:
handle_with_ancestor = current_handle
for event in event_list:
if event.is_step_event:
solid_handle = cast(NodeHandle, event.solid_handle)
if solid_handle.is_or_descends_from(handle_with_ancestor):
events.append(event)
return events
def __new__(
cls,
source: Union[FromPendingDynamicStepOutput, FromUnresolvedStepOutput],
# deprecated, preserved for back-compat
solid_handle: Optional[NodeHandle] = None,
input_name: Optional[str] = None,
):
return super().__new__(
cls,
source=source,
# add placeholder values for back-compat
solid_handle=check.opt_inst_param(solid_handle,
"solid_handle",
NodeHandle,
default=NodeHandle("", None)),
input_name=check.opt_str_param(input_name,
"input_handle",
default=""),
)
def result_for_handle(self, handle):
"""Get the result of a solid by its solid handle.
This allows indexing into top-level solids to retrieve the results of children of
composite solids.
Args:
handle (Union[str,NodeHandle]): The handle for the solid.
Returns:
Union[CompositeSolidExecutionResult, SolidExecutionResult]: The result of the given
solid.
"""
if isinstance(handle, str):
handle = NodeHandle.from_string(handle)
else:
check.inst_param(handle, "handle", NodeHandle)
solid = self.container.get_solid(handle)
return self._result_for_handle(solid, handle)
def output_values(self) -> Dict[str, Any]:
"""
The values for any outputs that this associated graph maps.
"""
values = {}
for output_name in self._graph_def.output_dict:
output_mapping = self._graph_def.get_output_mapping(output_name)
inner_solid_values = self._result_for_handle(
self._graph_def.solid_named(
output_mapping.maps_from.solid_name),
NodeHandle(output_mapping.maps_from.solid_name, None),
).output_values
if inner_solid_values is not None: # may be None if inner solid was skipped
if output_mapping.maps_from.output_name in inner_solid_values:
values[output_name] = inner_solid_values[
output_mapping.maps_from.output_name]
return values
def _build_solid_handles(represented_pipeline, current_dep_index, parent=None):
check.inst_param(represented_pipeline, "represented_pipeline", RepresentedPipeline)
check.opt_inst_param(parent, "parent", GrapheneSolidHandle)
all_handle = []
for solid_invocation in current_dep_index.solid_invocations:
solid_name, solid_def_name = solid_invocation.solid_name, solid_invocation.solid_def_name
handle = GrapheneSolidHandle(
solid=GrapheneSolid(represented_pipeline, solid_name, current_dep_index),
handle=NodeHandle(solid_name, parent.handleID if parent else None),
parent=parent if parent else None,
)
solid_def_snap = represented_pipeline.get_node_def_snap(solid_def_name)
if isinstance(solid_def_snap, CompositeSolidDefSnap):
all_handle += _build_solid_handles(
represented_pipeline,
represented_pipeline.get_dep_structure_index(solid_def_name),
handle,
)
all_handle.append(handle)
return all_handle
def output_value(self, output_name: str = DEFAULT_OUTPUT) -> Any:
"""Retrieves output of top-level job, if an output is returned.
If the top-level job has no output, calling this method will result in a
DagsterInvariantViolationError.
Args:
output_name (Optional[str]): The name of the output to retrieve. Defaults to `result`,
the default output name in dagster.
Returns:
Any: The value of the retrieved output.
"""
check.str_param(output_name, "output_name")
graph_def = self._node_def.ensure_graph_def()
if not graph_def.has_output(output_name) and len(
graph_def.output_mappings) == 0:
raise DagsterInvariantViolationError(
f"Attempted to retrieve top-level outputs for '{graph_def.name}', which has no outputs."
)
elif not graph_def.has_output(output_name):
raise DagsterInvariantViolationError(
f"Could not find top-level output '{output_name}' in '{graph_def.name}'."
)
# Resolve the first layer of mapping
output_mapping = graph_def.get_output_mapping(output_name)
mapped_node = graph_def.solid_named(
output_mapping.maps_from.solid_name)
origin_output_def, origin_handle = mapped_node.definition.resolve_output_to_origin(
output_mapping.maps_from.output_name,
NodeHandle(mapped_node.name, None),
)
# Get output from origin node
return _filter_outputs_by_handle(self._output_capture, origin_handle,
origin_output_def.name)
def __new__(
cls,
unresolved_step_output_handle: UnresolvedStepOutputHandle,
# deprecated, preserved for back-compat
solid_handle: Optional[NodeHandle] = None,
input_name: Optional[str] = None,
):
return super().__new__(
cls,
unresolved_step_output_handle=check.inst_param(
unresolved_step_output_handle,
"unresolved_step_output_handle",
UnresolvedStepOutputHandle,
),
# add placeholder values for back-compat
solid_handle=check.opt_inst_param(solid_handle,
"solid_handle",
NodeHandle,
default=NodeHandle("", None)),
input_name=check.opt_str_param(input_name,
"input_handle",
default=""),
)
def __new__(
cls,
step_output_handle: StepOutputHandle,
fan_in: bool,
# deprecated, preserved for back-compat
solid_handle: Optional[NodeHandle] = None,
input_name: Optional[str] = None,
):
return super().__new__(
cls,
step_output_handle=check.inst_param(step_output_handle,
"step_output_handle",
StepOutputHandle),
fan_in=check.bool_param(fan_in, "fan_in"),
# add placeholder values for back-compat
solid_handle=check.opt_inst_param(solid_handle,
"solid_handle",
NodeHandle,
default=NodeHandle("", None)),
input_name=check.opt_str_param(input_name,
"input_handle",
default=""),
)
def _get_solid_handle_from_output(
step_output_handle: StepOutputHandle) -> Optional[NodeHandle]:
return NodeHandle.from_string(step_output_handle.step_key)
