def create_unresolved_step(solid, solid_handle, step_inputs, pipeline_name,
environment_config):
check.inst_param(solid, "solid", Solid)
check.inst_param(solid_handle, "solid_handle", SolidHandle)
check.list_param(step_inputs,
"step_inputs",
of_type=(StepInput, UnresolvedStepInput))
check.str_param(pipeline_name, "pipeline_name")
return UnresolvedExecutionStep(
handle=UnresolvedStepHandle(solid_handle),
solid=solid,
step_inputs=step_inputs,
step_outputs=_create_step_outputs(solid, solid_handle,
environment_config),
pipeline_name=pipeline_name,
)
def _build_from_sorted_solids(
self,
solids: List[Solid],
dependency_structure: DependencyStructure,
parent_handle: Optional[SolidHandle] = None,
parent_step_inputs: Optional[List[
Union[StepInput, UnresolvedMappedStepInput,
UnresolvedCollectStepInput]]] = None,
):
for solid in solids:
handle = SolidHandle(solid.name, parent_handle)
### 1. INPUTS
# Create and add execution plan steps for solid inputs
has_unresolved_input = False
has_pending_input = False
step_inputs: List[Union[StepInput, UnresolvedMappedStepInput,
UnresolvedCollectStepInput]] = []
for input_name, input_def in solid.definition.input_dict.items():
step_input_source = get_step_input_source(
self,
solid,
input_name,
input_def,
dependency_structure,
handle,
parent_step_inputs,
)
# If an input with dagster_type "Nothing" doesn't have a value
# we don't create a StepInput
if step_input_source is None:
continue
if isinstance(
step_input_source,
(FromPendingDynamicStepOutput, FromUnresolvedStepOutput),
):
has_unresolved_input = True
step_inputs.append(
UnresolvedMappedStepInput(
name=input_name,
dagster_type_key=input_def.dagster_type.key,
source=step_input_source,
))
elif isinstance(step_input_source, FromDynamicCollect):
has_pending_input = True
step_inputs.append(
UnresolvedCollectStepInput(
name=input_name,
dagster_type_key=input_def.dagster_type.key,
source=step_input_source,
))
else:
check.inst_param(step_input_source, "step_input_source",
StepInputSource)
step_inputs.append(
StepInput(
name=input_name,
dagster_type_key=input_def.dagster_type.key,
source=step_input_source,
))
### 2a. COMPUTE FUNCTION
# Create and add execution plan step for the solid compute function
if isinstance(solid.definition, SolidDefinition):
step_outputs = create_step_outputs(solid, handle,
self.environment_config)
if has_pending_input and has_unresolved_input:
check.failed(
"Can not have pending and unresolved step inputs")
elif has_unresolved_input:
new_step: IExecutionStep = UnresolvedMappedExecutionStep(
handle=UnresolvedStepHandle(solid_handle=handle),
pipeline_name=self.pipeline_name,
step_inputs=cast(
List[Union[StepInput, UnresolvedMappedStepInput]],
step_inputs),
step_outputs=step_outputs,
tags=solid.tags,
)
elif has_pending_input:
new_step = UnresolvedCollectExecutionStep(
handle=StepHandle(solid_handle=handle),
pipeline_name=self.pipeline_name,
step_inputs=cast(
List[Union[StepInput, UnresolvedCollectStepInput]],
step_inputs),
step_outputs=step_outputs,
tags=solid.tags,
)
else:
new_step = ExecutionStep(
handle=StepHandle(solid_handle=handle),
pipeline_name=self.pipeline_name,
step_inputs=cast(List[StepInput], step_inputs),
step_outputs=step_outputs,
tags=solid.tags,
)
self.add_step(new_step)
### 2b. RECURSE
# Recurse over the solids contained in an instance of GraphDefinition
elif isinstance(solid.definition, GraphDefinition):
self._build_from_sorted_solids(
solid.definition.solids_in_topological_order,
solid.definition.dependency_structure,
parent_handle=handle,
parent_step_inputs=step_inputs,
)
else:
check.invariant(
False,
"Unexpected solid type {type} encountered during execution planning"
.format(type=type(solid.definition)),
)
### 3. OUTPUTS
# Create output handles for solid outputs
for name, output_def in solid.definition.output_dict.items():
output_handle = solid.output_handle(name)
# Punch through layers of composition scope to map to the output of the
# actual compute step
resolved_output_def, resolved_handle = solid.definition.resolve_output_to_origin(
output_def.name, handle)
step = self.get_step_by_solid_handle(resolved_handle)
if isinstance(step,
(ExecutionStep, UnresolvedCollectExecutionStep)):
step_output_handle: Union[
StepOutputHandle,
UnresolvedStepOutputHandle] = StepOutputHandle(
step.key, resolved_output_def.name)
elif isinstance(step, UnresolvedMappedExecutionStep):
step_output_handle = UnresolvedStepOutputHandle(
step.handle,
resolved_output_def.name,
step.resolved_by_step_key,
step.resolved_by_output_name,
)
else:
check.failed(f"Unexpected step type {step}")
self.set_output_handle(output_handle, step_output_handle)