def _do_construct(solids, dependencies):
pipeline_def = PipelineDefinition(name="test", solid_defs=solids, dependencies=dependencies)
solids = {
s.name: Solid(name=s.name, definition=s, graph_definition=pipeline_def) for s in solids
}
dependency_structure = DependencyStructure.from_definitions(solids, dependencies)
return _create_adjacency_lists(list(solids.values()), dependency_structure)
def _do_construct(solids, dependencies):
solids = {
s.name: Solid(
name=s.name, definition=s, resource_mapper_fn=SolidInvocation.default_resource_mapper_fn
)
for s in solids
}
dependency_structure = DependencyStructure.from_definitions(solids, dependencies)
return _create_adjacency_lists(list(solids.values()), dependency_structure)
def get_step_input_source(
plan_builder: _PlanBuilder,
solid: Solid,
input_name: str,
input_def: InputDefinition,
dependency_structure: DependencyStructure,
handle: SolidHandle,
parent_step_inputs: Optional[List[Union[StepInput,
UnresolvedMappedStepInput,
UnresolvedCollectStepInput]]],
):
check.inst_param(plan_builder, "plan_builder", _PlanBuilder)
check.inst_param(solid, "solid", Solid)
check.str_param(input_name, "input_name")
check.inst_param(input_def, "input_def", InputDefinition)
check.inst_param(dependency_structure, "dependency_structure",
DependencyStructure)
check.opt_inst_param(handle, "handle", SolidHandle)
check.opt_list_param(
parent_step_inputs,
"parent_step_inputs",
of_type=(StepInput, UnresolvedMappedStepInput,
UnresolvedCollectStepInput),
)
input_handle = solid.input_handle(input_name)
solid_config = plan_builder.environment_config.solids.get(str(handle))
input_def = solid.definition.input_def_named(input_name)
if input_def.root_manager_key and not dependency_structure.has_deps(
input_handle):
return FromRootInputManager(solid_handle=handle, input_name=input_name)
if dependency_structure.has_direct_dep(input_handle):
solid_output_handle = dependency_structure.get_direct_dep(input_handle)
step_output_handle = plan_builder.get_output_handle(
solid_output_handle)
if isinstance(step_output_handle, UnresolvedStepOutputHandle):
return FromUnresolvedStepOutput(
unresolved_step_output_handle=step_output_handle,
solid_handle=handle,
input_name=input_name,
)
if solid_output_handle.output_def.is_dynamic:
return FromPendingDynamicStepOutput(
step_output_handle=step_output_handle,
solid_handle=handle,
input_name=input_name,
)
return FromStepOutput(
step_output_handle=step_output_handle,
solid_handle=handle,
input_name=input_name,
fan_in=False,
)
if dependency_structure.has_fan_in_deps(input_handle):
sources: List[StepInputSource] = []
deps = dependency_structure.get_fan_in_deps(input_handle)
for idx, handle_or_placeholder in enumerate(deps):
if isinstance(handle_or_placeholder, SolidOutputHandle):
step_output_handle = plan_builder.get_output_handle(
handle_or_placeholder)
if (isinstance(step_output_handle, UnresolvedStepOutputHandle)
or handle_or_placeholder.output_def.is_dynamic):
check.failed(
"Unexpected dynamic output dependency in regular fan in, "
"should have been caught at definition time.")
sources.append(
FromStepOutput(
step_output_handle=step_output_handle,
solid_handle=handle,
input_name=input_name,
fan_in=True,
))
else:
check.invariant(
handle_or_placeholder is MappedInputPlaceholder,
f"Expected SolidOutputHandle or MappedInputPlaceholder, got {handle_or_placeholder}",
)
if parent_step_inputs is None:
check.failed(
"unexpected error in composition descent during plan building"
)
parent_name = solid.container_mapped_fan_in_input(
input_name, idx).definition.name
parent_inputs = {
step_input.name: step_input
for step_input in parent_step_inputs
}
parent_input = parent_inputs[parent_name]
source = parent_input.source
if not isinstance(source, StepInputSource):
check.failed(
f"Unexpected parent mapped input source type {source}")
sources.append(source)
return FromMultipleSources(solid_handle=handle,
input_name=input_name,
sources=sources)
if dependency_structure.has_dynamic_fan_in_dep(input_handle):
solid_output_handle = dependency_structure.get_dynamic_fan_in_dep(
input_handle)
step_output_handle = plan_builder.get_output_handle(
solid_output_handle)
if isinstance(step_output_handle, UnresolvedStepOutputHandle):
return FromDynamicCollect(
solid_handle=handle,
input_name=input_name,
source=FromUnresolvedStepOutput(
unresolved_step_output_handle=step_output_handle,
solid_handle=handle,
input_name=input_name,
),
)
elif solid_output_handle.output_def.is_dynamic:
return FromDynamicCollect(
solid_handle=handle,
input_name=input_name,
source=FromPendingDynamicStepOutput(
step_output_handle=step_output_handle,
solid_handle=handle,
input_name=input_name,
),
)
if solid_config and input_name in solid_config.inputs:
return FromConfig(solid_handle=handle, input_name=input_name)
if solid.container_maps_input(input_name):
if parent_step_inputs is None:
check.failed(
"unexpected error in composition descent during plan building")
parent_name = solid.container_mapped_input(input_name).definition.name
parent_inputs = {
step_input.name: step_input
for step_input in parent_step_inputs
}
if parent_name in parent_inputs:
parent_input = parent_inputs[parent_name]
return parent_input.source
# else fall through to Nothing case or raise
if solid.definition.input_has_default(input_name):
return FromDefaultValue(solid_handle=handle, input_name=input_name)
# At this point we have an input that is not hooked up to
# the output of another solid or provided via environment config.
# We will allow this for "Nothing" type inputs and continue.
if input_def.dagster_type.kind == DagsterTypeKind.NOTHING:
return None
# Otherwise we throw an error.
raise DagsterInvariantViolationError(
("In pipeline {pipeline_name} solid {solid_name}, input {input_name} "
"must get a value either (a) from a dependency or (b) from the "
"inputs section of its configuration.").format(
pipeline_name=plan_builder.pipeline_name,
solid_name=solid.name,
input_name=input_name))
def _do_construct(solids, dependencies):
solids = {s.name: Solid(name=s.name, definition=s) for s in solids}
dependency_structure = DependencyStructure.from_definitions(
solids, dependencies)
return _create_adjacency_lists(list(solids.values()), dependency_structure)