def _do_construct(solids, dependencies): pipeline_def = PipelineDefinition(name="test", solid_defs=solids, dependencies=dependencies) solids = { s.name: Solid(name=s.name, definition=s, graph_definition=pipeline_def) for s in solids } dependency_structure = DependencyStructure.from_definitions(solids, dependencies) return _create_adjacency_lists(list(solids.values()), dependency_structure)
def _do_construct(solids, dependencies): solids = { s.name: Solid( name=s.name, definition=s, resource_mapper_fn=SolidInvocation.default_resource_mapper_fn ) for s in solids } dependency_structure = DependencyStructure.from_definitions(solids, dependencies) return _create_adjacency_lists(list(solids.values()), dependency_structure)
def get_step_input_source( plan_builder: _PlanBuilder, solid: Solid, input_name: str, input_def: InputDefinition, dependency_structure: DependencyStructure, handle: SolidHandle, parent_step_inputs: Optional[List[Union[StepInput, UnresolvedMappedStepInput, UnresolvedCollectStepInput]]], ): check.inst_param(plan_builder, "plan_builder", _PlanBuilder) check.inst_param(solid, "solid", Solid) check.str_param(input_name, "input_name") check.inst_param(input_def, "input_def", InputDefinition) check.inst_param(dependency_structure, "dependency_structure", DependencyStructure) check.opt_inst_param(handle, "handle", SolidHandle) check.opt_list_param( parent_step_inputs, "parent_step_inputs", of_type=(StepInput, UnresolvedMappedStepInput, UnresolvedCollectStepInput), ) input_handle = solid.input_handle(input_name) solid_config = plan_builder.environment_config.solids.get(str(handle)) input_def = solid.definition.input_def_named(input_name) if input_def.root_manager_key and not dependency_structure.has_deps( input_handle): return FromRootInputManager(solid_handle=handle, input_name=input_name) if dependency_structure.has_direct_dep(input_handle): solid_output_handle = dependency_structure.get_direct_dep(input_handle) step_output_handle = plan_builder.get_output_handle( solid_output_handle) if isinstance(step_output_handle, UnresolvedStepOutputHandle): return FromUnresolvedStepOutput( unresolved_step_output_handle=step_output_handle, solid_handle=handle, input_name=input_name, ) if solid_output_handle.output_def.is_dynamic: return FromPendingDynamicStepOutput( step_output_handle=step_output_handle, solid_handle=handle, input_name=input_name, ) return FromStepOutput( step_output_handle=step_output_handle, solid_handle=handle, input_name=input_name, fan_in=False, ) if dependency_structure.has_fan_in_deps(input_handle): sources: List[StepInputSource] = [] deps = dependency_structure.get_fan_in_deps(input_handle) for idx, handle_or_placeholder in enumerate(deps): if isinstance(handle_or_placeholder, SolidOutputHandle): step_output_handle = plan_builder.get_output_handle( handle_or_placeholder) if (isinstance(step_output_handle, UnresolvedStepOutputHandle) or handle_or_placeholder.output_def.is_dynamic): check.failed( "Unexpected dynamic output dependency in regular fan in, " "should have been caught at definition time.") sources.append( FromStepOutput( step_output_handle=step_output_handle, solid_handle=handle, input_name=input_name, fan_in=True, )) else: check.invariant( handle_or_placeholder is MappedInputPlaceholder, f"Expected SolidOutputHandle or MappedInputPlaceholder, got {handle_or_placeholder}", ) if parent_step_inputs is None: check.failed( "unexpected error in composition descent during plan building" ) parent_name = solid.container_mapped_fan_in_input( input_name, idx).definition.name parent_inputs = { step_input.name: step_input for step_input in parent_step_inputs } parent_input = parent_inputs[parent_name] source = parent_input.source if not isinstance(source, StepInputSource): check.failed( f"Unexpected parent mapped input source type {source}") sources.append(source) return FromMultipleSources(solid_handle=handle, input_name=input_name, sources=sources) if dependency_structure.has_dynamic_fan_in_dep(input_handle): solid_output_handle = dependency_structure.get_dynamic_fan_in_dep( input_handle) step_output_handle = plan_builder.get_output_handle( solid_output_handle) if isinstance(step_output_handle, UnresolvedStepOutputHandle): return FromDynamicCollect( solid_handle=handle, input_name=input_name, source=FromUnresolvedStepOutput( unresolved_step_output_handle=step_output_handle, solid_handle=handle, input_name=input_name, ), ) elif solid_output_handle.output_def.is_dynamic: return FromDynamicCollect( solid_handle=handle, input_name=input_name, source=FromPendingDynamicStepOutput( step_output_handle=step_output_handle, solid_handle=handle, input_name=input_name, ), ) if solid_config and input_name in solid_config.inputs: return FromConfig(solid_handle=handle, input_name=input_name) if solid.container_maps_input(input_name): if parent_step_inputs is None: check.failed( "unexpected error in composition descent during plan building") parent_name = solid.container_mapped_input(input_name).definition.name parent_inputs = { step_input.name: step_input for step_input in parent_step_inputs } if parent_name in parent_inputs: parent_input = parent_inputs[parent_name] return parent_input.source # else fall through to Nothing case or raise if solid.definition.input_has_default(input_name): return FromDefaultValue(solid_handle=handle, input_name=input_name) # At this point we have an input that is not hooked up to # the output of another solid or provided via environment config. # We will allow this for "Nothing" type inputs and continue. if input_def.dagster_type.kind == DagsterTypeKind.NOTHING: return None # Otherwise we throw an error. raise DagsterInvariantViolationError( ("In pipeline {pipeline_name} solid {solid_name}, input {input_name} " "must get a value either (a) from a dependency or (b) from the " "inputs section of its configuration.").format( pipeline_name=plan_builder.pipeline_name, solid_name=solid.name, input_name=input_name))
def _do_construct(solids, dependencies): solids = {s.name: Solid(name=s.name, definition=s) for s in solids} dependency_structure = DependencyStructure.from_definitions( solids, dependencies) return _create_adjacency_lists(list(solids.values()), dependency_structure)