def load_input_object(
self,
step_context: "StepExecutionContext") -> Iterator["DagsterEvent"]:
from dagster.core.events import DagsterEvent
from dagster.core.storage.intermediate_storage import IntermediateStorageAdapter
source_handle = self.step_output_handle
manager_key = step_context.execution_plan.get_manager_key(
source_handle, step_context.pipeline_def)
input_manager = step_context.get_io_manager(source_handle)
check.invariant(
isinstance(input_manager, IOManager),
f'Input "{self.input_name}" for step "{step_context.step.key}" is depending on '
f'the manager of upstream output "{source_handle.output_name}" from step '
f'"{source_handle.step_key}" to load it, but that manager is not an IOManager. '
f"Please ensure that the resource returned for resource key "
f'"{manager_key}" is an IOManager.',
)
yield _load_input_with_input_manager(
input_manager, self.get_load_context(step_context))
yield DagsterEvent.loaded_input(
step_context,
input_name=self.input_name,
manager_key=manager_key,
upstream_output_name=source_handle.output_name,
upstream_step_key=source_handle.step_key,
message_override=
f'Loaded input "{self.input_name}" using intermediate storage'
if isinstance(input_manager, IntermediateStorageAdapter) else None,
)
def load_input_object(
self,
step_context: "StepExecutionContext") -> Iterator["DagsterEvent"]:
from dagster.core.events import DagsterEvent
input_def = self.get_input_def(step_context.pipeline_def)
solid_config = step_context.environment_config.solids.get(
str(self.solid_handle))
config_data = solid_config.inputs.get(
self.input_name) if solid_config else None
loader = getattr(step_context.resources, input_def.root_manager_key)
load_input_context = step_context.for_input_manager(
input_def.name,
config_data,
metadata=input_def.metadata,
dagster_type=input_def.dagster_type,
resource_config=step_context.environment_config.resources[
input_def.root_manager_key].config,
resources=build_resources_for_manager(input_def.root_manager_key,
step_context),
)
yield _load_input_with_input_manager(loader, load_input_context)
yield DagsterEvent.loaded_input(
step_context,
input_name=input_def.name,
manager_key=input_def.root_manager_key,
)
def load_input_object(
self,
step_context: "StepExecutionContext",
input_def: InputDefinition,
) -> Iterator["DagsterEvent"]:
from dagster.core.events import DagsterEvent
check.invariant(
step_context.solid_handle == self.solid_handle
and input_def.name == self.input_name,
"RootInputManager source must be op input and not one along compostion mapping. "
f"Loading for op {step_context.solid_handle}.{input_def.name} "
f"but source is {self.solid_handle}.{self.input_name}.",
)
input_def = step_context.solid_def.input_def_named(input_def.name)
solid_config = step_context.resolved_run_config.solids.get(
str(self.solid_handle))
config_data = solid_config.inputs.get(
self.input_name) if solid_config else None
loader = getattr(step_context.resources, input_def.root_manager_key)
load_input_context = step_context.for_input_manager(
input_def.name,
config_data,
metadata=input_def.metadata,
dagster_type=input_def.dagster_type,
resource_config=step_context.resolved_run_config.resources[
input_def.root_manager_key].config,
resources=build_resources_for_manager(input_def.root_manager_key,
step_context),
)
yield from _load_input_with_input_manager(loader, load_input_context)
metadata_entries = load_input_context.consume_metadata_entries()
yield DagsterEvent.loaded_input(
step_context,
input_name=input_def.name,
manager_key=input_def.root_manager_key,
metadata_entries=[
entry for entry in metadata_entries
if isinstance(entry, MetadataEntry)
],
)
def load_input_object(self, step_context):
from dagster.core.events import DagsterEvent
loader = getattr(step_context.resources, self.input_def.root_manager_key)
load_input_context = step_context.for_input_manager(
self.input_def.name,
self.config_data,
metadata=self.input_def.metadata,
dagster_type=self.input_def.dagster_type,
resource_config=step_context.environment_config.resources[
self.input_def.root_manager_key
].get("config", {}),
resources=build_resources_for_manager(self.input_def.root_manager_key, step_context),
)
yield _load_input_with_input_manager(loader, load_input_context)
yield DagsterEvent.loaded_input(
step_context,
input_name=self.input_def.name,
manager_key=self.input_def.root_manager_key,
)
def load_input_object(self, step_context):
from dagster.core.events import DagsterEvent
source_handle = self.step_output_handle
manager_key = step_context.execution_plan.get_manager_key(source_handle)
input_manager = step_context.get_output_manager(source_handle)
check.invariant(
isinstance(input_manager, IOManager),
f'Input "{self.input_def.name}" for step "{step_context.step.key}" is depending on '
f'the manager of upstream output "{source_handle.output_name}" from step '
f'"{source_handle.step_key}" to load it, but that manager is not an IOManager. '
f"Please ensure that the resource returned for resource key "
f'"{manager_key}" is an IOManager.',
)
yield _load_input_with_input_manager(input_manager, self.get_load_context(step_context))
yield DagsterEvent.loaded_input(
step_context,
input_name=self.input_def.name,
manager_key=manager_key,
upstream_output_name=source_handle.output_name,
upstream_step_key=source_handle.step_key,
)
def load_input_object(
self,
step_context: "StepExecutionContext",
input_def: InputDefinition,
) -> Iterator["DagsterEvent"]:
from dagster.core.events import DagsterEvent
source_handle = self.step_output_handle
manager_key = step_context.execution_plan.get_manager_key(
source_handle, step_context.pipeline_def)
input_manager = step_context.get_io_manager(source_handle)
check.invariant(
isinstance(input_manager, IOManager),
f'Input "{input_def.name}" for step "{step_context.step.key}" is depending on '
f'the manager of upstream output "{source_handle.output_name}" from step '
f'"{source_handle.step_key}" to load it, but that manager is not an IOManager. '
f"Please ensure that the resource returned for resource key "
f'"{manager_key}" is an IOManager.',
)
load_input_context = self.get_load_context(step_context, input_def)
yield from _load_input_with_input_manager(input_manager,
load_input_context)
metadata_entries = load_input_context.consume_metadata_entries()
yield DagsterEvent.loaded_input(
step_context,
input_name=input_def.name,
manager_key=manager_key,
upstream_output_name=source_handle.output_name,
upstream_step_key=source_handle.step_key,
metadata_entries=[
entry for entry in metadata_entries
if isinstance(entry, MetadataEntry)
],
)
