def load_input_object( self, step_context: "StepExecutionContext") -> Iterator["DagsterEvent"]: from dagster.core.events import DagsterEvent from dagster.core.storage.intermediate_storage import IntermediateStorageAdapter source_handle = self.step_output_handle manager_key = step_context.execution_plan.get_manager_key( source_handle, step_context.pipeline_def) input_manager = step_context.get_io_manager(source_handle) check.invariant( isinstance(input_manager, IOManager), f'Input "{self.input_name}" for step "{step_context.step.key}" is depending on ' f'the manager of upstream output "{source_handle.output_name}" from step ' f'"{source_handle.step_key}" to load it, but that manager is not an IOManager. ' f"Please ensure that the resource returned for resource key " f'"{manager_key}" is an IOManager.', ) yield _load_input_with_input_manager( input_manager, self.get_load_context(step_context)) yield DagsterEvent.loaded_input( step_context, input_name=self.input_name, manager_key=manager_key, upstream_output_name=source_handle.output_name, upstream_step_key=source_handle.step_key, message_override= f'Loaded input "{self.input_name}" using intermediate storage' if isinstance(input_manager, IntermediateStorageAdapter) else None, )
def load_input_object( self, step_context: "StepExecutionContext") -> Iterator["DagsterEvent"]: from dagster.core.events import DagsterEvent input_def = self.get_input_def(step_context.pipeline_def) solid_config = step_context.environment_config.solids.get( str(self.solid_handle)) config_data = solid_config.inputs.get( self.input_name) if solid_config else None loader = getattr(step_context.resources, input_def.root_manager_key) load_input_context = step_context.for_input_manager( input_def.name, config_data, metadata=input_def.metadata, dagster_type=input_def.dagster_type, resource_config=step_context.environment_config.resources[ input_def.root_manager_key].config, resources=build_resources_for_manager(input_def.root_manager_key, step_context), ) yield _load_input_with_input_manager(loader, load_input_context) yield DagsterEvent.loaded_input( step_context, input_name=input_def.name, manager_key=input_def.root_manager_key, )
def load_input_object( self, step_context: "StepExecutionContext", input_def: InputDefinition, ) -> Iterator["DagsterEvent"]: from dagster.core.events import DagsterEvent check.invariant( step_context.solid_handle == self.solid_handle and input_def.name == self.input_name, "RootInputManager source must be op input and not one along compostion mapping. " f"Loading for op {step_context.solid_handle}.{input_def.name} " f"but source is {self.solid_handle}.{self.input_name}.", ) input_def = step_context.solid_def.input_def_named(input_def.name) solid_config = step_context.resolved_run_config.solids.get( str(self.solid_handle)) config_data = solid_config.inputs.get( self.input_name) if solid_config else None loader = getattr(step_context.resources, input_def.root_manager_key) load_input_context = step_context.for_input_manager( input_def.name, config_data, metadata=input_def.metadata, dagster_type=input_def.dagster_type, resource_config=step_context.resolved_run_config.resources[ input_def.root_manager_key].config, resources=build_resources_for_manager(input_def.root_manager_key, step_context), ) yield from _load_input_with_input_manager(loader, load_input_context) metadata_entries = load_input_context.consume_metadata_entries() yield DagsterEvent.loaded_input( step_context, input_name=input_def.name, manager_key=input_def.root_manager_key, metadata_entries=[ entry for entry in metadata_entries if isinstance(entry, MetadataEntry) ], )
def load_input_object(self, step_context): from dagster.core.events import DagsterEvent loader = getattr(step_context.resources, self.input_def.root_manager_key) load_input_context = step_context.for_input_manager( self.input_def.name, self.config_data, metadata=self.input_def.metadata, dagster_type=self.input_def.dagster_type, resource_config=step_context.environment_config.resources[ self.input_def.root_manager_key ].get("config", {}), resources=build_resources_for_manager(self.input_def.root_manager_key, step_context), ) yield _load_input_with_input_manager(loader, load_input_context) yield DagsterEvent.loaded_input( step_context, input_name=self.input_def.name, manager_key=self.input_def.root_manager_key, )
def load_input_object(self, step_context): from dagster.core.events import DagsterEvent source_handle = self.step_output_handle manager_key = step_context.execution_plan.get_manager_key(source_handle) input_manager = step_context.get_output_manager(source_handle) check.invariant( isinstance(input_manager, IOManager), f'Input "{self.input_def.name}" for step "{step_context.step.key}" is depending on ' f'the manager of upstream output "{source_handle.output_name}" from step ' f'"{source_handle.step_key}" to load it, but that manager is not an IOManager. ' f"Please ensure that the resource returned for resource key " f'"{manager_key}" is an IOManager.', ) yield _load_input_with_input_manager(input_manager, self.get_load_context(step_context)) yield DagsterEvent.loaded_input( step_context, input_name=self.input_def.name, manager_key=manager_key, upstream_output_name=source_handle.output_name, upstream_step_key=source_handle.step_key, )
def load_input_object( self, step_context: "StepExecutionContext", input_def: InputDefinition, ) -> Iterator["DagsterEvent"]: from dagster.core.events import DagsterEvent source_handle = self.step_output_handle manager_key = step_context.execution_plan.get_manager_key( source_handle, step_context.pipeline_def) input_manager = step_context.get_io_manager(source_handle) check.invariant( isinstance(input_manager, IOManager), f'Input "{input_def.name}" for step "{step_context.step.key}" is depending on ' f'the manager of upstream output "{source_handle.output_name}" from step ' f'"{source_handle.step_key}" to load it, but that manager is not an IOManager. ' f"Please ensure that the resource returned for resource key " f'"{manager_key}" is an IOManager.', ) load_input_context = self.get_load_context(step_context, input_def) yield from _load_input_with_input_manager(input_manager, load_input_context) metadata_entries = load_input_context.consume_metadata_entries() yield DagsterEvent.loaded_input( step_context, input_name=input_def.name, manager_key=manager_key, upstream_output_name=source_handle.output_name, upstream_step_key=source_handle.step_key, metadata_entries=[ entry for entry in metadata_entries if isinstance(entry, MetadataEntry) ], )