def create_intermediate_storage(
context_creation_data: ContextCreationData,
intermediate_storage_data: Optional[IntermediateStorage],
scoped_resources_builder: ScopedResourcesBuilder,
) -> IntermediateStorage:
environment_config, pipeline_def, intermediate_storage_def, pipeline_run = (
context_creation_data.environment_config,
context_creation_data.pipeline_def,
context_creation_data.intermediate_storage_def,
context_creation_data.pipeline_run,
)
intermediate_storage_data = (
intermediate_storage_data
if intermediate_storage_data else construct_intermediate_storage_data(
InitIntermediateStorageContext(
pipeline_def=pipeline_def,
mode_def=context_creation_data.mode_def,
intermediate_storage_def=intermediate_storage_def,
intermediate_storage_config=environment_config.
intermediate_storage.intermediate_storage_config,
pipeline_run=pipeline_run,
instance=context_creation_data.instance,
environment_config=environment_config,
type_storage_plugin_registry=
construct_type_storage_plugin_registry(
pipeline_def, intermediate_storage_def),
resources=scoped_resources_builder.build(
context_creation_data.intermediate_storage_def.
required_resource_keys, ),
)))
return intermediate_storage_data
def __new__(
cls,
resource_config: Any,
resource_def: ResourceDefinition,
pipeline_run: Optional[PipelineRun] = None,
log_manager: Optional[DagsterLogManager] = None,
resource_instance_dict: Optional[Dict[str, Any]] = None,
required_resource_keys: Optional[AbstractSet[str]] = None,
instance: Optional[DagsterInstance] = None,
pipeline_def_for_backwards_compat: Optional[PipelineDefinition] = None,
):
check.opt_dict_param(resource_instance_dict, "resource_instance_dict")
required_resource_keys = check.opt_set_param(required_resource_keys,
"required_resource_keys")
scoped_resources_builder = ScopedResourcesBuilder(
resource_instance_dict)
return super(InitResourceContext, cls).__new__(
cls,
resource_config,
check.inst_param(resource_def, "resource_def", ResourceDefinition),
check.opt_inst_param(log_manager, "log_manager",
DagsterLogManager),
resources=scoped_resources_builder.build(required_resource_keys),
instance=check.opt_inst_param(instance, "instance",
DagsterInstance),
pipeline_def_for_backwards_compat=check.opt_inst_param(
pipeline_def_for_backwards_compat,
"pipeline_def_for_backwards_compat",
PipelineDefinition,
),
pipeline_run=check.opt_inst_param(pipeline_run, "pipeline_run",
PipelineRun),
)
def __new__(
cls,
resource_config,
pipeline_def,
resource_def,
run_id,
log_manager=None,
resource_instance_dict=None,
required_resource_keys=None,
):
check.opt_dict_param(resource_instance_dict, "resource_instance_dict")
required_resource_keys = check.opt_set_param(required_resource_keys,
"required_resource_keys")
scoped_resources_builder = ScopedResourcesBuilder(
resource_instance_dict)
return super(InitResourceContext, cls).__new__(
cls,
resource_config,
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition),
check.inst_param(resource_def, "resource_def", ResourceDefinition),
check.str_param(run_id, "run_id"),
check.opt_inst_param(log_manager, "log_manager",
DagsterLogManager),
resources=scoped_resources_builder.build(required_resource_keys),
)
def __init__(
self,
run_id: str,
log_manager: DagsterLogManager,
scoped_resources_builder: ScopedResourcesBuilder,
dagster_type: DagsterType,
):
self._run_id = run_id
self._log = log_manager
self._resources = scoped_resources_builder.build(dagster_type.required_resource_keys)
def __new__(
cls,
resource_config,
resource_def,
pipeline_run,
log_manager=None,
resource_instance_dict=None,
required_resource_keys=None,
instance_for_backwards_compat=None,
pipeline_def_for_backwards_compat=None,
):
check.opt_dict_param(resource_instance_dict, "resource_instance_dict")
required_resource_keys = check.opt_set_param(required_resource_keys,
"required_resource_keys")
scoped_resources_builder = ScopedResourcesBuilder(
resource_instance_dict)
return super(InitResourceContext, cls).__new__(
cls,
resource_config,
check.inst_param(resource_def, "resource_def", ResourceDefinition),
check.inst_param(pipeline_run, "pipeline_run", PipelineRun),
check.opt_inst_param(log_manager, "log_manager",
DagsterLogManager),
resources=scoped_resources_builder.build(required_resource_keys),
# The following are used internally for adapting intermediate storage defs to resources
instance_for_backwards_compat=check.opt_inst_param(
instance_for_backwards_compat, "instance_for_backwards_compat",
DagsterInstance),
pipeline_def_for_backwards_compat=check.opt_inst_param(
pipeline_def_for_backwards_compat,
"pipeline_def_for_backwards_compat",
PipelineDefinition,
),
)
