def build_resources(
resource_defs: Dict[str, ResourceDefinition],
instance: DagsterInstance,
run_config: Optional[Dict[str, Any]] = None,
pipeline_run: Optional[PipelineRun] = None,
log_manager: Optional[DagsterLogManager] = None,
) -> Generator[NamedTuple, None, None]:
"""Context manager that yields resources using provided resource definitions and run config.
This API allows for using resources in an independent context. Resources will be initialized
with the provided run config, and optionally, pipeline_run. The resulting resources will be
yielded on a dictionary keyed identically to that provided for `resource_defs`. Upon exiting the
context, resources will also be torn down safely.
Args:
resource_defs (Dict[str, ResourceDefinition]): Resource definitions to build. All
required resource dependencies to a given resource must be contained within this
dictionary, or the resource build will fail.
instance (DagsterInstance): The dagster instance configured to instantiate resources on.
run_config (Optional[Dict[str, Any]]): A dict representing the configuration to be provided
to each resource during initialization and teardown.
pipeline_run (Optional[PipelineRun]): The pipeline run to provide during resource
initialization and teardown. If the provided resources require either the `pipeline_run`
or `run_id` attributes of the provided context during resource initialization and/or
teardown, this must be provided, or initialization will fail.
log_manager (Optional[DagsterLogManager]): Log Manager to use during resource
initialization. Defaults to system log manager.
"""
resource_defs = check.dict_param(resource_defs,
"resource_defs",
key_type=str,
value_type=ResourceDefinition)
instance = check.inst_param(instance, "instance", DagsterInstance)
run_config = check.opt_dict_param(run_config, "run_config", key_type=str)
log_manager = check.opt_inst_param(log_manager, "log_manager",
DagsterLogManager)
mapped_resource_config = _get_mapped_resource_config(
resource_defs, run_config)
resources_manager = resource_initialization_manager(
resource_defs=resource_defs,
resource_configs=mapped_resource_config,
log_manager=log_manager
if log_manager else initialize_console_manager(pipeline_run),
execution_plan=None,
pipeline_run=pipeline_run,
resource_keys_to_init=set(resource_defs.keys()),
instance=instance,
resource_instances_to_override=None,
emit_persistent_events=False,
)
try:
list(resources_manager.generate_setup_events())
resources = check.inst(resources_manager.get_object(),
ScopedResourcesBuilder)
yield resources.build(set(resources.resource_instance_dict.keys()))
finally:
list(resources_manager.generate_teardown_events())
def build_resources(
resources: Dict[str, Any],
instance: Optional[DagsterInstance] = None,
resource_config: Optional[Dict[str, Any]] = None,
pipeline_run: Optional[PipelineRun] = None,
log_manager: Optional[DagsterLogManager] = None,
) -> Generator[Resources, None, None]:
"""Context manager that yields resources using provided resource definitions and run config.
This API allows for using resources in an independent context. Resources will be initialized
with the provided run config, and optionally, pipeline_run. The resulting resources will be
yielded on a dictionary keyed identically to that provided for `resource_defs`. Upon exiting the
context, resources will also be torn down safely.
Args:
resources (Dict[str, Any]): Resource instances or definitions to build. All
required resource dependencies to a given resource must be contained within this
dictionary, or the resource build will fail.
instance (Optional[DagsterInstance]): The dagster instance configured to instantiate
resources on.
resource_config (Optional[Dict[str, Any]]): A dict representing the config to be
provided to each resource during initialization and teardown.
pipeline_run (Optional[PipelineRun]): The pipeline run to provide during resource
initialization and teardown. If the provided resources require either the `pipeline_run`
or `run_id` attributes of the provided context during resource initialization and/or
teardown, this must be provided, or initialization will fail.
log_manager (Optional[DagsterLogManager]): Log Manager to use during resource
initialization. Defaults to system log manager.
Examples:
.. code-block:: python
from dagster import resource, build_resources
@resource
def the_resource():
return "foo"
with build_resources(resources={"from_def": the_resource, "from_val": "bar"}) as resources:
assert resources.from_def == "foo"
assert resources.from_val == "bar"
"""
resources = check.dict_param(resources, "resource_defs", key_type=str)
instance = check.opt_inst_param(instance, "instance", DagsterInstance)
resource_config = check.opt_dict_param(resource_config,
"resource_config",
key_type=str)
log_manager = check.opt_inst_param(log_manager, "log_manager",
DagsterLogManager)
resource_defs = wrap_resources_for_execution(resources)
mapped_resource_config = _get_mapped_resource_config(
resource_defs, resource_config)
with ephemeral_instance_if_missing(instance) as dagster_instance:
resources_manager = resource_initialization_manager(
resource_defs=resource_defs,
resource_configs=mapped_resource_config,
log_manager=log_manager
if log_manager else initialize_console_manager(pipeline_run),
execution_plan=None,
pipeline_run=pipeline_run,
resource_keys_to_init=set(resource_defs.keys()),
instance=dagster_instance,
emit_persistent_events=False,
pipeline_def_for_backwards_compat=None,
)
try:
list(resources_manager.generate_setup_events())
instantiated_resources = check.inst(resources_manager.get_object(),
ScopedResourcesBuilder)
yield instantiated_resources.build(
set(instantiated_resources.resource_instance_dict.keys()))
finally:
list(resources_manager.generate_teardown_events())
