def test_resource_init_requires_instance():
@resource
def basic_resource(init_context):
assert init_context.instance
with DagsterInstance.ephemeral() as instance:
build_resources({"basic_resource": basic_resource}, instance)
def cleanup_memoized_results(pipeline_def, mode_str, instance, run_config):
# Clean up any memoized outputs from the s3 bucket
from dagster_aws.s3 import s3_resource, s3_pickle_io_manager
execution_plan = create_execution_plan(pipeline_def,
run_config=run_config,
instance_ref=instance.get_ref(),
mode=mode_str)
with build_resources(
{
"s3": s3_resource,
"io_manager": s3_pickle_io_manager
},
resource_config=run_config["resources"],
) as resources:
io_manager = resources.io_manager
for step_output_handle, version in execution_plan.step_output_versions.items(
):
output_context = build_output_context(
step_key=step_output_handle.step_key,
name=step_output_handle.output_name,
version=version,
)
# pylint: disable=protected-access
key = io_manager._get_path(output_context)
io_manager._rm_object(key)
def __init__(
self,
solid_config: Any,
resources_dict: Optional[Dict[str, Any]],
instance: Optional[DagsterInstance],
): # pylint: disable=super-init-not-called
from dagster.core.execution.context_creation_pipeline import initialize_console_manager
from dagster.core.execution.api import ephemeral_instance_if_missing
self._solid_config = solid_config
self._instance_provided = (
check.opt_inst_param(instance, "instance", DagsterInstance) is not None
)
# Construct ephemeral instance if missing
self._instance_cm = ephemeral_instance_if_missing(instance)
# Pylint can't infer that the ephemeral_instance context manager has an __enter__ method,
# so ignore lint error
self._instance = self._instance_cm.__enter__() # pylint: disable=no-member
# Open resource context manager
self._resources_cm = build_resources(
check.opt_dict_param(resources_dict, "resources_dict", key_type=str), instance
)
self._resources = self._resources_cm.__enter__() # pylint: disable=no-member
self._resources_contain_cm = isinstance(self._resources, IContainsGenerator)
self._log = initialize_console_manager(None)
self._pdb: Optional[ForkedPdb] = None
self._cm_scope_entered = False
def test_basic_resource():
@resource
def basic_resource(_):
return "foo"
with build_resources(resources={"basic_resource":
basic_resource}, ) as resources:
assert resources.basic_resource == "foo"
def test_basic_resource():
@resource
def basic_resource(_):
return "foo"
with build_resources(
resource_defs={"basic_resource": basic_resource}) as resources:
assert resources.resource_instance_dict["basic_resource"] == "foo"
def test_basic_resource():
@resource
def basic_resource(_):
return "foo"
with build_resources(resources={"basic_resource":
basic_resource}, ) as resources:
assert not isinstance(resources, IContainsGenerator)
assert resources.basic_resource == "foo"
def test_resource_init_values():
@resource(required_resource_keys={"bar"})
def foo_resource(init_context):
assert init_context.resources.bar == "bar"
return "foo"
with build_resources({"foo": foo_resource, "bar": "bar"}) as resources:
assert resources.foo == "foo"
assert resources.bar == "bar"
def test_basic_resource():
@resource
def basic_resource(_):
return "foo"
with DagsterInstance.ephemeral() as dagster_instance:
with build_resources(
resource_defs={"basic_resource": basic_resource},
instance=dagster_instance,
) as resources:
assert resources.basic_resource == "foo"
def test_error_in_resource_initialization():
@resource
def i_will_fail(_):
raise Exception("Failed.")
with pytest.raises(
DagsterResourceFunctionError,
match="Error executing resource_fn on ResourceDefinition i_will_fail",
):
with build_resources(resources={"i_will_fail": i_will_fail}):
pass
def __init__(
self,
step_key: Optional[str] = None,
name: Optional[str] = None,
pipeline_name: Optional[str] = None,
run_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
mapping_key: Optional[str] = None,
config: Optional[Any] = None,
solid_def: Optional["SolidDefinition"] = None,
dagster_type: Optional["DagsterType"] = None,
log_manager: Optional["DagsterLogManager"] = None,
version: Optional[str] = None,
resource_config: Optional[Dict[str, Any]] = None,
resources: Optional[Union["Resources", Dict[str, Any]]] = None,
step_context: Optional["StepExecutionContext"] = None,
op_def: Optional["OpDefinition"] = None,
):
from dagster.core.definitions.resource_definition import IContainsGenerator, Resources
from dagster.core.execution.build_resources import build_resources
self._step_key = step_key
self._name = name
self._pipeline_name = pipeline_name
self._run_id = run_id
self._metadata = metadata
self._mapping_key = mapping_key
self._config = config
check.invariant(solid_def is None or op_def is None,
"Can't provide both a solid_def and an op_def arg")
self._solid_def = solid_def or op_def
self._dagster_type = dagster_type
self._log = log_manager
self._version = version
self._resource_config = resource_config
self._step_context = step_context
if isinstance(resources, Resources):
self._resources_cm = None
self._resources = resources
else:
self._resources_cm = build_resources(
check.opt_dict_param(resources, "resources", key_type=str))
self._resources = self._resources_cm.__enter__() # pylint: disable=no-member
self._resources_contain_cm = isinstance(self._resources,
IContainsGenerator)
self._cm_scope_entered = False
self._events: List["DagsterEvent"] = []
self._user_events: List[Union[AssetMaterialization, AssetObservation,
Materialization]] = []
self._metadata_entries: Optional[List[Union[
MetadataEntry, PartitionMetadataEntry]]] = None
def test_resource_init_values():
@resource(required_resource_keys={"bar"})
def foo_resource(init_context):
assert init_context.resources.bar == "bar"
return "foo"
with DagsterInstance.ephemeral() as instance:
with build_resources({
"foo": foo_resource,
"bar": "bar"
}, instance) as resources:
assert resources.foo == "foo"
assert resources.bar == "bar"
def test_resource_with_config():
@resource(
config_schema={"plant": str, "animal": Field(str, is_required=False, default_value="dog")}
)
def basic_resource(init_context):
plant = init_context.resource_config["plant"]
animal = init_context.resource_config["animal"]
return f"plant: {plant}, animal: {animal}"
with build_resources(
resources={"basic_resource": basic_resource},
run_config={"basic_resource": {"config": {"plant": "maple tree"}}},
) as resources:
assert resources.basic_resource == "plant: maple tree, animal: dog"
def test_error_in_resource_initialization():
@resource
def i_will_fail(_):
raise Exception("Failed.")
with DagsterInstance.ephemeral() as dagster_instance:
with pytest.raises(
DagsterResourceFunctionError,
match=
"Error executing resource_fn on ResourceDefinition i_will_fail",
):
with build_resources(resource_defs={"i_will_fail": i_will_fail},
instance=dagster_instance):
pass
def test_context_manager_resource():
tore_down = []
@resource
def cm_resource(_):
try:
yield "foo"
finally:
tore_down.append("yes")
with build_resources({"cm_resource": cm_resource}) as resources:
assert isinstance(resources, IContainsGenerator)
assert resources.cm_resource == "foo"
assert tore_down == ["yes"]
def test_resource_with_dependencies():
@resource(config_schema={"animal": str})
def no_deps(init_context):
return init_context.resource_config["animal"]
@resource(required_resource_keys={"no_deps"})
def has_deps(init_context):
return f"{init_context.resources.no_deps} is an animal."
with build_resources(
resources={"no_deps": no_deps, "has_deps": has_deps},
run_config={"no_deps": {"config": {"animal": "dog"}}},
) as resources:
assert resources.no_deps == "dog"
assert resources.has_deps == "dog is an animal."
def __init__(
self,
step_key: Optional[str] = None,
name: Optional[str] = None,
pipeline_name: Optional[str] = None,
run_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
mapping_key: Optional[str] = None,
config: Optional[Any] = None,
solid_def: Optional["SolidDefinition"] = None,
dagster_type: Optional["DagsterType"] = None,
log_manager: Optional["DagsterLogManager"] = None,
version: Optional[str] = None,
resource_config: Optional[Dict[str, Any]] = None,
resources: Optional[Union["Resources", Dict[str, Any]]] = None,
step_context: Optional["StepExecutionContext"] = None,
):
from dagster.core.definitions.resource import Resources, IContainsGenerator
from dagster.core.execution.build_resources import build_resources
self._step_key = step_key
self._name = name
self._pipeline_name = pipeline_name
self._run_id = run_id
self._metadata = metadata
self._mapping_key = mapping_key
self._config = config
self._solid_def = solid_def
self._dagster_type = dagster_type
self._log = log_manager
self._version = version
self._resource_config = resource_config
self._step_context = step_context
if isinstance(resources, Resources):
self._resources_cm = None
self._resources = resources
else:
self._resources_cm = build_resources(
check.opt_dict_param(resources, "resources", key_type=str))
self._resources = self._resources_cm.__enter__() # pylint: disable=no-member
self._resources_contain_cm = isinstance(self._resources,
IContainsGenerator)
self._cm_scope_entered = False
def __init__(
self,
name: Optional[str] = None,
pipeline_name: Optional[str] = None,
solid_def: Optional["SolidDefinition"] = None,
config: Optional[Any] = None,
metadata: Optional[Dict[str, Any]] = None,
upstream_output: Optional["OutputContext"] = None,
dagster_type: Optional["DagsterType"] = None,
log_manager: Optional["DagsterLogManager"] = None,
resource_config: Optional[Dict[str, Any]] = None,
resources: Optional[Union["Resources", Dict[str, Any]]] = None,
step_context: Optional["StepExecutionContext"] = None,
op_def: Optional["OpDefinition"] = None,
):
from dagster.core.definitions.resource_definition import Resources, IContainsGenerator
from dagster.core.execution.build_resources import build_resources
self._name = name
self._pipeline_name = pipeline_name
check.invariant(
solid_def is None or op_def is None, "Can't provide both a solid_def and an op_def arg"
)
self._solid_def = solid_def or op_def
self._config = config
self._metadata = metadata
self._upstream_output = upstream_output
self._dagster_type = dagster_type
self._log = log_manager
self._resource_config = resource_config
self._step_context = step_context
if isinstance(resources, Resources):
self._resources_cm = None
self._resources = resources
else:
self._resources_cm = build_resources(
check.opt_dict_param(resources, "resources", key_type=str)
)
self._resources = self._resources_cm.__enter__() # pylint: disable=no-member
self._resources_contain_cm = isinstance(self._resources, IContainsGenerator)
self._cm_scope_entered = False
def __init__(
self,
resource_config: Any,
resources: Optional[Union[Resources, Dict[str, Any]]],
instance: Optional[DagsterInstance],
):
from dagster.core.execution.build_resources import build_resources
from dagster.core.execution.api import ephemeral_instance_if_missing
from dagster.core.execution.context_creation_pipeline import initialize_console_manager
self._instance_provided = (check.opt_inst_param(
instance, "instance", DagsterInstance) is not None)
# Construct ephemeral instance if missing
self._instance_cm = ephemeral_instance_if_missing(instance)
# Pylint can't infer that the ephemeral_instance context manager has an __enter__ method,
# so ignore lint error
instance = self._instance_cm.__enter__() # pylint: disable=no-member
# If we are provided with a Resources instance, then we do not need to initialize
if isinstance(resources, Resources):
self._resources_cm = None
else:
self._resources_cm = build_resources(check.opt_dict_param(
resources, "resources", key_type=str),
instance=instance)
resources = self._resources_cm.__enter__() # pylint: disable=no-member
self._resources_contain_cm = isinstance(resources,
IContainsGenerator)
self._cm_scope_entered = False
super(UnboundInitResourceContext, self).__init__(
resource_config=resource_config,
resources=resources,
resource_def=None,
instance=instance,
pipeline_run=None,
log_manager=initialize_console_manager(None),
pipeline_def_for_backwards_compat=None,
)
def test_resource_init_io_manager():
build_resources({"io_manager": InMemoryIOManager})
def test_resource_init_requires_instance():
@resource
def basic_resource(init_context):
assert init_context.instance
build_resources({"basic_resource": basic_resource})
def test_resource_init_io_manager():
with DagsterInstance.ephemeral() as instance:
build_resources({"io_manager": InMemoryIOManager}, instance)