def test_resource_init_requires_instance(): @resource def basic_resource(init_context): assert init_context.instance with DagsterInstance.ephemeral() as instance: build_resources({"basic_resource": basic_resource}, instance)
def cleanup_memoized_results(pipeline_def, mode_str, instance, run_config): # Clean up any memoized outputs from the s3 bucket from dagster_aws.s3 import s3_resource, s3_pickle_io_manager execution_plan = create_execution_plan(pipeline_def, run_config=run_config, instance_ref=instance.get_ref(), mode=mode_str) with build_resources( { "s3": s3_resource, "io_manager": s3_pickle_io_manager }, resource_config=run_config["resources"], ) as resources: io_manager = resources.io_manager for step_output_handle, version in execution_plan.step_output_versions.items( ): output_context = build_output_context( step_key=step_output_handle.step_key, name=step_output_handle.output_name, version=version, ) # pylint: disable=protected-access key = io_manager._get_path(output_context) io_manager._rm_object(key)
def __init__( self, solid_config: Any, resources_dict: Optional[Dict[str, Any]], instance: Optional[DagsterInstance], ): # pylint: disable=super-init-not-called from dagster.core.execution.context_creation_pipeline import initialize_console_manager from dagster.core.execution.api import ephemeral_instance_if_missing self._solid_config = solid_config self._instance_provided = ( check.opt_inst_param(instance, "instance", DagsterInstance) is not None ) # Construct ephemeral instance if missing self._instance_cm = ephemeral_instance_if_missing(instance) # Pylint can't infer that the ephemeral_instance context manager has an __enter__ method, # so ignore lint error self._instance = self._instance_cm.__enter__() # pylint: disable=no-member # Open resource context manager self._resources_cm = build_resources( check.opt_dict_param(resources_dict, "resources_dict", key_type=str), instance ) self._resources = self._resources_cm.__enter__() # pylint: disable=no-member self._resources_contain_cm = isinstance(self._resources, IContainsGenerator) self._log = initialize_console_manager(None) self._pdb: Optional[ForkedPdb] = None self._cm_scope_entered = False
def test_basic_resource(): @resource def basic_resource(_): return "foo" with build_resources(resources={"basic_resource": basic_resource}, ) as resources: assert resources.basic_resource == "foo"
def test_basic_resource(): @resource def basic_resource(_): return "foo" with build_resources( resource_defs={"basic_resource": basic_resource}) as resources: assert resources.resource_instance_dict["basic_resource"] == "foo"
def test_basic_resource(): @resource def basic_resource(_): return "foo" with build_resources(resources={"basic_resource": basic_resource}, ) as resources: assert not isinstance(resources, IContainsGenerator) assert resources.basic_resource == "foo"
def test_resource_init_values(): @resource(required_resource_keys={"bar"}) def foo_resource(init_context): assert init_context.resources.bar == "bar" return "foo" with build_resources({"foo": foo_resource, "bar": "bar"}) as resources: assert resources.foo == "foo" assert resources.bar == "bar"
def test_basic_resource(): @resource def basic_resource(_): return "foo" with DagsterInstance.ephemeral() as dagster_instance: with build_resources( resource_defs={"basic_resource": basic_resource}, instance=dagster_instance, ) as resources: assert resources.basic_resource == "foo"
def test_error_in_resource_initialization(): @resource def i_will_fail(_): raise Exception("Failed.") with pytest.raises( DagsterResourceFunctionError, match="Error executing resource_fn on ResourceDefinition i_will_fail", ): with build_resources(resources={"i_will_fail": i_will_fail}): pass
def __init__( self, step_key: Optional[str] = None, name: Optional[str] = None, pipeline_name: Optional[str] = None, run_id: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, mapping_key: Optional[str] = None, config: Optional[Any] = None, solid_def: Optional["SolidDefinition"] = None, dagster_type: Optional["DagsterType"] = None, log_manager: Optional["DagsterLogManager"] = None, version: Optional[str] = None, resource_config: Optional[Dict[str, Any]] = None, resources: Optional[Union["Resources", Dict[str, Any]]] = None, step_context: Optional["StepExecutionContext"] = None, op_def: Optional["OpDefinition"] = None, ): from dagster.core.definitions.resource_definition import IContainsGenerator, Resources from dagster.core.execution.build_resources import build_resources self._step_key = step_key self._name = name self._pipeline_name = pipeline_name self._run_id = run_id self._metadata = metadata self._mapping_key = mapping_key self._config = config check.invariant(solid_def is None or op_def is None, "Can't provide both a solid_def and an op_def arg") self._solid_def = solid_def or op_def self._dagster_type = dagster_type self._log = log_manager self._version = version self._resource_config = resource_config self._step_context = step_context if isinstance(resources, Resources): self._resources_cm = None self._resources = resources else: self._resources_cm = build_resources( check.opt_dict_param(resources, "resources", key_type=str)) self._resources = self._resources_cm.__enter__() # pylint: disable=no-member self._resources_contain_cm = isinstance(self._resources, IContainsGenerator) self._cm_scope_entered = False self._events: List["DagsterEvent"] = [] self._user_events: List[Union[AssetMaterialization, AssetObservation, Materialization]] = [] self._metadata_entries: Optional[List[Union[ MetadataEntry, PartitionMetadataEntry]]] = None
def test_resource_init_values(): @resource(required_resource_keys={"bar"}) def foo_resource(init_context): assert init_context.resources.bar == "bar" return "foo" with DagsterInstance.ephemeral() as instance: with build_resources({ "foo": foo_resource, "bar": "bar" }, instance) as resources: assert resources.foo == "foo" assert resources.bar == "bar"
def test_resource_with_config(): @resource( config_schema={"plant": str, "animal": Field(str, is_required=False, default_value="dog")} ) def basic_resource(init_context): plant = init_context.resource_config["plant"] animal = init_context.resource_config["animal"] return f"plant: {plant}, animal: {animal}" with build_resources( resources={"basic_resource": basic_resource}, run_config={"basic_resource": {"config": {"plant": "maple tree"}}}, ) as resources: assert resources.basic_resource == "plant: maple tree, animal: dog"
def test_error_in_resource_initialization(): @resource def i_will_fail(_): raise Exception("Failed.") with DagsterInstance.ephemeral() as dagster_instance: with pytest.raises( DagsterResourceFunctionError, match= "Error executing resource_fn on ResourceDefinition i_will_fail", ): with build_resources(resource_defs={"i_will_fail": i_will_fail}, instance=dagster_instance): pass
def test_context_manager_resource(): tore_down = [] @resource def cm_resource(_): try: yield "foo" finally: tore_down.append("yes") with build_resources({"cm_resource": cm_resource}) as resources: assert isinstance(resources, IContainsGenerator) assert resources.cm_resource == "foo" assert tore_down == ["yes"]
def test_resource_with_dependencies(): @resource(config_schema={"animal": str}) def no_deps(init_context): return init_context.resource_config["animal"] @resource(required_resource_keys={"no_deps"}) def has_deps(init_context): return f"{init_context.resources.no_deps} is an animal." with build_resources( resources={"no_deps": no_deps, "has_deps": has_deps}, run_config={"no_deps": {"config": {"animal": "dog"}}}, ) as resources: assert resources.no_deps == "dog" assert resources.has_deps == "dog is an animal."
def __init__( self, step_key: Optional[str] = None, name: Optional[str] = None, pipeline_name: Optional[str] = None, run_id: Optional[str] = None, metadata: Optional[Dict[str, Any]] = None, mapping_key: Optional[str] = None, config: Optional[Any] = None, solid_def: Optional["SolidDefinition"] = None, dagster_type: Optional["DagsterType"] = None, log_manager: Optional["DagsterLogManager"] = None, version: Optional[str] = None, resource_config: Optional[Dict[str, Any]] = None, resources: Optional[Union["Resources", Dict[str, Any]]] = None, step_context: Optional["StepExecutionContext"] = None, ): from dagster.core.definitions.resource import Resources, IContainsGenerator from dagster.core.execution.build_resources import build_resources self._step_key = step_key self._name = name self._pipeline_name = pipeline_name self._run_id = run_id self._metadata = metadata self._mapping_key = mapping_key self._config = config self._solid_def = solid_def self._dagster_type = dagster_type self._log = log_manager self._version = version self._resource_config = resource_config self._step_context = step_context if isinstance(resources, Resources): self._resources_cm = None self._resources = resources else: self._resources_cm = build_resources( check.opt_dict_param(resources, "resources", key_type=str)) self._resources = self._resources_cm.__enter__() # pylint: disable=no-member self._resources_contain_cm = isinstance(self._resources, IContainsGenerator) self._cm_scope_entered = False
def __init__( self, name: Optional[str] = None, pipeline_name: Optional[str] = None, solid_def: Optional["SolidDefinition"] = None, config: Optional[Any] = None, metadata: Optional[Dict[str, Any]] = None, upstream_output: Optional["OutputContext"] = None, dagster_type: Optional["DagsterType"] = None, log_manager: Optional["DagsterLogManager"] = None, resource_config: Optional[Dict[str, Any]] = None, resources: Optional[Union["Resources", Dict[str, Any]]] = None, step_context: Optional["StepExecutionContext"] = None, op_def: Optional["OpDefinition"] = None, ): from dagster.core.definitions.resource_definition import Resources, IContainsGenerator from dagster.core.execution.build_resources import build_resources self._name = name self._pipeline_name = pipeline_name check.invariant( solid_def is None or op_def is None, "Can't provide both a solid_def and an op_def arg" ) self._solid_def = solid_def or op_def self._config = config self._metadata = metadata self._upstream_output = upstream_output self._dagster_type = dagster_type self._log = log_manager self._resource_config = resource_config self._step_context = step_context if isinstance(resources, Resources): self._resources_cm = None self._resources = resources else: self._resources_cm = build_resources( check.opt_dict_param(resources, "resources", key_type=str) ) self._resources = self._resources_cm.__enter__() # pylint: disable=no-member self._resources_contain_cm = isinstance(self._resources, IContainsGenerator) self._cm_scope_entered = False
def __init__( self, resource_config: Any, resources: Optional[Union[Resources, Dict[str, Any]]], instance: Optional[DagsterInstance], ): from dagster.core.execution.build_resources import build_resources from dagster.core.execution.api import ephemeral_instance_if_missing from dagster.core.execution.context_creation_pipeline import initialize_console_manager self._instance_provided = (check.opt_inst_param( instance, "instance", DagsterInstance) is not None) # Construct ephemeral instance if missing self._instance_cm = ephemeral_instance_if_missing(instance) # Pylint can't infer that the ephemeral_instance context manager has an __enter__ method, # so ignore lint error instance = self._instance_cm.__enter__() # pylint: disable=no-member # If we are provided with a Resources instance, then we do not need to initialize if isinstance(resources, Resources): self._resources_cm = None else: self._resources_cm = build_resources(check.opt_dict_param( resources, "resources", key_type=str), instance=instance) resources = self._resources_cm.__enter__() # pylint: disable=no-member self._resources_contain_cm = isinstance(resources, IContainsGenerator) self._cm_scope_entered = False super(UnboundInitResourceContext, self).__init__( resource_config=resource_config, resources=resources, resource_def=None, instance=instance, pipeline_run=None, log_manager=initialize_console_manager(None), pipeline_def_for_backwards_compat=None, )
def test_resource_init_io_manager(): build_resources({"io_manager": InMemoryIOManager})
def test_resource_init_requires_instance(): @resource def basic_resource(init_context): assert init_context.instance build_resources({"basic_resource": basic_resource})
def test_resource_init_io_manager(): with DagsterInstance.ephemeral() as instance: build_resources({"io_manager": InMemoryIOManager}, instance)