def resource(config_schema=None, description=None, config=None):
'''Define a resource.
The decorated function should accept an :py:class:`InitResourceContext` and return an instance of
the resource. This function will become the ``resource_fn`` of an underlying
:py:class:`ResourceDefinition`.
If the decorated function yields once rather than returning (in the manner of functions
decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then
the body of the function after the yield will be run after execution resolves, allowing users
to write their own teardown/cleanup logic.
Args:
config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in
`init_context.resource_config`.
description(Optional[str]): A human-readable description of the resource.
'''
# This case is for when decorator is used bare, without arguments.
# E.g. @resource versus @resource()
if callable(
config_schema) and not is_callable_valid_config_arg(config_schema):
return _ResourceDecoratorCallable()(config_schema)
def _wrap(resource_fn):
return _ResourceDecoratorCallable(
config_schema=canonicalize_backcompat_args(config_schema,
'config_schema', config,
'config', '0.9.0'),
description=description,
)(resource_fn)
return _wrap
def logger(config=None, description=None):
'''Define a logger.
The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of
:py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying
:py:class:`LoggerDefinition`.
Args:
config (Optional[Any]): The schema for the config. Configuration data available in
`init_context.logger_config`.
This value can be a:
- :py:class:`Field`
- Python primitive types that resolve to dagster config types
- int, float, bool, str, list.
- A dagster config type: Int, Float, Bool, List, Optional, :py:class:`Selector`, :py:class:`Dict`
- A bare python dictionary, which is wrapped in Field(Dict(...)). Any values
in the dictionary get resolved by the same rules, recursively.
config_field (Optional[Field]): Used in the rare case of a top level config type other than
a dictionary.
Only one of config or config_field can be provided.
description (Optional[str]): A human-readable description of the logger.
'''
# This case is for when decorator is used bare, without arguments.
# E.g. @logger versus @logger()
if callable(config) and not is_callable_valid_config_arg(config):
return LoggerDefinition(logger_fn=config)
def _wrap(logger_fn):
return LoggerDefinition(logger_fn, config, description)
return _wrap
def logger(
config_schema: Any = None,
description: Optional[str] = None
) -> Union["LoggerDefinition", Callable[["InitLoggerFunction"],
"LoggerDefinition"]]:
"""Define a logger.
The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of
:py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying
:py:class:`LoggerDefinition`.
Args:
config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in
`init_context.logger_config`. If not set, Dagster will accept any config provided.
description (Optional[str]): A human-readable description of the logger.
"""
# This case is for when decorator is used bare, without arguments.
# E.g. @logger versus @logger()
if callable(
config_schema) and not is_callable_valid_config_arg(config_schema):
return LoggerDefinition(logger_fn=config_schema)
def _wrap(logger_fn: "InitLoggerFunction") -> "LoggerDefinition":
return LoggerDefinition(
logger_fn=logger_fn,
config_schema=config_schema,
description=description,
)
return _wrap
def resource(config=None, description=None):
'''Define a resource.
The decorated function should accept an :py:class:`InitResourceContext` and return an instance of
the resource. This function will become the ``resource_fn`` of an underlying
:py:class:`ResourceDefinition`.
If the decorated function yields once rather than returning (in the manner of functions
decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then
the body of the function after the yield will be run after execution resolves, allowing users
to write their own teardown/cleanup logic.
Args:
config (Optional[Any]): The schema for the config. Configuration data available in
`init_context.resource_config`.
This value can be a:
- :py:class:`Field`
- Python primitive types that resolve to dagster config types
- int, float, bool, str, list.
- A dagster config type: Int, Float, Bool, List, Optional, :py:class:`Selector`, :py:class:`Dict`
- A bare python dictionary, which is wrapped in Field(Dict(...)). Any values of
in the dictionary get resolved by the same rules, recursively.
description(Optional[str]): A human-readable description of the resource.
'''
# This case is for when decorator is used bare, without arguments.
# E.g. @resource versus @resource()
if callable(config) and not is_callable_valid_config_arg(config):
return ResourceDefinition(resource_fn=config)
def _wrap(resource_fn):
return ResourceDefinition(resource_fn, config, description)
return _wrap
def resource(config_schema=None, description=None, version=None):
"""Define a resource.
The decorated function should accept an :py:class:`InitResourceContext` and return an instance of
the resource. This function will become the ``resource_fn`` of an underlying
:py:class:`ResourceDefinition`.
If the decorated function yields once rather than returning (in the manner of functions
decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then
the body of the function after the yield will be run after execution resolves, allowing users
to write their own teardown/cleanup logic.
Args:
config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in
`init_context.resource_config`.
description(Optional[str]): A human-readable description of the resource.
version (Optional[str]): (Experimental) The version of a resource function. Two wrapped
resource functions should only have the same version if they produce the same resource
definition when provided with the same inputs.
"""
# This case is for when decorator is used bare, without arguments.
# E.g. @resource versus @resource()
if callable(config_schema) and not is_callable_valid_config_arg(config_schema):
return _ResourceDecoratorCallable()(config_schema)
def _wrap(resource_fn):
return _ResourceDecoratorCallable(
config_schema=config_schema, description=description, version=version,
)(resource_fn)
return _wrap
def logger(config_schema=None, description=None, config=None):
'''Define a logger.
The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of
:py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying
:py:class:`LoggerDefinition`.
Args:
config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in
`init_context.logger_config`.
description (Optional[str]): A human-readable description of the logger.
'''
# This case is for when decorator is used bare, without arguments.
# E.g. @logger versus @logger()
if callable(
config_schema) and not is_callable_valid_config_arg(config_schema):
return LoggerDefinition(logger_fn=config_schema)
def _wrap(logger_fn):
return LoggerDefinition(
logger_fn=logger_fn,
config_schema=canonicalize_backcompat_args(config_schema,
'config_schema', config,
'config', '0.9.0'),
description=description,
)
return _wrap
def input_manager(config_schema=None,
description=None,
input_config_schema=None,
version=None):
"""Define an input manager.
The decorated function should accept a :py:class:`InputContext` and resource config, and return
a loaded object that will be passed into one of the inputs of a solid.
The decorator produces an :py:class:`InputManagerDefinition`.
Args:
config_schema (Optional[ConfigSchema]): The schema for the resource-level config.
description (Optional[str]): A human-readable description of the resource.
input_config_schema (Optional[ConfigSchema]): A schema for the input-level config. Each
input that uses this input manager can be configured separately using this config.
version (Optional[str]): (Experimental) the version of the input manager definition.
**Examples:**
.. code-block:: python
@input_manager
def csv_loader(_, _resource_config):
return read_csv("some/path")
@solid(input_defs=[InputDefinition("input1", manager_key="csv_loader_key")])
def my_solid(_, input1):
do_stuff(input1)
@pipeline(mode_defs=[ModeDefinition(resource_defs={"csv_loader_key": csv_loader})])
def my_pipeline():
my_solid()
@input_manager(config_schema={"base_dir": str})
def csv_loader(_, resource_config):
return read_csv(resource_config["base_dir"] + "/some/path")
@input_manager(input_config_schema={"path": str})
def csv_loader(context, _resource_config):
return read_csv(context.input_config["path"])
"""
if callable(
config_schema) and not is_callable_valid_config_arg(config_schema):
return _InputManagerDecoratorCallable()(config_schema)
def _wrap(load_fn):
return _InputManagerDecoratorCallable(
config_schema=config_schema,
description=description,
version=version,
input_config_schema=input_config_schema,
)(load_fn)
return _wrap
def resource(config=None, description=None):
'''Define a resource.
The decorated function should accept an :py:class:`InitResourceContext` and return an instance of
the resource. This function will become the ``resource_fn`` of an underlying
:py:class:`ResourceDefinition`.
If the decorated function yields once rather than returning (in the manner of functions
decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then
the body of the function after the yield will be run after execution resolves, allowing users
to write their own teardown/cleanup logic.
Args:
config (Optional[Any]): The schema for the config. Configuration data available in
`init_context.resource_config`.
1. A Python primitive type that resolves to a Dagster config type
(:py:class:`~python:int`, :py:class:`~python:float`, :py:class:`~python:bool`,
:py:class:`~python:str`, or :py:class:`~python:list`).
2. A Dagster config type: :py:data:`~dagster.Int`, :py:data:`~dagster.Float`,
:py:data:`~dagster.Bool`, :py:data:`~dagster.String`,
:py:data:`~dagster.StringSource`, :py:data:`~dagster.Path`, :py:data:`~dagster.Any`,
:py:class:`~dagster.Array`, :py:data:`~dagster.Noneable`, :py:data:`~dagster.Enum`,
:py:class:`~dagster.Selector`, :py:class:`~dagster.Shape`, or
:py:class:`~dagster.Permissive`.
3. A bare python dictionary, which will be automatically wrapped in
:py:class:`~dagster.Shape`. Values of the dictionary are resolved recursively
according to the same rules.
4. A bare python list of length one which itself is config type.
Becomes :py:class:`Array` with list element as an argument.
5. An instance of :py:class:`~dagster.Field`.
description(Optional[str]): A human-readable description of the resource.
'''
# This case is for when decorator is used bare, without arguments.
# E.g. @resource versus @resource()
if callable(config) and not is_callable_valid_config_arg(config):
return _ResourceDecoratorCallable()(config)
def _wrap(resource_fn):
return _ResourceDecoratorCallable(config=config,
description=description)(resource_fn)
return _wrap
def resource(
config_schema: Optional[Union[Callable[["InitResourceContext"], Any],
IDefinitionConfigSchema, Dict[str,
Any]]] = None,
description: Optional[str] = None,
required_resource_keys: Optional[AbstractSet[str]] = None,
version=None,
) -> Union[Callable[[Callable[["InitResourceContext"], Any]],
"ResourceDefinition"], "ResourceDefinition"]:
"""Define a resource.
The decorated function should accept an :py:class:`InitResourceContext` and return an instance of
the resource. This function will become the ``resource_fn`` of an underlying
:py:class:`ResourceDefinition`.
If the decorated function yields once rather than returning (in the manner of functions
decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then
the body of the function after the yield will be run after execution resolves, allowing users
to write their own teardown/cleanup logic.
Args:
config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in
`init_context.resource_config`. If not set, Dagster will accept any config provided.
description(Optional[str]): A human-readable description of the resource.
version (Optional[str]): (Experimental) The version of a resource function. Two wrapped
resource functions should only have the same version if they produce the same resource
definition when provided with the same inputs.
required_resource_keys (Optional[Set[str]]): Keys for the resources required by this resource.
"""
# This case is for when decorator is used bare, without arguments.
# E.g. @resource versus @resource()
if callable(
config_schema) and not is_callable_valid_config_arg(config_schema):
return _ResourceDecoratorCallable()(config_schema)
def _wrap(
resource_fn: Callable[["InitResourceContext"], Any]
) -> "ResourceDefinition":
return _ResourceDecoratorCallable(
config_schema=cast(Optional[Dict[str, Any]], config_schema),
description=description,
required_resource_keys=required_resource_keys,
version=version,
)(resource_fn)
return _wrap
def logger(config=None, description=None):
'''Define a logger.
The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of
:py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying
:py:class:`LoggerDefinition`.
Args:
config (Optional[Any]): The schema for the config. Configuration data available in
`init_context.logger_config`.
This value can be any of:
1. A Python primitive type that resolves to a Dagster config type
(:py:class:`~python:int`, :py:class:`~python:float`, :py:class:`~python:bool`,
:py:class:`~python:str`, or :py:class:`~python:list`).
2. A Dagster config type: :py:data:`~dagster.Int`, :py:data:`~dagster.Float`,
:py:data:`~dagster.Bool`, :py:data:`~dagster.String`,
:py:data:`~dagster.StringSource`, :py:data:`~dagster.Path`, :py:data:`~dagster.Any`,
:py:class:`~dagster.Array`, :py:data:`~dagster.Noneable`, :py:data:`~dagster.Enum`,
:py:class:`~dagster.Selector`, :py:class:`~dagster.Shape`, or
:py:class:`~dagster.Permissive`.
3. A bare python dictionary, which will be automatically wrapped in
:py:class:`~dagster.Shape`. Values of the dictionary are resolved recursively
according to the same rules.
4. A bare python list of length one which itself is config type.
Becomes :py:class:`Array` with list element as an argument.
5. An instance of :py:class:`~dagster.Field`.
description (Optional[str]): A human-readable description of the logger.
'''
# This case is for when decorator is used bare, without arguments.
# E.g. @logger versus @logger()
if callable(config) and not is_callable_valid_config_arg(config):
return LoggerDefinition(logger_fn=config)
def _wrap(logger_fn):
return LoggerDefinition(logger_fn, config, description)
return _wrap
def logger(config=None, description=None):
'''Define a logger.
The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of
:py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying
:py:class:`LoggerDefinition`.
Args:
config (Optional[Any]): The schema for the config. Configuration data available in
`init_context.logger_config`.
This value can be:
1. A Python primitive type that resolve to dagster config
types: int, float, bool, str.
2. A dagster config type: Int, Float, Bool,
:py:class:`Array`, :py:class:`Noneable`, :py:class:`Selector`,
:py:class:`Shape`, :py:class:`Permissive`, etc
3. A bare python dictionary, which is wrapped in :py:class:`Shape`. Any
values in the dictionary get resolved by the same rules, recursively.
4. A bare python list of length one which itself is config type.
Becomes :py:class:`Array` with list element as an argument.
5. A instance of :py:class:`Field`.
description (Optional[str]): A human-readable description of the logger.
'''
# This case is for when decorator is used bare, without arguments.
# E.g. @logger versus @logger()
if callable(config) and not is_callable_valid_config_arg(config):
return LoggerDefinition(logger_fn=config)
def _wrap(logger_fn):
return LoggerDefinition(logger_fn, config, description)
return _wrap
def asset_storage(config_schema=None, description=None, type_policies=None):
first_arg = config_schema
first_arg_is_fn = callable(config_schema) and not is_callable_valid_config_arg(config_schema)
actual_config_schema = None if first_arg_is_fn else config_schema
def inner(fn):
check.callable_param(fn, 'fn')
resource_def = AssetStorageDefinition(
resource_fn=fn,
config_schema=actual_config_schema,
description=description,
type_policies=type_policies,
)
update_wrapper(resource_def, wrapped=fn)
return resource_def
if first_arg_is_fn:
return inner(first_arg)
return inner
def io_manager(
config_schema=None,
description=None,
output_config_schema=None,
input_config_schema=None,
required_resource_keys=None,
version=None,
):
"""
Define an IO manager.
IOManagers are used to store solid outputs and load them as inputs to downstream solids.
The decorated function should accept an :py:class:`InitResourceContext` and return an
:py:class:`IOManager`.
Args:
config_schema (Optional[ConfigSchema]): The schema for the resource config. Configuration
data available in `init_context.resource_config`.
description(Optional[str]): A human-readable description of the resource.
output_config_schema (Optional[ConfigSchema]): The schema for per-output config.
input_config_schema (Optional[ConfigSchema]): The schema for per-input config.
required_resource_keys (Optional[Set[str]]): Keys for the resources required by the object
manager.
version (Optional[str]): (Experimental) The version of a resource function. Two wrapped
resource functions should only have the same version if they produce the same resource
definition when provided with the same inputs.
**Examples:**
.. code-block:: python
class MyIOManager(IOManager):
def handle_output(self, context, obj):
write_csv("some/path")
def load_input(self, context):
return read_csv("some/path")
@io_manager
def my_io_manager(init_context):
return MyIOManager()
@solid(output_defs=[OutputDefinition(io_manager_key="my_io_manager_key")])
def my_solid(_):
return do_stuff()
@pipeline(
mode_defs=[ModeDefinition(resource_defs={"my_io_manager_key": my_io_manager})]
)
def my_pipeline():
my_solid()
execute_pipeline(my_pipeline)
"""
if callable(config_schema) and not is_callable_valid_config_arg(config_schema):
return _IOManagerDecoratorCallable()(config_schema)
def _wrap(resource_fn):
return _IOManagerDecoratorCallable(
config_schema=config_schema,
description=description,
required_resource_keys=required_resource_keys,
version=version,
output_config_schema=output_config_schema,
input_config_schema=input_config_schema,
)(resource_fn)
return _wrap
def root_input_manager(
config_schema=None,
description=None,
input_config_schema=None,
required_resource_keys=None,
version=None,
):
"""Define a root input manager.
Root input managers load op inputs that aren't connected to upstream outputs.
The decorated function should accept a :py:class:`InputContext` and resource config, and return
a loaded object that will be passed into one of the inputs of an op.
The decorator produces an :py:class:`RootInputManagerDefinition`.
Args:
config_schema (Optional[ConfigSchema]): The schema for the resource-level config. If not
set, Dagster will accept any config provided.
description (Optional[str]): A human-readable description of the resource.
input_config_schema (Optional[ConfigSchema]): A schema for the input-level config. Each
input that uses this input manager can be configured separately using this config.
If not set, Dagster will accept any config provided.
required_resource_keys (Optional[Set[str]]): Keys for the resources required by the input
manager.
version (Optional[str]): (Experimental) the version of the input manager definition.
**Examples:**
.. code-block:: python
from dagster import root_input_manager, op, job, In
@root_input_manager
def csv_loader(_):
return read_csv("some/path")
@op(ins={"input1": In(root_manager_key="csv_loader_key")})
def my_op(_, input1):
do_stuff(input1)
@job(resource_defs={"csv_loader_key": csv_loader})
def my_job():
my_op()
@root_input_manager(config_schema={"base_dir": str})
def csv_loader(context):
return read_csv(context.resource_config["base_dir"] + "/some/path")
@root_input_manager(input_config_schema={"path": str})
def csv_loader(context):
return read_csv(context.config["path"])
"""
if callable(
config_schema) and not is_callable_valid_config_arg(config_schema):
return _InputManagerDecoratorCallable()(config_schema)
def _wrap(load_fn):
return _InputManagerDecoratorCallable(
config_schema=config_schema,
description=description,
version=version,
input_config_schema=input_config_schema,
required_resource_keys=required_resource_keys,
)(load_fn)
return _wrap
def output_manager(config_schema=None,
description=None,
output_config_schema=None,
version=None):
"""Define an output manager.
The decorated function should accept a :py:class:`OutputContext` and resource config, and
handle an object that is yielded as one of the outputs of a solid.
The decorator produces an :py:class:`OutputManagerDefinition`.
Args:
config_schema (Optional[ConfigSchema]): The schema for the resource-level config.
description (Optional[str]): A human-readable description of the resource.
output_config_schema (Optional[ConfigSchema]): A schema for the output-level config. Each
output that uses this output manager can be configured separately using this config.
version (Optional[str]): (Experimental) the version of the output manager definition.
**Examples:**
.. code-block:: python
@output_manager
def csv_materializer(_, _resource_config):
write_csv("some/path")
@solid(output_defs=[OutputDefinition(manager_key="csv_materializer_key")])
def my_solid(_):
return do_stuff()
@pipeline(
mode_defs=[ModeDefinition(resource_defs={"csv_materializer_key": csv_materializer})]
)
def my_pipeline():
my_solid()
execute_pipeline(my_pipeline)
OutputManager with resource config:
.. code-block:: python
@output_manager(config_schema={"base_dir": str})
def csv_materializer(_, resource_config):
write_csv(resource_config["base_dir"] + "/some/path")
...
execute_pipeline(
my_pipeline,
run_config={"resources": {"csv_materializer_key": {"config": {"base_dir"" "a/b/c"}}}},
)
OutputManager with per-output config:
.. code-block:: python
@output_manager(output_config_schema={"path": str})
def csv_materializer(context, _resource_config):
write_csv(context.config["path"])
...
execute_pipeline(
my_pipeline,
run_config={"solids": {"my_solid": {"outputs": {"result": {"path"" "a/b/c/d"}}}}},
)
"""
if callable(
config_schema) and not is_callable_valid_config_arg(config_schema):
return _OutputManagerDecoratorCallable()(config_schema)
def _wrap(load_fn):
return _OutputManagerDecoratorCallable(
config_schema=config_schema,
description=description,
version=version,
output_config_schema=output_config_schema,
)(load_fn)
return _wrap
