def resource(config_schema=None, description=None, config=None): '''Define a resource. The decorated function should accept an :py:class:`InitResourceContext` and return an instance of the resource. This function will become the ``resource_fn`` of an underlying :py:class:`ResourceDefinition`. If the decorated function yields once rather than returning (in the manner of functions decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then the body of the function after the yield will be run after execution resolves, allowing users to write their own teardown/cleanup logic. Args: config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in `init_context.resource_config`. description(Optional[str]): A human-readable description of the resource. ''' # This case is for when decorator is used bare, without arguments. # E.g. @resource versus @resource() if callable( config_schema) and not is_callable_valid_config_arg(config_schema): return _ResourceDecoratorCallable()(config_schema) def _wrap(resource_fn): return _ResourceDecoratorCallable( config_schema=canonicalize_backcompat_args(config_schema, 'config_schema', config, 'config', '0.9.0'), description=description, )(resource_fn) return _wrap
def logger(config=None, description=None): '''Define a logger. The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of :py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying :py:class:`LoggerDefinition`. Args: config (Optional[Any]): The schema for the config. Configuration data available in `init_context.logger_config`. This value can be a: - :py:class:`Field` - Python primitive types that resolve to dagster config types - int, float, bool, str, list. - A dagster config type: Int, Float, Bool, List, Optional, :py:class:`Selector`, :py:class:`Dict` - A bare python dictionary, which is wrapped in Field(Dict(...)). Any values in the dictionary get resolved by the same rules, recursively. config_field (Optional[Field]): Used in the rare case of a top level config type other than a dictionary. Only one of config or config_field can be provided. description (Optional[str]): A human-readable description of the logger. ''' # This case is for when decorator is used bare, without arguments. # E.g. @logger versus @logger() if callable(config) and not is_callable_valid_config_arg(config): return LoggerDefinition(logger_fn=config) def _wrap(logger_fn): return LoggerDefinition(logger_fn, config, description) return _wrap
def logger( config_schema: Any = None, description: Optional[str] = None ) -> Union["LoggerDefinition", Callable[["InitLoggerFunction"], "LoggerDefinition"]]: """Define a logger. The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of :py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying :py:class:`LoggerDefinition`. Args: config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in `init_context.logger_config`. If not set, Dagster will accept any config provided. description (Optional[str]): A human-readable description of the logger. """ # This case is for when decorator is used bare, without arguments. # E.g. @logger versus @logger() if callable( config_schema) and not is_callable_valid_config_arg(config_schema): return LoggerDefinition(logger_fn=config_schema) def _wrap(logger_fn: "InitLoggerFunction") -> "LoggerDefinition": return LoggerDefinition( logger_fn=logger_fn, config_schema=config_schema, description=description, ) return _wrap
def resource(config=None, description=None): '''Define a resource. The decorated function should accept an :py:class:`InitResourceContext` and return an instance of the resource. This function will become the ``resource_fn`` of an underlying :py:class:`ResourceDefinition`. If the decorated function yields once rather than returning (in the manner of functions decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then the body of the function after the yield will be run after execution resolves, allowing users to write their own teardown/cleanup logic. Args: config (Optional[Any]): The schema for the config. Configuration data available in `init_context.resource_config`. This value can be a: - :py:class:`Field` - Python primitive types that resolve to dagster config types - int, float, bool, str, list. - A dagster config type: Int, Float, Bool, List, Optional, :py:class:`Selector`, :py:class:`Dict` - A bare python dictionary, which is wrapped in Field(Dict(...)). Any values of in the dictionary get resolved by the same rules, recursively. description(Optional[str]): A human-readable description of the resource. ''' # This case is for when decorator is used bare, without arguments. # E.g. @resource versus @resource() if callable(config) and not is_callable_valid_config_arg(config): return ResourceDefinition(resource_fn=config) def _wrap(resource_fn): return ResourceDefinition(resource_fn, config, description) return _wrap
def resource(config_schema=None, description=None, version=None): """Define a resource. The decorated function should accept an :py:class:`InitResourceContext` and return an instance of the resource. This function will become the ``resource_fn`` of an underlying :py:class:`ResourceDefinition`. If the decorated function yields once rather than returning (in the manner of functions decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then the body of the function after the yield will be run after execution resolves, allowing users to write their own teardown/cleanup logic. Args: config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in `init_context.resource_config`. description(Optional[str]): A human-readable description of the resource. version (Optional[str]): (Experimental) The version of a resource function. Two wrapped resource functions should only have the same version if they produce the same resource definition when provided with the same inputs. """ # This case is for when decorator is used bare, without arguments. # E.g. @resource versus @resource() if callable(config_schema) and not is_callable_valid_config_arg(config_schema): return _ResourceDecoratorCallable()(config_schema) def _wrap(resource_fn): return _ResourceDecoratorCallable( config_schema=config_schema, description=description, version=version, )(resource_fn) return _wrap
def logger(config_schema=None, description=None, config=None): '''Define a logger. The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of :py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying :py:class:`LoggerDefinition`. Args: config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in `init_context.logger_config`. description (Optional[str]): A human-readable description of the logger. ''' # This case is for when decorator is used bare, without arguments. # E.g. @logger versus @logger() if callable( config_schema) and not is_callable_valid_config_arg(config_schema): return LoggerDefinition(logger_fn=config_schema) def _wrap(logger_fn): return LoggerDefinition( logger_fn=logger_fn, config_schema=canonicalize_backcompat_args(config_schema, 'config_schema', config, 'config', '0.9.0'), description=description, ) return _wrap
def input_manager(config_schema=None, description=None, input_config_schema=None, version=None): """Define an input manager. The decorated function should accept a :py:class:`InputContext` and resource config, and return a loaded object that will be passed into one of the inputs of a solid. The decorator produces an :py:class:`InputManagerDefinition`. Args: config_schema (Optional[ConfigSchema]): The schema for the resource-level config. description (Optional[str]): A human-readable description of the resource. input_config_schema (Optional[ConfigSchema]): A schema for the input-level config. Each input that uses this input manager can be configured separately using this config. version (Optional[str]): (Experimental) the version of the input manager definition. **Examples:** .. code-block:: python @input_manager def csv_loader(_, _resource_config): return read_csv("some/path") @solid(input_defs=[InputDefinition("input1", manager_key="csv_loader_key")]) def my_solid(_, input1): do_stuff(input1) @pipeline(mode_defs=[ModeDefinition(resource_defs={"csv_loader_key": csv_loader})]) def my_pipeline(): my_solid() @input_manager(config_schema={"base_dir": str}) def csv_loader(_, resource_config): return read_csv(resource_config["base_dir"] + "/some/path") @input_manager(input_config_schema={"path": str}) def csv_loader(context, _resource_config): return read_csv(context.input_config["path"]) """ if callable( config_schema) and not is_callable_valid_config_arg(config_schema): return _InputManagerDecoratorCallable()(config_schema) def _wrap(load_fn): return _InputManagerDecoratorCallable( config_schema=config_schema, description=description, version=version, input_config_schema=input_config_schema, )(load_fn) return _wrap
def resource(config=None, description=None): '''Define a resource. The decorated function should accept an :py:class:`InitResourceContext` and return an instance of the resource. This function will become the ``resource_fn`` of an underlying :py:class:`ResourceDefinition`. If the decorated function yields once rather than returning (in the manner of functions decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then the body of the function after the yield will be run after execution resolves, allowing users to write their own teardown/cleanup logic. Args: config (Optional[Any]): The schema for the config. Configuration data available in `init_context.resource_config`. 1. A Python primitive type that resolves to a Dagster config type (:py:class:`~python:int`, :py:class:`~python:float`, :py:class:`~python:bool`, :py:class:`~python:str`, or :py:class:`~python:list`). 2. A Dagster config type: :py:data:`~dagster.Int`, :py:data:`~dagster.Float`, :py:data:`~dagster.Bool`, :py:data:`~dagster.String`, :py:data:`~dagster.StringSource`, :py:data:`~dagster.Path`, :py:data:`~dagster.Any`, :py:class:`~dagster.Array`, :py:data:`~dagster.Noneable`, :py:data:`~dagster.Enum`, :py:class:`~dagster.Selector`, :py:class:`~dagster.Shape`, or :py:class:`~dagster.Permissive`. 3. A bare python dictionary, which will be automatically wrapped in :py:class:`~dagster.Shape`. Values of the dictionary are resolved recursively according to the same rules. 4. A bare python list of length one which itself is config type. Becomes :py:class:`Array` with list element as an argument. 5. An instance of :py:class:`~dagster.Field`. description(Optional[str]): A human-readable description of the resource. ''' # This case is for when decorator is used bare, without arguments. # E.g. @resource versus @resource() if callable(config) and not is_callable_valid_config_arg(config): return _ResourceDecoratorCallable()(config) def _wrap(resource_fn): return _ResourceDecoratorCallable(config=config, description=description)(resource_fn) return _wrap
def resource( config_schema: Optional[Union[Callable[["InitResourceContext"], Any], IDefinitionConfigSchema, Dict[str, Any]]] = None, description: Optional[str] = None, required_resource_keys: Optional[AbstractSet[str]] = None, version=None, ) -> Union[Callable[[Callable[["InitResourceContext"], Any]], "ResourceDefinition"], "ResourceDefinition"]: """Define a resource. The decorated function should accept an :py:class:`InitResourceContext` and return an instance of the resource. This function will become the ``resource_fn`` of an underlying :py:class:`ResourceDefinition`. If the decorated function yields once rather than returning (in the manner of functions decorable with :py:func:`@contextlib.contextmanager <python:contextlib.contextmanager>`) then the body of the function after the yield will be run after execution resolves, allowing users to write their own teardown/cleanup logic. Args: config_schema (Optional[ConfigSchema]): The schema for the config. Configuration data available in `init_context.resource_config`. If not set, Dagster will accept any config provided. description(Optional[str]): A human-readable description of the resource. version (Optional[str]): (Experimental) The version of a resource function. Two wrapped resource functions should only have the same version if they produce the same resource definition when provided with the same inputs. required_resource_keys (Optional[Set[str]]): Keys for the resources required by this resource. """ # This case is for when decorator is used bare, without arguments. # E.g. @resource versus @resource() if callable( config_schema) and not is_callable_valid_config_arg(config_schema): return _ResourceDecoratorCallable()(config_schema) def _wrap( resource_fn: Callable[["InitResourceContext"], Any] ) -> "ResourceDefinition": return _ResourceDecoratorCallable( config_schema=cast(Optional[Dict[str, Any]], config_schema), description=description, required_resource_keys=required_resource_keys, version=version, )(resource_fn) return _wrap
def logger(config=None, description=None): '''Define a logger. The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of :py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying :py:class:`LoggerDefinition`. Args: config (Optional[Any]): The schema for the config. Configuration data available in `init_context.logger_config`. This value can be any of: 1. A Python primitive type that resolves to a Dagster config type (:py:class:`~python:int`, :py:class:`~python:float`, :py:class:`~python:bool`, :py:class:`~python:str`, or :py:class:`~python:list`). 2. A Dagster config type: :py:data:`~dagster.Int`, :py:data:`~dagster.Float`, :py:data:`~dagster.Bool`, :py:data:`~dagster.String`, :py:data:`~dagster.StringSource`, :py:data:`~dagster.Path`, :py:data:`~dagster.Any`, :py:class:`~dagster.Array`, :py:data:`~dagster.Noneable`, :py:data:`~dagster.Enum`, :py:class:`~dagster.Selector`, :py:class:`~dagster.Shape`, or :py:class:`~dagster.Permissive`. 3. A bare python dictionary, which will be automatically wrapped in :py:class:`~dagster.Shape`. Values of the dictionary are resolved recursively according to the same rules. 4. A bare python list of length one which itself is config type. Becomes :py:class:`Array` with list element as an argument. 5. An instance of :py:class:`~dagster.Field`. description (Optional[str]): A human-readable description of the logger. ''' # This case is for when decorator is used bare, without arguments. # E.g. @logger versus @logger() if callable(config) and not is_callable_valid_config_arg(config): return LoggerDefinition(logger_fn=config) def _wrap(logger_fn): return LoggerDefinition(logger_fn, config, description) return _wrap
def logger(config=None, description=None): '''Define a logger. The decorated function should accept an :py:class:`InitLoggerContext` and return an instance of :py:class:`python:logging.Logger`. This function will become the ``logger_fn`` of an underlying :py:class:`LoggerDefinition`. Args: config (Optional[Any]): The schema for the config. Configuration data available in `init_context.logger_config`. This value can be: 1. A Python primitive type that resolve to dagster config types: int, float, bool, str. 2. A dagster config type: Int, Float, Bool, :py:class:`Array`, :py:class:`Noneable`, :py:class:`Selector`, :py:class:`Shape`, :py:class:`Permissive`, etc 3. A bare python dictionary, which is wrapped in :py:class:`Shape`. Any values in the dictionary get resolved by the same rules, recursively. 4. A bare python list of length one which itself is config type. Becomes :py:class:`Array` with list element as an argument. 5. A instance of :py:class:`Field`. description (Optional[str]): A human-readable description of the logger. ''' # This case is for when decorator is used bare, without arguments. # E.g. @logger versus @logger() if callable(config) and not is_callable_valid_config_arg(config): return LoggerDefinition(logger_fn=config) def _wrap(logger_fn): return LoggerDefinition(logger_fn, config, description) return _wrap
def asset_storage(config_schema=None, description=None, type_policies=None): first_arg = config_schema first_arg_is_fn = callable(config_schema) and not is_callable_valid_config_arg(config_schema) actual_config_schema = None if first_arg_is_fn else config_schema def inner(fn): check.callable_param(fn, 'fn') resource_def = AssetStorageDefinition( resource_fn=fn, config_schema=actual_config_schema, description=description, type_policies=type_policies, ) update_wrapper(resource_def, wrapped=fn) return resource_def if first_arg_is_fn: return inner(first_arg) return inner
def io_manager( config_schema=None, description=None, output_config_schema=None, input_config_schema=None, required_resource_keys=None, version=None, ): """ Define an IO manager. IOManagers are used to store solid outputs and load them as inputs to downstream solids. The decorated function should accept an :py:class:`InitResourceContext` and return an :py:class:`IOManager`. Args: config_schema (Optional[ConfigSchema]): The schema for the resource config. Configuration data available in `init_context.resource_config`. description(Optional[str]): A human-readable description of the resource. output_config_schema (Optional[ConfigSchema]): The schema for per-output config. input_config_schema (Optional[ConfigSchema]): The schema for per-input config. required_resource_keys (Optional[Set[str]]): Keys for the resources required by the object manager. version (Optional[str]): (Experimental) The version of a resource function. Two wrapped resource functions should only have the same version if they produce the same resource definition when provided with the same inputs. **Examples:** .. code-block:: python class MyIOManager(IOManager): def handle_output(self, context, obj): write_csv("some/path") def load_input(self, context): return read_csv("some/path") @io_manager def my_io_manager(init_context): return MyIOManager() @solid(output_defs=[OutputDefinition(io_manager_key="my_io_manager_key")]) def my_solid(_): return do_stuff() @pipeline( mode_defs=[ModeDefinition(resource_defs={"my_io_manager_key": my_io_manager})] ) def my_pipeline(): my_solid() execute_pipeline(my_pipeline) """ if callable(config_schema) and not is_callable_valid_config_arg(config_schema): return _IOManagerDecoratorCallable()(config_schema) def _wrap(resource_fn): return _IOManagerDecoratorCallable( config_schema=config_schema, description=description, required_resource_keys=required_resource_keys, version=version, output_config_schema=output_config_schema, input_config_schema=input_config_schema, )(resource_fn) return _wrap
def root_input_manager( config_schema=None, description=None, input_config_schema=None, required_resource_keys=None, version=None, ): """Define a root input manager. Root input managers load op inputs that aren't connected to upstream outputs. The decorated function should accept a :py:class:`InputContext` and resource config, and return a loaded object that will be passed into one of the inputs of an op. The decorator produces an :py:class:`RootInputManagerDefinition`. Args: config_schema (Optional[ConfigSchema]): The schema for the resource-level config. If not set, Dagster will accept any config provided. description (Optional[str]): A human-readable description of the resource. input_config_schema (Optional[ConfigSchema]): A schema for the input-level config. Each input that uses this input manager can be configured separately using this config. If not set, Dagster will accept any config provided. required_resource_keys (Optional[Set[str]]): Keys for the resources required by the input manager. version (Optional[str]): (Experimental) the version of the input manager definition. **Examples:** .. code-block:: python from dagster import root_input_manager, op, job, In @root_input_manager def csv_loader(_): return read_csv("some/path") @op(ins={"input1": In(root_manager_key="csv_loader_key")}) def my_op(_, input1): do_stuff(input1) @job(resource_defs={"csv_loader_key": csv_loader}) def my_job(): my_op() @root_input_manager(config_schema={"base_dir": str}) def csv_loader(context): return read_csv(context.resource_config["base_dir"] + "/some/path") @root_input_manager(input_config_schema={"path": str}) def csv_loader(context): return read_csv(context.config["path"]) """ if callable( config_schema) and not is_callable_valid_config_arg(config_schema): return _InputManagerDecoratorCallable()(config_schema) def _wrap(load_fn): return _InputManagerDecoratorCallable( config_schema=config_schema, description=description, version=version, input_config_schema=input_config_schema, required_resource_keys=required_resource_keys, )(load_fn) return _wrap
def output_manager(config_schema=None, description=None, output_config_schema=None, version=None): """Define an output manager. The decorated function should accept a :py:class:`OutputContext` and resource config, and handle an object that is yielded as one of the outputs of a solid. The decorator produces an :py:class:`OutputManagerDefinition`. Args: config_schema (Optional[ConfigSchema]): The schema for the resource-level config. description (Optional[str]): A human-readable description of the resource. output_config_schema (Optional[ConfigSchema]): A schema for the output-level config. Each output that uses this output manager can be configured separately using this config. version (Optional[str]): (Experimental) the version of the output manager definition. **Examples:** .. code-block:: python @output_manager def csv_materializer(_, _resource_config): write_csv("some/path") @solid(output_defs=[OutputDefinition(manager_key="csv_materializer_key")]) def my_solid(_): return do_stuff() @pipeline( mode_defs=[ModeDefinition(resource_defs={"csv_materializer_key": csv_materializer})] ) def my_pipeline(): my_solid() execute_pipeline(my_pipeline) OutputManager with resource config: .. code-block:: python @output_manager(config_schema={"base_dir": str}) def csv_materializer(_, resource_config): write_csv(resource_config["base_dir"] + "/some/path") ... execute_pipeline( my_pipeline, run_config={"resources": {"csv_materializer_key": {"config": {"base_dir"" "a/b/c"}}}}, ) OutputManager with per-output config: .. code-block:: python @output_manager(output_config_schema={"path": str}) def csv_materializer(context, _resource_config): write_csv(context.config["path"]) ... execute_pipeline( my_pipeline, run_config={"solids": {"my_solid": {"outputs": {"result": {"path"" "a/b/c/d"}}}}}, ) """ if callable( config_schema) and not is_callable_valid_config_arg(config_schema): return _OutputManagerDecoratorCallable()(config_schema) def _wrap(load_fn): return _OutputManagerDecoratorCallable( config_schema=config_schema, description=description, version=version, output_config_schema=output_config_schema, )(load_fn) return _wrap