def __init__(self,
name=None,
is_persistent=True,
config_schema=None,
required_resource_keys=None):
self.name = check.opt_str_param(name, 'name')
self.is_persistent = check.bool_param(is_persistent, 'is_persistent')
self.config_schema = check_user_facing_opt_config_param(
config_schema, 'config_schema')
self.required_resource_keys = check.opt_set_param(
required_resource_keys, 'required_resource_keys', of_type=str)
def __init__(self,
name,
config=None,
executor_creation_fn=None,
required_resource_keys=None):
self._name = check.str_param(name, 'name')
self._config_field = check_user_facing_opt_config_param(
config, 'config')
self._executor_creation_fn = check.opt_callable_param(
executor_creation_fn, 'executor_creation_fn')
self._required_resource_keys = check.opt_set_param(
required_resource_keys, 'required_resource_keys', of_type=str)
def __init__(
self,
resource_fn=None,
config_schema=None,
description=None,
config=None,
process_config_fn=None,
):
self._resource_fn = check.opt_callable_param(resource_fn,
'resource_fn')
self._config_schema = canonicalize_backcompat_args(
check_user_facing_opt_config_param(config_schema, 'config_schema'),
'config_schema',
check_user_facing_opt_config_param(config, 'config'),
'config',
'0.9.0',
)
self._description = check.opt_str_param(description, 'description')
self._process_config_fn = check.opt_callable_param(
process_config_fn,
'process_config_fn') or (lambda conf: process_config(
{'config': self._config_schema or {}}, conf))
def __init__(
self,
name,
config_schema=None,
executor_creation_fn=None,
required_resource_keys=None,
config=None,
):
self._name = check.str_param(name, 'name')
self._config_schema = canonicalize_backcompat_args(
check_user_facing_opt_config_param(config_schema, 'config_schema'),
'config_schema',
check_user_facing_opt_config_param(config, 'config'),
'config',
'0.9.0',
)
self._executor_creation_fn = check.opt_callable_param(
executor_creation_fn, 'executor_creation_fn')
self._required_resource_keys = frozenset(
check.opt_set_param(required_resource_keys,
'required_resource_keys',
of_type=str))
def __init__(
self,
logger_fn,
config_schema=None,
description=None,
_configured_config_mapping_fn=None,
):
self._logger_fn = check.callable_param(logger_fn, "logger_fn")
self._config_schema = check_user_facing_opt_config_param(
config_schema, "config_schema")
self._description = check.opt_str_param(description, "description")
self.__configured_config_mapping_fn = check.opt_callable_param(
_configured_config_mapping_fn, "config_mapping_fn")
def define_dagstermill_solid(
name,
notebook_path,
input_defs=None,
output_defs=None,
config_schema=None,
required_resource_keys=None,
output_notebook=None,
):
"""Wrap a Jupyter notebook in a solid.
Arguments:
name (str): The name of the solid.
notebook_path (str): Path to the backing notebook.
input_defs (Optional[List[InputDefinition]]): The solid's inputs.
output_defs (Optional[List[OutputDefinition]]): The solid's outputs. Your notebook should
call :py:func:`~dagstermill.yield_result` to yield each of these outputs.
required_resource_keys (Optional[Set[str]]): The string names of any required resources.
output_notebook (Optional[str]): If set, will be used as the name of an injected output of
type :py:class:`~dagster.FileHandle` that will point to the executed notebook (in
addition to the :py:class:`~dagster.AssetMaterialization` that is always created). This
respects the :py:class:`~dagster.core.storage.file_manager.FileManager` configured on
the pipeline system storage, so, e.g., if :py:class:`~dagster_aws.s3.s3_system_storage`
is configured, the output will be a :py:class:`~dagster_aws.s3.S3FileHandle`.
Returns:
:py:class:`~dagster.SolidDefinition`
"""
check.str_param(name, "name")
check.str_param(notebook_path, "notebook_path")
input_defs = check.opt_list_param(input_defs, "input_defs", of_type=InputDefinition)
output_defs = check.opt_list_param(output_defs, "output_defs", of_type=OutputDefinition)
required_resource_keys = check.opt_set_param(
required_resource_keys, "required_resource_keys", of_type=str
)
return SolidDefinition(
name=name,
input_defs=input_defs,
compute_fn=_dm_solid_compute(name, notebook_path, output_notebook),
output_defs=output_defs
+ (
[OutputDefinition(dagster_type=FileHandle, name=output_notebook)]
if output_notebook
else []
),
config_schema=check_user_facing_opt_config_param(config_schema, "config_schema"),
required_resource_keys=required_resource_keys,
description="This solid is backed by the notebook at {path}".format(path=notebook_path),
tags={"notebook_path": notebook_path, "kind": "ipynb"},
)
def configured(configurable, config_schema=None, **kwargs):
"""
A decorator that makes it easy to create a function-configured version of an object.
The following definition types can be configured using this function:
* :py:class:`CompositeSolidDefinition`
* :py:class:`ExecutorDefinition`
* :py:class:`IntermediateStorageDefinition`
* :py:class:`LoggerDefinition`
* :py:class:`ResourceDefinition`
* :py:class:`SolidDefinition`
* :py:class:`SystemStorageDefinition`
If the config that will be supplied to the object is constant, you may alternatively invoke this
and call the result with a dict of config values to be curried. Examples of both strategies
below.
Args:
configurable (IConfigMappable): An object that can be configured.
config_schema (ConfigSchema): The config schema that the inputs to the decorated function
must satisfy.
**kwargs: Arbitrary keyword arguments that will be passed to the initializer of the returned
object.
Returns:
(Callable[[Union[Any, Callable[[Any], Any]]], IConfigMappable])
**Examples:**
.. code-block:: python
dev_s3 = configured(s3_resource, name="dev_s3")({'bucket': 'dev'})
@configured(s3_resource):
def dev_s3(_):
return {'bucket': 'dev'}
@configured(s3_resource, {'bucket_prefix', str}):
def dev_s3(config):
return {'bucket': config['bucket_prefix'] + 'dev'}
"""
_check_configurable_param(configurable)
config_schema = check_user_facing_opt_config_param(config_schema, "config_schema")
def _configured(config_or_config_fn):
return configurable.configured(
config_schema=config_schema, config_or_config_fn=config_or_config_fn, **kwargs
)
return _configured
def __init__(
self,
resource_fn=None,
config_schema=None,
description=None,
config=None,
process_config_fn=None,
):
EXPECTED_POSITIONALS = ['*']
fn_positionals, _ = split_function_parameters(resource_fn,
EXPECTED_POSITIONALS)
missing_positional = validate_decorated_fn_positionals(
fn_positionals, EXPECTED_POSITIONALS)
if missing_positional:
raise DagsterInvalidDefinitionError(
"@resource '{resource_name}' decorated function does not have required "
"positional parameter '{missing_param}'. Resource functions should only have keyword "
"arguments that match input names and a first positional parameter."
.format(resource_name=resource_fn.__name__,
missing_param=missing_positional))
self._resource_fn = check.opt_callable_param(resource_fn,
'resource_fn')
self._config_schema = canonicalize_backcompat_args(
check_user_facing_opt_config_param(config_schema, 'config_schema'),
'config_schema',
check_user_facing_opt_config_param(config, 'config'),
'config',
'0.9.0',
)
self._description = check.opt_str_param(description, 'description')
self._process_config_fn = check.opt_callable_param(
process_config_fn,
'process_config_fn') or (lambda conf: process_config(
{'config': self._config_schema or {}}, conf))
def __init__(
self,
name,
input_defs,
compute_fn,
output_defs,
config_schema=None,
description=None,
tags=None,
required_resource_keys=None,
positional_inputs=None,
config=None,
):
self._compute_fn = check.callable_param(compute_fn, 'compute_fn')
self._config_schema = canonicalize_backcompat_args(
check_user_facing_opt_config_param(config_schema, 'config_schema'),
'config_schema',
check_user_facing_opt_config_param(config, 'config'),
'config',
'0.9.0',
)
self._required_resource_keys = frozenset(
check.opt_set_param(required_resource_keys,
'required_resource_keys',
of_type=str))
super(SolidDefinition, self).__init__(
name=name,
input_defs=check.list_param(input_defs, 'input_defs',
InputDefinition),
output_defs=check.list_param(output_defs, 'output_defs',
OutputDefinition),
description=description,
tags=check.opt_dict_param(tags, 'tags', key_type=str),
positional_inputs=positional_inputs,
)
def define_dagstermill_solid(
name,
notebook_path,
input_defs=None,
output_defs=None,
config=None,
required_resource_keys=None,
):
'''Wrap a Jupyter notebook in a solid.
Arguments:
name (str): The name of the solid.
notebook_path (str): Path to the backing notebook.
input_defs (Optional[list[:class:`dagster.InputDefinition`]]): The solid's inputs.
output_defs (Optional[list[:class:`dagster.OutputDefinition`]]): The solid's outputs.
required_resource_keys (Optional[set[str]]): The string names of any required resources.
Returns:
:class:`dagster.SolidDefinition`
'''
check.str_param(name, 'name')
check.str_param(notebook_path, 'notebook_path')
input_defs = check.opt_list_param(input_defs,
'input_defs',
of_type=InputDefinition)
output_defs = check.opt_list_param(output_defs,
'output_defs',
of_type=OutputDefinition)
required_resource_keys = check.opt_set_param(required_resource_keys,
'required_resource_keys',
of_type=str)
return SolidDefinition(
name=name,
input_defs=input_defs,
compute_fn=_dm_solid_compute(name, notebook_path),
output_defs=output_defs,
config=check_user_facing_opt_config_param(config, 'config'),
required_resource_keys=required_resource_keys,
description='This solid is backed by the notebook at {path}'.format(
path=notebook_path),
tags={
'notebook_path': notebook_path,
'kind': 'ipynb'
},
)
def __init__(
self,
name,
config_schema=None,
executor_creation_fn=None,
required_resource_keys=None,
_configured_config_mapping_fn=None,
):
self._name = check.str_param(name, "name")
self._config_schema = check_user_facing_opt_config_param(config_schema, "config_schema")
self._executor_creation_fn = check.opt_callable_param(
executor_creation_fn, "executor_creation_fn"
)
self._required_resource_keys = frozenset(
check.opt_set_param(required_resource_keys, "required_resource_keys", of_type=str)
)
self.__configured_config_mapping_fn = check.opt_callable_param(
_configured_config_mapping_fn, "config_mapping_fn"
)
def __new__(
cls,
name,
is_persistent,
required_resource_keys,
config_schema=None,
system_storage_creation_fn=None,
):
return super(SystemStorageDefinition, cls).__new__(
cls,
name=check.str_param(name, 'name'),
is_persistent=check.bool_param(is_persistent, 'is_persistent'),
config_schema=check_user_facing_opt_config_param(config_schema, 'config_schema'),
system_storage_creation_fn=check.opt_callable_param(
system_storage_creation_fn, 'system_storage_creation_fn'
),
required_resource_keys=frozenset(
check.set_param(required_resource_keys, 'required_resource_keys', of_type=str)
),
)
def __init__(
self,
name,
is_persistent,
required_resource_keys,
config_schema=None,
system_storage_creation_fn=None,
_configured_config_mapping_fn=None,
):
self._name = check_valid_name(name)
self._is_persistent = check.bool_param(is_persistent, "is_persistent")
self._config_schema = check_user_facing_opt_config_param(
config_schema, "config_schema")
self._system_storage_creation_fn = check.opt_callable_param(
system_storage_creation_fn, "system_storage_creation_fn")
self._required_resource_keys = frozenset(
check.set_param(required_resource_keys,
"required_resource_keys",
of_type=str))
self.__configured_config_mapping_fn = check.opt_callable_param(
_configured_config_mapping_fn, "config_mapping_fn")
def __init__(
self,
name,
input_defs,
compute_fn,
output_defs,
config_schema=None,
description=None,
tags=None,
required_resource_keys=None,
positional_inputs=None,
version=None,
_configured_config_mapping_fn=None,
):
self._compute_fn = check.callable_param(compute_fn, "compute_fn")
self._config_schema = check_user_facing_opt_config_param(
config_schema, "config_schema")
self._required_resource_keys = frozenset(
check.opt_set_param(required_resource_keys,
"required_resource_keys",
of_type=str))
self.__configured_config_mapping_fn = check.opt_callable_param(
_configured_config_mapping_fn, "config_mapping_fn")
self._version = check.opt_str_param(version, "version")
if version:
experimental_arg_warning("version", "SolidDefinition.__init__")
super(SolidDefinition, self).__init__(
name=name,
input_defs=check.list_param(input_defs, "input_defs",
InputDefinition),
output_defs=check.list_param(output_defs, "output_defs",
OutputDefinition),
description=description,
tags=check.opt_dict_param(tags, "tags", key_type=str),
positional_inputs=positional_inputs,
)
def __init__(
self,
name,
is_persistent,
required_resource_keys,
config_schema=None,
intermediate_storage_creation_fn=None,
_configured_config_mapping_fn=None,
):
self._name = check.str_param(name, "name")
self._is_persistent = check.bool_param(is_persistent, "is_persistent")
self._config_schema = check_user_facing_opt_config_param(
config_schema, "config_schema")
self._intermediate_storage_creation_fn = check.opt_callable_param(
intermediate_storage_creation_fn,
"intermediate_storage_creation_fn")
self._required_resource_keys = frozenset(
check.set_param(
required_resource_keys if required_resource_keys else set(),
"required_resource_keys",
of_type=str,
))
self.__configured_config_mapping_fn = check.opt_callable_param(
_configured_config_mapping_fn, "config_mapping_fn")
def __init__(self, logger_fn, config=None, description=None):
self._logger_fn = check.callable_param(logger_fn, 'logger_fn')
self._config_field = check_user_facing_opt_config_param(
config, 'config')
self._description = check.opt_str_param(description, 'description')
def __init__(self, resource_fn, config=None, description=None):
self._resource_fn = check.callable_param(resource_fn, 'resource_fn')
self._config_field = check_user_facing_opt_config_param(config, 'config',)
self._description = check.opt_str_param(description, 'description')
def __new__(cls, config_fn, config_schema=None):
return super(ConfigMapping, cls).__new__(
cls,
config_fn=check.callable_param(config_fn, 'config_fn'),
config_schema=check_user_facing_opt_config_param(config_schema, 'config_schema'),
)
def __init__(self, config_schema=None, description=None):
self.config_schema = check_user_facing_opt_config_param(
config_schema, 'config_schema')
self.description = check.opt_str_param(description, 'description')
def __init__(self, config_schema=None, description=None, version=None):
self.config_schema = check_user_facing_opt_config_param(config_schema, "config_schema")
self.description = check.opt_str_param(description, "description")
self.version = check.opt_str_param(version, "version")
