def input_selector_schema(config_cls, required_resource_keys=None):
"""
Deprecated in favor of dagster_type_loader.
A decorator for annotating a function that can take the selected properties
from a ``config_value`` in to an instance of a custom type.
Args:
config_cls (Selector)
"""
rename_warning("dagster_type_loader", "input_selector_schema", "0.10.0")
from dagster.config.field import resolve_to_config_type
config_type = resolve_to_config_type(config_cls)
check.param_invariant(config_type.kind == ConfigTypeKind.SELECTOR,
"config_cls")
def _wrap(func):
def _selector(context, config_value):
selector_key, selector_value = ensure_single_item(config_value)
return func(context, selector_key, selector_value)
return _create_type_loader_for_decorator(config_type, _selector,
required_resource_keys)
return _wrap
def output_selector_schema(config_cls, required_resource_keys=None):
'''
Deprecated in favor of dagster_type_materializer.
A decorator for a annotating a function that can take the selected properties
of a ``config_value`` and an instance of a custom type and materialize it.
Args:
config_cls (Selector):
'''
rename_warning('dagster_type_materializer', 'output_selector_schema', '0.10.0')
from dagster.config.field import resolve_to_config_type
config_type = resolve_to_config_type(config_cls)
check.param_invariant(config_type.kind == ConfigTypeKind.SELECTOR, 'config_cls')
def _wrap(func):
def _selector(context, config_value, runtime_value):
selector_key, selector_value = ensure_single_item(config_value)
return func(context, selector_key, selector_value, runtime_value)
return _create_output_materializer_for_decorator(
config_type, _selector, required_resource_keys
)
return _wrap
def input_hydration_config(config_schema=None, required_resource_keys=None, config_cls=None):
'''Deprecated in favor of dagster_type_loader'''
rename_warning('dagster_type_loader', 'input_hydration_config', '0.10.0')
config_schema = canonicalize_backcompat_args(
config_schema, 'config_schema', config_cls, 'config_cls', '0.10.0',
)
return dagster_type_loader(config_schema, required_resource_keys)
def output_materialization_config(config_schema=None, required_resource_keys=None, config_cls=None):
"""Deprecated in favor of dagster_type_materializer"""
rename_warning("dagster_type_materializer", "output_materialization_config", "0.10.0")
config_schema = canonicalize_backcompat_args(
config_schema, "config_schema", config_cls, "config_cls", "0.10.0",
)
return dagster_type_materializer(config_schema, required_resource_keys)
def all_runtime_types(self):
rename_warning(
new_name='all_dagster_types', old_name='all_runtime_types', breaking_version='0.8.0'
)
for tt in self.all_input_output_types():
yield tt
for solid_def in self._solid_defs:
for ttype in solid_def.all_dagster_types():
yield ttype
def __getattr__(name):
if name in _DEPRECATED:
value, breaking_version = _DEPRECATED[name]
stacklevel = 3 if sys.version_info >= (3, 7) else 4
rename_warning(value.__name__,
name,
breaking_version,
stacklevel=stacklevel)
return value
else:
raise AttributeError("module '{}' has no attribute '{}'".format(
__name__, name))
def test_no_additional_warn_text():
with mock.patch('warnings.warn') as warn_mock:
rename_warning('a_new_name', 'an_old_name', '0.3.0')
warn_mock.assert_called_once_with(
'"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" instead.',
stacklevel=3,
)
with mock.patch('warnings.warn') as warn_mock:
rename_warning('a_new_name', 'an_old_name', '0.3.0',
'Additional compelling text.')
warn_mock.assert_called_once_with(
('"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" '
'instead. Additional compelling text.'),
stacklevel=3,
)
def test_no_additional_warn_text():
with pytest.warns(
UserWarning,
match=re.escape(
'"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" instead.'
),
):
rename_warning('a_new_name', 'an_old_name', '0.3.0')
with pytest.warns(
UserWarning,
match=re.escape(
'"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" '
'instead. Additional compelling text.'),
):
rename_warning('a_new_name', 'an_old_name', '0.3.0',
'Additional compelling text.')
def test_no_additional_warn_text():
with pytest.warns(
DeprecationWarning,
match=re.escape(
'"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" instead.'
),
):
rename_warning("a_new_name", "an_old_name", "0.3.0")
with pytest.warns(
DeprecationWarning,
match=re.escape(
'"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" '
"instead. Additional compelling text."),
):
rename_warning("a_new_name", "an_old_name", "0.3.0",
"Additional compelling text.")
def __init__(self):
rename_warning('DagsterTypeMaterializer', 'OutputMaterializationConfig', '0.10.0')
super(OutputMaterializationConfig, self).__init__()
def __init__(self):
rename_warning('DagsterTypeLoader', 'InputHydrationConfig', '0.10.0')
super(InputHydrationConfig, self).__init__()
def runtime_type(self):
rename_warning(new_name='dagster_type',
old_name='runtime_type',
breaking_version='0.8.0')
return self._dagster_type
def __init__(self):
rename_warning("DagsterTypeMaterializer",
"OutputMaterializationConfig", "0.10.0")
super(OutputMaterializationConfig, self).__init__()
def __init__(self):
rename_warning("DagsterTypeLoader", "InputHydrationConfig", "0.10.0")
super(InputHydrationConfig, self).__init__()
def get_environment_dict(self, context):
rename_warning('get_run_config', 'get_environment_dict', '0.9.0')
return self.get_run_config(context)
def runtime_type_named(self, name):
rename_warning(new_name='dagster_type_named',
old_name='runtime_type_named',
breaking_version='0.8.0')
check.str_param(name, 'name')
return self._dagster_type_dict[name]
def config_field(self):
rename_warning("config_schema", "config_field", "0.9.0")
return self.config_schema
def config_field(self):
rename_warning('config_schema', 'config_field', '0.9.0')
return self.config_schema
def define_dagstermill_solid(
name: str,
notebook_path: str,
input_defs: Optional[Sequence[InputDefinition]] = None,
output_defs: Optional[Sequence[OutputDefinition]] = None,
config_schema: Optional[Union[Any, Dict[str, Any]]] = None,
required_resource_keys: Optional[Set[str]] = None,
output_notebook: Optional[str] = None,
output_notebook_name: Optional[str] = None,
asset_key_prefix: Optional[Union[List[str], str]] = None,
description: Optional[str] = None,
tags: Optional[Dict[str, Any]] = None,
):
"""Wrap a Jupyter notebook in a solid.
Arguments:
name (str): The name of the solid.
notebook_path (str): Path to the backing notebook.
input_defs (Optional[List[InputDefinition]]): The solid's inputs.
output_defs (Optional[List[OutputDefinition]]): The solid's outputs. Your notebook should
call :py:func:`~dagstermill.yield_result` to yield each of these outputs.
required_resource_keys (Optional[Set[str]]): The string names of any required resources.
output_notebook (Optional[str]): If set, will be used as the name of an injected output of
type :py:class:`~dagster.FileHandle` that will point to the executed notebook (in
addition to the :py:class:`~dagster.AssetMaterialization` that is always created). This
respects the :py:class:`~dagster.core.storage.file_manager.FileManager` configured on
the pipeline resources via the "file_manager" resource key, so, e.g.,
if :py:class:`~dagster_aws.s3.s3_file_manager` is configured, the output will be a :
py:class:`~dagster_aws.s3.S3FileHandle`.
output_notebook_name: (Optional[str]): If set, will be used as the name of an injected output
of type of :py:class:`~dagster.BufferedIOBase` that is the file object of the executed
notebook (in addition to the :py:class:`~dagster.AssetMaterialization` that is always
created). It allows the downstream solids to access the executed notebook via a file
object.
asset_key_prefix (Optional[Union[List[str], str]]): If set, will be used to prefix the
asset keys for materialized notebooks.
description (Optional[str]): If set, description used for solid.
tags (Optional[Dict[str, str]]): If set, additional tags used to annotate solid.
Dagster uses the tag keys `notebook_path` and `kind`, which cannot be
overwritten by the user.
Returns:
:py:class:`~dagster.SolidDefinition`
"""
check.str_param(name, "name")
check.str_param(notebook_path, "notebook_path")
input_defs = check.opt_list_param(input_defs,
"input_defs",
of_type=InputDefinition)
output_defs = check.opt_list_param(output_defs,
"output_defs",
of_type=OutputDefinition)
required_resource_keys = check.opt_set_param(required_resource_keys,
"required_resource_keys",
of_type=str)
extra_output_defs = []
if output_notebook_name is not None:
required_resource_keys.add("output_notebook_io_manager")
extra_output_defs.append(
OutputDefinition(name=output_notebook_name,
io_manager_key="output_notebook_io_manager"))
# backcompact
if output_notebook is not None:
rename_warning(new_name="output_notebook_name",
old_name="output_notebook",
breaking_version="0.14.0")
required_resource_keys.add("file_manager")
extra_output_defs.append(
OutputDefinition(dagster_type=FileHandle, name=output_notebook))
if isinstance(asset_key_prefix, str):
asset_key_prefix = [asset_key_prefix]
asset_key_prefix = check.opt_list_param(asset_key_prefix,
"asset_key_prefix",
of_type=str)
default_description = f"This solid is backed by the notebook at {notebook_path}"
description = check.opt_str_param(description,
"description",
default=default_description)
user_tags = validate_tags(tags)
if tags is not None:
check.invariant(
"notebook_path" not in tags,
"user-defined solid tags contains the `notebook_path` key, but the `notebook_path` key is reserved for use by Dagster",
)
check.invariant(
"kind" not in tags,
"user-defined solid tags contains the `kind` key, but the `kind` key is reserved for use by Dagster",
)
default_tags = {"notebook_path": notebook_path, "kind": "ipynb"}
return SolidDefinition(
name=name,
input_defs=input_defs,
compute_fn=_dm_compute(
"define_dagstermill_solid",
name,
notebook_path,
output_notebook_name,
asset_key_prefix=asset_key_prefix,
output_notebook=output_notebook, # backcompact
),
output_defs=output_defs + extra_output_defs,
config_schema=config_schema,
required_resource_keys=required_resource_keys,
description=description,
tags={
**user_tags,
**default_tags
},
)
def output_materialization_config(self):
rename_warning("materializer", "output_materialization_config",
"0.10.0")
return self.materializer
def input_hydration_config(self):
rename_warning("loader", "input_hydration_config", "0.10.0")
return self.loader
def output_materialization_schema_key(self):
rename_warning('materializer_schema_key', 'output_materialization_schema_key', '0.10.0')
return self.materializer_schema_key
def environment_dict_for_partition(self, partition):
rename_warning('run_config_for_partition', 'environment_dict_for_partition', '0.9.0')
return self.run_config_for_partition(partition)
def input_hydration_schema_key(self):
rename_warning('loader_schema_key', 'input_hydration_schema_key', '0.10.0')
return self.loader_schema_key
def environment_dict(self):
# Backcompat
rename_warning('run_config', 'environment_dict', '0.9.0')
return self.run_config
def input_hydration_schema_key(self):
rename_warning("loader_schema_key", "input_hydration_schema_key",
"0.10.0")
return self.loader_schema_key
def has_runtime_type(self, name):
rename_warning(new_name='has_dagster_type',
old_name='has_runtime_type',
breaking_version='0.8.0')
check.str_param(name, 'name')
return name in self._dagster_type_dict
def output_materialization_schema_key(self):
rename_warning("materializer_schema_key",
"output_materialization_schema_key", "0.10.0")
return self.materializer_schema_key
def all_runtime_types(self):
rename_warning(new_name='all_dagster_types',
old_name='all_runtime_types',
breaking_version='0.8.0')
return self._dagster_type_dict.values()
def output_materialization_config(self):
rename_warning('materializer', 'output_materialization_config', '0.10.0')
return self.materializer
