def input_selector_schema(config_cls, required_resource_keys=None): """ Deprecated in favor of dagster_type_loader. A decorator for annotating a function that can take the selected properties from a ``config_value`` in to an instance of a custom type. Args: config_cls (Selector) """ rename_warning("dagster_type_loader", "input_selector_schema", "0.10.0") from dagster.config.field import resolve_to_config_type config_type = resolve_to_config_type(config_cls) check.param_invariant(config_type.kind == ConfigTypeKind.SELECTOR, "config_cls") def _wrap(func): def _selector(context, config_value): selector_key, selector_value = ensure_single_item(config_value) return func(context, selector_key, selector_value) return _create_type_loader_for_decorator(config_type, _selector, required_resource_keys) return _wrap
def output_selector_schema(config_cls, required_resource_keys=None): ''' Deprecated in favor of dagster_type_materializer. A decorator for a annotating a function that can take the selected properties of a ``config_value`` and an instance of a custom type and materialize it. Args: config_cls (Selector): ''' rename_warning('dagster_type_materializer', 'output_selector_schema', '0.10.0') from dagster.config.field import resolve_to_config_type config_type = resolve_to_config_type(config_cls) check.param_invariant(config_type.kind == ConfigTypeKind.SELECTOR, 'config_cls') def _wrap(func): def _selector(context, config_value, runtime_value): selector_key, selector_value = ensure_single_item(config_value) return func(context, selector_key, selector_value, runtime_value) return _create_output_materializer_for_decorator( config_type, _selector, required_resource_keys ) return _wrap
def input_hydration_config(config_schema=None, required_resource_keys=None, config_cls=None): '''Deprecated in favor of dagster_type_loader''' rename_warning('dagster_type_loader', 'input_hydration_config', '0.10.0') config_schema = canonicalize_backcompat_args( config_schema, 'config_schema', config_cls, 'config_cls', '0.10.0', ) return dagster_type_loader(config_schema, required_resource_keys)
def output_materialization_config(config_schema=None, required_resource_keys=None, config_cls=None): """Deprecated in favor of dagster_type_materializer""" rename_warning("dagster_type_materializer", "output_materialization_config", "0.10.0") config_schema = canonicalize_backcompat_args( config_schema, "config_schema", config_cls, "config_cls", "0.10.0", ) return dagster_type_materializer(config_schema, required_resource_keys)
def all_runtime_types(self): rename_warning( new_name='all_dagster_types', old_name='all_runtime_types', breaking_version='0.8.0' ) for tt in self.all_input_output_types(): yield tt for solid_def in self._solid_defs: for ttype in solid_def.all_dagster_types(): yield ttype
def __getattr__(name): if name in _DEPRECATED: value, breaking_version = _DEPRECATED[name] stacklevel = 3 if sys.version_info >= (3, 7) else 4 rename_warning(value.__name__, name, breaking_version, stacklevel=stacklevel) return value else: raise AttributeError("module '{}' has no attribute '{}'".format( __name__, name))
def test_no_additional_warn_text(): with mock.patch('warnings.warn') as warn_mock: rename_warning('a_new_name', 'an_old_name', '0.3.0') warn_mock.assert_called_once_with( '"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" instead.', stacklevel=3, ) with mock.patch('warnings.warn') as warn_mock: rename_warning('a_new_name', 'an_old_name', '0.3.0', 'Additional compelling text.') warn_mock.assert_called_once_with( ('"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" ' 'instead. Additional compelling text.'), stacklevel=3, )
def test_no_additional_warn_text(): with pytest.warns( UserWarning, match=re.escape( '"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" instead.' ), ): rename_warning('a_new_name', 'an_old_name', '0.3.0') with pytest.warns( UserWarning, match=re.escape( '"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" ' 'instead. Additional compelling text.'), ): rename_warning('a_new_name', 'an_old_name', '0.3.0', 'Additional compelling text.')
def test_no_additional_warn_text(): with pytest.warns( DeprecationWarning, match=re.escape( '"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" instead.' ), ): rename_warning("a_new_name", "an_old_name", "0.3.0") with pytest.warns( DeprecationWarning, match=re.escape( '"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" ' "instead. Additional compelling text."), ): rename_warning("a_new_name", "an_old_name", "0.3.0", "Additional compelling text.")
def __init__(self): rename_warning('DagsterTypeMaterializer', 'OutputMaterializationConfig', '0.10.0') super(OutputMaterializationConfig, self).__init__()
def __init__(self): rename_warning('DagsterTypeLoader', 'InputHydrationConfig', '0.10.0') super(InputHydrationConfig, self).__init__()
def runtime_type(self): rename_warning(new_name='dagster_type', old_name='runtime_type', breaking_version='0.8.0') return self._dagster_type
def __init__(self): rename_warning("DagsterTypeMaterializer", "OutputMaterializationConfig", "0.10.0") super(OutputMaterializationConfig, self).__init__()
def __init__(self): rename_warning("DagsterTypeLoader", "InputHydrationConfig", "0.10.0") super(InputHydrationConfig, self).__init__()
def get_environment_dict(self, context): rename_warning('get_run_config', 'get_environment_dict', '0.9.0') return self.get_run_config(context)
def runtime_type_named(self, name): rename_warning(new_name='dagster_type_named', old_name='runtime_type_named', breaking_version='0.8.0') check.str_param(name, 'name') return self._dagster_type_dict[name]
def config_field(self): rename_warning("config_schema", "config_field", "0.9.0") return self.config_schema
def config_field(self): rename_warning('config_schema', 'config_field', '0.9.0') return self.config_schema
def define_dagstermill_solid( name: str, notebook_path: str, input_defs: Optional[Sequence[InputDefinition]] = None, output_defs: Optional[Sequence[OutputDefinition]] = None, config_schema: Optional[Union[Any, Dict[str, Any]]] = None, required_resource_keys: Optional[Set[str]] = None, output_notebook: Optional[str] = None, output_notebook_name: Optional[str] = None, asset_key_prefix: Optional[Union[List[str], str]] = None, description: Optional[str] = None, tags: Optional[Dict[str, Any]] = None, ): """Wrap a Jupyter notebook in a solid. Arguments: name (str): The name of the solid. notebook_path (str): Path to the backing notebook. input_defs (Optional[List[InputDefinition]]): The solid's inputs. output_defs (Optional[List[OutputDefinition]]): The solid's outputs. Your notebook should call :py:func:`~dagstermill.yield_result` to yield each of these outputs. required_resource_keys (Optional[Set[str]]): The string names of any required resources. output_notebook (Optional[str]): If set, will be used as the name of an injected output of type :py:class:`~dagster.FileHandle` that will point to the executed notebook (in addition to the :py:class:`~dagster.AssetMaterialization` that is always created). This respects the :py:class:`~dagster.core.storage.file_manager.FileManager` configured on the pipeline resources via the "file_manager" resource key, so, e.g., if :py:class:`~dagster_aws.s3.s3_file_manager` is configured, the output will be a : py:class:`~dagster_aws.s3.S3FileHandle`. output_notebook_name: (Optional[str]): If set, will be used as the name of an injected output of type of :py:class:`~dagster.BufferedIOBase` that is the file object of the executed notebook (in addition to the :py:class:`~dagster.AssetMaterialization` that is always created). It allows the downstream solids to access the executed notebook via a file object. asset_key_prefix (Optional[Union[List[str], str]]): If set, will be used to prefix the asset keys for materialized notebooks. description (Optional[str]): If set, description used for solid. tags (Optional[Dict[str, str]]): If set, additional tags used to annotate solid. Dagster uses the tag keys `notebook_path` and `kind`, which cannot be overwritten by the user. Returns: :py:class:`~dagster.SolidDefinition` """ check.str_param(name, "name") check.str_param(notebook_path, "notebook_path") input_defs = check.opt_list_param(input_defs, "input_defs", of_type=InputDefinition) output_defs = check.opt_list_param(output_defs, "output_defs", of_type=OutputDefinition) required_resource_keys = check.opt_set_param(required_resource_keys, "required_resource_keys", of_type=str) extra_output_defs = [] if output_notebook_name is not None: required_resource_keys.add("output_notebook_io_manager") extra_output_defs.append( OutputDefinition(name=output_notebook_name, io_manager_key="output_notebook_io_manager")) # backcompact if output_notebook is not None: rename_warning(new_name="output_notebook_name", old_name="output_notebook", breaking_version="0.14.0") required_resource_keys.add("file_manager") extra_output_defs.append( OutputDefinition(dagster_type=FileHandle, name=output_notebook)) if isinstance(asset_key_prefix, str): asset_key_prefix = [asset_key_prefix] asset_key_prefix = check.opt_list_param(asset_key_prefix, "asset_key_prefix", of_type=str) default_description = f"This solid is backed by the notebook at {notebook_path}" description = check.opt_str_param(description, "description", default=default_description) user_tags = validate_tags(tags) if tags is not None: check.invariant( "notebook_path" not in tags, "user-defined solid tags contains the `notebook_path` key, but the `notebook_path` key is reserved for use by Dagster", ) check.invariant( "kind" not in tags, "user-defined solid tags contains the `kind` key, but the `kind` key is reserved for use by Dagster", ) default_tags = {"notebook_path": notebook_path, "kind": "ipynb"} return SolidDefinition( name=name, input_defs=input_defs, compute_fn=_dm_compute( "define_dagstermill_solid", name, notebook_path, output_notebook_name, asset_key_prefix=asset_key_prefix, output_notebook=output_notebook, # backcompact ), output_defs=output_defs + extra_output_defs, config_schema=config_schema, required_resource_keys=required_resource_keys, description=description, tags={ **user_tags, **default_tags }, )
def output_materialization_config(self): rename_warning("materializer", "output_materialization_config", "0.10.0") return self.materializer
def input_hydration_config(self): rename_warning("loader", "input_hydration_config", "0.10.0") return self.loader
def output_materialization_schema_key(self): rename_warning('materializer_schema_key', 'output_materialization_schema_key', '0.10.0') return self.materializer_schema_key
def environment_dict_for_partition(self, partition): rename_warning('run_config_for_partition', 'environment_dict_for_partition', '0.9.0') return self.run_config_for_partition(partition)
def input_hydration_schema_key(self): rename_warning('loader_schema_key', 'input_hydration_schema_key', '0.10.0') return self.loader_schema_key
def environment_dict(self): # Backcompat rename_warning('run_config', 'environment_dict', '0.9.0') return self.run_config
def input_hydration_schema_key(self): rename_warning("loader_schema_key", "input_hydration_schema_key", "0.10.0") return self.loader_schema_key
def has_runtime_type(self, name): rename_warning(new_name='has_dagster_type', old_name='has_runtime_type', breaking_version='0.8.0') check.str_param(name, 'name') return name in self._dagster_type_dict
def output_materialization_schema_key(self): rename_warning("materializer_schema_key", "output_materialization_schema_key", "0.10.0") return self.materializer_schema_key
def all_runtime_types(self): rename_warning(new_name='all_dagster_types', old_name='all_runtime_types', breaking_version='0.8.0') return self._dagster_type_dict.values()
def output_materialization_config(self): rename_warning('materializer', 'output_materialization_config', '0.10.0') return self.materializer