def __init__(self, name=None, is_persistent=True, config_schema=None, required_resource_keys=None): self.name = check.opt_str_param(name, 'name') self.is_persistent = check.bool_param(is_persistent, 'is_persistent') self.config_schema = check_user_facing_opt_config_param( config_schema, 'config_schema') self.required_resource_keys = check.opt_set_param( required_resource_keys, 'required_resource_keys', of_type=str)
def __init__(self, name, config=None, executor_creation_fn=None, required_resource_keys=None): self._name = check.str_param(name, 'name') self._config_field = check_user_facing_opt_config_param( config, 'config') self._executor_creation_fn = check.opt_callable_param( executor_creation_fn, 'executor_creation_fn') self._required_resource_keys = check.opt_set_param( required_resource_keys, 'required_resource_keys', of_type=str)
def __init__( self, resource_fn=None, config_schema=None, description=None, config=None, process_config_fn=None, ): self._resource_fn = check.opt_callable_param(resource_fn, 'resource_fn') self._config_schema = canonicalize_backcompat_args( check_user_facing_opt_config_param(config_schema, 'config_schema'), 'config_schema', check_user_facing_opt_config_param(config, 'config'), 'config', '0.9.0', ) self._description = check.opt_str_param(description, 'description') self._process_config_fn = check.opt_callable_param( process_config_fn, 'process_config_fn') or (lambda conf: process_config( {'config': self._config_schema or {}}, conf))
def __init__( self, name, config_schema=None, executor_creation_fn=None, required_resource_keys=None, config=None, ): self._name = check.str_param(name, 'name') self._config_schema = canonicalize_backcompat_args( check_user_facing_opt_config_param(config_schema, 'config_schema'), 'config_schema', check_user_facing_opt_config_param(config, 'config'), 'config', '0.9.0', ) self._executor_creation_fn = check.opt_callable_param( executor_creation_fn, 'executor_creation_fn') self._required_resource_keys = frozenset( check.opt_set_param(required_resource_keys, 'required_resource_keys', of_type=str))
def __init__( self, logger_fn, config_schema=None, description=None, _configured_config_mapping_fn=None, ): self._logger_fn = check.callable_param(logger_fn, "logger_fn") self._config_schema = check_user_facing_opt_config_param( config_schema, "config_schema") self._description = check.opt_str_param(description, "description") self.__configured_config_mapping_fn = check.opt_callable_param( _configured_config_mapping_fn, "config_mapping_fn")
def define_dagstermill_solid( name, notebook_path, input_defs=None, output_defs=None, config_schema=None, required_resource_keys=None, output_notebook=None, ): """Wrap a Jupyter notebook in a solid. Arguments: name (str): The name of the solid. notebook_path (str): Path to the backing notebook. input_defs (Optional[List[InputDefinition]]): The solid's inputs. output_defs (Optional[List[OutputDefinition]]): The solid's outputs. Your notebook should call :py:func:`~dagstermill.yield_result` to yield each of these outputs. required_resource_keys (Optional[Set[str]]): The string names of any required resources. output_notebook (Optional[str]): If set, will be used as the name of an injected output of type :py:class:`~dagster.FileHandle` that will point to the executed notebook (in addition to the :py:class:`~dagster.AssetMaterialization` that is always created). This respects the :py:class:`~dagster.core.storage.file_manager.FileManager` configured on the pipeline system storage, so, e.g., if :py:class:`~dagster_aws.s3.s3_system_storage` is configured, the output will be a :py:class:`~dagster_aws.s3.S3FileHandle`. Returns: :py:class:`~dagster.SolidDefinition` """ check.str_param(name, "name") check.str_param(notebook_path, "notebook_path") input_defs = check.opt_list_param(input_defs, "input_defs", of_type=InputDefinition) output_defs = check.opt_list_param(output_defs, "output_defs", of_type=OutputDefinition) required_resource_keys = check.opt_set_param( required_resource_keys, "required_resource_keys", of_type=str ) return SolidDefinition( name=name, input_defs=input_defs, compute_fn=_dm_solid_compute(name, notebook_path, output_notebook), output_defs=output_defs + ( [OutputDefinition(dagster_type=FileHandle, name=output_notebook)] if output_notebook else [] ), config_schema=check_user_facing_opt_config_param(config_schema, "config_schema"), required_resource_keys=required_resource_keys, description="This solid is backed by the notebook at {path}".format(path=notebook_path), tags={"notebook_path": notebook_path, "kind": "ipynb"}, )
def configured(configurable, config_schema=None, **kwargs): """ A decorator that makes it easy to create a function-configured version of an object. The following definition types can be configured using this function: * :py:class:`CompositeSolidDefinition` * :py:class:`ExecutorDefinition` * :py:class:`IntermediateStorageDefinition` * :py:class:`LoggerDefinition` * :py:class:`ResourceDefinition` * :py:class:`SolidDefinition` * :py:class:`SystemStorageDefinition` If the config that will be supplied to the object is constant, you may alternatively invoke this and call the result with a dict of config values to be curried. Examples of both strategies below. Args: configurable (IConfigMappable): An object that can be configured. config_schema (ConfigSchema): The config schema that the inputs to the decorated function must satisfy. **kwargs: Arbitrary keyword arguments that will be passed to the initializer of the returned object. Returns: (Callable[[Union[Any, Callable[[Any], Any]]], IConfigMappable]) **Examples:** .. code-block:: python dev_s3 = configured(s3_resource, name="dev_s3")({'bucket': 'dev'}) @configured(s3_resource): def dev_s3(_): return {'bucket': 'dev'} @configured(s3_resource, {'bucket_prefix', str}): def dev_s3(config): return {'bucket': config['bucket_prefix'] + 'dev'} """ _check_configurable_param(configurable) config_schema = check_user_facing_opt_config_param(config_schema, "config_schema") def _configured(config_or_config_fn): return configurable.configured( config_schema=config_schema, config_or_config_fn=config_or_config_fn, **kwargs ) return _configured
def __init__( self, resource_fn=None, config_schema=None, description=None, config=None, process_config_fn=None, ): EXPECTED_POSITIONALS = ['*'] fn_positionals, _ = split_function_parameters(resource_fn, EXPECTED_POSITIONALS) missing_positional = validate_decorated_fn_positionals( fn_positionals, EXPECTED_POSITIONALS) if missing_positional: raise DagsterInvalidDefinitionError( "@resource '{resource_name}' decorated function does not have required " "positional parameter '{missing_param}'. Resource functions should only have keyword " "arguments that match input names and a first positional parameter." .format(resource_name=resource_fn.__name__, missing_param=missing_positional)) self._resource_fn = check.opt_callable_param(resource_fn, 'resource_fn') self._config_schema = canonicalize_backcompat_args( check_user_facing_opt_config_param(config_schema, 'config_schema'), 'config_schema', check_user_facing_opt_config_param(config, 'config'), 'config', '0.9.0', ) self._description = check.opt_str_param(description, 'description') self._process_config_fn = check.opt_callable_param( process_config_fn, 'process_config_fn') or (lambda conf: process_config( {'config': self._config_schema or {}}, conf))
def __init__( self, name, input_defs, compute_fn, output_defs, config_schema=None, description=None, tags=None, required_resource_keys=None, positional_inputs=None, config=None, ): self._compute_fn = check.callable_param(compute_fn, 'compute_fn') self._config_schema = canonicalize_backcompat_args( check_user_facing_opt_config_param(config_schema, 'config_schema'), 'config_schema', check_user_facing_opt_config_param(config, 'config'), 'config', '0.9.0', ) self._required_resource_keys = frozenset( check.opt_set_param(required_resource_keys, 'required_resource_keys', of_type=str)) super(SolidDefinition, self).__init__( name=name, input_defs=check.list_param(input_defs, 'input_defs', InputDefinition), output_defs=check.list_param(output_defs, 'output_defs', OutputDefinition), description=description, tags=check.opt_dict_param(tags, 'tags', key_type=str), positional_inputs=positional_inputs, )
def define_dagstermill_solid( name, notebook_path, input_defs=None, output_defs=None, config=None, required_resource_keys=None, ): '''Wrap a Jupyter notebook in a solid. Arguments: name (str): The name of the solid. notebook_path (str): Path to the backing notebook. input_defs (Optional[list[:class:`dagster.InputDefinition`]]): The solid's inputs. output_defs (Optional[list[:class:`dagster.OutputDefinition`]]): The solid's outputs. required_resource_keys (Optional[set[str]]): The string names of any required resources. Returns: :class:`dagster.SolidDefinition` ''' check.str_param(name, 'name') check.str_param(notebook_path, 'notebook_path') input_defs = check.opt_list_param(input_defs, 'input_defs', of_type=InputDefinition) output_defs = check.opt_list_param(output_defs, 'output_defs', of_type=OutputDefinition) required_resource_keys = check.opt_set_param(required_resource_keys, 'required_resource_keys', of_type=str) return SolidDefinition( name=name, input_defs=input_defs, compute_fn=_dm_solid_compute(name, notebook_path), output_defs=output_defs, config=check_user_facing_opt_config_param(config, 'config'), required_resource_keys=required_resource_keys, description='This solid is backed by the notebook at {path}'.format( path=notebook_path), tags={ 'notebook_path': notebook_path, 'kind': 'ipynb' }, )
def __init__( self, name, config_schema=None, executor_creation_fn=None, required_resource_keys=None, _configured_config_mapping_fn=None, ): self._name = check.str_param(name, "name") self._config_schema = check_user_facing_opt_config_param(config_schema, "config_schema") self._executor_creation_fn = check.opt_callable_param( executor_creation_fn, "executor_creation_fn" ) self._required_resource_keys = frozenset( check.opt_set_param(required_resource_keys, "required_resource_keys", of_type=str) ) self.__configured_config_mapping_fn = check.opt_callable_param( _configured_config_mapping_fn, "config_mapping_fn" )
def __new__( cls, name, is_persistent, required_resource_keys, config_schema=None, system_storage_creation_fn=None, ): return super(SystemStorageDefinition, cls).__new__( cls, name=check.str_param(name, 'name'), is_persistent=check.bool_param(is_persistent, 'is_persistent'), config_schema=check_user_facing_opt_config_param(config_schema, 'config_schema'), system_storage_creation_fn=check.opt_callable_param( system_storage_creation_fn, 'system_storage_creation_fn' ), required_resource_keys=frozenset( check.set_param(required_resource_keys, 'required_resource_keys', of_type=str) ), )
def __init__( self, name, is_persistent, required_resource_keys, config_schema=None, system_storage_creation_fn=None, _configured_config_mapping_fn=None, ): self._name = check_valid_name(name) self._is_persistent = check.bool_param(is_persistent, "is_persistent") self._config_schema = check_user_facing_opt_config_param( config_schema, "config_schema") self._system_storage_creation_fn = check.opt_callable_param( system_storage_creation_fn, "system_storage_creation_fn") self._required_resource_keys = frozenset( check.set_param(required_resource_keys, "required_resource_keys", of_type=str)) self.__configured_config_mapping_fn = check.opt_callable_param( _configured_config_mapping_fn, "config_mapping_fn")
def __init__( self, name, input_defs, compute_fn, output_defs, config_schema=None, description=None, tags=None, required_resource_keys=None, positional_inputs=None, version=None, _configured_config_mapping_fn=None, ): self._compute_fn = check.callable_param(compute_fn, "compute_fn") self._config_schema = check_user_facing_opt_config_param( config_schema, "config_schema") self._required_resource_keys = frozenset( check.opt_set_param(required_resource_keys, "required_resource_keys", of_type=str)) self.__configured_config_mapping_fn = check.opt_callable_param( _configured_config_mapping_fn, "config_mapping_fn") self._version = check.opt_str_param(version, "version") if version: experimental_arg_warning("version", "SolidDefinition.__init__") super(SolidDefinition, self).__init__( name=name, input_defs=check.list_param(input_defs, "input_defs", InputDefinition), output_defs=check.list_param(output_defs, "output_defs", OutputDefinition), description=description, tags=check.opt_dict_param(tags, "tags", key_type=str), positional_inputs=positional_inputs, )
def __init__( self, name, is_persistent, required_resource_keys, config_schema=None, intermediate_storage_creation_fn=None, _configured_config_mapping_fn=None, ): self._name = check.str_param(name, "name") self._is_persistent = check.bool_param(is_persistent, "is_persistent") self._config_schema = check_user_facing_opt_config_param( config_schema, "config_schema") self._intermediate_storage_creation_fn = check.opt_callable_param( intermediate_storage_creation_fn, "intermediate_storage_creation_fn") self._required_resource_keys = frozenset( check.set_param( required_resource_keys if required_resource_keys else set(), "required_resource_keys", of_type=str, )) self.__configured_config_mapping_fn = check.opt_callable_param( _configured_config_mapping_fn, "config_mapping_fn")
def __init__(self, logger_fn, config=None, description=None): self._logger_fn = check.callable_param(logger_fn, 'logger_fn') self._config_field = check_user_facing_opt_config_param( config, 'config') self._description = check.opt_str_param(description, 'description')
def __init__(self, resource_fn, config=None, description=None): self._resource_fn = check.callable_param(resource_fn, 'resource_fn') self._config_field = check_user_facing_opt_config_param(config, 'config',) self._description = check.opt_str_param(description, 'description')
def __new__(cls, config_fn, config_schema=None): return super(ConfigMapping, cls).__new__( cls, config_fn=check.callable_param(config_fn, 'config_fn'), config_schema=check_user_facing_opt_config_param(config_schema, 'config_schema'), )
def __init__(self, config_schema=None, description=None): self.config_schema = check_user_facing_opt_config_param( config_schema, 'config_schema') self.description = check.opt_str_param(description, 'description')
def __init__(self, config_schema=None, description=None, version=None): self.config_schema = check_user_facing_opt_config_param(config_schema, "config_schema") self.description = check.opt_str_param(description, "description") self.version = check.opt_str_param(version, "version")