def __init__(
self,
name,
input_defs,
compute_fn,
output_defs,
config_field=None,
description=None,
metadata=None,
required_resource_keys=None,
step_metadata_fn=None,
):
self._compute_fn = check.callable_param(compute_fn, 'compute_fn')
self._config_field = check_user_facing_opt_field_param(
config_field,
'config_field',
'of a SolidDefinition or @solid named "{name}"'.format(name=name),
)
self._required_resource_keys = check.opt_set_param(
required_resource_keys, 'required_resource_keys', of_type=str)
self._step_metadata_fn = step_metadata_fn
super(SolidDefinition, self).__init__(
name=name,
input_defs=check.list_param(input_defs, 'input_defs',
InputDefinition),
output_defs=check.list_param(output_defs, 'output_defs',
OutputDefinition),
description=description,
metadata=metadata,
)
def __init__(
self,
name,
inputs,
compute_fn,
outputs,
config_field=None,
description=None,
metadata=None,
resources=None,
step_metadata_fn=None,
):
self.name = check_valid_name(name)
self.compute_fn = check.callable_param(compute_fn, 'compute_fn')
self.description = check.opt_str_param(description, 'description')
self.config_field = check_user_facing_opt_field_param(
config_field,
'config_field',
'of a SolidDefinition or @solid named "{name}"'.format(name=name),
)
self.metadata = check.opt_dict_param(metadata, 'metadata', key_type=str)
self.resources = check.opt_set_param(resources, 'resources', of_type=str)
self.step_metadata_fn = step_metadata_fn
input_dict = {inp.name: inp for inp in check.list_param(inputs, 'inputs', InputDefinition)}
output_dict = {
output.name: output for output in check.list_param(outputs, 'outputs', OutputDefinition)
}
super(SolidDefinition, self).__init__(name, input_dict, output_dict, description, metadata)
def define_dagstermill_solid(
name,
notebook_path,
input_defs=None,
output_defs=None,
config_field=None,
required_resource_keys=None,
):
'''Wrap a Jupyter notebook in a solid.
Arguments:
name (str): The name of the solid (passed to SolidDefinition)
notebook_path (str): Path to the backing notebook.
input_defs (Optional[list[InputDefinition]]): The solid's inputs (passed to SolidDefinition).
output_defs (Optional[list[OutputDefinition]]): The solid's outputs
(passed to SolidDefinition).
required_resource_keys (Optional[set[str]]): The string names of any required resources
(passed to SolidDefinition).
Returns:
SolidDefinition
'''
check.str_param(name, 'name')
check.str_param(notebook_path, 'notebook_path')
input_defs = check.opt_list_param(input_defs,
'input_defs',
of_type=InputDefinition)
output_defs = check.opt_list_param(output_defs,
'output_defs',
of_type=OutputDefinition)
required_resource_keys = check.opt_set_param(required_resource_keys,
'required_resource_keys',
of_type=str)
config_field = check_user_facing_opt_field_param(
config_field, 'config_field',
'of a dagstermill solid named "{name}"'.format(name=name))
return SolidDefinition(
name=name,
input_defs=input_defs,
compute_fn=_dm_solid_transform(name, notebook_path),
output_defs=output_defs,
config_field=config_field,
required_resource_keys=required_resource_keys,
description='This solid is backed by the notebook at {path}'.format(
path=notebook_path),
metadata={
'notebook_path': notebook_path,
'kind': 'ipynb'
},
)
def __init__(self,
name,
config_field=None,
executor_creation_fn=None,
required_resource_keys=None):
self._name = check.str_param(name, 'name')
self._config_field = check_user_facing_opt_field_param(
config_field,
'config_field',
'of an ExecutorDefinition named {name}'.format(name=self.name),
)
self._executor_creation_fn = check.opt_callable_param(
executor_creation_fn, 'executor_creation_fn')
self._required_resource_keys = check.opt_set_param(
required_resource_keys, 'required_resource_keys', of_type=str)
def __init__(
self,
name,
is_persistent,
config_field=None,
system_storage_creation_fn=None,
required_resource_keys=None,
):
self.name = check.str_param(name, 'name')
self.is_persistent = check.bool_param(is_persistent, 'is_persistent')
self.config_field = check_user_facing_opt_field_param(
config_field,
'config_field',
'of a SystemStorageDefinition named {name}'.format(name=name),
)
self.system_storage_creation_fn = check.opt_callable_param(
system_storage_creation_fn, 'system_storage_creation_fn')
self.required_resource_keys = check.opt_set_param(
required_resource_keys, 'required_resource_keys', of_type=str)
def __init__(
self,
name,
inputs,
transform_fn,
outputs,
config_field=None,
description=None,
metadata=None,
):
self.name = check_valid_name(name)
self.input_defs = check.list_param(inputs, 'inputs', InputDefinition)
self.transform_fn = check.callable_param(transform_fn, 'transform_fn')
self.output_defs = check.list_param(outputs, 'outputs', OutputDefinition)
self.description = check.opt_str_param(description, 'description')
self.config_field = check_user_facing_opt_field_param(
config_field,
'config_field',
'of a SolidDefinition or @solid named "{name}"'.format(name=name),
)
self.metadata = check.opt_dict_param(metadata, 'metadata', key_type=str)
self._input_dict = {inp.name: inp for inp in inputs}
self._output_dict = {output.name: output for output in outputs}
def __init__(self,
context_fn=None,
config_field=None,
resources=None,
description=None):
'''
'''
if config_field is None and context_fn is None:
config_field = _default_config_field()
context_fn = _default_context_fn
self.config_field = check_user_facing_opt_field_param(
config_field, 'config_field', 'of a PipelineContextDefinition')
self.context_fn = check.opt_callable_param(
context_fn, 'context_fn',
lambda *args, **kwargs: ExecutionContext())
self.resources = check.opt_dict_param(resources,
'resources',
key_type=str,
value_type=ResourceDefinition)
self.description = description
self.resources_type = namedtuple('Resources', list(
resources.keys())) if resources else None
def __init__(self, logger_fn, config_field=None, description=None):
self._logger_fn = check.callable_param(logger_fn, 'logger_fn')
self._config_field = check_user_facing_opt_field_param(
config_field, 'config_field', 'of a LoggerDefinition or @logger')
self._description = check.opt_str_param(description, 'description')
def __init__(self, resource_fn, config_field=None, description=None):
self.resource_fn = check.callable_param(resource_fn, 'resource_fn')
self.config_field = check_user_facing_opt_field_param(
config_field, 'config_field',
'of a ResourceDefinition or @resource')
self.description = check.opt_str_param(description, 'description')
