def __init__( self, name, input_defs, compute_fn, output_defs, config_field=None, description=None, metadata=None, required_resource_keys=None, step_metadata_fn=None, ): self._compute_fn = check.callable_param(compute_fn, 'compute_fn') self._config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of a SolidDefinition or @solid named "{name}"'.format(name=name), ) self._required_resource_keys = check.opt_set_param( required_resource_keys, 'required_resource_keys', of_type=str) self._step_metadata_fn = step_metadata_fn super(SolidDefinition, self).__init__( name=name, input_defs=check.list_param(input_defs, 'input_defs', InputDefinition), output_defs=check.list_param(output_defs, 'output_defs', OutputDefinition), description=description, metadata=metadata, )
def __init__( self, name, inputs, compute_fn, outputs, config_field=None, description=None, metadata=None, resources=None, step_metadata_fn=None, ): self.name = check_valid_name(name) self.compute_fn = check.callable_param(compute_fn, 'compute_fn') self.description = check.opt_str_param(description, 'description') self.config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of a SolidDefinition or @solid named "{name}"'.format(name=name), ) self.metadata = check.opt_dict_param(metadata, 'metadata', key_type=str) self.resources = check.opt_set_param(resources, 'resources', of_type=str) self.step_metadata_fn = step_metadata_fn input_dict = {inp.name: inp for inp in check.list_param(inputs, 'inputs', InputDefinition)} output_dict = { output.name: output for output in check.list_param(outputs, 'outputs', OutputDefinition) } super(SolidDefinition, self).__init__(name, input_dict, output_dict, description, metadata)
def define_dagstermill_solid( name, notebook_path, input_defs=None, output_defs=None, config_field=None, required_resource_keys=None, ): '''Wrap a Jupyter notebook in a solid. Arguments: name (str): The name of the solid (passed to SolidDefinition) notebook_path (str): Path to the backing notebook. input_defs (Optional[list[InputDefinition]]): The solid's inputs (passed to SolidDefinition). output_defs (Optional[list[OutputDefinition]]): The solid's outputs (passed to SolidDefinition). required_resource_keys (Optional[set[str]]): The string names of any required resources (passed to SolidDefinition). Returns: SolidDefinition ''' check.str_param(name, 'name') check.str_param(notebook_path, 'notebook_path') input_defs = check.opt_list_param(input_defs, 'input_defs', of_type=InputDefinition) output_defs = check.opt_list_param(output_defs, 'output_defs', of_type=OutputDefinition) required_resource_keys = check.opt_set_param(required_resource_keys, 'required_resource_keys', of_type=str) config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of a dagstermill solid named "{name}"'.format(name=name)) return SolidDefinition( name=name, input_defs=input_defs, compute_fn=_dm_solid_transform(name, notebook_path), output_defs=output_defs, config_field=config_field, required_resource_keys=required_resource_keys, description='This solid is backed by the notebook at {path}'.format( path=notebook_path), metadata={ 'notebook_path': notebook_path, 'kind': 'ipynb' }, )
def __init__(self, name, config_field=None, executor_creation_fn=None, required_resource_keys=None): self._name = check.str_param(name, 'name') self._config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of an ExecutorDefinition named {name}'.format(name=self.name), ) self._executor_creation_fn = check.opt_callable_param( executor_creation_fn, 'executor_creation_fn') self._required_resource_keys = check.opt_set_param( required_resource_keys, 'required_resource_keys', of_type=str)
def __init__( self, name, is_persistent, config_field=None, system_storage_creation_fn=None, required_resource_keys=None, ): self.name = check.str_param(name, 'name') self.is_persistent = check.bool_param(is_persistent, 'is_persistent') self.config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of a SystemStorageDefinition named {name}'.format(name=name), ) self.system_storage_creation_fn = check.opt_callable_param( system_storage_creation_fn, 'system_storage_creation_fn') self.required_resource_keys = check.opt_set_param( required_resource_keys, 'required_resource_keys', of_type=str)
def __init__( self, name, inputs, transform_fn, outputs, config_field=None, description=None, metadata=None, ): self.name = check_valid_name(name) self.input_defs = check.list_param(inputs, 'inputs', InputDefinition) self.transform_fn = check.callable_param(transform_fn, 'transform_fn') self.output_defs = check.list_param(outputs, 'outputs', OutputDefinition) self.description = check.opt_str_param(description, 'description') self.config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of a SolidDefinition or @solid named "{name}"'.format(name=name), ) self.metadata = check.opt_dict_param(metadata, 'metadata', key_type=str) self._input_dict = {inp.name: inp for inp in inputs} self._output_dict = {output.name: output for output in outputs}
def __init__(self, context_fn=None, config_field=None, resources=None, description=None): ''' ''' if config_field is None and context_fn is None: config_field = _default_config_field() context_fn = _default_context_fn self.config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of a PipelineContextDefinition') self.context_fn = check.opt_callable_param( context_fn, 'context_fn', lambda *args, **kwargs: ExecutionContext()) self.resources = check.opt_dict_param(resources, 'resources', key_type=str, value_type=ResourceDefinition) self.description = description self.resources_type = namedtuple('Resources', list( resources.keys())) if resources else None
def __init__(self, logger_fn, config_field=None, description=None): self._logger_fn = check.callable_param(logger_fn, 'logger_fn') self._config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of a LoggerDefinition or @logger') self._description = check.opt_str_param(description, 'description')
def __init__(self, resource_fn, config_field=None, description=None): self.resource_fn = check.callable_param(resource_fn, 'resource_fn') self.config_field = check_user_facing_opt_field_param( config_field, 'config_field', 'of a ResourceDefinition or @resource') self.description = check.opt_str_param(description, 'description')