def __init__(self, name=None, inputs=None, outputs=None, description=None, config_field=None):
self.name = check.opt_str_param(name, 'name')
self.input_defs = check.opt_list_param(inputs, 'inputs', InputDefinition)
outputs = outputs or ([OutputDefinition()] if outputs is None else [])
self.outputs = check.list_param(outputs, 'outputs', OutputDefinition)
self.description = check.opt_str_param(description, 'description')
self.config_field = check_opt_field_param(config_field, 'config_field')
def define_specific_context_config_cls(name, config_field, resources):
check.str_param(name, 'name')
check_opt_field_param(config_field, 'config_field')
check.dict_param(resources,
'resources',
key_type=str,
value_type=ResourceDefinition)
return SystemNamedDict(
name,
fields=remove_none_entries({
'config':
config_field,
'resources':
Field(
define_resource_dictionary_cls(
'{name}.Resources'.format(name=name), resources)),
}),
)
def define_solid_config_cls(name, config_field, inputs_field, outputs_field):
check.str_param(name, 'name')
check_opt_field_param(config_field, 'config_field')
check_opt_field_param(inputs_field, 'inputs_field')
check_opt_field_param(outputs_field, 'outputs_field')
return SystemNamedDict(
name,
remove_none_entries(
{'config': config_field, 'inputs': inputs_field, 'outputs': outputs_field}
),
)
def __init__(
self,
name,
inputs,
transform_fn,
outputs,
config_field=None,
description=None,
metadata=None,
):
self.name = check_valid_name(name)
self.input_defs = check.list_param(inputs, 'inputs', InputDefinition)
self.transform_fn = check.callable_param(transform_fn, 'transform_fn')
self.output_defs = check.list_param(outputs, 'outputs', OutputDefinition)
self.description = check.opt_str_param(description, 'description')
self.config_field = check_opt_field_param(config_field, 'config_field')
self.metadata = check.opt_dict_param(metadata, 'metadata', key_type=str)
self._input_dict = {inp.name: inp for inp in inputs}
self._output_dict = {output.name: output for output in outputs}
def __init__(self,
context_fn=None,
config_field=None,
resources=None,
description=None):
'''
Args:
context_fn (callable):
Signature of context_fn:
(pipeline: PipelineDefintion, config_value: Any) => ExecutionContext
Returns *or* yields an ExecutionContext.
If it yields a context, the code after the yield executes after pipeline
completion, just like a python context manager.
Environment-specific resources should be placed in the "resources" argument
to an execution context. This argument can be *anything* and it is made
avaiable to every solid in the pipeline. A typical pattern is to have this
resources object be a namedtuple, where each property is an object that
manages a particular resource, e.g. aws, a local filesystem manager, etc.
config_field (Field):
Define the configuration for the context
description (str): Description of the context definition.
'''
if config_field is None and context_fn is None:
config_field = _default_config_field()
context_fn = _default_context_fn
self.config_field = check_opt_field_param(config_field, 'config_field')
self.context_fn = check.opt_callable_param(
context_fn, 'context_fn',
lambda *args, **kwargs: ExecutionContext())
self.resources = check.opt_dict_param(resources,
'resources',
key_type=str,
value_type=ResourceDefinition)
self.description = description
self.resources_type = namedtuple('Resources', list(
resources.keys())) if resources else None
def define_solid_config_cls(name, config_field, inputs_field, outputs_field):
check.str_param(name, 'name')
check_opt_field_param(config_field, 'config_field')
check_opt_field_param(inputs_field, 'inputs_field')
check_opt_field_param(outputs_field, 'outputs_field')
return NamedDict(
name,
remove_none_entries({
'config': config_field,
'inputs': inputs_field,
'outputs': outputs_field
}),
type_attributes=ConfigTypeAttributes(is_system_config=True),
)
def __init__(self, resource_fn, config_field=None, description=None):
self.resource_fn = check.callable_param(resource_fn, 'resource_fn')
self.config_field = check_opt_field_param(config_field, 'config_field')
self.description = check.opt_str_param(description, 'description')