def __init__(self, name=None, inputs=None, outputs=None, description=None, config_field=None): self.name = check.opt_str_param(name, 'name') self.input_defs = check.opt_list_param(inputs, 'inputs', InputDefinition) outputs = outputs or ([OutputDefinition()] if outputs is None else []) self.outputs = check.list_param(outputs, 'outputs', OutputDefinition) self.description = check.opt_str_param(description, 'description') self.config_field = check_opt_field_param(config_field, 'config_field')
def define_specific_context_config_cls(name, config_field, resources): check.str_param(name, 'name') check_opt_field_param(config_field, 'config_field') check.dict_param(resources, 'resources', key_type=str, value_type=ResourceDefinition) return SystemNamedDict( name, fields=remove_none_entries({ 'config': config_field, 'resources': Field( define_resource_dictionary_cls( '{name}.Resources'.format(name=name), resources)), }), )
def define_solid_config_cls(name, config_field, inputs_field, outputs_field): check.str_param(name, 'name') check_opt_field_param(config_field, 'config_field') check_opt_field_param(inputs_field, 'inputs_field') check_opt_field_param(outputs_field, 'outputs_field') return SystemNamedDict( name, remove_none_entries( {'config': config_field, 'inputs': inputs_field, 'outputs': outputs_field} ), )
def __init__( self, name, inputs, transform_fn, outputs, config_field=None, description=None, metadata=None, ): self.name = check_valid_name(name) self.input_defs = check.list_param(inputs, 'inputs', InputDefinition) self.transform_fn = check.callable_param(transform_fn, 'transform_fn') self.output_defs = check.list_param(outputs, 'outputs', OutputDefinition) self.description = check.opt_str_param(description, 'description') self.config_field = check_opt_field_param(config_field, 'config_field') self.metadata = check.opt_dict_param(metadata, 'metadata', key_type=str) self._input_dict = {inp.name: inp for inp in inputs} self._output_dict = {output.name: output for output in outputs}
def __init__(self, context_fn=None, config_field=None, resources=None, description=None): ''' Args: context_fn (callable): Signature of context_fn: (pipeline: PipelineDefintion, config_value: Any) => ExecutionContext Returns *or* yields an ExecutionContext. If it yields a context, the code after the yield executes after pipeline completion, just like a python context manager. Environment-specific resources should be placed in the "resources" argument to an execution context. This argument can be *anything* and it is made avaiable to every solid in the pipeline. A typical pattern is to have this resources object be a namedtuple, where each property is an object that manages a particular resource, e.g. aws, a local filesystem manager, etc. config_field (Field): Define the configuration for the context description (str): Description of the context definition. ''' if config_field is None and context_fn is None: config_field = _default_config_field() context_fn = _default_context_fn self.config_field = check_opt_field_param(config_field, 'config_field') self.context_fn = check.opt_callable_param( context_fn, 'context_fn', lambda *args, **kwargs: ExecutionContext()) self.resources = check.opt_dict_param(resources, 'resources', key_type=str, value_type=ResourceDefinition) self.description = description self.resources_type = namedtuple('Resources', list( resources.keys())) if resources else None
def define_solid_config_cls(name, config_field, inputs_field, outputs_field): check.str_param(name, 'name') check_opt_field_param(config_field, 'config_field') check_opt_field_param(inputs_field, 'inputs_field') check_opt_field_param(outputs_field, 'outputs_field') return NamedDict( name, remove_none_entries({ 'config': config_field, 'inputs': inputs_field, 'outputs': outputs_field }), type_attributes=ConfigTypeAttributes(is_system_config=True), )
def __init__(self, resource_fn, config_field=None, description=None): self.resource_fn = check.callable_param(resource_fn, 'resource_fn') self.config_field = check_opt_field_param(config_field, 'config_field') self.description = check.opt_str_param(description, 'description')