def test_opt_two_dim_dict_parm():
assert check.opt_two_dim_dict_param({}, "foo") == {}
assert check.opt_two_dim_dict_param({"key": {}}, "foo")
assert check.opt_two_dim_dict_param({"key": {"key2": 2}}, "foo")
assert check.opt_two_dim_dict_param(None, "foo") == {}
with pytest.raises(CheckError):
assert check.opt_two_dim_dict_param("str", "foo")
def test_opt_two_dim_dict_parm():
assert check.opt_two_dim_dict_param({}, 'foo') == {}
assert check.opt_two_dim_dict_param({'key': {}}, 'foo')
assert check.opt_two_dim_dict_param({'key': {'key2': 2}}, 'foo')
assert check.opt_two_dim_dict_param(None, 'foo') == {}
with pytest.raises(CheckError):
assert check.opt_two_dim_dict_param('str', 'foo')
def with_input_marshalling(included_step_keys, marshalled_inputs=None):
'''
Create an execution subset that uses the marshalling infrastruture in order
to provide values to includes.
inputs dictionary is a Dict[str,Dict[str,Any]] mapping
step_key => input_name => marshalling_key
'''
check.list_param(included_step_keys, 'included_step_keys', of_type=str)
marshalled_inputs = check.opt_two_dim_dict_param(
marshalled_inputs, 'marshalled_inputs')
input_step_factory_fns = defaultdict(dict)
def _create_unmarshal_input_factory_fn(marshalling_key):
return lambda pipeline_context, step, step_input: create_unmarshal_input_step(
pipeline_context, step, step_input, marshalling_key)
for step_key, input_marshal_dict in marshalled_inputs.items():
for input_name, key in input_marshal_dict.items():
input_step_factory_fns[step_key][
input_name] = _create_unmarshal_input_factory_fn(key)
return ExecutionPlanSubsetInfo(included_step_keys,
input_step_factory_fns)
def __new__(cls, included_step_keys, input_step_factory_fns):
'''
This should not be called directly, but instead through the static methods
on this class.
included_step_keys: list of step keys to include in the subset
input_step_factory_fns: A 2D dictinoary step_key => input_name => callable
callable should have signature of
(StepBuilderState, ExecutionStep, StepInput): StepOutputHandle
Full type signature is
Dict[str,
Dict[str,
Callable[StepBuilderState, ExecutionStep, StepInput]: StepOutputHandle
]
]
'''
return super(ExecutionPlanSubsetInfo, cls).__new__(
cls,
set(
check.list_param(included_step_keys,
'included_step_keys',
of_type=str)),
check.opt_two_dim_dict_param(input_step_factory_fns,
'input_step_factory_fns',
key_type=str),
)
def test_opt_two_dim_dict_parm():
assert check.opt_two_dim_dict_param({}, 'foo') == {}
assert check.opt_two_dim_dict_param({'key': {}}, 'foo')
assert check.opt_two_dim_dict_param({'key': {'key2': 2}}, 'foo')
assert check.opt_two_dim_dict_param(None, 'foo') == {}
def __init__(self,
solids,
name=None,
description=None,
context_definitions=None,
dependencies=None):
'''
Args:
solids (List[SolidDefinition]): Solids in the pipeline
name (str): Name. This is optional, mostly for situations that require ephemeral
pipeline definitions for fast scaffolding or testing.
description (str): Description of the pipline.
context_definitions (Dict[str, PipelineContextDefinition]): See class description.
dependencies: (Dict[str, Dict[str, DependencyDefinition]]): See class description.
'''
self.name = check.opt_str_param(name, 'name', '<<unnamed>>')
self.description = check.opt_str_param(description, 'description')
check.list_param(solids, 'solids')
if context_definitions is None:
context_definitions = default_pipeline_context_definitions()
self.context_definitions = check.dict_param(
context_definitions,
'context_definitions',
key_type=str,
value_type=PipelineContextDefinition,
)
self.dependencies = check.opt_two_dim_dict_param(
dependencies,
'dependencies',
key_type=six.string_types + (SolidInstance, ),
value_type=DependencyDefinition,
)
dependency_structure, pipeline_solid_dict = create_execution_structure(
solids, self.dependencies)
self._solid_dict = pipeline_solid_dict
self.dependency_structure = dependency_structure
self.environment_cls = define_environment_cls(
EnvironmentClassCreationData(
self.name,
list(self._solid_dict.values()),
context_definitions,
dependency_structure,
))
self.environment_type = self.environment_cls.inst()
self.context_cls = define_context_cls(self)
self.context_type = self.context_cls.inst()
(
self._config_type_dict_by_name,
self._config_type_dict_by_key,
) = construct_config_type_dictionary(solids, self.context_definitions,
self.environment_type)
self._runtime_type_dict = construct_runtime_type_dictionary(solids)
