def _create_handle_dict(solid_dict, dep_dict):
from .composition import MappedInputPlaceholder
check.dict_param(solid_dict, "solid_dict", key_type=str, value_type=Solid)
check.two_dim_dict_param(dep_dict, "dep_dict", value_type=IDependencyDefinition)
handle_dict = InputToOutputHandleDict()
for solid_name, input_dict in dep_dict.items():
from_solid = solid_dict[solid_name]
for input_name, dep_def in input_dict.items():
if dep_def.is_multi():
handles = []
for inner_dep in dep_def.get_dependencies_and_mappings():
if isinstance(inner_dep, DependencyDefinition):
handles.append(solid_dict[inner_dep.solid].output_handle(inner_dep.output))
elif inner_dep is MappedInputPlaceholder:
handles.append(inner_dep)
else:
check.failed(
"Unexpected MultiDependencyDefinition dependencies type {}".format(
inner_dep
)
)
handle_dict[from_solid.input_handle(input_name)] = handles
else:
handle_dict[from_solid.input_handle(input_name)] = solid_dict[
dep_def.solid
].output_handle(dep_def.output)
return handle_dict
def _create_handle_dict(
solid_dict: Dict[str, Solid],
dep_dict: Dict[str, Dict[str, IDependencyDefinition]],
) -> InputToOutputHandleDict:
from .composition import MappedInputPlaceholder
check.dict_param(solid_dict, "solid_dict", key_type=str, value_type=Solid)
check.two_dim_dict_param(dep_dict,
"dep_dict",
value_type=IDependencyDefinition)
handle_dict: InputToOutputHandleDict = {}
for solid_name, input_dict in dep_dict.items():
from_solid = solid_dict[solid_name]
for input_name, dep_def in input_dict.items():
if isinstance(dep_def, MultiDependencyDefinition):
handles: List[Union[SolidOutputHandle,
Type[MappedInputPlaceholder]]] = []
for inner_dep in dep_def.get_dependencies_and_mappings():
if isinstance(inner_dep, DependencyDefinition):
handles.append(
solid_dict[inner_dep.solid].output_handle(
inner_dep.output))
elif inner_dep is MappedInputPlaceholder:
handles.append(inner_dep)
else:
check.failed(
"Unexpected MultiDependencyDefinition dependencies type {}"
.format(inner_dep))
handle_dict[from_solid.input_handle(input_name)] = (
DependencyType.FAN_IN, handles)
elif isinstance(dep_def, DependencyDefinition):
handle_dict[from_solid.input_handle(input_name)] = (
DependencyType.DIRECT,
solid_dict[dep_def.solid].output_handle(dep_def.output),
)
elif isinstance(dep_def, DynamicCollectDependencyDefinition):
handle_dict[from_solid.input_handle(input_name)] = (
DependencyType.DYNAMIC_COLLECT,
solid_dict[dep_def.solid_name].output_handle(
dep_def.output_name),
)
else:
check.failed(f"Unknown dependency type {dep_def}")
return handle_dict
def _create_handle_dict(solid_dict, dep_dict):
check.dict_param(solid_dict, 'solid_dict', key_type=str, value_type=Solid)
check.two_dim_dict_param(dep_dict,
'dep_dict',
value_type=DependencyDefinition)
handle_dict = InputToOutputHandleDict()
for solid_name, input_dict in dep_dict.items():
for input_name, dep_def in input_dict.items():
from_solid = solid_dict[solid_name]
to_solid = solid_dict[dep_def.solid]
handle_dict[from_solid.input_handle(
input_name)] = to_solid.output_handle(dep_def.output)
return handle_dict
def with_input_values(included_step_keys, inputs):
'''
Create an execution subset with hardcoded values as inputs. This will create
execution steps with the key "{step_key}.input.{input_name}.value" that simply
emit the value
inputs dictionary is a Dict[str,Dict[str,Any]] mapping
step_key => input_name => value
Example:
create_execution_plan(
define_two_int_pipeline(),
subset_info=ExecutionPlanSubsetInfo.with_input_values(
['add_one.transform'], {'add_one.transform': {'num': 2}}
),
)
'''
check.list_param(included_step_keys, 'included_step_keys', of_type=str)
check.two_dim_dict_param(inputs, 'inputs', key_type=str)
def _create_injected_value_factory_fn(input_value):
return lambda pipeline_context, step, step_input: create_value_thunk_step(
pipeline_context=pipeline_context,
solid=step.solid,
runtime_type=step_input.runtime_type,
step_key='{step_key}.input.{input_name}.value'.format(
step_key=step.key, input_name=step_input.name),
value=input_value,
)
input_step_factory_fns = defaultdict(dict)
for step_key, input_dict in inputs.items():
for input_name, input_value in input_dict.items():
input_step_factory_fns[step_key][
input_name] = _create_injected_value_factory_fn(
input_value)
return ExecutionPlanSubsetInfo(included_step_keys,
input_step_factory_fns)
def _create_handle_dict(solid_dict, dep_dict):
check.dict_param(solid_dict, 'solid_dict', key_type=str, value_type=Solid)
check.two_dim_dict_param(dep_dict, 'dep_dict', value_type=IDependencyDefinition)
handle_dict = InputToOutputHandleDict()
for solid_name, input_dict in dep_dict.items():
from_solid = solid_dict[solid_name]
for input_name, dep_def in input_dict.items():
if dep_def.is_multi():
handle_dict[from_solid.input_handle(input_name)] = [
solid_dict[dep.solid].output_handle(dep.output)
for dep in dep_def.get_definitions()
]
else:
handle_dict[from_solid.input_handle(input_name)] = solid_dict[
dep_def.solid
].output_handle(dep_def.output)
return handle_dict
def test_two_dim_dict():
assert check.two_dim_dict_param({}, "foo") == {}
assert check.two_dim_dict_param({"key": {}}, "foo")
assert check.two_dim_dict_param({"key": {"key2": 2}}, "foo")
# make sure default dict passes
default_dict = defaultdict(dict)
default_dict["key"]["key2"] = 2
assert check.two_dim_dict_param(default_dict, "foo")
with raises_with_message(
CheckError,
"""Param "foo" is not a dict. Got None which is type <class 'NoneType'>."""
if is_python_three() else
"""Param "foo" is not a dict. Got None which is type <type 'NoneType'>.""",
):
check.two_dim_dict_param(None, "foo")
with raises_with_message(
CheckError,
"Value in dictionary mismatches expected type for key int_value. Expected value "
"of type <class 'dict'>. Got value 2 of type <class 'int'>."
if is_python_three() else
"Value in dictionary mismatches expected type for key int_value. Expected value "
"of type <type 'dict'>. Got value 2 of type <type 'int'>.",
):
check.two_dim_dict_param({"int_value": 2}, "foo")
with raises_with_message(
CheckError,
"Value in dictionary mismatches expected type for key level_two_value_mismatch. "
"Expected value of type (<class 'str'>,). Got value 2 of type <class 'int'>."
if is_python_three() else
"Value in dictionary mismatches expected type for key level_two_value_mismatch. "
"Expected value of type (<type 'basestring'>,). Got value 2 of type <type 'int'>.",
):
check.two_dim_dict_param(
{"level_one_key": {
"level_two_value_mismatch": 2
}},
"foo",
value_type=str)
with raises_with_message(
CheckError,
"Key in dictionary mismatches type. Expected <class 'int'>. Got 'key'"
if is_python_three() else
"Key in dictionary mismatches type. Expected <type 'int'>. Got 'key'",
):
assert check.two_dim_dict_param({"key": {}}, "foo", key_type=int)
with raises_with_message(
CheckError,
"Key in dictionary mismatches type. Expected <class 'int'>. Got 'level_two_key'"
if is_python_three() else
"Key in dictionary mismatches type. Expected <type 'int'>. Got 'level_two_key'",
):
assert check.two_dim_dict_param({1: {
"level_two_key": "something"
}},
"foo",
key_type=int)
def test_two_dim_dict():
assert check.two_dim_dict_param({}, 'foo') == {}
assert check.two_dim_dict_param({'key': {}}, 'foo')
assert check.two_dim_dict_param({'key': {'key2': 2}}, 'foo')
# make sure default dict passes
default_dict = defaultdict(dict)
default_dict['key']['key2'] = 2
assert check.two_dim_dict_param(default_dict, 'foo')
with raises_with_message(
CheckError,
'''Param "foo" is not a dict. Got None which is type <class 'NoneType'>.'''
if is_python_three() else
'''Param "foo" is not a dict. Got None which is type <type 'NoneType'>.''',
):
check.two_dim_dict_param(None, 'foo')
with raises_with_message(
CheckError,
"Value in dictionary mismatches expected type for key int_value. Expected value "
"of type <class 'dict'>. Got value 2 of type <class 'int'>."
if is_python_three() else
"Value in dictionary mismatches expected type for key int_value. Expected value "
"of type <type 'dict'>. Got value 2 of type <type 'int'>.",
):
check.two_dim_dict_param({'int_value': 2}, 'foo')
with raises_with_message(
CheckError,
"Value in dictionary mismatches expected type for key level_two_value_mismatch. "
"Expected value of type (<class 'str'>,). Got value 2 of type <class 'int'>."
if is_python_three() else
"Value in dictionary mismatches expected type for key level_two_value_mismatch. "
"Expected value of type (<type 'basestring'>,). Got value 2 of type <type 'int'>.",
):
check.two_dim_dict_param(
{'level_one_key': {
'level_two_value_mismatch': 2
}},
'foo',
value_type=str)
with raises_with_message(
CheckError,
"Key in dictionary mismatches type. Expected <class 'int'>. Got 'key'"
if is_python_three() else
"Key in dictionary mismatches type. Expected <type 'int'>. Got 'key'",
):
assert check.two_dim_dict_param({'key': {}}, 'foo', key_type=int)
with raises_with_message(
CheckError,
"Key in dictionary mismatches type. Expected <class 'int'>. Got 'level_two_key'"
if is_python_three() else
"Key in dictionary mismatches type. Expected <type 'int'>. Got 'level_two_key'",
):
assert check.two_dim_dict_param({1: {
'level_two_key': 'something'
}},
'foo',
key_type=int)
