def _create_handle_dict(solid_dict, dep_dict): from .composition import MappedInputPlaceholder check.dict_param(solid_dict, "solid_dict", key_type=str, value_type=Solid) check.two_dim_dict_param(dep_dict, "dep_dict", value_type=IDependencyDefinition) handle_dict = InputToOutputHandleDict() for solid_name, input_dict in dep_dict.items(): from_solid = solid_dict[solid_name] for input_name, dep_def in input_dict.items(): if dep_def.is_multi(): handles = [] for inner_dep in dep_def.get_dependencies_and_mappings(): if isinstance(inner_dep, DependencyDefinition): handles.append(solid_dict[inner_dep.solid].output_handle(inner_dep.output)) elif inner_dep is MappedInputPlaceholder: handles.append(inner_dep) else: check.failed( "Unexpected MultiDependencyDefinition dependencies type {}".format( inner_dep ) ) handle_dict[from_solid.input_handle(input_name)] = handles else: handle_dict[from_solid.input_handle(input_name)] = solid_dict[ dep_def.solid ].output_handle(dep_def.output) return handle_dict
def _create_handle_dict( solid_dict: Dict[str, Solid], dep_dict: Dict[str, Dict[str, IDependencyDefinition]], ) -> InputToOutputHandleDict: from .composition import MappedInputPlaceholder check.dict_param(solid_dict, "solid_dict", key_type=str, value_type=Solid) check.two_dim_dict_param(dep_dict, "dep_dict", value_type=IDependencyDefinition) handle_dict: InputToOutputHandleDict = {} for solid_name, input_dict in dep_dict.items(): from_solid = solid_dict[solid_name] for input_name, dep_def in input_dict.items(): if isinstance(dep_def, MultiDependencyDefinition): handles: List[Union[SolidOutputHandle, Type[MappedInputPlaceholder]]] = [] for inner_dep in dep_def.get_dependencies_and_mappings(): if isinstance(inner_dep, DependencyDefinition): handles.append( solid_dict[inner_dep.solid].output_handle( inner_dep.output)) elif inner_dep is MappedInputPlaceholder: handles.append(inner_dep) else: check.failed( "Unexpected MultiDependencyDefinition dependencies type {}" .format(inner_dep)) handle_dict[from_solid.input_handle(input_name)] = ( DependencyType.FAN_IN, handles) elif isinstance(dep_def, DependencyDefinition): handle_dict[from_solid.input_handle(input_name)] = ( DependencyType.DIRECT, solid_dict[dep_def.solid].output_handle(dep_def.output), ) elif isinstance(dep_def, DynamicCollectDependencyDefinition): handle_dict[from_solid.input_handle(input_name)] = ( DependencyType.DYNAMIC_COLLECT, solid_dict[dep_def.solid_name].output_handle( dep_def.output_name), ) else: check.failed(f"Unknown dependency type {dep_def}") return handle_dict
def _create_handle_dict(solid_dict, dep_dict): check.dict_param(solid_dict, 'solid_dict', key_type=str, value_type=Solid) check.two_dim_dict_param(dep_dict, 'dep_dict', value_type=DependencyDefinition) handle_dict = InputToOutputHandleDict() for solid_name, input_dict in dep_dict.items(): for input_name, dep_def in input_dict.items(): from_solid = solid_dict[solid_name] to_solid = solid_dict[dep_def.solid] handle_dict[from_solid.input_handle( input_name)] = to_solid.output_handle(dep_def.output) return handle_dict
def with_input_values(included_step_keys, inputs): ''' Create an execution subset with hardcoded values as inputs. This will create execution steps with the key "{step_key}.input.{input_name}.value" that simply emit the value inputs dictionary is a Dict[str,Dict[str,Any]] mapping step_key => input_name => value Example: create_execution_plan( define_two_int_pipeline(), subset_info=ExecutionPlanSubsetInfo.with_input_values( ['add_one.transform'], {'add_one.transform': {'num': 2}} ), ) ''' check.list_param(included_step_keys, 'included_step_keys', of_type=str) check.two_dim_dict_param(inputs, 'inputs', key_type=str) def _create_injected_value_factory_fn(input_value): return lambda pipeline_context, step, step_input: create_value_thunk_step( pipeline_context=pipeline_context, solid=step.solid, runtime_type=step_input.runtime_type, step_key='{step_key}.input.{input_name}.value'.format( step_key=step.key, input_name=step_input.name), value=input_value, ) input_step_factory_fns = defaultdict(dict) for step_key, input_dict in inputs.items(): for input_name, input_value in input_dict.items(): input_step_factory_fns[step_key][ input_name] = _create_injected_value_factory_fn( input_value) return ExecutionPlanSubsetInfo(included_step_keys, input_step_factory_fns)
def _create_handle_dict(solid_dict, dep_dict): check.dict_param(solid_dict, 'solid_dict', key_type=str, value_type=Solid) check.two_dim_dict_param(dep_dict, 'dep_dict', value_type=IDependencyDefinition) handle_dict = InputToOutputHandleDict() for solid_name, input_dict in dep_dict.items(): from_solid = solid_dict[solid_name] for input_name, dep_def in input_dict.items(): if dep_def.is_multi(): handle_dict[from_solid.input_handle(input_name)] = [ solid_dict[dep.solid].output_handle(dep.output) for dep in dep_def.get_definitions() ] else: handle_dict[from_solid.input_handle(input_name)] = solid_dict[ dep_def.solid ].output_handle(dep_def.output) return handle_dict
def test_two_dim_dict(): assert check.two_dim_dict_param({}, "foo") == {} assert check.two_dim_dict_param({"key": {}}, "foo") assert check.two_dim_dict_param({"key": {"key2": 2}}, "foo") # make sure default dict passes default_dict = defaultdict(dict) default_dict["key"]["key2"] = 2 assert check.two_dim_dict_param(default_dict, "foo") with raises_with_message( CheckError, """Param "foo" is not a dict. Got None which is type <class 'NoneType'>.""" if is_python_three() else """Param "foo" is not a dict. Got None which is type <type 'NoneType'>.""", ): check.two_dim_dict_param(None, "foo") with raises_with_message( CheckError, "Value in dictionary mismatches expected type for key int_value. Expected value " "of type <class 'dict'>. Got value 2 of type <class 'int'>." if is_python_three() else "Value in dictionary mismatches expected type for key int_value. Expected value " "of type <type 'dict'>. Got value 2 of type <type 'int'>.", ): check.two_dim_dict_param({"int_value": 2}, "foo") with raises_with_message( CheckError, "Value in dictionary mismatches expected type for key level_two_value_mismatch. " "Expected value of type (<class 'str'>,). Got value 2 of type <class 'int'>." if is_python_three() else "Value in dictionary mismatches expected type for key level_two_value_mismatch. " "Expected value of type (<type 'basestring'>,). Got value 2 of type <type 'int'>.", ): check.two_dim_dict_param( {"level_one_key": { "level_two_value_mismatch": 2 }}, "foo", value_type=str) with raises_with_message( CheckError, "Key in dictionary mismatches type. Expected <class 'int'>. Got 'key'" if is_python_three() else "Key in dictionary mismatches type. Expected <type 'int'>. Got 'key'", ): assert check.two_dim_dict_param({"key": {}}, "foo", key_type=int) with raises_with_message( CheckError, "Key in dictionary mismatches type. Expected <class 'int'>. Got 'level_two_key'" if is_python_three() else "Key in dictionary mismatches type. Expected <type 'int'>. Got 'level_two_key'", ): assert check.two_dim_dict_param({1: { "level_two_key": "something" }}, "foo", key_type=int)
def test_two_dim_dict(): assert check.two_dim_dict_param({}, 'foo') == {} assert check.two_dim_dict_param({'key': {}}, 'foo') assert check.two_dim_dict_param({'key': {'key2': 2}}, 'foo') # make sure default dict passes default_dict = defaultdict(dict) default_dict['key']['key2'] = 2 assert check.two_dim_dict_param(default_dict, 'foo') with raises_with_message( CheckError, '''Param "foo" is not a dict. Got None which is type <class 'NoneType'>.''' if is_python_three() else '''Param "foo" is not a dict. Got None which is type <type 'NoneType'>.''', ): check.two_dim_dict_param(None, 'foo') with raises_with_message( CheckError, "Value in dictionary mismatches expected type for key int_value. Expected value " "of type <class 'dict'>. Got value 2 of type <class 'int'>." if is_python_three() else "Value in dictionary mismatches expected type for key int_value. Expected value " "of type <type 'dict'>. Got value 2 of type <type 'int'>.", ): check.two_dim_dict_param({'int_value': 2}, 'foo') with raises_with_message( CheckError, "Value in dictionary mismatches expected type for key level_two_value_mismatch. " "Expected value of type (<class 'str'>,). Got value 2 of type <class 'int'>." if is_python_three() else "Value in dictionary mismatches expected type for key level_two_value_mismatch. " "Expected value of type (<type 'basestring'>,). Got value 2 of type <type 'int'>.", ): check.two_dim_dict_param( {'level_one_key': { 'level_two_value_mismatch': 2 }}, 'foo', value_type=str) with raises_with_message( CheckError, "Key in dictionary mismatches type. Expected <class 'int'>. Got 'key'" if is_python_three() else "Key in dictionary mismatches type. Expected <type 'int'>. Got 'key'", ): assert check.two_dim_dict_param({'key': {}}, 'foo', key_type=int) with raises_with_message( CheckError, "Key in dictionary mismatches type. Expected <class 'int'>. Got 'level_two_key'" if is_python_three() else "Key in dictionary mismatches type. Expected <type 'int'>. Got 'level_two_key'", ): assert check.two_dim_dict_param({1: { 'level_two_key': 'something' }}, 'foo', key_type=int)