def extract_value( ctx: FlyteContext, input_val: Any, val_type: type, flyte_literal_type: _type_models.LiteralType ) -> _literal_models.Literal: if isinstance(input_val, list): if flyte_literal_type.collection_type is None: raise Exception( f"Not a collection type {flyte_literal_type} but got a list {input_val}" ) try: sub_type = ListTransformer.get_sub_type(val_type) except ValueError: if len(input_val) == 0: raise sub_type = type(input_val[0]) literals = [ extract_value(ctx, v, sub_type, flyte_literal_type.collection_type) for v in input_val ] return _literal_models.Literal( collection=_literal_models.LiteralCollection( literals=literals)) elif isinstance(input_val, dict): if (flyte_literal_type.map_value_type is None and flyte_literal_type.simple != _type_models.SimpleType.STRUCT): raise Exception( f"Not a map type {flyte_literal_type} but got a map {input_val}" ) k_type, sub_type = DictTransformer.get_dict_types(val_type) if flyte_literal_type.simple == _type_models.SimpleType.STRUCT: return TypeEngine.to_literal(ctx, input_val, type(input_val), flyte_literal_type) else: literals = { k: extract_value(ctx, v, sub_type, flyte_literal_type.map_value_type) for k, v in input_val.items() } return _literal_models.Literal(map=_literal_models.LiteralMap( literals=literals)) elif isinstance(input_val, Promise): # In the example above, this handles the "in2=a" type of argument return input_val.val elif isinstance(input_val, VoidPromise): raise AssertionError( f"Outputs of a non-output producing task {input_val.task_name} cannot be passed to another task." ) elif isinstance(input_val, tuple): raise AssertionError( "Tuples are not a supported type for individual values in Flyte - got a tuple -" f" {input_val}. If using named tuple in an inner task, please, de-reference the" "actual attribute that you want to use. For example, in NamedTuple('OP', x=int) then" "return v.x, instead of v, even if this has a single element") else: # This handles native values, the 5 example return TypeEngine.to_literal(ctx, input_val, val_type, flyte_literal_type)
def binding_data_from_python_std( ctx: _flyte_context.FlyteContext, expected_literal_type: _type_models.LiteralType, t_value: typing.Any, t_value_type: type, ) -> _literals_models.BindingData: # This handles the case where the given value is the output of another task if isinstance(t_value, Promise): if not t_value.is_ready: return _literals_models.BindingData(promise=t_value.ref) elif isinstance(t_value, VoidPromise): raise AssertionError( f"Cannot pass output from task {t_value.task_name} that produces no outputs to a downstream task" ) elif isinstance(t_value, list): if expected_literal_type.collection_type is None: raise AssertionError( f"this should be a list and it is not: {type(t_value)} vs {expected_literal_type}" ) sub_type = ListTransformer.get_sub_type(t_value_type) collection = _literals_models.BindingDataCollection(bindings=[ binding_data_from_python_std( ctx, expected_literal_type.collection_type, t, sub_type) for t in t_value ]) return _literals_models.BindingData(collection=collection) elif isinstance(t_value, dict): if (expected_literal_type.map_value_type is None and expected_literal_type.simple != _type_models.SimpleType.STRUCT): raise AssertionError( f"this should be a Dictionary type and it is not: {type(t_value)} vs {expected_literal_type}" ) k_type, v_type = DictTransformer.get_dict_types(t_value_type) if expected_literal_type.simple == _type_models.SimpleType.STRUCT: lit = TypeEngine.to_literal(ctx, t_value, type(t_value), expected_literal_type) return _literals_models.BindingData(scalar=lit.scalar) else: m = _literals_models.BindingDataMap( bindings={ k: binding_data_from_python_std( ctx, expected_literal_type.map_value_type, v, v_type) for k, v in t_value.items() }) return _literals_models.BindingData(map=m) # This is the scalar case - e.g. my_task(in1=5) scalar = TypeEngine.to_literal(ctx, t_value, t_value_type, expected_literal_type).scalar return _literals_models.BindingData(scalar=scalar)
def extract_value( ctx: FlyteContext, input_val: Any, val_type: type, flyte_literal_type: _type_models.LiteralType ) -> _literal_models.Literal: if isinstance(input_val, list): if flyte_literal_type.collection_type is None: raise Exception( f"Not a collection type {flyte_literal_type} but got a list {input_val}" ) try: sub_type = ListTransformer.get_sub_type(val_type) except ValueError: if len(input_val) == 0: raise sub_type = type(input_val[0]) literals = [ extract_value(ctx, v, sub_type, flyte_literal_type.collection_type) for v in input_val ] return _literal_models.Literal( collection=_literal_models.LiteralCollection( literals=literals)) elif isinstance(input_val, dict): if (flyte_literal_type.map_value_type is None and flyte_literal_type.simple != _type_models.SimpleType.STRUCT): raise Exception( f"Not a map type {flyte_literal_type} but got a map {input_val}" ) k_type, sub_type = DictTransformer.get_dict_types(val_type) if flyte_literal_type.simple == _type_models.SimpleType.STRUCT: return TypeEngine.to_literal(ctx, input_val, type(input_val), flyte_literal_type) else: literals = { k: extract_value(ctx, v, sub_type, flyte_literal_type.map_value_type) for k, v in input_val.items() } return _literal_models.Literal(map=_literal_models.LiteralMap( literals=literals)) elif isinstance(input_val, Promise): # In the example above, this handles the "in2=a" type of argument return input_val.val elif isinstance(input_val, VoidPromise): raise AssertionError( f"Outputs of a non-output producing task {input_val.task_name} cannot be passed to another task." ) else: # This handles native values, the 5 example return TypeEngine.to_literal(ctx, input_val, val_type, flyte_literal_type)
def test_dict_transformer(): d = DictTransformer() def assert_struct(lit: LiteralType): assert lit is not None assert lit.simple == SimpleType.STRUCT def recursive_assert(lit: LiteralType, expected: LiteralType, expected_depth: int = 1, curr_depth: int = 0): assert curr_depth <= expected_depth assert lit is not None if lit.map_value_type is None: assert lit == expected return recursive_assert(lit.map_value_type, expected, expected_depth, curr_depth + 1) # Type inference assert_struct(d.get_literal_type(dict)) assert_struct(d.get_literal_type(typing.Dict[int, int])) recursive_assert(d.get_literal_type(typing.Dict[str, str]), LiteralType(simple=SimpleType.STRING)) recursive_assert(d.get_literal_type(typing.Dict[str, int]), LiteralType(simple=SimpleType.INTEGER)) recursive_assert(d.get_literal_type(typing.Dict[str, datetime.datetime]), LiteralType(simple=SimpleType.DATETIME)) recursive_assert(d.get_literal_type(typing.Dict[str, datetime.timedelta]), LiteralType(simple=SimpleType.DURATION)) recursive_assert(d.get_literal_type(typing.Dict[str, dict]), LiteralType(simple=SimpleType.STRUCT)) recursive_assert( d.get_literal_type(typing.Dict[str, typing.Dict[str, str]]), LiteralType(simple=SimpleType.STRING), expected_depth=2, ) recursive_assert( d.get_literal_type(typing.Dict[str, typing.Dict[int, str]]), LiteralType(simple=SimpleType.STRUCT), expected_depth=2, ) recursive_assert( d.get_literal_type(typing.Dict[str, typing.Dict[str, typing.Dict[str, str]]]), LiteralType(simple=SimpleType.STRING), expected_depth=3, ) recursive_assert( d.get_literal_type(typing.Dict[str, typing.Dict[str, typing.Dict[str, dict]]]), LiteralType(simple=SimpleType.STRUCT), expected_depth=3, ) recursive_assert( d.get_literal_type(typing.Dict[str, typing.Dict[str, typing.Dict[int, dict]]]), LiteralType(simple=SimpleType.STRUCT), expected_depth=2, ) ctx = FlyteContext.current_context() lit = d.to_literal(ctx, {}, typing.Dict, LiteralType(SimpleType.STRUCT)) pv = d.to_python_value(ctx, lit, typing.Dict) assert pv == {} # Literal to python with pytest.raises(TypeError): d.to_python_value( ctx, Literal(scalar=Scalar(primitive=Primitive(integer=10))), dict) with pytest.raises(TypeError): d.to_python_value(ctx, Literal(), dict) with pytest.raises(TypeError): d.to_python_value(ctx, Literal(map=LiteralMap(literals={"x": None})), dict) with pytest.raises(TypeError): d.to_python_value(ctx, Literal(map=LiteralMap(literals={"x": None})), typing.Dict[int, str]) d.to_python_value( ctx, Literal(map=LiteralMap( literals={ "x": Literal(scalar=Scalar(primitive=Primitive(integer=1))) })), typing.Dict[str, int], )
def binding_data_from_python_std( ctx: _flyte_context.FlyteContext, expected_literal_type: _type_models.LiteralType, t_value: typing.Any, t_value_type: type, ) -> _literals_models.BindingData: # This handles the case where the given value is the output of another task if isinstance(t_value, Promise): if not t_value.is_ready: return _literals_models.BindingData(promise=t_value.ref) elif isinstance(t_value, VoidPromise): raise AssertionError( f"Cannot pass output from task {t_value.task_name} that produces no outputs to a downstream task" ) elif isinstance(t_value, list): if expected_literal_type.collection_type is None: raise AssertionError( f"this should be a list and it is not: {type(t_value)} vs {expected_literal_type}" ) sub_type = ListTransformer.get_sub_type(t_value_type) collection = _literals_models.BindingDataCollection(bindings=[ binding_data_from_python_std( ctx, expected_literal_type.collection_type, t, sub_type) for t in t_value ]) return _literals_models.BindingData(collection=collection) elif isinstance(t_value, dict): if (expected_literal_type.map_value_type is None and expected_literal_type.simple != _type_models.SimpleType.STRUCT): raise AssertionError( f"this should be a Dictionary type and it is not: {type(t_value)} vs {expected_literal_type}" ) k_type, v_type = DictTransformer.get_dict_types(t_value_type) if expected_literal_type.simple == _type_models.SimpleType.STRUCT: lit = TypeEngine.to_literal(ctx, t_value, type(t_value), expected_literal_type) return _literals_models.BindingData(scalar=lit.scalar) else: m = _literals_models.BindingDataMap( bindings={ k: binding_data_from_python_std( ctx, expected_literal_type.map_value_type, v, v_type) for k, v in t_value.items() }) return _literals_models.BindingData(map=m) elif isinstance(t_value, tuple): raise AssertionError( "Tuples are not a supported type for individual values in Flyte - got a tuple -" f" {t_value}. If using named tuple in an inner task, please, de-reference the" "actual attribute that you want to use. For example, in NamedTuple('OP', x=int) then" "return v.x, instead of v, even if this has a single element") # This is the scalar case - e.g. my_task(in1=5) scalar = TypeEngine.to_literal(ctx, t_value, t_value_type, expected_literal_type).scalar return _literals_models.BindingData(scalar=scalar)