def test_zero_floats():
ctx = FlyteContext.current_context()
l0 = Literal(scalar=Scalar(primitive=Primitive(integer=0)))
l1 = Literal(scalar=Scalar(primitive=Primitive(float_value=0.0)))
assert TypeEngine.to_python_value(ctx, l0, float) == 0
assert TypeEngine.to_python_value(ctx, l1, float) == 0
def test_protos():
ctx = FlyteContext.current_context()
pb = errors_pb2.ContainerError(code="code", message="message")
lt = TypeEngine.to_literal_type(errors_pb2.ContainerError)
assert lt.simple == SimpleType.STRUCT
assert lt.metadata["pb_type"] == "flyteidl.core.errors_pb2.ContainerError"
lit = TypeEngine.to_literal(ctx, pb, errors_pb2.ContainerError, lt)
new_python_val = TypeEngine.to_python_value(ctx, lit, errors_pb2.ContainerError)
assert new_python_val == pb
# Test error
l0 = Literal(scalar=Scalar(primitive=Primitive(integer=4)))
with pytest.raises(AssertionError):
TypeEngine.to_python_value(ctx, l0, errors_pb2.ContainerError)
def test_engine_file_output():
basic_blob_type = _core_types.BlobType(
format="",
dimensionality=_core_types.BlobType.BlobDimensionality.SINGLE,
)
fs = FileAccessProvider(local_sandbox_dir="/tmp/flytetesting")
with context_manager.FlyteContext.current_context(
).new_file_access_context(file_access_provider=fs) as ctx:
# Write some text to a file not in that directory above
test_file_location = "/tmp/sample.txt"
with open(test_file_location, "w") as fh:
fh.write("Hello World\n")
lit = TypeEngine.to_literal(ctx, test_file_location, os.PathLike,
LiteralType(blob=basic_blob_type))
# Since we're using local as remote, we should be able to just read the file from the 'remote' location.
with open(lit.scalar.blob.uri, "r") as fh:
assert fh.readline() == "Hello World\n"
# We should also be able to turn the thing back into regular python native thing.
redownloaded_local_file_location = TypeEngine.to_python_value(
ctx, lit, os.PathLike)
with open(redownloaded_local_file_location, "r") as fh:
assert fh.readline() == "Hello World\n"
def test_list_transformer():
l0 = Literal(scalar=Scalar(primitive=Primitive(integer=3)))
l1 = Literal(scalar=Scalar(primitive=Primitive(integer=4)))
lc = LiteralCollection(literals=[l0, l1])
lit = Literal(collection=lc)
ctx = FlyteContext.current_context()
xx = TypeEngine.to_python_value(ctx, lit, typing.List[int])
assert xx == [3, 4]
def test_pb_guess_python_type():
artifact_tag = catalog_pb2.CatalogArtifactTag(artifact_id="artifact_1", name="artifact_name")
x = {"a": artifact_tag}
lt = TypeEngine.to_literal_type(catalog_pb2.CatalogArtifactTag)
gt = TypeEngine.guess_python_type(lt)
assert gt == catalog_pb2.CatalogArtifactTag
ctx = FlyteContextManager.current_context()
lm = TypeEngine.dict_to_literal_map(ctx, x, {"a": gt})
pv = TypeEngine.to_python_value(ctx, lm.literals["a"], gt)
assert pv == artifact_tag
def test_enum_type():
t = TypeEngine.to_literal_type(Color)
assert t is not None
assert t.enum_type is not None
assert t.enum_type.values
assert t.enum_type.values == [c.value for c in Color]
ctx = FlyteContextManager.current_context()
lv = TypeEngine.to_literal(ctx, Color.RED, Color, TypeEngine.to_literal_type(Color))
assert lv
assert lv.scalar
assert lv.scalar.primitive.string_value == "red"
v = TypeEngine.to_python_value(ctx, lv, Color)
assert v
assert v == Color.RED
v = TypeEngine.to_python_value(ctx, lv, str)
assert v
assert v == "red"
with pytest.raises(ValueError):
TypeEngine.to_python_value(ctx, Literal(scalar=Scalar(primitive=Primitive(string_value=str(Color.RED)))), Color)
with pytest.raises(ValueError):
TypeEngine.to_python_value(ctx, Literal(scalar=Scalar(primitive=Primitive(string_value="bad"))), Color)
with pytest.raises(AssertionError):
TypeEngine.to_literal_type(UnsupportedEnumValues)
def create_native_named_tuple(ctx: FlyteContext,
promises: Union[Promise, typing.List[Promise]],
entity_interface: Interface) -> Optional[Tuple]:
"""
Creates and returns a Named tuple with all variables that match the expected named outputs. this makes
it possible to run things locally and expect a more native behavior, i.e. address elements of a named tuple
by name.
"""
if entity_interface is None:
raise ValueError(
"Interface of the entity is required to generate named outputs")
if promises is None:
return None
if isinstance(promises, Promise):
v = [v for k, v in entity_interface.outputs.items()
][0] # get output native type
return TypeEngine.to_python_value(ctx, promises.val, v)
if len(promises) == 0:
return None
named_tuple_name = "DefaultNamedTupleOutput"
if entity_interface.output_tuple_name:
named_tuple_name = entity_interface.output_tuple_name
outputs = {}
for p in promises:
if not isinstance(p, Promise):
raise AssertionError(
"Workflow outputs can only be promises that are returned by tasks. Found a value of"
f"type {type(p)}. Workflows cannot return local variables or constants."
)
outputs[p.var] = TypeEngine.to_python_value(
ctx, p.val, entity_interface.outputs[p.var])
# Should this class be part of the Interface?
t = collections.namedtuple(named_tuple_name, list(outputs.keys()))
return t(**outputs)
def test_dataclass_complex_transform(two_sample_inputs):
my_input = two_sample_inputs[0]
my_input_2 = two_sample_inputs[1]
ctx = FlyteContextManager.current_context()
literal_type = TypeEngine.to_literal_type(MyInput)
first_literal = TypeEngine.to_literal(ctx, my_input, MyInput, literal_type)
assert first_literal.scalar.generic["apriori_config"] is not None
converted_back_1 = TypeEngine.to_python_value(ctx, first_literal, MyInput)
assert converted_back_1.apriori_config is not None
second_literal = TypeEngine.to_literal(ctx, converted_back_1, MyInput, literal_type)
assert second_literal.scalar.generic["apriori_config"] is not None
converted_back_2 = TypeEngine.to_python_value(ctx, second_literal, MyInput)
assert converted_back_2.apriori_config is not None
input_list = [my_input, my_input_2]
input_list_type = TypeEngine.to_literal_type(List[MyInput])
literal_list = TypeEngine.to_literal(ctx, input_list, List[MyInput], input_list_type)
assert literal_list.collection.literals[0].scalar.generic["apriori_config"] is not None
assert literal_list.collection.literals[1].scalar.generic["apriori_config"] is not None
def __call__(self, *args, **kwargs):
"""
The call pattern for Workflows is close to, but not exactly, the call pattern for Tasks. For local execution,
it goes
__call__ -> _local_execute -> execute
From execute, different things happen for the two Workflow styles. For PythonFunctionWorkflows, the Python
function is run, for the ImperativeWorkflow, each node is run one at a time.
"""
if len(args) > 0:
raise AssertionError("Only Keyword Arguments are supported for Workflow executions")
ctx = FlyteContextManager.current_context()
# Get default agruements and override with kwargs passed in
input_kwargs = self.python_interface.default_inputs_as_kwargs
input_kwargs.update(kwargs)
# The first condition is compilation.
if ctx.compilation_state is not None:
return create_and_link_node(ctx, entity=self, interface=self.python_interface, **input_kwargs)
# This condition is hit when this workflow (self) is being called as part of a parent's workflow local run.
# The context specifying the local workflow execution has already been set.
elif (
ctx.execution_state is not None and ctx.execution_state.mode == ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION
):
if ctx.execution_state.branch_eval_mode == BranchEvalMode.BRANCH_SKIPPED:
if self.python_interface and self.python_interface.output_tuple_name:
variables = [k for k in self.python_interface.outputs.keys()]
output_tuple = collections.namedtuple(self.python_interface.output_tuple_name, variables)
nones = [None for _ in self.python_interface.outputs.keys()]
return output_tuple(*nones)
else:
return None
# We are already in a local execution, just continue the execution context
return self._local_execute(ctx, **input_kwargs)
# Last is starting a local workflow execution
else:
# Run some sanity checks
# Even though the _local_execute call generally expects inputs to be Promises, we don't have to do the
# conversion here in this loop. The reason is because we don't prevent users from specifying inputs
# as direct scalars, which means there's another Promise-generating loop inside _local_execute too
for k, v in input_kwargs.items():
if k not in self.interface.inputs:
raise ValueError(f"Received unexpected keyword argument {k}")
if isinstance(v, Promise):
raise ValueError(f"Received a promise for a workflow call, when expecting a native value for {k}")
result = None
with FlyteContextManager.with_context(
ctx.with_execution_state(
ctx.new_execution_state().with_params(mode=ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION)
)
) as child_ctx:
result = self._local_execute(child_ctx, **input_kwargs)
expected_outputs = len(self.python_interface.outputs)
if expected_outputs == 0:
if result is None or isinstance(result, VoidPromise):
return None
else:
raise Exception(f"Workflow local execution expected 0 outputs but something received {result}")
if (1 < expected_outputs == len(result)) or (result is not None and expected_outputs == 1):
if isinstance(result, Promise):
v = [v for k, v in self.python_interface.outputs.items()][0] # get output native type
return TypeEngine.to_python_value(ctx, result.val, v)
else:
for prom in result:
if not isinstance(prom, Promise):
raise Exception("should be promises")
native_list = [
TypeEngine.to_python_value(ctx, promise.val, self.python_interface.outputs[promise.var])
for promise in result
]
return tuple(native_list)
raise ValueError("expected outputs and actual outputs do not match")
def __call__(self, *args, **kwargs):
if len(args) > 0:
raise AssertionError(
"Only Keyword Arguments are supported for Workflow executions")
ctx = FlyteContext.current_context()
# Handle subworkflows in compilation
if ctx.compilation_state is not None:
input_kwargs = self._native_interface.default_inputs_as_kwargs
input_kwargs.update(kwargs)
return create_and_link_node(ctx,
entity=self,
interface=self._native_interface,
**input_kwargs)
elif (ctx.execution_state is not None and ctx.execution_state.mode
== ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION):
# We are already in a local execution, just continue the execution context
return self._local_execute(ctx, **kwargs)
# When someone wants to run the workflow function locally. Assume that the inputs given are given as Python
# native values. _local_execute will always translate Python native literals to Flyte literals, so no worries
# there, but it'll return Promise objects.
else:
# Run some sanity checks
# Even though the _local_execute call generally expects inputs to be Promises, we don't have to do the
# conversion here in this loop. The reason is because we don't prevent users from specifying inputs
# as direct scalars, which means there's another Promise-generating loop inside _local_execute too
for k, v in kwargs.items():
if k not in self.interface.inputs:
raise ValueError(
f"Received unexpected keyword argument {k}")
if isinstance(v, Promise):
raise ValueError(
f"Received a promise for a workflow call, when expecting a native value for {k}"
)
with ctx.new_execution_context(
mode=ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION) as ctx:
result = self._local_execute(ctx, **kwargs)
expected_outputs = len(self._native_interface.outputs)
if expected_outputs == 0:
if result is None or isinstance(result, VoidPromise):
return None
else:
raise Exception(
f"Workflow local execution expected 0 outputs but something received {result}"
)
if (expected_outputs > 1 and len(result) == expected_outputs) or (
expected_outputs == 1 and result is not None):
if isinstance(result, Promise):
v = [v
for k, v in self._native_interface.outputs.items()][0]
return TypeEngine.to_python_value(ctx, result.val, v)
else:
for prom in result:
if not isinstance(prom, Promise):
raise Exception("should be promises")
native_list = [
TypeEngine.to_python_value(
ctx, promise.val,
self._native_interface.outputs[promise.var])
for promise in result
]
return tuple(native_list)
raise ValueError(
"expected outputs and actual outputs do not match")
def convert(self, ctx, param, value) -> typing.Union[Literal, typing.Any]:
lit = self.convert_to_literal(ctx, param, value)
if not self._remote:
return TypeEngine.to_python_value(self._flyte_ctx, lit,
self._python_type)
return lit
