def test_zero_floats(): ctx = FlyteContext.current_context() l0 = Literal(scalar=Scalar(primitive=Primitive(integer=0))) l1 = Literal(scalar=Scalar(primitive=Primitive(float_value=0.0))) assert TypeEngine.to_python_value(ctx, l0, float) == 0 assert TypeEngine.to_python_value(ctx, l1, float) == 0
def test_protos(): ctx = FlyteContext.current_context() pb = errors_pb2.ContainerError(code="code", message="message") lt = TypeEngine.to_literal_type(errors_pb2.ContainerError) assert lt.simple == SimpleType.STRUCT assert lt.metadata["pb_type"] == "flyteidl.core.errors_pb2.ContainerError" lit = TypeEngine.to_literal(ctx, pb, errors_pb2.ContainerError, lt) new_python_val = TypeEngine.to_python_value(ctx, lit, errors_pb2.ContainerError) assert new_python_val == pb # Test error l0 = Literal(scalar=Scalar(primitive=Primitive(integer=4))) with pytest.raises(AssertionError): TypeEngine.to_python_value(ctx, l0, errors_pb2.ContainerError)
def test_engine_file_output(): basic_blob_type = _core_types.BlobType( format="", dimensionality=_core_types.BlobType.BlobDimensionality.SINGLE, ) fs = FileAccessProvider(local_sandbox_dir="/tmp/flytetesting") with context_manager.FlyteContext.current_context( ).new_file_access_context(file_access_provider=fs) as ctx: # Write some text to a file not in that directory above test_file_location = "/tmp/sample.txt" with open(test_file_location, "w") as fh: fh.write("Hello World\n") lit = TypeEngine.to_literal(ctx, test_file_location, os.PathLike, LiteralType(blob=basic_blob_type)) # Since we're using local as remote, we should be able to just read the file from the 'remote' location. with open(lit.scalar.blob.uri, "r") as fh: assert fh.readline() == "Hello World\n" # We should also be able to turn the thing back into regular python native thing. redownloaded_local_file_location = TypeEngine.to_python_value( ctx, lit, os.PathLike) with open(redownloaded_local_file_location, "r") as fh: assert fh.readline() == "Hello World\n"
def test_list_transformer(): l0 = Literal(scalar=Scalar(primitive=Primitive(integer=3))) l1 = Literal(scalar=Scalar(primitive=Primitive(integer=4))) lc = LiteralCollection(literals=[l0, l1]) lit = Literal(collection=lc) ctx = FlyteContext.current_context() xx = TypeEngine.to_python_value(ctx, lit, typing.List[int]) assert xx == [3, 4]
def test_pb_guess_python_type(): artifact_tag = catalog_pb2.CatalogArtifactTag(artifact_id="artifact_1", name="artifact_name") x = {"a": artifact_tag} lt = TypeEngine.to_literal_type(catalog_pb2.CatalogArtifactTag) gt = TypeEngine.guess_python_type(lt) assert gt == catalog_pb2.CatalogArtifactTag ctx = FlyteContextManager.current_context() lm = TypeEngine.dict_to_literal_map(ctx, x, {"a": gt}) pv = TypeEngine.to_python_value(ctx, lm.literals["a"], gt) assert pv == artifact_tag
def test_enum_type(): t = TypeEngine.to_literal_type(Color) assert t is not None assert t.enum_type is not None assert t.enum_type.values assert t.enum_type.values == [c.value for c in Color] ctx = FlyteContextManager.current_context() lv = TypeEngine.to_literal(ctx, Color.RED, Color, TypeEngine.to_literal_type(Color)) assert lv assert lv.scalar assert lv.scalar.primitive.string_value == "red" v = TypeEngine.to_python_value(ctx, lv, Color) assert v assert v == Color.RED v = TypeEngine.to_python_value(ctx, lv, str) assert v assert v == "red" with pytest.raises(ValueError): TypeEngine.to_python_value(ctx, Literal(scalar=Scalar(primitive=Primitive(string_value=str(Color.RED)))), Color) with pytest.raises(ValueError): TypeEngine.to_python_value(ctx, Literal(scalar=Scalar(primitive=Primitive(string_value="bad"))), Color) with pytest.raises(AssertionError): TypeEngine.to_literal_type(UnsupportedEnumValues)
def create_native_named_tuple(ctx: FlyteContext, promises: Union[Promise, typing.List[Promise]], entity_interface: Interface) -> Optional[Tuple]: """ Creates and returns a Named tuple with all variables that match the expected named outputs. this makes it possible to run things locally and expect a more native behavior, i.e. address elements of a named tuple by name. """ if entity_interface is None: raise ValueError( "Interface of the entity is required to generate named outputs") if promises is None: return None if isinstance(promises, Promise): v = [v for k, v in entity_interface.outputs.items() ][0] # get output native type return TypeEngine.to_python_value(ctx, promises.val, v) if len(promises) == 0: return None named_tuple_name = "DefaultNamedTupleOutput" if entity_interface.output_tuple_name: named_tuple_name = entity_interface.output_tuple_name outputs = {} for p in promises: if not isinstance(p, Promise): raise AssertionError( "Workflow outputs can only be promises that are returned by tasks. Found a value of" f"type {type(p)}. Workflows cannot return local variables or constants." ) outputs[p.var] = TypeEngine.to_python_value( ctx, p.val, entity_interface.outputs[p.var]) # Should this class be part of the Interface? t = collections.namedtuple(named_tuple_name, list(outputs.keys())) return t(**outputs)
def test_dataclass_complex_transform(two_sample_inputs): my_input = two_sample_inputs[0] my_input_2 = two_sample_inputs[1] ctx = FlyteContextManager.current_context() literal_type = TypeEngine.to_literal_type(MyInput) first_literal = TypeEngine.to_literal(ctx, my_input, MyInput, literal_type) assert first_literal.scalar.generic["apriori_config"] is not None converted_back_1 = TypeEngine.to_python_value(ctx, first_literal, MyInput) assert converted_back_1.apriori_config is not None second_literal = TypeEngine.to_literal(ctx, converted_back_1, MyInput, literal_type) assert second_literal.scalar.generic["apriori_config"] is not None converted_back_2 = TypeEngine.to_python_value(ctx, second_literal, MyInput) assert converted_back_2.apriori_config is not None input_list = [my_input, my_input_2] input_list_type = TypeEngine.to_literal_type(List[MyInput]) literal_list = TypeEngine.to_literal(ctx, input_list, List[MyInput], input_list_type) assert literal_list.collection.literals[0].scalar.generic["apriori_config"] is not None assert literal_list.collection.literals[1].scalar.generic["apriori_config"] is not None
def __call__(self, *args, **kwargs): """ The call pattern for Workflows is close to, but not exactly, the call pattern for Tasks. For local execution, it goes __call__ -> _local_execute -> execute From execute, different things happen for the two Workflow styles. For PythonFunctionWorkflows, the Python function is run, for the ImperativeWorkflow, each node is run one at a time. """ if len(args) > 0: raise AssertionError("Only Keyword Arguments are supported for Workflow executions") ctx = FlyteContextManager.current_context() # Get default agruements and override with kwargs passed in input_kwargs = self.python_interface.default_inputs_as_kwargs input_kwargs.update(kwargs) # The first condition is compilation. if ctx.compilation_state is not None: return create_and_link_node(ctx, entity=self, interface=self.python_interface, **input_kwargs) # This condition is hit when this workflow (self) is being called as part of a parent's workflow local run. # The context specifying the local workflow execution has already been set. elif ( ctx.execution_state is not None and ctx.execution_state.mode == ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION ): if ctx.execution_state.branch_eval_mode == BranchEvalMode.BRANCH_SKIPPED: if self.python_interface and self.python_interface.output_tuple_name: variables = [k for k in self.python_interface.outputs.keys()] output_tuple = collections.namedtuple(self.python_interface.output_tuple_name, variables) nones = [None for _ in self.python_interface.outputs.keys()] return output_tuple(*nones) else: return None # We are already in a local execution, just continue the execution context return self._local_execute(ctx, **input_kwargs) # Last is starting a local workflow execution else: # Run some sanity checks # Even though the _local_execute call generally expects inputs to be Promises, we don't have to do the # conversion here in this loop. The reason is because we don't prevent users from specifying inputs # as direct scalars, which means there's another Promise-generating loop inside _local_execute too for k, v in input_kwargs.items(): if k not in self.interface.inputs: raise ValueError(f"Received unexpected keyword argument {k}") if isinstance(v, Promise): raise ValueError(f"Received a promise for a workflow call, when expecting a native value for {k}") result = None with FlyteContextManager.with_context( ctx.with_execution_state( ctx.new_execution_state().with_params(mode=ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION) ) ) as child_ctx: result = self._local_execute(child_ctx, **input_kwargs) expected_outputs = len(self.python_interface.outputs) if expected_outputs == 0: if result is None or isinstance(result, VoidPromise): return None else: raise Exception(f"Workflow local execution expected 0 outputs but something received {result}") if (1 < expected_outputs == len(result)) or (result is not None and expected_outputs == 1): if isinstance(result, Promise): v = [v for k, v in self.python_interface.outputs.items()][0] # get output native type return TypeEngine.to_python_value(ctx, result.val, v) else: for prom in result: if not isinstance(prom, Promise): raise Exception("should be promises") native_list = [ TypeEngine.to_python_value(ctx, promise.val, self.python_interface.outputs[promise.var]) for promise in result ] return tuple(native_list) raise ValueError("expected outputs and actual outputs do not match")
def __call__(self, *args, **kwargs): if len(args) > 0: raise AssertionError( "Only Keyword Arguments are supported for Workflow executions") ctx = FlyteContext.current_context() # Handle subworkflows in compilation if ctx.compilation_state is not None: input_kwargs = self._native_interface.default_inputs_as_kwargs input_kwargs.update(kwargs) return create_and_link_node(ctx, entity=self, interface=self._native_interface, **input_kwargs) elif (ctx.execution_state is not None and ctx.execution_state.mode == ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION): # We are already in a local execution, just continue the execution context return self._local_execute(ctx, **kwargs) # When someone wants to run the workflow function locally. Assume that the inputs given are given as Python # native values. _local_execute will always translate Python native literals to Flyte literals, so no worries # there, but it'll return Promise objects. else: # Run some sanity checks # Even though the _local_execute call generally expects inputs to be Promises, we don't have to do the # conversion here in this loop. The reason is because we don't prevent users from specifying inputs # as direct scalars, which means there's another Promise-generating loop inside _local_execute too for k, v in kwargs.items(): if k not in self.interface.inputs: raise ValueError( f"Received unexpected keyword argument {k}") if isinstance(v, Promise): raise ValueError( f"Received a promise for a workflow call, when expecting a native value for {k}" ) with ctx.new_execution_context( mode=ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION) as ctx: result = self._local_execute(ctx, **kwargs) expected_outputs = len(self._native_interface.outputs) if expected_outputs == 0: if result is None or isinstance(result, VoidPromise): return None else: raise Exception( f"Workflow local execution expected 0 outputs but something received {result}" ) if (expected_outputs > 1 and len(result) == expected_outputs) or ( expected_outputs == 1 and result is not None): if isinstance(result, Promise): v = [v for k, v in self._native_interface.outputs.items()][0] return TypeEngine.to_python_value(ctx, result.val, v) else: for prom in result: if not isinstance(prom, Promise): raise Exception("should be promises") native_list = [ TypeEngine.to_python_value( ctx, promise.val, self._native_interface.outputs[promise.var]) for promise in result ] return tuple(native_list) raise ValueError( "expected outputs and actual outputs do not match")
def convert(self, ctx, param, value) -> typing.Union[Literal, typing.Any]: lit = self.convert_to_literal(ctx, param, value) if not self._remote: return TypeEngine.to_python_value(self._flyte_ctx, lit, self._python_type) return lit