def test_translate_inputs_to_literals(input): @dataclass_json @dataclass class MyDataclass(object): i: int a: typing.List[str] @task def t1(a: typing.Union[float, typing.List[int], MyDataclass]): print(a) ctx = context_manager.FlyteContext.current_context() translate_inputs_to_literals(ctx, {"a": input}, t1.interface.inputs, t1.python_interface.inputs)
def _local_execute(self, ctx: FlyteContext, **kwargs) -> Union[Tuple[Promise], Promise, VoidPromise]: """ Please see the _local_execute comments in the main task. """ # Unwrap the kwargs values. After this, we essentially have a LiteralMap # The reason why we need to do this is because the inputs during local execute can be of 2 types # - Promises or native constants # Promises as essentially inputs from previous task executions # native constants are just bound to this specific task (default values for a task input) # Also alongwith promises and constants, there could be dictionary or list of promises or constants kwargs = translate_inputs_to_literals( ctx, incoming_values=kwargs, flyte_interface_types=self.interface.inputs, native_types=self.python_interface.inputs, ) input_literal_map = _literal_models.LiteralMap(literals=kwargs) outputs_literal_map = self.unwrap_literal_map_and_execute(ctx, input_literal_map) # After running, we again have to wrap the outputs, if any, back into Promise objects outputs_literals = outputs_literal_map.literals output_names = list(self.python_interface.outputs.keys()) if len(output_names) != len(outputs_literals): # Length check, clean up exception raise AssertionError(f"Length difference {len(output_names)} {len(outputs_literals)}") # Tasks that don't return anything still return a VoidPromise if len(output_names) == 0: return VoidPromise(self.name) vals = [Promise(var, outputs_literals[var]) for var in output_names] return create_task_output(vals, self.python_interface)
def _local_execute(self, ctx: FlyteContext, **kwargs) -> Union[Tuple[Promise], Promise, VoidPromise]: """ This code is used only in the case when we want to dispatch_execute with outputs from a previous node For regular execution, dispatch_execute is invoked directly. """ # Unwrap the kwargs values. After this, we essentially have a LiteralMap # The reason why we need to do this is because the inputs during local execute can be of 2 types # - Promises or native constants # Promises as essentially inputs from previous task executions # native constants are just bound to this specific task (default values for a task input) # Also alongwith promises and constants, there could be dictionary or list of promises or constants kwargs = translate_inputs_to_literals( ctx, input_kwargs=kwargs, interface=self.interface, native_input_types=self.get_input_types() ) input_literal_map = _literal_models.LiteralMap(literals=kwargs) outputs_literal_map = self.dispatch_execute(ctx, input_literal_map) outputs_literals = outputs_literal_map.literals # TODO maybe this is the part that should be done for local execution, we pass the outputs to some special # location, otherwise we dont really need to right? The higher level execute could just handle literalMap # After running, we again have to wrap the outputs, if any, back into Promise objects output_names = list(self.interface.outputs.keys()) if len(output_names) != len(outputs_literals): # Length check, clean up exception raise AssertionError(f"Length difference {len(output_names)} {len(outputs_literals)}") # Tasks that don't return anything still return a VoidPromise if len(output_names) == 0: return VoidPromise(self.name) vals = [Promise(var, outputs_literals[var]) for var in output_names] return create_task_output(vals, self.python_interface)
def create( cls, name: str, workflow: _annotated_workflow.WorkflowBase, default_inputs: Dict[str, Any] = None, fixed_inputs: Dict[str, Any] = None, schedule: _schedule_model.Schedule = None, notifications: List[_common_models.Notification] = None, auth_role: _common_models.AuthRole = None, ) -> LaunchPlan: ctx = FlyteContextManager.current_context() default_inputs = default_inputs or {} fixed_inputs = fixed_inputs or {} # Default inputs come from two places, the original signature of the workflow function, and the default_inputs # argument to this function. We'll take the latter as having higher precedence. wf_signature_parameters = transform_inputs_to_parameters( ctx, workflow.python_interface) # Construct a new Interface object with just the default inputs given to get Parameters, maybe there's an # easier way to do this, think about it later. temp_inputs = {} for k, v in default_inputs.items(): temp_inputs[k] = (workflow.python_interface.inputs[k], v) temp_interface = Interface(inputs=temp_inputs, outputs={}) temp_signature = transform_inputs_to_parameters(ctx, temp_interface) wf_signature_parameters._parameters.update(temp_signature.parameters) # These are fixed inputs that cannot change at launch time. If the same argument is also in default inputs, # it'll be taken out from defaults in the LaunchPlan constructor fixed_literals = translate_inputs_to_literals( ctx, incoming_values=fixed_inputs, flyte_interface_types=workflow.interface.inputs, native_types=workflow.python_interface.inputs, ) fixed_lm = _literal_models.LiteralMap(literals=fixed_literals) lp = cls( name=name, workflow=workflow, parameters=wf_signature_parameters, fixed_inputs=fixed_lm, schedule=schedule, notifications=notifications, auth_role=auth_role, ) # This is just a convenience - we'll need the fixed inputs LiteralMap for when serializing the Launch Plan out # to protobuf, but for local execution and such, why not save the original Python native values as well so # we don't have to reverse it back every time. default_inputs.update(fixed_inputs) lp._saved_inputs = default_inputs if name in cls.CACHE: raise AssertionError( f"Launch plan named {name} was already created! Make sure your names are unique." ) cls.CACHE[name] = lp return lp
def _local_execute(self, ctx: FlyteContext, **kwargs) -> Union[Tuple[Promise], Promise, VoidPromise]: # This is done to support the invariant that Workflow local executions always work with Promise objects # holding Flyte literal values. Even in a wf, a user can call a sub-workflow with a Python native value. for k, v in kwargs.items(): if not isinstance(v, Promise): t = self.python_interface.inputs[k] kwargs[k] = Promise(var=k, val=TypeEngine.to_literal(ctx, v, t, self.interface.inputs[k].type)) # The output of this will always be a combination of Python native values and Promises containing Flyte # Literals. function_outputs = self.execute(**kwargs) # First handle the empty return case. # A workflow function may return a task that doesn't return anything # def wf(): # return t1() # or it may not return at all # def wf(): # t1() # In the former case we get the task's VoidPromise, in the latter we get None if isinstance(function_outputs, VoidPromise) or function_outputs is None: if len(self.python_interface.outputs) != 0: raise FlyteValueException( function_outputs, f"{function_outputs} received but interface has {len(self.python_interface.outputs)} outputs.", ) return VoidPromise(self.name) # Because we should've already returned in the above check, we just raise an error here. if len(self.python_interface.outputs) == 0: raise FlyteValueException( function_outputs, f"{function_outputs} received but should've been VoidPromise or None." ) expected_output_names = list(self.python_interface.outputs.keys()) if len(expected_output_names) == 1: # Here we have to handle the fact that the wf could've been declared with a typing.NamedTuple of # length one. That convention is used for naming outputs - and single-length-NamedTuples are # particularly troublesome but elegant handling of them is not a high priority # Again, we're using the output_tuple_name as a proxy. if self.python_interface.output_tuple_name and isinstance(function_outputs, tuple): wf_outputs_as_map = {expected_output_names[0]: function_outputs[0]} else: wf_outputs_as_map = {expected_output_names[0]: function_outputs} else: wf_outputs_as_map = {expected_output_names[i]: function_outputs[i] for i, _ in enumerate(function_outputs)} # Basically we need to repackage the promises coming from the tasks into Promises that match the workflow's # interface. We do that by extracting out the literals, and creating new Promises wf_outputs_as_literal_dict = translate_inputs_to_literals( ctx, wf_outputs_as_map, flyte_interface_types=self.interface.outputs, native_types=self.python_interface.outputs, ) # Recreate new promises that use the workflow's output names. new_promises = [Promise(var, wf_outputs_as_literal_dict[var]) for var in expected_output_names] return create_task_output(new_promises, self.python_interface)
def test_translate_inputs_to_literals_with_wrong_types(): ctx = context_manager.FlyteContext.current_context() with pytest.raises(TypeError, match="Not a map type union_type"): @task def t1(a: typing.Union[float, typing.List[int]]): print(a) translate_inputs_to_literals(ctx, {"a": {"a": 3}}, t1.interface.inputs, t1.python_interface.inputs) with pytest.raises(TypeError, match="Not a collection type union_type"): @task def t1(a: typing.Union[float, typing.Dict[str, int]]): print(a) translate_inputs_to_literals(ctx, {"a": [1, 2, 3]}, t1.interface.inputs, t1.python_interface.inputs) with pytest.raises( AssertionError, match="Outputs of a non-output producing task n0 cannot be passed to another task" ): @task def t1(a: typing.Union[float, typing.Dict[str, int]]): print(a) translate_inputs_to_literals(ctx, {"a": VoidPromise("n0")}, t1.interface.inputs, t1.python_interface.inputs)
def local_execute(self, ctx: FlyteContext, **kwargs) -> Union[Tuple[Promise], Promise, VoidPromise]: """ This function is used only in the local execution path and is responsible for calling dispatch execute. Use this function when calling a task with native values (or Promises containing Flyte literals derived from Python native values). """ # Unwrap the kwargs values. After this, we essentially have a LiteralMap # The reason why we need to do this is because the inputs during local execute can be of 2 types # - Promises or native constants # Promises as essentially inputs from previous task executions # native constants are just bound to this specific task (default values for a task input) # Also along with promises and constants, there could be dictionary or list of promises or constants kwargs = translate_inputs_to_literals( ctx, incoming_values=kwargs, flyte_interface_types=self.interface.inputs, # type: ignore native_types=self.get_input_types(), ) input_literal_map = _literal_models.LiteralMap(literals=kwargs) # if metadata.cache is set, check memoized version if self.metadata.cache: # TODO: how to get a nice `native_inputs` here? logger.info( f"Checking cache for task named {self.name}, cache version {self.metadata.cache_version} " f"and inputs: {input_literal_map}") outputs_literal_map = LocalTaskCache.get( self.name, self.metadata.cache_version, input_literal_map) # The cache returns None iff the key does not exist in the cache if outputs_literal_map is None: logger.info("Cache miss, task will be executed now") outputs_literal_map = self.dispatch_execute( ctx, input_literal_map) # TODO: need `native_inputs` LocalTaskCache.set(self.name, self.metadata.cache_version, input_literal_map, outputs_literal_map) logger.info( f"Cache set for task named {self.name}, cache version {self.metadata.cache_version} " f"and inputs: {input_literal_map}") else: logger.info("Cache hit") else: es = ctx.execution_state b = es.user_space_params.with_task_sandbox() ctx = ctx.current_context().with_execution_state( es.with_params(user_space_params=b.build())).build() outputs_literal_map = self.dispatch_execute(ctx, input_literal_map) outputs_literals = outputs_literal_map.literals # TODO maybe this is the part that should be done for local execution, we pass the outputs to some special # location, otherwise we dont really need to right? The higher level execute could just handle literalMap # After running, we again have to wrap the outputs, if any, back into Promise objects output_names = list(self.interface.outputs.keys()) # type: ignore if len(output_names) != len(outputs_literals): # Length check, clean up exception raise AssertionError( f"Length difference {len(output_names)} {len(outputs_literals)}" ) # Tasks that don't return anything still return a VoidPromise if len(output_names) == 0: return VoidPromise(self.name) vals = [Promise(var, outputs_literals[var]) for var in output_names] return create_task_output(vals, self.python_interface)