def inputs(self) -> Dict[str, Any]:
"""
Returns the inputs of the task execution in the standard Python format that is produced by
the type engine.
"""
from flytekit.control_plane.tasks.task import FlyteTask
if self._inputs is None:
client = _flyte_engine.get_client()
execution_data = client.get_task_execution_data(self.id)
# Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned.
input_map = _literal_models.LiteralMap({})
if bool(execution_data.full_inputs.literals):
input_map = execution_data.full_inputs
elif execution_data.inputs.bytes > 0:
with _common_utils.AutoDeletingTempDir() as tmp_dir:
tmp_name = os.path.join(tmp_dir.name, "inputs.pb")
_data_proxy.Data.get_data(execution_data.inputs.url,
tmp_name)
input_map = _literal_models.LiteralMap.from_flyte_idl(
_common_utils.load_proto_from_file(
_literals_pb2.LiteralMap, tmp_name))
task = FlyteTask.fetch(self.id.task_id.project,
self.id.task_id.domain,
self.id.task_id.name,
self.id.task_id.version)
self._inputs = TypeEngine.literal_map_to_kwargs(
ctx=FlyteContextManager.current_context(),
lm=input_map,
python_types=TypeEngine.guess_python_types(
task.interface.inputs),
)
return self._inputs
def inputs(self) -> Dict[str, Any]:
"""
Returns the inputs to the execution in the standard python format as dictated by the type engine.
"""
if self._inputs is None:
client = _flyte_engine.get_client()
execution_data = client.get_execution_data(self.id)
# Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned.
input_map: _literal_models.LiteralMap = _literal_models.LiteralMap(
{})
if bool(execution_data.full_inputs.literals):
input_map = execution_data.full_inputs
elif execution_data.inputs.bytes > 0:
with _common_utils.AutoDeletingTempDir() as tmp_dir:
tmp_name = _os.path.join(tmp_dir.name, "inputs.pb")
_data_proxy.Data.get_data(execution_data.inputs.url,
tmp_name)
input_map = _literal_models.LiteralMap.from_flyte_idl(
_common_utils.load_proto_from_file(
_literals_pb2.Literalmap, tmp_name))
lp_id = self.spec.launch_plan
workflow = _workflow.FlyteWorkflow.fetch(lp_id.project,
lp_id.domain, lp_id.name,
lp_id.version)
self._inputs = TypeEngine.literal_map_to_kwargs(
ctx=FlyteContextManager.current_context(),
lm=input_map,
python_types=TypeEngine.guess_python_types(
workflow.interface.inputs),
)
return self._inputs
def outputs(self) -> Dict[str, Any]:
"""
Returns the outputs of the task execution, if available, in the standard Python format that is produced by
the type engine.
:raises: ``FlyteAssertion`` error if execution is in progress or execution ended in error.
"""
from flytekit.control_plane.tasks.task import FlyteTask
if not self.is_complete:
raise _user_exceptions.FlyteAssertion(
"Please what until the task execution has completed before requesting the outputs."
)
if self.error:
raise _user_exceptions.FlyteAssertion(
"Outputs could not be found because the execution ended in failure."
)
if self._outputs is None:
client = _flyte_engine.get_client()
execution_data = client.get_task_execution_data(self.id)
# Inputs are returned inline unless they are too big, in which case a url blob pointing to them is returned.
output_map = _literal_models.LiteralMap({})
if bool(execution_data.full_outputs.literals):
output_map = execution_data.full_outputs
elif execution_data.outputs.bytes > 0:
with _common_utils.AutoDeletingTempDir() as t:
tmp_name = os.path.join(t.name, "outputs.pb")
_data_proxy.Data.get_data(execution_data.outputs.url,
tmp_name)
output_map = _literal_models.LiteralMap.from_flyte_idl(
_common_utils.load_proto_from_file(
_literals_pb2.LiteralMap, tmp_name))
task = FlyteTask.fetch(self.id.task_id.project,
self.id.task_id.domain,
self.id.task_id.name,
self.id.task_id.version)
self._outputs = TypeEngine.literal_map_to_kwargs(
ctx=FlyteContextManager.current_context(),
lm=output_map,
python_types=TypeEngine.guess_python_types(
task.interface.outputs),
)
return self._outputs
def outputs(self) -> Dict[str, Any]:
"""
Returns the outputs to the execution in the standard python format as dictated by the type engine.
:raises: ``FlyteAssertion`` error if execution is in progress or execution ended in error.
"""
if not self.is_complete:
raise _user_exceptions.FlyteAssertion(
"Please wait until the node execution has completed before requesting the outputs."
)
if self.error:
raise _user_exceptions.FlyteAssertion(
"Outputs could not be found because the execution ended in failure."
)
if self._outputs is None:
client = _flyte_engine.get_client()
execution_data = client.get_execution_data(self.id)
# Outputs are returned inline unless they are too big, in which case a url blob pointing to them is returned.
output_map: _literal_models.LiteralMap = _literal_models.LiteralMap(
{})
if bool(execution_data.full_outputs.literals):
output_map = execution_data.full_outputs
elif execution_data.outputs.bytes > 0:
with _common_utils.AutoDeletingTempDir() as tmp_dir:
tmp_name = _os.path.join(tmp_dir.name, "outputs.pb")
_data_proxy.Data.get_data(execution_data.outputs.url,
tmp_name)
output_map = _literal_models.LiteralMap.from_flyte_idl(
_common_utils.load_proto_from_file(
_literals_pb2.LiteralMap, tmp_name))
lp_id = self.spec.launch_plan
workflow = _workflow.FlyteWorkflow.fetch(lp_id.project,
lp_id.domain, lp_id.name,
lp_id.version)
self._outputs = TypeEngine.literal_map_to_kwargs(
ctx=FlyteContextManager.current_context(),
lm=output_map,
python_types=TypeEngine.guess_python_types(
workflow.interface.outputs),
)
return self._outputs
def dispatch_execute(
self, ctx: FlyteContext, input_literal_map: _literal_models.LiteralMap
) -> Union[_literal_models.LiteralMap, _dynamic_job.DynamicJobSpec]:
"""
This function is mostly copied from the base PythonTask, but differs in that we have to infer the Python
interface before executing. Also, we refer to ``self.task_template`` rather than just ``self`` like in task
classes that derive from the base ``PythonTask``.
"""
# Invoked before the task is executed
new_user_params = self.pre_execute(ctx.user_space_params)
# Create another execution context with the new user params, but let's keep the same working dir
with ctx.new_execution_context(
mode=ctx.execution_state.mode,
execution_params=new_user_params,
working_dir=ctx.execution_state.working_dir,
) as exec_ctx:
# Added: Have to reverse the Python interface from the task template Flyte interface
# See docstring for more details.
guessed_python_input_types = TypeEngine.guess_python_types(
self.task_template.interface.inputs)
native_inputs = TypeEngine.literal_map_to_kwargs(
exec_ctx, input_literal_map, guessed_python_input_types)
logger.info(
f"Invoking FlyteTask executor {self.task_template.id.name} with inputs: {native_inputs}"
)
try:
native_outputs = self.execute(**native_inputs)
except Exception as e:
logger.exception(f"Exception when executing {e}")
raise e
logger.info(
f"Task executed successfully in user level, outputs: {native_outputs}"
)
# Lets run the post_execute method. This may result in a IgnoreOutputs Exception, which is
# bubbled up to be handled at the callee layer.
native_outputs = self.post_execute(new_user_params, native_outputs)
# Short circuit the translation to literal map because what's returned may be a dj spec (or an
# already-constructed LiteralMap if the dynamic task was a no-op), not python native values
if isinstance(native_outputs,
_literal_models.LiteralMap) or isinstance(
native_outputs, _dynamic_job.DynamicJobSpec):
return native_outputs
expected_output_names = list(
self.task_template.interface.outputs.keys())
if len(expected_output_names) == 1:
# Here we have to handle the fact that the task could've been declared with a typing.NamedTuple of
# length one. That convention is used for naming outputs - and single-length-NamedTuples are
# particularly troublesome but elegant handling of them is not a high priority
# Again, we're using the output_tuple_name as a proxy.
# Deleted some stuff
native_outputs_as_map = {
expected_output_names[0]: native_outputs
}
elif len(expected_output_names) == 0:
native_outputs_as_map = {}
else:
native_outputs_as_map = {
expected_output_names[i]: native_outputs[i]
for i, _ in enumerate(native_outputs)
}
# We manually construct a LiteralMap here because task inputs and outputs actually violate the assumption
# built into the IDL that all the values of a literal map are of the same type.
literals = {}
for k, v in native_outputs_as_map.items():
literal_type = self.task_template.interface.outputs[k].type
py_type = type(v)
if isinstance(v, tuple):
raise AssertionError(
f"Output({k}) in task{self.task_template.id.name} received a tuple {v}, instead of {py_type}"
)
try:
literals[k] = TypeEngine.to_literal(
exec_ctx, v, py_type, literal_type)
except Exception as e:
raise AssertionError(
f"failed to convert return value for var {k}") from e
outputs_literal_map = _literal_models.LiteralMap(literals=literals)
# After the execute has been successfully completed
return outputs_literal_map
