def outputs(self) -> Optional[LiteralsResolver]:
"""
:return: Returns the outputs LiteralsResolver to the execution
:raises: ``FlyteAssertion`` error if execution is in progress or execution ended in error.
"""
if not self.is_done:
raise user_exceptions.FlyteAssertion(
"Please wait until the execution has completed before requesting the outputs."
)
if self.error:
raise user_exceptions.FlyteAssertion("Outputs could not be found because the execution ended in failure.")
return self._outputs
def __call__(self, *args, **kwargs):
# When a Task is () aka __called__, there are three things we may do:
# a. Plain execution Mode - just run the execute function. If not overridden, we should raise an exception
# b. Compilation Mode - this happens when the function is called as part of a workflow (potentially
# dynamic task). Produce promise objects and create a node.
# c. Workflow Execution Mode - when a workflow is being run locally. Even though workflows are functions
# and everything should be able to be passed through naturally, we'll want to wrap output values of the
# function into objects, so that potential .with_cpu or other ancillary functions can be attached to do
# nothing. Subsequent tasks will have to know how to unwrap these. If by chance a non-Flyte task uses a
# task output as an input, things probably will fail pretty obviously.
# Since this is a reference entity, it still needs to be mocked otherwise an exception will be raised.
if len(args) > 0:
raise user_exceptions.FlyteAssertion(
f"Cannot call remotely fetched entity with args - detected {len(args)} positional args {args}"
)
ctx = FlyteContext.current_context()
if ctx.compilation_state is not None and ctx.compilation_state.mode == 1:
return self.compile(ctx, *args, **kwargs)
elif (ctx.execution_state is not None and ctx.execution_state.mode
== ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION):
if ctx.execution_state.branch_eval_mode == BranchEvalMode.BRANCH_SKIPPED:
return
return self.local_execute(ctx, **kwargs)
else:
logger.debug("Fetched entity, running raw execute.")
return self.execute(**kwargs)
def _read_legacy_config(self, location: str) -> _configparser.ConfigParser:
c = _configparser.ConfigParser()
c.read(self._location)
if c.has_section("internal"):
raise _user_exceptions.FlyteAssertion(
"The config file '{}' cannot contain a section for internal " "only configurations.".format(location)
)
return c
def test_flyte_assert():
try:
raise user.FlyteAssertion("I ASSERT THAT THIS IS WRONG!")
except user.FlyteAssertion as e:
assert str(e) == "I ASSERT THAT THIS IS WRONG!"
assert isinstance(e, AssertionError)
assert type(e).error_code == "USER:AssertionError"
assert isinstance(e, user.FlyteUserException)
def error(self) -> Optional[core_execution_models.ExecutionError]:
"""
If execution is in progress, raise an exception. Otherwise, return None if no error was present upon
reaching completion.
"""
if not self.is_done:
raise user_exceptions.FlyteAssertion(
"Please what until the task execution has completed before requesting error information."
)
return self.closure.error
def error(self) -> core_execution_models.ExecutionError:
"""
If execution is in progress, raise an exception. Otherwise, return None if no error was present upon
reaching completion.
"""
if not self.is_done:
raise user_exceptions.FlyteAssertion(
"Please wait until a workflow has completed before checking for an error."
)
return self.closure.error
def __init__(
self,
id,
upstream_nodes,
bindings,
metadata,
flyte_task: Optional["FlyteTask"] = None,
flyte_workflow: Optional["FlyteWorkflow"] = None,
flyte_launch_plan: Optional["FlyteLaunchPlan"] = None,
flyte_branch_node: Optional["FlyteBranchNode"] = None,
):
# todo: flyte_branch_node is the only non-entity here, feels wrong, it should probably be a Condition
# or the other ones changed.
non_none_entities = list(
filter(None, [
flyte_task, flyte_workflow, flyte_launch_plan,
flyte_branch_node
]))
if len(non_none_entities) != 1:
raise _user_exceptions.FlyteAssertion(
"An Flyte node must have one underlying entity specified at once. Received the following "
"entities: {}".format(non_none_entities))
# todo: wip - flyte_branch_node is a hack, it should be a Condition, but backing out a Condition object from
# the compiled IfElseBlock is cumbersome, shouldn't do it if we can get away with it.
self._flyte_entity = flyte_task or flyte_workflow or flyte_launch_plan or flyte_branch_node
workflow_node = None
if flyte_workflow is not None:
workflow_node = _component_nodes.FlyteWorkflowNode(
flyte_workflow=flyte_workflow)
elif flyte_launch_plan is not None:
workflow_node = _component_nodes.FlyteWorkflowNode(
flyte_launch_plan=flyte_launch_plan)
task_node = None
if flyte_task:
task_node = _component_nodes.FlyteTaskNode(flyte_task)
super(FlyteNode, self).__init__(
id=id,
metadata=metadata,
inputs=bindings,
upstream_node_ids=[n.id for n in upstream_nodes],
output_aliases=[],
task_node=task_node,
workflow_node=workflow_node,
branch_node=flyte_branch_node,
)
self._upstream = upstream_nodes
def get(self, from_path: str, to_path: str, recursive: bool = False):
if recursive:
raise user.FlyteAssertion("Reading data recursively from HTTP endpoint is not currently supported.")
rsp = requests.get(from_path)
if rsp.status_code != self._HTTP_OK:
raise user.FlyteValueException(
rsp.status_code,
"Request for data @ {} failed. Expected status code {}".format(from_path, type(self)._HTTP_OK),
)
head, _ = os.path.split(to_path)
if head and head.startswith("/"):
logger.debug(f"HttpPersistence creating {head} so that parent dirs exist")
pathlib.Path(head).mkdir(parents=True, exist_ok=True)
with open(to_path, "wb") as writer:
writer.write(rsp.content)
def test_base_scope():
with pytest.raises(ValueError) as e:
_user_func(ValueError("Bad value"))
assert "Bad value" in str(e.value)
with pytest.raises(ValueError) as e:
_system_func(ValueError("Bad value"))
assert "Bad value" in str(e.value)
with pytest.raises(user.FlyteAssertion) as e:
_user_func(user.FlyteAssertion("Bad assert"))
assert "Bad assert" in str(e.value)
with pytest.raises(system.FlyteSystemAssertion) as e:
_user_func(system.FlyteSystemAssertion("Bad assert"))
assert "Bad assert" in str(e.value)
with pytest.raises(user.FlyteAssertion) as e:
_system_func(user.FlyteAssertion("Bad assert"))
assert "Bad assert" in str(e.value)
with pytest.raises(system.FlyteSystemAssertion) as e:
_system_func(system.FlyteSystemAssertion("Bad assert"))
assert "Bad assert" in str(e.value)
def put(self, from_path: str, to_path: str, recursive: bool = False):
if recursive:
raise user.FlyteAssertion("Recursive writing data to HTTP endpoint is not currently supported.")
md5, _ = script_mode.hash_file(from_path)
encoded_md5 = base64.b64encode(md5)
with open(from_path, "+rb") as local_file:
content = local_file.read()
content_length = len(content)
rsp = requests.put(
to_path, data=content, headers={"Content-Length": str(content_length), "Content-MD5": encoded_md5}
)
if rsp.status_code != self._HTTP_OK:
raise user.FlyteValueException(
rsp.status_code,
f"Request to send data {to_path} failed.",
)
def test_intercepted_scope_flyte_user_exception():
assertion_error = user.FlyteAssertion("Bad assert")
with pytest.raises(scopes.FlyteScopedUserException) as e:
_user_func(assertion_error)
e = e.value
assert e.value == assertion_error
assert "Bad assert" in e.verbose_message
assert "User error." in e.verbose_message
assert e.error_code == "USER:AssertionError"
assert e.kind == _error_models.ContainerError.Kind.NON_RECOVERABLE
with pytest.raises(scopes.FlyteScopedUserException) as e:
_system_func(assertion_error)
e = e.value
assert e.value == assertion_error
assert "Bad assert" in e.verbose_message
assert "User error." in e.verbose_message
assert e.error_code == "USER:AssertionError"
assert e.kind == _error_models.ContainerError.Kind.NON_RECOVERABLE
def __init__(
self,
id: id_models.Identifier,
nodes: List[_nodes.FlyteNode],
interface,
output_bindings,
metadata,
metadata_defaults,
subworkflows: Optional[Dict[id_models.Identifier, _workflow_models.WorkflowTemplate]] = None,
tasks: Optional[Dict[id_models.Identifier, _task_models.TaskTemplate]] = None,
launch_plans: Optional[Dict[id_models.Identifier, launch_plan_models.LaunchPlanSpec]] = None,
compiled_closure: Optional[compiler_models.CompiledWorkflowClosure] = None,
):
# TODO: Remove check
for node in nodes:
for upstream in node.upstream_nodes:
if upstream.id is None:
raise _user_exceptions.FlyteAssertion(
"Some nodes contained in the workflow were not found in the workflow description. Please "
"ensure all nodes are either assigned to attributes within the class or an element in a "
"list, dict, or tuple which is stored as an attribute in the class."
)
super(FlyteWorkflow, self).__init__(
id=id,
metadata=metadata,
metadata_defaults=metadata_defaults,
interface=interface,
nodes=nodes,
outputs=output_bindings,
)
self._flyte_nodes = nodes
# Optional things that we save for ease of access when promoting from a model or CompiledWorkflowClosure
self._subworkflows = subworkflows
self._tasks = tasks
self._launch_plans = launch_plans
self._compiled_closure = compiled_closure
self._node_map = None
self._name = id.name
def create_node(
entity: Union[PythonTask, LaunchPlan, WorkflowBase,
RemoteEntity], *args,
**kwargs) -> Union[Node, VoidPromise, Type[collections.namedtuple]]:
"""
This is the function you want to call if you need to specify dependencies between tasks that don't consume and/or
don't produce outputs. For example, if you have t1() and t2(), both of which do not take in nor produce any
outputs, how do you specify that t2 should run before t1?
t1_node = create_node(t1)
t2_node = create_node(t2)
t2_node.runs_before(t1_node)
# OR
t2_node >> t1_node
This works for tasks that take inputs as well, say a ``t3(in1: int)``
t3_node = create_node(t3, in1=some_int) # basically calling t3(in1=some_int)
You can still use this method to handle setting certain overrides
t3_node = create_node(t3, in1=some_int).with_overrides(...)
Outputs, if there are any, will be accessible. A `t4() -> (int, str)`
t4_node = create_node(t4)
in compilation node.o0 has the promise.
t5(in1=t4_node.o0)
in local workflow execution, what is the node? Can it just be the named tuple?
t5(in1=t4_node.o0)
@workflow
def wf():
create_node(sub_wf)
create_node(wf2)
@dynamic
def sub_wf():
create_node(other_sub)
create_node(task)
If t1 produces only one output, note that in local execution, you still get a wrapper object that
needs to be dereferenced by the output name.
t1_node = create_node(t1)
t2(t1_node.o0)
"""
from flytekit.remote.remote_callable import RemoteEntity
if len(args) > 0:
raise _user_exceptions.FlyteAssertion(
f"Only keyword args are supported to pass inputs to workflows and tasks."
f"Aborting execution as detected {len(args)} positional args {args}"
)
if (not isinstance(entity, PythonTask)
and not isinstance(entity, WorkflowBase)
and not isinstance(entity, LaunchPlan)
and not isinstance(entity, RemoteEntity)):
raise AssertionError(
f"Should be a callable Flyte entity (either local or fetched) but is {type(entity)}"
)
# This function is only called from inside workflows and dynamic tasks.
# That means there are two scenarios we need to take care of, compilation and local workflow execution.
# When compiling, calling the entity will create a node.
ctx = FlyteContext.current_context()
if ctx.compilation_state is not None and ctx.compilation_state.mode == 1:
outputs = entity(**kwargs)
# This is always the output of create_and_link_node which returns create_task_output, which can be
# VoidPromise, Promise, or our custom namedtuple of Promises.
node = ctx.compilation_state.nodes[-1]
# In addition to storing the outputs on the object itself, we also want to set them in a map. When used by
# the imperative workflow patterns, users will probably find themselves doing things like
# n = create_node(...) # then
# output_name = "o0"
# n.outputs[output_name] # rather than
# n.o0
# That is, they'll likely have the name of the output stored as a string variable, and dicts provide cleaner
# access than getattr
node._outputs = {}
# If a VoidPromise, just return the node.
if isinstance(outputs, VoidPromise):
return node
# If a Promise or custom namedtuple of Promises, we need to attach each output as an attribute to the node.
# todo: fix the noqas below somehow... can't add abstract property to RemoteEntity because it has to come
# before the model Template classes in FlyteTask/Workflow/LaunchPlan
if entity.interface.outputs: # noqa
if isinstance(outputs, tuple):
for output_name in entity.interface.outputs.keys(): # noqa
attr = getattr(outputs, output_name)
if attr is None:
raise _user_exceptions.FlyteAssertion(
f"Output {output_name} in outputs when calling {entity.name} is empty {attr}."
)
if hasattr(node, output_name):
raise _user_exceptions.FlyteAssertion(
f"Node {node} already has attribute {output_name}, change the name of output."
)
setattr(node, output_name, attr)
node.outputs[output_name] = attr
else:
output_names = [k for k in entity.interface.outputs.keys()
] # noqa
if len(output_names) != 1:
raise _user_exceptions.FlyteAssertion(
f"Output of length 1 expected but {len(output_names)} found"
)
if hasattr(node, output_names[0]):
raise _user_exceptions.FlyteAssertion(
f"Node {node} already has attribute {output_names[0]}, change the name of output."
)
setattr(node, output_names[0],
outputs) # This should be a singular Promise
node.outputs[output_names[0]] = outputs
return node
# Handling local execution
elif ctx.execution_state is not None and ctx.execution_state.mode == ExecutionState.Mode.LOCAL_WORKFLOW_EXECUTION:
if isinstance(entity, RemoteEntity):
raise AssertionError(
f"Remote entities are not yet runnable locally {entity.name}")
if ctx.execution_state.branch_eval_mode == BranchEvalMode.BRANCH_SKIPPED:
logger.warning(
f"Manual node creation cannot be used in branch logic {entity.name}"
)
raise Exception(
"Being more restrictive for now and disallowing manual node creation in branch logic"
)
# This the output of __call__ under local execute conditions which means this is the output of local_execute
# which means this is the output of create_task_output with Promises containing values (or a VoidPromise)
results = entity(**kwargs)
# If it's a VoidPromise, let's just return it, it shouldn't get used anywhere and if it does, we want an error
# The reason we return it if it's a tuple is to handle the case where the task returns a typing.NamedTuple.
# In that case, it's already a tuple and we don't need to further tupletize.
if isinstance(results, VoidPromise) or isinstance(results, tuple):
return results
output_names = entity.python_interface.output_names
if not output_names:
raise Exception(
f"Non-VoidPromise received {results} but interface for {entity.name} doesn't have outputs"
)
if len(output_names) == 1:
# See explanation above for why we still tupletize a single element.
return entity.python_interface.output_tuple(results)
return entity.python_interface.output_tuple(*results)
else:
raise Exception(
f"Cannot use explicit run to call Flyte entities {entity.name}")
def construct_path(self, add_protocol: bool, add_prefix: bool, *paths) -> str:
raise user.FlyteAssertion(
"There are multiple ways of creating http links / paths, this is not supported by the persistence layer"
)