def my_wf(a: int, b: str) -> (str, str):
t1_node = create_node(
t1, a=a).with_overrides(aliases={"t1_str_output": "foo"})
t1_nt_node = create_node(t1_nt, a=a)
t2_node = create_node(
t2, a=[t1_node.t1_str_output, t1_nt_node.t1_str_output, b])
return t1_node.t1_str_output, t2_node.o0
def wrapper(*args, **kwargs):
# get the current flyte context to obtain access to the compilation state of the workflow DAG.
ctx = FlyteContextManager.current_context()
# defines before node
before_node = create_node(before)
# ctx.compilation_state.nodes == [before_node]
# under the hood, flytekit compiler defines and threads
# together nodes within the `my_workflow` function body
outputs = fn(*args, **kwargs)
# ctx.compilation_state.nodes == [before_node, *nodes_created_by_fn]
# defines the after node
after_node = create_node(after)
# ctx.compilation_state.nodes == [before_node, *nodes_created_by_fn, after_node]
# compile the workflow correctly by making sure `before_node`
# runs before the first workflow node and `after_node`
# runs after the last workflow node.
if ctx.compilation_state is not None:
# ctx.compilation_state.nodes is a list of nodes defined in the
# order of execution above
workflow_node0 = ctx.compilation_state.nodes[1]
workflow_node1 = ctx.compilation_state.nodes[-2]
before_node >> workflow_node0
workflow_node1 >> after_node
return outputs
def chain_tasks_wf() -> pd.DataFrame:
write_node = create_node(write)
read_node = create_node(read)
write_node >> read_node
return read_node.o0
def cached_dataframe_wf():
raw_data = uncached_data_reading_task()
# We execute `cached_data_processing_task` twice, but we force those
# two executions to happen serially to demonstrate how the second run
# hits the cache.
t1_node = create_node(cached_data_processing_task, df=raw_data)
t2_node = create_node(cached_data_processing_task, df=raw_data)
t1_node >> t2_node
# Confirm that the dataframes actually match
compare_dataframes(df1=t1_node.o0, df2=t2_node.o0)
def add_entity(self, entity: PythonAutoContainerTask, **kwargs) -> Node:
"""
Anytime you add an entity, all the inputs to the entity must be bound.
"""
# circular import
from flytekit.core.node_creation import create_node
ctx = FlyteContext.current_context()
if ctx.compilation_state is not None:
raise Exception("Can't already be compiling")
with FlyteContextManager.with_context(ctx.with_compilation_state(self.compilation_state)) as ctx:
n = create_node(entity=entity, **kwargs)
def get_input_values(input_value):
if isinstance(input_value, list):
input_promises = []
for x in input_value:
input_promises.extend(get_input_values(x))
return input_promises
if isinstance(input_value, dict):
input_promises = []
for _, v in input_value.items():
input_promises.extend(get_input_values(v))
return input_promises
else:
return [input_value]
# Every time an entity is added, mark it as used.
for input_value in get_input_values(kwargs):
if input_value in self._unbound_inputs:
self._unbound_inputs.remove(input_value)
return n
def add_entity(self, entity: Union[PythonTask, LaunchPlan, WorkflowBase],
**kwargs) -> Node:
"""
Anytime you add an entity, all the inputs to the entity must be bound.
"""
# circular import
from flytekit.core.node_creation import create_node
ctx = FlyteContext.current_context()
if ctx.compilation_state is not None:
raise Exception("Can't already be compiling")
with FlyteContextManager.with_context(
ctx.with_compilation_state(self.compilation_state)) as ctx:
n = create_node(entity=entity, **kwargs)
def get_input_values(input_value):
if isinstance(input_value, list):
input_promises = []
for x in input_value:
input_promises.extend(get_input_values(x))
return input_promises
if isinstance(input_value, dict):
input_promises = []
for _, v in input_value.items():
input_promises.extend(get_input_values(v))
return input_promises
else:
return [input_value]
# Every time an entity is added, mark it as used. The above function though will gather all the input
# values but we're only interested in the ones that are Promises so let's filter for those.
# There's probably a way to clean this up, maybe key off of the name instead of value?
all_input_values = get_input_values(kwargs)
for input_value in filter(lambda x: isinstance(x, Promise),
all_input_values):
if input_value in self._unbound_inputs:
self._unbound_inputs.remove(input_value)
return n
def empty_wf2():
create_node(t2, "foo")
def empty_wf2():
t2_node = create_node(t2)
t3_node = create_node(t3)
t3_node >> t2_node
def empty_wf():
t2_node = create_node(t2)
t3_node = create_node(t3)
t3_node.runs_before(t2_node)
def my_wf(a: str) -> (str, typing.List[str]):
t1_node = create_node(t1, a=a)
dyn_node = create_node(my_subwf, a=3)
return t1_node.o0, dyn_node.o0
def my_wf(a: int) -> str:
t1_node = create_node(t1, a=a)
return t1_node.outputs
def my_wf(a: str) -> str:
t1_node = create_node(t1, a=a)
return t1_node.o0
def my_wf(numbers: typing.List[int],
run_local_at_count: int) -> typing.List[int]:
t1_node = create_node(ft,
numbers=numbers,
run_local_at_count=run_local_at_count)
return t1_node.o0
def my_wf(a: typing.List[str]) -> typing.List[str]:
mappy = map_task(t1)
map_node = create_node(mappy, a=a).with_overrides(
requests=Resources(cpu="1", mem="100"),
limits=Resources(cpu="2", mem="200"))
return map_node.o0
def my_wf(a: int, b: str) -> (str, typing.List[str], int):
subwf_node = create_node(my_subwf, a=a)
t2_node = create_node(t2, a=b, b=b)
subwf_node.runs_before(t2_node)
subwf_node >> t2_node
return t2_node.o0, subwf_node.o0, subwf_node.o1
def my_wf(a: int, b: str) -> (str, str):
t1_node = create_node(t1, a=a)
t1_nt_node = create_node(t1_nt, a=a)
t2_node = create_node(
t2, a=[t1_node.t1_str_output, t1_nt_node.t1_str_output, b])
return t1_node.t1_str_output, t2_node.o0
