def my_wf(a: int, b: str) -> (str, str): t1_node = create_node( t1, a=a).with_overrides(aliases={"t1_str_output": "foo"}) t1_nt_node = create_node(t1_nt, a=a) t2_node = create_node( t2, a=[t1_node.t1_str_output, t1_nt_node.t1_str_output, b]) return t1_node.t1_str_output, t2_node.o0
def wrapper(*args, **kwargs): # get the current flyte context to obtain access to the compilation state of the workflow DAG. ctx = FlyteContextManager.current_context() # defines before node before_node = create_node(before) # ctx.compilation_state.nodes == [before_node] # under the hood, flytekit compiler defines and threads # together nodes within the `my_workflow` function body outputs = fn(*args, **kwargs) # ctx.compilation_state.nodes == [before_node, *nodes_created_by_fn] # defines the after node after_node = create_node(after) # ctx.compilation_state.nodes == [before_node, *nodes_created_by_fn, after_node] # compile the workflow correctly by making sure `before_node` # runs before the first workflow node and `after_node` # runs after the last workflow node. if ctx.compilation_state is not None: # ctx.compilation_state.nodes is a list of nodes defined in the # order of execution above workflow_node0 = ctx.compilation_state.nodes[1] workflow_node1 = ctx.compilation_state.nodes[-2] before_node >> workflow_node0 workflow_node1 >> after_node return outputs
def chain_tasks_wf() -> pd.DataFrame: write_node = create_node(write) read_node = create_node(read) write_node >> read_node return read_node.o0
def cached_dataframe_wf(): raw_data = uncached_data_reading_task() # We execute `cached_data_processing_task` twice, but we force those # two executions to happen serially to demonstrate how the second run # hits the cache. t1_node = create_node(cached_data_processing_task, df=raw_data) t2_node = create_node(cached_data_processing_task, df=raw_data) t1_node >> t2_node # Confirm that the dataframes actually match compare_dataframes(df1=t1_node.o0, df2=t2_node.o0)
def add_entity(self, entity: PythonAutoContainerTask, **kwargs) -> Node: """ Anytime you add an entity, all the inputs to the entity must be bound. """ # circular import from flytekit.core.node_creation import create_node ctx = FlyteContext.current_context() if ctx.compilation_state is not None: raise Exception("Can't already be compiling") with FlyteContextManager.with_context(ctx.with_compilation_state(self.compilation_state)) as ctx: n = create_node(entity=entity, **kwargs) def get_input_values(input_value): if isinstance(input_value, list): input_promises = [] for x in input_value: input_promises.extend(get_input_values(x)) return input_promises if isinstance(input_value, dict): input_promises = [] for _, v in input_value.items(): input_promises.extend(get_input_values(v)) return input_promises else: return [input_value] # Every time an entity is added, mark it as used. for input_value in get_input_values(kwargs): if input_value in self._unbound_inputs: self._unbound_inputs.remove(input_value) return n
def add_entity(self, entity: Union[PythonTask, LaunchPlan, WorkflowBase], **kwargs) -> Node: """ Anytime you add an entity, all the inputs to the entity must be bound. """ # circular import from flytekit.core.node_creation import create_node ctx = FlyteContext.current_context() if ctx.compilation_state is not None: raise Exception("Can't already be compiling") with FlyteContextManager.with_context( ctx.with_compilation_state(self.compilation_state)) as ctx: n = create_node(entity=entity, **kwargs) def get_input_values(input_value): if isinstance(input_value, list): input_promises = [] for x in input_value: input_promises.extend(get_input_values(x)) return input_promises if isinstance(input_value, dict): input_promises = [] for _, v in input_value.items(): input_promises.extend(get_input_values(v)) return input_promises else: return [input_value] # Every time an entity is added, mark it as used. The above function though will gather all the input # values but we're only interested in the ones that are Promises so let's filter for those. # There's probably a way to clean this up, maybe key off of the name instead of value? all_input_values = get_input_values(kwargs) for input_value in filter(lambda x: isinstance(x, Promise), all_input_values): if input_value in self._unbound_inputs: self._unbound_inputs.remove(input_value) return n
def empty_wf2(): create_node(t2, "foo")
def empty_wf2(): t2_node = create_node(t2) t3_node = create_node(t3) t3_node >> t2_node
def empty_wf(): t2_node = create_node(t2) t3_node = create_node(t3) t3_node.runs_before(t2_node)
def my_wf(a: str) -> (str, typing.List[str]): t1_node = create_node(t1, a=a) dyn_node = create_node(my_subwf, a=3) return t1_node.o0, dyn_node.o0
def my_wf(a: int) -> str: t1_node = create_node(t1, a=a) return t1_node.outputs
def my_wf(a: str) -> str: t1_node = create_node(t1, a=a) return t1_node.o0
def my_wf(numbers: typing.List[int], run_local_at_count: int) -> typing.List[int]: t1_node = create_node(ft, numbers=numbers, run_local_at_count=run_local_at_count) return t1_node.o0
def my_wf(a: typing.List[str]) -> typing.List[str]: mappy = map_task(t1) map_node = create_node(mappy, a=a).with_overrides( requests=Resources(cpu="1", mem="100"), limits=Resources(cpu="2", mem="200")) return map_node.o0
def my_wf(a: int, b: str) -> (str, typing.List[str], int): subwf_node = create_node(my_subwf, a=a) t2_node = create_node(t2, a=b, b=b) subwf_node.runs_before(t2_node) subwf_node >> t2_node return t2_node.o0, subwf_node.o0, subwf_node.o1
def my_wf(a: int, b: str) -> (str, str): t1_node = create_node(t1, a=a) t1_nt_node = create_node(t1_nt, a=a) t2_node = create_node( t2, a=[t1_node.t1_str_output, t1_nt_node.t1_str_output, b]) return t1_node.t1_str_output, t2_node.o0