def test_two_dependent_tasks(): """ x1 -> x2 Two identical tasks in a row """ f = Flow(name="test") f.add_edge(get_task("x1"), get_task("x2")) steps = f.generate_local_task_ids(_debug_steps=True) # step 1 isn't enough to differentiate the tasks assert count_unique_ids(steps[1]) == 1 # step 2 is able to differentiate them assert count_unique_ids(steps[2]) == 2 # no further processing assert steps[2] == steps[3] == steps[4] == steps[5]
def flow_from_chains(*chains): """ Builds a Flow from chains of task names. To build a flow that runs x, then y, then z, and also runs x2 after x: flow_from_chains( ['x', 'y', 'z'], ['x', 'x2'] ) The tasks in the returned flow are all completely identical. """ flow = Flow(name="test") for chain in chains: for name in chain: flow.add_task(get_task(name)) for u_name, d_name in zip(chain, chain[1:]): flow.add_edge(get_task(u_name), get_task(d_name), validate=False) return flow
def run(self, x): return x + 10 class Load(Task): def run(self, x): print(x) flow = Flow("testing-example") e = Extract() t = Transform() l = Load() flow.add_edge(upstream_task=e, downstream_task=t, key="x") flow.add_edge(upstream_task=t, downstream_task=l, key="x") state = flow.run() # Testing state assert state.is_successful assert state.result[e].is_successful assert state.result[t].is_successful assert state.result[l].is_successful # Testing results assert state.result[e].result == 10 assert state.result[t].result == 20
MAX_WIDTH = 11 PROB_EDGE = 0.1 VERSION = 6 random.seed(SEED) flow = Flow(f"{SEED} Seed Flow {VERSION}") LEVELS = dict() for level in range(N_LEVELS): width = random.randint(MIN_WIDTH, MAX_WIDTH) LEVELS[level] = [ Task(name=f"Task {level}-{i}") for i, _ in enumerate(range(width)) ] for task in LEVELS[level]: flow.add_task(task) if level: for a, b in itertools.product(LEVELS[level - 1], LEVELS[level]): if random.random() > PROB_EDGE: flow.add_edge(a, b) # flow.storage = Docker( # base_image="python:3.8", # python_dependencies=[], # registry_url="znicholasbrown", # image_name=f"random_seed-{VERSION}", # image_tag=f"random-seed-flow-{VERSION}", # ) flow.register(project_name="Community Support Flows")