def test_two_dependent_tasks():
"""
x1 -> x2
Two identical tasks in a row
"""
f = Flow(name="test")
f.add_edge(get_task("x1"), get_task("x2"))
steps = f.generate_local_task_ids(_debug_steps=True)
# step 1 isn't enough to differentiate the tasks
assert count_unique_ids(steps[1]) == 1
# step 2 is able to differentiate them
assert count_unique_ids(steps[2]) == 2
# no further processing
assert steps[2] == steps[3] == steps[4] == steps[5]
def flow_from_chains(*chains):
"""
Builds a Flow from chains of task names.
To build a flow that runs x, then y, then z, and also runs x2 after x:
flow_from_chains(
['x', 'y', 'z'],
['x', 'x2']
)
The tasks in the returned flow are all completely identical.
"""
flow = Flow(name="test")
for chain in chains:
for name in chain:
flow.add_task(get_task(name))
for u_name, d_name in zip(chain, chain[1:]):
flow.add_edge(get_task(u_name), get_task(d_name), validate=False)
return flow
def run(self, x):
return x + 10
class Load(Task):
def run(self, x):
print(x)
flow = Flow("testing-example")
e = Extract()
t = Transform()
l = Load()
flow.add_edge(upstream_task=e, downstream_task=t, key="x")
flow.add_edge(upstream_task=t, downstream_task=l, key="x")
state = flow.run()
# Testing state
assert state.is_successful
assert state.result[e].is_successful
assert state.result[t].is_successful
assert state.result[l].is_successful
# Testing results
assert state.result[e].result == 10
assert state.result[t].result == 20
MAX_WIDTH = 11
PROB_EDGE = 0.1
VERSION = 6
random.seed(SEED)
flow = Flow(f"{SEED} Seed Flow {VERSION}")
LEVELS = dict()
for level in range(N_LEVELS):
width = random.randint(MIN_WIDTH, MAX_WIDTH)
LEVELS[level] = [
Task(name=f"Task {level}-{i}") for i, _ in enumerate(range(width))
]
for task in LEVELS[level]:
flow.add_task(task)
if level:
for a, b in itertools.product(LEVELS[level - 1], LEVELS[level]):
if random.random() > PROB_EDGE:
flow.add_edge(a, b)
# flow.storage = Docker(
# base_image="python:3.8",
# python_dependencies=[],
# registry_url="znicholasbrown",
# image_name=f"random_seed-{VERSION}",
# image_tag=f"random-seed-flow-{VERSION}",
# )
flow.register(project_name="Community Support Flows")
