def test_simulate_run_group_group(self, some_sigs):
wf = Workflow()
group1 = celery_canvas.group(some_sigs[:4])
group2 = celery_canvas.group(some_sigs[4:9])
# What celery does here is actually # group() | group() -> single group
# this is rather unexpected as group 2 should wait for all tasks from
# group 1...
canvas = group1 | group2
wf.add_celery_canvas(canvas)
# use build_exec_asserts helper to rebuild asserts
# build_exec_asserts(wf)
assert wf.simulate_tick()
assert wf.running == {
'task-8': True,
'task-6': True,
'task-7': True,
'task-4': True,
'task-5': True,
'task-2': True,
'task-3': True,
'task-0': True,
'task-1': True
} # noqa
assert not wf.simulate_tick()
def test_simulate_run_group_group_fixed(self, some_sigs):
wf = Workflow()
group1 = celery_canvas.group(some_sigs[:4])
group2 = celery_canvas.group(some_sigs[4:9])
canvas = group1 | group2
wf.add_celery_canvas(canvas)
# use build_exec_asserts helper to rebuild asserts
# build_exec_asserts(wf)
assert wf.simulate_tick()
assert wf.running == {
'task-0': True,
'task-1': True,
'task-2': True,
'task-3': True
} # noqa
assert wf.simulate_tick()
assert wf.running == {
'task-4': True,
'task-5': True,
'task-6': True,
'task-7': True,
'task-8': True
} # noqa
assert not wf.simulate_tick()
def test_simulate_run_chain_group_chain(self, some_sigs):
wf = Workflow()
chain1 = celery_canvas.chain(some_sigs[:3])
group1 = celery_canvas.group(some_sigs[3:7])
chain2 = celery_canvas.chain(some_sigs[7:9])
canvas = chain1 | group1 | chain2
wf.add_celery_canvas(canvas)
# use build_exec_asserts helper to rebuild asserts
# build_exec_asserts(wf)
assert wf.simulate_tick()
assert wf.running == {'task-0': True} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-1': True} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-2': True} # noqa
assert wf.simulate_tick()
assert wf.running == {
'task-6': True,
'task-4': True,
'task-5': True,
'task-3': True
} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-7': True} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-8': True} # noqa
assert not wf.simulate_tick()
def get_canvas_order(self, canvas):
workflow = Workflow()
workflow.add_celery_canvas(canvas)
result = workflow.apply_async()
result.wait()
logs_result = read_logs.delay()
return self.parse_logs(logs_result.get())
def test_return_value(self, test_name):
canvas = return_value_task.s(test_name)
workflow = Workflow()
workflow.add_celery_canvas(canvas)
workflow.apply_async().wait()
task_id = list(workflow.nodes.keys())[0]
logs_result = read_logs.delay()
results = self.parse_logs(logs_result.get())[test_name]
assert task_id == results[0]
def test_add_chain_of_chords(self, some_sigs):
wf = Workflow()
sigs = some_sigs
c1 = celery_canvas.chord(sigs[:4], sigs[4])
c2 = celery_canvas.chord(sigs[5:9], sigs[9])
canvas = celery_canvas.chain([c1, c2])
res = wf.add_celery_canvas(canvas)
assert res == [wf.nodes[sigs[9].id]]
# use build_dependency_asserts(wf) to regenerate
# build_dependency_asserts(wf)
assert wf.nodes["task-0"].dependencies == {} # noqa
assert wf.nodes["task-1"].dependencies == {} # noqa
assert wf.nodes["task-2"].dependencies == {} # noqa
assert wf.nodes["task-3"].dependencies == {} # noqa
assert "task-0" in wf.nodes["task-4"].dependencies # noqa
assert "task-1" in wf.nodes["task-4"].dependencies # noqa
assert "task-2" in wf.nodes["task-4"].dependencies # noqa
assert "task-3" in wf.nodes["task-4"].dependencies # noqa
assert "task-4" in wf.nodes["task-5"].dependencies # noqa
assert "task-4" in wf.nodes["task-6"].dependencies # noqa
assert "task-4" in wf.nodes["task-7"].dependencies # noqa
assert "task-4" in wf.nodes["task-8"].dependencies # noqa
assert "task-5" in wf.nodes["task-9"].dependencies # noqa
assert "task-6" in wf.nodes["task-9"].dependencies # noqa
assert "task-7" in wf.nodes["task-9"].dependencies # noqa
assert "task-8" in wf.nodes["task-9"].dependencies # noqa
def test_to_from_dict(self, some_sigs):
wf = Workflow()
chain1 = celery_canvas.chain(some_sigs[:3])
wf.add_celery_canvas(chain1)
wf_dict = wf.to_dict()
wf2 = Workflow.from_dict(wf_dict)
assert wf.nodes.keys() == wf2.nodes.keys()
# use build_exec_asserts helper to rebuild asserts
# build_exec_asserts(wf)
assert wf.simulate_tick()
assert wf.running == {'task-0': True} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-1': True} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-2': True} # noqa
assert not wf.simulate_tick()
def test_add_empty_chain(self, some_sigs):
"""
task -> chain([]) -> task
Verify if dependencies are built correctly
"""
wf = Workflow()
c0 = some_sigs[0]
empty_chain = celery_canvas.chain([])
c1 = some_sigs[1]
canvas = c0 | empty_chain | c1
wf.add_celery_canvas(canvas)
# use build_dependency_asserts(wf) to regenerate
build_dependency_asserts(wf)
assert wf.nodes["task-0"].dependencies == {} # noqa
assert "task-0" in wf.nodes["task-1"].dependencies # noqa
def test_add_complex(self, some_sigs):
wf = Workflow()
c0 = some_sigs[0]
c1 = some_sigs[5]
chain1 = celery_canvas.chain(some_sigs[1:3])
chain2 = celery_canvas.chain(some_sigs[3:5])
chord = celery_canvas.chord([chain1, chain2], c1)
canvas = c0 | chord
wf.add_celery_canvas(canvas)
# use build_dependency_asserts(wf) to regenerate
# build_dependency_asserts(wf)
assert wf.nodes["task-0"].dependencies == {} # noqa
assert "task-0" in wf.nodes["task-1"].dependencies # noqa
assert "task-1" in wf.nodes["task-2"].dependencies # noqa
assert "task-0" in wf.nodes["task-3"].dependencies # noqa
assert "task-3" in wf.nodes["task-4"].dependencies # noqa
assert "task-2" in wf.nodes["task-5"].dependencies # noqa
assert "task-4" in wf.nodes["task-5"].dependencies # noqa
def test_simulate_run_chain_chord_chain(self, some_sigs):
wf = Workflow()
c0 = some_sigs[0]
c1 = some_sigs[5]
chain1 = celery_canvas.chain(some_sigs[1:3])
chain2 = celery_canvas.chain(some_sigs[3:5])
chord = celery_canvas.chord([chain1, chain2], c1)
canvas = c0 | chord
wf.add_celery_canvas(canvas)
# use build_exec_asserts helper to rebuild asserts
# build_exec_asserts(wf)
assert wf.simulate_tick()
assert wf.running == {'task-0': True} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-3': True, 'task-1': True} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-4': True, 'task-2': True} # noqa
assert wf.simulate_tick()
assert wf.running == {'task-5': True} # noqa
assert not wf.simulate_tick()
def test_add_empty_group(self, some_sigs):
"""
task -> group([])
Verify if dependencies are built correctly
"""
wf = Workflow()
c0, c1 = some_sigs[0:2]
empty_group = celery_canvas.group([])
empty_group2 = celery_canvas.group([])
# celery optimizes group | task into chord
# so lets use group | group to verify if dependencies
# are propagated properly
canvas = c0 | empty_group | empty_group2 | c1
wf.add_celery_canvas(canvas)
# use build_dependency_asserts(wf) to regenerate
# build_dependency_asserts(wf)
assert wf.nodes["task-0"].dependencies == {} # noqa
assert "task-0" in wf.nodes["task-1"].dependencies # noqa
