def test_simulate_run_group_group(self, some_sigs): wf = Workflow() group1 = celery_canvas.group(some_sigs[:4]) group2 = celery_canvas.group(some_sigs[4:9]) # What celery does here is actually # group() | group() -> single group # this is rather unexpected as group 2 should wait for all tasks from # group 1... canvas = group1 | group2 wf.add_celery_canvas(canvas) # use build_exec_asserts helper to rebuild asserts # build_exec_asserts(wf) assert wf.simulate_tick() assert wf.running == { 'task-8': True, 'task-6': True, 'task-7': True, 'task-4': True, 'task-5': True, 'task-2': True, 'task-3': True, 'task-0': True, 'task-1': True } # noqa assert not wf.simulate_tick()
def test_simulate_run_group_group_fixed(self, some_sigs): wf = Workflow() group1 = celery_canvas.group(some_sigs[:4]) group2 = celery_canvas.group(some_sigs[4:9]) canvas = group1 | group2 wf.add_celery_canvas(canvas) # use build_exec_asserts helper to rebuild asserts # build_exec_asserts(wf) assert wf.simulate_tick() assert wf.running == { 'task-0': True, 'task-1': True, 'task-2': True, 'task-3': True } # noqa assert wf.simulate_tick() assert wf.running == { 'task-4': True, 'task-5': True, 'task-6': True, 'task-7': True, 'task-8': True } # noqa assert not wf.simulate_tick()
def test_simulate_run_chain_group_chain(self, some_sigs): wf = Workflow() chain1 = celery_canvas.chain(some_sigs[:3]) group1 = celery_canvas.group(some_sigs[3:7]) chain2 = celery_canvas.chain(some_sigs[7:9]) canvas = chain1 | group1 | chain2 wf.add_celery_canvas(canvas) # use build_exec_asserts helper to rebuild asserts # build_exec_asserts(wf) assert wf.simulate_tick() assert wf.running == {'task-0': True} # noqa assert wf.simulate_tick() assert wf.running == {'task-1': True} # noqa assert wf.simulate_tick() assert wf.running == {'task-2': True} # noqa assert wf.simulate_tick() assert wf.running == { 'task-6': True, 'task-4': True, 'task-5': True, 'task-3': True } # noqa assert wf.simulate_tick() assert wf.running == {'task-7': True} # noqa assert wf.simulate_tick() assert wf.running == {'task-8': True} # noqa assert not wf.simulate_tick()
def get_canvas_order(self, canvas): workflow = Workflow() workflow.add_celery_canvas(canvas) result = workflow.apply_async() result.wait() logs_result = read_logs.delay() return self.parse_logs(logs_result.get())
def test_return_value(self, test_name): canvas = return_value_task.s(test_name) workflow = Workflow() workflow.add_celery_canvas(canvas) workflow.apply_async().wait() task_id = list(workflow.nodes.keys())[0] logs_result = read_logs.delay() results = self.parse_logs(logs_result.get())[test_name] assert task_id == results[0]
def test_add_chain_of_chords(self, some_sigs): wf = Workflow() sigs = some_sigs c1 = celery_canvas.chord(sigs[:4], sigs[4]) c2 = celery_canvas.chord(sigs[5:9], sigs[9]) canvas = celery_canvas.chain([c1, c2]) res = wf.add_celery_canvas(canvas) assert res == [wf.nodes[sigs[9].id]] # use build_dependency_asserts(wf) to regenerate # build_dependency_asserts(wf) assert wf.nodes["task-0"].dependencies == {} # noqa assert wf.nodes["task-1"].dependencies == {} # noqa assert wf.nodes["task-2"].dependencies == {} # noqa assert wf.nodes["task-3"].dependencies == {} # noqa assert "task-0" in wf.nodes["task-4"].dependencies # noqa assert "task-1" in wf.nodes["task-4"].dependencies # noqa assert "task-2" in wf.nodes["task-4"].dependencies # noqa assert "task-3" in wf.nodes["task-4"].dependencies # noqa assert "task-4" in wf.nodes["task-5"].dependencies # noqa assert "task-4" in wf.nodes["task-6"].dependencies # noqa assert "task-4" in wf.nodes["task-7"].dependencies # noqa assert "task-4" in wf.nodes["task-8"].dependencies # noqa assert "task-5" in wf.nodes["task-9"].dependencies # noqa assert "task-6" in wf.nodes["task-9"].dependencies # noqa assert "task-7" in wf.nodes["task-9"].dependencies # noqa assert "task-8" in wf.nodes["task-9"].dependencies # noqa
def test_to_from_dict(self, some_sigs): wf = Workflow() chain1 = celery_canvas.chain(some_sigs[:3]) wf.add_celery_canvas(chain1) wf_dict = wf.to_dict() wf2 = Workflow.from_dict(wf_dict) assert wf.nodes.keys() == wf2.nodes.keys() # use build_exec_asserts helper to rebuild asserts # build_exec_asserts(wf) assert wf.simulate_tick() assert wf.running == {'task-0': True} # noqa assert wf.simulate_tick() assert wf.running == {'task-1': True} # noqa assert wf.simulate_tick() assert wf.running == {'task-2': True} # noqa assert not wf.simulate_tick()
def test_add_empty_chain(self, some_sigs): """ task -> chain([]) -> task Verify if dependencies are built correctly """ wf = Workflow() c0 = some_sigs[0] empty_chain = celery_canvas.chain([]) c1 = some_sigs[1] canvas = c0 | empty_chain | c1 wf.add_celery_canvas(canvas) # use build_dependency_asserts(wf) to regenerate build_dependency_asserts(wf) assert wf.nodes["task-0"].dependencies == {} # noqa assert "task-0" in wf.nodes["task-1"].dependencies # noqa
def test_add_complex(self, some_sigs): wf = Workflow() c0 = some_sigs[0] c1 = some_sigs[5] chain1 = celery_canvas.chain(some_sigs[1:3]) chain2 = celery_canvas.chain(some_sigs[3:5]) chord = celery_canvas.chord([chain1, chain2], c1) canvas = c0 | chord wf.add_celery_canvas(canvas) # use build_dependency_asserts(wf) to regenerate # build_dependency_asserts(wf) assert wf.nodes["task-0"].dependencies == {} # noqa assert "task-0" in wf.nodes["task-1"].dependencies # noqa assert "task-1" in wf.nodes["task-2"].dependencies # noqa assert "task-0" in wf.nodes["task-3"].dependencies # noqa assert "task-3" in wf.nodes["task-4"].dependencies # noqa assert "task-2" in wf.nodes["task-5"].dependencies # noqa assert "task-4" in wf.nodes["task-5"].dependencies # noqa
def test_simulate_run_chain_chord_chain(self, some_sigs): wf = Workflow() c0 = some_sigs[0] c1 = some_sigs[5] chain1 = celery_canvas.chain(some_sigs[1:3]) chain2 = celery_canvas.chain(some_sigs[3:5]) chord = celery_canvas.chord([chain1, chain2], c1) canvas = c0 | chord wf.add_celery_canvas(canvas) # use build_exec_asserts helper to rebuild asserts # build_exec_asserts(wf) assert wf.simulate_tick() assert wf.running == {'task-0': True} # noqa assert wf.simulate_tick() assert wf.running == {'task-3': True, 'task-1': True} # noqa assert wf.simulate_tick() assert wf.running == {'task-4': True, 'task-2': True} # noqa assert wf.simulate_tick() assert wf.running == {'task-5': True} # noqa assert not wf.simulate_tick()
def test_add_empty_group(self, some_sigs): """ task -> group([]) Verify if dependencies are built correctly """ wf = Workflow() c0, c1 = some_sigs[0:2] empty_group = celery_canvas.group([]) empty_group2 = celery_canvas.group([]) # celery optimizes group | task into chord # so lets use group | group to verify if dependencies # are propagated properly canvas = c0 | empty_group | empty_group2 | c1 wf.add_celery_canvas(canvas) # use build_dependency_asserts(wf) to regenerate # build_dependency_asserts(wf) assert wf.nodes["task-0"].dependencies == {} # noqa assert "task-0" in wf.nodes["task-1"].dependencies # noqa