def test_max_parallel(self): future = Group( (running_task, "test1"), (running_task, "test2"), (running_task, "test3"), max_parallel=2 ).submit(executor) self.assertTrue(future.running) self.assertEquals(len(future.futures), 2) future = Group( (to_string, "test1"), (running_task, "test2"), (running_task, "test3"), (running_task, "test4"), max_parallel=2 ).submit(executor) self.assertTrue(future.running) self.assertEquals(len(future.futures), 3) self.assertEquals([f.state for f in future.futures], [futures.FINISHED, futures.RUNNING, futures.RUNNING]) future = Group( (to_string, "test1"), (to_string, "test2"), (to_string, "test3"), max_parallel=2 ).submit(executor) self.assertTrue(future.finished)
def run(self): future = self.submit( Step('my_step', Group( (multiply, 1), (multiply, 2), (multiply, 3), ))) futures.wait(future) # You can force the step even if already executed group = Group((multiply, 1), (multiply, 2), (multiply, 3)) step = Step('my_step_force', group, force=True) futures.wait(self.submit(step)) # You can play another activity group in the step was already computed group_done = Group(self.signal('DONE')) step = Step('my_step_with_callback_done', group, activities_if_step_already_done=group_done) futures.wait(self.submit(step)) # You can emit a signal with the identifier step.{step_name} # after the step is executed (cached or not) step = Step('my_step_with_signal', group, emit_signal=True) futures.wait(self.submit(step))
def run(self, *args, **kwargs): print(colorize(GREEN, "NotFailingWorkflow args: {}".format(args))) print(colorize(GREEN, "NotFailingWorkflow kwargs: {}".format(kwargs))) g = Group(raises_on_failure=False) g.append(FailingWorkflow) g.append(timeout_no_raise) f = self.submit(g) futures.wait(f)
def test(self): future = Group((to_string, 1), (to_string, 2)).submit(executor) self.assertTrue(future.finished) future = Group((to_string, "test1"), (running_task, "test2"), (sum_values, [1, 2])).submit(executor) self.assertTrue(future.running) self.assertEqual(future.count_finished_activities, 2) self.assertEqual(future._result, ["test1", None, 3]) with self.assertRaises(exceptions.ExecutionBlocked): future.result
def test_propagate_attribute(self): """ Test that attribute 'raises_on_failure' is well propagated through Group. """ inner_a = Group( (running_task, "test1"), (running_task, "test2"), ) inner_b = ActivityTask(running_task, "test3") Group(inner_a, inner_b, raises_on_failure=False).submit(executor) self.assertFalse(inner_b.activity.raises_on_failure) self.assertFalse(inner_a.activities[0].activity.raises_on_failure) self.assertFalse(inner_a.activities[1].activity.raises_on_failure)
def run(self): chain1 = Chain( Group(ActivityTask(func_a_1_1), ActivityTask(func_a_1_2)), self.signal("signal1"), Group(ActivityTask(func_a_2_1), ActivityTask(func_a_2_2)), ) chain2 = Chain( Group(ActivityTask(func_b_1_1), ActivityTask(func_b_1_2)), self.wait_signal("signal1"), Group(ActivityTask(func_b_2_1), ActivityTask(func_b_2_2)), ) my_group = Group(chain1, chain2) fut = self.submit(my_group) futures.wait(fut)
def test_exceptions(self): future = Group((to_string, 1), (to_string, 2)).submit(executor) self.assertIsNone(future.exception) future = Group( (zero_division), (zero_division), ).submit(executor) self.assertTrue(future.finished) self.assertIsInstance(future.exception, AggregateException) self.assertEqual(2, len(future.exception.exceptions)) self.assertIsInstance(future.exception.exceptions[0], ZeroDivisionError) self.assertIsInstance(future.exception.exceptions[1], ZeroDivisionError)
def test(self): complex_canvas = Chain((sum_values, [1, 2]), (sum_values, [1, 2]), Group( (to_int, 1), (to_int, 2), ), Chain( (sum_values, [1, 2]), running_task, ), (sum_values, [1, 2])) result = complex_canvas.submit(executor) self.assertFalse(result.finished) self.assertTrue(result.futures[0].finished) self.assertTrue(result.futures[1].finished) self.assertTrue(result.futures[2].finished) self.assertFalse(result.futures[3].finished) self.assertTrue(result.futures[3].futures[0].finished) self.assertFalse(result.futures[3].futures[1].finished) # As result.futures[3] is not finished, we shouldn't find other future self.assertEqual(len(result.futures), 4) # Change the state of the n-1 chain to make the whole # canvas done complex_canvas.activities[3].activities[1] = ActivityTask(to_int, 1) result = complex_canvas.submit(executor) self.assertTrue(result.finished) self.assertEqual(len(result.futures), 5)
def test_simplified_declaration(self): future = Group((to_string, 1), (to_string, 2)).submit(executor) self.assertTrue(future.finished) group = Group() group += [ (to_string, "test1"), running_task, (sum_values, [1, 2]), ] future = group.submit(executor) self.assertTrue(future.running) self.assertEqual(future.count_finished_activities, 2) self.assertEqual(future._result, ["test1", None, 3]) with self.assertRaises(exceptions.ExecutionBlocked): future.result
def run(self): x = 1 y = 2 z = 3 future = self.submit( Chain( Group( (increment_slowly, x), (increment_slowly, y), (increment_slowly, z), ), multiply, send_result=True ) ) futures.wait(future) res = future.result[-1] print('({}+1)*({}+1)*({}+1) = {}'.format(x, y, z, res)) # Canvas's and Group's can also be "optional" future = self.submit( Chain( (fail_incrementing, x), (increment_slowly, 1), # never executed raises_on_failure=False, ) ) futures.wait(future) print('SUCCESS!')
def run(self): return self.submit( Chain( Group( self.wait_signal('signal'), self.wait_signal('signal 2'), ), )).result
def test_run_context_for_child_workflows(self): """ Test that the local executor adds a different run_id for each child workflows. """ class ChildWorkflow3(Workflow): name = "ChildWorkflow3" @classmethod def get_workflow_id(*args, **kwargs): return kwargs["name"] def run(self, name, *args, **kwargs): return { "run_id": self.get_run_context()["run_id"], "workflow_id": self.get_run_context()["workflow_id"], } class ChildWorkflow2(Workflow): name = "ChildWorkflow2" def run(self, *args, **kwargs): return { "run_id": self.get_run_context()["run_id"], "workflow_id": self.get_run_context()["workflow_id"], } class ChildWorkflow1(Workflow): name = "ChildWorkflow" def run(self, *args, **kwargs): results = (Chain(ChildWorkflow2, (to_int, "2")).submit(self.executor).result) return { "workflow_id": self.get_run_context()["workflow_id"], "run_id": self.get_run_context()["run_id"], "child": results, } result = (Group( ChildWorkflow1, ChildWorkflow1, WorkflowTask(None, ChildWorkflow3, name="test_workflow_id"), ).submit(executor).result) child1_1, child1_2, child3 = result self.assertTrue( child1_1["workflow_id"].startswith("local_childworkflow")) self.assertTrue( child1_2["workflow_id"].startswith("local_childworkflow")) self.assertNotEqual(child1_1["run_id"], child1_2["run_id"]) child2_1 = child1_1["child"][0] child2_2 = child1_2["child"][0] self.assertTrue( child2_1["workflow_id"].startswith("local_childworkflow2")) self.assertTrue( child2_2["workflow_id"].startswith("local_childworkflow2")) self.assertNotEqual(child2_1["run_id"], child2_2["run_id"]) self.assertEqual(child3["workflow_id"], "test_workflow_id") self.assertEqual(child1_1["workflow_id"], "local_childworkflow")
def run(self): return self.submit( Chain( Group( ChildWorkflowWaitingSignals, self.signal("signal", propagate=True), self.signal("signal 2", propagate=True), ), ) ).result
def test_simplified_declaration(self): future = Group( (to_string, 1), (to_string, 2) ).submit(executor) self.assertTrue(future.finished) group = Group() group += [ (to_string, "test1"), running_task, (sum_values, [1, 2]), ] future = group.submit(executor) self.assertTrue(future.running) self.assertEqual(future.count_finished_activities, 2) self.assertEqual(future._result, ["test1", None, 3]) with self.assertRaises(exceptions.ExecutionBlocked): future.result
def run(self, *args, **kwargs): future = self.submit( Chain( Group( (self.wait_signal("signal 2"), ), (self.wait_signal("signal"), ), (wait_and_signal, ), (wait_and_signal, "signal 2"), ), (increment, 1), )) futures.wait(future)
def run(self, t1=30, t2=120): """ Cancel timer 2 after timer 1 is fired. """ future = self.submit( Group( self.start_timer("timer 2", t2), Chain(self.start_timer("timer 1", t1), self.cancel_timer("timer 2"),), ) ) if future.pending: print("Starting timers") futures.wait(future) print("Timer fired, exiting")
def run(self): x = 1 y = 2 z = 3 future = self.submit( Chain(Group( ActivityTask(increment_slowly, x), ActivityTask(increment_slowly, y), ActivityTask(increment_slowly, z), ), ActivityTask(multiply), send_result=True)) futures.wait(future) res = future.result[-1] print('({}+1)*({}+1)*({}+1) = {}'.format(x, y, z, res))
def test_group_with_workflow(self): """ Test that it is possible to provide a WorkflowTask to a Group(). """ class ChildWorkflowGroupWithWorkflow(Workflow): name = "ChildWorkflow" def run(self, str1, *args, **kwargs): return {"str1": str1, "kwargs": kwargs} future = Group( ChildWorkflowTask(ChildWorkflowGroupWithWorkflow, str1="str1", **{"hello": "world"})).submit(executor) self.assertTrue(future.finished) self.assertEqual(future.result, [{ "str1": "str1", "kwargs": { "hello": "world" } }])
def run(self): x = 1 y = 2 z = 3 future = self.submit( Chain(Group( (increment_slowly, x), (increment_slowly, y), (increment_slowly, z), ), multiply, send_result=True)) futures.wait(future) res = future.result[-1] print('({}+1)*({}+1)*({}+1) = {}'.format(x, y, z, res)) # Canvas's and Group's can also be "optional" future = self.submit( Chain( (fail_incrementing, x), (increment_slowly, 1), # never executed (multiply, [3, 2]), raises_on_failure=False, )) assert [None] == future.result, 'Unexpected result {!r}'.format( future.result) print('Chain with failure: {}'.format(future.result)) # Breaking the chain on failure is the default but can be bypassed future = self.submit( Chain( (fail_incrementing, x), (increment_slowly, 1), # executed (multiply, [3, 2]), break_on_failure=False, )) assert [None, 2, 6] == future.result, 'Unexpected result {!r}'.format( future.result) print('Chain ignoring failure: {}'.format(future.result)) # Failing inside a chain by default don't stop an upper chain future = self.submit( Chain( Chain( (fail_incrementing, x), raises_on_failure=False, ), (increment_slowly, 1), # executed (multiply, [3, 2]), )) assert [[None], 2, 6] == future.result, 'Unexpected result {!r}'.format( future.result) print('Chain with failure in subchain: {}'.format(future.result)) # But it can, too future = self.submit( Chain( Chain( Chain( (fail_incrementing, x), raises_on_failure=False, bubbles_exception_on_failure=True, ), (increment_slowly, 1), # not executed bubbles_exception_on_failure=False, ), (multiply, [3, 2]), # executed )) assert [[[None]], 6] == future.result, 'Unexpected result {!r}'.format( future.result) print('Chain with failure in sub-subchain: {}'.format(future.result)) print('Finished!')
def run(self): g = Group() g.append(ChainTestWorkflow, 4) future = self.submit(g) return future.result
def custom_func(previous_value): group = Group() for i in range(0, previous_value): group.append(to_int, i * 2) return group
def custom_func(): group = Group() for i in range(0, 2): group.append(zero_division) return group
def custom_func(previous_value): group = Group() for i in xrange(0, previous_value): group.append(ActivityTask(to_int, i * 2)) return group
def run(self, *args, **kwargs): return self.submit( Group(ChildWorkflowWaitingSignals, ChildWorkflowSendingSignals,) ).result