def test_max_parallel(self):
future = Group(
(running_task, "test1"),
(running_task, "test2"),
(running_task, "test3"),
max_parallel=2
).submit(executor)
self.assertTrue(future.running)
self.assertEquals(len(future.futures), 2)
future = Group(
(to_string, "test1"),
(running_task, "test2"),
(running_task, "test3"),
(running_task, "test4"),
max_parallel=2
).submit(executor)
self.assertTrue(future.running)
self.assertEquals(len(future.futures), 3)
self.assertEquals([f.state for f in future.futures],
[futures.FINISHED, futures.RUNNING, futures.RUNNING])
future = Group(
(to_string, "test1"),
(to_string, "test2"),
(to_string, "test3"),
max_parallel=2
).submit(executor)
self.assertTrue(future.finished)
def run(self):
future = self.submit(
Step('my_step', Group(
(multiply, 1),
(multiply, 2),
(multiply, 3),
)))
futures.wait(future)
# You can force the step even if already executed
group = Group((multiply, 1), (multiply, 2), (multiply, 3))
step = Step('my_step_force', group, force=True)
futures.wait(self.submit(step))
# You can play another activity group in the step was already computed
group_done = Group(self.signal('DONE'))
step = Step('my_step_with_callback_done',
group,
activities_if_step_already_done=group_done)
futures.wait(self.submit(step))
# You can emit a signal with the identifier step.{step_name}
# after the step is executed (cached or not)
step = Step('my_step_with_signal', group, emit_signal=True)
futures.wait(self.submit(step))
def run(self, *args, **kwargs):
print(colorize(GREEN, "NotFailingWorkflow args: {}".format(args)))
print(colorize(GREEN, "NotFailingWorkflow kwargs: {}".format(kwargs)))
g = Group(raises_on_failure=False)
g.append(FailingWorkflow)
g.append(timeout_no_raise)
f = self.submit(g)
futures.wait(f)
def test(self):
future = Group((to_string, 1), (to_string, 2)).submit(executor)
self.assertTrue(future.finished)
future = Group((to_string, "test1"), (running_task, "test2"),
(sum_values, [1, 2])).submit(executor)
self.assertTrue(future.running)
self.assertEqual(future.count_finished_activities, 2)
self.assertEqual(future._result, ["test1", None, 3])
with self.assertRaises(exceptions.ExecutionBlocked):
future.result
def test_propagate_attribute(self):
"""
Test that attribute 'raises_on_failure' is well propagated through Group.
"""
inner_a = Group(
(running_task, "test1"),
(running_task, "test2"),
)
inner_b = ActivityTask(running_task, "test3")
Group(inner_a, inner_b, raises_on_failure=False).submit(executor)
self.assertFalse(inner_b.activity.raises_on_failure)
self.assertFalse(inner_a.activities[0].activity.raises_on_failure)
self.assertFalse(inner_a.activities[1].activity.raises_on_failure)
def run(self):
chain1 = Chain(
Group(ActivityTask(func_a_1_1), ActivityTask(func_a_1_2)),
self.signal("signal1"),
Group(ActivityTask(func_a_2_1), ActivityTask(func_a_2_2)),
)
chain2 = Chain(
Group(ActivityTask(func_b_1_1), ActivityTask(func_b_1_2)),
self.wait_signal("signal1"),
Group(ActivityTask(func_b_2_1), ActivityTask(func_b_2_2)),
)
my_group = Group(chain1, chain2)
fut = self.submit(my_group)
futures.wait(fut)
def test_exceptions(self):
future = Group((to_string, 1), (to_string, 2)).submit(executor)
self.assertIsNone(future.exception)
future = Group(
(zero_division),
(zero_division),
).submit(executor)
self.assertTrue(future.finished)
self.assertIsInstance(future.exception, AggregateException)
self.assertEqual(2, len(future.exception.exceptions))
self.assertIsInstance(future.exception.exceptions[0],
ZeroDivisionError)
self.assertIsInstance(future.exception.exceptions[1],
ZeroDivisionError)
def test(self):
complex_canvas = Chain((sum_values, [1, 2]), (sum_values, [1, 2]),
Group(
(to_int, 1),
(to_int, 2),
), Chain(
(sum_values, [1, 2]),
running_task,
), (sum_values, [1, 2]))
result = complex_canvas.submit(executor)
self.assertFalse(result.finished)
self.assertTrue(result.futures[0].finished)
self.assertTrue(result.futures[1].finished)
self.assertTrue(result.futures[2].finished)
self.assertFalse(result.futures[3].finished)
self.assertTrue(result.futures[3].futures[0].finished)
self.assertFalse(result.futures[3].futures[1].finished)
# As result.futures[3] is not finished, we shouldn't find other future
self.assertEqual(len(result.futures), 4)
# Change the state of the n-1 chain to make the whole
# canvas done
complex_canvas.activities[3].activities[1] = ActivityTask(to_int, 1)
result = complex_canvas.submit(executor)
self.assertTrue(result.finished)
self.assertEqual(len(result.futures), 5)
def test_simplified_declaration(self):
future = Group((to_string, 1), (to_string, 2)).submit(executor)
self.assertTrue(future.finished)
group = Group()
group += [
(to_string, "test1"),
running_task,
(sum_values, [1, 2]),
]
future = group.submit(executor)
self.assertTrue(future.running)
self.assertEqual(future.count_finished_activities, 2)
self.assertEqual(future._result, ["test1", None, 3])
with self.assertRaises(exceptions.ExecutionBlocked):
future.result
def run(self):
x = 1
y = 2
z = 3
future = self.submit(
Chain(
Group(
(increment_slowly, x),
(increment_slowly, y),
(increment_slowly, z),
),
multiply,
send_result=True
)
)
futures.wait(future)
res = future.result[-1]
print('({}+1)*({}+1)*({}+1) = {}'.format(x, y, z, res))
# Canvas's and Group's can also be "optional"
future = self.submit(
Chain(
(fail_incrementing, x),
(increment_slowly, 1), # never executed
raises_on_failure=False,
)
)
futures.wait(future)
print('SUCCESS!')
def run(self):
return self.submit(
Chain(
Group(
self.wait_signal('signal'),
self.wait_signal('signal 2'),
), )).result
def test_run_context_for_child_workflows(self):
""" Test that the local executor adds a different run_id for each
child workflows.
"""
class ChildWorkflow3(Workflow):
name = "ChildWorkflow3"
@classmethod
def get_workflow_id(*args, **kwargs):
return kwargs["name"]
def run(self, name, *args, **kwargs):
return {
"run_id": self.get_run_context()["run_id"],
"workflow_id": self.get_run_context()["workflow_id"],
}
class ChildWorkflow2(Workflow):
name = "ChildWorkflow2"
def run(self, *args, **kwargs):
return {
"run_id": self.get_run_context()["run_id"],
"workflow_id": self.get_run_context()["workflow_id"],
}
class ChildWorkflow1(Workflow):
name = "ChildWorkflow"
def run(self, *args, **kwargs):
results = (Chain(ChildWorkflow2,
(to_int, "2")).submit(self.executor).result)
return {
"workflow_id": self.get_run_context()["workflow_id"],
"run_id": self.get_run_context()["run_id"],
"child": results,
}
result = (Group(
ChildWorkflow1,
ChildWorkflow1,
WorkflowTask(None, ChildWorkflow3, name="test_workflow_id"),
).submit(executor).result)
child1_1, child1_2, child3 = result
self.assertTrue(
child1_1["workflow_id"].startswith("local_childworkflow"))
self.assertTrue(
child1_2["workflow_id"].startswith("local_childworkflow"))
self.assertNotEqual(child1_1["run_id"], child1_2["run_id"])
child2_1 = child1_1["child"][0]
child2_2 = child1_2["child"][0]
self.assertTrue(
child2_1["workflow_id"].startswith("local_childworkflow2"))
self.assertTrue(
child2_2["workflow_id"].startswith("local_childworkflow2"))
self.assertNotEqual(child2_1["run_id"], child2_2["run_id"])
self.assertEqual(child3["workflow_id"], "test_workflow_id")
self.assertEqual(child1_1["workflow_id"], "local_childworkflow")
def run(self):
return self.submit(
Chain(
Group(
ChildWorkflowWaitingSignals,
self.signal("signal", propagate=True),
self.signal("signal 2", propagate=True),
),
)
).result
def test_simplified_declaration(self):
future = Group(
(to_string, 1),
(to_string, 2)
).submit(executor)
self.assertTrue(future.finished)
group = Group()
group += [
(to_string, "test1"),
running_task,
(sum_values, [1, 2]),
]
future = group.submit(executor)
self.assertTrue(future.running)
self.assertEqual(future.count_finished_activities, 2)
self.assertEqual(future._result, ["test1", None, 3])
with self.assertRaises(exceptions.ExecutionBlocked):
future.result
def run(self, *args, **kwargs):
future = self.submit(
Chain(
Group(
(self.wait_signal("signal 2"), ),
(self.wait_signal("signal"), ),
(wait_and_signal, ),
(wait_and_signal, "signal 2"),
),
(increment, 1),
))
futures.wait(future)
def run(self, *args, **kwargs):
print(colorize(GREEN, "NotFailingWorkflow args: {}".format(args)))
print(colorize(GREEN, "NotFailingWorkflow kwargs: {}".format(kwargs)))
g = Group(raises_on_failure=False)
g.append(FailingWorkflow)
g.append(timeout_no_raise)
f = self.submit(g)
futures.wait(f)
def run(self, t1=30, t2=120):
"""
Cancel timer 2 after timer 1 is fired.
"""
future = self.submit(
Group(
self.start_timer("timer 2", t2),
Chain(self.start_timer("timer 1", t1), self.cancel_timer("timer 2"),),
)
)
if future.pending:
print("Starting timers")
futures.wait(future)
print("Timer fired, exiting")
def run(self):
x = 1
y = 2
z = 3
future = self.submit(
Chain(Group(
ActivityTask(increment_slowly, x),
ActivityTask(increment_slowly, y),
ActivityTask(increment_slowly, z),
),
ActivityTask(multiply),
send_result=True))
futures.wait(future)
res = future.result[-1]
print('({}+1)*({}+1)*({}+1) = {}'.format(x, y, z, res))
def test_group_with_workflow(self):
""" Test that it is possible to provide a WorkflowTask to a Group(). """
class ChildWorkflowGroupWithWorkflow(Workflow):
name = "ChildWorkflow"
def run(self, str1, *args, **kwargs):
return {"str1": str1, "kwargs": kwargs}
future = Group(
ChildWorkflowTask(ChildWorkflowGroupWithWorkflow,
str1="str1",
**{"hello": "world"})).submit(executor)
self.assertTrue(future.finished)
self.assertEqual(future.result, [{
"str1": "str1",
"kwargs": {
"hello": "world"
}
}])
def run(self):
x = 1
y = 2
z = 3
future = self.submit(
Chain(Group(
(increment_slowly, x),
(increment_slowly, y),
(increment_slowly, z),
),
multiply,
send_result=True))
futures.wait(future)
res = future.result[-1]
print('({}+1)*({}+1)*({}+1) = {}'.format(x, y, z, res))
# Canvas's and Group's can also be "optional"
future = self.submit(
Chain(
(fail_incrementing, x),
(increment_slowly, 1), # never executed
(multiply, [3, 2]),
raises_on_failure=False,
))
assert [None] == future.result, 'Unexpected result {!r}'.format(
future.result)
print('Chain with failure: {}'.format(future.result))
# Breaking the chain on failure is the default but can be bypassed
future = self.submit(
Chain(
(fail_incrementing, x),
(increment_slowly, 1), # executed
(multiply, [3, 2]),
break_on_failure=False,
))
assert [None, 2, 6] == future.result, 'Unexpected result {!r}'.format(
future.result)
print('Chain ignoring failure: {}'.format(future.result))
# Failing inside a chain by default don't stop an upper chain
future = self.submit(
Chain(
Chain(
(fail_incrementing, x),
raises_on_failure=False,
),
(increment_slowly, 1), # executed
(multiply, [3, 2]),
))
assert [[None], 2,
6] == future.result, 'Unexpected result {!r}'.format(
future.result)
print('Chain with failure in subchain: {}'.format(future.result))
# But it can, too
future = self.submit(
Chain(
Chain(
Chain(
(fail_incrementing, x),
raises_on_failure=False,
bubbles_exception_on_failure=True,
),
(increment_slowly, 1), # not executed
bubbles_exception_on_failure=False,
),
(multiply, [3, 2]), # executed
))
assert [[[None]], 6] == future.result, 'Unexpected result {!r}'.format(
future.result)
print('Chain with failure in sub-subchain: {}'.format(future.result))
print('Finished!')
def run(self):
g = Group()
g.append(ChainTestWorkflow, 4)
future = self.submit(g)
return future.result
def custom_func(previous_value):
group = Group()
for i in range(0, previous_value):
group.append(to_int, i * 2)
return group
def custom_func():
group = Group()
for i in range(0, 2):
group.append(zero_division)
return group
def custom_func(previous_value):
group = Group()
for i in xrange(0, previous_value):
group.append(ActivityTask(to_int, i * 2))
return group
def run(self, *args, **kwargs):
return self.submit(
Group(ChildWorkflowWaitingSignals, ChildWorkflowSendingSignals,)
).result
def custom_func(previous_value):
group = Group()
for i in xrange(0, previous_value):
group.append(ActivityTask(to_int, i * 2))
return group
def run(self):
g = Group()
g.append(ChainTestWorkflow, 4)
future = self.submit(g)
return future.result
def custom_func():
group = Group()
for i in range(0, 2):
group.append(zero_division)
return group
def custom_func(previous_value):
group = Group()
for i in range(0, previous_value):
group.append(to_int, i * 2)
return group