def test_previous_value(self):
future = Chain(ActivityTask(sum_values, [1, 2]),
ActivityTask(sum_previous, [2, 3]),
ActivityTask(sum_previous, [4, 5]),
send_result=True).submit(executor)
self.assertTrue(future.finished)
self.assertEquals(future.result, [3, 8, 17])
def test_propagate_attribute(self):
"""
Test that attribute 'raises_on_failure' is well propagated through FuncGroup.
"""
first = ActivityTask(running_task, "test1")
intermediary_activities = Chain(
(running_task, "test2"),
(running_task, "test3"),
)
last = ActivityTask(running_task, "test4")
def custom_func(_):
return intermediary_activities
Chain(first,
FuncGroup(custom_func),
last,
send_result=True,
raises_on_failure=False).submit(executor)
self.assertFalse(first.activity.raises_on_failure)
self.assertFalse(last.activity.raises_on_failure)
self.assertFalse(
intermediary_activities.activities[0].activity.raises_on_failure)
self.assertFalse(
intermediary_activities.activities[1].activity.raises_on_failure)
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
future = futures.Future()
context = self.get_execution_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name)
)
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name, []).append(Marker(func.name, func.details))
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, 'activity', None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
try:
future._result = task.execute()
state = 'completed'
except Exception as err:
future._exception = err
logger.info('rescuing exception: {}'.format(err))
if isinstance(func, Activity) and func.raises_on_failure:
message = err.args[0] if err.args else ''
raise exceptions.TaskFailed(func.name, message)
state = 'failed'
finally:
future._state = futures.FINISHED
if func:
self._history.add_activity_task(
func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={'args': args, 'kwargs': kwargs},
result=future._result)
return future
def run(self, x=5):
future = self.submit(
Chain(ActivityTask(increment, x),
ActivityTask(double),
send_result=True))
print("Future: {}".format(future))
futures.wait(future)
print("Result: {}".format(future.result)) # future.result == [6, 12]
return future.result
def test(self):
future = Chain(ActivityTask(to_string, "test"),
ActivityTask(to_string, "test")).submit(executor)
self.assertTrue(future.finished)
self.assertEquals(future.count_finished_activities, 2)
future = Chain(ActivityTask(to_string, "test"),
ActivityTask(running_task, "test"),
ActivityTask(to_string, "test")).submit(executor)
self.assertTrue(future.running)
self.assertEquals(future.count_finished_activities, 1)
def test_previous_value_with_func(self):
def custom_func(previous_value):
group = Group()
for i in xrange(0, previous_value):
group.append(ActivityTask(to_int, i * 2))
return group
chain = Chain(ActivityTask(sum_values, [1, 2]),
FuncGroup(custom_func),
ActivityTask(sum_values),
send_result=True).submit(executor)
self.assertEquals(chain.result, [3, [0, 2, 4], 6])
def test_raises_on_failure(self):
chain = Chain(ActivityTask(to_string, "test1"),
ActivityTask(zero_division),
raises_on_failure=False)
self.assertFalse(chain.activities[0].activity.raises_on_failure)
self.assertFalse(chain.activities[1].activity.raises_on_failure)
chain = Chain(ActivityTask(to_string, "test1"),
ActivityTask(zero_division),
raises_on_failure=True)
self.assertTrue(chain.activities[0].activity.raises_on_failure)
self.assertTrue(chain.activities[1].activity.raises_on_failure)
def test(self):
future = Group(ActivityTask(to_string, 1),
ActivityTask(to_string, 2)).submit(executor)
self.assertTrue(future.finished)
future = Group(ActivityTask(to_string, "test1"),
ActivityTask(running_task, "test2"),
ActivityTask(sum_values, [1, 2])).submit(executor)
self.assertTrue(future.running)
self.assertEquals(future.count_finished_activities, 2)
self.assertEquals(future._result, ["test1", None, 3])
with self.assertRaises(exceptions.ExecutionBlocked):
future.result
def test(self):
complex_canvas = Chain(
ActivityTask(sum_values, [1, 2]), ActivityTask(sum_values, [1, 2]),
Group(
ActivityTask(to_int, 1),
ActivityTask(to_int, 2),
),
Chain(ActivityTask(sum_values, [1, 2]),
ActivityTask(running_task, 1)),
ActivityTask(sum_values, [1, 2]))
result = complex_canvas.submit(executor)
self.assertFalse(result.finished)
self.assertTrue(result.futures[0].finished)
self.assertTrue(result.futures[1].finished)
self.assertTrue(result.futures[2].finished)
self.assertFalse(result.futures[3].finished)
self.assertTrue(result.futures[3].futures[0].finished)
self.assertFalse(result.futures[3].futures[1].finished)
# As result.futures[3] is not finished, we shouldn't find other future
self.assertEquals(len(result.futures), 4)
# Change the state of the n-1 chain to make the whole
# canvas done
complex_canvas.activities[3].activities[1] = ActivityTask(to_int, 1)
result = complex_canvas.submit(executor)
self.assertTrue(result.finished)
self.assertEquals(len(result.futures), 5)
def test_exceptions(self):
future = Group(ActivityTask(to_string, 1),
ActivityTask(to_string, 2)).submit(executor)
self.assertIsNone(future.exception)
future = Group(
ActivityTask(zero_division),
ActivityTask(zero_division),
).submit(executor)
self.assertTrue(future.finished)
self.assertIsInstance(future.exception, AggregateException)
self.assertEqual(2, len(future.exception.exceptions))
self.assertIsInstance(future.exception.exceptions[0],
ZeroDivisionError)
self.assertIsInstance(future.exception.exceptions[1],
ZeroDivisionError)
def run(self):
x = 1
y = 2
z = 3
future = self.submit(
Chain(Group(
ActivityTask(increment_slowly, x),
ActivityTask(increment_slowly, y),
ActivityTask(increment_slowly, z),
),
ActivityTask(multiply),
send_result=True))
futures.wait(future)
res = future.result[-1]
print('({}+1)*({}+1)*({}+1) = {}'.format(x, y, z, res))
def test_exceptions(self):
future = Chain(ActivityTask(to_string, 1),
ActivityTask(to_string, 2)).submit(executor)
self.assertIsNone(future.exception)
future = Chain(
ActivityTask(zero_division),
ActivityTask(zero_division),
).submit(executor)
self.assertTrue(future.finished)
self.assertIsInstance(future.exception, AggregateException)
# Both tasks were tried and failed (being in a chain doesn't change this)
self.assertEqual(2, len(future.exception.exceptions))
self.assertIsInstance(future.exception.exceptions[0],
ZeroDivisionError)
self.assertIsInstance(future.exception.exceptions[1],
ZeroDivisionError)
def run(self):
all = [
self.submit(ChildWorkUntilSignalWorkflow),
self.submit(
Chain(
ActivityTask(func_a_1_2),
self.signal('signal1'),
)),
]
futures.wait(*all)
def run(self):
chain1 = Chain(
Group(ActivityTask(func_a_1_1), ActivityTask(func_a_1_2)),
self.signal("signal1"),
Group(ActivityTask(func_a_2_1), ActivityTask(func_a_2_2)),
)
chain2 = Chain(
Group(ActivityTask(func_b_1_1), ActivityTask(func_b_1_2)),
self.wait_signal("signal1"),
Group(ActivityTask(func_b_2_1), ActivityTask(func_b_2_2)),
)
my_group = Group(chain1, chain2)
fut = self.submit(my_group)
futures.wait(fut)
def test_max_parallel(self):
future = Group(ActivityTask(running_task, "test1"),
ActivityTask(running_task, "test2"),
ActivityTask(running_task, "test3"),
max_parallel=2).submit(executor)
self.assertTrue(future.running)
self.assertEquals(len(future.futures), 2)
future = Group(ActivityTask(to_string, "test1"),
ActivityTask(running_task, "test2"),
ActivityTask(running_task, "test3"),
ActivityTask(running_task, "test4"),
max_parallel=2).submit(executor)
self.assertTrue(future.running)
self.assertEquals(len(future.futures), 3)
self.assertEquals([f.state for f in future.futures],
[futures.FINISHED, futures.RUNNING, futures.RUNNING])
future = Group(ActivityTask(to_string, "test1"),
ActivityTask(to_string, "test2"),
ActivityTask(to_string, "test3"),
max_parallel=2).submit(executor)
self.assertTrue(future.finished)
def test_propagate_attribute(self):
"""
Test that attribute 'raises_on_failure' is well propagated through Chain.
"""
inner_a = Chain(
(running_task, "test1"),
(running_task, "test2"),
)
inner_b = ActivityTask(running_task, "test3")
Chain(inner_a, inner_b, raises_on_failure=False).submit(executor)
self.assertFalse(inner_b.activity.raises_on_failure)
self.assertFalse(inner_a.activities[0].activity.raises_on_failure)
self.assertFalse(inner_a.activities[1].activity.raises_on_failure)
def test_simplified_declaration(self):
future = Group((to_string, 1), (to_string, 2)).submit(executor)
self.assertTrue(future.finished)
group = Group()
group += [
ActivityTask(to_string, "test1"),
running_task,
(sum_values, [1, 2]),
]
future = group.submit(executor)
self.assertTrue(future.running)
self.assertEquals(future.count_finished_activities, 2)
self.assertEquals(future._result, ["test1", None, 3])
with self.assertRaises(exceptions.ExecutionBlocked):
future.result
def custom_func():
group = Group()
for i in range(0, 2):
group.append(ActivityTask(zero_division))
return group
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
future = futures.Future()
context = self.get_run_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
'wait_signal({}) before signal was sent: unsupported by the local executor'
.format(signal_name))
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name,
[]).append(Marker(func.name,
func.details))
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, 'activity', None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(type(func), func))
if isinstance(task, WorkflowTask):
self.on_new_workflow(task)
try:
future._result = task.execute()
if hasattr(task, 'post_execute'):
task.post_execute()
state = 'completed'
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
future._exception = exc_value
logger.exception('rescuing exception: {}'.format(exc_value))
if (isinstance(func, Activity)
or issubclass_(func, Workflow)) and getattr(
func, 'raises_on_failure', None):
tb = traceback.format_tb(exc_traceback)
message = format_exc(exc_value)
details = json_dumps(
{
'error': exc_type.__name__,
'message': str(exc_value),
'traceback': tb,
},
default=repr)
raise exceptions.TaskFailed(
func.name,
message,
details,
)
state = 'failed'
finally:
if isinstance(task, WorkflowTask):
self.on_completed_workflow()
future._state = futures.FINISHED
if func:
self._history.add_activity_task(func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={
'args': args,
'kwargs': kwargs
},
result=future.result)
return future
def submit(self, func, *args, **kwargs):
logger.info('executing task {}(args={}, kwargs={})'.format(
func, args, kwargs))
future = futures.Future()
context = self.get_run_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name)
)
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name, []).append(Marker(func.name, func.details))
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, 'activity', None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError('invalid type {} for {}'.format(
type(func), func))
if isinstance(task, WorkflowTask):
self.on_new_workflow(task)
try:
future._result = task.execute()
if hasattr(task, 'post_execute'):
task.post_execute()
state = 'completed'
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
future._exception = exc_value
logger.exception('rescuing exception: {}'.format(exc_value))
if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr(func, 'raises_on_failure', None):
tb = traceback.format_tb(exc_traceback)
message = format_exc(exc_value)
details = json_dumps(
{
'error': exc_type.__name__,
'message': str(exc_value),
'traceback': tb,
},
default=repr
)
raise exceptions.TaskFailed(
func.name,
message,
details,
)
state = 'failed'
finally:
if isinstance(task, WorkflowTask):
self.on_completed_workflow()
future._state = futures.FINISHED
if func:
self._history.add_activity_task(
func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={'args': args, 'kwargs': kwargs},
result=future.result)
return future
def custom_func(previous_value):
group = Group()
for i in xrange(0, previous_value):
group.append(ActivityTask(to_int, i * 2))
return group
def submit(self, func, *args, **kwargs):
logger.info("executing task {}(args={}, kwargs={})".format(func, args, kwargs))
future = futures.Future()
context = self.get_run_context()
context["activity_id"] = str(self.nb_activities)
self.nb_activities += 1
# Ensure signals ordering
if isinstance(func, SignalTask):
self.signals_sent.add(func.name)
elif isinstance(func, WaitForSignal):
signal_name = func.signal_name
if signal_name not in self.signals_sent:
raise NotImplementedError(
"wait_signal({}) before signal was sent: unsupported by the local executor".format(
signal_name
)
)
elif isinstance(func, MarkerTask):
self._markers.setdefault(func.name, []).append(
Marker(func.name, func.details)
)
if isinstance(func, Submittable):
task = func # *args, **kwargs already resolved.
task.context = context
func = getattr(task, "activity", None)
elif isinstance(func, Activity):
task = ActivityTask(func, context=context, *args, **kwargs)
elif issubclass(func, Workflow):
task = WorkflowTask(self, func, *args, **kwargs)
else:
raise TypeError("invalid type {} for {}".format(type(func), func))
if isinstance(task, WorkflowTask):
self.on_new_workflow(task)
try:
future._result = task.execute()
if hasattr(task, "post_execute"):
task.post_execute()
state = "completed"
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.format_tb(exc_traceback)
task_failed = exceptions.TaskFailed(
name=getattr(task, "name", "unknown"),
reason=format_exc(exc_value),
details=json_dumps(
{
"error": exc_type.__name__,
"error_type": format_exc_type(exc_type),
"message": str(exc_value),
"traceback": tb,
},
default=repr,
),
)
future.set_exception(task_failed)
logger.exception("rescuing exception: {}".format(exc_value))
if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr(
func, "raises_on_failure", None
):
raise task_failed
state = "failed"
finally:
if isinstance(task, WorkflowTask):
self.on_completed_workflow()
future._state = futures.FINISHED
if func:
self._history.add_activity_task(
func,
decision_id=None,
last_state=state,
activity_id=context["activity_id"],
input={"args": args, "kwargs": kwargs},
result=future.result,
)
return future
