def test_previous_value(self): future = Chain(ActivityTask(sum_values, [1, 2]), ActivityTask(sum_previous, [2, 3]), ActivityTask(sum_previous, [4, 5]), send_result=True).submit(executor) self.assertTrue(future.finished) self.assertEquals(future.result, [3, 8, 17])
def test_propagate_attribute(self): """ Test that attribute 'raises_on_failure' is well propagated through FuncGroup. """ first = ActivityTask(running_task, "test1") intermediary_activities = Chain( (running_task, "test2"), (running_task, "test3"), ) last = ActivityTask(running_task, "test4") def custom_func(_): return intermediary_activities Chain(first, FuncGroup(custom_func), last, send_result=True, raises_on_failure=False).submit(executor) self.assertFalse(first.activity.raises_on_failure) self.assertFalse(last.activity.raises_on_failure) self.assertFalse( intermediary_activities.activities[0].activity.raises_on_failure) self.assertFalse( intermediary_activities.activities[1].activity.raises_on_failure)
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_execution_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) try: future._result = task.execute() state = 'completed' except Exception as err: future._exception = err logger.info('rescuing exception: {}'.format(err)) if isinstance(func, Activity) and func.raises_on_failure: message = err.args[0] if err.args else '' raise exceptions.TaskFailed(func.name, message) state = 'failed' finally: future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={'args': args, 'kwargs': kwargs}, result=future._result) return future
def run(self, x=5): future = self.submit( Chain(ActivityTask(increment, x), ActivityTask(double), send_result=True)) print("Future: {}".format(future)) futures.wait(future) print("Result: {}".format(future.result)) # future.result == [6, 12] return future.result
def test(self): future = Chain(ActivityTask(to_string, "test"), ActivityTask(to_string, "test")).submit(executor) self.assertTrue(future.finished) self.assertEquals(future.count_finished_activities, 2) future = Chain(ActivityTask(to_string, "test"), ActivityTask(running_task, "test"), ActivityTask(to_string, "test")).submit(executor) self.assertTrue(future.running) self.assertEquals(future.count_finished_activities, 1)
def test_previous_value_with_func(self): def custom_func(previous_value): group = Group() for i in xrange(0, previous_value): group.append(ActivityTask(to_int, i * 2)) return group chain = Chain(ActivityTask(sum_values, [1, 2]), FuncGroup(custom_func), ActivityTask(sum_values), send_result=True).submit(executor) self.assertEquals(chain.result, [3, [0, 2, 4], 6])
def test_raises_on_failure(self): chain = Chain(ActivityTask(to_string, "test1"), ActivityTask(zero_division), raises_on_failure=False) self.assertFalse(chain.activities[0].activity.raises_on_failure) self.assertFalse(chain.activities[1].activity.raises_on_failure) chain = Chain(ActivityTask(to_string, "test1"), ActivityTask(zero_division), raises_on_failure=True) self.assertTrue(chain.activities[0].activity.raises_on_failure) self.assertTrue(chain.activities[1].activity.raises_on_failure)
def test(self): future = Group(ActivityTask(to_string, 1), ActivityTask(to_string, 2)).submit(executor) self.assertTrue(future.finished) future = Group(ActivityTask(to_string, "test1"), ActivityTask(running_task, "test2"), ActivityTask(sum_values, [1, 2])).submit(executor) self.assertTrue(future.running) self.assertEquals(future.count_finished_activities, 2) self.assertEquals(future._result, ["test1", None, 3]) with self.assertRaises(exceptions.ExecutionBlocked): future.result
def test(self): complex_canvas = Chain( ActivityTask(sum_values, [1, 2]), ActivityTask(sum_values, [1, 2]), Group( ActivityTask(to_int, 1), ActivityTask(to_int, 2), ), Chain(ActivityTask(sum_values, [1, 2]), ActivityTask(running_task, 1)), ActivityTask(sum_values, [1, 2])) result = complex_canvas.submit(executor) self.assertFalse(result.finished) self.assertTrue(result.futures[0].finished) self.assertTrue(result.futures[1].finished) self.assertTrue(result.futures[2].finished) self.assertFalse(result.futures[3].finished) self.assertTrue(result.futures[3].futures[0].finished) self.assertFalse(result.futures[3].futures[1].finished) # As result.futures[3] is not finished, we shouldn't find other future self.assertEquals(len(result.futures), 4) # Change the state of the n-1 chain to make the whole # canvas done complex_canvas.activities[3].activities[1] = ActivityTask(to_int, 1) result = complex_canvas.submit(executor) self.assertTrue(result.finished) self.assertEquals(len(result.futures), 5)
def test_exceptions(self): future = Group(ActivityTask(to_string, 1), ActivityTask(to_string, 2)).submit(executor) self.assertIsNone(future.exception) future = Group( ActivityTask(zero_division), ActivityTask(zero_division), ).submit(executor) self.assertTrue(future.finished) self.assertIsInstance(future.exception, AggregateException) self.assertEqual(2, len(future.exception.exceptions)) self.assertIsInstance(future.exception.exceptions[0], ZeroDivisionError) self.assertIsInstance(future.exception.exceptions[1], ZeroDivisionError)
def run(self): x = 1 y = 2 z = 3 future = self.submit( Chain(Group( ActivityTask(increment_slowly, x), ActivityTask(increment_slowly, y), ActivityTask(increment_slowly, z), ), ActivityTask(multiply), send_result=True)) futures.wait(future) res = future.result[-1] print('({}+1)*({}+1)*({}+1) = {}'.format(x, y, z, res))
def test_exceptions(self): future = Chain(ActivityTask(to_string, 1), ActivityTask(to_string, 2)).submit(executor) self.assertIsNone(future.exception) future = Chain( ActivityTask(zero_division), ActivityTask(zero_division), ).submit(executor) self.assertTrue(future.finished) self.assertIsInstance(future.exception, AggregateException) # Both tasks were tried and failed (being in a chain doesn't change this) self.assertEqual(2, len(future.exception.exceptions)) self.assertIsInstance(future.exception.exceptions[0], ZeroDivisionError) self.assertIsInstance(future.exception.exceptions[1], ZeroDivisionError)
def run(self): all = [ self.submit(ChildWorkUntilSignalWorkflow), self.submit( Chain( ActivityTask(func_a_1_2), self.signal('signal1'), )), ] futures.wait(*all)
def run(self): chain1 = Chain( Group(ActivityTask(func_a_1_1), ActivityTask(func_a_1_2)), self.signal("signal1"), Group(ActivityTask(func_a_2_1), ActivityTask(func_a_2_2)), ) chain2 = Chain( Group(ActivityTask(func_b_1_1), ActivityTask(func_b_1_2)), self.wait_signal("signal1"), Group(ActivityTask(func_b_2_1), ActivityTask(func_b_2_2)), ) my_group = Group(chain1, chain2) fut = self.submit(my_group) futures.wait(fut)
def test_max_parallel(self): future = Group(ActivityTask(running_task, "test1"), ActivityTask(running_task, "test2"), ActivityTask(running_task, "test3"), max_parallel=2).submit(executor) self.assertTrue(future.running) self.assertEquals(len(future.futures), 2) future = Group(ActivityTask(to_string, "test1"), ActivityTask(running_task, "test2"), ActivityTask(running_task, "test3"), ActivityTask(running_task, "test4"), max_parallel=2).submit(executor) self.assertTrue(future.running) self.assertEquals(len(future.futures), 3) self.assertEquals([f.state for f in future.futures], [futures.FINISHED, futures.RUNNING, futures.RUNNING]) future = Group(ActivityTask(to_string, "test1"), ActivityTask(to_string, "test2"), ActivityTask(to_string, "test3"), max_parallel=2).submit(executor) self.assertTrue(future.finished)
def test_propagate_attribute(self): """ Test that attribute 'raises_on_failure' is well propagated through Chain. """ inner_a = Chain( (running_task, "test1"), (running_task, "test2"), ) inner_b = ActivityTask(running_task, "test3") Chain(inner_a, inner_b, raises_on_failure=False).submit(executor) self.assertFalse(inner_b.activity.raises_on_failure) self.assertFalse(inner_a.activities[0].activity.raises_on_failure) self.assertFalse(inner_a.activities[1].activity.raises_on_failure)
def test_simplified_declaration(self): future = Group((to_string, 1), (to_string, 2)).submit(executor) self.assertTrue(future.finished) group = Group() group += [ ActivityTask(to_string, "test1"), running_task, (sum_values, [1, 2]), ] future = group.submit(executor) self.assertTrue(future.running) self.assertEquals(future.count_finished_activities, 2) self.assertEquals(future._result, ["test1", None, 3]) with self.assertRaises(exceptions.ExecutionBlocked): future.result
def custom_func(): group = Group() for i in range(0, 2): group.append(ActivityTask(zero_division)) return group
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor' .format(signal_name)) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format(type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, 'post_execute'): task.post_execute() state = 'completed' except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() future._exception = exc_value logger.exception('rescuing exception: {}'.format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr( func, 'raises_on_failure', None): tb = traceback.format_tb(exc_traceback) message = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr) raise exceptions.TaskFailed( func.name, message, details, ) state = 'failed' finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task(func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={ 'args': args, 'kwargs': kwargs }, result=future.result) return future
def submit(self, func, *args, **kwargs): logger.info('executing task {}(args={}, kwargs={})'.format( func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( 'wait_signal({}) before signal was sent: unsupported by the local executor'.format(signal_name) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append(Marker(func.name, func.details)) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, 'activity', None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError('invalid type {} for {}'.format( type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, 'post_execute'): task.post_execute() state = 'completed' except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() future._exception = exc_value logger.exception('rescuing exception: {}'.format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr(func, 'raises_on_failure', None): tb = traceback.format_tb(exc_traceback) message = format_exc(exc_value) details = json_dumps( { 'error': exc_type.__name__, 'message': str(exc_value), 'traceback': tb, }, default=repr ) raise exceptions.TaskFailed( func.name, message, details, ) state = 'failed' finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={'args': args, 'kwargs': kwargs}, result=future.result) return future
def custom_func(previous_value): group = Group() for i in xrange(0, previous_value): group.append(ActivityTask(to_int, i * 2)) return group
def submit(self, func, *args, **kwargs): logger.info("executing task {}(args={}, kwargs={})".format(func, args, kwargs)) future = futures.Future() context = self.get_run_context() context["activity_id"] = str(self.nb_activities) self.nb_activities += 1 # Ensure signals ordering if isinstance(func, SignalTask): self.signals_sent.add(func.name) elif isinstance(func, WaitForSignal): signal_name = func.signal_name if signal_name not in self.signals_sent: raise NotImplementedError( "wait_signal({}) before signal was sent: unsupported by the local executor".format( signal_name ) ) elif isinstance(func, MarkerTask): self._markers.setdefault(func.name, []).append( Marker(func.name, func.details) ) if isinstance(func, Submittable): task = func # *args, **kwargs already resolved. task.context = context func = getattr(task, "activity", None) elif isinstance(func, Activity): task = ActivityTask(func, context=context, *args, **kwargs) elif issubclass(func, Workflow): task = WorkflowTask(self, func, *args, **kwargs) else: raise TypeError("invalid type {} for {}".format(type(func), func)) if isinstance(task, WorkflowTask): self.on_new_workflow(task) try: future._result = task.execute() if hasattr(task, "post_execute"): task.post_execute() state = "completed" except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) task_failed = exceptions.TaskFailed( name=getattr(task, "name", "unknown"), reason=format_exc(exc_value), details=json_dumps( { "error": exc_type.__name__, "error_type": format_exc_type(exc_type), "message": str(exc_value), "traceback": tb, }, default=repr, ), ) future.set_exception(task_failed) logger.exception("rescuing exception: {}".format(exc_value)) if (isinstance(func, Activity) or issubclass_(func, Workflow)) and getattr( func, "raises_on_failure", None ): raise task_failed state = "failed" finally: if isinstance(task, WorkflowTask): self.on_completed_workflow() future._state = futures.FINISHED if func: self._history.add_activity_task( func, decision_id=None, last_state=state, activity_id=context["activity_id"], input={"args": args, "kwargs": kwargs}, result=future.result, ) return future