def test_process_error_handling(self, mock_get_lock): expected_errors = [ { "message": "Execution failed. See result for details.", "type": "error", "task_id": "task1", }, { "type": "error", "message": "ToozConnectionError: foobar", "task_id": "task1", "route": 0, }, ] mock_get_lock.side_effect = coordination_service.NoOpLock(name="noop") wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = action_service.request(lv_ac_db) # Assert action execution is running. lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] self.assertEqual(wf_ex_db.status, action_constants.LIVEACTION_STATUS_RUNNING) # Process task1. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id))[0] mock_get_lock.side_effect = [ coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination_service.NoOpLock(name="noop"), coordination_service.NoOpLock(name="noop"), ] workflows.get_engine().process(t1_ac_ex_db) # Assert the task is marked as failed. t1_ex_db = wf_db_access.TaskExecution.get_by_id(str(t1_ex_db.id)) self.assertEqual(t1_ex_db.status, wf_statuses.FAILED) # Assert the workflow has failed with expected errors. wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) self.assertEqual(wf_ex_db.status, wf_statuses.FAILED) self.assertListEqual(wf_ex_db.errors, expected_errors) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED)
def test_process_error_handling_has_error(self, mock_get_lock): mock_get_lock.side_effect = coordination_service.NoOpLock(name="noop") wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = action_service.request(lv_ac_db) # Assert action execution is running. lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] self.assertEqual(wf_ex_db.status, action_constants.LIVEACTION_STATUS_RUNNING) # Process task1. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id))[0] mock_get_lock.side_effect = [ coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), ] self.assertRaisesRegexp(Exception, "Unexpected error.", workflows.get_engine().process, t1_ac_ex_db) self.assertTrue( workflows.WorkflowExecutionHandler.fail_workflow_execution.called) mock_get_lock.side_effect = coordination_service.NoOpLock(name="noop") # Since error handling failed, the workflow will have status of running. wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) self.assertEqual(wf_ex_db.status, wf_statuses.RUNNING) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) # Sleep up to the test config gc_max_idle_sec before running gc. eventlet.sleep(cfg.CONF.workflow_engine.gc_max_idle_sec) # Run garbage collection. gc = garbage_collector.GarbageCollectorService() gc._purge_orphaned_workflow_executions() # Assert workflow execution is cleaned up and canceled. lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_CANCELED)
def _start(self): super(FileDriver, self)._start() for a_dir in self._reserved_dirs: try: fileutils.ensure_tree(a_dir) except OSError as e: raise coordination.ToozConnectionError(e)
def test_coordinator_offline(self): crd = self.get_coordinator.return_value crd.start.side_effect = tooz_coordination.ToozConnectionError('err') agent = coordination.Coordinator() self.assertRaises(tooz_coordination.ToozError, agent.start) self.assertFalse(agent.started)
def _start(self): for a_dir in self._reserved_dirs: try: fileutils.ensure_tree(a_dir) except OSError as e: raise coordination.ToozConnectionError(e) self._executor.start()
def start(self): try: self._coord.start(timeout=self.timeout) except self._coord.handler.timeout_exception as e: raise coordination.ToozConnectionError("operation error: %s" % (e)) try: self._coord.ensure_path(self.paths_join("/", self._TOOZ_NAMESPACE)) except exceptions.KazooException as e: raise coordination.ToozError("operation error: %s" % (e)) self._group_members = collections.defaultdict(set) self._watchers = six.moves.queue.Queue() self._leader_locks = {}
def test_retries_exhausted_from_coordinator_connection_error( self, mock_get_lock): mock_get_lock.side_effect = coord_svc.NoOpLock(name="noop") wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] # Process task1 but retries exhaused with connection errors. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id))[0] tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) mock_get_lock.side_effect = [ coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), ] # The connection error should raise if retries are exhaused. self.assertRaises( coordination.ToozConnectionError, wf_svc.handle_action_execution_completion, tk1_ac_ex_db, )
def test_recover_from_coordinator_connection_error(self, mock_get_lock): mock_get_lock.side_effect = coord_svc.NoOpLock(name="noop") wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] # Process task1 and expect acquiring lock returns a few connection errors. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id))[0] tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) mock_get_lock.side_effect = [ coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("foobar"), coord_svc.NoOpLock(name="noop"), coord_svc.NoOpLock(name="noop"), coord_svc.NoOpLock(name="noop"), coord_svc.NoOpLock(name="noop"), coord_svc.NoOpLock(name="noop"), ] wf_svc.handle_action_execution_completion(tk1_ac_ex_db) mock_get_lock.side_effect = coord_svc.NoOpLock(name="noop") # Workflow service should recover from retries and task1 should succeed. tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) self.assertEqual(tk1_ex_db.status, wf_statuses.SUCCEEDED)
def _start(self): try: self.client.self_stats() except requests.exceptions.ConnectionError as e: raise coordination.ToozConnectionError( encodeutils.exception_to_unicode(e))
class WorkflowExecutionHandlerTest(st2tests.WorkflowTestCase): @classmethod def setUpClass(cls): super(WorkflowExecutionHandlerTest, cls).setUpClass() # Register runners. runnersregistrar.register_runners() # Register test pack(s). actions_registrar = actionsregistrar.ActionsRegistrar( use_pack_cache=False, fail_on_failure=True) for pack in PACKS: actions_registrar.register_from_pack(pack) def test_process(self): wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, 'sequential.yaml') lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name']) lv_ac_db, ac_ex_db = action_service.request(lv_ac_db) # Assert action execution is running. lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] self.assertEqual(wf_ex_db.status, action_constants.LIVEACTION_STATUS_RUNNING) # Process task1. query_filters = { 'workflow_execution': str(wf_ex_db.id), 'task_id': 'task1' } t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id))[0] workflows.get_engine().process(t1_ac_ex_db) t1_ex_db = wf_db_access.TaskExecution.get_by_id(t1_ex_db.id) self.assertEqual(t1_ex_db.status, wf_statuses.SUCCEEDED) # Process task2. query_filters = { 'workflow_execution': str(wf_ex_db.id), 'task_id': 'task2' } t2_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] t2_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t2_ex_db.id))[0] workflows.get_engine().process(t2_ac_ex_db) t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id) self.assertEqual(t2_ex_db.status, wf_statuses.SUCCEEDED) # Process task3. query_filters = { 'workflow_execution': str(wf_ex_db.id), 'task_id': 'task3' } t3_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] t3_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t3_ex_db.id))[0] workflows.get_engine().process(t3_ac_ex_db) t3_ex_db = wf_db_access.TaskExecution.get_by_id(t3_ex_db.id) self.assertEqual(t3_ex_db.status, wf_statuses.SUCCEEDED) # Assert the workflow has completed successfully with expected output. expected_output = {'msg': 'Stanley, All your base are belong to us!'} wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) self.assertEqual(wf_ex_db.status, wf_statuses.SUCCEEDED) self.assertDictEqual(wf_ex_db.output, expected_output) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) @mock.patch.object( coordination_service.NoOpDriver, 'get_lock', mock.MagicMock(side_effect=coordination.ToozConnectionError('foobar'))) def test_process_error_handling(self): expected_errors = [{ 'message': 'Execution failed. See result for details.', 'type': 'error', 'task_id': 'task1' }, { 'type': 'error', 'message': 'ToozConnectionError: foobar', 'task_id': 'task1', 'route': 0 }] wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, 'sequential.yaml') lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name']) lv_ac_db, ac_ex_db = action_service.request(lv_ac_db) # Assert action execution is running. lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] self.assertEqual(wf_ex_db.status, action_constants.LIVEACTION_STATUS_RUNNING) # Process task1. query_filters = { 'workflow_execution': str(wf_ex_db.id), 'task_id': 'task1' } t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id))[0] workflows.get_engine().process(t1_ac_ex_db) # Assert the task is marked as failed. t1_ex_db = wf_db_access.TaskExecution.get_by_id(str(t1_ex_db.id)) self.assertEqual(t1_ex_db.status, wf_statuses.FAILED) # Assert the workflow has failed with expected errors. wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) self.assertEqual(wf_ex_db.status, wf_statuses.FAILED) self.assertListEqual(wf_ex_db.errors, expected_errors) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_FAILED) @mock.patch.object( coordination_service.NoOpDriver, 'get_lock', mock.MagicMock(side_effect=coordination.ToozConnectionError('foobar'))) @mock.patch.object( workflows.WorkflowExecutionHandler, 'fail_workflow_execution', mock.MagicMock(side_effect=Exception('Unexpected error.'))) def test_process_error_handling_has_error(self): wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, 'sequential.yaml') lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name']) lv_ac_db, ac_ex_db = action_service.request(lv_ac_db) # Assert action execution is running. lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] self.assertEqual(wf_ex_db.status, action_constants.LIVEACTION_STATUS_RUNNING) # Process task1. query_filters = { 'workflow_execution': str(wf_ex_db.id), 'task_id': 'task1' } t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] t1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(t1_ex_db.id))[0] self.assertRaisesRegexp(Exception, 'Unexpected error.', workflows.get_engine().process, t1_ac_ex_db) self.assertTrue( workflows.WorkflowExecutionHandler.fail_workflow_execution.called) # Since error handling failed, the workflow will have status of running. wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id) self.assertEqual(wf_ex_db.status, wf_statuses.RUNNING) lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_RUNNING) # Sleep up to the test config gc_max_idle_sec before running gc. eventlet.sleep(cfg.CONF.workflow_engine.gc_max_idle_sec) # Run garbage collection. gc = garbage_collector.GarbageCollectorService() gc._purge_orphaned_workflow_executions() # Assert workflow execution is cleaned up and canceled. lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id)) self.assertEqual(lv_ac_db.status, action_constants.LIVEACTION_STATUS_CANCELED)
class OrquestaServiceRetryTest(st2tests.WorkflowTestCase): ensure_indexes = True ensure_indexes_models = [ wf_db_models.WorkflowExecutionDB, wf_db_models.TaskExecutionDB, ex_q_db_models.ActionExecutionSchedulingQueueItemDB, ] @classmethod def setUpClass(cls): super(OrquestaServiceRetryTest, cls).setUpClass() # Register runners. runnersregistrar.register_runners() # Register test pack(s). actions_registrar = actionsregistrar.ActionsRegistrar( use_pack_cache=False, fail_on_failure=True) for pack in PACKS: actions_registrar.register_from_pack(pack) @mock.patch.object( coord_svc.NoOpDriver, "get_lock", mock.MagicMock(side_effect=[ coordination.ToozConnectionError("foobar"), coordination.ToozConnectionError("fubar"), coord_svc.NoOpLock(name="noop"), ]), ) def test_recover_from_coordinator_connection_error(self): wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] # Process task1 and expect acquiring lock returns a few connection errors. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id))[0] tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) # Workflow service should recover from retries and task1 should succeed. tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) self.assertEqual(tk1_ex_db.status, wf_statuses.SUCCEEDED) @mock.patch.object( coord_svc.NoOpDriver, "get_lock", mock.MagicMock(side_effect=coordination.ToozConnectionError("foobar")), ) def test_retries_exhausted_from_coordinator_connection_error(self): wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] # Process task1 but retries exhaused with connection errors. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id))[0] tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # The connection error should raise if retries are exhaused. self.assertRaises( coordination.ToozConnectionError, wf_svc.handle_action_execution_completion, tk1_ac_ex_db, ) @mock.patch.object( wf_svc, "update_task_state", mock.MagicMock(side_effect=[ mongoengine.connection.ConnectionFailure(), mongoengine.connection.ConnectionFailure(), None, ]), ) def test_recover_from_database_connection_error(self): wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] # Process task1 and expect acquiring lock returns a few connection errors. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id))[0] tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) wf_svc.handle_action_execution_completion(tk1_ac_ex_db) # Workflow service should recover from retries and task1 should succeed. tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id) self.assertEqual(tk1_ex_db.status, wf_statuses.SUCCEEDED) @mock.patch.object( wf_svc, "update_task_state", mock.MagicMock(side_effect=mongoengine.connection.ConnectionFailure()), ) def test_retries_exhausted_from_database_connection_error(self): wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] # Process task1 but retries exhaused with connection errors. query_filters = { "workflow_execution": str(wf_ex_db.id), "task_id": "task1" } tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0] tk1_ac_ex_db = ex_db_access.ActionExecution.query( task_execution=str(tk1_ex_db.id))[0] tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id( tk1_ac_ex_db.liveaction["id"]) self.assertEqual(tk1_lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) # The connection error should raise if retries are exhaused. self.assertRaises( mongoengine.connection.ConnectionFailure, wf_svc.handle_action_execution_completion, tk1_ac_ex_db, ) @mock.patch.object( wf_db_access.WorkflowExecution, "update", mock.MagicMock(side_effect=mock_wf_db_update_conflict), ) def test_recover_from_database_write_conflicts(self): # Create a temporary file which will be used to signal # which task(s) to mock the DB write conflict. temp_file_path = TEMP_DIR_PATH + "/task4" if not os.path.exists(temp_file_path): with open(temp_file_path, "w"): pass # Manually create the liveaction and action execution objects without publishing. wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "join.yaml") lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"]) lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db) wf_ex_db = wf_db_access.WorkflowExecution.query( action_execution=str(ac_ex_db.id))[0] # Manually request task executions. task_route = 0 self.run_workflow_step(wf_ex_db, "task1", task_route) self.assert_task_running("task2", task_route) self.assert_task_running("task4", task_route) self.run_workflow_step(wf_ex_db, "task2", task_route) self.assert_task_running("task3", task_route) self.run_workflow_step(wf_ex_db, "task4", task_route) self.assert_task_running("task5", task_route) self.run_workflow_step(wf_ex_db, "task3", task_route) self.assert_task_not_started("task6", task_route) self.run_workflow_step(wf_ex_db, "task5", task_route) self.assert_task_running("task6", task_route) self.run_workflow_step(wf_ex_db, "task6", task_route) self.assert_task_running("task7", task_route) self.run_workflow_step(wf_ex_db, "task7", task_route) self.assert_workflow_completed(str(wf_ex_db.id), status=wf_statuses.SUCCEEDED) # Ensure retry happened. self.assertFalse(os.path.exists(temp_file_path))
def test_retry_on_connection_errors(self): exc = coordination.ToozConnectionError("foobar") self.assertTrue(wf_exc.retry_on_connection_errors(exc)) exc = mongoengine.connection.MongoEngineConnectionError() self.assertTrue(wf_exc.retry_on_connection_errors(exc))
def raiser(cond): if not cond: raise tooz_coordination.ToozConnectionError('err')