def test_process_error_handling(self, mock_get_lock):
expected_errors = [
{
"message": "Execution failed. See result for details.",
"type": "error",
"task_id": "task1",
},
{
"type": "error",
"message": "ToozConnectionError: foobar",
"task_id": "task1",
"route": 0,
},
]
mock_get_lock.side_effect = coordination_service.NoOpLock(name="noop")
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = action_service.request(lv_ac_db)
# Assert action execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
self.assertEqual(wf_ex_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Process task1.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))[0]
mock_get_lock.side_effect = [
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination_service.NoOpLock(name="noop"),
coordination_service.NoOpLock(name="noop"),
]
workflows.get_engine().process(t1_ac_ex_db)
# Assert the task is marked as failed.
t1_ex_db = wf_db_access.TaskExecution.get_by_id(str(t1_ex_db.id))
self.assertEqual(t1_ex_db.status, wf_statuses.FAILED)
# Assert the workflow has failed with expected errors.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
self.assertListEqual(wf_ex_db.errors, expected_errors)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_FAILED)
def test_process_error_handling_has_error(self, mock_get_lock):
mock_get_lock.side_effect = coordination_service.NoOpLock(name="noop")
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = action_service.request(lv_ac_db)
# Assert action execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
self.assertEqual(wf_ex_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Process task1.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))[0]
mock_get_lock.side_effect = [
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
]
self.assertRaisesRegexp(Exception, "Unexpected error.",
workflows.get_engine().process, t1_ac_ex_db)
self.assertTrue(
workflows.WorkflowExecutionHandler.fail_workflow_execution.called)
mock_get_lock.side_effect = coordination_service.NoOpLock(name="noop")
# Since error handling failed, the workflow will have status of running.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.RUNNING)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Sleep up to the test config gc_max_idle_sec before running gc.
eventlet.sleep(cfg.CONF.workflow_engine.gc_max_idle_sec)
# Run garbage collection.
gc = garbage_collector.GarbageCollectorService()
gc._purge_orphaned_workflow_executions()
# Assert workflow execution is cleaned up and canceled.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_CANCELED)
def _start(self):
super(FileDriver, self)._start()
for a_dir in self._reserved_dirs:
try:
fileutils.ensure_tree(a_dir)
except OSError as e:
raise coordination.ToozConnectionError(e)
def test_coordinator_offline(self):
crd = self.get_coordinator.return_value
crd.start.side_effect = tooz_coordination.ToozConnectionError('err')
agent = coordination.Coordinator()
self.assertRaises(tooz_coordination.ToozError, agent.start)
self.assertFalse(agent.started)
def _start(self):
for a_dir in self._reserved_dirs:
try:
fileutils.ensure_tree(a_dir)
except OSError as e:
raise coordination.ToozConnectionError(e)
self._executor.start()
def start(self):
try:
self._coord.start(timeout=self.timeout)
except self._coord.handler.timeout_exception as e:
raise coordination.ToozConnectionError("operation error: %s" % (e))
try:
self._coord.ensure_path(self.paths_join("/", self._TOOZ_NAMESPACE))
except exceptions.KazooException as e:
raise coordination.ToozError("operation error: %s" % (e))
self._group_members = collections.defaultdict(set)
self._watchers = six.moves.queue.Queue()
self._leader_locks = {}
def test_retries_exhausted_from_coordinator_connection_error(
self, mock_get_lock):
mock_get_lock.side_effect = coord_svc.NoOpLock(name="noop")
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
# Process task1 but retries exhaused with connection errors.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
tk1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(tk1_ex_db.id))[0]
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(
tk1_ac_ex_db.liveaction["id"])
self.assertEqual(tk1_lv_ac_db.status,
ac_const.LIVEACTION_STATUS_SUCCEEDED)
mock_get_lock.side_effect = [
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
]
# The connection error should raise if retries are exhaused.
self.assertRaises(
coordination.ToozConnectionError,
wf_svc.handle_action_execution_completion,
tk1_ac_ex_db,
)
def test_recover_from_coordinator_connection_error(self, mock_get_lock):
mock_get_lock.side_effect = coord_svc.NoOpLock(name="noop")
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
# Process task1 and expect acquiring lock returns a few connection errors.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
tk1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(tk1_ex_db.id))[0]
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(
tk1_ac_ex_db.liveaction["id"])
self.assertEqual(tk1_lv_ac_db.status,
ac_const.LIVEACTION_STATUS_SUCCEEDED)
mock_get_lock.side_effect = [
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("foobar"),
coord_svc.NoOpLock(name="noop"),
coord_svc.NoOpLock(name="noop"),
coord_svc.NoOpLock(name="noop"),
coord_svc.NoOpLock(name="noop"),
coord_svc.NoOpLock(name="noop"),
]
wf_svc.handle_action_execution_completion(tk1_ac_ex_db)
mock_get_lock.side_effect = coord_svc.NoOpLock(name="noop")
# Workflow service should recover from retries and task1 should succeed.
tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id)
self.assertEqual(tk1_ex_db.status, wf_statuses.SUCCEEDED)
def _start(self):
try:
self.client.self_stats()
except requests.exceptions.ConnectionError as e:
raise coordination.ToozConnectionError(
encodeutils.exception_to_unicode(e))
class WorkflowExecutionHandlerTest(st2tests.WorkflowTestCase):
@classmethod
def setUpClass(cls):
super(WorkflowExecutionHandlerTest, cls).setUpClass()
# Register runners.
runnersregistrar.register_runners()
# Register test pack(s).
actions_registrar = actionsregistrar.ActionsRegistrar(
use_pack_cache=False, fail_on_failure=True)
for pack in PACKS:
actions_registrar.register_from_pack(pack)
def test_process(self):
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
'sequential.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = action_service.request(lv_ac_db)
# Assert action execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
self.assertEqual(wf_ex_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Process task1.
query_filters = {
'workflow_execution': str(wf_ex_db.id),
'task_id': 'task1'
}
t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))[0]
workflows.get_engine().process(t1_ac_ex_db)
t1_ex_db = wf_db_access.TaskExecution.get_by_id(t1_ex_db.id)
self.assertEqual(t1_ex_db.status, wf_statuses.SUCCEEDED)
# Process task2.
query_filters = {
'workflow_execution': str(wf_ex_db.id),
'task_id': 'task2'
}
t2_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t2_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(t2_ex_db.id))[0]
workflows.get_engine().process(t2_ac_ex_db)
t2_ex_db = wf_db_access.TaskExecution.get_by_id(t2_ex_db.id)
self.assertEqual(t2_ex_db.status, wf_statuses.SUCCEEDED)
# Process task3.
query_filters = {
'workflow_execution': str(wf_ex_db.id),
'task_id': 'task3'
}
t3_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t3_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(t3_ex_db.id))[0]
workflows.get_engine().process(t3_ac_ex_db)
t3_ex_db = wf_db_access.TaskExecution.get_by_id(t3_ex_db.id)
self.assertEqual(t3_ex_db.status, wf_statuses.SUCCEEDED)
# Assert the workflow has completed successfully with expected output.
expected_output = {'msg': 'Stanley, All your base are belong to us!'}
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.SUCCEEDED)
self.assertDictEqual(wf_ex_db.output, expected_output)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_SUCCEEDED)
@mock.patch.object(
coordination_service.NoOpDriver, 'get_lock',
mock.MagicMock(side_effect=coordination.ToozConnectionError('foobar')))
def test_process_error_handling(self):
expected_errors = [{
'message': 'Execution failed. See result for details.',
'type': 'error',
'task_id': 'task1'
}, {
'type': 'error',
'message': 'ToozConnectionError: foobar',
'task_id': 'task1',
'route': 0
}]
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
'sequential.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = action_service.request(lv_ac_db)
# Assert action execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
self.assertEqual(wf_ex_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Process task1.
query_filters = {
'workflow_execution': str(wf_ex_db.id),
'task_id': 'task1'
}
t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))[0]
workflows.get_engine().process(t1_ac_ex_db)
# Assert the task is marked as failed.
t1_ex_db = wf_db_access.TaskExecution.get_by_id(str(t1_ex_db.id))
self.assertEqual(t1_ex_db.status, wf_statuses.FAILED)
# Assert the workflow has failed with expected errors.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.FAILED)
self.assertListEqual(wf_ex_db.errors, expected_errors)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_FAILED)
@mock.patch.object(
coordination_service.NoOpDriver, 'get_lock',
mock.MagicMock(side_effect=coordination.ToozConnectionError('foobar')))
@mock.patch.object(
workflows.WorkflowExecutionHandler, 'fail_workflow_execution',
mock.MagicMock(side_effect=Exception('Unexpected error.')))
def test_process_error_handling_has_error(self):
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
'sequential.yaml')
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta['name'])
lv_ac_db, ac_ex_db = action_service.request(lv_ac_db)
# Assert action execution is running.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
self.assertEqual(wf_ex_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Process task1.
query_filters = {
'workflow_execution': str(wf_ex_db.id),
'task_id': 'task1'
}
t1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
t1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(t1_ex_db.id))[0]
self.assertRaisesRegexp(Exception, 'Unexpected error.',
workflows.get_engine().process, t1_ac_ex_db)
self.assertTrue(
workflows.WorkflowExecutionHandler.fail_workflow_execution.called)
# Since error handling failed, the workflow will have status of running.
wf_ex_db = wf_db_access.WorkflowExecution.get_by_id(wf_ex_db.id)
self.assertEqual(wf_ex_db.status, wf_statuses.RUNNING)
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_RUNNING)
# Sleep up to the test config gc_max_idle_sec before running gc.
eventlet.sleep(cfg.CONF.workflow_engine.gc_max_idle_sec)
# Run garbage collection.
gc = garbage_collector.GarbageCollectorService()
gc._purge_orphaned_workflow_executions()
# Assert workflow execution is cleaned up and canceled.
lv_ac_db = lv_db_access.LiveAction.get_by_id(str(lv_ac_db.id))
self.assertEqual(lv_ac_db.status,
action_constants.LIVEACTION_STATUS_CANCELED)
class OrquestaServiceRetryTest(st2tests.WorkflowTestCase):
ensure_indexes = True
ensure_indexes_models = [
wf_db_models.WorkflowExecutionDB,
wf_db_models.TaskExecutionDB,
ex_q_db_models.ActionExecutionSchedulingQueueItemDB,
]
@classmethod
def setUpClass(cls):
super(OrquestaServiceRetryTest, cls).setUpClass()
# Register runners.
runnersregistrar.register_runners()
# Register test pack(s).
actions_registrar = actionsregistrar.ActionsRegistrar(
use_pack_cache=False, fail_on_failure=True)
for pack in PACKS:
actions_registrar.register_from_pack(pack)
@mock.patch.object(
coord_svc.NoOpDriver,
"get_lock",
mock.MagicMock(side_effect=[
coordination.ToozConnectionError("foobar"),
coordination.ToozConnectionError("fubar"),
coord_svc.NoOpLock(name="noop"),
]),
)
def test_recover_from_coordinator_connection_error(self):
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
# Process task1 and expect acquiring lock returns a few connection errors.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
tk1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(tk1_ex_db.id))[0]
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(
tk1_ac_ex_db.liveaction["id"])
self.assertEqual(tk1_lv_ac_db.status,
ac_const.LIVEACTION_STATUS_SUCCEEDED)
wf_svc.handle_action_execution_completion(tk1_ac_ex_db)
# Workflow service should recover from retries and task1 should succeed.
tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id)
self.assertEqual(tk1_ex_db.status, wf_statuses.SUCCEEDED)
@mock.patch.object(
coord_svc.NoOpDriver,
"get_lock",
mock.MagicMock(side_effect=coordination.ToozConnectionError("foobar")),
)
def test_retries_exhausted_from_coordinator_connection_error(self):
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
# Process task1 but retries exhaused with connection errors.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
tk1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(tk1_ex_db.id))[0]
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(
tk1_ac_ex_db.liveaction["id"])
self.assertEqual(tk1_lv_ac_db.status,
ac_const.LIVEACTION_STATUS_SUCCEEDED)
# The connection error should raise if retries are exhaused.
self.assertRaises(
coordination.ToozConnectionError,
wf_svc.handle_action_execution_completion,
tk1_ac_ex_db,
)
@mock.patch.object(
wf_svc,
"update_task_state",
mock.MagicMock(side_effect=[
mongoengine.connection.ConnectionFailure(),
mongoengine.connection.ConnectionFailure(),
None,
]),
)
def test_recover_from_database_connection_error(self):
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
# Process task1 and expect acquiring lock returns a few connection errors.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
tk1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(tk1_ex_db.id))[0]
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(
tk1_ac_ex_db.liveaction["id"])
self.assertEqual(tk1_lv_ac_db.status,
ac_const.LIVEACTION_STATUS_SUCCEEDED)
wf_svc.handle_action_execution_completion(tk1_ac_ex_db)
# Workflow service should recover from retries and task1 should succeed.
tk1_ex_db = wf_db_access.TaskExecution.get_by_id(tk1_ex_db.id)
self.assertEqual(tk1_ex_db.status, wf_statuses.SUCCEEDED)
@mock.patch.object(
wf_svc,
"update_task_state",
mock.MagicMock(side_effect=mongoengine.connection.ConnectionFailure()),
)
def test_retries_exhausted_from_database_connection_error(self):
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH,
"sequential.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
# Process task1 but retries exhaused with connection errors.
query_filters = {
"workflow_execution": str(wf_ex_db.id),
"task_id": "task1"
}
tk1_ex_db = wf_db_access.TaskExecution.query(**query_filters)[0]
tk1_ac_ex_db = ex_db_access.ActionExecution.query(
task_execution=str(tk1_ex_db.id))[0]
tk1_lv_ac_db = lv_db_access.LiveAction.get_by_id(
tk1_ac_ex_db.liveaction["id"])
self.assertEqual(tk1_lv_ac_db.status,
ac_const.LIVEACTION_STATUS_SUCCEEDED)
# The connection error should raise if retries are exhaused.
self.assertRaises(
mongoengine.connection.ConnectionFailure,
wf_svc.handle_action_execution_completion,
tk1_ac_ex_db,
)
@mock.patch.object(
wf_db_access.WorkflowExecution,
"update",
mock.MagicMock(side_effect=mock_wf_db_update_conflict),
)
def test_recover_from_database_write_conflicts(self):
# Create a temporary file which will be used to signal
# which task(s) to mock the DB write conflict.
temp_file_path = TEMP_DIR_PATH + "/task4"
if not os.path.exists(temp_file_path):
with open(temp_file_path, "w"):
pass
# Manually create the liveaction and action execution objects without publishing.
wf_meta = self.get_wf_fixture_meta_data(TEST_PACK_PATH, "join.yaml")
lv_ac_db = lv_db_models.LiveActionDB(action=wf_meta["name"])
lv_ac_db, ac_ex_db = ac_svc.request(lv_ac_db)
wf_ex_db = wf_db_access.WorkflowExecution.query(
action_execution=str(ac_ex_db.id))[0]
# Manually request task executions.
task_route = 0
self.run_workflow_step(wf_ex_db, "task1", task_route)
self.assert_task_running("task2", task_route)
self.assert_task_running("task4", task_route)
self.run_workflow_step(wf_ex_db, "task2", task_route)
self.assert_task_running("task3", task_route)
self.run_workflow_step(wf_ex_db, "task4", task_route)
self.assert_task_running("task5", task_route)
self.run_workflow_step(wf_ex_db, "task3", task_route)
self.assert_task_not_started("task6", task_route)
self.run_workflow_step(wf_ex_db, "task5", task_route)
self.assert_task_running("task6", task_route)
self.run_workflow_step(wf_ex_db, "task6", task_route)
self.assert_task_running("task7", task_route)
self.run_workflow_step(wf_ex_db, "task7", task_route)
self.assert_workflow_completed(str(wf_ex_db.id),
status=wf_statuses.SUCCEEDED)
# Ensure retry happened.
self.assertFalse(os.path.exists(temp_file_path))
def test_retry_on_connection_errors(self):
exc = coordination.ToozConnectionError("foobar")
self.assertTrue(wf_exc.retry_on_connection_errors(exc))
exc = mongoengine.connection.MongoEngineConnectionError()
self.assertTrue(wf_exc.retry_on_connection_errors(exc))
def raiser(cond):
if not cond:
raise tooz_coordination.ToozConnectionError('err')
