def test_garbage_collection(self):
# Workflow that is still running with no task and expired.
wf_ex_set_1 = self.mock_workflow_records(completed=False, expired=True)
# Workflow that is still running with task completed and expired.
wf_ex_set_2 = self.mock_workflow_records(completed=False, expired=True)
self.mock_task_records(wf_ex_set_2,
"task1",
completed=True,
expired=True)
# Ensure these workflows are identified as orphans.
orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows()
self.assertEqual(len(orphaned_ac_ex_dbs), 2)
self.assertIn(orphaned_ac_ex_dbs[0].id,
[wf_ex_set_1[2].id, wf_ex_set_2[2].id])
self.assertIn(orphaned_ac_ex_dbs[1].id,
[wf_ex_set_1[2].id, wf_ex_set_2[2].id])
# Run garbage collection.
ex_gc.purge_orphaned_workflow_executions(logger=LOG)
# Ensure these workflows are processed and not returned as orphans.
orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows()
self.assertEqual(len(orphaned_ac_ex_dbs), 0)
def test_no_orphans(self):
# Workflow that is still running and not expired.
self.mock_workflow_records(completed=False, expired=False)
# Workflow that is still running with task completed and not expired.
wf_ex_set_2 = self.mock_workflow_records(completed=False,
expired=False)
self.mock_task_records(wf_ex_set_2,
"task1",
completed=True,
expired=False)
# Workflow that is still running with task running and not expired.
wf_ex_set_3 = self.mock_workflow_records(completed=False,
expired=False)
self.mock_task_records(wf_ex_set_3,
"task1",
completed=False,
expired=False)
# Workflow that is completed and not expired.
self.mock_workflow_records(completed=True, expired=False)
# Workflow that is completed with task completed and not expired.
wf_ex_set_5 = self.mock_workflow_records(completed=True, expired=False)
self.mock_task_records(wf_ex_set_5,
"task1",
completed=True,
expired=False)
orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows()
self.assertEqual(len(orphaned_ac_ex_dbs), 0)
def test_action_execution_with_missing_log_entries(self):
# Workflow that is still running and expired. However the state change logs are missing.
wf_ex_set_1 = self.mock_workflow_records(completed=False, expired=True, log=False)
self.mock_task_records(wf_ex_set_1, 'task1', completed=True, expired=True)
orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows()
self.assertEqual(len(orphaned_ac_ex_dbs), 0)
def test_identify_orphans_with_task_executions(self):
# Workflow that is still running with task completed and expired.
wf_ex_set_1 = self.mock_workflow_records(completed=False, expired=True)
self.mock_task_records(wf_ex_set_1, 'task1', completed=True, expired=True)
# Workflow that is still running with task completed and not expired.
wf_ex_set_2 = self.mock_workflow_records(completed=False, expired=False)
self.mock_task_records(wf_ex_set_2, 'task1', completed=True, expired=False)
# Workflow that is still running with task running and not expired.
wf_ex_set_3 = self.mock_workflow_records(completed=False, expired=False)
self.mock_task_records(wf_ex_set_3, 'task1', completed=False, expired=False)
# Workflow that is still running with multiple tasks and not expired.
# One of the task completed passed expiry date but another task is still running.
wf_ex_set_4 = self.mock_workflow_records(completed=False, expired=False)
self.mock_task_records(wf_ex_set_4, 'task1', completed=True, expired=True)
self.mock_task_records(wf_ex_set_4, 'task2', completed=False, expired=False)
# Workflow that is still running with multiple tasks and not expired.
# Both of the tasks are completed with one completed only recently.
wf_ex_set_5 = self.mock_workflow_records(completed=False, expired=False)
self.mock_task_records(wf_ex_set_5, 'task1', completed=True, expired=True)
self.mock_task_records(wf_ex_set_5, 'task2', completed=True, expired=False)
# Workflow that is still running with multiple tasks and not expired.
# One of the task completed recently and another task is still running.
wf_ex_set_6 = self.mock_workflow_records(completed=False, expired=False)
self.mock_task_records(wf_ex_set_6, 'task1', completed=True, expired=False)
self.mock_task_records(wf_ex_set_6, 'task2', completed=False, expired=False)
orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows()
self.assertEqual(len(orphaned_ac_ex_dbs), 1)
self.assertEqual(orphaned_ac_ex_dbs[0].id, wf_ex_set_1[2].id)
def purge_orphaned_workflow_executions(logger):
"""
Purge workflow executions that are idled and identified as orphans.
"""
# Cancel workflow executions that are identified as orphans. The workflow executions are
# marked as canceled instead of failed because error handling during normal operation
# failed and the system does not know what state the workflow execution is in. A failed
# workflow execution can be rerun from failed task(s). Since we do not know what state
# the workflow execution is in because correct data may not be recorded in the database
# as a result of the original failure, the garbage collection routine here cancels
# the workflow execution so it cannot be rerun from failed task(s).
for ac_ex_db in workflow_service.identify_orphaned_workflows():
lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction['id'])
action_service.request_cancellation(lv_ac_db, None)
def test_identify_orphans_with_no_task_executions(self):
# Workflow that is still running and expired.
wf_ex_set_1 = self.mock_workflow_records(completed=False, expired=True)
# Workflow that is completed and expired.
self.mock_workflow_records(completed=True, expired=True)
# Workflow that is still running and not expired.
self.mock_workflow_records(completed=False, expired=False)
# Workflow that is completed and not expired.
self.mock_workflow_records(completed=True, expired=False)
orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows()
self.assertEqual(len(orphaned_ac_ex_dbs), 1)
self.assertEqual(orphaned_ac_ex_dbs[0].id, wf_ex_set_1[2].id)
