def test_garbage_collection(self): # Workflow that is still running with no task and expired. wf_ex_set_1 = self.mock_workflow_records(completed=False, expired=True) # Workflow that is still running with task completed and expired. wf_ex_set_2 = self.mock_workflow_records(completed=False, expired=True) self.mock_task_records(wf_ex_set_2, "task1", completed=True, expired=True) # Ensure these workflows are identified as orphans. orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows() self.assertEqual(len(orphaned_ac_ex_dbs), 2) self.assertIn(orphaned_ac_ex_dbs[0].id, [wf_ex_set_1[2].id, wf_ex_set_2[2].id]) self.assertIn(orphaned_ac_ex_dbs[1].id, [wf_ex_set_1[2].id, wf_ex_set_2[2].id]) # Run garbage collection. ex_gc.purge_orphaned_workflow_executions(logger=LOG) # Ensure these workflows are processed and not returned as orphans. orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows() self.assertEqual(len(orphaned_ac_ex_dbs), 0)
def test_no_orphans(self): # Workflow that is still running and not expired. self.mock_workflow_records(completed=False, expired=False) # Workflow that is still running with task completed and not expired. wf_ex_set_2 = self.mock_workflow_records(completed=False, expired=False) self.mock_task_records(wf_ex_set_2, "task1", completed=True, expired=False) # Workflow that is still running with task running and not expired. wf_ex_set_3 = self.mock_workflow_records(completed=False, expired=False) self.mock_task_records(wf_ex_set_3, "task1", completed=False, expired=False) # Workflow that is completed and not expired. self.mock_workflow_records(completed=True, expired=False) # Workflow that is completed with task completed and not expired. wf_ex_set_5 = self.mock_workflow_records(completed=True, expired=False) self.mock_task_records(wf_ex_set_5, "task1", completed=True, expired=False) orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows() self.assertEqual(len(orphaned_ac_ex_dbs), 0)
def test_action_execution_with_missing_log_entries(self): # Workflow that is still running and expired. However the state change logs are missing. wf_ex_set_1 = self.mock_workflow_records(completed=False, expired=True, log=False) self.mock_task_records(wf_ex_set_1, 'task1', completed=True, expired=True) orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows() self.assertEqual(len(orphaned_ac_ex_dbs), 0)
def test_identify_orphans_with_task_executions(self): # Workflow that is still running with task completed and expired. wf_ex_set_1 = self.mock_workflow_records(completed=False, expired=True) self.mock_task_records(wf_ex_set_1, 'task1', completed=True, expired=True) # Workflow that is still running with task completed and not expired. wf_ex_set_2 = self.mock_workflow_records(completed=False, expired=False) self.mock_task_records(wf_ex_set_2, 'task1', completed=True, expired=False) # Workflow that is still running with task running and not expired. wf_ex_set_3 = self.mock_workflow_records(completed=False, expired=False) self.mock_task_records(wf_ex_set_3, 'task1', completed=False, expired=False) # Workflow that is still running with multiple tasks and not expired. # One of the task completed passed expiry date but another task is still running. wf_ex_set_4 = self.mock_workflow_records(completed=False, expired=False) self.mock_task_records(wf_ex_set_4, 'task1', completed=True, expired=True) self.mock_task_records(wf_ex_set_4, 'task2', completed=False, expired=False) # Workflow that is still running with multiple tasks and not expired. # Both of the tasks are completed with one completed only recently. wf_ex_set_5 = self.mock_workflow_records(completed=False, expired=False) self.mock_task_records(wf_ex_set_5, 'task1', completed=True, expired=True) self.mock_task_records(wf_ex_set_5, 'task2', completed=True, expired=False) # Workflow that is still running with multiple tasks and not expired. # One of the task completed recently and another task is still running. wf_ex_set_6 = self.mock_workflow_records(completed=False, expired=False) self.mock_task_records(wf_ex_set_6, 'task1', completed=True, expired=False) self.mock_task_records(wf_ex_set_6, 'task2', completed=False, expired=False) orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows() self.assertEqual(len(orphaned_ac_ex_dbs), 1) self.assertEqual(orphaned_ac_ex_dbs[0].id, wf_ex_set_1[2].id)
def purge_orphaned_workflow_executions(logger): """ Purge workflow executions that are idled and identified as orphans. """ # Cancel workflow executions that are identified as orphans. The workflow executions are # marked as canceled instead of failed because error handling during normal operation # failed and the system does not know what state the workflow execution is in. A failed # workflow execution can be rerun from failed task(s). Since we do not know what state # the workflow execution is in because correct data may not be recorded in the database # as a result of the original failure, the garbage collection routine here cancels # the workflow execution so it cannot be rerun from failed task(s). for ac_ex_db in workflow_service.identify_orphaned_workflows(): lv_ac_db = LiveAction.get(id=ac_ex_db.liveaction['id']) action_service.request_cancellation(lv_ac_db, None)
def test_identify_orphans_with_no_task_executions(self): # Workflow that is still running and expired. wf_ex_set_1 = self.mock_workflow_records(completed=False, expired=True) # Workflow that is completed and expired. self.mock_workflow_records(completed=True, expired=True) # Workflow that is still running and not expired. self.mock_workflow_records(completed=False, expired=False) # Workflow that is completed and not expired. self.mock_workflow_records(completed=True, expired=False) orphaned_ac_ex_dbs = wf_svc.identify_orphaned_workflows() self.assertEqual(len(orphaned_ac_ex_dbs), 1) self.assertEqual(orphaned_ac_ex_dbs[0].id, wf_ex_set_1[2].id)