def test_cleanup_inconsistent_task(self, notify, active_tasks, considered_jobs, reapable_jobs, running_tasks, waiting_tasks, mocker): tm = TaskManager() tm.get_running_tasks = mocker.Mock(return_value=(running_tasks, waiting_tasks)) tm.get_active_tasks = mocker.Mock(return_value=active_tasks) tm.cleanup_inconsistent_celery_tasks() for j in considered_jobs: if j not in reapable_jobs: j.save.assert_not_called() assert notify.call_count == 4 notify.assert_has_calls( [mock.call(j, 'failed') for j in reapable_jobs], any_order=True) for j in reapable_jobs: j.websocket_emit_status.assert_called_once_with('failed') assert j.status == 'failed' assert j.job_explanation == ( 'Task was marked as running in Tower but was not present in Celery, so it has been marked as failed.' )
def test_instance_does_not_exist(self, logger_mock, *args): logger_mock.error = mock.MagicMock(side_effect=RuntimeError("mocked")) tm = TaskManager() with pytest.raises(RuntimeError) as excinfo: tm.cleanup_inconsistent_celery_tasks() assert "mocked" in str(excinfo.value) logger_mock.error.assert_called_once_with("Execution node Instance host1 not found in database. " "The node is currently executing jobs ['job 2 (new)', " "'job 3 (new)']")
def test_save_failed(self, logger_mock, get_running_tasks, *args): logger_mock.error = mock.MagicMock() job = Job(id=2, modified=tz_now(), status='running', celery_task_id='blah', execution_node='host1') job.websocket_emit_status = mock.MagicMock() get_running_tasks.return_value = ({'host1': [job]}, []) tm = TaskManager() with mock.patch.object(job, 'save', side_effect=DatabaseError): tm.cleanup_inconsistent_celery_tasks() job.save.assert_called_once() logger_mock.error.assert_called_once_with("Task job 2 (failed) DB error in marking failed. Job possibly deleted.")