def test_purge_hung_worker_override_timeout_time(self, mock_time): w = Worker(worker_processes=2, wait_interval=0.01, timeout=5) mock_time.time.return_value = 0 w.add(HungWorker(10)) w._run_task('HungWorker(worker_timeout=10)') mock_time.time.return_value = 10 w._handle_next_task() self.assertEqual(1, len(w._running_tasks)) mock_time.time.return_value = 11 w._handle_next_task() self.assertEqual(0, len(w._running_tasks))
def test_purge_hung_worker_default_timeout_time(self, mock_time): w = Worker(worker_processes=2, wait_interval=0.01, worker_timeout=5) mock_time.time.return_value = 0 w.add(HungWorker()) w._run_task('HungWorker(worker_timeout=None)') mock_time.time.return_value = 5 w._handle_next_task() self.assertEqual(1, len(w._running_tasks)) mock_time.time.return_value = 6 w._handle_next_task() self.assertEqual(0, len(w._running_tasks))
def test_purge_multiple_workers(self): w = Worker(worker_processes=2, wait_interval=0.01) t1 = SuicidalWorker(signal.SIGTERM) t2 = SuicidalWorker(signal.SIGKILL) w.add(t1) w.add(t2) w._run_task(t1.task_id) w._run_task(t2.task_id) time.sleep(1.0) w._handle_next_task() w._handle_next_task() w._handle_next_task()
def test_purge_hung_worker_default_timeout_time(self, mock_time): w = Worker(worker_processes=2, wait_interval=0.01, timeout=5) mock_time.time.return_value = 0 task = HangTheWorkerTask() w.add(task) w._run_task(task.task_id) mock_time.time.return_value = 5 w._handle_next_task() self.assertEqual(1, len(w._running_tasks)) mock_time.time.return_value = 6 w._handle_next_task() self.assertEqual(0, len(w._running_tasks))
def test_stop_worker_kills_subprocesses(self): w = Worker(worker_processes=2) hung_task = HungWorker() w.add(hung_task) w._run_task(hung_task.task_id) pids = [p.pid for p in w._running_tasks.values()] self.assertEqual(1, len(pids)) pid = pids[0] def is_running(): return pid in {p.pid for p in psutil.Process().children()} self.assertTrue(is_running()) w.stop() self.assertFalse(is_running())
def test_timeout_handler(self, mock_time): result = [] @HangTheWorkerTask.event_handler(Event.TIMEOUT) def store_task(t, error_msg): self.assertTrue(error_msg) result.append(t) w = Worker(worker_processes=2, wait_interval=0.01, timeout=5) mock_time.time.return_value = 0 task = HangTheWorkerTask(worker_timeout=1) w.add(task) w._run_task(task.task_id) mock_time.time.return_value = 3 w._handle_next_task() self.assertEqual(result, [task])
def test_process_killed_handler(self, task_proc): result = [] @HangTheWorkerTask.event_handler(Event.PROCESS_FAILURE) def store_task(t, error_msg): self.assertTrue(error_msg) result.append(t) w = Worker() task = HangTheWorkerTask() task_process = mock.MagicMock(is_alive=lambda: False, exitcode=-14, task=task) task_proc.return_value = task_process w.add(task) w._run_task(task.task_id) w._handle_next_task() self.assertEqual(result, [task])