class MailQueueTest(TestCase, QueueHelperMixin): def setUp(self): self.queue = MailQueue() self.queue.MAX_WORKERS = 1 def test_default_attributes(self): """The needed attributes are there""" self.assertIsInstance(self.queue.queue, list) self.assertIsInstance(self.queue.entries, dict) self.assertEqual(self.queue.processed_count, 0) def test_add_returns_mail_queue_entry(self): identifier = 'a' entry = self.queue.add(identifier) self.assertIsInstance(entry, MailQueueEntry) self.assertEqual(entry.identifier, identifier) def test_add_twice(self): """Duplicate add() is ignored""" self.queue.add('a') self.queue.add('a') self.assertEqual(len(self.queue.queue), 1) def test_remove(self): """remove() cancels the effects of add()""" self.queue.add('a') self.queue.remove('a') self.assertQueueIsEmpty() self.assertEqual(len(self.queue.entries), 0) def test_remove_increases_processed_count(self): """remove() counts the number of processed mails""" self.queue.add('a') self.assertEqual(self.queue.processed_count, 0) self.queue.remove('a') self.assertEqual(self.queue.processed_count, 1) def test_remove_non_existing(self): """remove() is a no-op for a non-existing entry""" self.queue.remove('a') @mock.patch('os.listdir') def test_initialize(self, mock_listdir): """ Initialize calls os.listdir() on the Maildir/new and populates the queue attribute with it """ mock_listdir.return_value = ['a', 'b', 'c'] new = os.path.join(settings.DISTRO_TRACKER_MAILDIR_DIRECTORY, 'new') self.queue.initialize() mock_listdir.assert_called_with(new) self.assertListEqual( list(map(lambda x: x.identifier, self.queue.queue)), mock_listdir.return_value) def test_pool_is_multiprocessing_pool(self): self.assertIsInstance(self.queue.pool, multiprocessing.pool.Pool) def test_pool_is_singleton(self): self.assertEqual(self.queue.pool, self.queue.pool) def test_close_pool_drops_cached_object(self): self.queue.pool self.queue.close_pool() self.assertIsNone(self.queue._pool) def test_close_pool_works_without_pool(self): self.queue.close_pool() def test_close_pool_really_closes_the_pool(self): pool = self.queue.pool self.queue.close_pool() if six.PY2: expected_exception = AssertionError # assert self._state == RUN else: expected_exception = ValueError # Pool not running exception with self.assertRaises(expected_exception): pool.apply_async(time.sleep, 0) def test_process_queue_handles_preexisting_mails(self): """Pre-existing mails are processed""" self.patch_mail_processor() self.add_mails_to_queue('a', 'b') self.queue.process_queue() self.queue.close_pool() self.assertQueueIsEmpty() def test_process_queue_does_not_start_tasks_for_entries_with_task(self): """Mails being processed are not re-queued""" self.patch_mail_processor() entry_a, entry_b = self.add_mails_to_queue('a', 'b') self.patch_methods(entry_a, processing_task_started=True, processing_task_finished=False, start_processing_task=None) self.patch_methods(entry_b, processing_task_started=False, processing_task_finished=False, start_processing_task=None) self.queue.process_queue() self.assertFalse(entry_a.start_processing_task.called) entry_b.start_processing_task.assert_called_once_with() def test_process_queue_handles_processing_task_result(self): """Mails being processed are handled when finished""" self.patch_mail_processor() entry_a, entry_b = self.add_mails_to_queue('a', 'b') self.patch_methods(entry_a, processing_task_started=True, processing_task_finished=False, handle_processing_task_result=None) self.patch_methods(entry_b, processing_task_started=True, processing_task_finished=True, handle_processing_task_result=None) self.queue.process_queue() entry_a.processing_task_finished.assert_called_once_with() entry_b.processing_task_finished.assert_called_once_with() self.assertFalse(entry_a.handle_processing_task_result.called) entry_b.handle_processing_task_result.assert_called_once_with() def test_process_queue_works_when_queue_items_are_removed(self): """The processing of entries results in entries being dropped. This should not confuse process_queue which should still properly process all entries""" queue = ['a', 'b', 'c', 'd', 'e', 'f'] self.queue._count_mock_calls = 0 for entry in self.add_mails_to_queue(*queue): def side_effect(): entry.queue._count_mock_calls += 1 entry.queue.remove(entry.identifier) return False self.patch_methods(entry, processing_task_started=True, processing_task_finished=side_effect) self.queue.process_queue() self.assertEqual(self.queue._count_mock_calls, len(queue)) def test_sleep_timeout_mailqueue_empty(self): self.assertEqual(self.queue.sleep_timeout(), self.queue.SLEEP_TIMEOUT_EMPTY) def _add_entry_task_running(self, name): entry = self.add_mails_to_queue(name)[0] self.patch_methods(entry, processing_task_started=True, processing_task_finished=False) return entry def test_sleep_timeout_task_started_not_finished(self): self._add_entry_task_running('a') self.assertEqual(self.queue.sleep_timeout(), self.queue.SLEEP_TIMEOUT_TASK_RUNNING) def _add_entry_task_finished(self, name): entry = self.add_mails_to_queue(name)[0] self.patch_methods(entry, processing_task_started=True, processing_task_finished=True) return entry def test_sleep_timeout_task_finished(self): self._add_entry_task_finished('a') self.assertEqual(self.queue.sleep_timeout(), self.queue.SLEEP_TIMEOUT_TASK_FINISHED) def _add_entry_task_waiting_next_try(self, name): entry = self.add_mails_to_queue(name)[0] entry.schedule_next_try() return entry def _get_wait_time(self, entry): wait_time = entry.get_data('next_try_time') - self.current_datetime return wait_time.total_seconds() def test_sleep_timeout_task_waiting_next_try(self): self.patch_now() entry = self._add_entry_task_waiting_next_try('a') wait_time = self._get_wait_time(entry) self.assertEqual(self.queue.sleep_timeout(), wait_time) def _add_entry_task_runnable(self, name): entry = self.add_mails_to_queue(name)[0] return entry def test_sleep_timeout_task_runnable(self): self._add_entry_task_runnable('a') self.assertEqual(self.queue.sleep_timeout(), self.queue.SLEEP_TIMEOUT_TASK_RUNNABLE) def test_sleep_timeout_picks_the_shorter_wait_time(self): self.patch_now() self._add_entry_task_running('a') self._add_entry_task_runnable('b') self._add_entry_task_finished('c') entry_d = self._add_entry_task_waiting_next_try('d') wait_time = self._get_wait_time(entry_d) self.assertEqual(self.queue.sleep_timeout(), min(self.queue.SLEEP_TIMEOUT_TASK_RUNNING, self.queue.SLEEP_TIMEOUT_TASK_RUNNABLE, self.queue.SLEEP_TIMEOUT_TASK_FINISHED, wait_time)) def start_process_loop(self, stop_after=None): """ Start process_loop() in a dedicated process and ensure it's ready to proocess new files before returning """ self.mkdir(self.queue._get_maildir()) lock = multiprocessing.Lock() lock.acquire() def process_loop(lock): def release_lock(): lock.release() queue = MailQueue() queue.process_loop(stop_after=stop_after, ready_cb=release_lock) process = multiprocessing.Process(target=process_loop, args=(lock,)) process.start() lock.acquire() return process def test_process_loop_processes_one_mail(self): self.patch_mail_processor() process = self.start_process_loop(stop_after=1) # The mail is created after process_loop() is ready path = self.create_mail('a') # We wait the end of the task for max 1 second process.join(1) # Process finished successfully (and we're not here due to timeout) if process.is_alive(): process.terminate() self.fail("process_loop did not terminate") self.assertFalse(process.is_alive()) self.assertEqual(process.exitcode, 0) # And it did its job by handling the mail self.assertFalse(os.path.exists(path)) @mock.patch('distro_tracker.mail.processor.MailQueueWatcher') def test_process_loop_calls_sleep_timeout(self, mock_watcher): """Ensure we feed the sleep timeout to watcher.process_events""" self.mkdir(self.queue._get_maildir()) self.patch_methods(self.queue, sleep_timeout=mock.sentinel.DELAY) self.queue.process_loop(stop_after=0) mock_watcher.return_value.process_events.assert_called_with( timeout=mock.sentinel.DELAY) @mock.patch('distro_tracker.mail.processor.MailQueueWatcher') def test_process_loop_calls_initialize(self, mock_watcher): """Ensure we call the initialize method before announcing readyness""" self.patch_methods(self.queue, initialize=None) def check_when_ready(): self.queue.initialize.assert_called_with() self.queue.process_loop(stop_after=0, ready_cb=check_when_ready)
def process_loop(lock): def release_lock(): lock.release() queue = MailQueue() queue.process_loop(stop_after=stop_after, ready_cb=release_lock)
def handle(self, *args, **kwargs): queue = MailQueue() queue.process_loop() # Never returns