class MailQueueTest(TestCase, QueueHelperMixin):
def setUp(self):
self.queue = MailQueue()
self.queue.MAX_WORKERS = 1
def test_default_attributes(self):
"""The needed attributes are there"""
self.assertIsInstance(self.queue.queue, list)
self.assertIsInstance(self.queue.entries, dict)
self.assertEqual(self.queue.processed_count, 0)
def test_add_returns_mail_queue_entry(self):
identifier = 'a'
entry = self.queue.add(identifier)
self.assertIsInstance(entry, MailQueueEntry)
self.assertEqual(entry.identifier, identifier)
def test_add_twice(self):
"""Duplicate add() is ignored"""
self.queue.add('a')
self.queue.add('a')
self.assertEqual(len(self.queue.queue), 1)
def test_remove(self):
"""remove() cancels the effects of add()"""
self.queue.add('a')
self.queue.remove('a')
self.assertQueueIsEmpty()
self.assertEqual(len(self.queue.entries), 0)
def test_remove_increases_processed_count(self):
"""remove() counts the number of processed mails"""
self.queue.add('a')
self.assertEqual(self.queue.processed_count, 0)
self.queue.remove('a')
self.assertEqual(self.queue.processed_count, 1)
def test_remove_non_existing(self):
"""remove() is a no-op for a non-existing entry"""
self.queue.remove('a')
@mock.patch('os.listdir')
def test_initialize(self, mock_listdir):
"""
Initialize calls os.listdir() on the Maildir/new and populates
the queue attribute with it
"""
mock_listdir.return_value = ['a', 'b', 'c']
new = os.path.join(settings.DISTRO_TRACKER_MAILDIR_DIRECTORY, 'new')
self.queue.initialize()
mock_listdir.assert_called_with(new)
self.assertListEqual(
list(map(lambda x: x.identifier, self.queue.queue)),
mock_listdir.return_value)
def test_pool_is_multiprocessing_pool(self):
self.assertIsInstance(self.queue.pool, multiprocessing.pool.Pool)
def test_pool_is_singleton(self):
self.assertEqual(self.queue.pool, self.queue.pool)
def test_close_pool_drops_cached_object(self):
self.queue.pool
self.queue.close_pool()
self.assertIsNone(self.queue._pool)
def test_close_pool_works_without_pool(self):
self.queue.close_pool()
def test_close_pool_really_closes_the_pool(self):
pool = self.queue.pool
self.queue.close_pool()
if six.PY2:
expected_exception = AssertionError # assert self._state == RUN
else:
expected_exception = ValueError # Pool not running exception
with self.assertRaises(expected_exception):
pool.apply_async(time.sleep, 0)
def test_process_queue_handles_preexisting_mails(self):
"""Pre-existing mails are processed"""
self.patch_mail_processor()
self.add_mails_to_queue('a', 'b')
self.queue.process_queue()
self.queue.close_pool()
self.assertQueueIsEmpty()
def test_process_queue_does_not_start_tasks_for_entries_with_task(self):
"""Mails being processed are not re-queued"""
self.patch_mail_processor()
entry_a, entry_b = self.add_mails_to_queue('a', 'b')
self.patch_methods(entry_a, processing_task_started=True,
processing_task_finished=False,
start_processing_task=None)
self.patch_methods(entry_b, processing_task_started=False,
processing_task_finished=False,
start_processing_task=None)
self.queue.process_queue()
self.assertFalse(entry_a.start_processing_task.called)
entry_b.start_processing_task.assert_called_once_with()
def test_process_queue_handles_processing_task_result(self):
"""Mails being processed are handled when finished"""
self.patch_mail_processor()
entry_a, entry_b = self.add_mails_to_queue('a', 'b')
self.patch_methods(entry_a, processing_task_started=True,
processing_task_finished=False,
handle_processing_task_result=None)
self.patch_methods(entry_b, processing_task_started=True,
processing_task_finished=True,
handle_processing_task_result=None)
self.queue.process_queue()
entry_a.processing_task_finished.assert_called_once_with()
entry_b.processing_task_finished.assert_called_once_with()
self.assertFalse(entry_a.handle_processing_task_result.called)
entry_b.handle_processing_task_result.assert_called_once_with()
def test_process_queue_works_when_queue_items_are_removed(self):
"""The processing of entries results in entries being dropped. This
should not confuse process_queue which should still properly
process all entries"""
queue = ['a', 'b', 'c', 'd', 'e', 'f']
self.queue._count_mock_calls = 0
for entry in self.add_mails_to_queue(*queue):
def side_effect():
entry.queue._count_mock_calls += 1
entry.queue.remove(entry.identifier)
return False
self.patch_methods(entry, processing_task_started=True,
processing_task_finished=side_effect)
self.queue.process_queue()
self.assertEqual(self.queue._count_mock_calls, len(queue))
def test_sleep_timeout_mailqueue_empty(self):
self.assertEqual(self.queue.sleep_timeout(),
self.queue.SLEEP_TIMEOUT_EMPTY)
def _add_entry_task_running(self, name):
entry = self.add_mails_to_queue(name)[0]
self.patch_methods(entry, processing_task_started=True,
processing_task_finished=False)
return entry
def test_sleep_timeout_task_started_not_finished(self):
self._add_entry_task_running('a')
self.assertEqual(self.queue.sleep_timeout(),
self.queue.SLEEP_TIMEOUT_TASK_RUNNING)
def _add_entry_task_finished(self, name):
entry = self.add_mails_to_queue(name)[0]
self.patch_methods(entry, processing_task_started=True,
processing_task_finished=True)
return entry
def test_sleep_timeout_task_finished(self):
self._add_entry_task_finished('a')
self.assertEqual(self.queue.sleep_timeout(),
self.queue.SLEEP_TIMEOUT_TASK_FINISHED)
def _add_entry_task_waiting_next_try(self, name):
entry = self.add_mails_to_queue(name)[0]
entry.schedule_next_try()
return entry
def _get_wait_time(self, entry):
wait_time = entry.get_data('next_try_time') - self.current_datetime
return wait_time.total_seconds()
def test_sleep_timeout_task_waiting_next_try(self):
self.patch_now()
entry = self._add_entry_task_waiting_next_try('a')
wait_time = self._get_wait_time(entry)
self.assertEqual(self.queue.sleep_timeout(), wait_time)
def _add_entry_task_runnable(self, name):
entry = self.add_mails_to_queue(name)[0]
return entry
def test_sleep_timeout_task_runnable(self):
self._add_entry_task_runnable('a')
self.assertEqual(self.queue.sleep_timeout(),
self.queue.SLEEP_TIMEOUT_TASK_RUNNABLE)
def test_sleep_timeout_picks_the_shorter_wait_time(self):
self.patch_now()
self._add_entry_task_running('a')
self._add_entry_task_runnable('b')
self._add_entry_task_finished('c')
entry_d = self._add_entry_task_waiting_next_try('d')
wait_time = self._get_wait_time(entry_d)
self.assertEqual(self.queue.sleep_timeout(),
min(self.queue.SLEEP_TIMEOUT_TASK_RUNNING,
self.queue.SLEEP_TIMEOUT_TASK_RUNNABLE,
self.queue.SLEEP_TIMEOUT_TASK_FINISHED,
wait_time))
def start_process_loop(self, stop_after=None):
"""
Start process_loop() in a dedicated process and ensure it's
ready to proocess new files before returning
"""
self.mkdir(self.queue._get_maildir())
lock = multiprocessing.Lock()
lock.acquire()
def process_loop(lock):
def release_lock():
lock.release()
queue = MailQueue()
queue.process_loop(stop_after=stop_after, ready_cb=release_lock)
process = multiprocessing.Process(target=process_loop, args=(lock,))
process.start()
lock.acquire()
return process
def test_process_loop_processes_one_mail(self):
self.patch_mail_processor()
process = self.start_process_loop(stop_after=1)
# The mail is created after process_loop() is ready
path = self.create_mail('a')
# We wait the end of the task for max 1 second
process.join(1)
# Process finished successfully (and we're not here due to timeout)
if process.is_alive():
process.terminate()
self.fail("process_loop did not terminate")
self.assertFalse(process.is_alive())
self.assertEqual(process.exitcode, 0)
# And it did its job by handling the mail
self.assertFalse(os.path.exists(path))
@mock.patch('distro_tracker.mail.processor.MailQueueWatcher')
def test_process_loop_calls_sleep_timeout(self, mock_watcher):
"""Ensure we feed the sleep timeout to watcher.process_events"""
self.mkdir(self.queue._get_maildir())
self.patch_methods(self.queue, sleep_timeout=mock.sentinel.DELAY)
self.queue.process_loop(stop_after=0)
mock_watcher.return_value.process_events.assert_called_with(
timeout=mock.sentinel.DELAY)
@mock.patch('distro_tracker.mail.processor.MailQueueWatcher')
def test_process_loop_calls_initialize(self, mock_watcher):
"""Ensure we call the initialize method before announcing readyness"""
self.patch_methods(self.queue, initialize=None)
def check_when_ready():
self.queue.initialize.assert_called_with()
self.queue.process_loop(stop_after=0, ready_cb=check_when_ready)
def process_loop(lock):
def release_lock():
lock.release()
queue = MailQueue()
queue.process_loop(stop_after=stop_after, ready_cb=release_lock)
def handle(self, *args, **kwargs):
queue = MailQueue()
queue.process_loop() # Never returns
