def test_RunnableQueue_resubmitted_jobs(mocker):
'''
Verify that jobs that fail due to timeout are resubmitted without modifying order_number.
'''
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
mock_session_maker = mocker.MagicMock(return_value=mock_session)
job_cls_high_priority = factory.dummy_job_factory(mocker, 'mock')
job_cls_low_priority = factory.dummy_job_factory(mocker, 'mock')
queue = RunnableQueue(mock_api_client, mock_session_maker)
queue.JOB_PRIORITIES = {job_cls_high_priority: 1, job_cls_low_priority: 2}
job1 = job_cls_high_priority()
job2 = job_cls_low_priority()
job3 = job_cls_high_priority()
job4 = job_cls_low_priority()
queue.add_job(job1)
queue.add_job(job2)
queue.add_job(job3)
queue.add_job(job4)
# Expected order of execution is job1 -> job3 -> job2 -> job4
assert queue.queue.get(block=True) == (1, job1)
# Now resubmit job1 via put_nowait. It should execute prior to job2-4.
queue.re_add_job(job1)
assert queue.queue.get(block=True) == (1, job1)
assert queue.queue.get(block=True) == (1, job3)
assert queue.queue.get(block=True) == (2, job2)
assert queue.queue.get(block=True) == (2, job4)
def test_RunnableQueue_high_priority_jobs_run_first_and_in_fifo_order(mocker):
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
mock_session_maker = mocker.MagicMock(return_value=mock_session)
return_value = 'wat'
job_cls_high_priority = factory.dummy_job_factory(mocker, return_value)
job_cls_low_priority = factory.dummy_job_factory(mocker, return_value)
queue = RunnableQueue(mock_api_client, mock_session_maker)
queue.JOB_PRIORITIES = {job_cls_high_priority: 1, job_cls_low_priority: 2}
job1 = job_cls_high_priority()
job2 = job_cls_low_priority()
job3 = job_cls_high_priority()
job4 = job_cls_low_priority()
queue.add_job(job1)
queue.add_job(job2)
queue.add_job(job3)
queue.add_job(job4)
# Expected order of execution is job1 -> job3 -> job2 -> job4
assert queue.queue.get(block=True) == (1, job1)
assert queue.queue.get(block=True) == (1, job3)
assert queue.queue.get(block=True) == (2, job2)
assert queue.queue.get(block=True) == (2, job4)
def test_ApiJobQueue_enqueue_when_queues_are_not_running(mocker):
mock_client = mocker.MagicMock()
mock_session_maker = mocker.MagicMock()
job_queue = ApiJobQueue(mock_client, mock_session_maker)
job_priority = 2
dummy_job = factory.dummy_job_factory(mocker, 'mock')()
job_queue.JOB_PRIORITIES = {
FileDownloadJob: job_priority,
type(dummy_job): job_priority
}
mock_download_file_queue = mocker.patch.object(job_queue,
'download_file_queue')
mock_main_queue = mocker.patch.object(job_queue, 'main_queue')
mock_download_file_add_job = mocker.patch.object(mock_download_file_queue,
'add_job')
mock_main_queue_add_job = mocker.patch.object(mock_main_queue, 'add_job')
job_queue.main_queue.api_client = 'has a value'
job_queue.download_file_queue.api_client = 'has a value'
job_queue.stop(
) # queues are already not running, but just in case the code changes one day
dl_job = FileDownloadJob('mock', 'mock', 'mock')
job_queue.enqueue(dl_job)
mock_download_file_add_job.assert_not_called()
mock_main_queue_add_job.assert_not_called()
def test_RunnableQueue_job_timeout(mocker, exception):
'''
Add two jobs to the queue. The first times out, and then gets resubmitted for the next pass
through the loop.
'''
queue = RunnableQueue(mocker.MagicMock(), mocker.MagicMock())
queue.pause = mocker.MagicMock()
job_cls = factory.dummy_job_factory(mocker,
exception(),
remaining_attempts=5)
job1 = job_cls()
job2 = job_cls()
queue.JOB_PRIORITIES = {PauseQueueJob: 0, job_cls: 1}
# RequestTimeoutError or ServerConnectionError will cause the queue to pause,
# use our fake pause method instead
def fake_pause() -> None:
queue.add_job(PauseQueueJob())
queue.pause.emit = fake_pause
# Add two jobs that timeout during processing to the queue
queue.add_job(job1)
queue.add_job(job2)
# attempt to process job1 knowing that it times out
queue.process()
assert queue.queue.qsize() == 2 # queue contains: job1, job2
# now process after making it so job1 no longer times out
job1.return_value = 'mock'
queue.process()
assert queue.queue.qsize() == 1 # queue contains: job2
assert queue.queue.get(block=True) == (1, job2)
def test_ApiJob_order_number_unset(mocker):
return_value = "wat"
api_job_cls = dummy_job_factory(mocker, return_value)
api_job_1 = api_job_cls()
api_job_2 = api_job_cls()
with pytest.raises(ValueError):
api_job_1 < api_job_2
def test_ApiJob_comparison(mocker):
return_value = "wat"
api_job_cls = dummy_job_factory(mocker, return_value)
api_job_1 = api_job_cls()
api_job_1.order_number = 1
api_job_2 = api_job_cls()
api_job_2.order_number = 2
assert api_job_1 < api_job_2
def test_ApiJob_success(mocker):
return_value = "wat"
api_job_cls = dummy_job_factory(mocker, return_value)
api_job = api_job_cls()
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
api_job._do_call_api(mock_api_client, mock_session)
api_job.success_signal.emit.assert_called_once_with(return_value)
assert not api_job.failure_signal.emit.called
def test_ApiJob_no_api(mocker):
return_value = "wat"
api_job_cls = dummy_job_factory(mocker, return_value)
api_job = api_job_cls()
mock_session = mocker.MagicMock()
with pytest.raises(ApiInaccessibleError):
api_job._do_call_api(None, mock_session)
assert not api_job.success_signal.emit.called
assert not api_job.failure_signal.emit.called
def test_ApiJob_other_error(mocker):
return_value = Exception()
api_job_cls = dummy_job_factory(mocker, return_value)
api_job = api_job_cls()
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
with pytest.raises(Exception):
api_job._do_call_api(mock_api_client, mock_session)
assert not api_job.success_signal.emit.called
api_job.failure_signal.emit.assert_called_once_with(return_value)
def test_ApiJob_timeout_error(mocker):
return_value = RequestTimeoutError()
api_job_cls = dummy_job_factory(mocker, return_value)
api_job = api_job_cls()
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
with pytest.raises(RequestTimeoutError):
api_job._do_call_api(mock_api_client, mock_session)
assert not api_job.success_signal.emit.called
assert api_job.failure_signal.emit.called
def test_ApiJob_auth_error(mocker):
return_value = AuthError("oh no")
api_job_cls = dummy_job_factory(mocker, return_value)
api_job = api_job_cls()
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
with pytest.raises(ApiInaccessibleError):
api_job._do_call_api(mock_api_client, mock_session)
assert not api_job.success_signal.emit.called
assert not api_job.failure_signal.emit.called
def test_ApiJobQueue_enqueue_when_queues_are_running(mocker):
mock_client = mocker.MagicMock()
mock_session_maker = mocker.MagicMock()
job_queue = ApiJobQueue(mock_client, mock_session_maker)
job_priority = 2
dummy_job = factory.dummy_job_factory(mocker, 'mock')()
job_queue.JOB_PRIORITIES = {
FileDownloadJob: job_priority,
type(dummy_job): job_priority
}
mock_download_file_queue = mocker.patch.object(job_queue,
'download_file_queue')
mock_main_queue = mocker.patch.object(job_queue, 'main_queue')
mock_download_file_add_job = mocker.patch.object(mock_download_file_queue,
'add_job')
mock_main_queue_add_job = mocker.patch.object(mock_main_queue, 'add_job')
job_queue.main_queue.api_client = 'has a value'
job_queue.download_file_queue.api_client = 'has a value'
job_queue.main_thread.isRunning = mocker.MagicMock(return_value=True)
job_queue.download_file_thread.isRunning = mocker.MagicMock(
return_value=True)
dl_job = FileDownloadJob('mock', 'mock', 'mock')
job_queue.enqueue(dl_job)
mock_download_file_add_job.assert_called_once_with(dl_job)
assert not mock_main_queue_add_job.called
# reset for next test
mock_download_file_queue.reset_mock()
mock_download_file_add_job.reset_mock()
mock_main_queue.reset_mock()
mock_main_queue_add_job.reset_mock()
job_queue.enqueue(FileDownloadJob('mock', 'mock', 'mock'))
assert not mock_main_queue_add_job.called
# reset for next test
mock_download_file_queue.reset_mock()
mock_download_file_add_job.reset_mock()
mock_main_queue.reset_mock()
mock_main_queue_add_job.reset_mock()
job_queue.enqueue(dummy_job)
mock_main_queue_add_job.assert_called_once_with(dummy_job)
assert not mock_download_file_add_job.called
def test_RunnableQueue_job_generic_exception(mocker):
'''
Add two jobs to the queue, the first of which will cause a generic exception, which is handled
in _do_call_api. Ensure that the queue continues processing jobs after dropping a job that
runs into a generic exception.
'''
job1_cls = factory.dummy_job_factory(mocker,
Exception()) # processing skips job
job2_cls = factory.dummy_job_factory(mocker, 'mock')
job1 = job1_cls()
job2 = job2_cls()
queue = RunnableQueue(mocker.MagicMock(), mocker.MagicMock())
queue.JOB_PRIORITIES = {PauseQueueJob: 3, job1_cls: 2, job2_cls: 2}
queue.add_job(job1)
queue.add_job(job2)
queue.add_job(
PauseQueueJob()) # Pause queue so our test exits the processing loop
queue.process()
# check that all jobs are gone
assert queue.queue.empty()
def test_ApiJob_timeout_error(mocker, exception):
"""If the server times out or is unreachable, the corresponding
exception should be raised"""
return_value = exception()
api_job_cls = dummy_job_factory(mocker, return_value)
api_job = api_job_cls()
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
with pytest.raises(exception):
api_job._do_call_api(mock_api_client, mock_session)
assert not api_job.success_signal.emit.called
assert api_job.failure_signal.emit.called
def test_ApiJob_retry_suceeds_after_failed_attempt(mocker, exception):
"""Retry logic: after failed attempt should succeed"""
number_of_attempts = 5
success_return_value = "now works"
return_values = [exception(), success_return_value]
api_job_cls = dummy_job_factory(mocker, return_values, remaining_attempts=number_of_attempts)
api_job = api_job_cls()
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
api_job._do_call_api(mock_api_client, mock_session)
assert api_job.remaining_attempts == number_of_attempts - 2
assert not api_job.failure_signal.emit.called
api_job.success_signal.emit.assert_called_once_with(success_return_value)
def test_ApiJob_retry_exactly_n_attempts_times(mocker, exception):
"""Retry logic: boundary value case - 5th attempt should succeed"""
number_of_attempts = 5
success_return_value = "now works"
return_values = [exception()] * (number_of_attempts - 1) + [success_return_value]
api_job_cls = dummy_job_factory(mocker, return_values, remaining_attempts=number_of_attempts)
api_job = api_job_cls()
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
api_job._do_call_api(mock_api_client, mock_session)
assert api_job.remaining_attempts == 0
assert not api_job.failure_signal.emit.called
api_job.success_signal.emit.assert_called_once_with(success_return_value)
def test_RunnableQueue_does_not_run_jobs_when_not_authed(mocker):
'''
Check that a job that sees an ApiInaccessibleError does not get resubmitted since it is not
authorized and that its api_client is None.
'''
queue = RunnableQueue(mocker.MagicMock(), mocker.MagicMock())
job_cls = factory.dummy_job_factory(mocker, ApiInaccessibleError())
queue.JOB_PRIORITIES = {PauseQueueJob: 0, job_cls: 1}
# Add a job that results in an ApiInaccessibleError to the queue
job = job_cls()
queue.add_job(job)
# attempt to process job knowing that it errors
queue.process()
assert queue.queue.qsize(
) == 0 # queue should not contain job since it was not resubmitted
assert queue.api_client is None
def test_ApiJob_retry_timeout(mocker, exception):
"""Retry logic: If we exceed the number of attempts, the job will still fail"""
number_of_attempts = 5
return_values = [exception()] * (number_of_attempts + 1)
api_job_cls = dummy_job_factory(mocker, return_values, remaining_attempts=number_of_attempts)
api_job = api_job_cls()
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
with pytest.raises(exception):
api_job._do_call_api(mock_api_client, mock_session)
assert api_job.remaining_attempts == 0
assert not api_job.success_signal.emit.called
assert api_job.failure_signal.emit.called
def test_RunnableQueue_happy_path(mocker):
'''
Add one job to the queue, run it.
'''
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
mock_session_maker = mocker.MagicMock(return_value=mock_session)
return_value = 'foo'
dummy_job_cls = factory.dummy_job_factory(mocker, return_value)
queue = RunnableQueue(mock_api_client, mock_session_maker)
queue.JOB_PRIORITIES = {dummy_job_cls: 1, PauseQueueJob: 2}
queue.add_job(dummy_job_cls())
queue.add_job(
PauseQueueJob()) # Pause queue so our test exits the processing loop
queue.process()
assert queue.queue.empty()
def test_ApiJobQueue_enqueue_no_auth(mocker):
mock_client = mocker.MagicMock()
mock_session_maker = mocker.MagicMock()
job_queue = ApiJobQueue(mock_client, mock_session_maker)
mock_download_file_queue = mocker.patch.object(job_queue,
'download_file_queue')
mock_main_queue = mocker.patch.object(job_queue, 'main_queue')
mock_download_file_add_job = mocker.patch.object(mock_download_file_queue,
'add_job')
mock_main_queue_add_job = mocker.patch.object(mock_main_queue, 'add_job')
job_queue.main_queue.api_client = None
job_queue.download_file_queue.api_client = None
dummy_job = factory.dummy_job_factory(mocker, 'mock')()
job_queue.JOB_PRIORITIES = {type(dummy_job): 1}
job_queue.enqueue(dummy_job)
assert mock_download_file_add_job.call_count == 0
assert mock_main_queue_add_job.call_count == 0
def test_RunnableQueue_does_not_run_jobs_when_not_authed(mocker):
'''
Add a job to the queue, ensure we don't run it when not authenticated.
'''
queue = RunnableQueue(mocker.MagicMock(), mocker.MagicMock())
queue.pause = mocker.MagicMock()
job_cls = factory.dummy_job_factory(mocker, ApiInaccessibleError())
queue.JOB_PRIORITIES = {PauseQueueJob: 0, job_cls: 1}
# ApiInaccessibleError will cause the queue to pause, use our fake pause method instead
def fake_pause() -> None:
queue.add_job(PauseQueueJob())
queue.pause.emit = fake_pause
# Add a job that results in an ApiInaccessibleError to the queue
job = job_cls()
queue.add_job(job)
# attempt to process job1 knowing that it times out
queue.process()
assert queue.queue.qsize() == 1 # queue contains: job1
