def test_RunnableQueue_resubmitted_jobs(mocker):
'''
Verify that jobs that fail due to timeout are resubmitted without modifying order_number.
'''
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
mock_session_maker = mocker.MagicMock(return_value=mock_session)
job_cls_high_priority = factory.dummy_job_factory(mocker, 'mock')
job_cls_low_priority = factory.dummy_job_factory(mocker, 'mock')
queue = RunnableQueue(mock_api_client, mock_session_maker)
queue.JOB_PRIORITIES = {job_cls_high_priority: 1, job_cls_low_priority: 2}
job1 = job_cls_high_priority()
job2 = job_cls_low_priority()
job3 = job_cls_high_priority()
job4 = job_cls_low_priority()
queue.add_job(job1)
queue.add_job(job2)
queue.add_job(job3)
queue.add_job(job4)
# Expected order of execution is job1 -> job3 -> job2 -> job4
assert queue.queue.get(block=True) == (1, job1)
# Now resubmit job1 via put_nowait. It should execute prior to job2-4.
queue.re_add_job(job1)
assert queue.queue.get(block=True) == (1, job1)
assert queue.queue.get(block=True) == (1, job3)
assert queue.queue.get(block=True) == (2, job2)
assert queue.queue.get(block=True) == (2, job4)
def test_RunnableQueue_job_timeout(mocker, exception):
'''
Add two jobs to the queue. The first times out, and then gets resubmitted for the next pass
through the loop.
'''
queue = RunnableQueue(mocker.MagicMock(), mocker.MagicMock())
queue.pause = mocker.MagicMock()
job_cls = factory.dummy_job_factory(mocker,
exception(),
remaining_attempts=5)
job1 = job_cls()
job2 = job_cls()
queue.JOB_PRIORITIES = {PauseQueueJob: 0, job_cls: 1}
# RequestTimeoutError or ServerConnectionError will cause the queue to pause,
# use our fake pause method instead
def fake_pause() -> None:
queue.add_job(PauseQueueJob())
queue.pause.emit = fake_pause
# Add two jobs that timeout during processing to the queue
queue.add_job(job1)
queue.add_job(job2)
# attempt to process job1 knowing that it times out
queue.process()
assert queue.queue.qsize() == 2 # queue contains: job1, job2
# now process after making it so job1 no longer times out
job1.return_value = 'mock'
queue.process()
assert queue.queue.qsize() == 1 # queue contains: job2
assert queue.queue.get(block=True) == (1, job2)
def test_RunnableQueue_high_priority_jobs_run_first_and_in_fifo_order(mocker):
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
mock_session_maker = mocker.MagicMock(return_value=mock_session)
return_value = 'wat'
job_cls_high_priority = factory.dummy_job_factory(mocker, return_value)
job_cls_low_priority = factory.dummy_job_factory(mocker, return_value)
queue = RunnableQueue(mock_api_client, mock_session_maker)
queue.JOB_PRIORITIES = {job_cls_high_priority: 1, job_cls_low_priority: 2}
job1 = job_cls_high_priority()
job2 = job_cls_low_priority()
job3 = job_cls_high_priority()
job4 = job_cls_low_priority()
queue.add_job(job1)
queue.add_job(job2)
queue.add_job(job3)
queue.add_job(job4)
# Expected order of execution is job1 -> job3 -> job2 -> job4
assert queue.queue.get(block=True) == (1, job1)
assert queue.queue.get(block=True) == (1, job3)
assert queue.queue.get(block=True) == (2, job2)
assert queue.queue.get(block=True) == (2, job4)
def test_RunnableQueue_process_PauseQueueJob(mocker):
api_client = mocker.MagicMock()
session_maker = mocker.MagicMock(return_value=mocker.MagicMock())
queue = RunnableQueue(api_client, session_maker)
queue.JOB_PRIORITIES = {PauseQueueJob: 11}
queue.add_job(PauseQueueJob())
queue.process()
assert queue.queue.empty()
def test_RunnableQueue_does_not_run_jobs_when_not_authed(mocker):
'''
Check that a job that sees an ApiInaccessibleError does not get resubmitted since it is not
authorized and that its api_client is None.
'''
queue = RunnableQueue(mocker.MagicMock(), mocker.MagicMock())
job_cls = factory.dummy_job_factory(mocker, ApiInaccessibleError())
queue.JOB_PRIORITIES = {PauseQueueJob: 0, job_cls: 1}
# Add a job that results in an ApiInaccessibleError to the queue
job = job_cls()
queue.add_job(job)
# attempt to process job knowing that it errors
queue.process()
assert queue.queue.qsize(
) == 0 # queue should not contain job since it was not resubmitted
assert queue.api_client is None
def test_RunnableQueue_happy_path(mocker):
'''
Add one job to the queue, run it.
'''
mock_api_client = mocker.MagicMock()
mock_session = mocker.MagicMock()
mock_session_maker = mocker.MagicMock(return_value=mock_session)
return_value = 'foo'
dummy_job_cls = factory.dummy_job_factory(mocker, return_value)
queue = RunnableQueue(mock_api_client, mock_session_maker)
queue.JOB_PRIORITIES = {dummy_job_cls: 1, PauseQueueJob: 2}
queue.add_job(dummy_job_cls())
queue.add_job(
PauseQueueJob()) # Pause queue so our test exits the processing loop
queue.process()
assert queue.queue.empty()
def test_RunnableQueue_does_not_run_jobs_when_not_authed(mocker):
'''
Add a job to the queue, ensure we don't run it when not authenticated.
'''
queue = RunnableQueue(mocker.MagicMock(), mocker.MagicMock())
queue.pause = mocker.MagicMock()
job_cls = factory.dummy_job_factory(mocker, ApiInaccessibleError())
queue.JOB_PRIORITIES = {PauseQueueJob: 0, job_cls: 1}
# ApiInaccessibleError will cause the queue to pause, use our fake pause method instead
def fake_pause() -> None:
queue.add_job(PauseQueueJob())
queue.pause.emit = fake_pause
# Add a job that results in an ApiInaccessibleError to the queue
job = job_cls()
queue.add_job(job)
# attempt to process job1 knowing that it times out
queue.process()
assert queue.queue.qsize() == 1 # queue contains: job1
def test_RunnableQueue_job_generic_exception(mocker):
'''
Add two jobs to the queue, the first of which will cause a generic exception, which is handled
in _do_call_api. Ensure that the queue continues processing jobs after dropping a job that
runs into a generic exception.
'''
job1_cls = factory.dummy_job_factory(mocker,
Exception()) # processing skips job
job2_cls = factory.dummy_job_factory(mocker, 'mock')
job1 = job1_cls()
job2 = job2_cls()
queue = RunnableQueue(mocker.MagicMock(), mocker.MagicMock())
queue.JOB_PRIORITIES = {PauseQueueJob: 3, job1_cls: 2, job2_cls: 2}
queue.add_job(job1)
queue.add_job(job2)
queue.add_job(
PauseQueueJob()) # Pause queue so our test exits the processing loop
queue.process()
# check that all jobs are gone
assert queue.queue.empty()