def test_running_job(broker):
running_jobs_key = broker._to_namespaced(
RUNNING_JOBS_KEY.format(broker._id))
# Non-idempotent job
job = Job('foo_task', 'foo_queue', datetime.now(timezone.utc), 0)
broker.enqueue_jobs([job])
assert broker._r.hget(running_jobs_key, str(job.id)) is None
broker.get_jobs_from_queue('foo_queue', 1)
assert broker._r.hget(running_jobs_key, str(job.id)) is None
# Try to remove it, even if it doesn't exist in running
broker.remove_job_from_running(job)
# Idempotent job - get from queue
job = Job('foo_task', 'foo_queue', datetime.now(timezone.utc), 10)
broker.enqueue_jobs([job])
assert broker._r.hget(running_jobs_key, str(job.id)) is None
broker.get_jobs_from_queue('foo_queue', 1)
job.status = JobStatus.RUNNING
assert (Job.deserialize(
broker._r.hget(running_jobs_key, str(job.id)).decode()) == job)
# Idempotent job - re-enqueue after job ran with error
job.retries += 1
broker.enqueue_jobs([job])
assert broker._r.hget(running_jobs_key, str(job.id)) is None
broker.get_jobs_from_queue('foo_queue', 1)
job.status = JobStatus.RUNNING
assert (Job.deserialize(
broker._r.hget(running_jobs_key, str(job.id)).decode()) == job)
# Idempotent job - job succeeded
broker.remove_job_from_running(job)
assert broker._r.hget(running_jobs_key, str(job.id)) is None
assert broker.get_jobs_from_queue('foo_queue', 1) == []
def test_idempotency_token(_, broker):
job_1 = Job('foo_task', 'foo_queue', datetime.now(timezone.utc), 0)
job_2 = Job('foo_task', 'foo_queue', datetime.now(timezone.utc), 0)
broker.enqueue_jobs([job_1])
broker.enqueue_jobs([job_2])
jobs = broker.get_jobs_from_queue('foo_queue', max_jobs=10)
job_1.status = JobStatus.RUNNING
assert jobs == [job_1]
def test_normal_job(broker):
job = Job('foo_task',
'foo_queue',
datetime.now(timezone.utc),
0,
task_args=(1, 2),
task_kwargs={'foo': 'bar'})
broker.enqueue_jobs([job])
assert job.status == JobStatus.QUEUED
job.status = JobStatus.RUNNING
assert broker.get_jobs_from_queue('foo_queue', 5) == [job]
assert broker.get_jobs_from_queue('foo_queue', 1) == []
def test_job_ran(broker):
now = datetime.now(timezone.utc)
job = Job('foo_task',
'foo_queue',
now,
0,
task_args=(1, 2),
task_kwargs={'foo': 'bar'})
job.status = JobStatus.RUNNING
broker.job_ran(job, None)
assert job.status is JobStatus.SUCCEEDED
job.status = JobStatus.RUNNING
broker.job_ran(job, RuntimeError('Error'))
assert job.status is JobStatus.FAILED
job.status = JobStatus.RUNNING
job.max_retries = 10
broker.job_ran(job, RuntimeError('Error'))
assert job.status is JobStatus.WAITING
assert job.at > now
def test_decrements_concurrency_count_when_job_fails(broker):
set_concurrency_keys(broker)
job = Job(
CONCURRENT_TASK_NAME,
'foo_queue',
datetime.now(timezone.utc),
max_retries=10,
)
broker.enqueue_jobs([job])
broker.get_jobs_from_queue('foo_queue', 1)
current = broker._r.hget(broker._to_namespaced(CURRENT_CONCURRENCY_KEY),
CONCURRENT_TASK_NAME)
assert current == b'1'
job.status = JobStatus.NOT_SET
job.retries += 1
broker.enqueue_jobs([job], from_failure=True)
current = broker._r.hget(broker._to_namespaced(CURRENT_CONCURRENCY_KEY),
CONCURRENT_TASK_NAME)
assert current == b'0'
def test_enqueue_jobs_from_dead_broker(broker, broker_2):
# Enqueue one idempotent job and one non-idempotent job
job_1 = Job('foo_task', 'foo_queue', datetime.now(timezone.utc), 0)
job_2 = Job('foo_task', 'foo_queue', datetime.now(timezone.utc), 10)
broker.enqueue_jobs([job_1, job_2])
# Simulate broker starting the jobs
broker.get_jobs_from_queue('foo_queue', 100)
# Mark broker as dead, should re-enqueue only the idempotent job
assert broker_2.enqueue_jobs_from_dead_broker(broker._id) == 1
# Simulate broker 2 getting jobs from the queue
job_2.status = JobStatus.RUNNING
job_2.retries = 1
assert broker_2.get_jobs_from_queue('foo_queue', 100) == [job_2]
# Check that a broker can be marked as dead multiple times
# without duplicating jobs
assert broker_2.enqueue_jobs_from_dead_broker(broker._id) == 0
def test_decrements_concurrency_count_when_job_ends(broker):
set_concurrency_keys(broker)
job = Job(
CONCURRENT_TASK_NAME,
'foo_queue',
datetime.now(timezone.utc),
1,
# kwargs help with debugging but are not part of the test.
task_kwargs=dict(name='job1'),
)
broker.enqueue_jobs([job])
returned_jobs = broker.get_jobs_from_queue('foo_queue', 2)
assert len(returned_jobs) == CONCURRENT_TASK_MAX_CONCURRENCY
current = broker._r.hget(broker._to_namespaced(CURRENT_CONCURRENCY_KEY),
CONCURRENT_TASK_NAME)
assert current == b'1'
broker.get_jobs_from_queue('foo_queue', 1)
job.status = JobStatus.RUNNING
broker.remove_job_from_running(job)
current = broker._r.hget(broker._to_namespaced(CURRENT_CONCURRENCY_KEY),
CONCURRENT_TASK_NAME)
assert current == b'0'
def test_future_job(broker, patch_now):
assert broker.next_future_job_delta is None
assert broker.move_future_jobs() == 0
job = Job('foo_task',
'foo_queue',
get_now() + timedelta(minutes=10),
0,
task_args=(1, 2),
task_kwargs={'foo': 'bar'})
broker.enqueue_jobs([job])
assert job.status == JobStatus.WAITING
assert broker.get_jobs_from_queue('foo_queue', 5) == []
assert broker.next_future_job_delta == 600
assert broker.move_future_jobs() == 0
set_now(datetime(2017, 9, 2, 9, 00, 56, 482169))
assert broker.next_future_job_delta == 0
assert broker.move_future_jobs() == 1
job.status = JobStatus.RUNNING
assert broker.get_jobs_from_queue('foo_queue', 5) == [job]