def test_freeze_objects_already_running(self, session, client,
museum_object_factory):
"""
Test freezing two objects that already have running jobs
"""
def successful_job():
return ":)"
confirm_queue = get_queue(QueueType.CONFIRM_SIP)
started_registry = StartedJobRegistry(queue=confirm_queue)
for i in [5, 10]:
museum_object_factory(id=i)
job = confirm_queue.enqueue(successful_job,
job_id=f"download_object_{i}")
started_registry.add(job, -1)
result = client.post("/web-ui/freeze-objects/",
data={
"reason": "Test reason",
"object_ids": "10\n5"
})
assert (escape(
"following object IDs have running jobs and can't be frozen: "
"5, 10").encode("utf-8") in result.data)
def test_get_expiration_time(self):
"""registry.get_expiration_time() returns correct datetime objects"""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
registry.add(job, 5)
self.assertEqual(registry.get_expiration_time(job),
(datetime.utcnow() +
timedelta(seconds=5)).replace(microsecond=0))
def test_started_jobs(self):
"""Ensure that active jobs page works properly."""
queue = get_queue('django_rq_test')
queue_index = get_queue_index('django_rq_test')
job = queue.enqueue(access_self)
registry = StartedJobRegistry(queue.name, queue.connection)
registry.add(job, 2)
response = self.client.get(
reverse('rq_started_jobs', args=[queue_index]))
self.assertEqual(response.context['jobs'], [job])
def test_add_and_remove_with_serializer(self):
"""Adding and removing job to StartedJobRegistry (with serializer)."""
# delete_job = True also works with job.id and custom serializer
queue = Queue(connection=self.testconn, serializer=JSONSerializer)
registry = StartedJobRegistry(connection=self.testconn,
serializer=JSONSerializer)
job = queue.enqueue(say_hello)
registry.add(job, -1)
registry.remove(job.id, delete_job=True)
self.assertIsNone(self.testconn.zscore(registry.key, job.id))
self.assertFalse(self.testconn.exists(job.key))
def test_contains(self):
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.assertFalse(job in registry)
self.assertFalse(job.id in registry)
registry.add(job, 5)
self.assertTrue(job in registry)
self.assertTrue(job.id in registry)
def test_get_expiration_time(self):
"""registry.get_expiration_time() returns correct datetime objects"""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
registry.add(job, 5)
time = registry.get_expiration_time(job)
expected_time = (datetime.utcnow() +
timedelta(seconds=5)).replace(microsecond=0)
self.assertGreaterEqual(time, expected_time - timedelta(seconds=2))
self.assertLessEqual(time, expected_time + timedelta(seconds=2))
def test_started_jobs(self):
"""Ensure that active jobs page works properly."""
queue = get_queue('django_rq_test')
queue_index = get_queue_index('django_rq_test')
job = queue.enqueue(access_self)
registry = StartedJobRegistry(queue.name, queue.connection)
registry.add(job, 2)
response = self.client.get(
reverse('rq_started_jobs', args=[queue_index])
)
self.assertEqual(response.context['jobs'], [job])
def test_contains(self):
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.assertFalse(job in registry)
self.assertFalse(job.id in registry)
registry.add(job, 5)
self.assertTrue(job in registry)
self.assertTrue(job.id in registry)
def test_freeze_objects_running_jobs(session, redis, freeze_objects,
museum_object_factory):
"""
Try freezing two objects when they have running jobs.
"""
def successful_job():
return ":)"
museum_object_factory(id=123456)
museum_object_factory(id=654321)
queue = get_queue(QueueType.DOWNLOAD_OBJECT)
started_registry = StartedJobRegistry(queue=queue)
job_a = queue.enqueue(successful_job, job_id="download_object_123456")
job_b = queue.enqueue(successful_job, job_id="download_object_654321")
started_registry.add(job_a, -1)
started_registry.add(job_b, -1)
with pytest.raises(WorkflowJobRunningError) as exc:
freeze_objects(["--reason", "Won't succeed", "654321", "123456"],
success=False)
assert "can't be frozen: 123456, 654321" in str(exc.value)
def test_navbar_stats(self, session, client):
# Create 1 'download_object' job
get_queue(QueueType.DOWNLOAD_OBJECT).enqueue(
successful_job, job_id="download_object_1")
# Create 2 'create_sip' jobs
for i in range(2, 4):
get_queue(QueueType.CREATE_SIP).enqueue(successful_job,
job_id=f"create_sip_{i}")
# Create 1 failed 'submit_sip' job
submit_queue = get_queue(QueueType.SUBMIT_SIP)
submit_queue.enqueue(failing_job, job_id="submit_sip_4")
SimpleWorker([submit_queue],
connection=submit_queue.connection).work(burst=True)
# Create 1 started 'confirm_sip' job
confirm_queue = get_queue(QueueType.CONFIRM_SIP)
started_registry = StartedJobRegistry(queue=confirm_queue)
job = confirm_queue.enqueue(successful_job, job_id="confirm_sip_5")
started_registry.add(job, -1)
result = client.get("/api/navbar-stats").json
assert result["queues"]["download_object"] \
== {"processing": 0, "pending": 1}
assert result["queues"]["create_sip"] == \
{"processing": 0, "pending": 2}
assert result["queues"]["submit_sip"] == \
{"processing": 0, "pending": 0}
# TODO: In practice, if one worker is working on a job and there are
# no pending jobs, this should be 'processing': 1, 'pending': 0.
# How can we mimic a similar situation in this test scenario?
assert result["queues"]["confirm_sip"] == \
{"processing": 1, "pending": 1}
assert result["failed"] == 1
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of -1
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), -1)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, timestamp + 10, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 20, 'bar')
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.testconn.zadd(self.registry.key, timestamp + 30, 'baz')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup(self):
"""Moving expired jobs to FailedQueue."""
failed_queue = FailedQueue(connection=self.testconn)
self.assertTrue(failed_queue.is_empty())
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, 2, job.id)
self.registry.cleanup(1)
self.assertNotIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2)
self.registry.cleanup()
self.assertIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None)
job.refresh()
self.assertEqual(job.status, JobStatus.FAILED)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, timestamp, 'foo')
self.testconn.zadd(self.registry.key, timestamp, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, 1, 'foo')
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, 1, 'foo')
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of -1
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), -1)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, timestamp + 10, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 20, 'bar')
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.testconn.zadd(self.registry.key, timestamp + 30, 'baz')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup(self):
"""Moving expired jobs to FailedQueue."""
failed_queue = FailedQueue(connection=self.testconn)
self.assertTrue(failed_queue.is_empty())
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, 2, job.id)
self.registry.cleanup(1)
self.assertNotIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2)
self.registry.cleanup()
self.assertIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None)
job.refresh()
self.assertEqual(job.get_status(), JobStatus.FAILED)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, timestamp, 'foo')
self.testconn.zadd(self.registry.key, timestamp, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, 1, 'foo')
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, 1, 'foo')
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_init(self):
"""Registry can be instantiated with queue or name/Redis connection"""
queue = Queue('foo', connection=self.testconn)
registry = StartedJobRegistry(queue=queue)
self.assertEqual(registry.name, queue.name)
self.assertEqual(registry.connection, queue.connection)
registry = StartedJobRegistry('bar', self.testconn)
self.assertEqual(registry.name, 'bar')
self.assertEqual(registry.connection, self.testconn)
def test_key(self):
self.assertEqual(self.registry.key, 'rq:wip:default')
def test_custom_job_class(self):
registry = StartedJobRegistry(job_class=CustomJob)
self.assertFalse(registry.job_class == self.registry.job_class)
def test_contains(self):
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.assertFalse(job in registry)
self.assertFalse(job.id in registry)
registry.add(job, 5)
self.assertTrue(job in registry)
self.assertTrue(job.id in registry)
def test_get_expiration_time(self):
"""registry.get_expiration_time() returns correct datetime objects"""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
registry.add(job, 5)
self.assertEqual(
registry.get_expiration_time(job),
(datetime.utcnow() + timedelta(seconds=5)).replace(microsecond=0)
)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of inf
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), float('inf'))
# Ensure that job is removed from sorted set, but job key is not deleted
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertTrue(self.testconn.exists(job.key))
self.registry.add(job, -1)
# registry.remove() also accepts job.id
self.registry.remove(job.id)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.registry.add(job, -1)
# delete_job = True deletes job key
self.registry.remove(job, delete_job=True)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertFalse(self.testconn.exists(job.key))
job = queue.enqueue(say_hello)
self.registry.add(job, -1)
# delete_job = True also works with job.id
self.registry.remove(job.id, delete_job=True)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertFalse(self.testconn.exists(job.key))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': timestamp + 10})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 20})
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup_moves_jobs_to_failed_job_registry(self):
"""Moving expired jobs to FailedJobRegistry."""
queue = Queue(connection=self.testconn)
failed_job_registry = FailedJobRegistry(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, {job.id: 2})
# Job has not been moved to FailedJobRegistry
self.registry.cleanup(1)
self.assertNotIn(job, failed_job_registry)
self.assertIn(job, self.registry)
self.registry.cleanup()
self.assertIn(job.id, failed_job_registry)
self.assertNotIn(job, self.registry)
job.refresh()
self.assertEqual(job.get_status(), JobStatus.FAILED)
self.assertTrue(job.exc_info) # explanation is written to exc_info
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_started)
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_finished)
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
def test_job_deletion(self):
"""Ensure job is removed from StartedJobRegistry when deleted."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
job.delete()
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, {'foo': timestamp})
self.testconn.zadd(self.registry.key, {'bar': timestamp})
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, {'foo': 1})
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, {'foo': 1})
failed_job_registry = FailedJobRegistry(connection=self.testconn)
self.testconn.zadd(failed_job_registry.key, {'foo': 1})
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(failed_job_registry.key), 0)
def test_get_queue(self):
"""registry.get_queue() returns the right Queue object."""
registry = StartedJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_queue(), Queue(connection=self.testconn))
registry = StartedJobRegistry('foo', connection=self.testconn)
self.assertEqual(registry.get_queue(),
Queue('foo', connection=self.testconn))
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_init(self):
"""Registry can be instantiated with queue or name/Redis connection"""
queue = Queue('foo', connection=self.testconn)
registry = StartedJobRegistry(queue=queue)
self.assertEqual(registry.name, queue.name)
self.assertEqual(registry.connection, queue.connection)
registry = StartedJobRegistry('bar', self.testconn)
self.assertEqual(registry.name, 'bar')
self.assertEqual(registry.connection, self.testconn)
def test_key(self):
self.assertEqual(self.registry.key, 'rq:wip:default')
def test_custom_job_class(self):
registry = StartedJobRegistry(job_class=CustomJob)
self.assertFalse(registry.job_class == self.registry.job_class)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of inf
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id),
float('inf'))
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': timestamp + 10})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 20})
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup(self):
"""Moving expired jobs to FailedQueue."""
failed_queue = FailedQueue(connection=self.testconn)
self.assertTrue(failed_queue.is_empty())
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, {job.id: 2})
self.registry.cleanup(1)
self.assertNotIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2)
self.registry.cleanup()
self.assertIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None)
job.refresh()
self.assertEqual(job.get_status(), JobStatus.FAILED)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_started)
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_finished)
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
def test_job_deletion(self):
"""Ensure job is removed from StartedJobRegistry when deleted."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
job.delete()
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, {'foo': timestamp})
self.testconn.zadd(self.registry.key, {'bar': timestamp})
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, {'foo': 1})
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, {'foo': 1})
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
class TestQueue(RQTestCase):
def setUp(self):
super(TestQueue, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, 10, 'bar')
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
def test_cleanup(self):
"""Moving expired jobs to FailedQueue."""
failed_queue = FailedQueue(connection=self.testconn)
self.assertTrue(failed_queue.is_empty())
self.testconn.zadd(self.registry.key, 1, 'foo')
self.registry.move_expired_jobs_to_failed_queue()
self.assertIn('foo', failed_queue.job_ids)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, 10, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_init(self):
"""Registry can be instantiated with queue or name/Redis connection"""
queue = Queue('foo', connection=self.testconn)
registry = StartedJobRegistry(queue=queue)
self.assertEqual(registry.name, queue.name)
self.assertEqual(registry.connection, queue.connection)
registry = StartedJobRegistry('bar', self.testconn)
self.assertEqual(registry.name, 'bar')
self.assertEqual(registry.connection, self.testconn)
def test_key(self):
self.assertEqual(self.registry.key, 'rq:wip:default')
def test_custom_job_class(self):
registry = StartedJobRegistry(job_class=CustomJob)
self.assertFalse(registry.job_class == self.registry.job_class)
def test_contains(self):
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.assertFalse(job in registry)
self.assertFalse(job.id in registry)
registry.add(job, 5)
self.assertTrue(job in registry)
self.assertTrue(job.id in registry)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of inf
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), float('inf'))
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': timestamp + 10})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 20})
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup_moves_jobs_to_failed_job_registry(self):
"""Moving expired jobs to FailedJobRegistry."""
queue = Queue(connection=self.testconn)
failed_job_registry = FailedJobRegistry(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, {job.id: 2})
# Job has not been moved to FailedJobRegistry
self.registry.cleanup(1)
self.assertNotIn(job, failed_job_registry)
self.assertIn(job, self.registry)
self.registry.cleanup()
self.assertIn(job.id, failed_job_registry)
self.assertNotIn(job, self.registry)
job.refresh()
self.assertEqual(job.get_status(), JobStatus.FAILED)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_started)
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_finished)
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
def test_job_deletion(self):
"""Ensure job is removed from StartedJobRegistry when deleted."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
job.delete()
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, {'foo': timestamp})
self.testconn.zadd(self.registry.key, {'bar': timestamp})
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, {'foo': 1})
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, {'foo': 1})
failed_job_registry = FailedJobRegistry(connection=self.testconn)
self.testconn.zadd(failed_job_registry.key, {'foo': 1})
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(failed_job_registry.key), 0)
def test_get_queue(self):
"""registry.get_queue() returns the right Queue object."""
registry = StartedJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_queue(), Queue(connection=self.testconn))
registry = StartedJobRegistry('foo', connection=self.testconn)
self.assertEqual(registry.get_queue(),
Queue('foo', connection=self.testconn))
