def test_default_failure_ttl(self):
"""Job TTL defaults to DEFAULT_FAILURE_TTL"""
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
registry = FailedJobRegistry(connection=self.testconn)
key = registry.key
timestamp = current_timestamp()
registry.add(job)
score = self.testconn.zscore(key, job.id)
self.assertLess(score, timestamp + DEFAULT_FAILURE_TTL + 2)
self.assertGreater(score, timestamp + DEFAULT_FAILURE_TTL - 2)
# Job key will also expire
job_ttl = self.testconn.ttl(job.key)
self.assertLess(job_ttl, DEFAULT_FAILURE_TTL + 2)
self.assertGreater(job_ttl, DEFAULT_FAILURE_TTL - 2)
timestamp = current_timestamp()
ttl = 5
registry.add(job, ttl=ttl)
score = self.testconn.zscore(key, job.id)
self.assertLess(score, timestamp + ttl + 2)
self.assertGreater(score, timestamp + ttl - 2)
job_ttl = self.testconn.ttl(job.key)
self.assertLess(job_ttl, ttl + 2)
self.assertGreater(job_ttl, ttl - 2)
def test_default_failure_ttl(self):
"""Job TTL defaults to DEFAULT_FAILURE_TTL"""
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
registry = FailedJobRegistry(connection=self.testconn)
key = registry.key
timestamp = current_timestamp()
registry.add(job)
self.assertLess(
self.testconn.zscore(key, job.id),
timestamp + DEFAULT_FAILURE_TTL + 2
)
self.assertGreater(
self.testconn.zscore(key, job.id),
timestamp + DEFAULT_FAILURE_TTL - 2
)
timestamp = current_timestamp()
ttl = 5
registry.add(job, ttl=5)
self.assertLess(
self.testconn.zscore(key, job.id),
timestamp + ttl + 2
)
self.assertGreater(
self.testconn.zscore(key, job.id),
timestamp + ttl - 2
)
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, timestamp, 'foo')
self.testconn.zadd(self.registry.key, timestamp, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def hello():
with connect_to_redis(REDIS_URL_RQ) as r:
with Connection(connection=r):
### redis info
client_list = r.client_list()
local_clients = [cl for cl in client_list if is_local(cl['addr'])]
remote_clients = [cl for cl in client_list if not is_local(cl['addr'])]
ws = Worker.all()
q_2_w = defaultdict(int)
for w in ws:
for qn in w.queue_names():
q_2_w[qn] += 1
timestamp = current_timestamp()
return render_template(
'home.html',
queues=get_queue_info(r, q_2_w, timestamp),
info={
'timestamp': timestamp,
'local_clients': len(local_clients),
'remote_clients': len(remote_clients),
'workers': len(ws)
},
)
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, timestamp, 'foo')
self.testconn.zadd(self.registry.key, timestamp, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, "foo")
self.testconn.zadd(self.registry.key, timestamp + 10, "bar")
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ["bar"])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ['bar'])
def test_get_job_count(self):
"""
StartedJobRegistry returns the right number of job count. It should
return the number of non-expired jobs -- regardless of whether the job
exists or not
"""
foo = self.create_job('foo')
bar = self.create_job('foo')
keep_time = current_timestamp() + 10
remove_time = current_timestamp() - 10
self.testconn.zadd(self.registry.key, keep_time, foo.id)
self.testconn.zadd(self.registry.key, remove_time, bar.id)
self.testconn.zadd(self.registry.key, keep_time, 'missing1')
self.testconn.zadd(self.registry.key, remove_time, 'missing2')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def add(self, job, ttl=0, pipeline=None):
"""Adds a job to a registry with expiry time of now + ttl."""
score = ttl if ttl < 0 else current_timestamp() + ttl
# TODO: support pipeline mode
# if pipeline is not None:
# return pipeline.zadd(self.key, score, job.id)
return (yield from self.connection.zadd(self.key, score, job.id))
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id), timestamp + 1002)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.testconn.zadd(self.registry.key, timestamp + 30, 'baz')
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ['bar', 'baz'])
self.registry.cleanup(timestamp + 20)
self.assertEqual(self.registry.get_job_ids(), ['baz'])
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ['bar', 'baz'])
self.registry.cleanup(timestamp + 20)
self.assertEqual(self.registry.get_job_ids(), ['baz'])
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_jobs_to_enqueue(self):
"""Getting job ids to enqueue from ScheduledJobRegistry."""
queue = Queue(connection=self.testconn)
registry = ScheduledJobRegistry(queue=queue)
timestamp = current_timestamp()
self.testconn.zadd(registry.key, {'foo': 1})
self.testconn.zadd(registry.key, {'bar': timestamp + 10})
self.testconn.zadd(registry.key, {'baz': timestamp + 30})
self.assertEqual(registry.get_jobs_to_enqueue(), ['foo'])
self.assertEqual(registry.get_jobs_to_enqueue(timestamp + 20),
['foo', 'bar'])
def test_get_jobs_to_schedule_with_chunk_size(self):
"""Max amount of jobs returns by get_jobs_to_schedule() equal to chunk_size"""
queue = Queue(connection=self.testconn)
registry = ScheduledJobRegistry(queue=queue)
timestamp = current_timestamp()
chunk_size = 5
for index in range(0, chunk_size * 2):
self.testconn.zadd(registry.key, {'foo_{}'.format(index): 1})
self.assertEqual(len(registry.get_jobs_to_schedule(timestamp, chunk_size)),
chunk_size)
self.assertEqual(len(registry.get_jobs_to_schedule(timestamp, chunk_size * 2)),
chunk_size * 2)
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
# CanceledJobRegistry does not implement get_expired_job_ids()
registry = CanceledJobRegistry(connection=self.testconn)
self.assertRaises(NotImplementedError, registry.get_expired_job_ids)
def test_cleanup(self):
"""Finished job registry removes expired jobs."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.registry.cleanup()
self.assertEqual(self.registry.get_job_ids(), ['bar', 'baz'])
self.registry.cleanup(timestamp + 20)
self.assertEqual(self.registry.get_job_ids(), ['baz'])
# CanceledJobRegistry now implements noop cleanup, should not raise exception
registry = CanceledJobRegistry(connection=self.testconn)
registry.cleanup()
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of inf
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id),
float('inf'))
# Ensure that job is removed from sorted set, but job key is not deleted
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertTrue(self.testconn.exists(job.key))
self.registry.add(job, -1)
# registry.remove() also accepts job.id
self.registry.remove(job.id)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.registry.add(job, -1)
# delete_job = True deletes job key
self.registry.remove(job, delete_job=True)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertFalse(self.testconn.exists(job.key))
job = queue.enqueue(say_hello)
self.registry.add(job, -1)
# delete_job = True also works with job.id
self.registry.remove(job.id, delete_job=True)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertFalse(self.testconn.exists(job.key))
def printinfo(r):
timestamp = current_timestamp()
click.echo('current time {0}'.format(timestamp))
client_list = r.client_list()
local_clients = [cl for cl in client_list if is_local(cl['addr'])]
remote_clients = [cl for cl in client_list if not is_local(cl['addr'])]
click.echo(
'Number of active Redis connections: {remote} (+ {local} local)'.
format(remote=len(remote_clients), local=len(local_clients)))
print "-------------"
ws = Worker.all()
click.echo('Workers: {0}'.format(len(ws)))
q_2_w = defaultdict(int)
for w in ws:
for qn in w.queue_names():
q_2_w[qn] += 1
line = "{name:20} {workers:>3} {count:>10} {wip_count:>10} {expired_count:>10}".format(
name="Name",
count="Count",
workers="Wrk",
wip_count="WIP",
expired_count="Expired")
click.echo(line)
print "-------------"
qs = Queue.all()
for q in qs:
wip_key = 'rq:wip:{q.name}'.format(q=q)
expired_count = r.zcount(wip_key, 0, timestamp - 1)
wip_count = r.zcount(wip_key, timestamp, '+inf')
if all(x == 0
for x in (q.count, q_2_w[q.name], wip_count, expired_count)):
continue
line = "{q.name:20} {workers:>3} {q.count:>10} {wip_count:>10} {expired_count:>10}".format(
q=q,
workers=q_2_w[q.name],
wip_count=wip_count,
expired_count=expired_count,
)
click.echo(line)
def test_worker_handle_job_failure(self):
"""Failed jobs are added to FailedJobRegistry"""
q = Queue(connection=self.testconn)
w = Worker([q])
registry = FailedJobRegistry(connection=w.connection)
timestamp = current_timestamp()
job = q.enqueue(div_by_zero, failure_ttl=5)
w.handle_job_failure(job)
# job is added to FailedJobRegistry with default failure ttl
self.assertIn(job.id, registry.get_job_ids())
self.assertLess(self.testconn.zscore(registry.key, job.id),
timestamp + DEFAULT_FAILURE_TTL + 5)
# job is added to FailedJobRegistry with specified ttl
job = q.enqueue(div_by_zero, failure_ttl=5)
w.handle_job_failure(job)
self.assertLess(self.testconn.zscore(registry.key, job.id),
timestamp + 7)
def test_worker_handle_job_failure(self):
"""Failed jobs are added to FailedJobRegistry"""
q = Queue(connection=self.testconn)
w = Worker([q])
registry = FailedJobRegistry(connection=w.connection)
timestamp = current_timestamp()
job = q.enqueue(div_by_zero, failure_ttl=5)
w.handle_job_failure(job)
# job is added to FailedJobRegistry with default failure ttl
self.assertIn(job.id, registry.get_job_ids())
self.assertLess(self.testconn.zscore(registry.key, job.id),
timestamp + DEFAULT_FAILURE_TTL + 5)
# job is added to FailedJobRegistry with specified ttl
job = q.enqueue(div_by_zero, failure_ttl=5)
w.handle_job_failure(job)
self.assertLess(self.testconn.zscore(registry.key, job.id),
timestamp + 7)
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': timestamp + 10})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 20})
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, timestamp + 10, "foo")
self.testconn.zadd(self.registry.key, timestamp + 20, "bar")
self.assertEqual(self.registry.get_job_ids(), ["foo", "bar"])
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': timestamp + 10})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 20})
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
#! /usr/bin/env python import sys from redis import StrictRedis from cvgmeasure.conf import REDIS_URL_RQ from rq.registry import StartedJobRegistry from rq.utils import current_timestamp r = StrictRedis.from_url(REDIS_URL_RQ) sjr = StartedJobRegistry(sys.argv[1], connection=r) #sjr.cleanup() sjr.cleanup(timestamp=current_timestamp()+100000000000)