def test_clean_rq(self):
r = get_redis_connection()
self.assertEqual(len(r.keys("rq:job:*")), 0)
r.hmset("rq:job:abc", {"bar": "baz"})
r.hmset("rq:job:def", {"created_at": utcformat(utcnow())})
r.hmset("rq:job:123", {"created_at": utcformat(utcnow() - timedelta(days=10))})
self.assertEqual(len(r.keys("rq:job:*")), 3)
call_command("clean_rq")
self.assertEqual(len(r.keys("rq:job:*")), 2)
def test_clean_rq(self):
r = get_redis_connection()
self.assertEqual(len(r.keys('rq:job:*')), 0)
r.hmset('rq:job:abc', {'bar': 'baz'})
r.hmset('rq:job:def', {'created_at': utcformat(utcnow())})
r.hmset('rq:job:123', {
'created_at': utcformat(utcnow() - timedelta(days=10))})
self.assertEqual(len(r.keys('rq:job:*')), 3)
call_command('clean_rq')
self.assertEqual(len(r.keys('rq:job:*')), 2)
def test_clean_rq(self):
r = get_redis_connection()
self.assertEqual(len(r.keys('rq:job:*')), 0)
r.hmset('rq:job:abc', {'bar': 'baz'})
r.hmset('rq:job:def', {'created_at': utcformat(utcnow())})
r.hmset('rq:job:123',
{'created_at': utcformat(utcnow() - timedelta(days=10))})
self.assertEqual(len(r.keys('rq:job:*')), 3)
call_command('clean_rq')
self.assertEqual(len(r.keys('rq:job:*')), 2)
def register_death(self):
"""Registers its own death."""
logger.debug('Registering death')
# We cannot use self.state = 'dead' here, because that would
# rollback the pipeline
pipe = self.connection.multi_exec()
pipe.srem(self.redis_workers_keys, self.key)
pipe.hset(self.key, 'death', utcformat(utcnow()))
pipe.expire(self.key, 60)
yield from pipe.execute()
def test_persistence_of_typical_jobs(self):
"""Storing typical jobs."""
job = Job.create(func=fixtures.some_calculation, args=(3, 4), kwargs=dict(z=2))
job.save()
expected_date = strip_microseconds(job.created_at)
stored_date = self.testconn.hget(job.key, "created_at").decode("utf-8")
self.assertEqual(stored_date, utcformat(expected_date))
# ... and no other keys are stored
self.assertEqual(sorted(self.testconn.hkeys(job.key)), [b"created_at", b"data", b"description"])
def register_death(self):
"""Registers its own death."""
logger.debug('Registering death')
# We cannot use self.state = 'dead' here, because that would
# rollback the pipeline
pipe = self.connection.multi_exec()
pipe.srem(self.redis_workers_keys, self.key)
pipe.hset(self.key, 'death', utcformat(utcnow()))
pipe.expire(self.key, 60)
yield from pipe.execute()
def test_persistence_of_typical_jobs(self):
"""Storing typical jobs."""
job = Job.create(func=some_calculation, args=(3, 4), kwargs=dict(z=2))
job.save()
expected_date = strip_microseconds(job.created_at)
stored_date = self.testconn.hget(job.key, 'created_at').decode('utf-8')
self.assertEquals(stored_date, utcformat(expected_date))
# ... and no other keys are stored
self.assertEqual(sorted(self.testconn.hkeys(job.key)),
[b'created_at', b'data', b'description'])
def test_persistence_of_typical_jobs(self):
"""Storing typical jobs."""
job = Job.create(func=fixtures.some_calculation, args=(3, 4), kwargs=dict(z=2))
job.save()
stored_date = self.testconn.hget(job.key, 'created_at').decode('utf-8')
self.assertEqual(stored_date, utcformat(job.created_at))
# ... and no other keys are stored
self.assertEqual(
sorted(self.testconn.hkeys(job.key)),
[b'created_at', b'data', b'description'])
def test_persistence_of_typical_jobs(redis):
"""Storing typical jobs."""
job = Job.create(func=some_calculation, args=(3, 4), kwargs=dict(z=2))
yield from job.save()
expected_date = strip_microseconds(job.created_at)
stored_date = (yield from redis.hget(job.key, 'created_at')) \
.decode('utf-8')
assert stored_date == utcformat(expected_date)
# ... and no other keys are stored
assert sorted((yield from redis.hkeys(job.key))) \
== [b'created_at', b'data', b'description']
def prepare_job_execution(self, job):
"""Performs misc bookkeeping like updating states prior to job
execution.
"""
timeout = (job.timeout or 180) + 60
pipe = self.connection.multi_exec()
yield from self.set_state(WorkerStatus.BUSY, pipeline=pipe)
yield from self.set_current_job_id(job.id, pipeline=pipe)
yield from self.heartbeat(timeout, pipeline=pipe)
registry = StartedJobRegistry(job.origin, self.connection)
yield from registry.add(job, timeout, pipeline=pipe)
yield from job.set_status(JobStatus.STARTED, pipeline=pipe)
pipe.hset(job.key, 'started_at', utcformat(utcnow()))
yield from pipe.execute()
def prepare_job_execution(self, job):
"""Performs misc bookkeeping like updating states prior to job
execution.
"""
timeout = (job.timeout or 180) + 60
pipe = self.connection.multi_exec()
yield from self.set_state(WorkerStatus.BUSY, pipeline=pipe)
yield from self.set_current_job_id(job.id, pipeline=pipe)
yield from self.heartbeat(timeout, pipeline=pipe)
registry = StartedJobRegistry(job.origin, self.connection)
yield from registry.add(job, timeout, pipeline=pipe)
yield from job.set_status(JobStatus.STARTED, pipeline=pipe)
pipe.hset(job.key, 'started_at', utcformat(utcnow()))
yield from pipe.execute()
def register_birth(self):
"""Registers its own birth."""
logger.debug('Registering birth of worker {0}'.format(self.name))
another_is_present = yield from self.connection.exists(self.key)
another_is_dead = yield from self.connection.hexists(self.key, 'death')
if another_is_present and not another_is_dead:
msg = 'There exists an active worker named {0!r} already'
raise ValueError(msg.format(self.name))
key = self.key
queues = ','.join(self.queue_names())
pipe = self.connection.multi_exec()
pipe.delete(key)
pipe.hset(key, 'birth', utcformat(utcnow()))
pipe.hset(key, 'queues', queues)
pipe.sadd(self.redis_workers_keys, key)
pipe.expire(key, self.default_worker_ttl)
yield from pipe.execute()
def register_birth(self):
"""Registers its own birth."""
logger.debug('Registering birth of worker {0}'.format(self.name))
another_is_present = yield from self.connection.exists(self.key)
another_is_dead = yield from self.connection.hexists(self.key, 'death')
if another_is_present and not another_is_dead:
msg = 'There exists an active worker named {0!r} already'
raise ValueError(msg.format(self.name))
key = self.key
queues = ','.join(self.queue_names())
pipe = self.connection.multi_exec()
pipe.delete(key)
pipe.hset(key, 'birth', utcformat(utcnow()))
pipe.hset(key, 'queues', queues)
pipe.sadd(self.redis_workers_keys, key)
pipe.expire(key, self.default_worker_ttl)
yield from pipe.execute()
def test_persistence_of_typical_jobs(self):
"""Storing typical jobs."""
job = Job.create(func=fixtures.some_calculation, args=(3, 4), kwargs=dict(z=2))
job.save()
stored_date = self.testconn.hget(job.key, 'created_at').decode('utf-8')
self.assertEqual(stored_date, utcformat(job.created_at))
# ... and no other keys are stored
self.assertEqual(
set(self.testconn.hkeys(job.key)),
{b'created_at', b'data', b'description', b'ended_at', b'last_heartbeat', b'started_at',
b'worker_name', b'success_callback_name', b'failure_callback_name'}
)
self.assertEqual(job.last_heartbeat, None)
self.assertEqual(job.last_heartbeat, None)
ts = utcnow()
job.heartbeat(ts, 0)
self.assertEqual(job.last_heartbeat, ts)
