def consume(self, job_id=None):
# <ns>
keys = (self.namespace, )
# <client_name> <job_id> <datetime> <expires>
args = (self.name, job_id or "", utcunixts(), self.worker_expiration)
return self.call_script("consume", keys, args) \
.addCallback(lambda res: Job.from_string_list(res))
def test_schedule_1(self):
rjq = self.rjq
rjq.maxfailed(0)
jobgen = generate_jobs()
job1a = next(jobgen)
self.assertFalse(rjq.can_consume())
t = utcunixts()+1
rjq.schedule(job1a, t)
self.assertEqual(rjq.count(NS_SCHEDULED), 1)
self.assertFalse(rjq.can_consume())
job1b = rjq.get(job1a.id)
self.assertTrue(job1b.priority != t)
self.assertEqual(job1b.state, JOB_STATE_SCHEDULED)
time.sleep(1.2)
self.assertTrue(rjq.can_consume())
job1c = rjq.consume()
self.assertEqual(rjq.count(NS_SCHEDULED), 0)
self.assertEqual(rjq.count(NS_WORKING), 1)
self.assertEqual(job1c.state, JOB_STATE_WORKING)
self.assertFalse(rjq.can_consume())
rjq.ack(job1c)
def can_consume(self):
"""Returns a deferred that is called back with True if there are jobs
available to consume and False otherwise.
"""
keys = (self.namespace, )
args = (utcunixts(), )
return self.call_script("can_consume", keys, args)\
.addCallback(lambda res: res > 0)
def can_consume(self):
"""Returns True if there are jobs available to consume. False
otherwise.
"""
keys = (self.namespace, )
args = (utcunixts(), )
res = self.call_script("can_consume", keys, args)
return res > 0
def test_reschedule(self):
rjq = self.rjq
rjq.maxfailed(0)
jobgen = generate_jobs()
job1a = next(jobgen)
t = utcunixts()
rjq.schedule(job1a, t)
job1b, job1b_t = next(rjq.queue_iter(NS_SCHEDULED, withscores=True))
self.assertEqual(t, job1b_t)
self.assertTrue(rjq.can_consume())
t = utcunixts() + 1
rjq.reschedule(job1a, t)
self.assertFalse(rjq.can_consume())
job1b, job1b_t = next(rjq.queue_iter(NS_SCHEDULED, withscores=True))
self.assertEqual(t, job1b_t)
def fail(self, job, requeue_seconds=None):
# <ns>
keys = (self.namespace, )
# <jobid> <datetime> <requeue_seconds>
if requeue_seconds is None:
requeue_seconds = (REQUEUE_TIMEOUT * (job.failures**2))
args = (job.id, utcunixts(), requeue_seconds)
res = self.call_script("fail", keys, args)
return res
def test_tx_schedule_1(self):
can_consume = yield self.rjq.can_consume()
self.assertFalse(can_consume)
jobgen = generate_jobs()
job1a = next(jobgen)
t = utcunixts() + 1
yield self.rjq.schedule(job1a, t)
scheduled_jobs = yield self.rjq.count(NS_SCHEDULED)
self.assertEqual(scheduled_jobs, 1)
can_consume = yield self.rjq.can_consume()
self.assertFalse(can_consume)
yield task.deferLater(reactor, 1.2, lambda *args: None)
can_consume = yield self.rjq.can_consume()
self.assertTrue(can_consume)
job1b = yield self.rjq.consume()
self.assertTrue(job_cmp(job1a, job1b))
def recover(self, requeue_seconds=None):
"""
This function fails each abandoned job individually and sets their
scheduled requeue time to requeue_seconds. It is useful for the caller
to reschedule jobs individually after recovering.
"""
if requeue_seconds is None:
# TODO: This could be set to 0 to immediately requeue items
# What's the strategy when we don't know why a worker
# failed? Requeue right away? or requeue later?
requeue_seconds = REQUEUE_TIMEOUT
# <ns>
keys = (self.namespace, )
# <datetime> <requeue_seconds>
args = (utcunixts(), requeue_seconds)
# list of job IDs that have been recovered
recovered_jobs = self.call_script("recover", keys, args)
return recovered_jobs