def test_enqueue_consume_fail_1(self):
# """One reschedule attempt and then fail. maxfailed=1"""
rjq = self.rjq
# Consume, Fail->Reschedule, Consume, Fail->Failed
rjq.maxfailed(1)
jobgen = generate_jobs()
job1a = next(jobgen)
rjq.enqueue(job1a)
self.assertEqual(rjq.get(job1a.id).state, JOB_STATE_ENQUEUED)
# You can't fail a job that has not been consumed (that is not
# in the WORKING queue)
with self.assertRaises(UnknownJobId):
rjq.fail(job1a)
job1b = rjq.consume()
self.assertEqual(rjq.count(NS_WORKING), 1)
self.assertTrue(job_cmp(job1a, job1b))
self.assertEqual(job1b.state, JOB_STATE_WORKING)
self.assertEqual(rjq.get(job1a.id).state, JOB_STATE_WORKING)
# 1st time failure goes to SCHEDULED queue
self.assertEqual(rjq.count(NS_SCHEDULED), 0)
rjq.fail(job1b)
self.assertEqual(rjq.count(NS_SCHEDULED), 1)
job1b = rjq.get(job1b.id)
self.assertEqual(job1b.state, JOB_STATE_SCHEDULED)
self.assertEqual(job1b.failures, 1)
job1c = rjq.consume()
self.assertEqual(rjq.count(NS_SCHEDULED), 0)
self.assertEqual(rjq.count(NS_WORKING), 1)
self.assertTrue(job_cmp(job1b, job1c))
self.assertEqual(job1c.state, JOB_STATE_WORKING)
# 2nd time failure goes to FAILED queue
rjq.fail(job1c)
self.assertEqual(rjq.count(NS_WORKING), 0)
self.assertEqual(rjq.count(NS_QUEUED), 0)
self.assertEqual(rjq.count(NS_SCHEDULED), 0)
self.assertEqual(rjq.count(NS_FAILED), 1)
job1c = rjq.get(job1c.id)
self.assertEqual(job1c.state, JOB_STATE_FAILED)
self.assertEqual(job1c.failures, 2)
self.assertTrue(job_cmp(job1b, job1c))
def test_enqueue_consume_ack_1(self):
# """Enqueue, consume, ack. Test for state changes."""
rjq = self.rjq
jobgen = generate_jobs()
job1 = next(jobgen)
rjq.enqueue(job1)
kjob = rjq.key(NS_JOB, job1.id)
logger.info("Job Key: %s", kjob)
state = rjq.conn.hget(kjob, "state")
logger.info("Job state: %s", state)
self.assertEqual(state, JOB_STATE_ENQUEUED)
job2 = rjq.consume()
state = rjq.conn.hget(kjob, "state")
logger.info("Job state: %s", state)
self.assertEqual(state, JOB_STATE_WORKING)
# We should get the same job object as we enqueued.
self.assertTrue(job_cmp(job1, job2))
state = rjq.conn.hget(rjq.key(NS_JOB, job1.id), "state")
# the database should have the state attribute updated
self.assertEqual(state, JOB_STATE_WORKING)
# the object should have received the updated state attribute
self.assertEqual(job2.state, JOB_STATE_WORKING)
rjq.ack(job2)
# acking should remove the job entry from the queue and remove the hash
self.assertIsNone(rjq.conn.zscore(rjq.key(NS_QUEUED), job2.id))
self.assertDictEqual(rjq.conn.hgetall(rjq.key(NS_JOB, job2.id)), {})
def test_add_consume_cancel_ack_cancel(self):
can_consume = yield self.rjq.can_consume()
self.assertFalse(can_consume)
jobgen = generate_jobs()
job1a = next(jobgen)
# add
yield self.rjq.add(job1a)
can_consume = yield self.rjq.can_consume()
self.assertTrue(can_consume)
# consume
job1b = yield self.rjq.consume()
self.assertTrue(job_cmp(job1a, job1b))
# cancel should fail with JobInWork
try:
yield self.rjq.cancel(job1a)
except JobInWork as exc:
logger.info(exc)
yield self.rjq.ack(job1b)
# cancel should fail with UnknownJobId
try:
yield self.rjq.cancel(job1a)
except UnknownJobId as exc:
logger.info(exc)
def consume_fail(self, job1a):
yield self.rjq.enqueue(job1a)
scheduled_jobs = yield self.rjq.count(NS_SCHEDULED)
self.assertEqual(scheduled_jobs, 0)
queued_jobs = yield self.rjq.count(NS_QUEUED)
self.assertEqual(queued_jobs, 1)
job1b = yield self.rjq.consume()
self.assertTrue(job_cmp(job1a, job1b))
yield self.rjq.fail(job1b, requeue_seconds=1)
scheduled_jobs = yield self.rjq.count(NS_SCHEDULED)
self.assertEqual(scheduled_jobs, 1)
can_consume = yield self.rjq.can_consume()
self.assertFalse(can_consume)
yield task.deferLater(reactor, 1.2, lambda *args: None)
can_consume = yield self.rjq.can_consume()
self.assertTrue(can_consume)
job1c = yield self.rjq.consume()
self.assertTrue(job_cmp(job1a, job1c))
working_jobs = yield self.rjq.count(NS_WORKING)
self.assertEqual(working_jobs, 1)
defer.returnValue(job1c)
def test_add_consume_ack(self):
jobgen = generate_jobs()
for rjq in (self.rjq,
TxRedisJobQueueView(self.rjq,
self.rjq.namespace + ":MORE")):
job1 = next(jobgen)
res = yield rjq.enqueue(job1)
self.assertEqual(res, 1)
job1b = yield rjq.consume()
self.assertTrue(job_cmp(job1, job1b))
res = yield rjq.ack(job1b)
self.assertEqual(res, 1)
def test_peek(self):
rjq = self.rjq
jobgen = generate_jobs()
job0 = next(jobgen)
job0.priority = 1
job1 = next(jobgen)
job1.priority = 2
job2 = next(jobgen)
job2.priority = 3
yield rjq.add(job0)
yield rjq.add(job1)
yield rjq.add(job2)
jobs = []
yield rjq.peek(jobs.append, NS_QUEUED)
self.assertTrue(job_cmp(job0, jobs[0]))
self.assertTrue(job_cmp(job1, jobs[1]))
self.assertTrue(job_cmp(job2, jobs[2]))
jobs = []
yield rjq.peek(jobs.append, NS_QUEUED, count=2)
self.assertTrue(job_cmp(job0, jobs[0]))
self.assertTrue(job_cmp(job1, jobs[1]))
self.assertEqual(len(jobs), 2)
try:
yield rjq.peek(jobs.append, "INVALID_QUEUE_NAME")
except AbnormalOperationError as err:
assert isinstance(err, InvalidQueue)
# This method works like an iterator
jobit = yield rjq.queue_iter(NS_QUEUED)
job0b = yield next(jobit)
job1b = yield next(jobit)
job2b = yield next(jobit)
self.assertTrue(job_cmp(job0, job0b))
self.assertTrue(job_cmp(job1, job1b))
self.assertTrue(job_cmp(job2, job2b))
jobit = yield rjq.queue_iter(NS_QUEUED, count=2)
njobs = 0
for _ in jobit:
njobs += 1
self.assertEqual(njobs, 2)
def test_tx_schedule_1(self):
can_consume = yield self.rjq.can_consume()
self.assertFalse(can_consume)
jobgen = generate_jobs()
job1a = next(jobgen)
t = utcunixts() + 1
yield self.rjq.schedule(job1a, t)
scheduled_jobs = yield self.rjq.count(NS_SCHEDULED)
self.assertEqual(scheduled_jobs, 1)
can_consume = yield self.rjq.can_consume()
self.assertFalse(can_consume)
yield task.deferLater(reactor, 1.2, lambda *args: None)
can_consume = yield self.rjq.can_consume()
self.assertTrue(can_consume)
job1b = yield self.rjq.consume()
self.assertTrue(job_cmp(job1a, job1b))
def test_consume(self):
no = yield self.rjq.can_consume()
self.assertFalse(no)
jobgen = generate_jobs()
jobs = []
for i in range(10):
job = next(jobgen)
job.priority = i
jobs.append(job)
yield self.rjq.add(job)
yes = yield self.rjq.can_consume()
self.assertTrue(yes)
job0 = yield self.rjq.consume()
self.assertTrue(job_cmp(job0, jobs[0]))
@defer.inlineCallbacks
def get_jobs(queue):
d = yield self.rjq.queue_iter(queue)
jobs = []
for d_job in d:
job = yield d_job
jobs.append(job)
defer.returnValue(jobs)
working = yield get_jobs(NS_WORKING)
self.assertEqual(len(working), 1)
wct = yield self.rjq.count(NS_WORKING)
self.assertEqual(wct, 1)
yield self.rjq.ack(working[0])
working = yield get_jobs(NS_WORKING)
self.assertEqual(len(working), 0)
wct = yield self.rjq.count(NS_WORKING)
self.assertEqual(wct, 0)