def test_add_consume_cancel_ack_cancel(self): can_consume = yield self.rjq.can_consume() self.assertFalse(can_consume) jobgen = generate_jobs() job1a = next(jobgen) # add yield self.rjq.add(job1a) can_consume = yield self.rjq.can_consume() self.assertTrue(can_consume) # consume job1b = yield self.rjq.consume() self.assertTrue(job_cmp(job1a, job1b)) # cancel should fail with JobInWork try: yield self.rjq.cancel(job1a) except JobInWork as exc: logger.info(exc) yield self.rjq.ack(job1b) # cancel should fail with UnknownJobId try: yield self.rjq.cancel(job1a) except UnknownJobId as exc: logger.info(exc)
def test_enqueue_consume_fail_2(self): # """Enqueue, consume, fail, fail to consume. maxfailed=0""" rjq = self.rjq rjq.maxfailed(0) jobgen = generate_jobs() job1a = next(jobgen) self.assertFalse(rjq.can_consume()) # No jobs in any queue with self.assertRaises(NoItems): rjq.consume() rjq.enqueue(job1a) self.assertTrue(rjq.can_consume()) job1b = rjq.consume() rjq.fail(job1b) job1b = rjq.get(job1b.id) self.assertEqual(job1b.state, JOB_STATE_FAILED) self.assertEqual(job1b.failures, 1) self.assertFalse(rjq.can_consume()) # The only Job is in the FAILED queue with self.assertRaises(NoItems): rjq.consume()
def test_schedule_1(self): rjq = self.rjq rjq.maxfailed(0) jobgen = generate_jobs() job1a = next(jobgen) self.assertFalse(rjq.can_consume()) t = utcunixts()+1 rjq.schedule(job1a, t) self.assertEqual(rjq.count(NS_SCHEDULED), 1) self.assertFalse(rjq.can_consume()) job1b = rjq.get(job1a.id) self.assertTrue(job1b.priority != t) self.assertEqual(job1b.state, JOB_STATE_SCHEDULED) time.sleep(1.2) self.assertTrue(rjq.can_consume()) job1c = rjq.consume() self.assertEqual(rjq.count(NS_SCHEDULED), 0) self.assertEqual(rjq.count(NS_WORKING), 1) self.assertEqual(job1c.state, JOB_STATE_WORKING) self.assertFalse(rjq.can_consume()) rjq.ack(job1c)
def test_recover(self): # rjq = self.rjq jobgen = generate_jobs() job1a = next(jobgen) namespace = "RECOVER" conn = os.environ.get('REDIS_CONN', 'redis://localhost') rjq1 = RedisJobQueue(namespace="TEST:RJQ:%s" % namespace, worker_expiration=1, name="testrjq1", conn=conn) rjq2 = RedisJobQueue(namespace="TEST:RJQ:%s" % namespace, worker_expiration=1, name="testrjq2", conn=conn) rjq1.add(job1a) job1a = rjq1.consume() clients = {e['name'] for e in rjq1.conn.client_list()} self.assertIn(rjq1.name, clients) self.assertIn(rjq2.name, clients) del rjq1 time.sleep(1.2) try: job_ids = rjq2.recover() self.assertEqual(len(job_ids), 1) self.assertEqual(job_ids[0], job1a.id) finally: clear_ns(rjq2)
def test_enqueue_consume_ack_1(self): # """Enqueue, consume, ack. Test for state changes.""" rjq = self.rjq jobgen = generate_jobs() job1 = next(jobgen) rjq.enqueue(job1) kjob = rjq.key(NS_JOB, job1.id) logger.info("Job Key: %s", kjob) state = rjq.conn.hget(kjob, "state") logger.info("Job state: %s", state) self.assertEqual(state, JOB_STATE_ENQUEUED) job2 = rjq.consume() state = rjq.conn.hget(kjob, "state") logger.info("Job state: %s", state) self.assertEqual(state, JOB_STATE_WORKING) # We should get the same job object as we enqueued. self.assertTrue(job_cmp(job1, job2)) state = rjq.conn.hget(rjq.key(NS_JOB, job1.id), "state") # the database should have the state attribute updated self.assertEqual(state, JOB_STATE_WORKING) # the object should have received the updated state attribute self.assertEqual(job2.state, JOB_STATE_WORKING) rjq.ack(job2) # acking should remove the job entry from the queue and remove the hash self.assertIsNone(rjq.conn.zscore(rjq.key(NS_QUEUED), job2.id)) self.assertDictEqual(rjq.conn.hgetall(rjq.key(NS_JOB, job2.id)), {})
def test_tx_consume_fail_ack(self): yield self.rjq.maxfailed(1) jobgen = generate_jobs() job1a = next(jobgen) job1c = yield self.consume_fail(job1a) yield self.rjq.ack(job1c) working_jobs = yield self.rjq.count(NS_WORKING) self.assertEqual(working_jobs, 0)
def test_get_job_by_id(self): jobgen = generate_jobs() job = next(jobgen) yield self.rjq.add(job) job2 = yield self.rjq.get(job.id) self.assertEqual(job.id, job2.id) try: yield self.rjq.get(job.id + "INVALIDJOBID") except UnknownJobId: pass
def test_add_twice(self): jobgen = generate_jobs() job = next(jobgen) # First time is good yield self.rjq.add(job) try: # Second time raises `JobExists` yield self.rjq.add(job) except JobExists: pass
def test_add_cancel(self): can_consume = yield self.rjq.can_consume() self.assertFalse(can_consume) jobgen = generate_jobs() job1a = next(jobgen) yield self.rjq.add(job1a) can_consume = yield self.rjq.can_consume() self.assertTrue(can_consume) yield self.rjq.cancel(job1a) can_consume = yield self.rjq.can_consume() self.assertFalse(can_consume)
def test_enqueue_consume_fail_1(self): # """One reschedule attempt and then fail. maxfailed=1""" rjq = self.rjq # Consume, Fail->Reschedule, Consume, Fail->Failed rjq.maxfailed(1) jobgen = generate_jobs() job1a = next(jobgen) rjq.enqueue(job1a) self.assertEqual(rjq.get(job1a.id).state, JOB_STATE_ENQUEUED) # You can't fail a job that has not been consumed (that is not # in the WORKING queue) with self.assertRaises(UnknownJobId): rjq.fail(job1a) job1b = rjq.consume() self.assertEqual(rjq.count(NS_WORKING), 1) self.assertTrue(job_cmp(job1a, job1b)) self.assertEqual(job1b.state, JOB_STATE_WORKING) self.assertEqual(rjq.get(job1a.id).state, JOB_STATE_WORKING) # 1st time failure goes to SCHEDULED queue self.assertEqual(rjq.count(NS_SCHEDULED), 0) rjq.fail(job1b) self.assertEqual(rjq.count(NS_SCHEDULED), 1) job1b = rjq.get(job1b.id) self.assertEqual(job1b.state, JOB_STATE_SCHEDULED) self.assertEqual(job1b.failures, 1) job1c = rjq.consume() self.assertEqual(rjq.count(NS_SCHEDULED), 0) self.assertEqual(rjq.count(NS_WORKING), 1) self.assertTrue(job_cmp(job1b, job1c)) self.assertEqual(job1c.state, JOB_STATE_WORKING) # 2nd time failure goes to FAILED queue rjq.fail(job1c) self.assertEqual(rjq.count(NS_WORKING), 0) self.assertEqual(rjq.count(NS_QUEUED), 0) self.assertEqual(rjq.count(NS_SCHEDULED), 0) self.assertEqual(rjq.count(NS_FAILED), 1) job1c = rjq.get(job1c.id) self.assertEqual(job1c.state, JOB_STATE_FAILED) self.assertEqual(job1c.failures, 2) self.assertTrue(job_cmp(job1b, job1c))
def test_tx_consume_fail_fail(self): yield self.rjq.maxfailed(1) jobgen = generate_jobs() job1a = next(jobgen) job1c = yield self.consume_fail(job1a) yield self.rjq.fail(job1c) working_jobs = yield self.rjq.count(NS_WORKING) self.assertEqual(working_jobs, 0) scheduled_jobs = yield self.rjq.count(NS_SCHEDULED) self.assertEqual(scheduled_jobs, 0) failed_jobs = yield self.rjq.count(NS_FAILED) self.assertEqual(failed_jobs, 1)
def test_add_consume_ack(self): jobgen = generate_jobs() for rjq in (self.rjq, TxRedisJobQueueView(self.rjq, self.rjq.namespace + ":MORE")): job1 = next(jobgen) res = yield rjq.enqueue(job1) self.assertEqual(res, 1) job1b = yield rjq.consume() self.assertTrue(job_cmp(job1, job1b)) res = yield rjq.ack(job1b) self.assertEqual(res, 1)
def test_add_consume_add_again(self): rjq = self.rjq jobgen = generate_jobs() job1 = next(jobgen) yield rjq.enqueue(job1) job1consumed = yield rjq.consume() try: # Second time raises `JobExists` yield self.rjq.add(job1) except JobInWork: pass
def test_enqueue_consume_ack_2(self): # """Enqueue multiple, checking queue counts, consume all.""" rjq = self.rjq jobgen = generate_jobs() self.assertEqual(rjq.count(NS_QUEUED), 0) for i in range(1, 5): job = next(jobgen) rjq.add(job) self.assertEqual(rjq.count(NS_QUEUED), i) while rjq.can_consume(): job = rjq.consume() with self.assertRaises(NoItems): rjq.consume()
def test_tx_schedule_1(self): can_consume = yield self.rjq.can_consume() self.assertFalse(can_consume) jobgen = generate_jobs() job1a = next(jobgen) t = utcunixts() + 1 yield self.rjq.schedule(job1a, t) scheduled_jobs = yield self.rjq.count(NS_SCHEDULED) self.assertEqual(scheduled_jobs, 1) can_consume = yield self.rjq.can_consume() self.assertFalse(can_consume) yield task.deferLater(reactor, 1.2, lambda *args: None) can_consume = yield self.rjq.can_consume() self.assertTrue(can_consume) job1b = yield self.rjq.consume() self.assertTrue(job_cmp(job1a, job1b))
def test_queue_iter(self): rjq = self.rjq jobgen = generate_jobs() rjq.add(next(jobgen)) rjq.add(next(jobgen)) it = rjq.queue_iter(NS_QUEUED) next(it) job1 = rjq.consume() rjq.ack(job1) job2 = rjq.consume() rjq.ack(job2) with self.assertRaises(StopIteration): next(it)
def test_subscribe_post_consume(self): rjq = self.rjq jobgen = generate_jobs() job1 = next(jobgen) sub = rjq.subscribe() msg = next(sub) self.assertTrue(msg is None) rjq.enqueue(job1) msg = next(sub) self.assertEqual(msg[1], NS_QUEUED) self.assertEqual(msg[0], job1.id)
def test_peek(self): rjq = self.rjq jobgen = generate_jobs() job0 = next(jobgen) job0.priority = 1 job1 = next(jobgen) job1.priority = 2 job2 = next(jobgen) job2.priority = 3 yield rjq.add(job0) yield rjq.add(job1) yield rjq.add(job2) jobs = [] yield rjq.peek(jobs.append, NS_QUEUED) self.assertTrue(job_cmp(job0, jobs[0])) self.assertTrue(job_cmp(job1, jobs[1])) self.assertTrue(job_cmp(job2, jobs[2])) jobs = [] yield rjq.peek(jobs.append, NS_QUEUED, count=2) self.assertTrue(job_cmp(job0, jobs[0])) self.assertTrue(job_cmp(job1, jobs[1])) self.assertEqual(len(jobs), 2) try: yield rjq.peek(jobs.append, "INVALID_QUEUE_NAME") except AbnormalOperationError as err: assert isinstance(err, InvalidQueue) # This method works like an iterator jobit = yield rjq.queue_iter(NS_QUEUED) job0b = yield next(jobit) job1b = yield next(jobit) job2b = yield next(jobit) self.assertTrue(job_cmp(job0, job0b)) self.assertTrue(job_cmp(job1, job1b)) self.assertTrue(job_cmp(job2, job2b)) jobit = yield rjq.queue_iter(NS_QUEUED, count=2) njobs = 0 for _ in jobit: njobs += 1 self.assertEqual(njobs, 2)
def test_reschedule(self): rjq = self.rjq rjq.maxfailed(0) jobgen = generate_jobs() job1a = next(jobgen) t = utcunixts() rjq.schedule(job1a, t) job1b, job1b_t = next(rjq.queue_iter(NS_SCHEDULED, withscores=True)) self.assertEqual(t, job1b_t) self.assertTrue(rjq.can_consume()) t = utcunixts() + 1 rjq.reschedule(job1a, t) self.assertFalse(rjq.can_consume()) job1b, job1b_t = next(rjq.queue_iter(NS_SCHEDULED, withscores=True)) self.assertEqual(t, job1b_t)
def test_requeue(self): rjq = self.rjq rjq.maxfailed(0) jobgen = generate_jobs() job1a = next(jobgen) job1a.priority = 5 jobid = job1a.id rjq.add(job1a) job1a, prio = next(rjq.queue_iter(NS_QUEUED, withscores=True)) self.assertEqual(rjq.get(jobid).priority, 5) self.assertEqual(prio, 5) job1a.priority = 6 rjq.requeue(job1a) job1a, prio = next(rjq.queue_iter(NS_QUEUED, withscores=True)) self.assertEqual(rjq.get(jobid).priority, 6) self.assertEqual(prio, 6)
def test_scanner(self): jobgen = generate_jobs() for _ in range(10): job = next(jobgen) yield self.rjq.add(job) # count < available scanner = JobCollectScanner(self.rjq) yield scanner.scan(NS_QUEUED, count=2) self.assertEqual(len(scanner.jobs), 2) # count == available scanner = JobCollectScanner(self.rjq) yield scanner.scan(NS_QUEUED) self.assertEqual(len(scanner.jobs), 10) # count > available scanner = JobCollectScanner(self.rjq) yield scanner.scan(NS_QUEUED, count=20) self.assertEqual(len(scanner.jobs), 10)
def test_subscribe(self): rjq = self.rjq jobgen = generate_jobs() job1 = next(jobgen) job_signals = [] def collectJobSignals(job_signal): job_signals.append(job_signal) sub = yield rjq.subscribe( collectJobSignals) # subscribe to all queue events yield rjq.enqueue(job1) yield task.deferLater(reactor, 0.2, lambda *args: None) self.assertEqual(len(job_signals), 1) self.assertEqual(job_signals[0][0], job1.id) yield sub.unsubscribe("") yield sub.disconnect()
def test_consume(self): no = yield self.rjq.can_consume() self.assertFalse(no) jobgen = generate_jobs() jobs = [] for i in range(10): job = next(jobgen) job.priority = i jobs.append(job) yield self.rjq.add(job) yes = yield self.rjq.can_consume() self.assertTrue(yes) job0 = yield self.rjq.consume() self.assertTrue(job_cmp(job0, jobs[0])) @defer.inlineCallbacks def get_jobs(queue): d = yield self.rjq.queue_iter(queue) jobs = [] for d_job in d: job = yield d_job jobs.append(job) defer.returnValue(jobs) working = yield get_jobs(NS_WORKING) self.assertEqual(len(working), 1) wct = yield self.rjq.count(NS_WORKING) self.assertEqual(wct, 1) yield self.rjq.ack(working[0]) working = yield get_jobs(NS_WORKING) self.assertEqual(len(working), 0) wct = yield self.rjq.count(NS_WORKING) self.assertEqual(wct, 0)
def test_add_consume_clear(self): rjq = self.rjq yield rjq.maxfailed(1) jobgen = generate_jobs() for _ in range(10): yield rjq.enqueue(next(jobgen)) job = yield rjq.consume() number_cleared = yield rjq.clear() can_consume = yield rjq.can_consume() self.assertEqual(number_cleared, 9) self.assertFalse(can_consume) yield rjq.fail(job, requeue_seconds=0) can_consume = yield rjq.can_consume() self.assertTrue(can_consume) number_enqueued = yield rjq.count(NS_SCHEDULED) self.assertEqual(number_enqueued, 1)
def test_add_consume_clear_all(self): rjq = self.rjq yield rjq.maxfailed(1) jobgen = generate_jobs() for _ in range(10): yield rjq.enqueue(next(jobgen)) job = yield rjq.consume() number_cleared = yield rjq.clear(queues=(NS_QUEUED, NS_SCHEDULED, NS_FAILED, NS_WORKING)) self.assertEqual(number_cleared, 10) self.assertFalse((yield rjq.can_consume())) try: yield rjq.ack(job) except UnknownJobId: pass try: yield rjq.fail(job) except UnknownJobId: pass
def test_list_workers(self): rjq = self.rjq jobgen = generate_jobs() job1 = next(jobgen) rjq.enqueue(job1) nworkers = len(rjq.workers()) njobs = len(rjq.jobs_for_worker(rjq.name)) nactive = len(rjq.active()) self.assertEqual(nworkers, 0) self.assertEqual(njobs, 0) self.assertEqual(nactive, 0) job1consumed = rjq.consume() nworkers = len(rjq.workers()) njobs = len(rjq.jobs_for_worker(rjq.name)) nactive = len(rjq.active()) self.assertEqual(nworkers, 1) self.assertEqual(njobs, 1) self.assertEqual(nactive, 1)
def add_jobs(self, n=10): rjq = self.rjq jobgen = generate_jobs() for _ in range(n): yield rjq.enqueue(next(jobgen))