class MongoQueueTest(TestCase): def setUp(self): self.client = pymongo.MongoClient() self.db = self.client.test_queue self.queue = MongoQueue(self.db.queue_1, "consumer_1") def tearDown(self): self.client.drop_database("test_queue") def assert_job_equal(self, job, data): for k, v in data.items(): self.assertEqual(job.payload[k], v) def test_put_next(self): data = {"context_id": "alpha", "data": [1, 2, 3], "more-data": time.time()} self.queue.put(dict(data)) job = self.queue.next() self.assert_job_equal(job, data) job = self.queue.next() self.assertEqual(job, None) def test_atomic_next(self): data = {"context_id": "alpha321", "data": [1, 2, 3], "more-data": time.time()} self.queue.put(dict(data)) p = mp.Pool() q = self.queue jobs = p.map(dequeue, [1,2]) self.assertNotEqual(jobs[0], jobs[1]) def test_get_empty_queue(self): job = self.queue.next() self.assertEqual(job, None) def test_priority(self): self.queue.put({"name": "alice"}, priority=1) self.queue.put({"name": "bob"}, priority=2) self.queue.put({"name": "mike"}, priority=0) self.assertEqual( ["bob", "alice", "mike"], [self.queue.next().payload['name'], self.queue.next().payload['name'], self.queue.next().payload['name']]) job = self.queue.next() self.assertEqual(job, None) def test_complete(self): data = {"context_id": "alpha", "data": [1, 2, 3], "more-data": datetime.now()} self.queue.put(data) self.assertEqual(self.queue.size(), 1) job = self.queue.next() job.complete() self.assertEqual(self.queue.size(), 0) job = self.queue.next() self.assertEqual(job, None) def test_release(self): data = {"context_id": "alpha", "data": [1, 2, 3], "more-data": time.time()} self.queue.put(data) job = self.queue.next() job.release() self.assertEqual(self.queue.size(), 1) job = self.queue.next() self.assert_job_equal(job, data) job = self.queue.next() self.assertEqual(job, None) def test_max_attempts(self): data = {"context_id": "alpha", "ts": time.time()} self.queue.put(dict(data)) attempts = 0 for i in xrange(0, self.queue.max_attempts): job = self.queue.next() if not job: break with job: attempts += 1 raise Exception() self.assertEqual(attempts, self.queue.max_attempts) def test_error(self): pass def test_progress(self): pass def test_stats(self): for i in range(5): data = {"context_id": "alpha", "data": [1, 2, 3], "more-data": time.time()} self.queue.put(data) job = self.queue.next() job.error("problem") stats = self.queue.stats() self.assertEqual({'available': 5, 'total': 5, 'locked': 0, 'errors': 0}, stats) def test_context_manager_error(self): self.queue.put({"foobar": 1}) job = self.queue.next() try: with job as data: self.assertEqual(data['payload']["foobar"], 1) # Item is returned to the queue on error raise SyntaxError except SyntaxError: pass job = self.queue.next() self.assertEqual(job.data['attempts'], 1) def test_context_manager_complete(self): self.queue.put({"foobar": 1}) job = self.queue.next() with job as data: self.assertEqual(data['payload']["foobar"], 1) job = self.queue.next() self.assertEqual(job, None) def test_next_by_payload(self): self.queue.put({"type": "first_type", "param":"param1"}) self.queue.put({"type": "second_type", "param":"param2"}) self.queue.put({"type": "third_type", "param":"param3"}) job = self.queue.next({"payload.type": "second_type"}) with job as data: self.assertEqual(data["payload"]["param"], "param2") job = self.queue.next({"payload.type": "third_type"}) with job as data: self.assertEqual(data["payload"]["param"], "param3") job = self.queue.next({"payload.type": "fourth_type"}) self.assertEqual(job, None) job = self.queue.next() with job as data: self.assertEqual(data["payload"]["param"], "param1") job = self.queue.next() self.assertEqual(job, None)
class MongoQueueTest(TestCase): def setUp(self): self.client = pymongo.Connection(os.environ.get("TEST_MONGODB")) self.db = self.client.test_queue self.queue = MongoQueue(self.db.queue_1, "consumer_1") def tearDown(self): self.client.drop_database("test_queue") def assert_job_equal(self, job, data): for k, v in data.items(): self.assertEqual(job.payload[k], v) def test_put_next(self): data = {"context_id": "alpha", "data": [1, 2, 3], "more-data": time.time()} self.queue.put(dict(data)) job = self.queue.next() self.assert_job_equal(job, data) def test_get_empty_queue(self): job = self.queue.next() self.assertEqual(job, None) def test_priority(self): data = {"priority": 1, "name": "hello world"} self.queue.put({ "priority": 1, "name": "alice"}) self.queue.put({ "priority": 2, "name": "bob"}) self.queue.put({ "priority": 0, "name": "mike"}) self.assertEqual( ["bob", "alice", "mike"], [self.queue.next().payload['name'], self.queue.next().payload['name'], self.queue.next().payload['name']]) def test_complete(self): data = {"context_id": "alpha", "data": [1, 2, 3], "more-data": datetime.now()} self.queue.put(data) self.assertEqual(self.queue.size(), 1) job = self.queue.next() job.complete() self.assertEqual(self.queue.size(), 0) def test_release(self): data = {"context_id": "alpha", "data": [1, 2, 3], "more-data": time.time()} self.queue.put(data) job = self.queue.next() job.release() self.assertEqual(self.queue.size(), 1) job = self.queue.next() self.assert_job_equal(job, data) def test_error(self): pass def test_progress(self): pass def test_stats(self): for i in range(5): data = {"context_id": "alpha", "data": [1, 2, 3], "more-data": time.time()} self.queue.put(data) job = self.queue.next() job.error("problem") stats = self.queue.stats() self.assertEqual({'available': 5, 'total': 5, 'locked': 0, 'errors': 0}, stats) def test_context_manager_error(self): self.queue.put({"foobar": 1}) job = self.queue.next() try: with job as data: self.assertEqual(data['payload']["foobar"], 1) # Item is returned to the queue on error raise SyntaxError except SyntaxError: pass job = self.queue.next() self.assertEqual(job.data['attempts'], 1) def test_context_manager_complete(self): self.queue.put({"foobar": 1}) job = self.queue.next() with job as data: self.assertEqual(data['payload']["foobar"], 1) job = self.queue.next() self.assertEqual(job, None)
class MongoQueueTest(TestCase): def setUp(self): self.client = pymongo.MongoClient() self.db = self.client.test_queue self.queue = MongoQueue(self.db.queue_1, "consumer_1") def tearDown(self): self.client.drop_database("test_queue") def assert_job_equal(self, job, data): for k, v in data.items(): self.assertEqual(job.payload[k], v) def test_put_next(self): data = { "context_id": "alpha", "data": [1, 2, 3], "more-data": time.time() } self.queue.put(dict(data)) job = self.queue.next() self.assert_job_equal(job, data) job = self.queue.next() self.assertEqual(job, None) def test_atomic_next(self): data = { "context_id": "alpha321", "data": [1, 2, 3], "more-data": time.time() } self.queue.put(dict(data)) p = mp.Pool() q = self.queue jobs = p.map(dequeue, [1, 2]) self.assertNotEqual(jobs[0], jobs[1]) def test_get_empty_queue(self): job = self.queue.next() self.assertEqual(job, None) def test_priority(self): self.queue.put({"name": "alice"}, priority=1) self.queue.put({"name": "bob"}, priority=2) self.queue.put({"name": "mike"}, priority=0) self.assertEqual(["bob", "alice", "mike"], [ self.queue.next().payload['name'], self.queue.next().payload['name'], self.queue.next().payload['name'] ]) job = self.queue.next() self.assertEqual(job, None) def test_complete(self): data = { "context_id": "alpha", "data": [1, 2, 3], "more-data": datetime.now() } self.queue.put(data) self.assertEqual(self.queue.size(), 1) job = self.queue.next() job.complete() self.assertEqual(self.queue.size(), 0) job = self.queue.next() self.assertEqual(job, None) def test_release(self): data = { "context_id": "alpha", "data": [1, 2, 3], "more-data": time.time() } self.queue.put(data) job = self.queue.next() job.release() self.assertEqual(self.queue.size(), 1) job = self.queue.next() self.assert_job_equal(job, data) job = self.queue.next() self.assertEqual(job, None) def test_max_attempts(self): data = {"context_id": "alpha", "ts": time.time()} self.queue.put(dict(data)) attempts = 0 for i in xrange(0, self.queue.max_attempts): job = self.queue.next() if not job: break with job: attempts += 1 raise Exception() self.assertEqual(attempts, self.queue.max_attempts) def test_error(self): pass def test_progress(self): pass def test_stats(self): for i in range(5): data = { "context_id": "alpha", "data": [1, 2, 3], "more-data": time.time() } self.queue.put(data) job = self.queue.next() job.error("problem") stats = self.queue.stats() self.assertEqual({ 'available': 5, 'total': 5, 'locked': 0, 'errors': 0 }, stats) def test_context_manager_error(self): self.queue.put({"foobar": 1}) job = self.queue.next() try: with job as data: self.assertEqual(data['payload']["foobar"], 1) # Item is returned to the queue on error raise SyntaxError except SyntaxError: pass job = self.queue.next() self.assertEqual(job.data['attempts'], 1) def test_context_manager_complete(self): self.queue.put({"foobar": 1}) job = self.queue.next() with job as data: self.assertEqual(data['payload']["foobar"], 1) job = self.queue.next() self.assertEqual(job, None) def test_next_by_payload(self): self.queue.put({"type": "first_type", "param": "param1"}) self.queue.put({"type": "second_type", "param": "param2"}) self.queue.put({"type": "third_type", "param": "param3"}) job = self.queue.next({"payload.type": "second_type"}) with job as data: self.assertEqual(data["payload"]["param"], "param2") job = self.queue.next({"payload.type": "third_type"}) with job as data: self.assertEqual(data["payload"]["param"], "param3") job = self.queue.next({"payload.type": "fourth_type"}) self.assertEqual(job, None) job = self.queue.next() with job as data: self.assertEqual(data["payload"]["param"], "param1") job = self.queue.next() self.assertEqual(job, None)
class MongoQueueTest(TestCase): def setUp(self): self.client = pymongo.Connection(os.environ.get("TEST_MONGODB")) self.db = self.client.test_queue self.queue = MongoQueue(self.db.queue_1, "consumer_1") def tearDown(self): self.client.drop_database("test_queue") def assert_job_equal(self, job, data): for k, v in data.items(): self.assertEqual(job.payload[k], v) def test_put_next(self): data = { "context_id": "alpha", "data": [1, 2, 3], "more-data": time.time() } self.queue.put(dict(data)) job = self.queue.next() self.assert_job_equal(job, data) def test_get_empty_queue(self): job = self.queue.next() self.assertEqual(job, None) def test_priority(self): self.queue.put({"name": "alice"}, priority=1) self.queue.put({"name": "bob"}, priority=2) self.queue.put({"name": "mike"}, priority=0) self.assertEqual(["bob", "alice", "mike"], [ self.queue.next().payload['name'], self.queue.next().payload['name'], self.queue.next().payload['name'] ]) def test_complete(self): data = { "context_id": "alpha", "data": [1, 2, 3], "more-data": datetime.now() } self.queue.put(data) self.assertEqual(self.queue.size(), 1) job = self.queue.next() job.complete() self.assertEqual(self.queue.size(), 0) def test_release(self): data = { "context_id": "alpha", "data": [1, 2, 3], "more-data": time.time() } self.queue.put(data) job = self.queue.next() job.release() self.assertEqual(self.queue.size(), 1) job = self.queue.next() self.assert_job_equal(job, data) def test_error(self): pass def test_progress(self): pass def test_stats(self): for i in range(5): data = { "context_id": "alpha", "data": [1, 2, 3], "more-data": time.time() } self.queue.put(data) job = self.queue.next() job.error("problem") stats = self.queue.stats() self.assertEqual({ 'available': 5, 'total': 5, 'locked': 0, 'errors': 0 }, stats) def test_context_manager_error(self): self.queue.put({"foobar": 1}) job = self.queue.next() try: with job as data: self.assertEqual(data['payload']["foobar"], 1) # Item is returned to the queue on error raise SyntaxError except SyntaxError: pass job = self.queue.next() self.assertEqual(job.data['attempts'], 1) def test_context_manager_complete(self): self.queue.put({"foobar": 1}) job = self.queue.next() with job as data: self.assertEqual(data['payload']["foobar"], 1) job = self.queue.next() self.assertEqual(job, None)