Beispiel #1
0
class MongoQueueTest(TestCase):

    def setUp(self):
        self.client = pymongo.MongoClient()
        self.db = self.client.test_queue
        self.queue = MongoQueue(self.db.queue_1, "consumer_1")

    def tearDown(self):
        self.client.drop_database("test_queue")

    def assert_job_equal(self, job, data):
        for k, v in data.items():
            self.assertEqual(job.payload[k], v)

    def test_put_next(self):
        data = {"context_id": "alpha",
                "data": [1, 2, 3],
                "more-data": time.time()}
        self.queue.put(dict(data))
        job = self.queue.next()
        self.assert_job_equal(job, data)

        job = self.queue.next()
        self.assertEqual(job, None)

    def test_atomic_next(self):
        data = {"context_id": "alpha321",
                "data": [1, 2, 3],
                "more-data": time.time()}
        self.queue.put(dict(data))

        p = mp.Pool()
        q = self.queue
        jobs = p.map(dequeue, [1,2])
        self.assertNotEqual(jobs[0], jobs[1])

    def test_get_empty_queue(self):
        job = self.queue.next()
        self.assertEqual(job, None)

    def test_priority(self):
        self.queue.put({"name": "alice"}, priority=1)
        self.queue.put({"name": "bob"}, priority=2)
        self.queue.put({"name": "mike"}, priority=0)

        self.assertEqual(
            ["bob", "alice", "mike"],
            [self.queue.next().payload['name'],
             self.queue.next().payload['name'],
             self.queue.next().payload['name']])

        job = self.queue.next()
        self.assertEqual(job, None)

    def test_complete(self):
        data = {"context_id": "alpha",
                "data": [1, 2, 3],
                "more-data": datetime.now()}

        self.queue.put(data)
        self.assertEqual(self.queue.size(), 1)
        job = self.queue.next()
        job.complete()
        self.assertEqual(self.queue.size(), 0)

        job = self.queue.next()
        self.assertEqual(job, None)

    def test_release(self):
        data = {"context_id": "alpha",
                "data": [1, 2, 3],
                "more-data": time.time()}

        self.queue.put(data)
        job = self.queue.next()
        job.release()
        self.assertEqual(self.queue.size(), 1)
        job = self.queue.next()
        self.assert_job_equal(job, data)

        job = self.queue.next()
        self.assertEqual(job, None)

    def test_max_attempts(self):
        data = {"context_id": "alpha",
                "ts": time.time()}
        self.queue.put(dict(data))
        attempts = 0
        for i in xrange(0, self.queue.max_attempts):
            job = self.queue.next()
            if not job:
                break
            with job:
                attempts += 1
                raise Exception()
        self.assertEqual(attempts, self.queue.max_attempts)

    def test_error(self):
        pass

    def test_progress(self):
        pass

    def test_stats(self):

        for i in range(5):
            data = {"context_id": "alpha",
                    "data": [1, 2, 3],
                    "more-data": time.time()}
            self.queue.put(data)
        job = self.queue.next()
        job.error("problem")

        stats = self.queue.stats()
        self.assertEqual({'available': 5,
                          'total': 5,
                          'locked': 0,
                          'errors': 0}, stats)

    def test_context_manager_error(self):
        self.queue.put({"foobar": 1})
        job = self.queue.next()
        try:
            with job as data:
                self.assertEqual(data['payload']["foobar"], 1)
                # Item is returned to the queue on error
                raise SyntaxError
        except SyntaxError:
            pass

        job = self.queue.next()
        self.assertEqual(job.data['attempts'], 1)

    def test_context_manager_complete(self):
        self.queue.put({"foobar": 1})
        job = self.queue.next()
        with job as data:
            self.assertEqual(data['payload']["foobar"], 1)
        job = self.queue.next()
        self.assertEqual(job, None)

    def test_next_by_payload(self):
        self.queue.put({"type": "first_type", "param":"param1"})
        self.queue.put({"type": "second_type", "param":"param2"})
        self.queue.put({"type": "third_type", "param":"param3"})

        job = self.queue.next({"payload.type": "second_type"})
        with job as data:
            self.assertEqual(data["payload"]["param"], "param2")

        job = self.queue.next({"payload.type": "third_type"})
        with job as data:
            self.assertEqual(data["payload"]["param"], "param3")

        job = self.queue.next({"payload.type": "fourth_type"})
        self.assertEqual(job, None)

        job = self.queue.next()
        with job as data:
            self.assertEqual(data["payload"]["param"], "param1")

        job = self.queue.next()
        self.assertEqual(job, None)
Beispiel #2
0
class MongoQueueTest(TestCase):

    def setUp(self):
        self.client = pymongo.Connection(os.environ.get("TEST_MONGODB"))
        self.db = self.client.test_queue
        self.queue = MongoQueue(self.db.queue_1, "consumer_1")

    def tearDown(self):
        self.client.drop_database("test_queue")

    def assert_job_equal(self, job, data):
        for k, v in data.items():
            self.assertEqual(job.payload[k], v)

    def test_put_next(self):
        data = {"context_id": "alpha",
                "data": [1, 2, 3],
                "more-data": time.time()}
        self.queue.put(dict(data))
        job = self.queue.next()
        self.assert_job_equal(job, data)

    def test_get_empty_queue(self):
        job = self.queue.next()
        self.assertEqual(job, None)

    def test_priority(self):
        data = {"priority": 1,
                "name": "hello world"}
        self.queue.put({
            "priority": 1, "name": "alice"})
        self.queue.put({
            "priority": 2, "name": "bob"})
        self.queue.put({
            "priority": 0, "name": "mike"})
        self.assertEqual(
            ["bob", "alice", "mike"],
            [self.queue.next().payload['name'],
             self.queue.next().payload['name'],
             self.queue.next().payload['name']])

    def test_complete(self):
        data = {"context_id": "alpha",
                "data": [1, 2, 3],
                "more-data": datetime.now()}

        self.queue.put(data)
        self.assertEqual(self.queue.size(), 1)
        job = self.queue.next()
        job.complete()
        self.assertEqual(self.queue.size(), 0)

    def test_release(self):
        data = {"context_id": "alpha",
                "data": [1, 2, 3],
                "more-data": time.time()}

        self.queue.put(data)
        job = self.queue.next()
        job.release()
        self.assertEqual(self.queue.size(), 1)
        job = self.queue.next()
        self.assert_job_equal(job, data)

    def test_error(self):
        pass

    def test_progress(self):
        pass

    def test_stats(self):

        for i in range(5):
            data = {"context_id": "alpha",
                    "data": [1, 2, 3],
                    "more-data": time.time()}
            self.queue.put(data)
        job = self.queue.next()
        job.error("problem")

        stats = self.queue.stats()
        self.assertEqual({'available': 5,
                          'total': 5,
                          'locked': 0,
                          'errors': 0}, stats)

    def test_context_manager_error(self):
        self.queue.put({"foobar": 1})
        job = self.queue.next()
        try:
            with job as data:
                self.assertEqual(data['payload']["foobar"], 1)
                # Item is returned to the queue on error
                raise SyntaxError
        except SyntaxError:
            pass

        job = self.queue.next()
        self.assertEqual(job.data['attempts'], 1)

    def test_context_manager_complete(self):
        self.queue.put({"foobar": 1})
        job = self.queue.next()
        with job as data:
            self.assertEqual(data['payload']["foobar"], 1)
        job = self.queue.next()
        self.assertEqual(job, None)
Beispiel #3
0
class MongoQueueTest(TestCase):
    def setUp(self):
        self.client = pymongo.MongoClient()
        self.db = self.client.test_queue
        self.queue = MongoQueue(self.db.queue_1, "consumer_1")

    def tearDown(self):
        self.client.drop_database("test_queue")

    def assert_job_equal(self, job, data):
        for k, v in data.items():
            self.assertEqual(job.payload[k], v)

    def test_put_next(self):
        data = {
            "context_id": "alpha",
            "data": [1, 2, 3],
            "more-data": time.time()
        }
        self.queue.put(dict(data))
        job = self.queue.next()
        self.assert_job_equal(job, data)

        job = self.queue.next()
        self.assertEqual(job, None)

    def test_atomic_next(self):
        data = {
            "context_id": "alpha321",
            "data": [1, 2, 3],
            "more-data": time.time()
        }
        self.queue.put(dict(data))

        p = mp.Pool()
        q = self.queue
        jobs = p.map(dequeue, [1, 2])
        self.assertNotEqual(jobs[0], jobs[1])

    def test_get_empty_queue(self):
        job = self.queue.next()
        self.assertEqual(job, None)

    def test_priority(self):
        self.queue.put({"name": "alice"}, priority=1)
        self.queue.put({"name": "bob"}, priority=2)
        self.queue.put({"name": "mike"}, priority=0)

        self.assertEqual(["bob", "alice", "mike"], [
            self.queue.next().payload['name'],
            self.queue.next().payload['name'],
            self.queue.next().payload['name']
        ])

        job = self.queue.next()
        self.assertEqual(job, None)

    def test_complete(self):
        data = {
            "context_id": "alpha",
            "data": [1, 2, 3],
            "more-data": datetime.now()
        }

        self.queue.put(data)
        self.assertEqual(self.queue.size(), 1)
        job = self.queue.next()
        job.complete()
        self.assertEqual(self.queue.size(), 0)

        job = self.queue.next()
        self.assertEqual(job, None)

    def test_release(self):
        data = {
            "context_id": "alpha",
            "data": [1, 2, 3],
            "more-data": time.time()
        }

        self.queue.put(data)
        job = self.queue.next()
        job.release()
        self.assertEqual(self.queue.size(), 1)
        job = self.queue.next()
        self.assert_job_equal(job, data)

        job = self.queue.next()
        self.assertEqual(job, None)

    def test_max_attempts(self):
        data = {"context_id": "alpha", "ts": time.time()}
        self.queue.put(dict(data))
        attempts = 0
        for i in xrange(0, self.queue.max_attempts):
            job = self.queue.next()
            if not job:
                break
            with job:
                attempts += 1
                raise Exception()
        self.assertEqual(attempts, self.queue.max_attempts)

    def test_error(self):
        pass

    def test_progress(self):
        pass

    def test_stats(self):

        for i in range(5):
            data = {
                "context_id": "alpha",
                "data": [1, 2, 3],
                "more-data": time.time()
            }
            self.queue.put(data)
        job = self.queue.next()
        job.error("problem")

        stats = self.queue.stats()
        self.assertEqual({
            'available': 5,
            'total': 5,
            'locked': 0,
            'errors': 0
        }, stats)

    def test_context_manager_error(self):
        self.queue.put({"foobar": 1})
        job = self.queue.next()
        try:
            with job as data:
                self.assertEqual(data['payload']["foobar"], 1)
                # Item is returned to the queue on error
                raise SyntaxError
        except SyntaxError:
            pass

        job = self.queue.next()
        self.assertEqual(job.data['attempts'], 1)

    def test_context_manager_complete(self):
        self.queue.put({"foobar": 1})
        job = self.queue.next()
        with job as data:
            self.assertEqual(data['payload']["foobar"], 1)
        job = self.queue.next()
        self.assertEqual(job, None)

    def test_next_by_payload(self):
        self.queue.put({"type": "first_type", "param": "param1"})
        self.queue.put({"type": "second_type", "param": "param2"})
        self.queue.put({"type": "third_type", "param": "param3"})

        job = self.queue.next({"payload.type": "second_type"})
        with job as data:
            self.assertEqual(data["payload"]["param"], "param2")

        job = self.queue.next({"payload.type": "third_type"})
        with job as data:
            self.assertEqual(data["payload"]["param"], "param3")

        job = self.queue.next({"payload.type": "fourth_type"})
        self.assertEqual(job, None)

        job = self.queue.next()
        with job as data:
            self.assertEqual(data["payload"]["param"], "param1")

        job = self.queue.next()
        self.assertEqual(job, None)
Beispiel #4
0
class MongoQueueTest(TestCase):
    def setUp(self):
        self.client = pymongo.Connection(os.environ.get("TEST_MONGODB"))
        self.db = self.client.test_queue
        self.queue = MongoQueue(self.db.queue_1, "consumer_1")

    def tearDown(self):
        self.client.drop_database("test_queue")

    def assert_job_equal(self, job, data):
        for k, v in data.items():
            self.assertEqual(job.payload[k], v)

    def test_put_next(self):
        data = {
            "context_id": "alpha",
            "data": [1, 2, 3],
            "more-data": time.time()
        }
        self.queue.put(dict(data))
        job = self.queue.next()
        self.assert_job_equal(job, data)

    def test_get_empty_queue(self):
        job = self.queue.next()
        self.assertEqual(job, None)

    def test_priority(self):
        self.queue.put({"name": "alice"}, priority=1)
        self.queue.put({"name": "bob"}, priority=2)
        self.queue.put({"name": "mike"}, priority=0)

        self.assertEqual(["bob", "alice", "mike"], [
            self.queue.next().payload['name'],
            self.queue.next().payload['name'],
            self.queue.next().payload['name']
        ])

    def test_complete(self):
        data = {
            "context_id": "alpha",
            "data": [1, 2, 3],
            "more-data": datetime.now()
        }

        self.queue.put(data)
        self.assertEqual(self.queue.size(), 1)
        job = self.queue.next()
        job.complete()
        self.assertEqual(self.queue.size(), 0)

    def test_release(self):
        data = {
            "context_id": "alpha",
            "data": [1, 2, 3],
            "more-data": time.time()
        }

        self.queue.put(data)
        job = self.queue.next()
        job.release()
        self.assertEqual(self.queue.size(), 1)
        job = self.queue.next()
        self.assert_job_equal(job, data)

    def test_error(self):
        pass

    def test_progress(self):
        pass

    def test_stats(self):

        for i in range(5):
            data = {
                "context_id": "alpha",
                "data": [1, 2, 3],
                "more-data": time.time()
            }
            self.queue.put(data)
        job = self.queue.next()
        job.error("problem")

        stats = self.queue.stats()
        self.assertEqual({
            'available': 5,
            'total': 5,
            'locked': 0,
            'errors': 0
        }, stats)

    def test_context_manager_error(self):
        self.queue.put({"foobar": 1})
        job = self.queue.next()
        try:
            with job as data:
                self.assertEqual(data['payload']["foobar"], 1)
                # Item is returned to the queue on error
                raise SyntaxError
        except SyntaxError:
            pass

        job = self.queue.next()
        self.assertEqual(job.data['attempts'], 1)

    def test_context_manager_complete(self):
        self.queue.put({"foobar": 1})
        job = self.queue.next()
        with job as data:
            self.assertEqual(data['payload']["foobar"], 1)
        job = self.queue.next()
        self.assertEqual(job, None)