Example #1
0
    def test_Priority(self):
        """
            Check that connection-level, database-level, collection-level
            and query-level write concerns are respected with correct priority
        """
        conn = MongoConnection(mongo_host, mongo_port, w=1, wtimeout=500)

        try:
            with self.mock_gle() as mock:
                yield conn.mydb.mycol.insert({'x': 42})
                mock.assert_called_once_with("mydb", w=1, wtimeout=500)

            db_w0 = Database(conn, "mydb", write_concern=WriteConcern(w=0))
            with self.mock_gle() as mock:
                yield db_w0.mycol.insert({'x': 42})
                self.assertFalse(mock.called)

            coll = Collection(db_w0, "mycol", write_concern=WriteConcern(w=2))
            with self.mock_gle() as mock:
                yield coll.insert({'x': 42})
                mock.assert_called_once_with("mydb", w=2)

            with self.mock_gle() as mock:
                yield coll.insert({'x': 42}, j=True)
                mock.assert_called_once_with("mydb", j=True)

        finally:
            yield conn.mydb.mycol.drop()
            yield conn.disconnect()
Example #2
0
    def test_Priority(self):
        """
            Check that connection-level, database-level, collection-level
            and query-level write concerns are respected with correct priority
        """
        conn = MongoConnection(mongo_host, mongo_port, w=1, wtimeout=500)

        try:
            with self.mock_gle() as mock:
                yield conn.mydb.mycol.insert({'x': 42})
                mock.assert_called_once_with("mydb", w=1, wtimeout=500)

            db_w0 = Database(conn, "mydb", write_concern=WriteConcern(w=0))
            with self.mock_gle() as mock:
                yield db_w0.mycol.insert({'x': 42})
                self.assertFalse(mock.called)

            coll = Collection(db_w0, "mycol", write_concern=WriteConcern(w=2))
            with self.mock_gle() as mock:
                yield coll.insert({'x': 42})
                mock.assert_called_once_with("mydb", w=2)

            with self.mock_gle() as mock:
                yield coll.insert({'x': 42}, j=True)
                mock.assert_called_once_with("mydb", j=True)

        finally:
            yield conn.mydb.mycol.drop()
            yield conn.disconnect()
Example #3
0
 def test_WriteToMaster(self):
     conn = MongoConnection("localhost", self.ports[0])
     try:
         coll = conn.db.coll
         yield coll.insert({'x': 42}, safe=True)
         result = yield coll.find_one()
         self.assertEqual(result['x'], 42)
     finally:
         yield conn.disconnect()
Example #4
0
    def test_SwitchToMasterOnConnect(self):
        # Reverse hosts order
        try:
            conn = MongoConnection("localhost", self.ports[1])
            result = yield conn.db.coll.find({'x': 42})
            self.assertEqual(result, [])
        finally:
            yield conn.disconnect()

        # txmongo will do log.err() for AutoReconnects
        self.flushLoggedErrors(AutoReconnect)
Example #5
0
    def test_AllOperations(self):
        conn = MongoConnection(mongo_host, mongo_port, w=0)
        coll = conn.mydb.mycol

        try:
            yield self.__test_operation(coll, "insert", {'x': 42})
            yield self.__test_operation(coll, "update", {}, {'x': 42})
            yield self.__test_operation(coll, "save", {'x': 42})
            yield self.__test_operation(coll, "remove", {})
        finally:
            yield coll.drop()
            yield conn.disconnect()
Example #6
0
    def test_AllOperations(self):
        conn = MongoConnection(mongo_host, mongo_port, w=0)
        coll = conn.mydb.mycol

        try:
            yield self.__test_operation(coll, "insert", {'x': 42})
            yield self.__test_operation(coll, "update", {}, {'x': 42})
            yield self.__test_operation(coll, "save", {'x': 42})
            yield self.__test_operation(coll, "remove", {})
        finally:
            yield coll.drop()
            yield conn.disconnect()
Example #7
0
 def test_StaleConnection(self):
     conn = MongoConnection("localhost", self.ports[0])
     try:
         yield conn.db.coll.count()
         self.__mongod[0].kill(signal.SIGSTOP)
         yield self.__sleep(0.2)
         while True:
             try:
                 yield conn.db.coll.count()
                 break
             except AutoReconnect:
                 pass
     finally:
         self.__mongod[0].kill(signal.SIGCONT)
         yield conn.disconnect()
Example #8
0
    def test_Levels(self):
        as_son = CodecOptions(document_class=SON)

        doc = yield self.coll.find_one()
        self.assertIsInstance(doc, dict)

        try:
            conn = MongoConnection(mongo_host, mongo_port, codec_options=as_son)
            doc = yield conn.db.coll.find_one()
            self.assertIsInstance(doc, SON)
        finally:
            yield conn.disconnect()

        doc = yield Database(self.conn, "db", codec_options=as_son).coll.find_one()
        self.assertIsInstance(doc, SON)

        doc = yield self.coll.with_options(codec_options=as_son).find_one()
        self.assertIsInstance(doc, SON)
Example #9
0
 def test_StaleConnection(self):
     conn = MongoConnection("localhost", self.ports[0], ping_interval = 5, ping_timeout = 5)
     try:
         yield conn.db.coll.count()
         # check that 5s pingers won't break connection if it is healthy
         yield self.__sleep(6)
         yield conn.db.coll.count()
         self.__mongod[0].kill(signal.SIGSTOP)
         yield self.__sleep(0.2)
         while True:
             try:
                 yield conn.db.coll.count()
                 break
             except AutoReconnect:
                 pass
     finally:
         self.__mongod[0].kill(signal.SIGCONT)
         yield conn.disconnect()
Example #10
0
 def test_StaleConnection(self):
     conn = MongoConnection("localhost",
                            self.ports[0],
                            watchdog_interval=10,
                            watchdog_timeout=5)
     try:
         yield conn.db.coll.count()
         self.__mongod[0].kill(signal.SIGSTOP)
         yield self.__sleep(0.2)
         while True:
             try:
                 yield conn.db.coll.count()
                 break
             except AutoReconnect:
                 pass
     finally:
         self.__mongod[0].kill(signal.SIGCONT)
         yield conn.disconnect()
Example #11
0
    def test_SafeWithDefaultW0(self):
        conn = MongoConnection(mongo_host, mongo_port, w=0)
        coll = conn.mydb.mycol

        try:
            with self.mock_gle() as mock:
                yield coll.insert({'x': 42})
                self.assertFalse(mock.called)

            with self.mock_gle() as mock:
                yield coll.insert({'x': 42}, safe=False)
                self.assertFalse(mock.called)

            with self.mock_gle() as mock:
                yield coll.insert({'x': 42}, safe=True)
                mock.assert_called_once_with("mydb")
        finally:
            yield coll.drop()
            yield conn.disconnect()
Example #12
0
    def test_SafeWithDefaultW0(self):
        conn = MongoConnection(mongo_host, mongo_port, w=0)
        coll = conn.mydb.mycol

        try:
            with self.mock_gle() as mock:
                yield coll.insert({'x': 42})
                self.assertFalse(mock.called)

            with self.mock_gle() as mock:
                yield coll.insert({'x': 42}, safe=False)
                self.assertFalse(mock.called)

            with self.mock_gle() as mock:
                yield coll.insert({'x': 42}, safe=True)
                mock.assert_called_once_with("mydb")
        finally:
            yield coll.drop()
            yield conn.disconnect()
Example #13
0
class DBPipeline(object):
    def __init__(self, mongo_uri, mongo_db, collection_name):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db
        self.collection_name = collection_name

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items'),
            collection_name=crawler.settings.get('MONGO_COLLECTION_NAME'))

    def open_spider(self, spider):
        self.client = MongoConnection(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        if self.get_item({
                "company": item["company"],
                "title": item["title"]
        }) is None:
            self.insert_item(item)
        else:
            raise DropItem(
                u'Existing item with same company and title found, skipping.')

    def get_item(self, item):
        return self.db[self.collection_name].find_one(item)

    def insert_item(self, item):
        try:
            self.db[self.collection_name].insert_one(dict(item))
            return item
        except Exception as e:
            raise DropItem(
                u'Inserting of item into database failed with error: %s' %
                str(e))
Example #14
0
    def test_Levels(self):
        as_son = CodecOptions(document_class=SON)

        doc = yield self.coll.find_one()
        self.assertIsInstance(doc, dict)

        try:
            conn = MongoConnection(mongo_host,
                                   mongo_port,
                                   codec_options=as_son)
            doc = yield conn.db.coll.find_one()
            self.assertIsInstance(doc, SON)
        finally:
            yield conn.disconnect()

        doc = yield Database(self.conn, "db",
                             codec_options=as_son).coll.find_one()
        self.assertIsInstance(doc, SON)

        doc = yield self.coll.with_options(codec_options=as_son).find_one()
        self.assertIsInstance(doc, SON)
Example #15
0
class TestCodecOptions(unittest.TestCase):
    @defer.inlineCallbacks
    def setUp(self):
        self.conn = MongoConnection(mongo_host, mongo_port)
        self.db = self.conn.db
        self.coll = self.db.coll
        yield self.coll.insert_one({'x': 42, 'y': datetime.datetime.now()})

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop()
        yield self.conn.disconnect()

    @defer.inlineCallbacks
    def test_Levels(self):
        as_son = CodecOptions(document_class=SON)

        doc = yield self.coll.find_one()
        self.assertIsInstance(doc, dict)

        try:
            conn = MongoConnection(mongo_host,
                                   mongo_port,
                                   codec_options=as_son)
            doc = yield conn.db.coll.find_one()
            self.assertIsInstance(doc, SON)
        finally:
            yield conn.disconnect()

        doc = yield Database(self.conn, "db",
                             codec_options=as_son).coll.find_one()
        self.assertIsInstance(doc, SON)

        doc = yield self.coll.with_options(codec_options=as_son).find_one()
        self.assertIsInstance(doc, SON)

    @defer.inlineCallbacks
    def test_TzAware(self):
        doc = yield self.coll.find_one()
        self.assertIsNone(doc['y'].tzinfo, None)

        tz_aware = CodecOptions(tz_aware=True)
        doc = yield self.coll.with_options(codec_options=tz_aware).find_one()
        self.assertIsNotNone(doc['y'].tzinfo, None)
Example #16
0
class TestCodecOptions(unittest.TestCase):

    @defer.inlineCallbacks
    def setUp(self):
        self.conn = MongoConnection(mongo_host, mongo_port)
        self.db = self.conn.db
        self.coll = self.db.coll
        yield self.coll.insert_one({'x': 42, 'y': datetime.datetime.now()})

    @defer.inlineCallbacks
    def tearDown(self):
        yield self.coll.drop()
        yield self.conn.disconnect()

    @defer.inlineCallbacks
    def test_Levels(self):
        as_son = CodecOptions(document_class=SON)

        doc = yield self.coll.find_one()
        self.assertIsInstance(doc, dict)

        try:
            conn = MongoConnection(mongo_host, mongo_port, codec_options=as_son)
            doc = yield conn.db.coll.find_one()
            self.assertIsInstance(doc, SON)
        finally:
            yield conn.disconnect()

        doc = yield Database(self.conn, "db", codec_options=as_son).coll.find_one()
        self.assertIsInstance(doc, SON)

        doc = yield self.coll.with_options(codec_options=as_son).find_one()
        self.assertIsInstance(doc, SON)

    @defer.inlineCallbacks
    def test_TzAware(self):
        doc = yield self.coll.find_one()
        self.assertIsNone(doc['y'].tzinfo, None)

        tz_aware = CodecOptions(tz_aware=True)
        doc = yield self.coll.with_options(codec_options=tz_aware).find_one()
        self.assertIsNotNone(doc['y'].tzinfo, None)
Example #17
0
 def setUp(self):
     self.conn = MongoConnection(mongo_host, mongo_port)
     self.db = self.conn.db
     self.coll = self.db.coll
     yield self.coll.insert_one({'x': 42, 'y': datetime.datetime.now()})
Example #18
0
 def setUp(self):
     self.conn = MongoConnection(mongo_host, mongo_port)
     self.db = self.conn.db
     self.coll = self.db.coll
     yield self.coll.insert_one({'x': 42, 'y': datetime.datetime.now()})
Example #19
0
 def open_spider(self, spider):
     self.client = MongoConnection(self.mongo_uri)
     self.db = self.client[self.mongo_db]
Example #20
0
 def __init__(self):
     self.connection_host = os.getenv("DB_SERVICE_HOST", "172.22.22.20")
     self.connection_port = os.getenv("DB_SERVICE_PORT", 27017)
     self.connection = MongoConnection(self.connection_host,
                                       self.connection_port)
     self.db = self.connection.demo_chat