Esempio n. 1
0
class AsyncMongoDBPipeline(object):
    
    def __init__(self):
        host = settings['MONGODB_SERVER']
        port = settings['MONGODB_PORT']
        db = settings['MONGODB_EMAIL_DB']
        collection = settings['MONGODB_EMAIL_COLLECTION']

        self.connection = MongoConnectionPool(host=host, port=port)
        logger.info("Async MongoDB connection established...")
        self.db = self.connection[db]
        self.collection = self.db[collection]

    def evaluateIndex(self, indxs):
        if ('EmailIndex' in indxs) and (indxs['EmailIndex']['unique'] == True):
            return
        else:
            logging.info("Creating email index...")
            self.collection.create_index(
                qf.sort(qf.ASCENDING('Email')),
                name="EmailIndex", unique=True)

    @defer.inlineCallbacks
    def process_item(self, item, spider):
        try:
            yield self.collection.insert(dict(item))
        except:
            print("Caught data exception in pipeline for", item["PageUrl"])
    def close_spider(self, spider):
        spider.log("Spider %s closed, closing mongodb connection"
                   % (spider.name), level=logging.INFO)
        self.connection.disconnect()

    def printError(self, err):
        logger.debug(err.getErrorMessage())
Esempio n. 2
0
 def pool(self):
     if not getattr(self, '_pool', None):
         mongo = yield MongoConnectionPool(
             host=self.settings.get('oauth_mongo_host', '127.0.0.1'),
             port=self.settings.get('oauth_mongo_port', 27017),
             reconnect=self.settings.get('oauth_mongo_reconnect', True),
             pool_size=self.settings.get('oauth_mongo_pool_size', 5))
         self._pool = mongo
     defer.returnValue(self._pool)
Esempio n. 3
0
    def __init__(self):
        host = settings['MONGODB_SERVER']
        port = settings['MONGODB_PORT']
        db = settings['MONGODB_EMAIL_DB']
        collection = settings['MONGODB_EMAIL_COLLECTION']

        self.connection = MongoConnectionPool(host=host, port=port)
        logger.info("Async MongoDB connection established...")
        self.db = self.connection[db]
        self.collection = self.db[collection]
Esempio n. 4
0
 def __init__(self, hostname="127.0.0.1", port=27017):
     connection = MongoConnectionPool(host=hostname, port=port)
     self.collection = connection.benchmark.results