Exemplo n.º 1
0
    def _open(self):
        conninfo = self.connection.client
        mongoconn = Connection(host=conninfo.hostname, port=conninfo.port)
        dbname = conninfo.virtual_host
        version = mongoconn.server_info()["version"]
        if tuple(map(int, version.split(".")[:2])) < (1, 3):
            raise NotImplementedError(
                "Kombu requires MongoDB version 1.3+, but connected to %s" % (
                    version, ))
        if not dbname or dbname == "/":
            dbname = "kombu_default"
        database = getattr(mongoconn, dbname)
        if conninfo.userid:
            database.authenticate(conninfo.userid, conninfo.password)

        self.db = database
        col = database.messages
        col.ensure_index([("queue", 1)])

        if "messages.broadcast" not in database.collection_names():
            capsize = conninfo.transport_options.get(
                            "capped_queue_size") or 100000
            database.create_collection("messages.broadcast", size=capsize,
                                                             capped=True)

        self.bcast = getattr(database, "messages.broadcast")
        self.bcast.ensure_index([("queue", 1)])

        self.routing = getattr(database, "messages.routing")
        self.routing.ensure_index([("queue", 1), ("exchange", 1)])
        return database
    def test_document_class(self):
        c = Connection(self.host, self.port)
        db = c.pymongo_test
        db.test.insert({"x": 1})

        self.assertEqual(dict, c.document_class)
        self.assertTrue(isinstance(db.test.find_one(), dict))
        self.assertFalse(isinstance(db.test.find_one(), SON))

        c.document_class = SON

        self.assertEqual(SON, c.document_class)
        self.assertTrue(isinstance(db.test.find_one(), SON))
        self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON))

        c = Connection(self.host, self.port, document_class=SON)
        db = c.pymongo_test

        self.assertEqual(SON, c.document_class)
        self.assertTrue(isinstance(db.test.find_one(), SON))
        self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON))

        c.document_class = dict

        self.assertEqual(dict, c.document_class)
        self.assertTrue(isinstance(db.test.find_one(), dict))
        self.assertFalse(isinstance(db.test.find_one(), SON))
Exemplo n.º 3
0
class TestConfig(unittest.TestCase):

    def setUp(self):
        filename = dirname(__file__) + '/logging-test.config'
        fileConfig(filename)

        """ Create an empty database that could be used for logging """
        self.db_name = '_mongolog_test'
        self.collection_name = 'log_test'

        self.conn = Connection('localhost')
        self.conn.drop_database(self.db_name)

    def tearDown(self):
        """ Drop used database """
        self.conn.drop_database(self.db_name)

    def testLoggingFileConfiguration(self):
        log = logging.getLogger('example')
        log.debug('test')

        r = self.conn[self.db_name][self.collection_name]

        message = r.find_one({'level': 'debug', 'msg': 'test'})
        self.assertEquals(message['msg'], 'test')
Exemplo n.º 4
0
 def __init__(self):
     self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
     self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, 'config', 'processing.conf'))
     self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled")
     self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled")
     self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled")
     
     if self.__mongodbEnabled:
         #Anbindung an Datenbank MongoDB Collection Ragpicker herstellen
         try:
             mongodbHost = self.__cfgReporting.getOption("mongodb", "host")
             mongodbPort = self.__cfgReporting.getOption("mongodb", "port")
             self.__mongodbConnection = Connection(mongodbHost, mongodbPort)
             self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker
             self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families
             self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue
         except TypeError:
             raise Exception("MongoDB connection port in report.config must be integer")
         except ConnectionFailure:
             raise Exception("Cannot connect to MongoDB (ragpicker)")
     
     if self.__codedbEnabled:
         #Anbindung an Datenbank MongoDB Collection CodeDB herstellen
         try:
             codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host")
             codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port")
             self.__codedbConnection = Connection(codedbHost, codedbPort)
             self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB
         except TypeError:
             raise Exception("MongoDB connection port for CodeDB in report.config must be integer")
         except ConnectionFailure:
             raise Exception("Cannot connect to MongoDB (codeDB)")  
Exemplo n.º 5
0
    def test_ipv6(self):
        self.assertRaises(InvalidURI, _parse_uri, "::1", 27017)
        self.assertRaises(InvalidURI, _parse_uri, "[::1", 27017)
        self.assertRaises(InvalidURI, _parse_uri, "::1]:27017")
        self.assertRaises(InvalidURI, _parse_uri, "mongodb://::1", 27017)
        self.assert_(_parse_uri, "mongodb://[::1]:27017/?slaveOk=true")
        self.assert_(_parse_uri,
                     "[::1]:27017,[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"
                     ":27018,192.168.0.212:27019,localhost:27020")
        self.assert_(_parse_uri,
                     "mongodb://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"
                     ":27017/?slaveOk=true")
        try:
            connection = Connection("[::1]")
        except:
            # Either mongod was started without --ipv6
            # or the OS doesn't support it (or both).
            raise SkipTest()

        # Try a few simple things
        connection = Connection("mongodb://[::1]:27017")
        connection = Connection("mongodb://[::1]:27017/?slaveOk=true")
        connection = Connection("[::1]:27017,localhost:27017")
        connection = Connection("localhost:27017,[::1]:27017")
        connection.pymongo_test.test.save({"dummy": u"object"})
        connection.pymongo_test_bernie.test.save({"dummy": u"object"})

        dbs = connection.database_names()
        self.assert_("pymongo_test" in dbs)
        self.assert_("pymongo_test_bernie" in dbs)
Exemplo n.º 6
0
class plantmongo:
    def __init__(self, addr, user="******", password="******"):
        self.addr = addr
        self.user = user
        self.password = password
        self.conn = Connection(addr)
        self.db = conn.plant
        if not db.authenticate("plant", "plant"):
            print "unable to authenticate to mongodb"
            self.connected = False
            return None
        self.coll = self.db.test

    def publish(self, xbeeobj):
        post = {
            "time": time.time(),
            "temp": xbeeobj.temp,
            "moisture": xbeeobj.moisture,
            "dewpoint": xbeeobj.dewpoint,
            "pressure": xbeeobj.pressure,
            "light": xbeeobj.light,
            "batt": xbeeobj.batt,
            "rssi": xbeeobj.rssi,
        }
        self.coll.save(post)

    def close(self):
        self.conn.disconnect()
    def test_connection(self):
        c = Connection(host, port)
        self.assertTrue(c.auto_start_request)
        self.assertFalse(c.slave_okay)
        self.assertFalse(c.safe)
        self.assertEqual({}, c.get_lasterror_options())

        # Connection's writes are unacknowledged by default
        doc = {"_id": ObjectId()}
        coll = c.pymongo_test.write_concern_test
        coll.really_drop()
        coll.insert(doc)
        coll.insert(doc)

        c = Connection("mongodb://%s:%s/?safe=true" % (host, port))
        self.assertTrue(c.safe)

        # Connection's network_timeout argument is translated into
        # socketTimeoutMS
        self.assertEqual(123, Connection(
            host, port, network_timeout=123)._MongoClient__net_timeout)

        for network_timeout in 'foo', 0, -1:
            self.assertRaises(
                ConfigurationError,
                Connection, host, port, network_timeout=network_timeout)
 def test_unix_domain_socket(self):
     try:
         connection = Connection("/tmp/mongodb-27017.sock")
         connection.database_names()
     except:
         # Possibly the OS doesn't support unix domain sockets
         raise SkipTest("No Unix domain sockets")
Exemplo n.º 9
0
class MongoHandler(logging.Handler):
    """ Custom log handler

    Logs all messages to a mongo collection. This  handler is 
    designed to be used with the standard python logging mechanism.
    """

    @classmethod
    def to(cls, db, collection, host='localhost', port=None, level=logging.NOTSET):
        """ Create a handler for a given  """
        return cls(Connection(host, port)[db][collection], level)
        
    def __init__(self, collection, db='mongolog', host='localhost', port=None, level=logging.NOTSET):
        """ Init log handler and store the collection handle """
        logging.Handler.__init__(self, level)
        if (type(collection) == str):
            self.collection = Connection(host, port)[db][collection]
        else:
            self.collection = collection
        self.formatter = MongoFormatter()

    def emit(self,record):
        """ Store the record to the collection. Async insert """
        try:
            self.collection.save(self.format(record))
        except InvalidDocument, e:
            logging.error("Unable to save log record: %s", e.message ,exc_info=True)
Exemplo n.º 10
0
    def WriteTempFile(self, data, hash_name=None):
        
        if self.use_cache == True:
            if hash_name is None:
                hash = md5(self.url )
                hash_name = hash.hexdigest()
                self.last_hash_name = hash_name
                
            self.log.debug('write file to cache: ', hash_name)
            self.log.debug('use mongo: %s' % self.use_mongo)
#            open(self.download_temp+hash_name, 'wb').write(data)
            if self.use_mongo == False: 
                f_name = self.download_temp + hash_name + '.gz'
                f = gzip.open(f_name, 'wb')
                f.write(data)
                f.close()

            if self.use_mongo == True:
                connection = Connection("localhost", 27017)
                db = connection['parser']

                s = StringIO.StringIO()
                f = gzip.GzipFile(fileobj=s, mode='wb')
                f.write(data)
                f.close()
                val = s.getvalue()
                s.close()
                del (s)
                del (f)

                fs = GridFS(db)
                fp = fs.open(hash_name , 'w', self.download_temp.replace('/', '') )
                fp.write(val)
                fp.close()
                connection.disconnect()
Exemplo n.º 11
0
    def test_from_uri(self):
        c = Connection(self.host, self.port)

        self.assertRaises(InvalidURI, Connection.from_uri, "mongodb://localhost/baz")

        self.assertEqual(c, Connection.from_uri("mongodb://%s:%s" %
                                                (self.host, self.port)))

        c.admin.system.users.remove({})
        c.pymongo_test.system.users.remove({})

        c.admin.add_user("admin", "pass")
        c.pymongo_test.add_user("user", "pass")

        self.assertRaises(InvalidURI, Connection.from_uri,
                          "mongodb://*****:*****@%s:%s" % (self.host, self.port))
        self.assertRaises(InvalidURI, Connection.from_uri,
                          "mongodb://*****:*****@%s:%s" % (self.host, self.port))
        self.assertRaises(InvalidURI, Connection.from_uri,
                          "mongodb://*****:*****@%s:%s" % (self.host, self.port))
        Connection.from_uri("mongodb://*****:*****@%s:%s" % (self.host, self.port))

        self.assertRaises(InvalidURI, Connection.from_uri,
                          "mongodb://*****:*****@%s:%s/pymongo_test" %
                          (self.host, self.port))
        self.assertRaises(InvalidURI, Connection.from_uri,
                          "mongodb://*****:*****@%s:%s/pymongo_test" %
                          (self.host, self.port))
        Connection.from_uri("mongodb://*****:*****@%s:%s/pymongo_test" %
                            (self.host, self.port))
    def test_connection(self):
        c = Connection(host, port)
        self.assertTrue(c.auto_start_request)
        self.assertEqual(None, c.max_pool_size)
        self.assertFalse(c.slave_okay)
        self.assertFalse(c.safe)
        self.assertEqual({}, c.get_lasterror_options())

        # Connection's writes are unacknowledged by default
        doc = {"_id": ObjectId()}
        coll = c.pymongo_test.write_concern_test
        coll.drop()
        coll.insert(doc)
        coll.insert(doc)

        c = Connection("mongodb://%s:%s/?safe=true" % (host, port))
        self.assertTrue(c.safe)

        # To preserve legacy Connection's behavior, max_size should be None.
        # Pool should handle this without error.
        self.assertEqual(None, get_pool(c).max_size)
        c.end_request()

        # Connection's network_timeout argument is translated into
        # socketTimeoutMS
        self.assertEqual(123, Connection(
            host, port, network_timeout=123)._MongoClient__net_timeout)

        for network_timeout in 'foo', 0, -1:
            self.assertRaises(
                ConfigurationError,
                Connection, host, port, network_timeout=network_timeout)
Exemplo n.º 13
0
 def test_autoreconnect(self):
     def find_one(conn):
         return conn.test.stuff.find_one()
     # Simulate a temporary connection failure
     c = Connection('foo', _connect=False)
     self.assertRaises(AutoReconnect, find_one, c)
     c._Connection__nodes = set([('localhost', 27017)])
     self.assert_(find_one, c)
    def test_database_names(self):
        connection = Connection(self.host, self.port)

        connection.pymongo_test.test.save({"dummy": u"object"})
        connection.pymongo_test_mike.test.save({"dummy": u"object"})

        dbs = connection.database_names()
        self.assertTrue("pymongo_test" in dbs)
        self.assertTrue("pymongo_test_mike" in dbs)
Exemplo n.º 15
0
 def setUp(self):
     """
     set up DAS core module
     """
     debug = 0
     self.das = DASCore(debug=debug)
     config = deepcopy(das_readconfig())
     dburi = config['mongodb']['dburi']
     connection = Connection(dburi)
     connection.drop_database('das') 
Exemplo n.º 16
0
    def test_end_request(self):
        self.skip()
        connection = Connection([self.left, self.right])
        db = connection.pymongo_test

        for _ in range(100):
            db.test.remove({})
            db.test.insert({})
            self.assertTrue(db.test.find_one())
            connection.end_request()
Exemplo n.º 17
0
 def check_conn (ip = "localhost", port = 27017):
     try:
         port = int(port)
         conn = Connection (host = ip, port = port)
     except Exception as E:
         logging.error (E)
         return False
     else:
         conn.close ()
         return True
Exemplo n.º 18
0
    def GetTempFile(self):
        """
        Metoda pobiera/zapisuje stronke pobierana do cache'u
        """
        data = None
        if self.use_cache == True:
            hash = md5(self.url)
            self.hash_name = hash.hexdigest()
            self.page_from_cache = False
            
#            f_name = self.download_temp + self.hash_name
            f_name = self.download_temp + self.hash_name + '.gz'
            if self._devel == True:
                print 'seek cache: ',f_name, '::', self.url
                
            # czy plik lokalny jest gz
            if os.path.exists(f_name.replace('.gz', '') ):
                data = open(f_name.replace('.gz', ''), 'rb').read()
                f = gzip.open(f_name, 'wb')
                f.write(data)
                f.close()
                os.unlink( f_name.replace('.gz', '') )
                return data
            
            # teraz odczyt pliku gzip
            if self.read_cache == True:
                if self.use_mongo == True:
                    try:
                        connection = Connection("localhost", 27017)
                        db = connection['parser']

                        fs = GridFS(db)
                        fp = fs.open(self.hash_name , 'r', self.download_temp.replace('/', '') )
                        f = gzip.GzipFile(fileobj=fp, mode='rb')
                        data = f.read()
                        f.close()
                        fp.close()
                        del(f)
                        connection.disconnect()

                    except Exception, e:
                        print 'read cahce error: ', e
                        self.page_from_cache = False
                        return None

                elif os.path.exists(f_name):
                        f = gzip.open(f_name, 'rb')
                        data = f.read()
                        f.close()
            else:
                data = ''
                    
            if self._devel == True:
                print '# Found cache: ', self.hash_name
            self.page_from_cache = True
    def test_parse_uri(self):
        self.assertEqual(([("localhost", 27017)], None, None, None),
                         Connection._parse_uri("localhost"))
        self.assertRaises(InvalidURI, Connection._parse_uri, "http://foobar.com")
        self.assertRaises(InvalidURI, Connection._parse_uri, "http://[email protected]")

        self.assertEqual(([("localhost", 27017)], None, None, None),
                         Connection._parse_uri("mongodb://localhost"))
        self.assertEqual(([("localhost", 27017)], None, "fred", "foobar"),
                         Connection._parse_uri("mongodb://*****:*****@localhost"))
        self.assertEqual(([("localhost", 27017)], "baz", "fred", "foobar"),
                         Connection._parse_uri("mongodb://*****:*****@localhost/baz"))
        self.assertEqual(([("example1.com", 27017), ("example2.com", 27017)],
                          None, None, None),
                         Connection._parse_uri("mongodb://example1.com:27017,example2.com:27017"))
        self.assertEqual(([("localhost", 27017),
                           ("localhost", 27018),
                           ("localhost", 27019)], None, None, None),
                         Connection._parse_uri("mongodb://localhost,localhost:27018,localhost:27019"))

        self.assertEqual(([("localhost", 27018)], None, None, None),
                         Connection._parse_uri("localhost:27018"))
        self.assertEqual(([("localhost", 27017)], "foo", None, None),
                         Connection._parse_uri("localhost/foo"))
        self.assertEqual(([("localhost", 27017)], None, None, None),
                         Connection._parse_uri("localhost/"))
Exemplo n.º 20
0
    def test_tz_aware(self):
        aware = Connection(self.host, self.port, tz_aware=True)
        naive = Connection(self.host, self.port)
        aware.pymongo_test.drop_collection("test")

        now = datetime.datetime.utcnow()
        aware.pymongo_test.test.insert({"x": now}, safe=True)

        self.assertEqual(None, naive.pymongo_test.test.find_one()["x"].tzinfo)
        self.assertEqual(utc, aware.pymongo_test.test.find_one()["x"].tzinfo)
        self.assertEqual(
            aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None),
            naive.pymongo_test.test.find_one()["x"])
    def test_disconnect(self):
        c = Connection(self.host, self.port)
        coll = c.pymongo_test.bar

        c.disconnect()
        c.disconnect()

        coll.count()

        c.disconnect()
        c.disconnect()

        coll.count()
Exemplo n.º 22
0
 def __init__(self,
              collection,
              db='mongolog',
              host='localhost',
              port=None,
              level=logging.NOTSET):
     """ Init log handler and store the collection handle """
     logging.Handler.__init__(self, level)
     if (type(collection) == str):
         self.collection = Connection(host, port)[db][collection]
     else:
         self.collection = collection
     self.formatter = MongoFormatter()
Exemplo n.º 23
0
    def test_constants(self):
        Connection.HOST = self.host
        Connection.PORT = self.port
        self.assert_(Connection())

        Connection.HOST = "somedomainthatdoesntexist.org"
        Connection.PORT = 123456789
        self.assertRaises(ConnectionFailure, Connection)
        self.assert_(Connection(self.host, self.port))

        Connection.HOST = self.host
        Connection.PORT = self.port
        self.assert_(Connection())
Exemplo n.º 24
0
 def test_max_pool_size_validation(self):
     self.assertRaises(ValueError, Connection, max_pool_size=-1)
     self.assertRaises(TypeError, Connection, max_pool_size='foo')
     self.assertRaises(TypeError, Connection,
                       'mongodb://localhost/?maxPoolSize=-1')
     self.assertRaises(TypeError, Connection,
                       'mongodb://localhost/?maxPoolSize=foo')
     self.assertRaises(TypeError, Connection,
                       'mongodb://localhost/?maxPoolSize=5.5')
     c = Connection('mongodb://localhost/?maxPoolSize=5')
     self.assertEqual(c.max_pool_size, 5)
     c = Connection(max_pool_size=100)
     self.assertEqual(c.max_pool_size, 100)
Exemplo n.º 25
0
class MongoDBStore(SimpleStore):
    '''MongoDB-based object storage frontend.'''

    init = 'mongodb://'

    def __init__(self, engine, **kw):
        super(MongoDBStore, self).__init__(engine, **kw)
        spliturl = urlsplit(engine)
        _, dbpath, self._colpath = spliturl.path.split('/')
        self._conn = Connection(host=spliturl.hostname, port=spliturl.port)
        self._db = getattr(self._conn, dbpath)
        self._store = getattr(self._db, self._colpath)
        self._store.ensure_index('key', unique=True)

    def __getitem__(self, key):
        try:
            return self.loads(self._store.find_one(dict(key=key))['value'])
        except TypeError:
            raise KeyError(key)

    def __setitem__(self, key, value):
        self._store.save(dict(key=key, value=Binary(self.dumps(value))))

    def __delitem__(self, key):
        self._store.remove(dict(key=key))

    def __len__(self):
        return self._store.count()

    def __iter__(self):
        for key in self._store.find(
                dict(key={'$exists': True}),
                fields=['key'],
        ):
            yield key['key']

    def close(self):
        self._conn.close()

    def clear(self):
        self._store.drop()
        self._store = getattr(self._db, self._colpath)

    def items(self):
        for key in self._store.find({'key': {'$exists': True}}):
            yield (key['key'], key['value'])

    def values(self):
        loads = self.loads
        for value in self._store.distinct('key'):
            yield loads(value['value'])
Exemplo n.º 26
0
    def _open(self):
        """
        See mongodb uri documentation:
        http://www.mongodb.org/display/DOCS/Connections
        """
        client = self.connection.client
        hostname = client.hostname or DEFAULT_HOST
        authdb = dbname = client.virtual_host

        if dbname in ["/", None]:
            dbname = "kombu_default"
            authdb = "admin"

        if not client.userid:
            hostname = hostname.replace('/' + client.virtual_host, '/')
        else:
            hostname = hostname.replace('/' + client.virtual_host,
                                        '/' + authdb)

        mongo_uri = 'mongodb://' + hostname
        # At this point we expect the hostname to be something like
        # (considering replica set form too):
        #
        #   mongodb://[username:password@]host1[:port1][,host2[:port2],
        #   ...[,hostN[:portN]]][/[?options]]
        mongoconn = Connection(host=mongo_uri, ssl=client.ssl)
        database = getattr(mongoconn, dbname)

        version = mongoconn.server_info()['version']
        if tuple(map(int, version.split('.')[:2])) < (1, 3):
            raise NotImplementedError(
                'Kombu requires MongoDB version 1.3+ (server is {0})'.format(
                    version))

        self.db = database
        col = database.messages
        col.ensure_index([('queue', 1), ('_id', 1)], background=True)

        if 'messages.broadcast' not in database.collection_names():
            capsize = (client.transport_options.get('capped_queue_size')
                       or 100000)
            database.create_collection('messages.broadcast',
                                       size=capsize,
                                       capped=True)

        self.bcast = getattr(database, 'messages.broadcast')
        self.bcast.ensure_index([('queue', 1)])

        self.routing = getattr(database, 'messages.routing')
        self.routing.ensure_index([('queue', 1), ('exchange', 1)])
        return database
Exemplo n.º 27
0
class MongoDBStore(SimpleStore):

    '''MongoDB-based object storage frontend.'''

    init = 'mongodb://'

    def __init__(self, engine, **kw):
        super(MongoDBStore, self).__init__(engine, **kw)
        spliturl = urlsplit(engine)
        _, dbpath, self._colpath = spliturl.path.split('/')
        self._conn = Connection(host=spliturl.hostname, port=spliturl.port)
        self._db = getattr(self._conn, dbpath)
        self._store = getattr(self._db, self._colpath)
        self._store.ensure_index('key', unique=True)

    def __getitem__(self, key):
        try:
            return self.loads(self._store.find_one(dict(key=key))['value'], key)
        except TypeError:
            raise KeyError(key)

    def __setitem__(self, key, value):
        self._store.save(dict(key=key, value=Binary(self.dumps(value))))

    def __delitem__(self, key):
        self._store.remove(dict(key=key))

    def __len__(self):
        return self._store.count()

    def __iter__(self):
        for key in self._store.find(
            dict(key={'$exists': True}), fields=['key'],
        ):
            yield key['key']

    def close(self):
        self._conn.close()

    def clear(self):
        self._store.drop()
        self._store = getattr(self._db, self._colpath)

    def items(self):
        for key in self._store.find({'key': {'$exists': True}}):
            yield (key['key'], key['value'])

    def values(self):
        loads = self.loads
        for value in self._store.distinct('key'):
            yield loads(value['value'])
Exemplo n.º 28
0
    def test_connection(self):
        c = Connection(host, port)
        self.assertTrue(c.auto_start_request)
        self.assertEqual(None, c.max_pool_size)
        self.assertFalse(c.slave_okay)
        self.assertFalse(c.safe)
        self.assertEqual({}, c.get_lasterror_options())

        # Connection's writes are unacknowledged by default
        doc = {"_id": ObjectId()}
        coll = c.pymongo_test.write_concern_test
        coll.drop()
        coll.insert(doc)
        coll.insert(doc)

        c = Connection("mongodb://%s:%s/?safe=true" % (host, port))
        self.assertTrue(c.safe)

        # To preserve legacy Connection's behavior, max_size should be None.
        # Pool should handle this without error.
        self.assertEqual(None, c._MongoClient__pool.max_size)
        c.end_request()

        # Connection's network_timeout argument is translated into
        # socketTimeoutMS
        self.assertEqual(123, Connection(
            host, port, network_timeout=123)._MongoClient__net_timeout)

        for network_timeout in 'foo', 0, -1:
            self.assertRaises(
                ConfigurationError,
                Connection, host, port, network_timeout=network_timeout)
Exemplo n.º 29
0
    def init(self):
        logger.info("[Trending broker] I init the %s server connection to %s:%s (%s)" % (self.get_name(), self.backend, self.uri, self.replica_set))
        if self.replica_set:
            self.con = ReplicaSetConnection(self.uri, replicaSet=self.replica_set, safe=True)
        else:
            # Old versions of pymongo do not known about fsync
            if ReplicaSetConnection:
                self.con = Connection(self.uri, safe=True)
            else:
                self.con = Connection(self.uri, safe=True)

        # Open a connection
        self.db = getattr(self.con, self.database)
        self.col = self.db['trending']
Exemplo n.º 30
0
    def _open(self):
        """
        See mongodb uri documentation:
        http://www.mongodb.org/display/DOCS/Connections
        """
        client = self.connection.client
        hostname = client.hostname or DEFAULT_HOST
        authdb = dbname = client.virtual_host

        if dbname in ["/", None]:
            dbname = "kombu_default"
            authdb = "admin"

        if not client.userid:
            hostname = hostname.replace('/' + client.virtual_host, '/')
        else:
            hostname = hostname.replace('/' + client.virtual_host,
                                        '/' + authdb)

        mongo_uri = 'mongodb://' + hostname
        # At this point we expect the hostname to be something like
        # (considering replica set form too):
        #
        #   mongodb://[username:password@]host1[:port1][,host2[:port2],
        #   ...[,hostN[:portN]]][/[?options]]
        mongoconn = Connection(host=mongo_uri, ssl=client.ssl)
        database = getattr(mongoconn, dbname)

        version = mongoconn.server_info()['version']
        if tuple(map(int, version.split('.')[:2])) < (1, 3):
            raise NotImplementedError(
                'Kombu requires MongoDB version 1.3+ (server is {0})'.format(
                    version))

        self.db = database
        col = database.messages
        col.ensure_index([('queue', 1), ('_id', 1)], background=True)

        if 'messages.broadcast' not in database.collection_names():
            capsize = (client.transport_options.get('capped_queue_size')
                       or 100000)
            database.create_collection('messages.broadcast',
                                       size=capsize, capped=True)

        self.bcast = getattr(database, 'messages.broadcast')
        self.bcast.ensure_index([('queue', 1)])

        self.routing = getattr(database, 'messages.routing')
        self.routing.ensure_index([('queue', 1), ('exchange', 1)])
        return database
Exemplo n.º 31
0
    def test_alive(self):
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_random_secondary()
        primary_cx = Connection(primary, use_greenlets=use_greenlets)
        secondary_cx = Connection(secondary, use_greenlets=use_greenlets)
        rsc = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)

        try:
            self.assertTrue(primary_cx.alive())
            self.assertTrue(secondary_cx.alive())
            self.assertTrue(rsc.alive())
    
            ha_tools.kill_primary()
            time.sleep(0.5)

            self.assertFalse(primary_cx.alive())
            self.assertTrue(secondary_cx.alive())
            self.assertFalse(rsc.alive())
            
            ha_tools.kill_members([secondary], 2)
            time.sleep(0.5)

            self.assertFalse(primary_cx.alive())
            self.assertFalse(secondary_cx.alive())
            self.assertFalse(rsc.alive())
        finally:
            rsc.close()
Exemplo n.º 32
0
 def _open(self):
     conninfo = self.connection.client
     mongoconn = Connection(host=conninfo.hostname, port=conninfo.port)
     dbname = conninfo.virtual_host
     version = mongoconn.server_info()["version"]
     if tuple(map(int, version.split("."))) < (1, 3):
         raise NotImplementedError(
             "Kombu requires MongoDB version 1.3+, but connected to %s" % (
                 version, ))
     if not dbname or dbname == "/":
         dbname = "kombu_default"
     database = getattr(mongoconn, dbname)
     col = database.messages
     col.ensure_index([("queue", 1)])
     return col
Exemplo n.º 33
0
def main():
    connection = Connection()
    runner = BenchmarkRunner(10000, connection.server_info()["gitVersion"])

    runner.run_benchmark(Encode(small, "small"))
    runner.run_benchmark(Encode(medium, "medium"))
    runner.run_benchmark(Encode(large, "large"))

    runner.run_benchmark(Decode(BSON.from_dict(small), "small"))
    runner.run_benchmark(Decode(BSON.from_dict(medium), "medium"))
    runner.run_benchmark(Decode(BSON.from_dict(large), "large"))

    runner.run_benchmark(Insert(connection.benchmark, medium, "medium"))

    runner.run_benchmark(FindOne(connection.benchmark.medium_no_index, {"x": 5000}, "medium"))
Exemplo n.º 34
0
 def _open(self):
     conninfo = self.connection.client
     mongoconn = Connection(host=conninfo.hostname, port=conninfo.port)
     dbname = conninfo.virtual_host
     version = mongoconn.server_info()["version"]
     if tuple(map(int, version.split("."))) < (1, 3):
         raise NotImplementedError(
             "Kombu requires MongoDB version 1.3+, but connected to %s" %
             (version, ))
     if not dbname or dbname == "/":
         dbname = "kombu_default"
     database = getattr(mongoconn, dbname)
     col = database.messages
     col.ensure_index([("queue", 1)])
     return col
Exemplo n.º 35
0
    def connect(self):
        """Connects to Mongo database, loads options and set connectors.
        @raise CuckooReportError: if unable to connect.
        """
        host = self.options.get("host", "127.0.0.1")
        port = self.options.get("port", 27017)

        try:
            self.conn = Connection(host, port)
            self.db = self.conn.cuckoo
            self.fs = GridFS(self.db)
        except TypeError:
            raise CuckooReportError("Mongo connection port must be integer")
        except ConnectionFailure:
            raise CuckooReportError("Cannot connect to MongoDB")
    def test_constants(self):
        Connection.HOST = self.host
        Connection.PORT = self.port
        self.assertTrue(Connection())

        Connection.HOST = "somedomainthatdoesntexist.org"
        Connection.PORT = 123456789
        assertRaisesExactly(ConnectionFailure,
                            Connection,
                            connectTimeoutMS=600)
        self.assertTrue(Connection(self.host, self.port))

        Connection.HOST = self.host
        Connection.PORT = self.port
        self.assertTrue(Connection())
Exemplo n.º 37
0
 def setUp(self):
     self.host = os.environ.get("DB_IP", "localhost")
     self.port = int(os.environ.get("DB_PORT", 27017))
     default_connection = Connection(self.host, self.port)
     no_auto_connection = Connection(self.host, self.port,
                                     auto_start_request=False)
     pooled_connection = Connection(self.host, self.port, 
                                    pool_size=10, timeout=-1)
     no_auto_pooled_connection = Connection(self.host, self.port,
                                            pool_size=10, timeout=-1,
                                            auto_start_request=False)
     self.default_db = default_connection["pymongo_test"]
     self.pooled_db = pooled_connection["pymongo_test"]
     self.no_auto_db = no_auto_connection["pymongo_test"]
     self.no_auto_pooled_db = no_auto_pooled_connection["pymongo_test"]
Exemplo n.º 38
0
class TestDictConfig(unittest.TestCase):
    def setUp(self):
        """ Create an empty database that could be used for logging """
        self.db_name = '_mongolog_test_dict'
        self.collection_name = 'log_test'

        self.configDict = {
            'version': 1,
            'handlers': {
                'mongo': {
                    'class': 'mongolog.handlers.MongoHandler',
                    'db': self.db_name,
                    'collection': self.collection_name,
                    'level': 'INFO'
                }
            },
            'root': {
                'handlers': ['mongo'],
                'level': 'INFO'
            }
        }

        self.conn = Connection('localhost')
        self.conn.drop_database(self.db_name)

    def testLoggingDictConfiguration(self):
        dictConfig(self.configDict)
        log = logging.getLogger('dict_example')
        log.debug('testing dictionary config')

        r = self.conn[self.db_name][self.collection_name]

        message = r.find_one({'level': 'debug', 'msg': 'dict_example'})
        self.assertEquals(
            message, None,
            "Logger put debug message in when info level handler requested")

        log.info('dict_example')
        message = r.find_one({'level': 'info', 'msg': 'dict_example'})
        self.assertNotEquals(message, None,
                             "Logger didn't insert message into database")
        self.assertEquals(
            message['msg'], 'dict_example',
            "Logger didn't insert correct message into database")

    def tearDown(self):
        """ Drop used database """
        self.conn.drop_database(self.db_name)
Exemplo n.º 39
0
def generate_doc_content():
    generate_random_content()

    conn = Connection()
    conn.drop_database('peduli')

    print("Adding users.", sep="")
    for i in range(50):
        create_new_user()
        print(".", end="")
    print(".\nAdding contents.", end="")
    for i in range(500):
        create_new_content(i)
        print(".", end="")
        sys.stdout.flush()
    print(".")
Exemplo n.º 40
0
    def init_app(self, app, **kwargs):
        """Initializes `app`, a :class:`~flask.Flask` application, for use with
        the specified configuration variables. Keyword arguments passed to this
        override the configuration options.
        """
        options = {
            'max_pool_size': app.config.get('MONGO_MAX_POOL_SIZE', 10),
            'network_timeout': app.config.get('MONGO_NETWORK_TIMEOUT', None),
            'tz_aware': app.config.get('MONGO_TZ_AWARE', False),
            'slave_okay': app.config.get('MONGO_SLAVE_OKAY', False),
            'safe': app.config.get('MONGO_GETLASTERROR', False),
            'fsync': app.config.get('MONGO_GETLASTERROR_FSYNC', None),
            'j': app.config.get('MONGO_GETLASTERROR_J', None),
            'w': app.config.get('MONGO_GETLASTERROR_W', None),
            'wtimeout': app.config.get('MONGO_GETLASTERROR_W_TIMEOUT', None),
            'replicaset': app.config.get('MONGO_REPLICA_SET', None),
        }.update(kwargs)

        self.app = app
        self.hosts = app.config.get('MONGO_HOSTS', "mongodb://localhost:27017")
        self.connection = BaseConnection(self.hosts, options)

        @app.teardown_request
        def free_sockets(response):
            # release thread connection to pool so socket is reclaimed
            self.connection.end_request()
            return response

        for model, indices in _indices.iteritems():
            for index in indices:
                index.ensure(model.query)
Exemplo n.º 41
0
    def get_connection(self):
        """Connect to the MongoDB server."""
        from pymongo.connection import Connection

        if self._connection is None:
            self._connection = Connection(self.host, self.port)
            return self._connection
Exemplo n.º 42
0
def generate_doc_content():
    generate_random_content()

    conn = Connection()
    conn.drop_database("peduli")

    print("Adding users.", sep="")
    for i in range(50):
        create_new_user()
        print(".", end="")
    print(".\nAdding contents.", end="")
    for i in range(500):
        create_new_content(i)
        print(".", end="")
        sys.stdout.flush()
    print(".")
Exemplo n.º 43
0
def import_cdr_freeswitch_mongodb(shell=False):
    #TODO : dont use the args here
    # Browse settings.MG_IMPORT and for each IP check if the IP exist
    # in our Switch objects. If it does we will connect to that Database
    # and import the data as we do below

    print_shell(shell, "Starting the synchronization...")

    #loop within the Mongo CDR Import List
    for ipaddress in settings.MG_IMPORT:

        data = chk_ipaddress(ipaddress)
        ipaddress = data['ipaddress']
        switch = data['switch']

        #Connect on MongoDB Database
        host = settings.MG_IMPORT[ipaddress]['host']
        port = settings.MG_IMPORT[ipaddress]['port']
        db_name = settings.MG_IMPORT[ipaddress]['db_name']
        try:
            connection = Connection(host, port)
            DBCON = connection[db_name]
        except ConnectionFailure, e:
            sys.stderr.write("Could not connect to MongoDB: %s - %s" % \
                                                            (e, ipaddress))
            sys.exit(1)

        #Connect to Mongo
        importcdr_handler = DBCON[settings.MG_IMPORT[ipaddress]['collection']]

        #Start import for this mongoDB
        func_importcdr_aggregate(shell, importcdr_handler, switch, ipaddress)
Exemplo n.º 44
0
    def init_app(self, app, **kwargs):
        """Initializes `app`, a :class:`~flask.Flask` application, for use with
        the specified configuration variables. Keyword arguments passed to this
        override the configuration options.
        """
        options = {
            'max_pool_size': app.config.get('MONGO_MAX_POOL_SIZE', 10),
            'network_timeout': app.config.get('MONGO_NETWORK_TIMEOUT', None),
            'tz_aware': app.config.get('MONGO_TZ_AWARE', False),
            'slave_okay': app.config.get('MONGO_SLAVE_OKAY', False),
            'safe': app.config.get('MONGO_GETLASTERROR', False),
            'fsync': app.config.get('MONGO_GETLASTERROR_FSYNC', None),
            'j': app.config.get('MONGO_GETLASTERROR_J', None),
            'w': app.config.get('MONGO_GETLASTERROR_W', None),
            'wtimeout': app.config.get('MONGO_GETLASTERROR_W_TIMEOUT', None),
            'replicaset': app.config.get('MONGO_REPLICA_SET', None),
        }.update(kwargs)

        self.app = app
        self.hosts = app.config.get('MONGO_HOSTS', "mongodb://localhost:27017")
        self.connection = BaseConnection(self.hosts, options)

        @app.teardown_request
        def free_sockets(response):
            # release thread connection to pool so socket is reclaimed
            self.connection.end_request()
            return response

        for model, indices in _indices.iteritems():
            for index in indices:
                index.ensure(model.query)
Exemplo n.º 45
0
 def mongo_connection(klass):
     """ returns a pointer to the DB"""
     if not getattr(klass, 'MONGO_CONNECTION', None):
         from pymongo.connection import Connection
         connection = Connection(settings.MONGO_HOST, settings.MONGO_PORT)
         klass.MONGO_CONNECTION = connection
     return klass.MONGO_CONNECTION
Exemplo n.º 46
0
 def connect(self,
             database,
             timezone=None,
             cache_size=0,
             auto_ensure=True,
             *args,
             **kwds):
     ''' `connect` is a thin wrapper around __init__ which creates the 
         database connection that the session will use.
         
         :param database: the database name to use.  Should be an instance of \
                 :class:`basestring`
         :param safe: The value for the "safe" parameter of the Session \ 
             init function
         :param auto_ensure: Whether to implicitly call ensure_indexes on all write \
             operations.
         :param args: arguments for :class:`pymongo.connection.Connection`
         :param kwds: keyword arguments for :class:`pymongo.connection.Connection`
     '''
     safe = kwds.get('safe', False)
     if 'safe' in kwds:
         del kwds['safe']
     if timezone is not None:
         kwds['tz_aware'] = True
     conn = Connection(*args, **kwds)
     db = conn[database]
     return Session(db,
                    timezone=timezone,
                    safe=safe,
                    cache_size=cache_size,
                    auto_ensure=auto_ensure)
Exemplo n.º 47
0
def readSHPPoint(append):
    fileP = 'F:\\project\\Kalium_web\\shapefile\\co99_d00.shp'
    sf = shapefile.Reader(fileP)
    shapeRecs = sf.shapeRecords()

    mongodb_server='127.0.0.1'
    mongodb_port = 27017
    mongodb_collection ='test1'
    mongodb_db = 'gisdb'
    connection = Connection(mongodb_server, mongodb_port)
    print 'Getting database %s' % mongodb_db
    db = connection[mongodb_db]
    print 'Getting the collection %s' % mongodb_collection
    collection = db[mongodb_collection]
    if append == False:
        print 'Removing features from the collection...'
        collection.remove({})
    print 'Starting loading features...'
        
    for shaperec in shapeRecs:
        mongofeat = {}
        #'{x='',y=''}'
        strX = "%.3f" % shaperec.shape.points[0][0]
        strY = "%.3f" % shaperec.shape.points[0][1]
        mongogeom = '{x="'+strX+'",y="'+strY+'"}'
        print mongogeom
        mongofeat['geom'] = mongogeom
        mongofeat['name'] = shaperec.record[1]
        mongofeat['tmp'] = shaperec.record[2]
        collection.insert(mongofeat)
    #create 2d index
    collection.create_index([("geom", pymongo.GEO2D)])
Exemplo n.º 48
0
def main():
    parser = OptionParser(
        usage='mtop.py [options]\nSee also: https://github.com/beaufour/mtop')
    parser.add_option('-s',
                      '--server',
                      dest='server',
                      default='localhost',
                      help='connect to mongo on SERVER',
                      metavar='SERVER')
    parser.add_option('-d',
                      '--delay',
                      dest='delay',
                      type=int,
                      default=1000,
                      help='update every MS',
                      metavar='MS')

    (options, _) = parser.parse_args()

    try:
        if hasattr(
                pymongo, 'version_tuple'
        ) and pymongo.version_tuple[0] >= 2 and pymongo.version_tuple[1] >= 4:
            from pymongo import MongoClient
            from pymongo.read_preferences import ReadPreference
            connection = MongoClient(host=options.server,
                                     read_preference=ReadPreference.SECONDARY)
        else:
            from pymongo.connection import Connection
            connection = Connection(options.server, slave_okay=True)
    except AutoReconnect, ex:
        print 'Connection to %s failed: %s' % (options.server, str(ex))
        return -1
Exemplo n.º 49
0
    def setUp(self):
        """
        set up DAS core module
        """
        debug    = 0
        config   = deepcopy(das_readconfig())
        logger   = PrintManager('TestDASMongocache', verbose=debug)
        config['logger']  = logger
        config['verbose'] = debug
        dburi    = config['mongodb']['dburi']

        connection = Connection(dburi)
        connection.drop_database('das') 
        dasmapping = DASMapping(config)
        config['dasmapping'] = dasmapping
        self.dasmongocache = DASMongocache(config)
Exemplo n.º 50
0
def main():
    parser = SafeConfigParser()
    parser.read('settings.ini')

    connection = Connection(parser.get('mongodb', 'server'))
    db = None
    exec('db = connection.' + parser.get('mongodb', 'db'))

    ai = mc(db, "markov", exp=1.2)

    deck = []

    for i in range(100):
        c = ai.get()
        if (isinstance(c, type(""))):
            deck.append(c)

    deck.sort()

    deck_clean = bag(lambda: 0)

    for c in deck:
        deck_clean[c] += 1

    for c in set(deck):
        print("%2i  %s" % (deck_clean[c], c))
Exemplo n.º 51
0
    def test_repr(self):
        self.skip()
        connection = Connection.paired(self.left, self.right)

        self.assertEqual(
            repr(connection), "Connection(['%s:%s', '%s:%s'])" %
            (self.left[0], self.left[1], self.right[0], self.right[1]))
Exemplo n.º 52
0
    def _create_mongo_connection(self):
        host_uri = \
            'mongodb://%s' % (",".join(["%s:%s" % h for h in self.url_nodelist]))
        log.info("[MongoDBGridFS] Host URI: %s" % host_uri)

        params = {}
        params['slaveok'] = self.url_options.get("slaveok", False)
        if self.url_options.has_key("replicaset"):
            params['replicaset'] = self.url_options["replicaset"] or ""

        conn = Connection(host_uri, **params)
        db = conn[self.url_database]

        if self.url_username:
            log.info("[MongoDBGridFS] Attempting to authenticate %s/%s " %
                     (self.url_username, self.url_password))
            if not db.authenticate(self.url_username, self.url_password):
                raise InvalidCacheBackendError(
                    'Cannot authenticate to MongoDB.')

        collection = db["%s.files" % self.url_collection]
        collection.ensure_index([("namespace", ASCENDING),
                                 ("filename", ASCENDING)],
                                unique=True)
        collection.ensure_index([("namespace", ASCENDING)])

        return (db, GridFS(db, self.url_collection))
Exemplo n.º 53
0
def mongodb2shape(mongodb_server, mongodb_port, mongodb_db, mongodb_collection, output_shape, query_filter):
    """Convert a mongodb collection (all elements must have same attributes) to a shapefile"""
    print ' Converting a mongodb collection to a shapefile '
    connection = Connection(mongodb_server, mongodb_port)
    print 'Getting database MongoDB %s...' % mongodb_db
    db = connection[mongodb_db]
    print 'Getting the collection %s...' % mongodb_collection
    collection = db[mongodb_collection]
    print 'Exporting %s elements in collection to shapefile...' % collection.count()
    drv = ogr.GetDriverByName("ESRI Shapefile")
    ds = drv.CreateDataSource(output_shape)
    lyr = ds.CreateLayer('test', None, ogr.wkbPolygon)
    print 'Shapefile %s created...' % ds.name
    cursor = collection.find(query_filter)
    # define the progressbar
    pbar = ProgressBar(collection.count()).start()
    k=0
    # iterate the features in the collection and copy them to the shapefile
    # for simplicity we export only the geometry to the shapefile
    # if we would like to store also the other fields we should have created a metadata element with fields datatype info
    for element in cursor:
        element_geom = element['geom']
        feat = ogr.Feature(lyr.GetLayerDefn())
        feat.SetGeometry(ogr.CreateGeometryFromWkt(element_geom))
        lyr.CreateFeature(feat)
        feat.Destroy()
        k = k + 1
        pbar.update(k)
    pbar.finish()
    print '%s features loaded in shapefile from MongoDb.' % lyr.GetFeatureCount()
Exemplo n.º 54
0
 def __init__(self):
     try:
         self.connection = Connection()
         #define database name
         self.db = self.connection.todo_db
     except Exception, e:
         raise Exception("Error!! Please check db conenction: %s" % e)
Exemplo n.º 55
0
    def _connect(self):
        self._conn = None
        try:
            if len(self.master_args) == 2:
                self._conn = Connection.paired(
                    (str(self.master_args[0]['host']), int(self.master_args[0]['port'])),
                    (str(self.master_args[1]['host']), int(self.master_args[1]['port'])),
                    pool_size=int(self.master_args[0]['query'].get('pool_size','16')))
            else:
                if self.master_args:
                    try:
                        network_timeout = self.master_args[0]['query'].get('network_timeout')
                        if network_timeout is not None:
                            network_timeout = float(network_timeout)
                        master = Connection(str(self.master_args[0]['host']), int(self.master_args[0]['port']),
                                            pool_size=int(self.master_args[0]['query'].get('pool_size','16')),
                                            network_timeout=network_timeout)
                        ###authenticating  when the db requires it to be done.
                        if self.master_args[0].get('username') and self.master_args[0].get('password'):
                            db = database.Database(master, self.master_args[0]['path'][1:])
                            db.authenticate(self.master_args[0]['username'], 
                                            self.master_args[0]['password'])
                        ###
                    except:
                        if self.slave_args:
                            log.exception('Cannot connect to master: %s will use slave: %s' % (self.master_args, self.slave_args))
                            # and continue... to use the slave only
                            master = None
                        else:
                            raise
                else:
                    log.info('No master connection specified, using slaves only: %s' % self.slave_args)
                    master = None

                if self.slave_args:
                    slave = []
                    for a in self.slave_args:
                        network_timeout = a['query'].get('network_timeout')
                        if network_timeout is not None:
                            network_timeout = float(network_timeout)
                        slave.append(
                            Connection(str(a['host']), int(a['port']),
                                       pool_size=int(a['query'].get('pool_size','16')),
                                       slave_okay=True,
                                       network_timeout=network_timeout,
                                      )
                        )
                    if master:
                        self._conn = MasterSlaveConnection(master, slave)
                    else:
                        self._conn = slave[0]

                else:
                    self._conn = master
        except:
            log.exception('Cannot connect to %s %s' % (self.master_args, self.slave_args))
        else:
            #log.info('Connected to %s %s' % (self.master_args, self.slave_args))
            pass
        return self._conn
Exemplo n.º 56
0
def _connect_mongodb(settings, mongodb):
    # type: (Settings, MongoDB) -> None
    if Connection is None:
        raise Exception(
            'Could not initialize MongoDB (Python-Modules are missing)')
    mongodb.connection = Connection(*_mongodb_local_connection_opts(settings))
    mongodb.db = mongodb.connection.__getitem__(os.environ['OMD_SITE'])
Exemplo n.º 57
0
    def test_mongo_replica_set_client(self):
        c = Connection(pair)
        ismaster = c.admin.command('ismaster')
        if 'setName' in ismaster:
            setname = str(ismaster.get('setName'))
        else:
            raise SkipTest("Not connected to a replica set.")
        m = MongoReplicaSetClient(pair, replicaSet=setname, w=0)
        coll = m.pymongo_test.write_concern_test
        coll.drop()
        doc = {"_id": ObjectId()}
        coll.insert(doc)
        self.assertTrue(coll.insert(doc, safe=False))
        self.assertTrue(coll.insert(doc, w=0))
        self.assertTrue(coll.insert(doc))
        self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
        self.assertRaises(OperationFailure, coll.insert, doc, w=1)

        m = MongoReplicaSetClient(pair, replicaSet=setname)
        coll = m.pymongo_test.write_concern_test
        self.assertTrue(coll.insert(doc, safe=False))
        self.assertTrue(coll.insert(doc, w=0))
        self.assertRaises(OperationFailure, coll.insert, doc)
        self.assertRaises(OperationFailure, coll.insert, doc, safe=True)
        self.assertRaises(OperationFailure, coll.insert, doc, w=1)

        m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s" % (pair, setname))
        self.assertTrue(m.safe)
        coll = m.pymongo_test.write_concern_test
        self.assertRaises(OperationFailure, coll.insert, doc)
        m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s;w=0" % (pair, setname))
        self.assertFalse(m.safe)
        coll = m.pymongo_test.write_concern_test
        self.assertTrue(coll.insert(doc))
Exemplo n.º 58
0
 def test_postProcessing():
     mongodb_connection = Connection('localhost', 27017)
     crowds_db = mongodb_connection.set_ats_new
     crowds_collection = crowds_db.Crowd
     i = 1
     for crowd in crowds_collection.find():
         for k in ['start', 'end']:
             if crowd[k]:
                 if type(crowd[k]) == type(1.0):
                     crowd[k] = dt.utcfromtimestamp(crowd[k])
                 else:
                     print 'nop', crowd['_id']
             else:
                 crowd[k] = dt(2011, 6, 1)
         end = crowd['end']
         for user in crowd['users']:
             user['id'] = int(user['id'])
             for h in user['history']:
                 for x in xrange(2):
                     if h[x]:
                         if type(h[x]) == type(1.0):
                             h[x] = dt.utcfromtimestamp(h[x])
                     else:
                         h[x] = end
         print i, crowd['_id']
         i += 1
         crowds_collection.save(crowd)
Exemplo n.º 59
0
    def test_iteration(self):
        connection = Connection(self.host, self.port)

        def iterate():
            [a for a in connection]

        self.assertRaises(TypeError, iterate)
Exemplo n.º 60
0
def mongo_upgrade():
    """Migrate mongodb schema and data."""
    # Read reporting.conf to fetch mongo configuration.
    config = Config(os.path.join("..", "..", "conf", "reporting.conf"))
    # Run migration only if mongo is enabled as reporting module.
    if config.mongodb.enabled:
        host = config.mongodb.get("host", "127.0.0.1")
        port = config.mongodb.get("port", 27017)
        print "Mongo reporting is enabled, strarting mongo data migration."

        # Connect.
        try:
            conn = Connection(host, port)
            db = conn.cuckoo
        except TypeError:
            print "Mongo connection port must be integer"
            sys.exit()
        except ConnectionFailure:
            print "Cannot connect to MongoDB"
            sys.exit()

        # Check for schema version and create it.
        if "cuckoo_schema" in db.collection_names():
            print "Mongo schema version not expected"
            sys.exit()
        else:
            db.cuckoo_schema.save({"version": mongo_revision})

    else:
        print "Mongo reporting module not enabled, skipping mongo migration."