def _open(self): conninfo = self.connection.client mongoconn = Connection(host=conninfo.hostname, port=conninfo.port) dbname = conninfo.virtual_host version = mongoconn.server_info()["version"] if tuple(map(int, version.split(".")[:2])) < (1, 3): raise NotImplementedError( "Kombu requires MongoDB version 1.3+, but connected to %s" % ( version, )) if not dbname or dbname == "/": dbname = "kombu_default" database = getattr(mongoconn, dbname) if conninfo.userid: database.authenticate(conninfo.userid, conninfo.password) self.db = database col = database.messages col.ensure_index([("queue", 1)]) if "messages.broadcast" not in database.collection_names(): capsize = conninfo.transport_options.get( "capped_queue_size") or 100000 database.create_collection("messages.broadcast", size=capsize, capped=True) self.bcast = getattr(database, "messages.broadcast") self.bcast.ensure_index([("queue", 1)]) self.routing = getattr(database, "messages.routing") self.routing.ensure_index([("queue", 1), ("exchange", 1)]) return database
def test_document_class(self): c = Connection(self.host, self.port) db = c.pymongo_test db.test.insert({"x": 1}) self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = Connection(self.host, self.port, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON))
class TestConfig(unittest.TestCase): def setUp(self): filename = dirname(__file__) + '/logging-test.config' fileConfig(filename) """ Create an empty database that could be used for logging """ self.db_name = '_mongolog_test' self.collection_name = 'log_test' self.conn = Connection('localhost') self.conn.drop_database(self.db_name) def tearDown(self): """ Drop used database """ self.conn.drop_database(self.db_name) def testLoggingFileConfiguration(self): log = logging.getLogger('example') log.debug('test') r = self.conn[self.db_name][self.collection_name] message = r.find_one({'level': 'debug', 'msg': 'test'}) self.assertEquals(message['msg'], 'test')
def __init__(self): self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, 'config', 'processing.conf')) self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled") self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled") self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled") if self.__mongodbEnabled: #Anbindung an Datenbank MongoDB Collection Ragpicker herstellen try: mongodbHost = self.__cfgReporting.getOption("mongodb", "host") mongodbPort = self.__cfgReporting.getOption("mongodb", "port") self.__mongodbConnection = Connection(mongodbHost, mongodbPort) self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue except TypeError: raise Exception("MongoDB connection port in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (ragpicker)") if self.__codedbEnabled: #Anbindung an Datenbank MongoDB Collection CodeDB herstellen try: codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host") codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port") self.__codedbConnection = Connection(codedbHost, codedbPort) self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB except TypeError: raise Exception("MongoDB connection port for CodeDB in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (codeDB)")
def test_ipv6(self): self.assertRaises(InvalidURI, _parse_uri, "::1", 27017) self.assertRaises(InvalidURI, _parse_uri, "[::1", 27017) self.assertRaises(InvalidURI, _parse_uri, "::1]:27017") self.assertRaises(InvalidURI, _parse_uri, "mongodb://::1", 27017) self.assert_(_parse_uri, "mongodb://[::1]:27017/?slaveOk=true") self.assert_(_parse_uri, "[::1]:27017,[2001:0db8:85a3:0000:0000:8a2e:0370:7334]" ":27018,192.168.0.212:27019,localhost:27020") self.assert_(_parse_uri, "mongodb://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]" ":27017/?slaveOk=true") try: connection = Connection("[::1]") except: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest() # Try a few simple things connection = Connection("mongodb://[::1]:27017") connection = Connection("mongodb://[::1]:27017/?slaveOk=true") connection = Connection("[::1]:27017,localhost:27017") connection = Connection("localhost:27017,[::1]:27017") connection.pymongo_test.test.save({"dummy": u"object"}) connection.pymongo_test_bernie.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_bernie" in dbs)
class plantmongo: def __init__(self, addr, user="******", password="******"): self.addr = addr self.user = user self.password = password self.conn = Connection(addr) self.db = conn.plant if not db.authenticate("plant", "plant"): print "unable to authenticate to mongodb" self.connected = False return None self.coll = self.db.test def publish(self, xbeeobj): post = { "time": time.time(), "temp": xbeeobj.temp, "moisture": xbeeobj.moisture, "dewpoint": xbeeobj.dewpoint, "pressure": xbeeobj.pressure, "light": xbeeobj.light, "batt": xbeeobj.batt, "rssi": xbeeobj.rssi, } self.coll.save(post) def close(self): self.conn.disconnect()
def test_connection(self): c = Connection(host, port) self.assertTrue(c.auto_start_request) self.assertFalse(c.slave_okay) self.assertFalse(c.safe) self.assertEqual({}, c.get_lasterror_options()) # Connection's writes are unacknowledged by default doc = {"_id": ObjectId()} coll = c.pymongo_test.write_concern_test coll.really_drop() coll.insert(doc) coll.insert(doc) c = Connection("mongodb://%s:%s/?safe=true" % (host, port)) self.assertTrue(c.safe) # Connection's network_timeout argument is translated into # socketTimeoutMS self.assertEqual(123, Connection( host, port, network_timeout=123)._MongoClient__net_timeout) for network_timeout in 'foo', 0, -1: self.assertRaises( ConfigurationError, Connection, host, port, network_timeout=network_timeout)
def test_unix_domain_socket(self): try: connection = Connection("/tmp/mongodb-27017.sock") connection.database_names() except: # Possibly the OS doesn't support unix domain sockets raise SkipTest("No Unix domain sockets")
class MongoHandler(logging.Handler): """ Custom log handler Logs all messages to a mongo collection. This handler is designed to be used with the standard python logging mechanism. """ @classmethod def to(cls, db, collection, host='localhost', port=None, level=logging.NOTSET): """ Create a handler for a given """ return cls(Connection(host, port)[db][collection], level) def __init__(self, collection, db='mongolog', host='localhost', port=None, level=logging.NOTSET): """ Init log handler and store the collection handle """ logging.Handler.__init__(self, level) if (type(collection) == str): self.collection = Connection(host, port)[db][collection] else: self.collection = collection self.formatter = MongoFormatter() def emit(self,record): """ Store the record to the collection. Async insert """ try: self.collection.save(self.format(record)) except InvalidDocument, e: logging.error("Unable to save log record: %s", e.message ,exc_info=True)
def WriteTempFile(self, data, hash_name=None): if self.use_cache == True: if hash_name is None: hash = md5(self.url ) hash_name = hash.hexdigest() self.last_hash_name = hash_name self.log.debug('write file to cache: ', hash_name) self.log.debug('use mongo: %s' % self.use_mongo) # open(self.download_temp+hash_name, 'wb').write(data) if self.use_mongo == False: f_name = self.download_temp + hash_name + '.gz' f = gzip.open(f_name, 'wb') f.write(data) f.close() if self.use_mongo == True: connection = Connection("localhost", 27017) db = connection['parser'] s = StringIO.StringIO() f = gzip.GzipFile(fileobj=s, mode='wb') f.write(data) f.close() val = s.getvalue() s.close() del (s) del (f) fs = GridFS(db) fp = fs.open(hash_name , 'w', self.download_temp.replace('/', '') ) fp.write(val) fp.close() connection.disconnect()
def test_from_uri(self): c = Connection(self.host, self.port) self.assertRaises(InvalidURI, Connection.from_uri, "mongodb://localhost/baz") self.assertEqual(c, Connection.from_uri("mongodb://%s:%s" % (self.host, self.port))) c.admin.system.users.remove({}) c.pymongo_test.system.users.remove({}) c.admin.add_user("admin", "pass") c.pymongo_test.add_user("user", "pass") self.assertRaises(InvalidURI, Connection.from_uri, "mongodb://*****:*****@%s:%s" % (self.host, self.port)) self.assertRaises(InvalidURI, Connection.from_uri, "mongodb://*****:*****@%s:%s" % (self.host, self.port)) self.assertRaises(InvalidURI, Connection.from_uri, "mongodb://*****:*****@%s:%s" % (self.host, self.port)) Connection.from_uri("mongodb://*****:*****@%s:%s" % (self.host, self.port)) self.assertRaises(InvalidURI, Connection.from_uri, "mongodb://*****:*****@%s:%s/pymongo_test" % (self.host, self.port)) self.assertRaises(InvalidURI, Connection.from_uri, "mongodb://*****:*****@%s:%s/pymongo_test" % (self.host, self.port)) Connection.from_uri("mongodb://*****:*****@%s:%s/pymongo_test" % (self.host, self.port))
def test_connection(self): c = Connection(host, port) self.assertTrue(c.auto_start_request) self.assertEqual(None, c.max_pool_size) self.assertFalse(c.slave_okay) self.assertFalse(c.safe) self.assertEqual({}, c.get_lasterror_options()) # Connection's writes are unacknowledged by default doc = {"_id": ObjectId()} coll = c.pymongo_test.write_concern_test coll.drop() coll.insert(doc) coll.insert(doc) c = Connection("mongodb://%s:%s/?safe=true" % (host, port)) self.assertTrue(c.safe) # To preserve legacy Connection's behavior, max_size should be None. # Pool should handle this without error. self.assertEqual(None, get_pool(c).max_size) c.end_request() # Connection's network_timeout argument is translated into # socketTimeoutMS self.assertEqual(123, Connection( host, port, network_timeout=123)._MongoClient__net_timeout) for network_timeout in 'foo', 0, -1: self.assertRaises( ConfigurationError, Connection, host, port, network_timeout=network_timeout)
def test_autoreconnect(self): def find_one(conn): return conn.test.stuff.find_one() # Simulate a temporary connection failure c = Connection('foo', _connect=False) self.assertRaises(AutoReconnect, find_one, c) c._Connection__nodes = set([('localhost', 27017)]) self.assert_(find_one, c)
def test_database_names(self): connection = Connection(self.host, self.port) connection.pymongo_test.test.save({"dummy": u"object"}) connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = connection.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_mike" in dbs)
def setUp(self): """ set up DAS core module """ debug = 0 self.das = DASCore(debug=debug) config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] connection = Connection(dburi) connection.drop_database('das')
def test_end_request(self): self.skip() connection = Connection([self.left, self.right]) db = connection.pymongo_test for _ in range(100): db.test.remove({}) db.test.insert({}) self.assertTrue(db.test.find_one()) connection.end_request()
def check_conn (ip = "localhost", port = 27017): try: port = int(port) conn = Connection (host = ip, port = port) except Exception as E: logging.error (E) return False else: conn.close () return True
def GetTempFile(self): """ Metoda pobiera/zapisuje stronke pobierana do cache'u """ data = None if self.use_cache == True: hash = md5(self.url) self.hash_name = hash.hexdigest() self.page_from_cache = False # f_name = self.download_temp + self.hash_name f_name = self.download_temp + self.hash_name + '.gz' if self._devel == True: print 'seek cache: ',f_name, '::', self.url # czy plik lokalny jest gz if os.path.exists(f_name.replace('.gz', '') ): data = open(f_name.replace('.gz', ''), 'rb').read() f = gzip.open(f_name, 'wb') f.write(data) f.close() os.unlink( f_name.replace('.gz', '') ) return data # teraz odczyt pliku gzip if self.read_cache == True: if self.use_mongo == True: try: connection = Connection("localhost", 27017) db = connection['parser'] fs = GridFS(db) fp = fs.open(self.hash_name , 'r', self.download_temp.replace('/', '') ) f = gzip.GzipFile(fileobj=fp, mode='rb') data = f.read() f.close() fp.close() del(f) connection.disconnect() except Exception, e: print 'read cahce error: ', e self.page_from_cache = False return None elif os.path.exists(f_name): f = gzip.open(f_name, 'rb') data = f.read() f.close() else: data = '' if self._devel == True: print '# Found cache: ', self.hash_name self.page_from_cache = True
def test_parse_uri(self): self.assertEqual(([("localhost", 27017)], None, None, None), Connection._parse_uri("localhost")) self.assertRaises(InvalidURI, Connection._parse_uri, "http://foobar.com") self.assertRaises(InvalidURI, Connection._parse_uri, "http://[email protected]") self.assertEqual(([("localhost", 27017)], None, None, None), Connection._parse_uri("mongodb://localhost")) self.assertEqual(([("localhost", 27017)], None, "fred", "foobar"), Connection._parse_uri("mongodb://*****:*****@localhost")) self.assertEqual(([("localhost", 27017)], "baz", "fred", "foobar"), Connection._parse_uri("mongodb://*****:*****@localhost/baz")) self.assertEqual(([("example1.com", 27017), ("example2.com", 27017)], None, None, None), Connection._parse_uri("mongodb://example1.com:27017,example2.com:27017")) self.assertEqual(([("localhost", 27017), ("localhost", 27018), ("localhost", 27019)], None, None, None), Connection._parse_uri("mongodb://localhost,localhost:27018,localhost:27019")) self.assertEqual(([("localhost", 27018)], None, None, None), Connection._parse_uri("localhost:27018")) self.assertEqual(([("localhost", 27017)], "foo", None, None), Connection._parse_uri("localhost/foo")) self.assertEqual(([("localhost", 27017)], None, None, None), Connection._parse_uri("localhost/"))
def test_tz_aware(self): aware = Connection(self.host, self.port, tz_aware=True) naive = Connection(self.host, self.port) aware.pymongo_test.drop_collection("test") now = datetime.datetime.utcnow() aware.pymongo_test.test.insert({"x": now}, safe=True) self.assertEqual(None, naive.pymongo_test.test.find_one()["x"].tzinfo) self.assertEqual(utc, aware.pymongo_test.test.find_one()["x"].tzinfo) self.assertEqual( aware.pymongo_test.test.find_one()["x"].replace(tzinfo=None), naive.pymongo_test.test.find_one()["x"])
def test_disconnect(self): c = Connection(self.host, self.port) coll = c.pymongo_test.bar c.disconnect() c.disconnect() coll.count() c.disconnect() c.disconnect() coll.count()
def __init__(self, collection, db='mongolog', host='localhost', port=None, level=logging.NOTSET): """ Init log handler and store the collection handle """ logging.Handler.__init__(self, level) if (type(collection) == str): self.collection = Connection(host, port)[db][collection] else: self.collection = collection self.formatter = MongoFormatter()
def test_constants(self): Connection.HOST = self.host Connection.PORT = self.port self.assert_(Connection()) Connection.HOST = "somedomainthatdoesntexist.org" Connection.PORT = 123456789 self.assertRaises(ConnectionFailure, Connection) self.assert_(Connection(self.host, self.port)) Connection.HOST = self.host Connection.PORT = self.port self.assert_(Connection())
def test_max_pool_size_validation(self): self.assertRaises(ValueError, Connection, max_pool_size=-1) self.assertRaises(TypeError, Connection, max_pool_size='foo') self.assertRaises(TypeError, Connection, 'mongodb://localhost/?maxPoolSize=-1') self.assertRaises(TypeError, Connection, 'mongodb://localhost/?maxPoolSize=foo') self.assertRaises(TypeError, Connection, 'mongodb://localhost/?maxPoolSize=5.5') c = Connection('mongodb://localhost/?maxPoolSize=5') self.assertEqual(c.max_pool_size, 5) c = Connection(max_pool_size=100) self.assertEqual(c.max_pool_size, 100)
class MongoDBStore(SimpleStore): '''MongoDB-based object storage frontend.''' init = 'mongodb://' def __init__(self, engine, **kw): super(MongoDBStore, self).__init__(engine, **kw) spliturl = urlsplit(engine) _, dbpath, self._colpath = spliturl.path.split('/') self._conn = Connection(host=spliturl.hostname, port=spliturl.port) self._db = getattr(self._conn, dbpath) self._store = getattr(self._db, self._colpath) self._store.ensure_index('key', unique=True) def __getitem__(self, key): try: return self.loads(self._store.find_one(dict(key=key))['value']) except TypeError: raise KeyError(key) def __setitem__(self, key, value): self._store.save(dict(key=key, value=Binary(self.dumps(value)))) def __delitem__(self, key): self._store.remove(dict(key=key)) def __len__(self): return self._store.count() def __iter__(self): for key in self._store.find( dict(key={'$exists': True}), fields=['key'], ): yield key['key'] def close(self): self._conn.close() def clear(self): self._store.drop() self._store = getattr(self._db, self._colpath) def items(self): for key in self._store.find({'key': {'$exists': True}}): yield (key['key'], key['value']) def values(self): loads = self.loads for value in self._store.distinct('key'): yield loads(value['value'])
def _open(self): """ See mongodb uri documentation: http://www.mongodb.org/display/DOCS/Connections """ client = self.connection.client hostname = client.hostname or DEFAULT_HOST authdb = dbname = client.virtual_host if dbname in ["/", None]: dbname = "kombu_default" authdb = "admin" if not client.userid: hostname = hostname.replace('/' + client.virtual_host, '/') else: hostname = hostname.replace('/' + client.virtual_host, '/' + authdb) mongo_uri = 'mongodb://' + hostname # At this point we expect the hostname to be something like # (considering replica set form too): # # mongodb://[username:password@]host1[:port1][,host2[:port2], # ...[,hostN[:portN]]][/[?options]] mongoconn = Connection(host=mongo_uri, ssl=client.ssl) database = getattr(mongoconn, dbname) version = mongoconn.server_info()['version'] if tuple(map(int, version.split('.')[:2])) < (1, 3): raise NotImplementedError( 'Kombu requires MongoDB version 1.3+ (server is {0})'.format( version)) self.db = database col = database.messages col.ensure_index([('queue', 1), ('_id', 1)], background=True) if 'messages.broadcast' not in database.collection_names(): capsize = (client.transport_options.get('capped_queue_size') or 100000) database.create_collection('messages.broadcast', size=capsize, capped=True) self.bcast = getattr(database, 'messages.broadcast') self.bcast.ensure_index([('queue', 1)]) self.routing = getattr(database, 'messages.routing') self.routing.ensure_index([('queue', 1), ('exchange', 1)]) return database
class MongoDBStore(SimpleStore): '''MongoDB-based object storage frontend.''' init = 'mongodb://' def __init__(self, engine, **kw): super(MongoDBStore, self).__init__(engine, **kw) spliturl = urlsplit(engine) _, dbpath, self._colpath = spliturl.path.split('/') self._conn = Connection(host=spliturl.hostname, port=spliturl.port) self._db = getattr(self._conn, dbpath) self._store = getattr(self._db, self._colpath) self._store.ensure_index('key', unique=True) def __getitem__(self, key): try: return self.loads(self._store.find_one(dict(key=key))['value'], key) except TypeError: raise KeyError(key) def __setitem__(self, key, value): self._store.save(dict(key=key, value=Binary(self.dumps(value)))) def __delitem__(self, key): self._store.remove(dict(key=key)) def __len__(self): return self._store.count() def __iter__(self): for key in self._store.find( dict(key={'$exists': True}), fields=['key'], ): yield key['key'] def close(self): self._conn.close() def clear(self): self._store.drop() self._store = getattr(self._db, self._colpath) def items(self): for key in self._store.find({'key': {'$exists': True}}): yield (key['key'], key['value']) def values(self): loads = self.loads for value in self._store.distinct('key'): yield loads(value['value'])
def test_connection(self): c = Connection(host, port) self.assertTrue(c.auto_start_request) self.assertEqual(None, c.max_pool_size) self.assertFalse(c.slave_okay) self.assertFalse(c.safe) self.assertEqual({}, c.get_lasterror_options()) # Connection's writes are unacknowledged by default doc = {"_id": ObjectId()} coll = c.pymongo_test.write_concern_test coll.drop() coll.insert(doc) coll.insert(doc) c = Connection("mongodb://%s:%s/?safe=true" % (host, port)) self.assertTrue(c.safe) # To preserve legacy Connection's behavior, max_size should be None. # Pool should handle this without error. self.assertEqual(None, c._MongoClient__pool.max_size) c.end_request() # Connection's network_timeout argument is translated into # socketTimeoutMS self.assertEqual(123, Connection( host, port, network_timeout=123)._MongoClient__net_timeout) for network_timeout in 'foo', 0, -1: self.assertRaises( ConfigurationError, Connection, host, port, network_timeout=network_timeout)
def init(self): logger.info("[Trending broker] I init the %s server connection to %s:%s (%s)" % (self.get_name(), self.backend, self.uri, self.replica_set)) if self.replica_set: self.con = ReplicaSetConnection(self.uri, replicaSet=self.replica_set, safe=True) else: # Old versions of pymongo do not known about fsync if ReplicaSetConnection: self.con = Connection(self.uri, safe=True) else: self.con = Connection(self.uri, safe=True) # Open a connection self.db = getattr(self.con, self.database) self.col = self.db['trending']
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = Connection(primary, use_greenlets=use_greenlets) secondary_cx = Connection(secondary, use_greenlets=use_greenlets) rsc = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) try: self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) finally: rsc.close()
def _open(self): conninfo = self.connection.client mongoconn = Connection(host=conninfo.hostname, port=conninfo.port) dbname = conninfo.virtual_host version = mongoconn.server_info()["version"] if tuple(map(int, version.split("."))) < (1, 3): raise NotImplementedError( "Kombu requires MongoDB version 1.3+, but connected to %s" % ( version, )) if not dbname or dbname == "/": dbname = "kombu_default" database = getattr(mongoconn, dbname) col = database.messages col.ensure_index([("queue", 1)]) return col
def main(): connection = Connection() runner = BenchmarkRunner(10000, connection.server_info()["gitVersion"]) runner.run_benchmark(Encode(small, "small")) runner.run_benchmark(Encode(medium, "medium")) runner.run_benchmark(Encode(large, "large")) runner.run_benchmark(Decode(BSON.from_dict(small), "small")) runner.run_benchmark(Decode(BSON.from_dict(medium), "medium")) runner.run_benchmark(Decode(BSON.from_dict(large), "large")) runner.run_benchmark(Insert(connection.benchmark, medium, "medium")) runner.run_benchmark(FindOne(connection.benchmark.medium_no_index, {"x": 5000}, "medium"))
def _open(self): conninfo = self.connection.client mongoconn = Connection(host=conninfo.hostname, port=conninfo.port) dbname = conninfo.virtual_host version = mongoconn.server_info()["version"] if tuple(map(int, version.split("."))) < (1, 3): raise NotImplementedError( "Kombu requires MongoDB version 1.3+, but connected to %s" % (version, )) if not dbname or dbname == "/": dbname = "kombu_default" database = getattr(mongoconn, dbname) col = database.messages col.ensure_index([("queue", 1)]) return col
def connect(self): """Connects to Mongo database, loads options and set connectors. @raise CuckooReportError: if unable to connect. """ host = self.options.get("host", "127.0.0.1") port = self.options.get("port", 27017) try: self.conn = Connection(host, port) self.db = self.conn.cuckoo self.fs = GridFS(self.db) except TypeError: raise CuckooReportError("Mongo connection port must be integer") except ConnectionFailure: raise CuckooReportError("Cannot connect to MongoDB")
def test_constants(self): Connection.HOST = self.host Connection.PORT = self.port self.assertTrue(Connection()) Connection.HOST = "somedomainthatdoesntexist.org" Connection.PORT = 123456789 assertRaisesExactly(ConnectionFailure, Connection, connectTimeoutMS=600) self.assertTrue(Connection(self.host, self.port)) Connection.HOST = self.host Connection.PORT = self.port self.assertTrue(Connection())
def setUp(self): self.host = os.environ.get("DB_IP", "localhost") self.port = int(os.environ.get("DB_PORT", 27017)) default_connection = Connection(self.host, self.port) no_auto_connection = Connection(self.host, self.port, auto_start_request=False) pooled_connection = Connection(self.host, self.port, pool_size=10, timeout=-1) no_auto_pooled_connection = Connection(self.host, self.port, pool_size=10, timeout=-1, auto_start_request=False) self.default_db = default_connection["pymongo_test"] self.pooled_db = pooled_connection["pymongo_test"] self.no_auto_db = no_auto_connection["pymongo_test"] self.no_auto_pooled_db = no_auto_pooled_connection["pymongo_test"]
class TestDictConfig(unittest.TestCase): def setUp(self): """ Create an empty database that could be used for logging """ self.db_name = '_mongolog_test_dict' self.collection_name = 'log_test' self.configDict = { 'version': 1, 'handlers': { 'mongo': { 'class': 'mongolog.handlers.MongoHandler', 'db': self.db_name, 'collection': self.collection_name, 'level': 'INFO' } }, 'root': { 'handlers': ['mongo'], 'level': 'INFO' } } self.conn = Connection('localhost') self.conn.drop_database(self.db_name) def testLoggingDictConfiguration(self): dictConfig(self.configDict) log = logging.getLogger('dict_example') log.debug('testing dictionary config') r = self.conn[self.db_name][self.collection_name] message = r.find_one({'level': 'debug', 'msg': 'dict_example'}) self.assertEquals( message, None, "Logger put debug message in when info level handler requested") log.info('dict_example') message = r.find_one({'level': 'info', 'msg': 'dict_example'}) self.assertNotEquals(message, None, "Logger didn't insert message into database") self.assertEquals( message['msg'], 'dict_example', "Logger didn't insert correct message into database") def tearDown(self): """ Drop used database """ self.conn.drop_database(self.db_name)
def generate_doc_content(): generate_random_content() conn = Connection() conn.drop_database('peduli') print("Adding users.", sep="") for i in range(50): create_new_user() print(".", end="") print(".\nAdding contents.", end="") for i in range(500): create_new_content(i) print(".", end="") sys.stdout.flush() print(".")
def init_app(self, app, **kwargs): """Initializes `app`, a :class:`~flask.Flask` application, for use with the specified configuration variables. Keyword arguments passed to this override the configuration options. """ options = { 'max_pool_size': app.config.get('MONGO_MAX_POOL_SIZE', 10), 'network_timeout': app.config.get('MONGO_NETWORK_TIMEOUT', None), 'tz_aware': app.config.get('MONGO_TZ_AWARE', False), 'slave_okay': app.config.get('MONGO_SLAVE_OKAY', False), 'safe': app.config.get('MONGO_GETLASTERROR', False), 'fsync': app.config.get('MONGO_GETLASTERROR_FSYNC', None), 'j': app.config.get('MONGO_GETLASTERROR_J', None), 'w': app.config.get('MONGO_GETLASTERROR_W', None), 'wtimeout': app.config.get('MONGO_GETLASTERROR_W_TIMEOUT', None), 'replicaset': app.config.get('MONGO_REPLICA_SET', None), }.update(kwargs) self.app = app self.hosts = app.config.get('MONGO_HOSTS', "mongodb://localhost:27017") self.connection = BaseConnection(self.hosts, options) @app.teardown_request def free_sockets(response): # release thread connection to pool so socket is reclaimed self.connection.end_request() return response for model, indices in _indices.iteritems(): for index in indices: index.ensure(model.query)
def get_connection(self): """Connect to the MongoDB server.""" from pymongo.connection import Connection if self._connection is None: self._connection = Connection(self.host, self.port) return self._connection
def generate_doc_content(): generate_random_content() conn = Connection() conn.drop_database("peduli") print("Adding users.", sep="") for i in range(50): create_new_user() print(".", end="") print(".\nAdding contents.", end="") for i in range(500): create_new_content(i) print(".", end="") sys.stdout.flush() print(".")
def import_cdr_freeswitch_mongodb(shell=False): #TODO : dont use the args here # Browse settings.MG_IMPORT and for each IP check if the IP exist # in our Switch objects. If it does we will connect to that Database # and import the data as we do below print_shell(shell, "Starting the synchronization...") #loop within the Mongo CDR Import List for ipaddress in settings.MG_IMPORT: data = chk_ipaddress(ipaddress) ipaddress = data['ipaddress'] switch = data['switch'] #Connect on MongoDB Database host = settings.MG_IMPORT[ipaddress]['host'] port = settings.MG_IMPORT[ipaddress]['port'] db_name = settings.MG_IMPORT[ipaddress]['db_name'] try: connection = Connection(host, port) DBCON = connection[db_name] except ConnectionFailure, e: sys.stderr.write("Could not connect to MongoDB: %s - %s" % \ (e, ipaddress)) sys.exit(1) #Connect to Mongo importcdr_handler = DBCON[settings.MG_IMPORT[ipaddress]['collection']] #Start import for this mongoDB func_importcdr_aggregate(shell, importcdr_handler, switch, ipaddress)
def mongo_connection(klass): """ returns a pointer to the DB""" if not getattr(klass, 'MONGO_CONNECTION', None): from pymongo.connection import Connection connection = Connection(settings.MONGO_HOST, settings.MONGO_PORT) klass.MONGO_CONNECTION = connection return klass.MONGO_CONNECTION
def connect(self, database, timezone=None, cache_size=0, auto_ensure=True, *args, **kwds): ''' `connect` is a thin wrapper around __init__ which creates the database connection that the session will use. :param database: the database name to use. Should be an instance of \ :class:`basestring` :param safe: The value for the "safe" parameter of the Session \ init function :param auto_ensure: Whether to implicitly call ensure_indexes on all write \ operations. :param args: arguments for :class:`pymongo.connection.Connection` :param kwds: keyword arguments for :class:`pymongo.connection.Connection` ''' safe = kwds.get('safe', False) if 'safe' in kwds: del kwds['safe'] if timezone is not None: kwds['tz_aware'] = True conn = Connection(*args, **kwds) db = conn[database] return Session(db, timezone=timezone, safe=safe, cache_size=cache_size, auto_ensure=auto_ensure)
def readSHPPoint(append): fileP = 'F:\\project\\Kalium_web\\shapefile\\co99_d00.shp' sf = shapefile.Reader(fileP) shapeRecs = sf.shapeRecords() mongodb_server='127.0.0.1' mongodb_port = 27017 mongodb_collection ='test1' mongodb_db = 'gisdb' connection = Connection(mongodb_server, mongodb_port) print 'Getting database %s' % mongodb_db db = connection[mongodb_db] print 'Getting the collection %s' % mongodb_collection collection = db[mongodb_collection] if append == False: print 'Removing features from the collection...' collection.remove({}) print 'Starting loading features...' for shaperec in shapeRecs: mongofeat = {} #'{x='',y=''}' strX = "%.3f" % shaperec.shape.points[0][0] strY = "%.3f" % shaperec.shape.points[0][1] mongogeom = '{x="'+strX+'",y="'+strY+'"}' print mongogeom mongofeat['geom'] = mongogeom mongofeat['name'] = shaperec.record[1] mongofeat['tmp'] = shaperec.record[2] collection.insert(mongofeat) #create 2d index collection.create_index([("geom", pymongo.GEO2D)])
def main(): parser = OptionParser( usage='mtop.py [options]\nSee also: https://github.com/beaufour/mtop') parser.add_option('-s', '--server', dest='server', default='localhost', help='connect to mongo on SERVER', metavar='SERVER') parser.add_option('-d', '--delay', dest='delay', type=int, default=1000, help='update every MS', metavar='MS') (options, _) = parser.parse_args() try: if hasattr( pymongo, 'version_tuple' ) and pymongo.version_tuple[0] >= 2 and pymongo.version_tuple[1] >= 4: from pymongo import MongoClient from pymongo.read_preferences import ReadPreference connection = MongoClient(host=options.server, read_preference=ReadPreference.SECONDARY) else: from pymongo.connection import Connection connection = Connection(options.server, slave_okay=True) except AutoReconnect, ex: print 'Connection to %s failed: %s' % (options.server, str(ex)) return -1
def setUp(self): """ set up DAS core module """ debug = 0 config = deepcopy(das_readconfig()) logger = PrintManager('TestDASMongocache', verbose=debug) config['logger'] = logger config['verbose'] = debug dburi = config['mongodb']['dburi'] connection = Connection(dburi) connection.drop_database('das') dasmapping = DASMapping(config) config['dasmapping'] = dasmapping self.dasmongocache = DASMongocache(config)
def main(): parser = SafeConfigParser() parser.read('settings.ini') connection = Connection(parser.get('mongodb', 'server')) db = None exec('db = connection.' + parser.get('mongodb', 'db')) ai = mc(db, "markov", exp=1.2) deck = [] for i in range(100): c = ai.get() if (isinstance(c, type(""))): deck.append(c) deck.sort() deck_clean = bag(lambda: 0) for c in deck: deck_clean[c] += 1 for c in set(deck): print("%2i %s" % (deck_clean[c], c))
def test_repr(self): self.skip() connection = Connection.paired(self.left, self.right) self.assertEqual( repr(connection), "Connection(['%s:%s', '%s:%s'])" % (self.left[0], self.left[1], self.right[0], self.right[1]))
def _create_mongo_connection(self): host_uri = \ 'mongodb://%s' % (",".join(["%s:%s" % h for h in self.url_nodelist])) log.info("[MongoDBGridFS] Host URI: %s" % host_uri) params = {} params['slaveok'] = self.url_options.get("slaveok", False) if self.url_options.has_key("replicaset"): params['replicaset'] = self.url_options["replicaset"] or "" conn = Connection(host_uri, **params) db = conn[self.url_database] if self.url_username: log.info("[MongoDBGridFS] Attempting to authenticate %s/%s " % (self.url_username, self.url_password)) if not db.authenticate(self.url_username, self.url_password): raise InvalidCacheBackendError( 'Cannot authenticate to MongoDB.') collection = db["%s.files" % self.url_collection] collection.ensure_index([("namespace", ASCENDING), ("filename", ASCENDING)], unique=True) collection.ensure_index([("namespace", ASCENDING)]) return (db, GridFS(db, self.url_collection))
def mongodb2shape(mongodb_server, mongodb_port, mongodb_db, mongodb_collection, output_shape, query_filter): """Convert a mongodb collection (all elements must have same attributes) to a shapefile""" print ' Converting a mongodb collection to a shapefile ' connection = Connection(mongodb_server, mongodb_port) print 'Getting database MongoDB %s...' % mongodb_db db = connection[mongodb_db] print 'Getting the collection %s...' % mongodb_collection collection = db[mongodb_collection] print 'Exporting %s elements in collection to shapefile...' % collection.count() drv = ogr.GetDriverByName("ESRI Shapefile") ds = drv.CreateDataSource(output_shape) lyr = ds.CreateLayer('test', None, ogr.wkbPolygon) print 'Shapefile %s created...' % ds.name cursor = collection.find(query_filter) # define the progressbar pbar = ProgressBar(collection.count()).start() k=0 # iterate the features in the collection and copy them to the shapefile # for simplicity we export only the geometry to the shapefile # if we would like to store also the other fields we should have created a metadata element with fields datatype info for element in cursor: element_geom = element['geom'] feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetGeometry(ogr.CreateGeometryFromWkt(element_geom)) lyr.CreateFeature(feat) feat.Destroy() k = k + 1 pbar.update(k) pbar.finish() print '%s features loaded in shapefile from MongoDb.' % lyr.GetFeatureCount()
def __init__(self): try: self.connection = Connection() #define database name self.db = self.connection.todo_db except Exception, e: raise Exception("Error!! Please check db conenction: %s" % e)
def _connect(self): self._conn = None try: if len(self.master_args) == 2: self._conn = Connection.paired( (str(self.master_args[0]['host']), int(self.master_args[0]['port'])), (str(self.master_args[1]['host']), int(self.master_args[1]['port'])), pool_size=int(self.master_args[0]['query'].get('pool_size','16'))) else: if self.master_args: try: network_timeout = self.master_args[0]['query'].get('network_timeout') if network_timeout is not None: network_timeout = float(network_timeout) master = Connection(str(self.master_args[0]['host']), int(self.master_args[0]['port']), pool_size=int(self.master_args[0]['query'].get('pool_size','16')), network_timeout=network_timeout) ###authenticating when the db requires it to be done. if self.master_args[0].get('username') and self.master_args[0].get('password'): db = database.Database(master, self.master_args[0]['path'][1:]) db.authenticate(self.master_args[0]['username'], self.master_args[0]['password']) ### except: if self.slave_args: log.exception('Cannot connect to master: %s will use slave: %s' % (self.master_args, self.slave_args)) # and continue... to use the slave only master = None else: raise else: log.info('No master connection specified, using slaves only: %s' % self.slave_args) master = None if self.slave_args: slave = [] for a in self.slave_args: network_timeout = a['query'].get('network_timeout') if network_timeout is not None: network_timeout = float(network_timeout) slave.append( Connection(str(a['host']), int(a['port']), pool_size=int(a['query'].get('pool_size','16')), slave_okay=True, network_timeout=network_timeout, ) ) if master: self._conn = MasterSlaveConnection(master, slave) else: self._conn = slave[0] else: self._conn = master except: log.exception('Cannot connect to %s %s' % (self.master_args, self.slave_args)) else: #log.info('Connected to %s %s' % (self.master_args, self.slave_args)) pass return self._conn
def _connect_mongodb(settings, mongodb): # type: (Settings, MongoDB) -> None if Connection is None: raise Exception( 'Could not initialize MongoDB (Python-Modules are missing)') mongodb.connection = Connection(*_mongodb_local_connection_opts(settings)) mongodb.db = mongodb.connection.__getitem__(os.environ['OMD_SITE'])
def test_mongo_replica_set_client(self): c = Connection(pair) ismaster = c.admin.command('ismaster') if 'setName' in ismaster: setname = str(ismaster.get('setName')) else: raise SkipTest("Not connected to a replica set.") m = MongoReplicaSetClient(pair, replicaSet=setname, w=0) coll = m.pymongo_test.write_concern_test coll.drop() doc = {"_id": ObjectId()} coll.insert(doc) self.assertTrue(coll.insert(doc, safe=False)) self.assertTrue(coll.insert(doc, w=0)) self.assertTrue(coll.insert(doc)) self.assertRaises(OperationFailure, coll.insert, doc, safe=True) self.assertRaises(OperationFailure, coll.insert, doc, w=1) m = MongoReplicaSetClient(pair, replicaSet=setname) coll = m.pymongo_test.write_concern_test self.assertTrue(coll.insert(doc, safe=False)) self.assertTrue(coll.insert(doc, w=0)) self.assertRaises(OperationFailure, coll.insert, doc) self.assertRaises(OperationFailure, coll.insert, doc, safe=True) self.assertRaises(OperationFailure, coll.insert, doc, w=1) m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s" % (pair, setname)) self.assertTrue(m.safe) coll = m.pymongo_test.write_concern_test self.assertRaises(OperationFailure, coll.insert, doc) m = MongoReplicaSetClient("mongodb://%s/?replicaSet=%s;w=0" % (pair, setname)) self.assertFalse(m.safe) coll = m.pymongo_test.write_concern_test self.assertTrue(coll.insert(doc))
def test_postProcessing(): mongodb_connection = Connection('localhost', 27017) crowds_db = mongodb_connection.set_ats_new crowds_collection = crowds_db.Crowd i = 1 for crowd in crowds_collection.find(): for k in ['start', 'end']: if crowd[k]: if type(crowd[k]) == type(1.0): crowd[k] = dt.utcfromtimestamp(crowd[k]) else: print 'nop', crowd['_id'] else: crowd[k] = dt(2011, 6, 1) end = crowd['end'] for user in crowd['users']: user['id'] = int(user['id']) for h in user['history']: for x in xrange(2): if h[x]: if type(h[x]) == type(1.0): h[x] = dt.utcfromtimestamp(h[x]) else: h[x] = end print i, crowd['_id'] i += 1 crowds_collection.save(crowd)
def test_iteration(self): connection = Connection(self.host, self.port) def iterate(): [a for a in connection] self.assertRaises(TypeError, iterate)
def mongo_upgrade(): """Migrate mongodb schema and data.""" # Read reporting.conf to fetch mongo configuration. config = Config(os.path.join("..", "..", "conf", "reporting.conf")) # Run migration only if mongo is enabled as reporting module. if config.mongodb.enabled: host = config.mongodb.get("host", "127.0.0.1") port = config.mongodb.get("port", 27017) print "Mongo reporting is enabled, strarting mongo data migration." # Connect. try: conn = Connection(host, port) db = conn.cuckoo except TypeError: print "Mongo connection port must be integer" sys.exit() except ConnectionFailure: print "Cannot connect to MongoDB" sys.exit() # Check for schema version and create it. if "cuckoo_schema" in db.collection_names(): print "Mongo schema version not expected" sys.exit() else: db.cuckoo_schema.save({"version": mongo_revision}) else: print "Mongo reporting module not enabled, skipping mongo migration."