def test_init_disconnected(self): c = MongoClient(host, port, _connect=False) ctx = catch_warnings() try: warnings.simplefilter("ignore", DeprecationWarning) self.assertIsInstance(c.is_primary, bool) self.assertIsInstance(c.is_mongos, bool) self.assertIsInstance(c.max_pool_size, int) self.assertIsInstance(c.use_greenlets, bool) self.assertIsInstance(c.nodes, frozenset) self.assertIsInstance(c.auto_start_request, bool) self.assertEqual(dict, c.get_document_class()) self.assertIsInstance(c.tz_aware, bool) self.assertIsInstance(c.max_bson_size, int) self.assertIsInstance(c.min_wire_version, int) self.assertIsInstance(c.max_wire_version, int) self.assertIsInstance(c.max_write_batch_size, int) self.assertEqual(None, c.host) self.assertEqual(None, c.port) finally: ctx.exit() c.pymongo_test.test.find_one() # Auto-connect. self.assertEqual((host, port), c.address) if version.at_least(c, (2, 5, 4, -1)): self.assertTrue(c.max_wire_version > 0) else: self.assertEqual(c.max_wire_version, 0) self.assertTrue(c.min_wire_version >= 0) bad_host = "somedomainthatdoesntexist.org" c = MongoClient(bad_host, port, connectTimeoutMS=1, _connect=False) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
def test_unix_socket(self): if not hasattr(socket, "AF_UNIX"): raise SkipTest("UNIX-sockets are not supported on this system") mongodb_socket = '/tmp/mongodb-27017.sock' encoded_socket = '%2Ftmp%2Fmongodb-27017.sock' if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessible") if client_context.auth_enabled: uri = "mongodb://%s:%s@%s" % (db_user, db_pwd, encoded_socket) else: uri = "mongodb://%s" % encoded_socket # Confirm we can do operations via the socket. client = MongoClient(uri) client.pymongo_test.test.insert_one({"dummy": "object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) # Confirm it fails with a missing socket. self.assertRaises( ConnectionFailure, connected, MongoClient("mongodb://%2Ftmp%2Fnon-existent.sock", serverSelectionTimeoutMS=100))
def test_ipv6(self): c = MongoClient("mongodb://[::1]:%d" % (port,), replicaSet=self.name) # Client switches to IPv4 once it has first ismaster response. msg = 'discovered primary with IPv4 address "%r"' % (self.primary,) wait_until(lambda: c.primary == self.primary, msg) # Same outcome with both IPv4 and IPv6 seeds. c = MongoClient("[::1]:%d,localhost:%d" % (port, port), replicaSet=self.name) wait_until(lambda: c.primary == self.primary, msg) if client_context.auth_enabled: auth_str = "%s:%s@" % (db_user, db_pwd) else: auth_str = "" uri = "mongodb://%slocalhost:%d,[::1]:%d" % (auth_str, port, port) client = MongoClient(uri, replicaSet=self.name) client.pymongo_test.test.insert_one({"dummy": u("object")}) client.pymongo_test_bernie.test.insert_one({"dummy": u("object")}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_bernie" in dbs) client.close()
def test_unix_socket(self): if not hasattr(socket, "AF_UNIX"): raise SkipTest("UNIX-sockets are not supported on this system") client = MongoClient(host, port) if (sys.platform == 'darwin' and server_started_with_auth(client) and not version.at_least(client, (2, 7, 1))): raise SkipTest("SERVER-8492") mongodb_socket = '/tmp/mongodb-27017.sock' if not os.access(mongodb_socket, os.R_OK): raise SkipTest("Socket file is not accessable") self.assertTrue(MongoClient("mongodb://%s" % mongodb_socket)) client = MongoClient("mongodb://%s" % mongodb_socket) client.pymongo_test.test.save({"dummy": "object"}) # Confirm we can read via the socket dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) # Confirm it fails with a missing socket self.assertRaises(ConnectionFailure, MongoClient, "mongodb:///tmp/none-existent.sock")
def test_document_class(self): c = MongoClient(host, port) db = c.pymongo_test db.test.insert({"x": 1}) ctx = catch_warnings() try: warnings.simplefilter("ignore", DeprecationWarning) self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = MongoClient(host, port, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict db = c.pymongo_test self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) finally: ctx.exit()
def test_max_idle_time_checkout(self): # Use high frequency to test _get_socket_no_auth. with client_knobs(kill_cursor_frequency=99999999): client = MongoClient(host, port, maxIdleTimeMS=.5) server = client._get_topology().select_server(any_server_selector) with server._pool.get_socket({}) as sock_info: pass self.assertEqual(1, len(server._pool.sockets)) time.sleep(1) # Sleep so that the socket becomes stale. with server._pool.get_socket({}) as new_sock_info: self.assertNotEqual(sock_info, new_sock_info) self.assertEqual(1, len(server._pool.sockets)) self.assertFalse(sock_info in server._pool.sockets) self.assertTrue(new_sock_info in server._pool.sockets) # Test that sockets are reused if maxIdleTimeMS is not set. client = MongoClient(host, port) server = client._get_topology().select_server(any_server_selector) with server._pool.get_socket({}) as sock_info: pass self.assertEqual(1, len(server._pool.sockets)) time.sleep(1) with server._pool.get_socket({}) as new_sock_info: self.assertEqual(sock_info, new_sock_info) self.assertEqual(1, len(server._pool.sockets))
def test_init_disconnected(self): c = MongoClient(host, port, _connect=False) self.assertIsInstance(c.is_primary, bool) self.assertIsInstance(c.is_mongos, bool) self.assertIsInstance(c.max_pool_size, int) self.assertIsInstance(c.use_greenlets, bool) self.assertIsInstance(c.nodes, frozenset) self.assertIsInstance(c.auto_start_request, bool) self.assertEqual(dict, c.get_document_class()) self.assertIsInstance(c.tz_aware, bool) self.assertIsInstance(c.max_bson_size, int) self.assertIsInstance(c.min_wire_version, int) self.assertIsInstance(c.max_wire_version, int) self.assertEqual(None, c.host) self.assertEqual(None, c.port) c.pymongo_test.test.find_one() # Auto-connect. self.assertEqual(host, c.host) self.assertEqual(port, c.port) if version.at_least(c, (2, 5, 4, -1)): self.assertTrue(c.max_wire_version > 0) else: self.assertEqual(c.max_wire_version, 0) self.assertTrue(c.min_wire_version >= 0) bad_host = "somedomainthatdoesntexist.org" c = MongoClient(bad_host, port, connectTimeoutMS=1, _connect=False) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
def open_spider(self, spider): self._build_unique_key() if self._replica_set is not None: self.connection = MongoReplicaSetClient( self._uri, replicaSet=self._replica_set, w=self._write_concern, fsync=self._fsync, read_preference=ReadPreference.PRIMARY_PREFERRED) else: self.connection = MongoClient( self._uri, fsync=self._fsync, read_preference=ReadPreference.PRIMARY) self.database = self.connection[self._database] self.collection = self.database[self._collection] log.msg('Connected to MongoDB "%s", using "%s/%s"' % self._uri, self._database, self._collection) # ensure index if self._unique_key: log.msg('Creating index for key %s' % self._unique_key) self.collection.ensure_index(self._unique_key.items(), unique=True, sparse=True)
def test_read_with_failover(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") def iter_cursor(cursor): for _ in cursor: pass return True w = len(c.secondaries) + 1 db = c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) # Force replication db.test.insert_many([{'foo': i} for i in xrange(10)]) self.assertEqual(10, db.test.count()) db.read_preference = SECONDARY_PREFERRED cursor = db.test.find().batch_size(5) next(cursor) self.assertEqual(5, cursor._Cursor__retrieved) self.assertTrue(cursor.address in c.secondaries) ha_tools.kill_primary() # Primary failure shouldn't interrupt the cursor self.assertTrue(iter_cursor(cursor)) self.assertEqual(10, cursor._Cursor__retrieved)
def mongo_text_test(): cli = MongoClient() test = cli.get_database("test").get_collection("test_search") test.create_index( [("super_type", ASCENDING), ("resource_state", ASCENDING), ("uuid", TEXT), ("name", TEXT), ("description", TEXT)], background=True ) docs = [{ "uuid": str(uuid.uuid4()), "name": str(i), "description": "Nature, \"time, and patience are " "the three great -physicians.", "super_type": "super_vol", "type": "vol", "create_time": int( time.time()), "resource_state": "inUse" } for i in range(1, 11)] test.insert_many(docs) text = [" -physicians"] print test.find({ "description": { "$regex": "|".join([re.sub( r"(\*|\.|\?|\+|\$|\^|\[|\]|\(|\)|\{|\}|\||\\|/)", r"\\\1", g ) for g in text]), "$options": "i" } }).count()
def test_document_class(self): c = MongoClient(host, port) db = c.pymongo_test db.test.insert({"x": 1}) self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = MongoClient(host, port, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON))
def __init__(self): connection = MongoClient( settings['MONGO_SERVER'], settings['MONGO_PORT'] ) db = connection.get_database(settings['MONGO_DB']) self.collection = db[settings['MONGO_COLLECTION']]
def stash(results): """ 暂存到mongo数据库中。 """ summary = {} mongo = MongoClient(**config.mongo) try: for item_model, objs in results: collection_name = item_model['name'] db = mongo.get_database('theforce') collection = db.get_collection(collection_name) collection.insert_many(objs) summary[collection_name] = len( objs) if collection_name not in summary else len( objs) + summary[collection_name] print print "=" * 40 print ' ' * 15, u'Stash' print "=" * 40 print print u"数据已成功保存到MongoDB的theforce库中,其中新增数据:" for name, length in summary.items(): print name, length finally: mongo.close()
def test_database_names(self): client = MongoClient(host, port) client.pymongo_test.test.save({"dummy": u"object"}) client.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_mike" in dbs)
def doEverything(): # certfile = '/home/bryan/Downloads/baratheon.pem' conn = MongoClient(url) db = conn[database] commands = [] collectionName = "pythonMongo" commands.append("Creating collection " + collectionName) collection = db[collectionName] #insert 1 commands.append("# 1 Inserts") commands.append("# 1.1 Insert a single document to a collection") collection.insert({"name": "test1", "value": 1}) commands.append("Inserted {\"name\": \"test1\", \"value\": 1}") #insert many commands.append("#1.2 Inserting multiple entries into collection") multiPost = [{"name": "test1", "value": 1},{"name": "test2", "value": 2}, {"name": "test3", "value": 3}] collection.insert(multiPost) commands.append("Inserted \n {\"name\": \"test1\", \"value\": 1} \n {\"name\": \"test2\", \"value\": 2} \n {\"name\": \"test3\", \"value\": 3}") # Find commands.append("#2 Queries") commands.append("#2.1 Find one that matches a query condition") commands.append(collection.find_one({"name": "test1"})) # Find all commands.append("#2.2 Find all that match a query condition") for doc in collection.find({"name": "test1"}): commands.append(doc) # Display all documents commands.append( "#2.3 Find all documents in collection") for doc in collection.find(): commands.append(doc) # update document commands.append("#3 Updating Documents") collection.update({"name": "test3"}, {"$set": { "value": 4}}) commands.append("Updated test3 with value 4") # delete document commands.append("#4 Delete Documents") collection.remove({"name": "test2"}) commands.append("Deleted all with name test2") # Display all collection names commands.append("#5 Get a list of all of the collections") commands.append( db.collection_names()) commands.append("#6 Drop a collection") db.drop_collection(collectionName) conn.close() commands.append("Connection to database has been closed") return commands
def makeDBConnection(port, **kwargs): global _C if not _C: logging.info("establishing db connection at port %s ..." % port) import pymongo logging.info("using pymongo version %s" % pymongo.version) from pymongo.mongo_client import MongoClient _C = MongoClient(port = port, **kwargs) mongo_info = _C.server_info() logging.info("mongodb version: %s" % mongo_info["version"])
def test_alive(self): ctx = catch_warnings() try: warnings.simplefilter("ignore", DeprecationWarning) self.assertTrue(get_client().alive()) client = MongoClient('doesnt exist', _connect=False) self.assertFalse(client.alive()) finally: ctx.exit()
def test_stale_getmore(self): # A cursor is created, but its member goes down and is removed from # the topology before the getMore message is sent. Test that # MongoClient._send_message_with_response handles the error. with self.assertRaises(AutoReconnect): client = MongoClient(host, port, connect=False, serverSelectionTimeoutMS=100, replicaSet=client_context.replica_set_name) client._send_message_with_response( operation=message._GetMore('collection', 101, 1234), address=('not-a-member', 27017))
def test_disconnect(self): c = MongoClient(host, port) coll = c.pymongo_test.bar c.disconnect() c.disconnect() coll.count() c.disconnect() c.disconnect() coll.count()
def test_mongo(self): client = MongoClient() db = client.test_database collection = db.test_collection import datetime post = {"author": "Mike", "text": "My first blog post!", "tags": ["mongodb", "python", "pymongo"], "date": datetime.datetime.utcnow()} posts = db.posts post_id = posts.insert(post) print(post_id) print(client.server_info())
def test_properties(self): c = client_context.rs_client c.admin.command('ping') wait_until(lambda: c.primary == self.primary, "discover primary") wait_until(lambda: c.arbiters == self.arbiters, "discover arbiters") wait_until(lambda: c.secondaries == self.secondaries, "discover secondaries") self.assertEqual(c.primary, self.primary) self.assertEqual(c.secondaries, self.secondaries) self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.max_pool_size, 100) # Make sure MongoClient's properties are copied to Database and # Collection. for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.codec_options, CodecOptions()) self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.write_concern, WriteConcern()) cursor = c.pymongo_test.test.find() self.assertEqual( ReadPreference.PRIMARY, cursor._Cursor__read_preference) tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}] secondary = Secondary(tag_sets=tag_sets) c = MongoClient( pair, replicaSet=self.name, maxPoolSize=25, document_class=SON, tz_aware=True, read_preference=secondary, localThresholdMS=77, j=True) self.assertEqual(c.max_pool_size, 25) for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.codec_options, CodecOptions(SON, True)) self.assertEqual(obj.read_preference, secondary) self.assertEqual(obj.write_concern, WriteConcern(j=True)) cursor = c.pymongo_test.test.find() self.assertEqual( secondary, cursor._Cursor__read_preference) nearest = Nearest(tag_sets=[{'dc': 'ny'}, {}]) cursor = c.pymongo_test.get_collection( "test", read_preference=nearest).find() self.assertEqual(nearest, cursor._Cursor__read_preference) self.assertEqual(c.max_bson_size, 16777216) c.close()
def test_slave_okay_metadata_commands(self): secondaries = iter(self._get_client().secondaries) host, port = secondaries.next() # Direct connection to a secondary. client = MongoClient(host, port) self.assertFalse(client.is_primary) self.assertEqual(client.read_preference, ReadPreference.PRIMARY) # No error. client.database_names() client.pymongo_test.collection_names() client.pymongo_test.test.options() client.pymongo_test.test.index_information()
def get_empty_db_da(_database_name, _json_schema_folders=None, _uri_handlers = None): """ Create an empty database. Drops any existing. :param _database_name: The name of the database :return: A database access object for the database :param _json_schema_folders: A list of application specific JSON schema folders """ _client = MongoClient() if _database_name in _client.database_names(): _client.drop_database(_client[_database_name]) _database = _client[_database_name] return DatabaseAccess(_database=_database, _json_schema_folders=_json_schema_folders, _uri_handlers=_uri_handlers)
def test_kill_cursors_with_server_unavailable(self): with client_knobs(kill_cursor_frequency=9999999): client = MongoClient("doesnt exist", connect=False, serverSelectionTimeoutMS=0) # Wait for the first tick of the periodic kill-cursors to pass. time.sleep(1) # Enqueue a kill-cursors message. client.close_cursor(1234, ("doesnt-exist", 27017)) with warnings.catch_warnings(record=True) as user_warnings: client._process_kill_cursors_queue() self.assertIn("couldn't close cursor on ('doesnt-exist', 27017)", str(user_warnings[0].message))
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = MongoClient(primary, use_greenlets=use_greenlets) secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets) rsc = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) try: self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) finally: rsc.close()
class TestMongosLoadBalancing(HATestCase): def setUp(self): super(TestMongosLoadBalancing, self).setUp() seed_list = ha_tools.create_sharded_cluster() self.assertIsNotNone(seed_list) self.dbname = 'pymongo_mongos_ha' self.client = MongoClient( seed_list, serverSelectionTimeoutMS=self.server_selection_timeout) self.client.drop_database(self.dbname) def test_mongos_load_balancing(self): wait_until(lambda: len(ha_tools.routers) == len(self.client.nodes), 'discover all mongoses') # Can't access "address" when load balancing. with self.assertRaises(InvalidOperation): self.client.address coll = self.client[self.dbname].test coll.insert_one({'foo': 'bar'}) live_routers = list(ha_tools.routers) ha_tools.kill_mongos(live_routers.pop()) while live_routers: try: self.assertEqual(1, coll.count()) except ConnectionFailure: # If first attempt happened to select the dead mongos. self.assertEqual(1, coll.count()) wait_until(lambda: len(live_routers) == len(self.client.nodes), 'remove dead mongos', timeout=30) ha_tools.kill_mongos(live_routers.pop()) # Make sure the last one's really dead. time.sleep(1) # I'm alone. self.assertRaises(ConnectionFailure, coll.count) wait_until(lambda: 0 == len(self.client.nodes), 'remove dead mongos', timeout=30) ha_tools.restart_mongos(one(ha_tools.routers)) # Find new mongos self.assertEqual(1, coll.count())
def test_last_error(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: c.secondaries, "discover secondary") ha_tools.stepdown_primary() db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=0)) db.test.insert_one({}) response = db.error() self.assertTrue('err' in response and 'not master' in response['err']) wait_until(lambda: len(c.secondaries) == 2, "discover two secondaries")
def connect(self): self.client = MongoClient( self.config['HOST'], self.config['PORT'] ) self.db = self.client[self.config['COLLECTION']]
def test_write_concern(self): c = MongoClient(pair) self.assertEqual({}, c.write_concern) wc = {"w": 2, "wtimeout": 1000} c.write_concern = wc self.assertEqual(wc, c.write_concern) wc = {"w": 3, "wtimeout": 1000} c.write_concern["w"] = 3 self.assertEqual(wc, c.write_concern) wc = {"w": 3} del c.write_concern["wtimeout"] self.assertEqual(wc, c.write_concern) wc = {"w": 3, "wtimeout": 1000} c = MongoClient(pair, w=3, wtimeout=1000) self.assertEqual(wc, c.write_concern) wc = {"w": 2, "wtimeout": 1000} c.write_concern = wc self.assertEqual(wc, c.write_concern) db = c.pymongo_test self.assertEqual(wc, db.write_concern) coll = db.test self.assertEqual(wc, coll.write_concern) coll.write_concern = {"j": True} self.assertEqual({"j": True}, coll.write_concern) self.assertEqual(wc, db.write_concern) wc = SON([("w", 2)]) coll.write_concern = wc self.assertEqual(wc.to_dict(), coll.write_concern) def f(): c.write_concern = {"foo": "bar"} self.assertRaises(ConfigurationError, f) def f(): c.write_concern["foo"] = "bar" self.assertRaises(ConfigurationError, f) def f(): c.write_concern = [("foo", "bar")] self.assertRaises(ConfigurationError, f)
def test_min_pool_size(self): with client_knobs(kill_cursor_frequency=.1): client = MongoClient(host, port) server = client._get_topology().select_server(any_server_selector) self.assertEqual(0, len(server._pool.sockets)) # Assert that pool started up at minPoolSize client = MongoClient(host, port, minPoolSize=10) server = client._get_topology().select_server(any_server_selector) wait_until(lambda: 10 == len(server._pool.sockets), "pool initialized with 10 sockets") # Assert that if a socket is closed, a new one takes its place with server._pool.get_socket({}) as sock_info: sock_info.close() wait_until(lambda: 10 == len(server._pool.sockets), "a closed socket gets replaced from the pool") self.assertFalse(sock_info in server._pool.sockets)
def setUp(self): client = MongoClient(pair) response = client.admin.command('ismaster') if 'setName' in response: raise SkipTest("Connected to a replica set, not a standalone mongod")