def test_parallel_scan(self): if not (yield at_least(self.cx, (2, 5, 5))): raise SkipTest("Requires MongoDB >= 2.5.5") yield skip_if_mongos(self.cx) collection = self.collection # Enough documents that each cursor requires multiple batches. yield collection.remove() yield collection.insert(({ '_id': i } for i in range(8000)), w=test.env.w) if test.env.is_replica_set: client = self.motor_rsc() # Test that getMore messages are sent to the right server. client.read_preference = ReadPreference.SECONDARY collection = client.motor_test.test_collection docs = [] @gen.coroutine def f(cursor): self.assertTrue( isinstance(cursor, motor.motor_tornado.MotorCommandCursor)) while (yield cursor.fetch_next): docs.append(cursor.next_object()) cursors = yield collection.parallel_scan(3) yield [f(cursor) for cursor in cursors] self.assertEqual(len(docs), (yield collection.count()))
def test_authenticate(self): # self.db is logged in as root. with ignore_deprecations(): yield self.db.add_user("mike", "password") client = motor.MotorClient(host, port, **self.get_client_kwargs()) db = client.motor_test try: # Authenticate many times at once to test concurrency. yield [db.authenticate("mike", "password") for _ in range(10)] # Just make sure there are no exceptions here. yield db.remove_user("mike") yield db.logout() if (yield at_least(self.cx, (2, 5, 4))): info = yield self.db.command("usersInfo", "mike") users = info.get('users', []) else: users = yield self.db.system.users.find().to_list(length=10) self.assertFalse("mike" in [u['user'] for u in users]) finally: yield remove_all_users(self.db) test.env.sync_cx.disconnect()
def test_parallel_scan(self): if not (yield at_least(self.cx, (2, 5, 5))): raise SkipTest("Requires MongoDB >= 2.5.5") yield skip_if_mongos(self.cx) collection = self.collection # Enough documents that each cursor requires multiple batches. yield collection.remove() yield collection.insert(({'_id': i} for i in range(8000)), w=test.env.w) if test.env.is_replica_set: client = self.motor_rsc() # Test that getMore messages are sent to the right server. client.read_preference = ReadPreference.SECONDARY collection = client.motor_test.test_collection docs = [] @gen.coroutine def f(cursor): self.assertTrue(isinstance(cursor, motor.motor_tornado.MotorCommandCursor)) while (yield cursor.fetch_next): docs.append(cursor.next_object()) cursors = yield collection.parallel_scan(3) yield [f(cursor) for cursor in cursors] self.assertEqual(len(docs), (yield collection.count()))
def test_authenticate(self): # self.db is logged in as root. with ignore_deprecations(): yield self.db.add_user("mike", "password") client = motor.MotorClient(env.host, env.port, **self.get_client_kwargs()) db = client.motor_test try: # Authenticate many times at once to test concurrency. yield [db.authenticate("mike", "password") for _ in range(10)] # Just make sure there are no exceptions here. yield db.remove_user("mike") yield db.logout() if (yield at_least(self.cx, (2, 5, 4))): info = yield self.db.command("usersInfo", "mike") users = info.get('users', []) else: users = yield self.db.system.users.find().to_list(length=10) self.assertFalse("mike" in [u['user'] for u in users]) finally: yield remove_all_users(self.db) test.env.sync_cx.close()
def test_find_is_async(self): # Need parallel Javascript. if not (yield at_least(self.cx, (3,))): raise SkipTest("Requires MongoDB >= 3.0") # Confirm find() is async by launching two operations which will finish # out of order. Also test that MotorClient doesn't reuse sockets # incorrectly. # Launch find operations for _id's 1 and 2 which will finish in order # 2, then 1. coll = self.collection yield coll.insert_many([{'_id': 1}, {'_id': 2}]) results = [] futures = [Future(), Future()] def callback(result, error): if result: results.append(result) futures.pop().set_result(None) # This find() takes 0.5 seconds. coll.find({'_id': 1, '$where': delay(0.5)}).limit(1).each(callback) # Very fast lookup. coll.find({'_id': 2}).limit(1).each(callback) yield futures # Results were appended in order 2, 1. self.assertEqual([{'_id': 2}, {'_id': 1}], results)
def test_find_is_async(self): # Need parallel Javascript. if not (yield at_least(self.cx, (3, ))): raise SkipTest("Requires MongoDB >= 3.0") # Confirm find() is async by launching two operations which will finish # out of order. Also test that MotorClient doesn't reuse sockets # incorrectly. # Launch find operations for _id's 1 and 2 which will finish in order # 2, then 1. coll = self.collection yield coll.insert_many([{'_id': 1}, {'_id': 2}]) results = [] futures = [Future(), Future()] def callback(result, error): if result: results.append(result) futures.pop().set_result(None) # This find() takes 0.5 seconds. coll.find({'_id': 1, '$where': delay(0.5)}).limit(1).each(callback) # Very fast lookup. coll.find({'_id': 2}).limit(1).each(callback) yield futures # Results were appended in order 2, 1. self.assertEqual([{'_id': 2}, {'_id': 1}], results)
def test_mongodb_x509_auth(self): if 'EVERGREEN' in os.environ: raise SkipTest("TODO: fix on Evergreen") # Expects the server to be running with SSL config described above, # and with "--auth". if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") # self.env.uri includes username and password. authenticated_client = motor.MotorClient(test.env.uri, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) if not (yield at_least(authenticated_client, (2, 5, 3, -1))): raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer") if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # Give admin all necessary privileges. yield authenticated_client['$external'].add_user( MONGODB_X509_USERNAME, roles=[{ 'role': 'readWriteAnyDatabase', 'db': 'admin' }, { 'role': 'userAdminAnyDatabase', 'db': 'admin' }]) # Not authenticated. client = motor.MotorClient("server", test.env.port, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) with self.assertRaises(OperationFailure): yield client.motor_test.test.count() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % (quote_plus(MONGODB_X509_USERNAME), "server", test.env.port)) # SSL options aren't supported in the URI.... auth_uri_client = motor.MotorClient(uri, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield auth_uri_client.db.collection.find_one() # Cleanup. yield remove_all_users(authenticated_client['$external']) yield authenticated_client['$external'].logout()
def test_mongodb_x509_auth(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests as well as # --auth: # # --sslPEMKeyFile=jstests/libs/server.pem # --sslCAFile=jstests/libs/ca.pem # --sslCRLFile=jstests/libs/crl.pem # --auth if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") authenticated_client = motor.MotorClient(test.env.uri, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop) if not (yield at_least(authenticated_client, (2, 5, 3, -1))): raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer") if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # Give admin all necessary privileges. yield authenticated_client['$external'].add_user( MONGODB_X509_USERNAME, roles=[{ 'role': 'readWriteAnyDatabase', 'db': 'admin' }, { 'role': 'userAdminAnyDatabase', 'db': 'admin' }]) client = motor.MotorClient(host, port, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop) with test.assert_raises(OperationFailure): yield client.motor_test.test.count() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % (quote_plus(MONGODB_X509_USERNAME), host, port)) # SSL options aren't supported in the URI.... auth_uri_client = motor.MotorClient(uri, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop) yield auth_uri_client.db.collection.find_one() # Cleanup. yield remove_all_users(authenticated_client['$external']) yield authenticated_client['$external'].logout()
def test_to_list_with_chained_collation(self): if not (yield at_least(self.cx, (3, 4))): raise SkipTest("collation requires MongoDB >= 3.4") yield self.make_test_data() cursor = self.collection.find({}, {'_id': 1}) \ .sort([('_id', pymongo.ASCENDING)]) \ .collation(Collation("en")) expected = [{'_id': i} for i in range(200)] result = yield cursor.to_list(length=1000) self.assertEqual(expected, result)
def test_mongodb_x509_auth(self): if 'EVERGREEN' in os.environ: raise SkipTest("TODO: fix on Evergreen") # Expects the server to be running with SSL config described above, # and with "--auth". if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") # self.env.uri includes username and password. authenticated_client = motor.MotorClient( test.env.uri, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) if not (yield at_least(authenticated_client, (2, 5, 3, -1))): raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer") if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # Give admin all necessary privileges. yield authenticated_client['$external'].add_user( MONGODB_X509_USERNAME, roles=[ {'role': 'readWriteAnyDatabase', 'db': 'admin'}, {'role': 'userAdminAnyDatabase', 'db': 'admin'}]) # Not authenticated. client = motor.MotorClient( "server", test.env.port, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) with self.assertRaises(OperationFailure): yield client.motor_test.test.count() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), "server", test.env.port)) # SSL options aren't supported in the URI.... auth_uri_client = motor.MotorClient( uri, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield auth_uri_client.db.collection.find_one() # Cleanup. yield remove_all_users(authenticated_client['$external']) yield authenticated_client['$external'].logout()
def maybe_skip(self): if (yield server_is_mongos(self.cx)): raise SkipTest("mongos has no maxTimeAlwaysTimeOut fail point") if not (yield at_least(self.cx, (2, 5, 3, -1))): raise SkipTest("maxTimeMS requires MongoDB >= 2.5.3") cmdline = yield get_command_line(self.cx) if '1' != safe_get(cmdline, 'parsed.setParameter.enableTestCommands'): if 'enableTestCommands=1' not in cmdline['argv']: raise SkipTest("testing maxTimeMS requires failpoints")
def test_command_cursor_attrs(self): if not (yield at_least(self.cx, (2, 6))): raise SkipTest("Requires MongoDB >= 2.6") motor_agg_cursor_only = set( ['collection', 'start', 'args', 'kwargs', 'pipeline']).union(motor_cursor_only) pymongo_cursor = env.sync_cx.test.test.aggregate([], cursor={}) motor_cursor = self.cx.test.test.aggregate([]) self.assertEqual( attrs(pymongo_cursor) - pymongo_cursor_only, attrs(motor_cursor) - motor_agg_cursor_only)
def test_aggregation_cursor(self): if not (yield at_least(self.cx, (2, 6))): raise SkipTest("Requires MongoDB >= 2.6") db = self.db # A small collection which returns only an initial batch, # and a larger one that requires a getMore. for collection_size in (10, 1000): expected_sum = yield self._make_test_data(collection_size) cursor = db.test.aggregate(self.pipeline) docs = yield cursor.to_list(collection_size) self.assertAllDocs(expected_sum, docs)
def test_command_cursor_attrs(self): if not (yield at_least(self.cx, (2, 6))): raise SkipTest("Requires MongoDB >= 2.6") motor_agg_cursor_only = set([ 'collection', 'kwargs', 'pipeline' ]).union(motor_cursor_only) pymongo_cursor = env.sync_cx.test.test.aggregate([], cursor={}) motor_cursor = self.cx.test.test.aggregate([]) self.assertEqual( attrs(pymongo_cursor) - pymongo_cursor_only, attrs(motor_cursor) - motor_agg_cursor_only)
def test_aggregation_cursor(self): mongo_2_5_1 = yield at_least(self.cx, (2, 5, 1)) db = self.db # A small collection which returns only an initial batch, # and a larger one that requires a getMore. for collection_size in (10, 1000): expected_sum = yield self._make_test_data(collection_size) reply = yield db.test.aggregate(self.pipeline, cursor=False) self.assertAllDocs(expected_sum, reply['result']) if mongo_2_5_1: cursor = db.test.aggregate(self.pipeline) docs = yield cursor.to_list(collection_size) self.assertAllDocs(expected_sum, docs)
def test_mongodb_x509_auth(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests as well as # --auth: # # --sslPEMKeyFile=jstests/libs/server.pem # --sslCAFile=jstests/libs/ca.pem # --sslCRLFile=jstests/libs/crl.pem # --auth if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") authenticated_client = motor.MotorClient( test.env.uri, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop) if not (yield at_least(authenticated_client, (2, 5, 3, -1))): raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer") if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # Give admin all necessary privileges. yield authenticated_client['$external'].add_user( MONGODB_X509_USERNAME, roles=[ {'role': 'readWriteAnyDatabase', 'db': 'admin'}, {'role': 'userAdminAnyDatabase', 'db': 'admin'}]) client = motor.MotorClient( host, port, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop) with test.assert_raises(OperationFailure): yield client.motor_test.test.count() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), host, port)) # SSL options aren't supported in the URI.... auth_uri_client = motor.MotorClient( uri, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop) yield auth_uri_client.db.collection.find_one() # Cleanup. yield remove_all_users(authenticated_client['$external']) yield authenticated_client['$external'].logout()
def test_with_aggregate(self): if not (yield at_least(self.cx, (2, 6))): raise SkipTest("Requires MongoDB >= 2.6") coll = self.cx.motor_test.son_manipulator_test_collection _id = yield coll.insert({'foo': 'bar'}) coll.database.add_son_manipulator(CustomSONManipulator()) # Test aggregation cursor, both with fetch_next and to_list. cursor = coll.aggregate([]) assert (yield cursor.fetch_next) self.assertEqual( {'_id': _id, 'foo': 'bar', 'added_field': 42}, cursor.next_object()) cursor = coll.aggregate([]) self.assertEqual( [{'_id': _id, 'foo': 'bar', 'added_field': 42}], (yield cursor.to_list(length=None)))
def test_aggregate_callback(self): mongo_2_5_1 = yield at_least(self.cx, (2, 5, 1)) future = Future() def cb(result, error): if error: future.set_exception(error) else: future.set_result(result) # Callback is allowed if cursor=False. self.db.test.aggregate(self.pipeline, cursor=False, callback=cb) yield future # Completes without error. if mongo_2_5_1: # Pass a callback to to_list or each, not to aggregate. with self.assertRaises(InvalidOperation): self.db.test.aggregate(self.pipeline, callback=cb)
def test_aggregation_cursor(self): if not (yield at_least(self.cx, (2, 5, 1))): raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1") db = self.db # A small collection which returns only an initial batch, # and a larger one that requires a getMore. for collection_size in (10, 1000): yield db.drop_collection("test") yield db.test.insert([{'_id': i} for i in range(collection_size)]) expected_sum = sum(range(collection_size)) cursor = yield db.test.aggregate({'$project': { '_id': '$_id' }}, cursor={}) docs = yield cursor.to_list(collection_size) self.assertEqual(expected_sum, sum(doc['_id'] for doc in docs))
def test_aggregation_cursor(self): if not (yield at_least(self.cx, (2, 5, 1))): raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1") db = self.db # A small collection which returns only an initial batch, # and a larger one that requires a getMore. for collection_size in (10, 1000): yield db.drop_collection("test") yield db.test.insert([{'_id': i} for i in range(collection_size)]) expected_sum = sum(range(collection_size)) cursor = yield db.test.aggregate( {'$project': {'_id': '$_id'}}, cursor={}) docs = yield cursor.to_list(collection_size) self.assertEqual( expected_sum, sum(doc['_id'] for doc in docs))
def test_high_concurrency(self): if env.mongod_started_with_ssl: if not (yield at_least(self.cx, (2, 6))): raise SkipTest("Concurrent SSL is unreliable in 2.4") yield self.make_test_data() concurrency = 25 cx = self.motor_client(maxPoolSize=concurrency) expected_finds = 200 * concurrency n_inserts = 25 collection = cx.motor_test.test_collection insert_collection = cx.motor_test.insert_collection yield insert_collection.delete_many({}) ndocs = [0] insert_future = Future() @gen.coroutine def find(): cursor = collection.find() while (yield cursor.fetch_next): cursor.next_object() ndocs[0] += 1 # Half-way through, start an insert loop if ndocs[0] == expected_finds / 2: insert() @gen.coroutine def insert(): for i in range(n_inserts): yield insert_collection.insert_one({'s': hex(i)}) insert_future.set_result(None) # Finished yield [find() for _ in range(concurrency)] yield insert_future self.assertEqual(expected_finds, ndocs[0]) self.assertEqual(n_inserts, (yield insert_collection.count())) yield collection.delete_many({})
def test_aggregation_cursor_exc_info(self): if not (yield at_least(self.cx, (2, 6))): raise SkipTest("Requires MongoDB >= 2.6") if sys.version_info < (3, ): raise SkipTest("Requires Python 3") yield self._make_test_data(200) cursor = self.db.test.aggregate(self.pipeline) yield cursor.to_list(length=10) yield self.db.test.drop() try: yield cursor.to_list(length=None) except OperationFailure: _, _, tb = sys.exc_info() # The call tree should include PyMongo code we ran on a thread. formatted = '\n'.join(traceback.format_tb(tb)) self.assertTrue('_unpack_response' in formatted or '_check_command_response' in formatted)
def test_aggregation_cursor_exc_info(self): if not (yield at_least(self.cx, (2, 6))): raise SkipTest("Requires MongoDB >= 2.6") if sys.version_info < (3,): raise SkipTest("Requires Python 3") yield self._make_test_data(200) cursor = self.db.test.aggregate(self.pipeline) yield cursor.to_list(length=10) yield self.db.test.drop() try: yield cursor.to_list(length=None) except OperationFailure: _, _, tb = sys.exc_info() # The call tree should include PyMongo code we ran on a thread. formatted = '\n'.join(traceback.format_tb(tb)) self.assertTrue('_unpack_response' in formatted or '_check_command_response' in formatted)
def test_aggregation_cursor_to_list_callback(self): if not (yield at_least(self.cx, (2, 5, 1))): raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1") db = self.db # A small collection which returns only an initial batch, # and a larger one that requires a getMore. for collection_size in (10, 1000): expected_sum = yield self._make_test_data(collection_size) cursor = db.test.aggregate(self.pipeline) future = Future() def cb(result, error): if error: future.set_exception(error) else: future.set_result(result) cursor.to_list(collection_size, callback=cb) docs = yield future self.assertAllDocs(expected_sum, docs)