def test_parallel_scan(self):
if not (yield at_least(self.cx, (2, 5, 5))):
raise SkipTest("Requires MongoDB >= 2.5.5")
yield skip_if_mongos(self.cx)
collection = self.collection
# Enough documents that each cursor requires multiple batches.
yield collection.remove()
yield collection.insert(({
'_id': i
} for i in range(8000)),
w=test.env.w)
if test.env.is_replica_set:
client = self.motor_rsc()
# Test that getMore messages are sent to the right server.
client.read_preference = ReadPreference.SECONDARY
collection = client.motor_test.test_collection
docs = []
@gen.coroutine
def f(cursor):
self.assertTrue(
isinstance(cursor, motor.motor_tornado.MotorCommandCursor))
while (yield cursor.fetch_next):
docs.append(cursor.next_object())
cursors = yield collection.parallel_scan(3)
yield [f(cursor) for cursor in cursors]
self.assertEqual(len(docs), (yield collection.count()))
def test_authenticate(self):
# self.db is logged in as root.
with ignore_deprecations():
yield self.db.add_user("mike", "password")
client = motor.MotorClient(host, port, **self.get_client_kwargs())
db = client.motor_test
try:
# Authenticate many times at once to test concurrency.
yield [db.authenticate("mike", "password") for _ in range(10)]
# Just make sure there are no exceptions here.
yield db.remove_user("mike")
yield db.logout()
if (yield at_least(self.cx, (2, 5, 4))):
info = yield self.db.command("usersInfo", "mike")
users = info.get('users', [])
else:
users = yield self.db.system.users.find().to_list(length=10)
self.assertFalse("mike" in [u['user'] for u in users])
finally:
yield remove_all_users(self.db)
test.env.sync_cx.disconnect()
def test_parallel_scan(self):
if not (yield at_least(self.cx, (2, 5, 5))):
raise SkipTest("Requires MongoDB >= 2.5.5")
yield skip_if_mongos(self.cx)
collection = self.collection
# Enough documents that each cursor requires multiple batches.
yield collection.remove()
yield collection.insert(({'_id': i} for i in range(8000)), w=test.env.w)
if test.env.is_replica_set:
client = self.motor_rsc()
# Test that getMore messages are sent to the right server.
client.read_preference = ReadPreference.SECONDARY
collection = client.motor_test.test_collection
docs = []
@gen.coroutine
def f(cursor):
self.assertTrue(isinstance(cursor,
motor.motor_tornado.MotorCommandCursor))
while (yield cursor.fetch_next):
docs.append(cursor.next_object())
cursors = yield collection.parallel_scan(3)
yield [f(cursor) for cursor in cursors]
self.assertEqual(len(docs), (yield collection.count()))
def test_authenticate(self):
# self.db is logged in as root.
with ignore_deprecations():
yield self.db.add_user("mike", "password")
client = motor.MotorClient(env.host, env.port,
**self.get_client_kwargs())
db = client.motor_test
try:
# Authenticate many times at once to test concurrency.
yield [db.authenticate("mike", "password") for _ in range(10)]
# Just make sure there are no exceptions here.
yield db.remove_user("mike")
yield db.logout()
if (yield at_least(self.cx, (2, 5, 4))):
info = yield self.db.command("usersInfo", "mike")
users = info.get('users', [])
else:
users = yield self.db.system.users.find().to_list(length=10)
self.assertFalse("mike" in [u['user'] for u in users])
finally:
yield remove_all_users(self.db)
test.env.sync_cx.close()
def test_find_is_async(self):
# Need parallel Javascript.
if not (yield at_least(self.cx, (3,))):
raise SkipTest("Requires MongoDB >= 3.0")
# Confirm find() is async by launching two operations which will finish
# out of order. Also test that MotorClient doesn't reuse sockets
# incorrectly.
# Launch find operations for _id's 1 and 2 which will finish in order
# 2, then 1.
coll = self.collection
yield coll.insert_many([{'_id': 1}, {'_id': 2}])
results = []
futures = [Future(), Future()]
def callback(result, error):
if result:
results.append(result)
futures.pop().set_result(None)
# This find() takes 0.5 seconds.
coll.find({'_id': 1, '$where': delay(0.5)}).limit(1).each(callback)
# Very fast lookup.
coll.find({'_id': 2}).limit(1).each(callback)
yield futures
# Results were appended in order 2, 1.
self.assertEqual([{'_id': 2}, {'_id': 1}], results)
def test_find_is_async(self):
# Need parallel Javascript.
if not (yield at_least(self.cx, (3, ))):
raise SkipTest("Requires MongoDB >= 3.0")
# Confirm find() is async by launching two operations which will finish
# out of order. Also test that MotorClient doesn't reuse sockets
# incorrectly.
# Launch find operations for _id's 1 and 2 which will finish in order
# 2, then 1.
coll = self.collection
yield coll.insert_many([{'_id': 1}, {'_id': 2}])
results = []
futures = [Future(), Future()]
def callback(result, error):
if result:
results.append(result)
futures.pop().set_result(None)
# This find() takes 0.5 seconds.
coll.find({'_id': 1, '$where': delay(0.5)}).limit(1).each(callback)
# Very fast lookup.
coll.find({'_id': 2}).limit(1).each(callback)
yield futures
# Results were appended in order 2, 1.
self.assertEqual([{'_id': 2}, {'_id': 1}], results)
def test_mongodb_x509_auth(self):
if 'EVERGREEN' in os.environ:
raise SkipTest("TODO: fix on Evergreen")
# Expects the server to be running with SSL config described above,
# and with "--auth".
if not test.env.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
# self.env.uri includes username and password.
authenticated_client = motor.MotorClient(test.env.uri,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.io_loop)
if not (yield at_least(authenticated_client, (2, 5, 3, -1))):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# Give admin all necessary privileges.
yield authenticated_client['$external'].add_user(
MONGODB_X509_USERNAME,
roles=[{
'role': 'readWriteAnyDatabase',
'db': 'admin'
}, {
'role': 'userAdminAnyDatabase',
'db': 'admin'
}])
# Not authenticated.
client = motor.MotorClient("server",
test.env.port,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.io_loop)
with self.assertRaises(OperationFailure):
yield client.motor_test.test.count()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' %
(quote_plus(MONGODB_X509_USERNAME), "server", test.env.port))
# SSL options aren't supported in the URI....
auth_uri_client = motor.MotorClient(uri,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.io_loop)
yield auth_uri_client.db.collection.find_one()
# Cleanup.
yield remove_all_users(authenticated_client['$external'])
yield authenticated_client['$external'].logout()
def test_mongodb_x509_auth(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests as well as
# --auth:
#
# --sslPEMKeyFile=jstests/libs/server.pem
# --sslCAFile=jstests/libs/ca.pem
# --sslCRLFile=jstests/libs/crl.pem
# --auth
if not test.env.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
authenticated_client = motor.MotorClient(test.env.uri,
ssl_certfile=CLIENT_PEM,
io_loop=self.io_loop)
if not (yield at_least(authenticated_client, (2, 5, 3, -1))):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# Give admin all necessary privileges.
yield authenticated_client['$external'].add_user(
MONGODB_X509_USERNAME,
roles=[{
'role': 'readWriteAnyDatabase',
'db': 'admin'
}, {
'role': 'userAdminAnyDatabase',
'db': 'admin'
}])
client = motor.MotorClient(host,
port,
ssl_certfile=CLIENT_PEM,
io_loop=self.io_loop)
with test.assert_raises(OperationFailure):
yield client.motor_test.test.count()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' %
(quote_plus(MONGODB_X509_USERNAME), host, port))
# SSL options aren't supported in the URI....
auth_uri_client = motor.MotorClient(uri,
ssl_certfile=CLIENT_PEM,
io_loop=self.io_loop)
yield auth_uri_client.db.collection.find_one()
# Cleanup.
yield remove_all_users(authenticated_client['$external'])
yield authenticated_client['$external'].logout()
def test_to_list_with_chained_collation(self):
if not (yield at_least(self.cx, (3, 4))):
raise SkipTest("collation requires MongoDB >= 3.4")
yield self.make_test_data()
cursor = self.collection.find({}, {'_id': 1}) \
.sort([('_id', pymongo.ASCENDING)]) \
.collation(Collation("en"))
expected = [{'_id': i} for i in range(200)]
result = yield cursor.to_list(length=1000)
self.assertEqual(expected, result)
def test_mongodb_x509_auth(self):
if 'EVERGREEN' in os.environ:
raise SkipTest("TODO: fix on Evergreen")
# Expects the server to be running with SSL config described above,
# and with "--auth".
if not test.env.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
# self.env.uri includes username and password.
authenticated_client = motor.MotorClient(
test.env.uri,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.io_loop)
if not (yield at_least(authenticated_client, (2, 5, 3, -1))):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# Give admin all necessary privileges.
yield authenticated_client['$external'].add_user(
MONGODB_X509_USERNAME, roles=[
{'role': 'readWriteAnyDatabase', 'db': 'admin'},
{'role': 'userAdminAnyDatabase', 'db': 'admin'}])
# Not authenticated.
client = motor.MotorClient(
"server", test.env.port,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.io_loop)
with self.assertRaises(OperationFailure):
yield client.motor_test.test.count()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' % (
quote_plus(MONGODB_X509_USERNAME), "server", test.env.port))
# SSL options aren't supported in the URI....
auth_uri_client = motor.MotorClient(
uri,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.io_loop)
yield auth_uri_client.db.collection.find_one()
# Cleanup.
yield remove_all_users(authenticated_client['$external'])
yield authenticated_client['$external'].logout()
def maybe_skip(self):
if (yield server_is_mongos(self.cx)):
raise SkipTest("mongos has no maxTimeAlwaysTimeOut fail point")
if not (yield at_least(self.cx, (2, 5, 3, -1))):
raise SkipTest("maxTimeMS requires MongoDB >= 2.5.3")
cmdline = yield get_command_line(self.cx)
if '1' != safe_get(cmdline, 'parsed.setParameter.enableTestCommands'):
if 'enableTestCommands=1' not in cmdline['argv']:
raise SkipTest("testing maxTimeMS requires failpoints")
def maybe_skip(self):
if (yield server_is_mongos(self.cx)):
raise SkipTest("mongos has no maxTimeAlwaysTimeOut fail point")
if not (yield at_least(self.cx, (2, 5, 3, -1))):
raise SkipTest("maxTimeMS requires MongoDB >= 2.5.3")
cmdline = yield get_command_line(self.cx)
if '1' != safe_get(cmdline, 'parsed.setParameter.enableTestCommands'):
if 'enableTestCommands=1' not in cmdline['argv']:
raise SkipTest("testing maxTimeMS requires failpoints")
def test_command_cursor_attrs(self):
if not (yield at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
motor_agg_cursor_only = set(
['collection', 'start', 'args', 'kwargs',
'pipeline']).union(motor_cursor_only)
pymongo_cursor = env.sync_cx.test.test.aggregate([], cursor={})
motor_cursor = self.cx.test.test.aggregate([])
self.assertEqual(
attrs(pymongo_cursor) - pymongo_cursor_only,
attrs(motor_cursor) - motor_agg_cursor_only)
def test_aggregation_cursor(self):
if not (yield at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
expected_sum = yield self._make_test_data(collection_size)
cursor = db.test.aggregate(self.pipeline)
docs = yield cursor.to_list(collection_size)
self.assertAllDocs(expected_sum, docs)
def test_aggregation_cursor(self):
if not (yield at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
expected_sum = yield self._make_test_data(collection_size)
cursor = db.test.aggregate(self.pipeline)
docs = yield cursor.to_list(collection_size)
self.assertAllDocs(expected_sum, docs)
def test_command_cursor_attrs(self):
if not (yield at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
motor_agg_cursor_only = set([
'collection',
'kwargs',
'pipeline'
]).union(motor_cursor_only)
pymongo_cursor = env.sync_cx.test.test.aggregate([], cursor={})
motor_cursor = self.cx.test.test.aggregate([])
self.assertEqual(
attrs(pymongo_cursor) - pymongo_cursor_only,
attrs(motor_cursor) - motor_agg_cursor_only)
def test_aggregation_cursor(self):
mongo_2_5_1 = yield at_least(self.cx, (2, 5, 1))
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
expected_sum = yield self._make_test_data(collection_size)
reply = yield db.test.aggregate(self.pipeline, cursor=False)
self.assertAllDocs(expected_sum, reply['result'])
if mongo_2_5_1:
cursor = db.test.aggregate(self.pipeline)
docs = yield cursor.to_list(collection_size)
self.assertAllDocs(expected_sum, docs)
def test_aggregation_cursor(self):
mongo_2_5_1 = yield at_least(self.cx, (2, 5, 1))
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
expected_sum = yield self._make_test_data(collection_size)
reply = yield db.test.aggregate(self.pipeline, cursor=False)
self.assertAllDocs(expected_sum, reply['result'])
if mongo_2_5_1:
cursor = db.test.aggregate(self.pipeline)
docs = yield cursor.to_list(collection_size)
self.assertAllDocs(expected_sum, docs)
def test_mongodb_x509_auth(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests as well as
# --auth:
#
# --sslPEMKeyFile=jstests/libs/server.pem
# --sslCAFile=jstests/libs/ca.pem
# --sslCRLFile=jstests/libs/crl.pem
# --auth
if not test.env.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
authenticated_client = motor.MotorClient(
test.env.uri, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop)
if not (yield at_least(authenticated_client, (2, 5, 3, -1))):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# Give admin all necessary privileges.
yield authenticated_client['$external'].add_user(
MONGODB_X509_USERNAME, roles=[
{'role': 'readWriteAnyDatabase', 'db': 'admin'},
{'role': 'userAdminAnyDatabase', 'db': 'admin'}])
client = motor.MotorClient(
host, port, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop)
with test.assert_raises(OperationFailure):
yield client.motor_test.test.count()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' % (
quote_plus(MONGODB_X509_USERNAME), host, port))
# SSL options aren't supported in the URI....
auth_uri_client = motor.MotorClient(
uri, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop)
yield auth_uri_client.db.collection.find_one()
# Cleanup.
yield remove_all_users(authenticated_client['$external'])
yield authenticated_client['$external'].logout()
def test_with_aggregate(self):
if not (yield at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
coll = self.cx.motor_test.son_manipulator_test_collection
_id = yield coll.insert({'foo': 'bar'})
coll.database.add_son_manipulator(CustomSONManipulator())
# Test aggregation cursor, both with fetch_next and to_list.
cursor = coll.aggregate([])
assert (yield cursor.fetch_next)
self.assertEqual(
{'_id': _id, 'foo': 'bar', 'added_field': 42},
cursor.next_object())
cursor = coll.aggregate([])
self.assertEqual(
[{'_id': _id, 'foo': 'bar', 'added_field': 42}],
(yield cursor.to_list(length=None)))
def test_aggregate_callback(self):
mongo_2_5_1 = yield at_least(self.cx, (2, 5, 1))
future = Future()
def cb(result, error):
if error:
future.set_exception(error)
else:
future.set_result(result)
# Callback is allowed if cursor=False.
self.db.test.aggregate(self.pipeline, cursor=False, callback=cb)
yield future # Completes without error.
if mongo_2_5_1:
# Pass a callback to to_list or each, not to aggregate.
with self.assertRaises(InvalidOperation):
self.db.test.aggregate(self.pipeline, callback=cb)
def test_aggregation_cursor(self):
if not (yield at_least(self.cx, (2, 5, 1))):
raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
yield db.drop_collection("test")
yield db.test.insert([{'_id': i} for i in range(collection_size)])
expected_sum = sum(range(collection_size))
cursor = yield db.test.aggregate({'$project': {
'_id': '$_id'
}},
cursor={})
docs = yield cursor.to_list(collection_size)
self.assertEqual(expected_sum, sum(doc['_id'] for doc in docs))
def test_aggregate_callback(self):
mongo_2_5_1 = yield at_least(self.cx, (2, 5, 1))
future = Future()
def cb(result, error):
if error:
future.set_exception(error)
else:
future.set_result(result)
# Callback is allowed if cursor=False.
self.db.test.aggregate(self.pipeline, cursor=False, callback=cb)
yield future # Completes without error.
if mongo_2_5_1:
# Pass a callback to to_list or each, not to aggregate.
with self.assertRaises(InvalidOperation):
self.db.test.aggregate(self.pipeline, callback=cb)
def test_aggregation_cursor(self):
if not (yield at_least(self.cx, (2, 5, 1))):
raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
yield db.drop_collection("test")
yield db.test.insert([{'_id': i} for i in range(collection_size)])
expected_sum = sum(range(collection_size))
cursor = yield db.test.aggregate(
{'$project': {'_id': '$_id'}}, cursor={})
docs = yield cursor.to_list(collection_size)
self.assertEqual(
expected_sum,
sum(doc['_id'] for doc in docs))
def test_high_concurrency(self):
if env.mongod_started_with_ssl:
if not (yield at_least(self.cx, (2, 6))):
raise SkipTest("Concurrent SSL is unreliable in 2.4")
yield self.make_test_data()
concurrency = 25
cx = self.motor_client(maxPoolSize=concurrency)
expected_finds = 200 * concurrency
n_inserts = 25
collection = cx.motor_test.test_collection
insert_collection = cx.motor_test.insert_collection
yield insert_collection.delete_many({})
ndocs = [0]
insert_future = Future()
@gen.coroutine
def find():
cursor = collection.find()
while (yield cursor.fetch_next):
cursor.next_object()
ndocs[0] += 1
# Half-way through, start an insert loop
if ndocs[0] == expected_finds / 2:
insert()
@gen.coroutine
def insert():
for i in range(n_inserts):
yield insert_collection.insert_one({'s': hex(i)})
insert_future.set_result(None) # Finished
yield [find() for _ in range(concurrency)]
yield insert_future
self.assertEqual(expected_finds, ndocs[0])
self.assertEqual(n_inserts, (yield insert_collection.count()))
yield collection.delete_many({})
def test_aggregation_cursor_exc_info(self):
if not (yield at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
if sys.version_info < (3, ):
raise SkipTest("Requires Python 3")
yield self._make_test_data(200)
cursor = self.db.test.aggregate(self.pipeline)
yield cursor.to_list(length=10)
yield self.db.test.drop()
try:
yield cursor.to_list(length=None)
except OperationFailure:
_, _, tb = sys.exc_info()
# The call tree should include PyMongo code we ran on a thread.
formatted = '\n'.join(traceback.format_tb(tb))
self.assertTrue('_unpack_response' in formatted
or '_check_command_response' in formatted)
def test_aggregation_cursor_exc_info(self):
if not (yield at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
if sys.version_info < (3,):
raise SkipTest("Requires Python 3")
yield self._make_test_data(200)
cursor = self.db.test.aggregate(self.pipeline)
yield cursor.to_list(length=10)
yield self.db.test.drop()
try:
yield cursor.to_list(length=None)
except OperationFailure:
_, _, tb = sys.exc_info()
# The call tree should include PyMongo code we ran on a thread.
formatted = '\n'.join(traceback.format_tb(tb))
self.assertTrue('_unpack_response' in formatted
or '_check_command_response' in formatted)
def test_aggregation_cursor_to_list_callback(self):
if not (yield at_least(self.cx, (2, 5, 1))):
raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
expected_sum = yield self._make_test_data(collection_size)
cursor = db.test.aggregate(self.pipeline)
future = Future()
def cb(result, error):
if error:
future.set_exception(error)
else:
future.set_result(result)
cursor.to_list(collection_size, callback=cb)
docs = yield future
self.assertAllDocs(expected_sum, docs)
def test_aggregation_cursor_to_list_callback(self):
if not (yield at_least(self.cx, (2, 5, 1))):
raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
expected_sum = yield self._make_test_data(collection_size)
cursor = db.test.aggregate(self.pipeline)
future = Future()
def cb(result, error):
if error:
future.set_exception(error)
else:
future.set_result(result)
cursor.to_list(collection_size, callback=cb)
docs = yield future
self.assertAllDocs(expected_sum, docs)
