def test_parallel_scan(self):
if not (yield from at_least(self.cx, (2, 5, 5))):
raise SkipTest("Requires MongoDB >= 2.5.5")
yield from skip_if_mongos(self.cx)
collection = self.collection
# Enough documents that each cursor requires multiple batches.
yield from collection.remove()
yield from collection.insert(({'_id': i} for i in range(8000)),
w=test.env.w)
if test.env.is_replica_set:
client = self.asyncio_rsc()
# Test that getMore messages are sent to the right server.
client.read_preference = ReadPreference.SECONDARY
collection = client.motor_test.test_collection
docs = []
@asyncio.coroutine
def f(cursor):
self.assertTrue(isinstance(cursor, AsyncIOMotorCommandCursor))
while (yield from cursor.fetch_next):
docs.append(cursor.next_object())
cursors = yield from collection.parallel_scan(3)
yield from asyncio.wait(
[f(cursor) for cursor in cursors],
loop=self.loop)
self.assertEqual(len(docs), (yield from collection.count()))
def test_parallel_scan(self):
if not (yield from at_least(self.cx, (2, 5, 5))):
raise SkipTest("Requires MongoDB >= 2.5.5")
yield from skip_if_mongos(self.cx)
collection = self.collection
# Enough documents that each cursor requires multiple batches.
yield from collection.remove()
yield from collection.insert(({'_id': i} for i in range(8000)),
w=test.env.w)
if test.env.is_replica_set:
client = self.asyncio_rsc()
# Test that getMore messages are sent to the right server.
client.read_preference = ReadPreference.SECONDARY
collection = client.motor_test.test_collection
docs = []
@asyncio.coroutine
def f(cursor):
self.assertTrue(isinstance(cursor, AsyncIOMotorCommandCursor))
while (yield from cursor.fetch_next):
docs.append(cursor.next_object())
cursors = yield from collection.parallel_scan(3)
yield from asyncio.wait(
[f(cursor) for cursor in cursors],
loop=self.loop)
self.assertEqual(len(docs), (yield from collection.count()))
def test_authenticate(self):
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# self.db is logged in as root.
yield from self.db.add_user("jesse", "password")
db = AsyncIOMotorClient(host, port, io_loop=self.loop).motor_test
try:
# Authenticate many times at once to test concurrency.
yield from asyncio.wait(
[db.authenticate("jesse", "password") for _ in range(10)],
loop=self.loop)
# Just make sure there are no exceptions here.
yield from db.remove_user("jesse")
yield from db.logout()
if (yield from at_least(self.cx, (2, 5, 4))):
info = yield from self.db.command("usersInfo", "jesse")
users = info.get('users', [])
else:
users = yield from self.db.system.users.find().to_list(10)
self.assertFalse("jesse" in [u['user'] for u in users])
finally:
yield from remove_all_users(self.db)
test.env.sync_cx.disconnect()
def test_find_is_async(self):
# Need parallel Javascript.
if not (yield from at_least(self.cx, (3, ))):
raise SkipTest("Requires MongoDB >= 3.0")
# Confirm find() is async by launching two operations which will finish
# out of order. Also test that AsyncIOMotorClient doesn't reuse sockets
# incorrectly.
# Launch find operations for _id's 1 and 2 which will finish in order
# 2, then 1.
coll = self.collection
yield from coll.insert_many([{'_id': 1}, {'_id': 2}])
results = []
futures = [
asyncio.Future(loop=self.loop),
asyncio.Future(loop=self.loop)
]
def callback(result, error):
if result:
results.append(result)
futures.pop().set_result(None)
# This find() takes 0.5 seconds.
coll.find({'_id': 1, '$where': delay(0.5)}).limit(1).each(callback)
# Very fast lookup.
coll.find({'_id': 2}).limit(1).each(callback)
yield from asyncio.gather(*futures, loop=self.loop)
# Results were appended in order 2, 1.
self.assertEqual([{'_id': 2}, {'_id': 1}], results)
def test_authenticate(self):
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# self.db is logged in as root.
yield from self.db.add_user("jesse", "password")
db = AsyncIOMotorClient(host, port, io_loop=self.loop).motor_test
try:
# Authenticate many times at once to test concurrency.
yield from asyncio.wait(
[db.authenticate("jesse", "password") for _ in range(10)],
loop=self.loop)
# Just make sure there are no exceptions here.
yield from db.remove_user("jesse")
yield from db.logout()
if (yield from at_least(self.cx, (2, 5, 4))):
info = yield from self.db.command("usersInfo", "jesse")
users = info.get('users', [])
else:
users = yield from self.db.system.users.find().to_list(10)
self.assertFalse("jesse" in [u['user'] for u in users])
finally:
yield from remove_all_users(self.db)
test.env.sync_cx.disconnect()
def test_find_is_async(self):
# Need parallel Javascript.
if not (yield from at_least(self.cx, (3,))):
raise SkipTest("Requires MongoDB >= 3.0")
# Confirm find() is async by launching two operations which will finish
# out of order. Also test that AsyncIOMotorClient doesn't reuse sockets
# incorrectly.
# Launch find operations for _id's 1 and 2 which will finish in order
# 2, then 1.
coll = self.collection
yield from coll.insert_many([{'_id': 1}, {'_id': 2}])
results = []
futures = [asyncio.Future(loop=self.loop),
asyncio.Future(loop=self.loop)]
def callback(result, error):
if result:
results.append(result)
futures.pop().set_result(None)
# This find() takes 0.5 seconds.
coll.find({'_id': 1, '$where': delay(0.5)}).limit(1).each(callback)
# Very fast lookup.
coll.find({'_id': 2}).limit(1).each(callback)
yield from asyncio.gather(*futures, loop=self.loop)
# Results were appended in order 2, 1.
self.assertEqual([{'_id': 2}, {'_id': 1}], results)
def test_mongodb_x509_auth(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests as well as
# --auth:
#
# --sslPEMKeyFile=jstests/libs/server.pem
# --sslCAFile=jstests/libs/ca.pem
# --sslCRLFile=jstests/libs/crl.pem
# --auth
if not test.env.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
client = AsyncIOMotorClient(test.env.uri,
ssl_certfile=CLIENT_PEM,
io_loop=self.loop)
if not (yield from at_least(client, (2, 5, 3, -1))):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# Give admin all necessary privileges.
yield from client['$external'].add_user(MONGODB_X509_USERNAME,
roles=[{
'role':
'readWriteAnyDatabase',
'db': 'admin'
}, {
'role':
'userAdminAnyDatabase',
'db': 'admin'
}])
collection = client.motor_test.test
with self.assertRaises(OperationFailure):
yield from collection.count()
yield from client.admin.authenticate(MONGODB_X509_USERNAME,
mechanism='MONGODB-X509')
yield from collection.remove()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' %
(quote_plus(MONGODB_X509_USERNAME), host, port))
# SSL options aren't supported in the URI....
auth_uri_client = AsyncIOMotorClient(uri,
ssl_certfile=CLIENT_PEM,
io_loop=self.loop)
yield from auth_uri_client.db.collection.find_one()
# Cleanup.
yield from remove_all_users(client['$external'])
yield from client['$external'].logout()
def maybe_skip(self):
if (yield from server_is_mongos(self.cx)):
raise SkipTest("mongos has no maxTimeAlwaysTimeOut fail point")
if not (yield from at_least(self.cx, (2, 5, 3, -1))):
raise SkipTest("maxTimeMS requires MongoDB >= 2.5.3")
cmdline = yield from get_command_line(self.cx)
if '1' != safe_get(cmdline, 'parsed.setParameter.enableTestCommands'):
if 'enableTestCommands=1' not in cmdline['argv']:
raise SkipTest("testing maxTimeMS requires failpoints")
def maybe_skip(self):
if (yield from server_is_mongos(self.cx)):
raise SkipTest("mongos has no maxTimeAlwaysTimeOut fail point")
if not (yield from at_least(self.cx, (2, 5, 3, -1))):
raise SkipTest("maxTimeMS requires MongoDB >= 2.5.3")
cmdline = yield from get_command_line(self.cx)
if '1' != safe_get(cmdline, 'parsed.setParameter.enableTestCommands'):
if 'enableTestCommands=1' not in cmdline['argv']:
raise SkipTest("testing maxTimeMS requires failpoints")
def test_mongodb_x509_auth(self):
# Expects the server to be running with SSL config described above,
# and with "--auth".
if not test.env.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
# self.env.uri includes username and password.
authenticated_client = AsyncIOMotorClient(test.env.uri,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.loop)
if not (yield from at_least(authenticated_client, (2, 5, 3, -1))):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# Give admin all necessary privileges.
yield from authenticated_client['$external'].add_user(
MONGODB_X509_USERNAME, roles=[
{'role': 'readWriteAnyDatabase', 'db': 'admin'},
{'role': 'userAdminAnyDatabase', 'db': 'admin'}])
# Not authenticated.
client = AsyncIOMotorClient(env.host, env.port,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.loop)
collection = client.motor_test.test
with self.assertRaises(OperationFailure):
yield from collection.count()
yield from client.admin.authenticate(
MONGODB_X509_USERNAME, mechanism='MONGODB-X509')
yield from collection.delete_many({})
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' % (
quote_plus(MONGODB_X509_USERNAME), env.host, env.port))
# SSL options aren't supported in the URI....
auth_uri_client = AsyncIOMotorClient(uri,
ssl_certfile=CLIENT_PEM,
ssl_ca_certs=CA_PEM,
io_loop=self.loop)
yield from auth_uri_client.db.collection.find_one()
# Cleanup.
yield from remove_all_users(authenticated_client['$external'])
yield from authenticated_client['$external'].logout()
def test_aggregation_cursor(self):
if not (yield from at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
expected_sum = yield from self._make_test_data(collection_size)
cursor = db.test.aggregate(self.pipeline)
docs = yield from cursor.to_list(collection_size)
self.assertEqual(expected_sum, sum(doc['_id'] for doc in docs))
def test_aggregation_cursor(self):
if not (yield from at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
expected_sum = yield from self._make_test_data(collection_size)
cursor = db.test.aggregate(self.pipeline)
docs = yield from cursor.to_list(collection_size)
self.assertEqual(expected_sum,
sum(doc['_id'] for doc in docs))
def test_mongodb_x509_auth(self):
# Expects the server to be running with the server.pem, ca.pem
# and crl.pem provided in mongodb and the server tests as well as
# --auth:
#
# --sslPEMKeyFile=jstests/libs/server.pem
# --sslCAFile=jstests/libs/ca.pem
# --sslCRLFile=jstests/libs/crl.pem
# --auth
if not test.env.mongod_validates_client_cert:
raise SkipTest("No mongod available over SSL with certs")
client = AsyncIOMotorClient(test.env.uri,
ssl_certfile=CLIENT_PEM,
io_loop=self.loop)
if not (yield from at_least(client, (2, 5, 3, -1))):
raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer")
if not test.env.auth:
raise SkipTest('Authentication is not enabled on server')
# Give admin all necessary privileges.
yield from client['$external'].add_user(MONGODB_X509_USERNAME, roles=[
{'role': 'readWriteAnyDatabase', 'db': 'admin'},
{'role': 'userAdminAnyDatabase', 'db': 'admin'}])
collection = client.motor_test.test
with test.assert_raises(OperationFailure):
yield from collection.count()
yield from client.admin.authenticate(
MONGODB_X509_USERNAME, mechanism='MONGODB-X509')
yield from collection.remove()
uri = ('mongodb://%s@%s:%d/?authMechanism='
'MONGODB-X509' % (
quote_plus(MONGODB_X509_USERNAME), host, port))
# SSL options aren't supported in the URI....
auth_uri_client = AsyncIOMotorClient(uri,
ssl_certfile=CLIENT_PEM,
io_loop=self.loop)
yield from auth_uri_client.db.collection.find_one()
# Cleanup.
yield from remove_all_users(client['$external'])
yield from client['$external'].logout()
def test_aggregation_cursor_exc_info(self):
if not (yield from at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
yield from self._make_test_data(200)
cursor = self.db.test.aggregate(self.pipeline)
yield from cursor.to_list(length=10)
yield from self.db.test.drop()
try:
yield from cursor.to_list(length=None)
except OperationFailure:
_, _, tb = sys.exc_info()
# The call tree should include PyMongo code we ran on a thread.
formatted = '\n'.join(traceback.format_tb(tb))
self.assertTrue('_unpack_response' in formatted
or '_check_command_response' in formatted)
def test_aggregation_cursor_exc_info(self):
if not (yield from at_least(self.cx, (2, 6))):
raise SkipTest("Requires MongoDB >= 2.6")
yield from self._make_test_data(200)
cursor = self.db.test.aggregate(self.pipeline)
yield from cursor.to_list(length=10)
yield from self.db.test.drop()
try:
yield from cursor.to_list(length=None)
except OperationFailure:
_, _, tb = sys.exc_info()
# The call tree should include PyMongo code we ran on a thread.
formatted = '\n'.join(traceback.format_tb(tb))
self.assertTrue('_unpack_response' in formatted
or '_check_command_response' in formatted)
def test_high_concurrency(self):
if env.mongod_started_with_ssl:
if not (yield from at_least(self.cx, (2, 6))):
raise SkipTest("Concurrent SSL is unreliable in 2.4")
yield from self.make_test_data()
concurrency = 25
cx = self.asyncio_client(maxPoolSize=concurrency)
expected_finds = 200 * concurrency
n_inserts = 25
collection = cx.motor_test.test_collection
insert_collection = cx.motor_test.insert_collection
yield from insert_collection.delete_many({})
ndocs = 0
insert_future = asyncio.Future(loop=self.loop)
@asyncio.coroutine
def find():
nonlocal ndocs
cursor = collection.find()
while (yield from cursor.fetch_next):
cursor.next_object()
ndocs += 1
# Half-way through, start an insert loop
if ndocs == expected_finds / 2:
asyncio.Task(insert(), loop=self.loop)
@asyncio.coroutine
def insert():
for i in range(n_inserts):
yield from insert_collection.insert_one({'s': hex(i)})
insert_future.set_result(None) # Finished
yield from asyncio.gather(*[find() for _ in range(concurrency)],
loop=self.loop)
yield from insert_future
self.assertEqual(expected_finds, ndocs)
self.assertEqual(n_inserts, (yield from insert_collection.count()))
yield from collection.delete_many({})
def test_aggregation_cursor(self):
if not (yield from at_least(self.cx, (2, 5, 1))):
raise SkipTest("Aggregation cursor requires MongoDB >= 2.5.1")
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
yield from db.drop_collection("test")
yield from db.test.insert([{'_id': i} for i in range(collection_size)])
expected_sum = sum(range(collection_size))
cursor = yield from db.test.aggregate(
{'$project': {'_id': '$_id'}}, cursor={})
docs = yield from cursor.to_list(collection_size)
self.assertEqual(
expected_sum,
sum(doc['_id'] for doc in docs))
def test_aggregation_cursor(self):
mongo_2_5_1 = yield from at_least(self.cx, (2, 5, 1))
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
yield from db.drop_collection("test")
yield from db.test.insert([{'_id': i} for i in
range(collection_size)])
pipeline = {'$project': {'_id': '$_id'}}
expected_sum = sum(range(collection_size))
reply = yield from db.test.aggregate(pipeline, cursor=False)
self.assertEqual(expected_sum,
sum(doc['_id'] for doc in reply['result']))
if mongo_2_5_1:
cursor = db.test.aggregate(pipeline)
docs = yield from cursor.to_list(collection_size)
self.assertEqual(expected_sum,
sum(doc['_id'] for doc in docs))
def test_aggregation_cursor(self):
mongo_2_5_1 = yield from at_least(self.cx, (2, 5, 1))
db = self.db
# A small collection which returns only an initial batch,
# and a larger one that requires a getMore.
for collection_size in (10, 1000):
yield from db.drop_collection("test")
yield from db.test.insert([{'_id': i} for i in
range(collection_size)])
pipeline = {'$project': {'_id': '$_id'}}
expected_sum = sum(range(collection_size))
reply = yield from db.test.aggregate(pipeline, cursor=False)
self.assertEqual(expected_sum,
sum(doc['_id'] for doc in reply['result']))
if mongo_2_5_1:
cursor = db.test.aggregate(pipeline)
docs = yield from cursor.to_list(collection_size)
self.assertEqual(expected_sum,
sum(doc['_id'] for doc in docs))
def maybe_skip(self):
if not (yield from at_least(self.cx, (2, 5, 3, -1))):
raise SkipTest("maxTimeMS requires MongoDB >= 2.5.3")
cmd_line = yield from get_command_line(self.cx)
if "enableTestCommands=1" not in cmd_line:
raise SkipTest("testing maxTimeMS requires failpoints")
def maybe_skip(self):
if not (yield from at_least(self.cx, (2, 5, 3, -1))):
raise SkipTest("maxTimeMS requires MongoDB >= 2.5.3")
cmd_line = yield from get_command_line(self.cx)
if "enableTestCommands=1" not in cmd_line:
raise SkipTest("testing maxTimeMS requires failpoints")