def test_threaded_resolver(self): try: import concurrent.futures except ImportError: raise SkipTest('concurrent.futures module not available') yield self._test_resolver('tornado.netutil.ThreadedResolver')
def test_mongos(self): shard = client_context.client.config.shards.find_one()['host'] num_members = shard.count(',') + 1 if num_members == 1: raise SkipTest("Need a replica set shard to test.") coll = client_context.client.pymongo_test.get_collection( "test", write_concern=WriteConcern(w=num_members)) coll.drop() res = coll.insert_many([{} for _ in range(5)]) first_id = res.inserted_ids[0] last_id = res.inserted_ids[-1] # Note - this isn't a perfect test since there's no way to # tell what shard member a query ran on. for pref in (Primary(), PrimaryPreferred(), Secondary(), SecondaryPreferred(), Nearest()): qcoll = coll.with_options(read_preference=pref) results = list(qcoll.find().sort([("_id", 1)])) self.assertEqual(first_id, results[0]["_id"]) self.assertEqual(last_id, results[-1]["_id"]) results = list(qcoll.find().sort([("_id", -1)])) self.assertEqual(first_id, results[-1]["_id"]) self.assertEqual(last_id, results[0]["_id"])
def test_pickle_backwards_compatability(self): # For a full discussion see http://bugs.python.org/issue6137 if sys.version.startswith('3.0'): raise SkipTest("Python 3.0.x can't unpickle " "objects pickled in Python 2.x.") # This string was generated by pickling an ObjectId in pymongo # version 1.9 pickled_with_1_9 = ( b"ccopy_reg\n_reconstructor\np0\n" b"(cbson.objectid\nObjectId\np1\nc__builtin__\n" b"object\np2\nNtp3\nRp4\n" b"(dp5\nS'_ObjectId__id'\np6\n" b"S'M\\x9afV\\x13v\\xc0\\x0b\\x88\\x00\\x00\\x00'\np7\nsb.") # We also test against a hardcoded "New" pickle format so that we # make sure we're backward compatible with the current version in # the future as well. pickled_with_1_10 = ( b"ccopy_reg\n_reconstructor\np0\n" b"(cbson.objectid\nObjectId\np1\nc__builtin__\n" b"object\np2\nNtp3\nRp4\n" b"S'M\\x9afV\\x13v\\xc0\\x0b\\x88\\x00\\x00\\x00'\np5\nb.") if PY3: # Have to load using 'latin-1' since these were pickled in python2.x. oid_1_9 = pickle.loads(pickled_with_1_9, encoding='latin-1') oid_1_10 = pickle.loads(pickled_with_1_10, encoding='latin-1') else: oid_1_9 = pickle.loads(pickled_with_1_9) oid_1_10 = pickle.loads(pickled_with_1_10) self.assertEqual(oid_1_9, ObjectId("4d9a66561376c00b88000000")) self.assertEqual(oid_1_9, oid_1_10)
def test_kill_cursors_with_tuple(self): if (client_context.is_mongos and not client_context.version.at_least(2, 4, 7)): # Old mongos sends incorrectly formatted error response when # cursor isn't found, see SERVER-9738. raise SkipTest("Can't test kill_cursors against old mongos") self.collection = self.client.pymongo_test.test self.collection.drop() self.collection.insert_many([{'_id': i} for i in range(200)]) cursor = self.collection.find().batch_size(1) next(cursor) self.client.kill_cursors([cursor.cursor_id], self.client.address) # Prevent killcursors from reaching the server while a getmore is in # progress -- the server logs "Assertion: 16089:Cannot kill active # cursor." time.sleep(2) def raises_cursor_not_found(): try: next(cursor) return False except CursorNotFound: return True wait_until(raises_cursor_not_found, 'close cursor')
def test_cert_ssl_validation_optional(self): # Expects the server to be running with server.pem and ca.pem # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # client = MongoClient('localhost', ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_OPTIONAL, ssl_ca_certs=CA_PEM) response = client.admin.command('ismaster') if 'setName' in response: if response['primary'].split(":")[0] != 'localhost': raise SkipTest("No hosts in the replicaset for 'localhost'. " "Cannot validate hostname in the certificate") client = MongoClient('localhost', replicaSet=response['setName'], w=len(response['hosts']), ssl=True, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_OPTIONAL, ssl_ca_certs=CA_PEM) self.assertClientWorks(client)
def test_profiling_info(self): db = self.client.pymongo_test db.system.profile.drop() db.set_profiling_level(ALL) db.test.find_one() db.set_profiling_level(OFF) info = db.profiling_info() self.assertTrue(isinstance(info, list)) # Check if we're going to fail because of SERVER-4754, in which # profiling info isn't collected if mongod was started with --auth if server_started_with_auth(self.client): raise SkipTest( "We need SERVER-4754 fixed for the rest of this test to pass" ) self.assertTrue(len(info) >= 1) # These basically clue us in to server changes. self.assertTrue(isinstance(info[0]['responseLength'], int)) self.assertTrue(isinstance(info[0]['millis'], int)) self.assertTrue(isinstance(info[0]['client'], string_type)) self.assertTrue(isinstance(info[0]['user'], string_type)) self.assertTrue(isinstance(info[0]['ns'], string_type)) self.assertTrue(isinstance(info[0]['op'], string_type)) self.assertTrue(isinstance(info[0]["ts"], datetime.datetime))
async def test_ipv6(self): assert env.host in ( "localhost", "127.0.0.1", ), "This unittest isn't written to test IPv6 with host %s" % repr( env.host) try: connected( MongoClient("[::1]", username=db_user, password=db_password, serverSelectionTimeoutMS=100)) except ConnectionFailure: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest("No IPV6") if test.env.auth: cx_string = "mongodb://%s:%s@[::1]:%d" % (db_user, db_password, env.port) else: cx_string = "mongodb://[::1]:%d" % env.port cx = self.asyncio_client(uri=cx_string) collection = cx.motor_test.test_collection await collection.insert_one({"dummy": "object"}) self.assertTrue((await collection.find_one({"dummy": "object"})))
def test_parallel_scan(self): if not (yield at_least(self.cx, (2, 5, 5))): raise SkipTest("Requires MongoDB >= 2.5.5") yield skip_if_mongos(self.cx) collection = self.collection # Enough documents that each cursor requires multiple batches. yield collection.remove() yield collection.insert(({ '_id': i } for i in range(8000)), w=test.env.w) if test.env.is_replica_set: client = self.motor_rsc() # Test that getMore messages are sent to the right server. client.read_preference = ReadPreference.SECONDARY collection = client.motor_test.test_collection docs = [] @gen.coroutine def f(cursor): self.assertTrue( isinstance(cursor, motor.motor_tornado.MotorCommandCursor)) while (yield cursor.fetch_next): docs.append(cursor.next_object()) cursors = yield collection.parallel_scan(3) yield [f(cursor) for cursor in cursors] self.assertEqual(len(docs), (yield collection.count()))
def test_del_on_child_greenlet(self): if tornado_version < (4, 0, 0, 0): raise SkipTest("Tornado 3") # Since __del__ can happen on any greenlet, cursor must be # prepared to close itself correctly on main or a child. client, server = self.client_server(auto_ismaster=True) self.cursor = client.test.collection.find() future = self.cursor.fetch_next request = yield self.run_thread(server.receives, OpQuery) request.replies({'_id': 1}, cursor_id=123) yield future # Complete the first fetch. def f(): # Last ref, should trigger __del__ immediately in CPython and # allow eventual __del__ in PyPy. del self.cursor greenlet.greenlet(f).switch() # Let the event loop iterate once more to clear its references to # callbacks, allowing the cursor to be freed. yield self.pause(0) if 'PyPy' in sys.version: gc.collect() yield self.run_thread(server.receives, OpKillCursors)
def test_auth_from_uri(self): if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # self.db is logged in as root. yield remove_all_users(self.db) db = self.db try: yield db.add_user('mike', 'password', roles=['userAdmin', 'readWrite']) client = motor.MotorClient('mongodb://*****:*****@%s:%d' % (host, port), io_loop=self.io_loop) # Note: open() only calls ismaster, doesn't throw auth errors. yield client.open() with assert_raises(OperationFailure): yield client.db.collection.find_one() client = motor.MotorClient('mongodb://*****:*****@%s:%d/%s' % (host, port, db.name), io_loop=self.io_loop) yield client[db.name].collection.find_one() finally: yield db.remove_user('mike')
def test_mongodb_x509_auth(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests as well as # --auth: # # --sslPEMKeyFile=jstests/libs/server.pem # --sslCAFile=jstests/libs/ca.pem # --sslCRLFile=jstests/libs/crl.pem # --auth if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") client = motor.MotorClient( host, port, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop) if not (yield version.at_least(client, (2, 5, 3, -1))): raise SkipTest("MONGODB-X509 tests require MongoDB 2.5.3 or newer") if not test.env.auth: raise SkipTest('Authentication is not enabled on server') # Give admin all necessary privileges. yield client['$external'].add_user(MONGODB_X509_USERNAME, roles=[ {'role': 'readWriteAnyDatabase', 'db': 'admin'}, {'role': 'userAdminAnyDatabase', 'db': 'admin'}]) collection = client.motor_test.test with test.assert_raises(OperationFailure): yield collection.count() yield client.admin.authenticate( MONGODB_X509_USERNAME, mechanism='MONGODB-X509') yield collection.remove() uri = ('mongodb://%s@%s:%d/?authMechanism=' 'MONGODB-X509' % ( quote_plus(MONGODB_X509_USERNAME), host, port)) # SSL options aren't supported in the URI.... auth_uri_client = motor.MotorClient( uri, ssl_certfile=CLIENT_PEM, io_loop=self.io_loop) yield auth_uri_client.db.collection.find_one() # Cleanup. yield remove_all_users(client['$external']) yield client['$external'].logout()
def test_must_connect(self): if 'PYMONGO_MUST_CONNECT' not in os.environ: raise SkipTest('PYMONGO_MUST_CONNECT is not set') self.assertTrue( client_context.connected, 'client context must be connected when ' 'PYMONGO_MUST_CONNECT is set. Failed attempts:\n%s' % (client_context.connection_attempt_info(), ))
def test_ordered_dict(self): try: from collections import OrderedDict except ImportError: raise SkipTest("No OrderedDict") d = OrderedDict([("one", 1), ("two", 2), ("three", 3), ("four", 4)]) self.assertEqual( d, decode(encode(d), CodecOptions(document_class=OrderedDict)))
async def test_open_client_rs(self): if not test.env.is_replica_set: raise SkipTest("Not connected to a replica set") rsc = self.asyncio_rsc() with ignore_deprecations(): self.assertEqual(rsc, await rsc.open())
def test_validation_with_system_ca_certs(self): # Expects the server to be running with server.pem and ca.pem. # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslWeakCertificateValidation # # Also requires an /etc/hosts entry where "server" is resolvable if not CERT_SSL: raise SkipTest("No mongod available over SSL with certs") if not SERVER_IS_RESOLVABLE: raise SkipTest("No hosts entry for 'server'. Cannot validate " "hostname in the certificate") if sys.platform == "win32": raise SkipTest("Can't test system ca certs on Windows.") if sys.version_info < (2, 7, 9): raise SkipTest("Can't load system CA certificates.") # Tell OpenSSL where CA certificates live. os.environ['SSL_CERT_FILE'] = CA_PEM try: with self.assertRaises(ConnectionFailure): # Server cert is verified but hostname matching fails connected( MongoClient(pair, ssl=True, serverSelectionTimeoutMS=100)) # Server cert is verified. Disable hostname matching. connected( MongoClient(pair, ssl=True, ssl_match_hostname=False, serverSelectionTimeoutMS=100)) # Server cert and hostname are verified. connected( MongoClient('server', ssl=True, serverSelectionTimeoutMS=100)) # Server cert and hostname are verified. connected( MongoClient( 'mongodb://server/?ssl=true&serverSelectionTimeoutMS=100')) finally: os.environ.pop('SSL_CERT_FILE')
def test_enableTestCommands_is_disabled(self): if 'PYMONGO_DISABLE_TEST_COMMANDS' not in os.environ: raise SkipTest('PYMONGO_DISABLE_TEST_COMMANDS is not set') self.assertFalse( client_context.test_commands_enabled, 'enableTestCommands must be disabled when ' 'PYMONGO_DISABLE_TEST_COMMANDS is set.')
def test_validation_with_system_ca_certs(self): # Expects the server to be running with server.pem and ca.pem. # # --sslPEMKeyFile=/path/to/pymongo/test/certificates/server.pem # --sslCAFile=/path/to/pymongo/test/certificates/ca.pem # --sslWeakCertificateValidation # if sys.platform == "win32": raise SkipTest("Can't test system ca certs on Windows.") if sys.version_info < (2, 7, 9): raise SkipTest("Can't load system CA certificates.") # Tell OpenSSL where CA certificates live. os.environ['SSL_CERT_FILE'] = CA_PEM try: with self.assertRaises(ConnectionFailure): # Server cert is verified but hostname matching fails connected( MongoClient('server', ssl=True, serverSelectionTimeoutMS=100, **self.credentials)) # Server cert is verified. Disable hostname matching. connected( MongoClient('server', ssl=True, ssl_match_hostname=False, serverSelectionTimeoutMS=100, **self.credentials)) # Server cert and hostname are verified. connected( MongoClient('localhost', ssl=True, serverSelectionTimeoutMS=100, **self.credentials)) # Server cert and hostname are verified. connected( MongoClient( 'mongodb://localhost/?ssl=true&serverSelectionTimeoutMS=100', **self.credentials)) finally: os.environ.pop('SSL_CERT_FILE')
def test_fork(self): # Test using a client before and after a fork. if sys.platform == "win32": raise SkipTest("Can't fork on windows") try: import multiprocessing except ImportError: raise SkipTest("No multiprocessing module") db = self.client.pymongo_test # Ensure a socket is opened before the fork. db.test.find_one() def f(pipe): try: kill_cursors_executor = self.client._kill_cursors_executor servers = self.client._topology.select_servers( any_server_selector) # In child, only the thread that called fork() is alive. # The first operation should revive the rest. db.test.find_one() wait_until( lambda: all(s._monitor._executor._thread.is_alive() for s in servers), "restart monitor threads") wait_until(lambda: kill_cursors_executor._thread.is_alive(), "restart kill-cursors executor") except: traceback.print_exc() # Aid debugging. pipe.send(True) parent_pipe, child_pipe = multiprocessing.Pipe() p = multiprocessing.Process(target=f, args=(child_pipe,)) p.start() p.join(10) child_pipe.close() # Pipe will only have data if the child process failed. try: parent_pipe.recv() self.fail() except EOFError: pass
def test_cares_resolver(self): try: import pycares except ImportError: raise SkipTest('pycares not installed') yield self._test_resolver( 'tornado.platform.caresresolver.CaresResolver', False)
def setUpClass(cls): super(MotorChangeStreamTest, cls).setUpClass() if env.is_standalone: raise SkipTest("Standalone") # Ensure the collection exists. env.sync_cx.motor_test.test_collection.delete_many({}) env.sync_cx.motor_test.test_collection.insert_one({'_id': 1})
def test_simple_ssl(self): if test.env.mongod_validates_client_cert: raise SkipTest("mongod validates SSL certs") # Expects the server to be running with ssl and # --sslWeakCertificateValidation. motor_client() adds appropriate # ssl args and auth. if test.env.auth: raise SkipTest("can't test with auth") # Expects the server to be running with ssl and with # no --sslPEMKeyFile or with --sslWeakCertificateValidation. client = motor.MotorClient(host, port, ssl=True, io_loop=self.io_loop) yield client.db.collection.find_one() response = yield client.admin.command('ismaster') if 'setName' in response: client = self.motor_rsc() yield client.db.collection.find_one()
def test_serverless(self): if 'TEST_SERVERLESS' not in os.environ: raise SkipTest('TEST_SERVERLESS is not set') self.assertTrue( client_context.connected and client_context.serverless, 'client context must be connected to serverless when ' 'TEST_SERVERLESS is set. Failed attempts:\n%s' % (client_context.connection_attempt_info(), ))
async def test_cert_ssl(self): if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if test.env.auth: raise SkipTest("can't test with auth") client = motor.MotorClient(env.host, env.port, ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) await client.db.collection.find_one() response = await client.admin.command('ismaster') if 'setName' in response: client = self.motor_rsc(ssl_certfile=CLIENT_PEM, ssl_ca_certs=CA_PEM) await client.db.collection.find_one()
def test_wincertstore(self): if sys.platform != "win32": raise SkipTest("Only valid on Windows.") if hasattr(ssl, "SSLContext"): # SSLSocket doesn't provide ca_certs attribute on pythons # with SSLContext and SSLContext provides no information # about ca_certs. raise SkipTest("Can't test when SSLContext available.") if not ssl_support.HAVE_WINCERTSTORE: raise SkipTest("Need wincertstore to test wincertstore.") ctx = get_ssl_context(None, None, None, CA_PEM, ssl.CERT_REQUIRED, None) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, CA_PEM) ctx = get_ssl_context(None, None, None, None, None, None) ssl_sock = ctx.wrap_socket(socket.socket()) self.assertEqual(ssl_sock.ca_certs, ssl_support._WINCERTS.name)
def test_comment(self): if server_started_with_auth(self.db.client): raise SkipTest("SERVER-4754 - This test uses profiling.") def run_with_profiling(func): self.db.set_profiling_level(OFF) self.db.system.profile.drop() self.db.set_profiling_level(ALL) func() self.db.set_profiling_level(OFF) def find(): list(self.db.test.find().comment('foo')) op = self.db.system.profile.find({ 'ns': 'pymongo_test.test', 'op': 'query', 'query.$comment': 'foo' }) self.assertEqual(op.count(), 1) run_with_profiling(find) # MongoDB 3.1.5 changed the ns for commands. regex = {'$regex': 'pymongo_test.(\$cmd|test)'} def count(): self.db.test.find().comment('foo').count() op = self.db.system.profile.find({ 'ns': regex, 'op': 'command', 'command.count': 'test', 'command.$comment': 'foo' }) self.assertEqual(op.count(), 1) run_with_profiling(count) def distinct(): self.db.test.find().comment('foo').distinct('type') op = self.db.system.profile.find({ 'ns': regex, 'op': 'command', 'command.distinct': 'test', 'command.$comment': 'foo' }) self.assertEqual(op.count(), 1) run_with_profiling(distinct) self.db.test.insert_many([{}, {}]) cursor = self.db.test.find() next(cursor) self.assertRaises(InvalidOperation, cursor.comment, 'hello') self.db.system.profile.drop()
def test_cert_ssl_validation(self): # Expects the server to be running with the server.pem, ca.pem # and crl.pem provided in mongodb and the server tests e.g.: # # --sslPEMKeyFile=jstests/libs/server.pem # --sslCAFile=jstests/libs/ca.pem # --sslCRLFile=jstests/libs/crl.pem # # Also requires an /etc/hosts entry where "server" is resolvable. if not test.env.mongod_validates_client_cert: raise SkipTest("No mongod available over SSL with certs") if not test.env.server_is_resolvable: raise SkipTest("No hosts entry for 'server'. Cannot validate " "hostname in the certificate") if test.env.auth: raise SkipTest("can't test with auth") client = motor.MotorClient(test.env.fake_hostname_uri, ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield client.db.collection.find_one() response = yield client.admin.command('ismaster') if 'setName' in response: if response['primary'].split(":")[0] != 'server': raise SkipTest("No hosts in the replicaset for 'server'. " "Cannot validate hostname in the certificate") client = motor.MotorReplicaSetClient( test.env.fake_hostname_uri, replicaSet=response['setName'], ssl_certfile=CLIENT_PEM, ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_ca_certs=CA_PEM, io_loop=self.io_loop) yield client.db.collection.find_one()
def test_gssapi_threaded(self): client = MongoClient(GSSAPI_HOST, GSSAPI_PORT, username=GSSAPI_PRINCIPAL, password=GSSAPI_PASS, authMechanism='GSSAPI', authMechanismProperties=self.mech_properties) # Authentication succeeded? client.server_info() db = client[GSSAPI_DB] # Need one document in the collection. AutoAuthenticateThread does # collection.find_one with a 1-second delay, forcing it to check out # multiple sockets from the pool concurrently, proving that # auto-authentication works with GSSAPI. collection = db.test if not collection.count_documents({}): try: collection.drop() collection.insert_one({'_id': 1}) except OperationFailure: raise SkipTest("User must be able to write.") threads = [] for _ in range(4): threads.append(AutoAuthenticateThread(collection)) for thread in threads: thread.start() for thread in threads: thread.join() self.assertTrue(thread.success) set_name = client.admin.command('ismaster').get('setName') if set_name: client = MongoClient(GSSAPI_HOST, GSSAPI_PORT, username=GSSAPI_PRINCIPAL, password=GSSAPI_PASS, authMechanism='GSSAPI', authMechanismProperties=self.mech_properties, replicaSet=set_name) # Succeeded? client.server_info() threads = [] for _ in range(4): threads.append(AutoAuthenticateThread(collection)) for thread in threads: thread.start() for thread in threads: thread.join() self.assertTrue(thread.success)
def test_fetch_next_delete(self): if sys.version_info < (3, 4): raise SkipTest("requires Python 3.4") if 'PyPy' in sys.version: raise SkipTest('PyPy') client, server = self.client_server(auto_ismaster={'ismaster': True}) cursor = client.test.collection.find() # With Tornado, simply accessing fetch_next starts the fetch. cursor.fetch_next request = yield self.run_thread(server.receives, OpQuery) request.replies({'_id': 1}, cursor_id=123) # Decref'ing the cursor eventually closes it on the server. del cursor # Clear Runner's reference. This is Tornado 3 substitute for gen.moment. yield gen.Task(self.io_loop.add_callback) yield self.run_thread(server.receives, OpKillCursors(cursor_ids=[123]))
def test_actionable_error_message(self): if client_context.storage_engine != 'mmapv1': raise SkipTest('This cluster is not running MMAPv1') expected_msg = ("This MongoDB deployment does not support retryable " "writes. Please add retryWrites=false to your " "connection string.") for method, args, kwargs in retryable_single_statement_ops( self.db.retryable_write_test): with self.assertRaisesRegex(OperationFailure, expected_msg): method(*args, **kwargs)
def test_uuid_subtype(self): if pymongo.version_tuple < (2, 9, 4): raise SkipTest("PYTHON-1145") cx = self.motor_rsc(uuidRepresentation='javaLegacy') with ignore_deprecations(): self.assertEqual(cx.uuid_subtype, JAVA_LEGACY) cx.uuid_subtype = UUID_SUBTYPE self.assertEqual(cx.uuid_subtype, UUID_SUBTYPE) self.assertEqual(cx.delegate.uuid_subtype, UUID_SUBTYPE)