def setUp(self): raise SkipTest("master-slave setups are unsupported in TokuMX") self.master = MongoClient(host, port) self.slaves = [] try: self.slaves.append( MongoClient(host2, port2, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append( MongoClient(host3, port3, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest("Not connected to master-slave set") self.client = MasterSlaveConnection(self.master, self.slaves) self.db = self.client.pymongo_test
def test_document_class(self): c = MasterSlaveConnection(self.master, self.slaves) db = c.pymongo_test w = 1 + len(self.slaves) db.test.insert({"x": 1}, w=w) self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = MasterSlaveConnection(self.master, self.slaves, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON))
def setUp(self): self.master = MongoClient(host, port) self.slaves = [] try: self.slaves.append( MongoClient(host2, port2, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append( MongoClient(host3, port3, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest("Not connected to master-slave set") self.ctx = catch_warnings() warnings.simplefilter("ignore", DeprecationWarning) self.client = MasterSlaveConnection(self.master, self.slaves) self.db = self.client.pymongo_test
def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append( Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append( Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest("Not connected to master-slave set") self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test
def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append( Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), slave_okay=True)) except ConnectionFailure: pass try: self.slaves.append( Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), slave_okay=True)) except ConnectionFailure: pass if not self.slaves: raise SkipTest() self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test
def _master_slave_connection(master, port, slaves): ''' Master slave connection. Read from secondary. ''' m_con = Connection(master, port) s_con = [] for slave in slaves.split(','): items = slave.split(':') host = items[0] port = int(items[1]) or 27017 if len(items) > 1 else 27017 s_con.append(Connection(host, port)) con = MasterSlaveConnection(m_con, s_con) con.read_prefrence = ReadPreference.SECONDARY return con
def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: slave = Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), slave_okay=True) # If slave_okay is True and we've only provided one host we # don't check if that host is reachable while initializing # the connection object. Check if the host is reachable here. if slave.admin.command('ping'): self.slaves.append(slave) except ConnectionFailure: pass try: slave = Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), slave_okay=True) if slave.admin.command('ping'): self.slaves.append(slave) except ConnectionFailure: pass if not self.slaves: raise SkipTest() self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test
def setUp(self): # For TestRequestMixin: self.host = host self.port = port self.master = Connection(host, port) self.slaves = [] try: self.slaves.append(Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append(Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest("Not connected to master-slave set") self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test
def master_slave_factory(master, slaves): master = _to_connection(master) slaves = map(_to_connection, slaves) return MasterSlaveConnection(master, slaves, document_class=document_class, tz_aware=tz_aware)
def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append(Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), slave_okay=True)) except ConnectionFailure: pass try: self.slaves.append(Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), slave_okay=True)) except ConnectionFailure: pass if not self.slaves: raise SkipTest() self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test
def __init__(self, master, slaves=[]): """ The MasterSlaveConnection is a wrapper around the pymongo.master_slave_connection implementation. The constructor accepts the connection parameter for the master MongoDB server and a non-empty list of connection parameters for one or more slaves. The connection parameters are expressed as a dictionary where the keys match the signature of the constructor of a standard pymongo.connection.Connection instance ('host', 'port' etc.). For the 'slaves' it is not necessary to specify the 'slave_okay' parameter (will be added internally automatically). The purpose of the MasterSlaveConnection is to hide a master-slave setup with one master and several slave servers. The slave server(s) will be used for read and write will be made to the master (and re-synced to the slave automatically as part of the master-slave setup). """ # Run both inits. MongoKitConnection specific one. Then the Pymongo one at the end MongoKitConnection.__init__(self) # I am the master if not isinstance(master, dict): raise TypeError( '"master" must be a dict containing pymongo.Connection parameters' ) master_connection = PymongoConnection(**master) # You are my dirty slaves if not slaves: raise ValueError('You must specify at least one slave connection') slave_connections = list() for slave in slaves: if not isinstance(slave, dict): raise TypeError( '"slaves" must be list of dicts containing pymongo.Connection parameters' ) slave['slave_okay'] = True slave_connections.append(PymongoConnection(**slave)) # Specifying that it should use the pymongo init PymongoMasterSlaveConnection.__init__(self, master_connection, slave_connections)
def test_tz_aware(self): dt = datetime.datetime.utcnow() conn = MasterSlaveConnection(self.master, self.slaves) self.assertEqual(False, conn.tz_aware) db = conn.pymongo_test w = 1 + len(self.slaves) db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo) conn = MasterSlaveConnection(self.master, self.slaves, tz_aware=True) self.assertEqual(True, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(utc, db.tztest.find_one()['dt'].tzinfo) conn = MasterSlaveConnection(self.master, self.slaves, tz_aware=False) self.assertEqual(False, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo)
def test_use_greenlets(self): self.assertFalse(self.client.use_greenlets) if thread_util.have_gevent: master = MongoClient(host, port, use_greenlets=True) slaves = [ MongoClient(slave.host, slave.port, use_greenlets=True) for slave in self.slaves ] self.assertTrue( MasterSlaveConnection(master, slaves).use_greenlets)
def __init__(self, master, slaves=[]): """ The MasterSlaveConnection is a wrapper around the pymongo.master_slave_connection implementation. The constructor accepts the connection parameter for the master MongoDB server and a non-empty list of connection parameters for one or more slaves. The connection parameters are expressed as a dictionary where the keys match the signature of the constructor of a standard pymongo.connection.Connection instance ('host', 'port' etc.). For the 'slaves' it is not necessary to specify the 'slave_okay' parameter (will be added internally automatically). The purpose of the MasterSlaveConnection is to hide a master-slave setup with one master and several slave servers. The slave server(s) will be used for read and write will be made to the master (and re-synced to the slave automatically as part of the master-slave setup). """ # Run both inits. MongoKitConnection specific one. Then the Pymongo one at the end MongoKitConnection.__init__(self) # I am the master if not isinstance(master, dict): raise TypeError('"master" must be a dict containing pymongo.Connection parameters') master_connection = PymongoConnection(**master) # You are my dirty slaves if not slaves: raise ValueError('You must specify at least one slave connection') slave_connections = list() for slave in slaves: if not isinstance(slave, dict): raise TypeError('"slaves" must be list of dicts containing pymongo.Connection parameters') slave['slave_okay'] = True slave_connections.append(PymongoConnection(**slave)) # Specifying that it should use the pymongo init PymongoMasterSlaveConnection.__init__(self, master_connection, slave_connections)
def setUp(self): self.master = MongoClient(host, port) self.slaves = [] try: self.slaves.append(MongoClient(host2, port2, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append(MongoClient(host3, port3, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest("Not connected to master-slave set") self.client = MasterSlaveConnection(self.master, self.slaves) self.db = self.client.pymongo_test
class TestMasterSlaveConnection(unittest.TestCase): def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append( Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), slave_okay=True)) except ConnectionFailure: pass try: self.slaves.append( Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), slave_okay=True)) except ConnectionFailure: pass if not self.slaves: raise SkipTest() self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test def test_types(self): self.assertRaises(TypeError, MasterSlaveConnection, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, [1]) def test_repr(self): self.assertEqual( repr(self.connection), "MasterSlaveConnection(%r, %r)" % (self.master, self.slaves)) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.connection, "") self.assertRaises(InvalidName, make_db, self.connection, "te$t") self.assertRaises(InvalidName, make_db, self.connection, "te.t") self.assertRaises(InvalidName, make_db, self.connection, "te\\t") self.assertRaises(InvalidName, make_db, self.connection, "te/t") self.assertRaises(InvalidName, make_db, self.connection, "te st") self.assert_(isinstance(self.connection.test, Database)) self.assertEqual(self.connection.test, self.connection["test"]) self.assertEqual(self.connection.test, Database(self.connection, "test")) def test_database_names(self): self.connection.pymongo_test.test.save({"dummy": u"object"}) self.connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.connection.drop_database, 5) self.assertRaises(TypeError, self.connection.drop_database, None) self.connection.pymongo_test.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.connection.drop_database("pymongo_test") dbs = self.connection.database_names() self.assert_("pymongo_test" not in dbs) self.connection.pymongo_test.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.connection.drop_database(self.connection.pymongo_test) dbs = self.connection.database_names() self.assert_("pymongo_test" not in dbs) def test_iteration(self): def iterate(): [a for a in self.connection] self.assertRaises(TypeError, iterate) def test_insert_find_one_in_request(self): count = 0 for i in range(100): self.connection.start_request() self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.connection.end_request() self.failIf(count) def test_insert_find_one_no_slaves(self): if self.slaves: raise SkipTest() count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.failIf(count) # This was failing because commands were being sent to the slaves def test_create_collection(self): self.connection.drop_database('pymongo_test') collection = self.db.create_collection('test') self.assert_(isinstance(collection, Collection)) self.assertRaises(CollectionInvalid, self.db.create_collection, 'test') # Believe this was failing for the same reason... def test_unique_index(self): self.connection.drop_database('pymongo_test') self.db.test.create_index('username', unique=True) self.db.test.save({'username': '******'}, safe=True) self.assertRaises(OperationFailure, self.db.test.save, {'username': '******'}, safe=True) # NOTE this test is non-deterministic, but I expect # some failures unless the db is pulling instantaneously... def test_insert_find_one_with_slaves(self): if not self.slaves: raise SkipTest() count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assert_(count) # NOTE this test is non-deterministic, but hopefully we pause long enough # for the slaves to pull... def test_insert_find_one_with_pause(self): if not self.slaves: raise SkipTest() count = 0 self.db.test.remove({}) self.db.test.insert({"x": 5586}) time.sleep(7) for _ in range(10): try: if 5586 != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.failIf(count) def test_kill_cursors(self): def cursor_count(): count = 0 res = self.connection.master.test_pymongo._command( {"cursorInfo": 1}) count += res["clientCursors_size"] for slave in self.connection.slaves: res = slave.test_pymongo._command({"cursorInfo": 1}) count += res["clientCursors_size"] return count self.connection.test_pymongo.drop_collection("test") db = self.db before = cursor_count() for i in range(10000): db.test.insert({"i": i}) time.sleep(6) # need to sleep to be sure this gets pulled... self.assertEqual(before, cursor_count()) for _ in range(10): db.test.find_one() self.assertEqual(before, cursor_count()) for _ in range(10): for x in db.test.find(): break self.assertEqual(before, cursor_count()) a = db.test.find() for x in a: break self.assertNotEqual(before, cursor_count()) del a self.assertEqual(before, cursor_count()) a = db.test.find().limit(10) for x in a: break self.assertEqual(before, cursor_count())
class TestMasterSlaveConnection(unittest.TestCase): def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append( Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), slave_okay=True)) except ConnectionFailure: pass try: self.slaves.append( Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), slave_okay=True)) except ConnectionFailure: pass if not self.slaves: raise SkipTest() self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test def test_types(self): self.assertRaises(TypeError, MasterSlaveConnection, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, [1]) def test_repr(self): self.assertEqual( repr(self.connection), "MasterSlaveConnection(%r, %r)" % (self.master, self.slaves)) def test_disconnect(self): class Connection(object): def __init__(self): self._disconnects = 0 def disconnect(self): self._disconnects += 1 self.connection._MasterSlaveConnection__master = Connection() self.connection._MasterSlaveConnection__slaves = [ Connection(), Connection() ] self.connection.disconnect() self.assertEquals( 1, self.connection._MasterSlaveConnection__master._disconnects) self.assertEquals( 1, self.connection._MasterSlaveConnection__slaves[0]._disconnects) self.assertEquals( 1, self.connection._MasterSlaveConnection__slaves[1]._disconnects) def test_continue_until_slave_works(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): last_idx = -1 def __init__(self): self._items = [ Slave(True), Slave(True), Slave(False), Slave(True) ] def __len__(self): return len(self._items) def __getitem__(self, idx): NotRandomList.last_idx = idx return self._items.pop(0) self.connection._MasterSlaveConnection__slaves = NotRandomList() response = self.connection._send_message_with_response('message') self.assertEquals((NotRandomList.last_idx, 'sent'), response) self.assertNotEquals(-1, NotRandomList.last_idx) self.assertEquals(3, Slave.calls) def test_raise_autoreconnect_if_all_slaves_fail(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): def __init__(self): self._items = [ Slave(True), Slave(True), Slave(True), Slave(True) ] def __len__(self): return len(self._items) def __getitem__(self, idx): return self._items.pop(0) self.connection._MasterSlaveConnection__slaves = NotRandomList() self.assertRaises(AutoReconnect, self.connection._send_message_with_response, 'message') self.assertEquals(4, Slave.calls) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.connection, "") self.assertRaises(InvalidName, make_db, self.connection, "te$t") self.assertRaises(InvalidName, make_db, self.connection, "te.t") self.assertRaises(InvalidName, make_db, self.connection, "te\\t") self.assertRaises(InvalidName, make_db, self.connection, "te/t") self.assertRaises(InvalidName, make_db, self.connection, "te st") self.assert_(isinstance(self.connection.test, Database)) self.assertEqual(self.connection.test, self.connection["test"]) self.assertEqual(self.connection.test, Database(self.connection, "test")) def test_database_names(self): self.connection.pymongo_test.test.save({"dummy": u"object"}) self.connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.connection.drop_database, 5) self.assertRaises(TypeError, self.connection.drop_database, None) self.connection.pymongo_test.test.save({"dummy": u"object"}, safe=True) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.connection.drop_database("pymongo_test") dbs = self.connection.database_names() self.assert_("pymongo_test" not in dbs) self.connection.pymongo_test.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.connection.drop_database(self.connection.pymongo_test) dbs = self.connection.database_names() self.assert_("pymongo_test" not in dbs) def test_iteration(self): def iterate(): [a for a in self.connection] self.assertRaises(TypeError, iterate) def test_insert_find_one_in_request(self): count = 0 for i in range(100): self.connection.start_request() self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.connection.end_request() self.assertFalse(count) # This was failing because commands were being sent to the slaves def test_create_collection(self): self.connection.drop_database('pymongo_test') collection = self.db.create_collection('test') self.assert_(isinstance(collection, Collection)) self.assertRaises(CollectionInvalid, self.db.create_collection, 'test') # Believe this was failing for the same reason... def test_unique_index(self): self.connection.drop_database('pymongo_test') self.db.test.create_index('username', unique=True) self.db.test.save({'username': '******'}, safe=True) self.assertRaises(OperationFailure, self.db.test.save, {'username': '******'}, safe=True) # NOTE this test is non-deterministic, but I expect # some failures unless the db is pulling instantaneously... def test_insert_find_one_with_slaves(self): count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assert_(count) # NOTE this test is non-deterministic, but hopefully we pause long enough # for the slaves to pull... def test_insert_find_one_with_pause(self): count = 0 self.db.test.remove({}) self.db.test.insert({"x": 5586}) time.sleep(11) for _ in range(10): try: if 5586 != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assertFalse(count) def test_kill_cursors(self): def cursor_count(): count = 0 res = self.connection.master.test_pymongo.command("cursorInfo") count += res["clientCursors_size"] for slave in self.connection.slaves: res = slave.test_pymongo.command("cursorInfo") count += res["clientCursors_size"] return count self.connection.test_pymongo.drop_collection("test") db = self.db before = cursor_count() for i in range(10000): db.test.insert({"i": i}) time.sleep(11) # need to sleep to be sure this gets pulled... self.assertEqual(before, cursor_count()) for _ in range(10): db.test.find_one() self.assertEqual(before, cursor_count()) for _ in range(10): for x in db.test.find(): break self.assertEqual(before, cursor_count()) a = db.test.find() for x in a: break self.assertNotEqual(before, cursor_count()) del a self.assertEqual(before, cursor_count()) a = db.test.find().limit(10) for x in a: break self.assertEqual(before, cursor_count())
class TestMasterSlaveConnection(unittest.TestCase, TestRequestMixin): def setUp(self): self.master = MongoClient(host, port) self.slaves = [] try: self.slaves.append( MongoClient(host2, port2, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append( MongoClient(host3, port3, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest("Not connected to master-slave set") self.ctx = catch_warnings() warnings.simplefilter("ignore", DeprecationWarning) self.client = MasterSlaveConnection(self.master, self.slaves) self.db = self.client.pymongo_test def tearDown(self): self.ctx.exit() try: self.db.test.drop_indexes() except Exception: # Tests like test_disconnect can monkey with the client in ways # that make this fail pass self.master = self.slaves = self.db = self.client = None super(TestMasterSlaveConnection, self).tearDown() def test_types(self): self.assertRaises(TypeError, MasterSlaveConnection, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, [1]) def test_use_greenlets(self): self.assertFalse(self.client.use_greenlets) if thread_util.have_gevent: master = MongoClient(host, port, use_greenlets=True) slaves = [ MongoClient(slave.host, slave.port, use_greenlets=True) for slave in self.slaves ] self.assertTrue( MasterSlaveConnection(master, slaves).use_greenlets) def test_repr(self): self.assertEqual( repr(self.client), "MasterSlaveConnection(%r, %r)" % (self.master, self.slaves)) def test_disconnect(self): class MongoClient(object): def __init__(self): self._disconnects = 0 def disconnect(self): self._disconnects += 1 self.client._MasterSlaveConnection__master = MongoClient() self.client._MasterSlaveConnection__slaves = [ MongoClient(), MongoClient() ] self.client.disconnect() self.assertEqual( 1, self.client._MasterSlaveConnection__master._disconnects) self.assertEqual( 1, self.client._MasterSlaveConnection__slaves[0]._disconnects) self.assertEqual( 1, self.client._MasterSlaveConnection__slaves[1]._disconnects) def test_continue_until_slave_works(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return (None, 'sent') class NotRandomList(object): last_idx = -1 def __init__(self): self._items = [ Slave(True), Slave(True), Slave(False), Slave(True) ] def __len__(self): return len(self._items) def __getitem__(self, idx): NotRandomList.last_idx = idx return self._items.pop(0) self.client._MasterSlaveConnection__slaves = NotRandomList() response = self.client._send_message_with_response('message') self.assertEqual((NotRandomList.last_idx, 'sent'), response) self.assertNotEqual(-1, NotRandomList.last_idx) self.assertEqual(3, Slave.calls) def test_raise_autoreconnect_if_all_slaves_fail(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): def __init__(self): self._items = [ Slave(True), Slave(True), Slave(True), Slave(True) ] def __len__(self): return len(self._items) def __getitem__(self, idx): return self._items.pop(0) self.client._MasterSlaveConnection__slaves = NotRandomList() self.assertRaises(AutoReconnect, self.client._send_message_with_response, 'message') self.assertEqual(4, Slave.calls) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.client, "") self.assertRaises(InvalidName, make_db, self.client, "te$t") self.assertRaises(InvalidName, make_db, self.client, "te.t") self.assertRaises(InvalidName, make_db, self.client, "te\\t") self.assertRaises(InvalidName, make_db, self.client, "te/t") self.assertRaises(InvalidName, make_db, self.client, "te st") self.assertTrue(isinstance(self.client.test, Database)) self.assertEqual(self.client.test, self.client["test"]) self.assertEqual(self.client.test, Database(self.client, "test")) def test_database_names(self): self.client.pymongo_test.test.save({"dummy": u"object"}) self.client.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = self.client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.client.drop_database, 5) self.assertRaises(TypeError, self.client.drop_database, None) raise SkipTest("This test often fails due to SERVER-2329") self.client.pymongo_test.test.save({"dummy": u"object"}) dbs = self.client.database_names() self.assertTrue("pymongo_test" in dbs) self.client.drop_database("pymongo_test") dbs = self.client.database_names() self.assertTrue("pymongo_test" not in dbs) self.client.pymongo_test.test.save({"dummy": u"object"}) dbs = self.client.database_names() self.assertTrue("pymongo_test" in dbs) self.client.drop_database(self.client.pymongo_test) dbs = self.client.database_names() self.assertTrue("pymongo_test" not in dbs) def test_iteration(self): def iterate(): [a for a in self.client] self.assertRaises(TypeError, iterate) def test_insert_find_one_in_request(self): count = 0 for i in range(100): self.client.start_request() self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.client.end_request() self.assertFalse(count) def test_nested_request(self): client = self.client def assertRequest(in_request): self.assertEqual(in_request, client.in_request()) self.assertEqual(in_request, client.master.in_request()) # MasterSlaveConnection is special, alas - it has no auto_start_request # and it begins *not* in a request. When it's in a request, it sends # all queries to primary. self.assertFalse(client.in_request()) self.assertFalse(client.master.in_request()) # Start and end request client.start_request() assertRequest(True) client.end_request() assertRequest(False) # Double-nesting client.start_request() client.start_request() client.end_request() assertRequest(True) client.end_request() assertRequest(False) def test_request_threads(self): client = self.client # In a request, all ops go through master pool = get_pool(client.master) client.master.end_request() self.assertNotInRequestAndDifferentSock(client, pool) started_request, ended_request = threading.Event(), threading.Event() checked_request = threading.Event() thread_done = [False] # Starting a request in one thread doesn't put the other thread in a # request def f(): self.assertNotInRequestAndDifferentSock(client, pool) client.start_request() self.assertInRequestAndSameSock(client, pool) started_request.set() checked_request.wait() checked_request.clear() self.assertInRequestAndSameSock(client, pool) client.end_request() self.assertNotInRequestAndDifferentSock(client, pool) ended_request.set() checked_request.wait() thread_done[0] = True t = threading.Thread(target=f) t.setDaemon(True) t.start() started_request.wait() self.assertNotInRequestAndDifferentSock(client, pool) checked_request.set() ended_request.wait() self.assertNotInRequestAndDifferentSock(client, pool) checked_request.set() t.join() self.assertNotInRequestAndDifferentSock(client, pool) self.assertTrue(thread_done[0], "Thread didn't complete") # This was failing because commands were being sent to the slaves def test_create_collection(self): self.client.pymongo_test.test.drop() collection = self.db.create_collection('test') self.assertTrue(isinstance(collection, Collection)) self.assertRaises(CollectionInvalid, self.db.create_collection, 'test') # Believe this was failing for the same reason... def test_unique_index(self): self.client.pymongo_test.test.drop() self.db.test.create_index('username', unique=True) self.db.test.save({'username': '******'}) self.assertRaises(OperationFailure, self.db.test.save, {'username': '******'}) def test_insert(self): w = len(self.slaves) + 1 self.db.test.remove(w=w) self.assertEqual(0, self.db.test.count()) doc = {} self.db.test.insert(doc, w=w) self.assertEqual(doc, self.db.test.find_one()) def test_kill_cursor_explicit(self): c = self.client c.read_preference = ReadPreference.SECONDARY_PREFERRED db = c.pymongo_test test = db.master_slave_test_kill_cursor_explicit test.drop() for i in range(20): test.insert({"i": i}, w=1 + len(self.slaves)) st = time.time() while time.time() - st < 120: # Wait for replication -- the 'w' parameter should obviate this # loop but it's not working reliably in Jenkins right now if list(test.find({"i": 19})): break time.sleep(0.5) else: self.fail("Replication timeout, test coll has %s records" % (len(list(test.find())))) # Partially evaluate cursor so it's left alive, then kill it cursor = test.find().batch_size(10) self.assertNotEqual( cursor._Cursor__connection_id, -1, "Expected cursor connected to a slave, not master") self.assertTrue(cursor.next()) self.assertNotEqual(0, cursor.cursor_id) cursor_id = cursor.cursor_id # Cursor dead on server - trigger a getMore on the same cursor_id and # check that the server returns an error. cursor2 = cursor.clone() cursor2._Cursor__id = cursor_id if (sys.platform.startswith('java') or 'PyPy' in sys.version): # Explicitly kill cursor. cursor.close() else: # Implicitly kill it in CPython. del cursor self.assertRaises(OperationFailure, lambda: list(cursor2)) def test_base_object(self): ctx = catch_warnings() try: warnings.simplefilter("ignore", DeprecationWarning) c = self.client self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(c.safe) self.assertEqual({}, c.get_lasterror_options()) db = c.pymongo_test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(db.safe) self.assertEqual({}, db.get_lasterror_options()) coll = db.test coll.drop() self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(coll.safe) self.assertEqual({}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) w = 1 + len(self.slaves) wtimeout = 10000 # Wait 10 seconds for replication to complete c.set_lasterror_options(w=w, wtimeout=wtimeout) self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(c.safe) self.assertEqual({ 'w': w, 'wtimeout': wtimeout }, c.get_lasterror_options()) db = c.pymongo_test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(db.safe) self.assertEqual({ 'w': w, 'wtimeout': wtimeout }, db.get_lasterror_options()) coll = db.test self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(coll.safe) self.assertEqual({ 'w': w, 'wtimeout': wtimeout }, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) coll.insert({'foo': 'bar'}) self.assertEqual(1, coll.find({'foo': 'bar'}).count()) self.assertTrue(coll.find({'foo': 'bar'})) coll.remove({'foo': 'bar'}) self.assertEqual(0, coll.find({'foo': 'bar'}).count()) c.safe = False c.unset_lasterror_options() self.assertFalse(self.client.slave_okay) self.assertTrue(bool(self.client.read_preference)) self.assertFalse(self.client.safe) self.assertEqual({}, self.client.get_lasterror_options()) finally: ctx.exit() def test_document_class(self): c = MasterSlaveConnection(self.master, self.slaves) db = c.pymongo_test w = 1 + len(self.slaves) db.test.insert({"x": 1}, w=w) self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = MasterSlaveConnection(self.master, self.slaves, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) def test_tz_aware(self): dt = datetime.datetime.utcnow() client = MasterSlaveConnection(self.master, self.slaves) self.assertEqual(False, client.tz_aware) db = client.pymongo_test w = 1 + len(self.slaves) db.tztest.insert({'dt': dt}, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo) client = MasterSlaveConnection(self.master, self.slaves, tz_aware=True) self.assertEqual(True, client.tz_aware) db = client.pymongo_test db.tztest.insert({'dt': dt}, w=w) self.assertEqual(utc, db.tztest.find_one()['dt'].tzinfo) client = MasterSlaveConnection(self.master, self.slaves, tz_aware=False) self.assertEqual(False, client.tz_aware) db = client.pymongo_test db.tztest.insert({'dt': dt}, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo)
class TestMasterSlaveConnection(unittest.TestCase): def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append(Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append(Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest("Not connected to master-slave set") self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test def tearDown(self): try: self.db.test.drop_indexes() except Exception: # Tests like test_disconnect can monkey with the connection in ways # that make this fail pass super(TestMasterSlaveConnection, self).tearDown() def test_types(self): self.assertRaises(TypeError, MasterSlaveConnection, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, [1]) def test_repr(self): self.assertEqual(repr(self.connection), "MasterSlaveConnection(%r, %r)" % (self.master, self.slaves)) def test_disconnect(self): class Connection(object): def __init__(self): self._disconnects = 0 def disconnect(self): self._disconnects += 1 self.connection._MasterSlaveConnection__master = Connection() self.connection._MasterSlaveConnection__slaves = [Connection(), Connection()] self.connection.disconnect() self.assertEqual(1, self.connection._MasterSlaveConnection__master._disconnects) self.assertEqual(1, self.connection._MasterSlaveConnection__slaves[0]._disconnects) self.assertEqual(1, self.connection._MasterSlaveConnection__slaves[1]._disconnects) def test_continue_until_slave_works(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): last_idx = -1 def __init__(self): self._items = [Slave(True), Slave(True), Slave(False), Slave(True)] def __len__(self): return len(self._items) def __getitem__(self, idx): NotRandomList.last_idx = idx return self._items.pop(0) self.connection._MasterSlaveConnection__slaves = NotRandomList() response = self.connection._send_message_with_response('message') self.assertEqual((NotRandomList.last_idx, 'sent'), response) self.assertNotEqual(-1, NotRandomList.last_idx) self.assertEqual(3, Slave.calls) def test_raise_autoreconnect_if_all_slaves_fail(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): def __init__(self): self._items = [Slave(True), Slave(True), Slave(True), Slave(True)] def __len__(self): return len(self._items) def __getitem__(self, idx): return self._items.pop(0) self.connection._MasterSlaveConnection__slaves = NotRandomList() self.assertRaises(AutoReconnect, self.connection._send_message_with_response, 'message') self.assertEqual(4, Slave.calls) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.connection, "") self.assertRaises(InvalidName, make_db, self.connection, "te$t") self.assertRaises(InvalidName, make_db, self.connection, "te.t") self.assertRaises(InvalidName, make_db, self.connection, "te\\t") self.assertRaises(InvalidName, make_db, self.connection, "te/t") self.assertRaises(InvalidName, make_db, self.connection, "te st") self.assertTrue(isinstance(self.connection.test, Database)) self.assertEqual(self.connection.test, self.connection["test"]) self.assertEqual(self.connection.test, Database(self.connection, "test")) def test_database_names(self): self.connection.pymongo_test.test.save({"dummy": u"object"}) self.connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.connection.drop_database, 5) self.assertRaises(TypeError, self.connection.drop_database, None) raise SkipTest("This test often fails due to SERVER-2329") self.connection.pymongo_test.test.save({"dummy": u"object"}, safe=True) dbs = self.connection.database_names() self.assertTrue("pymongo_test" in dbs) self.connection.drop_database("pymongo_test") dbs = self.connection.database_names() self.assertTrue("pymongo_test" not in dbs) self.connection.pymongo_test.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assertTrue("pymongo_test" in dbs) self.connection.drop_database(self.connection.pymongo_test) dbs = self.connection.database_names() self.assertTrue("pymongo_test" not in dbs) def test_iteration(self): def iterate(): [a for a in self.connection] self.assertRaises(TypeError, iterate) def test_insert_find_one_in_request(self): count = 0 for i in range(100): self.connection.start_request() self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.connection.end_request() self.assertFalse(count) # This was failing because commands were being sent to the slaves def test_create_collection(self): self.connection.pymongo_test.test.drop() collection = self.db.create_collection('test') self.assertTrue(isinstance(collection, Collection)) self.assertRaises(CollectionInvalid, self.db.create_collection, 'test') # Believe this was failing for the same reason... def test_unique_index(self): self.connection.pymongo_test.test.drop() self.db.test.create_index('username', unique=True) self.db.test.save({'username': '******'}, safe=True) self.assertRaises(OperationFailure, self.db.test.save, {'username': '******'}, safe=True) # NOTE this test is non-deterministic, but I expect # some failures unless the db is pulling instantaneously... def test_insert_find_one_with_slaves(self): count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assertTrue(count) # NOTE this test is non-deterministic, but hopefully we pause long enough # for the slaves to pull... def test_insert_find_one_with_pause(self): count = 0 self.db.test.remove({}) self.db.test.insert({"x": 5586}) time.sleep(11) for _ in range(10): try: if 5586 != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assertFalse(count) def test_kill_cursor_explicit(self): c = self.connection c.slave_okay = True db = c.pymongo_test test = db.master_slave_test_kill_cursor_explicit test.drop() for i in range(20): test.insert({"i": i}, w=1 + len(self.slaves)) st = time.time() while time.time() - st < 120: # Wait for replication -- the 'w' parameter should obviate this # loop but it's not working reliably in Jenkins right now if list(test.find({"i": 19})): break time.sleep(0.5) else: self.fail("Replication timeout, test coll has %s records" % ( len(list(test.find())) )) # Partially evaluate cursor so it's left alive, then kill it cursor = test.find().batch_size(10) self.assertNotEqual( cursor._Cursor__connection_id, -1, "Expected cursor connected to a slave, not master") self.assertTrue(cursor.next()) self.assertNotEqual(0, cursor.cursor_id) cursor_id = cursor.cursor_id # Cursor dead on server - trigger a getMore on the same cursor_id and # check that the server returns an error. cursor2 = cursor.clone() cursor2._Cursor__id = cursor_id if (sys.platform.startswith('java') or 'PyPy' in sys.version): # Explicitly kill cursor. cursor.close() else: # Implicitly kill it in CPython. del cursor self.assertRaises(OperationFailure, lambda: list(cursor2)) def test_base_object(self): c = self.connection self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(c.safe) self.assertEqual({}, c.get_lasterror_options()) db = c.test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(db.safe) self.assertEqual({}, db.get_lasterror_options()) coll = db.test coll.drop() self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(coll.safe) self.assertEqual({}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) c.safe = True w = 1 + len(self.slaves) wtimeout=10000 # Wait 10 seconds for replication to complete c.set_lasterror_options(w=w, wtimeout=wtimeout) self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(c.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, c.get_lasterror_options()) db = c.test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(db.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, db.get_lasterror_options()) coll = db.test self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(coll.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) coll.insert({'foo': 'bar'}) self.assertEqual(1, coll.find({'foo': 'bar'}).count()) self.assertTrue(coll.find({'foo': 'bar'})) coll.remove({'foo': 'bar'}) self.assertEqual(0, coll.find({'foo': 'bar'}).count()) # Set self.connection back to defaults c.safe = False c.unset_lasterror_options() self.assertFalse(self.connection.slave_okay) self.assertTrue(bool(self.connection.read_preference)) self.assertFalse(self.connection.safe) self.assertEqual({}, self.connection.get_lasterror_options()) def test_document_class(self): c = MasterSlaveConnection(self.master, self.slaves) db = c.pymongo_test w = 1 + len(self.slaves) db.test.insert({"x": 1}, safe=True, w=w) self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = MasterSlaveConnection(self.master, self.slaves, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) def test_tz_aware(self): dt = datetime.datetime.utcnow() conn = MasterSlaveConnection(self.master, self.slaves) self.assertEqual(False, conn.tz_aware) db = conn.pymongo_test w = 1 + len(self.slaves) db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo) conn = MasterSlaveConnection(self.master, self.slaves, tz_aware=True) self.assertEqual(True, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(utc, db.tztest.find_one()['dt'].tzinfo) conn = MasterSlaveConnection(self.master, self.slaves, tz_aware=False) self.assertEqual(False, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo)
class TestMasterSlaveConnection(unittest.TestCase, TestRequestMixin): def setUp(self): raise SkipTest("master-slave setups are unsupported in TokuMX") self.master = MongoClient(host, port) self.slaves = [] try: self.slaves.append(MongoClient( host2, port2, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append(MongoClient( host3, port3, read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest("Not connected to master-slave set") self.client = MasterSlaveConnection(self.master, self.slaves) self.db = self.client.pymongo_test def tearDown(self): try: self.db.test.drop_indexes() except Exception: # Tests like test_disconnect can monkey with the client in ways # that make this fail pass self.master = self.slaves = self.db = self.client = None super(TestMasterSlaveConnection, self).tearDown() def test_types(self): self.assertRaises(TypeError, MasterSlaveConnection, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, [1]) def test_use_greenlets(self): self.assertFalse(self.client.use_greenlets) if thread_util.have_gevent: master = MongoClient(host, port, use_greenlets=True) slaves = [ MongoClient(slave.host, slave.port, use_greenlets=True) for slave in self.slaves] self.assertTrue( MasterSlaveConnection(master, slaves).use_greenlets) def test_repr(self): self.assertEqual(repr(self.client), "MasterSlaveConnection(%r, %r)" % (self.master, self.slaves)) def test_disconnect(self): class MongoClient(object): def __init__(self): self._disconnects = 0 def disconnect(self): self._disconnects += 1 self.client._MasterSlaveConnection__master = MongoClient() self.client._MasterSlaveConnection__slaves = [MongoClient(), MongoClient()] self.client.disconnect() self.assertEqual(1, self.client._MasterSlaveConnection__master._disconnects) self.assertEqual(1, self.client._MasterSlaveConnection__slaves[0]._disconnects) self.assertEqual(1, self.client._MasterSlaveConnection__slaves[1]._disconnects) def test_continue_until_slave_works(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return (None, 'sent') class NotRandomList(object): last_idx = -1 def __init__(self): self._items = [Slave(True), Slave(True), Slave(False), Slave(True)] def __len__(self): return len(self._items) def __getitem__(self, idx): NotRandomList.last_idx = idx return self._items.pop(0) self.client._MasterSlaveConnection__slaves = NotRandomList() response = self.client._send_message_with_response('message') self.assertEqual((NotRandomList.last_idx, 'sent'), response) self.assertNotEqual(-1, NotRandomList.last_idx) self.assertEqual(3, Slave.calls) def test_raise_autoreconnect_if_all_slaves_fail(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): def __init__(self): self._items = [Slave(True), Slave(True), Slave(True), Slave(True)] def __len__(self): return len(self._items) def __getitem__(self, idx): return self._items.pop(0) self.client._MasterSlaveConnection__slaves = NotRandomList() self.assertRaises(AutoReconnect, self.client._send_message_with_response, 'message') self.assertEqual(4, Slave.calls) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.client, "") self.assertRaises(InvalidName, make_db, self.client, "te$t") self.assertRaises(InvalidName, make_db, self.client, "te.t") self.assertRaises(InvalidName, make_db, self.client, "te\\t") self.assertRaises(InvalidName, make_db, self.client, "te/t") self.assertRaises(InvalidName, make_db, self.client, "te st") self.assertTrue(isinstance(self.client.test, Database)) self.assertEqual(self.client.test, self.client["test"]) self.assertEqual(self.client.test, Database(self.client, "test")) def test_database_names(self): self.client.pymongo_test.test.save({"dummy": u"object"}) self.client.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = self.client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.client.drop_database, 5) self.assertRaises(TypeError, self.client.drop_database, None) raise SkipTest("This test often fails due to SERVER-2329") self.client.pymongo_test.test.save({"dummy": u"object"}) dbs = self.client.database_names() self.assertTrue("pymongo_test" in dbs) self.client.drop_database("pymongo_test") dbs = self.client.database_names() self.assertTrue("pymongo_test" not in dbs) self.client.pymongo_test.test.save({"dummy": u"object"}) dbs = self.client.database_names() self.assertTrue("pymongo_test" in dbs) self.client.drop_database(self.client.pymongo_test) dbs = self.client.database_names() self.assertTrue("pymongo_test" not in dbs) def test_iteration(self): def iterate(): [a for a in self.client] self.assertRaises(TypeError, iterate) def test_insert_find_one_in_request(self): count = 0 for i in range(100): self.client.start_request() self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.client.end_request() self.assertFalse(count) def test_nested_request(self): client = self.client def assertRequest(in_request): self.assertEqual(in_request, client.in_request()) self.assertEqual(in_request, client.master.in_request()) # MasterSlaveConnection is special, alas - it has no auto_start_request # and it begins *not* in a request. When it's in a request, it sends # all queries to primary. self.assertFalse(client.in_request()) self.assertFalse(client.master.in_request()) # Start and end request client.start_request() assertRequest(True) client.end_request() assertRequest(False) # Double-nesting client.start_request() client.start_request() client.end_request() assertRequest(True) client.end_request() assertRequest(False) def test_request_threads(self): client = self.client # In a request, all ops go through master pool = get_pool(client.master) client.master.end_request() self.assertNotInRequestAndDifferentSock(client, pool) started_request, ended_request = threading.Event(), threading.Event() checked_request = threading.Event() thread_done = [False] # Starting a request in one thread doesn't put the other thread in a # request def f(): self.assertNotInRequestAndDifferentSock(client, pool) client.start_request() self.assertInRequestAndSameSock(client, pool) started_request.set() checked_request.wait() checked_request.clear() self.assertInRequestAndSameSock(client, pool) client.end_request() self.assertNotInRequestAndDifferentSock(client, pool) ended_request.set() checked_request.wait() thread_done[0] = True t = threading.Thread(target=f) t.setDaemon(True) t.start() started_request.wait() self.assertNotInRequestAndDifferentSock(client, pool) checked_request.set() ended_request.wait() self.assertNotInRequestAndDifferentSock(client, pool) checked_request.set() t.join() self.assertNotInRequestAndDifferentSock(client, pool) self.assertTrue(thread_done[0], "Thread didn't complete") # This was failing because commands were being sent to the slaves def test_create_collection(self): self.client.pymongo_test.test.drop() collection = self.db.create_collection('test') self.assertTrue(isinstance(collection, Collection)) self.assertRaises(CollectionInvalid, self.db.create_collection, 'test') # Believe this was failing for the same reason... def test_unique_index(self): self.client.pymongo_test.test.drop() self.db.test.create_index('username', unique=True) self.db.test.save({'username': '******'}) self.assertRaises(OperationFailure, self.db.test.save, {'username': '******'}) # NOTE this test is non-deterministic, but I expect # some failures unless the db is pulling instantaneously... def test_insert_find_one_with_slaves(self): count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assertTrue(count) # NOTE this test is non-deterministic, but hopefully we pause long enough # for the slaves to pull... def test_insert_find_one_with_pause(self): count = 0 self.db.test.remove({}) self.db.test.insert({"x": 5586}) time.sleep(11) for _ in range(10): try: if 5586 != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assertFalse(count) def test_kill_cursor_explicit(self): c = self.client c.read_preference = ReadPreference.SECONDARY_PREFERRED db = c.pymongo_test test = db.master_slave_test_kill_cursor_explicit test.drop() for i in range(20): test.insert({"i": i}, w=1 + len(self.slaves)) st = time.time() while time.time() - st < 120: # Wait for replication -- the 'w' parameter should obviate this # loop but it's not working reliably in Jenkins right now if list(test.find({"i": 19})): break time.sleep(0.5) else: self.fail("Replication timeout, test coll has %s records" % ( len(list(test.find())) )) # Partially evaluate cursor so it's left alive, then kill it cursor = test.find().batch_size(10) self.assertNotEqual( cursor._Cursor__connection_id, -1, "Expected cursor connected to a slave, not master") self.assertTrue(cursor.next()) self.assertNotEqual(0, cursor.cursor_id) cursor_id = cursor.cursor_id # Cursor dead on server - trigger a getMore on the same cursor_id and # check that the server returns an error. cursor2 = cursor.clone() cursor2._Cursor__id = cursor_id if (sys.platform.startswith('java') or 'PyPy' in sys.version): # Explicitly kill cursor. cursor.close() else: # Implicitly kill it in CPython. del cursor self.assertRaises(OperationFailure, lambda: list(cursor2)) def test_base_object(self): ctx = catch_warnings() try: warnings.simplefilter("ignore", DeprecationWarning) c = self.client self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(c.safe) self.assertEqual({}, c.get_lasterror_options()) db = c.pymongo_test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(db.safe) self.assertEqual({}, db.get_lasterror_options()) coll = db.test coll.drop() self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(coll.safe) self.assertEqual({}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) w = 1 + len(self.slaves) wtimeout=10000 # Wait 10 seconds for replication to complete c.set_lasterror_options(w=w, wtimeout=wtimeout) self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(c.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, c.get_lasterror_options()) db = c.pymongo_test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(db.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, db.get_lasterror_options()) coll = db.test self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(coll.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) coll.insert({'foo': 'bar'}) self.assertEqual(1, coll.find({'foo': 'bar'}).count()) self.assertTrue(coll.find({'foo': 'bar'})) coll.remove({'foo': 'bar'}) self.assertEqual(0, coll.find({'foo': 'bar'}).count()) c.safe = False c.unset_lasterror_options() self.assertFalse(self.client.slave_okay) self.assertTrue(bool(self.client.read_preference)) self.assertFalse(self.client.safe) self.assertEqual({}, self.client.get_lasterror_options()) finally: ctx.exit() def test_document_class(self): c = MasterSlaveConnection(self.master, self.slaves) db = c.pymongo_test w = 1 + len(self.slaves) db.test.insert({"x": 1}, w=w) self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = MasterSlaveConnection(self.master, self.slaves, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) def test_tz_aware(self): dt = datetime.datetime.utcnow() client = MasterSlaveConnection(self.master, self.slaves) self.assertEqual(False, client.tz_aware) db = client.pymongo_test w = 1 + len(self.slaves) db.tztest.insert({'dt': dt}, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo) client = MasterSlaveConnection(self.master, self.slaves, tz_aware=True) self.assertEqual(True, client.tz_aware) db = client.pymongo_test db.tztest.insert({'dt': dt}, w=w) self.assertEqual(utc, db.tztest.find_one()['dt'].tzinfo) client = MasterSlaveConnection(self.master, self.slaves, tz_aware=False) self.assertEqual(False, client.tz_aware) db = client.pymongo_test db.tztest.insert({'dt': dt}, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo)
class TestMasterSlaveConnection(unittest.TestCase): def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append(Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), slave_okay=True)) except ConnectionFailure: pass try: self.slaves.append(Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), slave_okay=True)) except ConnectionFailure: pass if not self.slaves: raise SkipTest() self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test def test_types(self): self.assertRaises(TypeError, MasterSlaveConnection, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, [1]) def test_repr(self): self.assertEqual(repr(self.connection), "MasterSlaveConnection(%r, %r)" % (self.master, self.slaves)) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.connection, "") self.assertRaises(InvalidName, make_db, self.connection, "te$t") self.assertRaises(InvalidName, make_db, self.connection, "te.t") self.assertRaises(InvalidName, make_db, self.connection, "te\\t") self.assertRaises(InvalidName, make_db, self.connection, "te/t") self.assertRaises(InvalidName, make_db, self.connection, "te st") self.assert_(isinstance(self.connection.test, Database)) self.assertEqual(self.connection.test, self.connection["test"]) self.assertEqual(self.connection.test, Database(self.connection, "test")) def test_database_names(self): self.connection.pymongo_test.test.save({"dummy": u"object"}) self.connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.connection.drop_database, 5) self.assertRaises(TypeError, self.connection.drop_database, None) self.connection.pymongo_test.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.connection.drop_database("pymongo_test") dbs = self.connection.database_names() self.assert_("pymongo_test" not in dbs) self.connection.pymongo_test.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.connection.drop_database(self.connection.pymongo_test) dbs = self.connection.database_names() self.assert_("pymongo_test" not in dbs) def test_iteration(self): def iterate(): [a for a in self.connection] self.assertRaises(TypeError, iterate) def test_insert_find_one_in_request(self): count = 0 for i in range(100): self.connection.start_request() self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.connection.end_request() self.failIf(count) def test_insert_find_one_no_slaves(self): if self.slaves: raise SkipTest() count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.failIf(count) # This was failing because commands were being sent to the slaves def test_create_collection(self): self.connection.drop_database('pymongo_test') collection = self.db.create_collection('test') self.assert_(isinstance(collection, Collection)) self.assertRaises(CollectionInvalid, self.db.create_collection, 'test') # Believe this was failing for the same reason... def test_unique_index(self): self.connection.drop_database('pymongo_test') self.db.test.create_index('username', unique=True) self.db.test.save({'username': '******'}, safe=True) self.assertRaises(OperationFailure, self.db.test.save, {'username': '******'}, safe=True) # NOTE this test is non-deterministic, but I expect # some failures unless the db is pulling instantaneously... def test_insert_find_one_with_slaves(self): if not self.slaves: raise SkipTest() count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assert_(count) # NOTE this test is non-deterministic, but hopefully we pause long enough # for the slaves to pull... def test_insert_find_one_with_pause(self): if not self.slaves: raise SkipTest() count = 0 self.db.test.remove({}) self.db.test.insert({"x": 5586}) time.sleep(7) for _ in range(10): try: if 5586 != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.failIf(count) def test_kill_cursors(self): def cursor_count(): count = 0 res = self.connection.master.test_pymongo.command({ "cursorInfo": 1}) count += res["clientCursors_size"] for slave in self.connection.slaves: res = slave.test_pymongo.command({"cursorInfo": 1}) count += res["clientCursors_size"] return count self.connection.test_pymongo.drop_collection("test") db = self.db before = cursor_count() for i in range(10000): db.test.insert({"i": i}) time.sleep(6) # need to sleep to be sure this gets pulled... self.assertEqual(before, cursor_count()) for _ in range(10): db.test.find_one() self.assertEqual(before, cursor_count()) for _ in range(10): for x in db.test.find(): break self.assertEqual(before, cursor_count()) a = db.test.find() for x in a: break self.assertNotEqual(before, cursor_count()) del a self.assertEqual(before, cursor_count()) a = db.test.find().limit(10) for x in a: break self.assertEqual(before, cursor_count())
class TestMasterSlaveConnection(unittest.TestCase): def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append(Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append(Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest() self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test def test_types(self): self.assertRaises(TypeError, MasterSlaveConnection, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, [1]) def test_repr(self): self.assertEqual(repr(self.connection), "MasterSlaveConnection(%r, %r)" % (self.master, self.slaves)) def test_disconnect(self): class Connection(object): def __init__(self): self._disconnects = 0 def disconnect(self): self._disconnects += 1 self.connection._MasterSlaveConnection__master = Connection() self.connection._MasterSlaveConnection__slaves = [Connection(), Connection()] self.connection.disconnect() self.assertEqual(1, self.connection._MasterSlaveConnection__master._disconnects) self.assertEqual(1, self.connection._MasterSlaveConnection__slaves[0]._disconnects) self.assertEqual(1, self.connection._MasterSlaveConnection__slaves[1]._disconnects) def test_continue_until_slave_works(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): last_idx = -1 def __init__(self): self._items = [Slave(True), Slave(True), Slave(False), Slave(True)] def __len__(self): return len(self._items) def __getitem__(self, idx): NotRandomList.last_idx = idx return self._items.pop(0) self.connection._MasterSlaveConnection__slaves = NotRandomList() response = self.connection._send_message_with_response('message') self.assertEqual((NotRandomList.last_idx, 'sent'), response) self.assertNotEqual(-1, NotRandomList.last_idx) self.assertEqual(3, Slave.calls) def test_raise_autoreconnect_if_all_slaves_fail(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): def __init__(self): self._items = [Slave(True), Slave(True), Slave(True), Slave(True)] def __len__(self): return len(self._items) def __getitem__(self, idx): return self._items.pop(0) self.connection._MasterSlaveConnection__slaves = NotRandomList() self.assertRaises(AutoReconnect, self.connection._send_message_with_response, 'message') self.assertEqual(4, Slave.calls) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.connection, "") self.assertRaises(InvalidName, make_db, self.connection, "te$t") self.assertRaises(InvalidName, make_db, self.connection, "te.t") self.assertRaises(InvalidName, make_db, self.connection, "te\\t") self.assertRaises(InvalidName, make_db, self.connection, "te/t") self.assertRaises(InvalidName, make_db, self.connection, "te st") self.assertTrue(isinstance(self.connection.test, Database)) self.assertEqual(self.connection.test, self.connection["test"]) self.assertEqual(self.connection.test, Database(self.connection, "test")) def test_database_names(self): self.connection.pymongo_test.test.save({"dummy": u"object"}) self.connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.connection.drop_database, 5) self.assertRaises(TypeError, self.connection.drop_database, None) raise SkipTest("This test often fails due to SERVER-2329") self.connection.pymongo_test.test.save({"dummy": u"object"}, safe=True) dbs = self.connection.database_names() self.assertTrue("pymongo_test" in dbs) self.connection.drop_database("pymongo_test") dbs = self.connection.database_names() self.assertTrue("pymongo_test" not in dbs) self.connection.pymongo_test.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assertTrue("pymongo_test" in dbs) self.connection.drop_database(self.connection.pymongo_test) dbs = self.connection.database_names() self.assertTrue("pymongo_test" not in dbs) def test_iteration(self): def iterate(): [a for a in self.connection] self.assertRaises(TypeError, iterate) def test_insert_find_one_in_request(self): count = 0 for i in range(100): self.connection.start_request() self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.connection.end_request() self.assertFalse(count) # This was failing because commands were being sent to the slaves def test_create_collection(self): self.connection.pymongo_test.test.drop() collection = self.db.create_collection('test') self.assertTrue(isinstance(collection, Collection)) self.assertRaises(CollectionInvalid, self.db.create_collection, 'test') # Believe this was failing for the same reason... def test_unique_index(self): self.connection.pymongo_test.test.drop() self.db.test.create_index('username', unique=True) self.db.test.save({'username': '******'}, safe=True) self.assertRaises(OperationFailure, self.db.test.save, {'username': '******'}, safe=True) # NOTE this test is non-deterministic, but I expect # some failures unless the db is pulling instantaneously... def test_insert_find_one_with_slaves(self): count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assertTrue(count) # NOTE this test is non-deterministic, but hopefully we pause long enough # for the slaves to pull... def test_insert_find_one_with_pause(self): count = 0 self.db.test.remove({}) self.db.test.insert({"x": 5586}) time.sleep(11) for _ in range(10): try: if 5586 != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assertFalse(count) def test_kill_cursor_explicit(self): c = self.connection c.slave_okay = True db = c.pymongo_test test = db.master_slave_test_kill_cursor_explicit test.drop() for i in range(20): test.insert({"i": i}, w=1 + len(self.slaves)) st = time.time() while time.time() - st < 120: # Wait for replication -- the 'w' parameter should obviate this # loop but it's not working reliably in Jenkins right now if list(test.find({"i": 19})): break time.sleep(0.5) else: self.fail("Replication timeout, test coll has %s records" % ( len(list(test.find())) )) # Partially evaluate cursor so it's left alive, then kill it cursor = test.find().batch_size(10) self.assertNotEqual( cursor._Cursor__connection_id, -1, "Expected cursor connected to a slave, not master") self.assertTrue(cursor.next()) self.assertNotEqual(0, cursor.cursor_id) cursor_id = cursor.cursor_id # Cursor dead on server - trigger a getMore on the same cursor_id and # check that the server returns an error. cursor2 = cursor.clone() cursor2._Cursor__id = cursor_id if (sys.platform.startswith('java') or 'PyPy' in sys.version): # Explicitly kill cursor. cursor.close() else: # Implicitly kill it in CPython. del cursor self.assertRaises(OperationFailure, lambda: list(cursor2)) def test_base_object(self): c = self.connection self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(c.safe) self.assertEqual({}, c.get_lasterror_options()) db = c.test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(db.safe) self.assertEqual({}, db.get_lasterror_options()) coll = db.test coll.drop() self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(coll.safe) self.assertEqual({}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) c.safe = True w = 1 + len(self.slaves) wtimeout=10000 # Wait 10 seconds for replication to complete c.set_lasterror_options(w=w, wtimeout=wtimeout) self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(c.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, c.get_lasterror_options()) db = c.test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(db.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, db.get_lasterror_options()) coll = db.test self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(coll.safe) self.assertEqual({'w': w, 'wtimeout': wtimeout}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) coll.insert({'foo': 'bar'}) self.assertEqual(1, coll.find({'foo': 'bar'}).count()) self.assertTrue(coll.find({'foo': 'bar'})) coll.remove({'foo': 'bar'}) self.assertEqual(0, coll.find({'foo': 'bar'}).count()) # Set self.connection back to defaults c.safe = False c.unset_lasterror_options() self.assertFalse(self.connection.slave_okay) self.assertTrue(bool(self.connection.read_preference)) self.assertFalse(self.connection.safe) self.assertEqual({}, self.connection.get_lasterror_options()) def test_document_class(self): c = MasterSlaveConnection(self.master, self.slaves) db = c.pymongo_test w = 1 + len(self.slaves) db.test.insert({"x": 1}, safe=True, w=w) self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = MasterSlaveConnection(self.master, self.slaves, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assertTrue(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict self.assertEqual(dict, c.document_class) self.assertTrue(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) def test_tz_aware(self): dt = datetime.datetime.utcnow() conn = MasterSlaveConnection(self.master, self.slaves) self.assertEqual(False, conn.tz_aware) db = conn.pymongo_test w = 1 + len(self.slaves) db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo) conn = MasterSlaveConnection(self.master, self.slaves, tz_aware=True) self.assertEqual(True, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(utc, db.tztest.find_one()['dt'].tzinfo) conn = MasterSlaveConnection(self.master, self.slaves, tz_aware=False) self.assertEqual(False, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}, safe=True, w=w) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo)
class TestMasterSlaveConnection(unittest.TestCase): def setUp(self): host = os.environ.get("DB_IP", "localhost") self.master = Connection(host, int(os.environ.get("DB_PORT", 27017))) self.slaves = [] try: self.slaves.append(Connection(os.environ.get("DB_IP2", host), int(os.environ.get("DB_PORT2", 27018)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass try: self.slaves.append(Connection(os.environ.get("DB_IP3", host), int(os.environ.get("DB_PORT3", 27019)), read_preference=ReadPreference.SECONDARY)) except ConnectionFailure: pass if not self.slaves: raise SkipTest() self.connection = MasterSlaveConnection(self.master, self.slaves) self.db = self.connection.pymongo_test def test_types(self): self.assertRaises(TypeError, MasterSlaveConnection, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, 1) self.assertRaises(TypeError, MasterSlaveConnection, self.master, [1]) def test_repr(self): self.assertEqual(repr(self.connection), "MasterSlaveConnection(%r, %r)" % (self.master, self.slaves)) def test_disconnect(self): class Connection(object): def __init__(self): self._disconnects = 0 def disconnect(self): self._disconnects += 1 self.connection._MasterSlaveConnection__master = Connection() self.connection._MasterSlaveConnection__slaves = [Connection(), Connection()] self.connection.disconnect() self.assertEquals(1, self.connection._MasterSlaveConnection__master._disconnects) self.assertEquals(1, self.connection._MasterSlaveConnection__slaves[0]._disconnects) self.assertEquals(1, self.connection._MasterSlaveConnection__slaves[1]._disconnects) def test_continue_until_slave_works(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): last_idx = -1 def __init__(self): self._items = [Slave(True), Slave(True), Slave(False), Slave(True)] def __len__(self): return len(self._items) def __getitem__(self, idx): NotRandomList.last_idx = idx return self._items.pop(0) self.connection._MasterSlaveConnection__slaves = NotRandomList() response = self.connection._send_message_with_response('message') self.assertEquals((NotRandomList.last_idx, 'sent'), response) self.assertNotEquals(-1, NotRandomList.last_idx) self.assertEquals(3, Slave.calls) def test_raise_autoreconnect_if_all_slaves_fail(self): class Slave(object): calls = 0 def __init__(self, fail): self._fail = fail def _send_message_with_response(self, *args, **kwargs): Slave.calls += 1 if self._fail: raise AutoReconnect() return 'sent' class NotRandomList(object): def __init__(self): self._items = [Slave(True), Slave(True), Slave(True), Slave(True)] def __len__(self): return len(self._items) def __getitem__(self, idx): return self._items.pop(0) self.connection._MasterSlaveConnection__slaves = NotRandomList() self.assertRaises(AutoReconnect, self.connection._send_message_with_response, 'message') self.assertEquals(4, Slave.calls) def test_get_db(self): def make_db(base, name): return base[name] self.assertRaises(InvalidName, make_db, self.connection, "") self.assertRaises(InvalidName, make_db, self.connection, "te$t") self.assertRaises(InvalidName, make_db, self.connection, "te.t") self.assertRaises(InvalidName, make_db, self.connection, "te\\t") self.assertRaises(InvalidName, make_db, self.connection, "te/t") self.assertRaises(InvalidName, make_db, self.connection, "te st") self.assert_(isinstance(self.connection.test, Database)) self.assertEqual(self.connection.test, self.connection["test"]) self.assertEqual(self.connection.test, Database(self.connection, "test")) def test_database_names(self): self.connection.pymongo_test.test.save({"dummy": u"object"}) self.connection.pymongo_test_mike.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.assert_("pymongo_test_mike" in dbs) def test_drop_database(self): self.assertRaises(TypeError, self.connection.drop_database, 5) self.assertRaises(TypeError, self.connection.drop_database, None) self.connection.pymongo_test.test.save({"dummy": u"object"}, safe=True) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.connection.drop_database("pymongo_test") dbs = self.connection.database_names() self.assert_("pymongo_test" not in dbs) self.connection.pymongo_test.test.save({"dummy": u"object"}) dbs = self.connection.database_names() self.assert_("pymongo_test" in dbs) self.connection.drop_database(self.connection.pymongo_test) dbs = self.connection.database_names() self.assert_("pymongo_test" not in dbs) def test_iteration(self): def iterate(): [a for a in self.connection] self.assertRaises(TypeError, iterate) def test_insert_find_one_in_request(self): count = 0 for i in range(100): self.connection.start_request() self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.connection.end_request() self.assertFalse(count) # This was failing because commands were being sent to the slaves def test_create_collection(self): self.connection.drop_database('pymongo_test') collection = self.db.create_collection('test') self.assert_(isinstance(collection, Collection)) self.assertRaises(CollectionInvalid, self.db.create_collection, 'test') # Believe this was failing for the same reason... def test_unique_index(self): self.connection.drop_database('pymongo_test') self.db.test.create_index('username', unique=True) self.db.test.save({'username': '******'}, safe=True) self.assertRaises(OperationFailure, self.db.test.save, {'username': '******'}, safe=True) # NOTE this test is non-deterministic, but I expect # some failures unless the db is pulling instantaneously... def test_insert_find_one_with_slaves(self): count = 0 for i in range(100): self.db.test.remove({}) self.db.test.insert({"x": i}) try: if i != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assert_(count) # NOTE this test is non-deterministic, but hopefully we pause long enough # for the slaves to pull... def test_insert_find_one_with_pause(self): count = 0 self.db.test.remove({}) self.db.test.insert({"x": 5586}) time.sleep(11) for _ in range(10): try: if 5586 != self.db.test.find_one()["x"]: count += 1 except: count += 1 self.assertFalse(count) def test_kill_cursors(self): def cursor_count(): count = 0 res = self.connection.master.test_pymongo.command("cursorInfo") count += res["clientCursors_size"] for slave in self.connection.slaves: res = slave.test_pymongo.command("cursorInfo") count += res["clientCursors_size"] return count self.connection.test_pymongo.drop_collection("test") db = self.db before = cursor_count() for i in range(10000): db.test.insert({"i": i}) time.sleep(11) # need to sleep to be sure this gets pulled... self.assertEqual(before, cursor_count()) for _ in range(10): db.test.find_one() self.assertEqual(before, cursor_count()) for _ in range(10): for x in db.test.find(): break self.assertEqual(before, cursor_count()) a = db.test.find() for x in a: break self.assertNotEqual(before, cursor_count()) del a self.assertEqual(before, cursor_count()) a = db.test.find().limit(10) for x in a: break self.assertEqual(before, cursor_count()) def test_base_object(self): c = self.connection self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(c.safe) self.assertEqual({}, c.get_lasterror_options()) db = c.test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(db.safe) self.assertEqual({}, db.get_lasterror_options()) coll = db.test self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertFalse(coll.safe) self.assertEqual({}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) c.safe = True c.set_lasterror_options(w=2, wtimeout=100) self.assertFalse(c.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(c.safe) self.assertEqual({'w': 2, 'wtimeout': 100}, c.get_lasterror_options()) db = c.test self.assertFalse(db.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(db.safe) self.assertEqual({'w': 2, 'wtimeout': 100}, db.get_lasterror_options()) coll = db.test self.assertFalse(coll.slave_okay) self.assertTrue(bool(c.read_preference)) self.assertTrue(coll.safe) self.assertEqual({'w': 2, 'wtimeout': 100}, coll.get_lasterror_options()) cursor = coll.find() self.assertFalse(cursor._Cursor__slave_okay) self.assertTrue(bool(cursor._Cursor__read_preference)) coll.insert({'foo': 'bar'}) self.assertEquals(1, coll.find({'foo': 'bar'}).count()) self.assert_(coll.find({'foo': 'bar'})) coll.remove({'foo': 'bar'}, safe=True) self.assertEquals(0, coll.find({'foo': 'bar'}).count()) # Set self.connection back to defaults c.safe = False c.unset_lasterror_options() self.assertFalse(self.connection.slave_okay) self.assertTrue(bool(self.connection.read_preference)) self.assertFalse(self.connection.safe) self.assertEqual({}, self.connection.get_lasterror_options()) def test_document_class(self): c = MasterSlaveConnection(self.master, self.slaves) db = c.pymongo_test db.test.insert({"x": 1}) time.sleep(1) self.assertEqual(dict, c.document_class) self.assert_(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) c.document_class = SON self.assertEqual(SON, c.document_class) self.assert_(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c = MasterSlaveConnection(self.master, self.slaves, document_class=SON) db = c.pymongo_test self.assertEqual(SON, c.document_class) self.assert_(isinstance(db.test.find_one(), SON)) self.assertFalse(isinstance(db.test.find_one(as_class=dict), SON)) c.document_class = dict self.assertEqual(dict, c.document_class) self.assert_(isinstance(db.test.find_one(), dict)) self.assertFalse(isinstance(db.test.find_one(), SON)) def test_tz_aware(self): dt = datetime.datetime.utcnow() conn = MasterSlaveConnection(self.master, self.slaves) self.assertEquals(False, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}, safe=True) time.sleep(0.5) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo) conn = MasterSlaveConnection(self.master, self.slaves, tz_aware=True) self.assertEquals(True, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}, safe=True) time.sleep(0.5) self.assertEqual(utc, db.tztest.find_one()['dt'].tzinfo) conn = MasterSlaveConnection(self.master, self.slaves, tz_aware=False) self.assertEquals(False, conn.tz_aware) db = conn.pymongo_test db.tztest.insert({'dt': dt}) time.sleep(0.5) self.assertEqual(None, db.tztest.find_one()['dt'].tzinfo)
# -*- coding: utf-8 -*- import gridfs from pymongo import Connection from pymongo.master_slave_connection import MasterSlaveConnection import dbconf try: dbconf.slave except: conn = Connection(host=dbconf.master) else: master_conn = Connection(host=dbconf.master) slave_conns = [Connection(host=i) for i in dbconf.slave] conn = MasterSlaveConnection(master_conn, slave_conns) androidesk = conn['androidesk'] imgfs = gridfs.GridFS(conn['imgfs']) """user collection has email: 注册邮箱 (str) nickname: 昵称 (str) passwd: 密码,明文 (str) avatar: 头像 (gridfs id) artist: 是否有创建主题权限 (bool) super: 是否是超级用户 (bool) wallpaper: 当前使用壁纸 (gridfs id) lastlogin: 最后一次登录 (datetime) IEME: IEME (str) phone_number: 电话号码 (str) logined: 是否已登录(str)