def test_get_default_database(self): host = one(self.hosts) uri = "mongodb://%s:%d/foo?replicaSet=%s" % ( host[0], host[1], self.name) c = MongoReplicaSetClient(uri, _connect=False) self.assertEqual(Database(c, 'foo'), c.get_default_database())
class TestPassiveAndHidden(HATestCase): def setUp(self): members = [{}, {'priority': 0}, {'arbiterOnly': True}, {'priority': 0, 'hidden': True}, {'priority': 0, 'slaveDelay': 5} ] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_passive_and_hidden(self): self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) passives = ha_tools.get_passives() passives = partition_nodes(passives) self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: utils.assertReadFromAll(self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) sleep(2 * MONITOR_INTERVAL) utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED) def tearDown(self): self.c.close() super(TestPassiveAndHidden, self).tearDown()
def test_alive(self): client = self._get_client() self.assertTrue(client.alive()) client = MongoReplicaSetClient("doesnt exist", replicaSet="rs", _connect=False) self.assertFalse(client.alive())
class TestMonitorRemovesRecoveringMember(HATestCase): # Members in STARTUP2 or RECOVERING states are shown in the primary's # isMaster response, but aren't secondaries and shouldn't be read from. # Verify that if a secondary goes into RECOVERING mode, the Monitor removes # it from the set of readers. def setUp(self): members = [{}, {'priority': 0}, {'priority': 0}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_monitor_removes_recovering_member(self): self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = partition_nodes(secondaries) utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) sleep(2 * MONITOR_INTERVAL) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member utils.assertReadFrom(self, self.c, _partition_node(secondary), mode) def tearDown(self): self.c.close() super(TestMonitorRemovesRecoveringMember, self).tearDown()
def test_get_default_database_with_authsource(self): # Ensure we distinguish database name from authSource. host = one(self.hosts) uri = "mongodb://%s:%d/foo?replicaSet=%s&authSource=src" % (host[0], host[1], self.name) c = MongoReplicaSetClient(uri, _connect=False) self.assertEqual(Database(c, "foo"), c.get_default_database())
class TestReplicaSetRequest(HATestCase): def setUp(self): members = [{}, {}, {'arbiterOnly': True}] res = ha_tools.start_replica_set(members) self.c = MongoReplicaSetClient(res[0], replicaSet=res[1], use_greenlets=use_greenlets, auto_start_request=True) def test_request_during_failover(self): primary = _partition_node(ha_tools.get_primary()) secondary = _partition_node(ha_tools.get_random_secondary()) self.assertTrue(self.c.auto_start_request) self.assertTrue(self.c.in_request()) rs_state = self.c._MongoReplicaSetClient__rs_state primary_pool = rs_state.get(primary).pool secondary_pool = rs_state.get(secondary).pool # Trigger start_request on primary pool utils.assertReadFrom(self, self.c, primary, PRIMARY) self.assertTrue(primary_pool.in_request()) # Fail over ha_tools.kill_primary() sleep(5) patience_seconds = 60 for _ in range(patience_seconds): try: if ha_tools.ha_tools_debug: print 'Waiting for failover' if ha_tools.get_primary(): # We have a new primary break except ConnectionFailure: pass sleep(1) else: self.fail("Problem with test: No new primary after %s seconds" % patience_seconds) try: # Trigger start_request on secondary_pool, which is becoming new # primary self.c.test.test.find_one() except AutoReconnect: # We've noticed the failover now pass # The old secondary is now primary utils.assertReadFrom(self, self.c, secondary, PRIMARY) self.assertTrue(self.c.in_request()) self.assertTrue(secondary_pool.in_request()) def tearDown(self): self.c.close() super(TestReplicaSetRequest, self).tearDown()
def _get_client(self, **kwargs): kwargs.setdefault('connectTimeoutMS', 500) # Assume there are no open mongods listening on a.com, b.com, .... bad_seeds = ['%s.com' % chr(ord('a') + i) for i in range(5)] seeds = ','.join(bad_seeds + [pair]) client = MongoReplicaSetClient(seeds, replicaSet=self.name, **kwargs) # In case of a slow test machine. client._refresh_timeout_sec = 30 return client
def test_ipv6(self): try: client = MongoReplicaSetClient("[::1]:%d" % (port,), replicaSet=self.name) except: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest("No IPv6") # Try a few simple things client = MongoReplicaSetClient("mongodb://[::1]:%d" % (port,), replicaSet=self.name) client = MongoReplicaSetClient("mongodb://[::1]:%d/?w=0;" "replicaSet=%s" % (port, self.name)) client = MongoReplicaSetClient("[::1]:%d,localhost:" "%d" % (port, port), replicaSet=self.name) client = MongoReplicaSetClient("localhost:%d,[::1]:" "%d" % (port, port), replicaSet=self.name) client.pymongo_test.test.save({"dummy": u"object"}) client.pymongo_test_bernie.test.save({"dummy": u"object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_bernie" in dbs) client.close()
class TestReplicaSetAuth(HATestCase): def setUp(self): members = [ {}, {'priority': 0}, {'priority': 0}, ] res = ha_tools.start_replica_set(members, auth=True) self.c = MongoReplicaSetClient(res[0], replicaSet=res[1], use_greenlets=use_greenlets) # Add an admin user to enable auth self.c.admin.add_user('admin', 'adminpass') self.c.admin.authenticate('admin', 'adminpass') self.db = self.c.pymongo_ha_auth self.db.add_user('user', 'userpass') self.c.admin.logout() def test_auth_during_failover(self): self.assertTrue(self.db.authenticate('user', 'userpass')) self.assertTrue(self.db.foo.insert({'foo': 'bar'}, safe=True, w=3, wtimeout=3000)) self.db.logout() self.assertRaises(OperationFailure, self.db.foo.find_one) primary = self.c.primary ha_tools.kill_members(['%s:%d' % primary], 2) # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) self.assertFalse(primary == self.c.primary) # Make sure we can still authenticate self.assertTrue(self.db.authenticate('user', 'userpass')) # And still query. self.db.read_preference = PRIMARY_PREFERRED self.assertEqual('bar', self.db.foo.find_one()['foo']) def tearDown(self): self.c.close() super(TestReplicaSetAuth, self).tearDown()
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = MongoClient(primary, use_greenlets=use_greenlets) secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets) rsc = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) try: self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) finally: rsc.close()
def test_passive_and_hidden(self): self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) passives = ha_tools.get_passives() passives = partition_nodes(passives) self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: utils.assertReadFromAll(self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) sleep(2 * MONITOR_INTERVAL) utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def setUp(self): members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], w=self.w) self.clear_ping_times()
class TestLastErrorDefaults(HATestCase): def setUp(self): members = [{}, {}] res = ha_tools.start_replica_set(members) self.seed, self.name = res self.c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) def test_get_last_error_defaults(self): if not version.at_least(self.c, (1, 9, 0)): raise SkipTest("Need MongoDB >= 1.9.0 to test getLastErrorDefaults") replset = self.c.local.system.replset.find_one() settings = replset.get('settings', {}) # This should cause a WTimeoutError for every write command settings['getLastErrorDefaults'] = { 'w': 3, 'wtimeout': 1 } replset['settings'] = settings replset['version'] = replset.get("version", 1) + 1 self.c.admin.command("replSetReconfig", replset) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.insert, {'_id': 0}) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.save, {'_id': 0, "a": 5}) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.update, {'_id': 0}, {"$set": {"a": 10}}) self.assertRaises(WTimeoutError, self.c.pymongo_test.test.remove, {'_id': 0}) def tearDown(self): self.c.close() super(TestLastErrorDefaults, self).tearDown()
def test_monitor_removes_recovering_member(self): self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = partition_nodes(secondaries) utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) sleep(2 * MONITOR_INTERVAL) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def setUp(self): members = [ {}, {'priority': 0}, {'priority': 0}, ] res = ha_tools.start_replica_set(members, auth=True) self.c = MongoReplicaSetClient(res[0], replicaSet=res[1], use_greenlets=use_greenlets) # Add an admin user to enable auth self.c.admin.add_user('admin', 'adminpass') self.c.admin.authenticate('admin', 'adminpass') self.db = self.c.pymongo_ha_auth self.db.add_user('user', 'userpass') self.c.admin.logout()
def test_properties(self): c = MongoReplicaSetClient(pair, replicaSet=self.name) c.admin.command('ping') self.assertEqual(c.primary, self.primary) self.assertEqual(c.hosts, self.hosts) self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.max_pool_size, 100) self.assertEqual(c.document_class, dict) self.assertEqual(c.tz_aware, False) # Make sure MRSC's properties are copied to Database and Collection for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.write_concern, {}) cursor = c.pymongo_test.test.find() self.assertEqual( ReadPreference.PRIMARY, cursor._Cursor__read_preference) c.close() tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}] secondary = Secondary(tag_sets) c = MongoReplicaSetClient(pair, replicaSet=self.name, max_pool_size=25, document_class=SON, tz_aware=True, read_preference=secondary, acceptablelatencyms=77) c.admin.command('ping') self.assertEqual(c.primary, self.primary) self.assertEqual(c.hosts, self.hosts) self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.max_pool_size, 25) self.assertEqual(c.document_class, SON) self.assertEqual(c.tz_aware, True) for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.read_preference, secondary) cursor = c.pymongo_test.test.find() self.assertEqual( secondary, cursor._Cursor__read_preference) nearest = Nearest([{'dc': 'ny'}, {}]) cursor = c.pymongo_test.test.find(read_preference=nearest) self.assertEqual( nearest, cursor._Cursor__read_preference) if version.at_least(c, (1, 7, 4)): self.assertEqual(c.max_bson_size, 16777216) else: self.assertEqual(c.max_bson_size, 4194304) c.close()
def setUp(self): members = [{}, {}, {'arbiterOnly': True}] res = ha_tools.start_replica_set(members) self.c = MongoReplicaSetClient(res[0], replicaSet=res[1], use_greenlets=use_greenlets, auto_start_request=True)
def test_deprecated(self): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) with self.assertRaises(DeprecationWarning): MongoReplicaSetClient()
def test_ship_of_theseus(self): c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) db = c.pymongo_test db.test.insert({}, w=len(c.secondaries) + 1) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [ha_tools.add_member() for _ in range(3)] # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary. for _ in xrange(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break sleep(1) else: self.fail("No failover") sleep(2 * MONITOR_INTERVAL) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. sleep(2 * MONITOR_INTERVAL) find_one(read_preference=SECONDARY) # Kill new members and switch back to original two members. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members([primary, secondary1]) # Should be able to reconnect to set again. sleep(2 * MONITOR_INTERVAL) find_one(read_preference=SECONDARY)
def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll( [primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000*1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(MongoClient( unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED ).admin.command('ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(MongoClient( unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED ).admin.command('ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times()
def test_init_disconnected_with_auth_failure(self): c = MongoReplicaSetClient( "mongodb://*****:*****@somedomainthatdoesntexist", replicaSet="rs", connectTimeoutMS=1, _connect=False) self.assertRaises(ConnectionFailure, c.pymongo_test.test.find_one)
def test_secondary_connection(self): self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test # Wait for replication... w = len(self.c.secondaries) + 1 db.test.remove({}, w=w) db.test.insert({'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # MongoClient succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, {'slave_okay': True} ]: client = MongoClient(primary_host, primary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(primary_host, client.host) self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(client.pymongo_test.test.find_one()) client = MongoClient(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, client.host) self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(client.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, client.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MongoClient # should do the same for unacknowledged writes. try: client.pymongo_test.test.insert({}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % (client,)) # Test direct connection to an arbiter client = MongoClient(arbiter_host, arbiter_port, **kwargs) self.assertEqual(arbiter_host, client.host) self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: client.pymongo_test.test.insert({}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.args[0])
def setUp(self): members = [{}, {}] res = ha_tools.start_replica_set(members) self.seed, self.name = res self.c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
def test_alive(self): client = MongoReplicaSetClient( 'doesnt exist', replicaSet='rs', _connect=False) self.assertFalse(client.alive())
class TestReadPreference(HATestCase): def setUp(self): members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], w=self.w) self.clear_ping_times() def set_ping_time(self, host, ping_time_seconds): Member._host_to_ping_time[host] = ping_time_seconds def clear_ping_times(self): Member._host_to_ping_time.clear() def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll( [primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000*1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(MongoClient( unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED ).admin.command('ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(MongoClient( unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED ).admin.command('ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times() def test_pinning(self): # To make the code terser, copy modes into local scope PRIMARY = ReadPreference.PRIMARY PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED SECONDARY = ReadPreference.SECONDARY SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED NEAREST = ReadPreference.NEAREST c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=True) # Verify that changing the mode unpins the member. We'll try it for # every relevant change of mode. for mode0, mode1 in itertools.permutations( (PRIMARY, SECONDARY, SECONDARY_PREFERRED, NEAREST), 2 ): # Try reading and then changing modes and reading again, see if we # read from a different host for _ in range(1000): # pin to this host host = utils.read_from_which_host(c, mode0) # unpin? new_host = utils.read_from_which_host(c, mode1) if host != new_host: # Reading with a different mode unpinned, hooray! break else: self.fail( "Changing from mode %s to mode %s never unpinned" % ( modes[mode0], modes[mode1])) # Now verify changing the tag_sets unpins the member. tags0 = [{'a': 'a'}, {}] tags1 = [{'a': 'x'}, {}] for _ in range(1000): host = utils.read_from_which_host(c, NEAREST, tags0) new_host = utils.read_from_which_host(c, NEAREST, tags1) if host != new_host: break else: self.fail( "Changing from tags %s to tags %s never unpinned" % ( tags0, tags1)) # Finally, verify changing the secondary_acceptable_latency_ms unpins # the member. for _ in range(1000): host = utils.read_from_which_host(c, SECONDARY, None, 15) new_host = utils.read_from_which_host(c, SECONDARY, None, 20) if host != new_host: break else: self.fail( "Changing secondary_acceptable_latency_ms from 15 to 20" " never unpinned") def tearDown(self): self.c.close() super(TestReadPreference, self).tearDown()
def test_properties(self): c = MongoReplicaSetClient(pair, replicaSet=self.name) c.admin.command("ping") self.assertEqual(c.primary, self.primary) self.assertEqual(c.hosts, self.hosts) self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.max_pool_size, 10) self.assertEqual(c.document_class, dict) self.assertEqual(c.tz_aware, False) # Make sure MRSC's properties are copied to Database and Collection for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.tag_sets, [{}]) self.assertEqual(obj.secondary_acceptable_latency_ms, 15) self.assertEqual(obj.slave_okay, False) self.assertEqual(obj.write_concern, {}) cursor = c.pymongo_test.test.find() self.assertEqual(ReadPreference.PRIMARY, cursor._Cursor__read_preference) self.assertEqual([{}], cursor._Cursor__tag_sets) self.assertEqual(15, cursor._Cursor__secondary_acceptable_latency_ms) self.assertEqual(False, cursor._Cursor__slave_okay) c.close() tag_sets = [{"dc": "la", "rack": "2"}, {"foo": "bar"}] c = MongoReplicaSetClient( pair, replicaSet=self.name, max_pool_size=25, document_class=SON, tz_aware=True, slaveOk=False, read_preference=ReadPreference.SECONDARY, tag_sets=copy.deepcopy(tag_sets), secondary_acceptable_latency_ms=77, ) c.admin.command("ping") self.assertEqual(c.primary, self.primary) self.assertEqual(c.hosts, self.hosts) self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.max_pool_size, 25) self.assertEqual(c.document_class, SON) self.assertEqual(c.tz_aware, True) for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.read_preference, ReadPreference.SECONDARY) self.assertEqual(obj.tag_sets, tag_sets) self.assertEqual(obj.secondary_acceptable_latency_ms, 77) self.assertEqual(obj.slave_okay, False) self.assertEqual(obj.safe, True) cursor = c.pymongo_test.test.find() self.assertEqual(ReadPreference.SECONDARY, cursor._Cursor__read_preference) self.assertEqual(tag_sets, cursor._Cursor__tag_sets) self.assertEqual(77, cursor._Cursor__secondary_acceptable_latency_ms) self.assertEqual(False, cursor._Cursor__slave_okay) cursor = c.pymongo_test.test.find( read_preference=ReadPreference.NEAREST, tag_sets=[{"dc": "ny"}, {}], secondary_acceptable_latency_ms=123 ) self.assertEqual(ReadPreference.NEAREST, cursor._Cursor__read_preference) self.assertEqual([{"dc": "ny"}, {}], cursor._Cursor__tag_sets) self.assertEqual(123, cursor._Cursor__secondary_acceptable_latency_ms) self.assertEqual(False, cursor._Cursor__slave_okay) if version.at_least(c, (1, 7, 4)): self.assertEqual(c.max_bson_size, 16777216) else: self.assertEqual(c.max_bson_size, 4194304) c.close()
def setUp(self): members = [ # primary { 'tags': { 'dc': 'ny', 'name': 'primary' } }, # secondary { 'tags': { 'dc': 'la', 'name': 'secondary' }, 'priority': 0 }, # other_secondary { 'tags': { 'dc': 'ny', 'name': 'other_secondary' }, 'priority': 0 }, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary' ] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary' ] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, w=self.w) self.db.test.insert([{'foo': i} for i in xrange(10)], w=self.w) self.clear_ping_times()
def test_ipv6(self): try: client = MongoReplicaSetClient("[::1]:%d" % (port,), replicaSet=self.name) except: # Either mongod was started without --ipv6 # or the OS doesn't support it (or both). raise SkipTest("No IPv6") # Try a few simple things client = MongoReplicaSetClient("mongodb://[::1]:%d" % (port,), replicaSet=self.name) client = MongoReplicaSetClient("mongodb://[::1]:%d/?safe=true;" "replicaSet=%s" % (port, self.name)) client = MongoReplicaSetClient("[::1]:%d,localhost:" "%d" % (port, port), replicaSet=self.name) client = MongoReplicaSetClient("localhost:%d,[::1]:" "%d" % (port, port), replicaSet=self.name) client.pymongo_test.test.save({"dummy": u"object"}) client.pymongo_test_bernie.test.save({"dummy": u"object"}) dbs = client.database_names() self.assertTrue("pymongo_test" in dbs) self.assertTrue("pymongo_test_bernie" in dbs) client.close()
class TestReadPreference(HATestCase): def setUp(self): members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], w=self.w) self.clear_ping_times() def set_ping_time(self, host, ping_time_seconds): Member._host_to_ping_time[host] = ping_time_seconds def clear_ping_times(self): Member._host_to_ping_time.clear() def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll( [primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000*1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(MongoClient( unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED ).admin.command('ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(MongoClient( unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED ).admin.command('ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times() def test_pinning(self): # To make the code terser, copy modes into local scope PRIMARY = ReadPreference.PRIMARY PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED SECONDARY = ReadPreference.SECONDARY SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED NEAREST = ReadPreference.NEAREST c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=True) # Verify that changing the mode unpins the member. We'll try it for # every relevant change of mode. for mode0, mode1 in permutations( (PRIMARY, SECONDARY, SECONDARY_PREFERRED, NEAREST), 2 ): # Try reading and then changing modes and reading again, see if we # read from a different host for _ in range(1000): # pin to this host host = utils.read_from_which_host(c, mode0) # unpin? new_host = utils.read_from_which_host(c, mode1) if host != new_host: # Reading with a different mode unpinned, hooray! break else: self.fail( "Changing from mode %s to mode %s never unpinned" % ( modes[mode0], modes[mode1])) # Now verify changing the tag_sets unpins the member. tags0 = [{'a': 'a'}, {}] tags1 = [{'a': 'x'}, {}] for _ in range(1000): host = utils.read_from_which_host(c, NEAREST, tags0) new_host = utils.read_from_which_host(c, NEAREST, tags1) if host != new_host: break else: self.fail( "Changing from tags %s to tags %s never unpinned" % ( tags0, tags1)) # Finally, verify changing the secondary_acceptable_latency_ms unpins # the member. for _ in range(1000): host = utils.read_from_which_host(c, SECONDARY, None, 15) new_host = utils.read_from_which_host(c, SECONDARY, None, 20) if host != new_host: break else: self.fail( "Changing secondary_acceptable_latency_ms from 15 to 20" " never unpinned") def tearDown(self): self.c.close() super(TestReadPreference, self).tearDown()
def __init__(self, hosts): MockClientBase.__init__(self) MongoReplicaSetClient.__init__( self, hosts, replicaSet=MOCK_RS_NAME)
def test_properties(self): c = MongoReplicaSetClient(pair, replicaSet=self.name) c.admin.command('ping') self.assertEqual(c.primary, self.primary) self.assertEqual(c.hosts, self.hosts) self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.max_pool_size, 10) self.assertEqual(c.document_class, dict) self.assertEqual(c.tz_aware, False) # Make sure MRSC's properties are copied to Database and Collection for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.tag_sets, [{}]) self.assertEqual(obj.secondary_acceptable_latency_ms, 15) self.assertEqual(obj.slave_okay, False) self.assertEqual(obj.write_concern, {}) cursor = c.pymongo_test.test.find() self.assertEqual( ReadPreference.PRIMARY, cursor._Cursor__read_preference) self.assertEqual([{}], cursor._Cursor__tag_sets) self.assertEqual(15, cursor._Cursor__secondary_acceptable_latency_ms) self.assertEqual(False, cursor._Cursor__slave_okay) c.close() tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}] c = MongoReplicaSetClient(pair, replicaSet=self.name, max_pool_size=25, document_class=SON, tz_aware=True, slaveOk=False, read_preference=ReadPreference.SECONDARY, tag_sets=copy.deepcopy(tag_sets), secondary_acceptable_latency_ms=77) c.admin.command('ping') self.assertEqual(c.primary, self.primary) self.assertEqual(c.hosts, self.hosts) self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.max_pool_size, 25) self.assertEqual(c.document_class, SON) self.assertEqual(c.tz_aware, True) for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.read_preference, ReadPreference.SECONDARY) self.assertEqual(obj.tag_sets, tag_sets) self.assertEqual(obj.secondary_acceptable_latency_ms, 77) self.assertEqual(obj.slave_okay, False) self.assertEqual(obj.safe, True) cursor = c.pymongo_test.test.find() self.assertEqual( ReadPreference.SECONDARY, cursor._Cursor__read_preference) self.assertEqual(tag_sets, cursor._Cursor__tag_sets) self.assertEqual(77, cursor._Cursor__secondary_acceptable_latency_ms) self.assertEqual(False, cursor._Cursor__slave_okay) cursor = c.pymongo_test.test.find( read_preference=ReadPreference.NEAREST, tag_sets=[{'dc':'ny'}, {}], secondary_acceptable_latency_ms=123) self.assertEqual( ReadPreference.NEAREST, cursor._Cursor__read_preference) self.assertEqual([{'dc':'ny'}, {}], cursor._Cursor__tag_sets) self.assertEqual(123, cursor._Cursor__secondary_acceptable_latency_ms) self.assertEqual(False, cursor._Cursor__slave_okay) if version.at_least(c, (1, 7, 4)): self.assertEqual(c.max_bson_size, 16777216) else: self.assertEqual(c.max_bson_size, 4194304) c.close()
def get_collection(): mrsc_options = config['metadata'].get('options', {}) mrsc_options['read_preference'] = pymongo.ReadPreference.PRIMARY meta_db = MongoReplicaSetClient(config['metadata']['url'], **mrsc_options) return meta_db[config['metadata']['jobs']['db']]['jobs']
def _get_client(self, **kwargs): return MongoReplicaSetClient(pair, replicaSet=self.name, **kwargs)