def test_primary_stepdown(self): c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary ha_tools.stepdown_primary() # Wait for new primary patience_seconds = 30 for _ in xrange(patience_seconds): sleep(1) rs_state = c._MongoReplicaSetClient__rs_state if rs_state.writer and rs_state.writer != primary: if ha_tools.get_primary(): # New primary stepped up new_primary = _partition_node(ha_tools.get_primary()) self.assertEqual(new_primary, rs_state.writer) new_secondaries = partition_nodes( ha_tools.get_secondaries()) self.assertEqual(set(new_secondaries), rs_state.secondaries) break else: self.fail( "No new primary after %s seconds. Old primary was %s, current" " is %s" % (patience_seconds, primary, ha_tools.get_primary()))
def test_ship_of_theseus(self): c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) db = c.pymongo_test db.test.insert({}, w=len(c.secondaries) + 1) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [ha_tools.add_member() for _ in range(3)] # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary. for _ in xrange(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break sleep(1) else: self.fail("No failover") sleep(2 * MONITOR_INTERVAL) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. sleep(2 * MONITOR_INTERVAL) find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self, done): loop = IOLoop.instance() c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) c.open_sync() db = c.pymongo_test w = len(c.secondaries) + 1 db.test.insert({}, w=w) primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() ha_tools.add_member() ha_tools.add_member() ha_tools.add_member() # Wait for new members to join for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break yield gen.Task(loop.add_timeout, time.time() + 1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary for _ in xrange(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break yield gen.Task(loop.add_timeout, time.time() + 1) else: self.fail("No failover") # Ensure monitor picks up new members yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL) try: yield motor.Op(db.test.find_one) except AutoReconnect: # Might take one try to reconnect yield gen.Task(loop.add_timeout, time.time() + 1) # No error yield motor.Op(db.test.find_one) yield motor.Op(db.test.find_one, read_preference=SECONDARY) done()
def test_secondary_connection(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test db.test.remove({}, safe=True, w=len(self.c.secondaries)) # Force replication... w = len(self.c.secondaries) + 1 db.test.insert({'foo': 'bar'}, safe=True, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets).is_primary) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.NEAREST).is_primary) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY).is_primary) for kwargs in [ {'read_preference': ReadPreference.PRIMARY_PREFERRED}, {'read_preference': ReadPreference.SECONDARY}, {'read_preference': ReadPreference.SECONDARY_PREFERRED}, {'read_preference': ReadPreference.NEAREST}, {'slave_okay': True}, ]: conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) self.assert_(conn.pymongo_test.test.find_one()) # Test direct connection to an arbiter secondary_host = ha_tools.get_arbiters()[0] host, port = ha_tools.get_arbiters()[0].split(':') port = int(port) conn = Connection(host, port) self.assertEqual(host, conn.host) self.assertEqual(port, conn.port)
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = yield motor.MotorClient(primary).open() secondary_cx = yield motor.MotorClient(secondary).open() rsc = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield rsc.open() try: self.assertTrue((yield primary_cx.alive())) self.assertTrue((yield secondary_cx.alive())) self.assertTrue((yield rsc.alive())) ha_tools.kill_primary() yield self.pause(0.5) self.assertFalse((yield primary_cx.alive())) self.assertTrue((yield secondary_cx.alive())) self.assertFalse((yield rsc.alive())) ha_tools.kill_members([secondary], 2) yield self.pause(0.5) self.assertFalse((yield primary_cx.alive())) self.assertFalse((yield secondary_cx.alive())) self.assertFalse((yield rsc.alive())) finally: rsc.close()
def test_alive(self, done): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = motor.MotorClient(primary).open_sync() secondary_cx = motor.MotorClient(secondary).open_sync() rsc = motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open_sync() loop = IOLoop.instance() try: yield AssertTrue(primary_cx.alive) yield AssertTrue(secondary_cx.alive) yield AssertTrue(rsc.alive) ha_tools.kill_primary() yield gen.Task(loop.add_timeout, time.time() + 0.5) yield AssertFalse(primary_cx.alive) yield AssertTrue(secondary_cx.alive) # Sometimes KeyError: https://jira.mongodb.org/browse/PYTHON-467 yield AssertFalse(rsc.alive) ha_tools.kill_members([secondary], 2) yield gen.Task(loop.add_timeout, time.time() + 0.5) yield AssertFalse(primary_cx.alive) yield AssertFalse(secondary_cx.alive) yield AssertFalse(rsc.alive) finally: rsc.close() done()
def test_stepdown_triggers_refresh(self, done): c_find_one = motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open_sync() # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual( one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1) # Trigger a refresh yield AssertRaises(AutoReconnect, c_find_one.test.test.find_one) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1) # We've detected the stepdown self.assertTrue( not c_find_one.primary or primary != _partition_node(c_find_one.primary)) done()
def test_stepdown_triggers_refresh(self): c_find_one = yield motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open() # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual(one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes yield self.pause(1) # Trigger a refresh with assert_raises(AutoReconnect): yield c_find_one.test.test.find_one() # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield self.pause(1) # We've detected the stepdown self.assertTrue(not c_find_one.primary or primary != _partition_node(c_find_one.primary))
def test_recovering_member_triggers_refresh(self): # To test that find_one() and count() trigger immediate refreshes, # we'll create a separate client for each self.c_find_one, self.c_count = yield [ motor.MotorReplicaSetClient(self.seed, replicaSet=self.name, read_preference=SECONDARY).open() for _ in xrange(2) ] # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] # Pre-condition: just make sure they all connected OK for c in self.c_find_one, self.c_count: self.assertEqual(one(c.secondaries), _partition_node(secondary)) ha_tools.set_maintenance(secondary, True) # Trigger a refresh in various ways with assert_raises(AutoReconnect): yield self.c_find_one.test.test.find_one() with assert_raises(AutoReconnect): yield self.c_count.test.test.count() # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield self.pause(1) for c in self.c_find_one, self.c_count: self.assertFalse(c.secondaries) self.assertEqual(_partition_node(primary), c.primary)
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = connected( MongoClient( primary, serverSelectionTimeoutMS=self.server_selection_timeout)), secondary_cx = connected( MongoClient( secondary, serverSelectionTimeoutMS=self.server_selection_timeout)) rsc = connected( MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout)) self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive())
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = MongoClient(primary, use_greenlets=use_greenlets) secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets) rsc = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) try: self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) finally: rsc.close()
def test_stepdown_triggers_refresh(self): c_find_one = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual( one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes sleep(1) # Trigger a refresh self.assertRaises(AutoReconnect, c_find_one.test.test.find_one) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled sleep(1) # We've detected the stepdown self.assertTrue( not c_find_one.primary or _partition_node(primary) != c_find_one.primary)
def test_recovering_member_triggers_refresh(self): # To test that find_one() and count() trigger immediate refreshes, # we'll create a separate client for each self.c_find_one, self.c_count = yield [ motor.MotorReplicaSetClient( self.seed, replicaSet=self.name, read_preference=SECONDARY ).open() for _ in xrange(2)] # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] # Pre-condition: just make sure they all connected OK for c in self.c_find_one, self.c_count: self.assertEqual(one(c.secondaries), _partition_node(secondary)) ha_tools.set_maintenance(secondary, True) # Trigger a refresh in various ways with assert_raises(AutoReconnect): yield self.c_find_one.test.test.find_one() with assert_raises(AutoReconnect): yield self.c_count.test.test.count() # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield self.pause(1) for c in self.c_find_one, self.c_count: self.assertFalse(c.secondaries) self.assertEqual(_partition_node(primary), c.primary)
def test_stepdown_triggers_refresh(self): c_find_one = yield motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open() # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual( one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes yield self.pause(1) # Trigger a refresh with assert_raises(AutoReconnect): yield c_find_one.test.test.find_one() # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield self.pause(1) # We've detected the stepdown self.assertTrue( not c_find_one.primary or primary != _partition_node(c_find_one.primary))
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = MongoClient(primary, use_greenlets=use_greenlets) secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets) rsc = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) try: self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) finally: rsc.close()
def test_stepdown_triggers_refresh(self): c_find_one = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual(one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes sleep(1) # Trigger a refresh self.assertRaises(AutoReconnect, c_find_one.test.test.find_one) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled sleep(1) # We've detected the stepdown self.assertTrue(not c_find_one.primary or primary != _partition_node(c_find_one.primary))
def test_request_during_failover(self): primary = _partition_node(ha_tools.get_primary()) secondary = _partition_node(ha_tools.get_random_secondary()) self.assertTrue(self.c.auto_start_request) self.assertTrue(self.c.in_request()) rs_state = self.c._MongoReplicaSetClient__rs_state primary_pool = rs_state.get(primary).pool secondary_pool = rs_state.get(secondary).pool # Trigger start_request on primary pool utils.assertReadFrom(self, self.c, primary, PRIMARY) self.assertTrue(primary_pool.in_request()) # Fail over ha_tools.kill_primary() sleep(5) patience_seconds = 60 for _ in range(patience_seconds): try: if ha_tools.ha_tools_debug: print 'Waiting for failover' if ha_tools.get_primary(): # We have a new primary break except ConnectionFailure: pass sleep(1) else: self.fail("Problem with test: No new primary after %s seconds" % patience_seconds) try: # Trigger start_request on secondary_pool, which is becoming new # primary self.c.test.test.find_one() except AutoReconnect: # We've noticed the failover now pass # The old secondary is now primary utils.assertReadFrom(self, self.c, secondary, PRIMARY) self.assertTrue(self.c.in_request()) self.assertTrue(secondary_pool.in_request())
def test_ship_of_theseus(self): c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() db = c.motor_test w = len(c.secondaries) + 1 db.test.insert({}, w=w) primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() ha_tools.add_member() ha_tools.add_member() ha_tools.add_member() # Wait for new members to join for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break yield self.pause(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary for _ in xrange(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break yield self.pause(1) else: self.fail("No failover") # Ensure monitor picks up new members yield self.pause(2 * MONITOR_INTERVAL) try: yield db.test.find_one() except AutoReconnect: # Might take one try to reconnect yield self.pause(1) # No error yield db.test.find_one() yield db.test.find_one(read_preference=SECONDARY)
def test_primary_stepdown(self): c = MongoClient(self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") ha_tools.stepdown_primary() # Wait for new primary. wait_until(lambda: (ha_tools.get_primary() and c.primary == partition_node(ha_tools.get_primary())), "discover new primary", timeout=30) wait_until(lambda: len(c.secondaries) == 2, "discover new secondaries", timeout=30)
def setUp(self): super(TestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) self.w = len(self.c.secondaries) + 1 self.db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=self.w)) self.db.test.delete_many({}) self.db.test.insert_many([{'foo': i} for i in xrange(10)]) self.clear_ping_times()
def test_primary_stepdown(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: c.primary, "discover primary") wait_until(lambda: len(c.secondaries) == 2, "discover secondaries") ha_tools.stepdown_primary() # Wait for new primary. wait_until(lambda: (ha_tools.get_primary() and c.primary == partition_node(ha_tools.get_primary())), "discover new primary", timeout=30) wait_until(lambda: len(c.secondaries) == 2, "discover new secondaries", timeout=30)
def setUp(self): super(MotorTestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} # Synchronous PyMongo interfaces for convenience self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient( self.seed, replicaSet=self.name) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], w=self.w) self.clear_ping_times()
def setUp(self): super(MotorTestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} # Synchronous PyMongo interfaces for convenience self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient( self.seed, replicaSet=self.name) self.db = self.c.motor_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], w=self.w) self.clear_ping_times()
def test_primary_stepdown(self): c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary ha_tools.stepdown_primary() # Wait for new primary patience_seconds = 30 for _ in xrange(patience_seconds): sleep(1) rs_state = c._MongoReplicaSetClient__rs_state if rs_state.writer and rs_state.writer != primary: # New primary stepped up new_primary = _partition_node(ha_tools.get_primary()) self.assertEqual(new_primary, rs_state.writer) new_secondaries = partition_nodes(ha_tools.get_secondaries()) self.assertEqual(set(new_secondaries), rs_state.secondaries) break else: self.fail( "No new primary after %s seconds. Old primary was %s, current" " is %s" % (patience_seconds, primary, ha_tools.get_primary()))
def setUp(self): members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, safe=True, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], safe=True, w=self.w) self.clear_ping_times()
def setUp(self): members = [ # primary {"tags": {"dc": "ny", "name": "primary"}}, # secondary {"tags": {"dc": "la", "name": "secondary"}, "priority": 0}, # other_secondary {"tags": {"dc": "ny", "name": "other_secondary"}, "priority": 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual("primary", self.primary_tags["name"]) self.primary_dc = {"dc": self.primary_tags["dc"]} secondaries = ha_tools.get_secondaries() (secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "secondary"] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {"dc": self.secondary_tags["dc"]} (other_secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "other_secondary"] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {"dc": self.other_secondary_tags["dc"]} self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, safe=True, w=self.w) self.db.test.insert([{"foo": i} for i in xrange(10)], safe=True, w=self.w) self.clear_ping_times()
def test_recovering_member_triggers_refresh(self): # To test that find_one() and count() trigger immediate refreshes, # we'll create a separate client for each self.c_find_one, self.c_count = [ MongoClient( self.seed, replicaSet=self.name, read_preference=SECONDARY, serverSelectionTimeoutMS=self.server_selection_timeout) for _ in xrange(2)] # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] # Pre-condition: just make sure they all connected OK for c in self.c_find_one, self.c_count: wait_until( lambda: c.primary == partition_node(primary), 'connect to the primary') wait_until( lambda: one(c.secondaries) == partition_node(secondary), 'connect to the secondary') ha_tools.set_maintenance(secondary, True) # Trigger a refresh in various ways self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one) self.assertRaises(AutoReconnect, self.c_count.test.test.count) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled time.sleep(1) self.assertFalse(self.c_find_one.secondaries) self.assertEqual(partition_node(primary), self.c_find_one.primary) self.assertFalse(self.c_count.secondaries) self.assertEqual(partition_node(primary), self.c_count.primary)
def test_secondary_connection(self): self.c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) wait_until(lambda: len(self.c.secondaries), "discover secondary") # Wait for replication... w = len(self.c.secondaries) + 1 db = self.c.get_database("pymongo_test", write_concern=WriteConcern(w=w)) db.test.delete_many({}) db.test.insert_one({'foo': 'bar'}) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # MongoClient succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, ]: client = MongoClient( primary_host, primary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: primary_host == client.host, "connect to primary") self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(client.pymongo_test.test.find_one()) client = MongoClient( secondary_host, secondary_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: secondary_host == client.host, "connect to secondary") self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(client.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, client.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MongoClient # should do the same for unacknowledged writes. try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % (client,)) # Test direct connection to an arbiter client = MongoClient( arbiter_host, arbiter_port, serverSelectionTimeoutMS=self.server_selection_timeout, **kwargs) wait_until(lambda: arbiter_host == client.host, "connect to arbiter") self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: client.get_database( "pymongo_test", write_concern=WriteConcern(w=0)).test.insert_one({}) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into arbiter client %s should' 'have raised exception' % (client,))
def test_secondary_connection(self): self.c = yield motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open() self.assertTrue(bool(len(self.c.secondaries))) db = self.c.motor_test yield db.test.remove({}, w=len(self.c.secondaries)) # Wait for replication... w = len(self.c.secondaries) + 1 yield db.test.insert({'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # A connection succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, ]: client = yield motor.MotorClient( primary_host, primary_port, **kwargs).open() self.assertEqual(primary_host, client.host) self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue((yield client.motor_test.test.find_one())) client = yield motor.MotorClient( secondary_host, secondary_port, **kwargs).open() self.assertEqual(secondary_host, client.host) self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(( yield client.motor_test.test.find_one())) else: with assert_raises(AutoReconnect): yield client.motor_test.test.find_one() # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MotorClient # should do the same for unacknowledged writes. try: yield client.motor_test.test.insert({}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % client) # Test direct connection to an arbiter client = yield motor.MotorClient( arbiter_host, arbiter_port, **kwargs).open() self.assertEqual(arbiter_host, client.host) self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: yield client.motor_test.test.insert({}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.message)
def test_read_preference(self): # This is long, but we put all the tests in one function to save time # on setUp, which takes about 30 seconds to bring up a replica set. # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() @gen.coroutine def read_from_which_host( rsc, mode, tag_sets=None, latency=15, ): db = rsc.motor_test db.read_preference = mode if isinstance(tag_sets, dict): tag_sets = [tag_sets] db.tag_sets = tag_sets or [{}] db.secondary_acceptable_latency_ms = latency cursor = db.test.find() try: yield cursor.fetch_next raise gen.Return(cursor.delegate._Cursor__connection_id) except AutoReconnect: raise gen.Return(None) @gen.coroutine def assert_read_from(member, *args, **kwargs): for _ in range(10): used = yield read_from_which_host(c, *args, **kwargs) self.assertEqual(member, used) @gen.coroutine def assert_read_from_all(members, *args, **kwargs): members = set(members) all_used = set() for _ in range(100): used = yield read_from_which_host(c, *args, **kwargs) all_used.add(used) if members == all_used: raise gen.Return() # Success # This will fail self.assertEqual(members, all_used) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag) yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY yield assert_read_from_all( [secondary, other_secondary], SECONDARY, latency=9999999) # SECONDARY_PREFERRED yield assert_read_from_all( [secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999) # Multiple tags yield assert_read_from( secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags yield assert_read_from( primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set yield assert_read_from_all( [secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}], latency=9999999) yield assert_read_from( other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() yield assert_read_from_all( [primary, secondary, other_secondary], NEAREST, latency=9999999) yield assert_read_from_all( [primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}], latency=9999999) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 milliseconds. self.set_ping_time(other_secondary, 10) # Nearest member, no tags yield assert_read_from(primary, NEAREST) # Tags override nearness yield assert_read_from(primary, NEAREST, {'name': 'primary'}) yield assert_read_from(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 milliseconds. self.set_ping_time(secondary, 0) yield assert_read_from(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) yield assert_read_from(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members yield assert_read_from_all( [primary, secondary, other_secondary], NEAREST, latency=9999999) self.clear_ping_times() yield assert_read_from_all( [primary, other_secondary], NEAREST, [{'dc': 'ny'}], latency=9999999) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone yield self.pause(2 * MONITOR_INTERVAL) # PRIMARY yield assert_read_from(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary yield assert_read_from_all( [secondary, other_secondary], PRIMARY_PREFERRED, latency=9999999) yield assert_read_from( secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY yield assert_read_from_all( [secondary, other_secondary], SECONDARY, latency=9999999) # Only primary matches yield assert_read_from(None, SECONDARY, {'name': 'primary'}) # No matching secondaries yield assert_read_from(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED yield assert_read_from_all( [secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999) # Mode and tags both match yield assert_read_from( secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() yield assert_read_from_all( [secondary, other_secondary], NEAREST, latency=9999999) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break yield self.pause(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(pymongo.MongoClient( unpartition_node(primary), slave_okay=True ).admin.command('ismaster')['ismaster']) yield self.pause(2 * MONITOR_INTERVAL) # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED yield assert_read_from(primary, PRIMARY_PREFERRED) # SECONDARY yield assert_read_from(other_secondary, SECONDARY) yield assert_read_from( other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches yield assert_read_from(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED yield assert_read_from(other_secondary, SECONDARY_PREFERRED) yield assert_read_from( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary yield assert_read_from( primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST yield assert_read_from_all( [primary, other_secondary], NEAREST, latency=9999999) yield assert_read_from( other_secondary, NEAREST, {'name': 'other_secondary'}) yield assert_read_from(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(pymongo.MongoClient( unpartition_node(primary), slave_okay=True ).admin.command('ismaster')['ismaster']) # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED yield assert_read_from(primary, PRIMARY_PREFERRED) yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY yield assert_read_from(None, SECONDARY) yield assert_read_from(None, SECONDARY, self.other_secondary_dc) yield assert_read_from(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED yield assert_read_from(primary, SECONDARY_PREFERRED) yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc) yield assert_read_from( primary, SECONDARY_PREFERRED, {'name': 'secondary'}) yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST yield assert_read_from(primary, NEAREST) yield assert_read_from(None, NEAREST, self.secondary_dc) yield assert_read_from(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) yield assert_read_from(primary, NEAREST) yield assert_read_from(None, NEAREST, self.secondary_dc) self.clear_ping_times()
def test_ship_of_theseus(self): c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) db = c.pymongo_test db.test.insert({}, w=len(c.secondaries) + 1) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [] for i in range(3): new_hosts.append(ha_tools.add_member()) # RS closes all connections after reconfig. for j in xrange(30): try: if ha_tools.get_primary(): break except (ConnectionFailure, OperationFailure): pass sleep(1) else: self.fail("Couldn't recover from reconfig") # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) sleep(5) # Wait for primary. for _ in xrange(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break sleep(1) else: self.fail("No failover") sleep(2 * MONITOR_INTERVAL) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. sleep(2 * MONITOR_INTERVAL) find_one(read_preference=SECONDARY) # Kill new members and switch back to original two members. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members([primary, secondary1]) # Wait for members to figure out they're secondaries. for _ in xrange(30): try: if len(ha_tools.get_secondaries()) == 2: break except ConnectionFailure: pass sleep(1) else: self.fail("Original members didn't become secondaries") # Should be able to reconnect to set again. sleep(2 * MONITOR_INTERVAL) find_one(read_preference=SECONDARY)
def test_secondary_connection(self): self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test db.test.remove({}, safe=True, w=len(self.c.secondaries)) # Force replication... w = len(self.c.secondaries) + 1 db.test.insert({'foo': 'bar'}, safe=True, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) self.assertTrue( Connection(primary_host, primary_port, use_greenlets=use_greenlets).is_primary) self.assertTrue( Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary) self.assertTrue( Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary) self.assertTrue( Connection(primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.NEAREST).is_primary) self.assertTrue( Connection(primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY).is_primary) for kwargs in [ { 'read_preference': ReadPreference.PRIMARY_PREFERRED }, { 'read_preference': ReadPreference.SECONDARY }, { 'read_preference': ReadPreference.SECONDARY_PREFERRED }, { 'read_preference': ReadPreference.NEAREST }, { 'slave_okay': True }, ]: conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) self.assert_(conn.pymongo_test.test.find_one()) # Test direct connection to an arbiter host, port = ha_tools.get_arbiters()[0].split(':') port = int(port) conn = Connection(host, port) self.assertEqual(host, conn.host) self.assertEqual(port, conn.port)
def test_ship_of_theseus(self): c = MongoReplicaSetClient( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) db = c.pymongo_test db.test.insert({}, w=len(c.secondaries) + 1) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [ha_tools.add_member() for _ in range(3)] # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary. for _ in xrange(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break sleep(1) else: self.fail("No failover") sleep(2 * MONITOR_INTERVAL) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. sleep(2 * MONITOR_INTERVAL) find_one(read_preference=SECONDARY) # Kill new members and switch back to original two members. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members([primary, secondary1]) # Should be able to reconnect to set again. sleep(2 * MONITOR_INTERVAL) find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self): c = MongoClient( self.seed, replicaSet=self.name, serverSelectionTimeoutMS=self.server_selection_timeout) db = c.get_database( "pymongo_test", write_concern=WriteConcern(w=len(c.secondaries) + 1)) db.test.insert_one({}) find_one = db.test.find_one primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() new_hosts = [] for i in range(3): new_hosts.append(ha_tools.add_member()) # RS closes all connections after reconfig. for j in xrange(30): try: if ha_tools.get_primary(): break except (ConnectionFailure, OperationFailure): pass time.sleep(1) else: self.fail("Couldn't recover from reconfig") # Wait for new members to join. for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break time.sleep(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) time.sleep(5) wait_until(lambda: (ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2), "fail over", timeout=30) time.sleep(2 * self.heartbeat_frequency) # No error. find_one() find_one(read_preference=SECONDARY) # All members down. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members(new_hosts) # Should be able to reconnect to set even though original seed # list is useless. Use SECONDARY so we don't have to wait for # the election, merely for the client to detect members are up. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY) # Kill new members and switch back to original two members. ha_tools.kill_members(new_hosts, 9) self.assertRaises( ConnectionFailure, find_one, read_preference=SECONDARY) ha_tools.restart_members([primary, secondary1]) # Wait for members to figure out they're secondaries. wait_until(lambda: len(ha_tools.get_secondaries()) == 2, "detect two secondaries", timeout=30) # Should be able to reconnect to set again. time.sleep(2 * self.heartbeat_frequency) find_one(read_preference=SECONDARY)
def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = MongoReplicaSetClient(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy hosts into local scope primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, { 'dc': 'ny' }]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, { 'dc': 'ny' }]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll([primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000 * 1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue( MongoClient(unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED).admin.command( 'ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom(other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue( MongoClient(unpartition_node(primary), use_greenlets=use_greenlets, read_preference=PRIMARY_PREFERRED).admin.command( 'ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times()
def test_secondary_connection(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test db.test.remove({}, safe=True, w=len(self.c.secondaries)) # Wait for replication... w = len(self.c.secondaries) + 1 db.test.insert({'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # A Connection succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, {'slave_okay': True} ]: conn = Connection(primary_host, primary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(primary_host, conn.host) self.assertEqual(primary_port, conn.port) self.assertTrue(conn.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(conn.pymongo_test.test.find_one()) conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(conn.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, conn.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), Connection # should do the same for unacknowledged writes. try: conn.pymongo_test.test.insert({}, safe=False) except AutoReconnect, e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary connection %s should' 'have raised exception' % conn) # Test direct connection to an arbiter conn = Connection(arbiter_host, arbiter_port, **kwargs) self.assertEqual(arbiter_host, conn.host) self.assertEqual(arbiter_port, conn.port) self.assertFalse(conn.is_primary) # See explanation above try: conn.pymongo_test.test.insert({}, safe=False) except AutoReconnect, e: self.assertEqual('not master', e.args[0])
def test_secondary_connection(self): self.c = yield motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open() self.assertTrue(bool(len(self.c.secondaries))) db = self.c.motor_test yield db.test.remove({}, w=len(self.c.secondaries)) # Wait for replication... w = len(self.c.secondaries) + 1 yield db.test.insert({'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # A connection succeeds no matter the read preference for kwargs in [ { 'read_preference': PRIMARY }, { 'read_preference': PRIMARY_PREFERRED }, { 'read_preference': SECONDARY }, { 'read_preference': SECONDARY_PREFERRED }, { 'read_preference': NEAREST }, ]: client = yield motor.MotorClient(primary_host, primary_port, **kwargs).open() self.assertEqual(primary_host, client.host) self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue((yield client.motor_test.test.find_one())) client = yield motor.MotorClient(secondary_host, secondary_port, **kwargs).open() self.assertEqual(secondary_host, client.host) self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue((yield client.motor_test.test.find_one())) else: with assert_raises(AutoReconnect): yield client.motor_test.test.find_one() # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MotorClient # should do the same for unacknowledged writes. try: yield client.motor_test.test.insert({}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % client) # Test direct connection to an arbiter client = yield motor.MotorClient(arbiter_host, arbiter_port, **kwargs).open() self.assertEqual(arbiter_host, client.host) self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: yield client.motor_test.test.insert({}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.message)
def test_read_preference(self): # This is long, but we put all the tests in one function to save time # on setUp, which takes about 30 seconds to bring up a replica set. # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() @gen.coroutine def read_from_which_host( rsc, mode, tag_sets=None, latency=15, ): db = rsc.motor_test db.read_preference = mode if isinstance(tag_sets, dict): tag_sets = [tag_sets] db.tag_sets = tag_sets or [{}] db.secondary_acceptable_latency_ms = latency cursor = db.test.find() try: yield cursor.fetch_next raise gen.Return(cursor.delegate._Cursor__connection_id) except AutoReconnect: raise gen.Return(None) @gen.coroutine def assert_read_from(member, *args, **kwargs): for _ in range(10): used = yield read_from_which_host(c, *args, **kwargs) self.assertEqual(member, used) @gen.coroutine def assert_read_from_all(members, *args, **kwargs): members = set(members) all_used = set() for _ in range(100): used = yield read_from_which_host(c, *args, **kwargs) all_used.add(used) if members == all_used: raise gen.Return() # Success # This will fail self.assertEqual(members, all_used) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag) yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY yield assert_read_from_all([secondary, other_secondary], SECONDARY, latency=9999999) # SECONDARY_PREFERRED yield assert_read_from_all([secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999) # Multiple tags yield assert_read_from(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags yield assert_read_from(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set yield assert_read_from_all([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}], latency=9999999) yield assert_read_from(other_secondary, SECONDARY_PREFERRED, [bad_tag, { 'dc': 'ny' }]) # NEAREST self.clear_ping_times() yield assert_read_from_all([primary, secondary, other_secondary], NEAREST, latency=9999999) yield assert_read_from_all([primary, other_secondary], NEAREST, [bad_tag, { 'dc': 'ny' }], latency=9999999) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 milliseconds. self.set_ping_time(other_secondary, 10) # Nearest member, no tags yield assert_read_from(primary, NEAREST) # Tags override nearness yield assert_read_from(primary, NEAREST, {'name': 'primary'}) yield assert_read_from(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 milliseconds. self.set_ping_time(secondary, 0) yield assert_read_from(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) yield assert_read_from(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members yield assert_read_from_all([primary, secondary, other_secondary], NEAREST, latency=9999999) self.clear_ping_times() yield assert_read_from_all([primary, other_secondary], NEAREST, [{ 'dc': 'ny' }], latency=9999999) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone yield self.pause(2 * MONITOR_INTERVAL) # PRIMARY yield assert_read_from(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary yield assert_read_from_all([secondary, other_secondary], PRIMARY_PREFERRED, latency=9999999) yield assert_read_from(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY yield assert_read_from_all([secondary, other_secondary], SECONDARY, latency=9999999) # Only primary matches yield assert_read_from(None, SECONDARY, {'name': 'primary'}) # No matching secondaries yield assert_read_from(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED yield assert_read_from_all([secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999) # Mode and tags both match yield assert_read_from(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() yield assert_read_from_all([secondary, other_secondary], NEAREST, latency=9999999) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break yield self.pause(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue( pymongo.MongoClient( unpartition_node(primary), slave_okay=True).admin.command('ismaster')['ismaster']) yield self.pause(2 * MONITOR_INTERVAL) # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED yield assert_read_from(primary, PRIMARY_PREFERRED) # SECONDARY yield assert_read_from(other_secondary, SECONDARY) yield assert_read_from(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches yield assert_read_from(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED yield assert_read_from(other_secondary, SECONDARY_PREFERRED) yield assert_read_from(other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary yield assert_read_from(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST yield assert_read_from_all([primary, other_secondary], NEAREST, latency=9999999) yield assert_read_from(other_secondary, NEAREST, {'name': 'other_secondary'}) yield assert_read_from(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue( pymongo.MongoClient( unpartition_node(primary), slave_okay=True).admin.command('ismaster')['ismaster']) # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED yield assert_read_from(primary, PRIMARY_PREFERRED) yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY yield assert_read_from(None, SECONDARY) yield assert_read_from(None, SECONDARY, self.other_secondary_dc) yield assert_read_from(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED yield assert_read_from(primary, SECONDARY_PREFERRED) yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc) yield assert_read_from(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST yield assert_read_from(primary, NEAREST) yield assert_read_from(None, NEAREST, self.secondary_dc) yield assert_read_from(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) yield assert_read_from(primary, NEAREST) yield assert_read_from(None, NEAREST, self.secondary_dc) self.clear_ping_times()
def test_secondary_connection(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test db.test.remove({}, safe=True, w=len(self.c.secondaries)) # Wait for replication... w = len(self.c.secondaries) + 1 db.test.insert({'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # A Connection succeeds no matter the read preference for kwargs in [ {'read_preference': ReadPreference.PRIMARY}, {'read_preference': ReadPreference.PRIMARY_PREFERRED}, {'read_preference': ReadPreference.SECONDARY}, {'read_preference': ReadPreference.SECONDARY_PREFERRED}, {'read_preference': ReadPreference.NEAREST}, {'slave_okay': True} ]: conn = Connection(primary_host, primary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(primary_host, conn.host) self.assertEqual(primary_port, conn.port) self.assertTrue(conn.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(conn.pymongo_test.test.find_one()) conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != ReadPreference.PRIMARY: self.assertTrue(conn.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, conn.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), Connection # should do the same for unacknowledged writes. try: conn.pymongo_test.test.insert({}, safe=False) except AutoReconnect, e: self.assertEqual('not master', e.message) else: self.fail( 'Unacknowledged insert into secondary connection %s should' 'have raised exception' % conn) # Test direct connection to an arbiter conn = Connection(arbiter_host, arbiter_port, **kwargs) self.assertEqual(arbiter_host, conn.host) self.assertEqual(arbiter_port, conn.port) self.assertFalse(conn.is_primary) # See explanation above try: conn.pymongo_test.test.insert({}, safe=False) except AutoReconnect, e: self.assertEqual('not master', e.message)
def test_read_preference(self): # This is long, but we put all the tests in one function to save time # on setUp, which takes about 30 seconds to bring up a replica set. # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=False) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy modes and hosts into local scope PRIMARY = ReadPreference.PRIMARY PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED SECONDARY = ReadPreference.SECONDARY SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED NEAREST = ReadPreference.NEAREST primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll( [primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000*1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(Connection( unpartition_node(primary), use_greenlets=use_greenlets, slave_okay=True ).admin.command('ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(Connection( unpartition_node(primary), use_greenlets=use_greenlets, slave_okay=True ).admin.command('ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times()
def post(self, op): """ Respond to a POST """ # Start RS if op == "start": request = self._parse_json(self.request.body) members = request["members"] try: # start RS and get its' params res = ha_tools.start_replica_set(members) except: raise tornado.httpserver._BadRequestException("Couldn't start RS!") rs_uri, rs_name = res rs_id = uuid.uuid4() self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_uri=rs_uri, rs_name=rs_name)) # Stop rs elif op == "stop": try: ha_tools.kill_all_members() except: raise tornado.httpserver._BadRequestException("Couldn't stop RS!") # Get primary elif op == "get_primary": request = self._parse_json(self.request.body) rs_id = request["rs"]["id"] rs_primary_uri = ha_tools.get_primary() self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_primary_uri=rs_primary_uri)) # Get secondaries elif op == "get_secondaries": request = self._parse_json(self.request.body) rs_id = request["rs"]["id"] rs_secondaries_uris = ha_tools.get_secondaries() self.write( self._template.load(op + self._ext).generate(rs_id=rs_id, rs_secondaries_uris=rs_secondaries_uris) ) # Get arbiters elif op == "get_arbiters": request = self._parse_json(self.request.body) rs_id = request["rs"]["id"] rs_arbiters_uris = ha_tools.get_arbiters() self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_arbiters_uris=rs_arbiters_uris)) # Kill primary elif op == "kill_primary": request = self._parse_json(self.request.body) rs_id = request["rs"]["id"] rs_killed_primary_uri = ha_tools.kill_primary() self.write( self._template.load(op + self._ext).generate(rs_id=rs_id, rs_killed_primary_uri=rs_killed_primary_uri) ) # Kill secondary elif op == "kill_secondary": request = self._parse_json(self.request.body) rs_id = request["rs"]["id"] rs_killed_secondary_uri = ha_tools.kill_secondary() self.write( self._template.load(op + self._ext).generate( rs_id=rs_id, rs_killed_secondary_uri=rs_killed_secondary_uri ) ) # Kill all secondaries elif op == "kill_all_secondaries": request = self._parse_json(self.request.body) rs_id = request["rs"]["id"] rs_killed_secondaries_uris = ha_tools.kill_all_secondaries() self.write( self._template.load(op + self._ext).generate( rs_id=rs_id, rs_killed_secondaries_uris=rs_killed_secondaries_uris ) )