def test_ship_of_theseus(self, done): loop = IOLoop.instance() c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) c.open_sync() db = c.pymongo_test w = len(c.secondaries) + 1 db.test.insert({}, w=w) primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() ha_tools.add_member() ha_tools.add_member() ha_tools.add_member() # Wait for new members to join for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break yield gen.Task(loop.add_timeout, time.time() + 1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary for _ in xrange(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break yield gen.Task(loop.add_timeout, time.time() + 1) else: self.fail("No failover") # Ensure monitor picks up new members yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL) try: yield motor.Op(db.test.find_one) except AutoReconnect: # Might take one try to reconnect yield gen.Task(loop.add_timeout, time.time() + 1) # No error yield motor.Op(db.test.find_one) yield motor.Op(db.test.find_one, read_preference=SECONDARY) done()
def test_stepdown_triggers_refresh(self): c_find_one = yield motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open() # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual(one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes yield self.pause(1) # Trigger a refresh with assert_raises(AutoReconnect): yield c_find_one.test.test.find_one() # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield self.pause(1) # We've detected the stepdown self.assertTrue(not c_find_one.primary or primary != c_find_one.primary)
def test_recovering_member_triggers_refresh(self): # To test that find_one() and count() trigger immediate refreshes, # we'll create a separate client for each self.c_find_one, self.c_count = yield [ motor.MotorReplicaSetClient(self.seed, replicaSet=self.name, read_preference=SECONDARY).open() for _ in range(2) ] # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] # Pre-condition: just make sure they all connected OK for c in self.c_find_one, self.c_count: self.assertEqual(one(c.secondaries), _partition_node(secondary)) ha_tools.set_maintenance(secondary, True) # Trigger a refresh in various ways with assert_raises(AutoReconnect): yield self.c_find_one.test.test.find_one() with assert_raises(AutoReconnect): yield self.c_count.test.test.count() # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield self.pause(1) for c in self.c_find_one, self.c_count: self.assertFalse(c.secondaries) self.assertEqual(_partition_node(primary), c.primary)
def test_recovering_member_triggers_refresh(self, done): # To test that find_one() and count() trigger immediate refreshes, # we'll create a separate connection for each self.c_find_one, self.c_count = [ motor.MotorReplicaSetClient( self.seed, replicaSet=self.name, read_preference=SECONDARY ).open_sync() for _ in xrange(2)] # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] # Pre-condition: just make sure they all connected OK for c in self.c_find_one, self.c_count: self.assertEqual(one(c.secondaries), _partition_node(secondary)) ha_tools.set_maintenance(secondary, True) # Trigger a refresh in various ways yield AssertRaises(AutoReconnect, self.c_find_one.test.test.find_one) yield AssertRaises(AutoReconnect, self.c_count.test.test.count) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield gen.Task( IOLoop.instance().add_timeout, time.time() + 1) for c in self.c_find_one, self.c_count: self.assertFalse(c.secondaries) self.assertEqual(_partition_node(primary), c.primary) done()
def test_stepdown_triggers_refresh(self, done): c_find_one = motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open_sync() # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual( one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1) # Trigger a refresh yield AssertRaises(AutoReconnect, c_find_one.test.test.find_one) # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1) # We've detected the stepdown self.assertTrue( not c_find_one.primary or primary != _partition_node(c_find_one.primary)) done()
def test_stepdown_triggers_refresh(self): c_find_one = yield motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open() # We've started the primary and one secondary primary = ha_tools.get_primary() secondary = ha_tools.get_secondaries()[0] self.assertEqual( one(c_find_one.secondaries), _partition_node(secondary)) ha_tools.stepdown_primary() # Make sure the stepdown completes yield self.pause(1) # Trigger a refresh with self.assertRaises(AutoReconnect): yield c_find_one.test.test.find_one() # Wait for the immediate refresh to complete - we're not waiting for # the periodic refresh, which has been disabled yield self.pause(1) # We've detected the stepdown self.assertTrue( not c_find_one.primary or primary != c_find_one.primary)
def test_ship_of_theseus(self): c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() db = c.motor_test w = len(c.secondaries) + 1 db.test.insert({}, w=w) primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() ha_tools.add_member() ha_tools.add_member() ha_tools.add_member() # Wait for new members to join for _ in range(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break yield self.pause(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary for _ in range(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break yield self.pause(1) else: self.fail("No failover") # Ensure monitor picks up new members yield self.pause(2 * MONITOR_INTERVAL) try: yield db.test.find_one() except AutoReconnect: # Might take one try to reconnect yield self.pause(1) # No error yield db.test.find_one() yield db.test.find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self): c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() db = c.motor_test w = len(c.secondaries) + 1 db.test.insert({}, w=w) primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() ha_tools.add_member() ha_tools.add_member() ha_tools.add_member() # Wait for new members to join for _ in range(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break yield self.pause(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary for _ in range(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break yield self.pause(1) else: self.fail("No failover") # Ensure monitor picks up new members yield self.pause(2 * MONITOR_INTERVAL) try: yield db.test.find_one() except AutoReconnect: # Might take one try to reconnect yield self.pause(1) # No error yield db.test.find_one() yield db.test.find_one(read_preference=SECONDARY)
def setUp(self): super(MotorTestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} # Synchronous PyMongo interfaces for convenience self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient( self.seed, replicaSet=self.name) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], w=self.w) self.clear_ping_times()
def setUp(self): super(MotorTestReadPreference, self).setUp() members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} # Synchronous PyMongo interfaces for convenience self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient( self.seed, replicaSet=self.name) self.db = self.c.motor_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, w=self.w) self.db.test.insert( [{'foo': i} for i in range(10)], w=self.w) self.clear_ping_times()
def test_monitor_removes_recovering_member(self): self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield self.c.open() secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = [_partition_node(s) for s in secondaries] yield assert_read_from_all(self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) yield self.pause(2 * MONITOR_INTERVAL) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member yield assert_read_from(self, self.c, _partition_node(secondary), mode)
def test_monitor_removes_recovering_member(self): self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield self.c.open() secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = [_partition_node(s) for s in secondaries] yield assert_read_from_all( self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) yield self.pause(2 * MONITOR_INTERVAL) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member yield assert_read_from( self, self.c, _partition_node(secondary), mode)
def test_monitor_removes_recovering_member(self, done): self.c = motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open_sync() secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = [_partition_node(s) for s in secondaries] yield motor.Op(assertReadFromAll, self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) yield gen.Task( IOLoop.instance().add_timeout, time.time() + 2 * MONITOR_INTERVAL) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member yield motor.Op(assertReadFrom, self, self.c, _partition_node(secondary), mode) done()
def test_secondary_connection(self): self.c = yield motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open() self.assertTrue(bool(len(self.c.secondaries))) db = self.c.motor_test yield db.test.remove({}, w=len(self.c.secondaries)) # Wait for replication... w = len(self.c.secondaries) + 1 yield db.test.insert({'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # A connection succeeds no matter the read preference for kwargs in [ { 'read_preference': PRIMARY }, { 'read_preference': PRIMARY_PREFERRED }, { 'read_preference': SECONDARY }, { 'read_preference': SECONDARY_PREFERRED }, { 'read_preference': NEAREST }, ]: client = yield motor.MotorClient(primary_host, primary_port, **kwargs).open() self.assertEqual(primary_host, client.host) self.assertEqual(primary_port, client.port) self.assertTrue(client.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue((yield client.motor_test.test.find_one())) client = yield motor.MotorClient(secondary_host, secondary_port, **kwargs).open() self.assertEqual(secondary_host, client.host) self.assertEqual(secondary_port, client.port) self.assertFalse(client.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue((yield client.motor_test.test.find_one())) else: with assert_raises(AutoReconnect): yield client.motor_test.test.find_one() # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MotorClient # should do the same for unacknowledged writes. try: yield client.motor_test.test.insert({}, w=0) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary client %s should' 'have raised exception' % client) # Test direct connection to an arbiter client = yield motor.MotorClient(arbiter_host, arbiter_port, **kwargs).open() self.assertEqual(arbiter_host, client.host) self.assertEqual(arbiter_port, client.port) self.assertFalse(client.is_primary) # See explanation above try: yield client.motor_test.test.insert({}, w=0) except AutoReconnect as e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into arbiter connection %s should' 'have raised exception' % client)
def test_secondary_connection(self, done): self.c = motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open_sync() self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test yield motor.Op(db.test.remove, {}, w=len(self.c.secondaries)) # Wait for replication... w = len(self.c.secondaries) + 1 yield motor.Op(db.test.insert, {'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # A connection succeeds no matter the read preference for kwargs in [ {'read_preference': PRIMARY}, {'read_preference': PRIMARY_PREFERRED}, {'read_preference': SECONDARY}, {'read_preference': SECONDARY_PREFERRED}, {'read_preference': NEAREST}, {'slave_okay': True} ]: conn = yield motor.Op(motor.MotorClient( primary_host, primary_port, **kwargs).open) self.assertEqual(primary_host, conn.host) self.assertEqual(primary_port, conn.port) self.assertTrue(conn.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue((yield motor.Op(conn.pymongo_test.test.find_one))) conn = yield motor.Op(motor.MotorClient( secondary_host, secondary_port, **kwargs).open) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != PRIMARY: self.assertTrue(( yield motor.Op(conn.pymongo_test.test.find_one))) else: yield AssertRaises( AutoReconnect, conn.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), MotorClient # should do the same for unacknowledged writes. try: yield motor.Op(conn.pymongo_test.test.insert, {}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.args[0]) else: self.fail( 'Unacknowledged insert into secondary connection %s should' 'have raised exception' % conn) # Test direct connection to an arbiter conn = yield motor.Op(motor.MotorClient( arbiter_host, arbiter_port, **kwargs).open) self.assertEqual(arbiter_host, conn.host) self.assertEqual(arbiter_port, conn.port) self.assertFalse(conn.is_primary) # See explanation above try: yield motor.Op(conn.pymongo_test.test.insert, {}, w=0) except AutoReconnect, e: self.assertEqual('not master', e.message)