def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = yield motor.MotorClient(primary).open() secondary_cx = yield motor.MotorClient(secondary).open() rsc = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield rsc.open() try: self.assertTrue((yield primary_cx.alive())) self.assertTrue((yield secondary_cx.alive())) self.assertTrue((yield rsc.alive())) ha_tools.kill_primary() yield self.pause(2) self.assertFalse((yield primary_cx.alive())) self.assertTrue((yield secondary_cx.alive())) self.assertFalse((yield rsc.alive())) ha_tools.kill_members([secondary], 2) yield self.pause(2) self.assertFalse((yield primary_cx.alive())) self.assertFalse((yield secondary_cx.alive())) self.assertFalse((yield rsc.alive())) finally: rsc.close()
def test_auth_during_failover(self, done): loop = IOLoop.instance() c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) c.open_sync() db = c.pymongo_ha_auth res = yield motor.Op(db.authenticate, 'user', 'userpass') self.assertTrue(res) yield motor.Op(db.foo.insert, {'foo': 'bar'}, w=3, wtimeout=1000) yield motor.Op(db.logout) yield AssertRaises(OperationFailure, db.foo.find_one) primary = '%s:%d' % self.c.primary ha_tools.kill_members([primary], 2) # Let monitor notice primary's gone yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL) # Make sure we can still authenticate res = yield motor.Op(db.authenticate, 'user', 'userpass') self.assertTrue(res) # And still query. db.read_preference = PRIMARY_PREFERRED res = yield motor.Op(db.foo.find_one) self.assertEqual('bar', res['foo']) c.close() done()
def test_auth_during_failover(self): c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() db = c.pymongo_ha_auth res = yield db.authenticate('user', 'userpass') self.assertTrue(res) yield db.foo.insert({'foo': 'bar'}, w=3, wtimeout=30000) yield db.logout() with assert_raises(OperationFailure): yield db.foo.find_one() primary = '%s:%d' % self.c.primary ha_tools.kill_members([primary], 2) # Let monitor notice primary's gone yield self.pause(2 * MONITOR_INTERVAL) # Make sure we can still authenticate res = yield db.authenticate('user', 'userpass') self.assertTrue(res) # And still query. db.read_preference = PRIMARY_PREFERRED res = yield db.foo.find_one() self.assertEqual('bar', res['foo']) c.close()
def test_alive(self, done): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = motor.MotorClient(primary).open_sync() secondary_cx = motor.MotorClient(secondary).open_sync() rsc = motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open_sync() loop = IOLoop.instance() try: yield AssertTrue(primary_cx.alive) yield AssertTrue(secondary_cx.alive) yield AssertTrue(rsc.alive) ha_tools.kill_primary() yield gen.Task(loop.add_timeout, time.time() + 0.5) yield AssertFalse(primary_cx.alive) yield AssertTrue(secondary_cx.alive) # Sometimes KeyError: https://jira.mongodb.org/browse/PYTHON-467 yield AssertFalse(rsc.alive) ha_tools.kill_members([secondary], 2) yield gen.Task(loop.add_timeout, time.time() + 0.5) yield AssertFalse(primary_cx.alive) yield AssertFalse(secondary_cx.alive) yield AssertFalse(rsc.alive) finally: rsc.close() done()
def test_auth_during_failover(self): c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() db = c.pymongo_ha_auth res = yield db.authenticate('user', 'userpass') self.assertTrue(res) yield db.foo.insert({'foo': 'bar'}, w=3, wtimeout=30000) yield db.logout() with self.assertRaises(OperationFailure): yield db.foo.find_one() primary = '%s:%d' % self.c.primary ha_tools.kill_members([primary], 2) # Let monitor notice primary's gone yield self.pause(2 * MONITOR_INTERVAL) # Make sure we can still authenticate res = yield db.authenticate('user', 'userpass') self.assertTrue(res) # And still query. db.read_preference = PRIMARY_PREFERRED res = yield db.foo.find_one() self.assertEqual('bar', res['foo']) c.close()
def test_ship_of_theseus(self, done): loop = IOLoop.instance() c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) c.open_sync() db = c.pymongo_test w = len(c.secondaries) + 1 db.test.insert({}, w=w) primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() ha_tools.add_member() ha_tools.add_member() ha_tools.add_member() # Wait for new members to join for _ in xrange(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break yield gen.Task(loop.add_timeout, time.time() + 1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary for _ in xrange(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break yield gen.Task(loop.add_timeout, time.time() + 1) else: self.fail("No failover") # Ensure monitor picks up new members yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL) try: yield motor.Op(db.test.find_one) except AutoReconnect: # Might take one try to reconnect yield gen.Task(loop.add_timeout, time.time() + 1) # No error yield motor.Op(db.test.find_one) yield motor.Op(db.test.find_one, read_preference=SECONDARY) done()
def test_passive_and_hidden(self): self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield self.c.open() passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: yield assert_read_from_all(self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) yield self.pause(2 * MONITOR_INTERVAL) yield assert_read_from(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def test_passive_and_hidden(self): self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield self.c.open() passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: yield assert_read_from_all(self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) yield self.pause(2 * MONITOR_INTERVAL) yield assert_read_from( self, self.c, self.c.primary, SECONDARY_PREFERRED)
def test_ship_of_theseus(self): c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() db = c.motor_test w = len(c.secondaries) + 1 db.test.insert({}, w=w) primary = ha_tools.get_primary() secondary1 = ha_tools.get_random_secondary() ha_tools.add_member() ha_tools.add_member() ha_tools.add_member() # Wait for new members to join for _ in range(120): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4: break yield self.pause(1) else: self.fail("New secondaries didn't join") ha_tools.kill_members([primary, secondary1], 9) # Wait for primary for _ in range(30): if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2: break yield self.pause(1) else: self.fail("No failover") # Ensure monitor picks up new members yield self.pause(2 * MONITOR_INTERVAL) try: yield db.test.find_one() except AutoReconnect: # Might take one try to reconnect yield self.pause(1) # No error yield db.test.find_one() yield db.test.find_one(read_preference=SECONDARY)
def test_passive_and_hidden(self, done): loop = IOLoop.instance() self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) self.c.open_sync() passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: yield motor.Op(assertReadFromAll, self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL) yield motor.Op(assertReadFrom, self, self.c, self.c.primary, SECONDARY_PREFERRED) done()
def test_read_preference(self): # This is long, but we put all the tests in one function to save time # on setUp, which takes about 30 seconds to bring up a replica set. # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name) yield c.open() @gen.coroutine def read_from_which_host( rsc, mode, tag_sets=None, latency=15, ): db = rsc.motor_test db.read_preference = mode if isinstance(tag_sets, dict): tag_sets = [tag_sets] db.tag_sets = tag_sets or [{}] db.secondary_acceptable_latency_ms = latency cursor = db.test.find() try: yield cursor.fetch_next raise gen.Return(cursor.delegate._Cursor__connection_id) except AutoReconnect: raise gen.Return(None) @gen.coroutine def assert_read_from(member, *args, **kwargs): for _ in range(10): used = yield read_from_which_host(c, *args, **kwargs) self.assertEqual(member, used) @gen.coroutine def assert_read_from_all(members, *args, **kwargs): members = set(members) all_used = set() for _ in range(100): used = yield read_from_which_host(c, *args, **kwargs) all_used.add(used) if members == all_used: raise gen.Return() # Success # This will fail self.assertEqual(members, all_used) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag) yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY yield assert_read_from_all([secondary, other_secondary], SECONDARY, latency=9999999) # SECONDARY_PREFERRED yield assert_read_from_all([secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999) # Multiple tags yield assert_read_from(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags yield assert_read_from(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set yield assert_read_from_all([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}], latency=9999999) yield assert_read_from(other_secondary, SECONDARY_PREFERRED, [bad_tag, { 'dc': 'ny' }]) # NEAREST self.clear_ping_times() yield assert_read_from_all([primary, secondary, other_secondary], NEAREST, latency=9999999) yield assert_read_from_all([primary, other_secondary], NEAREST, [bad_tag, { 'dc': 'ny' }], latency=9999999) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 milliseconds. self.set_ping_time(other_secondary, 10) # Nearest member, no tags yield assert_read_from(primary, NEAREST) # Tags override nearness yield assert_read_from(primary, NEAREST, {'name': 'primary'}) yield assert_read_from(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 milliseconds. self.set_ping_time(secondary, 0) yield assert_read_from(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) yield assert_read_from(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members yield assert_read_from_all([primary, secondary, other_secondary], NEAREST, latency=9999999) self.clear_ping_times() yield assert_read_from_all([primary, other_secondary], NEAREST, [{ 'dc': 'ny' }], latency=9999999) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone yield self.pause(2 * MONITOR_INTERVAL) # PRIMARY yield assert_read_from(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary yield assert_read_from_all([secondary, other_secondary], PRIMARY_PREFERRED, latency=9999999) yield assert_read_from(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY yield assert_read_from_all([secondary, other_secondary], SECONDARY, latency=9999999) # Only primary matches yield assert_read_from(None, SECONDARY, {'name': 'primary'}) # No matching secondaries yield assert_read_from(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED yield assert_read_from_all([secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999) # Mode and tags both match yield assert_read_from(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() yield assert_read_from_all([secondary, other_secondary], NEAREST, latency=9999999) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break yield self.pause(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue( pymongo.MongoClient( unpartition_node(primary), slave_okay=True).admin.command('ismaster')['ismaster']) yield self.pause(2 * MONITOR_INTERVAL) # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED yield assert_read_from(primary, PRIMARY_PREFERRED) # SECONDARY yield assert_read_from(other_secondary, SECONDARY) yield assert_read_from(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches yield assert_read_from(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED yield assert_read_from(other_secondary, SECONDARY_PREFERRED) yield assert_read_from(other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary yield assert_read_from(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST yield assert_read_from_all([primary, other_secondary], NEAREST, latency=9999999) yield assert_read_from(other_secondary, NEAREST, {'name': 'other_secondary'}) yield assert_read_from(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue( pymongo.MongoClient( unpartition_node(primary), slave_okay=True).admin.command('ismaster')['ismaster']) # PRIMARY yield assert_read_from(primary, PRIMARY) # PRIMARY_PREFERRED yield assert_read_from(primary, PRIMARY_PREFERRED) yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY yield assert_read_from(None, SECONDARY) yield assert_read_from(None, SECONDARY, self.other_secondary_dc) yield assert_read_from(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED yield assert_read_from(primary, SECONDARY_PREFERRED) yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc) yield assert_read_from(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST yield assert_read_from(primary, NEAREST) yield assert_read_from(None, NEAREST, self.secondary_dc) yield assert_read_from(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) yield assert_read_from(primary, NEAREST) yield assert_read_from(None, NEAREST, self.secondary_dc) self.clear_ping_times()
def test_read_preference(self, done): # This is long, but we put all the tests in one function to save time # on setUp, which takes about 30 seconds to bring up a replica set. # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST loop = IOLoop.instance() c = motor.MotorReplicaSetClient( self.seed, replicaSet=self.name).open_sync() @gen.engine def read_from_which_host( rsc, mode, tag_sets=None, latency=15, callback=None ): db = rsc.pymongo_test db.read_preference = mode if isinstance(tag_sets, dict): tag_sets = [tag_sets] db.tag_sets = tag_sets or [{}] db.secondary_acceptable_latency_ms = latency cursor = db.test.find() try: yield cursor.fetch_next callback(cursor.delegate._Cursor__connection_id) except AutoReconnect: callback(None) @gen.engine def assertReadFrom(member, *args, **kwargs): callback = kwargs.pop('callback') for _ in range(10): used = yield gen.Task( read_from_which_host, c, *args, **kwargs) self.assertEqual(member, used) # Done callback() @gen.engine def assertReadFromAll(members, *args, **kwargs): callback = kwargs.pop('callback') members = set(members) all_used = set() for _ in range(100): used = yield gen.Task( read_from_which_host, c, *args, **kwargs) all_used.add(used) if members == all_used: # Success callback() break # This will fail self.assertEqual(members, all_used) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY yield gen.Task(assertReadFrom, primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, bad_tag) yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY, latency=9999999) # SECONDARY_PREFERRED yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999) # Multiple tags yield gen.Task(assertReadFrom, secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}], latency=9999999) yield gen.Task(assertReadFrom, other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() yield gen.Task(assertReadFromAll, [primary, secondary, other_secondary], NEAREST, latency=9999999) yield gen.Task(assertReadFromAll, [primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}], latency=9999999) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags yield gen.Task(assertReadFrom, primary, NEAREST) # Tags override nearness yield gen.Task(assertReadFrom, primary, NEAREST, {'name': 'primary'}) yield gen.Task(assertReadFrom, secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) yield gen.Task(assertReadFrom, secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) yield gen.Task(assertReadFrom, other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members yield gen.Task(assertReadFromAll, [primary, secondary, other_secondary], NEAREST, latency=9999999) self.clear_ping_times() yield gen.Task(assertReadFromAll, [primary, other_secondary], NEAREST, [{'dc': 'ny'}], latency=9999999) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL) # PRIMARY yield gen.Task(assertReadFrom, None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary yield gen.Task(assertReadFromAll, [secondary, other_secondary], PRIMARY_PREFERRED, latency=9999999) yield gen.Task(assertReadFrom, secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary yield gen.Task(assertReadFrom, None, PRIMARY_PREFERRED, bad_tag) # SECONDARY yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY, latency=9999999) # Only primary matches yield gen.Task(assertReadFrom, None, SECONDARY, {'name': 'primary'}) # No matching secondaries yield gen.Task(assertReadFrom, None, SECONDARY, bad_tag) # SECONDARY_PREFERRED yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999) # Mode and tags both match yield gen.Task(assertReadFrom, secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() yield gen.Task(assertReadFromAll, [secondary, other_secondary], NEAREST, latency=9999999) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break yield gen.Task(loop.add_timeout, time.time() + 1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(pymongo.MongoClient( unpartition_node(primary), slave_okay=True ).admin.command('ismaster')['ismaster']) yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL) # PRIMARY yield gen.Task(assertReadFrom, primary, PRIMARY) # PRIMARY_PREFERRED yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED) # SECONDARY yield gen.Task(assertReadFrom, other_secondary, SECONDARY) yield gen.Task(assertReadFrom, other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches yield gen.Task(assertReadFrom, None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED yield gen.Task(assertReadFrom, other_secondary, SECONDARY_PREFERRED) yield gen.Task(assertReadFrom, other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST yield gen.Task(assertReadFromAll, [primary, other_secondary], NEAREST, latency=9999999) yield gen.Task(assertReadFrom, other_secondary, NEAREST, {'name': 'other_secondary'}) yield gen.Task(assertReadFrom, primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(pymongo.MongoClient( unpartition_node(primary), slave_okay=True ).admin.command('ismaster')['ismaster']) # PRIMARY yield gen.Task(assertReadFrom, primary, PRIMARY) # PRIMARY_PREFERRED yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED) yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY yield gen.Task(assertReadFrom, None, SECONDARY) yield gen.Task(assertReadFrom, None, SECONDARY, self.other_secondary_dc) yield gen.Task(assertReadFrom, None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED) yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, self.secondary_dc) yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, {'name': 'secondary'}) yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST yield gen.Task(assertReadFrom, primary, NEAREST) yield gen.Task(assertReadFrom, None, NEAREST, self.secondary_dc) yield gen.Task(assertReadFrom, None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) yield gen.Task(assertReadFrom, primary, NEAREST) yield gen.Task(assertReadFrom, None, NEAREST, self.secondary_dc) self.clear_ping_times() done()