def test_ship_of_theseus(self, done):
loop = IOLoop.instance()
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
c.open_sync()
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL)
try:
yield motor.Op(db.test.find_one)
except AutoReconnect:
# Might take one try to reconnect
yield gen.Task(loop.add_timeout, time.time() + 1)
# No error
yield motor.Op(db.test.find_one)
yield motor.Op(db.test.find_one, read_preference=SECONDARY)
done()
def test_recovering_member_triggers_refresh(self, done):
# To test that find_one() and count() trigger immediate refreshes,
# we'll create a separate connection for each
self.c_find_one, self.c_count = [
motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name, read_preference=SECONDARY
).open_sync() for _ in xrange(2)]
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
# Pre-condition: just make sure they all connected OK
for c in self.c_find_one, self.c_count:
self.assertEqual(one(c.secondaries), _partition_node(secondary))
ha_tools.set_maintenance(secondary, True)
# Trigger a refresh in various ways
yield AssertRaises(AutoReconnect, self.c_find_one.test.test.find_one)
yield AssertRaises(AutoReconnect, self.c_count.test.test.count)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield gen.Task(
IOLoop.instance().add_timeout, time.time() + 1)
for c in self.c_find_one, self.c_count:
self.assertFalse(c.secondaries)
self.assertEqual(_partition_node(primary), c.primary)
done()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = yield motor.MotorClient(primary).open()
secondary_cx = yield motor.MotorClient(secondary).open()
rsc = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield rsc.open()
try:
self.assertTrue((yield primary_cx.alive()))
self.assertTrue((yield secondary_cx.alive()))
self.assertTrue((yield rsc.alive()))
ha_tools.kill_primary()
yield self.pause(2)
self.assertFalse((yield primary_cx.alive()))
self.assertTrue((yield secondary_cx.alive()))
self.assertFalse((yield rsc.alive()))
ha_tools.kill_members([secondary], 2)
yield self.pause(2)
self.assertFalse((yield primary_cx.alive()))
self.assertFalse((yield secondary_cx.alive()))
self.assertFalse((yield rsc.alive()))
finally:
rsc.close()
def test_stepdown_triggers_refresh(self, done):
c_find_one = motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open_sync()
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(
one(c_find_one.secondaries), _partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1)
# Trigger a refresh
yield AssertRaises(AutoReconnect, c_find_one.test.test.find_one)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1)
# We've detected the stepdown
self.assertTrue(
not c_find_one.primary
or primary != _partition_node(c_find_one.primary))
done()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = yield motor.MotorClient(primary).open()
secondary_cx = yield motor.MotorClient(secondary).open()
rsc = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield rsc.open()
try:
self.assertTrue((yield primary_cx.alive()))
self.assertTrue((yield secondary_cx.alive()))
self.assertTrue((yield rsc.alive()))
ha_tools.kill_primary()
yield self.pause(2)
self.assertFalse((yield primary_cx.alive()))
self.assertTrue((yield secondary_cx.alive()))
self.assertFalse((yield rsc.alive()))
ha_tools.kill_members([secondary], 2)
yield self.pause(2)
self.assertFalse((yield primary_cx.alive()))
self.assertFalse((yield secondary_cx.alive()))
self.assertFalse((yield rsc.alive()))
finally:
rsc.close()
def test_stepdown_triggers_refresh(self):
c_find_one = yield motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open()
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(
one(c_find_one.secondaries), _partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
yield self.pause(1)
# Trigger a refresh
with self.assertRaises(AutoReconnect):
yield c_find_one.test.test.find_one()
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield self.pause(1)
# We've detected the stepdown
self.assertTrue(
not c_find_one.primary
or primary != c_find_one.primary)
def test_recovering_member_triggers_refresh(self):
# To test that find_one() and count() trigger immediate refreshes,
# we'll create a separate client for each
self.c_find_one, self.c_count = yield [
motor.MotorReplicaSetClient(self.seed,
replicaSet=self.name,
read_preference=SECONDARY).open()
for _ in range(2)
]
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
# Pre-condition: just make sure they all connected OK
for c in self.c_find_one, self.c_count:
self.assertEqual(one(c.secondaries), _partition_node(secondary))
ha_tools.set_maintenance(secondary, True)
# Trigger a refresh in various ways
with assert_raises(AutoReconnect):
yield self.c_find_one.test.test.find_one()
with assert_raises(AutoReconnect):
yield self.c_count.test.test.count()
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield self.pause(1)
for c in self.c_find_one, self.c_count:
self.assertFalse(c.secondaries)
self.assertEqual(_partition_node(primary), c.primary)
def test_stepdown_triggers_refresh(self):
c_find_one = yield motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open()
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(one(c_find_one.secondaries),
_partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
yield self.pause(1)
# Trigger a refresh
with assert_raises(AutoReconnect):
yield c_find_one.test.test.find_one()
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield self.pause(1)
# We've detected the stepdown
self.assertTrue(not c_find_one.primary
or primary != c_find_one.primary)
def test_alive(self, done):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = motor.MotorClient(primary).open_sync()
secondary_cx = motor.MotorClient(secondary).open_sync()
rsc = motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open_sync()
loop = IOLoop.instance()
try:
yield AssertTrue(primary_cx.alive)
yield AssertTrue(secondary_cx.alive)
yield AssertTrue(rsc.alive)
ha_tools.kill_primary()
yield gen.Task(loop.add_timeout, time.time() + 0.5)
yield AssertFalse(primary_cx.alive)
yield AssertTrue(secondary_cx.alive)
# Sometimes KeyError: https://jira.mongodb.org/browse/PYTHON-467
yield AssertFalse(rsc.alive)
ha_tools.kill_members([secondary], 2)
yield gen.Task(loop.add_timeout, time.time() + 0.5)
yield AssertFalse(primary_cx.alive)
yield AssertFalse(secondary_cx.alive)
yield AssertFalse(rsc.alive)
finally:
rsc.close()
done()
def test_ship_of_theseus(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
db = c.motor_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in range(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield self.pause(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in range(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield self.pause(1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield self.pause(2 * MONITOR_INTERVAL)
try:
yield db.test.find_one()
except AutoReconnect:
# Might take one try to reconnect
yield self.pause(1)
# No error
yield db.test.find_one()
yield db.test.find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
db = c.motor_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in range(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield self.pause(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in range(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield self.pause(1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield self.pause(2 * MONITOR_INTERVAL)
try:
yield db.test.find_one()
except AutoReconnect:
# Might take one try to reconnect
yield self.pause(1)
# No error
yield db.test.find_one()
yield db.test.find_one(read_preference=SECONDARY)
def setUp(self):
super(MotorTestReadPreference, self).setUp()
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
# Synchronous PyMongo interfaces for convenience
self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient(
self.seed, replicaSet=self.name)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], w=self.w)
self.clear_ping_times()
def setUp(self):
super(MotorTestReadPreference, self).setUp()
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
# Synchronous PyMongo interfaces for convenience
self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient(
self.seed, replicaSet=self.name)
self.db = self.c.motor_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, w=self.w)
self.db.test.insert(
[{'foo': i} for i in range(10)], w=self.w)
self.clear_ping_times()
def test_read_preference(self, done):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
loop = IOLoop.instance()
c = motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open_sync()
@gen.engine
def read_from_which_host(
rsc,
mode,
tag_sets=None,
latency=15,
callback=None
):
db = rsc.pymongo_test
db.read_preference = mode
if isinstance(tag_sets, dict):
tag_sets = [tag_sets]
db.tag_sets = tag_sets or [{}]
db.secondary_acceptable_latency_ms = latency
cursor = db.test.find()
try:
yield cursor.fetch_next
callback(cursor.delegate._Cursor__connection_id)
except AutoReconnect:
callback(None)
@gen.engine
def assertReadFrom(member, *args, **kwargs):
callback = kwargs.pop('callback')
for _ in range(10):
used = yield gen.Task(
read_from_which_host, c, *args, **kwargs)
self.assertEqual(member, used)
# Done
callback()
@gen.engine
def assertReadFromAll(members, *args, **kwargs):
callback = kwargs.pop('callback')
members = set(members)
all_used = set()
for _ in range(100):
used = yield gen.Task(
read_from_which_host, c, *args, **kwargs)
all_used.add(used)
if members == all_used:
# Success
callback()
break
# This will fail
self.assertEqual(members, all_used)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
yield gen.Task(assertReadFrom, primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, bad_tag)
yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY, latency=9999999)
# SECONDARY_PREFERRED
yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999)
# Multiple tags
yield gen.Task(assertReadFrom, secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
yield gen.Task(assertReadFromAll, [secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}], latency=9999999)
yield gen.Task(assertReadFrom, other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
yield gen.Task(assertReadFromAll, [primary, secondary, other_secondary], NEAREST, latency=9999999)
yield gen.Task(assertReadFromAll, [primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}], latency=9999999)
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
yield gen.Task(assertReadFrom, primary, NEAREST)
# Tags override nearness
yield gen.Task(assertReadFrom, primary, NEAREST, {'name': 'primary'})
yield gen.Task(assertReadFrom, secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
yield gen.Task(assertReadFrom, secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
yield gen.Task(assertReadFrom, other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
yield gen.Task(assertReadFromAll,
[primary, secondary, other_secondary],
NEAREST, latency=9999999)
self.clear_ping_times()
yield gen.Task(assertReadFromAll, [primary, other_secondary], NEAREST, [{'dc': 'ny'}], latency=9999999)
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL)
# PRIMARY
yield gen.Task(assertReadFrom, None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
yield gen.Task(assertReadFromAll, [secondary, other_secondary], PRIMARY_PREFERRED, latency=9999999)
yield gen.Task(assertReadFrom, secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
yield gen.Task(assertReadFrom, None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY, latency=9999999)
# Only primary matches
yield gen.Task(assertReadFrom, None, SECONDARY, {'name': 'primary'})
# No matching secondaries
yield gen.Task(assertReadFrom, None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
yield gen.Task(assertReadFromAll, [secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999)
# Mode and tags both match
yield gen.Task(assertReadFrom, secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
yield gen.Task(assertReadFromAll, [secondary, other_secondary], NEAREST, latency=9999999)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True
).admin.command('ismaster')['ismaster'])
yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL)
# PRIMARY
yield gen.Task(assertReadFrom, primary, PRIMARY)
# PRIMARY_PREFERRED
yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED)
# SECONDARY
yield gen.Task(assertReadFrom, other_secondary, SECONDARY)
yield gen.Task(assertReadFrom, other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
yield gen.Task(assertReadFrom, None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
yield gen.Task(assertReadFrom, other_secondary, SECONDARY_PREFERRED)
yield gen.Task(assertReadFrom,
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
yield gen.Task(assertReadFromAll, [primary, other_secondary], NEAREST, latency=9999999)
yield gen.Task(assertReadFrom, other_secondary, NEAREST, {'name': 'other_secondary'})
yield gen.Task(assertReadFrom, primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True
).admin.command('ismaster')['ismaster'])
# PRIMARY
yield gen.Task(assertReadFrom, primary, PRIMARY)
# PRIMARY_PREFERRED
yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED)
yield gen.Task(assertReadFrom, primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
yield gen.Task(assertReadFrom, None, SECONDARY)
yield gen.Task(assertReadFrom, None, SECONDARY, self.other_secondary_dc)
yield gen.Task(assertReadFrom, None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED)
yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, self.secondary_dc)
yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, {'name': 'secondary'})
yield gen.Task(assertReadFrom, primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
yield gen.Task(assertReadFrom, primary, NEAREST)
yield gen.Task(assertReadFrom, None, NEAREST, self.secondary_dc)
yield gen.Task(assertReadFrom, None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
yield gen.Task(assertReadFrom, primary, NEAREST)
yield gen.Task(assertReadFrom, None, NEAREST, self.secondary_dc)
self.clear_ping_times()
done()
def test_secondary_connection(self):
self.c = yield motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open()
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.motor_test
yield db.test.remove({}, w=len(self.c.secondaries))
# Wait for replication...
w = len(self.c.secondaries) + 1
yield db.test.insert({'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# A connection succeeds no matter the read preference
for kwargs in [
{
'read_preference': PRIMARY
},
{
'read_preference': PRIMARY_PREFERRED
},
{
'read_preference': SECONDARY
},
{
'read_preference': SECONDARY_PREFERRED
},
{
'read_preference': NEAREST
},
]:
client = yield motor.MotorClient(primary_host, primary_port,
**kwargs).open()
self.assertEqual(primary_host, client.host)
self.assertEqual(primary_port, client.port)
self.assertTrue(client.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue((yield client.motor_test.test.find_one()))
client = yield motor.MotorClient(secondary_host, secondary_port,
**kwargs).open()
self.assertEqual(secondary_host, client.host)
self.assertEqual(secondary_port, client.port)
self.assertFalse(client.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue((yield client.motor_test.test.find_one()))
else:
with assert_raises(AutoReconnect):
yield client.motor_test.test.find_one()
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), MotorClient
# should do the same for unacknowledged writes.
try:
yield client.motor_test.test.insert({}, w=0)
except AutoReconnect as e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary client %s should'
'have raised exception' % client)
# Test direct connection to an arbiter
client = yield motor.MotorClient(arbiter_host, arbiter_port,
**kwargs).open()
self.assertEqual(arbiter_host, client.host)
self.assertEqual(arbiter_port, client.port)
self.assertFalse(client.is_primary)
# See explanation above
try:
yield client.motor_test.test.insert({}, w=0)
except AutoReconnect as e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into arbiter connection %s should'
'have raised exception' % client)
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
@gen.coroutine
def read_from_which_host(
rsc,
mode,
tag_sets=None,
latency=15,
):
db = rsc.motor_test
db.read_preference = mode
if isinstance(tag_sets, dict):
tag_sets = [tag_sets]
db.tag_sets = tag_sets or [{}]
db.secondary_acceptable_latency_ms = latency
cursor = db.test.find()
try:
yield cursor.fetch_next
raise gen.Return(cursor.delegate._Cursor__connection_id)
except AutoReconnect:
raise gen.Return(None)
@gen.coroutine
def assert_read_from(member, *args, **kwargs):
for _ in range(10):
used = yield read_from_which_host(c, *args, **kwargs)
self.assertEqual(member, used)
@gen.coroutine
def assert_read_from_all(members, *args, **kwargs):
members = set(members)
all_used = set()
for _ in range(100):
used = yield read_from_which_host(c, *args, **kwargs)
all_used.add(used)
if members == all_used:
raise gen.Return() # Success
# This will fail
self.assertEqual(members, all_used)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag)
yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
yield assert_read_from_all([secondary, other_secondary],
SECONDARY,
latency=9999999)
# SECONDARY_PREFERRED
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED,
latency=9999999)
# Multiple tags
yield assert_read_from(secondary, SECONDARY_PREFERRED,
self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'primary'})
# No matching secondaries
yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}],
latency=9999999)
yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
[bad_tag, {
'dc': 'ny'
}])
# NEAREST
self.clear_ping_times()
yield assert_read_from_all([primary, secondary, other_secondary],
NEAREST,
latency=9999999)
yield assert_read_from_all([primary, other_secondary],
NEAREST, [bad_tag, {
'dc': 'ny'
}],
latency=9999999)
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 milliseconds.
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
yield assert_read_from(primary, NEAREST)
# Tags override nearness
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
yield assert_read_from(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 milliseconds.
self.set_ping_time(secondary, 0)
yield assert_read_from(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
yield assert_read_from(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
yield assert_read_from_all([primary, secondary, other_secondary],
NEAREST,
latency=9999999)
self.clear_ping_times()
yield assert_read_from_all([primary, other_secondary],
NEAREST, [{
'dc': 'ny'
}],
latency=9999999)
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
yield assert_read_from_all([secondary, other_secondary],
PRIMARY_PREFERRED,
latency=9999999)
yield assert_read_from(secondary, PRIMARY_PREFERRED,
{'name': 'secondary'})
# No primary or matching secondary
yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
yield assert_read_from_all([secondary, other_secondary],
SECONDARY,
latency=9999999)
# Only primary matches
yield assert_read_from(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
yield assert_read_from(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED,
latency=9999999)
# Mode and tags both match
yield assert_read_from(secondary, SECONDARY_PREFERRED,
{'name': 'secondary'})
# NEAREST
self.clear_ping_times()
yield assert_read_from_all([secondary, other_secondary],
NEAREST,
latency=9999999)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
yield self.pause(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(
pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True).admin.command('ismaster')['ismaster'])
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
# SECONDARY
yield assert_read_from(other_secondary, SECONDARY)
yield assert_read_from(other_secondary, SECONDARY,
self.other_secondary_dc)
# Only the down secondary matches
yield assert_read_from(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
yield assert_read_from(other_secondary, SECONDARY_PREFERRED)
yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
self.other_secondary_dc)
# The secondary matching the tag is down, use primary
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'secondary'})
# NEAREST
yield assert_read_from_all([primary, other_secondary],
NEAREST,
latency=9999999)
yield assert_read_from(other_secondary, NEAREST,
{'name': 'other_secondary'})
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(
pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True).admin.command('ismaster')['ismaster'])
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
yield assert_read_from(None, SECONDARY)
yield assert_read_from(None, SECONDARY, self.other_secondary_dc)
yield assert_read_from(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
yield assert_read_from(primary, SECONDARY_PREFERRED)
yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc)
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'secondary'})
yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
yield assert_read_from(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_secondary_connection(self, done):
self.c = motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open_sync()
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
yield motor.Op(db.test.remove, {}, w=len(self.c.secondaries))
# Wait for replication...
w = len(self.c.secondaries) + 1
yield motor.Op(db.test.insert, {'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# A connection succeeds no matter the read preference
for kwargs in [
{'read_preference': PRIMARY},
{'read_preference': PRIMARY_PREFERRED},
{'read_preference': SECONDARY},
{'read_preference': SECONDARY_PREFERRED},
{'read_preference': NEAREST},
{'slave_okay': True}
]:
conn = yield motor.Op(motor.MotorClient(
primary_host, primary_port, **kwargs).open)
self.assertEqual(primary_host, conn.host)
self.assertEqual(primary_port, conn.port)
self.assertTrue(conn.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue((yield motor.Op(conn.pymongo_test.test.find_one)))
conn = yield motor.Op(motor.MotorClient(
secondary_host, secondary_port, **kwargs).open)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue((
yield motor.Op(conn.pymongo_test.test.find_one)))
else:
yield AssertRaises(
AutoReconnect, conn.pymongo_test.test.find_one)
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), MotorClient
# should do the same for unacknowledged writes.
try:
yield motor.Op(conn.pymongo_test.test.insert, {}, w=0)
except AutoReconnect, e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary connection %s should'
'have raised exception' % conn)
# Test direct connection to an arbiter
conn = yield motor.Op(motor.MotorClient(
arbiter_host, arbiter_port, **kwargs).open)
self.assertEqual(arbiter_host, conn.host)
self.assertEqual(arbiter_port, conn.port)
self.assertFalse(conn.is_primary)
# See explanation above
try:
yield motor.Op(conn.pymongo_test.test.insert, {}, w=0)
except AutoReconnect, e:
self.assertEqual('not master', e.message)
