def test_passive_and_hidden(self):
self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield self.c.open()
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
yield assert_read_from_all(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
yield self.pause(2 * MONITOR_INTERVAL)
yield assert_read_from(
self, self.c, self.c.primary, SECONDARY_PREFERRED)
def test_passive_and_hidden(self):
self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield self.c.open()
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
yield assert_read_from_all(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
yield self.pause(2 * MONITOR_INTERVAL)
yield assert_read_from(self, self.c, self.c.primary,
SECONDARY_PREFERRED)
def test_monitor_removes_recovering_member(self):
self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield self.c.open()
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
yield assert_read_from_all(
self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
yield self.pause(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
yield assert_read_from(
self, self.c, _partition_node(secondary), mode)
def test_monitor_removes_recovering_member(self):
self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield self.c.open()
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
yield assert_read_from_all(self, self.c, partitioned_secondaries,
mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
yield self.pause(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
yield assert_read_from(self, self.c, _partition_node(secondary),
mode)
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
@gen.coroutine
def read_from_which_host(
rsc,
mode,
tag_sets=None,
latency=15,
):
db = rsc.motor_test
db.read_preference = mode
if isinstance(tag_sets, dict):
tag_sets = [tag_sets]
db.tag_sets = tag_sets or [{}]
db.secondary_acceptable_latency_ms = latency
cursor = db.test.find()
try:
yield cursor.fetch_next
raise gen.Return(cursor.delegate._Cursor__connection_id)
except AutoReconnect:
raise gen.Return(None)
@gen.coroutine
def assert_read_from(member, *args, **kwargs):
for _ in range(10):
used = yield read_from_which_host(c, *args, **kwargs)
self.assertEqual(member, used)
@gen.coroutine
def assert_read_from_all(members, *args, **kwargs):
members = set(members)
all_used = set()
for _ in range(100):
used = yield read_from_which_host(c, *args, **kwargs)
all_used.add(used)
if members == all_used:
raise gen.Return() # Success
# This will fail
self.assertEqual(members, all_used)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag)
yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
yield assert_read_from_all(
[secondary, other_secondary], SECONDARY, latency=9999999)
# SECONDARY_PREFERRED
yield assert_read_from_all(
[secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999)
# Multiple tags
yield assert_read_from(
secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
yield assert_read_from(
primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
yield assert_read_from_all(
[secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}], latency=9999999)
yield assert_read_from(
other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
yield assert_read_from_all(
[primary, secondary, other_secondary], NEAREST, latency=9999999)
yield assert_read_from_all(
[primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}], latency=9999999)
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 milliseconds.
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
yield assert_read_from(primary, NEAREST)
# Tags override nearness
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
yield assert_read_from(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 milliseconds.
self.set_ping_time(secondary, 0)
yield assert_read_from(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
yield assert_read_from(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
yield assert_read_from_all(
[primary, secondary, other_secondary],
NEAREST, latency=9999999)
self.clear_ping_times()
yield assert_read_from_all(
[primary, other_secondary], NEAREST, [{'dc': 'ny'}],
latency=9999999)
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
yield assert_read_from_all(
[secondary, other_secondary], PRIMARY_PREFERRED, latency=9999999)
yield assert_read_from(
secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
yield assert_read_from_all(
[secondary, other_secondary], SECONDARY, latency=9999999)
# Only primary matches
yield assert_read_from(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
yield assert_read_from(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
yield assert_read_from_all(
[secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999)
# Mode and tags both match
yield assert_read_from(
secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
yield assert_read_from_all(
[secondary, other_secondary], NEAREST, latency=9999999)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
yield self.pause(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True
).admin.command('ismaster')['ismaster'])
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
# SECONDARY
yield assert_read_from(other_secondary, SECONDARY)
yield assert_read_from(
other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
yield assert_read_from(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
yield assert_read_from(other_secondary, SECONDARY_PREFERRED)
yield assert_read_from(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
yield assert_read_from(
primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
yield assert_read_from_all(
[primary, other_secondary], NEAREST, latency=9999999)
yield assert_read_from(
other_secondary, NEAREST, {'name': 'other_secondary'})
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True
).admin.command('ismaster')['ismaster'])
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
yield assert_read_from(None, SECONDARY)
yield assert_read_from(None, SECONDARY, self.other_secondary_dc)
yield assert_read_from(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
yield assert_read_from(primary, SECONDARY_PREFERRED)
yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc)
yield assert_read_from(
primary, SECONDARY_PREFERRED, {'name': 'secondary'})
yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
yield assert_read_from(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
@gen.coroutine
def read_from_which_host(
rsc,
mode,
tag_sets=None,
latency=15,
):
db = rsc.motor_test
db.read_preference = mode
if isinstance(tag_sets, dict):
tag_sets = [tag_sets]
db.tag_sets = tag_sets or [{}]
db.secondary_acceptable_latency_ms = latency
cursor = db.test.find()
try:
yield cursor.fetch_next
raise gen.Return(cursor.delegate._Cursor__connection_id)
except AutoReconnect:
raise gen.Return(None)
@gen.coroutine
def assert_read_from(member, *args, **kwargs):
for _ in range(10):
used = yield read_from_which_host(c, *args, **kwargs)
self.assertEqual(member, used)
@gen.coroutine
def assert_read_from_all(members, *args, **kwargs):
members = set(members)
all_used = set()
for _ in range(100):
used = yield read_from_which_host(c, *args, **kwargs)
all_used.add(used)
if members == all_used:
raise gen.Return() # Success
# This will fail
self.assertEqual(members, all_used)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag)
yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
yield assert_read_from_all([secondary, other_secondary],
SECONDARY,
latency=9999999)
# SECONDARY_PREFERRED
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED,
latency=9999999)
# Multiple tags
yield assert_read_from(secondary, SECONDARY_PREFERRED,
self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'primary'})
# No matching secondaries
yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}],
latency=9999999)
yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
[bad_tag, {
'dc': 'ny'
}])
# NEAREST
self.clear_ping_times()
yield assert_read_from_all([primary, secondary, other_secondary],
NEAREST,
latency=9999999)
yield assert_read_from_all([primary, other_secondary],
NEAREST, [bad_tag, {
'dc': 'ny'
}],
latency=9999999)
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 milliseconds.
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
yield assert_read_from(primary, NEAREST)
# Tags override nearness
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
yield assert_read_from(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 milliseconds.
self.set_ping_time(secondary, 0)
yield assert_read_from(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
yield assert_read_from(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
yield assert_read_from_all([primary, secondary, other_secondary],
NEAREST,
latency=9999999)
self.clear_ping_times()
yield assert_read_from_all([primary, other_secondary],
NEAREST, [{
'dc': 'ny'
}],
latency=9999999)
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
yield assert_read_from_all([secondary, other_secondary],
PRIMARY_PREFERRED,
latency=9999999)
yield assert_read_from(secondary, PRIMARY_PREFERRED,
{'name': 'secondary'})
# No primary or matching secondary
yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
yield assert_read_from_all([secondary, other_secondary],
SECONDARY,
latency=9999999)
# Only primary matches
yield assert_read_from(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
yield assert_read_from(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED,
latency=9999999)
# Mode and tags both match
yield assert_read_from(secondary, SECONDARY_PREFERRED,
{'name': 'secondary'})
# NEAREST
self.clear_ping_times()
yield assert_read_from_all([secondary, other_secondary],
NEAREST,
latency=9999999)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
yield self.pause(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(
pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True).admin.command('ismaster')['ismaster'])
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
# SECONDARY
yield assert_read_from(other_secondary, SECONDARY)
yield assert_read_from(other_secondary, SECONDARY,
self.other_secondary_dc)
# Only the down secondary matches
yield assert_read_from(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
yield assert_read_from(other_secondary, SECONDARY_PREFERRED)
yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
self.other_secondary_dc)
# The secondary matching the tag is down, use primary
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'secondary'})
# NEAREST
yield assert_read_from_all([primary, other_secondary],
NEAREST,
latency=9999999)
yield assert_read_from(other_secondary, NEAREST,
{'name': 'other_secondary'})
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(
pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True).admin.command('ismaster')['ismaster'])
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
yield assert_read_from(None, SECONDARY)
yield assert_read_from(None, SECONDARY, self.other_secondary_dc)
yield assert_read_from(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
yield assert_read_from(primary, SECONDARY_PREFERRED)
yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc)
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'secondary'})
yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
yield assert_read_from(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
self.clear_ping_times()