class TestMonitorRemovesRecoveringMember(unittest.TestCase):
# Members in STARTUP2 or RECOVERING states are shown in the primary's
# isMaster response, but aren't secondaries and shouldn't be read from.
# Verify that if a secondary goes into RECOVERING mode, the Monitor removes
# it from the set of readers.
def setUp(self):
members = [{}, {'priority': 0}, {'priority': 0}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_monitor_removes_recovering_member(self):
self.c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
utils.assertReadFromAll(self, self.c, partitioned_secondaries,
mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary),
mode)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = MongoClient(primary, use_greenlets=use_greenlets)
secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets)
rsc = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
class TestMonitorRemovesRecoveringMember(unittest.TestCase):
# Members in STARTUP2 or RECOVERING states are shown in the primary's
# isMaster response, but aren't secondaries and shouldn't be read from.
# Verify that if a secondary goes into RECOVERING mode, the Monitor removes
# it from the set of readers.
def setUp(self):
members = [{}, {'priority': 0}, {'priority': 0}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_monitor_removes_recovering_member(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestPassiveAndHidden(unittest.TestCase):
def setUp(self):
members = [{}, {
'priority': 0
}, {
'arbiterOnly': True
}, {
'priority': 0,
'hidden': True
}, {
'priority': 0,
'slaveDelay': 5
}]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_passive_and_hidden(self):
self.c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
utils.assertReadFromAll(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = MongoClient(primary, use_greenlets=use_greenlets)
secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets)
rsc = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
class TestPassiveAndHidden(unittest.TestCase):
def setUp(self):
members = [{},
{'priority': 0},
{'arbiterOnly': True},
{'priority': 0, 'hidden': True},
{'priority': 0, 'slaveDelay': 5}
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
def test_passive_and_hidden(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
passives = ha_tools.get_passives()
passives = [_partition_node(member) for member in passives]
self.assertEqual(self.c.secondaries, set(passives))
for mode in SECONDARY, SECONDARY_PREFERRED:
utils.assertReadFromAll(self, self.c, passives, mode)
ha_tools.kill_members(ha_tools.get_passives(), 2)
sleep(2 * MONITOR_INTERVAL)
utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class Connection():
def __init__(self):
self.connection = None
def connect(self,hosts="localhost", replicaSet="", readPreference=ReadPreference.SECONDARY_PREFERRED):
# if the replicaset is empty, we use a normal connection, if not, we use a MongoReplicaSetClient
# in wich case we set the read preference
# by default, SECONDARY_PREFERRED since we don't big have consistancy issues. Fail safe when the primary/secondary goes down.
if replicaSet == "":
print("Using mongodb host " + hosts + " for read/writes.")
self.connection = MongoConnection(hosts)
else:
print("Using mongodb hosts " + hosts + " with replica set " + replicaSet + " and read pref " + str(readPreference))
self.connection = MongoReplicaSetClient(hosts, replicaSet=replicaSet, read_preference=readPreference)
print("Replica set:")
print("PRIMARY: " + str(self.connection.primary))
print("SECONDARIES: " + str(self.connection.secondaries))
print("ARBITERS: " + str(self.connection.arbiters))
def close(self):
self.connection.close()
self.connection = None
# \return the requested db setting the readPreference if set.
def getDatabase(self, dbname, readPreference=None):
db = self.connection[dbname]
if readPreference != None:
db.read_preference = readPreference
return db
class TestReplicaSetRequest(unittest.TestCase):
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = MongoReplicaSetClient(res[0],
replicaSet=res[1],
use_greenlets=use_greenlets,
auto_start_request=True)
def test_request_during_failover(self):
primary = _partition_node(ha_tools.get_primary())
secondary = _partition_node(ha_tools.get_random_secondary())
self.assertTrue(self.c.auto_start_request)
self.assertTrue(self.c.in_request())
primary_pool = self.c._MongoReplicaSetClient__members[primary].pool
secondary_pool = self.c._MongoReplicaSetClient__members[secondary].pool
# Trigger start_request on primary pool
utils.assertReadFrom(self, self.c, primary, PRIMARY)
self.assertTrue(primary_pool.in_request())
# Fail over
ha_tools.kill_primary()
patience_seconds = 60
for _ in range(patience_seconds):
sleep(1)
try:
if ha_tools.ha_tools_debug:
print 'Waiting for failover'
if ha_tools.get_primary():
# We have a new primary
break
except ConnectionFailure:
pass
else:
self.fail("Problem with test: No new primary after %s seconds" %
patience_seconds)
try:
# Trigger start_request on secondary_pool, which is becoming new
# primary
self.c.test.test.find_one()
except AutoReconnect:
# We've noticed the failover now
pass
# The old secondary is now primary
utils.assertReadFrom(self, self.c, secondary, PRIMARY)
self.assertTrue(self.c.in_request())
self.assertTrue(secondary_pool.in_request())
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestReplicaSetRequest(unittest.TestCase):
def setUp(self):
members = [{}, {}, {'arbiterOnly': True}]
res = ha_tools.start_replica_set(members)
self.c = MongoReplicaSetClient(res[0], replicaSet=res[1],
use_greenlets=use_greenlets,
auto_start_request=True)
def test_request_during_failover(self):
primary = _partition_node(ha_tools.get_primary())
secondary = _partition_node(ha_tools.get_random_secondary())
self.assertTrue(self.c.auto_start_request)
self.assertTrue(self.c.in_request())
primary_pool = self.c._MongoReplicaSetClient__members[primary].pool
secondary_pool = self.c._MongoReplicaSetClient__members[secondary].pool
# Trigger start_request on primary pool
utils.assertReadFrom(self, self.c, primary, PRIMARY)
self.assertTrue(primary_pool.in_request())
# Fail over
ha_tools.kill_primary()
patience_seconds = 60
for _ in range(patience_seconds):
sleep(1)
try:
if ha_tools.ha_tools_debug:
print 'Waiting for failover'
if ha_tools.get_primary():
# We have a new primary
break
except ConnectionFailure:
pass
else:
self.fail("Problem with test: No new primary after %s seconds"
% patience_seconds)
try:
# Trigger start_request on secondary_pool, which is becoming new
# primary
self.c.test.test.find_one()
except AutoReconnect:
# We've noticed the failover now
pass
# The old secondary is now primary
utils.assertReadFrom(self, self.c, secondary, PRIMARY)
self.assertTrue(self.c.in_request())
self.assertTrue(secondary_pool.in_request())
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestReplicaSetAuth(unittest.TestCase):
def setUp(self):
members = [
{},
{
'priority': 0
},
{
'priority': 0
},
]
res = ha_tools.start_replica_set(members, auth=True)
self.c = MongoReplicaSetClient(res[0],
replicaSet=res[1],
use_greenlets=use_greenlets)
# Add an admin user to enable auth
self.c.admin.add_user('admin', 'adminpass')
self.c.admin.authenticate('admin', 'adminpass')
self.db = self.c.pymongo_ha_auth
self.db.add_user('user', 'userpass')
self.c.admin.logout()
def test_auth_during_failover(self):
self.assertTrue(self.db.authenticate('user', 'userpass'))
self.assertTrue(
self.db.foo.insert({'foo': 'bar'}, safe=True, w=3, wtimeout=1000))
self.db.logout()
self.assertRaises(OperationFailure, self.db.foo.find_one)
primary = '%s:%d' % self.c.primary
ha_tools.kill_members([primary], 2)
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# Make sure we can still authenticate
self.assertTrue(self.db.authenticate('user', 'userpass'))
# And still query.
self.db.read_preference = PRIMARY_PREFERRED
self.assertEqual('bar', self.db.foo.find_one()['foo'])
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
class TestReplicaSetAuth(unittest.TestCase):
def setUp(self):
members = [
{},
{'priority': 0},
{'priority': 0},
]
res = ha_tools.start_replica_set(members, auth=True)
self.c = MongoReplicaSetClient(res[0], replicaSet=res[1],
use_greenlets=use_greenlets)
# Add an admin user to enable auth
self.c.admin.add_user('admin', 'adminpass')
self.c.admin.authenticate('admin', 'adminpass')
self.db = self.c.pymongo_ha_auth
self.db.add_user('user', 'userpass')
self.c.admin.logout()
def test_auth_during_failover(self):
self.assertTrue(self.db.authenticate('user', 'userpass'))
self.assertTrue(self.db.foo.insert({'foo': 'bar'},
safe=True, w=3, wtimeout=1000))
self.db.logout()
self.assertRaises(OperationFailure, self.db.foo.find_one)
primary = '%s:%d' % self.c.primary
ha_tools.kill_members([primary], 2)
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# Make sure we can still authenticate
self.assertTrue(self.db.authenticate('user', 'userpass'))
# And still query.
self.db.read_preference = PRIMARY_PREFERRED
self.assertEqual('bar', self.db.foo.find_one()['foo'])
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
sys.stderr.write("Operation %s" % e)
error_output_enabled = re.search("enabled",str(e))
error_output_initialized = re.search("initialized",str(e))
if error_output_enabled:
print "sharding already enabled"
pass
if error_output_initialized:
print "initialized"
pass
try:
#initiate Sharding
config = {_id: replicaset, members: [
{_id: 0, host: servers[0]},
{_id: 1, host: servers[1]},
{_id: 2, host: servers[2]}]
}
mongod.admin.command("replSetInitiate" , config)
except pymongo.errors.OperationFailure, e:
error_output = re.search("initialized",str(e))
if error_output:
print "sharding already initialized"
pass
mongod.close()
mongos.close()
class TestReadPreference(unittest.TestCase):
def setUp(self):
members = [
# primary
{
'tags': {
'dc': 'ny',
'name': 'primary'
}
},
# secondary
{
'tags': {
'dc': 'la',
'name': 'secondary'
},
'priority': 0
},
# other_secondary
{
'tags': {
'dc': 'ny',
'name': 'other_secondary'
},
'priority': 0
},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary'
]
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary'
]
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, w=self.w)
self.db.test.insert([{'foo': i} for i in xrange(10)], w=self.w)
self.clear_ping_times()
def set_ping_time(self, host, ping_time_seconds):
Member._host_to_ping_time[host] = ping_time_seconds
def clear_ping_times(self):
Member._host_to_ping_time.clear()
def test_read_preference(self):
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy hosts into local scope
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED,
[bad_tag, {}])
assertReadFrom(other_secondary, SECONDARY_PREFERRED,
[bad_tag, {
'dc': 'ny'
}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary], NEAREST,
[bad_tag, {
'dc': 'ny'
}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll([primary, secondary, other_secondary],
NEAREST,
secondary_acceptable_latency_ms=1000 * 1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(
MongoClient(unpartition_node(primary),
use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED).admin.command(
'ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(other_secondary, SECONDARY_PREFERRED,
self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(
MongoClient(unpartition_node(primary),
use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED).admin.command(
'ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_pinning(self):
# To make the code terser, copy modes into local scope
PRIMARY = ReadPreference.PRIMARY
PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED
SECONDARY = ReadPreference.SECONDARY
SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
NEAREST = ReadPreference.NEAREST
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets,
auto_start_request=True)
# Verify that changing the mode unpins the member. We'll try it for
# every relevant change of mode.
for mode0, mode1 in itertools.permutations(
(PRIMARY, SECONDARY, SECONDARY_PREFERRED, NEAREST), 2):
# Try reading and then changing modes and reading again, see if we
# read from a different host
for _ in range(1000):
# pin to this host
host = utils.read_from_which_host(c, mode0)
# unpin?
new_host = utils.read_from_which_host(c, mode1)
if host != new_host:
# Reading with a different mode unpinned, hooray!
break
else:
self.fail("Changing from mode %s to mode %s never unpinned" %
(modes[mode0], modes[mode1]))
# Now verify changing the tag_sets unpins the member.
tags0 = [{'a': 'a'}, {}]
tags1 = [{'a': 'x'}, {}]
for _ in range(1000):
host = utils.read_from_which_host(c, NEAREST, tags0)
new_host = utils.read_from_which_host(c, NEAREST, tags1)
if host != new_host:
break
else:
self.fail("Changing from tags %s to tags %s never unpinned" %
(tags0, tags1))
# Finally, verify changing the secondary_acceptable_latency_ms unpins
# the member.
for _ in range(1000):
host = utils.read_from_which_host(c, SECONDARY, None, 15)
new_host = utils.read_from_which_host(c, SECONDARY, None, 20)
if host != new_host:
break
else:
self.fail("Changing secondary_acceptable_latency_ms from 15 to 20"
" never unpinned")
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
self.clear_ping_times()
class TestReadPreference(unittest.TestCase):
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], w=self.w)
self.clear_ping_times()
def set_ping_time(self, host, ping_time_seconds):
Member._host_to_ping_time[host] = ping_time_seconds
def clear_ping_times(self):
Member._host_to_ping_time.clear()
def test_read_preference(self):
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy hosts into local scope
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}])
assertReadFrom(other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll(
[primary, secondary, other_secondary],
NEAREST, secondary_acceptable_latency_ms=1000*1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(MongoClient(
unpartition_node(primary), use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED
).admin.command('ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(MongoClient(
unpartition_node(primary), use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED
).admin.command('ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_pinning(self):
# To make the code terser, copy modes into local scope
PRIMARY = ReadPreference.PRIMARY
PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED
SECONDARY = ReadPreference.SECONDARY
SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
NEAREST = ReadPreference.NEAREST
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=True)
# Verify that changing the mode unpins the member. We'll try it for
# every relevant change of mode.
for mode0, mode1 in itertools.permutations(
(PRIMARY, SECONDARY, SECONDARY_PREFERRED, NEAREST), 2
):
# Try reading and then changing modes and reading again, see if we
# read from a different host
for _ in range(1000):
# pin to this host
host = utils.read_from_which_host(c, mode0)
# unpin?
new_host = utils.read_from_which_host(c, mode1)
if host != new_host:
# Reading with a different mode unpinned, hooray!
break
else:
self.fail(
"Changing from mode %s to mode %s never unpinned" % (
modes[mode0], modes[mode1]))
# Now verify changing the tag_sets unpins the member.
tags0 = [{'a': 'a'}, {}]
tags1 = [{'a': 'x'}, {}]
for _ in range(1000):
host = utils.read_from_which_host(c, NEAREST, tags0)
new_host = utils.read_from_which_host(c, NEAREST, tags1)
if host != new_host:
break
else:
self.fail(
"Changing from tags %s to tags %s never unpinned" % (
tags0, tags1))
# Finally, verify changing the secondary_acceptable_latency_ms unpins
# the member.
for _ in range(1000):
host = utils.read_from_which_host(c, SECONDARY, None, 15)
new_host = utils.read_from_which_host(c, SECONDARY, None, 20)
if host != new_host:
break
else:
self.fail(
"Changing secondary_acceptable_latency_ms from 15 to 20"
" never unpinned")
def tearDown(self):
self.c.close()
ha_tools.kill_all_members()
self.clear_ping_times()
class MongodbBroker(BaseModule):
def __init__(self, mod_conf):
BaseModule.__init__(self, mod_conf)
self._parse_conf(mod_conf)
self.queue = Queue.Queue(self.queue_size)
self.conn = None
# service notification log broks.
# ref: service.raise_notification_log_entry
self.service_notification = ('contact',
'host',
'service_description',
'state',
'command',
'output'
)
# host notification log broks.
# ref: host.raise_notification_log_entry
self.host_notification = ('contact',
'host',
'state',
'command',
'output')
self.timestamp_regex = re.compile('.*\[(?P<timestamp>\d+)\].*')
def _parse_conf(self, mod_conf):
self.high_availability = to_bool(getattr(mod_conf,
'high_availability', 'false'))
if not self.high_availability:
self.stand_alone = getattr(mod_conf, 'stand_alone', '')
if not self.stand_alone:
logger.error('[Mongodb-Notification-Broker] Mongodb is '
'configured with high availability be false but '
'stand_alone is not configured')
raise Exception('[Mongodb-Notification-Broker] Configuration '
'Error')
else:
replica_set_str = getattr(mod_conf, 'replica_set', '')
self._set_replica_set(replica_set_str)
self.database = getattr(mod_conf,
'database', 'shinken_broker_notification')
self.username = getattr(mod_conf,
'username', 'shinken_broker_notification')
self.password = getattr(mod_conf,
'password', 'shinken_broker_notification')
self.url_options = getattr(mod_conf, 'url_options', '')
try:
self.retry_per_log = int(getattr(mod_conf, 'retry_per_log'))
except:
self.retry_per_log = 5
try:
self.queue_size = int(getattr(mod_conf, 'queue_size'))
except:
self.queue_size = 10000
def _set_replica_set(self, replica_set_str):
raw_members = replica_set_str.split(',')
members = []
for member in raw_members:
members.append(member.strip())
self.replica_set = members
def _set_mongodb_url(self):
scheme = 'mongodb://'
db_and_options = '/%s?%s' % (self.database, self.url_options)
credential = ':'.join((self.username, '%s@' % self.password))
if not self.high_availability:
address = self.stand_alone
mongodb_url = ''.join((scheme, credential, address, db_and_options))
else:
address = ','.join(self.replica_set)
mongodb_url = ''.join((scheme, credential, address, db_and_options))
self.mongodb_url = mongodb_url
# Called by Broker to do init work
def init(self):
logger.info('[Mongodb-Notification-Broker] Initialization of '
'mongodb_notification_broker module')
self._set_mongodb_url()
logger.debug('[Mongodb-Notification-Broker] Mongodb connect url: %s'
% self.mongodb_url)
# In case notification broker process down occasionally, the self.conn
# object must be dropped cleanly in Broker daemon.
self.do_stop()
try:
if not self.high_availability:
self.conn = MongoClient(self.mongodb_url)
else:
self.conn = MongoReplicaSetClient(self.mongodb_url)
except ConnectionFailure:
logger.warn('[Mongodb-Notification-Broker] Can not make connection '
' with MongoDB')
raise
except (InvalidURI, ConfigurationError):
logger.warn('[Mongodb-Notification-Broker] Mongodb connect url '
'error')
logger.warn('[Mongodb-Notification-Broker] Mongodb connect url: %s'
% self.mongodb_url)
raise
self._get_collections()
def _get_collections(self):
db = self.conn[self.database]
self.hosts = db['hosts']
self.services = db['services']
self.notifications = db['notifications']
# Override the same function in basemodule.py for clean up
def do_stop(self):
if self.conn:
self.conn.close()
self.conn = None
# If we are confronted with AutoReconnect Exception, then we should always
# retry until the operation succeeds. However, if other exception is thrown,
# we should ignore the operation and go to next operation.
def _process_db_operation(self, operation, *param):
reconnect_start = time.time()
result = None
while True:
try:
result = operation(*param)
except AutoReconnect:
logger.warn('[Mongodb-Notification-Broker] Update error. '
'Reconnected last %d seconds' % (time.time() - reconnect_start))
# avoid to invoke too many write operations
time.sleep(self.retry_per_log)
except Exception:
logger.warn('[Mongodb-Notification-Broker] Update error. '
'operation %s, param %s' % (operation, param))
logger.warn('[Mongodb-Notification-Broker] %s' % traceback.format_exc())
break
else:
logger.debug('[Mongodb-Notification-Broker] Update success. '
'Operation %s, param %s' % (operation, param))
break
return result
# main function to update mongodb database
def _save(self, ref, ref_identity, notification):
self._process_db_operation(self.notifications.insert, notification)
if ref == 'service':
_id = ','.join((ref_identity.get('host'),
ref_identity.get('service_description')))
cursor = self._process_db_operation(self.services.find,
{'_id': _id})
elif ref == 'host':
_id = ref_identity.get('host')
cursor = self._process_db_operation(self.hosts.find, {'_id': _id})
# if service or host find error, 'cursor' will be None.
# then we can not make sure that whether specific host or service
# exists. In order to not make data be corrupted, we should stop here.
if cursor:
if not cursor.count():
# if notification insert error, then '_id' will not be in it and we
# then should ignore the notification.
ref_identity.setdefault('notification_ids',
[notification.get('_id')] if '_id' in notification else [])
ref_identity.setdefault('_id', _id)
if ref == 'service':
self._process_db_operation(self.services.insert, ref_identity)
elif ref == 'host':
self._process_db_operation(self.hosts.insert, ref_identity)
else:
document = cursor[0]
notification_ids = document.get('notification_ids')
# if notification insert error, then '_id' will not be in it and we
# then should ignore the notification
if '_id' in notification:
notification_ids.append(notification.get('_id'))
if ref == 'service':
self._process_db_operation(self.services.update,
{'_id': _id},
{'$set': {'notification_ids': notification_ids}})
elif ref == 'host':
self._process_db_operation(self.hosts.update,
{'_id': _id},
{'$set': {'notification_ids': notification_ids}})
else:
logger.warn('[Mongodb-Notification-Broker] Update notification '
'success, link with host or service error.')
logger.debug('[Mongodb-Notification-Broker] Update notification ends.')
# restore 'log' type notification to self.queue
def _do_loop_turn(self):
while not self.interrupted:
broks = self.to_q.get()
for brok in broks:
brok.prepare()
self._manage_brok(brok)
def _update_db(self):
while True:
# if self.queue is empty, get operation will be blocked.
brok = self.queue.get()
logger.debug('[Mongodb-Notification-Broker] '
'Update notification begins.')
msg = brok.data['log']
parts = msg.split(':', 1)
if 'SERVICE' in parts[0]:
service_identiry, notification = self._process_notification_brok('service',
self.service_notification,
parts[0], parts[1])
self._save('service', service_identiry, notification)
elif 'HOST' in parts[0]:
host_identity, notification = self._process_notification_brok('host',
self.host_notification,
parts[0], parts[1])
self._save('host', host_identity, notification)
def _process_notification_brok(self, ref, keys, header, notification_info):
elts = notification_info.split(';', len(keys))
timestamp = ''
match = self.timestamp_regex.match(header)
if match:
timestamp = match.group('timestamp')
info_map = dict(zip(keys, elts))
if ref == 'service':
ref_identity = {'host': info_map.get('host'),
'service_description': info_map.get('service_description')
}
elif ref == 'host':
ref_identity = {'host': info_map.get('host')}
notification = {'contact': info_map.get('contact'),
'command': info_map.get('command'),
'output': info_map.get('output'),
'timestamp': timestamp
}
return ref_identity, notification
def _manage_brok(self, brok):
if brok.type == 'log' and 'NOTIFICATION' in brok.data['log']:
try:
self.queue.put_nowait(brok)
except Queue.Full:
logger.warn('[Mongodb-Notification-Broker] Queue full. '
'Ignore broks.')
# invoked by basemodule._main
def main(self):
logger.info('[Mongodb-Notification-Broker] Start main function.')
worker = Thread(target=self._update_db)
worker.setDaemon(True)
worker.start()
self._do_loop_turn()
