def test_ship_of_theseus(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
db = c.pymongo_test
db.test.insert({}, w=len(c.secondaries) + 1)
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = [ha_tools.add_member() for _ in range(3)]
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary.
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
sleep(1)
else:
self.fail("No failover")
sleep(2 * MONITOR_INTERVAL)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self, done):
loop = IOLoop.instance()
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
c.open_sync()
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL)
try:
yield motor.Op(db.test.find_one)
except AutoReconnect:
# Might take one try to reconnect
yield gen.Task(loop.add_timeout, time.time() + 1)
# No error
yield motor.Op(db.test.find_one)
yield motor.Op(db.test.find_one, read_preference=SECONDARY)
done()
def test_stepdown_triggers_refresh(self):
c_find_one = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(one(c_find_one.secondaries),
_partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
sleep(1)
# Trigger a refresh
self.assertRaises(AutoReconnect, c_find_one.test.test.find_one)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
sleep(1)
# We've detected the stepdown
self.assertTrue(not c_find_one.primary
or primary != _partition_node(c_find_one.primary))
def test_recovering_member_triggers_refresh(self):
# To test that find_one() and count() trigger immediate refreshes,
# we'll create a separate client for each
self.c_find_one, self.c_count = yield [
motor.MotorReplicaSetClient(self.seed,
replicaSet=self.name,
read_preference=SECONDARY).open()
for _ in xrange(2)
]
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
# Pre-condition: just make sure they all connected OK
for c in self.c_find_one, self.c_count:
self.assertEqual(one(c.secondaries), _partition_node(secondary))
ha_tools.set_maintenance(secondary, True)
# Trigger a refresh in various ways
with assert_raises(AutoReconnect):
yield self.c_find_one.test.test.find_one()
with assert_raises(AutoReconnect):
yield self.c_count.test.test.count()
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield self.pause(1)
for c in self.c_find_one, self.c_count:
self.assertFalse(c.secondaries)
self.assertEqual(_partition_node(primary), c.primary)
def test_stepdown_triggers_refresh(self):
c_find_one = yield motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open()
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(one(c_find_one.secondaries),
_partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
yield self.pause(1)
# Trigger a refresh
with assert_raises(AutoReconnect):
yield c_find_one.test.test.find_one()
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield self.pause(1)
# We've detected the stepdown
self.assertTrue(not c_find_one.primary
or primary != _partition_node(c_find_one.primary))
def test_stepdown_triggers_refresh(self, done):
c_find_one = motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open_sync()
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(
one(c_find_one.secondaries), _partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1)
# Trigger a refresh
yield AssertRaises(AutoReconnect, c_find_one.test.test.find_one)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1)
# We've detected the stepdown
self.assertTrue(
not c_find_one.primary
or primary != _partition_node(c_find_one.primary))
done()
def test_stepdown_triggers_refresh(self):
c_find_one = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(
one(c_find_one.secondaries), _partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
sleep(1)
# Trigger a refresh
self.assertRaises(AutoReconnect, c_find_one.test.test.find_one)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
sleep(1)
# We've detected the stepdown
self.assertTrue(
not c_find_one.primary
or _partition_node(primary) != c_find_one.primary)
def test_recovering_member_triggers_refresh(self):
# To test that find_one() and count() trigger immediate refreshes,
# we'll create a separate client for each
self.c_find_one, self.c_count = yield [
motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name, read_preference=SECONDARY
).open() for _ in xrange(2)]
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
# Pre-condition: just make sure they all connected OK
for c in self.c_find_one, self.c_count:
self.assertEqual(one(c.secondaries), _partition_node(secondary))
ha_tools.set_maintenance(secondary, True)
# Trigger a refresh in various ways
with assert_raises(AutoReconnect):
yield self.c_find_one.test.test.find_one()
with assert_raises(AutoReconnect):
yield self.c_count.test.test.count()
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield self.pause(1)
for c in self.c_find_one, self.c_count:
self.assertFalse(c.secondaries)
self.assertEqual(_partition_node(primary), c.primary)
def test_stepdown_triggers_refresh(self):
c_find_one = yield motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open()
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
self.assertEqual(
one(c_find_one.secondaries), _partition_node(secondary))
ha_tools.stepdown_primary()
# Make sure the stepdown completes
yield self.pause(1)
# Trigger a refresh
with assert_raises(AutoReconnect):
yield c_find_one.test.test.find_one()
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
yield self.pause(1)
# We've detected the stepdown
self.assertTrue(
not c_find_one.primary
or primary != _partition_node(c_find_one.primary))
def test_primary_stepdown(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
ha_tools.stepdown_primary()
# Wait for new primary
patience_seconds = 30
for _ in xrange(patience_seconds):
sleep(1)
rs_state = c._MongoReplicaSetClient__rs_state
if rs_state.writer and rs_state.writer != primary:
if ha_tools.get_primary():
# New primary stepped up
new_primary = _partition_node(ha_tools.get_primary())
self.assertEqual(new_primary, rs_state.writer)
new_secondaries = partition_nodes(
ha_tools.get_secondaries())
self.assertEqual(set(new_secondaries),
rs_state.secondaries)
break
else:
self.fail(
"No new primary after %s seconds. Old primary was %s, current"
" is %s" % (patience_seconds, primary, ha_tools.get_primary()))
def test_secondary_connection(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
db.test.remove({}, safe=True, w=len(self.c.secondaries))
# Force replication...
w = len(self.c.secondaries) + 1
db.test.insert({'foo': 'bar'}, safe=True, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets).is_primary)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets,
read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets,
read_preference=ReadPreference.NEAREST).is_primary)
self.assertTrue(Connection(
primary_host, primary_port, use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY).is_primary)
for kwargs in [
{'read_preference': ReadPreference.PRIMARY_PREFERRED},
{'read_preference': ReadPreference.SECONDARY},
{'read_preference': ReadPreference.SECONDARY_PREFERRED},
{'read_preference': ReadPreference.NEAREST},
{'slave_okay': True},
]:
conn = Connection(secondary_host,
secondary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
self.assert_(conn.pymongo_test.test.find_one())
# Test direct connection to an arbiter
secondary_host = ha_tools.get_arbiters()[0]
host, port = ha_tools.get_arbiters()[0].split(':')
port = int(port)
conn = Connection(host, port)
self.assertEqual(host, conn.host)
self.assertEqual(port, conn.port)
def test_ship_of_theseus(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
db = c.motor_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield self.pause(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield self.pause(1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield self.pause(2 * MONITOR_INTERVAL)
try:
yield db.test.find_one()
except AutoReconnect:
# Might take one try to reconnect
yield self.pause(1)
# No error
yield db.test.find_one()
yield db.test.find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
db = c.motor_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield self.pause(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield self.pause(1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield self.pause(2 * MONITOR_INTERVAL)
try:
yield db.test.find_one()
except AutoReconnect:
# Might take one try to reconnect
yield self.pause(1)
# No error
yield db.test.find_one()
yield db.test.find_one(read_preference=SECONDARY)
def setUp(self):
super(TestReadPreference, self).setUp()
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
self.w = len(self.c.secondaries) + 1
self.db = self.c.get_database("pymongo_test",
write_concern=WriteConcern(w=self.w))
self.db.test.delete_many({})
self.db.test.insert_many([{'foo': i} for i in xrange(10)])
self.clear_ping_times()
def setUp(self):
super(TestReadPreference, self).setUp()
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
self.w = len(self.c.secondaries) + 1
self.db = self.c.get_database("pymongo_test",
write_concern=WriteConcern(w=self.w))
self.db.test.delete_many({})
self.db.test.insert_many([{'foo': i} for i in xrange(10)])
self.clear_ping_times()
def setUp(self):
super(MotorTestReadPreference, self).setUp()
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
# Synchronous PyMongo interfaces for convenience
self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient(
self.seed, replicaSet=self.name)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], w=self.w)
self.clear_ping_times()
def setUp(self):
super(MotorTestReadPreference, self).setUp()
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
# Synchronous PyMongo interfaces for convenience
self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient(
self.seed, replicaSet=self.name)
self.db = self.c.motor_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], w=self.w)
self.clear_ping_times()
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, safe=True, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], safe=True, w=self.w)
self.clear_ping_times()
def setUp(self):
members = [
# primary
{'tags': {'dc': 'ny', 'name': 'primary'}},
# secondary
{'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},
# other_secondary
{'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual('primary', self.primary_tags['name'])
self.primary_dc = {'dc': self.primary_tags['dc']}
secondaries = ha_tools.get_secondaries()
(secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'secondary']
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {'dc': self.secondary_tags['dc']}
(other_secondary, ) = [
s for s in secondaries
if ha_tools.get_tags(s)['name'] == 'other_secondary']
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, safe=True, w=self.w)
self.db.test.insert(
[{'foo': i} for i in xrange(10)], safe=True, w=self.w)
self.clear_ping_times()
def test_monitor_removes_recovering_member(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = partition_nodes(secondaries)
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def test_monitor_removes_recovering_member(self):
self.c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = partition_nodes(secondaries)
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def test_monitor_removes_recovering_member(self):
self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield self.c.open()
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
yield assert_read_from_all(self, self.c, partitioned_secondaries,
mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
yield self.pause(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
yield assert_read_from(self, self.c, _partition_node(secondary),
mode)
def test_monitor_removes_recovering_member(self):
self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield self.c.open()
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
yield assert_read_from_all(
self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
yield self.pause(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
yield assert_read_from(
self, self.c, _partition_node(secondary), mode)
def test_recovering(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=False)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
sleep(2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def test_monitor_removes_recovering_member(self):
self.c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = partition_nodes(secondaries)
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
time.sleep(2 * self.heartbeat_frequency)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, partition_node(secondary), mode)
def test_monitor_removes_recovering_member(self):
self.c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = partition_nodes(secondaries)
utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
time.sleep(2 * self.heartbeat_frequency)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
utils.assertReadFrom(self, self.c, partition_node(secondary), mode)
def setUp(self):
members = [
# primary
{"tags": {"dc": "ny", "name": "primary"}},
# secondary
{"tags": {"dc": "la", "name": "secondary"}, "priority": 0},
# other_secondary
{"tags": {"dc": "ny", "name": "other_secondary"}, "priority": 0},
]
res = ha_tools.start_replica_set(members)
self.seed, self.name = res
primary = ha_tools.get_primary()
self.primary = _partition_node(primary)
self.primary_tags = ha_tools.get_tags(primary)
# Make sure priority worked
self.assertEqual("primary", self.primary_tags["name"])
self.primary_dc = {"dc": self.primary_tags["dc"]}
secondaries = ha_tools.get_secondaries()
(secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "secondary"]
self.secondary = _partition_node(secondary)
self.secondary_tags = ha_tools.get_tags(secondary)
self.secondary_dc = {"dc": self.secondary_tags["dc"]}
(other_secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "other_secondary"]
self.other_secondary = _partition_node(other_secondary)
self.other_secondary_tags = ha_tools.get_tags(other_secondary)
self.other_secondary_dc = {"dc": self.other_secondary_tags["dc"]}
self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.db = self.c.pymongo_test
self.w = len(self.c.secondaries) + 1
self.db.test.remove({}, safe=True, w=self.w)
self.db.test.insert([{"foo": i} for i in xrange(10)], safe=True, w=self.w)
self.clear_ping_times()
def test_monitor_removes_recovering_member(self):
self.c = motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name)
secondaries = ha_tools.get_secondaries()
for mode in SECONDARY, SECONDARY_PREFERRED:
partitioned_secondaries = [_partition_node(s) for s in secondaries]
yield motor.Op(assertReadFromAll,
self, self.c, partitioned_secondaries, mode)
secondary, recovering_secondary = secondaries
ha_tools.set_maintenance(recovering_secondary, True)
yield gen.Task(
IOLoop.instance().add_timeout, time.time() + 2 * MONITOR_INTERVAL)
for mode in SECONDARY, SECONDARY_PREFERRED:
# Don't read from recovering member
yield motor.Op(assertReadFrom,
self, self.c, _partition_node(secondary), mode)
def test_recovering_member_triggers_refresh(self):
# To test that find_one() and count() trigger immediate refreshes,
# we'll create a separate client for each
self.c_find_one, self.c_count = [
MongoClient(
self.seed,
replicaSet=self.name,
read_preference=SECONDARY,
serverSelectionTimeoutMS=self.server_selection_timeout)
for _ in xrange(2)]
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
# Pre-condition: just make sure they all connected OK
for c in self.c_find_one, self.c_count:
wait_until(
lambda: c.primary == partition_node(primary),
'connect to the primary')
wait_until(
lambda: one(c.secondaries) == partition_node(secondary),
'connect to the secondary')
ha_tools.set_maintenance(secondary, True)
# Trigger a refresh in various ways
self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one)
self.assertRaises(AutoReconnect, self.c_count.test.test.count)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
time.sleep(1)
self.assertFalse(self.c_find_one.secondaries)
self.assertEqual(partition_node(primary), self.c_find_one.primary)
self.assertFalse(self.c_count.secondaries)
self.assertEqual(partition_node(primary), self.c_count.primary)
def test_recovering_member_triggers_refresh(self):
# To test that find_one() and count() trigger immediate refreshes,
# we'll create a separate client for each
self.c_find_one, self.c_count = [
MongoClient(
self.seed,
replicaSet=self.name,
read_preference=SECONDARY,
serverSelectionTimeoutMS=self.server_selection_timeout)
for _ in xrange(2)]
# We've started the primary and one secondary
primary = ha_tools.get_primary()
secondary = ha_tools.get_secondaries()[0]
# Pre-condition: just make sure they all connected OK
for c in self.c_find_one, self.c_count:
wait_until(
lambda: c.primary == partition_node(primary),
'connect to the primary')
wait_until(
lambda: one(c.secondaries) == partition_node(secondary),
'connect to the secondary')
ha_tools.set_maintenance(secondary, True)
# Trigger a refresh in various ways
self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one)
self.assertRaises(AutoReconnect, self.c_count.test.test.count)
# Wait for the immediate refresh to complete - we're not waiting for
# the periodic refresh, which has been disabled
time.sleep(1)
self.assertFalse(self.c_find_one.secondaries)
self.assertEqual(partition_node(primary), self.c_find_one.primary)
self.assertFalse(self.c_count.secondaries)
self.assertEqual(partition_node(primary), self.c_count.primary)
def test_primary_stepdown(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
ha_tools.stepdown_primary()
# Wait for new primary
patience_seconds = 30
for _ in xrange(patience_seconds):
sleep(1)
rs_state = c._MongoReplicaSetClient__rs_state
if rs_state.writer and rs_state.writer != primary:
# New primary stepped up
new_primary = _partition_node(ha_tools.get_primary())
self.assertEqual(new_primary, rs_state.writer)
new_secondaries = partition_nodes(ha_tools.get_secondaries())
self.assertEqual(set(new_secondaries), rs_state.secondaries)
break
else:
self.fail(
"No new primary after %s seconds. Old primary was %s, current"
" is %s" % (patience_seconds, primary, ha_tools.get_primary()))
def test_ship_of_theseus(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
db = c.get_database(
"pymongo_test",
write_concern=WriteConcern(w=len(c.secondaries) + 1))
db.test.insert_one({})
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = []
for i in range(3):
new_hosts.append(ha_tools.add_member())
# RS closes all connections after reconfig.
for j in xrange(30):
try:
if ha_tools.get_primary():
break
except (ConnectionFailure, OperationFailure):
pass
time.sleep(1)
else:
self.fail("Couldn't recover from reconfig")
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
time.sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
time.sleep(5)
wait_until(lambda: (ha_tools.get_primary()
and len(ha_tools.get_secondaries()) == 2),
"fail over",
timeout=30)
time.sleep(2 * self.heartbeat_frequency)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
time.sleep(2 * self.heartbeat_frequency)
find_one(read_preference=SECONDARY)
# Kill new members and switch back to original two members.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members([primary, secondary1])
# Wait for members to figure out they're secondaries.
wait_until(lambda: len(ha_tools.get_secondaries()) == 2,
"detect two secondaries",
timeout=30)
# Should be able to reconnect to set again.
time.sleep(2 * self.heartbeat_frequency)
find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
db = c.get_database(
"pymongo_test",
write_concern=WriteConcern(w=len(c.secondaries) + 1))
db.test.insert_one({})
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = []
for i in range(3):
new_hosts.append(ha_tools.add_member())
# RS closes all connections after reconfig.
for j in xrange(30):
try:
if ha_tools.get_primary():
break
except (ConnectionFailure, OperationFailure):
pass
time.sleep(1)
else:
self.fail("Couldn't recover from reconfig")
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
time.sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
time.sleep(5)
wait_until(lambda: (ha_tools.get_primary()
and len(ha_tools.get_secondaries()) == 2),
"fail over",
timeout=30)
time.sleep(2 * self.heartbeat_frequency)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
time.sleep(2 * self.heartbeat_frequency)
find_one(read_preference=SECONDARY)
# Kill new members and switch back to original two members.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members([primary, secondary1])
# Wait for members to figure out they're secondaries.
wait_until(lambda: len(ha_tools.get_secondaries()) == 2,
"detect two secondaries",
timeout=30)
# Should be able to reconnect to set again.
time.sleep(2 * self.heartbeat_frequency)
find_one(read_preference=SECONDARY)
def test_secondary_connection(self):
self.c = yield motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open()
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.motor_test
yield db.test.remove({}, w=len(self.c.secondaries))
# Wait for replication...
w = len(self.c.secondaries) + 1
yield db.test.insert({'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# A connection succeeds no matter the read preference
for kwargs in [
{
'read_preference': PRIMARY
},
{
'read_preference': PRIMARY_PREFERRED
},
{
'read_preference': SECONDARY
},
{
'read_preference': SECONDARY_PREFERRED
},
{
'read_preference': NEAREST
},
]:
client = yield motor.MotorClient(primary_host, primary_port,
**kwargs).open()
self.assertEqual(primary_host, client.host)
self.assertEqual(primary_port, client.port)
self.assertTrue(client.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue((yield client.motor_test.test.find_one()))
client = yield motor.MotorClient(secondary_host, secondary_port,
**kwargs).open()
self.assertEqual(secondary_host, client.host)
self.assertEqual(secondary_port, client.port)
self.assertFalse(client.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue((yield client.motor_test.test.find_one()))
else:
with assert_raises(AutoReconnect):
yield client.motor_test.test.find_one()
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), MotorClient
# should do the same for unacknowledged writes.
try:
yield client.motor_test.test.insert({}, w=0)
except AutoReconnect, e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary client %s should'
'have raised exception' % client)
# Test direct connection to an arbiter
client = yield motor.MotorClient(arbiter_host, arbiter_port,
**kwargs).open()
self.assertEqual(arbiter_host, client.host)
self.assertEqual(arbiter_port, client.port)
self.assertFalse(client.is_primary)
# See explanation above
try:
yield client.motor_test.test.insert({}, w=0)
except AutoReconnect, e:
self.assertEqual('not master', e.message)
def test_secondary_connection(self):
self.c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: len(self.c.secondaries), "discover secondary")
# Wait for replication...
w = len(self.c.secondaries) + 1
db = self.c.get_database("pymongo_test",
write_concern=WriteConcern(w=w))
db.test.delete_many({})
db.test.insert_one({'foo': 'bar'})
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# MongoClient succeeds no matter the read preference
for kwargs in [
{'read_preference': PRIMARY},
{'read_preference': PRIMARY_PREFERRED},
{'read_preference': SECONDARY},
{'read_preference': SECONDARY_PREFERRED},
{'read_preference': NEAREST},
]:
client = MongoClient(
primary_host,
primary_port,
serverSelectionTimeoutMS=self.server_selection_timeout,
**kwargs)
wait_until(lambda: primary_host == client.host,
"connect to primary")
self.assertEqual(primary_port, client.port)
self.assertTrue(client.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue(client.pymongo_test.test.find_one())
client = MongoClient(
secondary_host,
secondary_port,
serverSelectionTimeoutMS=self.server_selection_timeout,
**kwargs)
wait_until(lambda: secondary_host == client.host,
"connect to secondary")
self.assertEqual(secondary_port, client.port)
self.assertFalse(client.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue(client.pymongo_test.test.find_one())
else:
self.assertRaises(
AutoReconnect, client.pymongo_test.test.find_one)
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), MongoClient
# should do the same for unacknowledged writes.
try:
client.get_database(
"pymongo_test",
write_concern=WriteConcern(w=0)).test.insert_one({})
except AutoReconnect as e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary client %s should'
'have raised exception' % (client,))
# Test direct connection to an arbiter
client = MongoClient(
arbiter_host,
arbiter_port,
serverSelectionTimeoutMS=self.server_selection_timeout,
**kwargs)
wait_until(lambda: arbiter_host == client.host,
"connect to arbiter")
self.assertEqual(arbiter_port, client.port)
self.assertFalse(client.is_primary)
# See explanation above
try:
client.get_database(
"pymongo_test",
write_concern=WriteConcern(w=0)).test.insert_one({})
except AutoReconnect as e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into arbiter client %s should'
'have raised exception' % (client,))
def test_secondary_connection(self):
self.c = yield motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open()
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.motor_test
yield db.test.remove({}, w=len(self.c.secondaries))
# Wait for replication...
w = len(self.c.secondaries) + 1
yield db.test.insert({'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# A connection succeeds no matter the read preference
for kwargs in [
{'read_preference': PRIMARY},
{'read_preference': PRIMARY_PREFERRED},
{'read_preference': SECONDARY},
{'read_preference': SECONDARY_PREFERRED},
{'read_preference': NEAREST},
]:
client = yield motor.MotorClient(
primary_host, primary_port, **kwargs).open()
self.assertEqual(primary_host, client.host)
self.assertEqual(primary_port, client.port)
self.assertTrue(client.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue((yield client.motor_test.test.find_one()))
client = yield motor.MotorClient(
secondary_host, secondary_port, **kwargs).open()
self.assertEqual(secondary_host, client.host)
self.assertEqual(secondary_port, client.port)
self.assertFalse(client.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue((
yield client.motor_test.test.find_one()))
else:
with assert_raises(AutoReconnect):
yield client.motor_test.test.find_one()
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), MotorClient
# should do the same for unacknowledged writes.
try:
yield client.motor_test.test.insert({}, w=0)
except AutoReconnect, e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary client %s should'
'have raised exception' % client)
# Test direct connection to an arbiter
client = yield motor.MotorClient(
arbiter_host, arbiter_port, **kwargs).open()
self.assertEqual(arbiter_host, client.host)
self.assertEqual(arbiter_port, client.port)
self.assertFalse(client.is_primary)
# See explanation above
try:
yield client.motor_test.test.insert({}, w=0)
except AutoReconnect, e:
self.assertEqual('not master', e.message)
def test_ship_of_theseus(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
db = c.pymongo_test
db.test.insert({}, w=len(c.secondaries) + 1)
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = [ha_tools.add_member() for _ in range(3)]
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary.
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
sleep(1)
else:
self.fail("No failover")
sleep(2 * MONITOR_INTERVAL)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
# Kill new members and switch back to original two members.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members([primary, secondary1])
# Should be able to reconnect to set again.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
def test_secondary_connection(self):
self.c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: len(self.c.secondaries), "discover secondary")
# Wait for replication...
w = len(self.c.secondaries) + 1
db = self.c.get_database("pymongo_test",
write_concern=WriteConcern(w=w))
db.test.delete_many({})
db.test.insert_one({'foo': 'bar'})
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# MongoClient succeeds no matter the read preference
for kwargs in [
{'read_preference': PRIMARY},
{'read_preference': PRIMARY_PREFERRED},
{'read_preference': SECONDARY},
{'read_preference': SECONDARY_PREFERRED},
{'read_preference': NEAREST},
]:
client = MongoClient(
primary_host,
primary_port,
serverSelectionTimeoutMS=self.server_selection_timeout,
**kwargs)
wait_until(lambda: primary_host == client.host,
"connect to primary")
self.assertEqual(primary_port, client.port)
self.assertTrue(client.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue(client.pymongo_test.test.find_one())
client = MongoClient(
secondary_host,
secondary_port,
serverSelectionTimeoutMS=self.server_selection_timeout,
**kwargs)
wait_until(lambda: secondary_host == client.host,
"connect to secondary")
self.assertEqual(secondary_port, client.port)
self.assertFalse(client.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue(client.pymongo_test.test.find_one())
else:
self.assertRaises(
AutoReconnect, client.pymongo_test.test.find_one)
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), MongoClient
# should do the same for unacknowledged writes.
try:
client.get_database(
"pymongo_test",
write_concern=WriteConcern(w=0)).test.insert_one({})
except AutoReconnect as e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary client %s should'
'have raised exception' % (client,))
# Test direct connection to an arbiter
client = MongoClient(
arbiter_host,
arbiter_port,
serverSelectionTimeoutMS=self.server_selection_timeout,
**kwargs)
wait_until(lambda: arbiter_host == client.host,
"connect to arbiter")
self.assertEqual(arbiter_port, client.port)
self.assertFalse(client.is_primary)
# See explanation above
try:
client.get_database(
"pymongo_test",
write_concern=WriteConcern(w=0)).test.insert_one({})
except AutoReconnect as e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into arbiter client %s should'
'have raised exception' % (client,))
def test_secondary_connection(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
db.test.remove({}, safe=True, w=len(self.c.secondaries))
# Wait for replication...
w = len(self.c.secondaries) + 1
db.test.insert({'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# A Connection succeeds no matter the read preference
for kwargs in [
{'read_preference': PRIMARY},
{'read_preference': PRIMARY_PREFERRED},
{'read_preference': SECONDARY},
{'read_preference': SECONDARY_PREFERRED},
{'read_preference': NEAREST},
{'slave_okay': True}
]:
conn = Connection(primary_host,
primary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(primary_host, conn.host)
self.assertEqual(primary_port, conn.port)
self.assertTrue(conn.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue(conn.pymongo_test.test.find_one())
conn = Connection(secondary_host,
secondary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != PRIMARY:
self.assertTrue(conn.pymongo_test.test.find_one())
else:
self.assertRaises(
AutoReconnect, conn.pymongo_test.test.find_one)
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), Connection
# should do the same for unacknowledged writes.
try:
conn.pymongo_test.test.insert({}, safe=False)
except AutoReconnect, e:
self.assertEqual('not master', e.args[0])
else:
self.fail(
'Unacknowledged insert into secondary connection %s should'
'have raised exception' % conn)
# Test direct connection to an arbiter
conn = Connection(arbiter_host, arbiter_port, **kwargs)
self.assertEqual(arbiter_host, conn.host)
self.assertEqual(arbiter_port, conn.port)
self.assertFalse(conn.is_primary)
# See explanation above
try:
conn.pymongo_test.test.insert({}, safe=False)
except AutoReconnect, e:
self.assertEqual('not master', e.args[0])
def test_ship_of_theseus(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
db = c.pymongo_test
db.test.insert({}, w=len(c.secondaries) + 1)
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = []
for i in range(3):
new_hosts.append(ha_tools.add_member())
# RS closes all connections after reconfig.
for j in xrange(30):
try:
if ha_tools.get_primary():
break
except (ConnectionFailure, OperationFailure):
pass
sleep(1)
else:
self.fail("Couldn't recover from reconfig")
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
sleep(5)
# Wait for primary.
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
sleep(1)
else:
self.fail("No failover")
sleep(2 * MONITOR_INTERVAL)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
# Kill new members and switch back to original two members.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members([primary, secondary1])
# Wait for members to figure out they're secondaries.
for _ in xrange(30):
try:
if len(ha_tools.get_secondaries()) == 2:
break
except ConnectionFailure:
pass
sleep(1)
else:
self.fail("Original members didn't become secondaries")
# Should be able to reconnect to set again.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
def test_secondary_connection(self):
self.c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
db.test.remove({}, safe=True, w=len(self.c.secondaries))
# Wait for replication...
w = len(self.c.secondaries) + 1
db.test.insert({'foo': 'bar'}, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
arbiter_port = int(arbiter_port)
# A Connection succeeds no matter the read preference
for kwargs in [
{'read_preference': ReadPreference.PRIMARY},
{'read_preference': ReadPreference.PRIMARY_PREFERRED},
{'read_preference': ReadPreference.SECONDARY},
{'read_preference': ReadPreference.SECONDARY_PREFERRED},
{'read_preference': ReadPreference.NEAREST},
{'slave_okay': True}
]:
conn = Connection(primary_host,
primary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(primary_host, conn.host)
self.assertEqual(primary_port, conn.port)
self.assertTrue(conn.is_primary)
# Direct connection to primary can be queried with any read pref
self.assertTrue(conn.pymongo_test.test.find_one())
conn = Connection(secondary_host,
secondary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
# Direct connection to secondary can be queried with any read pref
# but PRIMARY
if kwargs.get('read_preference') != ReadPreference.PRIMARY:
self.assertTrue(conn.pymongo_test.test.find_one())
else:
self.assertRaises(
AutoReconnect, conn.pymongo_test.test.find_one)
# Since an attempt at an acknowledged write to a secondary from a
# direct connection raises AutoReconnect('not master'), Connection
# should do the same for unacknowledged writes.
try:
conn.pymongo_test.test.insert({}, safe=False)
except AutoReconnect, e:
self.assertEqual('not master', e.message)
else:
self.fail(
'Unacknowledged insert into secondary connection %s should'
'have raised exception' % conn)
# Test direct connection to an arbiter
conn = Connection(arbiter_host, arbiter_port, **kwargs)
self.assertEqual(arbiter_host, conn.host)
self.assertEqual(arbiter_port, conn.port)
self.assertFalse(conn.is_primary)
# See explanation above
try:
conn.pymongo_test.test.insert({}, safe=False)
except AutoReconnect, e:
self.assertEqual('not master', e.message)
def test_secondary_connection(self):
self.c = ReplicaSetConnection(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(self.c.secondaries)))
db = self.c.pymongo_test
db.test.remove({}, safe=True, w=len(self.c.secondaries))
# Force replication...
w = len(self.c.secondaries) + 1
db.test.insert({'foo': 'bar'}, safe=True, w=w)
# Test direct connection to a primary or secondary
primary_host, primary_port = ha_tools.get_primary().split(':')
primary_port = int(primary_port)
(secondary_host,
secondary_port) = ha_tools.get_secondaries()[0].split(':')
secondary_port = int(secondary_port)
self.assertTrue(
Connection(primary_host, primary_port,
use_greenlets=use_greenlets).is_primary)
self.assertTrue(
Connection(
primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary)
self.assertTrue(
Connection(
primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary)
self.assertTrue(
Connection(primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.NEAREST).is_primary)
self.assertTrue(
Connection(primary_host,
primary_port,
use_greenlets=use_greenlets,
read_preference=ReadPreference.SECONDARY).is_primary)
for kwargs in [
{
'read_preference': ReadPreference.PRIMARY_PREFERRED
},
{
'read_preference': ReadPreference.SECONDARY
},
{
'read_preference': ReadPreference.SECONDARY_PREFERRED
},
{
'read_preference': ReadPreference.NEAREST
},
{
'slave_okay': True
},
]:
conn = Connection(secondary_host,
secondary_port,
use_greenlets=use_greenlets,
**kwargs)
self.assertEqual(secondary_host, conn.host)
self.assertEqual(secondary_port, conn.port)
self.assertFalse(conn.is_primary)
self.assert_(conn.pymongo_test.test.find_one())
# Test direct connection to an arbiter
host, port = ha_tools.get_arbiters()[0].split(':')
port = int(port)
conn = Connection(host, port)
self.assertEqual(host, conn.host)
self.assertEqual(port, conn.port)
def post(self, op):
""" Respond to a POST
"""
# Start RS
if op == "start":
request = self._parse_json(self.request.body)
members = request["members"]
try:
# start RS and get its' params
res = ha_tools.start_replica_set(members)
except:
raise tornado.httpserver._BadRequestException("Couldn't start RS!")
rs_uri, rs_name = res
rs_id = uuid.uuid4()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_uri=rs_uri, rs_name=rs_name))
# Stop rs
elif op == "stop":
try:
ha_tools.kill_all_members()
except:
raise tornado.httpserver._BadRequestException("Couldn't stop RS!")
# Get primary
elif op == "get_primary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_primary_uri = ha_tools.get_primary()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_primary_uri=rs_primary_uri))
# Get secondaries
elif op == "get_secondaries":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_secondaries_uris = ha_tools.get_secondaries()
self.write(
self._template.load(op + self._ext).generate(rs_id=rs_id, rs_secondaries_uris=rs_secondaries_uris)
)
# Get arbiters
elif op == "get_arbiters":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_arbiters_uris = ha_tools.get_arbiters()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_arbiters_uris=rs_arbiters_uris))
# Kill primary
elif op == "kill_primary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_primary_uri = ha_tools.kill_primary()
self.write(
self._template.load(op + self._ext).generate(rs_id=rs_id, rs_killed_primary_uri=rs_killed_primary_uri)
)
# Kill secondary
elif op == "kill_secondary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_secondary_uri = ha_tools.kill_secondary()
self.write(
self._template.load(op + self._ext).generate(
rs_id=rs_id, rs_killed_secondary_uri=rs_killed_secondary_uri
)
)
# Kill all secondaries
elif op == "kill_all_secondaries":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_secondaries_uris = ha_tools.kill_all_secondaries()
self.write(
self._template.load(op + self._ext).generate(
rs_id=rs_id, rs_killed_secondaries_uris=rs_killed_secondaries_uris
)
)
