def test_alive(self, done):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = motor.MotorClient(primary).open_sync()
secondary_cx = motor.MotorClient(secondary).open_sync()
rsc = motor.MotorReplicaSetClient(
self.seed, replicaSet=self.name).open_sync()
loop = IOLoop.instance()
try:
yield AssertTrue(primary_cx.alive)
yield AssertTrue(secondary_cx.alive)
yield AssertTrue(rsc.alive)
ha_tools.kill_primary()
yield gen.Task(loop.add_timeout, time.time() + 0.5)
yield AssertFalse(primary_cx.alive)
yield AssertTrue(secondary_cx.alive)
# Sometimes KeyError: https://jira.mongodb.org/browse/PYTHON-467
yield AssertFalse(rsc.alive)
ha_tools.kill_members([secondary], 2)
yield gen.Task(loop.add_timeout, time.time() + 0.5)
yield AssertFalse(primary_cx.alive)
yield AssertFalse(secondary_cx.alive)
yield AssertFalse(rsc.alive)
finally:
rsc.close()
done()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = yield motor.MotorClient(primary).open()
secondary_cx = yield motor.MotorClient(secondary).open()
rsc = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield rsc.open()
try:
self.assertTrue((yield primary_cx.alive()))
self.assertTrue((yield secondary_cx.alive()))
self.assertTrue((yield rsc.alive()))
ha_tools.kill_primary()
yield self.pause(0.5)
self.assertFalse((yield primary_cx.alive()))
self.assertTrue((yield secondary_cx.alive()))
self.assertFalse((yield rsc.alive()))
ha_tools.kill_members([secondary], 2)
yield self.pause(0.5)
self.assertFalse((yield primary_cx.alive()))
self.assertFalse((yield secondary_cx.alive()))
self.assertFalse((yield rsc.alive()))
finally:
rsc.close()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = connected(
MongoClient(
primary,
serverSelectionTimeoutMS=self.server_selection_timeout)),
secondary_cx = connected(
MongoClient(
secondary,
serverSelectionTimeoutMS=self.server_selection_timeout))
rsc = connected(
MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout))
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
def test_read_with_failover(self, done):
loop = IOLoop.instance()
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
c.open_sync()
self.assertTrue(bool(len(c.secondaries)))
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
# Force replication
yield motor.Op(db.test.insert, [{'foo': i} for i in xrange(10)], w=w)
yield AssertEqual(10, db.test.count)
db.read_preference = SECONDARY
cursor = db.test.find().batch_size(5)
yield cursor.fetch_next
self.assertEqual(5, cursor.delegate._Cursor__retrieved)
for i in range(5):
cursor.next_object()
ha_tools.kill_primary()
yield gen.Task(loop.add_timeout, time.time() + 2)
# Primary failure shouldn't interrupt the cursor
yield cursor.fetch_next
self.assertEqual(10, cursor.delegate._Cursor__retrieved)
done()
def test_read_with_failover(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
def iter_cursor(cursor):
for _ in cursor:
pass
return True
w = len(c.secondaries) + 1
db = c.get_database("pymongo_test",
write_concern=WriteConcern(w=w))
db.test.delete_many({})
# Force replication
db.test.insert_many([{'foo': i} for i in xrange(10)])
self.assertEqual(10, db.test.count())
db.read_preference = SECONDARY_PREFERRED
cursor = db.test.find().batch_size(5)
next(cursor)
self.assertEqual(5, cursor._Cursor__retrieved)
self.assertTrue(cursor.address in c.secondaries)
ha_tools.kill_primary()
# Primary failure shouldn't interrupt the cursor
self.assertTrue(iter_cursor(cursor))
self.assertEqual(10, cursor._Cursor__retrieved)
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = MongoClient(primary, use_greenlets=use_greenlets)
secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets)
rsc = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = connected(
MongoClient(
primary,
serverSelectionTimeoutMS=self.server_selection_timeout)),
secondary_cx = connected(
MongoClient(
secondary,
serverSelectionTimeoutMS=self.server_selection_timeout))
rsc = connected(
MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout))
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = yield motor.MotorClient(primary).open()
secondary_cx = yield motor.MotorClient(secondary).open()
rsc = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield rsc.open()
try:
self.assertTrue((yield primary_cx.alive()))
self.assertTrue((yield secondary_cx.alive()))
self.assertTrue((yield rsc.alive()))
ha_tools.kill_primary()
yield self.pause(0.5)
self.assertFalse((yield primary_cx.alive()))
self.assertTrue((yield secondary_cx.alive()))
self.assertFalse((yield rsc.alive()))
ha_tools.kill_members([secondary], 2)
yield self.pause(0.5)
self.assertFalse((yield primary_cx.alive()))
self.assertFalse((yield secondary_cx.alive()))
self.assertFalse((yield rsc.alive()))
finally:
rsc.close()
def test_read_with_failover(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
def iter_cursor(cursor):
for _ in cursor:
pass
return True
w = len(c.secondaries) + 1
db = c.get_database("pymongo_test",
write_concern=WriteConcern(w=w))
db.test.delete_many({})
# Force replication
db.test.insert_many([{'foo': i} for i in xrange(10)])
self.assertEqual(10, db.test.count())
db.read_preference = SECONDARY_PREFERRED
cursor = db.test.find().batch_size(5)
next(cursor)
self.assertEqual(5, cursor._Cursor__retrieved)
self.assertTrue(cursor.address in c.secondaries)
ha_tools.kill_primary()
# Primary failure shouldn't interrupt the cursor
self.assertTrue(iter_cursor(cursor))
self.assertEqual(10, cursor._Cursor__retrieved)
def test_alive(self):
primary = ha_tools.get_primary()
secondary = ha_tools.get_random_secondary()
primary_cx = MongoClient(primary, use_greenlets=use_greenlets)
secondary_cx = MongoClient(secondary, use_greenlets=use_greenlets)
rsc = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
try:
self.assertTrue(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertTrue(rsc.alive())
ha_tools.kill_primary()
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertTrue(secondary_cx.alive())
self.assertFalse(rsc.alive())
ha_tools.kill_members([secondary], 2)
time.sleep(0.5)
self.assertFalse(primary_cx.alive())
self.assertFalse(secondary_cx.alive())
self.assertFalse(rsc.alive())
finally:
rsc.close()
def test_read_with_failover(self):
c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=False)
self.assertTrue(bool(len(c.secondaries)))
def iter_cursor(cursor):
for _ in cursor:
pass
return True
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, safe=True, w=w)
# Force replication
db.test.insert([{'foo': i} for i in xrange(10)], safe=True, w=w)
self.assertEqual(10, db.test.count())
db.read_preference = ReadPreference.SECONDARY_PREFERRED
cursor = db.test.find().batch_size(5)
cursor.next()
self.assertEqual(5, cursor._Cursor__retrieved)
self.assertTrue(cursor._Cursor__connection_id in c.secondaries)
ha_tools.kill_primary()
# Primary failure shouldn't interrupt the cursor
self.assertTrue(iter_cursor(cursor))
self.assertEqual(10, cursor._Cursor__retrieved)
def test_read_with_failover(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
def iter_cursor(cursor):
for _ in cursor:
pass
return True
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
# Force replication
db.test.insert([{'foo': i} for i in xrange(10)], w=w)
self.assertEqual(10, db.test.count())
db.read_preference = SECONDARY_PREFERRED
cursor = db.test.find().batch_size(5)
cursor.next()
self.assertEqual(5, cursor._Cursor__retrieved)
self.assertTrue(cursor._Cursor__connection_id in c.secondaries)
ha_tools.kill_primary()
# Primary failure shouldn't interrupt the cursor
self.assertTrue(iter_cursor(cursor))
self.assertEqual(10, cursor._Cursor__retrieved)
def test_writes_with_failover(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
db.test.insert({'foo': 'bar'}, w=w)
self.assertEqual('bar', db.test.find_one()['foo'])
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
# Wait past pool's check interval, so it throws an error from
# get_socket().
sleep(1)
# Verify that we only raise AutoReconnect, not some other error,
# while we wait for new primary.
for _ in xrange(10000):
try:
db.test.insert({'bar': 'baz'})
# No error, found primary.
break
except AutoReconnect:
sleep(.01)
else:
self.fail("Couldn't connect to new primary")
# Found new primary.
self.assertTrue(c.primary)
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_writes_with_failover(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
primary = c.primary
db = c.motor_test
w = len(c.secondaries) + 1
yield db.test.remove(w=w)
yield db.test.insert({'foo': 'bar'}, w=w)
result = yield db.test.find_one()
self.assertEqual('bar', result['foo'])
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
yield self.pause(2)
for _ in xrange(30):
try:
yield db.test.insert({'bar': 'baz'})
# Success
break
except AutoReconnect:
yield self.pause(1)
else:
self.fail("Couldn't insert after primary killed")
self.assertTrue(primary != c.primary)
result = yield db.test.find_one({'bar': 'baz'})
self.assertEqual('baz', result['bar'])
def test_writes_with_failover(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
primary = c.primary
db = c.motor_test
w = len(c.secondaries) + 1
yield db.test.remove(w=w)
yield db.test.insert({'foo': 'bar'}, w=w)
result = yield db.test.find_one()
self.assertEqual('bar', result['foo'])
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
yield self.pause(2)
for _ in xrange(30):
try:
yield db.test.insert({'bar': 'baz'})
# Success
break
except AutoReconnect:
yield self.pause(1)
else:
self.fail("Couldn't insert after primary killed")
self.assertTrue(primary != c.primary)
result = yield db.test.find_one({'bar': 'baz'})
self.assertEqual('baz', result['bar'])
def test_writes_with_failover(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
db.test.insert({'foo': 'bar'}, w=w)
self.assertEqual('bar', db.test.find_one()['foo'])
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
# Wait past pool's check interval, so it throws an error from
# get_socket().
sleep(1)
# Verify that we only raise AutoReconnect, not some other error,
# while we wait for new primary.
for _ in xrange(10000):
try:
db.test.insert({'bar': 'baz'})
# No error, found primary.
break
except AutoReconnect:
sleep(.01)
else:
self.fail("Couldn't connect to new primary")
# Found new primary.
self.assertTrue(c.primary)
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_writes_with_failover(self, done):
loop = IOLoop.instance()
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
c.open_sync()
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
yield motor.Op(db.test.remove, w=w)
yield motor.Op(db.test.insert, {'foo': 'bar'}, w=w)
result = yield motor.Op(db.test.find_one)
self.assertEqual('bar', result['foo'])
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
yield gen.Task(loop.add_timeout, time.time() + 2)
for _ in xrange(30):
try:
yield motor.Op(db.test.insert, {'bar': 'baz'})
# Success
break
except AutoReconnect:
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("Couldn't insert after primary killed")
self.assertTrue(primary != c.primary)
result = yield motor.Op(db.test.find_one, {'bar': 'baz'})
self.assertEqual('baz', result['bar'])
done()
def test_writes_with_failover(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
db.test.insert({'foo': 'bar'}, w=w)
self.assertEqual('bar', db.test.find_one()['foo'])
def try_write():
for _ in xrange(30):
try:
db.test.insert({'bar': 'baz'})
return True
except AutoReconnect:
sleep(1)
return False
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
self.assertTrue(try_write())
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_request_during_failover(self):
primary = _partition_node(ha_tools.get_primary())
secondary = _partition_node(ha_tools.get_random_secondary())
self.assertTrue(self.c.auto_start_request)
self.assertTrue(self.c.in_request())
rs_state = self.c._MongoReplicaSetClient__rs_state
primary_pool = rs_state.get(primary).pool
secondary_pool = rs_state.get(secondary).pool
# Trigger start_request on primary pool
utils.assertReadFrom(self, self.c, primary, PRIMARY)
self.assertTrue(primary_pool.in_request())
# Fail over
ha_tools.kill_primary()
sleep(5)
patience_seconds = 60
for _ in range(patience_seconds):
try:
if ha_tools.ha_tools_debug:
print 'Waiting for failover'
if ha_tools.get_primary():
# We have a new primary
break
except ConnectionFailure:
pass
sleep(1)
else:
self.fail("Problem with test: No new primary after %s seconds" %
patience_seconds)
try:
# Trigger start_request on secondary_pool, which is becoming new
# primary
self.c.test.test.find_one()
except AutoReconnect:
# We've noticed the failover now
pass
# The old secondary is now primary
utils.assertReadFrom(self, self.c, secondary, PRIMARY)
self.assertTrue(self.c.in_request())
self.assertTrue(secondary_pool.in_request())
def test_request_during_failover(self):
primary = _partition_node(ha_tools.get_primary())
secondary = _partition_node(ha_tools.get_random_secondary())
self.assertTrue(self.c.auto_start_request)
self.assertTrue(self.c.in_request())
rs_state = self.c._MongoReplicaSetClient__rs_state
primary_pool = rs_state.get(primary).pool
secondary_pool = rs_state.get(secondary).pool
# Trigger start_request on primary pool
utils.assertReadFrom(self, self.c, primary, PRIMARY)
self.assertTrue(primary_pool.in_request())
# Fail over
ha_tools.kill_primary()
sleep(5)
patience_seconds = 60
for _ in range(patience_seconds):
try:
if ha_tools.ha_tools_debug:
print 'Waiting for failover'
if ha_tools.get_primary():
# We have a new primary
break
except ConnectionFailure:
pass
sleep(1)
else:
self.fail("Problem with test: No new primary after %s seconds"
% patience_seconds)
try:
# Trigger start_request on secondary_pool, which is becoming new
# primary
self.c.test.test.find_one()
except AutoReconnect:
# We've noticed the failover now
pass
# The old secondary is now primary
utils.assertReadFrom(self, self.c, secondary, PRIMARY)
self.assertTrue(self.c.in_request())
self.assertTrue(secondary_pool.in_request())
def test_read_with_failover(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
db = c.motor_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
# Force replication
yield db.test.insert([{'foo': i} for i in xrange(10)], w=w)
self.assertEqual(10, (yield db.test.count()))
db.read_preference = SECONDARY
cursor = db.test.find().batch_size(5)
yield cursor.fetch_next
self.assertEqual(5, cursor.delegate._Cursor__retrieved)
for i in range(5):
cursor.next_object()
ha_tools.kill_primary()
yield self.pause(2)
# Primary failure shouldn't interrupt the cursor
yield cursor.fetch_next
self.assertEqual(10, cursor.delegate._Cursor__retrieved)
def test_read_with_failover(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
db = c.motor_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
# Force replication
yield db.test.insert([{'foo': i} for i in xrange(10)], w=w)
self.assertEqual(10, (yield db.test.count()))
db.read_preference = SECONDARY
cursor = db.test.find().batch_size(5)
yield cursor.fetch_next
self.assertEqual(5, cursor.delegate._Cursor__retrieved)
for i in range(5):
cursor.next_object()
ha_tools.kill_primary()
yield self.pause(2)
# Primary failure shouldn't interrupt the cursor
yield cursor.fetch_next
self.assertEqual(10, cursor.delegate._Cursor__retrieved)
def test_primary_failure(self):
c = MongoClient(self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
old_primary = c.primary
old_secondaries = c.secondaries
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
wait_until(lambda: c.primary and c.primary != old_primary,
"discover new primary",
timeout=30)
wait_until(lambda: c.secondaries != old_secondaries,
"discover new secondaries",
timeout=30)
def test_primary_failure(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
primary = c.primary
secondaries = c.secondaries
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
yield self.pause(1)
# Wait for new primary to step up, and for MotorReplicaSetClient
# to detect it.
for _ in xrange(30):
if c.primary != primary and c.secondaries != secondaries:
break
yield self.pause(1)
else:
self.fail("New primary not detected")
def test_primary_failure(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
primary = c.primary
secondaries = c.secondaries
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
yield self.pause(1)
# Wait for new primary to step up, and for MotorReplicaSetClient
# to detect it.
for _ in xrange(30):
if c.primary != primary and c.secondaries != secondaries:
break
yield self.pause(1)
else:
self.fail("New primary not detected")
def test_primary_failure(self):
c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
# Wait for new primary to be elected
def primary_changed():
for _ in xrange(30):
if c.primary and c.primary != primary:
return True
sleep(1)
return False
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
self.assertTrue(primary_changed())
self.assertNotEqual(secondaries, c.secondaries)
def test_primary_failure(self):
loop = IOLoop.instance()
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
c
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
yield gen.Task(loop.add_timeout, time.time() + 1)
# Wait for new primary to step up, and for MotorReplicaSetClient
# to detect it.
for _ in xrange(30):
if c.primary != primary and c.secondaries != secondaries:
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("New primary not detected")
def test_primary_failure(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
old_primary = c.primary
old_secondaries = c.secondaries
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
wait_until(lambda: c.primary and c.primary != old_primary,
"discover new primary",
timeout=30)
wait_until(lambda: c.secondaries != old_secondaries,
"discover new secondaries",
timeout=30)
def test_primary_failure(self):
c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
# Wait for new primary to be elected
def primary_changed():
for _ in xrange(30):
if c.primary and c.primary != primary:
return True
sleep(1)
return False
killed = ha_tools.kill_primary()
self.assertTrue(bool(len(killed)))
self.assertTrue(primary_changed())
self.assertNotEqual(secondaries, c.secondaries)
def test_writes_with_failover(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
primary = c.primary
w = len(c.secondaries) + 1
db = c.get_database("pymongo_test",
write_concern=WriteConcern(w=w))
db.test.delete_many({})
db.test.insert_one({'foo': 'bar'})
self.assertEqual('bar', db.test.find_one()['foo'])
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
# Wait past pool's check interval, so it throws an error from
# get_socket().
time.sleep(1)
# Verify that we only raise AutoReconnect, not some other error,
# while we wait for new primary.
for _ in xrange(10000):
try:
db.test.insert_one({'bar': 'baz'})
# No error, found primary.
break
except AutoReconnect:
time.sleep(.01)
else:
self.fail("Couldn't connect to new primary")
# Found new primary.
self.assertTrue(c.primary)
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_writes_with_failover(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
primary = c.primary
w = len(c.secondaries) + 1
db = c.get_database("pymongo_test",
write_concern=WriteConcern(w=w))
db.test.delete_many({})
db.test.insert_one({'foo': 'bar'})
self.assertEqual('bar', db.test.find_one()['foo'])
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
# Wait past pool's check interval, so it throws an error from
# get_socket().
time.sleep(1)
# Verify that we only raise AutoReconnect, not some other error,
# while we wait for new primary.
for _ in xrange(10000):
try:
db.test.insert_one({'bar': 'baz'})
# No error, found primary.
break
except AutoReconnect:
time.sleep(.01)
else:
self.fail("Couldn't connect to new primary")
# Found new primary.
self.assertTrue(c.primary)
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_writes_with_failover(self):
c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, safe=True, w=w)
db.test.insert({"foo": "bar"}, safe=True, w=w)
self.assertEqual("bar", db.test.find_one()["foo"])
def try_write():
for _ in xrange(30):
try:
db.test.insert({"bar": "baz"}, safe=True)
return True
except AutoReconnect:
sleep(1)
return False
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
self.assertTrue(try_write())
self.assertTrue(primary != c.primary)
self.assertEqual("baz", db.test.find_one({"bar": "baz"})["bar"])
def test_writes_with_failover(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
primary = c.primary
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.remove({}, w=w)
db.test.insert({'foo': 'bar'}, w=w)
self.assertEqual('bar', db.test.find_one()['foo'])
def try_write():
for _ in xrange(30):
try:
db.test.insert({'bar': 'baz'})
return True
except AutoReconnect:
sleep(1)
return False
killed = ha_tools.kill_primary(9)
self.assertTrue(bool(len(killed)))
self.assertTrue(try_write())
self.assertTrue(primary != c.primary)
self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
@gen.coroutine
def read_from_which_host(
rsc,
mode,
tag_sets=None,
latency=15,
):
db = rsc.motor_test
db.read_preference = mode
if isinstance(tag_sets, dict):
tag_sets = [tag_sets]
db.tag_sets = tag_sets or [{}]
db.secondary_acceptable_latency_ms = latency
cursor = db.test.find()
try:
yield cursor.fetch_next
raise gen.Return(cursor.delegate._Cursor__connection_id)
except AutoReconnect:
raise gen.Return(None)
@gen.coroutine
def assert_read_from(member, *args, **kwargs):
for _ in range(10):
used = yield read_from_which_host(c, *args, **kwargs)
self.assertEqual(member, used)
@gen.coroutine
def assert_read_from_all(members, *args, **kwargs):
members = set(members)
all_used = set()
for _ in range(100):
used = yield read_from_which_host(c, *args, **kwargs)
all_used.add(used)
if members == all_used:
raise gen.Return() # Success
# This will fail
self.assertEqual(members, all_used)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag)
yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
yield assert_read_from_all(
[secondary, other_secondary], SECONDARY, latency=9999999)
# SECONDARY_PREFERRED
yield assert_read_from_all(
[secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999)
# Multiple tags
yield assert_read_from(
secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
yield assert_read_from(
primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
yield assert_read_from_all(
[secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}], latency=9999999)
yield assert_read_from(
other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
yield assert_read_from_all(
[primary, secondary, other_secondary], NEAREST, latency=9999999)
yield assert_read_from_all(
[primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}], latency=9999999)
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 milliseconds.
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
yield assert_read_from(primary, NEAREST)
# Tags override nearness
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
yield assert_read_from(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 milliseconds.
self.set_ping_time(secondary, 0)
yield assert_read_from(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
yield assert_read_from(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
yield assert_read_from_all(
[primary, secondary, other_secondary],
NEAREST, latency=9999999)
self.clear_ping_times()
yield assert_read_from_all(
[primary, other_secondary], NEAREST, [{'dc': 'ny'}],
latency=9999999)
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
yield assert_read_from_all(
[secondary, other_secondary], PRIMARY_PREFERRED, latency=9999999)
yield assert_read_from(
secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
yield assert_read_from_all(
[secondary, other_secondary], SECONDARY, latency=9999999)
# Only primary matches
yield assert_read_from(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
yield assert_read_from(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
yield assert_read_from_all(
[secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999)
# Mode and tags both match
yield assert_read_from(
secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
yield assert_read_from_all(
[secondary, other_secondary], NEAREST, latency=9999999)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
yield self.pause(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True
).admin.command('ismaster')['ismaster'])
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
# SECONDARY
yield assert_read_from(other_secondary, SECONDARY)
yield assert_read_from(
other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
yield assert_read_from(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
yield assert_read_from(other_secondary, SECONDARY_PREFERRED)
yield assert_read_from(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
yield assert_read_from(
primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
yield assert_read_from_all(
[primary, other_secondary], NEAREST, latency=9999999)
yield assert_read_from(
other_secondary, NEAREST, {'name': 'other_secondary'})
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True
).admin.command('ismaster')['ismaster'])
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
yield assert_read_from(None, SECONDARY)
yield assert_read_from(None, SECONDARY, self.other_secondary_dc)
yield assert_read_from(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
yield assert_read_from(primary, SECONDARY_PREFERRED)
yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc)
yield assert_read_from(
primary, SECONDARY_PREFERRED, {'name': 'secondary'})
yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
yield assert_read_from(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_read_preference(self):
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy hosts into local scope
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}])
assertReadFrom(other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
time.sleep(2 * self.heartbeat_frequency)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
ha_tools.wait_for_primary()
ha_tools.kill_members([unpartition_node(secondary)], 2)
time.sleep(5)
ha_tools.wait_for_primary()
time.sleep(2 * self.heartbeat_frequency)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
@gen.coroutine
def read_from_which_host(
rsc,
mode,
tag_sets=None,
latency=15,
):
db = rsc.motor_test
db.read_preference = mode
if isinstance(tag_sets, dict):
tag_sets = [tag_sets]
db.tag_sets = tag_sets or [{}]
db.secondary_acceptable_latency_ms = latency
cursor = db.test.find()
try:
yield cursor.fetch_next
raise gen.Return(cursor.delegate._Cursor__connection_id)
except AutoReconnect:
raise gen.Return(None)
@gen.coroutine
def assert_read_from(member, *args, **kwargs):
for _ in range(10):
used = yield read_from_which_host(c, *args, **kwargs)
self.assertEqual(member, used)
@gen.coroutine
def assert_read_from_all(members, *args, **kwargs):
members = set(members)
all_used = set()
for _ in range(100):
used = yield read_from_which_host(c, *args, **kwargs)
all_used.add(used)
if members == all_used:
raise gen.Return() # Success
# This will fail
self.assertEqual(members, all_used)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag)
yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
yield assert_read_from_all([secondary, other_secondary],
SECONDARY,
latency=9999999)
# SECONDARY_PREFERRED
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED,
latency=9999999)
# Multiple tags
yield assert_read_from(secondary, SECONDARY_PREFERRED,
self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'primary'})
# No matching secondaries
yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}],
latency=9999999)
yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
[bad_tag, {
'dc': 'ny'
}])
# NEAREST
self.clear_ping_times()
yield assert_read_from_all([primary, secondary, other_secondary],
NEAREST,
latency=9999999)
yield assert_read_from_all([primary, other_secondary],
NEAREST, [bad_tag, {
'dc': 'ny'
}],
latency=9999999)
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 milliseconds.
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
yield assert_read_from(primary, NEAREST)
# Tags override nearness
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
yield assert_read_from(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 milliseconds.
self.set_ping_time(secondary, 0)
yield assert_read_from(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
yield assert_read_from(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
yield assert_read_from_all([primary, secondary, other_secondary],
NEAREST,
latency=9999999)
self.clear_ping_times()
yield assert_read_from_all([primary, other_secondary],
NEAREST, [{
'dc': 'ny'
}],
latency=9999999)
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
yield assert_read_from_all([secondary, other_secondary],
PRIMARY_PREFERRED,
latency=9999999)
yield assert_read_from(secondary, PRIMARY_PREFERRED,
{'name': 'secondary'})
# No primary or matching secondary
yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
yield assert_read_from_all([secondary, other_secondary],
SECONDARY,
latency=9999999)
# Only primary matches
yield assert_read_from(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
yield assert_read_from(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
yield assert_read_from_all([secondary, other_secondary],
SECONDARY_PREFERRED,
latency=9999999)
# Mode and tags both match
yield assert_read_from(secondary, SECONDARY_PREFERRED,
{'name': 'secondary'})
# NEAREST
self.clear_ping_times()
yield assert_read_from_all([secondary, other_secondary],
NEAREST,
latency=9999999)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
yield self.pause(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(
pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True).admin.command('ismaster')['ismaster'])
yield self.pause(2 * MONITOR_INTERVAL)
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
# SECONDARY
yield assert_read_from(other_secondary, SECONDARY)
yield assert_read_from(other_secondary, SECONDARY,
self.other_secondary_dc)
# Only the down secondary matches
yield assert_read_from(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
yield assert_read_from(other_secondary, SECONDARY_PREFERRED)
yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
self.other_secondary_dc)
# The secondary matching the tag is down, use primary
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'secondary'})
# NEAREST
yield assert_read_from_all([primary, other_secondary],
NEAREST,
latency=9999999)
yield assert_read_from(other_secondary, NEAREST,
{'name': 'other_secondary'})
yield assert_read_from(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(
pymongo.MongoClient(
unpartition_node(primary),
slave_okay=True).admin.command('ismaster')['ismaster'])
# PRIMARY
yield assert_read_from(primary, PRIMARY)
# PRIMARY_PREFERRED
yield assert_read_from(primary, PRIMARY_PREFERRED)
yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
yield assert_read_from(None, SECONDARY)
yield assert_read_from(None, SECONDARY, self.other_secondary_dc)
yield assert_read_from(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
yield assert_read_from(primary, SECONDARY_PREFERRED)
yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc)
yield assert_read_from(primary, SECONDARY_PREFERRED,
{'name': 'secondary'})
yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
yield assert_read_from(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
yield assert_read_from(primary, NEAREST)
yield assert_read_from(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_read_preference(self):
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy hosts into local scope
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}])
assertReadFrom(other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
time.sleep(2 * self.heartbeat_frequency)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
ha_tools.wait_for_primary()
ha_tools.kill_members([unpartition_node(secondary)], 2)
time.sleep(5)
ha_tools.wait_for_primary()
time.sleep(2 * self.heartbeat_frequency)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_read_preference(self):
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy hosts into local scope
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED,
[bad_tag, {}])
assertReadFrom(other_secondary, SECONDARY_PREFERRED,
[bad_tag, {
'dc': 'ny'
}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary], NEAREST,
[bad_tag, {
'dc': 'ny'
}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll([primary, secondary, other_secondary],
NEAREST,
secondary_acceptable_latency_ms=1000 * 1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(
MongoClient(unpartition_node(primary),
use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED).admin.command(
'ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(other_secondary, SECONDARY_PREFERRED,
self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(
MongoClient(unpartition_node(primary),
use_greenlets=use_greenlets,
read_preference=PRIMARY_PREFERRED).admin.command(
'ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def test_read_preference(self):
# This is long, but we put all the tests in one function to save time
# on setUp, which takes about 30 seconds to bring up a replica set.
# We pass through four states:
#
# 1. A primary and two secondaries
# 2. Primary down
# 3. Primary up, one secondary down
# 4. Primary up, all secondaries down
#
# For each state, we verify the behavior of PRIMARY,
# PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
auto_start_request=False)
def assertReadFrom(member, *args, **kwargs):
utils.assertReadFrom(self, c, member, *args, **kwargs)
def assertReadFromAll(members, *args, **kwargs):
utils.assertReadFromAll(self, c, members, *args, **kwargs)
def unpartition_node(node):
host, port = node
return '%s:%s' % (host, port)
# To make the code terser, copy modes and hosts into local scope
PRIMARY = ReadPreference.PRIMARY
PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED
SECONDARY = ReadPreference.SECONDARY
SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
NEAREST = ReadPreference.NEAREST
primary = self.primary
secondary = self.secondary
other_secondary = self.other_secondary
bad_tag = {'bad': 'tag'}
# 1. THREE MEMBERS UP -------------------------------------------------
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
# Trivial: mode and tags both match
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Secondary matches but not primary, choose primary
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)
# Chooses primary, ignoring tag sets
assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Multiple tags
assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)
# Fall back to primary if it's the only one matching the tags
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})
# No matching secondaries
assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)
# Fall back from non-matching tag set to matching set
assertReadFromAll([secondary, other_secondary],
SECONDARY_PREFERRED, [bad_tag, {}])
assertReadFrom(other_secondary,
SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])
# NEAREST
self.clear_ping_times()
assertReadFromAll([primary, secondary, other_secondary], NEAREST)
assertReadFromAll([primary, other_secondary],
NEAREST, [bad_tag, {'dc': 'ny'}])
self.set_ping_time(primary, 0)
self.set_ping_time(secondary, .03) # 30 ms
self.set_ping_time(other_secondary, 10)
# Nearest member, no tags
assertReadFrom(primary, NEAREST)
# Tags override nearness
assertReadFrom(primary, NEAREST, {'name': 'primary'})
assertReadFrom(secondary, NEAREST, self.secondary_dc)
# Make secondary fast
self.set_ping_time(primary, .03) # 30 ms
self.set_ping_time(secondary, 0)
assertReadFrom(secondary, NEAREST)
# Other secondary fast
self.set_ping_time(secondary, 10)
self.set_ping_time(other_secondary, 0)
assertReadFrom(other_secondary, NEAREST)
# High secondaryAcceptableLatencyMS, should read from all members
assertReadFromAll(
[primary, secondary, other_secondary],
NEAREST, secondary_acceptable_latency_ms=1000*1000)
self.clear_ping_times()
assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])
# 2. PRIMARY DOWN -----------------------------------------------------
killed = ha_tools.kill_primary()
# Let monitor notice primary's gone
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(None, PRIMARY)
# PRIMARY_PREFERRED
# No primary, choose matching secondary
assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})
# No primary or matching secondary
assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)
# SECONDARY
assertReadFromAll([secondary, other_secondary], SECONDARY)
# Only primary matches
assertReadFrom(None, SECONDARY, {'name': 'primary'})
# No matching secondaries
assertReadFrom(None, SECONDARY, bad_tag)
# SECONDARY_PREFERRED
assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)
# Mode and tags both match
assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
self.clear_ping_times()
assertReadFromAll([secondary, other_secondary], NEAREST)
# 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
ha_tools.restart_members([killed])
for _ in range(30):
if ha_tools.get_primary():
break
sleep(1)
else:
self.fail("Primary didn't come back up")
ha_tools.kill_members([unpartition_node(secondary)], 2)
self.assertTrue(Connection(
unpartition_node(primary), use_greenlets=use_greenlets,
slave_okay=True
).admin.command('ismaster')['ismaster'])
sleep(2 * MONITOR_INTERVAL)
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
# SECONDARY
assertReadFrom(other_secondary, SECONDARY)
assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)
# Only the down secondary matches
assertReadFrom(None, SECONDARY, {'name': 'secondary'})
# SECONDARY_PREFERRED
assertReadFrom(other_secondary, SECONDARY_PREFERRED)
assertReadFrom(
other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)
# The secondary matching the tag is down, use primary
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
# NEAREST
assertReadFromAll([primary, other_secondary], NEAREST)
assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
assertReadFrom(primary, NEAREST, {'name': 'primary'})
# 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
ha_tools.kill_members([unpartition_node(other_secondary)], 2)
self.assertTrue(Connection(
unpartition_node(primary), use_greenlets=use_greenlets,
slave_okay=True
).admin.command('ismaster')['ismaster'])
# PRIMARY
assertReadFrom(primary, PRIMARY)
# PRIMARY_PREFERRED
assertReadFrom(primary, PRIMARY_PREFERRED)
assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)
# SECONDARY
assertReadFrom(None, SECONDARY)
assertReadFrom(None, SECONDARY, self.other_secondary_dc)
assertReadFrom(None, SECONDARY, {'dc': 'ny'})
# SECONDARY_PREFERRED
assertReadFrom(primary, SECONDARY_PREFERRED)
assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})
# NEAREST
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
assertReadFrom(None, NEAREST, {'name': 'secondary'})
# Even if primary's slow, still read from it
self.set_ping_time(primary, 100)
assertReadFrom(primary, NEAREST)
assertReadFrom(None, NEAREST, self.secondary_dc)
self.clear_ping_times()
def post(self, op):
""" Respond to a POST
"""
# Start RS
if op == "start":
request = self._parse_json(self.request.body)
members = request["members"]
try:
# start RS and get its' params
res = ha_tools.start_replica_set(members)
except:
raise tornado.httpserver._BadRequestException("Couldn't start RS!")
rs_uri, rs_name = res
rs_id = uuid.uuid4()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_uri=rs_uri, rs_name=rs_name))
# Stop rs
elif op == "stop":
try:
ha_tools.kill_all_members()
except:
raise tornado.httpserver._BadRequestException("Couldn't stop RS!")
# Get primary
elif op == "get_primary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_primary_uri = ha_tools.get_primary()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_primary_uri=rs_primary_uri))
# Get secondaries
elif op == "get_secondaries":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_secondaries_uris = ha_tools.get_secondaries()
self.write(
self._template.load(op + self._ext).generate(rs_id=rs_id, rs_secondaries_uris=rs_secondaries_uris)
)
# Get arbiters
elif op == "get_arbiters":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_arbiters_uris = ha_tools.get_arbiters()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_arbiters_uris=rs_arbiters_uris))
# Kill primary
elif op == "kill_primary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_primary_uri = ha_tools.kill_primary()
self.write(
self._template.load(op + self._ext).generate(rs_id=rs_id, rs_killed_primary_uri=rs_killed_primary_uri)
)
# Kill secondary
elif op == "kill_secondary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_secondary_uri = ha_tools.kill_secondary()
self.write(
self._template.load(op + self._ext).generate(
rs_id=rs_id, rs_killed_secondary_uri=rs_killed_secondary_uri
)
)
# Kill all secondaries
elif op == "kill_all_secondaries":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_secondaries_uris = ha_tools.kill_all_secondaries()
self.write(
self._template.load(op + self._ext).generate(
rs_id=rs_id, rs_killed_secondaries_uris=rs_killed_secondaries_uris
)
)
