def test_ship_of_theseus(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
db = c.pymongo_test
db.test.insert({}, w=len(c.secondaries) + 1)
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = [ha_tools.add_member() for _ in range(3)]
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary.
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
sleep(1)
else:
self.fail("No failover")
sleep(2 * MONITOR_INTERVAL)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self, done):
loop = IOLoop.instance()
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
c.open_sync()
db = c.pymongo_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL)
try:
yield motor.Op(db.test.find_one)
except AutoReconnect:
# Might take one try to reconnect
yield gen.Task(loop.add_timeout, time.time() + 1)
# No error
yield motor.Op(db.test.find_one)
yield motor.Op(db.test.find_one, read_preference=SECONDARY)
done()
def test_ship_of_theseus(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
db = c.motor_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield self.pause(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield self.pause(1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield self.pause(2 * MONITOR_INTERVAL)
try:
yield db.test.find_one()
except AutoReconnect:
# Might take one try to reconnect
yield self.pause(1)
# No error
yield db.test.find_one()
yield db.test.find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
db = c.motor_test
w = len(c.secondaries) + 1
db.test.insert({}, w=w)
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
ha_tools.add_member()
ha_tools.add_member()
ha_tools.add_member()
# Wait for new members to join
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
yield self.pause(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
yield self.pause(1)
else:
self.fail("No failover")
# Ensure monitor picks up new members
yield self.pause(2 * MONITOR_INTERVAL)
try:
yield db.test.find_one()
except AutoReconnect:
# Might take one try to reconnect
yield self.pause(1)
# No error
yield db.test.find_one()
yield db.test.find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
db = c.pymongo_test
db.test.insert({}, w=len(c.secondaries) + 1)
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = []
for i in range(3):
new_hosts.append(ha_tools.add_member())
# RS closes all connections after reconfig.
for j in xrange(30):
try:
if ha_tools.get_primary():
break
except (ConnectionFailure, OperationFailure):
pass
sleep(1)
else:
self.fail("Couldn't recover from reconfig")
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
sleep(5)
# Wait for primary.
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
sleep(1)
else:
self.fail("No failover")
sleep(2 * MONITOR_INTERVAL)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
# Kill new members and switch back to original two members.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members([primary, secondary1])
# Wait for members to figure out they're secondaries.
for _ in xrange(30):
try:
if len(ha_tools.get_secondaries()) == 2:
break
except ConnectionFailure:
pass
sleep(1)
else:
self.fail("Original members didn't become secondaries")
# Should be able to reconnect to set again.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
db = c.get_database(
"pymongo_test",
write_concern=WriteConcern(w=len(c.secondaries) + 1))
db.test.insert_one({})
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = []
for i in range(3):
new_hosts.append(ha_tools.add_member())
# RS closes all connections after reconfig.
for j in xrange(30):
try:
if ha_tools.get_primary():
break
except (ConnectionFailure, OperationFailure):
pass
time.sleep(1)
else:
self.fail("Couldn't recover from reconfig")
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
time.sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
time.sleep(5)
wait_until(lambda: (ha_tools.get_primary()
and len(ha_tools.get_secondaries()) == 2),
"fail over",
timeout=30)
time.sleep(2 * self.heartbeat_frequency)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
time.sleep(2 * self.heartbeat_frequency)
find_one(read_preference=SECONDARY)
# Kill new members and switch back to original two members.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members([primary, secondary1])
# Wait for members to figure out they're secondaries.
wait_until(lambda: len(ha_tools.get_secondaries()) == 2,
"detect two secondaries",
timeout=30)
# Should be able to reconnect to set again.
time.sleep(2 * self.heartbeat_frequency)
find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
db = c.get_database(
"pymongo_test",
write_concern=WriteConcern(w=len(c.secondaries) + 1))
db.test.insert_one({})
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = []
for i in range(3):
new_hosts.append(ha_tools.add_member())
# RS closes all connections after reconfig.
for j in xrange(30):
try:
if ha_tools.get_primary():
break
except (ConnectionFailure, OperationFailure):
pass
time.sleep(1)
else:
self.fail("Couldn't recover from reconfig")
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
time.sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
time.sleep(5)
wait_until(lambda: (ha_tools.get_primary()
and len(ha_tools.get_secondaries()) == 2),
"fail over",
timeout=30)
time.sleep(2 * self.heartbeat_frequency)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
time.sleep(2 * self.heartbeat_frequency)
find_one(read_preference=SECONDARY)
# Kill new members and switch back to original two members.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members([primary, secondary1])
# Wait for members to figure out they're secondaries.
wait_until(lambda: len(ha_tools.get_secondaries()) == 2,
"detect two secondaries",
timeout=30)
# Should be able to reconnect to set again.
time.sleep(2 * self.heartbeat_frequency)
find_one(read_preference=SECONDARY)
def test_ship_of_theseus(self):
c = MongoReplicaSetClient(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
db = c.pymongo_test
db.test.insert({}, w=len(c.secondaries) + 1)
find_one = db.test.find_one
primary = ha_tools.get_primary()
secondary1 = ha_tools.get_random_secondary()
new_hosts = [ha_tools.add_member() for _ in range(3)]
# Wait for new members to join.
for _ in xrange(120):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
break
sleep(1)
else:
self.fail("New secondaries didn't join")
ha_tools.kill_members([primary, secondary1], 9)
# Wait for primary.
for _ in xrange(30):
if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
break
sleep(1)
else:
self.fail("No failover")
sleep(2 * MONITOR_INTERVAL)
# No error.
find_one()
find_one(read_preference=SECONDARY)
# All members down.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members(new_hosts)
# Should be able to reconnect to set even though original seed
# list is useless. Use SECONDARY so we don't have to wait for
# the election, merely for the client to detect members are up.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)
# Kill new members and switch back to original two members.
ha_tools.kill_members(new_hosts, 9)
self.assertRaises(
ConnectionFailure,
find_one, read_preference=SECONDARY)
ha_tools.restart_members([primary, secondary1])
# Should be able to reconnect to set again.
sleep(2 * MONITOR_INTERVAL)
find_one(read_preference=SECONDARY)