def test_secondary_failure(self):
c = ReplicaSetConnection(
self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
def readers_changed():
for _ in xrange(20):
if c.secondaries != secondaries:
return True
sleep(1)
return False
killed = ha_tools.kill_secondary()
sleep(2 * MONITOR_INTERVAL)
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
def test_secondary_failure(self):
c = MongoReplicaSetClient(self.seed,
replicaSet=self.name,
use_greenlets=use_greenlets)
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
def readers_changed():
for _ in xrange(20):
if c.secondaries != secondaries:
return True
sleep(1)
return False
killed = ha_tools.kill_secondary()
sleep(2 * MONITOR_INTERVAL)
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
self.assertTrue(readers_changed())
def test_secondary_failure(self, done):
loop = IOLoop.instance()
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
c.open_sync()
self.assertTrue(bool(len(c.secondaries)))
primary = c.primary
secondaries = c.secondaries
killed = ha_tools.kill_secondary()
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL)
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
# Wait for secondary to join, and for MotorReplicaSetClient
# to detect it.
for _ in xrange(30):
if c.secondaries != secondaries:
break
yield gen.Task(loop.add_timeout, time.time() + 1)
else:
self.fail("Dead secondary not detected")
done()
def test_secondary_failure(self):
c = MongoClient(self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
primary = c.primary
old_secondaries = c.secondaries
killed = ha_tools.kill_secondary()
time.sleep(2 * self.heartbeat_frequency)
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
wait_until(lambda: c.secondaries != old_secondaries,
"discover new secondaries",
timeout=30)
old_secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
wait_until(lambda: c.secondaries != old_secondaries,
"discover new secondaries",
timeout=30)
def test_secondary_failure(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
primary = c.primary
killed = ha_tools.kill_secondary()
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
yield self.pause(2 * MONITOR_INTERVAL)
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
# Wait for secondary to join, and for MotorReplicaSetClient
# to detect it.
for _ in xrange(30):
if c.secondaries != secondaries:
break
yield self.pause(1)
else:
self.fail("Dead secondary not detected")
def test_secondary_failure(self):
c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
yield c.open()
self.assertTrue(c.secondaries)
primary = c.primary
killed = ha_tools.kill_secondary()
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
yield self.pause(2 * MONITOR_INTERVAL)
secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
# Wait for secondary to join, and for MotorReplicaSetClient
# to detect it.
for _ in xrange(30):
if c.secondaries != secondaries:
break
yield self.pause(1)
else:
self.fail("Dead secondary not detected")
def test_secondary_failure(self):
c = MongoClient(
self.seed,
replicaSet=self.name,
serverSelectionTimeoutMS=self.server_selection_timeout)
wait_until(lambda: c.primary, "discover primary")
wait_until(lambda: len(c.secondaries) == 2, "discover secondaries")
primary = c.primary
old_secondaries = c.secondaries
killed = ha_tools.kill_secondary()
time.sleep(2 * self.heartbeat_frequency)
self.assertTrue(bool(len(killed)))
self.assertEqual(primary, c.primary)
wait_until(lambda: c.secondaries != old_secondaries,
"discover new secondaries",
timeout=30)
old_secondaries = c.secondaries
ha_tools.restart_members([killed])
self.assertEqual(primary, c.primary)
wait_until(lambda: c.secondaries != old_secondaries,
"discover new secondaries",
timeout=30)
def post(self, op):
""" Respond to a POST
"""
# Start RS
if op == "start":
request = self._parse_json(self.request.body)
members = request["members"]
try:
# start RS and get its' params
res = ha_tools.start_replica_set(members)
except:
raise tornado.httpserver._BadRequestException("Couldn't start RS!")
rs_uri, rs_name = res
rs_id = uuid.uuid4()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_uri=rs_uri, rs_name=rs_name))
# Stop rs
elif op == "stop":
try:
ha_tools.kill_all_members()
except:
raise tornado.httpserver._BadRequestException("Couldn't stop RS!")
# Get primary
elif op == "get_primary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_primary_uri = ha_tools.get_primary()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_primary_uri=rs_primary_uri))
# Get secondaries
elif op == "get_secondaries":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_secondaries_uris = ha_tools.get_secondaries()
self.write(
self._template.load(op + self._ext).generate(rs_id=rs_id, rs_secondaries_uris=rs_secondaries_uris)
)
# Get arbiters
elif op == "get_arbiters":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_arbiters_uris = ha_tools.get_arbiters()
self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_arbiters_uris=rs_arbiters_uris))
# Kill primary
elif op == "kill_primary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_primary_uri = ha_tools.kill_primary()
self.write(
self._template.load(op + self._ext).generate(rs_id=rs_id, rs_killed_primary_uri=rs_killed_primary_uri)
)
# Kill secondary
elif op == "kill_secondary":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_secondary_uri = ha_tools.kill_secondary()
self.write(
self._template.load(op + self._ext).generate(
rs_id=rs_id, rs_killed_secondary_uri=rs_killed_secondary_uri
)
)
# Kill all secondaries
elif op == "kill_all_secondaries":
request = self._parse_json(self.request.body)
rs_id = request["rs"]["id"]
rs_killed_secondaries_uris = ha_tools.kill_all_secondaries()
self.write(
self._template.load(op + self._ext).generate(
rs_id=rs_id, rs_killed_secondaries_uris=rs_killed_secondaries_uris
)
)
