def synchronize(slave, master):
"""Synchronize a slave with a master and after that stop the slave.
:param slave: Slave.
:param master: Master.
"""
_replication.sync_slave_with_master(slave, master, timeout=0)
def synchronize(slave, master):
"""Synchronize a slave with a master and after that stop the slave.
:param slave: Slave.
:param master: Master.
"""
_replication.sync_slave_with_master(slave, master, timeout=0)
def test_check_no_healthy_slave(self):
"""Test promoting when there is no healthy slave.
"""
# Configure replication.
instances = tests.utils.MySQLInstances()
user = instances.user
passwd = instances.passwd
instances.configure_instances({0 : [{1 : []}, {2 : []}]}, user, passwd)
master = instances.get_instance(0)
slave_1 = instances.get_instance(1)
slave_2 = instances.get_instance(2)
self.proxy.group.create("group_id", "")
self.proxy.group.add("group_id", master.address)
self.proxy.group.add("group_id", slave_1.address)
self.proxy.group.add("group_id", slave_2.address)
# Promote a master.
status = self.proxy.group.promote("group_id", str(master.uuid))
self.check_xmlrpc_command_result(status)
# Check replication.
status = self.proxy.group.health("group_id")
self.check_xmlrpc_simple(status, {
'status': _server.MySQLServer.SECONDARY
}, index=2, rowcount=3)
self.check_xmlrpc_simple(status, {
'status': _server.MySQLServer.SECONDARY
}, index=1, rowcount=3)
self.check_xmlrpc_simple(status, {
'status': _server.MySQLServer.PRIMARY
}, index=0, rowcount=3)
# Inject some events that make slaves break.
slave_1.set_session_binlog(False)
slave_1.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
slave_1.exec_stmt("USE test")
slave_1.exec_stmt("DROP TABLE IF EXISTS test")
slave_1.exec_stmt("CREATE TABLE test (id INTEGER)")
slave_1.set_session_binlog(True)
slave_2.set_session_binlog(False)
slave_2.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
slave_2.exec_stmt("USE test")
slave_2.exec_stmt("DROP TABLE IF EXISTS test")
slave_2.exec_stmt("CREATE TABLE test (id INTEGER)")
slave_2.set_session_binlog(True)
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master.exec_stmt("USE test")
master.exec_stmt("SET sql_log_bin=0")
master.exec_stmt("DROP TABLE IF EXISTS test")
master.exec_stmt("SET sql_log_bin=1")
master.exec_stmt("CREATE TABLE test (id INTEGER)")
# Synchronize replicas.
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_1, master, timeout=0)
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_2, master, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
for info in self.check_xmlrpc_iter(status):
if info['uuid'] in (str(slave_2.uuid), str(slave_1.uuid)):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
self.assertEqual(info['sql_not_running'], True)
elif info['uuid'] == str(master.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.PRIMARY
)
# Try to choose a new master through switch over.
status = self.proxy.group.promote("group_id")
self.check_xmlrpc_command_result(status, has_error=True)
# Try to reset the slave and restart slave.
_repl.stop_slave(slave_1, wait=True)
_repl.reset_slave(slave_1, clean=False)
try:
_repl.start_slave(slave_1, wait=True)
except _errors.DatabaseError as error:
self.assertEqual(
str(error), "Error 'Table 'test' already exists' "
"on query. Default database: 'test'. Query: 'CREATE "
"TABLE test (id INTEGER)'"
)
# Synchronize replica.
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_1, master, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
self.check_xmlrpc_simple(status, {
'status': _server.MySQLServer.SECONDARY,
"sql_not_running": True,
}, index=2, rowcount=3)
self.check_xmlrpc_simple(status, {
'status': _server.MySQLServer.SECONDARY,
"sql_not_running": True,
}, index=1, rowcount=3)
self.check_xmlrpc_simple(status, {
'status': _server.MySQLServer.PRIMARY,
}, index=0, rowcount=3)
# Try to drop the table on the slave.
_repl.stop_slave(slave_1, wait=True)
_repl.reset_slave(slave_1, clean=False)
slave_1.set_session_binlog(False)
slave_1.exec_stmt("DROP TABLE IF EXISTS test")
slave_1.set_session_binlog(True)
_repl.start_slave(slave_1, wait=True)
_repl.stop_slave(slave_2, wait=True)
_repl.reset_slave(slave_2, clean=False)
slave_2.set_session_binlog(False)
slave_2.exec_stmt("DROP TABLE IF EXISTS test")
slave_2.set_session_binlog(True)
_repl.start_slave(slave_2, wait=True)
# Synchronize replicas.
_repl.sync_slave_with_master(slave_1, master, timeout=0)
_repl.sync_slave_with_master(slave_2, master, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
for info in self.check_xmlrpc_iter(status, rowcount=3):
if info['uuid'] in (str(slave_2.uuid), str(slave_1.uuid)):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
self.assertEqual(info['sql_not_running'], False)
elif info['uuid'] == str(master.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.PRIMARY
)
def test_check_unhealthy_slave(self):
"""Test promoting a there is an unhealthy slave.
"""
# Configure replication.
instances = tests.utils.MySQLInstances()
user = instances.user
passwd = instances.passwd
instances.configure_instances({0 : [{1 : []}, {2 : []}]}, user, passwd)
master = instances.get_instance(0)
slave_1 = instances.get_instance(1)
slave_2 = instances.get_instance(2)
self.proxy.group.create("group_id", "")
self.proxy.group.add("group_id", master.address)
self.proxy.group.add("group_id", slave_1.address)
self.proxy.group.add("group_id", slave_2.address)
# Promote a master.
status = self.proxy.group.promote("group_id", str(master.uuid))
self.check_xmlrpc_command_result(status)
# Check replication.
status = self.proxy.group.health("group_id")
for info in self.check_xmlrpc_iter(status):
if info['uuid'] in (str(slave_1.uuid), str(slave_2.uuid)):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
elif info['uuid'] == str(master.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.PRIMARY
)
# Inject some events that makes a slave break.
slave_1.set_session_binlog(False)
slave_1.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
slave_1.exec_stmt("USE test")
slave_1.exec_stmt("DROP TABLE IF EXISTS test")
slave_1.exec_stmt("CREATE TABLE test (id INTEGER)")
slave_1.set_session_binlog(True)
slave_2.set_session_binlog(False)
slave_2.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
slave_2.exec_stmt("USE test")
slave_2.exec_stmt("DROP TABLE IF EXISTS test")
slave_2.set_session_binlog(True)
master.set_session_binlog(False)
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master.exec_stmt("USE test")
master.exec_stmt("DROP TABLE IF EXISTS test")
master.set_session_binlog(True)
master.exec_stmt("CREATE TABLE test (id INTEGER)")
# Synchronize replicas.
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_1, master, timeout=0)
_repl.sync_slave_with_master(slave_2, master, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
for info in self.check_xmlrpc_iter(status, rowcount=3):
if info['uuid'] == str(slave_1.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
self.assertEqual(info['sql_not_running'], True)
elif info['uuid'] == str(slave_2.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
elif info['uuid'] == str(master.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.PRIMARY
)
# Try to do a switch over to the faulty replica.
status = self.proxy.group.promote("group_id", str(slave_1.uuid))
self.check_xmlrpc_command_result(status, has_error=True)
# Choose a new master.
status = self.proxy.group.promote("group_id")
self.check_xmlrpc_command_result(status)
# Synchronize replicas.
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_1, slave_2, timeout=0)
_repl.sync_slave_with_master(master, slave_2, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
for info in self.check_xmlrpc_iter(status):
if info['uuid'] == str(slave_1.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
self.assertEqual(info['sql_not_running'], True)
elif info['uuid'] == str(slave_2.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.PRIMARY
)
elif info['uuid'] == str(master.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
self.assertEqual(info['sql_not_running'], False)
# Choose a new master.
status = self.proxy.group.promote("group_id")
self.check_xmlrpc_command_result(status)
# Synchronize replicas.
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_1, master, timeout=0)
_repl.sync_slave_with_master(slave_2, master, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
for info in self.check_xmlrpc_iter(status):
if info['uuid'] == str(slave_1.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
self.assertEqual(info['sql_not_running'], True)
elif info['uuid'] == str(slave_2.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.SECONDARY
)
elif info['uuid'] == str(master.uuid):
self.assertEqual(
info['status'],
_server.MySQLServer.PRIMARY
)
def test_check_no_healthy_slave(self):
"""Test promoting when there is no healthy slave.
"""
# Configure replication.
instances = tests.utils.MySQLInstances()
user = instances.user
passwd = instances.passwd
instances.configure_instances({0 : [{1 : []}, {2 : []}]}, user, passwd)
master = instances.get_instance(0)
slave_1 = instances.get_instance(1)
slave_2 = instances.get_instance(2)
self.proxy.group.create("group_id", "")
self.proxy.group.add("group_id", master.address)
self.proxy.group.add("group_id", slave_1.address)
self.proxy.group.add("group_id", slave_2.address)
# Promote a master.
status = self.proxy.group.promote("group_id", str(master.uuid))
self.assertStatus(status, _executor.Job.SUCCESS)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Executed action (_change_to_candidate).")
# Check replication.
status = self.proxy.group.health("group_id")
self.assertEqual(status[2][str(slave_1.uuid)]["threads"], {})
self.assertEqual(status[2][str(slave_1.uuid)]["status"],
_server.MySQLServer.SECONDARY)
self.assertEqual(status[2][str(slave_2.uuid)]["threads"], {})
self.assertEqual(status[2][str(slave_2.uuid)]["status"],
_server.MySQLServer.SECONDARY)
self.assertEqual(status[2][str(master.uuid)]["status"],
_server.MySQLServer.PRIMARY)
# Inject some events that make slaves break.
slave_1.set_session_binlog(False)
slave_1.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
slave_1.exec_stmt("USE test")
slave_1.exec_stmt("DROP TABLE IF EXISTS test")
slave_1.exec_stmt("CREATE TABLE test (id INTEGER)")
slave_1.set_session_binlog(True)
slave_2.set_session_binlog(False)
slave_2.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
slave_2.exec_stmt("USE test")
slave_2.exec_stmt("DROP TABLE IF EXISTS test")
slave_2.exec_stmt("CREATE TABLE test (id INTEGER)")
slave_2.set_session_binlog(True)
master.exec_stmt("CREATE DATABASE IF NOT EXISTS test")
master.exec_stmt("USE test")
master.exec_stmt("SET sql_log_bin=0")
master.exec_stmt("DROP TABLE IF EXISTS test")
master.exec_stmt("SET sql_log_bin=1")
master.exec_stmt("CREATE TABLE test (id INTEGER)")
# Synchronize replicas.
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_1, master, timeout=0)
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_2, master, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
self.assertEqual(status[2][str(slave_1.uuid)]["threads"],
{"sql_running": False, "sql_error": "Error 'Table 'test' "
"already exists' on query. Default database: 'test'. Query: "
"'CREATE TABLE test (id INTEGER)'"}
)
self.assertEqual(status[2][str(slave_1.uuid)]["status"],
_server.MySQLServer.SECONDARY)
self.assertEqual(status[2][str(slave_2.uuid)]["threads"],
{"sql_running": False, "sql_error": "Error 'Table 'test' "
"already exists' on query. Default database: 'test'. Query: "
"'CREATE TABLE test (id INTEGER)'"}
)
self.assertEqual(status[2][str(slave_2.uuid)]["status"],
_server.MySQLServer.SECONDARY)
self.assertEqual(status[2][str(master.uuid)]["status"],
_server.MySQLServer.PRIMARY)
# Try to choose a new master through switch over.
status = self.proxy.group.promote("group_id")
self.assertStatus(status, _executor.Job.ERROR)
self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE)
self.assertEqual(status[1][-1]["description"],
"Tried to execute action (_find_candidate_switch).")
# Try to reset the slave and restart slave.
_repl.stop_slave(slave_1, wait=True)
_repl.reset_slave(slave_1, clean=False)
try:
_repl.start_slave(slave_1, wait=True)
except _errors.DatabaseError as error:
self.assertEqual(
str(error), "Error 'Table 'test' already exists' "
"on query. Default database: 'test'. Query: 'CREATE "
"TABLE test (id INTEGER)'"
)
# Synchronize replica.
self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master,
slave_1, master, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
self.assertTrue(status[2][str(slave_1.uuid)]["threads"] ==
{"sql_running": False, "sql_error": "Error 'Table 'test' "
"already exists' on query. Default database: 'test'. Query: "
"'CREATE TABLE test (id INTEGER)'"}
)
self.assertEqual(status[2][str(slave_1.uuid)]["status"],
_server.MySQLServer.SECONDARY)
self.assertEqual(status[2][str(slave_2.uuid)]["threads"],
{"sql_running": False, "sql_error": "Error 'Table 'test' "
"already exists' on query. Default database: 'test'. Query: "
"'CREATE TABLE test (id INTEGER)'"}
)
self.assertEqual(status[2][str(slave_2.uuid)]["status"],
_server.MySQLServer.SECONDARY)
self.assertEqual(status[2][str(master.uuid)]["status"],
_server.MySQLServer.PRIMARY)
# Try to drop the table on the slave.
_repl.stop_slave(slave_1, wait=True)
_repl.reset_slave(slave_1, clean=False)
slave_1.set_session_binlog(False)
slave_1.exec_stmt("DROP TABLE IF EXISTS test")
slave_1.set_session_binlog(True)
_repl.start_slave(slave_1, wait=True)
_repl.stop_slave(slave_2, wait=True)
_repl.reset_slave(slave_2, clean=False)
slave_2.set_session_binlog(False)
slave_2.exec_stmt("DROP TABLE IF EXISTS test")
slave_2.set_session_binlog(True)
_repl.start_slave(slave_2, wait=True)
# Synchronize replicas.
_repl.sync_slave_with_master(slave_1, master, timeout=0)
_repl.sync_slave_with_master(slave_2, master, timeout=0)
# Check replication.
status = self.proxy.group.health("group_id")
self.assertEqual(status[2][str(slave_1.uuid)]["threads"], {})
self.assertEqual(status[2][str(slave_1.uuid)]["status"],
_server.MySQLServer.SECONDARY)
self.assertEqual(status[2][str(slave_2.uuid)]["threads"], {})
self.assertEqual(status[2][str(slave_2.uuid)]["status"],
_server.MySQLServer.SECONDARY)
self.assertEqual(status[2][str(master.uuid)]["status"],
_server.MySQLServer.PRIMARY)
