def synchronize(slave, master): """Synchronize a slave with a master and after that stop the slave. :param slave: Slave. :param master: Master. """ _replication.sync_slave_with_master(slave, master, timeout=0)
def test_check_no_healthy_slave(self): """Test promoting when there is no healthy slave. """ # Configure replication. instances = tests.utils.MySQLInstances() user = instances.user passwd = instances.passwd instances.configure_instances({0 : [{1 : []}, {2 : []}]}, user, passwd) master = instances.get_instance(0) slave_1 = instances.get_instance(1) slave_2 = instances.get_instance(2) self.proxy.group.create("group_id", "") self.proxy.group.add("group_id", master.address) self.proxy.group.add("group_id", slave_1.address) self.proxy.group.add("group_id", slave_2.address) # Promote a master. status = self.proxy.group.promote("group_id", str(master.uuid)) self.check_xmlrpc_command_result(status) # Check replication. status = self.proxy.group.health("group_id") self.check_xmlrpc_simple(status, { 'status': _server.MySQLServer.SECONDARY }, index=2, rowcount=3) self.check_xmlrpc_simple(status, { 'status': _server.MySQLServer.SECONDARY }, index=1, rowcount=3) self.check_xmlrpc_simple(status, { 'status': _server.MySQLServer.PRIMARY }, index=0, rowcount=3) # Inject some events that make slaves break. slave_1.set_session_binlog(False) slave_1.exec_stmt("CREATE DATABASE IF NOT EXISTS test") slave_1.exec_stmt("USE test") slave_1.exec_stmt("DROP TABLE IF EXISTS test") slave_1.exec_stmt("CREATE TABLE test (id INTEGER)") slave_1.set_session_binlog(True) slave_2.set_session_binlog(False) slave_2.exec_stmt("CREATE DATABASE IF NOT EXISTS test") slave_2.exec_stmt("USE test") slave_2.exec_stmt("DROP TABLE IF EXISTS test") slave_2.exec_stmt("CREATE TABLE test (id INTEGER)") slave_2.set_session_binlog(True) master.exec_stmt("CREATE DATABASE IF NOT EXISTS test") master.exec_stmt("USE test") master.exec_stmt("SET sql_log_bin=0") master.exec_stmt("DROP TABLE IF EXISTS test") master.exec_stmt("SET sql_log_bin=1") master.exec_stmt("CREATE TABLE test (id INTEGER)") # Synchronize replicas. self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_1, master, timeout=0) self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_2, master, timeout=0) # Check replication. status = self.proxy.group.health("group_id") for info in self.check_xmlrpc_iter(status): if info['uuid'] in (str(slave_2.uuid), str(slave_1.uuid)): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) self.assertEqual(info['sql_not_running'], True) elif info['uuid'] == str(master.uuid): self.assertEqual( info['status'], _server.MySQLServer.PRIMARY ) # Try to choose a new master through switch over. status = self.proxy.group.promote("group_id") self.check_xmlrpc_command_result(status, has_error=True) # Try to reset the slave and restart slave. _repl.stop_slave(slave_1, wait=True) _repl.reset_slave(slave_1, clean=False) try: _repl.start_slave(slave_1, wait=True) except _errors.DatabaseError as error: self.assertEqual( str(error), "Error 'Table 'test' already exists' " "on query. Default database: 'test'. Query: 'CREATE " "TABLE test (id INTEGER)'" ) # Synchronize replica. self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_1, master, timeout=0) # Check replication. status = self.proxy.group.health("group_id") self.check_xmlrpc_simple(status, { 'status': _server.MySQLServer.SECONDARY, "sql_not_running": True, }, index=2, rowcount=3) self.check_xmlrpc_simple(status, { 'status': _server.MySQLServer.SECONDARY, "sql_not_running": True, }, index=1, rowcount=3) self.check_xmlrpc_simple(status, { 'status': _server.MySQLServer.PRIMARY, }, index=0, rowcount=3) # Try to drop the table on the slave. _repl.stop_slave(slave_1, wait=True) _repl.reset_slave(slave_1, clean=False) slave_1.set_session_binlog(False) slave_1.exec_stmt("DROP TABLE IF EXISTS test") slave_1.set_session_binlog(True) _repl.start_slave(slave_1, wait=True) _repl.stop_slave(slave_2, wait=True) _repl.reset_slave(slave_2, clean=False) slave_2.set_session_binlog(False) slave_2.exec_stmt("DROP TABLE IF EXISTS test") slave_2.set_session_binlog(True) _repl.start_slave(slave_2, wait=True) # Synchronize replicas. _repl.sync_slave_with_master(slave_1, master, timeout=0) _repl.sync_slave_with_master(slave_2, master, timeout=0) # Check replication. status = self.proxy.group.health("group_id") for info in self.check_xmlrpc_iter(status, rowcount=3): if info['uuid'] in (str(slave_2.uuid), str(slave_1.uuid)): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) self.assertEqual(info['sql_not_running'], False) elif info['uuid'] == str(master.uuid): self.assertEqual( info['status'], _server.MySQLServer.PRIMARY )
def test_check_unhealthy_slave(self): """Test promoting a there is an unhealthy slave. """ # Configure replication. instances = tests.utils.MySQLInstances() user = instances.user passwd = instances.passwd instances.configure_instances({0 : [{1 : []}, {2 : []}]}, user, passwd) master = instances.get_instance(0) slave_1 = instances.get_instance(1) slave_2 = instances.get_instance(2) self.proxy.group.create("group_id", "") self.proxy.group.add("group_id", master.address) self.proxy.group.add("group_id", slave_1.address) self.proxy.group.add("group_id", slave_2.address) # Promote a master. status = self.proxy.group.promote("group_id", str(master.uuid)) self.check_xmlrpc_command_result(status) # Check replication. status = self.proxy.group.health("group_id") for info in self.check_xmlrpc_iter(status): if info['uuid'] in (str(slave_1.uuid), str(slave_2.uuid)): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) elif info['uuid'] == str(master.uuid): self.assertEqual( info['status'], _server.MySQLServer.PRIMARY ) # Inject some events that makes a slave break. slave_1.set_session_binlog(False) slave_1.exec_stmt("CREATE DATABASE IF NOT EXISTS test") slave_1.exec_stmt("USE test") slave_1.exec_stmt("DROP TABLE IF EXISTS test") slave_1.exec_stmt("CREATE TABLE test (id INTEGER)") slave_1.set_session_binlog(True) slave_2.set_session_binlog(False) slave_2.exec_stmt("CREATE DATABASE IF NOT EXISTS test") slave_2.exec_stmt("USE test") slave_2.exec_stmt("DROP TABLE IF EXISTS test") slave_2.set_session_binlog(True) master.set_session_binlog(False) master.exec_stmt("CREATE DATABASE IF NOT EXISTS test") master.exec_stmt("USE test") master.exec_stmt("DROP TABLE IF EXISTS test") master.set_session_binlog(True) master.exec_stmt("CREATE TABLE test (id INTEGER)") # Synchronize replicas. self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_1, master, timeout=0) _repl.sync_slave_with_master(slave_2, master, timeout=0) # Check replication. status = self.proxy.group.health("group_id") for info in self.check_xmlrpc_iter(status, rowcount=3): if info['uuid'] == str(slave_1.uuid): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) self.assertEqual(info['sql_not_running'], True) elif info['uuid'] == str(slave_2.uuid): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) elif info['uuid'] == str(master.uuid): self.assertEqual( info['status'], _server.MySQLServer.PRIMARY ) # Try to do a switch over to the faulty replica. status = self.proxy.group.promote("group_id", str(slave_1.uuid)) self.check_xmlrpc_command_result(status, has_error=True) # Choose a new master. status = self.proxy.group.promote("group_id") self.check_xmlrpc_command_result(status) # Synchronize replicas. self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_1, slave_2, timeout=0) _repl.sync_slave_with_master(master, slave_2, timeout=0) # Check replication. status = self.proxy.group.health("group_id") for info in self.check_xmlrpc_iter(status): if info['uuid'] == str(slave_1.uuid): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) self.assertEqual(info['sql_not_running'], True) elif info['uuid'] == str(slave_2.uuid): self.assertEqual( info['status'], _server.MySQLServer.PRIMARY ) elif info['uuid'] == str(master.uuid): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) self.assertEqual(info['sql_not_running'], False) # Choose a new master. status = self.proxy.group.promote("group_id") self.check_xmlrpc_command_result(status) # Synchronize replicas. self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_1, master, timeout=0) _repl.sync_slave_with_master(slave_2, master, timeout=0) # Check replication. status = self.proxy.group.health("group_id") for info in self.check_xmlrpc_iter(status): if info['uuid'] == str(slave_1.uuid): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) self.assertEqual(info['sql_not_running'], True) elif info['uuid'] == str(slave_2.uuid): self.assertEqual( info['status'], _server.MySQLServer.SECONDARY ) elif info['uuid'] == str(master.uuid): self.assertEqual( info['status'], _server.MySQLServer.PRIMARY )
def test_check_no_healthy_slave(self): """Test promoting when there is no healthy slave. """ # Configure replication. instances = tests.utils.MySQLInstances() user = instances.user passwd = instances.passwd instances.configure_instances({0 : [{1 : []}, {2 : []}]}, user, passwd) master = instances.get_instance(0) slave_1 = instances.get_instance(1) slave_2 = instances.get_instance(2) self.proxy.group.create("group_id", "") self.proxy.group.add("group_id", master.address) self.proxy.group.add("group_id", slave_1.address) self.proxy.group.add("group_id", slave_2.address) # Promote a master. status = self.proxy.group.promote("group_id", str(master.uuid)) self.assertStatus(status, _executor.Job.SUCCESS) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Executed action (_change_to_candidate).") # Check replication. status = self.proxy.group.health("group_id") self.assertEqual(status[2][str(slave_1.uuid)]["threads"], {}) self.assertEqual(status[2][str(slave_1.uuid)]["status"], _server.MySQLServer.SECONDARY) self.assertEqual(status[2][str(slave_2.uuid)]["threads"], {}) self.assertEqual(status[2][str(slave_2.uuid)]["status"], _server.MySQLServer.SECONDARY) self.assertEqual(status[2][str(master.uuid)]["status"], _server.MySQLServer.PRIMARY) # Inject some events that make slaves break. slave_1.set_session_binlog(False) slave_1.exec_stmt("CREATE DATABASE IF NOT EXISTS test") slave_1.exec_stmt("USE test") slave_1.exec_stmt("DROP TABLE IF EXISTS test") slave_1.exec_stmt("CREATE TABLE test (id INTEGER)") slave_1.set_session_binlog(True) slave_2.set_session_binlog(False) slave_2.exec_stmt("CREATE DATABASE IF NOT EXISTS test") slave_2.exec_stmt("USE test") slave_2.exec_stmt("DROP TABLE IF EXISTS test") slave_2.exec_stmt("CREATE TABLE test (id INTEGER)") slave_2.set_session_binlog(True) master.exec_stmt("CREATE DATABASE IF NOT EXISTS test") master.exec_stmt("USE test") master.exec_stmt("SET sql_log_bin=0") master.exec_stmt("DROP TABLE IF EXISTS test") master.exec_stmt("SET sql_log_bin=1") master.exec_stmt("CREATE TABLE test (id INTEGER)") # Synchronize replicas. self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_1, master, timeout=0) self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_2, master, timeout=0) # Check replication. status = self.proxy.group.health("group_id") self.assertEqual(status[2][str(slave_1.uuid)]["threads"], {"sql_running": False, "sql_error": "Error 'Table 'test' " "already exists' on query. Default database: 'test'. Query: " "'CREATE TABLE test (id INTEGER)'"} ) self.assertEqual(status[2][str(slave_1.uuid)]["status"], _server.MySQLServer.SECONDARY) self.assertEqual(status[2][str(slave_2.uuid)]["threads"], {"sql_running": False, "sql_error": "Error 'Table 'test' " "already exists' on query. Default database: 'test'. Query: " "'CREATE TABLE test (id INTEGER)'"} ) self.assertEqual(status[2][str(slave_2.uuid)]["status"], _server.MySQLServer.SECONDARY) self.assertEqual(status[2][str(master.uuid)]["status"], _server.MySQLServer.PRIMARY) # Try to choose a new master through switch over. status = self.proxy.group.promote("group_id") self.assertStatus(status, _executor.Job.ERROR) self.assertEqual(status[1][-1]["state"], _executor.Job.COMPLETE) self.assertEqual(status[1][-1]["description"], "Tried to execute action (_find_candidate_switch).") # Try to reset the slave and restart slave. _repl.stop_slave(slave_1, wait=True) _repl.reset_slave(slave_1, clean=False) try: _repl.start_slave(slave_1, wait=True) except _errors.DatabaseError as error: self.assertEqual( str(error), "Error 'Table 'test' already exists' " "on query. Default database: 'test'. Query: 'CREATE " "TABLE test (id INTEGER)'" ) # Synchronize replica. self.assertRaises(_errors.DatabaseError, _repl.sync_slave_with_master, slave_1, master, timeout=0) # Check replication. status = self.proxy.group.health("group_id") self.assertTrue(status[2][str(slave_1.uuid)]["threads"] == {"sql_running": False, "sql_error": "Error 'Table 'test' " "already exists' on query. Default database: 'test'. Query: " "'CREATE TABLE test (id INTEGER)'"} ) self.assertEqual(status[2][str(slave_1.uuid)]["status"], _server.MySQLServer.SECONDARY) self.assertEqual(status[2][str(slave_2.uuid)]["threads"], {"sql_running": False, "sql_error": "Error 'Table 'test' " "already exists' on query. Default database: 'test'. Query: " "'CREATE TABLE test (id INTEGER)'"} ) self.assertEqual(status[2][str(slave_2.uuid)]["status"], _server.MySQLServer.SECONDARY) self.assertEqual(status[2][str(master.uuid)]["status"], _server.MySQLServer.PRIMARY) # Try to drop the table on the slave. _repl.stop_slave(slave_1, wait=True) _repl.reset_slave(slave_1, clean=False) slave_1.set_session_binlog(False) slave_1.exec_stmt("DROP TABLE IF EXISTS test") slave_1.set_session_binlog(True) _repl.start_slave(slave_1, wait=True) _repl.stop_slave(slave_2, wait=True) _repl.reset_slave(slave_2, clean=False) slave_2.set_session_binlog(False) slave_2.exec_stmt("DROP TABLE IF EXISTS test") slave_2.set_session_binlog(True) _repl.start_slave(slave_2, wait=True) # Synchronize replicas. _repl.sync_slave_with_master(slave_1, master, timeout=0) _repl.sync_slave_with_master(slave_2, master, timeout=0) # Check replication. status = self.proxy.group.health("group_id") self.assertEqual(status[2][str(slave_1.uuid)]["threads"], {}) self.assertEqual(status[2][str(slave_1.uuid)]["status"], _server.MySQLServer.SECONDARY) self.assertEqual(status[2][str(slave_2.uuid)]["threads"], {}) self.assertEqual(status[2][str(slave_2.uuid)]["status"], _server.MySQLServer.SECONDARY) self.assertEqual(status[2][str(master.uuid)]["status"], _server.MySQLServer.PRIMARY)