def compare_databases(self, comment):
"""Compare databases.
This method compares the databases replicated.
comment[in] Test comment.
"""
# Compare command
compare_cmd = "mysqldbcompare.py {0} {1} {2}:{2} {3}"
from_conn = "--server1={0}".format(
self.build_connection_string(self.server1))
# Compare `inventory` database from master1
to_conn = "--server2={0}".format(
self.build_connection_string(self.server2))
res = self.run_test_case(
0,
compare_cmd.format(from_conn, to_conn, "inventory",
ssl_util_opts()),
"Comparing `inventory` database.")
if not res:
raise MUTLibError("{0}: failed".format(comment))
# Compare `import_test` database from master2
to_conn = "--server2={0}".format(
self.build_connection_string(self.server3))
res = self.run_test_case(
0,
compare_cmd.format(from_conn, to_conn, "import_test",
ssl_util_opts()),
"Comparing `import_test` database.")
if not res:
raise MUTLibError("{0}: failed".format(comment))
def compare_databases(self, comment):
"""Compare databases.
This method compares the databases replicated.
comment[in] Test comment.
"""
# Compare command
compare_cmd = "mysqldbcompare.py {0} {1} {2}:{2} {3}"
from_conn = "--server1={0}".format(
self.build_connection_string(self.server1))
# Compare `inventory` database from master1
to_conn = "--server2={0}".format(
self.build_connection_string(self.server2)
)
res = self.run_test_case(
0,
compare_cmd.format(from_conn, to_conn, "inventory",
ssl_util_opts()),
"Comparing `inventory` database."
)
if not res:
raise MUTLibError("{0}: failed".format(comment))
# Compare `import_test` database from master2
to_conn = "--server2={0}".format(
self.build_connection_string(self.server3)
)
res = self.run_test_case(
0,
compare_cmd.format(from_conn, to_conn, "import_test",
ssl_util_opts()),
"Comparing `import_test` database."
)
if not res:
raise MUTLibError("{0}: failed".format(comment))
def run(self):
slave_str = "--slave={0}[{1}]".format(self.config_file_path,
self.test_server_names[0])
masters_str = "--masters={0}[{1}],{0}[{2}]".format(
self.config_file_path, self.test_server_names[1],
self.test_server_names[2])
test_num = 1
rplms_cmd = ("python ../scripts/mysqlrplms.py --log={0} --interval=5 "
"--switchover-interval=30 --rpl-user=rpl:rpl {1} {2} {3}"
"".format(_RPLMS_LOG.format(test_num), slave_str,
masters_str, ssl_util_opts()))
comment = ("Test case {0} - Simple multi-source replication."
"".format(test_num))
self.test_rplms(rplms_cmd, _RPLMS_LOG.format(test_num), comment, True)
return True
def run(self):
slave_str = "--slave={0}[{1}]".format(self.config_file_path,
self.test_server_names[0])
masters_str = "--masters={0}[{1}],{0}[{2}]".format(
self.config_file_path,
self.test_server_names[1],
self.test_server_names[2]
)
test_num = 1
rplms_cmd = ("python ../scripts/mysqlrplms.py --log={0} --interval=5 "
"--switchover-interval=30 --rpl-user=rpl:rpl {1} {2} {3}"
"".format(_RPLMS_LOG.format(test_num), slave_str,
masters_str, ssl_util_opts()))
comment = ("Test case {0} - Simple multi-source replication."
"".format(test_num))
self.test_rplms(rplms_cmd, _RPLMS_LOG.format(test_num), comment, True)
return True
def run(self):
cmd_base = "mysqlrplsync.py"
master_con = self.build_connection_string(self.server1).strip(" ")
slave1_con = self.build_connection_string(self.server2).strip(" ")
slave2_con = self.build_connection_string(self.server3).strip(" ")
slaves_con = "{0},{1}".format(slave1_con, slave2_con)
# Check the data consistency of empty servers.
test_num = 1
comment = ("Test case {0} - check empty servers." "").format(test_num)
cmd = "{0} --master={1} --slaves={2} {3}".format(cmd_base, master_con, slaves_con, ssl_util_opts())
res = self.run_test_case(0, cmd, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
# Create example database on master and wait for it to be replicated
# on all slaves.
if self.debug:
print("\nCreate test database and tables on master.")
rpl_sync.create_test_db(self.server1)
# Wait for slaves to catch up, otherwise mysqlrplsync might find tables
# missing on slaves (which is normal).
if self.debug:
print("\nWait for slaves to catch up with master.")
self.wait_for_slaves()
# Load data on the master in a separated thread to perform the data
# consistency checks at the same time.
load_thread = threading.Thread(target=rpl_sync.load_test_data, args=(self.server1,))
load_thread.daemon = True
load_thread.start()
if self.debug:
print("\nThread to load/insert data started.")
start_time = time.time()
end_time = None
# Check data consistency specifying the master and using discovery.
test_num += 1
comment = (
"Test case {0} - data consistency check with active " "replication using master and slaves discovery." ""
).format(test_num)
cmd = ("{0} --master={1} --discover-slaves-login=root:root {2}" "").format(
cmd_base, master_con, ssl_util_opts()
)
res = self.run_test_case(0, cmd, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
if self.debug:
alive = load_thread.is_alive()
print("\nIs thread (loading data) still alive? " "{0}.".format(alive))
if not alive and not end_time:
end_time = time.time()
print("\nTime to load data took less than: {0} " "sec.".format(end_time - start_time))
else:
print("\nWaiting for thread (loading data) to finish.")
# Wait for all the data to finish to be loaded.
load_thread.join()
if self.debug and not end_time:
end_time = time.time()
print("\nTime to load data: {0} " "sec.".format(end_time - start_time))
# Drop test database and recreate it on master.
if self.debug:
print("\nDrop test database on master.")
rpl_sync.drop_test_db(self.server1)
if self.debug:
print("\nCreate test database and tables on master.")
rpl_sync.create_test_db(self.server1)
# Wait for slaves to catch up (avoiding non-deterministic results).
if self.debug:
print("\nWait for slaves to catch up with master.")
self.wait_for_slaves()
# Start a new thread to load/insert data (can only be started once).
load_thread = threading.Thread(target=rpl_sync.load_test_data, args=(self.server1,))
load_thread.daemon = True
load_thread.start()
if self.debug:
print("\nThread to load/insert data started.")
start_time = time.time()
end_time = None
# Check data consistency specifying only the slaves.
test_num += 1
comment = (
"Test case {0} - data consistency check with active " "replication only between slaves (no master)." ""
).format(test_num)
cmd = "{0} --slaves={1} {2}".format(cmd_base, slaves_con, ssl_util_opts())
res = self.run_test_case(0, cmd, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
if self.debug:
alive = load_thread.is_alive()
print("\nIs thread (loading data) still alive? " "{0}.".format(alive))
if not alive and not end_time:
end_time = time.time()
print("\nTime to load data took less than: {0} " "sec.".format(end_time - start_time))
else:
print("Waiting for thread (loading data) to finish.")
# Wait for all the data to finish to be loaded.
load_thread.join()
if self.debug and not end_time:
end_time = time.time()
print("\nTime to load data: {0} " "sec.".format(end_time - start_time))
# Perform data consistency check after loading all data.
test_num += 1
comment = ("Test case {0} - data consistency check between master and " "specified slaves.").format(test_num)
cmd = "{0} --master={1} --slaves={2} {3}".format(cmd_base, master_con, slaves_con, ssl_util_opts())
res = self.run_test_case(0, cmd, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
# Mask out non-deterministic data
self.do_masks()
return True
def run(self):
self.res_fname = "result.txt"
master_conn = self.build_connection_string(self.server1).strip(" ")
slave1_conn = self.build_connection_string(self.server2).strip(" ")
slave2_conn = self.build_connection_string(self.server3).strip(" ")
slave3_conn = self.build_connection_string(self.server4).strip(" ")
slave4_conn = self.build_connection_string(self.server5).strip(" ")
# Failover must work even with a slave that does not exist
slave5_conn = "doesNotExist@localhost:999999999999"
master_str = "--master={0}".format(master_conn)
slaves_str = "--slaves={0}".format(
",".join([slave1_conn, slave2_conn, slave3_conn, slave5_conn])
)
self.test_results = []
self.test_cases = []
failover_cmd = (
"{0} {1}"
"".format("python ../scripts/mysqlfailover.py --interval=10 "
"--daemon={0} --discover-slaves-login=root:root {1} "
"--failover-mode={2} --log={3} "
"--exec-post-fail=\"{4}\" --timeout=5{5}",
ssl_util_opts())
)
i = 1
cmd = failover_cmd.format("nodetach",
" ".join([master_str, slaves_str]), "auto",
_FAILOVER_LOG.format("1"),
self.fail_event_script,
" --candidates={0}".format(slave1_conn))
self.test_cases.append(
(self.server1, cmd, True, _FAILOVER_LOG.format("1"),
"Test case {0} - Simple failover with --daemon=nodetach "
"--failover=auto.".format(i),
_FAILOVER_COMPLETE, False)
)
i += 1
cmd = failover_cmd.format("nodetach",
"--master={0}".format(slave1_conn), "elect",
_FAILOVER_LOG.format("2"),
self.fail_event_script,
" --candidates={0} ".format(slave2_conn))
self.test_cases.append(
(self.server2, cmd, True, _FAILOVER_LOG.format("2"),
"Test case {0} - Simple failover with --failover=elect."
"".format(i), _FAILOVER_COMPLETE, True)
)
i += 1
cmd = failover_cmd.format("nodetach",
"--master={0}".format(slave2_conn), "fail",
_FAILOVER_LOG.format("3"),
self.fail_event_script, "")
self.test_cases.append(
(self.server3, cmd, False, _FAILOVER_LOG.format("3"),
"Test case {0} - Simple failover with --failover=fail.".format(i),
"Master has failed and automatic", True)
)
i += 1
cmd = failover_cmd.format("nodetach",
"--master={0}".format(slave3_conn), "fail",
_FAILOVER_LOG.format("4"),
self.fail_event_script, " --force")
self.test_cases.append(
(self.server4, cmd, False, _FAILOVER_LOG.format("4"),
"Test case {0} - Test with --daemon=nodetach and --force on "
"first run.".format(i), None, True)
)
# Run --daemon=nodetach tests
for test_case in self.test_cases:
res = self.test_failover_daemon_nodetach(test_case)
if res:
self.test_results.append(res)
else:
raise MUTLibError("{0}: failed".format(test_case[4]))
i += 1
comment = ("Test case {0} - Start failover with --daemon=start."
"".format(i))
cmd_extra = " --pidfile={0}".format(_FAILOVER_PID.format("5"))
cmd = failover_cmd.format("start",
"--master={0}".format(slave4_conn), "auto",
_FAILOVER_LOG.format("5"),
self.fail_event_script, cmd_extra)
res = self.test_failover_daemon(comment, cmd,
_FAILOVER_LOG.format("5"),
_FAILOVER_PID.format("5"), False)
if res:
self.test_results.append(res)
else:
raise MUTLibError("{0}: failed".format(comment))
i += 1
comment = ("Test case {0} - Restart failover by using --daemon="
"restart and then stop the daemon.".format(i))
cmd = failover_cmd.format("restart",
"--master={0}".format(slave4_conn), "auto",
_FAILOVER_LOG.format("5"),
self.fail_event_script, cmd_extra)
res = self.test_failover_daemon(comment, cmd,
_FAILOVER_LOG.format("5"),
_FAILOVER_PID.format("5"), True)
if res:
self.test_results.append(res)
else:
raise MUTLibError("{0}: failed".format(comment))
return True
def run(self):
cmd_base = "mysqlrplsync.py"
master_con = self.build_connection_string(self.server1).strip(' ')
slave1_con = self.build_connection_string(self.server2).strip(' ')
slave2_con = self.build_connection_string(self.server3).strip(' ')
slaves_con = "{0},{1}".format(slave1_con, slave2_con)
# Check the data consistency of empty servers.
test_num = 1
comment = ("Test case {0} - check empty servers."
"").format(test_num)
cmd = "{0} --master={1} --slaves={2} {3}".format(cmd_base, master_con,
slaves_con,
ssl_util_opts())
res = self.run_test_case(0, cmd, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
# Create example database on master and wait for it to be replicated
# on all slaves.
if self.debug:
print("\nCreate test database and tables on master.")
rpl_sync.create_test_db(self.server1)
# Wait for slaves to catch up, otherwise mysqlrplsync might find tables
# missing on slaves (which is normal).
if self.debug:
print("\nWait for slaves to catch up with master.")
self.wait_for_slaves()
# Load data on the master in a separated thread to perform the data
# consistency checks at the same time.
load_thread = threading.Thread(target=rpl_sync.load_test_data,
args=(self.server1,))
load_thread.daemon = True
load_thread.start()
if self.debug:
print("\nThread to load/insert data started.")
start_time = time.time()
end_time = None
# Check data consistency specifying the master and using discovery.
test_num += 1
comment = ("Test case {0} - data consistency check with active "
"replication using master and slaves discovery."
"").format(test_num)
cmd = ("{0} --master={1} --discover-slaves-login=root:root {2}"
"").format(cmd_base, master_con, ssl_util_opts())
res = self.run_test_case(0, cmd, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
if self.debug:
alive = load_thread.is_alive()
print("\nIs thread (loading data) still alive? "
"{0}.".format(alive))
if not alive and not end_time:
end_time = time.time()
print("\nTime to load data took less than: {0} "
"sec.".format(end_time - start_time))
else:
print("\nWaiting for thread (loading data) to finish.")
# Wait for all the data to finish to be loaded.
load_thread.join()
if self.debug and not end_time:
end_time = time.time()
print("\nTime to load data: {0} "
"sec.".format(end_time - start_time))
# Drop test database and recreate it on master.
if self.debug:
print("\nDrop test database on master.")
rpl_sync.drop_test_db(self.server1)
if self.debug:
print("\nCreate test database and tables on master.")
rpl_sync.create_test_db(self.server1)
# Wait for slaves to catch up (avoiding non-deterministic results).
if self.debug:
print("\nWait for slaves to catch up with master.")
self.wait_for_slaves()
# Start a new thread to load/insert data (can only be started once).
load_thread = threading.Thread(target=rpl_sync.load_test_data,
args=(self.server1,))
load_thread.daemon = True
load_thread.start()
if self.debug:
print("\nThread to load/insert data started.")
start_time = time.time()
end_time = None
# Check data consistency specifying only the slaves.
test_num += 1
comment = ("Test case {0} - data consistency check with active "
"replication only between slaves (no master)."
"").format(test_num)
cmd = "{0} --slaves={1} {2}".format(cmd_base, slaves_con,
ssl_util_opts())
res = self.run_test_case(0, cmd, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
if self.debug:
alive = load_thread.is_alive()
print("\nIs thread (loading data) still alive? "
"{0}.".format(alive))
if not alive and not end_time:
end_time = time.time()
print("\nTime to load data took less than: {0} "
"sec.".format(end_time - start_time))
else:
print("Waiting for thread (loading data) to finish.")
# Wait for all the data to finish to be loaded.
load_thread.join()
if self.debug and not end_time:
end_time = time.time()
print("\nTime to load data: {0} "
"sec.".format(end_time - start_time))
# Perform data consistency check after loading all data.
test_num += 1
comment = ("Test case {0} - data consistency check between master and "
"specified slaves.").format(test_num)
cmd = "{0} --master={1} --slaves={2} {3}".format(cmd_base, master_con,
slaves_con,
ssl_util_opts())
res = self.run_test_case(0, cmd, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
# Mask out non-deterministic data
self.do_masks()
return True
def run(self):
test_num = 1
master_conn = self.build_connection_string(self.server1).strip(' ')
slave1_conn = self.build_connection_string(self.server2).strip(' ')
slave2_conn = self.build_connection_string(self.server3).strip(' ')
slave3_conn = self.build_connection_string(self.server4).strip(' ')
comment = ("Test case {0} - SSL mysqlrplshow OLD Master "
"before demote".format(test_num))
cmd_str = "mysqlrplshow.py --master={0} {1} ".format(master_conn,
ssl_util_opts())
cmd_opts = "--discover-slaves={0} ".format(master_conn.split('@')[0])
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
test_num += 1
comment = ("Test case {0} - SSL "
"switchover demote-master ".format(test_num))
cmd_str = ("mysqlrpladmin.py --master={0} {1} "
).format(master_conn, ssl_util_opts())
cmd_opts = (" --new-master={0} --discover-slaves={1} "
"--rpl-user=rpluser:hispassword --demote-master "
"switchover".format(slave1_conn,
master_conn.split('@')[0]))
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplshow "
"NEW Master after demote".format(test_num))
cmd_str = "mysqlrplshow.py --master={0} {1}".format(slave1_conn,
ssl_util_opts())
cmd_opts = " --discover-slaves={0} ".format(master_conn.split('@')[0])
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_opts = "--slave={0} --show-slave-status".format(master_conn)
cmd_str = "mysqlrplcheck.py --master={0} {1} ".format(slave1_conn,
ssl_util_opts())
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_opts = "--slave={0} --show-slave-status".format(slave2_conn)
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_opts = "--slave={0} --show-slave-status".format(slave3_conn)
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL "
"failover ".format(test_num))
cmd_str = "mysqlrpladmin.py {0} ".format(ssl_util_opts())
slaves = ",".join([slave2_conn, slave3_conn,
master_conn])
cmd_opts = (" --slaves={0} --rpl-user=rpluser:hispassword "
"failover".format(slaves))
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplshow "
"NEW Master after failover".format(test_num))
cmd_str = "mysqlrplshow.py --master={0} {1}".format(slave2_conn,
ssl_util_opts())
cmd_opts = " --discover-slaves={0} ".format(master_conn.split('@')[0])
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_str = "mysqlrplcheck.py --master={0} {1} ".format(slave2_conn,
ssl_util_opts())
cmd_opts = "--slave={0} --show-slave-status".format(master_conn)
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_opts = "--slave={0} --show-slave-status".format(slave3_conn)
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
self.mask_results()
return True
def run_rpl_test(self,
slave,
master,
comment,
options=None,
save_for_compare=False,
expected_result=0,
save_results=True,
ssl=False,
config_path=False,
rpl_user='******',
rpl_pass='******',
ssl_opts=ssl_util_opts(),
use_rpl_user_group=False):
"""Run replication test.
slave[in] Slave instance.
master[in] Master instance.
comment[in] Test case comment.
options[in] Options.
save_for_compare[in] True for save compare
expected_result[in] Expected result.
save_results[in] True for save results.
ssl[in] use ssl by default False.
config_path[in] use config-path by default False.
rpl_user[in] the rpl user name to use by default 'rpl'.
rpl_pass[in] the rpl user password to use by default 'rpl'.
ssl_opts[in] the ssl certificate options.
use_rpl_user_group[in] use rpl user from config-path by default false.
"""
if not config_path:
master_str = "--master={0}".format(
self.build_connection_string(master))
else:
master_ = 'server_{0}'.format(master.port)
master_str = "--master={0}[{1}]".format(self.config_file_path,
master_)
if not config_path:
slave_str = " --slave={0}".format(
self.build_connection_string(slave))
else:
slave_ = 'server_{0}'.format(slave.port)
slave_str = " --slave={0}[{1}]".format(self.config_file_path,
slave_)
conn_str = '{0}{1}'.format(master_str, slave_str)
# Test case 1 - setup replication among two servers
if not save_for_compare:
self.results.append(comment)
if use_rpl_user_group:
cmd = ("mysqlreplicate.py --rpl-user={0}[{1}] {2}").format(
self.config_file_path, _RPL_USER_GROUP_NAME, conn_str)
else:
cmd = ("mysqlreplicate.py --rpl-user={0}:{1} {2}").format(
rpl_user, rpl_pass, conn_str)
if ssl:
cmd = "{0} {1}".format(cmd, ssl_opts)
if options:
cmd = "{0} {1}".format(cmd, options)
if not save_for_compare and save_results:
self.results.append(cmd)
res = self.exec_util(cmd, self.res_fname)
if not save_for_compare and save_results:
self.results.append(res)
if res != expected_result:
return False
# Now test the result and record the action.
try:
res = slave.exec_query("SHOW SLAVE STATUS")
if not save_for_compare and save_results:
self.results.append(res)
except UtilDBError as err:
raise MUTLibError("Cannot show slave status: "
"{0}".format(err.errmsg))
if save_for_compare:
self.results.append(comment + "\n")
with open(self.res_fname) as f:
for line in f:
# Don't save lines that have [Warning]
index = line.find("[Warning]")
if index <= 0:
self.results.append(line)
return True
def run(self):
self.res_fname = "result.txt"
master_conn = self.build_connection_string(self.server1).strip(" ")
slave1_conn = self.build_connection_string(self.server2).strip(" ")
slave2_conn = self.build_connection_string(self.server3).strip(" ")
slave3_conn = self.build_connection_string(self.server4).strip(" ")
slave4_conn = self.build_connection_string(self.server5).strip(" ")
# Failover must work even with a slave that does not exist
slave5_conn = "doesNotExist@localhost:999999999999"
master_str = "--master={0}".format(master_conn)
slaves_str = "--slaves={0}".format(",".join(
[slave1_conn, slave2_conn, slave3_conn, slave5_conn]))
self.test_results = []
self.test_cases = []
failover_cmd = (
"{0} {1}"
"".format(
"python ../scripts/mysqlfailover.py --interval=10 "
"--daemon={0} --discover-slaves-login=root:root {1} "
"--failover-mode={2} --log={3} "
"--exec-post-fail=\"{4}\" --timeout=5{5}", ssl_util_opts()))
i = 1
cmd = failover_cmd.format("nodetach",
" ".join([master_str, slaves_str]), "auto",
_FAILOVER_LOG.format("1"),
self.fail_event_script,
" --candidates={0}".format(slave1_conn))
self.test_cases.append(
(self.server1, cmd, True, _FAILOVER_LOG.format("1"),
"Test case {0} - Simple failover with --daemon=nodetach "
"--failover=auto.".format(i), _FAILOVER_COMPLETE, False))
i += 1
cmd = failover_cmd.format("nodetach",
"--master={0}".format(slave1_conn), "elect",
_FAILOVER_LOG.format("2"),
self.fail_event_script,
" --candidates={0} ".format(slave2_conn))
self.test_cases.append(
(self.server2, cmd, True, _FAILOVER_LOG.format("2"),
"Test case {0} - Simple failover with --failover=elect."
"".format(i), _FAILOVER_COMPLETE, True))
i += 1
cmd = failover_cmd.format("nodetach",
"--master={0}".format(slave2_conn), "fail",
_FAILOVER_LOG.format("3"),
self.fail_event_script, "")
self.test_cases.append(
(self.server3, cmd, False, _FAILOVER_LOG.format("3"),
"Test case {0} - Simple failover with --failover=fail.".format(i),
"Master has failed and automatic", True))
i += 1
cmd = failover_cmd.format("nodetach",
"--master={0}".format(slave3_conn), "fail",
_FAILOVER_LOG.format("4"),
self.fail_event_script, " --force")
self.test_cases.append(
(self.server4, cmd, False, _FAILOVER_LOG.format("4"),
"Test case {0} - Test with --daemon=nodetach and --force on "
"first run.".format(i), None, True))
# Run --daemon=nodetach tests
for test_case in self.test_cases:
res = self.test_failover_daemon_nodetach(test_case)
if res:
self.test_results.append(res)
else:
raise MUTLibError("{0}: failed".format(test_case[4]))
i += 1
comment = ("Test case {0} - Start failover with --daemon=start."
"".format(i))
cmd_extra = " --pidfile={0}".format(_FAILOVER_PID.format("5"))
cmd = failover_cmd.format("start", "--master={0}".format(slave4_conn),
"auto", _FAILOVER_LOG.format("5"),
self.fail_event_script, cmd_extra)
res = self.test_failover_daemon(comment, cmd,
_FAILOVER_LOG.format("5"),
_FAILOVER_PID.format("5"), False)
if res:
self.test_results.append(res)
else:
raise MUTLibError("{0}: failed".format(comment))
i += 1
comment = ("Test case {0} - Restart failover by using --daemon="
"restart and then stop the daemon.".format(i))
cmd = failover_cmd.format("restart",
"--master={0}".format(slave4_conn), "auto",
_FAILOVER_LOG.format("5"),
self.fail_event_script, cmd_extra)
res = self.test_failover_daemon(comment, cmd,
_FAILOVER_LOG.format("5"),
_FAILOVER_PID.format("5"), True)
if res:
self.test_results.append(res)
else:
raise MUTLibError("{0}: failed".format(comment))
return True
def run(self):
test_num = 1
master_conn = self.build_connection_string(self.server1).strip(' ')
slave1_conn = self.build_connection_string(self.server2).strip(' ')
slave2_conn = self.build_connection_string(self.server3).strip(' ')
slave3_conn = self.build_connection_string(self.server4).strip(' ')
comment = ("Test case {0} - SSL mysqlrplshow OLD Master "
"before demote".format(test_num))
cmd_str = "mysqlrplshow.py --master={0} {1} ".format(
master_conn, ssl_util_opts())
cmd_opts = "--discover-slaves={0} ".format(master_conn.split('@')[0])
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
test_num += 1
comment = ("Test case {0} - SSL "
"switchover demote-master ".format(test_num))
cmd_str = ("mysqlrpladmin.py --master={0} {1} ").format(
master_conn, ssl_util_opts())
cmd_opts = (" --new-master={0} --discover-slaves={1} "
"--rpl-user=rpluser:hispassword --demote-master "
"switchover".format(slave1_conn,
master_conn.split('@')[0]))
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplshow "
"NEW Master after demote".format(test_num))
cmd_str = "mysqlrplshow.py --master={0} {1}".format(
slave1_conn, ssl_util_opts())
cmd_opts = " --discover-slaves={0} ".format(master_conn.split('@')[0])
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_opts = "--slave={0} --show-slave-status".format(master_conn)
cmd_str = "mysqlrplcheck.py --master={0} {1} ".format(
slave1_conn, ssl_util_opts())
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_opts = "--slave={0} --show-slave-status".format(slave2_conn)
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_opts = "--slave={0} --show-slave-status".format(slave3_conn)
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL " "failover ".format(test_num))
cmd_str = "mysqlrpladmin.py {0} ".format(ssl_util_opts())
slaves = ",".join([slave2_conn, slave3_conn, master_conn])
cmd_opts = (" --slaves={0} --rpl-user=rpluser:hispassword "
"failover".format(slaves))
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplshow "
"NEW Master after failover".format(test_num))
cmd_str = "mysqlrplshow.py --master={0} {1}".format(
slave2_conn, ssl_util_opts())
cmd_opts = " --discover-slaves={0} ".format(master_conn.split('@')[0])
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_str = "mysqlrplcheck.py --master={0} {1} ".format(
slave2_conn, ssl_util_opts())
cmd_opts = "--slave={0} --show-slave-status".format(master_conn)
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
test_num += 1
comment = ("Test case {0} - SSL mysqlrplcheck "
"NEW Master after demote".format(test_num))
cmd_opts = "--slave={0} --show-slave-status".format(slave3_conn)
cmds = "{0} {1}".format(cmd_str, cmd_opts)
res = self.run_test_case(0, cmds, comment)
if not res:
raise MUTLibError("{0}: failed".format(comment))
self.results.append("\n")
self.mask_results()
return True
def run_rpl_test(self, slave, master, comment, options=None,
save_for_compare=False, expected_result=0,
save_results=True, ssl=False, config_path=False,
rpl_user='******', rpl_pass='******',
ssl_opts=ssl_util_opts(), use_rpl_user_group=False):
"""Run replication test.
slave[in] Slave instance.
master[in] Master instance.
comment[in] Test case comment.
options[in] Options.
save_for_compare[in] True for save compare
expected_result[in] Expected result.
save_results[in] True for save results.
ssl[in] use ssl by default False.
config_path[in] use config-path by default False.
rpl_user[in] the rpl user name to use by default 'rpl'.
rpl_pass[in] the rpl user password to use by default 'rpl'.
ssl_opts[in] the ssl certificate options.
use_rpl_user_group[in] use rpl user from config-path by default false.
"""
if not config_path:
master_str = "--master={0}".format(
self.build_connection_string(master))
else:
master_ = 'server_{0}'.format(master.port)
master_str = "--master={0}[{1}]".format(self.config_file_path,
master_)
if not config_path:
slave_str = " --slave={0}".format(
self.build_connection_string(slave))
else:
slave_ = 'server_{0}'.format(slave.port)
slave_str = " --slave={0}[{1}]".format(self.config_file_path,
slave_)
conn_str = '{0}{1}'.format(master_str, slave_str)
# Test case 1 - setup replication among two servers
if not save_for_compare:
self.results.append(comment)
if use_rpl_user_group:
cmd = ("mysqlreplicate.py --rpl-user={0}[{1}] {2}"
).format(self.config_file_path, _RPL_USER_GROUP_NAME,
conn_str)
else:
cmd = ("mysqlreplicate.py --rpl-user={0}:{1} {2}"
).format(rpl_user, rpl_pass, conn_str)
if ssl:
cmd = "{0} {1}".format(cmd, ssl_opts)
if options:
cmd = "{0} {1}".format(cmd, options)
if not save_for_compare and save_results:
self.results.append(cmd)
res = self.exec_util(cmd, self.res_fname)
if not save_for_compare and save_results:
self.results.append(res)
if res != expected_result:
return False
# Now test the result and record the action.
try:
res = slave.exec_query("SHOW SLAVE STATUS")
if not save_for_compare and save_results:
self.results.append(res)
except UtilDBError as err:
raise MUTLibError("Cannot show slave status: "
"{0}".format(err.errmsg))
if save_for_compare:
self.results.append(comment + "\n")
with open(self.res_fname) as f:
for line in f:
# Don't save lines that have [Warning]
index = line.find("[Warning]")
if index <= 0:
self.results.append(line)
return True
