def cleanup(self): from mysql.utilities.common.tools import delete_directory if self.server3: delete_directory(self.datadir3) self.server3 = None return server_info.test.cleanup(self)
def cleanup(self): if self.res_fname: os.unlink(self.res_fname) from mysql.utilities.common.tools import delete_directory if self.server3: delete_directory(self.datadir3) self.server3 = None return True
def run(self): cmd_base = "mysqlbinlogmove.py" master_con = self.build_connection_string(self.server1).strip(' ') slave1_con = self.build_connection_string(self.server2).strip(' ') if self.debug: print("\nCreate multiple binary logs on all servers " "(FLUSH LOGS)...") for srv in [self.server1, self.server2, self.server3]: for _ in range(5): srv.exec_query('FLUSH LOCAL LOGS') # Stop slaves to avoid the creation of more relay logs. for srv in [self.server2, self.server3]: srv.exec_query('STOP SLAVE') test_num = 1 comment = ("Test case {0}a - move binary logs from running server " "(master).").format(test_num) cmd = "{0} --server={1} {2}".format(cmd_base, master_con, self.master_dir) res = self.run_test_case(0, cmd, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) master_src = self.server1.select_variable('datadir') if self.debug: print("\nServer {0}:{1} source directory (datadir): " "{2}".format(self.server1.host, self.server1.port, master_src)) self.results.append("Test case {0}b - check moved files and changes " "in index file:\n".format(test_num)) self.check_moved_binlogs(self.master_dir, master_src, 'master-bin.index') test_num += 1 comment = ("Test case {0}a - move binary logs from running server " "(slave 1).").format(test_num) cmd = "{0} --server={1} {2}".format(cmd_base, slave1_con, self.slave1_dir) res = self.run_test_case(0, cmd, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) slave1_src = self.server2.select_variable('datadir') if self.debug: print("\nServer {0}:{1} source directory (datadir): " "{2}".format(self.server2.host, self.server2.port, slave1_src)) self.results.append("Test case {0}b - check moved files and changes " "in index file:\n".format(test_num)) self.check_moved_binlogs(self.slave1_dir, slave1_src, 'slave1-bin.index') # Get binary log source directory (datadir) for slave 2. slave2_src = self.server3.select_variable('datadir') if self.debug: print("\nServer {0}:{1} source directory (datadir): " "{2}".format(self.server3.host, self.server3.port, slave2_src)) print("\nStopping server {0}:{1}...".format( self.server3.host, self.server3.port)) # Stop slave 2 (to move binlogs with --source). self.servers.stop_server(self.server3, drop=False) test_num += 1 comment = ("Test case {0}a - move binary logs for stopped server " "(slave 2).").format(test_num) cmd = "{0} --binlog-dir={1} {2}".format(cmd_base, slave2_src, self.slave2_dir) res = self.run_test_case(0, cmd, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) self.results.append("Test case {0}b - check moved files and changes " "in index file:\n".format(test_num)) self.check_moved_binlogs(self.slave2_dir, slave2_src, 'slave2-bin.index') # Clean state data and filesystem for stopped server (slave 2). if self.debug: print("\nCleaning data for server {0}:{1} (already stopped)..." "".format(self.server3.host, self.server3.port)) self.servers.remove_server(self.server3.role) delete_directory(slave2_src) # Mask non-deterministic data. # Warning messages for older MySQL versions (variables not available). self.remove_result("# WARNING: Variable 'log_bin_basename' is not " "available for server ") return True
def test_failover_console(self, test_case, timeout=_TIMEOUT): """Tests failover console. test_case[in] Test case. """ server = test_case[0] cmd = test_case[1] kill_console = test_case[2] log_filename = test_case[3] comment = test_case[4] key_phrase = test_case[5] unregister = test_case[6] server_version = server.get_version() if unregister: # Unregister any failover instance from server try: server.exec_query("DROP TABLE IF EXISTS " "mysql.failover_console") except UtilError: pass # Since this test case expects the console to stop, we can launch it # via a subprocess and wait for it to finish. if self.debug: print(comment) print("# COMMAND: {0}".format(cmd)) # Cleanup in case previous test case failed if os.path.exists(self.failover_dir): try: os.system("rmdir {0}".format(self.failover_dir)) except OSError: pass # Launch the console in stealth mode proc, f_out = self.start_process(cmd) # Wait for console to load if self.debug: print("# Waiting for console to start.") i = 1 time.sleep(1) while proc.poll() is not None: time.sleep(1) i += 1 if i > timeout: if self.debug: print("# Timeout console to start.") raise MUTLibError("{0}: failed - timeout waiting for " "console to start.".format(comment)) # Wait for the failover console to register on master and start # its monitoring process phrase = "Failover console started" if self.debug: print("Waiting for failover console to register master and start " "its monitoring process") # Wait because of the warning message that may appear due to # mixing hostnames and IP addresses time.sleep(WARNING_SLEEP_TIME + 1) i = 0 # Wait for logfile file to be created if self.debug: print("# Waiting for logfile to be created.") for i in range(timeout): if os.path.exists(log_filename): break else: time.sleep(1) else: raise MUTLibError("{0}: failed - timeout waiting for " "logfile '{1}' to be " "created.".format(comment, log_filename)) with open(log_filename, 'r') as file_: while i < timeout: line = file_.readline() if not line: i += 1 time.sleep(1) elif phrase in line: break else: if self.debug: print("# Timeout waiting for failover console to register " "master and start its monitoring process") raise MUTLibError("{0}: failed - timeout waiting for console " "to register master and start its " "monitoring process".format(comment)) # Now, kill the master - wha-ha-ha! res = server.show_server_variable('pid_file') pid_file = open(res[0][1]) pid = int(pid_file.readline().strip('\n')) if self.debug: print("# Terminating server {0} via pid = {1}".format(server.port, pid)) pid_file.close() # Get server datadir to clean directory after kill. res = server.show_server_variable("datadir") datadir = res[0][1] # Stop the server server.disconnect() self.kill(pid) # Need to wait until the process is really dead. if self.debug: print("# Waiting for master to stop.") i = 0 while self.is_process_alive(pid, int(server.port) - 1, int(server.port) + 1): time.sleep(1) i += 1 if i > timeout: if self.debug: print("# Timeout master to fail.") raise MUTLibError("{0}: failed - timeout waiting for " "master to end.".format(comment)) # Remove server from the list (and clean data directory). if self.debug: print("# Removing server name '{0}'.".format(server.role)) delete_directory(datadir) self.servers.remove_server(server.role) # Now wait for interval to occur. if self.debug: print("# Waiting for failover to complete.") i = 0 while not os.path.isdir(self.failover_dir): time.sleep(5) i += 1 if i > timeout: if self.debug: print("# Timeout console failover.") raise MUTLibError("{0}: failed - timeout waiting for " "exec_post_fail.".format(comment)) # Need to poll here and wait for console to really end. ret_val = self.stop_process(proc, f_out, kill_console) # Wait for console to end if self.debug: print("# Waiting for console to end.") i = 0 while proc.poll() is None: time.sleep(1) i += 1 if i > timeout: if self.debug: print("# Timeout console to end.") raise MUTLibError("{0}: failed - timeout waiting for " "console to end.".format(comment)) if self.debug: print("# Return code from console termination = " "{0}".format(ret_val)) # Check result code from stop_process then read the log to find the # key phrase. found_row = False log_file = open(log_filename) rows = log_file.readlines() if self.debug: print("# Looking in log for: {0}".format(key_phrase)) for row in rows: if key_phrase in row: found_row = True if self.debug: print("# Found in row = '{0}'.".format(row[:len(row) - 1])) # Find MySQL Utilities version in the log if self.debug: print("# Looking in log for: {0}" "".format(_UTILITIES_VERSION_PHRASE)) for row in rows: if _UTILITIES_VERSION_PHRASE in row: found_row = True if self.debug: print("# Found in row = '{0}'.".format(row[:-1])) break # Find MySQL server version in the log host_port = "{host}:{port}".format(**get_connection_dictionary(server)) key_phrase = MSG_MYSQL_VERSION.format(server=host_port, version=server_version) if self.debug: print("# Looking in log for: {0}".format(key_phrase)) for row in rows: if key_phrase in row: found_row = True if self.debug: print("# Found in row = '{0}'.".format(row[:-1])) break log_file.close() if not found_row: print("# ERROR: Cannot find entry in log:") for row in rows: print row, # Cleanup after test case try: os.unlink(log_filename) except OSError: pass if os.path.exists(self.failover_dir): try: os.system("rmdir {0}".format(self.failover_dir)) except OSError: pass return comment, found_row
def stop_server(self, server, wait=10, drop=True): """Stop a running server. This method will stop a server using the mysqladmin utility to shutdown the server. It also destroys the datadir. server[in] Server instance to clone wait[in] Number of wait cycles for shutdown default = 10 drop[in] If True, drop datadir Returns - True = server shutdown, False - unknown state or error """ # Nothing to do if server is None if server is None: return True from mysql.utilities.common.tools import delete_directory # Build the shutdown command cmd = "" res = server.show_server_variable("basedir") mysqladmin_client = "mysqladmin" if not os.name == "posix": mysqladmin_client = "mysqladmin.exe" mysqladmin_path = os.path.normpath( os.path.join(res[0][1], "bin", mysqladmin_client)) if not os.path.exists(mysqladmin_path): mysqladmin_path = os.path.normpath( os.path.join(res[0][1], "client", mysqladmin_client)) if not os.path.exists(mysqladmin_path) and not os.name == 'posix': mysqladmin_path = os.path.normpath( os.path.join(res[0][1], "client/debug", mysqladmin_client)) if not os.path.exists(mysqladmin_path) and not os.name == 'posix': mysqladmin_path = os.path.normpath( os.path.join(res[0][1], "client/release", mysqladmin_client)) cmd += mysqladmin_path cmd += " shutdown " cmd += self.get_connection_parameters(server) res = server.show_server_variable("datadir") datadir = res[0][1] # Kill all connections so shutdown will work correctly res = server.exec_query("SHOW PROCESSLIST") for row in res: if not row[7] or not row[7].upper().startswith("SHOW PROCESS"): try: server.exec_query("KILL CONNECTION %s" % row[0]) except UtilDBError: # Ok to ignore KILL failures pass # disconnect user server.disconnect() # Stop the server file = os.devnull f_out = open(file, 'w') proc = subprocess.Popen(cmd, shell=True, stdout=f_out, stderr=f_out) ret_val = proc.wait() f_out.close() # if shutdown doesn't work, exit. if int(ret_val) != 0: return False # If datadir exists, delete it if drop: delete_directory(datadir) if os.path.exists("cmd.txt"): try: os.unlink("cmd.txt") except: pass return True
def run(self): self.res_fname = "result.txt" master_conn = self.build_connection_string(self.server1).strip(' ') slave1_conn = self.build_connection_string(self.server2).strip(' ') slave2_conn = self.build_connection_string(self.server3).strip(' ') master_str = "--master={0}".format(master_conn) failover_cmd = ("python ../scripts/mysqlfailover.py --interval=15 " " --discover-slaves-login=root:root --failover-" "mode={0} --log={1} {2} ") failover_cmd1 = failover_cmd.format("auto", "a" + _FAILOVER_LOG, master_str) failover_cmd2 = failover_cmd.format("auto", "b" + _FAILOVER_LOG, master_str) failover_cmd3 = failover_cmd.format("elect", "c" + _FAILOVER_LOG, master_str) failover_cmd3 += " --candidate={0}".format(slave1_conn) failover_cmd4 = failover_cmd.format("auto", "d" + _FAILOVER_LOG, "--master=" + slave2_conn) # We launch one console, wait for interval, then start another, # wait for interval, then kill both, and finally check log of each # for whether each logged the correct messages for multiple instance # check. interval = 15 test_num = 1 comment = ("Test case {0} : test multiple instances of failover " "console.".format(test_num)) if self.debug: print comment print "# First instance:", failover_cmd1 print "# Second instance:", failover_cmd2 proc1, f_out1 = failover.test.start_process(self, failover_cmd1) self._poll_console(True, "first", proc1, comment) # Now wait for interval to occur. if self.debug: print "# Waiting for interval to end." time.sleep(interval) proc2, f_out2 = failover.test.start_process(self, failover_cmd2) self._poll_console(True, "second", proc2, comment) failover.test.stop_process(self, proc1, f_out1, True) self._poll_console(False, "first", proc1, comment) failover.test.stop_process(self, proc2, f_out2, True) self._poll_console(False, "second", proc1, comment) # Check to see if second console changed modes. found_row = self._check_result("b", "Multiple instances of failover") self.results.append((comment, found_row)) test_num += 1 comment = "Test case {0} : test failed instance restart".format( test_num) if self.debug: print comment print "# Third instance:", failover_cmd3 print "# Fourth instance:", failover_cmd4 # Launch the console in stealth mode proc3, f_out3 = failover.test.start_process(self, failover_cmd3) self._poll_console(True, "third", proc3, comment) # Now, kill the master - wha-ha-ha! res = self.server1.show_server_variable('pid_file') pid_file = open(res[0][1]) pid = int(pid_file.readline().strip('\n')) if self.debug: print "# Terminating server", self.server1.port, "via pid =", pid pid_file.close() # Get server datadir to clean directory after kill. res = self.server1.show_server_variable("datadir") datadir = res[0][1] # Stop the server self.server1.disconnect() failover.test.kill(pid, True) delete_directory(datadir) self.servers.remove_server(self.server1.role) # Now wait for interval to occur. if self.debug: print "# Waiting for interval to end." time.sleep(interval) failover.test.stop_process(self, proc3, f_out3, True) self._poll_console(False, "third", proc3, comment) # Restart the console - should not demote the failover mode. proc4, f_out4 = failover.test.start_process(self, failover_cmd4) self._poll_console(True, "fourth", proc4, comment) failover.test.stop_process(self, proc4, f_out4, True) self._poll_console(False, "fourth", proc4, comment) found_row = self._check_result("d", "Multiple instances of failover") self.results.append((comment, not found_row)) rpl_admin_gtid.test.reset_topology(self) return True
def run(self): cmd_base = "mysqlbinlogmove.py" master_con = self.build_connection_string(self.server1).strip(' ') slave1_con = self.build_connection_string(self.server2).strip(' ') if self.debug: print("\nCreate multiple binary logs on all servers " "(FLUSH LOGS)...") for srv in [self.server1, self.server2, self.server3]: for _ in range(5): srv.exec_query('FLUSH LOCAL LOGS') # Stop slaves to avoid the creation of more relay logs. for srv in [self.server2, self.server3]: srv.exec_query('STOP SLAVE') test_num = 1 comment = ("Test case {0}a - move binary logs from running server " "(master).").format(test_num) cmd = "{0} --server={1} {2}".format(cmd_base, master_con, self.master_dir) res = self.run_test_case(0, cmd, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) master_src = self.server1.select_variable('datadir') if self.debug: print("\nServer {0}:{1} source directory (datadir): " "{2}".format(self.server1.host, self.server1.port, master_src)) self.results.append("Test case {0}b - check moved files and changes " "in index file:\n".format(test_num)) self.check_moved_binlogs(self.master_dir, master_src, 'master-bin.index') test_num += 1 comment = ("Test case {0}a - move binary logs from running server " "(slave 1).").format(test_num) cmd = "{0} --server={1} {2}".format(cmd_base, slave1_con, self.slave1_dir) res = self.run_test_case(0, cmd, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) slave1_src = self.server2.select_variable('datadir') if self.debug: print("\nServer {0}:{1} source directory (datadir): " "{2}".format(self.server2.host, self.server2.port, slave1_src)) self.results.append("Test case {0}b - check moved files and changes " "in index file:\n".format(test_num)) self.check_moved_binlogs(self.slave1_dir, slave1_src, 'slave1-bin.index') # Get binary log source directory (datadir) for slave 2. slave2_src = self.server3.select_variable('datadir') if self.debug: print("\nServer {0}:{1} source directory (datadir): " "{2}".format(self.server3.host, self.server3.port, slave2_src)) print("\nStopping server {0}:{1}...".format(self.server3.host, self.server3.port)) # Stop slave 2 (to move binlogs with --source). self.servers.stop_server(self.server3, drop=False) test_num += 1 comment = ("Test case {0}a - move binary logs for stopped server " "(slave 2).").format(test_num) cmd = "{0} --binlog-dir={1} {2}".format(cmd_base, slave2_src, self.slave2_dir) res = self.run_test_case(0, cmd, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) self.results.append("Test case {0}b - check moved files and changes " "in index file:\n".format(test_num)) self.check_moved_binlogs(self.slave2_dir, slave2_src, 'slave2-bin.index') # Clean state data and filesystem for stopped server (slave 2). if self.debug: print("\nCleaning data for server {0}:{1} (already stopped)..." "".format(self.server3.host, self.server3.port)) self.servers.remove_server(self.server3.role) delete_directory(slave2_src) # Mask non-deterministic data. # Warning messages for older MySQL versions (variables not available). self.remove_result("# WARNING: Variable 'log_bin_basename' is not " "available for server ") return True
def test_failover_console(self, test_case): """Tests failover console. test_case[in] Test case. """ server = test_case[0] cmd = test_case[1] kill_console = test_case[2] log_filename = test_case[3] comment = test_case[4] key_phrase = test_case[5] unregister = test_case[6] server_version = server.get_version() if unregister: # Unregister any failover instance from server try: server.exec_query("DROP TABLE IF EXISTS " "mysql.failover_console") except UtilError: pass # Since this test case expects the console to stop, we can launch it # via a subprocess and wait for it to finish. if self.debug: print(comment) print("# COMMAND: {0}".format(cmd)) # Cleanup in case previous test case failed if os.path.exists(self.failover_dir): try: os.system("rmdir {0}".format(self.failover_dir)) except OSError: pass # Launch the console in stealth mode proc, f_out = self.start_process(cmd) # Wait for console to load if self.debug: print("# Waiting for console to start.") i = 1 time.sleep(1) while proc.poll() is not None: time.sleep(1) i += 1 if i > _TIMEOUT: if self.debug: print("# Timeout console to start.") raise MUTLibError("{0}: failed - timeout waiting for " "console to start.".format(comment)) # Wait for the failover console to register on master and start # its monitoring process phrase = "Failover console started" if self.debug: print("Waiting for failover console to register master and start " "its monitoring process") # Wait because of the warning message that may appear due to # mixing hostnames and IP addresses time.sleep(WARNING_SLEEP_TIME + 1) i = 0 # Wait for logfile file to be created if self.debug: print("# Waiting for logfile to be created.") for i in range(_TIMEOUT): if os.path.exists(log_filename): break else: time.sleep(1) else: raise MUTLibError("{0}: failed - timeout waiting for " "logfile '{1}' to be " "created.".format(comment, log_filename)) with open(log_filename, 'r') as file_: while i < _TIMEOUT: line = file_.readline() if not line: i += 1 time.sleep(1) elif phrase in line: break else: if self.debug: print("# Timeout waiting for failover console to register " "master and start its monitoring process") raise MUTLibError("{0}: failed - timeout waiting for console " "to register master and start its " "monitoring process".format(comment)) # Now, kill the master - wha-ha-ha! res = server.show_server_variable('pid_file') pid_file = open(res[0][1]) pid = int(pid_file.readline().strip('\n')) if self.debug: print("# Terminating server {0} via pid = {1}".format(server.port, pid)) pid_file.close() # Get server datadir to clean directory after kill. res = server.show_server_variable("datadir") datadir = res[0][1] # Stop the server server.disconnect() self.kill(pid) # Need to wait until the process is really dead. if self.debug: print("# Waiting for master to stop.") i = 0 while self.is_process_alive(pid, int(server.port) - 1, int(server.port) + 1): time.sleep(1) i += 1 if i > _TIMEOUT: if self.debug: print("# Timeout master to fail.") raise MUTLibError("{0}: failed - timeout waiting for " "master to end.".format(comment)) # Remove server from the list (and clean data directory). if self.debug: print("# Removing server name '{0}'.".format(server.role)) delete_directory(datadir) self.servers.remove_server(server.role) # Now wait for interval to occur. if self.debug: print("# Waiting for failover to complete.") i = 0 while not os.path.isdir(self.failover_dir): time.sleep(5) i += 1 if i > _TIMEOUT: if self.debug: print("# Timeout console failover.") raise MUTLibError("{0}: failed - timeout waiting for " "exec_post_fail.".format(comment)) # Need to poll here and wait for console to really end. ret_val = self.stop_process(proc, f_out, kill_console) # Wait for console to end if self.debug: print("# Waiting for console to end.") i = 0 while proc.poll() is None: time.sleep(1) i += 1 if i > _TIMEOUT: if self.debug: print("# Timeout console to end.") raise MUTLibError("{0}: failed - timeout waiting for " "console to end.".format(comment)) if self.debug: print("# Return code from console termination = " "{0}".format(ret_val)) # Check result code from stop_process then read the log to find the # key phrase. found_row = False log_file = open(log_filename) rows = log_file.readlines() if self.debug: print("# Looking in log for: {0}".format(key_phrase)) for row in rows: if key_phrase in row: found_row = True if self.debug: print("# Found in row = '{0}'.".format(row[:len(row) - 1])) # Find MySQL Utilities version in the log if self.debug: print("# Looking in log for: {0}" "".format(_UTILITIES_VERSION_PHRASE)) for row in rows: if _UTILITIES_VERSION_PHRASE in row: found_row = True if self.debug: print("# Found in row = '{0}'.".format(row[:-1])) break # Find MySQL server version in the log host_port = "{host}:{port}".format(**get_connection_dictionary(server)) key_phrase = MSG_MYSQL_VERSION.format(server=host_port, version=server_version) if self.debug: print("# Looking in log for: {0}".format(key_phrase)) for row in rows: if key_phrase in row: found_row = True if self.debug: print("# Found in row = '{0}'.".format(row[:-1])) break log_file.close() if not found_row: print("# ERROR: Cannot find entry in log:") for row in rows: print row, # Cleanup after test case try: os.unlink(log_filename) except OSError: pass if os.path.exists(self.failover_dir): try: os.system("rmdir {0}".format(self.failover_dir)) except OSError: pass return comment, found_row
def stop_server(self, server, wait=10, drop=True): """Stop a running server. This method will stop a server using the mysqladmin utility to shutdown the server. It also destroys the datadir. server[in] Server instance to clone wait[in] Number of wait cycles for shutdown default = 10 drop[in] If True, drop datadir Returns - True = server shutdown, False - unknown state or error """ # Nothing to do if server is None if server is None: return True from mysql.utilities.common.tools import delete_directory # Build the shutdown command cmd = "" res = server.show_server_variable("basedir") mysqladmin_client = "mysqladmin" if not os.name == "posix": mysqladmin_client = "mysqladmin.exe" mysqladmin_path= os.path.normpath(os.path.join(res[0][1], "bin", mysqladmin_client)) if not os.path.exists(mysqladmin_path): mysqladmin_path= os.path.normpath(os.path.join(res[0][1], "client", mysqladmin_client)) if not os.path.exists(mysqladmin_path) and not os.name == 'posix': mysqladmin_path= os.path.normpath(os.path.join(res[0][1], "client/debug", mysqladmin_client)) if not os.path.exists(mysqladmin_path) and not os.name == 'posix': mysqladmin_path= os.path.normpath(os.path.join(res[0][1], "client/release", mysqladmin_client)) cmd += mysqladmin_path cmd += " shutdown " cmd += self.get_connection_parameters(server) res = server.show_server_variable("datadir") datadir = res[0][1] # Kill all connections so shutdown will work correctly res = server.exec_query("SHOW PROCESSLIST") for row in res: if not row[7] or not row[7].upper().startswith("SHOW PROCESS"): try: server.exec_query("KILL CONNECTION %s" % row[0]) except UtilDBError: # Ok to ignore KILL failures pass # disconnect user server.disconnect() # Stop the server file = os.devnull f_out = open(file, 'w') proc = subprocess.Popen(cmd, shell=True, stdout = f_out, stderr = f_out) ret_val = proc.wait() f_out.close() # if shutdown doesn't work, exit. if int(ret_val) != 0: return False # If datadir exists, delete it if drop: delete_directory(datadir) if os.path.exists("cmd.txt"): try: os.unlink("cmd.txt") except: pass return True