示例#1
0
    def test_failover_console(self, test_case, timeout=_TIMEOUT):
        """Tests failover console.

        test_case[in]     Test case.
        """
        server = test_case[0]
        cmd = test_case[1]
        kill_console = test_case[2]
        log_filename = test_case[3]
        comment = test_case[4]
        key_phrase = test_case[5]
        unregister = test_case[6]
        server_version = server.get_version()

        if unregister:
            # Unregister any failover instance from server
            try:
                server.exec_query("DROP TABLE IF EXISTS "
                                  "mysql.failover_console")
            except UtilError:
                pass

        # Since this test case expects the console to stop, we can launch it
        # via a subprocess and wait for it to finish.
        if self.debug:
            print(comment)
            print("# COMMAND: {0}".format(cmd))

        # Cleanup in case previous test case failed
        if os.path.exists(self.failover_dir):
            try:
                os.system("rmdir {0}".format(self.failover_dir))
            except OSError:
                pass

        # Launch the console in stealth mode
        proc, f_out = self.start_process(cmd)

        # Wait for console to load
        if self.debug:
            print("# Waiting for console to start.")
        i = 1
        time.sleep(1)
        while proc.poll() is not None:
            time.sleep(1)
            i += 1
            if i > timeout:
                if self.debug:
                    print("# Timeout console to start.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "console to start.".format(comment))

        # Wait for the failover console to register on master and start
        # its monitoring process

        phrase = "Failover console started"
        if self.debug:
            print("Waiting for failover console to register master and start "
                  "its monitoring process")
        # Wait because of the warning message that may appear due to
        # mixing hostnames and IP addresses
        time.sleep(WARNING_SLEEP_TIME + 1)
        i = 0

        # Wait for logfile file to be created
        if self.debug:
            print("# Waiting for logfile to be created.")
        for i in range(timeout):
            if os.path.exists(log_filename):
                break
            else:
                time.sleep(1)
        else:
            raise MUTLibError("{0}: failed - timeout waiting for "
                              "logfile '{1}' to be "
                              "created.".format(comment, log_filename))

        with open(log_filename, 'r') as file_:
            while i < timeout:
                line = file_.readline()
                if not line:
                    i += 1
                    time.sleep(1)
                elif phrase in line:
                    break
            else:
                if self.debug:
                    print("# Timeout waiting for failover console to register "
                          "master and start its monitoring process")
                raise MUTLibError("{0}: failed - timeout waiting for console "
                                  "to register master and start its "
                                  "monitoring process".format(comment))

        # Now, kill the master - wha-ha-ha!
        res = server.show_server_variable('pid_file')
        pid_file = open(res[0][1])
        pid = int(pid_file.readline().strip('\n'))
        if self.debug:
            print("# Terminating server {0} via pid = {1}".format(server.port,
                                                                  pid))
        pid_file.close()

        # Get server datadir to clean directory after kill.
        res = server.show_server_variable("datadir")
        datadir = res[0][1]

        # Stop the server
        server.disconnect()
        self.kill(pid)

        # Need to wait until the process is really dead.
        if self.debug:
            print("# Waiting for master to stop.")
        i = 0
        while self.is_process_alive(pid, int(server.port) - 1,
                                    int(server.port) + 1):
            time.sleep(1)
            i += 1
            if i > timeout:
                if self.debug:
                    print("# Timeout master to fail.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "master to end.".format(comment))

        # Remove server from the list (and clean data directory).
        if self.debug:
            print("# Removing server name '{0}'.".format(server.role))
        delete_directory(datadir)
        self.servers.remove_server(server.role)

        # Now wait for interval to occur.
        if self.debug:
            print("# Waiting for failover to complete.")
        i = 0
        while not os.path.isdir(self.failover_dir):
            time.sleep(5)
            i += 1
            if i > timeout:
                if self.debug:
                    print("# Timeout console failover.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "exec_post_fail.".format(comment))

        # Need to poll here and wait for console to really end.
        ret_val = self.stop_process(proc, f_out, kill_console)
        # Wait for console to end
        if self.debug:
            print("# Waiting for console to end.")
        i = 0
        while proc.poll() is None:
            time.sleep(1)
            i += 1
            if i > timeout:
                if self.debug:
                    print("# Timeout console to end.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "console to end.".format(comment))

        if self.debug:
            print("# Return code from console termination = "
                  "{0}".format(ret_val))

        # Check result code from stop_process then read the log to find the
        # key phrase.
        found_row = False
        log_file = open(log_filename)
        rows = log_file.readlines()
        if self.debug:
            print("# Looking in log for: {0}".format(key_phrase))
        for row in rows:
            if key_phrase in row:
                found_row = True
                if self.debug:
                    print("# Found in row = '{0}'.".format(row[:len(row) - 1]))

        # Find MySQL Utilities version in the log
        if self.debug:
            print("# Looking in log for: {0}"
                  "".format(_UTILITIES_VERSION_PHRASE))
        for row in rows:
            if _UTILITIES_VERSION_PHRASE in row:
                found_row = True
                if self.debug:
                    print("# Found in row = '{0}'.".format(row[:-1]))
                break

        # Find MySQL server version in the log
        host_port = "{host}:{port}".format(**get_connection_dictionary(server))
        key_phrase = MSG_MYSQL_VERSION.format(server=host_port,
                                              version=server_version)
        if self.debug:
            print("# Looking in log for: {0}".format(key_phrase))
        for row in rows:
            if key_phrase in row:
                found_row = True
                if self.debug:
                    print("# Found in row = '{0}'.".format(row[:-1]))
                break

        log_file.close()

        if not found_row:
            print("# ERROR: Cannot find entry in log:")
            for row in rows:
                print row,

        # Cleanup after test case
        try:
            os.unlink(log_filename)
        except OSError:
            pass

        if os.path.exists(self.failover_dir):
            try:
                os.system("rmdir {0}".format(self.failover_dir))
            except OSError:
                pass

        return comment, found_row
    def run(self):
        master_conn = self.build_connection_string(self.server1).strip(' ')
        slave_conn = self.build_connection_string(self.server2).strip(' ')

        # For this test, it's OK when master and slave are the same
        master_str = "--master={0}".format(master_conn)
        slave_str = "--slave={0}".format(slave_conn)

        # command used in test cases: replace 3 element with location of
        # log file.
        cmd = [
            "mysqlrpladmin.py",
            master_str,
            slave_str,
            "--log={0}".format(_LOGNAME),
            "health",
        ]

        # Test Case 1
        test_num = 1
        comment = "Test Case {0} - Log file is newly created".format(test_num)
        res = mutlib.System_test.run_test_case(self, 0, ' '.join(cmd), comment)
        if not res:
            raise MUTLibError("{0}: failed".format(comment))

        # Test Case 2
        test_num += 1
        comment = "Test Case {0} - Log file is reopened".format(test_num)
        res = mutlib.System_test.run_test_case(self, 0, ' '.join(cmd), comment)
        if not res:
            raise MUTLibError("{0}: failed".format(comment))

        # Test Case 3
        test_num += 1
        comment = ("Test Case {0} - Log file can not be "
                   "written to".format(test_num))
        os.chmod(_LOGNAME, stat.S_IREAD)  # Make log read-only
        res = mutlib.System_test.run_test_case(self, 2, ' '.join(cmd), comment)
        if not res:
            raise MUTLibError("{0}: failed".format(comment))

        # Find MySQL and Utilities versions in the log
        self.find_stop_phrase(
            _LOGNAME, comment,
            MSG_UTILITIES_VERSION.format(utility="mysqlrpladmin",
                                         version=VERSION_STRING))

        # Find master MySQL server version in the log
        master_host_port = ("{host}:{port}".format(
            **get_connection_dictionary(master_conn)))
        self.find_stop_phrase(
            _LOGNAME, comment,
            MSG_MYSQL_VERSION.format(server=master_host_port,
                                     version=self.server1.get_version()))

        # Find slave MySQL server version in the log
        slave_host_port = ("{host}:{port}".format(
            **get_connection_dictionary(slave_conn)))
        self.find_stop_phrase(
            _LOGNAME, comment,
            MSG_MYSQL_VERSION.format(server=slave_host_port,
                                     version=self.server2.get_version()))

        # Mask out non-deterministic data
        rpl_admin.test.do_masks(self)
        self.remove_result("NOTE: Log file 'temp_log.txt' does not exist. "
                           "Will be created.")

        return True
    def run(self):
        master_conn = self.build_connection_string(self.server1).strip(' ')
        slave_conn = self.build_connection_string(self.server2).strip(' ')

        # For this test, it's OK when master and slave are the same
        master_str = "--master={0}".format(master_conn)
        slave_str = "--slave={0}".format(slave_conn)

        # command used in test cases: replace 3 element with location of
        # log file.
        cmd = [
            "mysqlrpladmin.py",
            master_str,
            slave_str,
            "--log={0}".format(_LOGNAME),
            "health",
        ]

        # Test Case 1
        test_num = 1
        comment = "Test Case {0} - Log file is newly created".format(test_num)
        res = mutlib.System_test.run_test_case(
            self, 0, ' '.join(cmd), comment)
        if not res:
            raise MUTLibError("{0}: failed".format(comment))

        # Test Case 2
        test_num += 1
        comment = "Test Case {0} - Log file is reopened".format(test_num)
        res = mutlib.System_test.run_test_case(
            self, 0, ' '.join(cmd), comment)
        if not res:
            raise MUTLibError("{0}: failed".format(comment))

        # Test Case 3
        test_num += 1
        comment = ("Test Case {0} - Log file can not be "
                   "written to".format(test_num))
        os.chmod(_LOGNAME, stat.S_IREAD)  # Make log read-only
        res = mutlib.System_test.run_test_case(
            self, 2, ' '.join(cmd), comment)
        if not res:
            raise MUTLibError("{0}: failed".format(comment))

        # Find MySQL and Utilities versions in the log
        self.find_stop_phrase(_LOGNAME, comment, MSG_UTILITIES_VERSION.format(
            utility="mysqlrpladmin", version=VERSION_STRING))

        # Find master MySQL server version in the log
        master_host_port = (
            "{host}:{port}".format(**get_connection_dictionary(master_conn)))
        self.find_stop_phrase(_LOGNAME, comment, MSG_MYSQL_VERSION.format(
            server=master_host_port, version=self.server1.get_version()))

        # Find slave MySQL server version in the log
        slave_host_port = (
            "{host}:{port}".format(**get_connection_dictionary(slave_conn)))
        self.find_stop_phrase(_LOGNAME, comment, MSG_MYSQL_VERSION.format(
            server=slave_host_port, version=self.server2.get_version()))

        # Mask out non-deterministic data
        rpl_admin.test.do_masks(self)
        self.remove_result("NOTE: Log file 'temp_log.txt' does not exist. "
                           "Will be created.")

        return True
示例#4
0
    def test_rplms(self, cmd, logfile, comment, kill_process=True,
                   stop_phrase=None):
        """Test multi-source replication.

        cmd[in]           Command to be executed.
        logfile[in]       Log filename.
        comment[in]       Test comment.
        kill_process[in]  True if the process is to be killed.
        stop_phrase[in]   Stop phrase to be searched in the log.

        This method create a process by executing the command and waits for
        the round-robin scheduling to switch all the masters. At the end
        compares the databases.
        """
        # Since this test case expects the process to stop, we can launch it
        # via a subprocess and wait for it to finish.
        if self.debug:
            print(comment)
            print("# COMMAND: {0}".format(cmd))

        # Run command
        proc, f_out = self.start_process(cmd)

        # Wait for process to load
        if self.debug:
            print("# Waiting for process to start.")
        i = 1
        time.sleep(1)
        while proc.poll() is not None:
            time.sleep(1)
            i += 1
            if i > _TIMEOUT:
                if self.debug:
                    print("# Timeout process to start.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "process to start.".format(comment))

        # Wait for logfile file to be created
        if self.debug:
            print("# Waiting for logfile to be created.")
        for i in range(_TIMEOUT):
            if os.path.exists(logfile):
                break
            else:
                time.sleep(1)
        else:
            raise MUTLibError("{0}: failed - timeout waiting for "
                              "logfile '{1}' to be "
                              "created.".format(comment, logfile))

        # Wait for switching all the masters
        self.wait_for_switching_all_masters(logfile, comment)

        # Compare databases
        self.compare_databases(comment)

        # Kill process
        if kill_process:
            self.kill_process(proc, f_out, comment)

        # Find stop phrase
        if stop_phrase:
            self.find_stop_phrase(logfile, comment, stop_phrase)

        # Find MySQL Utilities version in the log
        utils_phrase = MSG_UTILITIES_VERSION.format(utility="mysqlrplms",
                                                    version=VERSION_STRING)
        self.find_stop_phrase(logfile, comment, utils_phrase)

        # Find MySQL servers versions in the log
        for server in (self.server1, self.server2, self.server3,):
            host_port = "{host}:{port}".format(
                **get_connection_dictionary(server))
            server_version = server.get_version()
            mysql_phrase = MSG_MYSQL_VERSION.format(server=host_port,
                                                    version=server_version)
            self.find_stop_phrase(logfile, comment, mysql_phrase)

        # Cleanup after test case
        try:
            os.unlink(logfile)
        except OSError:
            pass
示例#5
0
    def test_rplms(self, cmd, logfile, comment, kill_process=True,
                   stop_phrase=None):
        """Test multi-source replication.

        cmd[in]           Command to be executed.
        logfile[in]       Log filename.
        comment[in]       Test comment.
        kill_process[in]  True if the process is to be killed.
        stop_phrase[in]   Stop phrase to be searched in the log.

        This method create a process by executing the command and waits for
        the round-robin scheduling to switch all the masters. At the end
        compares the databases.
        """
        # Since this test case expects the process to stop, we can launch it
        # via a subprocess and wait for it to finish.
        if self.debug:
            print(comment)
            print("# COMMAND: {0}".format(cmd))

        # Run command
        proc, f_out = self.start_process(cmd)

        # Wait for process to load
        if self.debug:
            print("# Waiting for process to start.")
        i = 1
        time.sleep(1)
        while proc.poll() is not None:
            time.sleep(1)
            i += 1
            if i > _TIMEOUT:
                if self.debug:
                    print("# Timeout process to start.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "process to start.".format(comment))

        # Wait for logfile file to be created
        if self.debug:
            print("# Waiting for logfile to be created.")
        for i in range(_TIMEOUT):
            if os.path.exists(logfile):
                break
            else:
                time.sleep(1)
        else:
            raise MUTLibError("{0}: failed - timeout waiting for "
                              "logfile '{1}' to be "
                              "created.".format(comment, logfile))

        # Wait for switching all the masters
        self.wait_for_switching_all_masters(logfile, comment)

        # Compare databases
        self.compare_databases(comment)

        # Kill process
        if kill_process:
            self.kill_process(proc, f_out, comment)

        # Find stop phrase
        if stop_phrase:
            self.find_stop_phrase(logfile, comment, stop_phrase)

        # Find MySQL Utilities version in the log
        utils_phrase = MSG_UTILITIES_VERSION.format(utility="mysqlrplms",
                                                    version=VERSION_STRING)
        self.find_stop_phrase(logfile, comment, utils_phrase)

        # Find MySQL servers versions in the log
        for server in (self.server1, self.server2, self.server3,):
            host_port = "{host}:{port}".format(
                **get_connection_dictionary(server))
            server_version = server.get_version()
            mysql_phrase = MSG_MYSQL_VERSION.format(server=host_port,
                                                    version=server_version)
            self.find_stop_phrase(logfile, comment, mysql_phrase)

        # Cleanup after test case
        try:
            os.unlink(logfile)
        except OSError:
            pass
示例#6
0
    def test_failover_console(self, test_case):
        """Tests failover console.

        test_case[in]     Test case.
        """
        server = test_case[0]
        cmd = test_case[1]
        kill_console = test_case[2]
        log_filename = test_case[3]
        comment = test_case[4]
        key_phrase = test_case[5]
        unregister = test_case[6]
        server_version = server.get_version()

        if unregister:
            # Unregister any failover instance from server
            try:
                server.exec_query("DROP TABLE IF EXISTS "
                                  "mysql.failover_console")
            except UtilError:
                pass

        # Since this test case expects the console to stop, we can launch it
        # via a subprocess and wait for it to finish.
        if self.debug:
            print(comment)
            print("# COMMAND: {0}".format(cmd))

        # Cleanup in case previous test case failed
        if os.path.exists(self.failover_dir):
            try:
                os.system("rmdir {0}".format(self.failover_dir))
            except OSError:
                pass

        # Launch the console in stealth mode
        proc, f_out = self.start_process(cmd)

        # Wait for console to load
        if self.debug:
            print("# Waiting for console to start.")
        i = 1
        time.sleep(1)
        while proc.poll() is not None:
            time.sleep(1)
            i += 1
            if i > _TIMEOUT:
                if self.debug:
                    print("# Timeout console to start.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "console to start.".format(comment))

        # Wait for the failover console to register on master and start
        # its monitoring process

        phrase = "Failover console started"
        if self.debug:
            print("Waiting for failover console to register master and start "
                  "its monitoring process")
        # Wait because of the warning message that may appear due to
        # mixing hostnames and IP addresses
        time.sleep(WARNING_SLEEP_TIME + 1)
        i = 0

        # Wait for logfile file to be created
        if self.debug:
            print("# Waiting for logfile to be created.")
        for i in range(_TIMEOUT):
            if os.path.exists(log_filename):
                break
            else:
                time.sleep(1)
        else:
            raise MUTLibError("{0}: failed - timeout waiting for "
                              "logfile '{1}' to be "
                              "created.".format(comment, log_filename))

        with open(log_filename, 'r') as file_:
            while i < _TIMEOUT:
                line = file_.readline()
                if not line:
                    i += 1
                    time.sleep(1)
                elif phrase in line:
                    break
            else:
                if self.debug:
                    print("# Timeout waiting for failover console to register "
                          "master and start its monitoring process")
                raise MUTLibError("{0}: failed - timeout waiting for console "
                                  "to register master and start its "
                                  "monitoring process".format(comment))

        # Now, kill the master - wha-ha-ha!
        res = server.show_server_variable('pid_file')
        pid_file = open(res[0][1])
        pid = int(pid_file.readline().strip('\n'))
        if self.debug:
            print("# Terminating server {0} via pid = {1}".format(server.port,
                                                                  pid))
        pid_file.close()

        # Get server datadir to clean directory after kill.
        res = server.show_server_variable("datadir")
        datadir = res[0][1]

        # Stop the server
        server.disconnect()
        self.kill(pid)

        # Need to wait until the process is really dead.
        if self.debug:
            print("# Waiting for master to stop.")
        i = 0
        while self.is_process_alive(pid, int(server.port) - 1,
                                    int(server.port) + 1):
            time.sleep(1)
            i += 1
            if i > _TIMEOUT:
                if self.debug:
                    print("# Timeout master to fail.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "master to end.".format(comment))

        # Remove server from the list (and clean data directory).
        if self.debug:
            print("# Removing server name '{0}'.".format(server.role))
        delete_directory(datadir)
        self.servers.remove_server(server.role)

        # Now wait for interval to occur.
        if self.debug:
            print("# Waiting for failover to complete.")
        i = 0
        while not os.path.isdir(self.failover_dir):
            time.sleep(5)
            i += 1
            if i > _TIMEOUT:
                if self.debug:
                    print("# Timeout console failover.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "exec_post_fail.".format(comment))

        # Need to poll here and wait for console to really end.
        ret_val = self.stop_process(proc, f_out, kill_console)
        # Wait for console to end
        if self.debug:
            print("# Waiting for console to end.")
        i = 0
        while proc.poll() is None:
            time.sleep(1)
            i += 1
            if i > _TIMEOUT:
                if self.debug:
                    print("# Timeout console to end.")
                raise MUTLibError("{0}: failed - timeout waiting for "
                                  "console to end.".format(comment))

        if self.debug:
            print("# Return code from console termination = "
                  "{0}".format(ret_val))

        # Check result code from stop_process then read the log to find the
        # key phrase.
        found_row = False
        log_file = open(log_filename)
        rows = log_file.readlines()
        if self.debug:
            print("# Looking in log for: {0}".format(key_phrase))
        for row in rows:
            if key_phrase in row:
                found_row = True
                if self.debug:
                    print("# Found in row = '{0}'.".format(row[:len(row) - 1]))

        # Find MySQL Utilities version in the log
        if self.debug:
            print("# Looking in log for: {0}"
                  "".format(_UTILITIES_VERSION_PHRASE))
        for row in rows:
            if _UTILITIES_VERSION_PHRASE in row:
                found_row = True
                if self.debug:
                    print("# Found in row = '{0}'.".format(row[:-1]))
                break

        # Find MySQL server version in the log
        host_port = "{host}:{port}".format(**get_connection_dictionary(server))
        key_phrase = MSG_MYSQL_VERSION.format(server=host_port,
                                              version=server_version)
        if self.debug:
            print("# Looking in log for: {0}".format(key_phrase))
        for row in rows:
            if key_phrase in row:
                found_row = True
                if self.debug:
                    print("# Found in row = '{0}'.".format(row[:-1]))
                break

        log_file.close()

        if not found_row:
            print("# ERROR: Cannot find entry in log:")
            for row in rows:
                print row,

        # Cleanup after test case
        try:
            os.unlink(log_filename)
        except OSError:
            pass

        if os.path.exists(self.failover_dir):
            try:
                os.system("rmdir {0}".format(self.failover_dir))
            except OSError:
                pass

        return comment, found_row