def setUp(self): """ Setup server connections (for all tests) . """ self.source_cnx = {'conn_info': self.options[SERVER_CNX_OPT][0]} self.destination_cnx = {'conn_info': self.options[SERVER_CNX_OPT][1]} self.source = server.Server(self.source_cnx) self.destination = server.Server(self.destination_cnx) self.source.connect() self.destination.connect() try: self.source_gtids_enabled = self.source.supports_gtid() except exceptions.GadgetServerError: self.source_gtids_enabled = False try: self.destination_gtids_enabled = self.destination.supports_gtid() except exceptions.GadgetServerError: self.destination_gtids_enabled = False if self.destination_gtids_enabled: self.destination_has_no_gtid_executed = not \ self.destination.get_gtid_executed(skip_gtid_check=False) else: self.destination_has_no_gtid_executed = False self.mysqldump_exec = get_tool_path(None, "mysqldump", search_path=True, required=False) self.mysql_exec = get_tool_path(None, "mysql", search_path=True, required=False) # Search for mysqldump only on path (ignore MySQL default paths). self.mysqldump_exec_in_defaults = get_tool_path(None, "mysqldump", search_path=False, required=False) # setup source server with a sample database and users data_sql_path = os.path.normpath( os.path.join(__file__, "..", "std_data", "basic_data.sql")) self.source.read_and_exec_sql(data_sql_path, verbose=True) # Create user without super permission on both source and destination self.source.exec_query("create user no_super@'%'") self.source.exec_query("GRANT ALL on *.* to no_super@'%'") self.source.exec_query("REVOKE SUPER on *.* from no_super@'%'") self.destination.exec_query("create user no_super@'%'") self.destination.exec_query("GRANT ALL on *.* to no_super@'%'") self.destination.exec_query("REVOKE SUPER on *.* from no_super@'%'") if self.destination_has_no_gtid_executed: self.destination.exec_query("RESET MASTER")
def test_get_server_version(self): """Tests the get_server_version method.""" # get version from the server version_from_server = self.server.get_version() # try to find the executable path via basedir basedir = self.server.select_variable("basedir", "global") try: mysqld_path = tools.get_tool_path(basedir, "mysqld", required=True) except GadgetError: raise unittest.SkipTest("Couldn't find mysqld executable") else: version_from_binary = server.get_mysqld_version(mysqld_path) self.assertEqual(tuple(version_from_server), version_from_binary[0]) self.assertIn(".".join(str(i) for i in version_from_binary[0]), version_from_binary[1])
def test_get_server_version(self): """Tests the get_server_version method.""" # get version from the server version_from_server = self.server.get_version() # try to find the executable path via basedir basedir = self.server.select_variable("basedir", "global") try: mysqld_path = tools.get_tool_path(basedir, "mysqld", required=True) except GadgetError: raise unittest.SkipTest("Couldn't find mysqld executable") else: version_from_binary = server.get_mysqld_version(mysqld_path) self.assertEqual(tuple(version_from_server), version_from_binary[0]) self.assertIn(".".join(str(i) for i in version_from_binary[0]), version_from_binary[1])
def test_is_executable(self): """Test is_executable function""" # python is an executable file try: python_path = tools.get_tool_path(None, "python", required=True, search_path=True) except GadgetError: raise unittest.SkipTest("Couldn't find python executable.") self.assertTrue(tools.is_executable(python_path)) # a simple file is not executable, list this test file is not # executable if os.name == "posix": # on windows the test file is deemed as executable self.assertFalse(tools.is_executable(__file__)) # A file that does not exist is also not executable self.assertFalse(tools.is_executable(__file__ + "aa"))
def test_run_subprocess(self): """Test run_subprocess function""" # Run a simple echo command try: echo_path = tools.get_tool_path(None, "echo", required=True, search_path=True) except GadgetError: raise unittest.SkipTest("Couldn't find echo executable.") p = tools.run_subprocess("{0} output".format(echo_path), stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = p.communicate() ret_code = p.returncode # process ran ok self.assertEqual(ret_code, 0) self.assertEqual(out, "output\n") self.assertEqual("", err)
def test_get_tool_path(self): """ Test get_tool_path function. """ # Get server basedir. basedir = self.server.exec_query( "SHOW VARIABLES LIKE '%basedir%'")[0][1] # Find 'mysqld' using default option. res = tools.get_tool_path(basedir, 'mysqld') self.assertIn("mysqld", res, "The 'mysqld' file is expected to be found in basedir.") # Find 'mysqld' and return result with quotes. res = tools.get_tool_path(basedir, 'mysqld', quote=True) self.assertIn("mysqld", res, "The 'mysqld' file is expected to be found in basedir.") quote_char = "'" if os.name == "posix" else '"' self.assertEqual(res[0], quote_char, "Result expected to be quoted with " "({0}).".format(quote_char)) # Find 'mysqld' using the default path list. bindir = os.path.join(basedir, 'bin') res = tools.get_tool_path('.', 'mysqld', defaults_paths=[bindir]) self.assertIn("mysqld", res, "The 'mysqld' file is expected to be found in" "defaults_paths.") # Find 'mysqld' using a check tool function. res = tools.get_tool_path(basedir, 'mysqld', check_tool_func=server.is_valid_mysqld) self.assertIn("mysqld", res, "The 'mysqld' file is expected to be found in basedir.") # Find 'mysqld' using a check tool function and the default path list. bindir = os.path.join(basedir, 'bin') res = tools.get_tool_path('.', 'mysqld', defaults_paths=[bindir], check_tool_func=server.is_valid_mysqld) self.assertIn("mysqld", res, "The 'mysqld' file is expected to be found in" "defaults_paths.") # Find 'mysqld' using the PATH variable. path_var = os.environ['PATH'] os.environ['PATH'] = bindir try: res = tools.get_tool_path('.', 'mysqld', search_path=True) finally: # Make sure the PATH variable is restored. os.environ['PATH'] = path_var self.assertIn("mysqld", res, "The 'mysqld' file is expected to be found in PATH.") # Find 'mysqld' using a check tool function and the PATH variable. path_var = os.environ['PATH'] os.environ['PATH'] = bindir try: res = tools.get_tool_path('.', 'mysqld', search_path=True, check_tool_func=server.is_valid_mysqld) finally: # Make sure the PATH variable is restored. os.environ['PATH'] = path_var self.assertIn("mysqld", res, "The 'mysqld' file is expected to be found in PATH.") # Try to find a non existing tool (error raised by default). with self.assertRaises(GadgetError) as cm: tools.get_tool_path(basedir, 'non_existing_tool') self.assertEqual(cm.exception.errno, 1) # Cannot find a valid tool, based on check function (raise error) # Note: Use a lambda function that always return false. with self.assertRaises(GadgetError) as cm: tools.get_tool_path(basedir, 'mysqld', check_tool_func=lambda path: False) self.assertEqual(cm.exception.errno, 2) # Try to find a non existing tool with required set to False. res = tools.get_tool_path(basedir, 'non_existing_tool', required=False) self.assertIsNone(res, "None expected when trying to find" "'non_existing_tool'")
def clone_stream(connection_dict): """Clone the contents of source server into destination using a stream. :param connection_dict: dictionary of dictionaries of connection information: mysql users and host users. It can have the following keys: MYSQL_SOURCE, MYSQL_DEST, HOST_SOURCE and HOST_DEST. Each of these keys has as a value a dict with the following keys: user, hostname, port, passwd and their respective values. :type connection_dict: dict """ # Check tool requirements mysqldump_exe = get_tool_path(None, "mysqldump", search_path=True, required=False) if not mysqldump_exe: raise exceptions.GadgetError( "Could not find mysqldump executable. Make sure it is on " "{0}.".format(PATH_ENV_VAR)) mysqlc_exe = get_tool_path(None, "mysql", search_path=True, required=False) if not mysqlc_exe: raise exceptions.GadgetError( "Could not find mysql client executable. Make sure it is on " "{0}.".format(PATH_ENV_VAR)) # Creating Server instances for source and destination servers source_dict = connection_dict[MYSQL_SOURCE] destination_dict = connection_dict[MYSQL_DEST] try: source_server = Server({'conn_info': source_dict}) except exceptions.GadgetError as err: _LOGGER.error( "Unable to create a Server instance for source server." "Source dict was: %s", source_dict) raise err try: destination_server = Server({'conn_info': destination_dict}) except exceptions.GadgetError as err: _LOGGER.error( "Unable to create a Server instance for destination " "server. Destination dict was: %s", destination_dict) raise err # Connect to source and destination servers try: source_server.connect() except exceptions.GadgetServerError as err: raise exceptions.GadgetError( "Unable to connect to source server: {0}".format(str(err))) try: destination_server.connect() except exceptions.GadgetServerError as err: raise exceptions.GadgetError("Unable to connect to destination " "server: {0}.".format(str(err))) # Create config_file for mysqldump dump_config_file = Server.to_config_file(source_server, "mysqldump") # Create config_file for mysql client client_config_file = Server.to_config_file(destination_server, "client") # Create command list to create the backup backup_cmd = shlex.split( _MYSQLDUMP_STREAM_BACKUP_CMD.format(mysqldump_exec=mysqldump_exe, config_file=dump_config_file, quote=QUOTE_CHAR)) # Create command list to restore the backup restore_cmd = shlex.split( _MYSQLDUMP_STREAM_RESTORE_CMD.format( mysqlc_exec=mysqlc_exe, config_file=client_config_file, quote=QUOTE_CHAR)) # enable global read_lock _LOGGER.debug("Locking global read lock on source server to prevent " "modifications during clone.") source_server.toggle_global_read_lock(True) _LOGGER.debug("Source server locked (read-only=ON)") try: _LOGGER.debug( "Dumping contents of source server using command: " "%s", " ".join(backup_cmd)) dump_process = subprocess.Popen(backup_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) _LOGGER.debug( "Restoring contents to destination server using " "command: %s", " ".join(restore_cmd)) restore_process = subprocess.Popen(restore_cmd, stdin=dump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) # We call dump_process.stdout.close before # restore_process.communicate so that if restore_process dies # prematurely the SIGPIPE signal can be processed by dump_process # allowing it to exit. dump_process.stdout.close() # Wait for restore process to end and get the output. _, err = restore_process.communicate() dump_process.wait() error_msg = "" if dump_process.returncode: error_msg = ( "mysqldump exited with error code '{0}' and message: " "'{1}'. ".format(dump_process.returncode, dump_process.stderr.read().strip())) else: _LOGGER.info("Dump process successfully completed.") if restore_process.returncode: error_msg += ( "MySQL client exited with error code '{0}' and message: " "'{1}'".format(restore_process.returncode, err.strip())) else: _LOGGER.info("Restore process successfully completed.") # If there were errors, raise an exception to warn the user. if error_msg: raise exceptions.GadgetError(error_msg) finally: # disable global read_lock _LOGGER.debug("Unlocking global read lock on source server.") source_server.toggle_global_read_lock(False) _LOGGER.debug("Source server unlocked. (read-only=OFF)") # delete created configuration files try: _LOGGER.debug("Removing configuration file '%s'", dump_config_file) os.remove(dump_config_file) except OSError: _LOGGER.warning("Unable to remove configuration file '%s'", dump_config_file) else: _LOGGER.debug("Configuration file '%s' successfully removed", dump_config_file) try: _LOGGER.debug("Removing configuration file '%s'", client_config_file) os.remove(client_config_file) except OSError: _LOGGER.warning("Unable to remove configuration file '%s'", client_config_file) else: _LOGGER.debug("Configuration file '%s' successfully removed", client_config_file) _LOGGER.info("Contents loaded successfully into destination " "server")
def restore_from_image(connection_dict, image_path): """"Restore an image file into the destination server. :param connection_dict: dictionary of dictionaries of connection information: mysql users and host users. It can have the following keys: MYSQL_SOURCE, MYSQL_DEST, HOST_SOURCE and HOST_DEST. Each of these keys has as a value a dict with the following keys: user, hostname, port, passwd and their respective values. :type connection_dict: dict :param image_path: name/path of the image that we will be read to do the restore operation. :type image_path: string """ # Creating Server for destination server destination_dict = connection_dict[MYSQL_DEST] try: destination_server = Server({'conn_info': destination_dict}) except exceptions.GadgetError as err: _LOGGER.error( "Unable to create a Server instance for destination " "server. Destination dict was: %s", destination_dict) raise err # Connect to destination server try: destination_server.connect() except exceptions.GadgetServerError as err: raise exceptions.GadgetError("Unable to connect to destination " "server: {0}.".format(str(err))) mysqlc_exe = get_tool_path(None, "mysql", search_path=True, required=False) if not mysqlc_exe: raise exceptions.GadgetError( "Could not find MySQL client executable. Make sure it is on " "{0}.".format(PATH_ENV_VAR)) # Create config_file for mysql client client_config_file = Server.to_config_file(destination_server, "client") # Replace image_name backslashes with forward slashes to pass it to the # mysql source command if os.name == 'nt': image_path = '/'.join(image_path.split(os.sep)) # Create command list to restore the backup restore_cmd = shlex.split( _MYSQLDUMP_IMAGE_RESTORE_CMD.format(mysqlc_exec=mysqlc_exe, config_file=client_config_file, image_file=image_path, quote=QUOTE_CHAR)) try: _LOGGER.debug( "Restoring contents of destination server from " "image file %s using command: %s", image_path, " ".join(restore_cmd)) restore_process = subprocess.Popen(restore_cmd, stderr=subprocess.PIPE, universal_newlines=True) _, err = restore_process.communicate() if restore_process.returncode: raise exceptions.GadgetError( "MySQL client exited with error code '{0}' and message: " "'{1}'. ".format(restore_process.returncode, err.strip())) else: _LOGGER.info("Restoring from file was successful.") finally: # delete created configuration file try: _LOGGER.debug("Removing configuration file '%s'", client_config_file) os.remove(client_config_file) except OSError: _LOGGER.warning("Unable to remove configuration file '%s'", client_config_file) else: _LOGGER.debug("Configuration file '%s' successfully removed", client_config_file) _LOGGER.info( "Destination server contents successfully loaded from " "file '%s'.", image_path)
def backup_to_image(connection_dict, image_path): """"Backup the contents of source server into an image file. :param connection_dict: dictionary of dictionaries of connection information: mysql users and host users. It can have the following keys: MYSQL_SOURCE, MYSQL_DEST, HOST_SOURCE and HOST_DEST. Each of these keys has as a value a dict with the following keys: user, hostname, port, passwd and their respective values. :type connection_dict: dict :param image_path: name/path of the image that we will create with the backup. :type image_path: string """ # Check tool requirements mysqldump_exe = get_tool_path(None, "mysqldump", search_path=True, required=False) if not mysqldump_exe: raise exceptions.GadgetError( "Could not find mysqldump executable. Make sure it is on " "{0}.".format(PATH_ENV_VAR)) # Creating Server instance for source server source_dict = connection_dict[MYSQL_SOURCE] try: source_server = Server({'conn_info': source_dict}) except exceptions.GadgetError as err: _LOGGER.error( "Unable to create a Server instance for source " "server. Source dict was: %s", source_dict) raise err # Connect to source server try: source_server.connect() except exceptions.GadgetServerError as err: raise exceptions.GadgetError( "Unable to connect to source server: {0}".format(str(err))) # Create config_file for mysqldump dump_config_file = Server.to_config_file(source_server, "mysqldump") # Create command list for backup backup_cmd = shlex.split( _MYSQLDUMP_IMAGE_BACKUP_CMD.format(mysqldump_exec=mysqldump_exe, config_file=dump_config_file, image_file=image_path, quote=QUOTE_CHAR)) # enable global read_lock _LOGGER.debug("Locking global read lock on source server to prevent " "modifications during clone.") source_server.toggle_global_read_lock(True) _LOGGER.debug("Source server locked (read-only=ON)") # Do the backup try: _LOGGER.debug( "Dumping contents of source server to image file %s " "using command: %s", image_path, " ".join(backup_cmd)) dump_process = subprocess.Popen(backup_cmd, stderr=subprocess.PIPE, universal_newlines=True) _, err = dump_process.communicate() if dump_process.returncode: raise exceptions.GadgetError( "mysqldump exited with error code '{0}' and message: " "'{1}'. ".format(dump_process.returncode, err.strip())) else: _LOGGER.info("Dumping to file was successful.") finally: # disable global read_lock _LOGGER.debug("Unlocking global read lock on source server.") source_server.toggle_global_read_lock(False) _LOGGER.debug("Source server unlocked. (read-only=OFF)") # delete created configuration file try: _LOGGER.debug("Removing configuration file '%s'", dump_config_file) os.remove(dump_config_file) except OSError: _LOGGER.warning("Unable to remove configuration file '%s'", dump_config_file) else: _LOGGER.debug("Configuration file '%s' successfully removed", dump_config_file) _LOGGER.info( "Source server contents successfully cloned to file " "'%s'.", image_path)