def setUp(self):
""" Setup server connections (for all tests) .
"""
self.source_cnx = {'conn_info': self.options[SERVER_CNX_OPT][0]}
self.destination_cnx = {'conn_info': self.options[SERVER_CNX_OPT][1]}
self.source = server.Server(self.source_cnx)
self.destination = server.Server(self.destination_cnx)
self.source.connect()
self.destination.connect()
try:
self.source_gtids_enabled = self.source.supports_gtid()
except exceptions.GadgetServerError:
self.source_gtids_enabled = False
try:
self.destination_gtids_enabled = self.destination.supports_gtid()
except exceptions.GadgetServerError:
self.destination_gtids_enabled = False
if self.destination_gtids_enabled:
self.destination_has_no_gtid_executed = not \
self.destination.get_gtid_executed(skip_gtid_check=False)
else:
self.destination_has_no_gtid_executed = False
self.mysqldump_exec = get_tool_path(None,
"mysqldump",
search_path=True,
required=False)
self.mysql_exec = get_tool_path(None,
"mysql",
search_path=True,
required=False)
# Search for mysqldump only on path (ignore MySQL default paths).
self.mysqldump_exec_in_defaults = get_tool_path(None,
"mysqldump",
search_path=False,
required=False)
# setup source server with a sample database and users
data_sql_path = os.path.normpath(
os.path.join(__file__, "..", "std_data", "basic_data.sql"))
self.source.read_and_exec_sql(data_sql_path, verbose=True)
# Create user without super permission on both source and destination
self.source.exec_query("create user no_super@'%'")
self.source.exec_query("GRANT ALL on *.* to no_super@'%'")
self.source.exec_query("REVOKE SUPER on *.* from no_super@'%'")
self.destination.exec_query("create user no_super@'%'")
self.destination.exec_query("GRANT ALL on *.* to no_super@'%'")
self.destination.exec_query("REVOKE SUPER on *.* from no_super@'%'")
if self.destination_has_no_gtid_executed:
self.destination.exec_query("RESET MASTER")
def test_get_server_version(self):
"""Tests the get_server_version method."""
# get version from the server
version_from_server = self.server.get_version()
# try to find the executable path via basedir
basedir = self.server.select_variable("basedir", "global")
try:
mysqld_path = tools.get_tool_path(basedir, "mysqld", required=True)
except GadgetError:
raise unittest.SkipTest("Couldn't find mysqld executable")
else:
version_from_binary = server.get_mysqld_version(mysqld_path)
self.assertEqual(tuple(version_from_server),
version_from_binary[0])
self.assertIn(".".join(str(i) for i in version_from_binary[0]),
version_from_binary[1])
def test_get_server_version(self):
"""Tests the get_server_version method."""
# get version from the server
version_from_server = self.server.get_version()
# try to find the executable path via basedir
basedir = self.server.select_variable("basedir", "global")
try:
mysqld_path = tools.get_tool_path(basedir, "mysqld", required=True)
except GadgetError:
raise unittest.SkipTest("Couldn't find mysqld executable")
else:
version_from_binary = server.get_mysqld_version(mysqld_path)
self.assertEqual(tuple(version_from_server),
version_from_binary[0])
self.assertIn(".".join(str(i) for i in version_from_binary[0]),
version_from_binary[1])
def test_is_executable(self):
"""Test is_executable function"""
# python is an executable file
try:
python_path = tools.get_tool_path(None, "python", required=True,
search_path=True)
except GadgetError:
raise unittest.SkipTest("Couldn't find python executable.")
self.assertTrue(tools.is_executable(python_path))
# a simple file is not executable, list this test file is not
# executable
if os.name == "posix":
# on windows the test file is deemed as executable
self.assertFalse(tools.is_executable(__file__))
# A file that does not exist is also not executable
self.assertFalse(tools.is_executable(__file__ + "aa"))
def test_run_subprocess(self):
"""Test run_subprocess function"""
# Run a simple echo command
try:
echo_path = tools.get_tool_path(None, "echo", required=True,
search_path=True)
except GadgetError:
raise unittest.SkipTest("Couldn't find echo executable.")
p = tools.run_subprocess("{0} output".format(echo_path),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
out, err = p.communicate()
ret_code = p.returncode
# process ran ok
self.assertEqual(ret_code, 0)
self.assertEqual(out, "output\n")
self.assertEqual("", err)
def test_get_tool_path(self):
""" Test get_tool_path function.
"""
# Get server basedir.
basedir = self.server.exec_query(
"SHOW VARIABLES LIKE '%basedir%'")[0][1]
# Find 'mysqld' using default option.
res = tools.get_tool_path(basedir, 'mysqld')
self.assertIn("mysqld", res,
"The 'mysqld' file is expected to be found in basedir.")
# Find 'mysqld' and return result with quotes.
res = tools.get_tool_path(basedir, 'mysqld', quote=True)
self.assertIn("mysqld", res,
"The 'mysqld' file is expected to be found in basedir.")
quote_char = "'" if os.name == "posix" else '"'
self.assertEqual(res[0], quote_char,
"Result expected to be quoted with "
"({0}).".format(quote_char))
# Find 'mysqld' using the default path list.
bindir = os.path.join(basedir, 'bin')
res = tools.get_tool_path('.', 'mysqld', defaults_paths=[bindir])
self.assertIn("mysqld", res,
"The 'mysqld' file is expected to be found in"
"defaults_paths.")
# Find 'mysqld' using a check tool function.
res = tools.get_tool_path(basedir, 'mysqld',
check_tool_func=server.is_valid_mysqld)
self.assertIn("mysqld", res,
"The 'mysqld' file is expected to be found in basedir.")
# Find 'mysqld' using a check tool function and the default path list.
bindir = os.path.join(basedir, 'bin')
res = tools.get_tool_path('.', 'mysqld', defaults_paths=[bindir],
check_tool_func=server.is_valid_mysqld)
self.assertIn("mysqld", res,
"The 'mysqld' file is expected to be found in"
"defaults_paths.")
# Find 'mysqld' using the PATH variable.
path_var = os.environ['PATH']
os.environ['PATH'] = bindir
try:
res = tools.get_tool_path('.', 'mysqld', search_path=True)
finally:
# Make sure the PATH variable is restored.
os.environ['PATH'] = path_var
self.assertIn("mysqld", res,
"The 'mysqld' file is expected to be found in PATH.")
# Find 'mysqld' using a check tool function and the PATH variable.
path_var = os.environ['PATH']
os.environ['PATH'] = bindir
try:
res = tools.get_tool_path('.', 'mysqld', search_path=True,
check_tool_func=server.is_valid_mysqld)
finally:
# Make sure the PATH variable is restored.
os.environ['PATH'] = path_var
self.assertIn("mysqld", res,
"The 'mysqld' file is expected to be found in PATH.")
# Try to find a non existing tool (error raised by default).
with self.assertRaises(GadgetError) as cm:
tools.get_tool_path(basedir, 'non_existing_tool')
self.assertEqual(cm.exception.errno, 1)
# Cannot find a valid tool, based on check function (raise error)
# Note: Use a lambda function that always return false.
with self.assertRaises(GadgetError) as cm:
tools.get_tool_path(basedir, 'mysqld',
check_tool_func=lambda path: False)
self.assertEqual(cm.exception.errno, 2)
# Try to find a non existing tool with required set to False.
res = tools.get_tool_path(basedir, 'non_existing_tool', required=False)
self.assertIsNone(res, "None expected when trying to find"
"'non_existing_tool'")
def clone_stream(connection_dict):
"""Clone the contents of source server into destination using a stream.
:param connection_dict: dictionary of dictionaries of connection
information: mysql users and host users. It can
have the following keys: MYSQL_SOURCE,
MYSQL_DEST, HOST_SOURCE and HOST_DEST.
Each of these keys has as a value a dict with
the following keys: user, hostname, port,
passwd and their respective values.
:type connection_dict: dict
"""
# Check tool requirements
mysqldump_exe = get_tool_path(None,
"mysqldump",
search_path=True,
required=False)
if not mysqldump_exe:
raise exceptions.GadgetError(
"Could not find mysqldump executable. Make sure it is on "
"{0}.".format(PATH_ENV_VAR))
mysqlc_exe = get_tool_path(None,
"mysql",
search_path=True,
required=False)
if not mysqlc_exe:
raise exceptions.GadgetError(
"Could not find mysql client executable. Make sure it is on "
"{0}.".format(PATH_ENV_VAR))
# Creating Server instances for source and destination servers
source_dict = connection_dict[MYSQL_SOURCE]
destination_dict = connection_dict[MYSQL_DEST]
try:
source_server = Server({'conn_info': source_dict})
except exceptions.GadgetError as err:
_LOGGER.error(
"Unable to create a Server instance for source server."
"Source dict was: %s", source_dict)
raise err
try:
destination_server = Server({'conn_info': destination_dict})
except exceptions.GadgetError as err:
_LOGGER.error(
"Unable to create a Server instance for destination "
"server. Destination dict was: %s", destination_dict)
raise err
# Connect to source and destination servers
try:
source_server.connect()
except exceptions.GadgetServerError as err:
raise exceptions.GadgetError(
"Unable to connect to source server: {0}".format(str(err)))
try:
destination_server.connect()
except exceptions.GadgetServerError as err:
raise exceptions.GadgetError("Unable to connect to destination "
"server: {0}.".format(str(err)))
# Create config_file for mysqldump
dump_config_file = Server.to_config_file(source_server, "mysqldump")
# Create config_file for mysql client
client_config_file = Server.to_config_file(destination_server,
"client")
# Create command list to create the backup
backup_cmd = shlex.split(
_MYSQLDUMP_STREAM_BACKUP_CMD.format(mysqldump_exec=mysqldump_exe,
config_file=dump_config_file,
quote=QUOTE_CHAR))
# Create command list to restore the backup
restore_cmd = shlex.split(
_MYSQLDUMP_STREAM_RESTORE_CMD.format(
mysqlc_exec=mysqlc_exe,
config_file=client_config_file,
quote=QUOTE_CHAR))
# enable global read_lock
_LOGGER.debug("Locking global read lock on source server to prevent "
"modifications during clone.")
source_server.toggle_global_read_lock(True)
_LOGGER.debug("Source server locked (read-only=ON)")
try:
_LOGGER.debug(
"Dumping contents of source server using command: "
"%s", " ".join(backup_cmd))
dump_process = subprocess.Popen(backup_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
_LOGGER.debug(
"Restoring contents to destination server using "
"command: %s", " ".join(restore_cmd))
restore_process = subprocess.Popen(restore_cmd,
stdin=dump_process.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
# We call dump_process.stdout.close before
# restore_process.communicate so that if restore_process dies
# prematurely the SIGPIPE signal can be processed by dump_process
# allowing it to exit.
dump_process.stdout.close()
# Wait for restore process to end and get the output.
_, err = restore_process.communicate()
dump_process.wait()
error_msg = ""
if dump_process.returncode:
error_msg = (
"mysqldump exited with error code '{0}' and message: "
"'{1}'. ".format(dump_process.returncode,
dump_process.stderr.read().strip()))
else:
_LOGGER.info("Dump process successfully completed.")
if restore_process.returncode:
error_msg += (
"MySQL client exited with error code '{0}' and message: "
"'{1}'".format(restore_process.returncode, err.strip()))
else:
_LOGGER.info("Restore process successfully completed.")
# If there were errors, raise an exception to warn the user.
if error_msg:
raise exceptions.GadgetError(error_msg)
finally:
# disable global read_lock
_LOGGER.debug("Unlocking global read lock on source server.")
source_server.toggle_global_read_lock(False)
_LOGGER.debug("Source server unlocked. (read-only=OFF)")
# delete created configuration files
try:
_LOGGER.debug("Removing configuration file '%s'",
dump_config_file)
os.remove(dump_config_file)
except OSError:
_LOGGER.warning("Unable to remove configuration file '%s'",
dump_config_file)
else:
_LOGGER.debug("Configuration file '%s' successfully removed",
dump_config_file)
try:
_LOGGER.debug("Removing configuration file '%s'",
client_config_file)
os.remove(client_config_file)
except OSError:
_LOGGER.warning("Unable to remove configuration file '%s'",
client_config_file)
else:
_LOGGER.debug("Configuration file '%s' successfully removed",
client_config_file)
_LOGGER.info("Contents loaded successfully into destination " "server")
def restore_from_image(connection_dict, image_path):
""""Restore an image file into the destination server.
:param connection_dict: dictionary of dictionaries of connection
information: mysql users and host users. It can
have the following keys: MYSQL_SOURCE,
MYSQL_DEST, HOST_SOURCE and HOST_DEST.
Each of these keys has as a value a dict with
the following keys: user, hostname, port,
passwd and their respective values.
:type connection_dict: dict
:param image_path: name/path of the image that we will be read to do
the restore operation.
:type image_path: string
"""
# Creating Server for destination server
destination_dict = connection_dict[MYSQL_DEST]
try:
destination_server = Server({'conn_info': destination_dict})
except exceptions.GadgetError as err:
_LOGGER.error(
"Unable to create a Server instance for destination "
"server. Destination dict was: %s", destination_dict)
raise err
# Connect to destination server
try:
destination_server.connect()
except exceptions.GadgetServerError as err:
raise exceptions.GadgetError("Unable to connect to destination "
"server: {0}.".format(str(err)))
mysqlc_exe = get_tool_path(None,
"mysql",
search_path=True,
required=False)
if not mysqlc_exe:
raise exceptions.GadgetError(
"Could not find MySQL client executable. Make sure it is on "
"{0}.".format(PATH_ENV_VAR))
# Create config_file for mysql client
client_config_file = Server.to_config_file(destination_server,
"client")
# Replace image_name backslashes with forward slashes to pass it to the
# mysql source command
if os.name == 'nt':
image_path = '/'.join(image_path.split(os.sep))
# Create command list to restore the backup
restore_cmd = shlex.split(
_MYSQLDUMP_IMAGE_RESTORE_CMD.format(mysqlc_exec=mysqlc_exe,
config_file=client_config_file,
image_file=image_path,
quote=QUOTE_CHAR))
try:
_LOGGER.debug(
"Restoring contents of destination server from "
"image file %s using command: %s", image_path,
" ".join(restore_cmd))
restore_process = subprocess.Popen(restore_cmd,
stderr=subprocess.PIPE,
universal_newlines=True)
_, err = restore_process.communicate()
if restore_process.returncode:
raise exceptions.GadgetError(
"MySQL client exited with error code '{0}' and message: "
"'{1}'. ".format(restore_process.returncode, err.strip()))
else:
_LOGGER.info("Restoring from file was successful.")
finally:
# delete created configuration file
try:
_LOGGER.debug("Removing configuration file '%s'",
client_config_file)
os.remove(client_config_file)
except OSError:
_LOGGER.warning("Unable to remove configuration file '%s'",
client_config_file)
else:
_LOGGER.debug("Configuration file '%s' successfully removed",
client_config_file)
_LOGGER.info(
"Destination server contents successfully loaded from "
"file '%s'.", image_path)
def backup_to_image(connection_dict, image_path):
""""Backup the contents of source server into an image file.
:param connection_dict: dictionary of dictionaries of connection
information: mysql users and host users. It can
have the following keys: MYSQL_SOURCE,
MYSQL_DEST, HOST_SOURCE and HOST_DEST.
Each of these keys has as a value a dict with
the following keys: user, hostname, port,
passwd and their respective values.
:type connection_dict: dict
:param image_path: name/path of the image that we will create with the
backup.
:type image_path: string
"""
# Check tool requirements
mysqldump_exe = get_tool_path(None,
"mysqldump",
search_path=True,
required=False)
if not mysqldump_exe:
raise exceptions.GadgetError(
"Could not find mysqldump executable. Make sure it is on "
"{0}.".format(PATH_ENV_VAR))
# Creating Server instance for source server
source_dict = connection_dict[MYSQL_SOURCE]
try:
source_server = Server({'conn_info': source_dict})
except exceptions.GadgetError as err:
_LOGGER.error(
"Unable to create a Server instance for source "
"server. Source dict was: %s", source_dict)
raise err
# Connect to source server
try:
source_server.connect()
except exceptions.GadgetServerError as err:
raise exceptions.GadgetError(
"Unable to connect to source server: {0}".format(str(err)))
# Create config_file for mysqldump
dump_config_file = Server.to_config_file(source_server, "mysqldump")
# Create command list for backup
backup_cmd = shlex.split(
_MYSQLDUMP_IMAGE_BACKUP_CMD.format(mysqldump_exec=mysqldump_exe,
config_file=dump_config_file,
image_file=image_path,
quote=QUOTE_CHAR))
# enable global read_lock
_LOGGER.debug("Locking global read lock on source server to prevent "
"modifications during clone.")
source_server.toggle_global_read_lock(True)
_LOGGER.debug("Source server locked (read-only=ON)")
# Do the backup
try:
_LOGGER.debug(
"Dumping contents of source server to image file %s "
"using command: %s", image_path, " ".join(backup_cmd))
dump_process = subprocess.Popen(backup_cmd,
stderr=subprocess.PIPE,
universal_newlines=True)
_, err = dump_process.communicate()
if dump_process.returncode:
raise exceptions.GadgetError(
"mysqldump exited with error code '{0}' and message: "
"'{1}'. ".format(dump_process.returncode, err.strip()))
else:
_LOGGER.info("Dumping to file was successful.")
finally:
# disable global read_lock
_LOGGER.debug("Unlocking global read lock on source server.")
source_server.toggle_global_read_lock(False)
_LOGGER.debug("Source server unlocked. (read-only=OFF)")
# delete created configuration file
try:
_LOGGER.debug("Removing configuration file '%s'",
dump_config_file)
os.remove(dump_config_file)
except OSError:
_LOGGER.warning("Unable to remove configuration file '%s'",
dump_config_file)
else:
_LOGGER.debug("Configuration file '%s' successfully removed",
dump_config_file)
_LOGGER.info(
"Source server contents successfully cloned to file "
"'%s'.", image_path)