def setup(self):
self.res_fname = "result.txt"
self.server1 = self.servers.get_server(0)
if self.need_server:
try:
self.servers.spawn_new_servers(2)
except MUTLibError as err:
raise MUTLibError("Cannot spawn needed servers: {0}".format(
err.errmsg))
self.server2 = self.servers.get_server(1)
s1_conn = "--server1={0}".format(
self.build_connection_string(self.server1))
s2_conn = "--server2={0}".format(
self.build_connection_string(self.server2))
# pylint: disable=E1101
self.base_cmd = "{0} {1} {2} ".format(self.utility, s1_conn, s2_conn)
rows = self.server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine basedir of running "
"server.")
self.mysql_path = get_tool_path(basedir, "mysql", quote=True)
return True
def setup(self):
num_servers = self.servers.num_servers()
if self.need_servers:
try:
self.servers.spawn_new_servers(num_servers + 2)
except MUTLibError as err:
raise MUTLibError("Cannot spawn needed servers: {0}".format(
err.errmsg))
else:
num_servers -= 2 # Get last 2 servers in list
self.server1 = self.servers.get_server(num_servers)
self.server2 = self.servers.get_server(num_servers + 1)
self.drop_all()
data_file = os.path.normpath("./std_data/basic_data.sql")
try:
self.server1.read_and_exec_SQL(data_file, self.debug)
except MUTLibError as err:
raise MUTLibError(
"Failed to read commands from file {0}: {1}".format(
data_file, err.errmsg))
rows = self.server2.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine basedir of running "
"server.")
self.mysql_path = get_tool_path(basedir, "mysql", quote=True)
return True
def setup(self):
self.res_fname = "result.txt"
self.server1 = self.servers.get_server(0)
if self.need_server:
try:
self.servers.spawn_new_servers(2)
except MUTLibError as err:
raise MUTLibError("Cannot spawn needed servers: {0}".format(
err.errmsg))
self.server2 = self.servers.get_server(1)
s1_conn = "--server1={0}".format(
self.build_connection_string(self.server1))
s2_conn = "--server2={0}".format(
self.build_connection_string(self.server2))
# pylint: disable=E1101
self.base_cmd = "{0} {1} {2} ".format(self.utility, s1_conn, s2_conn)
rows = self.server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine basedir of running "
"server.")
self.mysql_path = get_tool_path(basedir, "mysql", quote=True)
return True
def search_my_print_defaults_tool(self, search_paths=None):
"""Search for the tool my_print_defaults.
"""
if not search_paths:
search_paths = []
# Set the default search paths (i.e., default location of the
# .mylogin.cnf file).
default_paths = [my_login_config_path()]
# Extend the list of path to search with the ones specified.
if search_paths:
default_paths.extend(search_paths)
# Search for the tool my_print_defaults.
try:
self._tool_path = get_tool_path(self._basedir,
_MY_PRINT_DEFAULTS_TOOL,
defaults_paths=default_paths,
search_PATH=True)
except UtilError as err:
raise UtilError("Unable to locate MySQL Client tools. "
"Please confirm that the path to the MySQL client "
"tools are included in the PATH. Error: %s"
% err.errmsg)
def search_my_print_defaults_tool(self, search_paths=None):
"""Search for the tool my_print_defaults.
"""
if not search_paths:
search_paths = []
# Set the default search paths (i.e., default location of the
# .mylogin.cnf file).
default_paths = [my_login_config_path()]
# Extend the list of path to search with the ones specified.
if search_paths:
default_paths.extend(search_paths)
# Search for the tool my_print_defaults.
try:
self._tool_path = get_tool_path(self._basedir,
_MY_PRINT_DEFAULTS_TOOL,
defaults_paths=default_paths,
search_PATH=True)
except UtilError as err:
raise UtilError("Unable to locate MySQL Client tools. "
"Please confirm that the path to the MySQL client "
"tools are included in the PATH. Error: %s" %
err.errmsg)
def _stop_server(server_val, basedir, options={}):
"""Stop an instance of a server started in read only mode
This method is used to stop the server started in read only mode. It will
launch mysqladmin to stop the server.
Caller must start the server with _start_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
options[in] dictionary of options (verbosity)
"""
from mysql.utilities.common.tools import get_tool_path
verbosity = options.get("verbosity", 0)
socket = server_val.get("unix_socket", None)
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
print "# Shutting down server ...",
if os.name == "posix":
cmd = mysqladmin_path + " shutdown -uroot "
if socket is not None:
cmd = cmd + " --socket=%s " % socket
else:
cmd = mysqladmin_path + " shutdown -uroot " + \
" --port=%(port)s" % server_val
if verbosity > 0:
proc = subprocess.Popen(cmd, shell=True)
else:
fnull = open(os.devnull, 'w')
proc = subprocess.Popen(cmd, shell=True,
stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
print "done."
def check_mylogin_requisites(self):
""" Check if the tools to manipulate mylogin.cnf are accessible.
This method verifies if the MySQL client tools my_print_defaults and
mysql_config_editor are accessible.
A MUTLibError exception is raised if the requisites are not met.
"""
try:
self.login_reader = MyDefaultsReader(
find_my_print_defaults_tool=True)
except UtilError as err:
raise MUTLibError("MySQL client tools must be accessible to run "
"this test (%s). E.g. Add the location of the "
"MySQL client tools to your PATH." % err.errmsg)
if not self.login_reader.check_login_path_support():
raise MUTLibError(
"ERROR: the used my_print_defaults tool does not "
"support login-path options. Used tool: %s" %
self.login_reader.tool_path)
try:
self.edit_tool_path = get_tool_path(None,
"mysql_config_editor",
search_PATH=True)
except UtilError as err:
raise MUTLibError("MySQL client tools must be accessible to run "
"this test (%s). E.g. Add the location of the "
"MySQL client tools to your PATH." % err.errmsg)
def check_mylogin_requisites(self):
""" Check if the tools to manipulate mylogin.cnf are accessible.
This method verifies if the MySQL client tools my_print_defaults and
mysql_config_editor are accessible.
A MUTLibError exception is raised if the requisites are not met.
"""
try:
self.login_reader = MyDefaultsReader(
find_my_print_defaults_tool=True)
except UtilError as err:
raise MUTLibError("MySQL client tools must be accessible to run "
"this test (%s). E.g. Add the location of the "
"MySQL client tools to your PATH." % err.errmsg)
if not self.login_reader.check_login_path_support():
raise MUTLibError("ERROR: the used my_print_defaults tool does not "
"support login-path options. Used tool: %s"
% self.login_reader.tool_path)
try:
self.edit_tool_path = get_tool_path(None, "mysql_config_editor",
search_PATH=True)
except UtilError as err:
raise MUTLibError("MySQL client tools must be accessible to run "
"this test (%s). E.g. Add the location of the "
"MySQL client tools to your PATH." % err.errmsg)
def _stop_server(server_val, basedir, options=None):
"""Stop an instance of a server started in read only mode
This method is used to stop the server started in read only mode. It will
launch mysqladmin to stop the server.
Caller must start the server with _start_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
socket = server_val.get("unix_socket", None)
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
print "# Shutting down server ...",
if os.name == "posix":
cmd = mysqladmin_path + " shutdown -uroot "
if socket is not None:
cmd = cmd + " --socket=%s " % socket
else:
cmd = mysqladmin_path + " shutdown -uroot " + \
" --port=%(port)s" % server_val
if verbosity > 0:
proc = subprocess.Popen(cmd, shell=True)
else:
fnull = open(os.devnull, 'w')
proc = subprocess.Popen(cmd, shell=True,
stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
proc.wait()
print "done."
def setUpClass(cls):
# Find mysql_config_editor to manipulate data from .mylogin.cnf
try:
cls.edit_tool_path = get_tool_path(None,
"mysql_config_editor",
search_PATH=True)
except UtilError as err:
raise UtilError("MySQL client tools must be accessible to run "
"this test (%s). Please add the location of the "
"MySQL client tools to your PATH." % err.errmsg)
# Create login-path data
cmd = ("{mysql_config_editor} set --login-path={login_path} "
"--host={host} --user={user}")
cmd = cmd.format(mysql_config_editor=cls.edit_tool_path,
login_path=_TEST_LOGIN_PATH,
host=_TEST_HOST,
user=_TEST_USER)
# Execute command to create login-path data
proc = subprocess.Popen(cmd.split(' '),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
# Overwrite login-path if already exists (i.e. answer 'y' to question)
proc.communicate(input='y')
def setup(self):
num_servers = self.servers.num_servers()
if self.need_servers:
try:
self.servers.spawn_new_servers(num_servers + 2)
except MUTLibError as err:
raise MUTLibError("Cannot spawn needed servers: {0}".format(
err.errmsg))
else:
num_servers -= 2 # Get last 2 servers in list
self.server1 = self.servers.get_server(num_servers)
self.server2 = self.servers.get_server(num_servers + 1)
self.drop_all()
data_file = os.path.normpath("./std_data/basic_data.sql")
try:
self.server1.read_and_exec_SQL(data_file, self.debug)
except MUTLibError as err:
raise MUTLibError(
"Failed to read commands from file {0}: {1}".format(
data_file, err.errmsg))
rows = self.server2.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine basedir of running "
"server.")
self.mysql_path = get_tool_path(basedir, "mysql", quote=True)
return True
def setup(self):
self.res_fname = "result.txt"
# Get path to mysql client tool from base server
base_server = self.servers.get_server(0)
if not base_server:
raise MUTLibError("Unable to get base server.")
rows = base_server.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine 'basedir' for base server.")
try:
self.mysql_path = get_tool_path(basedir, "mysql", quote=True)
except UtilError as err:
raise MUTLibError("Unable to find mysql client tool for server "
"{0}@{1} on basedir={2}. "
"ERROR: {3}".format(base_server.host,
base_server.port, basedir,
err.errmsg))
# Spawn required servers
num_servers = self.servers.num_servers()
if num_servers < 3:
try:
self.servers.spawn_new_servers(3)
except MUTLibError as err:
raise MUTLibError(
"Cannot spawn needed servers: {0}".format(err.errmsg))
# Set spawned servers
self.server1 = self.servers.get_server(1)
self.server2 = self.servers.get_server(2)
# SQL data to execute on servers
common_data = (
"CREATE DATABASE util_test_mysql_client;\n"
"CREATE TABLE util_test_mysql_client.t1 (a char(30), "
"PRIMARY KEY (a)) ENGINE=InnoDB DEFAULT CHARSET=latin1;\n"
)
# Write common data to file
with open(_INPUT_SQL_FILE, 'w') as sql_file:
sql_file.write(common_data)
# Execute common SQL data on both servers
self.exec_mysql_client_cmd(self.server1, _INPUT_SQL_FILE)
self.exec_mysql_client_cmd(self.server2, _INPUT_SQL_FILE)
return True
def setup(self):
self.res_fname = "result.txt"
# Get path to mysql client tool from base server
base_server = self.servers.get_server(0)
if not base_server:
raise MUTLibError("Unable to get base server.")
rows = base_server.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine 'basedir' for base server.")
try:
self.mysql_path = get_tool_path(basedir, "mysql", quote=True)
except UtilError as err:
raise MUTLibError("Unable to find mysql client tool for server "
"{0}@{1} on basedir={2}. "
"ERROR: {3}".format(base_server.host,
base_server.port, basedir,
err.errmsg))
# Spawn required servers
num_servers = self.servers.num_servers()
if num_servers < 3:
try:
self.servers.spawn_new_servers(3)
except MUTLibError as err:
raise MUTLibError("Cannot spawn needed servers: {0}".format(
err.errmsg))
# Set spawned servers
self.server1 = self.servers.get_server(1)
self.server2 = self.servers.get_server(2)
# SQL data to execute on servers
common_data = (
"CREATE DATABASE util_test_mysql_client;\n"
"CREATE TABLE util_test_mysql_client.t1 (a char(30), "
"PRIMARY KEY (a)) ENGINE=InnoDB DEFAULT CHARSET=latin1;\n")
# Write common data to file
with open(_INPUT_SQL_FILE, 'w') as sql_file:
sql_file.write(common_data)
# Execute common SQL data on both servers
self.exec_mysql_client_cmd(self.server1, _INPUT_SQL_FILE)
self.exec_mysql_client_cmd(self.server2, _INPUT_SQL_FILE)
return True
def run(self):
self.res_fname = "result.txt"
num_test = 1
comment = "Test case %s - get_tool mysqld-nt.exe \n" % num_test
self.results.append(comment)
try:
basedir = self.server1.show_server_variable("basedir")
# setting required=False to verify it founds mysqld-nt.exe
res = tools.get_tool_path(basedir[0][1], "mysqld", required=False)
# ensuring it founds mysqld-nt.exe.
if "mysqld-nt.exe" in res and not "Cannot find location of" in res:
self.results.append("Pass\n")
except UtilError, exc:
raise MUTLibError("%s: failed" % comment)
def run(self):
self.res_fname = "result.txt"
num_test = 1
comment = "Test case %s - get_tool mysqld-nt.exe \n" % num_test
self.results.append(comment)
try:
basedir = self.server1.show_server_variable("basedir")
#setting required=False to verify it founds mysqld-nt.exe
res = tools.get_tool_path(basedir[0][1], "mysqld", required=False)
#ensuring it founds mysqld-nt.exe.
if ("mysqld-nt.exe" in res and
not "Cannot find location of" in res):
self.results.append("Pass\n")
except UtilError, exc:
raise MUTLibError("%s: failed" % comment)
def run(self):
self.res_fname = "result.txt"
num_test = 1
comment = "Test case {0} - get_tool mysqld-nt.exe \n".format(num_test)
self.results.append(comment)
try:
basedir = self.server1.show_server_variable("basedir")
# setting required=False to verify it founds mysqld-nt.exe
res = tools.get_tool_path(basedir[0][1], "mysqld", required=False)
# ensuring it founds mysqld-nt.exe
if "mysqld-nt.exe" in res and "Cannot find location of" not in res:
self.results.append("Pass\n")
except UtilError:
raise MUTLibError("{0}: failed".format(comment))
return True
def setup(self):
proc_grep.test.setup(self)
rows = self.server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine 'basedir' for base server.")
try:
self.mysql_path = get_tool_path(basedir, "mysql")
except UtilError as err:
raise MUTLibError("Unable to find mysql client tool for server "
"{0}@{1} on basedir={2}. "
"ERROR: {3}".format(self.server1.host,
self.server1.port, basedir,
err.errmsg))
return True
def setup(self):
proc_grep.test.setup(self)
rows = self.server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine 'basedir' for base server.")
try:
self.mysql_path = get_tool_path(basedir, "mysql")
except UtilError as err:
raise MUTLibError("Unable to find mysql client tool for server "
"{0}@{1} on basedir={2}. "
"ERROR: {3}".format(self.server1.host,
self.server1.port, basedir,
err.errmsg))
return True
def setUpClass(cls):
# Find mysql_config_editor to manipulate data from .mylogin.cnf
try:
cls.edit_tool_path = get_tool_path(None, "mysql_config_editor",
search_PATH=True)
except UtilError as err:
raise UtilError("MySQL client tools must be accessible to run "
"this test (%s). Please add the location of the "
"MySQL client tools to your PATH." % err.errmsg)
# Create login-path data
cmd = ("{mysql_config_editor} set --login-path={login_path} "
"--host={host} --user={user}")
cmd = cmd.format(mysql_config_editor=cls.edit_tool_path,
login_path=_TEST_LOGIN_PATH,
host=_TEST_HOST, user=_TEST_USER)
# Execute command to create login-path data
proc = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
# Overwrite login-path if already exists (i.e. answer 'y' to question)
proc.communicate(input='y')
def _server_info(server_val, get_defaults=False, options=None):
"""Show information about a running server
This method gathers information from a running server. This information is
returned as a tuple to be displayed to the user in a format specified. The
information returned includes the following:
* server connection information
* version number of the server
* data directory path
* base directory path
* plugin directory path
* configuration file location and name
* current binary log file
* current binary log position
* current relay log file
* current relay log position
server_val[in] the server connection values or a connected server
get_defaults[in] if True, get the default settings for the server
options[in] options for connecting to the server
Return tuple - information about server
"""
if options is None:
options = {}
# Parse source connection values
source_values = parse_connection(server_val, None, options)
# Connect to the server
conn_options = {
'version': "5.1.30",
}
servers = connect_servers(source_values, None, conn_options)
server = servers[0]
params_dict = defaultdict(str)
# Initialize list of warnings
params_dict['warnings'] = []
# Identify server by string: 'host:port[:socket]'.
server_id = "{0}:{1}".format(source_values['host'], source_values['port'])
if source_values.get('socket', None):
server_id = "{0}:{1}".format(server_id, source_values.get('socket'))
params_dict['server'] = server_id
# Get _SERVER_VARIABLES values from the server
for server_var in _SERVER_VARIABLES:
res = server.show_server_variable(server_var)
if res:
params_dict[server_var] = res[0][1]
else:
raise UtilError("Unable to determine {0} of server '{1}'"
".".format(server_var, server_id))
# Get _LOG_FILES_VARIABLES values from the server
for msg, log_tpl in _LOG_FILES_VARIABLES.iteritems():
res = server.show_server_variable(log_tpl.log_name)
if res:
# Check if log is turned off
params_dict[log_tpl.log_name] = res[0][1]
# If logs are turned off, skip checking information about the file
if res[0][1] in ('', 'OFF'):
continue
# Logging is enabled, so we can get get information about log_file
# unless it is log_error because in that case we already have it.
if log_tpl.log_file is not None: # if it is not log_error
log_file = server.show_server_variable(log_tpl.log_file)[0][1]
params_dict[log_tpl.log_file] = log_file
else: # log error, so log_file_name is already on params_dict
log_file = params_dict[log_tpl.log_name]
# Now get the information about the size of the logs
# If log file is stderr, we cannot get the correct size.
if log_file not in ["stderr", "stdout"]:
# Now get the information about the size of the logs
try:
params_dict[log_tpl.log_file_size] = "{0} bytes".format(
os.path.getsize(log_file))
except os.error:
# if we are unable to get the log_file_size
params_dict[log_tpl.log_file_size] = ''
warning_msg = _WARNING_TEMPLATE.format(msg, log_file)
params_dict['warnings'].append(warning_msg)
else:
params_dict['warnings'].append(
"Unable to get information "
"regarding variable '{0}'").format(msg)
# if audit_log plugin is installed and enabled
if server.supports_plugin('audit'):
res = server.show_server_variable('audit_log_file')
if res:
# Audit_log variable might be a relative path to the datadir,
# so it needs to be treated accordingly
if not os.path.isabs(res[0][1]):
params_dict['audit_log_file'] = os.path.join(
params_dict['datadir'], res[0][1])
else:
params_dict['audit_log_file'] = res[0][1]
# Add audit_log field to the _COLUMNS List unless it is already
# there
if 'audit_log_file' not in _COLUMNS_SET:
_COLUMNS.append('audit_log_file')
_COLUMNS.append('audit_log_file_size')
_COLUMNS_SET.add('audit_log_file')
try:
params_dict['audit_log_file_size'] = "{0} bytes".format(
os.path.getsize(params_dict['audit_log_file']))
except os.error:
# If we are unable to get the size of the audit_log_file
params_dict['audit_log_file_size'] = ''
warning_msg = _WARNING_TEMPLATE.format(
"audit log", params_dict['audit_log_file'])
params_dict['warnings'].append(warning_msg)
# Build search path for config files
if os.name == "posix":
my_def_search = [
"/etc/my.cnf", "/etc/mysql/my.cnf",
os.path.join(params_dict['basedir'], "my.cnf"), "~/.my.cnf"
]
else:
my_def_search = [
r"c:\windows\my.ini", r"c:\my.ini", r"c:\my.cnf",
os.path.join(os.curdir, "my.ini")
]
my_def_search.append(os.path.join(os.curdir, "my.cnf"))
# Get server's default configuration values.
defaults = []
if get_defaults:
# Can only get defaults for local servers (need to access local data).
if server.is_alias('localhost'):
try:
my_def_path = get_tool_path(params_dict['basedir'],
"my_print_defaults",
quote=True)
except UtilError as err:
raise UtilError("Unable to retrieve the defaults data "
"(requires access to my_print_defaults): {0} "
"(basedir: {1})".format(
err.errmsg, params_dict['basedir']))
out_file = tempfile.TemporaryFile()
# Execute tool: <basedir>/my_print_defaults mysqld
cmd_list = shlex.split(my_def_path)
cmd_list.append("mysqld")
subprocess.call(cmd_list, stdout=out_file)
out_file.seek(0)
# Get defaults data from temp output file.
defaults.append("\nDefaults for server {0}".format(server_id))
for line in out_file.readlines():
defaults.append(line.rstrip())
else:
# Remote server; Cannot get the defaults data.
defaults.append("\nWARNING: The utility can not get defaults from "
"a remote host.")
# Find config file
config_file = ""
for search_path in my_def_search:
if os.path.exists(search_path):
if len(config_file) > 0:
config_file = "{0}, {1}".format(config_file, search_path)
else:
config_file = search_path
params_dict['config_file'] = config_file
# Find binary log, relay log
params_dict['binary_log'], params_dict['binary_log_pos'] = _get_binlog(
server)
params_dict['relay_log'], params_dict['relay_log_pos'] = _get_relay_log(
server)
server.disconnect()
return params_dict, defaults
def _start_server(server_val, basedir, datadir, options=None):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
start_timeout = options.get("start_timeout", 10)
mysqld_path = get_tool_path(basedir, "mysqld", quote=True)
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
if version is not None and int(version[0]) >= 5:
post_5_5 = int(version[1]) >= 5
post_5_6 = int(version[1]) >= 6
post_5_7_4 = int(version[1]) >= 7 and int(version[2]) > 4
else:
print("# Warning: cannot get server version.")
post_5_5 = False
post_5_6 = False
post_5_7_4 = False
# Get the user executing the utility to use in the mysqld options.
# Note: the option --user=user_name is mandatory to start mysqld as root.
user_name = getpass.getuser()
# Start the instance
if verbosity > 0:
print "# Starting read-only instance of the server ..."
print "# --- BEGIN (server output) ---"
else:
print "# Starting read-only instance of the server ...",
args = shlex.split(mysqld_path)
args.extend([
"--no-defaults",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
"--user={0}".format(user_name),
])
# It the server is 5.6 or later, we must use additional parameters
if post_5_5:
server_args = [
"--skip-slave-start",
"--default-storage-engine=MYISAM",
"--server-id=0",
]
if post_5_6:
server_args.append("--default-tmp-storage-engine=MYISAM")
if not post_5_7_4:
server_args.append("--skip-innodb")
args.extend(server_args)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
subprocess.Popen(args, shell=False)
else:
out = open(os.devnull, 'w')
subprocess.Popen(args, shell=False, stdout=out, stderr=out)
server_options = {
'conn_info': server_val,
'role': "read_only",
}
server = Server(server_options)
# Try to connect to the server, waiting for the server to become ready
# (retry start_timeout times and wait 1 sec between each attempt).
# Note: It can take up to 10 seconds for Windows machines.
i = 0
while i < start_timeout:
# Reset error and wait 1 second.
error = None
time.sleep(1)
try:
server.connect()
break # Server ready (connect succeed)! Exit the for loop.
except UtilError as err:
# Store exception to raise later (if needed).
error = err
i += 1
# Indicate end of the server output.
if verbosity > 0:
print "# --- END (server output) ---"
# Raise last known exception (if unable to connect to the server)
if error:
raise error # pylint: disable=E0702
# See: http://www.logilab.org/ticket/3207
if verbosity > 0:
print "# done (server started)."
else:
print "done."
return server
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
start_timeout[in] Number of seconds to wait for server to start
"""
new_data = os.path.abspath(options.get('new_data', None))
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
user = options.get('user', 'root')
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
start_timeout = int(options.get('start_timeout', 10))
mysqld_options = options.get('mysqld_options', '')
force = options.get('force', False)
quote_char = "'" if os.name == "posix" else '"'
if not check_port_in_use('localhost', int(new_port)):
raise UtilError("Port {0} in use. Please choose an "
"available port.".format(new_port))
# Check if path to database files is greater than MAX_DIR_SIZE char,
if len(new_data) > MAX_DATADIR_SIZE and not force:
raise UtilError("The --new-data path '{0}' is too long "
"(> {1} characters). Please use a smaller one. "
"You can use the --force option to skip this "
"check".format(new_data, MAX_DATADIR_SIZE))
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % \
conn_val["host"]
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = os.path.normpath(rows[0][1])
# Cloning downed or offline server
else:
basedir = os.path.abspath(options.get("basedir", None))
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
new_data_deleted = False
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
new_data_deleted = True
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
os.mkdir(new_data)
except OSError as err:
raise UtilError("Unable to create directory '{0}', reason: {1}"
"".format(new_data, err.strerror))
# After create the new data directory, check for free space, so the errors
# regarding invalid or inaccessible path had been dismissed already.
# If not force specified verify and stop if there is not enough free space
if not force and os.path.exists(new_data) and \
estimate_free_space(new_data) < REQ_FREE_SPACE:
# Don't leave empty folders, delete new_data if was previously deleted
if os.path.exists(new_data) and new_data_deleted:
shutil.rmtree(new_data, True)
raise UtilError(LOW_SPACE_ERRR_MSG.format(directory=new_data,
megabytes=REQ_FREE_SPACE))
# Check for warning of using --skip-innodb
mysqld_path = get_tool_path(basedir, "mysqld")
version_str = get_mysqld_version(mysqld_path)
# convert version_str from str tuple to integer tuple if possible
if version_str is not None:
version = tuple([int(digit) for digit in version_str])
else:
version = None
if mysqld_options is not None and ("--skip-innodb" in mysqld_options or
"--innodb" in mysqld_options) and version is not None and \
version >= (5, 7, 5):
print("# WARNING: {0}".format(WARN_OPT_SKIP_INNODB))
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
]
# From 5.7.6 version onwards, bootstrap is done via mysqld with the
# --initialize-insecure option, so no need to get information about the
# sql system tables that need to be loaded.
if version < (5, 7, 6):
system_tables = get_tool_path(basedir, "mysql_system_tables.sql",
False)
system_tables_data = get_tool_path(basedir,
"mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir,
"mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
locations.extend([("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
])
if verbosity >= 3 and not quiet:
print "# Location of files:"
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
fnull = open(os.devnull, 'w')
# For MySQL versions before 5.7.6, use regular bootstrap procedure.
if version < (5, 7, 6):
# Get bootstrap SQL statements
sql = list()
sql.append("CREATE DATABASE mysql;")
sql.append("USE mysql;")
innodb_disabled = False
if mysqld_options:
innodb_disabled = '--innodb=OFF' in mysqld_options
for sqlfile in [system_tables, system_tables_data, test_data_timezone,
help_data]:
lines = open(sqlfile, 'r').readlines()
# On MySQL 5.7.5, the root@localhost account creation was
# moved from the system_tables_data sql file into the
# mysql_install_db binary. Since we don't use mysql_install_db
# directly we need to create the root user account ourselves.
if (version is not None and version == (5, 7, 5) and
sqlfile == system_tables_data):
lines.extend(_CREATE_ROOT_USER)
for line in lines:
line = line.strip()
# Don't fail when InnoDB is turned off (Bug#16369955)
# (Ugly hack)
if (sqlfile == system_tables and
"SET @sql_mode_orig==@@SES" in line and innodb_disabled):
for line in lines:
if 'SET SESSION sql_mode=@@sql' in line:
break
sql.append(line)
# Bootstap to setup mysql tables
cmd = [
mysqld_path,
"--no-defaults",
"--bootstrap",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir)),
]
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
stdout=fnull, stderr=fnull)
proc.communicate('\n'.join(sql))
# From 5.7.6 onwards, mysql_install_db has been replaced by mysqld and
# the --initialize option
else:
cmd = [
mysqld_path,
"--initialize-insecure=on",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir))
]
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
raise UtilError("Failed to kill process with pid '{0}'"
"".format(proc.pid))
else:
ret_code = subprocess.call("taskkill /F /T /PID "
"{0}".format(proc.pid), shell=True)
# return code 0 means it was successful and 128 means it tried
# to kill a process that doesn't exist
if ret_code not in (0, 128):
raise UtilError("Failed to kill process with pid '{0}'. "
"Return code {1}".format(proc.pid,
ret_code))
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
# If the user is not the same as the user running the script...
# and this is a Posix system... and we are running as root
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_data])
subprocess.call(['chgrp', '-R', user, new_data])
socket_path = os.path.join(new_data, 'mysql.sock')
# If socket path is too long, use mkdtemp to create a tmp dir and
# use it instead to store the socket
if os.name == 'posix' and len(socket_path) > MAX_SOCKET_PATH_SIZE:
socket_path = os.path.join(tempfile.mkdtemp(), 'mysql.sock')
if not quiet:
print("# WARNING: The socket file path '{0}' is too long (>{1}), "
"using '{2}' instead".format(
os.path.join(new_data, 'mysql.sock'),
MAX_SOCKET_PATH_SIZE, socket_path))
cmd = {
'datadir': '--datadir={0}'.format(new_data),
'tmpdir': '--tmpdir={0}'.format(new_data),
'pid-file': '--pid-file={0}'.format(
os.path.join(new_data, "clone.pid")),
'port': '--port={0}'.format(new_port),
'server': '--server-id={0}'.format(options.get('new_id', 2)),
'basedir': '--basedir={0}'.format(mysql_basedir),
'socket': '--socket={0}'.format(socket_path),
}
if user:
cmd.update({'user': '******'.format(user)})
if mysqld_options:
if isinstance(mysqld_options, (list, tuple)):
cmd.update(dict(zip(mysqld_options, mysqld_options)))
else:
new_opts = mysqld_options.strip(" ")
# Drop the --mysqld=
if new_opts.startswith("--mysqld="):
new_opts = new_opts[9:]
if new_opts.startswith('"') and new_opts.endswith('"'):
list_ = shlex.split(new_opts.strip('"'))
cmd.update(dict(zip(list_, list_)))
elif new_opts.startswith("'") and new_opts.endswith("'"):
list_ = shlex.split(new_opts.strip("'"))
cmd.update(dict(zip(list_, list_)))
# Special case where there is only 1 option
elif len(new_opts.split("--")) == 1:
cmd.update({mysqld_options: mysqld_options})
else:
list_ = shlex.split(new_opts)
cmd.update(dict(zip(list_, list_)))
# set of options that must be surrounded with quotes
options_to_quote = set(["datadir", "tmpdir", "basedir", "socket",
"pid-file"])
# Strip spaces from each option
for key in cmd:
cmd[key] = cmd[key].strip(' ')
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
comment = " Startup command generated by mysqlserverclone.\n"
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("#{0}".format(comment))
elif os.name == 'nt' and cmd_file.endswith('.bat'):
cfile.write("REM{0}".format(comment))
else:
cfile.write("#{0}".format(comment))
start_cmd_lst = ["{0}{1}{0} --no-defaults".format(quote_char,
mysqld_path)]
# build start command
for key, val in cmd.iteritems():
if key in options_to_quote:
val = "{0}{1}{0}".format(quote_char, val)
start_cmd_lst.append(val)
cfile.write("{0}\n".format(" ".join(start_cmd_lst)))
cfile.close()
if os.name == "nt" and verbosity >= 1:
cmd.update({"console": "--console"})
start_cmd_lst = [mysqld_path, "--no-defaults"]
sorted_keys = sorted(cmd.keys())
start_cmd_lst.extend([cmd[val] for val in sorted_keys])
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print("# Startup command for new server:\n"
"{0}".format(" ".join(start_cmd_lst)))
proc = subprocess.Popen(start_cmd_lst, shell=False)
else:
proc = subprocess.Popen(start_cmd_lst, shell=False, stdout=fnull,
stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
if os.name == "posix":
new_sock = socket_path
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
i = 0
while i < start_timeout:
i += 1
time.sleep(1)
try:
server2.connect()
i = start_timeout + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == start_timeout:
raise UtilError("Unable to communicate with new instance. "
"Process id = {0}.".format(proc.pid))
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot']
if os.name == "posix":
cmd.append("--socket={0}".format(new_sock))
else:
cmd.append("--port={0}".format(int(new_port)))
cmd.extend(["password", root_pass])
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd, shell=False,
stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
start_timeout[in] Number of seconds to wait for server to start
"""
new_data = os.path.abspath(options.get('new_data', None))
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
user = options.get('user', 'root')
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
start_timeout = int(options.get('start_timeout', 10))
mysqld_options = options.get('mysqld_options', '')
force = options.get('force', False)
quote_char = "'" if os.name == "posix" else '"'
if not check_port_in_use('localhost', int(new_port)):
raise UtilError("Port {0} in use. Please choose an "
"available port.".format(new_port))
# Check if path to database files is greater than MAX_DIR_SIZE char,
if len(new_data) > MAX_DATADIR_SIZE and not force:
raise UtilError("The --new-data path '{0}' is too long "
"(> {1} characters). Please use a smaller one. "
"You can use the --force option to skip this "
"check".format(new_data, MAX_DATADIR_SIZE))
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % \
conn_val["host"]
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = os.path.normpath(rows[0][1])
# Cloning downed or offline server
else:
basedir = os.path.abspath(options.get("basedir", None))
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
new_data_deleted = False
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
new_data_deleted = True
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
os.mkdir(new_data)
except OSError as err:
raise UtilError("Unable to create directory '{0}', reason: {1}"
"".format(new_data, err.strerror))
# After create the new data directory, check for free space, so the errors
# regarding invalid or inaccessible path had been dismissed already.
# If not force specified verify and stop if there is not enough free space
if not force and os.path.exists(new_data) and \
estimate_free_space(new_data) < REQ_FREE_SPACE:
# Don't leave empty folders, delete new_data if was previously deleted
if os.path.exists(new_data) and new_data_deleted:
shutil.rmtree(new_data, True)
raise UtilError(
LOW_SPACE_ERRR_MSG.format(directory=new_data,
megabytes=REQ_FREE_SPACE))
# Check for warning of using --skip-innodb
mysqld_path = get_tool_path(basedir, "mysqld")
version_str = get_mysqld_version(mysqld_path)
# convert version_str from str tuple to integer tuple if possible
if version_str is not None:
version = tuple([int(digit) for digit in version_str])
else:
version = None
if mysqld_options is not None and ("--skip-innodb" in mysqld_options or
"--innodb" in mysqld_options) and version is not None and \
version >= (5, 7, 5):
print("# WARNING: {0}".format(WARN_OPT_SKIP_INNODB))
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
]
# From 5.7.6 version onwards, bootstrap is done via mysqld with the
# --initialize-insecure option, so no need to get information about the
# sql system tables that need to be loaded.
if version < (5, 7, 6):
system_tables = get_tool_path(basedir, "mysql_system_tables.sql",
False)
system_tables_data = get_tool_path(basedir,
"mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir,
"mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
locations.extend([
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
])
if verbosity >= 3 and not quiet:
print "# Location of files:"
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
fnull = open(os.devnull, 'w')
# For MySQL versions before 5.7.6, use regular bootstrap procedure.
if version < (5, 7, 6):
# Get bootstrap SQL statements
sql = list()
sql.append("CREATE DATABASE mysql;")
sql.append("USE mysql;")
innodb_disabled = False
if mysqld_options:
innodb_disabled = '--innodb=OFF' in mysqld_options
for sqlfile in [
system_tables, system_tables_data, test_data_timezone,
help_data
]:
lines = open(sqlfile, 'r').readlines()
# On MySQL 5.7.5, the root@localhost account creation was
# moved from the system_tables_data sql file into the
# mysql_install_db binary. Since we don't use mysql_install_db
# directly we need to create the root user account ourselves.
if (version is not None and version == (5, 7, 5)
and sqlfile == system_tables_data):
lines.extend(_CREATE_ROOT_USER)
for line in lines:
line = line.strip()
# Don't fail when InnoDB is turned off (Bug#16369955)
# (Ugly hack)
if (sqlfile == system_tables
and "SET @sql_mode_orig==@@SES" in line
and innodb_disabled):
for line in lines:
if 'SET SESSION sql_mode=@@sql' in line:
break
sql.append(line)
# Bootstap to setup mysql tables
cmd = [
mysqld_path,
"--no-defaults",
"--bootstrap",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir)),
]
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdin=subprocess.PIPE,
stdout=fnull,
stderr=fnull)
proc.communicate('\n'.join(sql))
# From 5.7.6 onwards, mysql_install_db has been replaced by mysqld and
# the --initialize option
else:
cmd = [
mysqld_path, "--initialize-insecure=on",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir))
]
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdin=subprocess.PIPE,
stdout=fnull,
stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
raise UtilError("Failed to kill process with pid '{0}'"
"".format(proc.pid))
else:
ret_code = subprocess.call("taskkill /F /T /PID "
"{0}".format(proc.pid),
shell=True)
# return code 0 means it was successful and 128 means it tried
# to kill a process that doesn't exist
if ret_code not in (0, 128):
raise UtilError("Failed to kill process with pid '{0}'. "
"Return code {1}".format(proc.pid, ret_code))
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
# If the user is not the same as the user running the script...
# and this is a Posix system... and we are running as root
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_data])
subprocess.call(['chgrp', '-R', user, new_data])
socket_path = os.path.join(new_data, 'mysql.sock')
# If socket path is too long, use mkdtemp to create a tmp dir and
# use it instead to store the socket
if os.name == 'posix' and len(socket_path) > MAX_SOCKET_PATH_SIZE:
socket_path = os.path.join(tempfile.mkdtemp(), 'mysql.sock')
if not quiet:
print(
"# WARNING: The socket file path '{0}' is too long (>{1}), "
"using '{2}' instead".format(
os.path.join(new_data, 'mysql.sock'), MAX_SOCKET_PATH_SIZE,
socket_path))
cmd = {
'datadir': '--datadir={0}'.format(new_data),
'tmpdir': '--tmpdir={0}'.format(new_data),
'pid-file':
'--pid-file={0}'.format(os.path.join(new_data, "clone.pid")),
'port': '--port={0}'.format(new_port),
'server': '--server-id={0}'.format(options.get('new_id', 2)),
'basedir': '--basedir={0}'.format(mysql_basedir),
'socket': '--socket={0}'.format(socket_path),
}
if user:
cmd.update({'user': '******'.format(user)})
if mysqld_options:
if isinstance(mysqld_options, (list, tuple)):
cmd.update(dict(zip(mysqld_options, mysqld_options)))
else:
new_opts = mysqld_options.strip(" ")
# Drop the --mysqld=
if new_opts.startswith("--mysqld="):
new_opts = new_opts[9:]
if new_opts.startswith('"') and new_opts.endswith('"'):
list_ = shlex.split(new_opts.strip('"'))
cmd.update(dict(zip(list_, list_)))
elif new_opts.startswith("'") and new_opts.endswith("'"):
list_ = shlex.split(new_opts.strip("'"))
cmd.update(dict(zip(list_, list_)))
# Special case where there is only 1 option
elif len(new_opts.split("--")) == 1:
cmd.update({mysqld_options: mysqld_options})
else:
list_ = shlex.split(new_opts)
cmd.update(dict(zip(list_, list_)))
# set of options that must be surrounded with quotes
options_to_quote = set(
["datadir", "tmpdir", "basedir", "socket", "pid-file"])
# Strip spaces from each option
for key in cmd:
cmd[key] = cmd[key].strip(' ')
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
comment = " Startup command generated by mysqlserverclone.\n"
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("#{0}".format(comment))
elif os.name == 'nt' and cmd_file.endswith('.bat'):
cfile.write("REM{0}".format(comment))
else:
cfile.write("#{0}".format(comment))
start_cmd_lst = [
"{0}{1}{0} --no-defaults".format(quote_char, mysqld_path)
]
# build start command
for key, val in cmd.iteritems():
if key in options_to_quote:
val = "{0}{1}{0}".format(quote_char, val)
start_cmd_lst.append(val)
cfile.write("{0}\n".format(" ".join(start_cmd_lst)))
cfile.close()
if os.name == "nt" and verbosity >= 1:
cmd.update({"console": "--console"})
start_cmd_lst = [mysqld_path, "--no-defaults"]
sorted_keys = sorted(cmd.keys())
start_cmd_lst.extend([cmd[val] for val in sorted_keys])
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print("# Startup command for new server:\n"
"{0}".format(" ".join(start_cmd_lst)))
proc = subprocess.Popen(start_cmd_lst, shell=False)
else:
proc = subprocess.Popen(start_cmd_lst,
shell=False,
stdout=fnull,
stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
if os.name == "posix":
new_sock = socket_path
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
i = 0
while i < start_timeout:
i += 1
time.sleep(1)
try:
server2.connect()
i = start_timeout + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == start_timeout:
raise UtilError("Unable to communicate with new instance. "
"Process id = {0}.".format(proc.pid))
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot']
if os.name == "posix":
cmd.append("--socket={0}".format(new_sock))
else:
cmd.append("--port={0}".format(int(new_port)))
cmd.extend(["password", root_pass])
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdout=fnull,
stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
class test(mutlib.System_test):
"""Template for diff_<object>_sql tests
This test executes a set of test cases for a given object definition pair.
It is a value result test.
The pair is stored as list of dictionary items in the following format:
test_object = {
'db1' : <name of first database>,
'db2' : <name of second database>,
'object_name' : <name of object>,
'server1_object' : <create statement for first server>,
'server2_object' : <create statement for second server>,
'comment' : <comment to be appended to test case name>,
'startup_cmds' : <array of commands to execute before test case>,
'shutdown_cmds' : <array of commands to execute after test case>,
# OPTIONAL - use for error code checking
'error_codes' : <array of 7 integers used for checking errors>,
# default = 1,0,0,1,0,0,1,1
# OPTIONAL - use for loading data (e.g. INSERT INTO)
'server1_data' : <array of insert commands for server1>
'server2_data' : <array of insert commands for server2>
}
self.test_objects.append(test_object)
For item in the dictionary of test objects, the following test cases are
executed:
- operation generated with --difftype=sql option, changes-for=server1
- SQL consumed
- operation rerun and check result against 'expected_result'
- operation generated with --difftype=sql option, changes-for=server2
- SQL consumed
- operation rerun and check result against 'expected_result'
- operation generated with --difftype=sql option,
changes-for=server1 with show-reverse
- operation generated with --difftype=sql option,
changes-for=server2 with show-reverse
To specify a new test object, do so in the setup method as described above
and call the template setup method. You can add multiple object pairs to
the list self.test_objects so that you can test different variants of the
CREATE statement to check for the various mechanisms of how diff generates
the SQL statements.
Note: the expected result for all test cases is 1 (errors found). It
shall be considered an error if this returns 0. Similarly, all
consumption runs is expected to generate a result of 0 and any other
value is considered an error.
You must supply the utility name via self.utility. Set it to either
'mysqldiff.py' for diff_sql* tests or 'mysqldbcompare.py' for
db_compare_sql* tests. Set it in the setup *before* calling
test_sql_template.test.setup().
"""
def check_prerequisites(self):
# if self.servers.get_server(0).check_version_compat(5, 6, 5):
# raise MUTLibError("Test requires server version prior to 5.6.5")
self.test_objects = []
# Need at least one server.
self.server1 = None
self.server2 = None
self.need_server = False
if not self.check_num_servers(2):
self.need_server = True
return self.check_num_servers(1)
def setup(self):
self.res_fname = "result.txt"
self.server1 = self.servers.get_server(0)
if self.need_server:
try:
self.servers.spawn_new_servers(2)
except MUTLibError, e:
raise MUTLibError("Cannot spawn needed servers: %s" % \
e.errmsg)
self.server2 = self.servers.get_server(1)
s1_conn = "--server1=" + self.build_connection_string(self.server1)
s2_conn = "--server2=" + self.build_connection_string(self.server2)
self.base_cmd = "%s %s %s " % (self.utility, s1_conn, s2_conn)
rows = self.server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine basedir of running "
"server.")
self.mysql_path = get_tool_path(basedir, "mysql")
return True
def _start_server(server_val, basedir, datadir, options={}):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
from mysql.utilities.common.tools import get_tool_path, get_mysqld_version
from mysql.utilities.common.server import Server
import time
verbosity = options.get("verbosity", 0)
mysqld_path = get_tool_path(basedir, "mysqld")
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
post_5_6 = version is not None and \
int(version[0]) >= 5 and int(version[1]) >= 6
# Start the instance
print "# Starting read-only instance of the server ...",
args = [
" -uroot",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
]
# It the server is 5.6 or later, we must use additional parameters
if post_5_6:
args_5_6 = [
"--skip-slave-start",
"--skip-innodb",
"--default-storage-engine=MYISAM",
"--default-tmp-storage-engine=MYISAM",
"--server-id=0",
]
args.extend(args_5_6)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
proc = subprocess.Popen(args, executable=mysqld_path)
else:
out = open(os.devnull, 'w')
proc = subprocess.Popen(args, executable=mysqld_path,
stdout=out, stderr=out)
server_options = {
'conn_info' : server_val,
'role' : "read_only",
}
server = Server(server_options)
# Now wait for the server to become ready - could be up to 10 seconds
# for Windows machines.
if os.name == "nt":
time.sleep(10)
else:
time.sleep(1)
server.connect()
print "done."
return server
def _start_server(server_val, basedir, datadir, options=None):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
start_timeout = options.get("start_timeout", 10)
mysqld_path = get_tool_path(basedir, "mysqld")
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
post_5_6 = version is not None and \
int(version[0]) >= 5 and int(version[1]) >= 6
# Start the instance
print "# Starting read-only instance of the server ...",
args = [
"--no-defaults",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
]
# It the server is 5.6 or later, we must use additional parameters
if post_5_6:
args_5_6 = [
"--skip-slave-start",
"--skip-innodb",
"--default-storage-engine=MYISAM",
"--default-tmp-storage-engine=MYISAM",
"--server-id=0",
]
args.extend(args_5_6)
args.insert(0, mysqld_path)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
subprocess.Popen(args, shell=False)
else:
out = open(os.devnull, 'w')
subprocess.Popen(args, shell=False, stdout=out, stderr=out)
server_options = {
'conn_info': server_val,
'role': "read_only",
}
server = Server(server_options)
# Try to connect to the server, waiting for the server to become ready
# (retry start_timeout times and wait 1 sec between each attempt).
# Note: It can take up to 10 seconds for Windows machines.
i = 0
while i < start_timeout:
# Reset error and wait 1 second.
error = None
time.sleep(1)
try:
server.connect()
break # Server ready (connect succeed)! Exit the for loop.
except UtilError as err:
# Store exception to raise later (if needed).
error = err
i += 1
# Raise last known exception (if unable to connect to the server)
if error:
raise error # pylint: disable=E0702
# See: http://www.logilab.org/ticket/3207
print "done."
return server
def _start_server(server_val, basedir, datadir, options={}):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
from mysql.utilities.common.tools import get_tool_path, get_mysqld_version
from mysql.utilities.common.server import Server
import time
verbosity = options.get("verbosity", 0)
mysqld_path = get_tool_path(basedir, "mysqld")
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
post_5_6 = version is not None and \
int(version[0]) >= 5 and int(version[1]) >= 6
# Start the instance
print "# Starting read-only instance of the server ...",
args = [
" -uroot",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
]
# It the server is 5.6 or later, we must use additional parameters
if post_5_6:
args_5_6 = [
"--skip-slave-start",
"--skip-innodb",
"--default-storage-engine=MYISAM",
"--default-tmp-storage-engine=MYISAM",
"--server-id=0",
]
args.extend(args_5_6)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
proc = subprocess.Popen(args, executable=mysqld_path)
else:
out = open(os.devnull, 'w')
proc = subprocess.Popen(args, executable=mysqld_path,
stdout=out, stderr=out)
server_options = {
'conn_info' : server_val,
'role' : "read_only",
}
server = Server(server_options)
# Now wait for the server to become ready - could be up to 10 seconds
# for Windows machines.
if os.name == "nt":
time.sleep(10)
else:
time.sleep(1)
server.connect()
print "done."
def _start_server(server_val, basedir, datadir, options=None):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
start_timeout = options.get("start_timeout", 10)
mysqld_path = get_tool_path(basedir, "mysqld", quote=True)
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
if version is not None and int(version[0]) >= 5:
post_5_5 = int(version[1]) >= 5
post_5_6 = int(version[1]) >= 6
post_5_7_4 = int(version[1]) >= 7 and int(version[2]) > 4
else:
print("# Warning: cannot get server version.")
post_5_5 = False
post_5_6 = False
post_5_7_4 = False
# Get the user executing the utility to use in the mysqld options.
# Note: the option --user=user_name is mandatory to start mysqld as root.
user_name = getpass.getuser()
# Start the instance
if verbosity > 0:
print "# Starting read-only instance of the server ..."
print "# --- BEGIN (server output) ---"
else:
print "# Starting read-only instance of the server ...",
args = shlex.split(mysqld_path)
args.extend([
"--no-defaults",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
"--user={0}".format(user_name),
])
# It the server is 5.6 or later, we must use additional parameters
if post_5_5:
server_args = [
"--skip-slave-start",
"--default-storage-engine=MYISAM",
"--server-id=0",
]
if post_5_6:
server_args.append("--default-tmp-storage-engine=MYISAM")
if not post_5_7_4:
server_args.append("--skip-innodb")
args.extend(server_args)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
subprocess.Popen(args, shell=False)
else:
out = open(os.devnull, 'w')
subprocess.Popen(args, shell=False, stdout=out, stderr=out)
server_options = {
'conn_info': server_val,
'role': "read_only",
}
server = Server(server_options)
# Try to connect to the server, waiting for the server to become ready
# (retry start_timeout times and wait 1 sec between each attempt).
# Note: It can take up to 10 seconds for Windows machines.
i = 0
while i < start_timeout:
# Reset error and wait 1 second.
error = None
time.sleep(1)
try:
server.connect()
break # Server ready (connect succeed)! Exit the for loop.
except UtilError as err:
# Store exception to raise later (if needed).
error = err
i += 1
# Indicate end of the server output.
if verbosity > 0:
print "# --- END (server output) ---"
# Raise last known exception (if unable to connect to the server)
if error:
# See: http://www.logilab.org/ticket/3207
# pylint: disable=E0702
raise error
if verbosity > 0:
print "# done (server started)."
else:
print "done."
return server
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
start_timeout[in] Number of seconds to wait for server to start
"""
new_data = os.path.abspath(options.get('new_data', None))
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
user = options.get('user', 'root')
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
start_timeout = int(options.get('start_timeout', 10))
mysqld_options = options.get('mysqld_options', '')
if not check_port_in_use('localhost', int(new_port)):
raise UtilError("Port {0} in use. Please choose an "
"available port.".format(new_port))
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % \
conn_val["host"]
basedir = ""
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = os.path.normpath(rows[0][1])
# Cloning downed or offline server
else:
basedir = os.path.abspath(options.get("basedir", None))
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
os.mkdir(new_data)
except:
raise UtilError("Unable to create directory '%s'" % new_data)
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqld_path = get_tool_path(basedir, "mysqld")
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys",
False, False)
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False)
system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
if verbosity >= 3 and not quiet:
print "# Location of files:"
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
]
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
# Get bootstrap SQL statements
sql = list()
sql.append("CREATE DATABASE mysql;")
sql.append("USE mysql;")
innodb_disabled = False
if mysqld_options:
innodb_disabled = '--innodb=OFF' in mysqld_options
for sqlfile in [system_tables, system_tables_data, test_data_timezone,
help_data]:
lines = open(sqlfile, 'r').readlines()
for line in lines:
line = line.strip()
# Don't fail when InnoDB is turned off (Bug#16369955) (Ugly hack)
if (sqlfile == system_tables and
"SET @sql_mode_orig==@@SES" in line and innodb_disabled):
for line in lines:
if 'SET SESSION sql_mode=@@sql' in line:
break
sql.append(line)
# Bootstap to setup mysql tables
fnull = open(os.devnull, 'w')
cmd = [
mysqld_path,
"--no-defaults",
"--bootstrap",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir)),
]
proc = None
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE,
stdout=fnull, stderr=fnull)
proc.communicate('\n'.join(sql))
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
raise UtilError("Failed to kill process with pid '{0}'"
"".format(proc.pid))
else:
ret_code = subprocess.call("taskkill /F /T /PID "
"{0}".format(proc.pid), shell=True)
# return code 0 means it was successful and 128 means it tried
# to kill a process that doesn't exist
if ret_code not in (0, 128):
raise UtilError("Failed to kill process with pid '{0}'. "
"Return code {1}".format(proc.pid,
ret_code))
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
# If the user is not the same as the user running the script...
# and this is a Posix system... and we are running as root
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_data])
subprocess.call(['chgrp', '-R', user, new_data])
cmd = [mysqld_path, '--no-defaults']
cmd.extend([
'--datadir={0}'.format(new_data),
'--tmpdir={0}'.format(new_data),
'--pid-file={0}'.format(os.path.join(new_data, "clone.pid")),
'--port={0}'.format(new_port),
'--server-id={0}'.format(options.get('new_id', 2)),
'--basedir={0}'.format(mysql_basedir),
'--socket={0}'.format(os.path.join(new_data, 'mysql.sock')),
])
if user:
cmd.append('--user={0}'.format(user))
if mysqld_options:
if isinstance(mysqld_options, (list, tuple)):
cmd.extend(mysqld_options)
else:
new_opts = mysqld_options.strip(" ")
# Drop the --mysqld=
if new_opts.startswith("--mysqld="):
new_opts = new_opts[9:]
if new_opts.startswith('"') and new_opts.endswith('"'):
cmd.extend(shlex.split(new_opts.strip('"')))
elif new_opts.startswith("'") and new_opts.endswith("'"):
cmd.extend(shlex.split(new_opts.strip("'")))
# Special case where there is only 1 option
elif len(new_opts.split("--")) == 1:
cmd.append(mysqld_options)
else:
cmd.extend(shlex.split(new_opts))
# Strip spaces from each option
cmd = [opt.strip(' ') for opt in cmd]
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("# Startup command generated by mysqlserverclone.\n")
cfile.write("%s\n" % cmd)
cfile.close()
if os.name == "nt" and verbosity >= 1:
cmd.append("--console")
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print("# Startup command for new server:\n"
"{0}".format(" ".join(cmd)))
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
port_int = None
if os.name == "posix":
new_sock = os.path.join(new_data, "mysql.sock")
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
i = 0
while i < start_timeout:
i += 1
time.sleep(1)
try:
server2.connect()
i = start_timeout + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == start_timeout:
raise UtilError("Unable to communicate with new instance. "
"Process id = {0}.".format(proc.pid))
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot']
if os.name == "posix":
cmd.append("--socket={0}".format(new_sock))
else:
cmd.append("--port={0}".format(int(new_port)))
cmd.extend(["password", root_pass])
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd, shell=False,
stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
start_timeout[in] Number of seconds to wait for server to start
"""
new_data = os.path.abspath(options.get('new_data', None))
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
user = options.get('user', 'root')
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
start_timeout = int(options.get('start_timeout', 10))
mysqld_options = options.get('mysqld_options', '')
if not check_port_in_use('localhost', int(new_port)):
raise UtilError("Port {0} in use. Please choose an "
"available port.".format(new_port))
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % \
conn_val["host"]
basedir = ""
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = os.path.normpath(rows[0][1])
# Cloning downed or offline server
else:
basedir = os.path.abspath(options.get("basedir", None))
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
os.mkdir(new_data)
except:
raise UtilError("Unable to create directory '%s'" % new_data)
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqld_path = get_tool_path(basedir, "mysqld")
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False,
False)
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False)
system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
if verbosity >= 3 and not quiet:
print "# Location of files:"
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
]
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
# Get bootstrap SQL statements
sql = list()
sql.append("CREATE DATABASE mysql;")
sql.append("USE mysql;")
innodb_disabled = False
if mysqld_options:
innodb_disabled = '--innodb=OFF' in mysqld_options
for sqlfile in [
system_tables, system_tables_data, test_data_timezone, help_data
]:
lines = open(sqlfile, 'r').readlines()
for line in lines:
line = line.strip()
# Don't fail when InnoDB is turned off (Bug#16369955) (Ugly hack)
if (sqlfile == system_tables
and "SET @sql_mode_orig==@@SES" in line
and innodb_disabled):
for line in lines:
if 'SET SESSION sql_mode=@@sql' in line:
break
sql.append(line)
# Bootstap to setup mysql tables
fnull = open(os.devnull, 'w')
cmd = [
mysqld_path,
"--no-defaults",
"--bootstrap",
"--datadir={0}".format(new_data),
"--basedir={0}".format(os.path.abspath(mysql_basedir)),
]
proc = None
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdin=subprocess.PIPE,
stdout=fnull,
stderr=fnull)
proc.communicate('\n'.join(sql))
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
raise UtilError("Failed to kill process with pid '{0}'"
"".format(proc.pid))
else:
ret_code = subprocess.call("taskkill /F /T /PID "
"{0}".format(proc.pid),
shell=True)
# return code 0 means it was successful and 128 means it tried
# to kill a process that doesn't exist
if ret_code not in (0, 128):
raise UtilError("Failed to kill process with pid '{0}'. "
"Return code {1}".format(proc.pid, ret_code))
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
# If the user is not the same as the user running the script...
# and this is a Posix system... and we are running as root
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_data])
subprocess.call(['chgrp', '-R', user, new_data])
cmd = [mysqld_path, '--no-defaults']
cmd.extend([
'--datadir={0}'.format(new_data),
'--tmpdir={0}'.format(new_data),
'--pid-file={0}'.format(os.path.join(new_data, "clone.pid")),
'--port={0}'.format(new_port),
'--server-id={0}'.format(options.get('new_id', 2)),
'--basedir={0}'.format(mysql_basedir),
'--socket={0}'.format(os.path.join(new_data, 'mysql.sock')),
])
if user:
cmd.append('--user={0}'.format(user))
if mysqld_options:
if isinstance(mysqld_options, (list, tuple)):
cmd.extend(mysqld_options)
else:
new_opts = mysqld_options.strip(" ")
# Drop the --mysqld=
if new_opts.startswith("--mysqld="):
new_opts = new_opts[9:]
if new_opts.startswith('"') and new_opts.endswith('"'):
cmd.extend(shlex.split(new_opts.strip('"')))
elif new_opts.startswith("'") and new_opts.endswith("'"):
cmd.extend(shlex.split(new_opts.strip("'")))
# Special case where there is only 1 option
elif len(new_opts.split("--")) == 1:
cmd.append(mysqld_options)
else:
cmd.extend(shlex.split(new_opts))
# Strip spaces from each option
cmd = [opt.strip(' ') for opt in cmd]
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("# Startup command generated by mysqlserverclone.\n")
cfile.write("%s\n" % cmd)
cfile.close()
if os.name == "nt" and verbosity >= 1:
cmd.append("--console")
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print("# Startup command for new server:\n"
"{0}".format(" ".join(cmd)))
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd, shell=False, stdout=fnull, stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
port_int = None
if os.name == "posix":
new_sock = os.path.join(new_data, "mysql.sock")
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
i = 0
while i < start_timeout:
i += 1
time.sleep(1)
try:
server2.connect()
i = start_timeout + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == start_timeout:
raise UtilError("Unable to communicate with new instance. "
"Process id = {0}.".format(proc.pid))
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
cmd = [mysqladmin_path, '--no-defaults', '-v', '-uroot']
if os.name == "posix":
cmd.append("--socket={0}".format(new_sock))
else:
cmd.append("--port={0}".format(int(new_port)))
cmd.extend(["password", root_pass])
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=False)
else:
proc = subprocess.Popen(cmd,
shell=False,
stdout=fnull,
stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
"""
from mysql.utilities.common.server import Server
from mysql.utilities.exception import UtilError
from mysql.utilities.common.tools import get_tool_path
new_data = options.get("new_data", None)
new_port = options.get("new_port", "3307")
root_pass = options.get("root_pass", None)
verbosity = options.get("verbosity", 0)
quiet = options.get("quiet", False)
cmd_file = options.get("cmd_file", None)
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {"conn_info": conn_val, "role": "source"}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % conn_val["host"]
basedir = ""
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = rows[0][1]
# Cloning downed or offline server
else:
basedir = options.get("basedir", None)
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
# If datadir exists, delete it
if os.path.exists(new_data):
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not quiet:
print "# Creating new data directory..."
if not os.path.exists(new_data):
try:
res = os.mkdir(new_data)
except:
raise UtilError("Unable to create directory '%s'" % new_data)
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqld_path = get_tool_path(basedir, "mysqld")
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False, False)
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False)
system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql", False)
test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql", False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
if verbosity >= 3 and not quiet:
print "# Location of files:"
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
]
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
# Create the bootstrap file
f_boot = open("bootstrap.sql", "w")
f_boot.write("CREATE DATABASE mysql;\n")
f_boot.write("USE mysql;\n")
f_boot.writelines(open(system_tables).readlines())
f_boot.writelines(open(system_tables_data).readlines())
f_boot.writelines(open(test_data_timezone).readlines())
f_boot.writelines(open(help_data).readlines())
f_boot.close()
# Bootstap to setup mysql tables
fnull = open(os.devnull, "w")
cmd = (
mysqld_path
+ " --no-defaults --bootstrap "
+ " --datadir=%s --basedir=%s " % (new_data, mysql_basedir)
+ " < bootstrap.sql"
)
proc = None
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
cmd = mysqld_path + " --no-defaults "
if options.get("mysqld_options", None):
cmd += options.get("mysqld_options") + " --user=root "
cmd += "--datadir=%s " % (new_data)
cmd += "--tmpdir=%s " % (new_data)
cmd += "--pid-file=%s " % os.path.join(new_data, "clone.pid")
cmd += "--port=%s " % (new_port)
cmd += "--server-id=%s " % (options.get("new_id", 2))
cmd += "--basedir=%s " % (mysql_basedir)
cmd += "--socket=%s/mysql.sock " % (new_data)
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, "w")
if os.name == "posix" and cmd_file.endswith(".sh"):
cfile.write("#!/bin/sh\n")
cfile.write("# Startup command generated by mysqlserverclone.\n")
cfile.write("%s\n" % cmd)
cfile.close()
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print "# Startup command for new server:\n%s" % cmd
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
port_int = None
if os.name == "posix":
new_sock = os.path.join(new_data, "mysql.sock")
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock,
}
server2_options = {"conn_info": conn, "role": "clone"}
server2 = Server(server2_options)
stop = 10 # stop after 10 attempts
i = 0
while i < stop:
i += 1
time.sleep(1)
try:
server2.connect()
i = stop + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == stop:
raise UtilError("Unable to communicate with new instance.")
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
if os.name == "posix":
cmd = mysqladmin_path + " --no-defaults -v -uroot " + "--socket=%s password %s " % (new_sock, root_pass)
else:
cmd = mysqladmin_path + " --no-defaults -v -uroot " + "password %s --port=%s" % (root_pass, int(new_port))
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
self.drop_all()
data_file = os.path.normpath("./std_data/basic_data.sql")
try:
res = self.server1.read_and_exec_SQL(data_file, self.debug)
except MUTLibError, e:
raise MUTLibError("Failed to read commands from file %s: " % \
data_file + e.errmsg)
rows = self.server2.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine basedir of running "
"server.")
self.mysql_path = get_tool_path(basedir, "mysql")
return True
def show_data(self, tbl):
comment = "Showing data for table %s \n" % tbl
self.results.append(comment)
if os.name == "posix":
cmd = "%s %s util_test -e 'SELECT * FROM %s'" % \
(self.mysql_path, self.server2_conn, tbl)
else:
cmd = '%s %s util_test -e "SELECT * FROM %s"' % \
(self.mysql_path, self.server2_conn, tbl)
res = self.exec_util(cmd, self.res_fname, True)
if res != 0:
raise MUTLibError("%s: failed" % comment)
def _server_info(server_val, get_defaults=False, options=None):
"""Show information about a running server
This method gathers information from a running server. This information is
returned as a tuple to be displayed to the user in a format specified. The
information returned includes the following:
* server connection information
* version number of the server
* data directory path
* base directory path
* plugin directory path
* configuration file location and name
* current binary log file
* current binary log position
* current relay log file
* current relay log position
server_val[in] the server connection values or a connected server
get_defaults[in] if True, get the default settings for the server
options[in] options for connecting to the server
Return tuple - information about server
"""
if options is None:
options = {}
# Parse source connection values
source_values = parse_connection(server_val, None, options)
# Connect to the server
conn_options = {
'version': "5.1.30",
}
servers = connect_servers(source_values, None, conn_options)
server = servers[0]
params_dict = defaultdict(str)
# Initialize list of warnings
params_dict['warnings'] = []
# Identify server by string: 'host:port[:socket]'.
server_id = "{0}:{1}".format(source_values['host'], source_values['port'])
if source_values.get('socket', None):
server_id = "{0}:{1}".format(server_id, source_values.get('socket'))
params_dict['server'] = server_id
# Get _SERVER_VARIABLES values from the server
for server_var in _SERVER_VARIABLES:
res = server.show_server_variable(server_var)
if res:
params_dict[server_var] = res[0][1]
else:
raise UtilError("Unable to determine {0} of server '{1}'"
".".format(server_var, server_id))
# Get _LOG_FILES_VARIABLES values from the server
for msg, log_tpl in _LOG_FILES_VARIABLES.iteritems():
res = server.show_server_variable(log_tpl.log_name)
if res:
# Check if log is turned off
params_dict[log_tpl.log_name] = res[0][1]
# If logs are turned off, skip checking information about the file
if res[0][1] in ('', 'OFF'):
continue
# Logging is enabled, so we can get get information about log_file
# unless it is log_error because in that case we already have it.
if log_tpl.log_file is not None: # if it is not log_error
log_file = server.show_server_variable(
log_tpl.log_file)[0][1]
params_dict[log_tpl.log_file] = log_file
else: # log error, so log_file_name is already on params_dict
log_file = params_dict[log_tpl.log_name]
# Now get the information about the size of the logs
try:
params_dict[log_tpl.log_file_size] = "{0} bytes".format(
os.path.getsize(log_file))
except os.error:
# if we are unable to get the log_file_size
params_dict[log_tpl.log_file_size] = ''
warning_msg = _WARNING_TEMPLATE.format(msg, log_file)
params_dict['warnings'].append(warning_msg)
else:
params_dict['warnings'].append("Unable to get information "
"regarding variable '{0}'"
).format(msg)
# if audit_log plugin is installed and enabled
if server.supports_plugin('audit'):
res = server.show_server_variable('audit_log_file')
if res:
# Audit_log variable might be a relative path to the datadir,
# so it needs to be treated accordingly
if not os.path.isabs(res[0][1]):
params_dict['audit_log_file'] = os.path.join(
params_dict['datadir'], res[0][1])
else:
params_dict['audit_log_file'] = res[0][1]
# Add audit_log field to the _COLUMNS List unless it is already
# there
if 'audit_log_file' not in _COLUMNS_SET:
_COLUMNS.append('audit_log_file')
_COLUMNS.append('audit_log_file_size')
_COLUMNS_SET.add('audit_log_file')
try:
params_dict['audit_log_file_size'] = "{0} bytes".format(
os.path.getsize(params_dict['audit_log_file']))
except os.error:
# If we are unable to get the size of the audit_log_file
params_dict['audit_log_file_size'] = ''
warning_msg = _WARNING_TEMPLATE.format(
"audit log",
params_dict['audit_log_file']
)
params_dict['warnings'].append(warning_msg)
# Build search path for config files
if os.name == "posix":
my_def_search = ["/etc/my.cnf", "/etc/mysql/my.cnf",
os.path.join(params_dict['basedir'], "my.cnf"),
"~/.my.cnf"]
else:
my_def_search = [r"c:\windows\my.ini", r"c:\my.ini", r"c:\my.cnf",
os.path.join(os.curdir, "my.ini")]
my_def_search.append(os.path.join(os.curdir, "my.cnf"))
# Get server's default configuration values.
defaults = []
if get_defaults:
# Can only get defaults for local servers (need to access local data).
if server.is_alias('localhost'):
try:
my_def_path = get_tool_path(params_dict['basedir'],
"my_print_defaults")
except UtilError as err:
raise UtilError("Unable to retrieve the defaults data "
"(requires access to my_print_defaults): {0} "
"(basedir: {1})".format(err.errmsg,
params_dict['basedir'])
)
out_file = tempfile.TemporaryFile()
# Execute tool: <basedir>/my_print_defaults mysqld
subprocess.call([my_def_path, "mysqld"], stdout=out_file)
out_file.seek(0)
# Get defaults data from temp output file.
defaults.append("\nDefaults for server {0}".format(server_id))
for line in out_file.readlines():
defaults.append(line.rstrip())
else:
# Remote server; Cannot get the defaults data.
defaults.append("\nWARNING: The utility can not get defaults from "
"a remote host.")
# Find config file
config_file = ""
for search_path in my_def_search:
if os.path.exists(search_path):
if len(config_file) > 0:
config_file = "{0}, {1}".format(config_file, search_path)
else:
config_file = search_path
params_dict['config_file'] = config_file
# Find binary log, relay log
params_dict['binary_log'], params_dict['binary_log_pos'] = _get_binlog(
server)
params_dict['relay_log'], params_dict['relay_log_pos'] = _get_relay_log(
server)
server.disconnect()
return params_dict, defaults
def _server_info(server_val, get_defaults=False, options={}):
"""Show information about a running server
This method gathers information from a running server. This information is
returned as a tuple to be displayed to the user in a format specified. The
information returned includes the following:
* server connection information
* version number of the server
* data directory path
* base directory path
* plugin directory path
* configuration file location and name
* current binary log file
* current binary log position
* current relay log file
* current relay log position
server_val[in] the server connection values or a connected server
get_defaults[in] if True, get the default settings for the server
options[in] options for connecting to the server
Return tuple - information about server
"""
import tempfile
from mysql.utilities.common.server import connect_servers
from mysql.utilities.common.tools import get_tool_path
verbosity = options.get("verbosity", 0)
# Parse source connection values
source_values = parse_connection(server_val)
# Connect to the server
conn_options = {
'version' : "5.1.30",
}
servers = connect_servers(source_values, None, conn_options)
server = servers[0]
rows = server.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise UtilError("Unable to determine basedir of running server.")
my_def_search = []
my_def_path = get_tool_path(basedir, "my_print_defaults")
if os.name == "posix":
my_def_search = ["/etc/my.cnf", "/etc/mysql/my.cnf",
os.path.join(basedir, "my.cnf"), "~/.my.cnf"]
else:
my_def_search = ["c:\windows\my.ini","c:\my.ini", "c:\my.cnf",
os.path.join(os.curdir, "my.ini")]
my_def_search.append(os.path.join(os.curdir, "my.cnf"))
# Make 'key' value
server_id = source_values['host']
# Use string mapping because port is an integer
server_id += ":%s" % source_values['port']
if source_values.get('socket', None) is not None:
server_id += ":" + source_values.get('socket')
defaults = []
if get_defaults:
if verbosity > 0:
file = tempfile.TemporaryFile()
else:
file = open(os.devnull, "w+b")
subprocess.call([my_def_path, "mysqld"], stdout=file)
file.seek(0)
defaults.append("\nDefaults for server " + server_id)
for line in file.readlines():
defaults.append(line.rstrip())
# Get server version
version = None
try:
res = server.show_server_variable('version')
version = res[0][1]
except:
raise UtilError("Cannot get version for server " + server_id)
# Find config file
config_file = ""
for search_path in my_def_search:
if os.path.exists(search_path):
if len(config_file) > 0:
config_file += ", " + search_path
else:
config_file = search_path
# Find datadir, basedir, plugin-dir, binary log, relay log
res = server.show_server_variable("datadir")
datadir = res[0][1]
res = server.show_server_variable("basedir")
basedir = res[0][1]
res = server.show_server_variable("plugin_dir")
plugin_dir = res[0][1]
binlog, binlog_pos = _get_binlog(server)
relay_log, relay_log_pos = _get_relay_log(server)
server.disconnect()
return ((server_id, version, datadir, basedir, plugin_dir, config_file,
binlog, binlog_pos, relay_log, relay_log_pos), defaults)
def _start_server(server_val, basedir, datadir, options=None):
"""Start an instance of a server in read only mode
This method is used to start the server in read only mode. It will launch
the server with --skip-grant-tables and --read_only options set.
Caller must stop the server with _stop_server().
server_val[in] dictionary of server connection values
basedir[in] the base directory for the server
datadir[in] the data directory for the server
options[in] dictionary of options (verbosity)
"""
if options is None:
options = {}
verbosity = options.get("verbosity", 0)
start_timeout = options.get("start_timeout", 10)
mysqld_path = get_tool_path(basedir, "mysqld")
print "# Server is offline."
# Check server version
print "# Checking server version ...",
version = get_mysqld_version(mysqld_path)
print "done."
post_5_6 = version is not None and \
int(version[0]) >= 5 and int(version[1]) >= 6
# Start the instance
print "# Starting read-only instance of the server ...",
args = [
"--no-defaults",
"--skip-grant-tables",
"--read_only",
"--port=%(port)s" % server_val,
"--basedir=" + basedir,
"--datadir=" + datadir,
]
# It the server is 5.6 or later, we must use additional parameters
if post_5_6:
args_5_6 = [
"--skip-slave-start",
"--skip-innodb",
"--default-storage-engine=MYISAM",
"--default-tmp-storage-engine=MYISAM",
"--server-id=0",
]
args.extend(args_5_6)
args.insert(0, mysqld_path)
socket = server_val.get('unix_socket', None)
if socket is not None:
args.append("--socket=%(unix_socket)s" % server_val)
if verbosity > 0:
subprocess.Popen(args, shell=False)
else:
out = open(os.devnull, 'w')
subprocess.Popen(args, shell=False, stdout=out, stderr=out)
server_options = {
'conn_info': server_val,
'role': "read_only",
}
server = Server(server_options)
# Try to connect to the server, waiting for the server to become ready
# (retry start_timeout times and wait 1 sec between each attempt).
# Note: It can take up to 10 seconds for Windows machines.
i = 0
while i < start_timeout:
# Reset error and wait 1 second.
error = None
time.sleep(1)
try:
server.connect()
break # Server ready (connect succeed)! Exit the for loop.
except UtilError as err:
# Store exception to raise later (if needed).
error = err
i += 1
# Raise last known exception (if unable to connect to the server)
if error:
raise error # pylint: disable=E0702
# See: http://www.logilab.org/ticket/3207
print "done."
return server
def mysql_tools_found():
# mysql_config_editor needs to be accessible to run the tests
try:
return get_tool_path(None, "mysql_config_editor", search_PATH=True)
except Exception:
return None
def _server_info(server_val, get_defaults=False, options={}):
"""Show information about a running server
This method gathers information from a running server. This information is
returned as a tuple to be displayed to the user in a format specified. The
information returned includes the following:
* server connection information
* version number of the server
* data directory path
* base directory path
* plugin directory path
* configuration file location and name
* current binary log file
* current binary log position
* current relay log file
* current relay log position
server_val[in] the server connection values or a connected server
get_defaults[in] if True, get the default settings for the server
options[in] options for connecting to the server
Return tuple - information about server
"""
import tempfile
from mysql.utilities.common.server import connect_servers
from mysql.utilities.common.tools import get_tool_path
verbosity = options.get("verbosity", 0)
# Parse source connection values
source_values = parse_connection(server_val, None, options)
# Connect to the server
conn_options = {
'version' : "5.1.30",
}
servers = connect_servers(source_values, None, conn_options)
server = servers[0]
rows = server.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise UtilError("Unable to determine basedir of running server.")
my_def_search = []
my_def_path = get_tool_path(basedir, "my_print_defaults")
if os.name == "posix":
my_def_search = ["/etc/my.cnf", "/etc/mysql/my.cnf",
os.path.join(basedir, "my.cnf"), "~/.my.cnf"]
else:
my_def_search = ["c:\windows\my.ini","c:\my.ini", "c:\my.cnf",
os.path.join(os.curdir, "my.ini")]
my_def_search.append(os.path.join(os.curdir, "my.cnf"))
# Make 'key' value
server_id = source_values['host']
# Use string mapping because port is an integer
server_id += ":%s" % source_values['port']
if source_values.get('socket', None) is not None:
server_id += ":" + source_values.get('socket')
defaults = []
if get_defaults:
if verbosity > 0:
file = tempfile.TemporaryFile()
else:
file = open(os.devnull, "w+b")
subprocess.call([my_def_path, "mysqld"], stdout=file)
file.seek(0)
defaults.append("\nDefaults for server " + server_id)
for line in file.readlines():
defaults.append(line.rstrip())
# Get server version
version = None
try:
res = server.show_server_variable('version')
version = res[0][1]
except:
raise UtilError("Cannot get version for server " + server_id)
# Find config file
config_file = ""
for search_path in my_def_search:
if os.path.exists(search_path):
if len(config_file) > 0:
config_file += ", " + search_path
else:
config_file = search_path
# Find datadir, basedir, plugin-dir, binary log, relay log
res = server.show_server_variable("datadir")
datadir = res[0][1]
res = server.show_server_variable("basedir")
basedir = res[0][1]
res = server.show_server_variable("plugin_dir")
plugin_dir = res[0][1]
binlog, binlog_pos = _get_binlog(server)
relay_log, relay_log_pos = _get_relay_log(server)
server.disconnect()
return ((server_id, version, datadir, basedir, plugin_dir, config_file,
binlog, binlog_pos, relay_log, relay_log_pos), defaults)
def mysql_tools_found():
# mysql_config_editor needs to be accessible to run the tests
try:
return get_tool_path(None, "mysql_config_editor", search_PATH=True)
except Exception:
return None
def clone_server(conn_val, options):
"""Clone an existing server
This method creates a new instance of a running server using a datadir
set to the new_data parametr, with a port set to new_port, server_id
set to new_id and a root password of root_pass. You can also specify
additional parameters for the mysqld command line as well as turn on
verbosity mode to display more diagnostic information during the clone
process.
The method will build a new base database installation from the .sql
files used to construct a new installation. Once the database is
created, the server will be started.
dest_val[in] a dictionary containing connection information
including:
(user, password, host, port, socket)
options[in] dictionary of options:
new_data[in] An existing path to create the new database and use
as datadir for new instance
(default = None)
new_port[in] Port number for new instance
(default = 3307)
new_id[in] Server_id for new instance
(default = 2)
root_pass[in] Password for root user on new instance (optional)
mysqld_options[in] Additional command line options for mysqld
verbosity[in] Print additional information during operation
(default is 0)
quiet[in] If True, do not print messages.
(default is False)
cmd_file[in] file name to write startup command
"""
from mysql.utilities.common.server import Server
from mysql.utilities.exception import UtilError
from mysql.utilities.common.tools import get_tool_path
new_data = options.get('new_data', None)
new_port = options.get('new_port', '3307')
root_pass = options.get('root_pass', None)
verbosity = options.get('verbosity', 0)
quiet = options.get('quiet', False)
cmd_file = options.get('cmd_file', None)
# Clone running server
if conn_val is not None:
# Try to connect to the MySQL database server.
server1_options = {
'conn_info': conn_val,
'role': "source",
}
server1 = Server(server1_options)
server1.connect()
if not quiet:
print "# Cloning the MySQL server running on %s." % conn_val["host"]
basedir = ""
# Get basedir
rows = server1.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = rows[0][1]
# Cloning downed or offline server
else:
basedir = options.get("basedir", None)
if not quiet:
print "# Cloning the MySQL server located at %s." % basedir
# If datadir exists, has data, and user said it was Ok, delete it
if os.path.exists(new_data) and options.get("delete", False) and \
os.listdir(new_data):
shutil.rmtree(new_data, True)
# Create new data directory if it does not exist
if not os.path.exists(new_data):
if not quiet:
print "# Creating new data directory..."
try:
res = os.mkdir(new_data)
except:
raise UtilError("Unable to create directory '%s'" % new_data)
if not quiet:
print "# Configuring new instance..."
print "# Locating mysql tools..."
mysqld_path = get_tool_path(basedir, "mysqld")
mysqladmin_path = get_tool_path(basedir, "mysqladmin")
mysql_basedir = get_tool_path(basedir, "share/english/errgmsg.sys", False,
False)
mysql_basedir = basedir
if os.path.exists(os.path.join(basedir, "local/mysql/share/")):
mysql_basedir = os.path.join(mysql_basedir, "local/mysql/")
# for source trees
elif os.path.exists(os.path.join(basedir, "/sql/share/english/")):
mysql_basedir = os.path.join(mysql_basedir, "/sql/")
system_tables = get_tool_path(basedir, "mysql_system_tables.sql", False)
system_tables_data = get_tool_path(basedir, "mysql_system_tables_data.sql",
False)
test_data_timezone = get_tool_path(basedir, "mysql_test_data_timezone.sql",
False)
help_data = get_tool_path(basedir, "fill_help_tables.sql", False)
if verbosity >= 3 and not quiet:
print "# Location of files:"
locations = [
("mysqld", mysqld_path),
("mysqladmin", mysqladmin_path),
("mysql_system_tables.sql", system_tables),
("mysql_system_tables_data.sql", system_tables_data),
("mysql_test_data_timezone.sql", test_data_timezone),
("fill_help_tables.sql", help_data),
]
if cmd_file is not None:
locations.append(("write startup command to", cmd_file))
for location in locations:
print "# % 28s: %s" % location
# Create the new mysql data with mysql_import_db-like process
if not quiet:
print "# Setting up empty database and mysql tables..."
# Create the bootstrap file
f_boot = open("bootstrap.sql", 'w')
f_boot.write("CREATE DATABASE mysql;\n")
f_boot.write("USE mysql;\n")
f_boot.writelines(open(system_tables).readlines())
f_boot.writelines(open(system_tables_data).readlines())
f_boot.writelines(open(test_data_timezone).readlines())
f_boot.writelines(open(help_data).readlines())
f_boot.close()
# Bootstap to setup mysql tables
fnull = open(os.devnull, 'w')
cmd = mysqld_path + " --no-defaults --bootstrap " + \
" --datadir=%s --basedir=%s " % (new_data, mysql_basedir) + \
" < bootstrap.sql"
proc = None
if verbosity >= 1 and not quiet:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
# Kill subprocess just in case it didn't finish - Ok if proc doesn't exist
if int(res) != 0:
if os.name == "posix":
try:
os.kill(proc.pid, subprocess.signal.SIGTERM)
except OSError:
pass
else:
try:
retval = subprocess.Popen("taskkill /F /T /PID %i" % proc.pid,
shell=True)
except:
pass
# Drop the bootstrap file
if os.path.isfile("bootstrap.sql"):
os.unlink("bootstrap.sql")
# Start the instance
if not quiet:
print "# Starting new instance of the server..."
cmd = mysqld_path + " --no-defaults "
if options.get('mysqld_options', None):
cmd += options.get('mysqld_options') + " --user=root "
cmd += "--datadir=%s " % (new_data)
cmd += "--tmpdir=%s " % (new_data)
cmd += "--pid-file=%s " % os.path.join(new_data, "clone.pid")
cmd += "--port=%s " % (new_port)
cmd += "--server-id=%s " % (options.get('new_id', 2))
cmd += "--basedir=%s " % (mysql_basedir)
cmd += "--socket=%s/mysql.sock " % (new_data)
# Write startup command if specified
if cmd_file is not None:
if verbosity >= 0 and not quiet:
print "# Writing startup command to file."
cfile = open(cmd_file, 'w')
if os.name == 'posix' and cmd_file.endswith('.sh'):
cfile.write("#!/bin/sh\n")
cfile.write("# Startup command generated by mysqlserverclone.\n")
cfile.write("%s\n" % cmd)
cfile.close()
if verbosity >= 1 and not quiet:
if verbosity >= 2:
print "# Startup command for new server:\n%s" % cmd
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd, shell=True, stdout=fnull, stderr=fnull)
# Try to connect to the new MySQL instance
if not quiet:
print "# Testing connection to new instance..."
new_sock = None
port_int = None
if os.name == "posix":
new_sock = os.path.join(new_data, "mysql.sock")
port_int = int(new_port)
conn = {
"user": "******",
"passwd": "",
"host": conn_val["host"] if conn_val is not None else "localhost",
"port": port_int,
"unix_socket": new_sock
}
server2_options = {
'conn_info': conn,
'role': "clone",
}
server2 = Server(server2_options)
stop = 10 # stop after 10 attempts
i = 0
while i < stop:
i += 1
time.sleep(1)
try:
server2.connect()
i = stop + 1
except:
pass
finally:
if verbosity >= 1 and not quiet:
print "# trying again..."
if i == stop:
raise UtilError("Unable to communicate with new instance.")
elif not quiet:
print "# Success!"
# Set the root password
if root_pass:
if not quiet:
print "# Setting the root password..."
if os.name == "posix":
cmd = mysqladmin_path + " --no-defaults -v -uroot " + \
"--socket=%s password %s " % (new_sock, root_pass)
else:
cmd = mysqladmin_path + " --no-defaults -v -uroot " + \
"password %s --port=%s" % (root_pass, int(new_port))
if verbosity > 0 and not quiet:
proc = subprocess.Popen(cmd, shell=True)
else:
proc = subprocess.Popen(cmd,
shell=True,
stdout=fnull,
stderr=fnull)
# Wait for subprocess to finish
res = proc.wait()
if not quiet:
conn_str = "# Connection Information:\n"
conn_str += "# -uroot"
if root_pass:
conn_str += " -p%s" % root_pass
if os.name == "posix":
conn_str += " --socket=%s" % new_sock
else:
conn_str += " --port=%s" % new_port
print conn_str
print "#...done."
fnull.close()
self.drop_all()
data_file = os.path.normpath("./std_data/basic_data.sql")
try:
res = self.server1.read_and_exec_SQL(data_file, self.debug)
except MUTLibError, e:
raise MUTLibError("Failed to read commands from file %s: " % \
data_file + e.errmsg)
rows = self.server2.exec_query("SHOW VARIABLES LIKE 'basedir'")
if rows:
basedir = rows[0][1]
else:
raise MUTLibError("Unable to determine basedir of running "
"server.")
self.mysql_path = get_tool_path(basedir, "mysql")
return True
def show_data(self, tbl):
comment = "Showing data for table %s \n" % tbl
self.results.append(comment)
if os.name == "posix":
cmd = "%s %s util_test -e 'SELECT * FROM %s'" % \
(self.mysql_path, self.server2_conn, tbl)
else:
cmd = '%s %s util_test -e "SELECT * FROM %s"' % \
(self.mysql_path, self.server2_conn, tbl)
res = self.exec_util(cmd, self.res_fname, True)
if res != 0:
raise MUTLibError("%s: failed" % comment)