def handle_config_path(config_path, group=None, use_default=True):
"""Retrieve the data associated to the given group.
config_path[in] the path to the configuration file.
group[in] The group name to retrieve the data from, if None
the 'client' group will be use if found and if
use_default is True.
use_default[in] Use the default 'client' group name, True by default,
used if no group is given.
Returns a dictionary with the options data.
"""
# first verify if the configuration file exist on the given config_path
# check config_path as near file as normalized path, then
# at the default location file.
if os.name == 'nt':
default_loc = os.path.join('c:\\', config_path)
else:
default_loc = os.path.join('/etc/mysql/', config_path)
# default group
default_group = 'client'
# first look at the given path, if not found look at the default location
paths = [os.path.normpath(config_path), os.path.normpath(default_loc)]
req_group = group
# if not group is given use default.
if not req_group and use_default:
req_group = default_group
for file_loc in paths:
if os.path.exists(file_loc) and os.path.isfile(file_loc):
opt_par = MySQLOptionsParser(file_loc)
dict_grp = opt_par.get_groups_as_dict(req_group)
if dict_grp:
return dict_grp[req_group]
else:
if group:
raise UtilError("The given group '{0}' was not found on "
"the configuration file '{1}'"
"".format(group, file_loc))
else:
raise UtilError("The default group '{0}' was not found "
"on the configuration file '{1}'"
"".format(req_group, file_loc))
# No configuration file was found
if paths[0] != paths[1]:
raise UtilError("Could not find a configuration file neither in the "
"given path '{0}' nor the default path '{1}'."
"".format(*paths))
raise UtilError("Could not find a configuration file in the given "
"path '{0}'.".format(paths[0]))
def grant_proxy_ssl_privileges(server,
user,
passw,
at='localhost',
privs="ALL PRIVILEGES",
grant_opt=True,
ssl=True,
grant_proxy=True,
proxy_user='******',
proxy_host='localhost'):
"""Grant privileges to an user in a server with GRANT OPTION or/and
REQUIRE SSL if required.
server[in] Server to execute the grant query at.
user_name[in] New user name.
passw[in] password of the new user.
at[in] Used in GRANT "TO '{0}'@'{1}'".format(user, at),
(default localhost)
grant_opt[in] if True, it will grant with GRANT OPTION (default True).
ssl[in] if True, it will set REQUIRE SSL (default True).
grant_proxy[in] if True, it will grant GRANT PROXY (default True).
proxy_user[in] username for the proxied account (default: root)
proxy_host[in] hostname for the proxied account (default: localhost)
Note: Raises UtilError on any Error.
"""
grant = [
"GRANT", privs, "ON *.*", "TO '{0}'@'{1}'".format(user, at),
"IDENTIFIED BY '{0}'".format(passw) if passw else "",
"REQUIRE SSL" if ssl else "", "WITH GRANT OPTION" if grant_opt else ""
]
try:
server.exec_query(" ".join(grant))
except UtilDBError as err:
raise UtilError("Cannot create new user {0} at {1}:{2} reason:"
"{3}".format(user, server.host, server.port,
err.errmsg))
if grant_proxy:
grant = ("GRANT PROXY ON '{0}'@'{1}' "
"TO '{2}'@'{3}' "
"WITH GRANT OPTION").format(proxy_user, proxy_host, user, at)
try:
server.exec_query(grant)
except UtilDBError as err:
raise UtilError("Cannot grant proxy to user {0} at {1}:{2} "
"reason:{3}".format(user, server.host, server.port,
err.errmsg))
def _setup_compare(table1, table2, span_key_size):
"""Create and populate the compare summary tables
This method creates the condensed hash table used to compare groups
(span) of records. It also creates the Table instance for each table
and populates values in the table information dictionary for use
in other methods.
The method also checks to ensure the tables have primary keys and that
the keys are the same (have the same columns). An error is raised if
neither of these are met.
table1[in] table1 Table instance
table2[in] table2 Table instance
span_key_size[in] the size of key used for the hash.
Returns tuple - string representations of the primary index columns
"""
server1 = table1.server
server2 = table2.server
# Get the primary key for the tables and make sure they are the same
table1_idx = table1.get_primary_index()
table2_idx = table2.get_primary_index()
if len(table1_idx) != len(table2_idx):
raise UtilError("Indexes are not the same.")
elif table1_idx == [] or table2_idx == []:
raise UtilError("No primary key found.")
# drop the temporary tables
_drop_compare_object(server1, table1.db_name, table1.tbl_name)
_drop_compare_object(server2, table2.db_name, table2.tbl_name)
# Build the primary key hash if needed
tbl1_table, pri_idx1 = _get_compare_objects(table1_idx, table1,
span_key_size)
tbl2_table, pri_idx2 = _get_compare_objects(table1_idx, table2,
span_key_size)
if tbl1_table is None or tbl2_table is None:
raise UtilError("Cannot create compare table.")
# Create the compare tables
server1.exec_query(tbl1_table)
server2.exec_query(tbl2_table)
return (pri_idx1, pri_idx2)
def search_my_print_defaults_tool(self, search_paths=None):
"""Search for the tool my_print_defaults.
"""
if not search_paths:
search_paths = []
# Set the default search paths (i.e., default location of the
# .mylogin.cnf file).
default_paths = [my_login_config_path()]
# Extend the list of path to search with the ones specified.
if search_paths:
default_paths.extend(search_paths)
# Search for the tool my_print_defaults.
try:
self._tool_path = get_tool_path(self._basedir,
_MY_PRINT_DEFAULTS_TOOL,
defaults_paths=default_paths,
search_PATH=True)
except UtilError as err:
raise UtilError("Unable to locate MySQL Client tools. "
"Please confirm that the path to the MySQL client "
"tools are included in the PATH. Error: %s" %
err.errmsg)
def rotate(self):
"""This Method runs the rotation.
This method will use the methods from the common library to rotate the
binary log.
"""
# Check required privileges
check_privileges(self.server, BINLOG_OP_ROTATE,
["RELOAD", "REPLICATION CLIENT"],
BINLOG_OP_ROTATE_DESC, self.verbosity, self._report)
active_binlog, binlog_size = get_active_binlog_and_size(self.server)
if self.verbosity:
self._report("# Active binlog file: '{0}' (size: {1} bytes)'"
"".format(active_binlog, binlog_size))
if self.binlog_min_size:
rotated = rotate(self.server,
self.binlog_min_size,
reporter=self._report)
else:
rotated = rotate(self.server, reporter=self._report)
if rotated:
new_active_binlog, _ = get_active_binlog_and_size(self.server)
if active_binlog == new_active_binlog:
raise UtilError("Unable to rotate binlog file.")
else:
self._report("# The binlog file has been rotated.")
if self.verbosity:
self._report("# New active binlog file: '{0}'"
"".format(new_active_binlog))
def _check_databases(server1, server2, db1, db2, options):
"""Check databases
server1[in] first server Server instance
server2[in] second server Server instance
db1[in] first database
db2[in] second database
options[in] options dictionary
Returns tuple - Database class instances for databases
"""
# Check database create for differences
if not options['no_diff']:
# temporarily make the diff quiet to retrieve errors
new_opt = {}
new_opt.update(options)
new_opt['quiet'] = True # do not print messages
new_opt['suppress_sql'] = True # do not print SQL statements either
res = diff_objects(server1, server2, db1, db2, new_opt, 'DATABASE')
if res is not None:
for row in res:
print row
print
if not options['run_all_tests']:
raise UtilError(_ERROR_DB_DIFF)
def _find_engine(server, target, message, fail, default):
if target is not None:
found = server.has_storage_engine(target)
if not found and fail:
raise UtilError(message)
elif not found and not quiet:
print message
def _make_select(objtype, pattern, database_pattern, check_body, use_regexp):
"""Generate a SELECT statement for finding an object.
"""
options = {
'pattern': obj2sql(pattern),
'regex': 'REGEXP' if use_regexp else 'LIKE',
'select_option': '',
'field_type': "'" + objtype.upper() + "'",
}
try:
options.update(_OBJMAP[objtype])
except KeyError:
raise UtilError("Invalid object type '{0}'. Use --help to see allowed "
"values.".format(objtype))
# Build a condition for inclusion in the select
condition = "{field_name} {regex} {pattern}".format(**options)
if check_body and "body_field" in options:
condition += " OR {body_field} {regex} {pattern}".format(**options)
if database_pattern:
options['database_pattern'] = obj2sql(database_pattern)
condition = ("({0}) AND {schema_field} {regex} {database_pattern}"
"".format(condition, **options))
options['condition'] = condition
return _SELECT_TYPE_FRM.format(**options)
def _exec_statements(statements, destination, format, options, dryrun=False):
"""Execute a list of SQL statements
statements[in] A list of SQL statements to execute
destination[in] A connection to the destination server
format[in] Format of import file
options[in] Option dictionary containing the --skip_* options
dryrun[in] If True, print the SQL statements and do not execute
Returns (bool) - True if all execute, raises error if one fails
"""
new_engine = options.get("new_engine", None)
def_engine = options.get("def_engine", None)
quiet = options.get("quiet", False)
for statement in statements:
if (new_engine is not None or def_engine is not None) and \
statement.upper()[0:12] == "CREATE TABLE":
i = statement.find(' ', 13)
tbl_name = statement[13:i]
statement = destination.substitute_engine(tbl_name, statement,
new_engine, def_engine,
quiet)
try:
if dryrun:
print statement
elif format != "sql" or not _skip_sql(statement, options):
res = destination.exec_query(statement)
# Here we capture any exception and raise UtilError to communicate to
# the script/user. Since all util errors (exceptions) derive from
# Exception, this is safe.
except Exception, e:
raise UtilError("Invalid statement:\n%s" % statement +
"\nERROR: %s" % e.errmsg)
def _get_compare_objects(index_cols, table1):
"""Build the compare table and identify the primary index
This method creates the compare table for use in forming the MD5 hash
of rows and a hash of the primary key. It also forms the primary key
list of columns.
index_cols[in] a list of columns that form the primary key in the form
(column_name, type)
table1[in] a Table instance of the original table
Returns tuple (table create statement, concatenated string of the
primary index columns)
"""
table = None
# build primary key col definition
index_str = ''.join("{0}, ".format(quote_with_backticks(col[0])) \
for col in index_cols)
index_defn = ''.join("{0} {1}, ".
format(quote_with_backticks(col[0]), col[1]) \
for col in index_cols)
if index_defn == "":
raise UtilError("Cannot generate index definition")
else:
# Quote compare table appropriately with backticks
q_tbl_name = quote_with_backticks(
_COMPARE_TABLE_NAME.format(tbl=table1.tbl_name))
table = _COMPARE_TABLE.format(db=table1.q_db_name,
compare_tbl=q_tbl_name,
pkdef=index_defn)
return (table, index_str)
def setUpClass(cls):
# Find mysql_config_editor to manipulate data from .mylogin.cnf
try:
cls.edit_tool_path = get_tool_path(None,
"mysql_config_editor",
search_PATH=True)
except UtilError as err:
raise UtilError("MySQL client tools must be accessible to run "
"this test (%s). Please add the location of the "
"MySQL client tools to your PATH." % err.errmsg)
# Create login-path data
cmd = ("{mysql_config_editor} set --login-path={login_path} "
"--host={host} --user={user}")
cmd = cmd.format(mysql_config_editor=cls.edit_tool_path,
login_path=_TEST_LOGIN_PATH,
host=_TEST_HOST,
user=_TEST_USER)
# Execute command to create login-path data
proc = subprocess.Popen(cmd.split(' '),
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
# Overwrite login-path if already exists (i.e. answer 'y' to question)
proc.communicate(input='y')
def _compare_objects(server1, server2, obj1, obj2, reporter, options):
"""Compare object definitions and produce difference
server1[in] first server Server instance
server2[in] second server Server instance
obj1[in] first object
obj2[in] second object
reporter[in] database compare reporter class instance
options[in] options dictionary
Returns list of errors
"""
from mysql.utilities.common.dbcompare import diff_objects
errors = []
if not options['no_diff']:
# For each database, compare objects
# temporarily make the diff quiet to retrieve errors
new_opt = {}
new_opt.update(options)
new_opt['quiet'] = True # do not print messages
new_opt['suppress_sql'] = True # do not print SQL statements either
res = diff_objects(server1, server2, obj1, obj2, new_opt)
if res is not None:
reporter.report_state('FAIL')
errors.extend(res)
if not options['run_all_tests']:
raise UtilError(_ERROR_DB_DIFF)
else:
reporter.report_state('pass')
else:
reporter.report_state('SKIP')
return errors
def determine_purgeable_binlogs(active_binlog_index,
slaves,
reporter,
verbosity=0):
"""Determine the purgeable binary logs.
This method will look at each slave given and will determinate the lowest
binary log file that is being in use.
active_binlog_index[in] Index of binlog currently in use by the
master server or the higher binlog index value
it wants to be purged.
slaves[in] Slaves list.
reporter[in] Method to call to report.
verbosity[in] The verbosity level for reporting information.
Returns the last index in use by the slaves, that is the newest binlog
index that has between read by all the slave servers.
"""
# Determine old no needed binlogs
master_log_file_in_use = []
index_last_in_use = active_binlog_index
# pylint: disable=R0101
if slaves:
for slave in slaves:
if reporter is not None and verbosity >= 1:
reporter("# Checking slave: {0}@{1}"
"".format(slave['host'], slave['port']))
res = slave['instance'].get_status()
if res:
master_log_file = res[0][5]
if reporter is not None and verbosity >= 1:
reporter("# I/O thread is currently reading: {0}"
"".format(master_log_file))
master_log_file_in_use.append(master_log_file)
reading_index_file = int(master_log_file.split('.')[1])
if index_last_in_use > reading_index_file:
index_last_in_use = reading_index_file
if reporter is not None and verbosity >= 2:
reporter("# File position of the I/O thread: {0}"
"".format(res[0][6]))
reporter("# Master binlog file with last event executed "
"by the SQL thread: {0}".format(res[0][9]))
reporter("# I/O thread running: {0}".format(res[0][10]))
reporter("# SQL thread running: {0}".format(res[0][11]))
if len(res[0]) > 52:
if res[0][51]:
reporter("# Retrieved GTid_Set: {0}"
"".format(res[0][51]))
if res[0][52]:
reporter("# Executed GTid_Set: {0}"
"".format(res[0][52]))
return index_last_in_use
else:
raise UtilError("None Slave is connected to master")
def purge(server,
purge_to_binlog,
server_binlogs_list=None,
reporter=None,
dryrun=False,
verbosity=0):
"""Purge the binary log for the given server.
This method purges all the binary logs from the given server that are older
than the given binlog file name specified by purge_to_binlog. The method
can receive a list of the binary logs listed on the server to avoid
querying the server again for this list. If The given purge_to_binlog is
not listed on the server_binlogs_list the purge will not occur. For
reporting capabilities if given the method report will be invoked to
report messages and the server name that appears on the messages can be
change with server_name.
server[in] server instance where to purge binlogs on
purge_to_binlog[in] purge binlog files older than this binlog file
name.
server_binlogs_list[in] A list of binlog files available on the given
server, if not given, the list will be retrieved
from the given server (default None).
server_name[in] This name will appear when reporting (default
'Server').
reporter[in] A method to invoke with messages and warnings
(default None).
dryrun[in] boolean value that indicates if the purge query
should be run on the server or reported only
(default False).
verbosity[in] The verbosity level for report messages.
"""
if server_binlogs_list is None:
server_binlogs_list = server.get_server_binlogs_list()
# The PURGE BINARY LOGS statement deletes all the binary log files listed
# in the log index file, prior to the specified log file name.
# Verify purge_to_binlog is listed on server binlog list and if not is the
# first in the list continue the purge, else there is no binlogs to purge
if (purge_to_binlog in server_binlogs_list
and purge_to_binlog != server_binlogs_list[0]):
purge_query = ("PURGE BINARY LOGS TO '{0}'").format(purge_to_binlog)
if dryrun:
reporter("# To manually purge purge the binary logs Execute the "
"following query:")
reporter(purge_query)
else:
if verbosity > 1:
reporter("# Executing query {0}".format(purge_query))
else:
reporter("# Purging binary logs prior to '{0}'"
"".format(purge_to_binlog))
try:
server.exec_query(purge_query)
except UtilDBError as err:
raise UtilError("Unable to purge binary log, reason: {0}"
"".format(err.errmsg))
else:
reporter("# No binlog files can be purged.")
def _check_row_counts(server1, server2, obj1, obj2, reporter, options):
"""Compare row counts for tables
server1[in] first server Server instance
server2[in] second server Server instance
obj1[in] first object
obj2[in] second object
reporter[in] database compare reporter class instance
options[in] options dictionary
Returns list of errors
"""
errors = []
if not options['no_row_count']:
rows1 = server1.exec_query("SELECT COUNT(*) FROM " + obj1)
rows2 = server2.exec_query("SELECT COUNT(*) FROM " + obj2)
if rows1 != rows2:
reporter.report_state('FAIL')
msg = _ERROR_ROW_COUNT.format(obj1, obj2)
if not options['run_all_tests']:
raise UtilError(msg)
else:
errors.append("# %s" % msg)
else:
reporter.report_state('pass')
else:
reporter.report_state('SKIP')
return errors
def get_next_record(self):
"""Get the next audit log record.
Generator function that return the next audit log record.
More precisely, it returns a tuple with a formatted record dict and
the original record.
"""
next_line = ""
for line in self.log:
if ((line.lstrip()).startswith('<AUDIT_RECORD')
and not line.endswith('/>\n')):
next_line = "{0} ".format(line.strip('\n'))
continue
elif len(next_line) > 0 and not line.endswith('/>\n'):
next_line = '{0}{1}'.format(next_line, line.strip('\n'))
continue
else:
next_line += line
log_entry = next_line
next_line = ""
try:
yield (self._make_record(xml.fromstring(log_entry)), log_entry)
except (ParseError, SyntaxError):
# SyntaxError is also caught for compatibility reasons with
# python 2.6. In case an ExpatError which does not inherits
# from SyntaxError is used as a ParseError.
if not self._validXML(log_entry):
raise UtilError("Malformed XML - Cannot parse log file: "
"'{0}'\nInvalid XML element: "
"{1!r}".format(self.log_name, log_entry))
def _check_privileges(server, options):
"""Check required privileges to move binary logs from server.
This method check if the used user possess the required privileges to
relocate binary logs from the server. More specifically, the following
privilege is required: RELOAD (to flush the binary logs).
An exception is thrown if the user doesn't have enough privileges.
server[in] Server instance to check.
options[in] Dictionary of options (skip_flush_binlogs, verbosity).
"""
skip_flush = options['skip_flush_binlogs']
if not skip_flush:
# Only need to check privileges if flush is not skipped.
verbosity = options['verbosity']
if verbosity > 0:
print("# Checking user permission to move binary logs...\n" "#")
# Check privileges
user_obj = User(server, "{0}@{1}".format(server.user, server.host))
if not user_obj.has_privilege('*', '*', 'RELOAD'):
raise UtilError(
ERROR_USER_WITHOUT_PRIVILEGES.format(
user=server.user,
host=server.host,
port=server.port,
operation='perform binary log move',
req_privileges='RELOAD'))
def _check_objects(server1, server2, db1, db2, db1_conn, db2_conn, options):
"""Check number of objects
server1[in] first server Server instance
server2[in] second server Server instance
db1[in] first database
db2[in] second database
db1_conn[in] first Database instance
db2_conn[in] second Database instance
options[in] options dictionary
Returns list of objects in both databases
"""
differs = False
# Check for same number of objects
in_both, in_db1, in_db2 = get_common_objects(server1, server2, db1, db2,
False, options)
in_both.sort()
if not options['no_object_check']:
server1_str = "server1." + db1
if server1 == server2:
server2_str = "server1." + db2
else:
server2_str = "server2." + db2
if len(in_db1) or len(in_db2):
if options['run_all_tests']:
if len(in_db1) > 0:
differs = True
print_missing_list(in_db1, server1_str, server2_str)
print "#"
if len(in_db2) > 0:
differs = True
print_missing_list(in_db2, server2_str, server1_str)
print "#"
else:
differs = True
raise UtilError(_ERROR_OBJECT_LIST.format(db1, db2))
# If in verbose mode, show count of object types.
if options['verbosity'] > 1:
objects = {
'TABLE': 0,
'VIEW': 0,
'TRIGGER': 0,
'PROCEDURE': 0,
'FUNCTION': 0,
'EVENT': 0,
}
for item in in_both:
obj_type = item[0]
objects[obj_type] += 1
print "Looking for object types table, view, trigger, procedure," + \
" function, and event."
print "Object types found common to both databases:"
for obj in objects:
print " {0:>12} : {1}".format(obj, objects[obj])
return (in_both, differs)
def _build_logfile_list(server, log_name, suffix='_file'):
"""Build a list of all log files based on the system variable by the
same name as log_name.
server[in] Connected server
log_name[in] Name of log (e.g. slow_query_log)
suffix[in] Suffix of log variable name (e.g. slow_query_log_file)
default = '_file'
return (tuple) (logfiles[], path to log files, total size)
"""
log_path = None
res = server.show_server_variable(log_name)
if res != [] and res[0][1].upper() == 'OFF':
print "# The %s is turned off on the server." % log_name
else:
res = server.show_server_variable(log_name + suffix)
if res == []:
raise UtilError("Cannot get %s_file setting." % log_name)
log_path = res[0][1]
if os.access(log_path, os.R_OK):
parts = os.path.split(log_path)
if len(parts) <= 1:
log_file = log_path
else:
log_file = parts[1]
log_path_size = os.path.getsize(log_path)
return (log_file, log_path, int(log_path_size))
return None, 0, 0
def get_option_matches(self, util_info, option_prefix, find_alias=False):
"""Get list of option dictionary entries for options that match
the prefix.
util_info[in] utility information
option_prefix[in] prefix for option name
find_alias[in] if True, match alias (default = False)
Returns list of dictionary items that match prefix
"""
# Check type of util_info
if util_info is None or util_info == {} or \
not isinstance(util_info, dict):
raise UtilError("Empty or invalide utility dictionary.")
matches = []
stop = len(option_prefix)
if isinstance(util_info['options'], str):
self.parse_all_options(util_info)
for option in util_info['options']:
if option is None:
continue
name = option.get('name', None)
if name is None:
continue
if find_alias:
if option.get('alias', '') == option_prefix:
matches.append(option)
else:
if name[0:stop] == option_prefix:
matches.append(option)
return matches
def connect(self):
"""Connect to server
Attempts to connect to the server as specified by the connection
parameters.
Note: This method must be called before executing queries.
Raises UtilError if error during connect
"""
try:
parameters = {
'user': self.user,
'host': self.host,
'port': self.port,
}
if self.socket and os.name == "posix":
parameters['unix_socket'] = self.socket
if self.passwd and self.passwd != "":
parameters['passwd'] = self.passwd
parameters['charset'] = self.charset
self.db_conn = mysql.connector.connect(**parameters)
except mysql.connector.Error, e:
# Reset any previous value if the connection cannot be established,
# before raising an exception. This prevents the use of a broken
# database connection.
self.db_conn = None
raise UtilError("Cannot connect to the %s server.\n"
"Error %s" % (self.role, e.msg), e.errno)
def get_connection_dictionary(conn_info):
"""Get the connection dictionary.
The method accepts one of the following types for conn_info:
- dictionary containing connection information including:
(user, passwd, host, port, socket)
- connection string in the form: user:pass@host:port:socket or
login-path:port:socket
- an instance of the Server class
conn_info[in] Connection information
Returns dict - dictionary for connection (user, passwd, host, port, socket)
"""
if conn_info is None:
return conn_info
conn_val = {}
if isinstance(conn_info, dict):
conn_val = conn_info
elif isinstance(conn_info, Server):
# get server's dictionary
conn_val = conn_info.get_connection_values()
elif isinstance(conn_info, basestring):
# parse the string
conn_val = parse_connection(conn_info, None)
else:
raise UtilError("Cannot determine connection information type.")
return conn_val
def _connect(self, conn):
"""Find the attached slaves for a list of server connections.
This method connects to each server in the list and retrieves its
slaves.
It can be called recursively if the recurse parameter is True.
conn[in] Connection dictionary used to connect to server
Returns tuple - master Server class instance, master:host string
"""
conn_options = {
'quiet': self.quiet,
'src_name': "master",
'dest_name': None,
'version': "5.0.0",
'unique': True,
'verbose': self.verbose,
}
certs_paths = {}
if 'ssl_ca' in dir(conn) and conn.ssl_ca is not None:
certs_paths['ssl_ca'] = conn.ssl_ca
if 'ssl_cert' in dir(conn) and conn.ssl_cert is not None:
certs_paths['ssl_cert'] = conn.ssl_cert
if 'ssl_key' in dir(conn) and conn.ssl_key is not None:
certs_paths['ssl_key'] = conn.ssl_key
conn_options.update(certs_paths)
master_info = "{0}:{1}".format(conn['host'], conn['port'])
master = None
# Increment num_retries if not set when --prompt is used
if self.prompt_user and self.num_retries == 0:
self.num_retries += 1
# Attempt to connect to the server given the retry limit
for i in range(0, self.num_retries + 1):
try:
servers = connect_servers(conn, None, conn_options)
master = servers[0]
break
except UtilError as e:
print("FAILED.\n")
if i < self.num_retries and self.prompt_user:
print("Connection to %s has failed.\n" % master_info + \
"Please enter the following information " + \
"to connect to this server.")
conn['user'] = raw_input("User name: ")
conn['passwd'] = getpass.getpass("Password: "******"{0}:{1}".format(conn['host'], master.port)
return (master, master_info)
def get_tool_path(basedir, tool, fix_ext=True, required=True,
defaults_paths=None, search_PATH=False):
"""Search for a MySQL tool and return the full path
basedir[in] The initial basedir to search (from mysql server)
tool[in] The name of the tool to find
fix_ext[in] If True (default is True), add .exe if running on
Windows.
required[in] If True (default is True), and error will be
generated and the utility aborted if the tool is
not found.
defaults_paths[in] Default list of paths to search for the tool.
By default an empty list is assumed, i.e. [].
search_PATH[in] Boolean value that indicates if the paths specified by
the PATH environment variable will be used to search
for the tool. By default the PATH will not be searched,
i.e. search_PATH=False.
Returns (string) full path to tool
"""
if not defaults_paths:
defaults_paths = []
search_paths = []
if basedir:
# Add specified basedir path to search paths
_add_basedir(search_paths, basedir)
if defaults_paths and len(defaults_paths):
# Add specified default paths to search paths
for path in defaults_paths:
search_paths.append(path)
else:
# Add default basedir paths to search paths
_add_basedir(search_paths, "/usr/local/mysql/")
_add_basedir(search_paths, "/usr/sbin/")
_add_basedir(search_paths, "/usr/share/")
# Search in path from the PATH environment variable
if search_PATH:
for path in os.environ['PATH'].split(os.pathsep):
search_paths.append(path)
if os.name == "nt" and fix_ext:
tool = tool + ".exe"
# Search for the tool
for path in search_paths:
norm_path = os.path.normpath(path)
if os.path.isdir(norm_path):
toolpath = os.path.join(norm_path, tool)
if os.path.isfile(toolpath):
return toolpath
else:
if tool == "mysqld.exe":
toolpath = os.path.join(norm_path, "mysqld-nt.exe")
if os.path.isfile(toolpath):
return toolpath
if required:
raise UtilError("Cannot find location of %s." % tool)
return None
def run(self):
self.server0 = self.servers.get_server(0)
cmd_str = "mysqlserverclone.py --server=%s --delete-data " % \
self.build_connection_string(self.server0)
port1 = int(self.servers.get_next_port())
cmd_str += " --new-port=%d --root-password=root " % port1
comment = "Test case 1 - clone a running server"
self.results.append(comment + "\n")
full_datadir = os.path.join(os.getcwd(), "temp_%s" % port1)
cmd_str += "--new-data=%s " % full_datadir
res = self.exec_util(cmd_str, "start.txt")
for line in open("start.txt").readlines():
# Don't save lines that have [Warning]
if "[Warning]" in line:
continue
self.results.append(line)
if res:
raise MUTLibError("%s: failed" % comment)
self.new_server = self.check_connect(port1, full_datadir)
basedir = ""
# Get basedir
rows = self.server0.exec_query("SHOW VARIABLES LIKE 'basedir'")
if not rows:
raise UtilError("Unable to determine basedir of running server.")
basedir = rows[0][1]
port2 = int(self.servers.get_next_port())
cmd_str = "mysqlserverclone.py --root-password=root --delete-data "
cmd_str += "--new-port=%d --basedir=%s " % (port2, basedir)
comment = "Test case 2 - clone a server from basedir"
self.results.append(comment + "\n")
full_datadir = os.path.join(os.getcwd(), "temp_%s" % port2)
cmd_str += "--new-data=%s " % full_datadir
res = self.exec_util(cmd_str, "start.txt")
for line in open("start.txt").readlines():
# Don't save lines that have [Warning]
if "[Warning]" in line:
continue
self.results.append(line)
if res:
raise MUTLibError("%s: failed" % comment)
server = self.check_connect(port2, full_datadir,
"cloned_server_basedir")
self.servers.stop_server(server)
self.servers.clear_last_port()
self.replace_result("# -uroot", "# -uroot [...]\n")
self.replace_result("# Cloning the MySQL server located at",
"# Cloning the MySQL server located at XXXX\n")
return True
def get_next_record(self):
"""Get the next audit log record.
Generator function that return the next audit log record.
More precisely, it returns a tuple with a formatted record dict and
the original record.
"""
next_line = ""
new_format = False
multiline = False
for line in self.log:
if line.lstrip().startswith('<AUDIT_RECORD>'):
# Found first record line in the new format.
new_format = True
multiline = True
next_line = line
continue
elif (line.lstrip().startswith('<AUDIT_RECORD') and
not line.endswith('/>\n')):
# Found (first) record line in the old format.
next_line = "{0} ".format(line.strip('\n'))
if not line.endswith('/>\n'):
multiline = True
continue
elif multiline:
if ((new_format and
line.strip().endswith('</AUDIT_RECORD>')) or
(not new_format and line.endswith('/>\n'))):
# Detect end of record in the old and new format and
# append last record line.
next_line += line
else:
if not line.strip().startswith('<'):
# Handle SQL queries broke into multiple lines,
# removing newline characters.
next_line = '{0}{1}'.format(next_line.strip('\n'),
line.strip('\n'))
else:
next_line += line
continue
else:
next_line += line
log_entry = next_line
next_line = ""
try:
yield (
self._make_record(xml.fromstring(log_entry), new_format),
log_entry
)
except (ParseError, SyntaxError):
# SyntaxError is also caught for compatibility reasons with
# python 2.6. In case an ExpatError which does not inherits
# from SyntaxError is used as a ParseError.
if not self._validXML(log_entry):
raise UtilError("Malformed XML - Cannot parse log file: "
"'{0}'\nInvalid XML element: "
"{1!r}".format(self.log_name, log_entry))
def open_log(self):
"""Open the audit log file.
"""
# Get the log from a remote server
# TODO : check to see if the log is local. If not, attempt
# to log into the server via rsh and copy the file locally.
self.remote_file = False
if not self.log_name or not os.path.exists(self.log_name):
raise UtilError("Cannot read log file '%s'." % self.log_name)
self.log = open(self.log_name)
def _get_transform(server1, server2, object1, object2, options,
object_type):
"""Get the transformation SQL statements
This method generates the SQL statements to transform the destination
object based on direction of the compare.
server1[in] first server connection
server2[in] second server connection
object1 the first object in the compare in the form: (db.name)
object2 the second object in the compare in the form: (db.name)
options[in] a dictionary containing the options for the operation:
(quiet, etc.)
object_type[in] type of the objects to be compared (e.g., TABLE,
PROCEDURE, etc.).
Returns tuple - (bool - same db name?, list of transformation statements)
"""
try:
m_obj1 = re.match(REGEXP_QUALIFIED_OBJ_NAME, object1)
db1, name1 = m_obj1.groups()
m_obj2 = re.match(REGEXP_QUALIFIED_OBJ_NAME, object2)
db2, name2 = m_obj2.groups()
except:
raise UtilError("Invalid object name arguments for _get_transform"
"(): %s, %s." % (object1, object2))
# If the second part of the object qualified name is None, then the format
# is not 'db_name.obj_name' for object1 and therefore must treat it as a
# database name. (supports backticks and the use of '.' (dots) in names.)
if not name1 or object_type == 'DATABASE':
# We are working with databases so db and name need to be set
# to the database name to tell the get_object_definition() method
# to retrieve the database information.
name1 = db1
name2 = db2
db_1 = Database(server1, db1, options)
db_2 = Database(server2, db2, options)
obj1 = db_1.get_object_definition(db1, name1, object_type)
obj2 = db_2.get_object_definition(db2, name2, object_type)
# Get the transformation based on direction.
transform_str = []
xform = SQLTransformer(db_1, db_2, obj1[0], obj2[0], object_type,
options.get('verbosity', 0))
differences = xform.transform_definition()
if differences and len(differences) > 0:
transform_str.extend(differences)
return transform_str
def _parse_grant_statement(statement, sql_mode=''):
""" Returns a namedtuple with the parsed GRANT information.
statement[in] Grant string in the sql format returned by the server.
Returns named tuple with GRANT information or None.
"""
grant_parse_re = re.compile(
r"""
GRANT\s(.+)?\sON\s # grant or list of grants
(?:(?:PROCEDURE\s)|(?:FUNCTION\s))? # optional for routines only
(?:(?:(\*|`?[^']+`?)\.(\*|`?[^']+`?)) # object where grant applies
| ('[^']*'@'[^']*')) # For proxy grants user/host
\sTO\s([^@]+@[\S]+) # grantee
(?:\sIDENTIFIED\sBY\sPASSWORD
(?:(?:\s<secret>)|(?:\s\'[^\']+\')?))? # optional pwd
(?:\sREQUIRE\sSSL)? # optional SSL
(\sWITH\sGRANT\sOPTION)? # optional grant option
$ # End of grant statement
""", re.VERBOSE)
grant_tpl_factory = namedtuple(
"grant_info", "privileges proxy_user "
"db object user")
match = re.match(grant_parse_re, statement)
if match:
# quote database name and object name with backticks
if match.group(1).upper() != 'PROXY':
db = match.group(2)
if not is_quoted_with_backticks(db, sql_mode) and db != '*':
db = quote_with_backticks(db, sql_mode)
obj = match.group(3)
if not is_quoted_with_backticks(obj, sql_mode) and obj != '*':
obj = quote_with_backticks(obj, sql_mode)
else: # if it is not a proxy grant
db = obj = None
grants = grant_tpl_factory(
# privileges
set([priv.strip() for priv in match.group(1).split(",")]),
match.group(4), # proxied user
db, # database
obj, # object
match.group(5), # user
)
# If user has grant option, add it to the list of privileges
if match.group(6) is not None:
grants.privileges.add("GRANT OPTION")
else:
raise UtilError("Unable to parse grant statement "
"{0}".format(statement))
return grants
def is_binary_log_filename(filename, log_type=LOG_TYPE_ALL, basename=None):
"""Check if the filename matches the name format for binary log files.
This function checks if the given filename corresponds to the filename
format of known binary log files, according to the specified log_type and
optional basename. The file extension is a sequence number (.nnnnnn). If
a basename is given then the filename for the binary log file must have
the format 'basename.nnnnnn'. Otherwise the default filename is assumed,
depending on the log_type: '*-bin.nnnnnn' for the 'bin' log type,
'*-relay-bin.nnnnnn' for the 'relay' log type, and both for the 'all' type.
filename[in] Filename to check.
log_type[in] Type of the binary log, must be one of the following
values: 'bin' for binlog files, 'relay' for relay log
files, 'all' for both binary log files. By default = 'all'.
basename[in] Basename defined for the binary log file. None by default,
meaning that the default server name formats are assumed
(according to the given log type).
"""
# Split file basename and extension.
f_base, f_ext = os.path.splitext(filename)
f_ext = f_ext[1:] # remove starting dot '.'
# Check file basename.
if basename:
if f_base != basename:
# Defined basename does not match.
return False
else:
# Check default serve basename for the given log_type.
if log_type == LOG_TYPE_BIN:
# *-bin.nnnnnn (excluding *-relay-bin.nnnnnn)
if not f_base.endswith('-bin') or f_base.endswith('-relay-bin'):
return False
elif log_type == LOG_TYPE_RELAY:
# *-relay-bin.nnnnnn
if not f_base.endswith('-relay-bin'):
return False
elif log_type == LOG_TYPE_ALL:
# *-bin.nnnnnn (including *-relay-bin.nnnnnn)
if not f_base.endswith('-bin'):
return False
else:
raise UtilError("Unsupported log-type: {0}".format(log_type))
# Check file extension.
try:
int(f_ext)
except ValueError:
# Extension is not a sequence number (error converting to integer).
return False
# Return true if basename and extension checks passed.
return True
