def get_diff(self, db_name, version):
file_schema = "%s/%04d-schema.img" % (db_name, int(version))
file_schema_swp = "%s/%04d-schema.swp" % (db_name, int(version))
tmp_db = "tmp_online_mig_%s" % (db_name)
query = "CREATE DATABASE %s;" % tmp_db
self.server.disable_foreign_key_checks()
self.server.exec_query(query)
f = open(file_schema, 'r')
f_swp = open(file_schema_swp, 'w')
f_swp.write("USE %s\n" % tmp_db)
buff = ""
for line in f.readlines():
#if re.search(';$', line, re.IGNORECASE):
if re.search(';$' + '(?i)', line):
buff = buff + line
f_swp.write(buff)
buff = ""
else:
buff = buff + line.strip()
f.close()
f_swp.close()
query_options = {'multi': True}
f_swp = open(file_schema_swp, 'r')
for line in f_swp.readlines():
self.server.exec_query(line, query_options)
f_swp.close()
os.remove(file_schema_swp)
query_options = {
'run_all_tests': True, 'reverse': False, 'verbosity': None,
'no_object_check': False, 'no_data': True, 'quiet': True,
'difftype': 'differ', 'width': 75, 'changes-for': 'server1',
'skip_grants': True}
source_values = parse_connection(server_connection)
destination_values = parse_connection(server_connection)
with capture() as stepback:
dbcompare.database_compare(source_values, destination_values, db_name, tmp_db, query_options)
buf = ""
found = 0
for line in stepback.getvalue().splitlines(True):
logging.debug(u"%s" % line)
if re.search('CREATE TABLE', line):
line = re.sub("CREATE" + '(?i)',"", line, 1)
#line = re.sub("CREATE","", line, 1, re.IGNORECASE)
#line = re.sub("\(\n","", line, 1, re.IGNORECASE)
line = re.sub("\(\n" + '(?i)',"", line, 1)
buf=line
if (re.search('^\+', line) or re.search('^\-', line)) and not re.search('CREATE DATABASE', line) and not re.search('CONSTRAINT.*FOREIGN KEY', line):
#line = re.sub("\n","", line, 1, re.IGNORECASE)
line = re.sub("\n" + '(?i)',"", line, 1)
if len(buf) > 0:
print u"%s" % buf
buf=""
print u"%s" % line
query = "DROP DATABASE %s" % tmp_db
self.server.exec_query(query)
self.server.disable_foreign_key_checks(disable=False)
def test_invalid(self):
"""Test parsing invalid versions of connection strings.
If the connection string is invalid, a FormatError should be thrown.
"""
for spec in self.invalid_specificers:
try:
parse_connection(spec)
except FormatError:
pass
except:
self.fail("Unexpected exception thrown.")
else:
self.fail("Exception not thrown for: '%s'." % spec)
def test_invalid(self):
"""Test parsing invalid versions of connection strings.
If the connection string is invalid, a FormatError should be thrown.
"""
for spec in self.invalid_specificers:
try:
parse_connection(spec)
except FormatError:
pass
except:
self.fail("Unexpected exception thrown.")
else:
self.fail("Exception not thrown for: '%s'." % spec)
def execute(self, connections, **kwrds):
"""Execute the search for processes, queries, or connections
This method searches for processes, queriers, or connections to
either kill or display the matches for one or more servers.
connections[in] list of connection parameters
kwrds[in] dictionary of options
output file stream to display information
default = sys.stdout
connector connector to use
default = mysql.connector
format format for display
default = GRID
"""
output = kwrds.get('output', sys.stdout)
connector = kwrds.get('connector', mysql.connector)
fmt = kwrds.get('format', "grid")
charset = kwrds.get('charset', None)
headers = ("Connection", "Id", "User", "Host", "Db", "Command", "Time",
"State", "Info")
entries = []
# Build SQL statement
for info in connections:
conn = parse_connection(info)
if not conn:
msg = "'%s' is not a valid connection specifier" % (info, )
raise FormatError(msg)
if charset:
conn['charset'] = charset
info = conn
connection = connector.connect(**info)
if not charset:
# If no charset provided, get it from the
# "character_set_client" server variable.
cursor = connection.cursor()
cursor.execute("SHOW VARIABLES LIKE 'character_set_client'")
res = cursor.fetchall()
connection.set_charset_collation(charset=str(res[0][1]))
cursor.close()
cursor = connection.cursor()
cursor.execute(self.__select)
for row in cursor:
if KILL_QUERY in self.__actions:
cursor.execute("KILL {0}".format(row[0]))
if KILL_CONNECTION in self.__actions:
cursor.execute("KILL {0}".format(row[0]))
if PRINT_PROCESS in self.__actions:
entries.append(tuple([_spec(info)] + list(row)))
# If output is None, nothing is printed
if len(entries) > 0 and output:
entries.sort(key=lambda fifth: fifth[5])
print_list(output, fmt, headers, entries)
elif PRINT_PROCESS in self.__actions:
raise EmptyResultError("No matches found")
def test_connection(self, test_num, test_data, with_credentials=False):
"""Test connection.
test_num[in] Test number.
test_data[in] Test data.
with_credentials True if with credentials.
"""
if self.debug:
print "\nTest case {0} - {1}".format(test_num + 1, test_data[0])
try:
if with_credentials:
conn_string = test_data[1]
else:
conn_string = "root@{0}:3306".format(test_data[1])
self.conn_vals = parse_connection(conn_string)
except FormatError as err:
if test_data[3]:
# This expected.
self.results.append("FAIL")
else:
raise MUTLibError("Test Case {0}: Parse failed! Error: "
"{1}".format(test_num + 1, err))
else:
if test_data[3]:
raise MUTLibError("Test Case {0}: Parse should have failed. "
"Got this instead: "
"{1}".format(test_num + 1,
self.conn_vals['host']))
elif with_credentials:
self.results.append(_spec(self.conn_vals))
else:
self.results.append(self.conn_vals['host'])
def execute(self, connections, **kwrds):
"""Execute the search for processes, queries, or connections
This method searches for processes, queriers, or connections to
either kill or display the matches for one or more servers.
connections[in] list of connection parameters
kwrds[in] dictionary of options
output file stream to display information
default = sys.stdout
connector connector to use
default = mysql.connector
format format for display
default = GRID
"""
output = kwrds.get('output', sys.stdout)
connector = kwrds.get('connector', mysql.connector)
fmt = kwrds.get('format', "grid")
charset = kwrds.get('charset', None)
headers = ("Connection", "Id", "User", "Host", "Db",
"Command", "Time", "State", "Info")
entries = []
# Build SQL statement
for info in connections:
conn = parse_connection(info)
if not conn:
msg = "'%s' is not a valid connection specifier" % (info,)
raise FormatError(msg)
if charset:
conn['charset'] = charset
info = conn
connection = connector.connect(**info)
if not charset:
# If no charset provided, get it from the
# "character_set_client" server variable.
cursor = connection.cursor()
cursor.execute("SHOW VARIABLES LIKE 'character_set_client'")
res = cursor.fetchall()
connection.set_charset_collation(charset=str(res[0][1]))
cursor.close()
cursor = connection.cursor()
cursor.execute(self.__select)
for row in cursor:
if KILL_QUERY in self.__actions:
cursor.execute("KILL {0}".format(row[0]))
if KILL_CONNECTION in self.__actions:
cursor.execute("KILL {0}".format(row[0]))
if PRINT_PROCESS in self.__actions:
entries.append(tuple([_spec(info)] + list(row)))
# If output is None, nothing is printed
if len(entries) > 0 and output:
entries.sort(key=lambda fifth: fifth[5])
print_list(output, fmt, headers, entries)
elif PRINT_PROCESS in self.__actions:
raise EmptyResultError("No matches found")
def run(self):
# Test parse_connection with login-paths
con_tests = [
"test_user@localhost:3306", "test_mylogin:3306",
"test_user@localhost:1000", "test_mylogin:1000",
"test-hyphen1234#:3306", "test' \\\"-hyphen:3306"
]
for test_ in con_tests:
con_dic = parse_connection(test_, options={"charset": "utf8"})
# Sort the keys to fix the issue where the keys are printed in
# different order on linux and windows.
self.results.append(sorted(con_dic.iteritems()))
# Test parse_user_password with login-paths
user_pass_tests = [
"test_user", "test_mylogin", "test_user:"******"user_x:",
"user_x:pass_y", "rpl:'L5!w1SJzVuj40(p?tF@(9Y70_@:z(HXc'"
]
for test_ in user_pass_tests:
try:
user_pass = parse_user_password(test_)
self.results.append(user_pass)
except FormatError as err:
self.results.append(err)
# Transform list of dictionaries into list of strings
self.results = ["{0!s}\n".format(con_dic) for con_dic in self.results]
# remove socket information from posix systems to have same output
# on both posix and and windows systems
self.replace_substring_portion(", ('unix_socket'", ".socket')", '')
return True
def test_connection(self, test_num, test_data, with_credentials=False):
"""Test connection.
test_num[in] Test number.
test_data[in] Test data.
with_credentials True if with credentials.
"""
if self.debug:
print "\nTest case {0} - {1}".format(test_num + 1, test_data[0])
try:
if with_credentials:
conn_string = test_data[1]
else:
conn_string = "root@{0}:3306".format(test_data[1])
self.conn_vals = parse_connection(conn_string)
except FormatError as err:
if test_data[3]:
# This expected.
self.results.append("FAIL")
else:
raise MUTLibError("Test Case {0}: Parse failed! Error: "
"{1}".format(test_num + 1, err))
else:
if test_data[3]:
raise MUTLibError("Test Case {0}: Parse should have failed. "
"Got this instead: "
"{1}".format(test_num + 1,
self.conn_vals['host']))
elif with_credentials:
self.results.append(_spec(self.conn_vals))
else:
self.results.append(self.conn_vals['host'])
def run(self):
# Test parse_connection with login-paths
con_tests = ["test_user@localhost:3306", "test_mylogin:3306",
"test_user@localhost:1000", "test_mylogin:1000",
"test_user@localhost:1000:/my.socket",
"test_mylogin:1000:/my.socket",
"test_user@localhost:3306:/my.socket",
"test_mylogin:3306:/my.socket", "test-hyphen1234#:3306",
"test-hyphen1234#:13000:my.socket",
"test' \\\"-hyphen:3306",
"test' \\\"-hyphen:3306:my.socket",
"test' \\\"-hyphen:13001:my.socket"]
for test_ in con_tests:
con_dic = parse_connection(test_, options={"charset": "utf8"})
# Sort the keys to fix the issue where the keys are printed in
# different order on linux and windows.
self.results.append(sorted(con_dic.iteritems()))
# Test parse_user_password with login-paths
user_pass_tests = ["test_user", "test_mylogin", "test_user:"******"user_x:", "user_x:pass_y"]
for test_ in user_pass_tests:
user_pass = parse_user_password(test_)
self.results.append(user_pass)
# Transform list of dictionaries into list of strings
self.results = ["{0!s}\n".format(con_dic) for con_dic in self.results]
# remove socket information from posix systems to have same output
# on both posix and and windows systems
self.replace_substring_portion(", ('unix_socket'", ".socket')", '')
return True
def test_valid(self):
"""Test parsing valid versions of connection strings.
"""
for source, expected in self.valid_specifiers:
result = _spec(parse_connection(source))
frm = u"{0}: was {1}, expected {2}"
msg = frm.format(source, result, expected)
self.assertEqual(expected, result, msg)
def test_valid(self):
"""Test parsing valid versions of connection strings.
"""
for source, expected in self.valid_specifiers:
result = _spec(parse_connection(source))
frm = u"{0}: was {1}, expected {2}"
msg = frm.format(source, result, expected)
self.assertEqual(expected, result, msg)
def execute(self, connections, output=sys.stdout,
connector=mysql.connector, **kwrds):
"""Execute the search for objects
This method searches for objects that match a search criteria for
one or more servers.
connections[in] list of connection parameters
output[in] file stream to display information
default = sys.stdout
connector[in] connector to use
default = mysql.connector
kwrds[in] dictionary of options
format format for display
default = GRID
"""
fmt = kwrds.get('format', "grid")
charset = kwrds.get('charset', None)
ssl_opts = kwrds.get('ssl_opts', {})
entries = []
for info in connections:
conn = parse_connection(info)
if not conn:
msg = "'%s' is not a valid connection specifier" % (info,)
raise FormatError(msg)
if charset:
conn['charset'] = charset
info = conn
conn['host'] = conn['host'].replace("[", "")
conn['host'] = conn['host'].replace("]", "")
if connector == mysql.connector:
set_ssl_opts_in_connection_info(ssl_opts, info)
connection = connector.connect(**info)
if not charset:
# If no charset provided, get it from the
# "character_set_client" server variable.
cursor = connection.cursor()
cursor.execute("SHOW VARIABLES LIKE 'character_set_client'")
res = cursor.fetchall()
connection.set_charset_collation(charset=str(res[0][1]))
cursor.close()
cursor = connection.cursor()
cursor.execute(self.__sql)
entries.extend([tuple([_spec(info)] + list(row))
for row in cursor])
headers = ["Connection"]
headers.extend(col[0].title() for col in cursor.description)
if len(entries) > 0 and output:
print_list(output, fmt, headers, entries)
else:
msg = "Nothing matches '%s' in any %s" % \
(self.__pattern, _join_words(self.__types, conjunction="or"))
raise EmptyResultError(msg)
def run(self):
# Test parse_connection with login-paths missing parameters.
con_tests = ["test_no_host", "test_no_user", "test_no_host:3333",
"test_no_user:3333",
"test_no_user:3306:/does/not/exist/mysql.sock",
"test_no_host:3333"]
for test_ in con_tests:
try:
parse_connection(test_, options={"charset": "utf8"})
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
# remove socket information to have same output
# on both posix and and windows operating systems.
self.replace_substring_portion("port or ", "socket", "port")
return True
def execute(self,
connections,
output=sys.stdout,
connector=mysql.connector,
**kwrds):
"""Execute the search for objects
This method searches for objects that match a search criteria for
one or more servers.
connections[in] list of connection parameters
output[in] file stream to display information
default = sys.stdout
connector[in] connector to use
default = mysql.connector
kwrds[in] dictionary of options
format format for display
default = GRID
"""
fmt = kwrds.get('format', "grid")
charset = kwrds.get('charset', None)
entries = []
for info in connections:
conn = parse_connection(info)
if not conn:
msg = "'%s' is not a valid connection specifier" % (info, )
raise FormatError(msg)
if charset:
conn['charset'] = charset
info = conn
conn['host'] = conn['host'].replace("[", "")
conn['host'] = conn['host'].replace("]", "")
connection = connector.connect(**info)
if not charset:
# If no charset provided, get it from the
# "character_set_client" server variable.
cursor = connection.cursor()
cursor.execute("SHOW VARIABLES LIKE 'character_set_client'")
res = cursor.fetchall()
connection.set_charset_collation(charset=str(res[0][1]))
cursor.close()
cursor = connection.cursor()
cursor.execute(self.__sql)
entries.extend(
[tuple([_spec(info)] + list(row)) for row in cursor])
headers = ["Connection"]
headers.extend(col[0].title() for col in cursor.description)
if len(entries) > 0 and output:
print_list(output, fmt, headers, entries)
else:
msg = "Nothing matches '%s' in any %s" % \
(self.__pattern, _join_words(self.__types, conjunction="or"))
raise EmptyResultError(msg)
def run(self):
# Test parse_connection with login-paths missing parameters.
con_tests = [
"test_user_only", "test_host_only", "test_port_only",
"test_user_only:3306", "test_host_only:3306",
"test_host_only:3306:/does/not/exist/mysql.sock",
"test_port_only:3306"
]
for test_ in con_tests:
try:
parse_connection(test_)
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
# Test login-paths only with socket information separately
# and change the output to be the same on both Windows and Posix
# operating systems.
# Missing username
try:
if os.name == "posix":
parse_connection("test_socket_only")
else:
parse_connection("test_host_only:3306")
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
# on posix systems if we use socket, and there is no hostname,
# hostname is assumed to be "localhost". So it is equivalent
# to have a host and a port on windows systems.
try:
if os.name == "posix":
parse_connection("test_port_only:/does/not/exist/mysql.sock")
else:
parse_connection("test_host_only:3306")
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
# remove socket warnings to have same output
# on both posix and windows operating systems.
self.replace_substring_portion("port or ", "socket", "port")
return True
def run(self):
# Test parse_connection with login-paths missing parameters.
con_tests = ["test_user_only", "test_host_only", "test_port_only",
"test_user_only:3306", "test_host_only:3306",
"test_host_only:3306:/does/not/exist/mysql.sock",
"test_port_only:3306"]
for test_ in con_tests:
try:
parse_connection(test_)
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
# Test login-paths only with socket information separately
# and change the output to be the same on both Windows and Posix
# operating systems.
# Missing username
try:
if os.name == "posix":
parse_connection("test_socket_only")
else:
parse_connection("test_host_only:3306")
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
# on posix systems if we use socket, and there is no hostname,
# hostname is assumed to be "localhost". So it is equivalent
# to have a host and a port on windows systems.
try:
if os.name == "posix":
parse_connection("test_port_only:/does/not/exist/mysql.sock")
else:
parse_connection("test_host_only:3306")
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
# remove socket warnings to have same output
# on both posix and windows operating systems.
self.replace_substring_portion("port or ", "socket", "port")
return True
def parse_user_host(user_name):
"""Parse user, passwd, host, port from user:passwd@host
user_name[in] MySQL user string (user:passwd@host)
"""
no_ticks = user_name.replace("'", "")
try:
conn_values = parse_connection(no_ticks)
except FormatError:
raise UtilError("Cannot parse user:pass@host : %s." % no_ticks)
return (conn_values['user'], conn_values['passwd'], conn_values['host'])
def get_schema_img(self, db_name):
with capture() as dbschema:
server_values = parse_connection(server_connection)
query_options = {'skip_data': True, 'skip_grants': True, 'skip_create': True,
'rpl_mode': None, 'quiet': True}
db_list = []
db_list.append(db_name)
with capture() as dbschema:
dbexport.export_databases(server_values, db_list, sys.stdout, query_options)
db_schema = dbschema.getvalue().splitlines(True)
return db_schema
def parse_user_host(user_name):
"""Parse user, passwd, host, port from user:passwd@host
user_name[in] MySQL user string (user:passwd@host)
"""
no_ticks = user_name.replace("'", "")
try:
conn_values = parse_connection(no_ticks)
except FormatError:
raise UtilError("Cannot parse user:pass@host : %s." %
no_ticks)
return (conn_values['user'], conn_values['passwd'], conn_values['host'])
def parse_user_host(user_name):
"""Parse user, passwd, host, port from user:passwd@host
user_name[in] MySQL user string (user:passwd@host)
returns - tuple - user, passwd, host
"""
# Check for anonymous user. If not, continue.
if user_name == "''@'%'":
return ('', None, '%')
no_ticks = user_name.replace("'", "")
try:
conn_values = parse_connection(no_ticks)
except FormatError:
raise UtilError("Cannot parse user:pass@host : %s." % no_ticks)
return (conn_values['user'], conn_values['passwd'], conn_values['host'])
def run(self):
self.res_fname = "result.txt"
# Test parse_connection with login-paths
con_tests = [
"test_user@localhost:3306", # use of normal parser
"test_mylogin:3306", # use of login-path
# collision with login-path:
"c:\\some_config.cnf", # & no existing windows file
"/etc_etc/some_config.cnf", # & no existing linux file
"test_default_group", # test default client group
"temp_1.cnf", # collision & default group
"temp_1.cnf[undeclared]", # undeclared group
"temp_2.cnf", # collision & default group w/ssl
"temp_2.cnf[missing_values]", # missing values
"temp_1.cnf[complete]", # complete group
]
test_n = 0
for test_ in con_tests:
test_n += 1
msg = "Test case {0} - {1}".format(test_n, test_)
if self.debug:
print(msg)
self.results.append("{0}\n".format(msg))
try:
conn = parse_connection(test_)
except UtilError as err:
if self.debug:
print(err.errmsg)
self.results.append("{0}\n".format(err.errmsg))
else:
if self.debug:
print(sorted(conn.iteritems()))
self.results.append("{0}\n".format(sorted(conn.iteritems())))
# Replacements
self.replace_substring_portion("port or ", "socket", "port")
self.replace_substring(".exe", "")
self.mask_result_portion(
"Unable to locate MySQL Client tools.",
"In addition, could not find a configuration "
"file", "\n", "In addition, could not find a "
"configuration file XXXX-XXXXX")
return True
def run(self):
self.res_fname = "result.txt"
# Test parse_connection with login-paths
con_tests = [
"test_user@localhost:3306", # use of normal parser
"test_mylogin:3306", # use of login-path
# collision with login-path:
"c:\\some_config.cnf", # & no existing windows file
"/etc_etc/some_config.cnf", # & no existing linux file
"test_default_group", # test default client group
"temp_1.cnf", # collision & default group
"temp_1.cnf[undeclared]", # undeclared group
"temp_2.cnf", # collision & default group w/ssl
"temp_2.cnf[missing_values]", # missing values
"temp_1.cnf[complete]", # complete group
]
test_n = 0
for test_ in con_tests:
test_n += 1
msg = "Test case {0} - {1}".format(test_n, test_)
if self.debug:
print(msg)
self.results.append("{0}\n".format(msg))
try:
conn = parse_connection(test_)
except UtilError as err:
if self.debug:
print(err.errmsg)
self.results.append("{0}\n".format(err.errmsg))
else:
if self.debug:
print(sorted(conn.iteritems()))
self.results.append("{0}\n".format(sorted(conn.iteritems())))
# Replacements
self.replace_substring_portion("port or ", "socket", "port")
self.replace_substring(".exe", "")
self.mask_result_portion("Unable to locate MySQL Client tools.",
"In addition, could not find a configuration "
"file", "\n", "In addition, could not find a "
"configuration file XXXX-XXXXX")
return True
def get_schema_img(self, db_name):
with capture() as dbschema:
server_values = parse_connection(server_connection)
query_options = {
'skip_data': True,
'skip_grants': True,
'skip_create': True,
'rpl_mode': None,
'skip_gtid': True,
'quiet': True
}
db_list = []
db_list.append(db_name)
with capture() as dbschema:
dbexport.export_databases(server_values, db_list, sys.stdout,
query_options)
db_schema = dbschema.getvalue().splitlines(True)
return db_schema
def run(self):
self.res_fname = "result.txt"
# Test parse_connection with login-paths
con_tests = [
"test_user@localhost:3306", # use of normal parser
"test/slash", # slash
# collision with login-path:
"test_group_simple_login", # simple collision
"c:\\some_config.cnf", # & no existing windows file
"/etc_etc/some_config.cnf", # & no existing linux file
"test-hyphen1234#", # another collision
"test_default_group", # test default client group
"temp_1.cnf", # collision & default group
"temp_1.cnf[undeclared]", # undeclared group
"temp_2.cnf", # collision & default group w/ssl
"temp_2.cnf[missing_values]" # missing values
]
test_n = 0
for test_ in con_tests:
test_n += 1
msg = "Test case {0} - {1}".format(test_n, test_)
if self.debug:
print(msg)
self.results.append("{0}\n".format(msg))
try:
conn = parse_connection(test_)
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
else:
self.results.append("{0}\n".format(sorted(conn.iteritems())))
# Replacements
self.replace_substring_portion("port or ", "socket", "port")
self.replace_substring(".exe", "")
return True
def run(self):
self.res_fname = "result.txt"
# Test parse_connection with login-paths
con_tests = [
"test_user@localhost:3306", # use of normal parser
"test/slash", # slash
# collision with login-path:
"test_group_simple_login", # simple collision
"c:\\some_config.cnf", # & no existing windows file
"/etc_etc/some_config.cnf", # & no existing linux file
"test-hyphen1234#", # another collision
"test_default_group", # test default client group
"temp_1.cnf", # collision & default group
"temp_1.cnf[undeclared]", # undeclared group
"temp_2.cnf", # collision & default group w/ssl
"temp_2.cnf[missing_values]" # missing values
]
test_n = 0
for test_ in con_tests:
test_n += 1
msg = "Test case {0} - {1}".format(test_n, test_)
if self.debug:
print(msg)
self.results.append("{0}\n".format(msg))
try:
conn = parse_connection(test_)
except UtilError as err:
self.results.append("{0}\n".format(err.errmsg))
else:
self.results.append("{0}\n".format(sorted(conn.iteritems())))
# Replacements
self.replace_substring_portion("port or ", "socket", "port")
self.replace_substring(".exe", "")
return True
parser.error("You must specify either a source user or use the --list "
"option. See --help for details.")
# Check security settings
check_password_security(opt, args)
# Fail if dump and quiet set
if opt.quiet and opt.dump:
parser.error("You cannot use --quiet and --dump together.")
# Warn if quiet and verbosity are both specified
check_verbosity(opt)
# Parse source connection values
try:
source_values = parse_connection(opt.source, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: %s." % err.errmsg)
if opt.list_users and opt.destination:
print(WARN_OPT_NOT_REQUIRED.format(opt='--destination', cmd="--list"))
opt.destination = None
if opt.dump and opt.destination:
print(WARN_OPT_NOT_REQUIRED.format(opt='--destination', cmd="--dump"))
opt.destination = None
def _server_info(server_val, get_defaults=False, options=None):
"""Show information about a running server
This method gathers information from a running server. This information is
returned as a tuple to be displayed to the user in a format specified. The
information returned includes the following:
* server connection information
* version number of the server
* data directory path
* base directory path
* plugin directory path
* configuration file location and name
* current binary log file
* current binary log position
* current relay log file
* current relay log position
server_val[in] the server connection values or a connected server
get_defaults[in] if True, get the default settings for the server
options[in] options for connecting to the server
Return tuple - information about server
"""
if options is None:
options = {}
# Parse source connection values
source_values = parse_connection(server_val, None, options)
# Connect to the server
conn_options = {
'version': "5.1.30",
}
servers = connect_servers(source_values, None, conn_options)
server = servers[0]
params_dict = defaultdict(str)
# Initialize list of warnings
params_dict['warnings'] = []
# Identify server by string: 'host:port[:socket]'.
server_id = "{0}:{1}".format(source_values['host'], source_values['port'])
if source_values.get('socket', None):
server_id = "{0}:{1}".format(server_id, source_values.get('socket'))
params_dict['server'] = server_id
# Get _SERVER_VARIABLES values from the server
for server_var in _SERVER_VARIABLES:
res = server.show_server_variable(server_var)
if res:
params_dict[server_var] = res[0][1]
else:
raise UtilError("Unable to determine {0} of server '{1}'"
".".format(server_var, server_id))
# Get _LOG_FILES_VARIABLES values from the server
for msg, log_tpl in _LOG_FILES_VARIABLES.iteritems():
res = server.show_server_variable(log_tpl.log_name)
if res:
# Check if log is turned off
params_dict[log_tpl.log_name] = res[0][1]
# If logs are turned off, skip checking information about the file
if res[0][1] in ('', 'OFF'):
continue
# Logging is enabled, so we can get get information about log_file
# unless it is log_error because in that case we already have it.
if log_tpl.log_file is not None: # if it is not log_error
log_file = server.show_server_variable(
log_tpl.log_file)[0][1]
params_dict[log_tpl.log_file] = log_file
else: # log error, so log_file_name is already on params_dict
log_file = params_dict[log_tpl.log_name]
# Now get the information about the size of the logs
try:
params_dict[log_tpl.log_file_size] = "{0} bytes".format(
os.path.getsize(log_file))
except os.error:
# if we are unable to get the log_file_size
params_dict[log_tpl.log_file_size] = ''
warning_msg = _WARNING_TEMPLATE.format(msg, log_file)
params_dict['warnings'].append(warning_msg)
else:
params_dict['warnings'].append("Unable to get information "
"regarding variable '{0}'"
).format(msg)
# if audit_log plugin is installed and enabled
if server.supports_plugin('audit'):
res = server.show_server_variable('audit_log_file')
if res:
# Audit_log variable might be a relative path to the datadir,
# so it needs to be treated accordingly
if not os.path.isabs(res[0][1]):
params_dict['audit_log_file'] = os.path.join(
params_dict['datadir'], res[0][1])
else:
params_dict['audit_log_file'] = res[0][1]
# Add audit_log field to the _COLUMNS List unless it is already
# there
if 'audit_log_file' not in _COLUMNS_SET:
_COLUMNS.append('audit_log_file')
_COLUMNS.append('audit_log_file_size')
_COLUMNS_SET.add('audit_log_file')
try:
params_dict['audit_log_file_size'] = "{0} bytes".format(
os.path.getsize(params_dict['audit_log_file']))
except os.error:
# If we are unable to get the size of the audit_log_file
params_dict['audit_log_file_size'] = ''
warning_msg = _WARNING_TEMPLATE.format(
"audit log",
params_dict['audit_log_file']
)
params_dict['warnings'].append(warning_msg)
# Build search path for config files
if os.name == "posix":
my_def_search = ["/etc/my.cnf", "/etc/mysql/my.cnf",
os.path.join(params_dict['basedir'], "my.cnf"),
"~/.my.cnf"]
else:
my_def_search = [r"c:\windows\my.ini", r"c:\my.ini", r"c:\my.cnf",
os.path.join(os.curdir, "my.ini")]
my_def_search.append(os.path.join(os.curdir, "my.cnf"))
# Get server's default configuration values.
defaults = []
if get_defaults:
# Can only get defaults for local servers (need to access local data).
if server.is_alias('localhost'):
try:
my_def_path = get_tool_path(params_dict['basedir'],
"my_print_defaults")
except UtilError as err:
raise UtilError("Unable to retrieve the defaults data "
"(requires access to my_print_defaults): {0} "
"(basedir: {1})".format(err.errmsg,
params_dict['basedir'])
)
out_file = tempfile.TemporaryFile()
# Execute tool: <basedir>/my_print_defaults mysqld
subprocess.call([my_def_path, "mysqld"], stdout=out_file)
out_file.seek(0)
# Get defaults data from temp output file.
defaults.append("\nDefaults for server {0}".format(server_id))
for line in out_file.readlines():
defaults.append(line.rstrip())
else:
# Remote server; Cannot get the defaults data.
defaults.append("\nWARNING: The utility can not get defaults from "
"a remote host.")
# Find config file
config_file = ""
for search_path in my_def_search:
if os.path.exists(search_path):
if len(config_file) > 0:
config_file = "{0}, {1}".format(config_file, search_path)
else:
config_file = search_path
params_dict['config_file'] = config_file
# Find binary log, relay log
params_dict['binary_log'], params_dict['binary_log_pos'] = _get_binlog(
server)
params_dict['relay_log'], params_dict['relay_log_pos'] = _get_relay_log(
server)
server.disconnect()
return params_dict, defaults
default_val = "root@localhost:3306"
print(WARN_OPT_USING_DEFAULT.format(default=default_val, opt="--master"))
# Print the WARNING to force determinism if a parser error occurs.
sys.stdout.flush()
# option --slave is required (mandatory)
if not opt.slave:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--slave"))
# option --rpl-user is required (mandatory)
if not opt.rpl_user:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--rpl-user"))
# Parse source connection values
try:
m_values = parse_connection(opt.master, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Master connection values invalid: {0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Master connection values invalid: {0}." "".format(err.errmsg))
# Parse source connection values
try:
s_values = parse_connection(opt.slave, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Slave connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
"debug": opt.verbosity >= 3,
"new_engine": opt.new_engine,
"def_engine": opt.def_engine,
"skip_rpl": opt.skip_rpl,
"skip_gtid": opt.skip_gtid,
"table": opt.table,
"charset": opt.charset,
"multiprocess": num_cpu if opt.multiprocess == 0 else opt.multiprocess,
"autocommit": opt.autocommit,
"max_bulk_insert": max_bulk_size,
}
# Parse server connection values
try:
options.update(get_ssl_dict(opt))
server_values = parse_connection(opt.server, None, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: {0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: "
"{0}.".format(err.errmsg))
# Check values for --format=raw_csv
if opt.format == "raw_csv":
if not opt.table:
print("ERROR: You must provide --table while using "
"--format=raw_csv.")
sys.exit(1)
# Validate table name using format <db>.<table>
"rpl_mode": opt.rpl_mode,
"verbosity": opt.verbosity,
"skip_gtid": opt.skip_gtid,
"charset": opt.charset,
"multiprocess": num_cpu if opt.multiprocess == 0 else opt.multiprocess,
}
options.update(get_ssl_dict(opt))
# Parse source connection values
try:
# Create a basic configuration reader first for optimization purposes.
# I.e., to avoid repeating the execution of some methods in further
# parse_connection methods (like, searching my_print_defaults tool).
config_reader = MyDefaultsReader(options, False)
source_values = parse_connection(opt.source, config_reader, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: {0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: "
"{0}.".format(err.errmsg))
# Parse destination connection values
try:
dest_values = parse_connection(opt.destination, config_reader, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Destination connection values invalid: "
"{0}.".format(err))
def get_diff(self, db_name, version):
file_schema = "%s/%04d-schema.img" % (db_name, int(version))
file_schema_swp = "%s/%04d-schema.swp" % (db_name, int(version))
tmp_db = "tmp_online_mig_%s" % (db_name)
query = "CREATE DATABASE %s;" % tmp_db
self.server.disable_foreign_key_checks()
self.server.exec_query(query)
f = open(file_schema, 'r')
f_swp = open(file_schema_swp, 'w')
f_swp.write("USE %s\n" % tmp_db)
buff = ""
for line in f.readlines():
#if re.search(';$', line, re.IGNORECASE):
if re.search(';$' + '(?i)', line):
buff = buff + line
f_swp.write(buff)
buff = ""
else:
buff = buff + line.strip()
f.close()
f_swp.close()
query_options = {'multi': True}
f_swp = open(file_schema_swp, 'r')
for line in f_swp.readlines():
self.server.exec_query(line, query_options)
f_swp.close()
os.remove(file_schema_swp)
query_options = {
'run_all_tests': True,
'reverse': False,
'verbosity': None,
'no_object_check': False,
'no_data': True,
'quiet': True,
'difftype': 'differ',
'width': 75,
'changes-for': 'server1',
'skip_grants': True,
'skip_gtid': True
}
source_values = parse_connection(server_connection)
destination_values = parse_connection(server_connection)
with capture() as stepback:
dbcompare.database_compare(source_values, destination_values,
db_name, tmp_db, query_options)
buf = ""
found = 0
for line in stepback.getvalue().splitlines(True):
logging.debug(u"%s" % line)
if re.search('CREATE TABLE', line):
line = re.sub("CREATE" + '(?i)', "", line, 1)
#line = re.sub("CREATE","", line, 1, re.IGNORECASE)
#line = re.sub("\(\n","", line, 1, re.IGNORECASE)
line = re.sub("\(\n" + '(?i)', "", line, 1)
buf = line
if (re.search('^\+', line)
or re.search('^\-', line)) and not re.search(
'CREATE DATABASE', line) and not re.search(
'CONSTRAINT.*FOREIGN KEY', line):
#line = re.sub("\n","", line, 1, re.IGNORECASE)
line = re.sub("\n" + '(?i)', "", line, 1)
if len(buf) > 0:
print u"%s" % buf
buf = ""
print u"%s" % line
query = "DROP DATABASE %s" % tmp_db
self.server.exec_query(query)
self.server.disable_foreign_key_checks(disable=False)
def migrate_up(self, db_name, last_version):
(ver, md5, comment) = self.read_meta(db_name, int(last_version))
if self.verify_checksum(db_name, last_version, md5) is False:
logging.warning(
"The current schema doesn't match the last applied migration")
version = self.new_migration_version(db_name)
if not os.path.exists("%s/%04d-up.mig" % (db_name, int(version))):
logging.info("No migration available")
else:
logging.info(u"Preparing migration to version %04d" % int(version))
if os.path.exists("%s/%04d-down.mig" % (db_name, int(version))):
os.remove("%s/%04d-down.mig" % (db_name, int(version)))
(ver, md5, comment) = self.read_meta(db_name, int(version))
query_options = {'skip_data': True, 'force': True}
db_list = []
grp = re.match("(\w+)(?:\:(\w+))?",
"%s:%s_%s" % (db_name, self.tmp_prefix, db_name))
db_entry = grp.groups()
db_list.append(db_entry)
source_values = parse_connection(server_connection)
destination_values = parse_connection(server_connection)
with capture() as stepback:
dbcopy.copy_db(source_values, destination_values, db_list,
query_options)
self.online_schema_change(
db_name, version, "%s/%04d-up.mig" % (db_name, int(version)))
if self.verify_checksum(db_name, version, md5) is True:
logging.info("Applied changes match the requested schema")
else:
logging.error(
"Something didn't run as expected, db schema doesn't match !"
)
self.change_migration_status(db_name, version,
'invalid checksum')
query_options = {
'run_all_tests': True,
'reverse': True,
'verbosity': None,
'no_object_check': False,
'no_data': True,
'quiet': True,
'difftype': 'sql',
'width': 75,
'changes-for': 'server1',
'skip_grants': True,
'skip_gtid': True
}
with capture() as stepback:
res = dbcompare.database_compare(
source_values, destination_values, db_name,
"%s_%s" % (self.tmp_prefix, db_name), query_options)
str = stepback.getvalue().splitlines(True)
to_add = 0
file_down = open("%s/%04d-down.tmp" % (db_name, int(version)), 'a')
for line in str:
if line[0] not in ['#', '\n', '+', '-', '@']:
# this if is required currently due to missing foreign keys in dbcopy
if not re.match("\s+DROP FOREIGN KEY", line):
#line = re.sub(" %s\." % db_name, " ", line, 1, re.IGNORECASE)
line = re.sub(" %s\." % db_name + '(?i)', " ", line, 1)
file_down.write("%s\n" % line.strip())
elif re.match("# WARNING: Objects in", line):
if re.match("# WARNING: Objects in \w+\.tmp_online_mig_",
line):
to_add = 2
else:
to_add = 1
else:
grp = re.match("#\s+TABLE\: (\w+)", line)
if grp:
if to_add == 2:
query = "SHOW CREATE TABLE tmp_online_mig_%s.%s;" % (
db_name, grp.group(1))
res = self.server.exec_query(query)
file_down.write("%s\n" % res[0][1])
elif to_add == 1:
file_down.write("DROP TABLE %s;\n" % grp.group(1))
file_down.close()
file_down_tmp = "%s/%04d-down.tmp" % (db_name, int(version))
self.create_migration_file(db_name, file_down_tmp, version, "down")
query = "DROP DATABASE %s_%s" % (self.tmp_prefix, db_name)
res = self.server.exec_query(query)
#os.remove(file_down_tmp)
file_schema = "%s/%04d-schema.img" % (db_name, int(version))
self.create_schema_img(db_name, file_schema)
def run(self):
# Test "--show" password in plain text option is available on this
# version of my_print_defaults.
test_num = 1
test_case = 'Test "--show" password in plain text option is available'
comment = "Test case {0} {1}".format(test_num, test_case)
my_defaults_reader = MyDefaultsReader()
if not my_defaults_reader.check_show_required():
raise MUTLibError("{0}: failed".format(comment))
else:
if self.debug:
print("{0}: pass".format(comment))
self.results.append("{0}: pass\n".format(comment))
# Test retrieve of expected values for passwords variables stored in
# test.mylogin.cnf
con_tests = {
"test1": "user_1:pass_1@localhost:10001",
"test2": "user_2:A_passw0rd@localhost:20002",
"test3": "user_3:M4g1cw0rd@localhost:30003",
"test4": "user_4:-*123 !%^@remotehost:40004",
}
for group in sorted(con_tests.keys()):
test_num += 1
test_case = 'Test Retrieve group: "{0}"'.format(group)
comment = "Test case {0} {1}".format(test_num, test_case)
self.results.append("{0}\n".format(comment))
if self.debug:
print(comment)
# Retrieve stored values in group from .mylogin.cnf
ret_dt = my_defaults_reader.get_group_data(group)
# If not values, group was not found
if ret_dt is None:
if self.debug:
print('Can not retrieve values for group: "{}"'
''.format(group))
raise MUTLibError("{0}: failed".format(comment))
con_dic = parse_connection(con_tests[group],
options={"charset": "utf8"})
con_dic.pop("charset")
self.results.append("Retrieved data:\n")
if self.debug:
print("Retrieved data:")
for data in sorted(ret_dt.iteritems()):
# Only password is saved as passwd key
if 'passw' in data[0]:
if con_dic['passwd'] != data[1]:
if self.debug:
print(
_VALUES_DIFFER_MSG.format(
"password", con_dic['passwd'], data[1]))
raise MUTLibError("{0}: failed".format(comment))
else:
msg = _VALUES_EQUAL_MSG.format("password",
con_dic['passwd'],
data[1])
if self.debug:
print(msg)
self.results.append("{0}\n".format(msg))
elif str(con_dic[data[0]]) != data[1]:
if self.debug:
print(
_VALUES_DIFFER_MSG.format(data[0],
con_dic[data[0]],
data[1]))
raise MUTLibError("{0}: failed".format(comment))
else:
msg = _VALUES_EQUAL_MSG.format(data[0], con_dic[data[0]],
data[1])
if self.debug:
print(msg)
self.results.append("{0}\n".format(msg))
# None check failed, mark as pass
self.results.append("Test result: pass\n")
if self.debug:
print("Result: pass")
return True
add_verbosity(parser, False)
# Now we process the rest of the arguments.
opt, args = parser.parse_args()
# Check security settings
check_password_security(opt, args)
# Check to make sure at least one table specified.
if len(args) == 0:
parser.error("You must specify at least one table or database to " "check.")
# Parse source connection values
try:
conn_options = get_ssl_dict(opt)
source_values = parse_connection(opt.server, options=conn_options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: %s." % err.errmsg)
# Check best, worst for validity
best = None
if opt.best is not None:
try:
best = int(opt.best)
except ValueError:
best = -1
if best is not None and best < 1:
"rpl_mode": opt.rpl_mode,
"verbosity": opt.verbosity,
"skip_gtid": opt.skip_gtid,
"charset": opt.charset,
"multiprocess": num_cpu if opt.multiprocess == 0 else opt.multiprocess,
}
options.update(get_ssl_dict(opt))
# Parse source connection values
try:
# Create a basic configuration reader first for optimization purposes.
# I.e., to avoid repeating the execution of some methods in further
# parse_connection methods (like, searching my_print_defaults tool).
config_reader = MyDefaultsReader(options, False)
source_values = parse_connection(opt.source, config_reader, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: {0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: " "{0}.".format(err.errmsg))
# Parse destination connection values
try:
dest_values = parse_connection(opt.destination, config_reader, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Destination connection values invalid: " "{0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
for slave_val in slaves_val:
if check_hostname_alias(master_val, slave_val):
err = ERROR_MASTER_IN_SLAVES.format(
master_host=master_val['host'],
master_port=master_val['port'],
slaves_candidates="slaves",
slave_host=slave_val['host'],
slave_port=slave_val['port'],
)
parser.error(err)
# Parse source connection values of --server
if opt.server:
try:
ssl_opts = get_ssl_dict(opt)
server_val = parse_connection(opt.server, None, ssl_opts)
except FormatError as err:
parser.error("ERROR: {0}\n".format(err.errmsg))
except UtilError as err:
parser.error("ERROR: {0}\n".format(err.errmsg))
# Create dictionary of options
options = {
'discover': opt.discover,
'verbosity': 0 if opt.verbosity is None else opt.verbosity,
'to_binlog_name': opt.binlog,
'dry_run': opt.dry_run,
}
try:
binlog_purge(server_val, master_val, slaves_val, options)
server_list = ServerList([], opt.start_port, opt.utildir, opt.verbosity)
basedir = None
# Print status of connections
print("\nServers:")
if not opt.servers:
print(" No servers specified.")
else:
i = 0
for server in opt.servers:
# add ssl options values.
conn_options = {}
conn_options.update(get_ssl_dict(opt))
try:
conn_val = parse_connection(server, options=conn_options)
except UtilError as err:
parser.error(err.errmsg)
except:
parser.error("Problem parsing server connection "
"'{0}'".format(server))
i += 1
# Fail if port is None
if conn_val["port"] is None:
parser.error("You must specify a port in the server "
"string: \n {0}".format(server))
sys.stdout.write(" Connecting to {0} as user {1} on port "
"{2}: ".format(conn_val["host"], conn_val["user"],
conn_val["port"]))
"locking": opt.locking,
"rpl_user": opt.rpl_user,
"rpl_mode": opt.rpl_mode,
"rpl_file": opt.rpl_file,
"comment_rpl": opt.comment_rpl,
"export": opt.export,
"skip_gtid": opt.skip_gtid,
"charset": opt.charset,
"multiprocess": num_cpu if opt.multiprocess == 0 else opt.multiprocess,
"output_filename": output_filename,
}
# Parse server connection values
try:
options.update(get_ssl_dict(opt))
server_values = parse_connection(opt.server, None, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: {0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: "
"{0}.".format(err.errmsg))
# Build list of databases to copy
db_list = []
for db in args:
# Remove backtick quotes (handled later)
db = remove_backtick_quoting(db) \
if is_quoted_with_backticks(db) else db
db_list.append(db)
# Add verbosity
add_verbosity(parser, quiet=False)
# Now we process the rest of the arguments.
opt, args = parser.parse_args()
# Check mandatory options: --server
if not opt.server:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--server"))
servers_val = None
# Parse source connection values if --server provided or default
try:
ssl_opts = get_ssl_dict(opt)
server_val = parse_connection(opt.server, None, ssl_opts)
except FormatError as err:
# pylint: disable=E1101
parser.error("ERROR: {0}\n".format(err.errmsg))
except UtilError as err:
parser.error("ERROR: {0}\n".format(err.errmsg))
# Create dictionary of options
options = {
'verbosity': 0 if opt.verbosity is None else opt.verbosity,
"min_size": opt.min_size,
}
try:
binlog_rotate(server_val, options)
except UtilError:
cmd=command,
only_cmd=only_used_cmds))
opt.candidates = None
# --format only used by 'health' or 'gtid' command
if opt.format and command not in ['health', 'gtid']:
only_used_cmds = 'health or gtid'
print(WARN_OPT_NOT_REQUIRED_ONLY_FOR.format(opt='--format',
cmd=command,
only_cmd=only_used_cmds))
opt.format = None
# Parse the --new-master connection string
if opt.new_master:
try:
new_master_val = parse_connection(opt.new_master, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("New master connection values invalid: "
"{0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("New master connection values invalid: "
"{0}.".format(err.errmsg))
else:
new_master_val = None
# Parse the master, slaves, and candidates connection parameters
try:
master_val, slaves_val, candidates_val = parse_topology_connections(
opt)
parser.add_option("-a", "--all", action="store_true", dest="do_all",
default=False, help="show all usage including empty "
"databases")
# Add verbosity mode
add_verbosity(parser, True)
# Now we process the rest of the arguments.
opt, args = parser.parse_args()
# Check security settings
check_password_security(opt, args)
# Parse source connection values
try:
source_values = parse_connection(opt.server, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: %s." % err.errmsg)
try:
conn_options = {
'version': "5.1.30",
}
servers = connect_servers(source_values, None)
except UtilError:
_, e, _ = sys.exc_info()
parser.error(e.errmsg)
def show_server_info(servers, options):
"""Show server information for a list of servers
This method will gather information about a running server. If the
show_defaults option is specified, the method will also read the
configuration file and return a list of the server default settings.
If the format option is set, the output will be in the format specified.
If the no_headers option is set, the output will not have a header row (no
column names) except for format = vertical.
If the basedir and start options are set, the method will attempt to start
the server in read only mode to get the information. Specifying only
basedir will not start the server. The extra start option is designed to
make sure the user wants to start the offline server. The user may not wish
to do this if there are certain error conditions and/or logs in place that
may be overwritten.
servers[in] list of server connections in the form
<user>:<password>@<host>:<port>:<socket>
options[in] dictionary of options (no_headers, format, basedir,
start, show_defaults)
Returns tuple ((server information), defaults)
"""
no_headers = options.get("no_headers", False)
fmt = options.get("format", "grid")
show_defaults = options.get("show_defaults", False)
basedir = options.get("basedir", None)
datadir = options.get("datadir", None)
start = options.get("start", False)
show_servers = options.get("show_servers", 0)
if show_servers:
if os.name == 'nt':
ports = options.get("ports", "3306:3333")
start_p, end_p = ports.split(":")
_show_running_servers(start_p, end_p)
else:
_show_running_servers()
ssl_dict = {}
ssl_dict['ssl_cert'] = options.get("ssl_cert", None)
ssl_dict['ssl_ca'] = options.get("ssl_ca", None)
ssl_dict['ssl_key'] = options.get("ssl_key", None)
ssl_dict['ssl'] = options.get("ssl", None)
row_dict_lst = []
warnings = []
server_val = {}
for server in servers:
new_server = None
try:
test_connect(server, throw_errors=True, ssl_dict=ssl_dict)
except UtilError as util_error:
conn_dict = get_connection_dictionary(server, ssl_dict=ssl_dict)
server1 = Server(options={'conn_info': conn_dict})
server_is_off = False
# If we got errno 2002 it means can not connect through the
# given socket.
if util_error.errno == CR_CONNECTION_ERROR:
socket = conn_dict.get("unix_socket", "")
if socket:
msg = ("Unable to connect to server using socket "
"'{0}'.".format(socket))
if os.path.isfile(socket):
err_msg = ["{0} Socket file is not valid.".format(msg)]
else:
err_msg = [
"{0} Socket file does not "
"exist.".format(msg)
]
# If we got errno 2003 and we do not have
# socket, instead we check if server is localhost.
elif (util_error.errno == CR_CONN_HOST_ERROR
and server1.is_alias("localhost")):
server_is_off = True
# If we got errno 1045 it means Access denied,
# notify the user if a password was used or not.
elif util_error.errno == ER_ACCESS_DENIED_ERROR:
use_pass = '******' if conn_dict['passwd'] else 'NO'
err_msg = ("Access denied for user '{0}'@'{1}' using "
"password: {2}".format(conn_dict['user'],
conn_dict['host'], use_pass))
# Use the error message from the connection attempt.
else:
err_msg = [util_error.errmsg]
# To propose to start a cloned server for extract the info,
# can not predict if the server is really off, but we can do it
# in case of socket error, or if one of the related
# parameter was given.
if server_is_off or basedir or datadir or start:
err_msg = [
"Server is offline. To connect, "
"you must also provide "
]
opts = ["basedir", "datadir", "start"]
for opt in tuple(opts):
try:
if locals()[opt] is not None:
opts.remove(opt)
except KeyError:
pass
if opts:
err_msg.append(", ".join(opts[0:-1]))
if len(opts) > 1:
err_msg.append(" and the ")
err_msg.append(opts[-1])
err_msg.append(" option")
raise UtilError("".join(err_msg))
if not start:
raise UtilError("".join(err_msg))
else:
try:
server_val = parse_connection(server, None, options)
except:
raise UtilError("Source connection values invalid"
" or cannot be parsed.")
new_server = _start_server(server_val, basedir, datadir,
options)
info_dict, defaults = _server_info(server, show_defaults, options)
warnings.extend(info_dict['warnings'])
if info_dict:
row_dict_lst.append(info_dict)
if new_server:
# Need to stop the server!
new_server.disconnect()
_stop_server(server_val, basedir, options)
# Get the row values stored in the dictionaries
rows = [[row_dict[key] for key in _COLUMNS] for row_dict in row_dict_lst]
print_list(sys.stdout, fmt, _COLUMNS, rows, no_headers)
if warnings:
print("\n# List of Warnings: \n")
for warning in warnings:
print("WARNING: {0}\n".format(warning))
# Print the default configurations.
if show_defaults and len(defaults) > 0:
for row in defaults:
print(" {0}".format(row))
server = None
if opt.server is None and opt.diagnostic:
print("# WARNING: Cannot generate character set or "
"collation names without the --server option.")
# Check start timeout for minimal value
if int(opt.start_timeout) < 10:
opt.start_timeout = 10
print("# WARNING: --start-timeout must be >= 10 seconds. Using "
"default value.")
# Parse source connection values if --server provided
if opt.server is not None and not opt.basedir:
try:
ssl_opts = get_ssl_dict(opt)
source_values = parse_connection(opt.server, None, ssl_opts)
except FormatError as err:
parser.error("Source connection values invalid: %s." % err)
except UtilError as err:
parser.error("Source connection values invalid: %s." % err.errmsg)
try:
conn_options = {
'version': "5.1.30",
'quiet': opt.quiet,
}
servers = connect_servers(source_values, None, conn_options)
except UtilError as error:
parser.error(error.errmsg)
server = servers[0]
# option --master is required (mandatory)
if not opt.master:
default_val = 'root@localhost:3306'
print(
WARN_OPT_USING_DEFAULT.format(default=default_val, opt='--master'))
# Print the WARNING to force determinism if a parser error occurs.
sys.stdout.flush()
# option --slave is required (mandatory)
if not opt.slave:
parser.error(PARSE_ERR_OPTS_REQ.format(opt='--slave'))
# Parse source connection values
try:
m_values = parse_connection(opt.master, options=opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Master connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Master connection values invalid: %s." % err.errmsg)
# Parse source connection values
try:
s_values = parse_connection(opt.slave, options=opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Slave connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
if not opt.slave:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--slave"))
# option --masters is required (mandatory)
if not opt.masters:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--masters"))
# option --rpl-user is required (mandatory)
if not opt.rpl_user:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--rpl-user"))
config_reader = MyDefaultsReader(opt, False)
# Parse slave connection values
try:
slave_vals = parse_connection(opt.slave, config_reader, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Slave connection values invalid: {0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Slave connection values invalid: {0}."
"".format(err.errmsg))
# Parse masters connection values
masters_vals = []
masters = opt.masters.split(",")
if len(masters) == 1:
parser.error("At least two masters are required for multi-source "
"replication.")
parser.error("You must specify either a source user or use the --list "
"option. See --help for details.")
# Check security settings
check_password_security(opt, args)
# Fail if dump and quiet set
if opt.quiet and opt.dump:
parser.error("You cannot use --quiet and --dump together.")
# Warn if quiet and verbosity are both specified
check_verbosity(opt)
# Parse source connection values
try:
source_values = parse_connection(opt.source, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Source connection values invalid: %s." % err.errmsg)
if opt.list_users and opt.destination:
print(WARN_OPT_NOT_REQUIRED.format(opt='--destination', cmd="--list"))
opt.destination = None
if opt.dump and opt.destination:
print(WARN_OPT_NOT_REQUIRED.format(opt='--destination', cmd="--dump"))
opt.destination = None
"difftype": opt.difftype,
"force": opt.force,
"width": opt.width,
"changes-for": opt.changes_for,
"reverse": opt.reverse,
"skip_table_opts": opt.skip_tbl_opts,
"compact": opt.compact,
"charset": opt.charset,
}
# add ssl options values.
options.update(get_ssl_dict(opt))
# Parse server connection values
try:
server1_values = parse_connection(opt.server1, None, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server1 connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server1 connection values invalid: %s." % err.errmsg)
if opt.server2 is not None:
try:
server2_values = parse_connection(opt.server2, None, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server2 connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server2 connection values invalid: %s." % err.errmsg)
"by databases and objects inside those databases. "
"Finally, if GLOBAL level is specified, normal "
"inheritance rules are applied, global grants "
"apply to both databases and objects and database "
"level grants apply to the objects.")
# Now we process the rest of the arguments.
opt, args = parser.parse_args()
if not opt.server:
parser.error("You need to specify a server using the --server option.")
# Parse server connection
server_val = None
try:
server_val = parse_connection(opt.server, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: " "{0!s}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: "
"{0!s}.".format(err.errmsg))
# The --show=users can only be used together with the privilege option
if opt.show_mode == 'users' and not opt.privileges:
parser.error("The --show=users can only be used if you specify a "
"list of privileges with the --privileges option.")
conn_opts = {
'quiet': True,
# Check mandatory option: --server.
if not opt.server:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--server"))
# Check security settings
check_password_security(opt, args)
# Check to make sure at least one table specified.
if len(args) == 0:
parser.error("You must specify at least one table or database to "
"check.")
# Parse source connection values
try:
conn_options = get_ssl_dict(opt)
source_values = parse_connection(opt.server, options=conn_options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: %s." % err.errmsg)
# Check best, worst for validity
best = None
if opt.best is not None:
try:
best = int(opt.best)
except ValueError:
best = -1
if best is not None and best < 1:
if not opt.slave:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--slave"))
# option --masters is required (mandatory)
if not opt.masters:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--masters"))
# option --rpl-user is required (mandatory)
if not opt.rpl_user:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--rpl-user"))
config_reader = MyDefaultsReader(opt, False)
# Parse slave connection values
try:
slave_vals = parse_connection(opt.slave, config_reader, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Slave connection values invalid: {0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Slave connection values invalid: {0}."
"".format(err.errmsg))
# Parse masters connection values
masters_vals = []
masters = opt.masters.split(",")
if len(masters) == 1:
parser.error("At least two masters are required for multi-source "
"replication.")
# Expand user paths and resolve relative paths
if opt.new_data and opt.new_data[0] == '~':
options['new_data'] = os.path.expanduser(opt.new_data)
if opt.basedir and opt.basedir[0] == '~':
options['basedir'] = os.path.expanduser(opt.basedir)
if opt.new_data and opt.new_data[0] == '.':
options['new_data'] = os.path.abspath(opt.new_data)
if opt.basedir and opt.basedir[0] == '.':
options['basedir'] = os.path.abspath(opt.basedir)
# Parse source connection values if we have a running server
if opt.basedir is None:
conn_options = get_ssl_dict(opt)
try:
conn = parse_connection(opt.server, options=conn_options)
# Now check for local server
server = Server({'conn_info': conn})
if not server.is_alias('localhost'):
parser.error("Server to be cloned must be running on the same "
"machine as mysqlserverclone.")
except exception.FormatError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: %s." % err)
except exception.UtilError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: %s." % err.errmsg)
else:
conn = None
def show_server_info(servers, options):
"""Show server information for a list of servers
This method will gather information about a running server. If the
show_defaults option is specified, the method will also read the
configuration file and return a list of the server default settings.
If the format option is set, the output will be in the format specified.
If the no_headers option is set, the output will not have a header row (no
column names) except for format = vertical.
If the basedir and start options are set, the method will attempt to start
the server in read only mode to get the information. Specifying only
basedir will not start the server. The extra start option is designed to
make sure the user wants to start the offline server. The user may not wish
to do this if there are certain error conditions and/or logs in place that
may be overwritten.
servers[in] list of server connections in the form
<user>:<password>@<host>:<port>:<socket>
options[in] dictionary of options (no_headers, format, basedir,
start, show_defaults)
Returns tuple ((server information), defaults)
"""
no_headers = options.get("no_headers", False)
fmt = options.get("format", "grid")
show_defaults = options.get("show_defaults", False)
basedir = options.get("basedir", None)
datadir = options.get("datadir", None)
start = options.get("start", False)
show_servers = options.get("show_servers", 0)
if show_servers:
if os.name == 'nt':
ports = options.get("ports", "3306:3333")
start_p, end_p = ports.split(":")
_show_running_servers(start_p, end_p)
else:
_show_running_servers()
row_dict_lst = []
warnings = []
server_val = {}
for server in servers:
new_server = None
try:
test_connect(server, True)
except UtilError as util_error:
conn_dict = get_connection_dictionary(server)
server1 = Server(options={'conn_info': conn_dict})
server_is_off = False
# If we got errno 2002 it means can not connect through the
# given socket, but if path to socket not empty, server could be
# turned off.
if util_error.errno == CR_CONNECTION_ERROR:
socket = conn_dict.get("unix_socket", "")
if socket:
mydir = os.path.split(socket)[0]
if os.path.isdir(mydir) and len(os.listdir(mydir)) != 0:
server_is_off = True
# If we got errno 2003 and this is a windows, we do not have
# socket, instead we check if server is localhost.
elif (util_error.errno == CR_CONN_HOST_ERROR and
os.name == 'nt' and server1.is_alias("localhost")):
server_is_off = True
# If we got errno 1045 it means Access denied,
# notify the user if a password was used or not.
elif util_error.errno == ER_ACCESS_DENIED_ERROR:
use_pass = '******' if conn_dict['passwd'] else 'NO'
er = ("Access denied for user '{0}'@'{1}' using password: {2}"
).format(conn_dict['user'], conn_dict['host'], use_pass)
# Use the error message from the connection attempt.
else:
er = [util_error.errmsg]
# To propose to start a cloned server for extract the info,
# can not predict if the server is really off, but we can do it
# in case of socket error, or if one of the related
# parameter was given.
if (server_is_off or basedir or datadir or start):
er = ["Server is offline. To connect, "
"you must also provide "]
opts = ["basedir", "datadir", "start"]
for opt in tuple(opts):
try:
if locals()[opt] is not None:
opts.remove(opt)
except KeyError:
pass
if opts:
er.append(", ".join(opts[0:-1]))
if len(opts) > 1:
er.append(" and the ")
er.append(opts[-1])
er.append(" option")
raise UtilError("".join(er))
if not start:
raise UtilError("".join(er))
else:
try:
server_val = parse_connection(server, None, options)
except:
raise UtilError("Source connection values invalid"
" or cannot be parsed.")
new_server = _start_server(server_val, basedir,
datadir, options)
info_dict, defaults = _server_info(server, show_defaults, options)
warnings.extend(info_dict['warnings'])
if info_dict:
row_dict_lst.append(info_dict)
if new_server:
# Need to stop the server!
new_server.disconnect()
_stop_server(server_val, basedir, options)
# Get the row values stored in the dictionaries
rows = [[row_dict[key] for key in _COLUMNS] for row_dict in row_dict_lst]
print_list(sys.stdout, fmt, _COLUMNS, rows, no_headers)
if warnings:
print("\n# List of Warnings: \n")
for warning in warnings:
print("WARNING: {0}\n".format(warning))
# Print the default configurations.
if show_defaults and len(defaults) > 0:
for row in defaults:
print(" {0}".format(row))
# option --master is required (mandatory)
if not opt.master:
default_val = 'root@localhost:3306'
print(WARN_OPT_USING_DEFAULT.format(default=default_val,
opt='--master'))
# Print the WARNING to force determinism if a parser error occurs.
sys.stdout.flush()
# option --slave is required (mandatory)
if not opt.slave:
parser.error(PARSE_ERR_OPTS_REQ.format(opt='--slave'))
# Parse source connection values
try:
m_values = parse_connection(opt.master, options=opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Master connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Master connection values invalid: %s." % err.errmsg)
# Parse source connection values
try:
s_values = parse_connection(opt.slave, options=opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Slave connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
"for any given object. Unless a user has all the "
"privileges listed for a specific object, "
"she will not appear in the list of users with "
"privileges for that specific object. To list "
"multiple privileges, use a comma-separated list.")
# Now we process the rest of the arguments.
opt, args = parser.parse_args()
if not opt.server:
parser.error("You need to specify a server using the --server option.")
# Parse server connection
server_val = None
try:
server_val = parse_connection(opt.server, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: "
"{0!s}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server connection values invalid: "
"{0!s}.".format(err.errmsg))
# The --show=users can only be used together with the privilege option
if opt.show_mode == 'users' and not opt.privileges:
parser.error("The --show=users can only be used if you specify a "
"list of privileges with the --privileges option.")
# Process list objects for which grants will be shown
def _server_info(server_val, get_defaults=False, options=None):
"""Show information about a running server
This method gathers information from a running server. This information is
returned as a tuple to be displayed to the user in a format specified. The
information returned includes the following:
* server connection information
* version number of the server
* data directory path
* base directory path
* plugin directory path
* configuration file location and name
* current binary log file
* current binary log position
* current relay log file
* current relay log position
server_val[in] the server connection values or a connected server
get_defaults[in] if True, get the default settings for the server
options[in] options for connecting to the server
Return tuple - information about server
"""
if options is None:
options = {}
# Parse source connection values
source_values = parse_connection(server_val, None, options)
# Connect to the server
conn_options = {
'version': "5.1.30",
}
servers = connect_servers(source_values, None, conn_options)
server = servers[0]
params_dict = defaultdict(str)
# Initialize list of warnings
params_dict['warnings'] = []
# Identify server by string: 'host:port[:socket]'.
server_id = "{0}:{1}".format(source_values['host'], source_values['port'])
if source_values.get('socket', None):
server_id = "{0}:{1}".format(server_id, source_values.get('socket'))
params_dict['server'] = server_id
# Get _SERVER_VARIABLES values from the server
for server_var in _SERVER_VARIABLES:
res = server.show_server_variable(server_var)
if res:
params_dict[server_var] = res[0][1]
else:
raise UtilError("Unable to determine {0} of server '{1}'"
".".format(server_var, server_id))
# Get _LOG_FILES_VARIABLES values from the server
for msg, log_tpl in _LOG_FILES_VARIABLES.iteritems():
res = server.show_server_variable(log_tpl.log_name)
if res:
# Check if log is turned off
params_dict[log_tpl.log_name] = res[0][1]
# If logs are turned off, skip checking information about the file
if res[0][1] in ('', 'OFF'):
continue
# Logging is enabled, so we can get get information about log_file
# unless it is log_error because in that case we already have it.
if log_tpl.log_file is not None: # if it is not log_error
log_file = server.show_server_variable(log_tpl.log_file)[0][1]
params_dict[log_tpl.log_file] = log_file
else: # log error, so log_file_name is already on params_dict
log_file = params_dict[log_tpl.log_name]
# Now get the information about the size of the logs
# If log file is stderr, we cannot get the correct size.
if log_file not in ["stderr", "stdout"]:
# Now get the information about the size of the logs
try:
params_dict[log_tpl.log_file_size] = "{0} bytes".format(
os.path.getsize(log_file))
except os.error:
# if we are unable to get the log_file_size
params_dict[log_tpl.log_file_size] = ''
warning_msg = _WARNING_TEMPLATE.format(msg, log_file)
params_dict['warnings'].append(warning_msg)
else:
params_dict['warnings'].append(
"Unable to get information "
"regarding variable '{0}'").format(msg)
# if audit_log plugin is installed and enabled
if server.supports_plugin('audit'):
res = server.show_server_variable('audit_log_file')
if res:
# Audit_log variable might be a relative path to the datadir,
# so it needs to be treated accordingly
if not os.path.isabs(res[0][1]):
params_dict['audit_log_file'] = os.path.join(
params_dict['datadir'], res[0][1])
else:
params_dict['audit_log_file'] = res[0][1]
# Add audit_log field to the _COLUMNS List unless it is already
# there
if 'audit_log_file' not in _COLUMNS_SET:
_COLUMNS.append('audit_log_file')
_COLUMNS.append('audit_log_file_size')
_COLUMNS_SET.add('audit_log_file')
try:
params_dict['audit_log_file_size'] = "{0} bytes".format(
os.path.getsize(params_dict['audit_log_file']))
except os.error:
# If we are unable to get the size of the audit_log_file
params_dict['audit_log_file_size'] = ''
warning_msg = _WARNING_TEMPLATE.format(
"audit log", params_dict['audit_log_file'])
params_dict['warnings'].append(warning_msg)
# Build search path for config files
if os.name == "posix":
my_def_search = [
"/etc/my.cnf", "/etc/mysql/my.cnf",
os.path.join(params_dict['basedir'], "my.cnf"), "~/.my.cnf"
]
else:
my_def_search = [
r"c:\windows\my.ini", r"c:\my.ini", r"c:\my.cnf",
os.path.join(os.curdir, "my.ini")
]
my_def_search.append(os.path.join(os.curdir, "my.cnf"))
# Get server's default configuration values.
defaults = []
if get_defaults:
# Can only get defaults for local servers (need to access local data).
if server.is_alias('localhost'):
try:
my_def_path = get_tool_path(params_dict['basedir'],
"my_print_defaults",
quote=True)
except UtilError as err:
raise UtilError("Unable to retrieve the defaults data "
"(requires access to my_print_defaults): {0} "
"(basedir: {1})".format(
err.errmsg, params_dict['basedir']))
out_file = tempfile.TemporaryFile()
# Execute tool: <basedir>/my_print_defaults mysqld
cmd_list = shlex.split(my_def_path)
cmd_list.append("mysqld")
subprocess.call(cmd_list, stdout=out_file)
out_file.seek(0)
# Get defaults data from temp output file.
defaults.append("\nDefaults for server {0}".format(server_id))
for line in out_file.readlines():
defaults.append(line.rstrip())
else:
# Remote server; Cannot get the defaults data.
defaults.append("\nWARNING: The utility can not get defaults from "
"a remote host.")
# Find config file
config_file = ""
for search_path in my_def_search:
if os.path.exists(search_path):
if len(config_file) > 0:
config_file = "{0}, {1}".format(config_file, search_path)
else:
config_file = search_path
params_dict['config_file'] = config_file
# Find binary log, relay log
params_dict['binary_log'], params_dict['binary_log_pos'] = _get_binlog(
server)
params_dict['relay_log'], params_dict['relay_log_pos'] = _get_relay_log(
server)
server.disconnect()
return params_dict, defaults
def migrate_up(self, db_name, last_version):
(ver, md5, comment) = self.read_meta(db_name, int(last_version))
if self.verify_checksum(db_name, last_version, md5) is False:
logging.warning("The current schema doesn't match the last applied migration")
version = self.new_migration_version(db_name)
if not os.path.exists("%s/%04d-up.mig" % (db_name, int(version))):
logging.info("No migration available")
else:
logging.info(u"Preparing migration to version %04d" % int(version))
if os.path.exists("%s/%04d-down.mig" % (db_name, int(version))):
os.remove("%s/%04d-down.mig" % (db_name, int(version)))
(ver, md5, comment) = self.read_meta(db_name, int(version))
query_options = {'skip_data': True, 'force': True}
db_list = []
grp = re.match("(\w+)(?:\:(\w+))?", "%s:%s_%s" % (db_name, self.tmp_prefix, db_name))
db_entry = grp.groups()
db_list.append(db_entry)
source_values = parse_connection(server_connection)
destination_values = parse_connection(server_connection)
with capture() as stepback:
dbcopy.copy_db(source_values, destination_values, db_list, query_options)
self.online_schema_change(db_name, version, "%s/%04d-up.mig" % (db_name, int(version)))
if self.verify_checksum(db_name, version, md5) is True:
logging.info("Applied changes match the requested schema")
else:
logging.error("Something didn't run as expected, db schema doesn't match !")
self.change_migration_status(db_name, version, 'invalid checksum')
query_options = {
'run_all_tests': True, 'reverse': True, 'verbosity': None,
'no_object_check': False, 'no_data': True, 'quiet': True,
'difftype': 'sql', 'width': 75, 'changes-for': 'server1',
'skip_grants': True}
with capture() as stepback:
res = dbcompare.database_compare(source_values, destination_values, db_name, "%s_%s" % (self.tmp_prefix, db_name), query_options)
str = stepback.getvalue().splitlines(True)
to_add = 0
file_down = open("%s/%04d-down.tmp" % (db_name, int(version)), 'a')
for line in str:
if line[0] not in ['#', '\n', '+', '-', '@']:
# this if is required currently due to missing foreign keys in dbcopy
if not re.match("\s+DROP FOREIGN KEY", line):
#line = re.sub(" %s\." % db_name, " ", line, 1, re.IGNORECASE)
line = re.sub(" %s\." % db_name + '(?i)', " ", line, 1)
file_down.write("%s\n" % line.strip())
elif re.match("# WARNING: Objects in", line):
if re.match("# WARNING: Objects in \w+\.tmp_online_mig_", line):
to_add = 2
else:
to_add = 1
else:
grp = re.match("#\s+TABLE\: (\w+)", line)
if grp:
if to_add == 2:
query = "SHOW CREATE TABLE tmp_online_mig_%s.%s;" % (db_name, grp.group(1))
res = self.server.exec_query(query)
file_down.write("%s\n" % res[0][1])
elif to_add == 1:
file_down.write("DROP TABLE %s;\n" % grp.group(1))
file_down.close()
file_down_tmp = "%s/%04d-down.tmp" % (db_name, int(version))
self.create_migration_file(db_name, file_down_tmp, version, "down")
query = "DROP DATABASE %s_%s" % (self.tmp_prefix, db_name)
res = self.server.exec_query(query)
#os.remove(file_down_tmp)
file_schema = "%s/%04d-schema.img" % (db_name, int(version))
self.create_schema_img(db_name, file_schema)
print(
WARN_OPT_USING_DEFAULT.format(default=default_val, opt='--master'))
# Print the WARNING to force determinism if a parser error occurs.
sys.stdout.flush()
# option --slave is required (mandatory)
if not opt.slave:
parser.error(PARSE_ERR_OPTS_REQ.format(opt='--slave'))
# option --rpl-user is required (mandatory)
if not opt.rpl_user:
parser.error(PARSE_ERR_OPTS_REQ.format(opt="--rpl-user"))
# Parse source connection values
try:
m_values = parse_connection(opt.master, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Master connection values invalid: {0}.".format(err))
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Master connection values invalid: {0}."
"".format(err.errmsg))
# Parse source connection values
try:
s_values = parse_connection(opt.slave, None, opt)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Slave connection values invalid: %s." % err)
except UtilError:
}
# Add ssl options to options instead of connection.
options.update(get_ssl_dict(opt))
# Check server options.
if not opt.server1:
parser.error(PARSE_ERR_OPTS_REQ.format(opt='--server1'))
if opt.all and not opt.server2:
parser.error(PARSE_ERR_OPTS_REQ.format(opt='--server2'))
# Parse server connection values
server1_values = None
server2_values = None
try:
server1_values = parse_connection(opt.server1, None, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server1 connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
parser.error("Server1 connection values invalid: %s." % err.errmsg)
if opt.server2:
try:
server2_values = parse_connection(opt.server2, None, options)
except FormatError:
_, err, _ = sys.exc_info()
parser.error("Server2 connection values invalid: %s." % err)
except UtilError:
_, err, _ = sys.exc_info()
server_list = ServerList([], opt.start_port, opt.utildir, opt.verbosity)
basedir = None
# Print status of connections
print("\nServers:")
if not opt.servers:
print(" No servers specified.")
else:
i = 0
for server in opt.servers:
# add ssl options values.
conn_options = {}
conn_options.update(get_ssl_dict(opt))
try:
conn_val = parse_connection(server, options=conn_options)
except UtilError as err:
parser.error(err.errmsg)
except:
parser.error("Problem parsing server connection "
"'{0}'".format(server))
i += 1
# Fail if port is None
if conn_val["port"] is None:
parser.error("You must specify a port in the server "
"string: \n {0}".format(server))
sys.stdout.write(" Connecting to {0} as user {1} on port "
"{2}: ".format(conn_val["host"], conn_val["user"],
conn_val["port"]))