def test_decode_encode(self):
"""Test valid encode/decode conversions of strings.
"""
for enc_str, dec_str in self.test_cases:
frm = "{0}: was {1}, expected {2}"
# First, do encode
result = encode(enc_str)
msg = frm.format(enc_str, result, dec_str)
self.assertEqual(dec_str, result, msg)
# Now, do decode
result = decode(dec_str)
msg = frm.format(enc_str, result, dec_str)
self.assertEqual(enc_str, result, msg)
def _build_db_list(server,
rows,
include_list,
datadir,
fmt=False,
have_read=False,
verbosity=0,
include_empty=True):
"""Build a list of all databases and their totals.
This method reads a list of databases and their calculated sizes
and builds a new list of the databases searching the datadir provided
and adds the size of the miscellaneous files.
The size of the database is calculated based on the ability of the user
to read the datadir. If user has read access to the datadir, the total
returned will be the calculation of the data_length+index_length from
INFORMATION_SCHEMA.TABLES plus the sum of all miscellaneous files (e.g.
trigger files, .frm files, etc.). If the user does not have read access
to the datadir, only the calculated size is returned.
If format is True, the columns and rows returned will be formatted to a
constant width using locale-specific options for printing numerals. For
example, US locale formats 12345 as 12,345.
The verbosity option controls how much data is shown:
0 : no additional information
> 0 : include data size (calculated) and size of misc files
>= 2 : also include database directory actual size
server[in] Connected server
rows[in] A list of databases and their calculated sizes
include_list[in] A list of databases included on the command line
datadir[in] The data directory
fmt[in] If True, format columns and rows to standard sizes
have_read[in] If True, user has read access to datadir path
verbosity[in] Controls how much data is shown
include_empty[in] Include empty databases in list
return (tuple) (column headers, rows, total size)
"""
total = 0
results = []
max_col = 0
# build the list
for row in rows:
# If user can read the datadir, calculate actual and misc file totals
if have_read:
# Encode database name (with strange characters) to the
# corresponding directory name.
db_dir = encode(row[0])
dbdir_size = _get_folder_size(os.path.join(datadir, db_dir))
misc_files = _get_db_dir_size(os.path.join(datadir, db_dir))
else:
dbdir_size = 0
misc_files = 0
if row[1] is None:
data_size = 0
db_total = 0
else:
data_size = int(row[1])
db_total = int(row[1]) + misc_files
# Count total for all databases
total += data_size + misc_files
if have_read:
if verbosity >= 2: # get all columns
results.append(
(row[0], dbdir_size, data_size, misc_files, db_total))
elif verbosity > 0:
results.append((row[0], data_size, misc_files, db_total))
else:
results.append((row[0], db_total))
else:
results.append((row[0], db_total))
if have_read and verbosity > 0:
num_cols = min(verbosity + 2, 4)
else:
num_cols = 1
# Build column list and format if necessary
col_list = ['db_name']
if num_cols == 4: # get all columns
col_list.append('db_dir_size')
col_list.append('data_size')
col_list.append('misc_files')
col_list.append('total')
elif num_cols == 3:
col_list.append('data_size')
col_list.append('misc_files')
col_list.append('total')
else:
col_list.append('total')
fmt_cols = []
max_col = [0, 0, 0, 0]
if fmt:
fmt_cols.append(col_list[0])
for i in range(0, num_cols):
max_col[i] = _get_formatted_max_width(results, col_list, i + 1)
fmt_cols.append("{0:>{1}}".format(col_list[i + 1], max_col[i]))
else:
fmt_cols = col_list
# format the list if needed
fmt_rows = []
if fmt:
for row in results:
fmt_data = ['', '', '', '', '']
# Put in commas and justify strings
for i in range(0, num_cols):
fmt_data[i] = locale.format("%d", row[i + 1], grouping=True)
if num_cols == 4: # get all columns
fmt_rows.append((row[0], fmt_data[0], fmt_data[1], fmt_data[2],
fmt_data[3]))
elif num_cols == 3:
fmt_rows.append(
(row[0], fmt_data[0], fmt_data[1], fmt_data[2]))
else:
fmt_rows.append((row[0], fmt_data[0]))
else:
fmt_rows = results
if include_empty:
dbs = server.exec_query("SHOW DATABASES")
if len(fmt_rows) != len(dbs) - 1:
# We have orphaned database - databases not listed in IS.TABLES
exclude_list = []
for row in fmt_rows:
exclude_list.append(row[0])
for db in dbs:
if db[0].upper() != "INFORMATION_SCHEMA" and \
db[0] not in exclude_list and \
(include_list is None or include_list == [] or
db[0] in include_list):
if fmt:
fmt_data = ['', '', '', '', '']
for i in range(0, num_cols):
if type(row[i + 1]) == type(int):
fmt_data[i] = locale.format("%s",
int(row[i + 1]),
grouping=True)
else:
fmt_data[i] = locale.format("%s",
row[i + 1],
grouping=True)
if num_cols == 4: # get all columns
fmt_rows.insert(0,
(db[0], fmt_data[0], fmt_data[1],
fmt_data[2], fmt_data[3]))
elif num_cols == 3:
fmt_rows.insert(
0,
(db[0], fmt_data[0], fmt_data[1], fmt_data[2]))
else:
fmt_rows.insert(0, (db[0], fmt_data[0]))
else:
if num_cols == 4:
fmt_rows.insert(0, (db[0], 0, 0, 0, 0))
elif num_cols == 3:
fmt_rows.insert(0, (db[0], 0, 0, 0))
else:
fmt_rows.insert(0, (db[0], 0))
return (fmt_cols, fmt_rows, total)
def _build_db_list(
server, rows, include_list, datadir, fmt=False, have_read=False, verbosity=0, include_empty=True, is_remote=False
):
"""Build a list of all databases and their totals.
This method reads a list of databases and their calculated sizes
and builds a new list of the databases searching the datadir provided
and adds the size of the miscellaneous files.
The size of the database is calculated based on the ability of the user
to read the datadir. If user has read access to the datadir, the total
returned will be the calculation of the data_length+index_length from
INFORMATION_SCHEMA.TABLES plus the sum of all miscellaneous files (e.g.
trigger files, .frm files, etc.). If the user does not have read access
to the datadir, only the calculated size is returned.
If format is True, the columns and rows returned will be formatted to a
constant width using locale-specific options for printing numerals. For
example, US locale formats 12345 as 12,345.
The verbosity option controls how much data is shown:
0 : no additional information
> 0 : include data size (calculated) and size of misc files
>= 2 : also include database directory actual size
server[in] Connected server
rows[in] A list of databases and their calculated sizes
include_list[in] A list of databases included on the command line
datadir[in] The data directory
fmt[in] If True, format columns and rows to standard sizes
have_read[in] If True, user has read access to datadir path
verbosity[in] Controls how much data is shown
include_empty[in] Include empty databases in list
is_remote[in] True is a remote server
return (tuple) (column headers, rows, total size)
"""
total = 0
results = []
max_col = 0
# build the list
for row in rows:
# If user can read the datadir, calculate actual and misc file totals
if have_read and not is_remote:
# Encode database name (with strange characters) to the
# corresponding directory name.
db_dir = encode(row[0])
dbdir_size = _get_folder_size(os.path.join(datadir, db_dir))
misc_files = _get_db_dir_size(os.path.join(datadir, db_dir))
else:
dbdir_size = 0
misc_files = 0
if row[1] is None:
data_size = 0
db_total = 0
else:
data_size = int(row[1])
db_total = int(row[1]) + misc_files
# Count total for all databases
total += data_size + misc_files
if have_read and not is_remote:
if verbosity >= 2: # get all columns
results.append((row[0], dbdir_size, data_size, misc_files, db_total))
elif verbosity > 0:
results.append((row[0], data_size, misc_files, db_total))
else:
results.append((row[0], db_total))
else:
results.append((row[0], db_total))
if have_read and not is_remote and verbosity > 0:
num_cols = min(verbosity + 2, 4)
else:
num_cols = 1
# Build column list and format if necessary
col_list = ["db_name"]
if num_cols == 4: # get all columns
col_list.append("db_dir_size")
col_list.append("data_size")
col_list.append("misc_files")
col_list.append("total")
elif num_cols == 3:
col_list.append("data_size")
col_list.append("misc_files")
col_list.append("total")
else:
col_list.append("total")
fmt_cols = []
max_col = [0, 0, 0, 0]
if fmt:
fmt_cols.append(col_list[0])
for i in range(0, num_cols):
max_col[i] = _get_formatted_max_width(results, col_list, i + 1)
fmt_cols.append("{0:>{1}}".format(col_list[i + 1], max_col[i]))
else:
fmt_cols = col_list
# format the list if needed
fmt_rows = []
if fmt:
for row in results:
fmt_data = ["", "", "", "", ""]
# Put in commas and justify strings
for i in range(0, num_cols):
fmt_data[i] = locale.format("%d", row[i + 1], grouping=True)
if num_cols == 4: # get all columns
fmt_rows.append((row[0], fmt_data[0], fmt_data[1], fmt_data[2], fmt_data[3]))
elif num_cols == 3:
fmt_rows.append((row[0], fmt_data[0], fmt_data[1], fmt_data[2]))
else:
fmt_rows.append((row[0], fmt_data[0]))
else:
fmt_rows = results
if include_empty:
dbs = server.exec_query("SHOW DATABASES")
if len(fmt_rows) != len(dbs) - 1:
# We have orphaned database - databases not listed in IS.TABLES
exclude_list = []
for row in fmt_rows:
exclude_list.append(row[0])
for db in dbs:
if (
db[0].upper() != "INFORMATION_SCHEMA"
and db[0] not in exclude_list
and (include_list is None or include_list == [] or db[0] in include_list)
):
if fmt:
fmt_data = ["", "", "", "", ""]
for i in range(0, num_cols):
if type(row[i + 1]) == type(int):
fmt_data[i] = locale.format("%s", int(row[i + 1]), grouping=True)
else:
fmt_data[i] = locale.format("%s", row[i + 1], grouping=True)
if num_cols == 4: # get all columns
fmt_rows.insert(0, (db[0], fmt_data[0], fmt_data[1], fmt_data[2], fmt_data[3]))
elif num_cols == 3:
fmt_rows.insert(0, (db[0], fmt_data[0], fmt_data[1], fmt_data[2]))
else:
fmt_rows.insert(0, (db[0], fmt_data[0]))
else:
if num_cols == 4:
fmt_rows.insert(0, (db[0], 0, 0, 0, 0))
elif num_cols == 3:
fmt_rows.insert(0, (db[0], 0, 0, 0))
else:
fmt_rows.insert(0, (db[0], 0))
return (fmt_cols, fmt_rows, total)
def _get_create_statement(server,
temp_datadir,
frm_file,
version,
options,
quiet=False):
"""Get the CREATE statement for the .frm file
This method attempts to read the CREATE statement by copying the .frm file,
altering the storage engine in the .frm file to MEMORY and issuing a SHOW
CREATE statement for the table/view.
If this method returns None, the operation was successful and the CREATE
statement was printed. If a string is returned, there was at least one
error (which will be printed) and the .frm file was not readable.
The returned frm file path can be used to tell the user to use the
diagnostic mode for reading files byte-by-byte. See the method
read_frm_files_diagnostic() above.
server[in] Server instance
temp_datadir[in] New data directory
frm_file[in] Tuple containing (db, table, path) for .frm file
version[in] Version string for the current server
options[in] Options from user
Returns string - None on success, path to frm file on error
"""
verbosity = int(options.get("verbosity", 0))
quiet = options.get("quiet", False)
new_engine = options.get("new_engine", None)
frm_dir = options.get("frm_dir", ".{0}".format(os.sep))
user = options.get('user', 'root')
if not quiet:
print "#\n# Reading the %s.frm file." % frm_file[1]
try:
# 1) copy the file
db = frm_file[0]
if not db or db == ".":
db = "test"
db_name = db + "_temp"
new_path = os.path.normpath(os.path.join(temp_datadir, db_name))
if not os.path.exists(new_path):
os.mkdir(new_path)
new_frm = os.path.join(new_path, frm_file[1] + ".frm")
# Check name for decoding and decode
try:
if requires_decoding(frm_file[1]):
new_frm_file = decode(frm_file[1])
frm_file = (frm_file[0], new_frm_file, frm_file[2])
shutil.copy(frm_file[2], new_path)
# Check name for encoding and encode
elif requires_encoding(frm_file[1]):
new_frm_file = encode(frm_file[1]) + ".frm"
new_frm = os.path.join(new_path, new_frm_file)
shutil.copy(frm_file[2], new_frm)
else:
shutil.copy(frm_file[2], new_path)
except:
_, e, _ = sys.exc_info()
print("ERROR: {0}".format(e))
# Set permissons on copied file if user context in play
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_path])
subprocess.call(['chgrp', '-R', user, new_path])
server.exec_query("CREATE DATABASE IF NOT EXISTS %s" % db_name)
frm = FrmReader(db_name, frm_file[1], new_frm, options)
frm_type = frm.get_type()
server.exec_query("FLUSH TABLES")
if frm_type == "TABLE":
# 2) change engine if it is a table
current_engine = frm.change_storage_engine()
# Abort read if restricted engine found
if current_engine[1].upper() in _CANNOT_READ_ENGINE:
print(
"ERROR: Cannot process tables with the %s storage "
"engine. Please use the diagnostic mode to read the "
"%s file." % (current_engine[1].upper(), frm_file[1]))
return frm_file[2]
# Check server version
server_version = None
if version and len(current_engine) > 1 and current_engine[2]:
server_version = (int(current_engine[2][0]),
int(current_engine[2][1:3]),
int(current_engine[2][3:]))
if verbosity > 1 and not quiet:
print("# Server version in file: %s.%s.%s" %
server_version)
if not server.check_version_compat(server_version[0],
server_version[1],
server_version[2]):
versions = (server_version[0], server_version[1],
server_version[2], version[0], version[1],
version[2])
print(
"ERROR: The server version for this "
"file is too low. It requires a server version "
"%s.%s.%s or higher but your server is version "
"%s.%s.%s. Try using a newer server or use "
"diagnostic mode." % versions)
return frm_file[2]
# 3) show CREATE TABLE
res = server.exec_query("SHOW CREATE TABLE `%s`.`%s`" %
(db_name, frm_file[1]))
create_str = res[0][1]
if new_engine:
create_str = create_str.replace("ENGINE=MEMORY",
"ENGINE=%s" % new_engine)
elif current_engine[1].upper() != "MEMORY":
create_str = create_str.replace(
"ENGINE=MEMORY", "ENGINE=%s" % current_engine[1])
if frm_file[0] and frm_file[0] != ".":
create_str = create_str.replace(
"CREATE TABLE ", "CREATE TABLE `%s`." % frm_file[0])
# if requested, generate the new .frm with the altered engine
if new_engine:
server.exec_query("ALTER TABLE `{0}`.`{1}` "
"ENGINE={2}".format(db_name, frm_file[1],
new_engine))
new_frm_file = os.path.join(frm_dir,
"{0}.frm".format(frm_file[1]))
if os.path.exists(new_frm_file):
print(
"#\n# WARNING: Unable to create new .frm file. "
"File exists.")
else:
try:
shutil.copyfile(new_frm, new_frm_file)
print(
"# Copy of .frm file with new storage "
"engine saved as {0}.".format(new_frm_file))
except (IOError, OSError, shutil.Error) as e:
print(
"# WARNING: Unable to create new .frm file. "
"Error: {0}".format(e))
elif frm_type == "VIEW":
# 5) show CREATE VIEW
res = server.exec_query("SHOW CREATE VIEW %s.%s" %
(db_name, frm_file[1]))
create_str = res[0][1]
if frm_file[0]:
create_str = create_str.replace(
"CREATE VIEW ", "CREATE VIEW `%s`." % frm_file[0])
# Now we must replace the string for storage engine!
print "#\n# CREATE statement for %s:\n#\n" % frm_file[2]
print create_str
print
if frm_type == "TABLE" and options.get("show_stats", False):
frm.show_statistics()
except:
print(
"ERROR: Failed to correctly read the .frm file. Please try "
"reading the file with the --diagnostic mode.")
return frm_file[2]
return None
def _get_create_statement(server, temp_datadir,
frm_file, version,
options, quiet=False):
"""Get the CREATE statement for the .frm file
This method attempts to read the CREATE statement by copying the .frm file,
altering the storage engine in the .frm file to MEMORY and issuing a SHOW
CREATE statement for the table/view.
If this method returns None, the operation was successful and the CREATE
statement was printed. If a string is returned, there was at least one
error (which will be printed) and the .frm file was not readable.
The returned frm file path can be used to tell the user to use the
diagnostic mode for reading files byte-by-byte. See the method
read_frm_files_diagnostic() above.
server[in] Server instance
temp_datadir[in] New data directory
frm_file[in] Tuple containing (db, table, path) for .frm file
version[in] Version string for the current server
options[in] Options from user
Returns string - None on success, path to frm file on error
"""
verbosity = int(options.get("verbosity", 0))
quiet = options.get("quiet", False)
new_engine = options.get("new_engine", None)
frm_dir = options.get("frm_dir", ".{0}".format(os.sep))
user = options.get('user', 'root')
if not quiet:
print "#\n# Reading the %s.frm file." % frm_file[1]
try:
# 1) copy the file
db = frm_file[0]
if not db or db == ".":
db = "test"
db_name = db + "_temp"
new_path = os.path.normpath(os.path.join(temp_datadir, db_name))
if not os.path.exists(new_path):
os.mkdir(new_path)
new_frm = os.path.join(new_path, frm_file[1] + ".frm")
# Check name for decoding and decode
try:
if requires_decoding(frm_file[1]):
new_frm_file = decode(frm_file[1])
frm_file = (frm_file[0], new_frm_file, frm_file[2])
shutil.copy(frm_file[2], new_path)
# Check name for encoding and encode
elif requires_encoding(frm_file[1]):
new_frm_file = encode(frm_file[1]) + ".frm"
new_frm = os.path.join(new_path, new_frm_file)
shutil.copy(frm_file[2], new_frm)
else:
shutil.copy(frm_file[2], new_path)
except:
_, e, _ = sys.exc_info()
print("ERROR: {0}".format(e))
# Set permissons on copied file if user context in play
if user_change_as_root(options):
subprocess.call(['chown', '-R', user, new_path])
subprocess.call(['chgrp', '-R', user, new_path])
server.exec_query("CREATE DATABASE IF NOT EXISTS %s" % db_name)
frm = FrmReader(db_name, frm_file[1], new_frm, options)
frm_type = frm.get_type()
server.exec_query("FLUSH TABLES")
if frm_type == "TABLE":
# 2) change engine if it is a table
current_engine = frm.change_storage_engine()
# Abort read if restricted engine found
if current_engine[1].upper() in _CANNOT_READ_ENGINE:
print ("ERROR: Cannot process tables with the %s storage "
"engine. Please use the diagnostic mode to read the "
"%s file." % (current_engine[1].upper(), frm_file[1]))
return frm_file[2]
# Check server version
server_version = None
if version and len(current_engine) > 1 and current_engine[2]:
server_version = (int(current_engine[2][0]),
int(current_engine[2][1:3]),
int(current_engine[2][3:]))
if verbosity > 1 and not quiet:
print ("# Server version in file: %s.%s.%s" %
server_version)
if not server.check_version_compat(server_version[0],
server_version[1],
server_version[2]):
versions = (server_version[0], server_version[1],
server_version[2], version[0], version[1],
version[2])
print ("ERROR: The server version for this "
"file is too low. It requires a server version "
"%s.%s.%s or higher but your server is version "
"%s.%s.%s. Try using a newer server or use "
"diagnostic mode." % versions)
return frm_file[2]
# 3) show CREATE TABLE
res = server.exec_query("SHOW CREATE TABLE `%s`.`%s`" %
(db_name, frm_file[1]))
create_str = res[0][1]
if new_engine:
create_str = create_str.replace("ENGINE=MEMORY",
"ENGINE=%s" % new_engine)
elif not current_engine[1].upper() == "MEMORY":
create_str = create_str.replace("ENGINE=MEMORY",
"ENGINE=%s" %
current_engine[1])
if frm_file[0] and not frm_file[0] == ".":
create_str = create_str.replace("CREATE TABLE ",
"CREATE TABLE `%s`." %
frm_file[0])
# if requested, generate the new .frm with the altered engine
if new_engine:
server.exec_query("ALTER TABLE `{0}`.`{1}` "
"ENGINE={2}".format(db_name,
frm_file[1],
new_engine))
new_frm_file = os.path.join(frm_dir,
"{0}.frm".format(frm_file[1]))
if os.path.exists(new_frm_file):
print("#\n# WARNING: Unable to create new .frm file. "
"File exists.")
else:
try:
shutil.copyfile(new_frm, new_frm_file)
print("# Copy of .frm file with new storage "
"engine saved as {0}.".format(new_frm_file))
except (IOError, OSError, shutil.Error) as e:
print("# WARNING: Unable to create new .frm file. "
"Error: {0}".format(e))
elif frm_type == "VIEW":
# 5) show CREATE VIEW
res = server.exec_query("SHOW CREATE VIEW %s.%s" %
(db_name, frm_file[1]))
create_str = res[0][1]
if frm_file[0]:
create_str = create_str.replace("CREATE VIEW ",
"CREATE VIEW `%s`." %
frm_file[0])
# Now we must replace the string for storage engine!
print "#\n# CREATE statement for %s:\n#\n" % frm_file[2]
print create_str
print
if frm_type == "TABLE" and options.get("show_stats", False):
frm.show_statistics()
except:
print ("ERROR: Failed to correctly read the .frm file. Please try "
"reading the file with the --diagnostic mode.")
return frm_file[2]
return None
