def test_decode_encode(self): """Test valid encode/decode conversions of strings. """ for enc_str, dec_str in self.test_cases: frm = "{0}: was {1}, expected {2}" # First, do encode result = encode(enc_str) msg = frm.format(enc_str, result, dec_str) self.assertEqual(dec_str, result, msg) # Now, do decode result = decode(dec_str) msg = frm.format(enc_str, result, dec_str) self.assertEqual(enc_str, result, msg)
def _build_db_list(server, rows, include_list, datadir, fmt=False, have_read=False, verbosity=0, include_empty=True): """Build a list of all databases and their totals. This method reads a list of databases and their calculated sizes and builds a new list of the databases searching the datadir provided and adds the size of the miscellaneous files. The size of the database is calculated based on the ability of the user to read the datadir. If user has read access to the datadir, the total returned will be the calculation of the data_length+index_length from INFORMATION_SCHEMA.TABLES plus the sum of all miscellaneous files (e.g. trigger files, .frm files, etc.). If the user does not have read access to the datadir, only the calculated size is returned. If format is True, the columns and rows returned will be formatted to a constant width using locale-specific options for printing numerals. For example, US locale formats 12345 as 12,345. The verbosity option controls how much data is shown: 0 : no additional information > 0 : include data size (calculated) and size of misc files >= 2 : also include database directory actual size server[in] Connected server rows[in] A list of databases and their calculated sizes include_list[in] A list of databases included on the command line datadir[in] The data directory fmt[in] If True, format columns and rows to standard sizes have_read[in] If True, user has read access to datadir path verbosity[in] Controls how much data is shown include_empty[in] Include empty databases in list return (tuple) (column headers, rows, total size) """ total = 0 results = [] max_col = 0 # build the list for row in rows: # If user can read the datadir, calculate actual and misc file totals if have_read: # Encode database name (with strange characters) to the # corresponding directory name. db_dir = encode(row[0]) dbdir_size = _get_folder_size(os.path.join(datadir, db_dir)) misc_files = _get_db_dir_size(os.path.join(datadir, db_dir)) else: dbdir_size = 0 misc_files = 0 if row[1] is None: data_size = 0 db_total = 0 else: data_size = int(row[1]) db_total = int(row[1]) + misc_files # Count total for all databases total += data_size + misc_files if have_read: if verbosity >= 2: # get all columns results.append( (row[0], dbdir_size, data_size, misc_files, db_total)) elif verbosity > 0: results.append((row[0], data_size, misc_files, db_total)) else: results.append((row[0], db_total)) else: results.append((row[0], db_total)) if have_read and verbosity > 0: num_cols = min(verbosity + 2, 4) else: num_cols = 1 # Build column list and format if necessary col_list = ['db_name'] if num_cols == 4: # get all columns col_list.append('db_dir_size') col_list.append('data_size') col_list.append('misc_files') col_list.append('total') elif num_cols == 3: col_list.append('data_size') col_list.append('misc_files') col_list.append('total') else: col_list.append('total') fmt_cols = [] max_col = [0, 0, 0, 0] if fmt: fmt_cols.append(col_list[0]) for i in range(0, num_cols): max_col[i] = _get_formatted_max_width(results, col_list, i + 1) fmt_cols.append("{0:>{1}}".format(col_list[i + 1], max_col[i])) else: fmt_cols = col_list # format the list if needed fmt_rows = [] if fmt: for row in results: fmt_data = ['', '', '', '', ''] # Put in commas and justify strings for i in range(0, num_cols): fmt_data[i] = locale.format("%d", row[i + 1], grouping=True) if num_cols == 4: # get all columns fmt_rows.append((row[0], fmt_data[0], fmt_data[1], fmt_data[2], fmt_data[3])) elif num_cols == 3: fmt_rows.append( (row[0], fmt_data[0], fmt_data[1], fmt_data[2])) else: fmt_rows.append((row[0], fmt_data[0])) else: fmt_rows = results if include_empty: dbs = server.exec_query("SHOW DATABASES") if len(fmt_rows) != len(dbs) - 1: # We have orphaned database - databases not listed in IS.TABLES exclude_list = [] for row in fmt_rows: exclude_list.append(row[0]) for db in dbs: if db[0].upper() != "INFORMATION_SCHEMA" and \ db[0] not in exclude_list and \ (include_list is None or include_list == [] or db[0] in include_list): if fmt: fmt_data = ['', '', '', '', ''] for i in range(0, num_cols): if type(row[i + 1]) == type(int): fmt_data[i] = locale.format("%s", int(row[i + 1]), grouping=True) else: fmt_data[i] = locale.format("%s", row[i + 1], grouping=True) if num_cols == 4: # get all columns fmt_rows.insert(0, (db[0], fmt_data[0], fmt_data[1], fmt_data[2], fmt_data[3])) elif num_cols == 3: fmt_rows.insert( 0, (db[0], fmt_data[0], fmt_data[1], fmt_data[2])) else: fmt_rows.insert(0, (db[0], fmt_data[0])) else: if num_cols == 4: fmt_rows.insert(0, (db[0], 0, 0, 0, 0)) elif num_cols == 3: fmt_rows.insert(0, (db[0], 0, 0, 0)) else: fmt_rows.insert(0, (db[0], 0)) return (fmt_cols, fmt_rows, total)
def _build_db_list( server, rows, include_list, datadir, fmt=False, have_read=False, verbosity=0, include_empty=True, is_remote=False ): """Build a list of all databases and their totals. This method reads a list of databases and their calculated sizes and builds a new list of the databases searching the datadir provided and adds the size of the miscellaneous files. The size of the database is calculated based on the ability of the user to read the datadir. If user has read access to the datadir, the total returned will be the calculation of the data_length+index_length from INFORMATION_SCHEMA.TABLES plus the sum of all miscellaneous files (e.g. trigger files, .frm files, etc.). If the user does not have read access to the datadir, only the calculated size is returned. If format is True, the columns and rows returned will be formatted to a constant width using locale-specific options for printing numerals. For example, US locale formats 12345 as 12,345. The verbosity option controls how much data is shown: 0 : no additional information > 0 : include data size (calculated) and size of misc files >= 2 : also include database directory actual size server[in] Connected server rows[in] A list of databases and their calculated sizes include_list[in] A list of databases included on the command line datadir[in] The data directory fmt[in] If True, format columns and rows to standard sizes have_read[in] If True, user has read access to datadir path verbosity[in] Controls how much data is shown include_empty[in] Include empty databases in list is_remote[in] True is a remote server return (tuple) (column headers, rows, total size) """ total = 0 results = [] max_col = 0 # build the list for row in rows: # If user can read the datadir, calculate actual and misc file totals if have_read and not is_remote: # Encode database name (with strange characters) to the # corresponding directory name. db_dir = encode(row[0]) dbdir_size = _get_folder_size(os.path.join(datadir, db_dir)) misc_files = _get_db_dir_size(os.path.join(datadir, db_dir)) else: dbdir_size = 0 misc_files = 0 if row[1] is None: data_size = 0 db_total = 0 else: data_size = int(row[1]) db_total = int(row[1]) + misc_files # Count total for all databases total += data_size + misc_files if have_read and not is_remote: if verbosity >= 2: # get all columns results.append((row[0], dbdir_size, data_size, misc_files, db_total)) elif verbosity > 0: results.append((row[0], data_size, misc_files, db_total)) else: results.append((row[0], db_total)) else: results.append((row[0], db_total)) if have_read and not is_remote and verbosity > 0: num_cols = min(verbosity + 2, 4) else: num_cols = 1 # Build column list and format if necessary col_list = ["db_name"] if num_cols == 4: # get all columns col_list.append("db_dir_size") col_list.append("data_size") col_list.append("misc_files") col_list.append("total") elif num_cols == 3: col_list.append("data_size") col_list.append("misc_files") col_list.append("total") else: col_list.append("total") fmt_cols = [] max_col = [0, 0, 0, 0] if fmt: fmt_cols.append(col_list[0]) for i in range(0, num_cols): max_col[i] = _get_formatted_max_width(results, col_list, i + 1) fmt_cols.append("{0:>{1}}".format(col_list[i + 1], max_col[i])) else: fmt_cols = col_list # format the list if needed fmt_rows = [] if fmt: for row in results: fmt_data = ["", "", "", "", ""] # Put in commas and justify strings for i in range(0, num_cols): fmt_data[i] = locale.format("%d", row[i + 1], grouping=True) if num_cols == 4: # get all columns fmt_rows.append((row[0], fmt_data[0], fmt_data[1], fmt_data[2], fmt_data[3])) elif num_cols == 3: fmt_rows.append((row[0], fmt_data[0], fmt_data[1], fmt_data[2])) else: fmt_rows.append((row[0], fmt_data[0])) else: fmt_rows = results if include_empty: dbs = server.exec_query("SHOW DATABASES") if len(fmt_rows) != len(dbs) - 1: # We have orphaned database - databases not listed in IS.TABLES exclude_list = [] for row in fmt_rows: exclude_list.append(row[0]) for db in dbs: if ( db[0].upper() != "INFORMATION_SCHEMA" and db[0] not in exclude_list and (include_list is None or include_list == [] or db[0] in include_list) ): if fmt: fmt_data = ["", "", "", "", ""] for i in range(0, num_cols): if type(row[i + 1]) == type(int): fmt_data[i] = locale.format("%s", int(row[i + 1]), grouping=True) else: fmt_data[i] = locale.format("%s", row[i + 1], grouping=True) if num_cols == 4: # get all columns fmt_rows.insert(0, (db[0], fmt_data[0], fmt_data[1], fmt_data[2], fmt_data[3])) elif num_cols == 3: fmt_rows.insert(0, (db[0], fmt_data[0], fmt_data[1], fmt_data[2])) else: fmt_rows.insert(0, (db[0], fmt_data[0])) else: if num_cols == 4: fmt_rows.insert(0, (db[0], 0, 0, 0, 0)) elif num_cols == 3: fmt_rows.insert(0, (db[0], 0, 0, 0)) else: fmt_rows.insert(0, (db[0], 0)) return (fmt_cols, fmt_rows, total)
def _get_create_statement(server, temp_datadir, frm_file, version, options, quiet=False): """Get the CREATE statement for the .frm file This method attempts to read the CREATE statement by copying the .frm file, altering the storage engine in the .frm file to MEMORY and issuing a SHOW CREATE statement for the table/view. If this method returns None, the operation was successful and the CREATE statement was printed. If a string is returned, there was at least one error (which will be printed) and the .frm file was not readable. The returned frm file path can be used to tell the user to use the diagnostic mode for reading files byte-by-byte. See the method read_frm_files_diagnostic() above. server[in] Server instance temp_datadir[in] New data directory frm_file[in] Tuple containing (db, table, path) for .frm file version[in] Version string for the current server options[in] Options from user Returns string - None on success, path to frm file on error """ verbosity = int(options.get("verbosity", 0)) quiet = options.get("quiet", False) new_engine = options.get("new_engine", None) frm_dir = options.get("frm_dir", ".{0}".format(os.sep)) user = options.get('user', 'root') if not quiet: print "#\n# Reading the %s.frm file." % frm_file[1] try: # 1) copy the file db = frm_file[0] if not db or db == ".": db = "test" db_name = db + "_temp" new_path = os.path.normpath(os.path.join(temp_datadir, db_name)) if not os.path.exists(new_path): os.mkdir(new_path) new_frm = os.path.join(new_path, frm_file[1] + ".frm") # Check name for decoding and decode try: if requires_decoding(frm_file[1]): new_frm_file = decode(frm_file[1]) frm_file = (frm_file[0], new_frm_file, frm_file[2]) shutil.copy(frm_file[2], new_path) # Check name for encoding and encode elif requires_encoding(frm_file[1]): new_frm_file = encode(frm_file[1]) + ".frm" new_frm = os.path.join(new_path, new_frm_file) shutil.copy(frm_file[2], new_frm) else: shutil.copy(frm_file[2], new_path) except: _, e, _ = sys.exc_info() print("ERROR: {0}".format(e)) # Set permissons on copied file if user context in play if user_change_as_root(options): subprocess.call(['chown', '-R', user, new_path]) subprocess.call(['chgrp', '-R', user, new_path]) server.exec_query("CREATE DATABASE IF NOT EXISTS %s" % db_name) frm = FrmReader(db_name, frm_file[1], new_frm, options) frm_type = frm.get_type() server.exec_query("FLUSH TABLES") if frm_type == "TABLE": # 2) change engine if it is a table current_engine = frm.change_storage_engine() # Abort read if restricted engine found if current_engine[1].upper() in _CANNOT_READ_ENGINE: print( "ERROR: Cannot process tables with the %s storage " "engine. Please use the diagnostic mode to read the " "%s file." % (current_engine[1].upper(), frm_file[1])) return frm_file[2] # Check server version server_version = None if version and len(current_engine) > 1 and current_engine[2]: server_version = (int(current_engine[2][0]), int(current_engine[2][1:3]), int(current_engine[2][3:])) if verbosity > 1 and not quiet: print("# Server version in file: %s.%s.%s" % server_version) if not server.check_version_compat(server_version[0], server_version[1], server_version[2]): versions = (server_version[0], server_version[1], server_version[2], version[0], version[1], version[2]) print( "ERROR: The server version for this " "file is too low. It requires a server version " "%s.%s.%s or higher but your server is version " "%s.%s.%s. Try using a newer server or use " "diagnostic mode." % versions) return frm_file[2] # 3) show CREATE TABLE res = server.exec_query("SHOW CREATE TABLE `%s`.`%s`" % (db_name, frm_file[1])) create_str = res[0][1] if new_engine: create_str = create_str.replace("ENGINE=MEMORY", "ENGINE=%s" % new_engine) elif current_engine[1].upper() != "MEMORY": create_str = create_str.replace( "ENGINE=MEMORY", "ENGINE=%s" % current_engine[1]) if frm_file[0] and frm_file[0] != ".": create_str = create_str.replace( "CREATE TABLE ", "CREATE TABLE `%s`." % frm_file[0]) # if requested, generate the new .frm with the altered engine if new_engine: server.exec_query("ALTER TABLE `{0}`.`{1}` " "ENGINE={2}".format(db_name, frm_file[1], new_engine)) new_frm_file = os.path.join(frm_dir, "{0}.frm".format(frm_file[1])) if os.path.exists(new_frm_file): print( "#\n# WARNING: Unable to create new .frm file. " "File exists.") else: try: shutil.copyfile(new_frm, new_frm_file) print( "# Copy of .frm file with new storage " "engine saved as {0}.".format(new_frm_file)) except (IOError, OSError, shutil.Error) as e: print( "# WARNING: Unable to create new .frm file. " "Error: {0}".format(e)) elif frm_type == "VIEW": # 5) show CREATE VIEW res = server.exec_query("SHOW CREATE VIEW %s.%s" % (db_name, frm_file[1])) create_str = res[0][1] if frm_file[0]: create_str = create_str.replace( "CREATE VIEW ", "CREATE VIEW `%s`." % frm_file[0]) # Now we must replace the string for storage engine! print "#\n# CREATE statement for %s:\n#\n" % frm_file[2] print create_str print if frm_type == "TABLE" and options.get("show_stats", False): frm.show_statistics() except: print( "ERROR: Failed to correctly read the .frm file. Please try " "reading the file with the --diagnostic mode.") return frm_file[2] return None
def _get_create_statement(server, temp_datadir, frm_file, version, options, quiet=False): """Get the CREATE statement for the .frm file This method attempts to read the CREATE statement by copying the .frm file, altering the storage engine in the .frm file to MEMORY and issuing a SHOW CREATE statement for the table/view. If this method returns None, the operation was successful and the CREATE statement was printed. If a string is returned, there was at least one error (which will be printed) and the .frm file was not readable. The returned frm file path can be used to tell the user to use the diagnostic mode for reading files byte-by-byte. See the method read_frm_files_diagnostic() above. server[in] Server instance temp_datadir[in] New data directory frm_file[in] Tuple containing (db, table, path) for .frm file version[in] Version string for the current server options[in] Options from user Returns string - None on success, path to frm file on error """ verbosity = int(options.get("verbosity", 0)) quiet = options.get("quiet", False) new_engine = options.get("new_engine", None) frm_dir = options.get("frm_dir", ".{0}".format(os.sep)) user = options.get('user', 'root') if not quiet: print "#\n# Reading the %s.frm file." % frm_file[1] try: # 1) copy the file db = frm_file[0] if not db or db == ".": db = "test" db_name = db + "_temp" new_path = os.path.normpath(os.path.join(temp_datadir, db_name)) if not os.path.exists(new_path): os.mkdir(new_path) new_frm = os.path.join(new_path, frm_file[1] + ".frm") # Check name for decoding and decode try: if requires_decoding(frm_file[1]): new_frm_file = decode(frm_file[1]) frm_file = (frm_file[0], new_frm_file, frm_file[2]) shutil.copy(frm_file[2], new_path) # Check name for encoding and encode elif requires_encoding(frm_file[1]): new_frm_file = encode(frm_file[1]) + ".frm" new_frm = os.path.join(new_path, new_frm_file) shutil.copy(frm_file[2], new_frm) else: shutil.copy(frm_file[2], new_path) except: _, e, _ = sys.exc_info() print("ERROR: {0}".format(e)) # Set permissons on copied file if user context in play if user_change_as_root(options): subprocess.call(['chown', '-R', user, new_path]) subprocess.call(['chgrp', '-R', user, new_path]) server.exec_query("CREATE DATABASE IF NOT EXISTS %s" % db_name) frm = FrmReader(db_name, frm_file[1], new_frm, options) frm_type = frm.get_type() server.exec_query("FLUSH TABLES") if frm_type == "TABLE": # 2) change engine if it is a table current_engine = frm.change_storage_engine() # Abort read if restricted engine found if current_engine[1].upper() in _CANNOT_READ_ENGINE: print ("ERROR: Cannot process tables with the %s storage " "engine. Please use the diagnostic mode to read the " "%s file." % (current_engine[1].upper(), frm_file[1])) return frm_file[2] # Check server version server_version = None if version and len(current_engine) > 1 and current_engine[2]: server_version = (int(current_engine[2][0]), int(current_engine[2][1:3]), int(current_engine[2][3:])) if verbosity > 1 and not quiet: print ("# Server version in file: %s.%s.%s" % server_version) if not server.check_version_compat(server_version[0], server_version[1], server_version[2]): versions = (server_version[0], server_version[1], server_version[2], version[0], version[1], version[2]) print ("ERROR: The server version for this " "file is too low. It requires a server version " "%s.%s.%s or higher but your server is version " "%s.%s.%s. Try using a newer server or use " "diagnostic mode." % versions) return frm_file[2] # 3) show CREATE TABLE res = server.exec_query("SHOW CREATE TABLE `%s`.`%s`" % (db_name, frm_file[1])) create_str = res[0][1] if new_engine: create_str = create_str.replace("ENGINE=MEMORY", "ENGINE=%s" % new_engine) elif not current_engine[1].upper() == "MEMORY": create_str = create_str.replace("ENGINE=MEMORY", "ENGINE=%s" % current_engine[1]) if frm_file[0] and not frm_file[0] == ".": create_str = create_str.replace("CREATE TABLE ", "CREATE TABLE `%s`." % frm_file[0]) # if requested, generate the new .frm with the altered engine if new_engine: server.exec_query("ALTER TABLE `{0}`.`{1}` " "ENGINE={2}".format(db_name, frm_file[1], new_engine)) new_frm_file = os.path.join(frm_dir, "{0}.frm".format(frm_file[1])) if os.path.exists(new_frm_file): print("#\n# WARNING: Unable to create new .frm file. " "File exists.") else: try: shutil.copyfile(new_frm, new_frm_file) print("# Copy of .frm file with new storage " "engine saved as {0}.".format(new_frm_file)) except (IOError, OSError, shutil.Error) as e: print("# WARNING: Unable to create new .frm file. " "Error: {0}".format(e)) elif frm_type == "VIEW": # 5) show CREATE VIEW res = server.exec_query("SHOW CREATE VIEW %s.%s" % (db_name, frm_file[1])) create_str = res[0][1] if frm_file[0]: create_str = create_str.replace("CREATE VIEW ", "CREATE VIEW `%s`." % frm_file[0]) # Now we must replace the string for storage engine! print "#\n# CREATE statement for %s:\n#\n" % frm_file[2] print create_str print if frm_type == "TABLE" and options.get("show_stats", False): frm.show_statistics() except: print ("ERROR: Failed to correctly read the .frm file. Please try " "reading the file with the --diagnostic mode.") return frm_file[2] return None