def handle_config_path(config_path, group=None, use_default=True): """Retrieve the data associated to the given group. config_path[in] the path to the configuration file. group[in] The group name to retrieve the data from, if None the 'client' group will be use if found and if use_default is True. use_default[in] Use the default 'client' group name, True by default, used if no group is given. Returns a dictionary with the options data. """ # first verify if the configuration file exist on the given config_path # check config_path as near file as normalized path, then # at the default location file. if os.name == 'nt': default_loc = os.path.join('c:\\', config_path) else: default_loc = os.path.join('/etc/mysql/', config_path) # default group default_group = 'client' # first look at the given path, if not found look at the default location paths = [os.path.normpath(config_path), os.path.normpath(default_loc)] req_group = group # if not group is given use default. if not req_group and use_default: req_group = default_group for file_loc in paths: if os.path.exists(file_loc) and os.path.isfile(file_loc): opt_par = MySQLOptionsParser(file_loc) dict_grp = opt_par.get_groups_as_dict(req_group) if dict_grp: return dict_grp[req_group] else: if group: raise UtilError("The given group '{0}' was not found on " "the configuration file '{1}'" "".format(group, file_loc)) else: raise UtilError("The default group '{0}' was not found " "on the configuration file '{1}'" "".format(req_group, file_loc)) # No configuration file was found if paths[0] != paths[1]: raise UtilError("Could not find a configuration file neither in the " "given path '{0}' nor the default path '{1}'." "".format(*paths)) raise UtilError("Could not find a configuration file in the given " "path '{0}'.".format(paths[0]))
def grant_proxy_ssl_privileges(server, user, passw, at='localhost', privs="ALL PRIVILEGES", grant_opt=True, ssl=True, grant_proxy=True, proxy_user='******', proxy_host='localhost'): """Grant privileges to an user in a server with GRANT OPTION or/and REQUIRE SSL if required. server[in] Server to execute the grant query at. user_name[in] New user name. passw[in] password of the new user. at[in] Used in GRANT "TO '{0}'@'{1}'".format(user, at), (default localhost) grant_opt[in] if True, it will grant with GRANT OPTION (default True). ssl[in] if True, it will set REQUIRE SSL (default True). grant_proxy[in] if True, it will grant GRANT PROXY (default True). proxy_user[in] username for the proxied account (default: root) proxy_host[in] hostname for the proxied account (default: localhost) Note: Raises UtilError on any Error. """ grant = [ "GRANT", privs, "ON *.*", "TO '{0}'@'{1}'".format(user, at), "IDENTIFIED BY '{0}'".format(passw) if passw else "", "REQUIRE SSL" if ssl else "", "WITH GRANT OPTION" if grant_opt else "" ] try: server.exec_query(" ".join(grant)) except UtilDBError as err: raise UtilError("Cannot create new user {0} at {1}:{2} reason:" "{3}".format(user, server.host, server.port, err.errmsg)) if grant_proxy: grant = ("GRANT PROXY ON '{0}'@'{1}' " "TO '{2}'@'{3}' " "WITH GRANT OPTION").format(proxy_user, proxy_host, user, at) try: server.exec_query(grant) except UtilDBError as err: raise UtilError("Cannot grant proxy to user {0} at {1}:{2} " "reason:{3}".format(user, server.host, server.port, err.errmsg))
def _setup_compare(table1, table2, span_key_size): """Create and populate the compare summary tables This method creates the condensed hash table used to compare groups (span) of records. It also creates the Table instance for each table and populates values in the table information dictionary for use in other methods. The method also checks to ensure the tables have primary keys and that the keys are the same (have the same columns). An error is raised if neither of these are met. table1[in] table1 Table instance table2[in] table2 Table instance span_key_size[in] the size of key used for the hash. Returns tuple - string representations of the primary index columns """ server1 = table1.server server2 = table2.server # Get the primary key for the tables and make sure they are the same table1_idx = table1.get_primary_index() table2_idx = table2.get_primary_index() if len(table1_idx) != len(table2_idx): raise UtilError("Indexes are not the same.") elif table1_idx == [] or table2_idx == []: raise UtilError("No primary key found.") # drop the temporary tables _drop_compare_object(server1, table1.db_name, table1.tbl_name) _drop_compare_object(server2, table2.db_name, table2.tbl_name) # Build the primary key hash if needed tbl1_table, pri_idx1 = _get_compare_objects(table1_idx, table1, span_key_size) tbl2_table, pri_idx2 = _get_compare_objects(table1_idx, table2, span_key_size) if tbl1_table is None or tbl2_table is None: raise UtilError("Cannot create compare table.") # Create the compare tables server1.exec_query(tbl1_table) server2.exec_query(tbl2_table) return (pri_idx1, pri_idx2)
def search_my_print_defaults_tool(self, search_paths=None): """Search for the tool my_print_defaults. """ if not search_paths: search_paths = [] # Set the default search paths (i.e., default location of the # .mylogin.cnf file). default_paths = [my_login_config_path()] # Extend the list of path to search with the ones specified. if search_paths: default_paths.extend(search_paths) # Search for the tool my_print_defaults. try: self._tool_path = get_tool_path(self._basedir, _MY_PRINT_DEFAULTS_TOOL, defaults_paths=default_paths, search_PATH=True) except UtilError as err: raise UtilError("Unable to locate MySQL Client tools. " "Please confirm that the path to the MySQL client " "tools are included in the PATH. Error: %s" % err.errmsg)
def rotate(self): """This Method runs the rotation. This method will use the methods from the common library to rotate the binary log. """ # Check required privileges check_privileges(self.server, BINLOG_OP_ROTATE, ["RELOAD", "REPLICATION CLIENT"], BINLOG_OP_ROTATE_DESC, self.verbosity, self._report) active_binlog, binlog_size = get_active_binlog_and_size(self.server) if self.verbosity: self._report("# Active binlog file: '{0}' (size: {1} bytes)'" "".format(active_binlog, binlog_size)) if self.binlog_min_size: rotated = rotate(self.server, self.binlog_min_size, reporter=self._report) else: rotated = rotate(self.server, reporter=self._report) if rotated: new_active_binlog, _ = get_active_binlog_and_size(self.server) if active_binlog == new_active_binlog: raise UtilError("Unable to rotate binlog file.") else: self._report("# The binlog file has been rotated.") if self.verbosity: self._report("# New active binlog file: '{0}'" "".format(new_active_binlog))
def _check_databases(server1, server2, db1, db2, options): """Check databases server1[in] first server Server instance server2[in] second server Server instance db1[in] first database db2[in] second database options[in] options dictionary Returns tuple - Database class instances for databases """ # Check database create for differences if not options['no_diff']: # temporarily make the diff quiet to retrieve errors new_opt = {} new_opt.update(options) new_opt['quiet'] = True # do not print messages new_opt['suppress_sql'] = True # do not print SQL statements either res = diff_objects(server1, server2, db1, db2, new_opt, 'DATABASE') if res is not None: for row in res: print row print if not options['run_all_tests']: raise UtilError(_ERROR_DB_DIFF)
def _find_engine(server, target, message, fail, default): if target is not None: found = server.has_storage_engine(target) if not found and fail: raise UtilError(message) elif not found and not quiet: print message
def _make_select(objtype, pattern, database_pattern, check_body, use_regexp): """Generate a SELECT statement for finding an object. """ options = { 'pattern': obj2sql(pattern), 'regex': 'REGEXP' if use_regexp else 'LIKE', 'select_option': '', 'field_type': "'" + objtype.upper() + "'", } try: options.update(_OBJMAP[objtype]) except KeyError: raise UtilError("Invalid object type '{0}'. Use --help to see allowed " "values.".format(objtype)) # Build a condition for inclusion in the select condition = "{field_name} {regex} {pattern}".format(**options) if check_body and "body_field" in options: condition += " OR {body_field} {regex} {pattern}".format(**options) if database_pattern: options['database_pattern'] = obj2sql(database_pattern) condition = ("({0}) AND {schema_field} {regex} {database_pattern}" "".format(condition, **options)) options['condition'] = condition return _SELECT_TYPE_FRM.format(**options)
def _exec_statements(statements, destination, format, options, dryrun=False): """Execute a list of SQL statements statements[in] A list of SQL statements to execute destination[in] A connection to the destination server format[in] Format of import file options[in] Option dictionary containing the --skip_* options dryrun[in] If True, print the SQL statements and do not execute Returns (bool) - True if all execute, raises error if one fails """ new_engine = options.get("new_engine", None) def_engine = options.get("def_engine", None) quiet = options.get("quiet", False) for statement in statements: if (new_engine is not None or def_engine is not None) and \ statement.upper()[0:12] == "CREATE TABLE": i = statement.find(' ', 13) tbl_name = statement[13:i] statement = destination.substitute_engine(tbl_name, statement, new_engine, def_engine, quiet) try: if dryrun: print statement elif format != "sql" or not _skip_sql(statement, options): res = destination.exec_query(statement) # Here we capture any exception and raise UtilError to communicate to # the script/user. Since all util errors (exceptions) derive from # Exception, this is safe. except Exception, e: raise UtilError("Invalid statement:\n%s" % statement + "\nERROR: %s" % e.errmsg)
def _get_compare_objects(index_cols, table1): """Build the compare table and identify the primary index This method creates the compare table for use in forming the MD5 hash of rows and a hash of the primary key. It also forms the primary key list of columns. index_cols[in] a list of columns that form the primary key in the form (column_name, type) table1[in] a Table instance of the original table Returns tuple (table create statement, concatenated string of the primary index columns) """ table = None # build primary key col definition index_str = ''.join("{0}, ".format(quote_with_backticks(col[0])) \ for col in index_cols) index_defn = ''.join("{0} {1}, ". format(quote_with_backticks(col[0]), col[1]) \ for col in index_cols) if index_defn == "": raise UtilError("Cannot generate index definition") else: # Quote compare table appropriately with backticks q_tbl_name = quote_with_backticks( _COMPARE_TABLE_NAME.format(tbl=table1.tbl_name)) table = _COMPARE_TABLE.format(db=table1.q_db_name, compare_tbl=q_tbl_name, pkdef=index_defn) return (table, index_str)
def setUpClass(cls): # Find mysql_config_editor to manipulate data from .mylogin.cnf try: cls.edit_tool_path = get_tool_path(None, "mysql_config_editor", search_PATH=True) except UtilError as err: raise UtilError("MySQL client tools must be accessible to run " "this test (%s). Please add the location of the " "MySQL client tools to your PATH." % err.errmsg) # Create login-path data cmd = ("{mysql_config_editor} set --login-path={login_path} " "--host={host} --user={user}") cmd = cmd.format(mysql_config_editor=cls.edit_tool_path, login_path=_TEST_LOGIN_PATH, host=_TEST_HOST, user=_TEST_USER) # Execute command to create login-path data proc = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE, stdin=subprocess.PIPE) # Overwrite login-path if already exists (i.e. answer 'y' to question) proc.communicate(input='y')
def _compare_objects(server1, server2, obj1, obj2, reporter, options): """Compare object definitions and produce difference server1[in] first server Server instance server2[in] second server Server instance obj1[in] first object obj2[in] second object reporter[in] database compare reporter class instance options[in] options dictionary Returns list of errors """ from mysql.utilities.common.dbcompare import diff_objects errors = [] if not options['no_diff']: # For each database, compare objects # temporarily make the diff quiet to retrieve errors new_opt = {} new_opt.update(options) new_opt['quiet'] = True # do not print messages new_opt['suppress_sql'] = True # do not print SQL statements either res = diff_objects(server1, server2, obj1, obj2, new_opt) if res is not None: reporter.report_state('FAIL') errors.extend(res) if not options['run_all_tests']: raise UtilError(_ERROR_DB_DIFF) else: reporter.report_state('pass') else: reporter.report_state('SKIP') return errors
def determine_purgeable_binlogs(active_binlog_index, slaves, reporter, verbosity=0): """Determine the purgeable binary logs. This method will look at each slave given and will determinate the lowest binary log file that is being in use. active_binlog_index[in] Index of binlog currently in use by the master server or the higher binlog index value it wants to be purged. slaves[in] Slaves list. reporter[in] Method to call to report. verbosity[in] The verbosity level for reporting information. Returns the last index in use by the slaves, that is the newest binlog index that has between read by all the slave servers. """ # Determine old no needed binlogs master_log_file_in_use = [] index_last_in_use = active_binlog_index # pylint: disable=R0101 if slaves: for slave in slaves: if reporter is not None and verbosity >= 1: reporter("# Checking slave: {0}@{1}" "".format(slave['host'], slave['port'])) res = slave['instance'].get_status() if res: master_log_file = res[0][5] if reporter is not None and verbosity >= 1: reporter("# I/O thread is currently reading: {0}" "".format(master_log_file)) master_log_file_in_use.append(master_log_file) reading_index_file = int(master_log_file.split('.')[1]) if index_last_in_use > reading_index_file: index_last_in_use = reading_index_file if reporter is not None and verbosity >= 2: reporter("# File position of the I/O thread: {0}" "".format(res[0][6])) reporter("# Master binlog file with last event executed " "by the SQL thread: {0}".format(res[0][9])) reporter("# I/O thread running: {0}".format(res[0][10])) reporter("# SQL thread running: {0}".format(res[0][11])) if len(res[0]) > 52: if res[0][51]: reporter("# Retrieved GTid_Set: {0}" "".format(res[0][51])) if res[0][52]: reporter("# Executed GTid_Set: {0}" "".format(res[0][52])) return index_last_in_use else: raise UtilError("None Slave is connected to master")
def purge(server, purge_to_binlog, server_binlogs_list=None, reporter=None, dryrun=False, verbosity=0): """Purge the binary log for the given server. This method purges all the binary logs from the given server that are older than the given binlog file name specified by purge_to_binlog. The method can receive a list of the binary logs listed on the server to avoid querying the server again for this list. If The given purge_to_binlog is not listed on the server_binlogs_list the purge will not occur. For reporting capabilities if given the method report will be invoked to report messages and the server name that appears on the messages can be change with server_name. server[in] server instance where to purge binlogs on purge_to_binlog[in] purge binlog files older than this binlog file name. server_binlogs_list[in] A list of binlog files available on the given server, if not given, the list will be retrieved from the given server (default None). server_name[in] This name will appear when reporting (default 'Server'). reporter[in] A method to invoke with messages and warnings (default None). dryrun[in] boolean value that indicates if the purge query should be run on the server or reported only (default False). verbosity[in] The verbosity level for report messages. """ if server_binlogs_list is None: server_binlogs_list = server.get_server_binlogs_list() # The PURGE BINARY LOGS statement deletes all the binary log files listed # in the log index file, prior to the specified log file name. # Verify purge_to_binlog is listed on server binlog list and if not is the # first in the list continue the purge, else there is no binlogs to purge if (purge_to_binlog in server_binlogs_list and purge_to_binlog != server_binlogs_list[0]): purge_query = ("PURGE BINARY LOGS TO '{0}'").format(purge_to_binlog) if dryrun: reporter("# To manually purge purge the binary logs Execute the " "following query:") reporter(purge_query) else: if verbosity > 1: reporter("# Executing query {0}".format(purge_query)) else: reporter("# Purging binary logs prior to '{0}'" "".format(purge_to_binlog)) try: server.exec_query(purge_query) except UtilDBError as err: raise UtilError("Unable to purge binary log, reason: {0}" "".format(err.errmsg)) else: reporter("# No binlog files can be purged.")
def _check_row_counts(server1, server2, obj1, obj2, reporter, options): """Compare row counts for tables server1[in] first server Server instance server2[in] second server Server instance obj1[in] first object obj2[in] second object reporter[in] database compare reporter class instance options[in] options dictionary Returns list of errors """ errors = [] if not options['no_row_count']: rows1 = server1.exec_query("SELECT COUNT(*) FROM " + obj1) rows2 = server2.exec_query("SELECT COUNT(*) FROM " + obj2) if rows1 != rows2: reporter.report_state('FAIL') msg = _ERROR_ROW_COUNT.format(obj1, obj2) if not options['run_all_tests']: raise UtilError(msg) else: errors.append("# %s" % msg) else: reporter.report_state('pass') else: reporter.report_state('SKIP') return errors
def get_next_record(self): """Get the next audit log record. Generator function that return the next audit log record. More precisely, it returns a tuple with a formatted record dict and the original record. """ next_line = "" for line in self.log: if ((line.lstrip()).startswith('<AUDIT_RECORD') and not line.endswith('/>\n')): next_line = "{0} ".format(line.strip('\n')) continue elif len(next_line) > 0 and not line.endswith('/>\n'): next_line = '{0}{1}'.format(next_line, line.strip('\n')) continue else: next_line += line log_entry = next_line next_line = "" try: yield (self._make_record(xml.fromstring(log_entry)), log_entry) except (ParseError, SyntaxError): # SyntaxError is also caught for compatibility reasons with # python 2.6. In case an ExpatError which does not inherits # from SyntaxError is used as a ParseError. if not self._validXML(log_entry): raise UtilError("Malformed XML - Cannot parse log file: " "'{0}'\nInvalid XML element: " "{1!r}".format(self.log_name, log_entry))
def _check_privileges(server, options): """Check required privileges to move binary logs from server. This method check if the used user possess the required privileges to relocate binary logs from the server. More specifically, the following privilege is required: RELOAD (to flush the binary logs). An exception is thrown if the user doesn't have enough privileges. server[in] Server instance to check. options[in] Dictionary of options (skip_flush_binlogs, verbosity). """ skip_flush = options['skip_flush_binlogs'] if not skip_flush: # Only need to check privileges if flush is not skipped. verbosity = options['verbosity'] if verbosity > 0: print("# Checking user permission to move binary logs...\n" "#") # Check privileges user_obj = User(server, "{0}@{1}".format(server.user, server.host)) if not user_obj.has_privilege('*', '*', 'RELOAD'): raise UtilError( ERROR_USER_WITHOUT_PRIVILEGES.format( user=server.user, host=server.host, port=server.port, operation='perform binary log move', req_privileges='RELOAD'))
def _check_objects(server1, server2, db1, db2, db1_conn, db2_conn, options): """Check number of objects server1[in] first server Server instance server2[in] second server Server instance db1[in] first database db2[in] second database db1_conn[in] first Database instance db2_conn[in] second Database instance options[in] options dictionary Returns list of objects in both databases """ differs = False # Check for same number of objects in_both, in_db1, in_db2 = get_common_objects(server1, server2, db1, db2, False, options) in_both.sort() if not options['no_object_check']: server1_str = "server1." + db1 if server1 == server2: server2_str = "server1." + db2 else: server2_str = "server2." + db2 if len(in_db1) or len(in_db2): if options['run_all_tests']: if len(in_db1) > 0: differs = True print_missing_list(in_db1, server1_str, server2_str) print "#" if len(in_db2) > 0: differs = True print_missing_list(in_db2, server2_str, server1_str) print "#" else: differs = True raise UtilError(_ERROR_OBJECT_LIST.format(db1, db2)) # If in verbose mode, show count of object types. if options['verbosity'] > 1: objects = { 'TABLE': 0, 'VIEW': 0, 'TRIGGER': 0, 'PROCEDURE': 0, 'FUNCTION': 0, 'EVENT': 0, } for item in in_both: obj_type = item[0] objects[obj_type] += 1 print "Looking for object types table, view, trigger, procedure," + \ " function, and event." print "Object types found common to both databases:" for obj in objects: print " {0:>12} : {1}".format(obj, objects[obj]) return (in_both, differs)
def _build_logfile_list(server, log_name, suffix='_file'): """Build a list of all log files based on the system variable by the same name as log_name. server[in] Connected server log_name[in] Name of log (e.g. slow_query_log) suffix[in] Suffix of log variable name (e.g. slow_query_log_file) default = '_file' return (tuple) (logfiles[], path to log files, total size) """ log_path = None res = server.show_server_variable(log_name) if res != [] and res[0][1].upper() == 'OFF': print "# The %s is turned off on the server." % log_name else: res = server.show_server_variable(log_name + suffix) if res == []: raise UtilError("Cannot get %s_file setting." % log_name) log_path = res[0][1] if os.access(log_path, os.R_OK): parts = os.path.split(log_path) if len(parts) <= 1: log_file = log_path else: log_file = parts[1] log_path_size = os.path.getsize(log_path) return (log_file, log_path, int(log_path_size)) return None, 0, 0
def get_option_matches(self, util_info, option_prefix, find_alias=False): """Get list of option dictionary entries for options that match the prefix. util_info[in] utility information option_prefix[in] prefix for option name find_alias[in] if True, match alias (default = False) Returns list of dictionary items that match prefix """ # Check type of util_info if util_info is None or util_info == {} or \ not isinstance(util_info, dict): raise UtilError("Empty or invalide utility dictionary.") matches = [] stop = len(option_prefix) if isinstance(util_info['options'], str): self.parse_all_options(util_info) for option in util_info['options']: if option is None: continue name = option.get('name', None) if name is None: continue if find_alias: if option.get('alias', '') == option_prefix: matches.append(option) else: if name[0:stop] == option_prefix: matches.append(option) return matches
def connect(self): """Connect to server Attempts to connect to the server as specified by the connection parameters. Note: This method must be called before executing queries. Raises UtilError if error during connect """ try: parameters = { 'user': self.user, 'host': self.host, 'port': self.port, } if self.socket and os.name == "posix": parameters['unix_socket'] = self.socket if self.passwd and self.passwd != "": parameters['passwd'] = self.passwd parameters['charset'] = self.charset self.db_conn = mysql.connector.connect(**parameters) except mysql.connector.Error, e: # Reset any previous value if the connection cannot be established, # before raising an exception. This prevents the use of a broken # database connection. self.db_conn = None raise UtilError("Cannot connect to the %s server.\n" "Error %s" % (self.role, e.msg), e.errno)
def get_connection_dictionary(conn_info): """Get the connection dictionary. The method accepts one of the following types for conn_info: - dictionary containing connection information including: (user, passwd, host, port, socket) - connection string in the form: user:pass@host:port:socket or login-path:port:socket - an instance of the Server class conn_info[in] Connection information Returns dict - dictionary for connection (user, passwd, host, port, socket) """ if conn_info is None: return conn_info conn_val = {} if isinstance(conn_info, dict): conn_val = conn_info elif isinstance(conn_info, Server): # get server's dictionary conn_val = conn_info.get_connection_values() elif isinstance(conn_info, basestring): # parse the string conn_val = parse_connection(conn_info, None) else: raise UtilError("Cannot determine connection information type.") return conn_val
def _connect(self, conn): """Find the attached slaves for a list of server connections. This method connects to each server in the list and retrieves its slaves. It can be called recursively if the recurse parameter is True. conn[in] Connection dictionary used to connect to server Returns tuple - master Server class instance, master:host string """ conn_options = { 'quiet': self.quiet, 'src_name': "master", 'dest_name': None, 'version': "5.0.0", 'unique': True, 'verbose': self.verbose, } certs_paths = {} if 'ssl_ca' in dir(conn) and conn.ssl_ca is not None: certs_paths['ssl_ca'] = conn.ssl_ca if 'ssl_cert' in dir(conn) and conn.ssl_cert is not None: certs_paths['ssl_cert'] = conn.ssl_cert if 'ssl_key' in dir(conn) and conn.ssl_key is not None: certs_paths['ssl_key'] = conn.ssl_key conn_options.update(certs_paths) master_info = "{0}:{1}".format(conn['host'], conn['port']) master = None # Increment num_retries if not set when --prompt is used if self.prompt_user and self.num_retries == 0: self.num_retries += 1 # Attempt to connect to the server given the retry limit for i in range(0, self.num_retries + 1): try: servers = connect_servers(conn, None, conn_options) master = servers[0] break except UtilError as e: print("FAILED.\n") if i < self.num_retries and self.prompt_user: print("Connection to %s has failed.\n" % master_info + \ "Please enter the following information " + \ "to connect to this server.") conn['user'] = raw_input("User name: ") conn['passwd'] = getpass.getpass("Password: "******"{0}:{1}".format(conn['host'], master.port) return (master, master_info)
def get_tool_path(basedir, tool, fix_ext=True, required=True, defaults_paths=None, search_PATH=False): """Search for a MySQL tool and return the full path basedir[in] The initial basedir to search (from mysql server) tool[in] The name of the tool to find fix_ext[in] If True (default is True), add .exe if running on Windows. required[in] If True (default is True), and error will be generated and the utility aborted if the tool is not found. defaults_paths[in] Default list of paths to search for the tool. By default an empty list is assumed, i.e. []. search_PATH[in] Boolean value that indicates if the paths specified by the PATH environment variable will be used to search for the tool. By default the PATH will not be searched, i.e. search_PATH=False. Returns (string) full path to tool """ if not defaults_paths: defaults_paths = [] search_paths = [] if basedir: # Add specified basedir path to search paths _add_basedir(search_paths, basedir) if defaults_paths and len(defaults_paths): # Add specified default paths to search paths for path in defaults_paths: search_paths.append(path) else: # Add default basedir paths to search paths _add_basedir(search_paths, "/usr/local/mysql/") _add_basedir(search_paths, "/usr/sbin/") _add_basedir(search_paths, "/usr/share/") # Search in path from the PATH environment variable if search_PATH: for path in os.environ['PATH'].split(os.pathsep): search_paths.append(path) if os.name == "nt" and fix_ext: tool = tool + ".exe" # Search for the tool for path in search_paths: norm_path = os.path.normpath(path) if os.path.isdir(norm_path): toolpath = os.path.join(norm_path, tool) if os.path.isfile(toolpath): return toolpath else: if tool == "mysqld.exe": toolpath = os.path.join(norm_path, "mysqld-nt.exe") if os.path.isfile(toolpath): return toolpath if required: raise UtilError("Cannot find location of %s." % tool) return None
def run(self): self.server0 = self.servers.get_server(0) cmd_str = "mysqlserverclone.py --server=%s --delete-data " % \ self.build_connection_string(self.server0) port1 = int(self.servers.get_next_port()) cmd_str += " --new-port=%d --root-password=root " % port1 comment = "Test case 1 - clone a running server" self.results.append(comment + "\n") full_datadir = os.path.join(os.getcwd(), "temp_%s" % port1) cmd_str += "--new-data=%s " % full_datadir res = self.exec_util(cmd_str, "start.txt") for line in open("start.txt").readlines(): # Don't save lines that have [Warning] if "[Warning]" in line: continue self.results.append(line) if res: raise MUTLibError("%s: failed" % comment) self.new_server = self.check_connect(port1, full_datadir) basedir = "" # Get basedir rows = self.server0.exec_query("SHOW VARIABLES LIKE 'basedir'") if not rows: raise UtilError("Unable to determine basedir of running server.") basedir = rows[0][1] port2 = int(self.servers.get_next_port()) cmd_str = "mysqlserverclone.py --root-password=root --delete-data " cmd_str += "--new-port=%d --basedir=%s " % (port2, basedir) comment = "Test case 2 - clone a server from basedir" self.results.append(comment + "\n") full_datadir = os.path.join(os.getcwd(), "temp_%s" % port2) cmd_str += "--new-data=%s " % full_datadir res = self.exec_util(cmd_str, "start.txt") for line in open("start.txt").readlines(): # Don't save lines that have [Warning] if "[Warning]" in line: continue self.results.append(line) if res: raise MUTLibError("%s: failed" % comment) server = self.check_connect(port2, full_datadir, "cloned_server_basedir") self.servers.stop_server(server) self.servers.clear_last_port() self.replace_result("# -uroot", "# -uroot [...]\n") self.replace_result("# Cloning the MySQL server located at", "# Cloning the MySQL server located at XXXX\n") return True
def get_next_record(self): """Get the next audit log record. Generator function that return the next audit log record. More precisely, it returns a tuple with a formatted record dict and the original record. """ next_line = "" new_format = False multiline = False for line in self.log: if line.lstrip().startswith('<AUDIT_RECORD>'): # Found first record line in the new format. new_format = True multiline = True next_line = line continue elif (line.lstrip().startswith('<AUDIT_RECORD') and not line.endswith('/>\n')): # Found (first) record line in the old format. next_line = "{0} ".format(line.strip('\n')) if not line.endswith('/>\n'): multiline = True continue elif multiline: if ((new_format and line.strip().endswith('</AUDIT_RECORD>')) or (not new_format and line.endswith('/>\n'))): # Detect end of record in the old and new format and # append last record line. next_line += line else: if not line.strip().startswith('<'): # Handle SQL queries broke into multiple lines, # removing newline characters. next_line = '{0}{1}'.format(next_line.strip('\n'), line.strip('\n')) else: next_line += line continue else: next_line += line log_entry = next_line next_line = "" try: yield ( self._make_record(xml.fromstring(log_entry), new_format), log_entry ) except (ParseError, SyntaxError): # SyntaxError is also caught for compatibility reasons with # python 2.6. In case an ExpatError which does not inherits # from SyntaxError is used as a ParseError. if not self._validXML(log_entry): raise UtilError("Malformed XML - Cannot parse log file: " "'{0}'\nInvalid XML element: " "{1!r}".format(self.log_name, log_entry))
def open_log(self): """Open the audit log file. """ # Get the log from a remote server # TODO : check to see if the log is local. If not, attempt # to log into the server via rsh and copy the file locally. self.remote_file = False if not self.log_name or not os.path.exists(self.log_name): raise UtilError("Cannot read log file '%s'." % self.log_name) self.log = open(self.log_name)
def _get_transform(server1, server2, object1, object2, options, object_type): """Get the transformation SQL statements This method generates the SQL statements to transform the destination object based on direction of the compare. server1[in] first server connection server2[in] second server connection object1 the first object in the compare in the form: (db.name) object2 the second object in the compare in the form: (db.name) options[in] a dictionary containing the options for the operation: (quiet, etc.) object_type[in] type of the objects to be compared (e.g., TABLE, PROCEDURE, etc.). Returns tuple - (bool - same db name?, list of transformation statements) """ try: m_obj1 = re.match(REGEXP_QUALIFIED_OBJ_NAME, object1) db1, name1 = m_obj1.groups() m_obj2 = re.match(REGEXP_QUALIFIED_OBJ_NAME, object2) db2, name2 = m_obj2.groups() except: raise UtilError("Invalid object name arguments for _get_transform" "(): %s, %s." % (object1, object2)) # If the second part of the object qualified name is None, then the format # is not 'db_name.obj_name' for object1 and therefore must treat it as a # database name. (supports backticks and the use of '.' (dots) in names.) if not name1 or object_type == 'DATABASE': # We are working with databases so db and name need to be set # to the database name to tell the get_object_definition() method # to retrieve the database information. name1 = db1 name2 = db2 db_1 = Database(server1, db1, options) db_2 = Database(server2, db2, options) obj1 = db_1.get_object_definition(db1, name1, object_type) obj2 = db_2.get_object_definition(db2, name2, object_type) # Get the transformation based on direction. transform_str = [] xform = SQLTransformer(db_1, db_2, obj1[0], obj2[0], object_type, options.get('verbosity', 0)) differences = xform.transform_definition() if differences and len(differences) > 0: transform_str.extend(differences) return transform_str
def _parse_grant_statement(statement, sql_mode=''): """ Returns a namedtuple with the parsed GRANT information. statement[in] Grant string in the sql format returned by the server. Returns named tuple with GRANT information or None. """ grant_parse_re = re.compile( r""" GRANT\s(.+)?\sON\s # grant or list of grants (?:(?:PROCEDURE\s)|(?:FUNCTION\s))? # optional for routines only (?:(?:(\*|`?[^']+`?)\.(\*|`?[^']+`?)) # object where grant applies | ('[^']*'@'[^']*')) # For proxy grants user/host \sTO\s([^@]+@[\S]+) # grantee (?:\sIDENTIFIED\sBY\sPASSWORD (?:(?:\s<secret>)|(?:\s\'[^\']+\')?))? # optional pwd (?:\sREQUIRE\sSSL)? # optional SSL (\sWITH\sGRANT\sOPTION)? # optional grant option $ # End of grant statement """, re.VERBOSE) grant_tpl_factory = namedtuple( "grant_info", "privileges proxy_user " "db object user") match = re.match(grant_parse_re, statement) if match: # quote database name and object name with backticks if match.group(1).upper() != 'PROXY': db = match.group(2) if not is_quoted_with_backticks(db, sql_mode) and db != '*': db = quote_with_backticks(db, sql_mode) obj = match.group(3) if not is_quoted_with_backticks(obj, sql_mode) and obj != '*': obj = quote_with_backticks(obj, sql_mode) else: # if it is not a proxy grant db = obj = None grants = grant_tpl_factory( # privileges set([priv.strip() for priv in match.group(1).split(",")]), match.group(4), # proxied user db, # database obj, # object match.group(5), # user ) # If user has grant option, add it to the list of privileges if match.group(6) is not None: grants.privileges.add("GRANT OPTION") else: raise UtilError("Unable to parse grant statement " "{0}".format(statement)) return grants
def is_binary_log_filename(filename, log_type=LOG_TYPE_ALL, basename=None): """Check if the filename matches the name format for binary log files. This function checks if the given filename corresponds to the filename format of known binary log files, according to the specified log_type and optional basename. The file extension is a sequence number (.nnnnnn). If a basename is given then the filename for the binary log file must have the format 'basename.nnnnnn'. Otherwise the default filename is assumed, depending on the log_type: '*-bin.nnnnnn' for the 'bin' log type, '*-relay-bin.nnnnnn' for the 'relay' log type, and both for the 'all' type. filename[in] Filename to check. log_type[in] Type of the binary log, must be one of the following values: 'bin' for binlog files, 'relay' for relay log files, 'all' for both binary log files. By default = 'all'. basename[in] Basename defined for the binary log file. None by default, meaning that the default server name formats are assumed (according to the given log type). """ # Split file basename and extension. f_base, f_ext = os.path.splitext(filename) f_ext = f_ext[1:] # remove starting dot '.' # Check file basename. if basename: if f_base != basename: # Defined basename does not match. return False else: # Check default serve basename for the given log_type. if log_type == LOG_TYPE_BIN: # *-bin.nnnnnn (excluding *-relay-bin.nnnnnn) if not f_base.endswith('-bin') or f_base.endswith('-relay-bin'): return False elif log_type == LOG_TYPE_RELAY: # *-relay-bin.nnnnnn if not f_base.endswith('-relay-bin'): return False elif log_type == LOG_TYPE_ALL: # *-bin.nnnnnn (including *-relay-bin.nnnnnn) if not f_base.endswith('-bin'): return False else: raise UtilError("Unsupported log-type: {0}".format(log_type)) # Check file extension. try: int(f_ext) except ValueError: # Extension is not a sequence number (error converting to integer). return False # Return true if basename and extension checks passed. return True