def get_diff(self, db_name, version): file_schema = "%s/%04d-schema.img" % (db_name, int(version)) file_schema_swp = "%s/%04d-schema.swp" % (db_name, int(version)) tmp_db = "tmp_online_mig_%s" % (db_name) query = "CREATE DATABASE %s;" % tmp_db self.server.disable_foreign_key_checks() self.server.exec_query(query) f = open(file_schema, 'r') f_swp = open(file_schema_swp, 'w') f_swp.write("USE %s\n" % tmp_db) buff = "" for line in f.readlines(): #if re.search(';$', line, re.IGNORECASE): if re.search(';$' + '(?i)', line): buff = buff + line f_swp.write(buff) buff = "" else: buff = buff + line.strip() f.close() f_swp.close() query_options = {'multi': True} f_swp = open(file_schema_swp, 'r') for line in f_swp.readlines(): self.server.exec_query(line, query_options) f_swp.close() os.remove(file_schema_swp) query_options = { 'run_all_tests': True, 'reverse': False, 'verbosity': None, 'no_object_check': False, 'no_data': True, 'quiet': True, 'difftype': 'differ', 'width': 75, 'changes-for': 'server1', 'skip_grants': True} source_values = parse_connection(server_connection) destination_values = parse_connection(server_connection) with capture() as stepback: dbcompare.database_compare(source_values, destination_values, db_name, tmp_db, query_options) buf = "" found = 0 for line in stepback.getvalue().splitlines(True): logging.debug(u"%s" % line) if re.search('CREATE TABLE', line): line = re.sub("CREATE" + '(?i)',"", line, 1) #line = re.sub("CREATE","", line, 1, re.IGNORECASE) #line = re.sub("\(\n","", line, 1, re.IGNORECASE) line = re.sub("\(\n" + '(?i)',"", line, 1) buf=line if (re.search('^\+', line) or re.search('^\-', line)) and not re.search('CREATE DATABASE', line) and not re.search('CONSTRAINT.*FOREIGN KEY', line): #line = re.sub("\n","", line, 1, re.IGNORECASE) line = re.sub("\n" + '(?i)',"", line, 1) if len(buf) > 0: print u"%s" % buf buf="" print u"%s" % line query = "DROP DATABASE %s" % tmp_db self.server.exec_query(query) self.server.disable_foreign_key_checks(disable=False)
def test_invalid(self): """Test parsing invalid versions of connection strings. If the connection string is invalid, a FormatError should be thrown. """ for spec in self.invalid_specificers: try: parse_connection(spec) except FormatError: pass except: self.fail("Unexpected exception thrown.") else: self.fail("Exception not thrown for: '%s'." % spec)
def execute(self, connections, **kwrds): """Execute the search for processes, queries, or connections This method searches for processes, queriers, or connections to either kill or display the matches for one or more servers. connections[in] list of connection parameters kwrds[in] dictionary of options output file stream to display information default = sys.stdout connector connector to use default = mysql.connector format format for display default = GRID """ output = kwrds.get('output', sys.stdout) connector = kwrds.get('connector', mysql.connector) fmt = kwrds.get('format', "grid") charset = kwrds.get('charset', None) headers = ("Connection", "Id", "User", "Host", "Db", "Command", "Time", "State", "Info") entries = [] # Build SQL statement for info in connections: conn = parse_connection(info) if not conn: msg = "'%s' is not a valid connection specifier" % (info, ) raise FormatError(msg) if charset: conn['charset'] = charset info = conn connection = connector.connect(**info) if not charset: # If no charset provided, get it from the # "character_set_client" server variable. cursor = connection.cursor() cursor.execute("SHOW VARIABLES LIKE 'character_set_client'") res = cursor.fetchall() connection.set_charset_collation(charset=str(res[0][1])) cursor.close() cursor = connection.cursor() cursor.execute(self.__select) for row in cursor: if KILL_QUERY in self.__actions: cursor.execute("KILL {0}".format(row[0])) if KILL_CONNECTION in self.__actions: cursor.execute("KILL {0}".format(row[0])) if PRINT_PROCESS in self.__actions: entries.append(tuple([_spec(info)] + list(row))) # If output is None, nothing is printed if len(entries) > 0 and output: entries.sort(key=lambda fifth: fifth[5]) print_list(output, fmt, headers, entries) elif PRINT_PROCESS in self.__actions: raise EmptyResultError("No matches found")
def test_connection(self, test_num, test_data, with_credentials=False): """Test connection. test_num[in] Test number. test_data[in] Test data. with_credentials True if with credentials. """ if self.debug: print "\nTest case {0} - {1}".format(test_num + 1, test_data[0]) try: if with_credentials: conn_string = test_data[1] else: conn_string = "root@{0}:3306".format(test_data[1]) self.conn_vals = parse_connection(conn_string) except FormatError as err: if test_data[3]: # This expected. self.results.append("FAIL") else: raise MUTLibError("Test Case {0}: Parse failed! Error: " "{1}".format(test_num + 1, err)) else: if test_data[3]: raise MUTLibError("Test Case {0}: Parse should have failed. " "Got this instead: " "{1}".format(test_num + 1, self.conn_vals['host'])) elif with_credentials: self.results.append(_spec(self.conn_vals)) else: self.results.append(self.conn_vals['host'])
def execute(self, connections, **kwrds): """Execute the search for processes, queries, or connections This method searches for processes, queriers, or connections to either kill or display the matches for one or more servers. connections[in] list of connection parameters kwrds[in] dictionary of options output file stream to display information default = sys.stdout connector connector to use default = mysql.connector format format for display default = GRID """ output = kwrds.get('output', sys.stdout) connector = kwrds.get('connector', mysql.connector) fmt = kwrds.get('format', "grid") charset = kwrds.get('charset', None) headers = ("Connection", "Id", "User", "Host", "Db", "Command", "Time", "State", "Info") entries = [] # Build SQL statement for info in connections: conn = parse_connection(info) if not conn: msg = "'%s' is not a valid connection specifier" % (info,) raise FormatError(msg) if charset: conn['charset'] = charset info = conn connection = connector.connect(**info) if not charset: # If no charset provided, get it from the # "character_set_client" server variable. cursor = connection.cursor() cursor.execute("SHOW VARIABLES LIKE 'character_set_client'") res = cursor.fetchall() connection.set_charset_collation(charset=str(res[0][1])) cursor.close() cursor = connection.cursor() cursor.execute(self.__select) for row in cursor: if KILL_QUERY in self.__actions: cursor.execute("KILL {0}".format(row[0])) if KILL_CONNECTION in self.__actions: cursor.execute("KILL {0}".format(row[0])) if PRINT_PROCESS in self.__actions: entries.append(tuple([_spec(info)] + list(row))) # If output is None, nothing is printed if len(entries) > 0 and output: entries.sort(key=lambda fifth: fifth[5]) print_list(output, fmt, headers, entries) elif PRINT_PROCESS in self.__actions: raise EmptyResultError("No matches found")
def run(self): # Test parse_connection with login-paths con_tests = [ "test_user@localhost:3306", "test_mylogin:3306", "test_user@localhost:1000", "test_mylogin:1000", "test-hyphen1234#:3306", "test' \\\"-hyphen:3306" ] for test_ in con_tests: con_dic = parse_connection(test_, options={"charset": "utf8"}) # Sort the keys to fix the issue where the keys are printed in # different order on linux and windows. self.results.append(sorted(con_dic.iteritems())) # Test parse_user_password with login-paths user_pass_tests = [ "test_user", "test_mylogin", "test_user:"******"user_x:", "user_x:pass_y", "rpl:'L5!w1SJzVuj40(p?tF@(9Y70_@:z(HXc'" ] for test_ in user_pass_tests: try: user_pass = parse_user_password(test_) self.results.append(user_pass) except FormatError as err: self.results.append(err) # Transform list of dictionaries into list of strings self.results = ["{0!s}\n".format(con_dic) for con_dic in self.results] # remove socket information from posix systems to have same output # on both posix and and windows systems self.replace_substring_portion(", ('unix_socket'", ".socket')", '') return True
def run(self): # Test parse_connection with login-paths con_tests = ["test_user@localhost:3306", "test_mylogin:3306", "test_user@localhost:1000", "test_mylogin:1000", "test_user@localhost:1000:/my.socket", "test_mylogin:1000:/my.socket", "test_user@localhost:3306:/my.socket", "test_mylogin:3306:/my.socket", "test-hyphen1234#:3306", "test-hyphen1234#:13000:my.socket", "test' \\\"-hyphen:3306", "test' \\\"-hyphen:3306:my.socket", "test' \\\"-hyphen:13001:my.socket"] for test_ in con_tests: con_dic = parse_connection(test_, options={"charset": "utf8"}) # Sort the keys to fix the issue where the keys are printed in # different order on linux and windows. self.results.append(sorted(con_dic.iteritems())) # Test parse_user_password with login-paths user_pass_tests = ["test_user", "test_mylogin", "test_user:"******"user_x:", "user_x:pass_y"] for test_ in user_pass_tests: user_pass = parse_user_password(test_) self.results.append(user_pass) # Transform list of dictionaries into list of strings self.results = ["{0!s}\n".format(con_dic) for con_dic in self.results] # remove socket information from posix systems to have same output # on both posix and and windows systems self.replace_substring_portion(", ('unix_socket'", ".socket')", '') return True
def test_valid(self): """Test parsing valid versions of connection strings. """ for source, expected in self.valid_specifiers: result = _spec(parse_connection(source)) frm = u"{0}: was {1}, expected {2}" msg = frm.format(source, result, expected) self.assertEqual(expected, result, msg)
def execute(self, connections, output=sys.stdout, connector=mysql.connector, **kwrds): """Execute the search for objects This method searches for objects that match a search criteria for one or more servers. connections[in] list of connection parameters output[in] file stream to display information default = sys.stdout connector[in] connector to use default = mysql.connector kwrds[in] dictionary of options format format for display default = GRID """ fmt = kwrds.get('format', "grid") charset = kwrds.get('charset', None) ssl_opts = kwrds.get('ssl_opts', {}) entries = [] for info in connections: conn = parse_connection(info) if not conn: msg = "'%s' is not a valid connection specifier" % (info,) raise FormatError(msg) if charset: conn['charset'] = charset info = conn conn['host'] = conn['host'].replace("[", "") conn['host'] = conn['host'].replace("]", "") if connector == mysql.connector: set_ssl_opts_in_connection_info(ssl_opts, info) connection = connector.connect(**info) if not charset: # If no charset provided, get it from the # "character_set_client" server variable. cursor = connection.cursor() cursor.execute("SHOW VARIABLES LIKE 'character_set_client'") res = cursor.fetchall() connection.set_charset_collation(charset=str(res[0][1])) cursor.close() cursor = connection.cursor() cursor.execute(self.__sql) entries.extend([tuple([_spec(info)] + list(row)) for row in cursor]) headers = ["Connection"] headers.extend(col[0].title() for col in cursor.description) if len(entries) > 0 and output: print_list(output, fmt, headers, entries) else: msg = "Nothing matches '%s' in any %s" % \ (self.__pattern, _join_words(self.__types, conjunction="or")) raise EmptyResultError(msg)
def run(self): # Test parse_connection with login-paths missing parameters. con_tests = ["test_no_host", "test_no_user", "test_no_host:3333", "test_no_user:3333", "test_no_user:3306:/does/not/exist/mysql.sock", "test_no_host:3333"] for test_ in con_tests: try: parse_connection(test_, options={"charset": "utf8"}) except UtilError as err: self.results.append("{0}\n".format(err.errmsg)) # remove socket information to have same output # on both posix and and windows operating systems. self.replace_substring_portion("port or ", "socket", "port") return True
def execute(self, connections, output=sys.stdout, connector=mysql.connector, **kwrds): """Execute the search for objects This method searches for objects that match a search criteria for one or more servers. connections[in] list of connection parameters output[in] file stream to display information default = sys.stdout connector[in] connector to use default = mysql.connector kwrds[in] dictionary of options format format for display default = GRID """ fmt = kwrds.get('format', "grid") charset = kwrds.get('charset', None) entries = [] for info in connections: conn = parse_connection(info) if not conn: msg = "'%s' is not a valid connection specifier" % (info, ) raise FormatError(msg) if charset: conn['charset'] = charset info = conn conn['host'] = conn['host'].replace("[", "") conn['host'] = conn['host'].replace("]", "") connection = connector.connect(**info) if not charset: # If no charset provided, get it from the # "character_set_client" server variable. cursor = connection.cursor() cursor.execute("SHOW VARIABLES LIKE 'character_set_client'") res = cursor.fetchall() connection.set_charset_collation(charset=str(res[0][1])) cursor.close() cursor = connection.cursor() cursor.execute(self.__sql) entries.extend( [tuple([_spec(info)] + list(row)) for row in cursor]) headers = ["Connection"] headers.extend(col[0].title() for col in cursor.description) if len(entries) > 0 and output: print_list(output, fmt, headers, entries) else: msg = "Nothing matches '%s' in any %s" % \ (self.__pattern, _join_words(self.__types, conjunction="or")) raise EmptyResultError(msg)
def run(self): # Test parse_connection with login-paths missing parameters. con_tests = [ "test_user_only", "test_host_only", "test_port_only", "test_user_only:3306", "test_host_only:3306", "test_host_only:3306:/does/not/exist/mysql.sock", "test_port_only:3306" ] for test_ in con_tests: try: parse_connection(test_) except UtilError as err: self.results.append("{0}\n".format(err.errmsg)) # Test login-paths only with socket information separately # and change the output to be the same on both Windows and Posix # operating systems. # Missing username try: if os.name == "posix": parse_connection("test_socket_only") else: parse_connection("test_host_only:3306") except UtilError as err: self.results.append("{0}\n".format(err.errmsg)) # on posix systems if we use socket, and there is no hostname, # hostname is assumed to be "localhost". So it is equivalent # to have a host and a port on windows systems. try: if os.name == "posix": parse_connection("test_port_only:/does/not/exist/mysql.sock") else: parse_connection("test_host_only:3306") except UtilError as err: self.results.append("{0}\n".format(err.errmsg)) # remove socket warnings to have same output # on both posix and windows operating systems. self.replace_substring_portion("port or ", "socket", "port") return True
def run(self): # Test parse_connection with login-paths missing parameters. con_tests = ["test_user_only", "test_host_only", "test_port_only", "test_user_only:3306", "test_host_only:3306", "test_host_only:3306:/does/not/exist/mysql.sock", "test_port_only:3306"] for test_ in con_tests: try: parse_connection(test_) except UtilError as err: self.results.append("{0}\n".format(err.errmsg)) # Test login-paths only with socket information separately # and change the output to be the same on both Windows and Posix # operating systems. # Missing username try: if os.name == "posix": parse_connection("test_socket_only") else: parse_connection("test_host_only:3306") except UtilError as err: self.results.append("{0}\n".format(err.errmsg)) # on posix systems if we use socket, and there is no hostname, # hostname is assumed to be "localhost". So it is equivalent # to have a host and a port on windows systems. try: if os.name == "posix": parse_connection("test_port_only:/does/not/exist/mysql.sock") else: parse_connection("test_host_only:3306") except UtilError as err: self.results.append("{0}\n".format(err.errmsg)) # remove socket warnings to have same output # on both posix and windows operating systems. self.replace_substring_portion("port or ", "socket", "port") return True
def parse_user_host(user_name): """Parse user, passwd, host, port from user:passwd@host user_name[in] MySQL user string (user:passwd@host) """ no_ticks = user_name.replace("'", "") try: conn_values = parse_connection(no_ticks) except FormatError: raise UtilError("Cannot parse user:pass@host : %s." % no_ticks) return (conn_values['user'], conn_values['passwd'], conn_values['host'])
def get_schema_img(self, db_name): with capture() as dbschema: server_values = parse_connection(server_connection) query_options = {'skip_data': True, 'skip_grants': True, 'skip_create': True, 'rpl_mode': None, 'quiet': True} db_list = [] db_list.append(db_name) with capture() as dbschema: dbexport.export_databases(server_values, db_list, sys.stdout, query_options) db_schema = dbschema.getvalue().splitlines(True) return db_schema
def parse_user_host(user_name): """Parse user, passwd, host, port from user:passwd@host user_name[in] MySQL user string (user:passwd@host) returns - tuple - user, passwd, host """ # Check for anonymous user. If not, continue. if user_name == "''@'%'": return ('', None, '%') no_ticks = user_name.replace("'", "") try: conn_values = parse_connection(no_ticks) except FormatError: raise UtilError("Cannot parse user:pass@host : %s." % no_ticks) return (conn_values['user'], conn_values['passwd'], conn_values['host'])
def run(self): self.res_fname = "result.txt" # Test parse_connection with login-paths con_tests = [ "test_user@localhost:3306", # use of normal parser "test_mylogin:3306", # use of login-path # collision with login-path: "c:\\some_config.cnf", # & no existing windows file "/etc_etc/some_config.cnf", # & no existing linux file "test_default_group", # test default client group "temp_1.cnf", # collision & default group "temp_1.cnf[undeclared]", # undeclared group "temp_2.cnf", # collision & default group w/ssl "temp_2.cnf[missing_values]", # missing values "temp_1.cnf[complete]", # complete group ] test_n = 0 for test_ in con_tests: test_n += 1 msg = "Test case {0} - {1}".format(test_n, test_) if self.debug: print(msg) self.results.append("{0}\n".format(msg)) try: conn = parse_connection(test_) except UtilError as err: if self.debug: print(err.errmsg) self.results.append("{0}\n".format(err.errmsg)) else: if self.debug: print(sorted(conn.iteritems())) self.results.append("{0}\n".format(sorted(conn.iteritems()))) # Replacements self.replace_substring_portion("port or ", "socket", "port") self.replace_substring(".exe", "") self.mask_result_portion( "Unable to locate MySQL Client tools.", "In addition, could not find a configuration " "file", "\n", "In addition, could not find a " "configuration file XXXX-XXXXX") return True
def run(self): self.res_fname = "result.txt" # Test parse_connection with login-paths con_tests = [ "test_user@localhost:3306", # use of normal parser "test_mylogin:3306", # use of login-path # collision with login-path: "c:\\some_config.cnf", # & no existing windows file "/etc_etc/some_config.cnf", # & no existing linux file "test_default_group", # test default client group "temp_1.cnf", # collision & default group "temp_1.cnf[undeclared]", # undeclared group "temp_2.cnf", # collision & default group w/ssl "temp_2.cnf[missing_values]", # missing values "temp_1.cnf[complete]", # complete group ] test_n = 0 for test_ in con_tests: test_n += 1 msg = "Test case {0} - {1}".format(test_n, test_) if self.debug: print(msg) self.results.append("{0}\n".format(msg)) try: conn = parse_connection(test_) except UtilError as err: if self.debug: print(err.errmsg) self.results.append("{0}\n".format(err.errmsg)) else: if self.debug: print(sorted(conn.iteritems())) self.results.append("{0}\n".format(sorted(conn.iteritems()))) # Replacements self.replace_substring_portion("port or ", "socket", "port") self.replace_substring(".exe", "") self.mask_result_portion("Unable to locate MySQL Client tools.", "In addition, could not find a configuration " "file", "\n", "In addition, could not find a " "configuration file XXXX-XXXXX") return True
def get_schema_img(self, db_name): with capture() as dbschema: server_values = parse_connection(server_connection) query_options = { 'skip_data': True, 'skip_grants': True, 'skip_create': True, 'rpl_mode': None, 'skip_gtid': True, 'quiet': True } db_list = [] db_list.append(db_name) with capture() as dbschema: dbexport.export_databases(server_values, db_list, sys.stdout, query_options) db_schema = dbschema.getvalue().splitlines(True) return db_schema
def run(self): self.res_fname = "result.txt" # Test parse_connection with login-paths con_tests = [ "test_user@localhost:3306", # use of normal parser "test/slash", # slash # collision with login-path: "test_group_simple_login", # simple collision "c:\\some_config.cnf", # & no existing windows file "/etc_etc/some_config.cnf", # & no existing linux file "test-hyphen1234#", # another collision "test_default_group", # test default client group "temp_1.cnf", # collision & default group "temp_1.cnf[undeclared]", # undeclared group "temp_2.cnf", # collision & default group w/ssl "temp_2.cnf[missing_values]" # missing values ] test_n = 0 for test_ in con_tests: test_n += 1 msg = "Test case {0} - {1}".format(test_n, test_) if self.debug: print(msg) self.results.append("{0}\n".format(msg)) try: conn = parse_connection(test_) except UtilError as err: self.results.append("{0}\n".format(err.errmsg)) else: self.results.append("{0}\n".format(sorted(conn.iteritems()))) # Replacements self.replace_substring_portion("port or ", "socket", "port") self.replace_substring(".exe", "") return True
parser.error("You must specify either a source user or use the --list " "option. See --help for details.") # Check security settings check_password_security(opt, args) # Fail if dump and quiet set if opt.quiet and opt.dump: parser.error("You cannot use --quiet and --dump together.") # Warn if quiet and verbosity are both specified check_verbosity(opt) # Parse source connection values try: source_values = parse_connection(opt.source, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Source connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Source connection values invalid: %s." % err.errmsg) if opt.list_users and opt.destination: print(WARN_OPT_NOT_REQUIRED.format(opt='--destination', cmd="--list")) opt.destination = None if opt.dump and opt.destination: print(WARN_OPT_NOT_REQUIRED.format(opt='--destination', cmd="--dump")) opt.destination = None
def _server_info(server_val, get_defaults=False, options=None): """Show information about a running server This method gathers information from a running server. This information is returned as a tuple to be displayed to the user in a format specified. The information returned includes the following: * server connection information * version number of the server * data directory path * base directory path * plugin directory path * configuration file location and name * current binary log file * current binary log position * current relay log file * current relay log position server_val[in] the server connection values or a connected server get_defaults[in] if True, get the default settings for the server options[in] options for connecting to the server Return tuple - information about server """ if options is None: options = {} # Parse source connection values source_values = parse_connection(server_val, None, options) # Connect to the server conn_options = { 'version': "5.1.30", } servers = connect_servers(source_values, None, conn_options) server = servers[0] params_dict = defaultdict(str) # Initialize list of warnings params_dict['warnings'] = [] # Identify server by string: 'host:port[:socket]'. server_id = "{0}:{1}".format(source_values['host'], source_values['port']) if source_values.get('socket', None): server_id = "{0}:{1}".format(server_id, source_values.get('socket')) params_dict['server'] = server_id # Get _SERVER_VARIABLES values from the server for server_var in _SERVER_VARIABLES: res = server.show_server_variable(server_var) if res: params_dict[server_var] = res[0][1] else: raise UtilError("Unable to determine {0} of server '{1}'" ".".format(server_var, server_id)) # Get _LOG_FILES_VARIABLES values from the server for msg, log_tpl in _LOG_FILES_VARIABLES.iteritems(): res = server.show_server_variable(log_tpl.log_name) if res: # Check if log is turned off params_dict[log_tpl.log_name] = res[0][1] # If logs are turned off, skip checking information about the file if res[0][1] in ('', 'OFF'): continue # Logging is enabled, so we can get get information about log_file # unless it is log_error because in that case we already have it. if log_tpl.log_file is not None: # if it is not log_error log_file = server.show_server_variable( log_tpl.log_file)[0][1] params_dict[log_tpl.log_file] = log_file else: # log error, so log_file_name is already on params_dict log_file = params_dict[log_tpl.log_name] # Now get the information about the size of the logs try: params_dict[log_tpl.log_file_size] = "{0} bytes".format( os.path.getsize(log_file)) except os.error: # if we are unable to get the log_file_size params_dict[log_tpl.log_file_size] = '' warning_msg = _WARNING_TEMPLATE.format(msg, log_file) params_dict['warnings'].append(warning_msg) else: params_dict['warnings'].append("Unable to get information " "regarding variable '{0}'" ).format(msg) # if audit_log plugin is installed and enabled if server.supports_plugin('audit'): res = server.show_server_variable('audit_log_file') if res: # Audit_log variable might be a relative path to the datadir, # so it needs to be treated accordingly if not os.path.isabs(res[0][1]): params_dict['audit_log_file'] = os.path.join( params_dict['datadir'], res[0][1]) else: params_dict['audit_log_file'] = res[0][1] # Add audit_log field to the _COLUMNS List unless it is already # there if 'audit_log_file' not in _COLUMNS_SET: _COLUMNS.append('audit_log_file') _COLUMNS.append('audit_log_file_size') _COLUMNS_SET.add('audit_log_file') try: params_dict['audit_log_file_size'] = "{0} bytes".format( os.path.getsize(params_dict['audit_log_file'])) except os.error: # If we are unable to get the size of the audit_log_file params_dict['audit_log_file_size'] = '' warning_msg = _WARNING_TEMPLATE.format( "audit log", params_dict['audit_log_file'] ) params_dict['warnings'].append(warning_msg) # Build search path for config files if os.name == "posix": my_def_search = ["/etc/my.cnf", "/etc/mysql/my.cnf", os.path.join(params_dict['basedir'], "my.cnf"), "~/.my.cnf"] else: my_def_search = [r"c:\windows\my.ini", r"c:\my.ini", r"c:\my.cnf", os.path.join(os.curdir, "my.ini")] my_def_search.append(os.path.join(os.curdir, "my.cnf")) # Get server's default configuration values. defaults = [] if get_defaults: # Can only get defaults for local servers (need to access local data). if server.is_alias('localhost'): try: my_def_path = get_tool_path(params_dict['basedir'], "my_print_defaults") except UtilError as err: raise UtilError("Unable to retrieve the defaults data " "(requires access to my_print_defaults): {0} " "(basedir: {1})".format(err.errmsg, params_dict['basedir']) ) out_file = tempfile.TemporaryFile() # Execute tool: <basedir>/my_print_defaults mysqld subprocess.call([my_def_path, "mysqld"], stdout=out_file) out_file.seek(0) # Get defaults data from temp output file. defaults.append("\nDefaults for server {0}".format(server_id)) for line in out_file.readlines(): defaults.append(line.rstrip()) else: # Remote server; Cannot get the defaults data. defaults.append("\nWARNING: The utility can not get defaults from " "a remote host.") # Find config file config_file = "" for search_path in my_def_search: if os.path.exists(search_path): if len(config_file) > 0: config_file = "{0}, {1}".format(config_file, search_path) else: config_file = search_path params_dict['config_file'] = config_file # Find binary log, relay log params_dict['binary_log'], params_dict['binary_log_pos'] = _get_binlog( server) params_dict['relay_log'], params_dict['relay_log_pos'] = _get_relay_log( server) server.disconnect() return params_dict, defaults
default_val = "root@localhost:3306" print(WARN_OPT_USING_DEFAULT.format(default=default_val, opt="--master")) # Print the WARNING to force determinism if a parser error occurs. sys.stdout.flush() # option --slave is required (mandatory) if not opt.slave: parser.error(PARSE_ERR_OPTS_REQ.format(opt="--slave")) # option --rpl-user is required (mandatory) if not opt.rpl_user: parser.error(PARSE_ERR_OPTS_REQ.format(opt="--rpl-user")) # Parse source connection values try: m_values = parse_connection(opt.master, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Master connection values invalid: {0}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Master connection values invalid: {0}." "".format(err.errmsg)) # Parse source connection values try: s_values = parse_connection(opt.slave, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Slave connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info()
"debug": opt.verbosity >= 3, "new_engine": opt.new_engine, "def_engine": opt.def_engine, "skip_rpl": opt.skip_rpl, "skip_gtid": opt.skip_gtid, "table": opt.table, "charset": opt.charset, "multiprocess": num_cpu if opt.multiprocess == 0 else opt.multiprocess, "autocommit": opt.autocommit, "max_bulk_insert": max_bulk_size, } # Parse server connection values try: options.update(get_ssl_dict(opt)) server_values = parse_connection(opt.server, None, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: {0}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: " "{0}.".format(err.errmsg)) # Check values for --format=raw_csv if opt.format == "raw_csv": if not opt.table: print("ERROR: You must provide --table while using " "--format=raw_csv.") sys.exit(1) # Validate table name using format <db>.<table>
"rpl_mode": opt.rpl_mode, "verbosity": opt.verbosity, "skip_gtid": opt.skip_gtid, "charset": opt.charset, "multiprocess": num_cpu if opt.multiprocess == 0 else opt.multiprocess, } options.update(get_ssl_dict(opt)) # Parse source connection values try: # Create a basic configuration reader first for optimization purposes. # I.e., to avoid repeating the execution of some methods in further # parse_connection methods (like, searching my_print_defaults tool). config_reader = MyDefaultsReader(options, False) source_values = parse_connection(opt.source, config_reader, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Source connection values invalid: {0}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Source connection values invalid: " "{0}.".format(err.errmsg)) # Parse destination connection values try: dest_values = parse_connection(opt.destination, config_reader, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Destination connection values invalid: " "{0}.".format(err))
def get_diff(self, db_name, version): file_schema = "%s/%04d-schema.img" % (db_name, int(version)) file_schema_swp = "%s/%04d-schema.swp" % (db_name, int(version)) tmp_db = "tmp_online_mig_%s" % (db_name) query = "CREATE DATABASE %s;" % tmp_db self.server.disable_foreign_key_checks() self.server.exec_query(query) f = open(file_schema, 'r') f_swp = open(file_schema_swp, 'w') f_swp.write("USE %s\n" % tmp_db) buff = "" for line in f.readlines(): #if re.search(';$', line, re.IGNORECASE): if re.search(';$' + '(?i)', line): buff = buff + line f_swp.write(buff) buff = "" else: buff = buff + line.strip() f.close() f_swp.close() query_options = {'multi': True} f_swp = open(file_schema_swp, 'r') for line in f_swp.readlines(): self.server.exec_query(line, query_options) f_swp.close() os.remove(file_schema_swp) query_options = { 'run_all_tests': True, 'reverse': False, 'verbosity': None, 'no_object_check': False, 'no_data': True, 'quiet': True, 'difftype': 'differ', 'width': 75, 'changes-for': 'server1', 'skip_grants': True, 'skip_gtid': True } source_values = parse_connection(server_connection) destination_values = parse_connection(server_connection) with capture() as stepback: dbcompare.database_compare(source_values, destination_values, db_name, tmp_db, query_options) buf = "" found = 0 for line in stepback.getvalue().splitlines(True): logging.debug(u"%s" % line) if re.search('CREATE TABLE', line): line = re.sub("CREATE" + '(?i)', "", line, 1) #line = re.sub("CREATE","", line, 1, re.IGNORECASE) #line = re.sub("\(\n","", line, 1, re.IGNORECASE) line = re.sub("\(\n" + '(?i)', "", line, 1) buf = line if (re.search('^\+', line) or re.search('^\-', line)) and not re.search( 'CREATE DATABASE', line) and not re.search( 'CONSTRAINT.*FOREIGN KEY', line): #line = re.sub("\n","", line, 1, re.IGNORECASE) line = re.sub("\n" + '(?i)', "", line, 1) if len(buf) > 0: print u"%s" % buf buf = "" print u"%s" % line query = "DROP DATABASE %s" % tmp_db self.server.exec_query(query) self.server.disable_foreign_key_checks(disable=False)
def migrate_up(self, db_name, last_version): (ver, md5, comment) = self.read_meta(db_name, int(last_version)) if self.verify_checksum(db_name, last_version, md5) is False: logging.warning( "The current schema doesn't match the last applied migration") version = self.new_migration_version(db_name) if not os.path.exists("%s/%04d-up.mig" % (db_name, int(version))): logging.info("No migration available") else: logging.info(u"Preparing migration to version %04d" % int(version)) if os.path.exists("%s/%04d-down.mig" % (db_name, int(version))): os.remove("%s/%04d-down.mig" % (db_name, int(version))) (ver, md5, comment) = self.read_meta(db_name, int(version)) query_options = {'skip_data': True, 'force': True} db_list = [] grp = re.match("(\w+)(?:\:(\w+))?", "%s:%s_%s" % (db_name, self.tmp_prefix, db_name)) db_entry = grp.groups() db_list.append(db_entry) source_values = parse_connection(server_connection) destination_values = parse_connection(server_connection) with capture() as stepback: dbcopy.copy_db(source_values, destination_values, db_list, query_options) self.online_schema_change( db_name, version, "%s/%04d-up.mig" % (db_name, int(version))) if self.verify_checksum(db_name, version, md5) is True: logging.info("Applied changes match the requested schema") else: logging.error( "Something didn't run as expected, db schema doesn't match !" ) self.change_migration_status(db_name, version, 'invalid checksum') query_options = { 'run_all_tests': True, 'reverse': True, 'verbosity': None, 'no_object_check': False, 'no_data': True, 'quiet': True, 'difftype': 'sql', 'width': 75, 'changes-for': 'server1', 'skip_grants': True, 'skip_gtid': True } with capture() as stepback: res = dbcompare.database_compare( source_values, destination_values, db_name, "%s_%s" % (self.tmp_prefix, db_name), query_options) str = stepback.getvalue().splitlines(True) to_add = 0 file_down = open("%s/%04d-down.tmp" % (db_name, int(version)), 'a') for line in str: if line[0] not in ['#', '\n', '+', '-', '@']: # this if is required currently due to missing foreign keys in dbcopy if not re.match("\s+DROP FOREIGN KEY", line): #line = re.sub(" %s\." % db_name, " ", line, 1, re.IGNORECASE) line = re.sub(" %s\." % db_name + '(?i)', " ", line, 1) file_down.write("%s\n" % line.strip()) elif re.match("# WARNING: Objects in", line): if re.match("# WARNING: Objects in \w+\.tmp_online_mig_", line): to_add = 2 else: to_add = 1 else: grp = re.match("#\s+TABLE\: (\w+)", line) if grp: if to_add == 2: query = "SHOW CREATE TABLE tmp_online_mig_%s.%s;" % ( db_name, grp.group(1)) res = self.server.exec_query(query) file_down.write("%s\n" % res[0][1]) elif to_add == 1: file_down.write("DROP TABLE %s;\n" % grp.group(1)) file_down.close() file_down_tmp = "%s/%04d-down.tmp" % (db_name, int(version)) self.create_migration_file(db_name, file_down_tmp, version, "down") query = "DROP DATABASE %s_%s" % (self.tmp_prefix, db_name) res = self.server.exec_query(query) #os.remove(file_down_tmp) file_schema = "%s/%04d-schema.img" % (db_name, int(version)) self.create_schema_img(db_name, file_schema)
def run(self): # Test "--show" password in plain text option is available on this # version of my_print_defaults. test_num = 1 test_case = 'Test "--show" password in plain text option is available' comment = "Test case {0} {1}".format(test_num, test_case) my_defaults_reader = MyDefaultsReader() if not my_defaults_reader.check_show_required(): raise MUTLibError("{0}: failed".format(comment)) else: if self.debug: print("{0}: pass".format(comment)) self.results.append("{0}: pass\n".format(comment)) # Test retrieve of expected values for passwords variables stored in # test.mylogin.cnf con_tests = { "test1": "user_1:pass_1@localhost:10001", "test2": "user_2:A_passw0rd@localhost:20002", "test3": "user_3:M4g1cw0rd@localhost:30003", "test4": "user_4:-*123 !%^@remotehost:40004", } for group in sorted(con_tests.keys()): test_num += 1 test_case = 'Test Retrieve group: "{0}"'.format(group) comment = "Test case {0} {1}".format(test_num, test_case) self.results.append("{0}\n".format(comment)) if self.debug: print(comment) # Retrieve stored values in group from .mylogin.cnf ret_dt = my_defaults_reader.get_group_data(group) # If not values, group was not found if ret_dt is None: if self.debug: print('Can not retrieve values for group: "{}"' ''.format(group)) raise MUTLibError("{0}: failed".format(comment)) con_dic = parse_connection(con_tests[group], options={"charset": "utf8"}) con_dic.pop("charset") self.results.append("Retrieved data:\n") if self.debug: print("Retrieved data:") for data in sorted(ret_dt.iteritems()): # Only password is saved as passwd key if 'passw' in data[0]: if con_dic['passwd'] != data[1]: if self.debug: print( _VALUES_DIFFER_MSG.format( "password", con_dic['passwd'], data[1])) raise MUTLibError("{0}: failed".format(comment)) else: msg = _VALUES_EQUAL_MSG.format("password", con_dic['passwd'], data[1]) if self.debug: print(msg) self.results.append("{0}\n".format(msg)) elif str(con_dic[data[0]]) != data[1]: if self.debug: print( _VALUES_DIFFER_MSG.format(data[0], con_dic[data[0]], data[1])) raise MUTLibError("{0}: failed".format(comment)) else: msg = _VALUES_EQUAL_MSG.format(data[0], con_dic[data[0]], data[1]) if self.debug: print(msg) self.results.append("{0}\n".format(msg)) # None check failed, mark as pass self.results.append("Test result: pass\n") if self.debug: print("Result: pass") return True
add_verbosity(parser, False) # Now we process the rest of the arguments. opt, args = parser.parse_args() # Check security settings check_password_security(opt, args) # Check to make sure at least one table specified. if len(args) == 0: parser.error("You must specify at least one table or database to " "check.") # Parse source connection values try: conn_options = get_ssl_dict(opt) source_values = parse_connection(opt.server, options=conn_options) except FormatError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: %s." % err.errmsg) # Check best, worst for validity best = None if opt.best is not None: try: best = int(opt.best) except ValueError: best = -1 if best is not None and best < 1:
"rpl_mode": opt.rpl_mode, "verbosity": opt.verbosity, "skip_gtid": opt.skip_gtid, "charset": opt.charset, "multiprocess": num_cpu if opt.multiprocess == 0 else opt.multiprocess, } options.update(get_ssl_dict(opt)) # Parse source connection values try: # Create a basic configuration reader first for optimization purposes. # I.e., to avoid repeating the execution of some methods in further # parse_connection methods (like, searching my_print_defaults tool). config_reader = MyDefaultsReader(options, False) source_values = parse_connection(opt.source, config_reader, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Source connection values invalid: {0}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Source connection values invalid: " "{0}.".format(err.errmsg)) # Parse destination connection values try: dest_values = parse_connection(opt.destination, config_reader, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Destination connection values invalid: " "{0}.".format(err)) except UtilError: _, err, _ = sys.exc_info()
for slave_val in slaves_val: if check_hostname_alias(master_val, slave_val): err = ERROR_MASTER_IN_SLAVES.format( master_host=master_val['host'], master_port=master_val['port'], slaves_candidates="slaves", slave_host=slave_val['host'], slave_port=slave_val['port'], ) parser.error(err) # Parse source connection values of --server if opt.server: try: ssl_opts = get_ssl_dict(opt) server_val = parse_connection(opt.server, None, ssl_opts) except FormatError as err: parser.error("ERROR: {0}\n".format(err.errmsg)) except UtilError as err: parser.error("ERROR: {0}\n".format(err.errmsg)) # Create dictionary of options options = { 'discover': opt.discover, 'verbosity': 0 if opt.verbosity is None else opt.verbosity, 'to_binlog_name': opt.binlog, 'dry_run': opt.dry_run, } try: binlog_purge(server_val, master_val, slaves_val, options)
server_list = ServerList([], opt.start_port, opt.utildir, opt.verbosity) basedir = None # Print status of connections print("\nServers:") if not opt.servers: print(" No servers specified.") else: i = 0 for server in opt.servers: # add ssl options values. conn_options = {} conn_options.update(get_ssl_dict(opt)) try: conn_val = parse_connection(server, options=conn_options) except UtilError as err: parser.error(err.errmsg) except: parser.error("Problem parsing server connection " "'{0}'".format(server)) i += 1 # Fail if port is None if conn_val["port"] is None: parser.error("You must specify a port in the server " "string: \n {0}".format(server)) sys.stdout.write(" Connecting to {0} as user {1} on port " "{2}: ".format(conn_val["host"], conn_val["user"], conn_val["port"]))
"locking": opt.locking, "rpl_user": opt.rpl_user, "rpl_mode": opt.rpl_mode, "rpl_file": opt.rpl_file, "comment_rpl": opt.comment_rpl, "export": opt.export, "skip_gtid": opt.skip_gtid, "charset": opt.charset, "multiprocess": num_cpu if opt.multiprocess == 0 else opt.multiprocess, "output_filename": output_filename, } # Parse server connection values try: options.update(get_ssl_dict(opt)) server_values = parse_connection(opt.server, None, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: {0}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: " "{0}.".format(err.errmsg)) # Build list of databases to copy db_list = [] for db in args: # Remove backtick quotes (handled later) db = remove_backtick_quoting(db) \ if is_quoted_with_backticks(db) else db db_list.append(db)
# Add verbosity add_verbosity(parser, quiet=False) # Now we process the rest of the arguments. opt, args = parser.parse_args() # Check mandatory options: --server if not opt.server: parser.error(PARSE_ERR_OPTS_REQ.format(opt="--server")) servers_val = None # Parse source connection values if --server provided or default try: ssl_opts = get_ssl_dict(opt) server_val = parse_connection(opt.server, None, ssl_opts) except FormatError as err: # pylint: disable=E1101 parser.error("ERROR: {0}\n".format(err.errmsg)) except UtilError as err: parser.error("ERROR: {0}\n".format(err.errmsg)) # Create dictionary of options options = { 'verbosity': 0 if opt.verbosity is None else opt.verbosity, "min_size": opt.min_size, } try: binlog_rotate(server_val, options) except UtilError:
cmd=command, only_cmd=only_used_cmds)) opt.candidates = None # --format only used by 'health' or 'gtid' command if opt.format and command not in ['health', 'gtid']: only_used_cmds = 'health or gtid' print(WARN_OPT_NOT_REQUIRED_ONLY_FOR.format(opt='--format', cmd=command, only_cmd=only_used_cmds)) opt.format = None # Parse the --new-master connection string if opt.new_master: try: new_master_val = parse_connection(opt.new_master, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("New master connection values invalid: " "{0}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("New master connection values invalid: " "{0}.".format(err.errmsg)) else: new_master_val = None # Parse the master, slaves, and candidates connection parameters try: master_val, slaves_val, candidates_val = parse_topology_connections( opt)
parser.add_option("-a", "--all", action="store_true", dest="do_all", default=False, help="show all usage including empty " "databases") # Add verbosity mode add_verbosity(parser, True) # Now we process the rest of the arguments. opt, args = parser.parse_args() # Check security settings check_password_security(opt, args) # Parse source connection values try: source_values = parse_connection(opt.server, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Source connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Source connection values invalid: %s." % err.errmsg) try: conn_options = { 'version': "5.1.30", } servers = connect_servers(source_values, None) except UtilError: _, e, _ = sys.exc_info() parser.error(e.errmsg)
def show_server_info(servers, options): """Show server information for a list of servers This method will gather information about a running server. If the show_defaults option is specified, the method will also read the configuration file and return a list of the server default settings. If the format option is set, the output will be in the format specified. If the no_headers option is set, the output will not have a header row (no column names) except for format = vertical. If the basedir and start options are set, the method will attempt to start the server in read only mode to get the information. Specifying only basedir will not start the server. The extra start option is designed to make sure the user wants to start the offline server. The user may not wish to do this if there are certain error conditions and/or logs in place that may be overwritten. servers[in] list of server connections in the form <user>:<password>@<host>:<port>:<socket> options[in] dictionary of options (no_headers, format, basedir, start, show_defaults) Returns tuple ((server information), defaults) """ no_headers = options.get("no_headers", False) fmt = options.get("format", "grid") show_defaults = options.get("show_defaults", False) basedir = options.get("basedir", None) datadir = options.get("datadir", None) start = options.get("start", False) show_servers = options.get("show_servers", 0) if show_servers: if os.name == 'nt': ports = options.get("ports", "3306:3333") start_p, end_p = ports.split(":") _show_running_servers(start_p, end_p) else: _show_running_servers() ssl_dict = {} ssl_dict['ssl_cert'] = options.get("ssl_cert", None) ssl_dict['ssl_ca'] = options.get("ssl_ca", None) ssl_dict['ssl_key'] = options.get("ssl_key", None) ssl_dict['ssl'] = options.get("ssl", None) row_dict_lst = [] warnings = [] server_val = {} for server in servers: new_server = None try: test_connect(server, throw_errors=True, ssl_dict=ssl_dict) except UtilError as util_error: conn_dict = get_connection_dictionary(server, ssl_dict=ssl_dict) server1 = Server(options={'conn_info': conn_dict}) server_is_off = False # If we got errno 2002 it means can not connect through the # given socket. if util_error.errno == CR_CONNECTION_ERROR: socket = conn_dict.get("unix_socket", "") if socket: msg = ("Unable to connect to server using socket " "'{0}'.".format(socket)) if os.path.isfile(socket): err_msg = ["{0} Socket file is not valid.".format(msg)] else: err_msg = [ "{0} Socket file does not " "exist.".format(msg) ] # If we got errno 2003 and we do not have # socket, instead we check if server is localhost. elif (util_error.errno == CR_CONN_HOST_ERROR and server1.is_alias("localhost")): server_is_off = True # If we got errno 1045 it means Access denied, # notify the user if a password was used or not. elif util_error.errno == ER_ACCESS_DENIED_ERROR: use_pass = '******' if conn_dict['passwd'] else 'NO' err_msg = ("Access denied for user '{0}'@'{1}' using " "password: {2}".format(conn_dict['user'], conn_dict['host'], use_pass)) # Use the error message from the connection attempt. else: err_msg = [util_error.errmsg] # To propose to start a cloned server for extract the info, # can not predict if the server is really off, but we can do it # in case of socket error, or if one of the related # parameter was given. if server_is_off or basedir or datadir or start: err_msg = [ "Server is offline. To connect, " "you must also provide " ] opts = ["basedir", "datadir", "start"] for opt in tuple(opts): try: if locals()[opt] is not None: opts.remove(opt) except KeyError: pass if opts: err_msg.append(", ".join(opts[0:-1])) if len(opts) > 1: err_msg.append(" and the ") err_msg.append(opts[-1]) err_msg.append(" option") raise UtilError("".join(err_msg)) if not start: raise UtilError("".join(err_msg)) else: try: server_val = parse_connection(server, None, options) except: raise UtilError("Source connection values invalid" " or cannot be parsed.") new_server = _start_server(server_val, basedir, datadir, options) info_dict, defaults = _server_info(server, show_defaults, options) warnings.extend(info_dict['warnings']) if info_dict: row_dict_lst.append(info_dict) if new_server: # Need to stop the server! new_server.disconnect() _stop_server(server_val, basedir, options) # Get the row values stored in the dictionaries rows = [[row_dict[key] for key in _COLUMNS] for row_dict in row_dict_lst] print_list(sys.stdout, fmt, _COLUMNS, rows, no_headers) if warnings: print("\n# List of Warnings: \n") for warning in warnings: print("WARNING: {0}\n".format(warning)) # Print the default configurations. if show_defaults and len(defaults) > 0: for row in defaults: print(" {0}".format(row))
server = None if opt.server is None and opt.diagnostic: print("# WARNING: Cannot generate character set or " "collation names without the --server option.") # Check start timeout for minimal value if int(opt.start_timeout) < 10: opt.start_timeout = 10 print("# WARNING: --start-timeout must be >= 10 seconds. Using " "default value.") # Parse source connection values if --server provided if opt.server is not None and not opt.basedir: try: ssl_opts = get_ssl_dict(opt) source_values = parse_connection(opt.server, None, ssl_opts) except FormatError as err: parser.error("Source connection values invalid: %s." % err) except UtilError as err: parser.error("Source connection values invalid: %s." % err.errmsg) try: conn_options = { 'version': "5.1.30", 'quiet': opt.quiet, } servers = connect_servers(source_values, None, conn_options) except UtilError as error: parser.error(error.errmsg) server = servers[0]
# option --master is required (mandatory) if not opt.master: default_val = 'root@localhost:3306' print( WARN_OPT_USING_DEFAULT.format(default=default_val, opt='--master')) # Print the WARNING to force determinism if a parser error occurs. sys.stdout.flush() # option --slave is required (mandatory) if not opt.slave: parser.error(PARSE_ERR_OPTS_REQ.format(opt='--slave')) # Parse source connection values try: m_values = parse_connection(opt.master, options=opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Master connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Master connection values invalid: %s." % err.errmsg) # Parse source connection values try: s_values = parse_connection(opt.slave, options=opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Slave connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info()
if not opt.slave: parser.error(PARSE_ERR_OPTS_REQ.format(opt="--slave")) # option --masters is required (mandatory) if not opt.masters: parser.error(PARSE_ERR_OPTS_REQ.format(opt="--masters")) # option --rpl-user is required (mandatory) if not opt.rpl_user: parser.error(PARSE_ERR_OPTS_REQ.format(opt="--rpl-user")) config_reader = MyDefaultsReader(opt, False) # Parse slave connection values try: slave_vals = parse_connection(opt.slave, config_reader, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Slave connection values invalid: {0}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Slave connection values invalid: {0}." "".format(err.errmsg)) # Parse masters connection values masters_vals = [] masters = opt.masters.split(",") if len(masters) == 1: parser.error("At least two masters are required for multi-source " "replication.")
"difftype": opt.difftype, "force": opt.force, "width": opt.width, "changes-for": opt.changes_for, "reverse": opt.reverse, "skip_table_opts": opt.skip_tbl_opts, "compact": opt.compact, "charset": opt.charset, } # add ssl options values. options.update(get_ssl_dict(opt)) # Parse server connection values try: server1_values = parse_connection(opt.server1, None, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Server1 connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Server1 connection values invalid: %s." % err.errmsg) if opt.server2 is not None: try: server2_values = parse_connection(opt.server2, None, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Server2 connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Server2 connection values invalid: %s." % err.errmsg)
"by databases and objects inside those databases. " "Finally, if GLOBAL level is specified, normal " "inheritance rules are applied, global grants " "apply to both databases and objects and database " "level grants apply to the objects.") # Now we process the rest of the arguments. opt, args = parser.parse_args() if not opt.server: parser.error("You need to specify a server using the --server option.") # Parse server connection server_val = None try: server_val = parse_connection(opt.server, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: " "{0!s}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: " "{0!s}.".format(err.errmsg)) # The --show=users can only be used together with the privilege option if opt.show_mode == 'users' and not opt.privileges: parser.error("The --show=users can only be used if you specify a " "list of privileges with the --privileges option.") conn_opts = { 'quiet': True,
# Check mandatory option: --server. if not opt.server: parser.error(PARSE_ERR_OPTS_REQ.format(opt="--server")) # Check security settings check_password_security(opt, args) # Check to make sure at least one table specified. if len(args) == 0: parser.error("You must specify at least one table or database to " "check.") # Parse source connection values try: conn_options = get_ssl_dict(opt) source_values = parse_connection(opt.server, options=conn_options) except FormatError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: %s." % err.errmsg) # Check best, worst for validity best = None if opt.best is not None: try: best = int(opt.best) except ValueError: best = -1 if best is not None and best < 1:
# Expand user paths and resolve relative paths if opt.new_data and opt.new_data[0] == '~': options['new_data'] = os.path.expanduser(opt.new_data) if opt.basedir and opt.basedir[0] == '~': options['basedir'] = os.path.expanduser(opt.basedir) if opt.new_data and opt.new_data[0] == '.': options['new_data'] = os.path.abspath(opt.new_data) if opt.basedir and opt.basedir[0] == '.': options['basedir'] = os.path.abspath(opt.basedir) # Parse source connection values if we have a running server if opt.basedir is None: conn_options = get_ssl_dict(opt) try: conn = parse_connection(opt.server, options=conn_options) # Now check for local server server = Server({'conn_info': conn}) if not server.is_alias('localhost'): parser.error("Server to be cloned must be running on the same " "machine as mysqlserverclone.") except exception.FormatError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: %s." % err) except exception.UtilError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: %s." % err.errmsg) else: conn = None
def show_server_info(servers, options): """Show server information for a list of servers This method will gather information about a running server. If the show_defaults option is specified, the method will also read the configuration file and return a list of the server default settings. If the format option is set, the output will be in the format specified. If the no_headers option is set, the output will not have a header row (no column names) except for format = vertical. If the basedir and start options are set, the method will attempt to start the server in read only mode to get the information. Specifying only basedir will not start the server. The extra start option is designed to make sure the user wants to start the offline server. The user may not wish to do this if there are certain error conditions and/or logs in place that may be overwritten. servers[in] list of server connections in the form <user>:<password>@<host>:<port>:<socket> options[in] dictionary of options (no_headers, format, basedir, start, show_defaults) Returns tuple ((server information), defaults) """ no_headers = options.get("no_headers", False) fmt = options.get("format", "grid") show_defaults = options.get("show_defaults", False) basedir = options.get("basedir", None) datadir = options.get("datadir", None) start = options.get("start", False) show_servers = options.get("show_servers", 0) if show_servers: if os.name == 'nt': ports = options.get("ports", "3306:3333") start_p, end_p = ports.split(":") _show_running_servers(start_p, end_p) else: _show_running_servers() row_dict_lst = [] warnings = [] server_val = {} for server in servers: new_server = None try: test_connect(server, True) except UtilError as util_error: conn_dict = get_connection_dictionary(server) server1 = Server(options={'conn_info': conn_dict}) server_is_off = False # If we got errno 2002 it means can not connect through the # given socket, but if path to socket not empty, server could be # turned off. if util_error.errno == CR_CONNECTION_ERROR: socket = conn_dict.get("unix_socket", "") if socket: mydir = os.path.split(socket)[0] if os.path.isdir(mydir) and len(os.listdir(mydir)) != 0: server_is_off = True # If we got errno 2003 and this is a windows, we do not have # socket, instead we check if server is localhost. elif (util_error.errno == CR_CONN_HOST_ERROR and os.name == 'nt' and server1.is_alias("localhost")): server_is_off = True # If we got errno 1045 it means Access denied, # notify the user if a password was used or not. elif util_error.errno == ER_ACCESS_DENIED_ERROR: use_pass = '******' if conn_dict['passwd'] else 'NO' er = ("Access denied for user '{0}'@'{1}' using password: {2}" ).format(conn_dict['user'], conn_dict['host'], use_pass) # Use the error message from the connection attempt. else: er = [util_error.errmsg] # To propose to start a cloned server for extract the info, # can not predict if the server is really off, but we can do it # in case of socket error, or if one of the related # parameter was given. if (server_is_off or basedir or datadir or start): er = ["Server is offline. To connect, " "you must also provide "] opts = ["basedir", "datadir", "start"] for opt in tuple(opts): try: if locals()[opt] is not None: opts.remove(opt) except KeyError: pass if opts: er.append(", ".join(opts[0:-1])) if len(opts) > 1: er.append(" and the ") er.append(opts[-1]) er.append(" option") raise UtilError("".join(er)) if not start: raise UtilError("".join(er)) else: try: server_val = parse_connection(server, None, options) except: raise UtilError("Source connection values invalid" " or cannot be parsed.") new_server = _start_server(server_val, basedir, datadir, options) info_dict, defaults = _server_info(server, show_defaults, options) warnings.extend(info_dict['warnings']) if info_dict: row_dict_lst.append(info_dict) if new_server: # Need to stop the server! new_server.disconnect() _stop_server(server_val, basedir, options) # Get the row values stored in the dictionaries rows = [[row_dict[key] for key in _COLUMNS] for row_dict in row_dict_lst] print_list(sys.stdout, fmt, _COLUMNS, rows, no_headers) if warnings: print("\n# List of Warnings: \n") for warning in warnings: print("WARNING: {0}\n".format(warning)) # Print the default configurations. if show_defaults and len(defaults) > 0: for row in defaults: print(" {0}".format(row))
# option --master is required (mandatory) if not opt.master: default_val = 'root@localhost:3306' print(WARN_OPT_USING_DEFAULT.format(default=default_val, opt='--master')) # Print the WARNING to force determinism if a parser error occurs. sys.stdout.flush() # option --slave is required (mandatory) if not opt.slave: parser.error(PARSE_ERR_OPTS_REQ.format(opt='--slave')) # Parse source connection values try: m_values = parse_connection(opt.master, options=opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Master connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Master connection values invalid: %s." % err.errmsg) # Parse source connection values try: s_values = parse_connection(opt.slave, options=opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Slave connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info()
"for any given object. Unless a user has all the " "privileges listed for a specific object, " "she will not appear in the list of users with " "privileges for that specific object. To list " "multiple privileges, use a comma-separated list.") # Now we process the rest of the arguments. opt, args = parser.parse_args() if not opt.server: parser.error("You need to specify a server using the --server option.") # Parse server connection server_val = None try: server_val = parse_connection(opt.server, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: " "{0!s}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Server connection values invalid: " "{0!s}.".format(err.errmsg)) # The --show=users can only be used together with the privilege option if opt.show_mode == 'users' and not opt.privileges: parser.error("The --show=users can only be used if you specify a " "list of privileges with the --privileges option.") # Process list objects for which grants will be shown
def _server_info(server_val, get_defaults=False, options=None): """Show information about a running server This method gathers information from a running server. This information is returned as a tuple to be displayed to the user in a format specified. The information returned includes the following: * server connection information * version number of the server * data directory path * base directory path * plugin directory path * configuration file location and name * current binary log file * current binary log position * current relay log file * current relay log position server_val[in] the server connection values or a connected server get_defaults[in] if True, get the default settings for the server options[in] options for connecting to the server Return tuple - information about server """ if options is None: options = {} # Parse source connection values source_values = parse_connection(server_val, None, options) # Connect to the server conn_options = { 'version': "5.1.30", } servers = connect_servers(source_values, None, conn_options) server = servers[0] params_dict = defaultdict(str) # Initialize list of warnings params_dict['warnings'] = [] # Identify server by string: 'host:port[:socket]'. server_id = "{0}:{1}".format(source_values['host'], source_values['port']) if source_values.get('socket', None): server_id = "{0}:{1}".format(server_id, source_values.get('socket')) params_dict['server'] = server_id # Get _SERVER_VARIABLES values from the server for server_var in _SERVER_VARIABLES: res = server.show_server_variable(server_var) if res: params_dict[server_var] = res[0][1] else: raise UtilError("Unable to determine {0} of server '{1}'" ".".format(server_var, server_id)) # Get _LOG_FILES_VARIABLES values from the server for msg, log_tpl in _LOG_FILES_VARIABLES.iteritems(): res = server.show_server_variable(log_tpl.log_name) if res: # Check if log is turned off params_dict[log_tpl.log_name] = res[0][1] # If logs are turned off, skip checking information about the file if res[0][1] in ('', 'OFF'): continue # Logging is enabled, so we can get get information about log_file # unless it is log_error because in that case we already have it. if log_tpl.log_file is not None: # if it is not log_error log_file = server.show_server_variable(log_tpl.log_file)[0][1] params_dict[log_tpl.log_file] = log_file else: # log error, so log_file_name is already on params_dict log_file = params_dict[log_tpl.log_name] # Now get the information about the size of the logs # If log file is stderr, we cannot get the correct size. if log_file not in ["stderr", "stdout"]: # Now get the information about the size of the logs try: params_dict[log_tpl.log_file_size] = "{0} bytes".format( os.path.getsize(log_file)) except os.error: # if we are unable to get the log_file_size params_dict[log_tpl.log_file_size] = '' warning_msg = _WARNING_TEMPLATE.format(msg, log_file) params_dict['warnings'].append(warning_msg) else: params_dict['warnings'].append( "Unable to get information " "regarding variable '{0}'").format(msg) # if audit_log plugin is installed and enabled if server.supports_plugin('audit'): res = server.show_server_variable('audit_log_file') if res: # Audit_log variable might be a relative path to the datadir, # so it needs to be treated accordingly if not os.path.isabs(res[0][1]): params_dict['audit_log_file'] = os.path.join( params_dict['datadir'], res[0][1]) else: params_dict['audit_log_file'] = res[0][1] # Add audit_log field to the _COLUMNS List unless it is already # there if 'audit_log_file' not in _COLUMNS_SET: _COLUMNS.append('audit_log_file') _COLUMNS.append('audit_log_file_size') _COLUMNS_SET.add('audit_log_file') try: params_dict['audit_log_file_size'] = "{0} bytes".format( os.path.getsize(params_dict['audit_log_file'])) except os.error: # If we are unable to get the size of the audit_log_file params_dict['audit_log_file_size'] = '' warning_msg = _WARNING_TEMPLATE.format( "audit log", params_dict['audit_log_file']) params_dict['warnings'].append(warning_msg) # Build search path for config files if os.name == "posix": my_def_search = [ "/etc/my.cnf", "/etc/mysql/my.cnf", os.path.join(params_dict['basedir'], "my.cnf"), "~/.my.cnf" ] else: my_def_search = [ r"c:\windows\my.ini", r"c:\my.ini", r"c:\my.cnf", os.path.join(os.curdir, "my.ini") ] my_def_search.append(os.path.join(os.curdir, "my.cnf")) # Get server's default configuration values. defaults = [] if get_defaults: # Can only get defaults for local servers (need to access local data). if server.is_alias('localhost'): try: my_def_path = get_tool_path(params_dict['basedir'], "my_print_defaults", quote=True) except UtilError as err: raise UtilError("Unable to retrieve the defaults data " "(requires access to my_print_defaults): {0} " "(basedir: {1})".format( err.errmsg, params_dict['basedir'])) out_file = tempfile.TemporaryFile() # Execute tool: <basedir>/my_print_defaults mysqld cmd_list = shlex.split(my_def_path) cmd_list.append("mysqld") subprocess.call(cmd_list, stdout=out_file) out_file.seek(0) # Get defaults data from temp output file. defaults.append("\nDefaults for server {0}".format(server_id)) for line in out_file.readlines(): defaults.append(line.rstrip()) else: # Remote server; Cannot get the defaults data. defaults.append("\nWARNING: The utility can not get defaults from " "a remote host.") # Find config file config_file = "" for search_path in my_def_search: if os.path.exists(search_path): if len(config_file) > 0: config_file = "{0}, {1}".format(config_file, search_path) else: config_file = search_path params_dict['config_file'] = config_file # Find binary log, relay log params_dict['binary_log'], params_dict['binary_log_pos'] = _get_binlog( server) params_dict['relay_log'], params_dict['relay_log_pos'] = _get_relay_log( server) server.disconnect() return params_dict, defaults
def migrate_up(self, db_name, last_version): (ver, md5, comment) = self.read_meta(db_name, int(last_version)) if self.verify_checksum(db_name, last_version, md5) is False: logging.warning("The current schema doesn't match the last applied migration") version = self.new_migration_version(db_name) if not os.path.exists("%s/%04d-up.mig" % (db_name, int(version))): logging.info("No migration available") else: logging.info(u"Preparing migration to version %04d" % int(version)) if os.path.exists("%s/%04d-down.mig" % (db_name, int(version))): os.remove("%s/%04d-down.mig" % (db_name, int(version))) (ver, md5, comment) = self.read_meta(db_name, int(version)) query_options = {'skip_data': True, 'force': True} db_list = [] grp = re.match("(\w+)(?:\:(\w+))?", "%s:%s_%s" % (db_name, self.tmp_prefix, db_name)) db_entry = grp.groups() db_list.append(db_entry) source_values = parse_connection(server_connection) destination_values = parse_connection(server_connection) with capture() as stepback: dbcopy.copy_db(source_values, destination_values, db_list, query_options) self.online_schema_change(db_name, version, "%s/%04d-up.mig" % (db_name, int(version))) if self.verify_checksum(db_name, version, md5) is True: logging.info("Applied changes match the requested schema") else: logging.error("Something didn't run as expected, db schema doesn't match !") self.change_migration_status(db_name, version, 'invalid checksum') query_options = { 'run_all_tests': True, 'reverse': True, 'verbosity': None, 'no_object_check': False, 'no_data': True, 'quiet': True, 'difftype': 'sql', 'width': 75, 'changes-for': 'server1', 'skip_grants': True} with capture() as stepback: res = dbcompare.database_compare(source_values, destination_values, db_name, "%s_%s" % (self.tmp_prefix, db_name), query_options) str = stepback.getvalue().splitlines(True) to_add = 0 file_down = open("%s/%04d-down.tmp" % (db_name, int(version)), 'a') for line in str: if line[0] not in ['#', '\n', '+', '-', '@']: # this if is required currently due to missing foreign keys in dbcopy if not re.match("\s+DROP FOREIGN KEY", line): #line = re.sub(" %s\." % db_name, " ", line, 1, re.IGNORECASE) line = re.sub(" %s\." % db_name + '(?i)', " ", line, 1) file_down.write("%s\n" % line.strip()) elif re.match("# WARNING: Objects in", line): if re.match("# WARNING: Objects in \w+\.tmp_online_mig_", line): to_add = 2 else: to_add = 1 else: grp = re.match("#\s+TABLE\: (\w+)", line) if grp: if to_add == 2: query = "SHOW CREATE TABLE tmp_online_mig_%s.%s;" % (db_name, grp.group(1)) res = self.server.exec_query(query) file_down.write("%s\n" % res[0][1]) elif to_add == 1: file_down.write("DROP TABLE %s;\n" % grp.group(1)) file_down.close() file_down_tmp = "%s/%04d-down.tmp" % (db_name, int(version)) self.create_migration_file(db_name, file_down_tmp, version, "down") query = "DROP DATABASE %s_%s" % (self.tmp_prefix, db_name) res = self.server.exec_query(query) #os.remove(file_down_tmp) file_schema = "%s/%04d-schema.img" % (db_name, int(version)) self.create_schema_img(db_name, file_schema)
print( WARN_OPT_USING_DEFAULT.format(default=default_val, opt='--master')) # Print the WARNING to force determinism if a parser error occurs. sys.stdout.flush() # option --slave is required (mandatory) if not opt.slave: parser.error(PARSE_ERR_OPTS_REQ.format(opt='--slave')) # option --rpl-user is required (mandatory) if not opt.rpl_user: parser.error(PARSE_ERR_OPTS_REQ.format(opt="--rpl-user")) # Parse source connection values try: m_values = parse_connection(opt.master, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Master connection values invalid: {0}.".format(err)) except UtilError: _, err, _ = sys.exc_info() parser.error("Master connection values invalid: {0}." "".format(err.errmsg)) # Parse source connection values try: s_values = parse_connection(opt.slave, None, opt) except FormatError: _, err, _ = sys.exc_info() parser.error("Slave connection values invalid: %s." % err) except UtilError:
} # Add ssl options to options instead of connection. options.update(get_ssl_dict(opt)) # Check server options. if not opt.server1: parser.error(PARSE_ERR_OPTS_REQ.format(opt='--server1')) if opt.all and not opt.server2: parser.error(PARSE_ERR_OPTS_REQ.format(opt='--server2')) # Parse server connection values server1_values = None server2_values = None try: server1_values = parse_connection(opt.server1, None, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Server1 connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info() parser.error("Server1 connection values invalid: %s." % err.errmsg) if opt.server2: try: server2_values = parse_connection(opt.server2, None, options) except FormatError: _, err, _ = sys.exc_info() parser.error("Server2 connection values invalid: %s." % err) except UtilError: _, err, _ = sys.exc_info()