def _format_health_data(self): """Return health data from topology. Returns tuple - (columns, rows). """ if self.topology: try: health_data = self.topology.get_health() current_master = self._get_master() # Get data for the remaining masters for master_vals in self.masters_vals: # Discard the current master if master_vals["host"] == current_master.host and \ master_vals["port"] == current_master.port: continue # Connect to the master conn_dict = { "conn_info": master_vals, "quiet": True, "verbose": self.verbosity > 0, } master = Master(conn_dict) master.connect() # Get master health rpl_health = master.check_rpl_health() master_data = [ master.host, master.port, "MASTER", get_server_state(master, master.host, 3, self.verbosity > 0), master.supports_gtid(), "OK" if rpl_health[0] else ", ".join(rpl_health[1]), ] # Get master status master_status = master.get_status() if len(master_status): master_log, master_log_pos = master_status[0][0:2] else: master_log = None master_log_pos = 0 # Show additional details if verbosity is turned on if self.verbosity > 0: master_data.extend([ master.get_version(), master_log, master_log_pos, "", "", "", "", "", "", "", "", "" ]) health_data[1].append(master_data) return health_data except UtilError as err: msg = "Cannot get health data: {0}".format(err) self._report(msg, logging.ERROR, False) raise UtilRplError(msg) return ([], [])
def setup(self): self.res_fname = "result.txt" result = replicate.test.setup(self) # Note: server1 is master, server2, server3 are slaves. # server3 is a new slave with nothing on it. self.server3 = self.servers.spawn_server( "new_slave", kill=True, mysqld='"--log-bin=mysql-bin"') self._drop_all() self.server1.exec_query("STOP SLAVE") self.server1.exec_query("RESET SLAVE") self.server2.exec_query("STOP SLAVE") self.server2.exec_query("RESET SLAVE") try: for cmd in _MASTER_DB_CMDS: self.server1.exec_query(cmd) except MUTLibError: raise data_file = os.path.normpath("./std_data/basic_data.sql") try: self.server1.read_and_exec_SQL(data_file, self.debug) self.server2.read_and_exec_SQL(data_file, self.debug) except MUTLibError as err: raise MUTLibError("Failed to read commands from file {0}: " "{1}".format(data_file, err.errmsg)) master_str = "--master={0}".format( self.build_connection_string(self.server1)) slave_str = " --slave={0}".format( self.build_connection_string(self.server2)) conn_str = master_str + slave_str cmd = "mysqlreplicate.py --rpl-user=rpl:rpl {0}".format(conn_str) try: self.exec_util(cmd, self.res_fname) except MUTLibError: raise # server1 is now a master server, lets treat it accordingly self.server1 = Master.fromServer(self.server1) try: self.server1.connect() except UtilError as err: raise MUTLibError("Cannot connect to spawned " "server: {0}".format(err.errmsg)) # server2 is now a slave, lets treat it accordingly self.server2 = Slave.fromServer(self.server2) try: self.server2.connect() except UtilError as err: raise MUTLibError("Cannot connect to spawned " "server: {0}".format(err.errmsg)) return result
def _format_health_data(self): """Return health data from topology. Returns tuple - (columns, rows). """ if self.topology: try: health_data = self.topology.get_health() current_master = self._get_master() # Get data for the remaining masters for master_vals in self.masters_vals: # Discard the current master if master_vals["host"] == current_master.host and \ master_vals["port"] == current_master.port: continue # Connect to the master conn_dict = { "conn_info": master_vals, "quiet": True, "verbose": self.verbosity > 0, } master = Master(conn_dict) master.connect() # Get master health rpl_health = master.check_rpl_health() master_data = [ master.host, master.port, "MASTER", get_server_state(master, master.host, 3, self.verbosity > 0), master.supports_gtid(), "OK" if rpl_health[0] else ", ".join(rpl_health[1]), ] # Get master status master_status = master.get_status() if len(master_status): master_log, master_log_pos = master_status[0][0:2] else: master_log = None master_log_pos = 0 # Show additional details if verbosity is turned on if self.verbosity > 0: master_data.extend([master.get_version(), master_log, master_log_pos, "", "", "", "", "", "", "", "", ""]) health_data[1].append(master_data) return health_data except UtilError as err: msg = "Cannot get health data: {0}".format(err) self._report(msg, logging.ERROR, False) raise UtilRplError(msg) return ([], [])
def get_server(name, values, quiet): """Connect to a server and return Server instance If the name is 'master' or 'slave', the connection will be made via the Master or Slave class else a normal Server class shall be used. name[in] name of the server values[in] dictionary of connection values quiet[in] if True, do not print messages Returns Server class instance """ from mysql.utilities.common.replication import Master from mysql.utilities.common.replication import Slave server_conn = None # Try to connect to the MySQL database server. if not quiet: _print_connection(name, values) server_options = { 'conn_info' : values, 'role' : name, } if name.lower() == 'master': server_conn = Master(server_options) elif name.lower() == 'slave': server_conn = Slave(server_options) else: server_conn = Server(server_options) server_conn.connect() return server_conn
def _check_server_versions(self): """Checks the server versions. """ if self.verbosity > 0: print("# Checking server versions.\n#") # Connection dictionary conn_dict = { "conn_info": None, "quiet": True, "verbose": self.verbosity > 0, } # Check masters version for master_vals in self.masters_vals: conn_dict["conn_info"] = master_vals master = Master(conn_dict) master.connect() if not master.check_version_compat(*_MIN_SERVER_VERSION): raise UtilRplError( ERROR_MIN_SERVER_VERSIONS.format( utility="mysqlrplms", min_version=".".join([str(val) for val in _MIN_SERVER_VERSION]), host=master.host, port=master.port ) ) master.disconnect() # Check slave version conn_dict["conn_info"] = self.slave_vals slave = Slave(conn_dict) slave.connect() if not slave.check_version_compat(*_MIN_SERVER_VERSION): raise UtilRplError( ERROR_MIN_SERVER_VERSIONS.format( utility="mysqlrplms", min_version=".".join([str(val) for val in _MIN_SERVER_VERSION]), host=slave.host, port=slave.port ) ) slave.disconnect()
def run_test_case(self, actual_result, test_num, master, source, destination, cmd_list, db_list, cmd_opts, comment, expected_results, restart_replication=False, skip_wait=False): results = [comment] # Drop all databases and reestablish replication if restart_replication: # Rollback here to avoid active transaction error for STOP SLAVE # with 5.5 servers (versions > 5.5.0). if self.servers.get_server(0).check_version_compat(5, 5, 0): destination.rollback() destination.exec_query("STOP SLAVE") destination.exec_query("RESET SLAVE") for db in db_list: self.drop_db(destination, db) master_str = "--master={0}".format( self.build_connection_string(master)) slave_str = " --slave={0}".format( self.build_connection_string(destination)) conn_str = master_str + slave_str cmd = "mysqlreplicate.py --rpl-user=rpl:rpl {0}".format(conn_str) try: self.exec_util(cmd, self.res_fname) except MUTLibError: raise # Convert object instance of master server to Master, if needed if not isinstance(master, Master): master = Master.fromServer(master) try: master.connect() except UtilError as err: raise MUTLibError("Cannot connect to spawned " "server: {0}".format(err.errmsg)) # Convert object instance of destination server to Slave, if needed if not isinstance(destination, Slave): destination = Slave.fromServer(destination) try: destination.connect() except UtilError as err: raise MUTLibError("Cannot connect to spawned " "server: {0}".format(err.errmsg)) # Check databases on slave and save results for 'BEFORE' check results.append(self._check_result(destination, "SHOW DATABASES " "LIKE 'util_test'")) results.append(self._check_result(destination, "SELECT COUNT(*) " "FROM util_test.t1")) results.append(self._check_result(destination, "SHOW DATABASES " "LIKE 'master_db1'")) results.append(self._check_result(destination, "SELECT COUNT(*) " "FROM master_db1.t1")) # Run the commands for cmd_str in cmd_list: try: res = self.exec_util(cmd_str + cmd_opts, self.res_fname) results.insert(1, res) # save result at front of list if res != actual_result: return False except MUTLibError: raise # Wait for slave to connect to master if not skip_wait: if self.debug: print "# Waiting for slave to connect to master", try: self.wait_for_slave_connection(destination, _MAX_ATTEMPTS) except MUTLibError: raise if self.debug: print "done." # Check databases on slave and save results for 'AFTER' check results.append(self._check_result(destination, "SHOW DATABASES " "LIKE 'util_test'")) results.append(self._check_result(destination, "SELECT COUNT(*) " "FROM util_test.t1")) results.append(self._check_result(destination, "SHOW DATABASES " "LIKE 'master_db1'")) results.append(self._check_result(destination, "SELECT COUNT(*) " "FROM master_db1.t1")) # Add something to master and check slave master.exec_query("INSERT INTO master_db1.t1 VALUES (10), (11)") # Wait for slave to catch up if not skip_wait: if self.debug: print "# Waiting for slave to sync", bin_info = master.get_binlog_info() if bin_info is None: # server is no longer acting as a master raise MUTLibError("The server '{0}' is no longer a master" "server".format(master.role)) # pylint: disable=W0633 binlog_file, binlog_pos = bin_info # Wait for slave to catch up with master, using the binlog # Note: This test requires servers without GTIDs (prior to 5.6.5) synced = destination.wait_for_slave(binlog_file, binlog_pos, _SYNC_TIMEOUT) if not synced: raise MUTLibError("Slave did not catch up with master") if self.debug: print "done." # ROLLBACK to close any active transaction leading to wrong values for # the next SELECT COUNT(*) with 5.5 servers (versions > 5.5.0). if self.servers.get_server(0).check_version_compat(5, 5, 0): destination.rollback() results.append(self._check_result(destination, "SELECT COUNT(*) " "FROM master_db1.t1")) if self.debug: print comment print "Expected Results:", expected_results[test_num - 1] print " Actual Results:", results[1:] self.results.append(results) return True
def _check_privileges(self): """Check required privileges to perform the multi-source replication. This method check if the used users for the slave and masters have the required privileges to perform the multi-source replication. The following privileges are required: - on slave: SUPER, SELECT, INSERT, UPDATE, REPLICATION SLAVE AND GRANT OPTION; - on the master: SUPER, SELECT, INSERT, UPDATE, REPLICATION SLAVE AND GRANT OPTION. An exception is thrown if users doesn't have enough privileges. """ if self.verbosity > 0: print("# Checking users privileges for replication.\n#") # Connection dictionary conn_dict = { "conn_info": None, "quiet": True, "verbose": self.verbosity > 0, } # Check privileges for master. master_priv = [('SUPER',), ('SELECT',), ('INSERT',), ('UPDATE',), ('REPLICATION SLAVE',), ('GRANT OPTION',)] master_priv_str = ("SUPER, SELECT, INSERT, UPDATE, REPLICATION SLAVE " "AND GRANT OPTION") for master_vals in self.masters_vals: conn_dict["conn_info"] = master_vals master = Master(conn_dict) master.connect() user_obj = User(master, "{0}@{1}".format(master.user, master.host)) for any_priv_tuple in master_priv: has_privilege = any( [user_obj.has_privilege('*', '*', priv) for priv in any_priv_tuple] ) if not has_privilege: msg = ERROR_USER_WITHOUT_PRIVILEGES.format( user=master.user, host=master.host, port=master.port, operation='perform replication', req_privileges=master_priv_str ) self._report(msg, logging.CRITICAL, False) raise UtilRplError(msg) master.disconnect() # Check privileges for slave slave_priv = [('SUPER',), ('SELECT',), ('INSERT',), ('UPDATE',), ('REPLICATION SLAVE',), ('GRANT OPTION',)] slave_priv_str = ("SUPER, SELECT, INSERT, UPDATE, REPLICATION SLAVE " "AND GRANT OPTION") conn_dict["conn_info"] = self.slave_vals slave = Slave(conn_dict) slave.connect() user_obj = User(slave, "{0}@{1}".format(slave.user, slave.host)) for any_priv_tuple in slave_priv: has_privilege = any( [user_obj.has_privilege('*', '*', priv) for priv in any_priv_tuple] ) if not has_privilege: msg = ("User '{0}' on '{1}@{2}' does not have sufficient " "privileges to perform replication (required: {3})." "".format(slave.user, slave.host, slave.port, slave_priv_str)) self._report(msg, logging.CRITICAL, False) raise UtilRplError(msg) slave.disconnect()
def run_test_case(self, actual_result, test_num, master, source, destination, cmd_list, db_list, cmd_opts, comment, expected_results, restart_replication=False, skip_wait=False): results = [comment] # Drop all databases and reestablish replication if restart_replication: # Rollback here to avoid active transaction error for STOP SLAVE # with 5.5 servers (versions > 5.5.0). if self.servers.get_server(0).check_version_compat(5, 5, 0): destination.rollback() destination.exec_query("STOP SLAVE") destination.exec_query("RESET SLAVE") for db in db_list: self.drop_db(destination, db) master_str = "--master={0}".format( self.build_connection_string(master)) slave_str = " --slave={0}".format( self.build_connection_string(destination)) conn_str = master_str + slave_str cmd = "mysqlreplicate.py --rpl-user=rpl:rpl {0}".format(conn_str) try: self.exec_util(cmd, self.res_fname) except MUTLibError: raise # Convert object instance of master server to Master, if needed if not isinstance(master, Master): master = Master.fromServer(master) try: master.connect() except UtilError as err: raise MUTLibError("Cannot connect to spawned " "server: {0}".format(err.errmsg)) # Convert object instance of destination server to Slave, if needed if not isinstance(destination, Slave): destination = Slave.fromServer(destination) try: destination.connect() except UtilError as err: raise MUTLibError("Cannot connect to spawned " "server: {0}".format(err.errmsg)) # Check databases on slave and save results for 'BEFORE' check results.append( self._check_result(destination, "SHOW DATABASES " "LIKE 'util_test'")) results.append( self._check_result(destination, "SELECT COUNT(*) " "FROM util_test.t1")) results.append( self._check_result(destination, "SHOW DATABASES " "LIKE 'master_db1'")) results.append( self._check_result(destination, "SELECT COUNT(*) " "FROM master_db1.t1")) # Run the commands for cmd_str in cmd_list: try: res = self.exec_util(cmd_str + cmd_opts, self.res_fname) results.insert(1, res) # save result at front of list if res != actual_result: return False except MUTLibError: raise # Wait for slave to connect to master if not skip_wait: if self.debug: print "# Waiting for slave to connect to master", try: self.wait_for_slave_connection(destination, _MAX_ATTEMPTS) except MUTLibError: raise if self.debug: print "done." # Check databases on slave and save results for 'AFTER' check results.append( self._check_result(destination, "SHOW DATABASES " "LIKE 'util_test'")) results.append( self._check_result(destination, "SELECT COUNT(*) " "FROM util_test.t1")) results.append( self._check_result(destination, "SHOW DATABASES " "LIKE 'master_db1'")) results.append( self._check_result(destination, "SELECT COUNT(*) " "FROM master_db1.t1")) # Add something to master and check slave master.exec_query("INSERT INTO master_db1.t1 VALUES (10), (11)") # Wait for slave to catch up if not skip_wait: if self.debug: print "# Waiting for slave to sync", bin_info = master.get_binlog_info() if bin_info is None: # server is no longer acting as a master raise MUTLibError("The server '{0}' is no longer a master" "server".format(master.role)) # pylint: disable=W0633 binlog_file, binlog_pos = bin_info # Wait for slave to catch up with master, using the binlog # Note: This test requires servers without GTIDs (prior to 5.6.5) synced = destination.wait_for_slave(binlog_file, binlog_pos, _SYNC_TIMEOUT) if not synced: raise MUTLibError("Slave did not catch up with master") if self.debug: print "done." # ROLLBACK to close any active transaction leading to wrong values for # the next SELECT COUNT(*) with 5.5 servers (versions > 5.5.0). if self.servers.get_server(0).check_version_compat(5, 5, 0): destination.rollback() results.append( self._check_result(destination, "SELECT COUNT(*) " "FROM master_db1.t1")) if self.debug: print comment print "Expected Results:", expected_results[test_num - 1] print " Actual Results:", results[1:] self.results.append(results) return True