def __init__(self, host, host_ip=None, ssh_user=None, ssh_port=None, ssh_options=None): config_helper = ConfigHelper(host) self._read_only_config = config_helper.get_read_only_config_file() self._super_read_only = config_helper.get_super_read_only() self._ssh_client = SSHHelper(host, host_ip, ssh_user, ssh_port, ssh_options)
class MHAHelper(object): FAILOVER_TYPE_ONLINE = 'online_failover' FAILOVER_TYPE_HARD = 'hard_failover' FAILOVER_STOP_CMD = 'stop' FAILOVER_STOPSSH_CMD = 'stopssh' FAILOVER_START_CMD = 'start' FAILOVER_STATUS_CMD = 'status' def __init__(self, failover_type): self.failover_type = failover_type if not self.__validate_failover_type(): raise ValueError # Setup configuration if not ConfigHelper.load_config(): raise ValueError self.orig_master_host = None self.new_master_host = None self.orig_master_config = None self.new_master_config = None def execute_command(self, **kwargs): for key, value in kwargs.iteritems(): setattr(self, key, value) try: command = getattr(self, "command") except Exception as e: print("No command supplied: %s" % str(e)) return False # Delegate the work to other functions if command == self.FAILOVER_STOP_CMD: if self.failover_type == self.FAILOVER_TYPE_ONLINE: if not self.__stop_command(): self.__rollback_stop_command() return False elif self.failover_type == self.FAILOVER_TYPE_HARD: return self.__stop_hard_command() return True elif command == self.FAILOVER_STOPSSH_CMD: return self.__stop_ssh_command() elif command == self.FAILOVER_START_CMD: return self.__start_command() elif command == self.FAILOVER_STATUS_CMD: return self.__status_command() # If we reach here that means no valid command was provided so we return an error here return False def __validate_failover_type(self): return (self.failover_type == MHAHelper.FAILOVER_TYPE_ONLINE or self.failover_type == MHAHelper.FAILOVER_TYPE_HARD) def __unescape_from_shell(self, unescaped): # This matches with mha4mysql-node::NodeUtil.pm::@shell_escape_chars # username and passsword provided by MHA are escaped like this escaped = re.sub(r'\\(?!\\)','',unescaped) # remove escaping (\) return escaped def __stop_command(self): try: self.orig_master_host = getattr(self, "orig_master_host") self.orig_master_config = ConfigHelper(self.orig_master_host) except Exception as e: print("Failed to read configuration for original master: %s" % str(e)) return False # Original master try: orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) orig_master_mysql_port = getattr(self, "orig_master_port", None) orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) orig_master_ssh_user = getattr(self, "orig_master_ssh_user", None) orig_master_mysql_user = getattr(self, "orig_master_user") orig_master_mysql_pass = getattr(self, "orig_master_password") except AttributeError as e: print("Failed to read one or more required original master parameter(s): %s" % str(e)) return False orig_master_mysql_user = self.__unescape_from_shell(orig_master_mysql_user) orig_master_mysql_pass = self.__unescape_from_shell(orig_master_mysql_pass) # Setup MySQL connections mysql_orig_master = MySQLHelper(orig_master_ip, orig_master_mysql_port, orig_master_mysql_user, orig_master_mysql_pass) try: print("Connecting to mysql on the original master '%s'" % self.orig_master_host) if not mysql_orig_master.connect(): return False if self.orig_master_config.get_manage_vip(): vip_type = self.orig_master_config.get_vip_type() print("Removing the vip using the '%s' provider from the original master '%s'" % (vip_type, self.orig_master_host)) if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, orig_master_ssh_port, self.FAILOVER_TYPE_ONLINE): return False if self.orig_master_config.get_super_read_only() == 'no': print("Setting read_only to '1' on the original master '%s'" % self.orig_master_host) if not mysql_orig_master.set_read_only() or not mysql_orig_master.is_read_only(): return False else: print("Setting super_read_only to '1' on the original master '%s'" % self.orig_master_host) if not mysql_orig_master.set_super_read_only() or not mysql_orig_master.is_super_read_only(): return False if self.orig_master_config.get_read_only_config_file(): mysqlconfig_helper = MySQLConfigHelper(self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, orig_master_ssh_port) mysqlconfig_helper.set_read_only_config() if not self.__mysql_kill_threads(self.orig_master_host, mysql_orig_master, self.orig_master_config.get_kill_after_timeout()): return False except Exception as e: print("Unexpected error: %s" % str(e)) return False finally: print("Disconnecting from mysql on the original master '%s'" % self.orig_master_host) mysql_orig_master.disconnect() return True def __stop_hard_command(self): try: self.orig_master_host = getattr(self, "orig_master_host") self.orig_master_config = ConfigHelper(self.orig_master_host) except Exception as e: print("Failed to read configuration for original master: %s" % str(e)) return False # Original master try: orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) except AttributeError as e: print("Failed to read one or more required original master parameter(s): %s" % str(e)) return False try: if self.orig_master_config.get_manage_vip(): vip_type = self.orig_master_config.get_vip_type() print("Removing the vip using the '%s' provider from the original master '%s'" % (vip_type, self.orig_master_host)) if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip, None, orig_master_ssh_port, self.FAILOVER_TYPE_HARD): return False except Exception as e: print("Unexpected error: %s" % str(e)) return False return True def __stop_ssh_command(self): try: self.orig_master_host = getattr(self, "orig_master_host") self.orig_master_config = ConfigHelper(self.orig_master_host) except Exception as e: print("Failed to read configuration for original master: %s" % str(e)) return False # Original master try: orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) orig_master_ssh_user = getattr(self, "ssh_user", None) except AttributeError as e: print("Failed to read one or more required original master parameter(s): %s" % str(e)) return False try: if self.orig_master_config.get_manage_vip(): vip_type = self.orig_master_config.get_vip_type() print("Removing the vip using the '%s' provider from the original master '%s'" % (vip_type, self.orig_master_host)) if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, orig_master_ssh_port, self.FAILOVER_TYPE_ONLINE): return False except Exception as e: print("Unexpected error: %s" % str(e)) return False return True def __start_command(self): try: self.orig_master_host = getattr(self, "orig_master_host") self.orig_master_config = ConfigHelper(self.orig_master_host) except Exception as e: print("Failed to read configuration for original master: %s" % str(e)) return False try: self.new_master_host = getattr(self, "new_master_host") self.new_master_config = ConfigHelper(self.new_master_host) except Exception as e: print("Failed to read configuration for new master: %s" % str(e)) return False # New master try: new_master_ip = getattr(self, "new_master_ip", self.new_master_host) new_master_mysql_port = getattr(self, "new_master_port", None) new_master_mysql_user = getattr(self, "new_master_user") new_master_mysql_pass = getattr(self, "new_master_password") new_master_ssh_ip = getattr(self, "new_master_ssh_ip", new_master_ip) new_master_ssh_port = getattr(self, "new_master_ssh_port", None) if self.failover_type == self.FAILOVER_TYPE_HARD: new_master_ssh_user = getattr(self, "ssh_user", None) else: new_master_ssh_user = getattr(self, "new_master_ssh_user", None) except AttributeError as e: print("Failed to read one or more required new master parameter(s): %s" % str(e)) return False new_master_mysql_user = self.__unescape_from_shell(new_master_mysql_user) new_master_mysql_pass = self.__unescape_from_shell(new_master_mysql_pass) # Setup MySQL connection mysql_new_master = MySQLHelper(new_master_ip, new_master_mysql_port, new_master_mysql_user, new_master_mysql_pass) try: print("Connecting to mysql on the new master '%s'" % self.new_master_host) if not mysql_new_master.connect(): return False if self.new_master_config.get_super_read_only() == 'no': print("Setting read_only to '0' on the new master '%s'" % self.new_master_host) if not mysql_new_master.unset_read_only() or mysql_new_master.is_read_only(): return False else: print("Setting super_read_only to '0' on the new master '%s'" % self.new_master_host) if not mysql_new_master.unset_super_read_only() or mysql_new_master.is_super_read_only(): return False if self.new_master_config.get_read_only_config_file(): mysqlconfig_helper = MySQLConfigHelper(self.new_master_host, new_master_ssh_ip, new_master_ssh_user, new_master_ssh_port) mysqlconfig_helper.unset_read_only_config() if self.new_master_config.get_manage_vip(): vip_type = self.new_master_config.get_vip_type() print("Adding the vip using the '%s' provider to the new master '%s'" % (vip_type, self.new_master_host)) if not self.__add_vip_to_host(vip_type, self.new_master_host, new_master_ssh_ip, new_master_ssh_user, new_master_ssh_port): return False except Exception as e: print("Unexpected error: %s" % str(e)) return False finally: print("Disconnecting from mysql on the new master '%s'" % self.new_master_host) mysql_new_master.disconnect() return True def __status_command(self): try: self.orig_master_host = getattr(self, "orig_master_host") self.orig_master_config = ConfigHelper(self.orig_master_host) except Exception as e: print("Failed to read configuration for original master: %s" % str(e)) return False # Original master try: orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) orig_master_ssh_user = getattr(self, "ssh_user", None) except AttributeError as e: print("Failed to read one or more required original master parameter(s): %s" % str(e)) return False try: if self.orig_master_config.get_manage_vip(): vip_type = self.orig_master_config.get_vip_type() print("Checking the vip using the '%s' provider on the original master '%s'" % (vip_type, self.orig_master_host)) if not self.__check_vip_on_host(vip_type, self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, orig_master_ssh_port): print("The VIP was not found on host %s\n" % (self.orig_master_host)) return False except Exception as e: print("Unexpected error: %s" % str(e)) return False return True def __rollback_stop_command(self): try: self.orig_master_host = getattr(self, "orig_master_host") self.orig_master_config = ConfigHelper(self.orig_master_host) except Exception as e: print("Failed to read configuration for original master: %s" % str(e)) return False # Original master try: orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host) orig_master_mysql_port = getattr(self, "orig_master_port", None) orig_master_mysql_user = getattr(self, "orig_master_user") orig_master_mysql_pass = getattr(self, "orig_master_password") orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip) orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None) orig_master_ssh_user = getattr(self, "orig_master_ssh_user", None) except AttributeError as e: print("Failed to read one or more required original master parameter(s): %s" % str(e)) return False orig_master_mysql_user = self.__unescape_from_shell(orig_master_mysql_user) orig_master_mysql_pass = self.__unescape_from_shell(orig_master_mysql_pass) # Setup MySQL connections mysql_orig_master = MySQLHelper(orig_master_ip, orig_master_mysql_port, orig_master_mysql_user, orig_master_mysql_pass) print("Rolling back the failover changes on the original master '%s'" % self.orig_master_host) try: if not mysql_orig_master.connect(): print("Failed to connect to mysql on the original master '%s'" % self.orig_master_host) return False if self.orig_master_config.get_super_read_only() == 'no': if not mysql_orig_master.unset_read_only() or mysql_orig_master.is_read_only(): print("Failed to reset read_only to '0' on the original master '%s'" % self.orig_master_host) return False print("Set read_only back to '0' on the original master '%s'" % self.orig_master_host) else: if not mysql_orig_master.unset_super_read_only() or mysql_orig_master.is_super_read_only(): print("Failed to reset super_read_only to '0' on the original master '%s'" % self.orig_master_host) return False print("Set super_read_only back to '0' on the original master '%s'" % self.orig_master_host) if self.orig_master_config.get_read_only_config_file(): mysqlconfig_helper = MySQLConfigHelper(self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, orig_master_ssh_port) mysqlconfig_helper.unset_read_only_config() if self.orig_master_config.get_manage_vip(): vip_type = self.orig_master_config.get_vip_type() if not self.__add_vip_to_host(vip_type, self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, orig_master_ssh_port): print("Failed to add back the vip using the '%s' provider to the original master '%s'" % (vip_type, self.orig_master_host)) return False print("Added back the vip to the original master '%s'" % self.orig_master_host) except Exception as e: print("Unexpected error: %s" % str(e)) return False finally: mysql_orig_master.disconnect() return True @classmethod def __remove_vip_from_host(cls, vip_type, host, host_ip, ssh_user, ssh_port, failover_type): if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL: vip_helper = VIPMetalHelper(host, host_ip, ssh_user, ssh_port) # If this is a hard failover and we cannot connect to the original master over SSH then we cannot really do # anything here at the moment. # TODO: At the moment we are not doing anything here but we would probably want to do something here if failover_type == cls.FAILOVER_TYPE_HARD: return True if not vip_helper.remove_vip(): return False elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS: pass elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS: pass else: # There are no other vip providers apart from what we are testing for above. Hence we throw an # error here return False return True @classmethod def __add_vip_to_host(cls, vip_type, host, host_ip, ssh_user, ssh_port): if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL: vip_helper = VIPMetalHelper(host, host_ip, ssh_user, ssh_port) if not vip_helper.assign_vip(): return False elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS: pass elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS: pass else: # There are no other vip providers apart from what we are testing for above. Hence we throw an # error here return False return True @classmethod def __check_vip_on_host(cls, vip_type, host, host_ip, ssh_user, ssh_port): if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL: return VIPMetalHelper(host, host_ip, ssh_user, ssh_port).has_vip() elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS: pass elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS: pass else: # There are no other vip providers apart from what we are testing for above. Hence we throw an # error here return False return True @classmethod def __mysql_kill_threads(cls, host, mysql_connection, timeout): sleep_interval = 0.1 start = datetime.datetime.now() print("Waiting %d seconds for application threads to disconnect from the MySQL server '%s'" % (timeout, host)) while True: try: mysql_threads = cls.__get_mysql_threads_list(mysql_connection) if len(mysql_threads) < 1: break except Exception as e: print("Unexpected error: %s" % str(e)) return False time.sleep(sleep_interval) now = datetime.datetime.now() if (now - start).seconds > timeout: break print("Terminating all application threads connected to the MySQL server '%s'" % host) try: for thread in iter(cls.__get_mysql_threads_list(mysql_connection)): print("Terminating thread Id => %s, User => %s, Host => %s" % (thread['Id'], thread['User'], thread['Host'])) mysql_connection.kill_connection(thread['Id']) except Exception as e: print("Unexpected error: %s" % str(e)) return False return True @classmethod def __get_mysql_threads_list(cls, mysql_connection): threads = list() try: for row in mysql_connection.get_processlist(): if (mysql_connection.get_connection_id() == row['Id'] or row['Command'] == "Binlog Dump" or row['Command'] == "Binlog Dump GTID" or row['User'] == "system user"): continue threads.append(row) except Exception as e: print("Failed to get list of processes from MySQL: %s" % str(e)) return False return threads