def __init__(self, host, host_ip=None, ssh_user=None, ssh_port=None, ssh_options=None):
config_helper = ConfigHelper(host)
self._read_only_config = config_helper.get_read_only_config_file()
self._super_read_only = config_helper.get_super_read_only()
self._ssh_client = SSHHelper(host, host_ip, ssh_user, ssh_port, ssh_options)
class MHAHelper(object):
FAILOVER_TYPE_ONLINE = 'online_failover'
FAILOVER_TYPE_HARD = 'hard_failover'
FAILOVER_STOP_CMD = 'stop'
FAILOVER_STOPSSH_CMD = 'stopssh'
FAILOVER_START_CMD = 'start'
FAILOVER_STATUS_CMD = 'status'
def __init__(self, failover_type):
self.failover_type = failover_type
if not self.__validate_failover_type():
raise ValueError
# Setup configuration
if not ConfigHelper.load_config():
raise ValueError
self.orig_master_host = None
self.new_master_host = None
self.orig_master_config = None
self.new_master_config = None
def execute_command(self, **kwargs):
for key, value in kwargs.iteritems():
setattr(self, key, value)
try:
command = getattr(self, "command")
except Exception as e:
print("No command supplied: %s" % str(e))
return False
# Delegate the work to other functions
if command == self.FAILOVER_STOP_CMD:
if self.failover_type == self.FAILOVER_TYPE_ONLINE:
if not self.__stop_command():
self.__rollback_stop_command()
return False
elif self.failover_type == self.FAILOVER_TYPE_HARD:
return self.__stop_hard_command()
return True
elif command == self.FAILOVER_STOPSSH_CMD:
return self.__stop_ssh_command()
elif command == self.FAILOVER_START_CMD:
return self.__start_command()
elif command == self.FAILOVER_STATUS_CMD:
return self.__status_command()
# If we reach here that means no valid command was provided so we return an error here
return False
def __validate_failover_type(self):
return (self.failover_type == MHAHelper.FAILOVER_TYPE_ONLINE or
self.failover_type == MHAHelper.FAILOVER_TYPE_HARD)
def __unescape_from_shell(self, unescaped):
# This matches with mha4mysql-node::NodeUtil.pm::@shell_escape_chars
# username and passsword provided by MHA are escaped like this
escaped = re.sub(r'\\(?!\\)','',unescaped) # remove escaping (\)
return escaped
def __stop_command(self):
try:
self.orig_master_host = getattr(self, "orig_master_host")
self.orig_master_config = ConfigHelper(self.orig_master_host)
except Exception as e:
print("Failed to read configuration for original master: %s" % str(e))
return False
# Original master
try:
orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host)
orig_master_mysql_port = getattr(self, "orig_master_port", None)
orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip)
orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None)
orig_master_ssh_user = getattr(self, "orig_master_ssh_user", None)
orig_master_mysql_user = getattr(self, "orig_master_user")
orig_master_mysql_pass = getattr(self, "orig_master_password")
except AttributeError as e:
print("Failed to read one or more required original master parameter(s): %s" % str(e))
return False
orig_master_mysql_user = self.__unescape_from_shell(orig_master_mysql_user)
orig_master_mysql_pass = self.__unescape_from_shell(orig_master_mysql_pass)
# Setup MySQL connections
mysql_orig_master = MySQLHelper(orig_master_ip, orig_master_mysql_port, orig_master_mysql_user,
orig_master_mysql_pass)
try:
print("Connecting to mysql on the original master '%s'" % self.orig_master_host)
if not mysql_orig_master.connect():
return False
if self.orig_master_config.get_manage_vip():
vip_type = self.orig_master_config.get_vip_type()
print("Removing the vip using the '%s' provider from the original master '%s'" %
(vip_type, self.orig_master_host))
if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip,
orig_master_ssh_user, orig_master_ssh_port,
self.FAILOVER_TYPE_ONLINE):
return False
if self.orig_master_config.get_super_read_only() == 'no':
print("Setting read_only to '1' on the original master '%s'" % self.orig_master_host)
if not mysql_orig_master.set_read_only() or not mysql_orig_master.is_read_only():
return False
else:
print("Setting super_read_only to '1' on the original master '%s'" % self.orig_master_host)
if not mysql_orig_master.set_super_read_only() or not mysql_orig_master.is_super_read_only():
return False
if self.orig_master_config.get_read_only_config_file():
mysqlconfig_helper = MySQLConfigHelper(self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, orig_master_ssh_port)
mysqlconfig_helper.set_read_only_config()
if not self.__mysql_kill_threads(self.orig_master_host, mysql_orig_master, self.orig_master_config.get_kill_after_timeout()):
return False
except Exception as e:
print("Unexpected error: %s" % str(e))
return False
finally:
print("Disconnecting from mysql on the original master '%s'" % self.orig_master_host)
mysql_orig_master.disconnect()
return True
def __stop_hard_command(self):
try:
self.orig_master_host = getattr(self, "orig_master_host")
self.orig_master_config = ConfigHelper(self.orig_master_host)
except Exception as e:
print("Failed to read configuration for original master: %s" % str(e))
return False
# Original master
try:
orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host)
orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip)
orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None)
except AttributeError as e:
print("Failed to read one or more required original master parameter(s): %s" % str(e))
return False
try:
if self.orig_master_config.get_manage_vip():
vip_type = self.orig_master_config.get_vip_type()
print("Removing the vip using the '%s' provider from the original master '%s'" %
(vip_type, self.orig_master_host))
if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip, None,
orig_master_ssh_port, self.FAILOVER_TYPE_HARD):
return False
except Exception as e:
print("Unexpected error: %s" % str(e))
return False
return True
def __stop_ssh_command(self):
try:
self.orig_master_host = getattr(self, "orig_master_host")
self.orig_master_config = ConfigHelper(self.orig_master_host)
except Exception as e:
print("Failed to read configuration for original master: %s" % str(e))
return False
# Original master
try:
orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host)
orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip)
orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None)
orig_master_ssh_user = getattr(self, "ssh_user", None)
except AttributeError as e:
print("Failed to read one or more required original master parameter(s): %s" % str(e))
return False
try:
if self.orig_master_config.get_manage_vip():
vip_type = self.orig_master_config.get_vip_type()
print("Removing the vip using the '%s' provider from the original master '%s'" %
(vip_type, self.orig_master_host))
if not self.__remove_vip_from_host(vip_type, self.orig_master_host, orig_master_ssh_ip,
orig_master_ssh_user, orig_master_ssh_port,
self.FAILOVER_TYPE_ONLINE):
return False
except Exception as e:
print("Unexpected error: %s" % str(e))
return False
return True
def __start_command(self):
try:
self.orig_master_host = getattr(self, "orig_master_host")
self.orig_master_config = ConfigHelper(self.orig_master_host)
except Exception as e:
print("Failed to read configuration for original master: %s" % str(e))
return False
try:
self.new_master_host = getattr(self, "new_master_host")
self.new_master_config = ConfigHelper(self.new_master_host)
except Exception as e:
print("Failed to read configuration for new master: %s" % str(e))
return False
# New master
try:
new_master_ip = getattr(self, "new_master_ip", self.new_master_host)
new_master_mysql_port = getattr(self, "new_master_port", None)
new_master_mysql_user = getattr(self, "new_master_user")
new_master_mysql_pass = getattr(self, "new_master_password")
new_master_ssh_ip = getattr(self, "new_master_ssh_ip", new_master_ip)
new_master_ssh_port = getattr(self, "new_master_ssh_port", None)
if self.failover_type == self.FAILOVER_TYPE_HARD:
new_master_ssh_user = getattr(self, "ssh_user", None)
else:
new_master_ssh_user = getattr(self, "new_master_ssh_user", None)
except AttributeError as e:
print("Failed to read one or more required new master parameter(s): %s" % str(e))
return False
new_master_mysql_user = self.__unescape_from_shell(new_master_mysql_user)
new_master_mysql_pass = self.__unescape_from_shell(new_master_mysql_pass)
# Setup MySQL connection
mysql_new_master = MySQLHelper(new_master_ip, new_master_mysql_port, new_master_mysql_user,
new_master_mysql_pass)
try:
print("Connecting to mysql on the new master '%s'" % self.new_master_host)
if not mysql_new_master.connect():
return False
if self.new_master_config.get_super_read_only() == 'no':
print("Setting read_only to '0' on the new master '%s'" % self.new_master_host)
if not mysql_new_master.unset_read_only() or mysql_new_master.is_read_only():
return False
else:
print("Setting super_read_only to '0' on the new master '%s'" % self.new_master_host)
if not mysql_new_master.unset_super_read_only() or mysql_new_master.is_super_read_only():
return False
if self.new_master_config.get_read_only_config_file():
mysqlconfig_helper = MySQLConfigHelper(self.new_master_host, new_master_ssh_ip, new_master_ssh_user, new_master_ssh_port)
mysqlconfig_helper.unset_read_only_config()
if self.new_master_config.get_manage_vip():
vip_type = self.new_master_config.get_vip_type()
print("Adding the vip using the '%s' provider to the new master '%s'" %
(vip_type, self.new_master_host))
if not self.__add_vip_to_host(vip_type, self.new_master_host, new_master_ssh_ip, new_master_ssh_user,
new_master_ssh_port):
return False
except Exception as e:
print("Unexpected error: %s" % str(e))
return False
finally:
print("Disconnecting from mysql on the new master '%s'" % self.new_master_host)
mysql_new_master.disconnect()
return True
def __status_command(self):
try:
self.orig_master_host = getattr(self, "orig_master_host")
self.orig_master_config = ConfigHelper(self.orig_master_host)
except Exception as e:
print("Failed to read configuration for original master: %s" % str(e))
return False
# Original master
try:
orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host)
orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip)
orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None)
orig_master_ssh_user = getattr(self, "ssh_user", None)
except AttributeError as e:
print("Failed to read one or more required original master parameter(s): %s" % str(e))
return False
try:
if self.orig_master_config.get_manage_vip():
vip_type = self.orig_master_config.get_vip_type()
print("Checking the vip using the '%s' provider on the original master '%s'" %
(vip_type, self.orig_master_host))
if not self.__check_vip_on_host(vip_type, self.orig_master_host, orig_master_ssh_ip,
orig_master_ssh_user, orig_master_ssh_port):
print("The VIP was not found on host %s\n" % (self.orig_master_host))
return False
except Exception as e:
print("Unexpected error: %s" % str(e))
return False
return True
def __rollback_stop_command(self):
try:
self.orig_master_host = getattr(self, "orig_master_host")
self.orig_master_config = ConfigHelper(self.orig_master_host)
except Exception as e:
print("Failed to read configuration for original master: %s" % str(e))
return False
# Original master
try:
orig_master_ip = getattr(self, "orig_master_ip", self.orig_master_host)
orig_master_mysql_port = getattr(self, "orig_master_port", None)
orig_master_mysql_user = getattr(self, "orig_master_user")
orig_master_mysql_pass = getattr(self, "orig_master_password")
orig_master_ssh_ip = getattr(self, "orig_master_ssh_ip", orig_master_ip)
orig_master_ssh_port = getattr(self, "orig_master_ssh_port", None)
orig_master_ssh_user = getattr(self, "orig_master_ssh_user", None)
except AttributeError as e:
print("Failed to read one or more required original master parameter(s): %s" % str(e))
return False
orig_master_mysql_user = self.__unescape_from_shell(orig_master_mysql_user)
orig_master_mysql_pass = self.__unescape_from_shell(orig_master_mysql_pass)
# Setup MySQL connections
mysql_orig_master = MySQLHelper(orig_master_ip, orig_master_mysql_port, orig_master_mysql_user,
orig_master_mysql_pass)
print("Rolling back the failover changes on the original master '%s'" % self.orig_master_host)
try:
if not mysql_orig_master.connect():
print("Failed to connect to mysql on the original master '%s'" % self.orig_master_host)
return False
if self.orig_master_config.get_super_read_only() == 'no':
if not mysql_orig_master.unset_read_only() or mysql_orig_master.is_read_only():
print("Failed to reset read_only to '0' on the original master '%s'" % self.orig_master_host)
return False
print("Set read_only back to '0' on the original master '%s'" % self.orig_master_host)
else:
if not mysql_orig_master.unset_super_read_only() or mysql_orig_master.is_super_read_only():
print("Failed to reset super_read_only to '0' on the original master '%s'" % self.orig_master_host)
return False
print("Set super_read_only back to '0' on the original master '%s'" % self.orig_master_host)
if self.orig_master_config.get_read_only_config_file():
mysqlconfig_helper = MySQLConfigHelper(self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user, orig_master_ssh_port)
mysqlconfig_helper.unset_read_only_config()
if self.orig_master_config.get_manage_vip():
vip_type = self.orig_master_config.get_vip_type()
if not self.__add_vip_to_host(vip_type, self.orig_master_host, orig_master_ssh_ip, orig_master_ssh_user,
orig_master_ssh_port):
print("Failed to add back the vip using the '%s' provider to the original master '%s'" %
(vip_type, self.orig_master_host))
return False
print("Added back the vip to the original master '%s'" % self.orig_master_host)
except Exception as e:
print("Unexpected error: %s" % str(e))
return False
finally:
mysql_orig_master.disconnect()
return True
@classmethod
def __remove_vip_from_host(cls, vip_type, host, host_ip, ssh_user, ssh_port, failover_type):
if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL:
vip_helper = VIPMetalHelper(host, host_ip, ssh_user, ssh_port)
# If this is a hard failover and we cannot connect to the original master over SSH then we cannot really do
# anything here at the moment.
# TODO: At the moment we are not doing anything here but we would probably want to do something here
if failover_type == cls.FAILOVER_TYPE_HARD:
return True
if not vip_helper.remove_vip():
return False
elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS:
pass
elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS:
pass
else:
# There are no other vip providers apart from what we are testing for above. Hence we throw an
# error here
return False
return True
@classmethod
def __add_vip_to_host(cls, vip_type, host, host_ip, ssh_user, ssh_port):
if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL:
vip_helper = VIPMetalHelper(host, host_ip, ssh_user, ssh_port)
if not vip_helper.assign_vip():
return False
elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS:
pass
elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS:
pass
else:
# There are no other vip providers apart from what we are testing for above. Hence we throw an
# error here
return False
return True
@classmethod
def __check_vip_on_host(cls, vip_type, host, host_ip, ssh_user, ssh_port):
if vip_type == ConfigHelper.VIP_PROVIDER_TYPE_METAL:
return VIPMetalHelper(host, host_ip, ssh_user, ssh_port).has_vip()
elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_AWS:
pass
elif vip_type == ConfigHelper.VIP_PROVIDER_TYPE_OS:
pass
else:
# There are no other vip providers apart from what we are testing for above. Hence we throw an
# error here
return False
return True
@classmethod
def __mysql_kill_threads(cls, host, mysql_connection, timeout):
sleep_interval = 0.1
start = datetime.datetime.now()
print("Waiting %d seconds for application threads to disconnect from the MySQL server '%s'" % (timeout, host))
while True:
try:
mysql_threads = cls.__get_mysql_threads_list(mysql_connection)
if len(mysql_threads) < 1:
break
except Exception as e:
print("Unexpected error: %s" % str(e))
return False
time.sleep(sleep_interval)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
break
print("Terminating all application threads connected to the MySQL server '%s'" % host)
try:
for thread in iter(cls.__get_mysql_threads_list(mysql_connection)):
print("Terminating thread Id => %s, User => %s, Host => %s" %
(thread['Id'], thread['User'], thread['Host']))
mysql_connection.kill_connection(thread['Id'])
except Exception as e:
print("Unexpected error: %s" % str(e))
return False
return True
@classmethod
def __get_mysql_threads_list(cls, mysql_connection):
threads = list()
try:
for row in mysql_connection.get_processlist():
if (mysql_connection.get_connection_id() == row['Id'] or row['Command'] == "Binlog Dump" or
row['Command'] == "Binlog Dump GTID" or row['User'] == "system user"):
continue
threads.append(row)
except Exception as e:
print("Failed to get list of processes from MySQL: %s" % str(e))
return False
return threads
