def failover(self):
"""Replace disconnected master with slave
If disconnected master comes back to live, it become slave.
"""
center = Center()
center.update_ip_port()
master_obj_list = center.get_master_obj_list()
msg = color.yellow(message.get('error_no_alive_slave_for_failover'))
all_alive = True
for node in master_obj_list:
if node['status'] != 'connected':
all_alive = False
success = False
for slave in node['slaves']:
if slave['status'] == 'connected':
msg2 = message.get('redis_failover').format(
slave_addr=slave['addr'],
master_addr=node['addr']
)
logger.info(msg2)
stdout = center.run_failover(
slave['addr'],
take_over=True
)
if stdout != 'OK':
continue
logger.info('OK')
success = True
break
if not success:
logger.info(msg.format(node['addr']))
if all_alive:
msg = message.get('already_all_master_alive')
logger.info(msg)
def _deploy_zero_downtime(cluster_id):
logger.debug("zero downtime update cluster {}".format(cluster_id))
center = Center()
center.update_ip_port()
m_hosts = center.master_host_list
m_ports = center.master_port_list
s_hosts = center.slave_host_list
s_ports = center.slave_port_list
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
# check master alive
m_count = len(m_hosts) * len(m_ports)
alive_m_count = center.get_alive_master_redis_count()
if alive_m_count < m_count:
logger.error(message.get('error_exist_disconnected_master'))
return
if not config.is_slave_enabled:
logger.error(message.get('error_need_to_slave'))
return
# select installer
installer_path = ask_util.installer()
installer_name = os.path.basename(installer_path)
# backup info
current_time = time.strftime("%Y%m%d%H%M%S", time.gmtime())
conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time)
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time)
local_ip = config.get_local_ip()
# backup conf
center.conf_backup(local_ip, cluster_id, conf_backup_dir)
# backup cluster
for host in s_hosts:
client = net.get_ssh(host)
center.cluster_backup(host, cluster_id, cluster_backup_dir)
client.close()
# transfer & install
logger.info(message.get('transfer_and_execute_installer'))
for host in m_hosts:
logger.info(' - {}'.format(host))
client = net.get_ssh(host)
cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path)
net.ssh_execute(client=client, command=cmd)
client.close()
DeployUtil().transfer_installer(host, cluster_id, installer_path)
try:
DeployUtil().install(host, cluster_id, installer_name)
except SSHCommandError as ex:
msg = message.get('error_execute_installer')
msg = msg.format(installer=installer_path)
logger.error(msg)
logger.exception(ex)
return
# restore conf
center.conf_restore(local_ip, cluster_id, conf_backup_dir)
# set deploy state complete
for node in m_hosts:
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(node)
cmd = 'rm -rf {}'.format(os.path.join(cluster_path, '.deploy.state'))
net.ssh_execute(client=client, command=cmd)
client.close()
# restart slave
center.stop_current_nodes(master=False, slave=True)
center.configure_redis()
center.sync_conf()
center.start_current_nodes(master=False, slave=True)
center.wait_until_all_redis_process_up()
slaves_for_failover = center.get_slave_nodes()
key = 'cluster-node-timeout'
origin_m_value = center.cli_config_get(key, m_hosts[0], m_ports[0])
origin_s_value = center.cli_config_get(key, s_hosts[0], s_ports[0])
logger.debug('config set: cluster-node-timeout 2000')
RedisCliConfig().set(key, '2000', all=True)
# cluster failover (with no option)
logger.info(message.get('failover_on_deploy'))
logger.debug(slaves_for_failover)
try_count = 0
while try_count < 10:
try_count += 1
success = True
for slave_addr in slaves_for_failover:
host, port = slave_addr.split(':')
stdout = center.run_failover("{}:{}".format(host, port))
logger.debug("failover {}:{} {}".format(host, port, stdout))
if stdout != "ERR You should send CLUSTER FAILOVER to a slave":
# In some cases, the cluster failover is not complete
# even if stdout is OK
# If redis changed to master completely,
# return 'ERR You should send CLUSTER FAILOVER to a slave'
success = False
if success:
break
msg = message.get('retry').format(try_count=try_count)
logger.info(msg)
time.sleep(5)
logger.debug('restore config: cluster-node-timeout')
center.cli_config_set_all(key, origin_m_value, m_hosts, m_ports)
center.cli_config_set_all(key, origin_s_value, s_hosts, s_ports)
if not success:
logger.error(message.get('error_redis_failover'))
return
# restart master (current slave)
center.stop_current_nodes(master=False, slave=True)
center.configure_redis(slave=False)
center.sync_conf()
center.start_current_nodes(master=False, slave=True)
center.wait_until_all_redis_process_up()
