def tree(self):
"""The results of 'cli cluster nodes' are displayed in tree format
"""
center = Center()
center.update_ip_port()
master_node_list = center.get_master_obj_list()
output_msg = []
for master_node in master_node_list:
addr = master_node['addr']
status = master_node['status']
msg = '{}({})'.format(addr, status)
if status == 'disconnected':
msg = color.red(msg)
if status == 'paused':
msg = color.yellow(msg)
output_msg.append(msg)
for slave_node in master_node['slaves']:
addr = slave_node['addr']
status = slave_node['status']
msg = '{}({})'.format(addr, status)
if status == 'disconnected':
msg = color.red(msg)
if status == 'paused':
msg = color.yellow(msg)
output_msg.append('|__ ' + msg)
output_msg.append('')
logger.info(color.ENDC + '\n'.join(output_msg))
def failover(self):
"""Replace disconnected master with slave
If disconnected master comes back to live, it become slave.
"""
center = Center()
center.update_ip_port()
master_obj_list = center.get_master_obj_list()
msg = color.yellow(message.get('error_no_alive_slave_for_failover'))
all_alive = True
for node in master_obj_list:
if node['status'] != 'connected':
all_alive = False
success = False
for slave in node['slaves']:
if slave['status'] == 'connected':
msg2 = message.get('redis_failover').format(
slave_addr=slave['addr'],
master_addr=node['addr']
)
logger.info(msg2)
stdout = center.run_failover(
slave['addr'],
take_over=True
)
if stdout != 'OK':
continue
logger.info('OK')
success = True
break
if not success:
logger.info(msg.format(node['addr']))
if all_alive:
msg = message.get('already_all_master_alive')
logger.info(msg)
def masters_with_dir(self, server, dir):
"""Find masters that use the specified directory path
:param server: IP or hostname
:param dir: directory path
"""
center = Center()
center.update_ip_port()
logger.debug('masters_with_dir')
master_nodes = center.get_master_obj_list()
ret = RedisCliUtil.command_all_async('config get dir', slave=True)
outs = ''
meta = []
m_endpoint = []
for node in master_nodes:
m_endpoint.append(node['addr'])
for _, host, port, res, stdout in ret:
if res == 'OK':
flat_stdout = '\n'.join([outs, stdout])
line = flat_stdout.splitlines()
if self.compare_ip(host, server) and dir in line[2]:
endpoint = '{}:{}'.format(socket.gethostbyname(host), port)
if endpoint in m_endpoint:
meta.append([host, port, line[2]])
else:
logger.warning("FAIL {}:{} {}".format(host, port, stdout))
utils.print_table([['HOST', 'PORT', 'PATH']] + meta)
def failover_with_dir(self, server, dir):
"""Find masters that use the specified directory path and do failover with its slave
:param server: IP or hostname
:param dir: directory path
"""
center = Center()
center.update_ip_port()
logger.debug('failover_with_dir')
master_nodes = center.get_master_obj_list()
cluster_id = config.get_cur_cluster_id()
lib_path = config.get_ld_library_path(cluster_id)
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_bin = path_of_fb['sr2_redis_bin']
env_cmd = [
'GLOBIGNORE=*;',
'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']),
'export DYLD_LIBRARY_PATH={};'.format(
lib_path['dyld_library_path']),
]
redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli')
# Find masters with dir
ret = RedisCliUtil.command_all_async('config get dir', slave=True)
outs = ''
meta = []
m_endpoint = []
for node in master_nodes:
m_endpoint.append(node['addr'])
for _, host, port, res, stdout in ret:
if res == 'OK':
flat_stdout = '\n'.join([outs, stdout])
line = flat_stdout.splitlines()
if self.compare_ip(host, server) and dir in line[2]:
endpoint = '{}:{}'.format(socket.gethostbyname(host), port)
if endpoint in m_endpoint:
meta.append(endpoint)
else:
logger.warning("FAIL {}:{} {}".format(host, port, stdout))
for endpoint in meta:
for master_node in master_nodes:
if endpoint == master_node['addr']:
for slave_node in master_node['slaves']:
addr = slave_node['addr']
(s_host, s_port) = addr.split(':')
sub_cmd = 'cluster failover takeover'
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
s_host,
s_port,
sub_cmd,
)
self._print(
message.get('try_failover_takeover').format(
slave=addr))
stdout = subprocess.check_output(command, shell=True)
self._print(stdout)
def rowcount(self):
"""Query and show cluster row count
"""
logger.debug('rowcount')
masters = []
center = Center()
center.update_ip_port()
master_nodes = center.get_master_obj_list()
for master_node in master_nodes:
node = master_node['addr']
masters.append(node)
# open-redis-cli-all info Tablespace | grep totalRows | awk -F ',
# ' '{print $4}' | awk -F '=' '{sum += $2} END {print sum}'
ret = RedisCliUtil.command_all_async('info Tablespace', slave=True)
outs = ''
for _, host, port, res, stdout in ret:
if res == 'OK':
endpoint = '{}:{}'.format(host, port)
if endpoint in masters:
outs = '\n'.join([outs, stdout])
else:
logger.warning("FAIL {}:{} {}".format(host, port, stdout))
lines = outs.splitlines()
key = 'totalRows'
partitions = 'partitions'
evictions = 'evictedRows'
filtered_lines = (filter(lambda x: key in x, lines))
#self._print(filtered_lines)
# Table list
table_list = []
result = []
for line in filtered_lines:
tableStats, _ = line.split(':')
tableId = tableStats.split('_')
if tableId[1] in table_list:
pass
else:
table_list.append(tableId[1])
for tid in table_list:
table_lines = (filter(lambda x: tid in x, filtered_lines))
ld = RedisCliUtil.to_list_of_dict(table_lines)
row_count = reduce(lambda x, y: x + int(y[key]), ld, 0)
partitions_count = reduce(lambda x, y: x + int(y[partitions]), ld,
0)
evictions_count = reduce(lambda x, y: x + int(y[evictions]), ld, 0)
result.append([tid, row_count, partitions_count, evictions_count])
utils.print_table(
[['Table_ID', 'ROW_COUNT', 'PARTITION_COUNT', 'EVICTED_ROWS']] +
result)
def force_failover(self, server):
""" Find all masters on the server and convert them to slaves. Finally, in the server, only slaves will be remained.
:param server: IP or hostname
"""
logger.debug('force_failover')
center = Center()
center.update_ip_port()
master_nodes = center.get_master_obj_list()
cluster_id = config.get_cur_cluster_id()
lib_path = config.get_ld_library_path(cluster_id)
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_bin = path_of_fb['sr2_redis_bin']
env_cmd = [
'GLOBIGNORE=*;',
'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']),
'export DYLD_LIBRARY_PATH={};'.format(
lib_path['dyld_library_path']),
]
redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli')
outs = ''
meta = []
m_endpoint = []
for node in master_nodes:
addr = node['addr']
(host, port) = addr.split(':')
# if host == server:
if self.compare_ip(host, server):
for slave_node in node['slaves']:
addr = slave_node['addr']
(s_host, s_port) = addr.split(':')
sub_cmd = 'cluster failover takeover'
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
s_host,
s_port,
sub_cmd,
)
self._print(
message.get('try_failover_takeover').format(
slave=addr))
stdout = subprocess.check_output(command, shell=True)
self._print(stdout)
def failback(self):
"""Restart disconnected redis
"""
center = Center()
center.update_ip_port()
master_obj_list = center.get_master_obj_list()
disconnected_list = []
paused_list = []
for master in master_obj_list:
if master['status'] == 'disconnected':
disconnected_list.append(master['addr'])
if master['status'] == 'paused':
paused_list.append(master['addr'])
for slave in master['slaves']:
if slave['status'] == 'disconnected':
disconnected_list.append(slave['addr'])
if slave['status'] == 'paused':
paused_list.append(slave['addr'])
classified_disconnected_list = {}
classified_paused_list = {}
for disconnected in disconnected_list:
host, port = disconnected.split(':')
if host not in classified_disconnected_list:
classified_disconnected_list[host] = []
classified_disconnected_list[host].append(port)
for paused in paused_list:
host, port = paused.split(':')
if host not in classified_paused_list:
classified_paused_list[host] = []
classified_paused_list[host].append(port)
current_time = time.strftime("%Y%m%d-%H%M", time.gmtime())
for host, ports in classified_disconnected_list.items():
msg = message.get('redis_run')
msg = msg.format(host=host, ports='|'.join(ports))
logger.info(msg)
center.run_redis_process(host, ports, False, current_time)
for host, ports in classified_paused_list.items():
msg = message.get('redis_restart')
msg = msg.format(host=host, ports='|'.join(ports))
logger.info(msg)
center.stop_redis_process(host, ports)
center.run_redis_process(host, ports, False, current_time)
if not classified_disconnected_list and not classified_paused_list:
msg = message.get('already_all_redis_alive')
logger.info(msg)
def failover_list(self):
""" Find failovered|no-slave|no-slot masters and failbacked slaves
"""
center = Center()
center.update_ip_port()
logger.debug('failover_list')
master_nodes = center.get_master_obj_list()
slave_nodes = center.get_slave_nodes()
master_ports = center.master_port_list
slave_ports = center.slave_port_list
output_msg = []
failovered_masters = []
for master_node in master_nodes:
addr = master_node['addr']
port = addr.split(':')[1]
try:
value = int(port)
if value in slave_ports:
failovered_masters.append(addr)
except ValueError:
pass
noslave_masters = []
for master_node in master_nodes:
if len(master_node['slaves']) == 0:
noslave_masters.append(master_node['addr'])
else:
for slave_node in master_node['slaves']:
if slave_node['status'] == 'disconnected':
noslave_masters.append(master_node['addr'])
break
noslot_masters = []
ret = RedisCliUtil.command_all_async('cluster nodes', slave=True)
outs = ''
for _, host, port, res, stdout in ret:
if res == 'OK':
outs = '\n'.join([outs, stdout])
lines = outs.splitlines()
filtered_nodes = (filter(lambda x: 'myself,master' in x,
lines))
else:
logger.warning("FAIL {}:{} {}".format(host, port, stdout))
for line in filtered_nodes:
words = line.split()
if len(words) == 8:
noslot_masters.append(line.split()[1])
failbacked_slaves = []
for slave_nodes in slave_nodes:
port = slave_nodes.split(':')[1]
try:
value = int(port)
if value in master_ports:
failbacked_slaves.append(slave_nodes)
except ValueError:
pass
output_msg.append('1) failovered masters:')
output_msg.extend(failovered_masters)
output_msg.append('')
output_msg.append('2) no-slave masters:')
output_msg.extend(noslave_masters)
output_msg.append('')
output_msg.append('3) no-slot masters:')
output_msg.extend(noslot_masters)
output_msg.append('')
output_msg.append('4) failbacked slaves:')
output_msg.extend(failbacked_slaves)
output_msg.append('')
logger.info(color.ENDC + '\n'.join(output_msg))
