def tree(self): """The results of 'cli cluster nodes' are displayed in tree format """ center = Center() center.update_ip_port() master_node_list = center.get_master_obj_list() output_msg = [] for master_node in master_node_list: addr = master_node['addr'] status = master_node['status'] msg = '{}({})'.format(addr, status) if status == 'disconnected': msg = color.red(msg) if status == 'paused': msg = color.yellow(msg) output_msg.append(msg) for slave_node in master_node['slaves']: addr = slave_node['addr'] status = slave_node['status'] msg = '{}({})'.format(addr, status) if status == 'disconnected': msg = color.red(msg) if status == 'paused': msg = color.yellow(msg) output_msg.append('|__ ' + msg) output_msg.append('') logger.info(color.ENDC + '\n'.join(output_msg))
def failover(self): """Replace disconnected master with slave If disconnected master comes back to live, it become slave. """ center = Center() center.update_ip_port() master_obj_list = center.get_master_obj_list() msg = color.yellow(message.get('error_no_alive_slave_for_failover')) all_alive = True for node in master_obj_list: if node['status'] != 'connected': all_alive = False success = False for slave in node['slaves']: if slave['status'] == 'connected': msg2 = message.get('redis_failover').format( slave_addr=slave['addr'], master_addr=node['addr'] ) logger.info(msg2) stdout = center.run_failover( slave['addr'], take_over=True ) if stdout != 'OK': continue logger.info('OK') success = True break if not success: logger.info(msg.format(node['addr'])) if all_alive: msg = message.get('already_all_master_alive') logger.info(msg)
def masters_with_dir(self, server, dir): """Find masters that use the specified directory path :param server: IP or hostname :param dir: directory path """ center = Center() center.update_ip_port() logger.debug('masters_with_dir') master_nodes = center.get_master_obj_list() ret = RedisCliUtil.command_all_async('config get dir', slave=True) outs = '' meta = [] m_endpoint = [] for node in master_nodes: m_endpoint.append(node['addr']) for _, host, port, res, stdout in ret: if res == 'OK': flat_stdout = '\n'.join([outs, stdout]) line = flat_stdout.splitlines() if self.compare_ip(host, server) and dir in line[2]: endpoint = '{}:{}'.format(socket.gethostbyname(host), port) if endpoint in m_endpoint: meta.append([host, port, line[2]]) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) utils.print_table([['HOST', 'PORT', 'PATH']] + meta)
def failover_with_dir(self, server, dir): """Find masters that use the specified directory path and do failover with its slave :param server: IP or hostname :param dir: directory path """ center = Center() center.update_ip_port() logger.debug('failover_with_dir') master_nodes = center.get_master_obj_list() cluster_id = config.get_cur_cluster_id() lib_path = config.get_ld_library_path(cluster_id) path_of_fb = config.get_path_of_fb(cluster_id) sr2_redis_bin = path_of_fb['sr2_redis_bin'] env_cmd = [ 'GLOBIGNORE=*;', 'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']), 'export DYLD_LIBRARY_PATH={};'.format( lib_path['dyld_library_path']), ] redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli') # Find masters with dir ret = RedisCliUtil.command_all_async('config get dir', slave=True) outs = '' meta = [] m_endpoint = [] for node in master_nodes: m_endpoint.append(node['addr']) for _, host, port, res, stdout in ret: if res == 'OK': flat_stdout = '\n'.join([outs, stdout]) line = flat_stdout.splitlines() if self.compare_ip(host, server) and dir in line[2]: endpoint = '{}:{}'.format(socket.gethostbyname(host), port) if endpoint in m_endpoint: meta.append(endpoint) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) for endpoint in meta: for master_node in master_nodes: if endpoint == master_node['addr']: for slave_node in master_node['slaves']: addr = slave_node['addr'] (s_host, s_port) = addr.split(':') sub_cmd = 'cluster failover takeover' command = '{} {} -h {} -p {} {}'.format( ' '.join(env_cmd), redis_cli_cmd, s_host, s_port, sub_cmd, ) self._print( message.get('try_failover_takeover').format( slave=addr)) stdout = subprocess.check_output(command, shell=True) self._print(stdout)
def rowcount(self): """Query and show cluster row count """ logger.debug('rowcount') masters = [] center = Center() center.update_ip_port() master_nodes = center.get_master_obj_list() for master_node in master_nodes: node = master_node['addr'] masters.append(node) # open-redis-cli-all info Tablespace | grep totalRows | awk -F ', # ' '{print $4}' | awk -F '=' '{sum += $2} END {print sum}' ret = RedisCliUtil.command_all_async('info Tablespace', slave=True) outs = '' for _, host, port, res, stdout in ret: if res == 'OK': endpoint = '{}:{}'.format(host, port) if endpoint in masters: outs = '\n'.join([outs, stdout]) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) lines = outs.splitlines() key = 'totalRows' partitions = 'partitions' evictions = 'evictedRows' filtered_lines = (filter(lambda x: key in x, lines)) #self._print(filtered_lines) # Table list table_list = [] result = [] for line in filtered_lines: tableStats, _ = line.split(':') tableId = tableStats.split('_') if tableId[1] in table_list: pass else: table_list.append(tableId[1]) for tid in table_list: table_lines = (filter(lambda x: tid in x, filtered_lines)) ld = RedisCliUtil.to_list_of_dict(table_lines) row_count = reduce(lambda x, y: x + int(y[key]), ld, 0) partitions_count = reduce(lambda x, y: x + int(y[partitions]), ld, 0) evictions_count = reduce(lambda x, y: x + int(y[evictions]), ld, 0) result.append([tid, row_count, partitions_count, evictions_count]) utils.print_table( [['Table_ID', 'ROW_COUNT', 'PARTITION_COUNT', 'EVICTED_ROWS']] + result)
def force_failover(self, server): """ Find all masters on the server and convert them to slaves. Finally, in the server, only slaves will be remained. :param server: IP or hostname """ logger.debug('force_failover') center = Center() center.update_ip_port() master_nodes = center.get_master_obj_list() cluster_id = config.get_cur_cluster_id() lib_path = config.get_ld_library_path(cluster_id) path_of_fb = config.get_path_of_fb(cluster_id) sr2_redis_bin = path_of_fb['sr2_redis_bin'] env_cmd = [ 'GLOBIGNORE=*;', 'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']), 'export DYLD_LIBRARY_PATH={};'.format( lib_path['dyld_library_path']), ] redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli') outs = '' meta = [] m_endpoint = [] for node in master_nodes: addr = node['addr'] (host, port) = addr.split(':') # if host == server: if self.compare_ip(host, server): for slave_node in node['slaves']: addr = slave_node['addr'] (s_host, s_port) = addr.split(':') sub_cmd = 'cluster failover takeover' command = '{} {} -h {} -p {} {}'.format( ' '.join(env_cmd), redis_cli_cmd, s_host, s_port, sub_cmd, ) self._print( message.get('try_failover_takeover').format( slave=addr)) stdout = subprocess.check_output(command, shell=True) self._print(stdout)
def failback(self): """Restart disconnected redis """ center = Center() center.update_ip_port() master_obj_list = center.get_master_obj_list() disconnected_list = [] paused_list = [] for master in master_obj_list: if master['status'] == 'disconnected': disconnected_list.append(master['addr']) if master['status'] == 'paused': paused_list.append(master['addr']) for slave in master['slaves']: if slave['status'] == 'disconnected': disconnected_list.append(slave['addr']) if slave['status'] == 'paused': paused_list.append(slave['addr']) classified_disconnected_list = {} classified_paused_list = {} for disconnected in disconnected_list: host, port = disconnected.split(':') if host not in classified_disconnected_list: classified_disconnected_list[host] = [] classified_disconnected_list[host].append(port) for paused in paused_list: host, port = paused.split(':') if host not in classified_paused_list: classified_paused_list[host] = [] classified_paused_list[host].append(port) current_time = time.strftime("%Y%m%d-%H%M", time.gmtime()) for host, ports in classified_disconnected_list.items(): msg = message.get('redis_run') msg = msg.format(host=host, ports='|'.join(ports)) logger.info(msg) center.run_redis_process(host, ports, False, current_time) for host, ports in classified_paused_list.items(): msg = message.get('redis_restart') msg = msg.format(host=host, ports='|'.join(ports)) logger.info(msg) center.stop_redis_process(host, ports) center.run_redis_process(host, ports, False, current_time) if not classified_disconnected_list and not classified_paused_list: msg = message.get('already_all_redis_alive') logger.info(msg)
def failover_list(self): """ Find failovered|no-slave|no-slot masters and failbacked slaves """ center = Center() center.update_ip_port() logger.debug('failover_list') master_nodes = center.get_master_obj_list() slave_nodes = center.get_slave_nodes() master_ports = center.master_port_list slave_ports = center.slave_port_list output_msg = [] failovered_masters = [] for master_node in master_nodes: addr = master_node['addr'] port = addr.split(':')[1] try: value = int(port) if value in slave_ports: failovered_masters.append(addr) except ValueError: pass noslave_masters = [] for master_node in master_nodes: if len(master_node['slaves']) == 0: noslave_masters.append(master_node['addr']) else: for slave_node in master_node['slaves']: if slave_node['status'] == 'disconnected': noslave_masters.append(master_node['addr']) break noslot_masters = [] ret = RedisCliUtil.command_all_async('cluster nodes', slave=True) outs = '' for _, host, port, res, stdout in ret: if res == 'OK': outs = '\n'.join([outs, stdout]) lines = outs.splitlines() filtered_nodes = (filter(lambda x: 'myself,master' in x, lines)) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) for line in filtered_nodes: words = line.split() if len(words) == 8: noslot_masters.append(line.split()[1]) failbacked_slaves = [] for slave_nodes in slave_nodes: port = slave_nodes.split(':')[1] try: value = int(port) if value in master_ports: failbacked_slaves.append(slave_nodes) except ValueError: pass output_msg.append('1) failovered masters:') output_msg.extend(failovered_masters) output_msg.append('') output_msg.append('2) no-slave masters:') output_msg.extend(noslave_masters) output_msg.append('') output_msg.append('3) no-slot masters:') output_msg.extend(noslot_masters) output_msg.append('') output_msg.append('4) failbacked slaves:') output_msg.extend(failbacked_slaves) output_msg.append('') logger.info(color.ENDC + '\n'.join(output_msg))