def set(self, key, value, all=False, save=False, host=None, port=None): """Command: redis-cli config set :param key: target key :param value: value to set :param save: If true, save value to config file :param all: If true, send command to all redis :param host: host info for redis :param port: port info for redis """ if not isinstance(all, bool): msg = m.get('error_option_type_not_boolean') msg = msg.format(options='all') logger.error(msg) return if not isinstance(save, bool): msg = m.get('error_option_type_not_boolean') msg = msg.format(options='save') logger.error(msg) return if (not host or not port) and not all: msg = m.get('use_host_port_or_option_all') logger.error(msg) return sub_cmd = 'config set {key} {value} 2>&1'.format(key=key, value=value) if all: meta = [] ret = RedisCliUtil.command_all_async(sub_cmd) ok_cnt = 0 for m_s, host, port, result, message in ret: addr = '{}:{}'.format(host, port) if result == 'OK': if utils.to_str(message) == 'OK': ok_cnt += 1 else: meta.append([m_s, addr, color.red(message)]) else: meta.append([m_s, addr, color.red('FAIL')]) if meta: utils.print_table([['TYPE', 'ADDR', 'RESULT']] + meta) logger.info('success {}/{}'.format(ok_cnt, len(ret))) else: output = RedisCliUtil.command(sub_cmd=sub_cmd, host=host, port=port, formatter=self.no_print) output = output.strip() if output == "OK": logger.info(output) else: logger.error(output) if save: RedisCliUtil.save_redis_template_config(key, value) center = Center() center.update_ip_port() success = center.check_hosts_connection() if not success: return center.configure_redis() center.sync_conf()
def ping(host=None, port=None, all=False): """Send ping command :param all: If true, send command to all :param host: host info for redis :param port: port info for redis """ if not isinstance(all, bool): msg = message.get('error_option_type_not_boolean').format(option='all') logger.error(msg) return if (not host or not port) and not all: msg = message.get('use_host_port_or_option_all') logger.error(msg) return if all: meta = [] ret = RedisCliUtil.command_all_async('ping 2>&1') pong_cnt = 0 for m_s, host, port, result, _ in ret: addr = '{}:{}'.format(host, port) if result == 'OK': pong_cnt += 1 else: meta.append([m_s, addr, color.red('FAIL')]) if meta: utils.print_table([['TYPE', 'ADDR', 'RESULT']] + meta) msg = message.get('counting_alive_redis') msg = msg.format(alive=pong_cnt, total=len(ret)) logger.info(msg) return if host and port: _command('ping', False, host, port)
def find_noaddr(self): """Find noaddr nodes that is not used anymore in cluster """ center = Center() center.update_ip_port() logger.debug('find_noaddr') ret = RedisCliUtil.command_all_async('cluster nodes', slave=True) outs = '' meta = [] for _, host, port, res, stdout in ret: if res == 'OK': outs = '\n'.join([outs, stdout]) lines = outs.splitlines() filtered_lines = (filter(lambda x: 'noaddr' in x, lines)) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) total_list = [] for line in filtered_lines: total_list.append(line.split()[0]) # Remove duplicates unique_list = list(set(total_list)) for uuid in unique_list: meta.append([uuid]) utils.print_table([['UUID']] + meta)
def masters_with_dir(self, server, dir): """Find masters that use the specified directory path :param server: IP or hostname :param dir: directory path """ center = Center() center.update_ip_port() logger.debug('masters_with_dir') master_nodes = center.get_master_obj_list() ret = RedisCliUtil.command_all_async('config get dir', slave=True) outs = '' meta = [] m_endpoint = [] for node in master_nodes: m_endpoint.append(node['addr']) for _, host, port, res, stdout in ret: if res == 'OK': flat_stdout = '\n'.join([outs, stdout]) line = flat_stdout.splitlines() if self.compare_ip(host, server) and dir in line[2]: endpoint = '{}:{}'.format(socket.gethostbyname(host), port) if endpoint in m_endpoint: meta.append([host, port, line[2]]) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) utils.print_table([['HOST', 'PORT', 'PATH']] + meta)
def rowcount(self): """Query and show cluster row count """ logger.debug('rowcount') masters = [] center = Center() center.update_ip_port() master_nodes = center.get_master_obj_list() for master_node in master_nodes: node = master_node['addr'] masters.append(node) # open-redis-cli-all info Tablespace | grep totalRows | awk -F ', # ' '{print $4}' | awk -F '=' '{sum += $2} END {print sum}' ret = RedisCliUtil.command_all_async('info Tablespace', slave=True) outs = '' for _, host, port, res, stdout in ret: if res == 'OK': endpoint = '{}:{}'.format(host, port) if endpoint in masters: outs = '\n'.join([outs, stdout]) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) lines = outs.splitlines() key = 'totalRows' partitions = 'partitions' evictions = 'evictedRows' filtered_lines = (filter(lambda x: key in x, lines)) #self._print(filtered_lines) # Table list table_list = [] result = [] for line in filtered_lines: tableStats, _ = line.split(':') tableId = tableStats.split('_') if tableId[1] in table_list: pass else: table_list.append(tableId[1]) for tid in table_list: table_lines = (filter(lambda x: tid in x, filtered_lines)) ld = RedisCliUtil.to_list_of_dict(table_lines) row_count = reduce(lambda x, y: x + int(y[key]), ld, 0) partitions_count = reduce(lambda x, y: x + int(y[partitions]), ld, 0) evictions_count = reduce(lambda x, y: x + int(y[evictions]), ld, 0) result.append([tid, row_count, partitions_count, evictions_count]) utils.print_table( [['Table_ID', 'ROW_COUNT', 'PARTITION_COUNT', 'EVICTED_ROWS']] + result)
def distribution(self): """Check the distribution of all masters and slaves """ center = Center() center.update_ip_port() logger.debug('distribution') ret = RedisCliUtil.command_all_async('cluster nodes', slave=True) outs = '' for _, host, port, res, stdout in ret: if res == 'OK': outs = '\n'.join([outs, stdout]) lines = outs.splitlines() myself_key = 'myself' filtered_lines = (filter(lambda x: myself_key in x, lines)) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) meta = [] total_masters = 0 total_slaves = 0 for nd in center.master_host_list: num_of_masters = 0 num_of_slaves = 0 node = socket.gethostbyname(nd) host_lines = (filter(lambda x: (node + ':') in x, filtered_lines)) for node in host_lines: params = node.split() endpoint = params[1] roles = params[2] host = endpoint.split(':')[0] role = roles.split(',')[1] if role == 'master': if len(params) == 9: num_of_masters += 1 else: num_of_slaves += 1 total_masters += num_of_masters total_slaves += num_of_slaves hostname = str(socket.gethostbyaddr(host)[0]) + str('(') + str( host) + str(')') meta.append([hostname, num_of_masters, num_of_slaves]) meta.append(['TOTAL', total_masters, total_slaves]) utils.print_table([['HOST', 'MASTER', 'SLAVE']] + meta)
def get(self, key, all=False, host=None, port=None): """Command: redis-cli config get :param key: redis config keyword :param all: If true, send command to all redis :param host: host info for redis :param port: port info for redis """ if not isinstance(all, bool): msg = m.get('error_option_type_not_boolean') msg = msg.format(options='all') logger.error(msg) return if (not host or not port) and not all: msg = m.get('use_host_port_or_option_all') logger.error(msg) return sub_cmd = 'config get "{key}" 2>&1'.format(key=key) if all: meta = [] ret = RedisCliUtil.command_all_async(sub_cmd) for m_s, host, port, result, message in ret: addr = '{}:{}'.format(host, port) if result == 'OK': if message: _, value = message.split('\n') meta.append([m_s, addr, value]) else: meta.append([m_s, addr, color.red('Invalid Key')]) else: meta.append([m_s, addr, color.red(result)]) utils.print_table([['TYPE', 'ADDR', 'RESULT']] + meta) else: output = RedisCliUtil.command(sub_cmd=sub_cmd, host=host, port=port, formatter=self.no_print) output = output.strip() if output: key, value = output.split('\n') logger.info(value) else: msg = m.get('error_invalid_key').format(key=key) logger.error(msg)
def nodes_with_dir(self, server, dir): """Find nodes that use the specified directory path :param server: IP or hostname :param dir: directory path """ center = Center() center.update_ip_port() logger.debug('nodes_with_dir') ret = RedisCliUtil.command_all_async('config get dir', slave=True) outs = '' meta = [] for _, host, port, res, stdout in ret: if res == 'OK': flat_stdout = '\n'.join([outs, stdout]) line = flat_stdout.splitlines() if self.compare_ip(host, server) and dir in line[2]: meta.append([host, port, line[2]]) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) utils.print_table([['HOST', 'PORT', 'PATH']] + meta)
def _deploy(cluster_id, history_save, clean): deploy_state = DeployUtil().get_state(cluster_id) if deploy_state == DEPLOYED: msg = message.get('ask_deploy_again') msg = msg.format(cluster_id=cluster_id) msg = color.yellow(msg) yes = ask_util.askBool(msg, default='n') if not yes: logger.info(message.get('cancel')) return restore_yes = None no_localhost = False current_time = time.strftime("%Y%m%d%H%M%S", time.gmtime()) cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time) conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time) tmp_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, 'tmp') meta = [['NAME', 'VALUE']] path_of_fb = config.get_path_of_fb(cluster_id) conf_path = path_of_fb['conf_path'] props_path = path_of_fb['redis_properties'] cluster_path = path_of_fb['cluster_path'] path_of_cli = config.get_path_of_cli(cluster_id) conf_backup_path = path_of_cli['conf_backup_path'] tmp_backup_path = os.path.join(conf_backup_path, tmp_backup_dir) local_ip = config.get_local_ip() # ask installer installer_path = ask_util.installer() installer_name = os.path.basename(installer_path) meta.append(['installer', installer_name]) # ask restore conf if deploy_state == DEPLOYED: restore_yes = ask_util.askBool(message.get('ask_restore_conf')) meta.append(['restore', restore_yes]) # input props hosts = [] if deploy_state == DEPLOYED: if restore_yes: meta += DeployUtil().get_meta_from_props(props_path) hosts = config.get_props(props_path, 'sr2_redis_master_hosts') else: if not os.path.isdir(conf_backup_path): os.mkdir(conf_backup_path) if os.path.exists(tmp_backup_path): msg = message.get('ask_load_history_of_previous_modification') yes = ask_util.askBool(msg) if not yes: shutil.rmtree(tmp_backup_path) if not os.path.exists(tmp_backup_path): os.mkdir(tmp_backup_path) shutil.copy(os.path.join(conf_path, 'redis.properties'), os.path.join(tmp_backup_path, 'redis.properties')) tmp_props_path = os.path.join(tmp_backup_path, 'redis.properties') editor.edit(tmp_props_path, syntax='sh') meta += DeployUtil().get_meta_from_props(tmp_props_path) hosts = config.get_props(tmp_props_path, 'sr2_redis_master_hosts') else: # new deploy props_dict = ask_util.props(cluster_id, save=history_save) hosts = props_dict['hosts'] meta += DeployUtil().get_meta_from_dict(props_dict) utils.print_table(meta) msg = message.get('confirm_deploy_information') yes = ask_util.askBool(msg) if not yes: logger.info(message.get('cancel')) return # check node status success = Center().check_hosts_connection(hosts, True) if not success: msg = message.get('error_exist_unavailable_host') logger.error(msg) return logger.debug('Connection of all hosts ok.') success = Center().check_include_localhost(hosts) if not success: no_localhost = True # get port info if deploy_state == DEPLOYED: if restore_yes: key = 'sr2_redis_master_ports' m_ports = config.get_props(props_path, key, []) key = 'sr2_redis_slave_ports' s_ports = config.get_props(props_path, key, []) replicas = len(s_ports) // len(m_ports) else: key = 'sr2_redis_master_ports' m_ports = config.get_props(tmp_props_path, key, []) key = 'sr2_redis_slave_ports' s_ports = config.get_props(tmp_props_path, key, []) replicas = len(s_ports) // len(m_ports) else: m_ports = props_dict['master_ports'] s_ports = props_dict['slave_ports'] replicas = props_dict['replicas'] while True: msg = message.get('check_port') logger.info(msg) host_ports_list = [] for host in hosts: host_ports_list.append((host, m_ports + s_ports)) conflict = Center().check_port_is_enable(host_ports_list) if not conflict: logger.info("OK") break utils.print_table([["HOST", "PORT"]] + conflict) msg = message.get('ask_port_collision') msg = color.yellow(msg) yes = ask_util.askBool(msg) if yes: logger.info("OK") break m_ports = ask_util.master_ports(False, cluster_id) replicas = ask_util.replicas(False) s_ports = ask_util.slave_ports(cluster_id, len(m_ports), replicas) if deploy_state == DEPLOYED: if restore_yes: key = 'sr2_redis_master_ports' value = cluster_util.convert_list_2_seq(m_ports) config.set_props(props_path, key, value) key = 'sr2_redis_slave_ports' value = cluster_util.convert_list_2_seq(s_ports) config.set_props(props_path, key, value) else: key = 'sr2_redis_master_ports' value = cluster_util.convert_list_2_seq(m_ports) config.set_props(tmp_props_path, key, value) key = 'sr2_redis_slave_ports' value = cluster_util.convert_list_2_seq(s_ports) config.set_props(tmp_props_path, key, value) else: props_dict['master_ports'] = m_ports props_dict['slave_ports'] = s_ports props_dict['replicas'] = replicas # if pending, delete legacy on each hosts if no_localhost: if DeployUtil().get_state(cluster_id, local_ip) == PENDING: client = net.get_ssh(local_ip) command = 'rm -rf {}'.format(cluster_path) net.ssh_execute(client=client, command=command) client.close() for host in hosts: if DeployUtil().get_state(cluster_id, host) == PENDING: client = net.get_ssh(host) command = 'rm -rf {}'.format(cluster_path) net.ssh_execute(client=client, command=command) client.close() # added_hosts = post_hosts - pre_hosts msg = message.get('check_cluster_exist') logger.info(msg) added_hosts = set(hosts) meta = [] if deploy_state == DEPLOYED: pre_hosts = config.get_props(props_path, 'sr2_redis_master_hosts') added_hosts -= set(pre_hosts) can_deploy = True if no_localhost: added_hosts |= set([local_ip]) for host in added_hosts: client = net.get_ssh(host) is_localhost = Center().is_localhost(host) if is_localhost: if no_localhost: continue if os.path.exists(cluster_path + '/remote'): meta.append([host, color.green('CLEAN')]) continue if net.is_exist(client, cluster_path): meta.append([host, color.red('CLUSTER EXIST')]) can_deploy = False continue meta.append([host, color.green('CLEAN')]) if meta: utils.print_table([['HOST', 'STATUS']] + meta) if not can_deploy: msg = message.get('error_cluster_collision') logger.error(msg) return # if not force: # logger.error("If you trying to force, use option '--force'") # return logger.info('OK') # cluster stop and clean if deploy_state == DEPLOYED and clean: center = Center() cur_cluster_id = config.get_cur_cluster_id(allow_empty_id=True) run_cluster_use(cluster_id) center.update_ip_port() center.stop_redis() center.remove_all_of_redis_log_force() center.cluster_clean() run_cluster_use(cur_cluster_id) # backup conf if deploy_state == DEPLOYED: Center().conf_backup(local_ip, cluster_id, conf_backup_dir) # backup cluster backup_hosts = [] if deploy_state == DEPLOYED: backup_hosts += set(pre_hosts) # if force: # backup_hosts += added_hosts for host in backup_hosts: cluster_path = path_of_fb['cluster_path'] client = net.get_ssh(host) Center().cluster_backup(host, cluster_id, cluster_backup_dir) client.close() # transfer & install msg = message.get('transfer_and_execute_installer') logger.info(msg) target_hosts = hosts + [local_ip] if no_localhost else hosts for host in target_hosts: if not (no_localhost and Center().is_localhost(host)): logger.info(' - {}'.format(host)) client = net.get_ssh(host) cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path) net.ssh_execute(client=client, command=cmd) client.close() DeployUtil().transfer_installer(host, cluster_id, installer_path) try: DeployUtil().install(host, cluster_id, installer_name) except SSHCommandError as ex: msg = message.get('error_execute_installer') msg = msg.format(installer=installer_path) logger.error(msg) logger.exception(ex) return # setup props if deploy_state == DEPLOYED: if restore_yes: tag = conf_backup_dir else: tag = tmp_backup_dir Center().conf_restore(local_ip, cluster_id, tag) else: key = 'sr2_redis_master_hosts' config.make_key_enable(props_path, key) config.set_props(props_path, key, props_dict['hosts']) key = 'sr2_redis_master_ports' config.make_key_enable(props_path, key) value = cluster_util.convert_list_2_seq(props_dict['master_ports']) config.set_props(props_path, key, value) key = 'sr2_redis_slave_hosts' config.make_key_enable(props_path, key) config.set_props(props_path, key, props_dict['hosts']) config.make_key_disable(props_path, key) if props_dict['replicas'] > 0: key = 'sr2_redis_slave_hosts' config.make_key_enable(props_path, key) key = 'sr2_redis_slave_ports' config.make_key_enable(props_path, key) value = cluster_util.convert_list_2_seq(props_dict['slave_ports']) config.set_props(props_path, key, value) key = 'ssd_count' config.make_key_enable(props_path, key) config.set_props(props_path, key, props_dict['ssd_count']) key = 'sr2_redis_data' config.make_key_enable(props_path, key, v1_flg=True) config.make_key_enable(props_path, key, v1_flg=True) config.make_key_disable(props_path, key) config.set_props(props_path, key, props_dict['prefix_of_db_path']) key = 'sr2_redis_db_path' config.make_key_enable(props_path, key, v1_flg=True) config.make_key_enable(props_path, key, v1_flg=True) config.make_key_disable(props_path, key) config.set_props(props_path, key, props_dict['prefix_of_db_path']) key = 'sr2_flash_db_path' config.make_key_enable(props_path, key, v1_flg=True) config.make_key_enable(props_path, key, v1_flg=True) config.make_key_disable(props_path, key) config.set_props(props_path, key, props_dict['prefix_of_db_path']) # synk props msg = message.get('sync_conf') logger.info(msg) for node in hosts: if socket.gethostbyname(node) in config.get_local_ip_list(): continue client = net.get_ssh(node) if not client: msg = message.get('error_ssh_connection').format(host=node) logger.error(msg) return net.copy_dir_to_remote(client, conf_path, conf_path) client.close() # set deploy state complete if os.path.exists(tmp_backup_path): shutil.rmtree(tmp_backup_path) for node in target_hosts: path_of_fb = config.get_path_of_fb(cluster_id) cluster_path = path_of_fb['cluster_path'] client = net.get_ssh(node) cmd = 'rm -rf {}'.format(os.path.join(cluster_path, '.deploy.state')) net.ssh_execute(client=client, command=cmd) client.close() if no_localhost: os.system('touch {}/remote'.format(cluster_path)) msg = message.get('complete_deploy').format(cluster_id=cluster_id) logger.info(msg) Cluster().use(cluster_id) msg = message.get('suggest_after_deploy') logger.info(msg)
def restore(self, cluster_id, tag=None): """Restore cluster :param cluster_id: target cluster id :param tag: Tag of backup, if omitted, restore the most recent backup file """ logger.debug('cluster restore: cluster_id={}, tag={}'.format( cluster_id, tag )) if not cluster_util.validate_id(cluster_id): raise ClusterIdError(cluster_id) # find restore folder with tag (local) path_of_fb = config.get_path_of_fb(cluster_id) cluster_backup_path = path_of_fb['cluster_backup_path'] if tag is None: backup_list = os.listdir(cluster_backup_path) pattern = 'cluster_{}_bak_'.format(cluster_id) filtered = filter(lambda x: x.startswith(pattern), backup_list) sorted_list = sorted(list(filtered)) if not sorted_list: msg = message.get('error_not_found_any_backup') logger.error('BackupNotExistError: ' + msg) return tag = sorted_list[-1] logger.debug("tag option is empty, auto select: {}".format(tag)) cluster_restore_dir = tag backup_path = os.path.join(cluster_backup_path, cluster_restore_dir) if not os.path.isdir(backup_path): msg = message.get('error_not_found_backup').format(tag=tag) logger.error('BackupNotExistError: ' + msg) return # get hosts from cluster props props_path = os.path.join( backup_path, 'tsr2-assembly-1.0.0-SNAPSHOT', 'conf', 'redis.properties' ) hosts = config.get_props(props_path, 'sr2_redis_master_hosts', []) # check status of hosts success = Center().check_hosts_connection(hosts, True) if not success: msg = message.get('error_exist_unavailable_host') logger.error(msg) return logger.debug('Connection of all hosts ok.') success = Center().check_include_localhost(hosts) if not success: msg = message.get('error_not_include_localhost') logger.error(msg) return # check all host tag folder: OK / NOT FOUND msg = message.get('check_backup_info') logger.info(msg) buf = [] for host in hosts: client = net.get_ssh(host) if not net.is_dir(client, backup_path): logger.debug('cannot find backup dir: {}-{}'.format( host, cluster_restore_dir )) buf.append([host, color.red('NOT FOUND')]) client.close() if buf: utils.print_table([['HOST', 'RESULT'] + buf]) return logger.info('OK') # backup cluster new_tag = time.strftime("%Y%m%d%H%M%S", time.gmtime()) cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, new_tag) for host in hosts: Center().cluster_backup(host, cluster_id, cluster_backup_dir) # restore cluster command = "cp -r {} {}/cluster_{}".format( backup_path, path_of_fb['base_directory'], cluster_id ) for host in hosts: msg = message.get('restore_cluster') msg = msg.format(tag=cluster_backup_dir, host=host) logger.info(msg) client = net.get_ssh(host) net.ssh_execute(client, command) client.close() logger.info("OK")