def run_delete(cluster_id=None): if not cluster_id: cluster_id = config.get_cur_cluster_id() valid = cluster_util.validate_id(cluster_id) if not valid: logger.error('Invalid cluster id: {}'.format(cluster_id)) return if cluster_id not in cluster_util.get_cluster_list(): logger.error('Cluster not exist: {}'.format(cluster_id)) return cluster = Cluster() cluster.stop(force=True, reset=True) # delete cluster folder cli_cluster_path = config.get_path_of_cli(cluster_id)["cluster_path"] path_of_fb = config.get_path_of_fb(cluster_id) fb_cluster_path = path_of_fb["cluster_path"] props_path = path_of_fb['redis_properties'] key = 'sr2_redis_master_hosts' nodes = config.get_props(props_path, key, []) for node in nodes: client = get_ssh(node) cmd = [ 'rm -rf {};'.format(fb_cluster_path), 'rm -rf {}'.format(cli_cluster_path), ] ssh_execute(client=client, command=' '.join(cmd)) client.close() if config.get_cur_cluster_id() == cluster_id: cluster.use(-1)
def get_alive_redis_process_count(self): """Calculate alive process count :return: alive process count """ self._update_ip_port() ps_list_command = get_ps_list_command(self.master_port_list + self.slave_port_list) command = '''{ps_list_command} | wc -l'''.format( ps_list_command=ps_list_command) logger.debug(command) command2 = """ps -ef | grep 'redis-rdb-to-slaves' | grep -v 'grep' | wc -l""" total_count = 0 for ip in self.ip_list: client = get_ssh(ip) exit_status, stdout_msg, stderr_msg = ssh_execute(client=client, command=command) c = int(stdout_msg) total_count += c exit_status, stdout_msg, stderr_msg = ssh_execute(client=client, command=command2) c = int(stdout_msg) total_count += c return total_count
def _rm_nodes_conf(self, ip_list, master_port_list, slave_port_list): for ip in ip_list: cmds = [] self.__append_nodes_conf(ip, master_port_list, cmds) self.__append_nodes_conf(ip, slave_port_list, cmds) command = ' '.join(cmds) logger.debug(command) client = get_ssh(ip) ssh_execute(client=client, command=command, allow_status=[0, 1])
def _send_stop_signal(self, force=False): ip, port, _ = self._get_thriftserver_info() pid_list = "ps -ef | grep 'thriftserver' | awk '{{print $2}}'" signal = 'SIGKILL' if force else 'SIGINT' command = 'kill -s {signal} $({pid_list})' \ .format(pid_list=pid_list, signal=signal) client = get_ssh(ip) ssh_execute(client=client, command=command, allow_status=[-1, 0, 1, 123, 130])
def _rm_data(self, ip_list, master_port_list, slave_port_list): for ip in ip_list: remove_dirs = [] self.__append_data_dirs(ip, master_port_list, remove_dirs) self.__append_data_dirs(ip, slave_port_list, remove_dirs) folders = ' '.join(remove_dirs) command = 'rm -fr {folders}'.format(folders=folders) logger.debug(command) client = get_ssh(ip) ssh_execute(client=client, command=command)
def install_v1(nodes, cluster_id, release, repo_path, tsr2_service_dir, report): installer = path_join(repo_path, 'releases', release) dest = path_join(tsr2_service_dir, 'cluster_%d' % cluster_id) command = '''chmod 755 {installer}; \ PATH=${{PATH}}:/usr/sbin; \ {installer} --full {dest}'''.format(installer=installer, dest=dest) for node in nodes: client = get_ssh(node) ssh_execute(client=client, command=command) report.success()
def create_redis_log_directory(self): """Create redis log directory using ssh and mkdir """ logger.debug('create_redis_log_directory start') self._update_ip_port() for ip in self.ip_list: envs = get_env_dict(self.ip_list, 0) client = get_ssh(ip) ssh_execute(client=client, command='mkdir -p %s' % envs['sr2_redis_log']) logger.debug('create_redis_log_directory complete')
def install(self, host, cluster_id, name): logger.debug('Deploy cluster {} at {}...'.format(cluster_id, host)) path_of_fb = config.get_path_of_fb(cluster_id) release_path = path_of_fb['release_path'] cluster_path = path_of_fb['cluster_path'] if '/' in name: name = os.path.basename(name) installer_path = path_join(release_path, name) command = '''chmod 755 {0}; \ PATH=${{PATH}}:/usr/sbin; \ {0} --full {1}'''.format(installer_path, cluster_path) client = get_ssh(host) ssh_execute(client=client, command=command) client.close() logger.debug('OK')
def overwrite_conf(report): logger.debug('_overwrite_conf') fb_config = config.get_config() nodes = fb_config['nodes'] cluster_home = config.get_repo_cluster_path() tsr2_home = config.get_tsr2_home() src = path_join(cluster_home, 'tsr2-conf') dest = path_join(tsr2_home, 'conf') command = '''cp -a {src}/* {dest}'''.format(src=src, dest=dest) for node in nodes: client = get_ssh(node) ssh_execute(client=client, command=command) logger.debug('overwrite_conf success [{node}]'.format(node=node)) if report: report.success()
def create_redis_data_directory(self): """Create redis data directory Create redis data directory using ssh and mkdir """ logger.debug('create_redis_data_directory start.') self._update_ip_port() targets = get_ip_port_tuple_list( self.ip_list, self.master_port_list + self.slave_port_list) for ip, port in targets: envs = get_env_dict(ip, port) redis_data = envs['sr2_redis_data'] flash_db_path = envs['sr2_flash_db_path'] client = get_ssh(ip) ssh_execute(client=client, command='mkdir -p %s %s' % (redis_data, flash_db_path)) logger.debug('create_redis_data_directory complete.')
def start(self): """Start thriftserver """ ip, port, cluster_id = self._get_thriftserver_info() logger.debug('Start thriftserver (%s:%d)' % (ip, port)) client = get_ssh(ip) if not client: logger.info('! ssh connection fail: %s' % ip) return exec_file = path_join(config.get_tsr2_home(cluster_id), 'sbin', 'thriftserver') env = '' # TODO: import env command = '''{env} ; {exec_file} start &'''.format(env=env, exec_file=exec_file) logger.info(command) ssh_execute(client=client, command=command)
def backup_server_logs(self): """Backup server logs""" logger.debug('backup_server_logs') self._update_ip_port() for ip in self.ip_list: backup_path = self.__get_redis_log_backup_path() client = get_ssh(ip) ssh_execute(client=client, command='mkdir -p %s' % backup_path) for port in (self.master_port_list + self.slave_port_list): redis_log_path = path_join(config.get_tsr2_home(), 'logs', 'redis') command = '''mv {redis_log_path}/*{port}.log {backup_path} &> /dev/null'''.format( redis_log_path=redis_log_path, port=port, backup_path=backup_path) ssh_execute(client=client, command=command, allow_status=[0, 1])
def cluster_backup(self, host, cluster_id, tag): logger.info('Backup cluster {} at {}...'.format(cluster_id, host)) # prepare path_of_fb = config.get_path_of_fb(cluster_id) cluster_path = path_of_fb['cluster_path'] cluster_backup_path = path_of_fb['cluster_backup_path'] cluster_backup_tag_path = path_join(cluster_backup_path, tag) # back up cluster client = get_ssh(host) if not net.is_dir(client, cluster_backup_path): sftp = get_sftp(client) sftp.mkdir(cluster_backup_path) sftp.close() command = 'mv {} {}'.format(cluster_path, cluster_backup_tag_path) ssh_execute(client=client, command=command) client.close() logger.info('OK, {}'.format(tag))
def _update_redis_conf(self, client, template_path, redis_conf_dir, ip, port_list): for port in port_list: target_conf = path_join(redis_conf_dir, 'redis-{port}.conf'.format(port=port)) envs = get_env_dict(ip, port) export_envs = ''' export SR2_REDIS_DATA={sr2_redis_data} export SR2_REDIS_HOST={sr2_redis_host} export SR2_REDIS_PORT={sr2_redis_port} export SR2_FLASH_DB_PATH={sr2_flash_db_path} '''.format(**envs) command = '''{export_envs} cat {template_path} | envsubst > {target_conf}'''.format( export_envs=export_envs, template_path=template_path, target_conf=target_conf) ssh_execute(client=client, command=command)
def update_redis_conf(self): """Update redis config """ logger.debug('update_redis_conf') self._update_ip_port() cluster_id = config.get_cur_cluster_id() path_of_fb = config.get_path_of_fb(cluster_id) master_template_path = path_of_fb['master_template'] slave_template_path = path_of_fb['slave_template'] for ip in self.ip_list: client = get_ssh(ip) redis_conf_dir = path_join(path_of_fb['conf_path'], 'redis') ssh_execute(client=client, command='mkdir -p %s' % redis_conf_dir) self._update_redis_conf(client, master_template_path, redis_conf_dir, ip, self.master_port_list) self._update_redis_conf(client, slave_template_path, redis_conf_dir, ip, self.slave_port_list) client.close() logger.info('update_redis_conf complete')
def _get_alive_process_count(self): ps = "ps -ef | grep 'thriftserver'" command = '''{ps} | wc -l'''.format(ps=ps) total_count = 0 ip, port, _ = self._get_thriftserver_info() client = get_ssh(ip) exit_status, stdout_msg, stderr_msg = ssh_execute(client=client, command=command) c = int(stdout_msg) total_count += c return total_count
def _start_redis_process(self, ip, port, m_or_s): logger.info('[%s] Start redis (%s:%d)' % (m_or_s, ip, port)) client = get_ssh(ip) redis_server = path_join(config.get_tsr2_home(), 'bin', 'redis-server') conf = path_join(config.get_tsr2_home(), 'conf', 'redis', 'redis-{port}.conf'.format(port=port)) log_file = self.__get_redis_log_path_file(port) env = make_export_envs(ip, port) command = '''{env} ; {redis_server} {conf} >> {log_file} &'''.format( env=env, redis_server=redis_server, conf=conf, log_file=log_file) exit_status, stdout_msg, stderr_msg = ssh_execute(client=client, command=command)
def stop_redis(self, force=False): """Stop redis :param force: If true, send SIGKILL. If not, send SIGINT """ logger.debug('stop_all_redis start') # d = self.get_ip_port_dict_using_cluster_nodes_cmd() d = self.ip_list for ip in d: ports = self.master_port_list + self.slave_port_list ps_list_command = get_ps_list_command(ports) pid_list = "{ps_list_command} | awk '{{print $2}}'".format( ps_list_command=ps_list_command) signal = 'SIGKILL' if force else 'SIGINT' command = 'kill -s {signal} $({pid_list})' \ .format(pid_list=pid_list, signal=signal) client = get_ssh(ip) ssh_execute(client=client, command=command, allow_status=[-1, 0, 1, 2, 123, 130]) logger.debug('stop_redis, send ps kill signal to redis processes')
def transfer_installer(self, host, cluster_id, installer_path): installer_path = os.path.expanduser(installer_path) path_of_fb = config.get_path_of_fb(cluster_id) name = os.path.basename(installer_path) dst = path_join(path_of_fb['release_path'], name) client = get_ssh(host) sftp = get_sftp(client) if not net.is_dir(client, path_of_fb['release_path']): logger.debug("Not exist releases directory at '{}'".format(host)) sftp.mkdir(path_of_fb['release_path']) logger.debug("Create releases directory at '{}'".format(host)) logger.debug('Check {}...'.format(name)) if not net.is_exist(client, dst): logger.debug('FAIL') logger.debug("Transfer '{}' to '{}'...".format(name, host)) sftp.put(installer_path, '{}.download'.format(dst)) command = 'mv {0}.download {0}'.format(dst) ssh_execute(client=client, command=command) sftp.close() client.close() logger.debug('OK')
def _deploy(cluster_id, history_save): deploy_state = DeployUtil().get_state(cluster_id) if deploy_state == DEPLOYED: q = [ color.YELLOW, '(Watch out) ', 'Cluster {} is already deployed. '.format(cluster_id), 'Do you want to deploy again?', color.ENDC, ] yes = askBool(''.join(q), default='n') if not yes: logger.info('Cancel deploy.') return restore_yes = None current_time = strftime("%Y%m%d%H%M%s", gmtime()) cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time) conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time) tmp_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, 'tmp') meta = [['NAME', 'VALUE']] path_of_fb = config.get_path_of_fb(cluster_id) conf_path = path_of_fb['conf_path'] props_path = path_of_fb['redis_properties'] cluster_path = path_of_fb['cluster_path'] path_of_cli = config.get_path_of_cli(cluster_id) conf_backup_path = path_of_cli['conf_backup_path'] tmp_backup_path = path_join(conf_backup_path, tmp_backup_dir) local_ip = config.get_local_ip() # ask installer installer_path = ask_util.installer() installer_name = os.path.basename(installer_path) meta.append(['installer', installer_name]) # ask restore conf if deploy_state == DEPLOYED: restore_yes = ask_util.askBool('Do you want to restore conf?') meta.append(['restore', restore_yes]) # input props hosts = [] if deploy_state == DEPLOYED: if restore_yes: meta += DeployUtil().get_meta_from_props(props_path) hosts = config.get_props(props_path, 'sr2_redis_master_hosts') else: if os.path.exists(tmp_backup_path): q = 'There is a history of modification. Do you want to load?' yes = ask_util.askBool(q) if not yes: shutil.rmtree(tmp_backup_path) if not os.path.exists(tmp_backup_path): shutil.copytree(conf_path, tmp_backup_path) tmp_props_path = path_join(tmp_backup_path, 'redis.properties') editor.edit(tmp_props_path, syntax='sh') meta += DeployUtil().get_meta_from_props(tmp_props_path) hosts = config.get_props(tmp_props_path, 'sr2_redis_master_hosts') else: props_dict = ask_util.props(cluster_id, save=history_save) hosts = props_dict['hosts'] meta += DeployUtil().get_meta_from_dict(props_dict) utils.print_table(meta) msg = [ 'Do you want to proceed with the deploy ', 'accroding to the above information?', ] yes = askBool(''.join(msg)) if not yes: logger.info("Cancel deploy.") return # check node status logger.info('Check status of hosts...') success = Center().check_hosts_connection(hosts, True) if not success: logger.error('There are unavailable host') return logger.debug('Connection of all hosts ok') success = Center().check_include_localhost(hosts) if not success: logger.error('Must include localhost') return # if pending, delete legacy on each hosts for host in hosts: if DeployUtil().get_state(cluster_id, host) == PENDING: client = get_ssh(host) command = 'rm -rf {}'.format(cluster_path) ssh_execute(client=client, command=command) client.close() # added_hosts = post_hosts - pre_hosts logger.info('Checking for cluster exist...') meta = [['HOST', 'STATUS']] added_hosts = set(hosts) if deploy_state == DEPLOYED: pre_hosts = config.get_props(props_path, 'sr2_redis_master_hosts') added_hosts -= set(pre_hosts) can_deploy = True for host in added_hosts: client = get_ssh(host) if net.is_exist(client, cluster_path): meta.append([host, color.red('CLUSTER EXIST')]) can_deploy = False continue meta.append([host, color.green('CLEAN')]) utils.print_table(meta) if not can_deploy: logger.error('Cluster information exist on some hosts.') return # if not force: # logger.error("If you trying to force, use option '--force'") # return # backup conf if deploy_state == DEPLOYED: Center().conf_backup(local_ip, cluster_id, conf_backup_dir) # backup cluster backup_hosts = [] if deploy_state == DEPLOYED: backup_hosts += set(pre_hosts) # if force: # backup_hosts += added_hosts for host in backup_hosts: cluster_path = path_of_fb['cluster_path'] client = get_ssh(host) Center().cluster_backup(host, cluster_id, cluster_backup_dir) client.close() # transfer & install logger.info('Transfer installer and execute...') for host in hosts: logger.info(host) client = get_ssh(host) cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path) ssh_execute(client=client, command=cmd) client.close() DeployUtil().transfer_installer(host, cluster_id, installer_path) DeployUtil().install(host, cluster_id, installer_name) # setup props if deploy_state == DEPLOYED: if restore_yes: tag = conf_backup_dir else: tag = tmp_backup_dir Center().conf_restore(local_ip, cluster_id, tag) else: key = 'sr2_redis_master_hosts' config.make_key_enable(props_path, key) config.set_props(props_path, key, props_dict['hosts']) key = 'sr2_redis_master_ports' config.make_key_enable(props_path, key) value = cluster_util.convert_list_2_seq(props_dict['master_ports']) config.set_props(props_path, key, value) key = 'sr2_redis_slave_hosts' config.make_key_enable(props_path, key) config.set_props(props_path, key, props_dict['hosts']) config.make_key_disable(props_path, key) if props_dict['replicas'] > 0: key = 'sr2_redis_slave_hosts' config.make_key_enable(props_path, key) key = 'sr2_redis_slave_ports' config.make_key_enable(props_path, key) value = cluster_util.convert_list_2_seq(props_dict['slave_ports']) config.set_props(props_path, key, value) key = 'ssd_count' config.make_key_enable(props_path, key) config.set_props(props_path, key, props_dict['ssd_count']) key = 'sr2_redis_data' config.make_key_enable(props_path, key, v1_flg=True) config.make_key_enable(props_path, key, v1_flg=True) config.make_key_disable(props_path, key) config.set_props(props_path, key, props_dict['prefix_of_rdp']) key = 'sr2_redis_db_path' config.make_key_enable(props_path, key, v1_flg=True) config.make_key_enable(props_path, key, v1_flg=True) config.make_key_disable(props_path, key) config.set_props(props_path, key, props_dict['prefix_of_rdbp']) key = 'sr2_flash_db_path' config.make_key_enable(props_path, key, v1_flg=True) config.make_key_enable(props_path, key, v1_flg=True) config.make_key_disable(props_path, key) config.set_props(props_path, key, props_dict['prefix_of_fdbp']) # synk props logger.info('Sync conf...') for node in hosts: if socket.gethostbyname(node) in config.get_local_ip_list(): continue client = get_ssh(node) if not client: logger.error("ssh connection fail: '{}'".format(node)) return net.copy_dir_to_remote(client, conf_path, conf_path) client.close() # set deploy state complete if os.path.exists(tmp_backup_path): shutil.rmtree(tmp_backup_path) for node in hosts: home_path = net.get_home_path(node) if not home_path: return path_of_fb = config.get_path_of_fb(cluster_id) cluster_path = path_of_fb['cluster_path'] client = get_ssh(node) command = 'rm -rf {}'.format(path_join(cluster_path, '.deploy.state')) ssh_execute(client=client, command=command) client.close() logger.info('Complete to deploy cluster {}'.format(cluster_id)) Cluster().use(cluster_id)