def run_delete(cluster_id=None):
if not cluster_id:
cluster_id = config.get_cur_cluster_id()
valid = cluster_util.validate_id(cluster_id)
if not valid:
logger.error('Invalid cluster id: {}'.format(cluster_id))
return
if cluster_id not in cluster_util.get_cluster_list():
logger.error('Cluster not exist: {}'.format(cluster_id))
return
cluster = Cluster()
cluster.stop(force=True, reset=True)
# delete cluster folder
cli_cluster_path = config.get_path_of_cli(cluster_id)["cluster_path"]
path_of_fb = config.get_path_of_fb(cluster_id)
fb_cluster_path = path_of_fb["cluster_path"]
props_path = path_of_fb['redis_properties']
key = 'sr2_redis_master_hosts'
nodes = config.get_props(props_path, key, [])
for node in nodes:
client = get_ssh(node)
cmd = [
'rm -rf {};'.format(fb_cluster_path),
'rm -rf {}'.format(cli_cluster_path),
]
ssh_execute(client=client, command=' '.join(cmd))
client.close()
if config.get_cur_cluster_id() == cluster_id:
cluster.use(-1)
def check_hosts_connection(self, hosts, show_result=False):
host_status = []
success_count = 0
for host in hosts:
try:
client = get_ssh(host)
client.close()
logger.debug('{} ssh... OK'.format(host))
success_count += 1
host_status.append([host, color.green('OK')])
except HostNameError:
host_status.append([host, color.red('UNKNOWN HOST')])
logger.debug('{} gethostbyname... FAIL'.format(host))
except HostConnectionError:
host_status.append([host, color.red('CONNECTION FAIL')])
logger.debug('{} connection... FAIL'.format(host))
except SSHConnectionError:
host_status.append([host, color.red('SSH FAIL')])
logger.debug('{} ssh... FAIL'.format(host))
if show_result:
table = AsciiTable([['HOST', 'STATUS']] + host_status)
print(table.table)
if len(hosts) != success_count:
return False
return True
def get_alive_redis_process_count(self):
"""Calculate alive process count
:return: alive process count
"""
self._update_ip_port()
ps_list_command = get_ps_list_command(self.master_port_list +
self.slave_port_list)
command = '''{ps_list_command} | wc -l'''.format(
ps_list_command=ps_list_command)
logger.debug(command)
command2 = """ps -ef | grep 'redis-rdb-to-slaves' | grep -v 'grep' | wc -l"""
total_count = 0
for ip in self.ip_list:
client = get_ssh(ip)
exit_status, stdout_msg, stderr_msg = ssh_execute(client=client,
command=command)
c = int(stdout_msg)
total_count += c
exit_status, stdout_msg, stderr_msg = ssh_execute(client=client,
command=command2)
c = int(stdout_msg)
total_count += c
return total_count
def rsync_fb_conf():
logger.info('Sync conf...')
cluster_id = config.get_cur_cluster_id()
if not validate_id(cluster_id):
logger.warn('Invalid cluster id: {}'.format(cluster_id))
return
cluster_list = get_cluster_list()
if cluster_id not in cluster_list:
logger.warn('Cluster not exist: {}'.format(cluster_id))
return
my_address = config.get_local_ip_list()
path_of_fb = config.get_path_of_fb(cluster_id)
props_path = path_of_fb['redis_properties']
key = 'sr2_redis_master_hosts'
nodes = config.get_props(props_path, key, [])
meta = [['HOST', 'RESULT']]
path_of_fb = config.get_path_of_fb(cluster_id)
conf_path = path_of_fb['conf_path']
cluster_path = path_of_fb['cluster_path']
for node in nodes:
if net.get_ip(node) in my_address:
meta.append([node, color.green('OK')])
continue
client = net.get_ssh(node)
if not client:
meta.append([node, color.red('SSH ERROR')])
continue
if not net.is_dir(client, cluster_path):
meta.append([node, color.red('NO CLUSTER')])
continue
net.copy_dir_to_remote(client, conf_path, conf_path)
meta.append([node, color.green('OK')])
utils.print_table(meta)
def _rm_nodes_conf(self, ip_list, master_port_list, slave_port_list):
for ip in ip_list:
cmds = []
self.__append_nodes_conf(ip, master_port_list, cmds)
self.__append_nodes_conf(ip, slave_port_list, cmds)
command = ' '.join(cmds)
logger.debug(command)
client = get_ssh(ip)
ssh_execute(client=client, command=command, allow_status=[0, 1])
def _rm_data(self, ip_list, master_port_list, slave_port_list):
for ip in ip_list:
remove_dirs = []
self.__append_data_dirs(ip, master_port_list, remove_dirs)
self.__append_data_dirs(ip, slave_port_list, remove_dirs)
folders = ' '.join(remove_dirs)
command = 'rm -fr {folders}'.format(folders=folders)
logger.debug(command)
client = get_ssh(ip)
ssh_execute(client=client, command=command)
def get_state(self, cluster_id, host='127.0.0.1'):
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
state_file = path_join(cluster_path, '.deploy.state')
client = get_ssh(host)
if net.is_exist(client, state_file):
return PENDING
if net.is_dir(client, cluster_path):
return DEPLOYED
return CLEAN
def _send_stop_signal(self, force=False):
ip, port, _ = self._get_thriftserver_info()
pid_list = "ps -ef | grep 'thriftserver' | awk '{{print $2}}'"
signal = 'SIGKILL' if force else 'SIGINT'
command = 'kill -s {signal} $({pid_list})' \
.format(pid_list=pid_list, signal=signal)
client = get_ssh(ip)
ssh_execute(client=client,
command=command,
allow_status=[-1, 0, 1, 123, 130])
def install_v1(nodes, cluster_id, release, repo_path, tsr2_service_dir,
report):
installer = path_join(repo_path, 'releases', release)
dest = path_join(tsr2_service_dir, 'cluster_%d' % cluster_id)
command = '''chmod 755 {installer}; \
PATH=${{PATH}}:/usr/sbin; \
{installer} --full {dest}'''.format(installer=installer, dest=dest)
for node in nodes:
client = get_ssh(node)
ssh_execute(client=client, command=command)
report.success()
def create_redis_log_directory(self):
"""Create redis log directory using ssh and mkdir
"""
logger.debug('create_redis_log_directory start')
self._update_ip_port()
for ip in self.ip_list:
envs = get_env_dict(self.ip_list, 0)
client = get_ssh(ip)
ssh_execute(client=client,
command='mkdir -p %s' % envs['sr2_redis_log'])
logger.debug('create_redis_log_directory complete')
def _get_alive_process_count(self):
ps = "ps -ef | grep 'thriftserver'"
command = '''{ps} | wc -l'''.format(ps=ps)
total_count = 0
ip, port, _ = self._get_thriftserver_info()
client = get_ssh(ip)
exit_status, stdout_msg, stderr_msg = ssh_execute(client=client,
command=command)
c = int(stdout_msg)
total_count += c
return total_count
def _start_redis_process(self, ip, port, m_or_s):
logger.info('[%s] Start redis (%s:%d)' % (m_or_s, ip, port))
client = get_ssh(ip)
redis_server = path_join(config.get_tsr2_home(), 'bin', 'redis-server')
conf = path_join(config.get_tsr2_home(), 'conf', 'redis',
'redis-{port}.conf'.format(port=port))
log_file = self.__get_redis_log_path_file(port)
env = make_export_envs(ip, port)
command = '''{env} ; {redis_server} {conf} >> {log_file} &'''.format(
env=env, redis_server=redis_server, conf=conf, log_file=log_file)
exit_status, stdout_msg, stderr_msg = ssh_execute(client=client,
command=command)
def is_pending(self, cluster_id, nodes=['127.0.0.1']):
if type(nodes) is type(str()):
nodes = [nodes]
for node in nodes:
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
deploy_state = path_join(cluster_path, '.deploy.state')
client = get_ssh(node)
if net.is_exist(client, deploy_state):
client.close()
return True
client.close()
return False
def conf_restore(self, host, cluster_id, tag):
logger.debug('Restore conf to cluster {}...'.format(cluster_id))
# prepare
path_of_fb = config.get_path_of_fb(cluster_id)
path_of_cli = config.get_path_of_cli(cluster_id)
conf_path = path_of_fb['conf_path']
conf_backup_path = path_of_cli['conf_backup_path']
conf_backup_tag_path = path_join(conf_backup_path, tag)
# restore conf
client = get_ssh(host)
net.copy_dir_to_remote(client, conf_backup_tag_path, conf_path)
client.close()
logger.debug('OK')
def install(self, host, cluster_id, name):
logger.debug('Deploy cluster {} at {}...'.format(cluster_id, host))
path_of_fb = config.get_path_of_fb(cluster_id)
release_path = path_of_fb['release_path']
cluster_path = path_of_fb['cluster_path']
if '/' in name:
name = os.path.basename(name)
installer_path = path_join(release_path, name)
command = '''chmod 755 {0}; \
PATH=${{PATH}}:/usr/sbin; \
{0} --full {1}'''.format(installer_path, cluster_path)
client = get_ssh(host)
ssh_execute(client=client, command=command)
client.close()
logger.debug('OK')
def overwrite_conf(report):
logger.debug('_overwrite_conf')
fb_config = config.get_config()
nodes = fb_config['nodes']
cluster_home = config.get_repo_cluster_path()
tsr2_home = config.get_tsr2_home()
src = path_join(cluster_home, 'tsr2-conf')
dest = path_join(tsr2_home, 'conf')
command = '''cp -a {src}/* {dest}'''.format(src=src, dest=dest)
for node in nodes:
client = get_ssh(node)
ssh_execute(client=client, command=command)
logger.debug('overwrite_conf success [{node}]'.format(node=node))
if report:
report.success()
def create_redis_data_directory(self):
"""Create redis data directory
Create redis data directory using ssh and mkdir
"""
logger.debug('create_redis_data_directory start.')
self._update_ip_port()
targets = get_ip_port_tuple_list(
self.ip_list, self.master_port_list + self.slave_port_list)
for ip, port in targets:
envs = get_env_dict(ip, port)
redis_data = envs['sr2_redis_data']
flash_db_path = envs['sr2_flash_db_path']
client = get_ssh(ip)
ssh_execute(client=client,
command='mkdir -p %s %s' % (redis_data, flash_db_path))
logger.debug('create_redis_data_directory complete.')
def start(self):
"""Start thriftserver
"""
ip, port, cluster_id = self._get_thriftserver_info()
logger.debug('Start thriftserver (%s:%d)' % (ip, port))
client = get_ssh(ip)
if not client:
logger.info('! ssh connection fail: %s' % ip)
return
exec_file = path_join(config.get_tsr2_home(cluster_id), 'sbin',
'thriftserver')
env = '' # TODO: import env
command = '''{env} ; {exec_file} start &'''.format(env=env,
exec_file=exec_file)
logger.info(command)
ssh_execute(client=client, command=command)
def backup_server_logs(self):
"""Backup server logs"""
logger.debug('backup_server_logs')
self._update_ip_port()
for ip in self.ip_list:
backup_path = self.__get_redis_log_backup_path()
client = get_ssh(ip)
ssh_execute(client=client, command='mkdir -p %s' % backup_path)
for port in (self.master_port_list + self.slave_port_list):
redis_log_path = path_join(config.get_tsr2_home(), 'logs',
'redis')
command = '''mv {redis_log_path}/*{port}.log {backup_path} &> /dev/null'''.format(
redis_log_path=redis_log_path,
port=port,
backup_path=backup_path)
ssh_execute(client=client,
command=command,
allow_status=[0, 1])
def cluster_backup(self, host, cluster_id, tag):
logger.info('Backup cluster {} at {}...'.format(cluster_id, host))
# prepare
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
cluster_backup_path = path_of_fb['cluster_backup_path']
cluster_backup_tag_path = path_join(cluster_backup_path, tag)
# back up cluster
client = get_ssh(host)
if not net.is_dir(client, cluster_backup_path):
sftp = get_sftp(client)
sftp.mkdir(cluster_backup_path)
sftp.close()
command = 'mv {} {}'.format(cluster_path, cluster_backup_tag_path)
ssh_execute(client=client, command=command)
client.close()
logger.info('OK, {}'.format(tag))
def update_redis_conf(self):
"""Update redis config
"""
logger.debug('update_redis_conf')
self._update_ip_port()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
master_template_path = path_of_fb['master_template']
slave_template_path = path_of_fb['slave_template']
for ip in self.ip_list:
client = get_ssh(ip)
redis_conf_dir = path_join(path_of_fb['conf_path'], 'redis')
ssh_execute(client=client, command='mkdir -p %s' % redis_conf_dir)
self._update_redis_conf(client, master_template_path,
redis_conf_dir, ip, self.master_port_list)
self._update_redis_conf(client, slave_template_path,
redis_conf_dir, ip, self.slave_port_list)
client.close()
logger.info('update_redis_conf complete')
def conf_backup(self, host, cluster_id, tag):
logger.info('Backup conf of cluster {}...'.format(cluster_id))
# prepare
path_of_fb = config.get_path_of_fb(cluster_id)
conf_path = path_of_fb['conf_path']
path_of_cli = config.get_path_of_cli(cluster_id)
conf_backup_path = path_of_cli['conf_backup_path']
conf_backup_tag_path = path_join(conf_backup_path, tag)
if not os.path.isdir(conf_backup_path):
os.mkdir(conf_backup_path)
# back up conf
os.mkdir(conf_backup_tag_path)
client = get_ssh(host)
net.copy_dir_from_remote(client, conf_path, conf_backup_tag_path)
client.close()
logger.info('OK, {}'.format(tag))
def run_monitor():
"""Run monitor command
Monitor remote logs
"""
ip_list = get_node_ip_list()
i = 1
msg = ''
for ip in ip_list:
msg += '%d) %s\n' % (i, ip)
i += 1
target_num = int(askInt(text=msg, default='1'))
logger.info('Ok. %s' % target_num)
target_index = target_num - 1
ip = ip_list[target_index]
client = get_ssh(ip)
envs = get_env_dict(ip, 0)
redis_log = envs['sr2_redis_log']
command = 'tail -f {redis_log}/*'.format(redis_log=redis_log)
ssh_execute_async(client=client, command=command)
def monitor(self):
"""Monitor thriftserver (using tail -f)
"""
cur_cluster_id = config.get_cur_cluster_id()
logger.info('Start monitor cluster_id: %s' % cur_cluster_id)
ip_list = config.get_node_ip_list()
i = 1
msg = ''
for ip in ip_list:
msg += '%d) %s\n' % (i, ip)
i += 1
target_num = int(askInt(text=msg, default='1'))
logger.info('Ok. %s' % target_num)
target_index = target_num - 1
ip = ip_list[target_index]
client = get_ssh(ip)
envs = config.get_env_dict(ip, 0)
tl = envs['sr2_redis_log'] # TODO: change redis log path
command = 'tail -f {tl}/*'.format(tl=tl)
ssh_execute_async(client=client, command=command)
def _initial_check():
client = get_ssh('localhost')
if not client:
logger.error('Need to ssh-keygen for localhost')
exit(1)
cli_config = config.get_cli_config()
try:
base_directory = cli_config['base_directory']
except KeyError:
pass
except TypeError:
root_of_cli_config = config.get_root_of_cli_config()
conf_path = path_join(root_of_cli_config, 'config')
os.system('rm {}'.format(conf_path))
base_directory = None
if not base_directory or not base_directory.startswith(('~', '/')):
base_directory = ask_util.base_directory()
base_directory = os.path.expanduser(base_directory)
if not os.path.isdir(base_directory):
os.system('mkdir -p {}'.format(base_directory))
def stop_redis(self, force=False):
"""Stop redis
:param force: If true, send SIGKILL. If not, send SIGINT
"""
logger.debug('stop_all_redis start')
# d = self.get_ip_port_dict_using_cluster_nodes_cmd()
d = self.ip_list
for ip in d:
ports = self.master_port_list + self.slave_port_list
ps_list_command = get_ps_list_command(ports)
pid_list = "{ps_list_command} | awk '{{print $2}}'".format(
ps_list_command=ps_list_command)
signal = 'SIGKILL' if force else 'SIGINT'
command = 'kill -s {signal} $({pid_list})' \
.format(pid_list=pid_list, signal=signal)
client = get_ssh(ip)
ssh_execute(client=client,
command=command,
allow_status=[-1, 0, 1, 2, 123, 130])
logger.debug('stop_redis, send ps kill signal to redis processes')
def transfer_installer(self, host, cluster_id, installer_path):
installer_path = os.path.expanduser(installer_path)
path_of_fb = config.get_path_of_fb(cluster_id)
name = os.path.basename(installer_path)
dst = path_join(path_of_fb['release_path'], name)
client = get_ssh(host)
sftp = get_sftp(client)
if not net.is_dir(client, path_of_fb['release_path']):
logger.debug("Not exist releases directory at '{}'".format(host))
sftp.mkdir(path_of_fb['release_path'])
logger.debug("Create releases directory at '{}'".format(host))
logger.debug('Check {}...'.format(name))
if not net.is_exist(client, dst):
logger.debug('FAIL')
logger.debug("Transfer '{}' to '{}'...".format(name, host))
sftp.put(installer_path, '{}.download'.format(dst))
command = 'mv {0}.download {0}'.format(dst)
ssh_execute(client=client, command=command)
sftp.close()
client.close()
logger.debug('OK')
def _deploy(cluster_id, history_save):
deploy_state = DeployUtil().get_state(cluster_id)
if deploy_state == DEPLOYED:
q = [
color.YELLOW,
'(Watch out) ',
'Cluster {} is already deployed. '.format(cluster_id),
'Do you want to deploy again?',
color.ENDC,
]
yes = askBool(''.join(q), default='n')
if not yes:
logger.info('Cancel deploy.')
return
restore_yes = None
current_time = strftime("%Y%m%d%H%M%s", gmtime())
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time)
conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time)
tmp_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, 'tmp')
meta = [['NAME', 'VALUE']]
path_of_fb = config.get_path_of_fb(cluster_id)
conf_path = path_of_fb['conf_path']
props_path = path_of_fb['redis_properties']
cluster_path = path_of_fb['cluster_path']
path_of_cli = config.get_path_of_cli(cluster_id)
conf_backup_path = path_of_cli['conf_backup_path']
tmp_backup_path = path_join(conf_backup_path, tmp_backup_dir)
local_ip = config.get_local_ip()
# ask installer
installer_path = ask_util.installer()
installer_name = os.path.basename(installer_path)
meta.append(['installer', installer_name])
# ask restore conf
if deploy_state == DEPLOYED:
restore_yes = ask_util.askBool('Do you want to restore conf?')
meta.append(['restore', restore_yes])
# input props
hosts = []
if deploy_state == DEPLOYED:
if restore_yes:
meta += DeployUtil().get_meta_from_props(props_path)
hosts = config.get_props(props_path, 'sr2_redis_master_hosts')
else:
if os.path.exists(tmp_backup_path):
q = 'There is a history of modification. Do you want to load?'
yes = ask_util.askBool(q)
if not yes:
shutil.rmtree(tmp_backup_path)
if not os.path.exists(tmp_backup_path):
shutil.copytree(conf_path, tmp_backup_path)
tmp_props_path = path_join(tmp_backup_path, 'redis.properties')
editor.edit(tmp_props_path, syntax='sh')
meta += DeployUtil().get_meta_from_props(tmp_props_path)
hosts = config.get_props(tmp_props_path, 'sr2_redis_master_hosts')
else:
props_dict = ask_util.props(cluster_id, save=history_save)
hosts = props_dict['hosts']
meta += DeployUtil().get_meta_from_dict(props_dict)
utils.print_table(meta)
msg = [
'Do you want to proceed with the deploy ',
'accroding to the above information?',
]
yes = askBool(''.join(msg))
if not yes:
logger.info("Cancel deploy.")
return
# check node status
logger.info('Check status of hosts...')
success = Center().check_hosts_connection(hosts, True)
if not success:
logger.error('There are unavailable host')
return
logger.debug('Connection of all hosts ok')
success = Center().check_include_localhost(hosts)
if not success:
logger.error('Must include localhost')
return
# if pending, delete legacy on each hosts
for host in hosts:
if DeployUtil().get_state(cluster_id, host) == PENDING:
client = get_ssh(host)
command = 'rm -rf {}'.format(cluster_path)
ssh_execute(client=client, command=command)
client.close()
# added_hosts = post_hosts - pre_hosts
logger.info('Checking for cluster exist...')
meta = [['HOST', 'STATUS']]
added_hosts = set(hosts)
if deploy_state == DEPLOYED:
pre_hosts = config.get_props(props_path, 'sr2_redis_master_hosts')
added_hosts -= set(pre_hosts)
can_deploy = True
for host in added_hosts:
client = get_ssh(host)
if net.is_exist(client, cluster_path):
meta.append([host, color.red('CLUSTER EXIST')])
can_deploy = False
continue
meta.append([host, color.green('CLEAN')])
utils.print_table(meta)
if not can_deploy:
logger.error('Cluster information exist on some hosts.')
return
# if not force:
# logger.error("If you trying to force, use option '--force'")
# return
# backup conf
if deploy_state == DEPLOYED:
Center().conf_backup(local_ip, cluster_id, conf_backup_dir)
# backup cluster
backup_hosts = []
if deploy_state == DEPLOYED:
backup_hosts += set(pre_hosts)
# if force:
# backup_hosts += added_hosts
for host in backup_hosts:
cluster_path = path_of_fb['cluster_path']
client = get_ssh(host)
Center().cluster_backup(host, cluster_id, cluster_backup_dir)
client.close()
# transfer & install
logger.info('Transfer installer and execute...')
for host in hosts:
logger.info(host)
client = get_ssh(host)
cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path)
ssh_execute(client=client, command=cmd)
client.close()
DeployUtil().transfer_installer(host, cluster_id, installer_path)
DeployUtil().install(host, cluster_id, installer_name)
# setup props
if deploy_state == DEPLOYED:
if restore_yes:
tag = conf_backup_dir
else:
tag = tmp_backup_dir
Center().conf_restore(local_ip, cluster_id, tag)
else:
key = 'sr2_redis_master_hosts'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['hosts'])
key = 'sr2_redis_master_ports'
config.make_key_enable(props_path, key)
value = cluster_util.convert_list_2_seq(props_dict['master_ports'])
config.set_props(props_path, key, value)
key = 'sr2_redis_slave_hosts'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['hosts'])
config.make_key_disable(props_path, key)
if props_dict['replicas'] > 0:
key = 'sr2_redis_slave_hosts'
config.make_key_enable(props_path, key)
key = 'sr2_redis_slave_ports'
config.make_key_enable(props_path, key)
value = cluster_util.convert_list_2_seq(props_dict['slave_ports'])
config.set_props(props_path, key, value)
key = 'ssd_count'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['ssd_count'])
key = 'sr2_redis_data'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_rdp'])
key = 'sr2_redis_db_path'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_rdbp'])
key = 'sr2_flash_db_path'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_fdbp'])
# synk props
logger.info('Sync conf...')
for node in hosts:
if socket.gethostbyname(node) in config.get_local_ip_list():
continue
client = get_ssh(node)
if not client:
logger.error("ssh connection fail: '{}'".format(node))
return
net.copy_dir_to_remote(client, conf_path, conf_path)
client.close()
# set deploy state complete
if os.path.exists(tmp_backup_path):
shutil.rmtree(tmp_backup_path)
for node in hosts:
home_path = net.get_home_path(node)
if not home_path:
return
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
client = get_ssh(node)
command = 'rm -rf {}'.format(path_join(cluster_path, '.deploy.state'))
ssh_execute(client=client, command=command)
client.close()
logger.info('Complete to deploy cluster {}'.format(cluster_id))
Cluster().use(cluster_id)
