def run_sync(host=None):
"""Import clusters from the host
"""
if host is None:
logger.error('host information is not available')
return None
cluster_base = config.get_base_directory()
if not os.path.exists(cluster_base):
logger.error('cluster does not exist on the localhost.')
os.mkdir(cluster_base)
cluster_set = set(
filter(lambda x: re.match(r'cluster_[\d]+', x),
os.listdir(cluster_base)))
client = net.get_ssh(host)
if not net.is_dir(client, cluster_base):
logger.error('cluster does not exist on the host({}).'.format(host))
return None
target_cluster_set = set(
filter(
lambda x: re.match(r'cluster_[\d]+', x),
net.ssh_execute(client, 'ls {}'.format(cluster_base))[1].split()))
conflict_cluster = cluster_set & target_cluster_set
import_target = (cluster_set ^ target_cluster_set) & target_cluster_set
for cluster in conflict_cluster:
msg = message.get('ask_cluster_overwrite').format(
cluster=" ".join(cluster.split('_')))
overwrite = ask_util.askBool(msg, default='n')
if overwrite:
import_target.add(cluster)
os.system("rm -rf {}".format(cluster_base + "/" + cluster))
for target in import_target:
os.system("rsync -a {} {}".format(
host + ":" + cluster_base + "/" + target, cluster_base))
logger.info("Importing cluster complete...")
def get_state(self, cluster_id, host='127.0.0.1'):
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
state_file = os.path.join(cluster_path, '.deploy.state')
client = net.get_ssh(host)
if net.is_exist(client, state_file):
return PENDING
if net.is_dir(client, cluster_path):
return DEPLOYED
return CLEAN
def is_pending(self, cluster_id, nodes=['127.0.0.1']):
if isinstance(nodes, str):
nodes = [nodes]
for node in nodes:
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
deploy_state = os.path.join(cluster_path, '.deploy.state')
client = net.get_ssh(node)
if net.is_exist(client, deploy_state):
client.close()
return True
client.close()
return False
def _initial_check():
try:
# Simple check to see if ssh access to localhost is possible
net.get_ssh('localhost')
except paramiko.ssh_exception.SSHException:
msg = message.get('error_ssh_connection').format(host='localhost')
logger.error(msg)
exit(1)
cli_config = config.get_cli_config()
try:
base_directory = cli_config['base_directory']
except KeyError:
pass
except TypeError:
root_of_cli_config = config.get_root_of_cli_config()
conf_path = os.path.join(root_of_cli_config, 'config')
os.system('rm {}'.format(conf_path))
base_directory = None
if not base_directory or not base_directory.startswith(('~', '/')):
base_directory = ask_util.base_directory()
base_directory = os.path.expanduser(base_directory)
if not os.path.isdir(base_directory):
os.system('mkdir -p {}'.format(base_directory))
def install(self, host, cluster_id, name):
logger.debug('Deploy cluster {} at {}...'.format(cluster_id, host))
path_of_fb = config.get_path_of_fb(cluster_id)
release_path = path_of_fb['release_path']
cluster_path = path_of_fb['cluster_path']
if '/' in name:
name = os.path.basename(name)
installer_path = os.path.join(release_path, name)
command = '''chmod 755 {0}; \
PATH=${{PATH}}:/usr/sbin; \
{0} --full {1}'''.format(installer_path, cluster_path)
client = net.get_ssh(host)
if not net.is_exist(client, installer_path):
raise FileNotExistError(installer_path, host=host)
net.ssh_execute(client=client, command=command)
client.close()
logger.debug('OK')
def run_monitor(n=10, t=2):
"""Monitoring logs of redis.
:param n: number of lines to print log
:param t: renewal cycle(sec)
"""
if not isinstance(n, int):
msg = message.get('error_option_type_not_number').format(option='n')
logger.error(msg)
return
if not isinstance(t, int) and not isinstance(t, float):
msg = message.get('error_option_type_not_float').format(option='t')
logger.error(msg)
return
try:
sp.check_output('which tail', shell=True)
except Exception:
msg = message.get('error_not_found_command_tail')
logger.error(msg)
return
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_log = path_of_fb['sr2_redis_log']
log_files = '{}/servers*'.format(sr2_redis_log)
host_list = config.get_master_host_list()
target_host = ask_util.host_for_monitor(host_list)
try:
sp.check_output('which watch', shell=True)
command = "ssh -t {} watch -n {} 'tail -n {} {}'".format(
target_host, t, n, log_files)
sp.call(command, shell=True)
except Exception:
msg = message.get('error_not_found_command_watch')
logger.warning(msg)
logger.info(message.get('message_for_exit'))
command = "tail -F -s {} {}".format(t, log_files)
client = net.get_ssh(target_host)
net.ssh_execute_async(client, command)
def transfer_installer(self, host, cluster_id, installer_path):
installer_path = os.path.expanduser(installer_path)
path_of_fb = config.get_path_of_fb(cluster_id)
name = os.path.basename(installer_path)
dst = os.path.join(path_of_fb['release_path'], name)
client = net.get_ssh(host)
sftp = net.get_sftp(client)
if not net.is_dir(client, path_of_fb['release_path']):
logger.debug("Not exist releases directory at '{}'".format(host))
sftp.mkdir(path_of_fb['release_path'])
logger.debug("Create releases directory at '{}'".format(host))
logger.debug('Check {}...'.format(name))
if not net.is_exist(client, dst):
logger.debug('FAIL')
logger.debug("Transfer '{}' to '{}'...".format(name, host))
sftp.put(installer_path, '{}.download'.format(dst))
command = 'mv {0}.download {0}'.format(dst)
net.ssh_execute(client=client, command=command)
sftp.close()
client.close()
logger.debug('OK')
def _deploy(cluster_id, history_save, clean):
deploy_state = DeployUtil().get_state(cluster_id)
if deploy_state == DEPLOYED:
msg = message.get('ask_deploy_again')
msg = msg.format(cluster_id=cluster_id)
msg = color.yellow(msg)
yes = ask_util.askBool(msg, default='n')
if not yes:
logger.info(message.get('cancel'))
return
restore_yes = None
no_localhost = False
current_time = time.strftime("%Y%m%d%H%M%S", time.gmtime())
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time)
conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time)
tmp_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, 'tmp')
meta = [['NAME', 'VALUE']]
path_of_fb = config.get_path_of_fb(cluster_id)
conf_path = path_of_fb['conf_path']
props_path = path_of_fb['redis_properties']
cluster_path = path_of_fb['cluster_path']
path_of_cli = config.get_path_of_cli(cluster_id)
conf_backup_path = path_of_cli['conf_backup_path']
tmp_backup_path = os.path.join(conf_backup_path, tmp_backup_dir)
local_ip = config.get_local_ip()
# ask installer
installer_path = ask_util.installer()
installer_name = os.path.basename(installer_path)
meta.append(['installer', installer_name])
# ask restore conf
if deploy_state == DEPLOYED:
restore_yes = ask_util.askBool(message.get('ask_restore_conf'))
meta.append(['restore', restore_yes])
# input props
hosts = []
if deploy_state == DEPLOYED:
if restore_yes:
meta += DeployUtil().get_meta_from_props(props_path)
hosts = config.get_props(props_path, 'sr2_redis_master_hosts')
else:
if not os.path.isdir(conf_backup_path):
os.mkdir(conf_backup_path)
if os.path.exists(tmp_backup_path):
msg = message.get('ask_load_history_of_previous_modification')
yes = ask_util.askBool(msg)
if not yes:
shutil.rmtree(tmp_backup_path)
if not os.path.exists(tmp_backup_path):
os.mkdir(tmp_backup_path)
shutil.copy(os.path.join(conf_path, 'redis.properties'),
os.path.join(tmp_backup_path, 'redis.properties'))
tmp_props_path = os.path.join(tmp_backup_path, 'redis.properties')
editor.edit(tmp_props_path, syntax='sh')
meta += DeployUtil().get_meta_from_props(tmp_props_path)
hosts = config.get_props(tmp_props_path, 'sr2_redis_master_hosts')
else:
# new deploy
props_dict = ask_util.props(cluster_id, save=history_save)
hosts = props_dict['hosts']
meta += DeployUtil().get_meta_from_dict(props_dict)
utils.print_table(meta)
msg = message.get('confirm_deploy_information')
yes = ask_util.askBool(msg)
if not yes:
logger.info(message.get('cancel'))
return
# check node status
success = Center().check_hosts_connection(hosts, True)
if not success:
msg = message.get('error_exist_unavailable_host')
logger.error(msg)
return
logger.debug('Connection of all hosts ok.')
success = Center().check_include_localhost(hosts)
if not success:
no_localhost = True
# get port info
if deploy_state == DEPLOYED:
if restore_yes:
key = 'sr2_redis_master_ports'
m_ports = config.get_props(props_path, key, [])
key = 'sr2_redis_slave_ports'
s_ports = config.get_props(props_path, key, [])
replicas = len(s_ports) // len(m_ports)
else:
key = 'sr2_redis_master_ports'
m_ports = config.get_props(tmp_props_path, key, [])
key = 'sr2_redis_slave_ports'
s_ports = config.get_props(tmp_props_path, key, [])
replicas = len(s_ports) // len(m_ports)
else:
m_ports = props_dict['master_ports']
s_ports = props_dict['slave_ports']
replicas = props_dict['replicas']
while True:
msg = message.get('check_port')
logger.info(msg)
host_ports_list = []
for host in hosts:
host_ports_list.append((host, m_ports + s_ports))
conflict = Center().check_port_is_enable(host_ports_list)
if not conflict:
logger.info("OK")
break
utils.print_table([["HOST", "PORT"]] + conflict)
msg = message.get('ask_port_collision')
msg = color.yellow(msg)
yes = ask_util.askBool(msg)
if yes:
logger.info("OK")
break
m_ports = ask_util.master_ports(False, cluster_id)
replicas = ask_util.replicas(False)
s_ports = ask_util.slave_ports(cluster_id, len(m_ports), replicas)
if deploy_state == DEPLOYED:
if restore_yes:
key = 'sr2_redis_master_ports'
value = cluster_util.convert_list_2_seq(m_ports)
config.set_props(props_path, key, value)
key = 'sr2_redis_slave_ports'
value = cluster_util.convert_list_2_seq(s_ports)
config.set_props(props_path, key, value)
else:
key = 'sr2_redis_master_ports'
value = cluster_util.convert_list_2_seq(m_ports)
config.set_props(tmp_props_path, key, value)
key = 'sr2_redis_slave_ports'
value = cluster_util.convert_list_2_seq(s_ports)
config.set_props(tmp_props_path, key, value)
else:
props_dict['master_ports'] = m_ports
props_dict['slave_ports'] = s_ports
props_dict['replicas'] = replicas
# if pending, delete legacy on each hosts
if no_localhost:
if DeployUtil().get_state(cluster_id, local_ip) == PENDING:
client = net.get_ssh(local_ip)
command = 'rm -rf {}'.format(cluster_path)
net.ssh_execute(client=client, command=command)
client.close()
for host in hosts:
if DeployUtil().get_state(cluster_id, host) == PENDING:
client = net.get_ssh(host)
command = 'rm -rf {}'.format(cluster_path)
net.ssh_execute(client=client, command=command)
client.close()
# added_hosts = post_hosts - pre_hosts
msg = message.get('check_cluster_exist')
logger.info(msg)
added_hosts = set(hosts)
meta = []
if deploy_state == DEPLOYED:
pre_hosts = config.get_props(props_path, 'sr2_redis_master_hosts')
added_hosts -= set(pre_hosts)
can_deploy = True
if no_localhost:
added_hosts |= set([local_ip])
for host in added_hosts:
client = net.get_ssh(host)
is_localhost = Center().is_localhost(host)
if is_localhost:
if no_localhost:
continue
if os.path.exists(cluster_path + '/remote'):
meta.append([host, color.green('CLEAN')])
continue
if net.is_exist(client, cluster_path):
meta.append([host, color.red('CLUSTER EXIST')])
can_deploy = False
continue
meta.append([host, color.green('CLEAN')])
if meta:
utils.print_table([['HOST', 'STATUS']] + meta)
if not can_deploy:
msg = message.get('error_cluster_collision')
logger.error(msg)
return
# if not force:
# logger.error("If you trying to force, use option '--force'")
# return
logger.info('OK')
# cluster stop and clean
if deploy_state == DEPLOYED and clean:
center = Center()
cur_cluster_id = config.get_cur_cluster_id(allow_empty_id=True)
run_cluster_use(cluster_id)
center.update_ip_port()
center.stop_redis()
center.remove_all_of_redis_log_force()
center.cluster_clean()
run_cluster_use(cur_cluster_id)
# backup conf
if deploy_state == DEPLOYED:
Center().conf_backup(local_ip, cluster_id, conf_backup_dir)
# backup cluster
backup_hosts = []
if deploy_state == DEPLOYED:
backup_hosts += set(pre_hosts)
# if force:
# backup_hosts += added_hosts
for host in backup_hosts:
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(host)
Center().cluster_backup(host, cluster_id, cluster_backup_dir)
client.close()
# transfer & install
msg = message.get('transfer_and_execute_installer')
logger.info(msg)
target_hosts = hosts + [local_ip] if no_localhost else hosts
for host in target_hosts:
if not (no_localhost and Center().is_localhost(host)):
logger.info(' - {}'.format(host))
client = net.get_ssh(host)
cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path)
net.ssh_execute(client=client, command=cmd)
client.close()
DeployUtil().transfer_installer(host, cluster_id, installer_path)
try:
DeployUtil().install(host, cluster_id, installer_name)
except SSHCommandError as ex:
msg = message.get('error_execute_installer')
msg = msg.format(installer=installer_path)
logger.error(msg)
logger.exception(ex)
return
# setup props
if deploy_state == DEPLOYED:
if restore_yes:
tag = conf_backup_dir
else:
tag = tmp_backup_dir
Center().conf_restore(local_ip, cluster_id, tag)
else:
key = 'sr2_redis_master_hosts'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['hosts'])
key = 'sr2_redis_master_ports'
config.make_key_enable(props_path, key)
value = cluster_util.convert_list_2_seq(props_dict['master_ports'])
config.set_props(props_path, key, value)
key = 'sr2_redis_slave_hosts'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['hosts'])
config.make_key_disable(props_path, key)
if props_dict['replicas'] > 0:
key = 'sr2_redis_slave_hosts'
config.make_key_enable(props_path, key)
key = 'sr2_redis_slave_ports'
config.make_key_enable(props_path, key)
value = cluster_util.convert_list_2_seq(props_dict['slave_ports'])
config.set_props(props_path, key, value)
key = 'ssd_count'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['ssd_count'])
key = 'sr2_redis_data'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
key = 'sr2_redis_db_path'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
key = 'sr2_flash_db_path'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
# synk props
msg = message.get('sync_conf')
logger.info(msg)
for node in hosts:
if socket.gethostbyname(node) in config.get_local_ip_list():
continue
client = net.get_ssh(node)
if not client:
msg = message.get('error_ssh_connection').format(host=node)
logger.error(msg)
return
net.copy_dir_to_remote(client, conf_path, conf_path)
client.close()
# set deploy state complete
if os.path.exists(tmp_backup_path):
shutil.rmtree(tmp_backup_path)
for node in target_hosts:
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(node)
cmd = 'rm -rf {}'.format(os.path.join(cluster_path, '.deploy.state'))
net.ssh_execute(client=client, command=cmd)
client.close()
if no_localhost:
os.system('touch {}/remote'.format(cluster_path))
msg = message.get('complete_deploy').format(cluster_id=cluster_id)
logger.info(msg)
Cluster().use(cluster_id)
msg = message.get('suggest_after_deploy')
logger.info(msg)
def _deploy_zero_downtime(cluster_id):
logger.debug("zero downtime update cluster {}".format(cluster_id))
center = Center()
center.update_ip_port()
m_hosts = center.master_host_list
m_ports = center.master_port_list
s_hosts = center.slave_host_list
s_ports = center.slave_port_list
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
# check master alive
m_count = len(m_hosts) * len(m_ports)
alive_m_count = center.get_alive_master_redis_count()
if alive_m_count < m_count:
logger.error(message.get('error_exist_disconnected_master'))
return
if not config.is_slave_enabled:
logger.error(message.get('error_need_to_slave'))
return
# select installer
installer_path = ask_util.installer()
installer_name = os.path.basename(installer_path)
# backup info
current_time = time.strftime("%Y%m%d%H%M%S", time.gmtime())
conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time)
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time)
local_ip = config.get_local_ip()
# backup conf
center.conf_backup(local_ip, cluster_id, conf_backup_dir)
# backup cluster
for host in s_hosts:
client = net.get_ssh(host)
center.cluster_backup(host, cluster_id, cluster_backup_dir)
client.close()
# transfer & install
logger.info(message.get('transfer_and_execute_installer'))
for host in m_hosts:
logger.info(' - {}'.format(host))
client = net.get_ssh(host)
cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path)
net.ssh_execute(client=client, command=cmd)
client.close()
DeployUtil().transfer_installer(host, cluster_id, installer_path)
try:
DeployUtil().install(host, cluster_id, installer_name)
except SSHCommandError as ex:
msg = message.get('error_execute_installer')
msg = msg.format(installer=installer_path)
logger.error(msg)
logger.exception(ex)
return
# restore conf
center.conf_restore(local_ip, cluster_id, conf_backup_dir)
# set deploy state complete
for node in m_hosts:
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(node)
cmd = 'rm -rf {}'.format(os.path.join(cluster_path, '.deploy.state'))
net.ssh_execute(client=client, command=cmd)
client.close()
# restart slave
center.stop_current_nodes(master=False, slave=True)
center.configure_redis()
center.sync_conf()
center.start_current_nodes(master=False, slave=True)
center.wait_until_all_redis_process_up()
slaves_for_failover = center.get_slave_nodes()
key = 'cluster-node-timeout'
origin_m_value = center.cli_config_get(key, m_hosts[0], m_ports[0])
origin_s_value = center.cli_config_get(key, s_hosts[0], s_ports[0])
logger.debug('config set: cluster-node-timeout 2000')
RedisCliConfig().set(key, '2000', all=True)
# cluster failover (with no option)
logger.info(message.get('failover_on_deploy'))
logger.debug(slaves_for_failover)
try_count = 0
while try_count < 10:
try_count += 1
success = True
for slave_addr in slaves_for_failover:
host, port = slave_addr.split(':')
stdout = center.run_failover("{}:{}".format(host, port))
logger.debug("failover {}:{} {}".format(host, port, stdout))
if stdout != "ERR You should send CLUSTER FAILOVER to a slave":
# In some cases, the cluster failover is not complete
# even if stdout is OK
# If redis changed to master completely,
# return 'ERR You should send CLUSTER FAILOVER to a slave'
success = False
if success:
break
msg = message.get('retry').format(try_count=try_count)
logger.info(msg)
time.sleep(5)
logger.debug('restore config: cluster-node-timeout')
center.cli_config_set_all(key, origin_m_value, m_hosts, m_ports)
center.cli_config_set_all(key, origin_s_value, s_hosts, s_ports)
if not success:
logger.error(message.get('error_redis_failover'))
return
# restart master (current slave)
center.stop_current_nodes(master=False, slave=True)
center.configure_redis(slave=False)
center.sync_conf()
center.start_current_nodes(master=False, slave=True)
center.wait_until_all_redis_process_up()
def restore(self, cluster_id, tag=None):
"""Restore cluster
:param cluster_id: target cluster id
:param tag: Tag of backup, if omitted, restore the most recent backup file
"""
logger.debug('cluster restore: cluster_id={}, tag={}'.format(
cluster_id,
tag
))
if not cluster_util.validate_id(cluster_id):
raise ClusterIdError(cluster_id)
# find restore folder with tag (local)
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_backup_path = path_of_fb['cluster_backup_path']
if tag is None:
backup_list = os.listdir(cluster_backup_path)
pattern = 'cluster_{}_bak_'.format(cluster_id)
filtered = filter(lambda x: x.startswith(pattern), backup_list)
sorted_list = sorted(list(filtered))
if not sorted_list:
msg = message.get('error_not_found_any_backup')
logger.error('BackupNotExistError: ' + msg)
return
tag = sorted_list[-1]
logger.debug("tag option is empty, auto select: {}".format(tag))
cluster_restore_dir = tag
backup_path = os.path.join(cluster_backup_path, cluster_restore_dir)
if not os.path.isdir(backup_path):
msg = message.get('error_not_found_backup').format(tag=tag)
logger.error('BackupNotExistError: ' + msg)
return
# get hosts from cluster props
props_path = os.path.join(
backup_path,
'tsr2-assembly-1.0.0-SNAPSHOT',
'conf',
'redis.properties'
)
hosts = config.get_props(props_path, 'sr2_redis_master_hosts', [])
# check status of hosts
success = Center().check_hosts_connection(hosts, True)
if not success:
msg = message.get('error_exist_unavailable_host')
logger.error(msg)
return
logger.debug('Connection of all hosts ok.')
success = Center().check_include_localhost(hosts)
if not success:
msg = message.get('error_not_include_localhost')
logger.error(msg)
return
# check all host tag folder: OK / NOT FOUND
msg = message.get('check_backup_info')
logger.info(msg)
buf = []
for host in hosts:
client = net.get_ssh(host)
if not net.is_dir(client, backup_path):
logger.debug('cannot find backup dir: {}-{}'.format(
host,
cluster_restore_dir
))
buf.append([host, color.red('NOT FOUND')])
client.close()
if buf:
utils.print_table([['HOST', 'RESULT'] + buf])
return
logger.info('OK')
# backup cluster
new_tag = time.strftime("%Y%m%d%H%M%S", time.gmtime())
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, new_tag)
for host in hosts:
Center().cluster_backup(host, cluster_id, cluster_backup_dir)
# restore cluster
command = "cp -r {} {}/cluster_{}".format(
backup_path,
path_of_fb['base_directory'],
cluster_id
)
for host in hosts:
msg = message.get('restore_cluster')
msg = msg.format(tag=cluster_backup_dir, host=host)
logger.info(msg)
client = net.get_ssh(host)
net.ssh_execute(client, command)
client.close()
logger.info("OK")