def delete(self, cluster_id):
"""Delete cluster
It is automatically backed up with timestamps as tags
:param cluster_id: target cluster id
"""
if not cluster_util.validate_id(cluster_id):
raise ClusterIdError(cluster_id)
center = Center()
center.update_ip_port()
success = center.check_hosts_connection()
if success:
center.stop_redis(force=True)
path_of_fb = config.get_path_of_fb(cluster_id)
props_path = path_of_fb['redis_properties']
hosts = config.get_props(props_path, 'sr2_redis_master_hosts', [])
if not center.check_include_localhost(hosts):
hosts += [config.get_local_ip()]
tag = time.strftime("%Y%m%d%H%M%S", time.gmtime())
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, tag)
for host in hosts:
center.cluster_backup(host, cluster_id, cluster_backup_dir)
msg = message.get('cluster_delete_complete')
msg = msg.format(cluster_id=cluster_id)
logger.info(msg)
def set(self, key, value, all=False, save=False, host=None, port=None):
"""Command: redis-cli config set
:param key: target key
:param value: value to set
:param save: If true, save value to config file
:param all: If true, send command to all redis
:param host: host info for redis
:param port: port info for redis
"""
if not isinstance(all, bool):
msg = m.get('error_option_type_not_boolean')
msg = msg.format(options='all')
logger.error(msg)
return
if not isinstance(save, bool):
msg = m.get('error_option_type_not_boolean')
msg = msg.format(options='save')
logger.error(msg)
return
if (not host or not port) and not all:
msg = m.get('use_host_port_or_option_all')
logger.error(msg)
return
sub_cmd = 'config set {key} {value} 2>&1'.format(key=key, value=value)
if all:
meta = []
ret = RedisCliUtil.command_all_async(sub_cmd)
ok_cnt = 0
for m_s, host, port, result, message in ret:
addr = '{}:{}'.format(host, port)
if result == 'OK':
if utils.to_str(message) == 'OK':
ok_cnt += 1
else:
meta.append([m_s, addr, color.red(message)])
else:
meta.append([m_s, addr, color.red('FAIL')])
if meta:
utils.print_table([['TYPE', 'ADDR', 'RESULT']] + meta)
logger.info('success {}/{}'.format(ok_cnt, len(ret)))
else:
output = RedisCliUtil.command(sub_cmd=sub_cmd,
host=host,
port=port,
formatter=self.no_print)
output = output.strip()
if output == "OK":
logger.info(output)
else:
logger.error(output)
if save:
RedisCliUtil.save_redis_template_config(key, value)
center = Center()
center.update_ip_port()
success = center.check_hosts_connection()
if not success:
return
center.configure_redis()
center.sync_conf()
def ping(host=None, port=None, all=False):
"""Send ping command
:param all: If true, send command to all
:param host: host info for redis
:param port: port info for redis
"""
if not isinstance(all, bool):
msg = message.get('error_option_type_not_boolean').format(option='all')
logger.error(msg)
return
if (not host or not port) and not all:
msg = message.get('use_host_port_or_option_all')
logger.error(msg)
return
if all:
meta = []
ret = RedisCliUtil.command_all_async('ping 2>&1')
pong_cnt = 0
for m_s, host, port, result, _ in ret:
addr = '{}:{}'.format(host, port)
if result == 'OK':
pong_cnt += 1
else:
meta.append([m_s, addr, color.red('FAIL')])
if meta:
utils.print_table([['TYPE', 'ADDR', 'RESULT']] + meta)
msg = message.get('counting_alive_redis')
msg = msg.format(alive=pong_cnt, total=len(ret))
logger.info(msg)
return
if host and port:
_command('ping', False, host, port)
def beeline(self, **kargs):
"""Connect to thriftserver command line
"""
logger.debug('thriftserver_command_beeline')
_check_spark()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
ths_props_path = path_of_fb['thrift_properties']
cmd = 'source {}; echo ${}'.format(ths_props_path, 'HIVE_HOST')
host = sp.check_output(cmd, shell=True).decode('utf-8').strip()
cmd = 'source {}; echo ${}'.format(ths_props_path, 'HIVE_PORT')
port = sp.check_output(cmd, shell=True).decode('utf-8').strip()
spark_env = _get_env()
base_cmd = '{}/beeline'.format(spark_env['spark_bin'])
options = {
'u': 'jdbc:hive2://{}:{}'.format(host, port),
'n': os.environ['USER']
}
for key, value in kargs.items():
options[key] = value
for key, value in options.items():
base_cmd += ' -{} {}'.format(key, value)
logger.debug(base_cmd)
msg = message.get('try_connection')
logger.info(msg)
os.system(base_cmd)
def command(sub_cmd,
cluster_id=-1,
mute=False,
formatter=None,
host=None,
port=0):
"""Send redis-cli command
:param sub_cmd: sub command
:param cluster_id: target cluster #
:param mute: without stdout
:param formatter: If set, call formatter with output string
:param host: host
:param port: port
:return: output string
"""
ip_list = config.get_node_ip_list(cluster_id)
port_list = config.get_master_port_list(cluster_id)
if host:
ip_list = [host]
port_list = [port]
outs = RedisCliUtil.command_raw(sub_cmd, ip_list, port_list)
if mute:
return outs
if formatter:
formatter(outs)
else:
logger.info(outs)
return outs
def run_sync(host=None):
"""Import clusters from the host
"""
if host is None:
logger.error('host information is not available')
return None
cluster_base = config.get_base_directory()
if not os.path.exists(cluster_base):
logger.error('cluster does not exist on the localhost.')
os.mkdir(cluster_base)
cluster_set = set(
filter(lambda x: re.match(r'cluster_[\d]+', x),
os.listdir(cluster_base)))
client = net.get_ssh(host)
if not net.is_dir(client, cluster_base):
logger.error('cluster does not exist on the host({}).'.format(host))
return None
target_cluster_set = set(
filter(
lambda x: re.match(r'cluster_[\d]+', x),
net.ssh_execute(client, 'ls {}'.format(cluster_base))[1].split()))
conflict_cluster = cluster_set & target_cluster_set
import_target = (cluster_set ^ target_cluster_set) & target_cluster_set
for cluster in conflict_cluster:
msg = message.get('ask_cluster_overwrite').format(
cluster=" ".join(cluster.split('_')))
overwrite = ask_util.askBool(msg, default='n')
if overwrite:
import_target.add(cluster)
os.system("rm -rf {}".format(cluster_base + "/" + cluster))
for target in import_target:
os.system("rsync -a {} {}".format(
host + ":" + cluster_base + "/" + target, cluster_base))
logger.info("Importing cluster complete...")
def tree(self):
"""The results of 'cli cluster nodes' are displayed in tree format
"""
center = Center()
center.update_ip_port()
master_node_list = center.get_master_obj_list()
output_msg = []
for master_node in master_node_list:
addr = master_node['addr']
status = master_node['status']
msg = '{}({})'.format(addr, status)
if status == 'disconnected':
msg = color.red(msg)
if status == 'paused':
msg = color.yellow(msg)
output_msg.append(msg)
for slave_node in master_node['slaves']:
addr = slave_node['addr']
status = slave_node['status']
msg = '{}({})'.format(addr, status)
if status == 'disconnected':
msg = color.red(msg)
if status == 'paused':
msg = color.yellow(msg)
output_msg.append('|__ ' + msg)
output_msg.append('')
logger.info(color.ENDC + '\n'.join(output_msg))
def version(self):
"""Get version of lightningDB
"""
cluster_id = config.get_cur_cluster_id()
tsr2_home = config.get_tsr2_home(cluster_id)
with open(os.path.join(tsr2_home, "VERSION"), "r") as version_file:
lines = version_file.readlines()
logger.info("".join(lines).strip())
def use(self, cluster_id):
"""Change selected cluster
:param cluster_id: target cluster #
"""
_change_cluster(cluster_id)
cluster_id = '-' if cluster_id == -1 else cluster_id
msg = message.get('use_cluster').format(cluster_id=cluster_id)
logger.info(msg)
def base_directory(default='~/tsr2'):
logger.debug('ask base directory')
result = ask(message.get('ask_base_directory'), default=default)
if not result.startswith(('~', '/')):
logger.error(message.get('error_invalid_path').format(value=result))
return base_directory()
logger.info('OK, {}'.format(result))
cli_config = config.get_cli_config()
cli_config['base_directory'] = result
config.save_cli_config(cli_config)
return result
def prefix_of_db_path(save, default=None):
logger.debug('ask_prefix_of_db_path')
deploy_history = config.get_deploy_history()
if not default:
default = deploy_history['prefix_of_db_path']
result = ask(message.get('ask_db_path'), default=default)
result = result.strip()
if save:
deploy_history['prefix_of_db_path'] = result
config.save_deploy_history(deploy_history)
logger.info('OK, {}'.format(result))
return result
def hosts(save, default=None):
logger.debug('ask host')
deploy_history = config.get_deploy_history()
if not default:
default = deploy_history['hosts']
msg = message.get('ask_hosts')
result = ask(msg, default=', '.join(default))
result = list(map(lambda x: x.strip(), result.split(',')))
if save:
deploy_history['hosts'] = result
config.save_deploy_history(deploy_history)
logger.info('OK, {}'.format(result))
return result
def save_redis_template_config(key, value):
"""Save redis template config to file
:param key: key
:param value: value
"""
key = key.strip()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
master_template = path_of_fb['master_template']
msg = message.get('save_config_to_template')
logger.info(msg)
RedisCliUtil._save_config(master_template, key, value)
def ssd_count(save, default=None):
logger.debug('ask ssd count')
deploy_history = config.get_deploy_history()
if not default:
default = deploy_history['ssd_count']
result = int(askInt(message.get('ask_ssd_count'), default=str(default)))
if result <= 0:
logger.error(message.get('error_ssd_count_less_than_1'))
return ssd_count(save=save, default=default)
if save:
deploy_history['ssd_count'] = result
config.save_deploy_history(deploy_history)
logger.info('OK, {}'.format(result))
return result
def failover(self):
"""Replace disconnected master with slave
If disconnected master comes back to live, it become slave.
"""
center = Center()
center.update_ip_port()
master_obj_list = center.get_master_obj_list()
msg = color.yellow(message.get('error_no_alive_slave_for_failover'))
all_alive = True
for node in master_obj_list:
if node['status'] != 'connected':
all_alive = False
success = False
for slave in node['slaves']:
if slave['status'] == 'connected':
msg2 = message.get('redis_failover').format(
slave_addr=slave['addr'],
master_addr=node['addr']
)
logger.info(msg2)
stdout = center.run_failover(
slave['addr'],
take_over=True
)
if stdout != 'OK':
continue
logger.info('OK')
success = True
break
if not success:
logger.info(msg.format(node['addr']))
if all_alive:
msg = message.get('already_all_master_alive')
logger.info(msg)
def replicas(save, default=None):
logger.debug('ask replicas')
deploy_history = config.get_deploy_history()
if not default:
default = deploy_history['replicas']
result = askInt(message.get('ask_replicas'), default=str(default))
result = int(result)
if result < 0:
logger.error(message.get('error_replicas_less_than_0'))
return replicas(save, default=default)
if save:
deploy_history['replicas'] = result
config.save_deploy_history(deploy_history)
logger.info('OK, {}'.format(result))
return result
def slave(self):
"""Edit 'redis-slave.conf.template'
"""
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
target_path = path_of_fb['slave_template']
self._edit_conf(target_path, syntax='sh')
center = Center()
center.update_ip_port()
success = center.check_hosts_connection()
if not success:
return
success = center.sync_file(target_path)
if success:
msg = message.get('complete_conf_edit')
logger.info(msg)
def thriftserver(self):
"""Edit 'thriftserver.properties'
"""
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
target_path = path_of_fb['thrift_properties']
self._edit_conf(target_path, syntax='sh')
center = Center()
center.update_ip_port()
success = center.check_hosts_connection()
if not success:
return
success = center.sync_file(target_path)
if success:
msg = message.get('complete_conf_edit')
logger.info(msg)
def main(cluster_id, debug, version):
if version:
print_version()
return
_initial_check()
if debug:
log.set_mode('debug')
logger.debug('Start ltcli')
cluster_id = _validate_cluster_id(cluster_id)
history = os.path.join(config.get_root_of_cli_config(), 'cli_history')
session = PromptSession(
# lexer=PygmentsLexer(SqlLexer),
history=FileHistory(history),
auto_suggest=AutoSuggestFromHistory(),
style=utils.style)
while True:
try:
exit_flg = False
p = prompt.get_cli_prompt()
text = session.prompt(p, style=utils.style)
command_list = text.split(';')
for cmd in command_list:
cmd = cmd.strip()
if cmd == "exit":
exit_flg = True
break
if 'ltcli' in cmd:
old = cmd
cmd = cmd.replace('ltcli', '').strip()
msg = message.get('notify_command_replacement_is_possible')
msg = msg.format(new=cmd, old=old)
logger.info(msg)
err_flg = _handle(cmd)
if err_flg:
break
if exit_flg:
break
except ClusterNotExistError:
run_cluster_use(-1)
continue
except KeyboardInterrupt:
continue
except EOFError:
break
def delete(self, cluster_id):
"""Delete cluster
It is automatically backed up with timestamps as tags
:param cluster_id: target cluster id
"""
if not cluster_util.validate_id(cluster_id):
raise ClusterIdError(cluster_id)
path_of_fb = config.get_path_of_fb(cluster_id)
props_path = path_of_fb['redis_properties']
hosts = config.get_props(props_path, 'sr2_redis_master_hosts', [])
tag = time.strftime("%Y%m%d%H%M%S", time.gmtime())
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, tag)
for host in hosts:
Center().cluster_backup(host, cluster_id, cluster_backup_dir)
msg = message.get('cluster_delete_complete')
msg = msg.format(cluster_id=cluster_id)
logger.info(msg)
def get(self, key, all=False, host=None, port=None):
"""Command: redis-cli config get
:param key: redis config keyword
:param all: If true, send command to all redis
:param host: host info for redis
:param port: port info for redis
"""
if not isinstance(all, bool):
msg = m.get('error_option_type_not_boolean')
msg = msg.format(options='all')
logger.error(msg)
return
if (not host or not port) and not all:
msg = m.get('use_host_port_or_option_all')
logger.error(msg)
return
sub_cmd = 'config get "{key}" 2>&1'.format(key=key)
if all:
meta = []
ret = RedisCliUtil.command_all_async(sub_cmd)
for m_s, host, port, result, message in ret:
addr = '{}:{}'.format(host, port)
if result == 'OK':
if message:
_, value = message.split('\n')
meta.append([m_s, addr, value])
else:
meta.append([m_s, addr, color.red('Invalid Key')])
else:
meta.append([m_s, addr, color.red(result)])
utils.print_table([['TYPE', 'ADDR', 'RESULT']] + meta)
else:
output = RedisCliUtil.command(sub_cmd=sub_cmd,
host=host,
port=port,
formatter=self.no_print)
output = output.strip()
if output:
key, value = output.split('\n')
logger.info(value)
else:
msg = m.get('error_invalid_key').format(key=key)
logger.error(msg)
def clean(self, logs=False):
"""Clean cluster
Delete redis config, data, node configuration.
:param log: Delete log of redis
"""
if not isinstance(logs, bool):
msg = message.get('error_option_type_not_boolean')
msg = msg.format(option='logs')
logger.error(msg)
return
center = Center()
center.update_ip_port()
if logs:
center.remove_all_of_redis_log_force()
return
center.cluster_clean()
msg = message.get('apply_after_restart')
logger.info(msg)
def download_file(url, file_path):
download_path = file_path + '.download'
file_name = os.path.basename(file_path)
try:
with open(download_path, 'wb') as f:
msg = message.get('file_download').format(file_name=file_name)
logger.info(msg)
logger.debug('url: {}'.format(url))
logger.debug('installer name: {}'.format(file_name))
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None:
f.write(response.content)
else:
done_length = 0
total_length = int(total_length)
for data in response.iter_content(chunk_size=4096):
done_length += len(data)
f.write(data)
done = int(100 * done_length / total_length)
comp = '=' * int(done / 2)
remain = ' ' * int(50 - int(done / 2))
progress = '\r[{}{}] {}%'.format(comp, remain, done)
sys.stdout.write(progress)
sys.stdout.flush()
print('')
shutil.move(download_path, file_path)
return True
except requests.exceptions.HTTPError as ex:
logger.warning(ex)
return False
except KeyboardInterrupt as ex:
print('')
raise ex
except BaseException as ex:
class_name = ex.__class__.__name__
logger.warning('{}: {}'.format(class_name, url))
return False
finally:
if os.path.isfile(download_path):
os.remove(download_path)
def run_monitor(n=10, t=2):
"""Monitoring logs of redis.
:param n: number of lines to print log
:param t: renewal cycle(sec)
"""
if not isinstance(n, int):
msg = message.get('error_option_type_not_number').format(option='n')
logger.error(msg)
return
if not isinstance(t, int) and not isinstance(t, float):
msg = message.get('error_option_type_not_float').format(option='t')
logger.error(msg)
return
try:
sp.check_output('which tail', shell=True)
except Exception:
msg = message.get('error_not_found_command_tail')
logger.error(msg)
return
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_log = path_of_fb['sr2_redis_log']
log_files = '{}/servers*'.format(sr2_redis_log)
host_list = config.get_master_host_list()
target_host = ask_util.host_for_monitor(host_list)
try:
sp.check_output('which watch', shell=True)
command = "ssh -t {} watch -n {} 'tail -n {} {}'".format(
target_host, t, n, log_files)
sp.call(command, shell=True)
except Exception:
msg = message.get('error_not_found_command_watch')
logger.warning(msg)
logger.info(message.get('message_for_exit'))
command = "tail -F -s {} {}".format(t, log_files)
client = net.get_ssh(target_host)
net.ssh_execute_async(client, command)
def monitor(self):
"""Monitoring log of thriftserver"
"""
logger.debug('thriftserver_command_monitor')
_check_spark()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
ths_props_path = path_of_fb['thrift_properties']
source_cmd = 'source {}'.format(ths_props_path)
spark_log = _get_env()['spark_log']
# log_file_path = ''.join([
# spark_log,
# '/spark-{}-org.apache'.format(os.environ['USER']),
# '.spark.sql.hive.thriftserver.HiveThriftServer2-1-*.out',
# ])
log_file_path = os.path.join(spark_log, NOHUP_LOGFILE)
if _find_files_with_regex(spark_log, ROLLING_LOGFILE_REGEX):
log_file_path = os.path.join(spark_log, ROLLING_LOGFILE)
base_cmd = 'tail -F {}'.format(log_file_path)
cmd = '{}; {}'.format(source_cmd, base_cmd)
logger.debug(cmd)
msg = message.get('message_for_exit')
logger.info(msg)
os.system(cmd)
def failback(self):
"""Restart disconnected redis
"""
center = Center()
center.update_ip_port()
master_obj_list = center.get_master_obj_list()
disconnected_list = []
paused_list = []
for master in master_obj_list:
if master['status'] == 'disconnected':
disconnected_list.append(master['addr'])
if master['status'] == 'paused':
paused_list.append(master['addr'])
for slave in master['slaves']:
if slave['status'] == 'disconnected':
disconnected_list.append(slave['addr'])
if slave['status'] == 'paused':
paused_list.append(slave['addr'])
classified_disconnected_list = {}
classified_paused_list = {}
for disconnected in disconnected_list:
host, port = disconnected.split(':')
if host not in classified_disconnected_list:
classified_disconnected_list[host] = []
classified_disconnected_list[host].append(port)
for paused in paused_list:
host, port = paused.split(':')
if host not in classified_paused_list:
classified_paused_list[host] = []
classified_paused_list[host].append(port)
current_time = time.strftime("%Y%m%d-%H%M", time.gmtime())
for host, ports in classified_disconnected_list.items():
msg = message.get('redis_run')
msg = msg.format(host=host, ports='|'.join(ports))
logger.info(msg)
center.run_redis_process(host, ports, False, current_time)
for host, ports in classified_paused_list.items():
msg = message.get('redis_restart')
msg = msg.format(host=host, ports='|'.join(ports))
logger.info(msg)
center.stop_redis_process(host, ports)
center.run_redis_process(host, ports, False, current_time)
if not classified_disconnected_list and not classified_paused_list:
msg = message.get('already_all_redis_alive')
logger.info(msg)
def create(host_port_list, max_slots=1024):
conns = []
try:
for host, port in set(host_port_list):
t = Connection(host, port)
conns.append(t)
_ensure_cluster_status_unset(t)
logging.info('Instance at %s:%d checked', t.host, t.port)
msg = message.get('cluster_meet')
logger.info(msg)
logger.info(' - {}:{}'.format(conns[0].host, conns[0].port))
first_conn = conns[0]
for i, t in enumerate(conns[1:]):
logger.info(' - {}:{}'.format(t.host, t.port))
_create(t, first_conn)
sleep(0.02)
slots_each = SLOT_COUNT // len(conns)
slots_residue = SLOT_COUNT - slots_each * len(conns)
slots_each += 1
prev = 0
for i, t in enumerate(conns[0:]):
if i == slots_residue:
slots_each -= 1
msg = ' - {}:{}, {}'.format(t.host, t.port, slots_each)
logger.info(msg)
_add_slots_range(t, prev, prev + slots_each, max_slots)
sleep(0.02)
logging.info('Add %d slots to %s:%d', slots_each, t.host, t.port)
prev = prev + slots_each
msg = message.get('check_cluster_state_assign_slot')
logger.info(msg)
for t in conns:
_poll_check_status(t)
logger.info('Ok')
finally:
for t in conns:
t.close()
def _deploy(cluster_id, history_save, clean):
deploy_state = DeployUtil().get_state(cluster_id)
if deploy_state == DEPLOYED:
msg = message.get('ask_deploy_again')
msg = msg.format(cluster_id=cluster_id)
msg = color.yellow(msg)
yes = ask_util.askBool(msg, default='n')
if not yes:
logger.info(message.get('cancel'))
return
restore_yes = None
no_localhost = False
current_time = time.strftime("%Y%m%d%H%M%S", time.gmtime())
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time)
conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time)
tmp_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, 'tmp')
meta = [['NAME', 'VALUE']]
path_of_fb = config.get_path_of_fb(cluster_id)
conf_path = path_of_fb['conf_path']
props_path = path_of_fb['redis_properties']
cluster_path = path_of_fb['cluster_path']
path_of_cli = config.get_path_of_cli(cluster_id)
conf_backup_path = path_of_cli['conf_backup_path']
tmp_backup_path = os.path.join(conf_backup_path, tmp_backup_dir)
local_ip = config.get_local_ip()
# ask installer
installer_path = ask_util.installer()
installer_name = os.path.basename(installer_path)
meta.append(['installer', installer_name])
# ask restore conf
if deploy_state == DEPLOYED:
restore_yes = ask_util.askBool(message.get('ask_restore_conf'))
meta.append(['restore', restore_yes])
# input props
hosts = []
if deploy_state == DEPLOYED:
if restore_yes:
meta += DeployUtil().get_meta_from_props(props_path)
hosts = config.get_props(props_path, 'sr2_redis_master_hosts')
else:
if not os.path.isdir(conf_backup_path):
os.mkdir(conf_backup_path)
if os.path.exists(tmp_backup_path):
msg = message.get('ask_load_history_of_previous_modification')
yes = ask_util.askBool(msg)
if not yes:
shutil.rmtree(tmp_backup_path)
if not os.path.exists(tmp_backup_path):
os.mkdir(tmp_backup_path)
shutil.copy(os.path.join(conf_path, 'redis.properties'),
os.path.join(tmp_backup_path, 'redis.properties'))
tmp_props_path = os.path.join(tmp_backup_path, 'redis.properties')
editor.edit(tmp_props_path, syntax='sh')
meta += DeployUtil().get_meta_from_props(tmp_props_path)
hosts = config.get_props(tmp_props_path, 'sr2_redis_master_hosts')
else:
# new deploy
props_dict = ask_util.props(cluster_id, save=history_save)
hosts = props_dict['hosts']
meta += DeployUtil().get_meta_from_dict(props_dict)
utils.print_table(meta)
msg = message.get('confirm_deploy_information')
yes = ask_util.askBool(msg)
if not yes:
logger.info(message.get('cancel'))
return
# check node status
success = Center().check_hosts_connection(hosts, True)
if not success:
msg = message.get('error_exist_unavailable_host')
logger.error(msg)
return
logger.debug('Connection of all hosts ok.')
success = Center().check_include_localhost(hosts)
if not success:
no_localhost = True
# get port info
if deploy_state == DEPLOYED:
if restore_yes:
key = 'sr2_redis_master_ports'
m_ports = config.get_props(props_path, key, [])
key = 'sr2_redis_slave_ports'
s_ports = config.get_props(props_path, key, [])
replicas = len(s_ports) // len(m_ports)
else:
key = 'sr2_redis_master_ports'
m_ports = config.get_props(tmp_props_path, key, [])
key = 'sr2_redis_slave_ports'
s_ports = config.get_props(tmp_props_path, key, [])
replicas = len(s_ports) // len(m_ports)
else:
m_ports = props_dict['master_ports']
s_ports = props_dict['slave_ports']
replicas = props_dict['replicas']
while True:
msg = message.get('check_port')
logger.info(msg)
host_ports_list = []
for host in hosts:
host_ports_list.append((host, m_ports + s_ports))
conflict = Center().check_port_is_enable(host_ports_list)
if not conflict:
logger.info("OK")
break
utils.print_table([["HOST", "PORT"]] + conflict)
msg = message.get('ask_port_collision')
msg = color.yellow(msg)
yes = ask_util.askBool(msg)
if yes:
logger.info("OK")
break
m_ports = ask_util.master_ports(False, cluster_id)
replicas = ask_util.replicas(False)
s_ports = ask_util.slave_ports(cluster_id, len(m_ports), replicas)
if deploy_state == DEPLOYED:
if restore_yes:
key = 'sr2_redis_master_ports'
value = cluster_util.convert_list_2_seq(m_ports)
config.set_props(props_path, key, value)
key = 'sr2_redis_slave_ports'
value = cluster_util.convert_list_2_seq(s_ports)
config.set_props(props_path, key, value)
else:
key = 'sr2_redis_master_ports'
value = cluster_util.convert_list_2_seq(m_ports)
config.set_props(tmp_props_path, key, value)
key = 'sr2_redis_slave_ports'
value = cluster_util.convert_list_2_seq(s_ports)
config.set_props(tmp_props_path, key, value)
else:
props_dict['master_ports'] = m_ports
props_dict['slave_ports'] = s_ports
props_dict['replicas'] = replicas
# if pending, delete legacy on each hosts
if no_localhost:
if DeployUtil().get_state(cluster_id, local_ip) == PENDING:
client = net.get_ssh(local_ip)
command = 'rm -rf {}'.format(cluster_path)
net.ssh_execute(client=client, command=command)
client.close()
for host in hosts:
if DeployUtil().get_state(cluster_id, host) == PENDING:
client = net.get_ssh(host)
command = 'rm -rf {}'.format(cluster_path)
net.ssh_execute(client=client, command=command)
client.close()
# added_hosts = post_hosts - pre_hosts
msg = message.get('check_cluster_exist')
logger.info(msg)
added_hosts = set(hosts)
meta = []
if deploy_state == DEPLOYED:
pre_hosts = config.get_props(props_path, 'sr2_redis_master_hosts')
added_hosts -= set(pre_hosts)
can_deploy = True
if no_localhost:
added_hosts |= set([local_ip])
for host in added_hosts:
client = net.get_ssh(host)
is_localhost = Center().is_localhost(host)
if is_localhost:
if no_localhost:
continue
if os.path.exists(cluster_path + '/remote'):
meta.append([host, color.green('CLEAN')])
continue
if net.is_exist(client, cluster_path):
meta.append([host, color.red('CLUSTER EXIST')])
can_deploy = False
continue
meta.append([host, color.green('CLEAN')])
if meta:
utils.print_table([['HOST', 'STATUS']] + meta)
if not can_deploy:
msg = message.get('error_cluster_collision')
logger.error(msg)
return
# if not force:
# logger.error("If you trying to force, use option '--force'")
# return
logger.info('OK')
# cluster stop and clean
if deploy_state == DEPLOYED and clean:
center = Center()
cur_cluster_id = config.get_cur_cluster_id(allow_empty_id=True)
run_cluster_use(cluster_id)
center.update_ip_port()
center.stop_redis()
center.remove_all_of_redis_log_force()
center.cluster_clean()
run_cluster_use(cur_cluster_id)
# backup conf
if deploy_state == DEPLOYED:
Center().conf_backup(local_ip, cluster_id, conf_backup_dir)
# backup cluster
backup_hosts = []
if deploy_state == DEPLOYED:
backup_hosts += set(pre_hosts)
# if force:
# backup_hosts += added_hosts
for host in backup_hosts:
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(host)
Center().cluster_backup(host, cluster_id, cluster_backup_dir)
client.close()
# transfer & install
msg = message.get('transfer_and_execute_installer')
logger.info(msg)
target_hosts = hosts + [local_ip] if no_localhost else hosts
for host in target_hosts:
if not (no_localhost and Center().is_localhost(host)):
logger.info(' - {}'.format(host))
client = net.get_ssh(host)
cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path)
net.ssh_execute(client=client, command=cmd)
client.close()
DeployUtil().transfer_installer(host, cluster_id, installer_path)
try:
DeployUtil().install(host, cluster_id, installer_name)
except SSHCommandError as ex:
msg = message.get('error_execute_installer')
msg = msg.format(installer=installer_path)
logger.error(msg)
logger.exception(ex)
return
# setup props
if deploy_state == DEPLOYED:
if restore_yes:
tag = conf_backup_dir
else:
tag = tmp_backup_dir
Center().conf_restore(local_ip, cluster_id, tag)
else:
key = 'sr2_redis_master_hosts'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['hosts'])
key = 'sr2_redis_master_ports'
config.make_key_enable(props_path, key)
value = cluster_util.convert_list_2_seq(props_dict['master_ports'])
config.set_props(props_path, key, value)
key = 'sr2_redis_slave_hosts'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['hosts'])
config.make_key_disable(props_path, key)
if props_dict['replicas'] > 0:
key = 'sr2_redis_slave_hosts'
config.make_key_enable(props_path, key)
key = 'sr2_redis_slave_ports'
config.make_key_enable(props_path, key)
value = cluster_util.convert_list_2_seq(props_dict['slave_ports'])
config.set_props(props_path, key, value)
key = 'ssd_count'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['ssd_count'])
key = 'sr2_redis_data'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
key = 'sr2_redis_db_path'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
key = 'sr2_flash_db_path'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
# synk props
msg = message.get('sync_conf')
logger.info(msg)
for node in hosts:
if socket.gethostbyname(node) in config.get_local_ip_list():
continue
client = net.get_ssh(node)
if not client:
msg = message.get('error_ssh_connection').format(host=node)
logger.error(msg)
return
net.copy_dir_to_remote(client, conf_path, conf_path)
client.close()
# set deploy state complete
if os.path.exists(tmp_backup_path):
shutil.rmtree(tmp_backup_path)
for node in target_hosts:
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(node)
cmd = 'rm -rf {}'.format(os.path.join(cluster_path, '.deploy.state'))
net.ssh_execute(client=client, command=cmd)
client.close()
if no_localhost:
os.system('touch {}/remote'.format(cluster_path))
msg = message.get('complete_deploy').format(cluster_id=cluster_id)
logger.info(msg)
Cluster().use(cluster_id)
msg = message.get('suggest_after_deploy')
logger.info(msg)
def _deploy_zero_downtime(cluster_id):
logger.debug("zero downtime update cluster {}".format(cluster_id))
center = Center()
center.update_ip_port()
m_hosts = center.master_host_list
m_ports = center.master_port_list
s_hosts = center.slave_host_list
s_ports = center.slave_port_list
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
# check master alive
m_count = len(m_hosts) * len(m_ports)
alive_m_count = center.get_alive_master_redis_count()
if alive_m_count < m_count:
logger.error(message.get('error_exist_disconnected_master'))
return
if not config.is_slave_enabled:
logger.error(message.get('error_need_to_slave'))
return
# select installer
installer_path = ask_util.installer()
installer_name = os.path.basename(installer_path)
# backup info
current_time = time.strftime("%Y%m%d%H%M%S", time.gmtime())
conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time)
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time)
local_ip = config.get_local_ip()
# backup conf
center.conf_backup(local_ip, cluster_id, conf_backup_dir)
# backup cluster
for host in s_hosts:
client = net.get_ssh(host)
center.cluster_backup(host, cluster_id, cluster_backup_dir)
client.close()
# transfer & install
logger.info(message.get('transfer_and_execute_installer'))
for host in m_hosts:
logger.info(' - {}'.format(host))
client = net.get_ssh(host)
cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path)
net.ssh_execute(client=client, command=cmd)
client.close()
DeployUtil().transfer_installer(host, cluster_id, installer_path)
try:
DeployUtil().install(host, cluster_id, installer_name)
except SSHCommandError as ex:
msg = message.get('error_execute_installer')
msg = msg.format(installer=installer_path)
logger.error(msg)
logger.exception(ex)
return
# restore conf
center.conf_restore(local_ip, cluster_id, conf_backup_dir)
# set deploy state complete
for node in m_hosts:
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(node)
cmd = 'rm -rf {}'.format(os.path.join(cluster_path, '.deploy.state'))
net.ssh_execute(client=client, command=cmd)
client.close()
# restart slave
center.stop_current_nodes(master=False, slave=True)
center.configure_redis()
center.sync_conf()
center.start_current_nodes(master=False, slave=True)
center.wait_until_all_redis_process_up()
slaves_for_failover = center.get_slave_nodes()
key = 'cluster-node-timeout'
origin_m_value = center.cli_config_get(key, m_hosts[0], m_ports[0])
origin_s_value = center.cli_config_get(key, s_hosts[0], s_ports[0])
logger.debug('config set: cluster-node-timeout 2000')
RedisCliConfig().set(key, '2000', all=True)
# cluster failover (with no option)
logger.info(message.get('failover_on_deploy'))
logger.debug(slaves_for_failover)
try_count = 0
while try_count < 10:
try_count += 1
success = True
for slave_addr in slaves_for_failover:
host, port = slave_addr.split(':')
stdout = center.run_failover("{}:{}".format(host, port))
logger.debug("failover {}:{} {}".format(host, port, stdout))
if stdout != "ERR You should send CLUSTER FAILOVER to a slave":
# In some cases, the cluster failover is not complete
# even if stdout is OK
# If redis changed to master completely,
# return 'ERR You should send CLUSTER FAILOVER to a slave'
success = False
if success:
break
msg = message.get('retry').format(try_count=try_count)
logger.info(msg)
time.sleep(5)
logger.debug('restore config: cluster-node-timeout')
center.cli_config_set_all(key, origin_m_value, m_hosts, m_ports)
center.cli_config_set_all(key, origin_s_value, s_hosts, s_ports)
if not success:
logger.error(message.get('error_redis_failover'))
return
# restart master (current slave)
center.stop_current_nodes(master=False, slave=True)
center.configure_redis(slave=False)
center.sync_conf()
center.start_current_nodes(master=False, slave=True)
center.wait_until_all_redis_process_up()
def installer():
'''
Select installer from list of '$FBPATH/releases'
or input absolute path of file directly
return installer path
'''
logger.debug('ask installer')
path_of_cli = config.get_path_of_cli(None)
release_path = path_of_cli['release_path']
if not os.path.exists(release_path):
os.mkdir(release_path)
installer_list = net.get_installers_from_fb_s3()
buf = os.listdir(release_path)
buf = list(filter(lambda x: x != '.gitignore', buf))
pattern = '.download'
buf = list(filter(lambda x: pattern not in x, buf))
for file_name in buf:
installer_list.append({
'name': file_name,
'url': os.path.join(release_path, file_name),
'type': 'local',
})
# formatting msg
formatted = []
for i, obj in enumerate(installer_list):
formatted.append(' ({index}) [{type}] {name}'.format(
index=i + 1,
name=obj['name'],
type=obj['type'].upper(),
))
if not formatted:
formatted.append(' (empty)')
stringfied_list = '\n'.join(formatted)
msg = '\n'.join(message.get('ask_installer')).format(list=stringfied_list)
result = ask(msg)
while True:
result = result.strip()
if installer_list and utils.is_number(result):
# case: select in list
result = int(result) - 1
if result in range(0, len(installer_list)):
selected = installer_list[result]
if selected['type'] == 'download':
url = selected['url']
file_name = selected['name']
installer_path = os.path.join(release_path, file_name)
success = net.download_file(url, installer_path)
if success:
logger.info('OK, {}'.format(file_name))
return installer_path
msg = message.get('error_download_installer')
msg = msg.format(url=url)
logger.error(msg)
if selected['type'] == 'local':
ret = selected['url']
logger.debug('Select insaller in list: {}'.format(ret))
logger.info('OK, {}'.format(selected['name']))
return os.path.expanduser(ret)
msg = message.get('error_select_number')
msg = msg.format(max_number=len(installer_list))
logger.error(msg)
elif result.startswith(('~', '/')):
# case: type path
if os.path.isfile(os.path.expanduser(result)):
logger.debug('Select insaller by path: {}'.format(result))
logger.info('OK, {}'.format(os.path.basename(result)))
return os.path.expanduser(result)
msg = message.get('error_type_installer_path')
msg = msg.format(file_path=result)
logger.error(msg)
elif result.startswith(('http://', 'https://')):
# case: type url
url = result
file_name = url.split('?')[0].split('/')[-1]
installer_path = os.path.join(release_path, file_name)
success = net.download_file(url, installer_path)
if success:
logger.info('OK, {}'.format(file_name))
return installer_path
msg = message.get('error_download_installer')
msg = msg.format(url=url)
logger.error(msg)
else:
msg = message.get('error_invalid_input')
msg = msg.format(value=result)
logger.error(msg)
result = ask('')
