def command_all_async(sub_cmd, slave=True):
def _async_target_func(m_s, pre_cmd, host, port, sub_cmd, ret):
try:
command = '{} -h {} -p {} {}'.format(pre_cmd, host, port,
sub_cmd)
logger.debug(command)
stdout = subprocess.check_output(command, shell=True)
stdout = stdout.decode('utf-8').strip()
ret.append((m_s, host, port, 'OK', stdout))
except Exception as ex:
stderr = str(ex)
logger.debug(stderr)
ret.append((m_s, host, port, 'FAIL', stderr))
cluster_id = config.get_cur_cluster_id()
master_host_list = config.get_master_host_list(cluster_id)
master_port_list = config.get_master_port_list(cluster_id)
if slave:
slave_host_list = config.get_slave_host_list(cluster_id)
slave_port_list = config.get_slave_port_list(cluster_id)
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_bin = path_of_fb['sr2_redis_bin']
logger.debug('command_all_async')
cluster_id = config.get_cur_cluster_id()
lib_path = config.get_ld_library_path(cluster_id)
env_cmd = [
'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']),
'export DYLD_LIBRARY_PATH={};'.format(
lib_path['dyld_library_path']),
]
env = ' '.join(env_cmd)
threads = []
ret = [] # (m/s, host, port, result, message)
pre_cmd = '{} {}/redis-cli -c'.format(env, sr2_redis_bin)
for host in master_host_list:
for port in master_port_list:
t = Thread(
target=_async_target_func,
args=('Master', pre_cmd, host, port, sub_cmd, ret),
)
threads.append(t)
if slave:
for host in slave_host_list:
for port in slave_port_list:
t = Thread(
target=_async_target_func,
args=('Slave', pre_cmd, host, port, sub_cmd, ret),
)
threads.append(t)
for th in threads:
th.start()
time.sleep(0.02)
for th in threads:
th.join()
logger.debug(ret)
return ret
def beeline(self, **kargs):
"""Connect to thriftserver command line
"""
logger.debug('thriftserver_command_beeline')
_check_spark()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
ths_props_path = path_of_fb['thrift_properties']
cmd = 'source {}; echo ${}'.format(ths_props_path, 'HIVE_HOST')
host = sp.check_output(cmd, shell=True).decode('utf-8').strip()
cmd = 'source {}; echo ${}'.format(ths_props_path, 'HIVE_PORT')
port = sp.check_output(cmd, shell=True).decode('utf-8').strip()
spark_env = _get_env()
base_cmd = '{}/beeline'.format(spark_env['spark_bin'])
options = {
'u': 'jdbc:hive2://{}:{}'.format(host, port),
'n': os.environ['USER']
}
for key, value in kargs.items():
options[key] = value
for key, value in options.items():
base_cmd += ' -{} {}'.format(key, value)
logger.debug(base_cmd)
msg = message.get('try_connection')
logger.info(msg)
os.system(base_cmd)
def open_vim_editor(target='config'):
"""Open vim editor
:param target: config | master | slave | thriftserver
"""
cluster_id = config.get_cur_cluster_id()
full_path = get_full_path_of_props(cluster_id, target)
editor.edit(full_path)
def command_all(sub_cmd, cluster_id=-1, formatter=None):
"""Send redis-cli command to all
:param sub_cmd: sub command
:param cluster_id: target cluster #
:param formatter: if set, call formatter with output string
"""
if cluster_id < 0:
cluster_id = config.get_cur_cluster_id()
master_host_list = config.get_master_host_list(cluster_id)
master_port_list = config.get_master_port_list(cluster_id)
slave_host_list = config.get_slave_host_list(cluster_id)
slave_port_list = config.get_slave_port_list()
outs, meta = RedisCliUtil.command_raw_all(sub_cmd, master_host_list,
master_port_list)
logger.debug(outs)
buf = meta[:]
outs, meta = RedisCliUtil.command_raw_all(sub_cmd, slave_host_list,
slave_port_list)
logger.debug(outs)
buf += meta[1:]
if formatter:
formatter(buf)
else:
logger.debug(outs)
def failover_with_dir(self, server, dir):
"""Find masters that use the specified directory path and do failover with its slave
:param server: IP or hostname
:param dir: directory path
"""
center = Center()
center.update_ip_port()
logger.debug('failover_with_dir')
master_nodes = center.get_master_obj_list()
cluster_id = config.get_cur_cluster_id()
lib_path = config.get_ld_library_path(cluster_id)
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_bin = path_of_fb['sr2_redis_bin']
env_cmd = [
'GLOBIGNORE=*;',
'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']),
'export DYLD_LIBRARY_PATH={};'.format(
lib_path['dyld_library_path']),
]
redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli')
# Find masters with dir
ret = RedisCliUtil.command_all_async('config get dir', slave=True)
outs = ''
meta = []
m_endpoint = []
for node in master_nodes:
m_endpoint.append(node['addr'])
for _, host, port, res, stdout in ret:
if res == 'OK':
flat_stdout = '\n'.join([outs, stdout])
line = flat_stdout.splitlines()
if self.compare_ip(host, server) and dir in line[2]:
endpoint = '{}:{}'.format(socket.gethostbyname(host), port)
if endpoint in m_endpoint:
meta.append(endpoint)
else:
logger.warning("FAIL {}:{} {}".format(host, port, stdout))
for endpoint in meta:
for master_node in master_nodes:
if endpoint == master_node['addr']:
for slave_node in master_node['slaves']:
addr = slave_node['addr']
(s_host, s_port) = addr.split(':')
sub_cmd = 'cluster failover takeover'
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
s_host,
s_port,
sub_cmd,
)
self._print(
message.get('try_failover_takeover').format(
slave=addr))
stdout = subprocess.check_output(command, shell=True)
self._print(stdout)
def version(self):
"""Get version of lightningDB
"""
cluster_id = config.get_cur_cluster_id()
tsr2_home = config.get_tsr2_home(cluster_id)
with open(os.path.join(tsr2_home, "VERSION"), "r") as version_file:
lines = version_file.readlines()
logger.info("".join(lines).strip())
def get_cli_prompt():
"""Return cli prompt
:param user: user name
:return: prompt string
"""
cluster_id = config.get_cur_cluster_id(allow_empty_id=True)
if cluster_id < 0:
cluster_id = '-'
user = environ['USER']
prompt_classname = u'class:ansiwhite'
prompt_message = u'{}@lightningdb:{}> '.format(user, cluster_id)
return [(prompt_classname, prompt_message)]
def stop(self):
"""Stop thriftserver
"""
logger.debug('thriftserver_command_stop')
_check_spark()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
ths_props_path = path_of_fb['thrift_properties']
source_cmd = 'source {}'.format(ths_props_path)
base_cmd = '$SPARK_SBIN/stop-thriftserver.sh'
cmd = '{}; {}'.format(source_cmd, base_cmd)
logger.debug(cmd)
os.system(cmd)
def save_redis_template_config(key, value):
"""Save redis template config to file
:param key: key
:param value: value
"""
key = key.strip()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
master_template = path_of_fb['master_template']
msg = message.get('save_config_to_template')
logger.info(msg)
RedisCliUtil._save_config(master_template, key, value)
def get_sql_prompt():
"""Return sql prompt
:param user: user name
:return: prompt string
"""
cluster_id = config.get_cur_cluster_id(allow_empty_id=True)
if cluster_id < 0:
cluster_id = '-'
user = environ['USER']
prompt_classname = u'class:ansigreen'
prompt_message = u'({}){}@lightningdb:sql> '.format(cluster_id, user)
return [(prompt_classname, prompt_message)]
def _validate_cluster_id(cluster_id):
try:
if cluster_id is None:
cluster_id = config.get_cur_cluster_id(allow_empty_id=True)
elif not utils.is_number(cluster_id):
raise ClusterIdError(cluster_id)
cluster_id = int(cluster_id)
run_cluster_use(cluster_id)
return cluster_id
except (ClusterIdError, ClusterNotExistError) as ex:
logger.warning(ex)
cluster_id = -1
run_cluster_use(cluster_id)
return cluster_id
def _get_env():
if 'SPARK_HOME' not in os.environ:
raise EnvError('you should set env SPARK_HOME')
ret = {
'spark_home': os.environ['SPARK_HOME'],
}
env_list = ['SPARK_BIN', 'SPARK_SBIN', 'SPARK_LOG', 'SPARK_CONF']
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
ths_props_path = path_of_fb['thrift_properties']
for env in env_list:
cmd = 'source {}; echo ${}'.format(ths_props_path, env.upper())
stdout = sp.check_output(cmd, shell=True).decode('utf-8').strip()
ret[env.lower()] = stdout
return ret
def thriftserver(self):
"""Edit 'thriftserver.properties'
"""
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
target_path = path_of_fb['thrift_properties']
self._edit_conf(target_path, syntax='sh')
center = Center()
center.update_ip_port()
success = center.check_hosts_connection()
if not success:
return
success = center.sync_file(target_path)
if success:
msg = message.get('complete_conf_edit')
logger.info(msg)
def slave(self):
"""Edit 'redis-slave.conf.template'
"""
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
target_path = path_of_fb['slave_template']
self._edit_conf(target_path, syntax='sh')
center = Center()
center.update_ip_port()
success = center.check_hosts_connection()
if not success:
return
success = center.sync_file(target_path)
if success:
msg = message.get('complete_conf_edit')
logger.info(msg)
def run_deploy(cluster_id=None,
history_save=True,
clean=False,
strategy="none"):
"""Install LightningDB package.
:param cluster_id: cluster id
:param history_save: save input history and use as default
:param clean: delete redis log, node configuration
:param strategy:
none(default): normal deploy,
zero-downtime: re-deploy without stop
"""
# validate cluster id
if cluster_id is None:
cluster_id = config.get_cur_cluster_id(allow_empty_id=True)
if cluster_id < 0:
msg = message.get('error_invalid_cluster_on_deploy')
logger.error(msg)
return
if not cluster_util.validate_id(cluster_id):
raise ClusterIdError(cluster_id)
# validate option
if not isinstance(history_save, bool):
msg = message.get('error_option_type_not_boolean')
msg = msg.format(option='history-save')
logger.error(msg)
return
logger.debug("option '--history-save': {}".format(history_save))
if not isinstance(clean, bool):
msg = message.get('error_option_type_not_boolean')
msg = msg.format(option='clean')
logger.error(msg)
return
logger.debug("option '--clean': {}".format(clean))
strategy_list = ["none", "zero-downtime"]
if strategy not in strategy_list:
msg = message.get('error_deploy_strategy').format(value=strategy,
list=strategy_list)
logger.error(msg)
return
if strategy == "zero-downtime":
run_cluster_use(cluster_id)
_deploy_zero_downtime(cluster_id)
return
_deploy(cluster_id, history_save, clean)
def force_failover(self, server):
""" Find all masters on the server and convert them to slaves. Finally, in the server, only slaves will be remained.
:param server: IP or hostname
"""
logger.debug('force_failover')
center = Center()
center.update_ip_port()
master_nodes = center.get_master_obj_list()
cluster_id = config.get_cur_cluster_id()
lib_path = config.get_ld_library_path(cluster_id)
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_bin = path_of_fb['sr2_redis_bin']
env_cmd = [
'GLOBIGNORE=*;',
'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']),
'export DYLD_LIBRARY_PATH={};'.format(
lib_path['dyld_library_path']),
]
redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli')
outs = ''
meta = []
m_endpoint = []
for node in master_nodes:
addr = node['addr']
(host, port) = addr.split(':')
# if host == server:
if self.compare_ip(host, server):
for slave_node in node['slaves']:
addr = slave_node['addr']
(s_host, s_port) = addr.split(':')
sub_cmd = 'cluster failover takeover'
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
s_host,
s_port,
sub_cmd,
)
self._print(
message.get('try_failover_takeover').format(
slave=addr))
stdout = subprocess.check_output(command, shell=True)
self._print(stdout)
def start(self):
"""Start thriftserver
"""
logger.debug('thriftserver_command_start')
_check_spark()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
ths_props_path = path_of_fb['thrift_properties']
source_cmd = 'source {}'.format(ths_props_path)
hive_opts = _get_hive_opts_str()
base_cmd = '$SPARK_SBIN/start-thriftserver.sh {}'.format(hive_opts)
cmd = '{}; {}'.format(source_cmd, base_cmd)
logger.debug(cmd)
os.system(cmd)
spark_log = os.path.join(os.environ['SPARK_HOME'], 'logs')
if _find_files_with_regex(spark_log, ROLLING_LOGFILE_REGEX):
for file in _find_files_with_regex(spark_log, NOHUP_LOGFILE_REGEX):
os.remove(os.path.join(spark_log, file))
def reset_distribution(self):
""" Reset the distribution of masters and slaves with original setting
"""
center = Center()
center.update_ip_port()
logger.debug('reset_distribution')
cluster_id = config.get_cur_cluster_id()
lib_path = config.get_ld_library_path(cluster_id)
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_bin = path_of_fb['sr2_redis_bin']
env_cmd = [
'GLOBIGNORE=*;',
'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']),
'export DYLD_LIBRARY_PATH={};'.format(
lib_path['dyld_library_path']),
]
redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli')
slave_nodes = center.get_slave_nodes()
master_ports = center.master_port_list
for slave_node in slave_nodes:
(host, port) = slave_node.split(':')
try:
value = int(port)
if value in master_ports:
# failover takeover
msg = message.get('try_failover_takeover').format(
slave=slave_node)
self._print(msg)
sub_cmd = 'cluster failover takeover'
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
host,
port,
sub_cmd,
)
stdout = subprocess.check_output(command, shell=True)
outs = ''
outs = '\n'.join([outs, stdout])
self._print(outs)
except ValueError:
pass
def run_monitor(n=10, t=2):
"""Monitoring logs of redis.
:param n: number of lines to print log
:param t: renewal cycle(sec)
"""
if not isinstance(n, int):
msg = message.get('error_option_type_not_number').format(option='n')
logger.error(msg)
return
if not isinstance(t, int) and not isinstance(t, float):
msg = message.get('error_option_type_not_float').format(option='t')
logger.error(msg)
return
try:
sp.check_output('which tail', shell=True)
except Exception:
msg = message.get('error_not_found_command_tail')
logger.error(msg)
return
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_log = path_of_fb['sr2_redis_log']
log_files = '{}/servers*'.format(sr2_redis_log)
host_list = config.get_master_host_list()
target_host = ask_util.host_for_monitor(host_list)
try:
sp.check_output('which watch', shell=True)
command = "ssh -t {} watch -n {} 'tail -n {} {}'".format(
target_host, t, n, log_files)
sp.call(command, shell=True)
except Exception:
msg = message.get('error_not_found_command_watch')
logger.warning(msg)
logger.info(message.get('message_for_exit'))
command = "tail -F -s {} {}".format(t, log_files)
client = net.get_ssh(target_host)
net.ssh_execute_async(client, command)
def monitor(self):
"""Monitoring log of thriftserver"
"""
logger.debug('thriftserver_command_monitor')
_check_spark()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
ths_props_path = path_of_fb['thrift_properties']
source_cmd = 'source {}'.format(ths_props_path)
spark_log = _get_env()['spark_log']
# log_file_path = ''.join([
# spark_log,
# '/spark-{}-org.apache'.format(os.environ['USER']),
# '.spark.sql.hive.thriftserver.HiveThriftServer2-1-*.out',
# ])
log_file_path = os.path.join(spark_log, NOHUP_LOGFILE)
if _find_files_with_regex(spark_log, ROLLING_LOGFILE_REGEX):
log_file_path = os.path.join(spark_log, ROLLING_LOGFILE)
base_cmd = 'tail -F {}'.format(log_file_path)
cmd = '{}; {}'.format(source_cmd, base_cmd)
logger.debug(cmd)
msg = message.get('message_for_exit')
logger.info(msg)
os.system(cmd)
def _deploy(cluster_id, history_save, clean):
deploy_state = DeployUtil().get_state(cluster_id)
if deploy_state == DEPLOYED:
msg = message.get('ask_deploy_again')
msg = msg.format(cluster_id=cluster_id)
msg = color.yellow(msg)
yes = ask_util.askBool(msg, default='n')
if not yes:
logger.info(message.get('cancel'))
return
restore_yes = None
no_localhost = False
current_time = time.strftime("%Y%m%d%H%M%S", time.gmtime())
cluster_backup_dir = 'cluster_{}_bak_{}'.format(cluster_id, current_time)
conf_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, current_time)
tmp_backup_dir = 'cluster_{}_conf_bak_{}'.format(cluster_id, 'tmp')
meta = [['NAME', 'VALUE']]
path_of_fb = config.get_path_of_fb(cluster_id)
conf_path = path_of_fb['conf_path']
props_path = path_of_fb['redis_properties']
cluster_path = path_of_fb['cluster_path']
path_of_cli = config.get_path_of_cli(cluster_id)
conf_backup_path = path_of_cli['conf_backup_path']
tmp_backup_path = os.path.join(conf_backup_path, tmp_backup_dir)
local_ip = config.get_local_ip()
# ask installer
installer_path = ask_util.installer()
installer_name = os.path.basename(installer_path)
meta.append(['installer', installer_name])
# ask restore conf
if deploy_state == DEPLOYED:
restore_yes = ask_util.askBool(message.get('ask_restore_conf'))
meta.append(['restore', restore_yes])
# input props
hosts = []
if deploy_state == DEPLOYED:
if restore_yes:
meta += DeployUtil().get_meta_from_props(props_path)
hosts = config.get_props(props_path, 'sr2_redis_master_hosts')
else:
if not os.path.isdir(conf_backup_path):
os.mkdir(conf_backup_path)
if os.path.exists(tmp_backup_path):
msg = message.get('ask_load_history_of_previous_modification')
yes = ask_util.askBool(msg)
if not yes:
shutil.rmtree(tmp_backup_path)
if not os.path.exists(tmp_backup_path):
os.mkdir(tmp_backup_path)
shutil.copy(os.path.join(conf_path, 'redis.properties'),
os.path.join(tmp_backup_path, 'redis.properties'))
tmp_props_path = os.path.join(tmp_backup_path, 'redis.properties')
editor.edit(tmp_props_path, syntax='sh')
meta += DeployUtil().get_meta_from_props(tmp_props_path)
hosts = config.get_props(tmp_props_path, 'sr2_redis_master_hosts')
else:
# new deploy
props_dict = ask_util.props(cluster_id, save=history_save)
hosts = props_dict['hosts']
meta += DeployUtil().get_meta_from_dict(props_dict)
utils.print_table(meta)
msg = message.get('confirm_deploy_information')
yes = ask_util.askBool(msg)
if not yes:
logger.info(message.get('cancel'))
return
# check node status
success = Center().check_hosts_connection(hosts, True)
if not success:
msg = message.get('error_exist_unavailable_host')
logger.error(msg)
return
logger.debug('Connection of all hosts ok.')
success = Center().check_include_localhost(hosts)
if not success:
no_localhost = True
# get port info
if deploy_state == DEPLOYED:
if restore_yes:
key = 'sr2_redis_master_ports'
m_ports = config.get_props(props_path, key, [])
key = 'sr2_redis_slave_ports'
s_ports = config.get_props(props_path, key, [])
replicas = len(s_ports) // len(m_ports)
else:
key = 'sr2_redis_master_ports'
m_ports = config.get_props(tmp_props_path, key, [])
key = 'sr2_redis_slave_ports'
s_ports = config.get_props(tmp_props_path, key, [])
replicas = len(s_ports) // len(m_ports)
else:
m_ports = props_dict['master_ports']
s_ports = props_dict['slave_ports']
replicas = props_dict['replicas']
while True:
msg = message.get('check_port')
logger.info(msg)
host_ports_list = []
for host in hosts:
host_ports_list.append((host, m_ports + s_ports))
conflict = Center().check_port_is_enable(host_ports_list)
if not conflict:
logger.info("OK")
break
utils.print_table([["HOST", "PORT"]] + conflict)
msg = message.get('ask_port_collision')
msg = color.yellow(msg)
yes = ask_util.askBool(msg)
if yes:
logger.info("OK")
break
m_ports = ask_util.master_ports(False, cluster_id)
replicas = ask_util.replicas(False)
s_ports = ask_util.slave_ports(cluster_id, len(m_ports), replicas)
if deploy_state == DEPLOYED:
if restore_yes:
key = 'sr2_redis_master_ports'
value = cluster_util.convert_list_2_seq(m_ports)
config.set_props(props_path, key, value)
key = 'sr2_redis_slave_ports'
value = cluster_util.convert_list_2_seq(s_ports)
config.set_props(props_path, key, value)
else:
key = 'sr2_redis_master_ports'
value = cluster_util.convert_list_2_seq(m_ports)
config.set_props(tmp_props_path, key, value)
key = 'sr2_redis_slave_ports'
value = cluster_util.convert_list_2_seq(s_ports)
config.set_props(tmp_props_path, key, value)
else:
props_dict['master_ports'] = m_ports
props_dict['slave_ports'] = s_ports
props_dict['replicas'] = replicas
# if pending, delete legacy on each hosts
if no_localhost:
if DeployUtil().get_state(cluster_id, local_ip) == PENDING:
client = net.get_ssh(local_ip)
command = 'rm -rf {}'.format(cluster_path)
net.ssh_execute(client=client, command=command)
client.close()
for host in hosts:
if DeployUtil().get_state(cluster_id, host) == PENDING:
client = net.get_ssh(host)
command = 'rm -rf {}'.format(cluster_path)
net.ssh_execute(client=client, command=command)
client.close()
# added_hosts = post_hosts - pre_hosts
msg = message.get('check_cluster_exist')
logger.info(msg)
added_hosts = set(hosts)
meta = []
if deploy_state == DEPLOYED:
pre_hosts = config.get_props(props_path, 'sr2_redis_master_hosts')
added_hosts -= set(pre_hosts)
can_deploy = True
if no_localhost:
added_hosts |= set([local_ip])
for host in added_hosts:
client = net.get_ssh(host)
is_localhost = Center().is_localhost(host)
if is_localhost:
if no_localhost:
continue
if os.path.exists(cluster_path + '/remote'):
meta.append([host, color.green('CLEAN')])
continue
if net.is_exist(client, cluster_path):
meta.append([host, color.red('CLUSTER EXIST')])
can_deploy = False
continue
meta.append([host, color.green('CLEAN')])
if meta:
utils.print_table([['HOST', 'STATUS']] + meta)
if not can_deploy:
msg = message.get('error_cluster_collision')
logger.error(msg)
return
# if not force:
# logger.error("If you trying to force, use option '--force'")
# return
logger.info('OK')
# cluster stop and clean
if deploy_state == DEPLOYED and clean:
center = Center()
cur_cluster_id = config.get_cur_cluster_id(allow_empty_id=True)
run_cluster_use(cluster_id)
center.update_ip_port()
center.stop_redis()
center.remove_all_of_redis_log_force()
center.cluster_clean()
run_cluster_use(cur_cluster_id)
# backup conf
if deploy_state == DEPLOYED:
Center().conf_backup(local_ip, cluster_id, conf_backup_dir)
# backup cluster
backup_hosts = []
if deploy_state == DEPLOYED:
backup_hosts += set(pre_hosts)
# if force:
# backup_hosts += added_hosts
for host in backup_hosts:
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(host)
Center().cluster_backup(host, cluster_id, cluster_backup_dir)
client.close()
# transfer & install
msg = message.get('transfer_and_execute_installer')
logger.info(msg)
target_hosts = hosts + [local_ip] if no_localhost else hosts
for host in target_hosts:
if not (no_localhost and Center().is_localhost(host)):
logger.info(' - {}'.format(host))
client = net.get_ssh(host)
cmd = 'mkdir -p {0} && touch {0}/.deploy.state'.format(cluster_path)
net.ssh_execute(client=client, command=cmd)
client.close()
DeployUtil().transfer_installer(host, cluster_id, installer_path)
try:
DeployUtil().install(host, cluster_id, installer_name)
except SSHCommandError as ex:
msg = message.get('error_execute_installer')
msg = msg.format(installer=installer_path)
logger.error(msg)
logger.exception(ex)
return
# setup props
if deploy_state == DEPLOYED:
if restore_yes:
tag = conf_backup_dir
else:
tag = tmp_backup_dir
Center().conf_restore(local_ip, cluster_id, tag)
else:
key = 'sr2_redis_master_hosts'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['hosts'])
key = 'sr2_redis_master_ports'
config.make_key_enable(props_path, key)
value = cluster_util.convert_list_2_seq(props_dict['master_ports'])
config.set_props(props_path, key, value)
key = 'sr2_redis_slave_hosts'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['hosts'])
config.make_key_disable(props_path, key)
if props_dict['replicas'] > 0:
key = 'sr2_redis_slave_hosts'
config.make_key_enable(props_path, key)
key = 'sr2_redis_slave_ports'
config.make_key_enable(props_path, key)
value = cluster_util.convert_list_2_seq(props_dict['slave_ports'])
config.set_props(props_path, key, value)
key = 'ssd_count'
config.make_key_enable(props_path, key)
config.set_props(props_path, key, props_dict['ssd_count'])
key = 'sr2_redis_data'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
key = 'sr2_redis_db_path'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
key = 'sr2_flash_db_path'
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_enable(props_path, key, v1_flg=True)
config.make_key_disable(props_path, key)
config.set_props(props_path, key, props_dict['prefix_of_db_path'])
# synk props
msg = message.get('sync_conf')
logger.info(msg)
for node in hosts:
if socket.gethostbyname(node) in config.get_local_ip_list():
continue
client = net.get_ssh(node)
if not client:
msg = message.get('error_ssh_connection').format(host=node)
logger.error(msg)
return
net.copy_dir_to_remote(client, conf_path, conf_path)
client.close()
# set deploy state complete
if os.path.exists(tmp_backup_path):
shutil.rmtree(tmp_backup_path)
for node in target_hosts:
path_of_fb = config.get_path_of_fb(cluster_id)
cluster_path = path_of_fb['cluster_path']
client = net.get_ssh(node)
cmd = 'rm -rf {}'.format(os.path.join(cluster_path, '.deploy.state'))
net.ssh_execute(client=client, command=cmd)
client.close()
if no_localhost:
os.system('touch {}/remote'.format(cluster_path))
msg = message.get('complete_deploy').format(cluster_id=cluster_id)
logger.info(msg)
Cluster().use(cluster_id)
msg = message.get('suggest_after_deploy')
logger.info(msg)
def do_replicate(self, slave, master):
""" Replicate a slave node to a master node.
Use like 'cluster replicate {slave's ip}:{slave's port} {master's ip}:{master's port}
:param slave: {slave's ip or hostname}:{slave's port}
:param master: {master's ip or hostname}:{master's port}
"""
logger.debug('do_replicate')
# Get master's uuid
s_hostname, s_port = slave.split(':')
m_hostname, m_port = master.split(':')
s_host = socket.gethostbyname(s_hostname)
m_host = socket.gethostbyname(m_hostname)
cluster_id = config.get_cur_cluster_id()
lib_path = config.get_ld_library_path(cluster_id)
path_of_fb = config.get_path_of_fb(cluster_id)
sr2_redis_bin = path_of_fb['sr2_redis_bin']
env_cmd = [
'GLOBIGNORE=*;',
'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']),
'export DYLD_LIBRARY_PATH={};'.format(
lib_path['dyld_library_path']),
]
redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli')
sub_cmd = 'cluster nodes'
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
s_host,
s_port,
sub_cmd,
)
stdout = subprocess.check_output(command, shell=True)
outs = ''
outs = '\n'.join([outs, stdout])
lines = outs.splitlines()
m_ip_port = m_host + str(':') + m_port
filtered_lines = (filter(lambda x: m_ip_port in x, lines))
if len(filtered_lines) == 0:
msg = message.get('error_need_cluster_meet')
self._print(msg)
msg = message.get('cluster_meet')
self._print(msg)
# Cluster meet
sub_cmd = 'cluster meet {ip} {port}'.format(ip=m_host, port=m_port)
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
s_host,
s_port,
sub_cmd,
)
stdout = subprocess.check_output(command, shell=True)
self._print(stdout)
# Get master's uuid
sub_cmd = 'cluster nodes'
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
s_host,
s_port,
sub_cmd,
)
stdout = subprocess.check_output(command, shell=True)
outs = ''
outs = '\n'.join([outs, stdout])
lines = outs.splitlines()
filtered_lines = (filter(lambda x: m_ip_port in x, lines))
m_uuid = filtered_lines[0].split()[0]
else:
m_uuid = filtered_lines[0].split()[0]
if len(m_uuid) == 0:
msg = message.get('error_no_uuid')
raise ClusterRedisError(msg)
# Replicate
msg = message.get('start_replicate')
self._print(msg)
sub_cmd = 'cluster replicate {uuid}'.format(uuid=m_uuid)
command = '{} {} -h {} -p {} {}'.format(
' '.join(env_cmd),
redis_cli_cmd,
s_host,
s_port,
sub_cmd,
)
stdout = subprocess.check_output(command, shell=True)
outs = ''
outs = '\n'.join([outs, stdout])
self._print(outs)
