def run_delete(cluster_id=None):
if not cluster_id:
cluster_id = config.get_cur_cluster_id()
valid = cluster_util.validate_id(cluster_id)
if not valid:
logger.error('Invalid cluster id: {}'.format(cluster_id))
return
if cluster_id not in cluster_util.get_cluster_list():
logger.error('Cluster not exist: {}'.format(cluster_id))
return
cluster = Cluster()
cluster.stop(force=True, reset=True)
# delete cluster folder
cli_cluster_path = config.get_path_of_cli(cluster_id)["cluster_path"]
path_of_fb = config.get_path_of_fb(cluster_id)
fb_cluster_path = path_of_fb["cluster_path"]
props_path = path_of_fb['redis_properties']
key = 'sr2_redis_master_hosts'
nodes = config.get_props(props_path, key, [])
for node in nodes:
client = get_ssh(node)
cmd = [
'rm -rf {};'.format(fb_cluster_path),
'rm -rf {}'.format(cli_cluster_path),
]
ssh_execute(client=client, command=' '.join(cmd))
client.close()
if config.get_cur_cluster_id() == cluster_id:
cluster.use(-1)
def rsync_fb_conf():
logger.info('Sync conf...')
cluster_id = config.get_cur_cluster_id()
if not validate_id(cluster_id):
logger.warn('Invalid cluster id: {}'.format(cluster_id))
return
cluster_list = get_cluster_list()
if cluster_id not in cluster_list:
logger.warn('Cluster not exist: {}'.format(cluster_id))
return
my_address = config.get_local_ip_list()
path_of_fb = config.get_path_of_fb(cluster_id)
props_path = path_of_fb['redis_properties']
key = 'sr2_redis_master_hosts'
nodes = config.get_props(props_path, key, [])
meta = [['HOST', 'RESULT']]
path_of_fb = config.get_path_of_fb(cluster_id)
conf_path = path_of_fb['conf_path']
cluster_path = path_of_fb['cluster_path']
for node in nodes:
if net.get_ip(node) in my_address:
meta.append([node, color.green('OK')])
continue
client = net.get_ssh(node)
if not client:
meta.append([node, color.red('SSH ERROR')])
continue
if not net.is_dir(client, cluster_path):
meta.append([node, color.red('NO CLUSTER')])
continue
net.copy_dir_to_remote(client, conf_path, conf_path)
meta.append([node, color.green('OK')])
utils.print_table(meta)
def open_vim_editor(target='config'):
"""Open vim editor
:param target: config | master | slave | thriftserver
"""
cluster_id = config.get_cur_cluster_id()
full_path = get_full_path_of_props(cluster_id, target)
editor.edit(full_path)
def rsync_and_update_conf():
fb_config = config.get_config()
cluster_id = config.get_cur_cluster_id()
nodes = fb_config['nodes']
repo_path = config.get_root_of_cli_config()
DeployUtil.yaml_to_redis_props(repo_path, cluster_id, None)
DeployUtil.rsync(repo_path, nodes, None)
DeployUtil.overwrite_conf(None)
def get_cli_prompt(user):
"""Return cli prompt
:param user: user name
:return: prompt string
"""
cluster_id = get_cur_cluster_id()
if cluster_id < 0:
cluster_id = '-'
return '{}@flashbase:{}>'.format(user, cluster_id)
def _get_cluster_ids_from_fb():
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
base_directory = path_of_fb['base_directory']
dirs = [
f for f in os.listdir(base_directory)
if not os.path.isfile(os.path.join(base_directory, f))
]
cluster_ids = [d.split('_')[1] for d in dirs if 'cluster_' in d]
return cluster_ids
def get_sql_prompt(user):
"""Return sql prompt
:param user: user name
:return: prompt string
"""
cluster_id = get_cur_cluster_id()
if cluster_id < 0:
cluster_id = '-'
return '({}){}@flashbase:sql>'.format(cluster_id, user)
def _update_table_map(self, rc):
table_map = self.get_table_map()
assert isinstance(table_map, dict), table_map
key = self.table_name
value = self.options['table']
cluster_id = config.get_cur_cluster_id()
table_map[key] = {
'table_id': value,
'cluster_id': cluster_id,
}
rc.set(TABLE_MAP_KEY, json.dumps(table_map))
def save_redis_template_config(key, value):
"""Save redis template config to file
:param key: key
:param value: value
"""
key = key.strip()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
master_template = path_of_fb['master_template']
slave_template = path_of_fb['slave_template']
RedisCliUtil._save_config(master_template, key, value)
RedisCliUtil._save_config(slave_template, key, value)
def _get_thriftserver_info(self):
key = 'thriftserver'
meta = utils.get_meta_data_after_ensure(key)
cur_cluster_id = config.get_cur_cluster_id()
if str(cur_cluster_id) in meta:
full_path = utils.get_full_path_of_props(cluster_id=cur_cluster_id,
target='thriftserver')
my_envs = _get_sh_envs(full_path)
ip = my_envs['HIVE_HOST']
port = int(my_envs['HIVE_PORT'])
ret = ip, port, cur_cluster_id
logger.info('thriftserver_info: ', ret)
return ret
assert False, 'You should edit thriftserver.properties before use it'
return 'localhost', 13000, 0
def edit(self):
"""Command: thriftserver edit
Open vim to edit thriftserver
"""
key = 'thriftserver'
utils.open_vim_editor(target=key)
meta = utils.get_meta_data_after_ensure(key, default_value={})
meta = json.loads(meta)
cur_cluster_id = str(config.get_cur_cluster_id())
now = datetime.datetime.now()
dt = now.strftime("%Y-%m-%d %H:%M:%S")
meta[cur_cluster_id] = dt
utils.set_meta_data(key, meta)
def main(user, password, cluster_id, file, debug):
_initial_check()
if debug:
log.set_mode('debug')
if file:
user_info['print_mode'] = 'file'
logger.debug('Start fbcli')
if not cluster_id:
cluster_id = get_cur_cluster_id()
run_cluster_use(cluster_id)
if cluster_id not in cluster_util.get_cluster_list() + [-1]:
root_of_cli_config = get_root_of_cli_config()
head_path = path_join(root_of_cli_config, 'HEAD')
with open(head_path, 'w') as fd:
fd.write('%s' % '-1')
if 'test' in sys.argv:
run_test()
exit(0)
if file:
_handle_file(user, file, cluster_id)
exit(0)
history = path_join(get_root_of_cli_config(), 'cli_history')
session = PromptSession(lexer=PygmentsLexer(SqlLexer),
completer=fb_completer,
history=FileHistory(history),
auto_suggest=AutoSuggestFromHistory(),
style=style)
while True:
try:
p = get_cli_prompt(user)
logger.info(p)
text = session.prompt()
if text == "exit":
break
if 'fbcli' in text:
old = text
text = text.replace('fbcli', '')
logger.info('> You can use "%s" instead of "%s"' % (text, old))
_handle(text)
except KeyboardInterrupt:
continue
except EOFError:
break
def command_all(sub_cmd, cluster_id=-1, formatter=None):
"""Send redis-cli command to all
:param sub_cmd: sub command
:param cluster_id: target cluster #
:param formatter: if set, call formatter with output string
"""
if cluster_id < 0:
cluster_id = config.get_cur_cluster_id()
ip_list = config.get_node_ip_list(cluster_id)
port_list = config.get_master_port_list(cluster_id)
outs, meta = RedisCliUtil.command_raw_all(
sub_cmd, ip_list, port_list)
if formatter:
formatter(meta)
else:
logger.debug(outs)
def update_redis_conf(self):
"""Update redis config
"""
logger.debug('update_redis_conf')
self._update_ip_port()
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
master_template_path = path_of_fb['master_template']
slave_template_path = path_of_fb['slave_template']
for ip in self.ip_list:
client = get_ssh(ip)
redis_conf_dir = path_join(path_of_fb['conf_path'], 'redis')
ssh_execute(client=client, command='mkdir -p %s' % redis_conf_dir)
self._update_redis_conf(client, master_template_path,
redis_conf_dir, ip, self.master_port_list)
self._update_redis_conf(client, slave_template_path,
redis_conf_dir, ip, self.slave_port_list)
client.close()
logger.info('update_redis_conf complete')
def edit(self):
"""Open vim to edit config file"""
cluster_id = config.get_cur_cluster_id()
path_of_fb = config.get_path_of_fb(cluster_id)
# path_of_cli = config.get_path_of_cli(cluster_id)
# if not os.path.exists(path_of_cli['cluster_path']):
# logger.debug(
# "FileNotExisted: '{}'".format(
# path_of_cli['cluster_path']))
# os.system('mkdir -p {}'.format(path_of_cli['cluster_path']))
# logger.debug("CreateDir: '{}'".format(path_of_cli['cluster_path']))
# if not os.path.exists(path_of_cli['conf_path']):
# logger.debug(
# "FileNotExisted: '{}'".format(
# path_of_cli['conf_path']))
# shutil.copytree(path_of_fb['conf_path'], path_of_cli['conf_path'])
# logger.debug("CopyTree: '{}'".format(path_of_cli['cluster_path']))
editor.edit(path_of_fb['redis_properties'], syntax='sh')
cluster_util.rsync_fb_conf()
def monitor(self):
"""Monitor thriftserver (using tail -f)
"""
cur_cluster_id = config.get_cur_cluster_id()
logger.info('Start monitor cluster_id: %s' % cur_cluster_id)
ip_list = config.get_node_ip_list()
i = 1
msg = ''
for ip in ip_list:
msg += '%d) %s\n' % (i, ip)
i += 1
target_num = int(askInt(text=msg, default='1'))
logger.info('Ok. %s' % target_num)
target_index = target_num - 1
ip = ip_list[target_index]
client = get_ssh(ip)
envs = config.get_env_dict(ip, 0)
tl = envs['sr2_redis_log'] # TODO: change redis log path
command = 'tail -f {tl}/*'.format(tl=tl)
ssh_execute_async(client=client, command=command)
def get_thrift_addr():
"""Get thrift address from file (thriftserver.properties)
:return: ip, port info
"""
cluster_id = config.get_cur_cluster_id()
path = config.get_repo_cluster_path(cluster_id)
p = path_join(path, 'tsr2-conf', 'thriftserver.properties')
ip = None
port = None
with open(p, 'r') as fd:
lines = fd.readlines()
for line in lines:
if 'HIVE_HOST' in line:
ip = line.split('=')[1].strip()
if '#' in ip:
ip = ip.split('#')[0].strip()
if 'HIVE_PORT' in line:
port = line.split('=')[1].strip()
if '#' in port:
port = port.split('#')[0].strip()
return ip, port
def run_deploy(cluster_id=None, history_save=True):
# validate cluster id
if cluster_id is None:
cluster_id = get_cur_cluster_id()
if cluster_id < 1:
msg = 'Select cluster first or type cluster id with argument'
logger.error(msg)
return
if not cluster_util.validate_id(cluster_id):
logger.error('Invalid cluster id: {}'.format(cluster_id))
return
# validate option
if not isinstance(history_save, bool):
logger.error("option '--history-save' can use only 'True' or 'False'")
return
logger.debug("option '--history-save': {}".format(history_save))
# if not isinstance(force, bool):
# logger.error("option '--force' can use only 'True' or 'False'")
# return
# logger.debug("option '--force': {}".format(force))
_deploy(cluster_id, history_save)