def command(
sub_cmd, cluster_id=-1, mute=False, formatter=None, host=None, port=0):
"""Send redis-cli command
:param sub_cmd: sub command
:param cluster_id: target cluster #
:param mute: without stdout
:param formatter: If set, call formatter with output string
:param host: host
:param port: port
:return: output string
"""
ip_list = config.get_node_ip_list(cluster_id)
port_list = config.get_master_port_list(cluster_id)
if host:
ip_list = [host]
port_list = [port]
outs = RedisCliUtil.command_raw(sub_cmd, ip_list, port_list)
if mute:
return outs
if formatter:
formatter(outs)
else:
logger.info(outs)
return outs
def test_redis_cluster():
cluster_id = 0
ip_list = config.get_node_ip_list(cluster_id)
port_list = config.get_master_port_list(cluster_id)
startup_nodes = utils.get_ip_port_dict_list(ip_list, port_list)
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
d = {'k': {'k2': [1, 2, 3]}}
# s = json.dumps(d)
rc.set('cols', d)
print(rc.get('cols'))
def create_cluster_connection_0():
"""Create cluster 0 for metadata db
:return: redis cluster instance
"""
cluster_id = 0
ip_list = config.get_node_ip_list(cluster_id)
port_list = config.get_master_port_list(cluster_id)
startup_nodes = get_ip_port_dict_list(ip_list, port_list)
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
return rc
def clean(self):
"""Clean (data)
Clean data
"""
logger.debug('clean start')
ip_list = config.get_node_ip_list()
master_port_list = config.get_master_port_list()
slave_port_list = config.get_slave_port_list()
self._rm_data(ip_list, master_port_list, slave_port_list)
self._rm_conf(ip_list, master_port_list, slave_port_list)
self._rm_nodes_conf(ip_list, master_port_list, slave_port_list)
logger.info('clean complete')
def rowcount(self):
"""Query and show cluster row count"""
logger.debug('rowcount')
# open-redis-cli-all info Tablespace | grep totalRows | awk -F ',
# ' '{print $4}' | awk -F '=' '{sum += $2} END {print sum}'
ip_list = config.get_node_ip_list()
port_list = config.get_master_port_list()
outs, meta = RedisCliUtil.command_raw_all('info Tablespace', ip_list,
port_list)
lines = outs.splitlines()
key = 'totalRows'
filtered_lines = (filter(lambda x: key in x, lines))
ld = RedisCliUtil.to_list_of_dict(filtered_lines)
# row_count = reduce(lambda x, y: {key: int(x[key]) + int(y[key])}, ld)
row_count = reduce(lambda x, y: x + int(y[key]), ld, 0)
self._print(row_count)
def command_all(sub_cmd, cluster_id=-1, formatter=None):
"""Send redis-cli command to all
:param sub_cmd: sub command
:param cluster_id: target cluster #
:param formatter: if set, call formatter with output string
"""
if cluster_id < 0:
cluster_id = config.get_cur_cluster_id()
ip_list = config.get_node_ip_list(cluster_id)
port_list = config.get_master_port_list(cluster_id)
outs, meta = RedisCliUtil.command_raw_all(
sub_cmd, ip_list, port_list)
if formatter:
formatter(meta)
else:
logger.debug(outs)
def monitor(self):
"""Monitor thriftserver (using tail -f)
"""
cur_cluster_id = config.get_cur_cluster_id()
logger.info('Start monitor cluster_id: %s' % cur_cluster_id)
ip_list = config.get_node_ip_list()
i = 1
msg = ''
for ip in ip_list:
msg += '%d) %s\n' % (i, ip)
i += 1
target_num = int(askInt(text=msg, default='1'))
logger.info('Ok. %s' % target_num)
target_index = target_num - 1
ip = ip_list[target_index]
client = get_ssh(ip)
envs = config.get_env_dict(ip, 0)
tl = envs['sr2_redis_log'] # TODO: change redis log path
command = 'tail -f {tl}/*'.format(tl=tl)
ssh_execute_async(client=client, command=command)
def run_monitor():
"""Run monitor command
Monitor remote logs
"""
ip_list = get_node_ip_list()
i = 1
msg = ''
for ip in ip_list:
msg += '%d) %s\n' % (i, ip)
i += 1
target_num = int(askInt(text=msg, default='1'))
logger.info('Ok. %s' % target_num)
target_index = target_num - 1
ip = ip_list[target_index]
client = get_ssh(ip)
envs = get_env_dict(ip, 0)
redis_log = envs['sr2_redis_log']
command = 'tail -f {redis_log}/*'.format(redis_log=redis_log)
ssh_execute_async(client=client, command=command)
def _update_ip_port(self):
self.ip_list = config.get_node_ip_list()
self.master_ip_list = self.ip_list
self.slave_ip_list = config.get_slave_ip_list()
self.master_port_list = config.get_master_port_list()
self.slave_port_list = config.get_slave_port_list()
def yaml_to_redis_props(repo_path, cluster_id, report=None):
node_ip_list = get_node_ip_list()
ip_list_str = ' '.join(node_ip_list)
conf_path = path_join(repo_path, 'clusters', str(cluster_id),
'tsr2-conf')
fb_config = config.get_config()
slave_enabled = config.is_slave_enabled()
slave_off = ''
if not slave_enabled:
slave_off = '#'
d = {
'ip_list_str': ip_list_str,
'master_from': fb_config['master_ports']['from'],
'master_to': fb_config['master_ports']['to'],
'slave_from': fb_config['slave_ports']['from'],
'slave_to': fb_config['slave_ports']['to'],
'slave_off': slave_off,
'ssd_count': fb_config['ssd']['count'],
'flash_db_path': fb_config['prefix']['flash_db_path'],
'redis_data': fb_config['prefix']['redis_data'],
'redis_db_path': fb_config['prefix']['redis_db_path'],
}
t = '''
#!/bin/bash
## Master hosts and ports
export SR2_REDIS_MASTER_HOSTS=( {ip_list_str} )
export SR2_REDIS_MASTER_PORTS=( $(seq {master_from} {master_to}) )
## Slave hosts and ports (optional)
{slave_off}export SR2_REDIS_SLAVE_HOSTS=( {ip_list_str} )
{slave_off}export SR2_REDIS_SLAVE_PORTS=( $(seq {slave_from} {slave_to}) )
## only single data directory in redis db and flash db
## Must exist below variables; 'SR2_REDIS_DATA', 'SR2_REDIS_DB_PATH' and 'SR2_FLASH_DB_PATH'
#export SR2_REDIS_DATA="/nvdrive0/nvkvs/redis"
#export SR2_REDIS_DB_PATH="/nvdrive0/nvkvs/redis"
#export SR2_FLASH_DB_PATH="/nvdrive0/nvkvs/flash"
## multiple data directory in redis db and flash db
export SSD_COUNT={ssd_count}
#export HDD_COUNT=3
export SR2_REDIS_DATA="{redis_data}"
export SR2_REDIS_DB_PATH="{redis_db_path}"
export SR2_FLASH_DB_PATH="{flash_db_path}"
#######################################################
# Example : only SSD data directory
#export SSD_COUNT=3
#export SR2_REDIS_DATA="/ssd_"
#export SR2_REDIS_DB_PATH="/ssd_"
#export SR2_FLASH_DB_PATH="/ssd_"
#######################################################
'''
s = t.format(**d)
logger.debug(s)
redis_properties = path_join(conf_path, 'redis.properties')
with open(redis_properties, 'w') as fd:
fd.write(s)
if report:
report.success()