def copy_dir_from_remote(client, remote_path, local_path): """copy directory from remote to local if already file exist, overwrite file copy all files recursively directory must exist :param client: SSHClient :param remote_path: absolute path of file :param local_path: absolute path of file """ logger.debug('copy FROM node:{} TO localhost:{}'.format( remote_path, local_path )) sftp = get_sftp(client) listdir = sftp.listdir(remote_path) for f in listdir: r_path = os.path.join(remote_path, f) l_path = os.path.join(local_path, f) if is_dir(client, r_path): if not os.path.exists(l_path): os.mkdir(l_path) copy_dir_from_remote(client, r_path, l_path) else: sftp.get(r_path, l_path)
def find_noaddr(self): """Find noaddr nodes that is not used anymore in cluster """ center = Center() center.update_ip_port() logger.debug('find_noaddr') ret = RedisCliUtil.command_all_async('cluster nodes', slave=True) outs = '' meta = [] for _, host, port, res, stdout in ret: if res == 'OK': outs = '\n'.join([outs, stdout]) lines = outs.splitlines() filtered_lines = (filter(lambda x: 'noaddr' in x, lines)) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) total_list = [] for line in filtered_lines: total_list.append(line.split()[0]) # Remove duplicates unique_list = list(set(total_list)) for uuid in unique_list: meta.append([uuid]) utils.print_table([['UUID']] + meta)
def masters_with_dir(self, server, dir): """Find masters that use the specified directory path :param server: IP or hostname :param dir: directory path """ center = Center() center.update_ip_port() logger.debug('masters_with_dir') master_nodes = center.get_master_obj_list() ret = RedisCliUtil.command_all_async('config get dir', slave=True) outs = '' meta = [] m_endpoint = [] for node in master_nodes: m_endpoint.append(node['addr']) for _, host, port, res, stdout in ret: if res == 'OK': flat_stdout = '\n'.join([outs, stdout]) line = flat_stdout.splitlines() if self.compare_ip(host, server) and dir in line[2]: endpoint = '{}:{}'.format(socket.gethostbyname(host), port) if endpoint in m_endpoint: meta.append([host, port, line[2]]) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) utils.print_table([['HOST', 'PORT', 'PATH']] + meta)
def convert_list_2_hyphen(ports): ''' converted as shown below [1, 2, 3, 5, 7, 8, 10] => ['1-3', '5', '7-8', 10] ''' logger.debug('ports: {}'.format(ports)) ret = [] s = ports[0] pre = ports[0] - 1 try: for port in ports: if pre != port - 1: if s != pre: ret.append('{}-{}'.format(s, pre)) else: ret.append(str(s)) s = port pre = port if s != pre: ret.append('{}-{}'.format(s, pre)) else: ret.append(str(s)) logger.debug('converted: {}'.format(ret)) return ret except Exception: raise ConvertError("Invalid ports: '{}'".format(ports))
def ping(host, duration=3): command = 'ping -c 1 -t {} {} > /dev/null 2>&1'.format(duration, host) response = os.system(command) print(response) logger.debug('ping to {}, respose: {}'.format(host, response)) if response is not 0: raise HostConnectionError(host)
def beeline(self, **kargs): """Connect to thriftserver command line """ logger.debug('thriftserver_command_beeline') _check_spark() cluster_id = config.get_cur_cluster_id() path_of_fb = config.get_path_of_fb(cluster_id) ths_props_path = path_of_fb['thrift_properties'] cmd = 'source {}; echo ${}'.format(ths_props_path, 'HIVE_HOST') host = sp.check_output(cmd, shell=True).decode('utf-8').strip() cmd = 'source {}; echo ${}'.format(ths_props_path, 'HIVE_PORT') port = sp.check_output(cmd, shell=True).decode('utf-8').strip() spark_env = _get_env() base_cmd = '{}/beeline'.format(spark_env['spark_bin']) options = { 'u': 'jdbc:hive2://{}:{}'.format(host, port), 'n': os.environ['USER'] } for key, value in kargs.items(): options[key] = value for key, value in options.items(): base_cmd += ' -{} {}'.format(key, value) logger.debug(base_cmd) msg = message.get('try_connection') logger.info(msg) os.system(base_cmd)
def copy_dir_to_remote(client, local_path, remote_path): """copy directory from local to remote if already file exist, overwrite file copy all files recursively directory must exist :param client: SSHClient :param local_path: absolute path of file :param remote_path: absolute path of file """ logger.debug('copy_dir_to_remote') logger.debug('copy FROM localhost:{} TO node:{}'.format( local_path, remote_path )) sftp = get_sftp(client) listdir = os.listdir(local_path) for f in listdir: r_path = os.path.join(remote_path, f) l_path = os.path.join(local_path, f) if os.path.isdir(l_path): if not is_exist(client, r_path): sftp.mkdir(r_path) copy_dir_to_remote(client, l_path, r_path) else: sftp.put(l_path, r_path)
def failover_with_dir(self, server, dir): """Find masters that use the specified directory path and do failover with its slave :param server: IP or hostname :param dir: directory path """ center = Center() center.update_ip_port() logger.debug('failover_with_dir') master_nodes = center.get_master_obj_list() cluster_id = config.get_cur_cluster_id() lib_path = config.get_ld_library_path(cluster_id) path_of_fb = config.get_path_of_fb(cluster_id) sr2_redis_bin = path_of_fb['sr2_redis_bin'] env_cmd = [ 'GLOBIGNORE=*;', 'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']), 'export DYLD_LIBRARY_PATH={};'.format( lib_path['dyld_library_path']), ] redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli') # Find masters with dir ret = RedisCliUtil.command_all_async('config get dir', slave=True) outs = '' meta = [] m_endpoint = [] for node in master_nodes: m_endpoint.append(node['addr']) for _, host, port, res, stdout in ret: if res == 'OK': flat_stdout = '\n'.join([outs, stdout]) line = flat_stdout.splitlines() if self.compare_ip(host, server) and dir in line[2]: endpoint = '{}:{}'.format(socket.gethostbyname(host), port) if endpoint in m_endpoint: meta.append(endpoint) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) for endpoint in meta: for master_node in master_nodes: if endpoint == master_node['addr']: for slave_node in master_node['slaves']: addr = slave_node['addr'] (s_host, s_port) = addr.split(':') sub_cmd = 'cluster failover takeover' command = '{} {} -h {} -p {} {}'.format( ' '.join(env_cmd), redis_cli_cmd, s_host, s_port, sub_cmd, ) self._print( message.get('try_failover_takeover').format( slave=addr)) stdout = subprocess.check_output(command, shell=True) self._print(stdout)
def start(self, profile=False, master=True, slave=True): """Start cluster :param master: If exclude master cluster, set False :param slave: If exclude slave cluster, set False """ logger.debug("command 'cluster start'") if not isinstance(profile, bool): msg = message.get('error_option_type_not_boolean') msg = msg.format(option='profile') logger.error(msg) return if not isinstance(master, bool): msg = message.get('error_option_type_not_boolean') msg = msg.format(option='master') logger.error(msg) return if not isinstance(slave, bool): msg = message.get('error_option_type_not_boolean') msg = msg.format(option='slave') logger.error(msg) return center = Center() center.update_ip_port() success = center.check_hosts_connection() if not success: return center.ensure_cluster_exist() if master: master_alive_count = center.get_alive_master_redis_count() master_alive_count_mine = center.get_alive_master_redis_count( check_owner=True ) not_mine_count = master_alive_count - master_alive_count_mine if not_mine_count > 0: msg = message.get('error_cluster_start_master_collision') msg = '\n'.join(msg).format(count=not_mine_count) raise LightningDBError(11, msg) if slave: slave_alive_count = center.get_alive_slave_redis_count() slave_alive_count_mine = center.get_alive_slave_redis_count( check_owner=True ) not_mine_count = slave_alive_count - slave_alive_count_mine if not_mine_count > 0: msg = message.get('error_cluster_start_slave_collision') msg = '\n'.join(msg).format(count=not_mine_count) raise LightningDBError(12, msg) center.backup_server_logs(master=master, slave=slave) center.create_redis_data_directory() # equal to cluster.configure() center.configure_redis() center.sync_conf(show_result=True) center.start_redis_process(profile, master=master, slave=slave) center.wait_until_all_redis_process_up(master=master, slave=slave)
def command_all_async(sub_cmd, slave=True): def _async_target_func(m_s, pre_cmd, host, port, sub_cmd, ret): try: command = '{} -h {} -p {} {}'.format(pre_cmd, host, port, sub_cmd) logger.debug(command) stdout = subprocess.check_output(command, shell=True) stdout = stdout.decode('utf-8').strip() ret.append((m_s, host, port, 'OK', stdout)) except Exception as ex: stderr = str(ex) logger.debug(stderr) ret.append((m_s, host, port, 'FAIL', stderr)) cluster_id = config.get_cur_cluster_id() master_host_list = config.get_master_host_list(cluster_id) master_port_list = config.get_master_port_list(cluster_id) if slave: slave_host_list = config.get_slave_host_list(cluster_id) slave_port_list = config.get_slave_port_list(cluster_id) path_of_fb = config.get_path_of_fb(cluster_id) sr2_redis_bin = path_of_fb['sr2_redis_bin'] logger.debug('command_all_async') cluster_id = config.get_cur_cluster_id() lib_path = config.get_ld_library_path(cluster_id) env_cmd = [ 'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']), 'export DYLD_LIBRARY_PATH={};'.format( lib_path['dyld_library_path']), ] env = ' '.join(env_cmd) threads = [] ret = [] # (m/s, host, port, result, message) pre_cmd = '{} {}/redis-cli -c'.format(env, sr2_redis_bin) for host in master_host_list: for port in master_port_list: t = Thread( target=_async_target_func, args=('Master', pre_cmd, host, port, sub_cmd, ret), ) threads.append(t) if slave: for host in slave_host_list: for port in slave_port_list: t = Thread( target=_async_target_func, args=('Slave', pre_cmd, host, port, sub_cmd, ret), ) threads.append(t) for th in threads: th.start() time.sleep(0.02) for th in threads: th.join() logger.debug(ret) return ret
def rowcount(self): """Query and show cluster row count """ logger.debug('rowcount') masters = [] center = Center() center.update_ip_port() master_nodes = center.get_master_obj_list() for master_node in master_nodes: node = master_node['addr'] masters.append(node) # open-redis-cli-all info Tablespace | grep totalRows | awk -F ', # ' '{print $4}' | awk -F '=' '{sum += $2} END {print sum}' ret = RedisCliUtil.command_all_async('info Tablespace', slave=True) outs = '' for _, host, port, res, stdout in ret: if res == 'OK': endpoint = '{}:{}'.format(host, port) if endpoint in masters: outs = '\n'.join([outs, stdout]) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) lines = outs.splitlines() key = 'totalRows' partitions = 'partitions' evictions = 'evictedRows' filtered_lines = (filter(lambda x: key in x, lines)) #self._print(filtered_lines) # Table list table_list = [] result = [] for line in filtered_lines: tableStats, _ = line.split(':') tableId = tableStats.split('_') if tableId[1] in table_list: pass else: table_list.append(tableId[1]) for tid in table_list: table_lines = (filter(lambda x: tid in x, filtered_lines)) ld = RedisCliUtil.to_list_of_dict(table_lines) row_count = reduce(lambda x, y: x + int(y[key]), ld, 0) partitions_count = reduce(lambda x, y: x + int(y[partitions]), ld, 0) evictions_count = reduce(lambda x, y: x + int(y[evictions]), ld, 0) result.append([tid, row_count, partitions_count, evictions_count]) utils.print_table( [['Table_ID', 'ROW_COUNT', 'PARTITION_COUNT', 'EVICTED_ROWS']] + result)
def base_directory(default='~/tsr2'): logger.debug('ask base directory') result = ask(message.get('ask_base_directory'), default=default) if not result.startswith(('~', '/')): logger.error(message.get('error_invalid_path').format(value=result)) return base_directory() logger.info('OK, {}'.format(result)) cli_config = config.get_cli_config() cli_config['base_directory'] = result config.save_cli_config(cli_config) return result
def get_sftp(client): """Open sftp :param client: SSHClient instance :return: opened sftp instance """ try: sftp = client.open_sftp() return sftp except Exception as e: logger.debug(e)
def prefix_of_db_path(save, default=None): logger.debug('ask_prefix_of_db_path') deploy_history = config.get_deploy_history() if not default: default = deploy_history['prefix_of_db_path'] result = ask(message.get('ask_db_path'), default=default) result = result.strip() if save: deploy_history['prefix_of_db_path'] = result config.save_deploy_history(deploy_history) logger.info('OK, {}'.format(result)) return result
def _async_target_func(m_s, pre_cmd, host, port, sub_cmd, ret): try: command = '{} -h {} -p {} {}'.format(pre_cmd, host, port, sub_cmd) logger.debug(command) stdout = subprocess.check_output(command, shell=True) stdout = stdout.decode('utf-8').strip() ret.append((m_s, host, port, 'OK', stdout)) except Exception as ex: stderr = str(ex) logger.debug(stderr) ret.append((m_s, host, port, 'FAIL', stderr))
def hosts(save, default=None): logger.debug('ask host') deploy_history = config.get_deploy_history() if not default: default = deploy_history['hosts'] msg = message.get('ask_hosts') result = ask(msg, default=', '.join(default)) result = list(map(lambda x: x.strip(), result.split(','))) if save: deploy_history['hosts'] = result config.save_deploy_history(deploy_history) logger.info('OK, {}'.format(result)) return result
def stop(self): """Stop thriftserver """ logger.debug('thriftserver_command_stop') _check_spark() cluster_id = config.get_cur_cluster_id() path_of_fb = config.get_path_of_fb(cluster_id) ths_props_path = path_of_fb['thrift_properties'] source_cmd = 'source {}'.format(ths_props_path) base_cmd = '$SPARK_SBIN/stop-thriftserver.sh' cmd = '{}; {}'.format(source_cmd, base_cmd) logger.debug(cmd) os.system(cmd)
def ssd_count(save, default=None): logger.debug('ask ssd count') deploy_history = config.get_deploy_history() if not default: default = deploy_history['ssd_count'] result = int(askInt(message.get('ask_ssd_count'), default=str(default))) if result <= 0: logger.error(message.get('error_ssd_count_less_than_1')) return ssd_count(save=save, default=default) if save: deploy_history['ssd_count'] = result config.save_deploy_history(deploy_history) logger.info('OK, {}'.format(result)) return result
def replicas(save, default=None): logger.debug('ask replicas') deploy_history = config.get_deploy_history() if not default: default = deploy_history['replicas'] result = askInt(message.get('ask_replicas'), default=str(default)) result = int(result) if result < 0: logger.error(message.get('error_replicas_less_than_0')) return replicas(save, default=default) if save: deploy_history['replicas'] = result config.save_deploy_history(deploy_history) logger.info('OK, {}'.format(result)) return result
def _save_config(f, key, value): inplace_count = 0 for line in fileinput.input(f, inplace=True): words = line.split() if words and words[0] == key: msg = '{key} {value}'.format(key=key, value=value) inplace_count += 1 print(msg) else: print(line, end='') logger.debug('inplace: %d (%s)' % (inplace_count, f)) if inplace_count == 1: logger.debug('save config(%s) success' % f) else: msg = message.get('error_save_config').format(key=key, file=f) logger.warning(msg)
def run_deploy(cluster_id=None, history_save=True, clean=False, strategy="none"): """Install LightningDB package. :param cluster_id: cluster id :param history_save: save input history and use as default :param clean: delete redis log, node configuration :param strategy: none(default): normal deploy, zero-downtime: re-deploy without stop """ # validate cluster id if cluster_id is None: cluster_id = config.get_cur_cluster_id(allow_empty_id=True) if cluster_id < 0: msg = message.get('error_invalid_cluster_on_deploy') logger.error(msg) return if not cluster_util.validate_id(cluster_id): raise ClusterIdError(cluster_id) # validate option if not isinstance(history_save, bool): msg = message.get('error_option_type_not_boolean') msg = msg.format(option='history-save') logger.error(msg) return logger.debug("option '--history-save': {}".format(history_save)) if not isinstance(clean, bool): msg = message.get('error_option_type_not_boolean') msg = msg.format(option='clean') logger.error(msg) return logger.debug("option '--clean': {}".format(clean)) strategy_list = ["none", "zero-downtime"] if strategy not in strategy_list: msg = message.get('error_deploy_strategy').format(value=strategy, list=strategy_list) logger.error(msg) return if strategy == "zero-downtime": run_cluster_use(cluster_id) _deploy_zero_downtime(cluster_id) return _deploy(cluster_id, history_save, clean)
def main(cluster_id, debug, version): if version: print_version() return _initial_check() if debug: log.set_mode('debug') logger.debug('Start ltcli') cluster_id = _validate_cluster_id(cluster_id) history = os.path.join(config.get_root_of_cli_config(), 'cli_history') session = PromptSession( # lexer=PygmentsLexer(SqlLexer), history=FileHistory(history), auto_suggest=AutoSuggestFromHistory(), style=utils.style) while True: try: exit_flg = False p = prompt.get_cli_prompt() text = session.prompt(p, style=utils.style) command_list = text.split(';') for cmd in command_list: cmd = cmd.strip() if cmd == "exit": exit_flg = True break if 'ltcli' in cmd: old = cmd cmd = cmd.replace('ltcli', '').strip() msg = message.get('notify_command_replacement_is_possible') msg = msg.format(new=cmd, old=old) logger.info(msg) err_flg = _handle(cmd) if err_flg: break if exit_flg: break except ClusterNotExistError: run_cluster_use(-1) continue except KeyboardInterrupt: continue except EOFError: break
def install(self, host, cluster_id, name): logger.debug('Deploy cluster {} at {}...'.format(cluster_id, host)) path_of_fb = config.get_path_of_fb(cluster_id) release_path = path_of_fb['release_path'] cluster_path = path_of_fb['cluster_path'] if '/' in name: name = os.path.basename(name) installer_path = os.path.join(release_path, name) command = '''chmod 755 {0}; \ PATH=${{PATH}}:/usr/sbin; \ {0} --full {1}'''.format(installer_path, cluster_path) client = net.get_ssh(host) if not net.is_exist(client, installer_path): raise FileNotExistError(installer_path, host=host) net.ssh_execute(client=client, command=command) client.close() logger.debug('OK')
def force_failover(self, server): """ Find all masters on the server and convert them to slaves. Finally, in the server, only slaves will be remained. :param server: IP or hostname """ logger.debug('force_failover') center = Center() center.update_ip_port() master_nodes = center.get_master_obj_list() cluster_id = config.get_cur_cluster_id() lib_path = config.get_ld_library_path(cluster_id) path_of_fb = config.get_path_of_fb(cluster_id) sr2_redis_bin = path_of_fb['sr2_redis_bin'] env_cmd = [ 'GLOBIGNORE=*;', 'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']), 'export DYLD_LIBRARY_PATH={};'.format( lib_path['dyld_library_path']), ] redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli') outs = '' meta = [] m_endpoint = [] for node in master_nodes: addr = node['addr'] (host, port) = addr.split(':') # if host == server: if self.compare_ip(host, server): for slave_node in node['slaves']: addr = slave_node['addr'] (s_host, s_port) = addr.split(':') sub_cmd = 'cluster failover takeover' command = '{} {} -h {} -p {} {}'.format( ' '.join(env_cmd), redis_cli_cmd, s_host, s_port, sub_cmd, ) self._print( message.get('try_failover_takeover').format( slave=addr)) stdout = subprocess.check_output(command, shell=True) self._print(stdout)
def distribution(self): """Check the distribution of all masters and slaves """ center = Center() center.update_ip_port() logger.debug('distribution') ret = RedisCliUtil.command_all_async('cluster nodes', slave=True) outs = '' for _, host, port, res, stdout in ret: if res == 'OK': outs = '\n'.join([outs, stdout]) lines = outs.splitlines() myself_key = 'myself' filtered_lines = (filter(lambda x: myself_key in x, lines)) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) meta = [] total_masters = 0 total_slaves = 0 for nd in center.master_host_list: num_of_masters = 0 num_of_slaves = 0 node = socket.gethostbyname(nd) host_lines = (filter(lambda x: (node + ':') in x, filtered_lines)) for node in host_lines: params = node.split() endpoint = params[1] roles = params[2] host = endpoint.split(':')[0] role = roles.split(',')[1] if role == 'master': if len(params) == 9: num_of_masters += 1 else: num_of_slaves += 1 total_masters += num_of_masters total_slaves += num_of_slaves hostname = str(socket.gethostbyaddr(host)[0]) + str('(') + str( host) + str(')') meta.append([hostname, num_of_masters, num_of_slaves]) meta.append(['TOTAL', total_masters, total_slaves]) utils.print_table([['HOST', 'MASTER', 'SLAVE']] + meta)
def start(self): """Start thriftserver """ logger.debug('thriftserver_command_start') _check_spark() cluster_id = config.get_cur_cluster_id() path_of_fb = config.get_path_of_fb(cluster_id) ths_props_path = path_of_fb['thrift_properties'] source_cmd = 'source {}'.format(ths_props_path) hive_opts = _get_hive_opts_str() base_cmd = '$SPARK_SBIN/start-thriftserver.sh {}'.format(hive_opts) cmd = '{}; {}'.format(source_cmd, base_cmd) logger.debug(cmd) os.system(cmd) spark_log = os.path.join(os.environ['SPARK_HOME'], 'logs') if _find_files_with_regex(spark_log, ROLLING_LOGFILE_REGEX): for file in _find_files_with_regex(spark_log, NOHUP_LOGFILE_REGEX): os.remove(os.path.join(spark_log, file))
def reset_distribution(self): """ Reset the distribution of masters and slaves with original setting """ center = Center() center.update_ip_port() logger.debug('reset_distribution') cluster_id = config.get_cur_cluster_id() lib_path = config.get_ld_library_path(cluster_id) path_of_fb = config.get_path_of_fb(cluster_id) sr2_redis_bin = path_of_fb['sr2_redis_bin'] env_cmd = [ 'GLOBIGNORE=*;', 'export LD_LIBRARY_PATH={};'.format(lib_path['ld_library_path']), 'export DYLD_LIBRARY_PATH={};'.format( lib_path['dyld_library_path']), ] redis_cli_cmd = os.path.join(sr2_redis_bin, 'redis-cli') slave_nodes = center.get_slave_nodes() master_ports = center.master_port_list for slave_node in slave_nodes: (host, port) = slave_node.split(':') try: value = int(port) if value in master_ports: # failover takeover msg = message.get('try_failover_takeover').format( slave=slave_node) self._print(msg) sub_cmd = 'cluster failover takeover' command = '{} {} -h {} -p {} {}'.format( ' '.join(env_cmd), redis_cli_cmd, host, port, sub_cmd, ) stdout = subprocess.check_output(command, shell=True) outs = '' outs = '\n'.join([outs, stdout]) self._print(outs) except ValueError: pass
def get_props_as_dict(props_path): ret = dict() with open(props_path, 'r') as f: lines = f.readlines() for i, line in enumerate(lines): if line.strip().startswith('#'): continue p = re.compile( r'export [^ \s\t\r\n\v\f]+=(\(.+\)|[^ \s\t\r\n\v\f]+)') m = p.search(line) if not m: continue s = m.start() e = m.end() key, value = line[s:e + 1].replace('export ', '').split('=') value = value.strip() key = key.lower() p = re.compile(r'\(.*\)') m = p.match(value) try: if m: cmd = [ 'FBCLI_TMP_ENV={}'.format(value), '&&', 'echo ${FBCLI_TMP_ENV[@]}' ] cmd = ' '.join(cmd) logger.debug('subprocess cmd: {}'.format(cmd)) value = subprocess.check_output(cmd, shell=True) value = to_str(value.strip()) logger.debug('subprocess result: {}'.format(value)) value = value.split(' ') value = map(lambda x: int(x) if is_number(x) else x, value) value = filter(lambda x: bool(x), value) value = list(value) else: cmd = 'echo {}'.format(value) value = subprocess.check_output(cmd, shell=True) value = to_str(value.strip()) value = int(value) if is_number(value) else value ret[key] = value except subprocess.CalledProcessError: raise PropsSyntaxError(value, i + 1) return ret
def download_file(url, file_path): download_path = file_path + '.download' file_name = os.path.basename(file_path) try: with open(download_path, 'wb') as f: msg = message.get('file_download').format(file_name=file_name) logger.info(msg) logger.debug('url: {}'.format(url)) logger.debug('installer name: {}'.format(file_name)) response = requests.get(url, stream=True) response.raise_for_status() total_length = response.headers.get('content-length') if total_length is None: f.write(response.content) else: done_length = 0 total_length = int(total_length) for data in response.iter_content(chunk_size=4096): done_length += len(data) f.write(data) done = int(100 * done_length / total_length) comp = '=' * int(done / 2) remain = ' ' * int(50 - int(done / 2)) progress = '\r[{}{}] {}%'.format(comp, remain, done) sys.stdout.write(progress) sys.stdout.flush() print('') shutil.move(download_path, file_path) return True except requests.exceptions.HTTPError as ex: logger.warning(ex) return False except KeyboardInterrupt as ex: print('') raise ex except BaseException as ex: class_name = ex.__class__.__name__ logger.warning('{}: {}'.format(class_name, url)) return False finally: if os.path.isfile(download_path): os.remove(download_path)
def rowcount(self): """Query and show cluster row count """ logger.debug('rowcount') # open-redis-cli-all info Tablespace | grep totalRows | awk -F ', # ' '{print $4}' | awk -F '=' '{sum += $2} END {print sum}' ret = RedisCliUtil.command_all_async('info Tablespace', slave=False) outs = '' for _, host, port, res, stdout in ret: if res == 'OK': outs = '\n'.join([outs, stdout]) else: logger.warning("FAIL {}:{} {}".format(host, port, stdout)) lines = outs.splitlines() key = 'totalRows' filtered_lines = (filter(lambda x: key in x, lines)) ld = RedisCliUtil.to_list_of_dict(filtered_lines) # row_count = reduce(lambda x, y: {key: int(x[key]) + int(y[key])}, ld) row_count = reduce(lambda x, y: x + int(y[key]), ld, 0) self._print(row_count)