def handle_osd_status(self, cmd): osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data', 'state']) osdmap = self.get("osd_map") filter_osds = set() bucket_filter = None if 'bucket' in cmd: self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket'])) bucket_filter = cmd['bucket'] crush = self.get("osd_map_crush") found = False for bucket in crush['buckets']: if fnmatch.fnmatch(bucket['name'], bucket_filter): found = True filter_osds.update([i['id'] for i in bucket['items']]) if not found: msg = "Bucket '{0}' not found".format(bucket_filter) return errno.ENOENT, msg, "" # Build dict of OSD ID to stats osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']]) for osd in osdmap['osds']: osd_id = osd['osd'] if bucket_filter and osd_id not in filter_osds: continue hostname = "" kb_used = 0 kb_avail = 0 if osd_id in osd_stats: metadata = self.get_metadata('osd', "%s" % osd_id) stats = osd_stats[osd_id] hostname = metadata['hostname'] kb_used = stats['kb_used'] * 1024 kb_avail = stats['kb_avail'] * 1024 osd_table.add_row([osd_id, hostname, mgr_util.format_bytes(kb_used, 5), mgr_util.format_bytes(kb_avail, 5), mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") + self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5), mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5), mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5), mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5), ','.join(osd['state']), ]) return 0, osd_table.get_string(), ""
def handle_osd_status(self, cmd): osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data', 'state']) osdmap = self.get("osd_map") filter_osds = set() bucket_filter = None if 'bucket' in cmd: self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket'])) bucket_filter = cmd['bucket'] crush = self.get("osd_map_crush") found = False for bucket in crush['buckets']: if fnmatch.fnmatch(bucket['name'], bucket_filter): found = True filter_osds.update([i['id'] for i in bucket['items']]) if not found: msg = "Bucket '{0}' not found".format(bucket_filter) return errno.ENOENT, msg, "" # Build dict of OSD ID to stats osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']]) for osd in osdmap['osds']: osd_id = osd['osd'] if bucket_filter and osd_id not in filter_osds: continue hostname = "" kb_used = 0 kb_avail = 0 if osd_id in osd_stats: metadata = self.get_metadata('osd', "%s" % osd_id) stats = osd_stats[osd_id] hostname = metadata['hostname'] kb_used = stats['kb_used'] * 1024 kb_avail = stats['kb_avail'] * 1024 osd_table.add_row([osd_id, hostname, mgr_util.format_bytes(kb_used, 5), mgr_util.format_bytes(kb_avail, 5), mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") + self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5), mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5), mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5), mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5), ','.join(osd['state']), ]) return 0, osd_table.get_string(), ""
def handle_fs_status(self, cmd): output = "" json_output = defaultdict(list) output_format = cmd.get('format', 'plain') fs_filter = cmd.get('fs', None) mds_versions = defaultdict(list) fsmap = self.get("fs_map") for filesystem in fsmap['filesystems']: if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter: continue rank_table = PrettyTable( ("RANK", "STATE", "MDS", "ACTIVITY", "DNS", "INOS"), border=False, ) rank_table.left_padding_width = 0 rank_table.right_padding_width = 2 mdsmap = filesystem['mdsmap'] client_count = 0 for rank in mdsmap["in"]: up = "mds_{0}".format(rank) in mdsmap["up"] if up: gid = mdsmap['up']["mds_{0}".format(rank)] info = mdsmap['info']['gid_{0}'.format(gid)] dns = self.get_latest("mds", info['name'], "mds_mem.dn") inos = self.get_latest("mds", info['name'], "mds_mem.ino") if rank == 0: client_count = self.get_latest( "mds", info['name'], "mds_sessions.session_count") elif client_count == 0: # In case rank 0 was down, look at another rank's # sessionmap to get an indication of clients. client_count = self.get_latest( "mds", info['name'], "mds_sessions.session_count") laggy = "laggy_since" in info state = info['state'].split(":")[1] if laggy: state += "(laggy)" if state == "active" and not laggy: c_state = mgr_util.colorize(state, mgr_util.GREEN) else: c_state = mgr_util.colorize(state, mgr_util.YELLOW) # Populate based on context of state, e.g. client # ops for an active daemon, replay progress, reconnect # progress activity = "" if state == "active": rate = self.get_rate( "mds", info['name'], "mds_server.handle_client_request") if output_format not in ('json', 'json-pretty'): activity = "Reqs: " + mgr_util.format_dimless( rate, 5) + "/s" metadata = self.get_metadata('mds', info['name']) mds_versions[metadata.get('ceph_version', "unknown")].append(info['name']) if output_format in ('json', 'json-pretty'): json_output['mdsmap'].append({ 'rank': rank, 'name': info['name'], 'state': state, 'rate': rate if state == "active" else "0", 'dns': dns, 'inos': inos }) else: rank_table.add_row([ mgr_util.bold(rank.__str__()), c_state, info['name'], activity, mgr_util.format_dimless(dns, 5), mgr_util.format_dimless(inos, 5) ]) else: if output_format in ('json', 'json-pretty'): json_output['mdsmap'].append({ 'rank': rank, 'state': "failed" }) else: rank_table.add_row([rank, "failed", "", "", "", ""]) # Find the standby replays for gid_str, daemon_info in six.iteritems(mdsmap['info']): if daemon_info['state'] != "up:standby-replay": continue inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn") events = self.get_rate("mds", daemon_info['name'], "mds_log.replayed") if output_format not in ('json', 'json-pretty'): activity = "Evts: " + mgr_util.format_dimless(events, 5) + "/s" metadata = self.get_metadata('mds', daemon_info['name']) mds_versions[metadata.get('ceph_version', "unknown")].append( daemon_info['name']) if output_format in ('json', 'json-pretty'): json_output['mdsmap'].append({ 'rank': rank, 'name': daemon_info['name'], 'state': 'standby-replay', 'events': events, 'dns': 5, 'inos': 5 }) else: rank_table.add_row([ "{0}-s".format(daemon_info['rank']), "standby-replay", daemon_info['name'], activity, mgr_util.format_dimless(dns, 5), mgr_util.format_dimless(inos, 5) ]) df = self.get("df") pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) osdmap = self.get("osd_map") pools = dict([(p['pool'], p) for p in osdmap['pools']]) metadata_pool_id = mdsmap['metadata_pool'] data_pool_ids = mdsmap['data_pools'] pools_table = PrettyTable(["POOL", "TYPE", "USED", "AVAIL"], border=False) pools_table.left_padding_width = 0 pools_table.right_padding_width = 2 for pool_id in [metadata_pool_id] + data_pool_ids: pool_type = "metadata" if pool_id == metadata_pool_id else "data" stats = pool_stats[pool_id] if output_format in ('json', 'json-pretty'): json_output['pools'].append({ 'id': pool_id, 'name': pools[pool_id]['pool_name'], 'type': pool_type, 'used': stats['bytes_used'], 'avail': stats['max_avail'] }) else: pools_table.add_row([ pools[pool_id]['pool_name'], pool_type, mgr_util.format_bytes(stats['bytes_used'], 5), mgr_util.format_bytes(stats['max_avail'], 5) ]) if output_format in ('json', 'json-pretty'): json_output['clients'].append({ 'fs': mdsmap['fs_name'], 'clients': client_count, }) else: output += "{0} - {1} clients\n".format(mdsmap['fs_name'], client_count) output += "=" * len(mdsmap['fs_name']) + "\n" output += rank_table.get_string() output += "\n" + pools_table.get_string() + "\n" if not output and not json_output and fs_filter is not None: return errno.EINVAL, "", "Invalid filesystem: " + fs_filter standby_table = PrettyTable(["STANDBY MDS"], border=False) standby_table.left_padding_width = 0 standby_table.right_padding_width = 2 for standby in fsmap['standbys']: metadata = self.get_metadata('mds', standby['name']) mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name']) if output_format in ('json', 'json-pretty'): json_output['mdsmap'].append({ 'name': standby['name'], 'state': "standby" }) else: standby_table.add_row([standby['name']]) if output_format not in ('json', 'json-pretty'): output += "\n" + standby_table.get_string() + "\n" if len(mds_versions) == 1: if output_format in ('json', 'json-pretty'): json_output['mds_version'] = list(mds_versions)[0] else: output += "MDS version: {0}".format(list(mds_versions)[0]) else: version_table = PrettyTable(["VERSION", "DAEMONS"], border=False) version_table.left_padding_width = 0 version_table.right_padding_width = 2 for version, daemons in six.iteritems(mds_versions): if output_format in ('json', 'json-pretty'): json_output['mds_version'].append({ 'version': version, 'daemons': daemons }) else: version_table.add_row([version, ", ".join(daemons)]) if output_format not in ('json', 'json-pretty'): output += version_table.get_string() + "\n" if output_format == "json": return HandleCommandResult( stdout=json.dumps(json_output, sort_keys=True)) elif output_format == "json-pretty": return HandleCommandResult(stdout=json.dumps( json_output, sort_keys=True, indent=4, separators=(',', ': '))) else: return HandleCommandResult(stdout=output)
def handle_osd_status(self, bucket: Optional[str] = None ) -> Tuple[int, str, str]: """ Show the status of OSDs within a bucket, or all """ osd_table = PrettyTable([ 'ID', 'HOST', 'USED', 'AVAIL', 'WR OPS', 'WR DATA', 'RD OPS', 'RD DATA', 'STATE' ], border=False) osd_table.align['ID'] = 'r' osd_table.align['HOST'] = 'l' osd_table.align['USED'] = 'r' osd_table.align['AVAIL'] = 'r' osd_table.align['WR OPS'] = 'r' osd_table.align['WR DATA'] = 'r' osd_table.align['RD OPS'] = 'r' osd_table.align['RD DATA'] = 'r' osd_table.align['STATE'] = 'l' osd_table.left_padding_width = 0 osd_table.right_padding_width = 2 osdmap = self.get("osd_map") filter_osds = set() bucket_filter = None if bucket is not None: self.log.debug(f"Filtering to bucket '{bucket}'") bucket_filter = bucket crush = self.get("osd_map_crush") found = False for bucket_ in crush['buckets']: if fnmatch.fnmatch(bucket_['name'], bucket_filter): found = True filter_osds.update([i['id'] for i in bucket_['items']]) if not found: msg = "Bucket '{0}' not found".format(bucket_filter) return errno.ENOENT, msg, "" # Build dict of OSD ID to stats osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']]) for osd in osdmap['osds']: osd_id = osd['osd'] if bucket_filter and osd_id not in filter_osds: continue hostname = "" kb_used = 0 kb_avail = 0 if osd_id in osd_stats: metadata = self.get_metadata('osd', str(osd_id), default=defaultdict(str)) stats = osd_stats[osd_id] assert metadata hostname = metadata['hostname'] kb_used = stats['kb_used'] * 1024 kb_avail = stats['kb_avail'] * 1024 wr_ops_rate = (self.get_rate("osd", osd_id.__str__(), "osd.op_w") + self.get_rate("osd", osd_id.__str__(), "osd.op_rw")) wr_byte_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes") rd_ops_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_r") rd_byte_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes") osd_table.add_row([ osd_id, hostname, mgr_util.format_bytes(kb_used, 5), mgr_util.format_bytes(kb_avail, 5), mgr_util.format_dimless(wr_ops_rate, 5), mgr_util.format_bytes(wr_byte_rate, 5), mgr_util.format_dimless(rd_ops_rate, 5), mgr_util.format_bytes(rd_byte_rate, 5), ','.join(osd['state']), ]) return 0, osd_table.get_string(), ""
def handle_fs_status(self, cmd): output = "" fs_filter = cmd.get('fs', None) mds_versions = defaultdict(list) fsmap = self.get("fs_map") for filesystem in fsmap['filesystems']: if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter: continue rank_table = PrettyTable( ("Rank", "State", "MDS", "Activity", "dns", "inos"), hrules=prettytable.FRAME ) mdsmap = filesystem['mdsmap'] client_count = 0 for rank in mdsmap["in"]: up = "mds_{0}".format(rank) in mdsmap["up"] if up: gid = mdsmap['up']["mds_{0}".format(rank)] info = mdsmap['info']['gid_{0}'.format(gid)] dns = self.get_latest("mds", info['name'], "mds_mem.dn") inos = self.get_latest("mds", info['name'], "mds_mem.ino") if rank == 0: client_count = self.get_latest("mds", info['name'], "mds_sessions.session_count") elif client_count == 0: # In case rank 0 was down, look at another rank's # sessionmap to get an indication of clients. client_count = self.get_latest("mds", info['name'], "mds_sessions.session_count") laggy = "laggy_since" in info state = info['state'].split(":")[1] if laggy: state += "(laggy)" if state == "active" and not laggy: c_state = self.colorize(state, self.GREEN) else: c_state = self.colorize(state, self.YELLOW) # Populate based on context of state, e.g. client # ops for an active daemon, replay progress, reconnect # progress activity = "" if state == "active": activity = "Reqs: " + mgr_util.format_dimless( self.get_rate("mds", info['name'], "mds_server.handle_client_request"), 5 ) + "/s" metadata = self.get_metadata('mds', info['name']) mds_versions[metadata.get('ceph_version', "unknown")].append(info['name']) rank_table.add_row([ self.bold(rank.__str__()), c_state, info['name'], activity, mgr_util.format_dimless(dns, 5), mgr_util.format_dimless(inos, 5) ]) else: rank_table.add_row([ rank, "failed", "", "", "", "" ]) # Find the standby replays for gid_str, daemon_info in six.iteritems(mdsmap['info']): if daemon_info['state'] != "up:standby-replay": continue inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn") activity = "Evts: " + mgr_util.format_dimless( self.get_rate("mds", daemon_info['name'], "mds_log.replayed"), 5 ) + "/s" metadata = self.get_metadata('mds', daemon_info['name']) mds_versions[metadata.get('ceph_version', "unknown")].append(daemon_info['name']) rank_table.add_row([ "{0}-s".format(daemon_info['rank']), "standby-replay", daemon_info['name'], activity, mgr_util.format_dimless(dns, 5), mgr_util.format_dimless(inos, 5) ]) df = self.get("df") pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) osdmap = self.get("osd_map") pools = dict([(p['pool'], p) for p in osdmap['pools']]) metadata_pool_id = mdsmap['metadata_pool'] data_pool_ids = mdsmap['data_pools'] pools_table = PrettyTable(["Pool", "type", "used", "avail"]) for pool_id in [metadata_pool_id] + data_pool_ids: pool_type = "metadata" if pool_id == metadata_pool_id else "data" stats = pool_stats[pool_id] pools_table.add_row([ pools[pool_id]['pool_name'], pool_type, mgr_util.format_bytes(stats['bytes_used'], 5), mgr_util.format_bytes(stats['max_avail'], 5) ]) output += "{0} - {1} clients\n".format( mdsmap['fs_name'], client_count) output += "=" * len(mdsmap['fs_name']) + "\n" output += rank_table.get_string() output += "\n" + pools_table.get_string() + "\n" if not output and fs_filter is not None: return errno.EINVAL, "", "Invalid filesystem: " + fs_filter standby_table = PrettyTable(["Standby MDS"]) for standby in fsmap['standbys']: metadata = self.get_metadata('mds', standby['name']) mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name']) standby_table.add_row([standby['name']]) output += "\n" + standby_table.get_string() + "\n" if len(mds_versions) == 1: output += "MDS version: {0}".format(list(mds_versions)[0]) else: version_table = PrettyTable(["version", "daemons"]) for version, daemons in six.iteritems(mds_versions): version_table.add_row([ version, ", ".join(daemons) ]) output += version_table.get_string() + "\n" return 0, output, ""
def handle_fs_status(self, cmd): output = "" fs_filter = cmd.get('fs', None) mds_versions = defaultdict(list) fsmap = self.get("fs_map") for filesystem in fsmap['filesystems']: if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter: continue rank_table = PrettyTable( ("Rank", "State", "MDS", "Activity", "dns", "inos"), hrules=prettytable.FRAME) mdsmap = filesystem['mdsmap'] client_count = 0 for rank in mdsmap["in"]: up = "mds_{0}".format(rank) in mdsmap["up"] if up: gid = mdsmap['up']["mds_{0}".format(rank)] info = mdsmap['info']['gid_{0}'.format(gid)] dns = self.get_latest("mds", info['name'], "mds_mem.dn") inos = self.get_latest("mds", info['name'], "mds_mem.ino") if rank == 0: client_count = self.get_latest( "mds", info['name'], "mds_sessions.session_count") elif client_count == 0: # In case rank 0 was down, look at another rank's # sessionmap to get an indication of clients. client_count = self.get_latest( "mds", info['name'], "mds_sessions.session_count") laggy = "laggy_since" in info state = info['state'].split(":")[1] if laggy: state += "(laggy)" if state == "active" and not laggy: c_state = mgr_util.colorize(state, mgr_util.GREEN) else: c_state = mgr_util.colorize(state, mgr_util.YELLOW) # Populate based on context of state, e.g. client # ops for an active daemon, replay progress, reconnect # progress activity = "" if state == "active": activity = "Reqs: " + mgr_util.format_dimless( self.get_rate("mds", info['name'], "mds_server.handle_client_request"), 5) + "/s" defaults = defaultdict(lambda: None, {'version': 'unknown'}) metadata = self.get_metadata('mds', info['name'], default=defaults) mds_versions[metadata['ceph_version']].append(info['name']) rank_table.add_row([ mgr_util.bold(rank.__str__()), c_state, info['name'], activity, mgr_util.format_dimless(dns, 5), mgr_util.format_dimless(inos, 5) ]) else: rank_table.add_row([rank, "failed", "", "", "", ""]) # Find the standby replays for gid_str, daemon_info in six.iteritems(mdsmap['info']): if daemon_info['state'] != "up:standby-replay": continue inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn") activity = "Evts: " + mgr_util.format_dimless( self.get_rate("mds", daemon_info['name'], "mds_log.replayed"), 5) + "/s" defaults = defaultdict(lambda: None, {'version': 'unknown'}) metadata = self.get_metadata('mds', daemon_info['name'], default=defaults) mds_versions[metadata['ceph_version']].append( daemon_info['name']) rank_table.add_row([ "{0}-s".format(daemon_info['rank']), "standby-replay", daemon_info['name'], activity, mgr_util.format_dimless(dns, 5), mgr_util.format_dimless(inos, 5) ]) df = self.get("df") pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) osdmap = self.get("osd_map") pools = dict([(p['pool'], p) for p in osdmap['pools']]) metadata_pool_id = mdsmap['metadata_pool'] data_pool_ids = mdsmap['data_pools'] pools_table = PrettyTable(["Pool", "type", "used", "avail"]) for pool_id in [metadata_pool_id] + data_pool_ids: pool_type = "metadata" if pool_id == metadata_pool_id else "data" stats = pool_stats[pool_id] pools_table.add_row([ pools[pool_id]['pool_name'], pool_type, mgr_util.format_bytes(stats['bytes_used'], 5), mgr_util.format_bytes(stats['max_avail'], 5) ]) output += "{0} - {1} clients\n".format(mdsmap['fs_name'], client_count) output += "=" * len(mdsmap['fs_name']) + "\n" output += rank_table.get_string() output += "\n" + pools_table.get_string() + "\n" if not output and fs_filter is not None: return errno.EINVAL, "", "Invalid filesystem: " + fs_filter standby_table = PrettyTable(["Standby MDS"]) for standby in fsmap['standbys']: defaults = defaultdict(lambda: None, {'version': 'unknown'}) metadata = self.get_metadata('mds', standby['name'], default=defaults) mds_versions[metadata['ceph_version']].append(standby['name']) standby_table.add_row([standby['name']]) output += "\n" + standby_table.get_string() + "\n" if len(mds_versions) == 1: output += "MDS version: {0}".format(list(mds_versions)[0]) else: version_table = PrettyTable(["version", "daemons"]) for version, daemons in six.iteritems(mds_versions): version_table.add_row([version, ", ".join(daemons)]) output += version_table.get_string() + "\n" return 0, output, ""