def load_daemon(): key = "daemon" state = colorize("running", color.GREEN) line = [ " "+colorize(key, color.BOLD), "%s" % state, "", str(data.get("pid", "")) if stats_data else "", "", fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), fmt_thr_mem_total(key, stats_data), "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: speaker = data["monitor"].get("nodes", {}).get(nodename, {}).get("speaker") if speaker: status = "up" status = colorize_status(status, lpad=0).replace(status, unicons[status]) else: status = "" line.append(status) out.append(line)
def load_collector(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) line = [ " "+colorize(key, color.BOLD), state, "", fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: speaker = data["monitor"].get("nodes", {}).get(nodename, {}).get("speaker") if speaker: status = "up" status = colorize_status(status, lpad=0).replace(status, unicons[status]) else: status = "" line.append(status) out.append(line)
def load_node_compat(): if "monitor" not in data: return if data["monitor"].get("compat") is True: # no need to clutter if the situation is normal return line = [ colorize(" compat", color.BOLD), colorize("warn", color.BROWN), "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: compat = data["monitor"]["nodes"].get(nodename, {}).get("compat", "") line.append(str(compat)) out.append(line)
def load_node_state(): if "monitor" not in data: return line = [ colorize(" state", color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: nmon_state = data["monitor"]["nodes"].get(nodename, {}).get("monitor", {}).get("status", "") if nmon_state == "idle": nmon_state = "" if data["monitor"]["nodes"].get(nodename, {}).get("frozen", ""): frozen = frozen_icon = colorize(unicons["frozen"], color.BLUE) else: frozen = "" global_expect = data["monitor"]["nodes"].get(nodename, {}).get("monitor", {}).get("global_expect") if global_expect: global_expect = colorize(" >" + str(global_expect), color.LIGHTBLUE) else: global_expect = "" line.append(str(nmon_state)+frozen+global_expect) out.append(line)
def load_node_version(): if "monitor" not in data: return line = [ colorize(" version", color.BOLD), colorize("warn", color.BROWN), "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] versions = [] for nodename in nodenames: agent = data["monitor"]["nodes"].get(nodename, {}).get("agent", "") line.append(str(agent)) if agent != "": versions.append(str(agent)) if len(set(versions)) > 1: out.append(line)
def colorize_status(status, lpad=10, agg_status=None): """ Return the colorized human readable status string. """ if isinstance(status, Status): status = str(status) elif isinstance(status, int): status = str(Status(status)) fmt = "%-" + str(lpad) + "s" if status is None: return colorize(fmt % "undef", color.LIGHTBLUE) elif status == "warn": return colorize(fmt % status, color.BROWN) elif status == "down" or status in ("err", "error"): if agg_status == "up": return colorize(fmt % status, color.LIGHTBLUE) else: return colorize(fmt % status, color.RED) elif status == "up" or status == "ok": return colorize(fmt % status, color.GREEN) elif status == "stdby up": if agg_status == "up": return colorize(fmt % status, color.LIGHTBLUE) else: return colorize(fmt % status, color.RED) elif status == "stdby down": return colorize(fmt % status, color.RED) elif status == "n/a": return colorize(fmt % status, color.LIGHTBLUE) else: return colorize(fmt % status, color.LIGHTBLUE)
def load_free_total(key): if "monitor" not in data: return line = [ colorize(" "+key, color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: total = data["monitor"]["nodes"].get(nodename, {}).get("stats", {}).get(key+"_total") avail = data["monitor"]["nodes"].get(nodename, {}).get("stats", {}).get(key+"_avail") limit = 100 - data["monitor"]["nodes"].get(nodename, {}).get("min_avail_"+key, 0) if avail is None or total in (0, None): line.append(colorize("-", color.LIGHTBLUE)) continue usage = 100 - avail total = print_size(total, unit="MB", compact=True) if limit: cell = "%d/%d%%:%s" % (usage, limit, total) else: cell = "%d%%:%s" % (usage, total) if usage > limit: cell = colorize(cell, color.RED) line.append(cell) out.append(line)
def get_svc_notice(data): svc_notice = [] if "cluster" in data: overall = data["cluster"].get("overall", "n/a") if overall == "warn": svc_notice.append(colorize(overall, STATUS_COLOR[overall])) placement = data["cluster"].get("placement", "n/a") if placement not in ("optimal", "n/a"): svc_notice.append(colorize(placement + " placement", color.RED)) compat = data["cluster"].get("compat", True) if not compat: svc_notice.append(colorize("incompatible versions", color.RED)) return ", ".join(svc_notice)
def load_hb(key, _data): state = _data.get("state", "") if state == "running": state = colorize(state, color.GREEN) else: state = colorize(state, color.RED) if _data.get("alerts"): state += colorize("!", color.BROWN) cf = _data.get("config", {}) addr = cf.get("addr", "") port = cf.get("port", "") dev = cf.get("dev", "") relay = cf.get("relay", "") if addr and port: config = addr + ":" + str(port) elif dev: config = os.path.basename(dev) elif relay: config = relay else: config = "" line = [ " " + colorize(key, color.BOLD), state, config, fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", "|" if nodenames else "", ] peers = _data.get("peers", {}) for nodename in nodenames: beating = peers.get(nodename, {}).get("beating") if beating is None: status = "n/a" elif beating: status = "up" else: status = "down" status = colorize_status(status, lpad=0).replace(status, unicons[status]) line.append(status) out.append(line)
def load_generic_thread(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) out.append(( " "+colorize(key, color.BOLD), state, "", fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", ))
def load_listener(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) out.append(( " "+colorize(key, color.BOLD), state, _data["config"]["addr"]+":"+str(_data["config"]["port"]), fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", ))
def format(self, record): record.message = record.getMessage() record.context = "" for xattr, key in self.attrs: if xattr == "sid" and not self.sid: continue try: val = getattr(record, xattr) if val in (None, ""): continue if val is True: val = "y" elif val is False: val = "n" record.context += "%s:%s " % (key, val) except AttributeError: pass record.context = record.context.rstrip() for pattern in core.extconfig.SECRETS: record.message = record.message.replace(pattern, "xxxx") if not self.human: return logging.Formatter.format(self, record) # Factorize context information, trim the level and colorize. buff = "" if self.last_context != record.context: buff += colorize("@ " + record.context, color.LIGHTBLUE) + "\n" self.last_context = record.context if record.levelname == "INFO": buff += " " + record.message elif record.levelname == "ERROR": buff += colorize("E " + record.message, color.RED) elif record.levelname == "WARNING": buff += colorize("W " + record.message, color.BROWN) elif record.levelname == "DEBUG": buff += colorize("D " + record.message, color.LIGHTBLUE) return buff
def format_cell(text, width, textcolor, separator, align): """ Format the table cell, happending the separator, coloring the text and applying the cell padding for alignment. """ if text in ("", None): return " " * width + separator if align == "right": fmt = "%" + str(width) + "s" else: fmt = "%-" + str(width) + "s" cell = fmt % text if textcolor: cell = colorize(cell, textcolor) return cell + separator
def load_monitor(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) transitions = _data.get("transitions", 0) if transitions: status = "%d transition" % transitions else: status = "" out.append(( " "+colorize(key, color.BOLD), state, status, fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", ))
def instance_notice(overall=None, frozen=None, node_frozen=None, constraints=None, provisioned=None, monitor=None): notice = [] if overall == "warn": notice.append(colorize(overall, STATUS_COLOR[overall])) if frozen: notice.append(colorize("frozen", color.BLUE)) if node_frozen: notice.append(colorize("node frozen", color.BLUE)) if not constraints: notice.append("constraints violation") if provisioned is False: notice.append(colorize("not provisioned", color.RED)) if monitor == {}: # encap monitor pass elif monitor: mon_status = monitor.get("status", "unknown") if monitor["status"] == "idle": notice.append(colorize(mon_status, color.LIGHTBLUE)) else: notice.append(colorize(mon_status, color.RED)) if monitor.get("local_expect") not in ("", None): notice.append( colorize(monitor.get("local_expect", ""), color.LIGHTBLUE)) if monitor.get("global_expect") not in ("", None): notice.append( colorize(">" + monitor.get("global_expect", ""), color.LIGHTBLUE)) else: notice.append(colorize("daemon down", color.RED)) return ", ".join(notice)
def load_loadavg(): if "monitor" not in data: return line = [ colorize(" load 15m", color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: line.append(str(data["monitor"]["nodes"].get(nodename, {}).get("stats", {}).get("load_15m", ""))) out.append(line)
def load_header(title=""): if isinstance(title, list): line = title else: line = [ title, "", "", "", "", "", "", "", "", "", "", "", "", ] for nodename in show_nodenames: line.append(colorize(nodename, color.BOLD)) out.append(line)
def load_arbitrators(): if "arbitrators" not in sections: return arbitrators = [] arbitrators_name = {} for nodename, ndata in data["monitor"]["nodes"].items(): for aid, adata in ndata.get("arbitrators", {}).items(): if aid not in arbitrators: arbitrators.append(aid) arbitrators_name[aid] = adata["name"] if len(arbitrators) == 0: return load_header("Arbitrators") for aid in arbitrators: line = [ colorize(" " + arbitrators_name[aid], color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: status = data["monitor"]["nodes"].get(nodename, {}).get( "arbitrators", {}).get(aid, {}).get("status", "undef") if status != "up": line[1] = colorize_status("warn", lpad=0) status = colorize_status(status, lpad=0).replace( status, unicons[status]) line.append(status) out.append(line) out.append([])
def load_svc(path, prefix=""): if path not in services: return data = services[path] if path in slave_parents and prefix == "": return try: topology = services[path].topology except KeyError: topology = "" if services[path].get("drp", False): topology = "drp " + topology # status status = colorize_status(data["avail"], lpad=0) if data["overall"] == "warn": status += colorize("!", color.BROWN) if data["placement"] == "non-optimal": status += colorize("^", color.RED) # info info = { "topology": data.get("topology", ""), "orchestrate": data.get("orchestrate", "-"), "status": "%d/1" % data["n_up"] if data["n_up"] is not None else 0, } if data.get("scale") is not None: info["status"] = "%d/%d" % (data["n_up"], data.get("scale")) elif data.get("wrapper"): info = { "topology": "", "orchestrate": "", "status": "", } elif topology == "flex": info["status"] = "%d/%d" % (data["n_up"], data["flex_target"]) if data["avail"] == "n/a": info["status"] = "" info = "%(orchestrate)-5s %(status)-5s" % info line = [ " "+colorize(prefix+strip_path(path, os.environ.get("OSVC_NAMESPACE")), color.BOLD), status, info, fmt_svc_uptime(path, stats_data), fmt_svc_tasks(path, prev_stats_data), fmt_svc_cpu_usage(path, prev_stats_data, stats_data), fmt_svc_cpu_time(path, stats_data), fmt_svc_mem_total(path, stats_data), fmt_svc_blk_rb(path, stats_data), fmt_svc_blk_wb(path, stats_data), fmt_svc_blk_rbps(path, prev_stats_data, stats_data), fmt_svc_blk_wbps(path, prev_stats_data, stats_data), "|" if nodenames else "", ] if not nodenames: states = [] if data["frozen"] == "frozen": frozen = "frozen" elif data["frozen"] == "thawed": frozen = "" elif data["frozen"] == "n/a": frozen = "" elif data["frozen"] == "mixed": frozen = "part-frozen" else: frozen = "" if frozen: states.append(frozen) if data["provisioned"] is True: provisioned = "" elif data["provisioned"] is False: provisioned = "unprovisioned" elif data["provisioned"] == "n/a": provisioned = "" elif data["provisioned"] == "mixed": provisioned = "part-provisioned" else: provisioned = "" if provisioned: states.append(provisioned) mon_status_counts = {} ge = None for nodename in avail_nodenames: try: _data = data["nodes"][nodename] except KeyError: continue if _data is None: continue ge = _data.get("global_expect") st = _data.get("mon") if st not in ("idle", None): if st not in mon_status_counts: mon_status_counts[st] = 1 else: mon_status_counts[st] += 1 if ge: states.append(">"+ge) for s, n in mon_status_counts.items(): states.append("%s(%d)" % (s, n)) line.append(", ".join(states)) for nodename in nodenames: if nodename not in data["nodes"]: line.append("") elif data["nodes"][nodename] is None: line.append(colorize("?", color.RED)) elif data["nodes"][nodename] is not None: val = [] # frozen unicon if data["nodes"][nodename]["frozen"]: frozen_icon = colorize(unicons["frozen"], color.BLUE) else: frozen_icon = "" # avail status unicon if data["wrapper"]: avail_icon = "" else: avail = data["nodes"][nodename]["avail"] if avail == "unknown": avail_icon = colorize("?", color.RED) else: avail_icon = colorize_status(avail, lpad=0, agg_status=data["avail"]).replace(avail, unicons[avail]) if data["nodes"][nodename].get("preserved"): avail_icon += colorize("?", color.LIGHTBLUE) # overall status unicon if data["wrapper"]: overall_icon = "" else: overall = data["nodes"][nodename]["overall"] if overall == "warn": overall_icon = colorize_status(overall, lpad=0).replace(overall, unicons[overall]) else: overall_icon = "" # mon status smon = data["nodes"][nodename]["mon"] if smon == "idle": # don't display 'idle', as its to normal status and thus repeated as nauseam smon = "" else: smon = " " + smon # global expect if smon == "": global_expect = data["nodes"][nodename]["global_expect"] if global_expect: global_expect = colorize(" >" + str(global_expect), color.LIGHTBLUE) else: global_expect = "" else: global_expect = "" # leader if data["wrapper"]: leader = "" else: if data["nodes"][nodename]["placement"] == "leader": leader = colorize("^", color.LIGHTBLUE) else: leader = "" # provisioned if data["nodes"][nodename].get("provisioned") is False: provisioned = colorize("P", color.RED) else: provisioned = "" val.append(avail_icon) val.append(overall_icon) val.append(leader) val.append(frozen_icon) val.append(provisioned) val.append(smon) val.append(global_expect) line.append("".join(val)) out.append(line) for child in sorted(list(data.get("slaves", []))): load_svc(child, prefix=prefix+" ")