def load_daemon():
key = "daemon"
state = colorize("running", color.GREEN)
line = [
" "+colorize(key, color.BOLD),
"%s" % state,
"",
str(data.get("pid", "")) if stats_data else "",
"",
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
fmt_thr_mem_total(key, stats_data),
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
speaker = data["monitor"].get("nodes", {}).get(nodename, {}).get("speaker")
if speaker:
status = "up"
status = colorize_status(status, lpad=0).replace(status, unicons[status])
else:
status = ""
line.append(status)
out.append(line)
def load_collector(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
line = [
" "+colorize(key, color.BOLD),
state,
"",
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
speaker = data["monitor"].get("nodes", {}).get(nodename, {}).get("speaker")
if speaker:
status = "up"
status = colorize_status(status, lpad=0).replace(status, unicons[status])
else:
status = ""
line.append(status)
out.append(line)
def load_node_compat():
if "monitor" not in data:
return
if data["monitor"].get("compat") is True:
# no need to clutter if the situation is normal
return
line = [
colorize(" compat", color.BOLD),
colorize("warn", color.BROWN),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
compat = data["monitor"]["nodes"].get(nodename, {}).get("compat", "")
line.append(str(compat))
out.append(line)
def load_node_state():
if "monitor" not in data:
return
line = [
colorize(" state", color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
nmon_state = data["monitor"]["nodes"].get(nodename, {}).get("monitor", {}).get("status", "")
if nmon_state == "idle":
nmon_state = ""
if data["monitor"]["nodes"].get(nodename, {}).get("frozen", ""):
frozen = frozen_icon = colorize(unicons["frozen"], color.BLUE)
else:
frozen = ""
global_expect = data["monitor"]["nodes"].get(nodename, {}).get("monitor", {}).get("global_expect")
if global_expect:
global_expect = colorize(" >" + str(global_expect), color.LIGHTBLUE)
else:
global_expect = ""
line.append(str(nmon_state)+frozen+global_expect)
out.append(line)
def load_node_version():
if "monitor" not in data:
return
line = [
colorize(" version", color.BOLD),
colorize("warn", color.BROWN),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
versions = []
for nodename in nodenames:
agent = data["monitor"]["nodes"].get(nodename, {}).get("agent", "")
line.append(str(agent))
if agent != "":
versions.append(str(agent))
if len(set(versions)) > 1:
out.append(line)
def colorize_status(status, lpad=10, agg_status=None):
"""
Return the colorized human readable status string.
"""
if isinstance(status, Status):
status = str(status)
elif isinstance(status, int):
status = str(Status(status))
fmt = "%-" + str(lpad) + "s"
if status is None:
return colorize(fmt % "undef", color.LIGHTBLUE)
elif status == "warn":
return colorize(fmt % status, color.BROWN)
elif status == "down" or status in ("err", "error"):
if agg_status == "up":
return colorize(fmt % status, color.LIGHTBLUE)
else:
return colorize(fmt % status, color.RED)
elif status == "up" or status == "ok":
return colorize(fmt % status, color.GREEN)
elif status == "stdby up":
if agg_status == "up":
return colorize(fmt % status, color.LIGHTBLUE)
else:
return colorize(fmt % status, color.RED)
elif status == "stdby down":
return colorize(fmt % status, color.RED)
elif status == "n/a":
return colorize(fmt % status, color.LIGHTBLUE)
else:
return colorize(fmt % status, color.LIGHTBLUE)
def load_free_total(key):
if "monitor" not in data:
return
line = [
colorize(" "+key, color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
total = data["monitor"]["nodes"].get(nodename, {}).get("stats", {}).get(key+"_total")
avail = data["monitor"]["nodes"].get(nodename, {}).get("stats", {}).get(key+"_avail")
limit = 100 - data["monitor"]["nodes"].get(nodename, {}).get("min_avail_"+key, 0)
if avail is None or total in (0, None):
line.append(colorize("-", color.LIGHTBLUE))
continue
usage = 100 - avail
total = print_size(total, unit="MB", compact=True)
if limit:
cell = "%d/%d%%:%s" % (usage, limit, total)
else:
cell = "%d%%:%s" % (usage, total)
if usage > limit:
cell = colorize(cell, color.RED)
line.append(cell)
out.append(line)
def get_svc_notice(data):
svc_notice = []
if "cluster" in data:
overall = data["cluster"].get("overall", "n/a")
if overall == "warn":
svc_notice.append(colorize(overall, STATUS_COLOR[overall]))
placement = data["cluster"].get("placement", "n/a")
if placement not in ("optimal", "n/a"):
svc_notice.append(colorize(placement + " placement", color.RED))
compat = data["cluster"].get("compat", True)
if not compat:
svc_notice.append(colorize("incompatible versions", color.RED))
return ", ".join(svc_notice)
def load_hb(key, _data):
state = _data.get("state", "")
if state == "running":
state = colorize(state, color.GREEN)
else:
state = colorize(state, color.RED)
if _data.get("alerts"):
state += colorize("!", color.BROWN)
cf = _data.get("config", {})
addr = cf.get("addr", "")
port = cf.get("port", "")
dev = cf.get("dev", "")
relay = cf.get("relay", "")
if addr and port:
config = addr + ":" + str(port)
elif dev:
config = os.path.basename(dev)
elif relay:
config = relay
else:
config = ""
line = [
" " + colorize(key, color.BOLD),
state,
config,
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
peers = _data.get("peers", {})
for nodename in nodenames:
beating = peers.get(nodename, {}).get("beating")
if beating is None:
status = "n/a"
elif beating:
status = "up"
else:
status = "down"
status = colorize_status(status,
lpad=0).replace(status, unicons[status])
line.append(status)
out.append(line)
def load_generic_thread(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
out.append((
" "+colorize(key, color.BOLD),
state,
"",
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
))
def load_listener(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
out.append((
" "+colorize(key, color.BOLD),
state,
_data["config"]["addr"]+":"+str(_data["config"]["port"]),
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
))
def format(self, record):
record.message = record.getMessage()
record.context = ""
for xattr, key in self.attrs:
if xattr == "sid" and not self.sid:
continue
try:
val = getattr(record, xattr)
if val in (None, ""):
continue
if val is True:
val = "y"
elif val is False:
val = "n"
record.context += "%s:%s " % (key, val)
except AttributeError:
pass
record.context = record.context.rstrip()
for pattern in core.extconfig.SECRETS:
record.message = record.message.replace(pattern, "xxxx")
if not self.human:
return logging.Formatter.format(self, record)
# Factorize context information, trim the level and colorize.
buff = ""
if self.last_context != record.context:
buff += colorize("@ " + record.context, color.LIGHTBLUE) + "\n"
self.last_context = record.context
if record.levelname == "INFO":
buff += " " + record.message
elif record.levelname == "ERROR":
buff += colorize("E " + record.message, color.RED)
elif record.levelname == "WARNING":
buff += colorize("W " + record.message, color.BROWN)
elif record.levelname == "DEBUG":
buff += colorize("D " + record.message, color.LIGHTBLUE)
return buff
def format_cell(text, width, textcolor, separator, align):
"""
Format the table cell, happending the separator, coloring the text and
applying the cell padding for alignment.
"""
if text in ("", None):
return " " * width + separator
if align == "right":
fmt = "%" + str(width) + "s"
else:
fmt = "%-" + str(width) + "s"
cell = fmt % text
if textcolor:
cell = colorize(cell, textcolor)
return cell + separator
def load_monitor(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
transitions = _data.get("transitions", 0)
if transitions:
status = "%d transition" % transitions
else:
status = ""
out.append((
" "+colorize(key, color.BOLD),
state,
status,
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
))
def instance_notice(overall=None,
frozen=None,
node_frozen=None,
constraints=None,
provisioned=None,
monitor=None):
notice = []
if overall == "warn":
notice.append(colorize(overall, STATUS_COLOR[overall]))
if frozen:
notice.append(colorize("frozen", color.BLUE))
if node_frozen:
notice.append(colorize("node frozen", color.BLUE))
if not constraints:
notice.append("constraints violation")
if provisioned is False:
notice.append(colorize("not provisioned", color.RED))
if monitor == {}:
# encap monitor
pass
elif monitor:
mon_status = monitor.get("status", "unknown")
if monitor["status"] == "idle":
notice.append(colorize(mon_status, color.LIGHTBLUE))
else:
notice.append(colorize(mon_status, color.RED))
if monitor.get("local_expect") not in ("", None):
notice.append(
colorize(monitor.get("local_expect", ""), color.LIGHTBLUE))
if monitor.get("global_expect") not in ("", None):
notice.append(
colorize(">" + monitor.get("global_expect", ""),
color.LIGHTBLUE))
else:
notice.append(colorize("daemon down", color.RED))
return ", ".join(notice)
def load_loadavg():
if "monitor" not in data:
return
line = [
colorize(" load 15m", color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
line.append(str(data["monitor"]["nodes"].get(nodename, {}).get("stats", {}).get("load_15m", "")))
out.append(line)
def load_header(title=""):
if isinstance(title, list):
line = title
else:
line = [
title,
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
]
for nodename in show_nodenames:
line.append(colorize(nodename, color.BOLD))
out.append(line)
def load_arbitrators():
if "arbitrators" not in sections:
return
arbitrators = []
arbitrators_name = {}
for nodename, ndata in data["monitor"]["nodes"].items():
for aid, adata in ndata.get("arbitrators", {}).items():
if aid not in arbitrators:
arbitrators.append(aid)
arbitrators_name[aid] = adata["name"]
if len(arbitrators) == 0:
return
load_header("Arbitrators")
for aid in arbitrators:
line = [
colorize(" " + arbitrators_name[aid], color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
status = data["monitor"]["nodes"].get(nodename, {}).get(
"arbitrators", {}).get(aid, {}).get("status", "undef")
if status != "up":
line[1] = colorize_status("warn", lpad=0)
status = colorize_status(status, lpad=0).replace(
status, unicons[status])
line.append(status)
out.append(line)
out.append([])
def load_svc(path, prefix=""):
if path not in services:
return
data = services[path]
if path in slave_parents and prefix == "":
return
try:
topology = services[path].topology
except KeyError:
topology = ""
if services[path].get("drp", False):
topology = "drp " + topology
# status
status = colorize_status(data["avail"], lpad=0)
if data["overall"] == "warn":
status += colorize("!", color.BROWN)
if data["placement"] == "non-optimal":
status += colorize("^", color.RED)
# info
info = {
"topology": data.get("topology", ""),
"orchestrate": data.get("orchestrate", "-"),
"status": "%d/1" % data["n_up"] if data["n_up"] is not None else 0,
}
if data.get("scale") is not None:
info["status"] = "%d/%d" % (data["n_up"], data.get("scale"))
elif data.get("wrapper"):
info = {
"topology": "",
"orchestrate": "",
"status": "",
}
elif topology == "flex":
info["status"] = "%d/%d" % (data["n_up"], data["flex_target"])
if data["avail"] == "n/a":
info["status"] = ""
info = "%(orchestrate)-5s %(status)-5s" % info
line = [
" "+colorize(prefix+strip_path(path, os.environ.get("OSVC_NAMESPACE")), color.BOLD),
status,
info,
fmt_svc_uptime(path, stats_data),
fmt_svc_tasks(path, prev_stats_data),
fmt_svc_cpu_usage(path, prev_stats_data, stats_data),
fmt_svc_cpu_time(path, stats_data),
fmt_svc_mem_total(path, stats_data),
fmt_svc_blk_rb(path, stats_data),
fmt_svc_blk_wb(path, stats_data),
fmt_svc_blk_rbps(path, prev_stats_data, stats_data),
fmt_svc_blk_wbps(path, prev_stats_data, stats_data),
"|" if nodenames else "",
]
if not nodenames:
states = []
if data["frozen"] == "frozen":
frozen = "frozen"
elif data["frozen"] == "thawed":
frozen = ""
elif data["frozen"] == "n/a":
frozen = ""
elif data["frozen"] == "mixed":
frozen = "part-frozen"
else:
frozen = ""
if frozen:
states.append(frozen)
if data["provisioned"] is True:
provisioned = ""
elif data["provisioned"] is False:
provisioned = "unprovisioned"
elif data["provisioned"] == "n/a":
provisioned = ""
elif data["provisioned"] == "mixed":
provisioned = "part-provisioned"
else:
provisioned = ""
if provisioned:
states.append(provisioned)
mon_status_counts = {}
ge = None
for nodename in avail_nodenames:
try:
_data = data["nodes"][nodename]
except KeyError:
continue
if _data is None:
continue
ge = _data.get("global_expect")
st = _data.get("mon")
if st not in ("idle", None):
if st not in mon_status_counts:
mon_status_counts[st] = 1
else:
mon_status_counts[st] += 1
if ge:
states.append(">"+ge)
for s, n in mon_status_counts.items():
states.append("%s(%d)" % (s, n))
line.append(", ".join(states))
for nodename in nodenames:
if nodename not in data["nodes"]:
line.append("")
elif data["nodes"][nodename] is None:
line.append(colorize("?", color.RED))
elif data["nodes"][nodename] is not None:
val = []
# frozen unicon
if data["nodes"][nodename]["frozen"]:
frozen_icon = colorize(unicons["frozen"], color.BLUE)
else:
frozen_icon = ""
# avail status unicon
if data["wrapper"]:
avail_icon = ""
else:
avail = data["nodes"][nodename]["avail"]
if avail == "unknown":
avail_icon = colorize("?", color.RED)
else:
avail_icon = colorize_status(avail, lpad=0, agg_status=data["avail"]).replace(avail, unicons[avail])
if data["nodes"][nodename].get("preserved"):
avail_icon += colorize("?", color.LIGHTBLUE)
# overall status unicon
if data["wrapper"]:
overall_icon = ""
else:
overall = data["nodes"][nodename]["overall"]
if overall == "warn":
overall_icon = colorize_status(overall, lpad=0).replace(overall, unicons[overall])
else:
overall_icon = ""
# mon status
smon = data["nodes"][nodename]["mon"]
if smon == "idle":
# don't display 'idle', as its to normal status and thus repeated as nauseam
smon = ""
else:
smon = " " + smon
# global expect
if smon == "":
global_expect = data["nodes"][nodename]["global_expect"]
if global_expect:
global_expect = colorize(" >" + str(global_expect), color.LIGHTBLUE)
else:
global_expect = ""
else:
global_expect = ""
# leader
if data["wrapper"]:
leader = ""
else:
if data["nodes"][nodename]["placement"] == "leader":
leader = colorize("^", color.LIGHTBLUE)
else:
leader = ""
# provisioned
if data["nodes"][nodename].get("provisioned") is False:
provisioned = colorize("P", color.RED)
else:
provisioned = ""
val.append(avail_icon)
val.append(overall_icon)
val.append(leader)
val.append(frozen_icon)
val.append(provisioned)
val.append(smon)
val.append(global_expect)
line.append("".join(val))
out.append(line)
for child in sorted(list(data.get("slaves", []))):
load_svc(child, prefix=prefix+" ")
