def load_collector(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
line = [
" " + colorize(key, color.BOLD),
state,
"",
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
speaker = data["monitor"].get("nodes", {}).get(nodename,
{}).get("speaker")
if speaker:
status = "up"
status = colorize_status(status, lpad=0).replace(
status, unicons[status])
else:
status = ""
line.append(status)
out.append(line)
def load_node_compat():
if "monitor" not in data:
return
if data["monitor"].get("compat") is True:
# no need to clutter if the situation is normal
return
line = [
colorize(" compat", color.BOLD),
colorize("warn", color.BROWN),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
compat = data["monitor"]["nodes"].get(nodename,
{}).get("compat", "")
line.append(str(compat))
out.append(line)
def load_node_version():
if "monitor" not in data:
return
line = [
colorize(" version", color.BOLD),
colorize("warn", color.BROWN),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
versions = []
for nodename in nodenames:
agent = data["monitor"]["nodes"].get(nodename, {}).get("agent", "")
line.append(str(agent))
if agent != "":
versions.append(str(agent))
if len(set(versions)) > 1:
out.append(line)
def print_table_default(data):
try:
data = validate_format(data)
except Exception as e:
return
labels = data[0]
max_label_len = reduce(lambda x,y: max(x,len(y)), labels, 0)+1
data = data[1:]
subsequent_indent = ""
for i in range(max_label_len+3):
subsequent_indent += " "
fmt = " %-"+str(max_label_len)+"s "
for j, d in enumerate(data):
print("-")
for i, label in enumerate(labels):
val = '\n'.join(wrap(convert(d[i]),
initial_indent = "",
subsequent_indent = subsequent_indent,
width=78
))
try:
print(colorize(fmt % (label+":"), color.LIGHTBLUE), val)
except UnicodeEncodeError:
print(colorize(fmt % (label+":"), color.LIGHTBLUE), val.encode("utf-8"))
def load_daemon():
key = "daemon"
state = colorize("running", color.GREEN)
line = [
" " + colorize(key, color.BOLD),
"%s" % state,
"",
str(data.get("pid", "")) if stats_data else "",
"",
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
fmt_thr_mem_total(key, stats_data),
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
speaker = data["monitor"].get("nodes", {}).get(nodename,
{}).get("speaker")
if speaker:
status = "up"
status = colorize_status(status, lpad=0).replace(
status, unicons[status])
else:
status = ""
line.append(status)
out.append(line)
def format(self, record):
record.message = record.getMessage()
record.context = ""
for xattr, key in self.attrs:
if xattr == "sid" and not self.sid:
continue
try:
val = getattr(record, xattr)
if val:
record.context += "%s:%s " % (key, getattr(record, xattr))
except AttributeError:
pass
record.context = record.context.rstrip()
for pattern in extconfig.SECRETS:
record.message = record.message.replace(pattern, "xxxx")
if not self.human:
return logging.Formatter.format(self, record)
# Factorize context information, trim the level and colorize.
buff = ""
if self.last_context != record.context:
buff += colorize("@ " + record.context, color.LIGHTBLUE) + "\n"
self.last_context = record.context
if record.levelname == "INFO":
buff += " " + record.message
elif record.levelname == "ERROR":
buff += colorize("E " + record.message, color.RED)
elif record.levelname == "WARNING":
buff += colorize("W " + record.message, color.BROWN)
elif record.levelname == "DEBUG":
buff += colorize("D " + record.message, color.LIGHTBLUE)
return buff
def colorize_status(status, lpad=10, agg_status=None):
"""
Return the colorized human readable status string.
"""
if isinstance(status, Status):
status = str(status)
elif isinstance(status, int):
status = str(Status(status))
fmt = "%-" + str(lpad) + "s"
if status is None:
return colorize(fmt % "undef", color.LIGHTBLUE)
elif status == "warn":
return colorize(fmt % status, color.BROWN)
elif status == "down" or status in ("err", "error"):
if agg_status == "up":
return colorize(fmt % status, color.LIGHTBLUE)
else:
return colorize(fmt % status, color.RED)
elif status == "up" or status == "ok":
return colorize(fmt % status, color.GREEN)
elif status == "stdby up":
if agg_status == "up":
return colorize(fmt % status, color.LIGHTBLUE)
else:
return colorize(fmt % status, color.RED)
elif status == "stdby down":
return colorize(fmt % status, color.RED)
elif status == "n/a":
return colorize(fmt % status, color.LIGHTBLUE)
else:
return colorize(fmt % status, color.LIGHTBLUE)
def instance_notice(overall=None, frozen=None, node_frozen=None, constraints=None,
provisioned=None, monitor=None):
notice = []
if overall == "warn":
notice.append(colorize(overall, STATUS_COLOR[overall]))
if frozen:
notice.append(colorize("frozen", color.BLUE))
if node_frozen:
notice.append(colorize("node frozen", color.BLUE))
if not constraints:
notice.append("constraints violation")
if provisioned is False:
notice.append(colorize("not provisioned", color.RED))
if monitor == {}:
# encap monitor
pass
elif monitor:
mon_status = monitor.get("status", "unknown")
if monitor["status"] == "idle":
notice.append(colorize(mon_status, color.LIGHTBLUE))
else:
notice.append(colorize(mon_status, color.RED))
if monitor.get("local_expect") not in ("", None):
notice.append(colorize(monitor.get("local_expect", ""), color.LIGHTBLUE))
if monitor.get("global_expect") not in ("", None):
notice.append(colorize(">"+monitor.get("global_expect", ""), color.LIGHTBLUE))
else:
notice.append(colorize("daemon down", color.RED))
return ", ".join(notice)
def print_rcode(self, r):
buff = "STATUS: "
if r == 1:
buff += colorize("nok", color.RED)
elif r == 0:
buff += colorize("ok", color.GREEN)
elif r == 2:
buff += "n/a"
else:
buff += "%d" % r
print(buff)
def get_svc_notice(data):
svc_notice = []
if "cluster" in data:
overall = data["cluster"].get("overall", "n/a")
if overall == "warn":
svc_notice.append(colorize(overall, STATUS_COLOR[overall]))
placement = data["cluster"].get("placement", "n/a")
if placement not in ("optimal", "n/a"):
svc_notice.append(colorize(placement + " placement", color.RED))
compat = data["cluster"].get("compat", True)
if not compat:
svc_notice.append(colorize("incompatible versions", color.RED))
return ", ".join(svc_notice)
def load_hb(key, _data):
state = _data.get("state", "")
if state == "running":
state = colorize(state, color.GREEN)
else:
state = colorize(state, color.RED)
if _data.get("alerts"):
state += colorize("!", color.BROWN)
cf = _data.get("config", {})
addr = cf.get("addr", "")
port = cf.get("port", "")
dev = cf.get("dev", "")
relay = cf.get("relay", "")
if addr and port:
config = addr + ":" + str(port)
elif dev:
config = os.path.basename(dev)
elif relay:
config = relay
else:
config = ""
line = [
" " + colorize(key, color.BOLD),
state,
config,
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
peers = _data.get("peers", {})
for nodename in nodenames:
beating = peers.get(nodename, {}).get("beating")
if beating is None:
status = "n/a"
elif beating:
status = "up"
else:
status = "down"
status = colorize_status(status,
lpad=0).replace(status, unicons[status])
line.append(status)
out.append(line)
def highlighter(line):
"""
Colorize interesting parts to help readability
"""
line = line.rstrip("\n")
if re.match(r'\[.+\]', line):
return colorize(line, color.BROWN)
line = re.sub(r"({[\.\w\-_#{}\[\]()\$\+]+})",
colorize(r"\1", color.GREEN), line)
line = re.sub(r"^(\s*\w+\s*)=",
colorize(r"\1", color.LIGHTBLUE) + "=", line)
line = re.sub(
r"^(\s*\w+)(@[\w\.]+\s*)=",
colorize(r"\1", color.LIGHTBLUE) + colorize(r"\2", color.RED) +
"=", line)
return line
def format_action(self, section, action, options=True):
"""
Format an candidate action for the help message.
The action message may or may include the possible options,
dependendin on the value of the options parameter.
"""
fancya = self.prog + " " + action.replace('_', ' ')
if self.colorize:
desc = " " + rcColor.colorize(fancya, rcColor.color.BOLD)
else:
desc = " " + fancya
desc += '\n\n'
if self.async_actions.get(action, {}).get("local"):
preamble = "Asynchronous orchestrated action, unless --local or --node <node> is specified.\n\n"
else:
preamble = ""
wrapper = textwrap.TextWrapper(width=self.width - self.indent,
replace_whitespace=False)
text = preamble + self.actions[section][action]["msg"]
text = text.replace("``", "`")
for phrase in text.splitlines():
for line in wrapper.wrap(phrase):
for _line in line.splitlines():
desc += self.subsequent_indent + _line
desc += '\n'
if options:
desc += self.format_options(section, action)
desc = wipe_rest_markup(desc)
desc += '\n'
return desc
def load_generic_thread(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
out.append((
" " + colorize(key, color.BOLD),
state,
"",
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
))
def load_listener(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
out.append((
" " + colorize(key, color.BOLD),
state,
_data["config"]["addr"] + ":" + str(_data["config"]["port"]),
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
))
def load_free_total(key):
if "monitor" not in data:
return
line = [
colorize(" " + key, color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
total = data["monitor"]["nodes"].get(nodename, {}).get(
"stats", {}).get(key + "_total")
avail = data["monitor"]["nodes"].get(nodename, {}).get(
"stats", {}).get(key + "_avail")
limit = 100 - data["monitor"]["nodes"].get(nodename, {}).get(
"min_avail_" + key, 0)
if avail is None or total in (0, None):
line.append("-")
continue
usage = 100 - avail
total = print_size(total, unit="MB", compact=True)
if limit:
cell = "%d/%d%%:%s" % (usage, limit, total)
else:
cell = "%d%%:%s" % (usage, total)
if usage > limit:
cell = colorize(cell, color.RED)
line.append(cell)
out.append(line)
def load_node_state():
if "monitor" not in data:
return
line = [
colorize(" state", color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
nmon_state = data["monitor"]["nodes"].get(nodename, {}).get(
"monitor", {}).get("status", "")
if nmon_state == "idle":
nmon_state = ""
if data["monitor"]["nodes"].get(nodename, {}).get("frozen", ""):
frozen = frozen_icon = colorize(unicons["frozen"], color.BLUE)
else:
frozen = ""
global_expect = data["monitor"]["nodes"].get(nodename, {}).get(
"monitor", {}).get("global_expect")
if global_expect:
global_expect = colorize(" >" + str(global_expect),
color.LIGHTBLUE)
else:
global_expect = ""
line.append(str(nmon_state) + frozen + global_expect)
out.append(line)
def format_cell(text, width, textcolor, separator, align):
"""
Format the table cell, happending the separator, coloring the text and
applying the cell padding for alignment.
"""
if text in ("", None):
return " " * width + separator
if align == "right":
fmt = "%" + str(width) + "s"
else:
fmt = "%-" + str(width) + "s"
cell = fmt % text
if textcolor:
cell = colorize(cell, textcolor)
return cell + separator
def load_monitor(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
transitions = _data.get("transitions", 0)
if transitions:
status = "%d transition" % transitions
else:
status = ""
out.append((
" " + colorize(key, color.BOLD),
state,
status,
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
))
def status_logs_str(self, color=False):
"""
Returns the formatted resource status log buffer entries.
"""
status_str = ""
for level, text in self.status_logs:
if len(text) == 0:
continue
entry = level + ": " + text + "\n"
if color:
if level == "warn":
color = rcColor.color.BROWN
elif level == "error":
color = rcColor.color.RED
else:
color = rcColor.color.LIGHTBLUE
status_str += rcColor.colorize(entry, color)
else:
status_str += entry
return status_str
def colorize_json(s):
import re
from rcStatus import colorize_status
s = re.sub(
r'(")(error|ok|err|up|down|warn|n/a|stdby up|stdby down)(")', lambda m:
m.group(1) + colorize_status(m.group(2), lpad=0) + m.group(3), s)
s = re.sub(r'((?!"DEFAULT":)("[\w: ,@-]+":))',
colorize(r'\1', color.LIGHTBLUE), s)
s = re.sub(r'("DEFAULT":)( {)', colorize(r'\1', color.BROWN) + r'\2', s)
s = re.sub(r'("[\w:-]+#[\w\.:-]+":)( {)',
colorize(r'\1', color.BROWN) + r'\2', s)
s = re.sub(r'(@[\w-]+)(":)',
colorize(r'\1', color.RED) + colorize(r'\2', color.LIGHTBLUE),
s)
s = re.sub(r'({.+})', colorize(r'\1', color.GREEN), s)
return s
def load_header(title=""):
if isinstance(title, list):
line = title
else:
line = [
title,
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
]
for nodename in show_nodenames:
line.append(colorize(nodename, color.BOLD))
out.append(line)
def load_arbitrators():
if "arbitrators" not in sections:
return
arbitrators = []
arbitrators_name = {}
for nodename, ndata in data["monitor"]["nodes"].items():
for aid, adata in ndata.get("arbitrators", {}).items():
if aid not in arbitrators:
arbitrators.append(aid)
arbitrators_name[aid] = adata["name"]
if len(arbitrators) == 0:
return
load_header("Arbitrators")
for aid in arbitrators:
line = [
colorize(" " + arbitrators_name[aid], color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
status = data["monitor"]["nodes"].get(nodename, {}).get(
"arbitrators", {}).get(aid, {}).get("status", "undef")
if status != "up":
line[1] = colorize_status("warn", lpad=0)
status = colorize_status(status, lpad=0).replace(
status, unicons[status])
line.append(status)
out.append(line)
out.append([])
def print_table_tabulate(data, width=20):
try:
data = validate_format(data)
except Exception as e:
return
from tabulate import tabulate
import rcColor
try:
table = tabulate(data, headers="firstrow", tablefmt="plain").splitlines()
except UnicodeEncodeError:
table = tabulate(data, headers="firstrow", tablefmt="plain").encode("utf-8").splitlines()
colors = [
rcColor.color.BGWHITE+rcColor.color.BLACK,
rcColor.color.E_BGODD+rcColor.color.BLACK,
]
idx = 0
tw = term_width()
for line_idx, line in enumerate(table):
if line.startswith("+-"):
idx = (idx + 1) % 2
continue
if line_idx == 0:
color = rcColor.color.BOLD
else:
color = colors[idx]
if line.endswith("|"):
cont = False
else:
cont = True
length = len(line)
if length > tw or cont:
rpad = tw - (length % tw)
line += " " * rpad
print(rcColor.colorize(line, color))
def load_loadavg():
if "monitor" not in data:
return
line = [
colorize(" load 15m", color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
line.append(
str(data["monitor"]["nodes"].get(nodename, {}).get(
"stats", {}).get("load_15m", "")))
out.append(line)
def colorize_log_line(line, last=None, auto=None):
"""
Format a log line, colorizing the log level.
Return the line as a string buffer.
"""
msg = os.linesep.join(line["m"])
extra = ""
for k, v in line["x"].items():
if k in ("n", "o") and auto:
try:
i = auto.index(v)
v = colorize(v, AUTO_COLORS[i % AUTO_COLORS_LEN])
except (ValueError, IndexError):
pass
extra += "%s:%s " % (k, v)
extra = extra.rstrip()
lvl = line["l"]
t = datetime.datetime.fromtimestamp(
line["t"]).strftime("%Y-%m-%d %H:%M:%S,%f")
if lvl == "INFO":
lvl = colorize("INFO ", color.LIGHTBLUE)
elif lvl == "WARNING":
lvl = colorize("WARNING", color.BROWN)
elif lvl == "ERROR":
lvl = colorize("ERROR ", color.RED)
elif lvl == "DEBUG":
lvl = "DEBUG "
if msg.startswith("do "):
msg = colorize(msg, color.BOLD)
if auto:
for i, word in enumerate(auto):
msg = re.sub(
"([\s,:@'\"]+)%s([\s,:@'\"]+)" % word,
lambda m: m.group(1) + colorize(
word, AUTO_COLORS[i % AUTO_COLORS_LEN]) + m.group(2), msg)
line = " ".join((t, lvl, extra, "|", msg))
return line
def print_bold(self, s):
print(colorize(s, color.BOLD))
def load_svc(path, prefix=""):
if path not in services:
return
data = services[path]
if path in slave_parents and prefix == "":
return
try:
topology = services[path].topology
except KeyError:
topology = ""
if services[path].get("drp", False):
topology = "drp " + topology
# status
status = colorize_status(data["avail"], lpad=0)
if data["overall"] == "warn":
status += colorize("!", color.BROWN)
if data["placement"] == "non-optimal":
status += colorize("^", color.RED)
# info
info = {
"topology": data.get("topology", ""),
"orchestrate": data.get("orchestrate", "-"),
"status": "%d/1" % data["n_up"] if data["n_up"] is not None else 0,
}
if data.get("scale") is not None:
info["status"] = "%d/%d" % (data["n_up"], data.get("scale"))
elif data.get("wrapper"):
info = {
"topology": "",
"orchestrate": "",
"status": "",
}
elif topology == "flex":
info["status"] = "%d/%d" % (data["n_up"], data["flex_target"])
if data["avail"] == "n/a":
info["status"] = ""
info = "%(orchestrate)-5s %(status)-5s" % info
line = [
" " + colorize(
prefix + strip_path(path, os.environ.get("OSVC_NAMESPACE")),
color.BOLD),
status,
info,
fmt_svc_uptime(path, stats_data),
fmt_svc_tasks(path, prev_stats_data),
fmt_svc_cpu_usage(path, prev_stats_data, stats_data),
fmt_svc_cpu_time(path, stats_data),
fmt_svc_mem_total(path, stats_data),
fmt_svc_blk_rb(path, stats_data),
fmt_svc_blk_wb(path, stats_data),
fmt_svc_blk_rbps(path, prev_stats_data, stats_data),
fmt_svc_blk_wbps(path, prev_stats_data, stats_data),
"|" if nodenames else "",
]
for nodename in nodenames:
if nodename not in data["nodes"]:
line.append("")
elif data["nodes"][nodename] is None:
line.append(colorize("?", color.RED))
elif data["nodes"][nodename] is not None:
val = []
# frozen unicon
if data["nodes"][nodename]["frozen"]:
frozen_icon = colorize(unicons["frozen"], color.BLUE)
else:
frozen_icon = ""
# avail status unicon
if data["wrapper"]:
avail_icon = ""
else:
avail = data["nodes"][nodename]["avail"]
if avail == "unknown":
avail_icon = colorize("?", color.RED)
else:
avail_icon = colorize_status(
avail, lpad=0, agg_status=data["avail"]).replace(
avail, unicons[avail])
# overall status unicon
if data["wrapper"]:
overall_icon = ""
else:
overall = data["nodes"][nodename]["overall"]
if overall == "warn":
overall_icon = colorize_status(
overall, lpad=0).replace(overall, unicons[overall])
else:
overall_icon = ""
# mon status
smon = data["nodes"][nodename]["mon"]
if smon == "idle":
# don't display 'idle', as its to normal status and thus repeated as nauseam
smon = ""
else:
smon = " " + smon
# global expect
if smon == "":
global_expect = data["nodes"][nodename]["global_expect"]
if global_expect:
global_expect = colorize(" >" + str(global_expect),
color.LIGHTBLUE)
else:
global_expect = ""
else:
global_expect = ""
# leader
if data["wrapper"]:
leader = ""
else:
if data["nodes"][nodename]["placement"] == "leader":
leader = colorize("^", color.LIGHTBLUE)
else:
leader = ""
# provisioned
if data["nodes"][nodename].get("provisioned") is False:
provisioned = colorize("P", color.RED)
else:
provisioned = ""
val.append(avail_icon)
val.append(overall_icon)
val.append(leader)
val.append(frozen_icon)
val.append(provisioned)
val.append(smon)
val.append(global_expect)
line.append("".join(val))
out.append(line)
for child in sorted(list(data.get("slaves", []))):
load_svc(child, prefix=prefix + " ")
