def init(self):
if self.username is not None and self.password is not None:
return
s = "array#" + self.head
try:
stype = self.node.oget(s, "type")
except Exception:
raise ex.excError("no array configuration for head %s" % self.head)
if stype != "nexenta":
raise ex.excError("array %s type is not nexanta" % self.head)
try:
self.username = self.node.oget(s, "username")
except Exception:
raise ex.excError("no username information for head %s" %
self.head)
try:
self.password = self.node.oget(s, "password")
except Exception:
raise ex.excError("no password information for head %s" %
self.head)
self.port = self.node.oget(s, "port")
try:
secname, namespace, _ = split_path(self.password)
self.password = factory("sec")(
secname, namespace=namespace,
volatile=True).decode_key("password")
except Exception as exc:
raise ex.excError("error decoding password: %s" % exc)
self.url = 'https://%(head)s:%(port)d/rest/nms/ <https://%(head)s:%(port)d/rest/nms/>' % dict(
head=self.head, port=self.port)
def get_service_slaves(self, path, slaves=None):
"""
Recursive lookup of object slaves.
"""
if slaves is None:
slaves = set()
_, namespace, _ = split_path(path)
def set_ns(path, parent_ns):
name, _namespace, kind = split_path(path)
if _namespace:
return path
else:
return fmt_path(name, parent_ns, kind)
for nodename in shared.CLUSTER_DATA:
try:
data = shared.CLUSTER_DATA[nodename]["services"]["status"][
path]
except KeyError:
continue
slaves.add(path)
new_slaves = set(data.get("slaves", [])) | set(
data.get("scaler_slaves", []))
new_slaves = set(
[set_ns(slave, namespace) for slave in new_slaves])
new_slaves -= slaves
for slave in new_slaves:
slaves |= self.get_service_slaves(slave, slaves)
return slaves
def ca(self):
capath = self.oget("DEFAULT", "ca")
name, namespace, kind = split_path(capath)
return factory("sec")(name,
namespace=namespace,
volatile=True,
node=self.node)
def on_create(self):
changes = []
has_ca = False
if not self.oget("DEFAULT", "cn"):
if self.namespace == "system":
changes.append("cn=%s" % self.name)
else:
changes.append("cn=%s" % self.fullname)
if self.namespace != "system":
try:
self.conf_get("DEFAULT", "validity")
except ex.OptNotFound:
changes.append("validity=%s" % DEFAULT_SACC_CERT_VALIDITY)
grant = "guest:" + self.namespace
changes.append("grant=%s" % grant)
if not self.oget("DEFAULT", "ca"):
capath = self.node.oget("cluster", "ca")
if capath is None:
capath = "system/sec/ca-" + self.node.cluster_name
name, namespace, kind = split_path(capath)
casec = factory("sec")(name,
namespace="system",
volatile=True,
log=self.log)
if casec.exists():
has_ca = True
changes.append("ca=%s" % capath)
else:
print("no cluster CA defined. skip certificate generation.")
if changes:
self.set_multi(changes)
if has_ca and "certificate" not in self.data_keys(
) and "private_key" in casec.data_keys():
self.gen_cert()
def format_service(path, idata, mon_data=None, discard_disabled=False, nodename=None):
name, namespace, kind = split_path(path)
svc_notice = get_svc_notice(idata)
tree = Forest(
separator=" ",
widths=(
(14, None),
None,
10,
None,
),
)
node_name = tree.add_node()
node_name.add_column(strip_path(path, os.environ.get("OSVC_NAMESPACE")), color.BOLD)
node_name.add_column()
if "cluster" in idata:
node_name.add_column(idata["cluster"].get("avail", "n/a"), STATUS_COLOR[idata["cluster"].get("avail", "n/a")])
else:
node_name.add_column()
node_name.add_column(svc_notice)
node_instances = node_name.add_node()
node_instances.add_column("instances")
add_instances(node_instances, path, nodename, mon_data)
if nodename in service_nodes(path, mon_data):
add_node_node(node_instances, nodename, idata, mon_data, discard_disabled=discard_disabled)
add_parents(node_name, idata, mon_data, namespace)
add_children(node_name, idata, mon_data, namespace)
add_scaler_slaves(node_name, idata, mon_data, namespace)
add_slaves(node_name, idata, mon_data, namespace)
tree.out()
def svc_ptr_record(self, qname):
if not qname.endswith(PTR_SUFFIX):
return []
names = []
ref = ".".join(reversed(qname[:-PTR_SUFFIX_LEN].split(".")))
with shared.CLUSTER_DATA_LOCK:
for nodename, node in shared.CLUSTER_DATA.items():
status = node.get("services", {}).get("status", {})
for path, svc in status.items():
name, namespace, kind = split_path(path)
if kind != "svc":
continue
if not namespace:
namespace = "root"
for rid, resource in status[path].get("resources",
{}).items():
addr = resource.get("info", {}).get("ipaddr")
if addr is None:
continue
if addr != ref:
continue
try:
hostname = resource.get(
"info",
{}).get("hostname").split(".")[0].lower()
except Exception:
hostname = None
gen_name = "%s.%s.%s.%s." % (name, namespace, kind,
self.cluster_name)
gen_name = gen_name.lower()
if hostname and hostname != name:
names.append("%s.%s" % (hostname, gen_name))
else:
names.append(gen_name)
return names
def rbac(self, nodename, thr=None, **kwargs):
options = self.parse_options(kwargs)
name, namespace, kind = split_path(options.path)
if kind == "cfg":
role = "guest"
else:
# sec, usr
role = "admin"
thr.rbac_requires(roles=[role], namespaces=[namespace], **kwargs)
def rbac(self, nodename, thr=None, **kwargs):
options = self.parse_options(kwargs)
if options.path:
name, namespace, kind = split_path(options.path)
thr.rbac_requires(roles=["operator"],
namespaces=[namespace],
**kwargs)
else:
thr.rbac_requires(roles=["operator"], namespaces="ANY", **kwargs)
def a_records(self):
data = self.get_cache("a")
if data is not None:
return data
names = {}
for nodename in self.cluster_nodes:
try:
node = shared.CLUSTER_DATA[nodename]
except KeyError:
continue
status = node.get("services", {}).get("status", {})
for path, svc in status.items():
name, namespace, kind = split_path(path)
if kind != "svc":
continue
if namespace:
namespace = namespace.lower()
else:
namespace = "root"
scaler_slave = svc.get("scaler_slave")
if scaler_slave:
_name = name[name.index(".") + 1:]
else:
_name = name
zone = "%s.%s.%s." % (namespace, kind, self.cluster_name)
qname = "%s.%s" % (_name, zone)
if qname not in names:
names[qname] = set()
for rid, resource in status.get(path, {}).get("resources",
{}).items():
addr = resource.get("info", {}).get("ipaddr")
if addr is None:
continue
hostname = resource.get("info", {}).get("hostname")
names[qname].add(addr)
rname = self.unique_name(addr) + "." + qname
if rname not in names:
names[rname] = set()
names[rname].add(addr)
if hostname:
name = hostname.split(".")[0] + "." + qname
if name not in names:
names[name] = set()
names[name].add(addr)
for i, ip in enumerate(shared.NODE.dns):
try:
dns = "%s.%s." % (shared.NODE.dnsnodes[i].split(".")[0],
self.cluster_name)
names[dns] = set([ip])
except IndexError:
self.log.warning("dns (%s) and dnsnodes (%s) are not aligned"
"" % (shared.NODE.dns, shared.NODE.dnsnodes))
break
self.set_cache("a", names)
return names
def __init__(self, objects=[], node=None):
self.objects = objects
self.filtering = len(objects) > 0
if node:
self.node = node
else:
self.node = Node()
done = []
for s in self.node.conf_sections(cat="array"):
try:
name = self.node.oget(s, "name")
except Exception:
name = None
if not name:
name = s.split("#", 1)[-1]
if name in done:
continue
if self.filtering and name not in self.objects:
continue
try:
stype = self.node.oget(s, "type")
except:
continue
if stype != "hds":
continue
try:
bin = self.node.oget(s, 'bin')
jre_path = self.node.oget(s, 'jre_path')
url = self.node.oget(s, 'url')
username = self.node.oget(s, 'username')
password = self.node.oget(s, 'password')
except Exception as exc:
print("error parsing section %s: %s" % (s, exc),
file=sys.stderr)
continue
try:
secname, namespace, _ = split_path(password)
password = factory("sec")(secname,
namespace=namespace,
volatile=True).decode_key("password")
except Exception as exc:
print("error decoding password: %s", exc, file=sys.stderr)
continue
self.arrays.append(
Array(name,
url,
username,
password,
bin=bin,
jre_path=jre_path,
node=self.node))
done.append(name)
def __init__(self, name):
self.node_flag = os.path.join(rcEnv.paths.pathvar, "node", "frozen")
if name == "node":
self.flag = self.node_flag
else:
name, namespace, kind = split_path(name)
if namespace:
self.flag = os.path.join(rcEnv.paths.pathvar, "namespaces",
namespace, kind, name, "frozen")
else:
self.flag = os.path.join(rcEnv.paths.pathvar, kind, name,
"frozen")
def __init__(self, objects=[], node=None):
self.objects = objects
if len(objects) > 0:
self.filtering = True
else:
self.filtering = False
self.arrays = []
if node:
self.node = node
else:
self.node = Node()
done = []
for s in self.node.conf_sections(cat="array"):
name = s.split("#", 1)[-1]
if name in done:
continue
if self.filtering and name not in self.objects:
continue
try:
stype = self.node.oget(s, "type")
except:
continue
if stype != "centera":
continue
try:
server = self.node.oget(s, "server")
username = self.node.oget(s, "username")
password = self.node.oget(s, "password")
jcass_dir = self.node.oget(s, "jcass_dir")
java_bin = self.node.oget(s, "java_bin")
except:
print("error parsing section", s, file=sys.stderr)
try:
secname, namespace, _ = split_path(password)
password = factory("sec")(secname,
namespace=namespace,
volatile=True).decode_key("password")
except Exception as exc:
print("error decoding password: %s", exc, file=sys.stderr)
continue
self.arrays.append(
Centera(name,
server=server,
username=username,
password=password,
java_bin=java_bin,
jcass_dir=jcass_dir,
node=self.node))
done.append(name)
def rbac(self, nodename, thr=None, **kwargs):
options = self.parse_options(kwargs)
name, namespace, kind = split_path(options.path)
if options.action in GUEST_ACTIONS:
role = "guest"
elif options.action in OPERATOR_ACTIONS:
role = "operator"
elif options.action in ADMIN_ACTIONS:
role = "admin"
else:
role = "root"
if options.action == "set":
# load current config
try:
cf = shared.SERVICES[options.path].print_config_data()
except Exception as exc:
cf = {}
# purge unwanted sections
try:
del cf["metadata"]
except Exception:
pass
# merge changes in current config
for buff in options.options.get("kw", []):
k, v = buff.split("=", 1)
if k[-1] in ("+", "-"):
k = k[:-1]
k = k.strip()
try:
s, k = k.split(".", 1)
except Exception:
s = "DEFAULT"
if s not in cf:
cf[s] = {}
cf[s][k] = v
# apply object create rbac to the amended config
payload = {options.path: cf}
errors = self.rbac_create_data(payload, thr=thr, **kwargs)
if errors:
raise HTTP(403, errors)
else:
thr.rbac_requires(roles=[role], namespaces=[namespace], **kwargs)
if options.cmd:
# compat, requires root
kwargs["roles"] = ["root"]
thr.rbac_requires(**kwargs)
def __init__(self, objects=[], node=None):
self.objects = objects
if len(objects) > 0:
self.filtering = True
else:
self.filtering = False
self.arrays = []
if node:
self.node = node
else:
self.node = Node()
done = []
for s in self.node.conf_sections(cat="array"):
name = s.split("#", 1)[-1]
if name in done:
continue
if self.filtering and name not in self.objects:
continue
try:
stype = self.node.oget(s, "type")
except:
continue
if stype != "emcvnx":
continue
try:
method = self.node.oget(s, "method")
scope = self.node.oget(s, "scope")
spa = self.node.oget(s, "spa")
spb = self.node.oget(s, "spb")
username = self.node.oget(s, "username")
password = self.node.oget(s, "password")
except Exception as exc:
print("error parsing section %s: %s" % (s, exc), file=sys.stderr)
continue
if method == "credentials":
if username is None or password is None:
print("error parsing section %s: username and password are mandatory" % s, file=sys.stderr)
continue
try:
secname, namespace, _ = split_path(password)
password = factory("sec")(secname, namespace=namespace, volatile=True).decode_key("password")
except Exception as exc:
print("error decoding password: %s", exc, file=sys.stderr)
continue
self.arrays.append(EmcVnx(name, method, scope, spa, spb, username=username, password=password, node=self.node))
done.append(name)
def dispatch_svcs(paths):
data = {}
for path in paths:
try:
_, _, kind = split_path(path)
except ValueError:
continue
try:
validate_kind(kind)
except ValueError as exc:
raise ex.excError(str(exc))
try:
data[kind].append(path)
except KeyError:
data[kind] = [path]
return data
def postinstall(self, key=None):
"""
Refresh installed keys
"""
for path in self.node.svcs_selector("*/svc/*",
namespace=self.namespace,
local=True):
name, _, _ = split_path(path)
svc = factory("svc")(name,
namespace=self.namespace,
volatile=True,
node=self.node,
log=self.log)
for vol in svc.get_resources("volume"):
if vol.has_data(self.kind, self.path,
key) and vol._status() == rcStatus.UP:
vol._install_data(self.kind)
def __init__(self, objects=[], node=None):
self.objects = objects
self.filtering = len(objects) > 0
if node:
self.node = node
else:
self.node = Node()
done = []
for s in self.node.conf_sections(cat="array"):
name = s.split("#", 1)[-1]
if name in done:
continue
try:
stype = self.node.oget(s, "type")
except:
continue
if stype != "eva":
continue
try:
manager = self.node.oget(s, 'manager')
username = self.node.oget(s, 'username')
password = self.node.oget(s, 'password')
sssubin = self.node.oget(s, 'bin')
except Exception as exc:
print("error parsing section %s: %s" % (s, exc), file=sys.stderr)
pass
try:
secname, namespace, _ = split_path(password)
password = factory("sec")(secname, namespace=namespace, volatile=True).decode_key("password")
except Exception as exc:
print("error decoding password: %s", exc, file=sys.stderr)
continue
out, err = sssu('ls system', manager, username, password, sssubin=sssubin)
_in = False
for line in out.split('\n'):
if 'Systems avail' in line:
_in = True
continue
if not _in:
continue
name = line.strip()
if self.filtering and name not in self.objects:
continue
self.arrays.append(Eva(name, manager, username, password, sssubin=sssubin))
done.append(name)
def __init__(self, objects=[], node=None):
self.objects = objects
self.filtering = len(objects) > 0
self.timeout = 10
if node:
self.node = node
else:
self.node = Node()
done = []
for s in self.node.conf_sections(cat="array"):
try:
name = self.node.oget(s, 'name')
except Exception:
name = None
if not name:
name = s.split("#", 1)[-1]
if name in done:
continue
if self.filtering and name not in self.objects:
continue
try:
stype = self.node.oget(s, "type")
except:
continue
if stype != "dorado":
continue
timeout = self.node.oget(s, "timeout")
try:
username = self.node.oget(s, "username")
password = self.node.oget(s, "password")
api = self.node.oget(s, "api")
except:
print("error parsing section", s, file=sys.stderr)
continue
try:
secname, namespace, _ = split_path(password)
password = factory("sec")(secname,
namespace=namespace,
volatile=True).decode_key("password")
except Exception as exc:
print("error decoding password: %s", exc, file=sys.stderr)
continue
self.arrays.append(
Dorado(name, api, username, password, timeout, node=self.node))
done.append(name)
def _data_status(self, kind):
for data in self.data_data(kind):
name, _, kind = split_path(data["obj"])
obj = factory(kind)(name,
namespace=self.svc.namespace,
volatile=True,
node=self.svc.node)
if not obj.exists():
self.status_log(
"referenced %s %s does not exist: "
"expected data %s can not be installed in the volume" %
(kind, name, data["key"]), "warn")
continue
keys = obj.resolve_key(data["key"])
if not keys and not is_glob(data["key"]):
self.status_log(
"%s %s has no key %s. "
"expected data can not be installed in the volume" %
(kind, name, data["key"]), "warn")
def action(self, nodename, thr=None, **kwargs):
"""
Care with locks
"""
thr.log_request("shutdown daemon", nodename, **kwargs)
with shared.THREADS_LOCK:
shared.THREADS["scheduler"].stop()
mon = shared.THREADS["monitor"]
if thr.stopped() or shared.NMON_DATA.status == "shutting":
thr.log.info("already shutting")
# wait for service shutdown to finish before releasing the dup client
while True:
if mon._shutdown:
break
time.sleep(0.3)
return {"status": 0}
try:
thr.set_nmon("shutting")
mon.kill_procs()
for path in shared.SMON_DATA:
_, _, kind = split_path(path)
if kind not in ("svc", "vol"):
continue
thr.set_smon(path, local_expect="shutdown")
self.wait_shutdown()
# send a last status to peers so they can takeover asap
mon.update_hb_data()
mon._shutdown = True
shared.wake_monitor("services shutdown done")
except Exception as exc:
thr.log.exception(exc)
thr.log.info("services are now shutdown")
while True:
with shared.THREADS_LOCK:
if not shared.THREADS["monitor"].is_alive():
break
time.sleep(0.3)
shared.DAEMON_STOP.set()
return {"status": 0}
def rbac(self, nodename, thr=None, **kwargs):
options = self.parse_options(kwargs)
name, namespace, kind = split_path(options.path)
role = "admin"
operator = (
# (local_expect, global_expect, reset_retries)
(None, None, True),
(None, "thawed", False),
(None, "frozen", False),
(None, "started", False),
(None, "stopped", False),
(None, "aborted", False),
(None, "placed", False),
(None, "shutdown", False),
)
_global_expect = options.global_expect.split(
"@")[0] if options.global_expect else options.global_expect
if (options.local_expect, _global_expect,
options.reset_retries) in operator:
role = "operator"
thr.rbac_requires(roles=[role], namespaces=[namespace], **kwargs)
def __init__(self, objects=[], node=None):
self.objects = objects
self.filtering = len(objects) > 0
self.arrays = []
if node:
self.node = node
else:
self.node = Node()
done = []
for s in self.node.conf_sections(cat="array"):
name = s.split("#", 1)[-1]
if name in done:
continue
if self.filtering and name not in self.objects:
continue
try:
stype = self.node.oget(s, "type")
except:
continue
if stype != "netapp":
continue
kwargs = {"node": self.node}
for key in ("server", "username", "key"):
try:
kwargs[key] = self.node.oget(s, key)
except:
print("missing parameter: %s", s)
if "server" not in kwargs or "username" not in kwargs or "key" not in kwargs:
continue
try:
secname, namespace, _ = split_path(kwargs["password"])
kwargs["password"] = factory("sec")(
secname, namespace=namespace,
volatile=True).decode_key("password")
except Exception as exc:
print("error decoding password: %s", exc, file=sys.stderr)
continue
self.arrays.append(Netapp(s, **kwargs))
def _install_data(self, kind):
for data in self.data_data(kind):
name, _, kind = split_path(data["obj"])
obj = factory(kind)(name,
namespace=self.svc.namespace,
volatile=True,
node=self.svc.node)
if not obj.exists():
self.log.warning(
"referenced %s %s does not exist: "
"expected data %s can not be installed in the volume",
kind, name, data["key"])
continue
keys = obj.resolve_key(data["key"])
if not keys and "*" not in data["key"] and "?" not in data["key"]:
self.log.warning(
"%s %s has no key %s. "
"expected data can not be installed in the volume", kind,
name, data["key"])
continue
for key in obj.resolve_key(data["key"]):
obj._install(key, data["path"])
def configure_volume(self,
volume,
size=None,
fmt=True,
access="rwo",
shared=False,
nodes=None,
env=None):
if self.template is None:
raise ex.excError("pool#%s.template is not set" % self.name)
if not is_service(self.template):
raise ex.excError("%s template volume not found" % self.template)
name = self.default_disk_name(volume)
tname, tnamespace, tkind = split_path(self.template)
if tkind != "vol":
raise ex.excError("%s template kind is not vol")
svc = factory(tkind)(tname, tnamespace, volatile=True, node=self.node)
config = svc.print_config_data()
try:
del config["DEFAULT"]["disable"]
except KeyError:
pass
if "DEFAULT" not in config:
config["DEFAULT"] = {}
if "env" not in config:
config["env"] = {}
config["DEFAULT"]["pool"] = self.name
config["DEFAULT"]["access"] = access
if access in ("rox", "rwx"):
config["DEFAULT"]["topology"] = "flex"
config["DEFAULT"]["flex_min"] = 0
if nodes:
config["DEFAULT"]["nodes"] = nodes
config["env"]["size"] = size
if env:
config["env"].update(env)
self.node.install_svc_conf_from_data(volume.name, volume.namespace,
volume.kind, config)
def gen_cert(self):
data = {}
for key in ("cn", "c", "st", "l", "o", "ou", "email", "alt_names",
"bits", "validity", "ca"):
val = self.oget("DEFAULT", key)
if val is not None:
data[key] = val
ca = data.get("ca")
casec = None
if ca is not None:
casecname, canamespace, _ = split_path(ca)
casec = factory("sec")(casecname,
namespace=canamespace,
log=self.log,
volatile=True)
if not casec.exists():
raise ex.excError("ca secret %s does not exist" % ca)
for key in ("crt", "key", "csr"):
data[key] = self.tempfilename()
if "alt_names" in data:
data["cnf"] = self.tempfilename()
try:
if casec:
for key, kw in (("cacrt", "certificate"), ("cakey",
"private_key")):
if kw not in casec.data_keys():
continue
data[key] = self.tempfilename()
buff = bdecode(casec.decode_key(kw))
with open(data[key], "w") as ofile:
ofile.write(buff)
gen_cert(log=self.log, **data)
self._add("private_key", value_from=data["key"])
if data.get("crt") is not None:
self._add("certificate", value_from=data["crt"])
if data.get("csr") is not None:
self._add("certificate_signing_request",
value_from=data["csr"])
if data.get("cakey") is None:
self._add("certificate_chain", value_from=data["crt"])
else:
# merge cacrt and crt
chain = self.tempfilename()
try:
with open(data["crt"], "r") as ofile:
buff = ofile.read()
with open(data["cacrt"], "r") as ofile:
buff += ofile.read()
with open(chain, "w") as ofile:
ofile.write(buff)
self._add("certificate_chain", value_from=chain)
finally:
try:
os.unlink(chain)
except Exception:
pass
self.add_key("fullpem", self._fullpem())
finally:
for key in ("crt", "key", "cacrt", "cakey", "csr", "cnf"):
if key not in data:
continue
try:
os.unlink(data[key])
except Exception:
pass
def format_cluster(paths=None,
node=None,
data=None,
prev_stats_data=None,
stats_data=None,
sections=None,
selector=None,
namespace=None):
if not data or data.get("status", 0) != 0:
return
if sections is None:
sections = DEFAULT_SECTIONS
out = []
nodenames = get_nodes(data)
nodenames = sorted([n for n in nodenames if n in node])
show_nodenames = abbrev([n for n in nodenames if n in node])
services = {}
def load_header(title=""):
if isinstance(title, list):
line = title
else:
line = [
title,
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
]
for nodename in show_nodenames:
line.append(colorize(nodename, color.BOLD))
out.append(line)
def load_svc(path, prefix=""):
if path not in services:
return
data = services[path]
if path in slave_parents and prefix == "":
return
try:
topology = services[path].topology
except KeyError:
topology = ""
if services[path].get("drp", False):
topology = "drp " + topology
# status
status = colorize_status(data["avail"], lpad=0)
if data["overall"] == "warn":
status += colorize("!", color.BROWN)
if data["placement"] == "non-optimal":
status += colorize("^", color.RED)
# info
info = {
"topology": data.get("topology", ""),
"orchestrate": data.get("orchestrate", "-"),
"status": "%d/1" % data["n_up"] if data["n_up"] is not None else 0,
}
if data.get("scale") is not None:
info["status"] = "%d/%d" % (data["n_up"], data.get("scale"))
elif data.get("wrapper"):
info = {
"topology": "",
"orchestrate": "",
"status": "",
}
elif topology == "flex":
info["status"] = "%d/%d" % (data["n_up"], data["flex_target"])
if data["avail"] == "n/a":
info["status"] = ""
info = "%(orchestrate)-5s %(status)-5s" % info
line = [
" " + colorize(
prefix + strip_path(path, os.environ.get("OSVC_NAMESPACE")),
color.BOLD),
status,
info,
fmt_svc_uptime(path, stats_data),
fmt_svc_tasks(path, prev_stats_data),
fmt_svc_cpu_usage(path, prev_stats_data, stats_data),
fmt_svc_cpu_time(path, stats_data),
fmt_svc_mem_total(path, stats_data),
fmt_svc_blk_rb(path, stats_data),
fmt_svc_blk_wb(path, stats_data),
fmt_svc_blk_rbps(path, prev_stats_data, stats_data),
fmt_svc_blk_wbps(path, prev_stats_data, stats_data),
"|" if nodenames else "",
]
for nodename in nodenames:
if nodename not in data["nodes"]:
line.append("")
elif data["nodes"][nodename] is None:
line.append(colorize("?", color.RED))
elif data["nodes"][nodename] is not None:
val = []
# frozen unicon
if data["nodes"][nodename]["frozen"]:
frozen_icon = colorize(unicons["frozen"], color.BLUE)
else:
frozen_icon = ""
# avail status unicon
if data["wrapper"]:
avail_icon = ""
else:
avail = data["nodes"][nodename]["avail"]
if avail == "unknown":
avail_icon = colorize("?", color.RED)
else:
avail_icon = colorize_status(
avail, lpad=0, agg_status=data["avail"]).replace(
avail, unicons[avail])
# overall status unicon
if data["wrapper"]:
overall_icon = ""
else:
overall = data["nodes"][nodename]["overall"]
if overall == "warn":
overall_icon = colorize_status(
overall, lpad=0).replace(overall, unicons[overall])
else:
overall_icon = ""
# mon status
smon = data["nodes"][nodename]["mon"]
if smon == "idle":
# don't display 'idle', as its to normal status and thus repeated as nauseam
smon = ""
else:
smon = " " + smon
# global expect
if smon == "":
global_expect = data["nodes"][nodename]["global_expect"]
if global_expect:
global_expect = colorize(" >" + str(global_expect),
color.LIGHTBLUE)
else:
global_expect = ""
else:
global_expect = ""
# leader
if data["wrapper"]:
leader = ""
else:
if data["nodes"][nodename]["placement"] == "leader":
leader = colorize("^", color.LIGHTBLUE)
else:
leader = ""
# provisioned
if data["nodes"][nodename].get("provisioned") is False:
provisioned = colorize("P", color.RED)
else:
provisioned = ""
val.append(avail_icon)
val.append(overall_icon)
val.append(leader)
val.append(frozen_icon)
val.append(provisioned)
val.append(smon)
val.append(global_expect)
line.append("".join(val))
out.append(line)
for child in sorted(list(data.get("slaves", []))):
load_svc(child, prefix=prefix + " ")
def load_hb(key, _data):
state = _data.get("state", "")
if state == "running":
state = colorize(state, color.GREEN)
else:
state = colorize(state, color.RED)
if _data.get("alerts"):
state += colorize("!", color.BROWN)
cf = _data.get("config", {})
addr = cf.get("addr", "")
port = cf.get("port", "")
dev = cf.get("dev", "")
relay = cf.get("relay", "")
if addr and port:
config = addr + ":" + str(port)
elif dev:
config = os.path.basename(dev)
elif relay:
config = relay
else:
config = ""
line = [
" " + colorize(key, color.BOLD),
state,
config,
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
peers = _data.get("peers", {})
for nodename in nodenames:
beating = peers.get(nodename, {}).get("beating")
if beating is None:
status = "n/a"
elif beating:
status = "up"
else:
status = "down"
status = colorize_status(status,
lpad=0).replace(status, unicons[status])
line.append(status)
out.append(line)
def load_monitor(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
transitions = _data.get("transitions", 0)
if transitions:
status = "%d transition" % transitions
else:
status = ""
out.append((
" " + colorize(key, color.BOLD),
state,
status,
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
))
def load_listener(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
out.append((
" " + colorize(key, color.BOLD),
state,
_data["config"]["addr"] + ":" + str(_data["config"]["port"]),
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
))
def load_scheduler(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
out.append((
" " + colorize(key, color.BOLD),
state,
"",
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
))
def load_daemon():
key = "daemon"
state = colorize("running", color.GREEN)
line = [
" " + colorize(key, color.BOLD),
"%s" % state,
"",
str(data.get("pid", "")) if stats_data else "",
"",
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
fmt_thr_mem_total(key, stats_data),
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
speaker = data["monitor"].get("nodes", {}).get(nodename,
{}).get("speaker")
if speaker:
status = "up"
status = colorize_status(status, lpad=0).replace(
status, unicons[status])
else:
status = ""
line.append(status)
out.append(line)
def load_collector(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
line = [
" " + colorize(key, color.BOLD),
state,
"",
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
speaker = data["monitor"].get("nodes", {}).get(nodename,
{}).get("speaker")
if speaker:
status = "up"
status = colorize_status(status, lpad=0).replace(
status, unicons[status])
else:
status = ""
line.append(status)
out.append(line)
def load_generic_thread(key, _data):
if _data["state"] == "running":
state = colorize(_data["state"], color.GREEN)
else:
state = colorize(_data["state"], color.RED)
out.append((
" " + colorize(key, color.BOLD),
state,
"",
fmt_tid(_data, stats_data),
fmt_thr_tasks(key, stats_data),
fmt_thr_cpu_usage(key, prev_stats_data, stats_data),
fmt_thr_cpu_time(key, stats_data),
"",
"",
"",
))
def load_threads():
if "threads" not in sections:
return
load_header([
"Threads",
"",
"",
"pid/tid" if stats_data else "",
"thr/sub" if stats_data else "",
"usage" if stats_data else "",
"time" if stats_data else "",
"rss" if stats_data else "",
"",
"",
"",
"",
"",
])
load_daemon()
for key in sorted([key for key in data if key != "cluster"]):
if key.startswith("hb#"):
load_hb(key, data[key])
elif key == "monitor":
load_monitor(key, data[key])
elif key == "scheduler":
load_scheduler(key, data[key])
elif key == "listener":
load_listener(key, data[key])
elif key == "collector":
load_collector(key, data[key])
else:
try:
load_generic_thread(key, data[key])
except Exception:
pass
out.append([])
def load_score():
if "monitor" not in data:
return
line = [
colorize(" score", color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
line.append(
str(data["monitor"]["nodes"].get(nodename,
{}).get("stats",
{}).get("score", "")))
out.append(line)
def load_loadavg():
if "monitor" not in data:
return
line = [
colorize(" load 15m", color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
line.append(
str(data["monitor"]["nodes"].get(nodename, {}).get(
"stats", {}).get("load_15m", "")))
out.append(line)
def load_free_total(key):
if "monitor" not in data:
return
line = [
colorize(" " + key, color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
total = data["monitor"]["nodes"].get(nodename, {}).get(
"stats", {}).get(key + "_total")
avail = data["monitor"]["nodes"].get(nodename, {}).get(
"stats", {}).get(key + "_avail")
limit = 100 - data["monitor"]["nodes"].get(nodename, {}).get(
"min_avail_" + key, 0)
if avail is None or total in (0, None):
line.append("-")
continue
usage = 100 - avail
total = print_size(total, unit="MB", compact=True)
if limit:
cell = "%d/%d%%:%s" % (usage, limit, total)
else:
cell = "%d%%:%s" % (usage, total)
if usage > limit:
cell = colorize(cell, color.RED)
line.append(cell)
out.append(line)
def load_node_state():
if "monitor" not in data:
return
line = [
colorize(" state", color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
nmon_state = data["monitor"]["nodes"].get(nodename, {}).get(
"monitor", {}).get("status", "")
if nmon_state == "idle":
nmon_state = ""
if data["monitor"]["nodes"].get(nodename, {}).get("frozen", ""):
frozen = frozen_icon = colorize(unicons["frozen"], color.BLUE)
else:
frozen = ""
global_expect = data["monitor"]["nodes"].get(nodename, {}).get(
"monitor", {}).get("global_expect")
if global_expect:
global_expect = colorize(" >" + str(global_expect),
color.LIGHTBLUE)
else:
global_expect = ""
line.append(str(nmon_state) + frozen + global_expect)
out.append(line)
def load_node_compat():
if "monitor" not in data:
return
if data["monitor"].get("compat") is True:
# no need to clutter if the situation is normal
return
line = [
colorize(" compat", color.BOLD),
colorize("warn", color.BROWN),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
compat = data["monitor"]["nodes"].get(nodename,
{}).get("compat", "")
line.append(str(compat))
out.append(line)
def load_node_version():
if "monitor" not in data:
return
line = [
colorize(" version", color.BOLD),
colorize("warn", color.BROWN),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
versions = []
for nodename in nodenames:
agent = data["monitor"]["nodes"].get(nodename, {}).get("agent", "")
line.append(str(agent))
if agent != "":
versions.append(str(agent))
if len(set(versions)) > 1:
out.append(line)
def load_arbitrators():
if "arbitrators" not in sections:
return
arbitrators = []
arbitrators_name = {}
for nodename, ndata in data["monitor"]["nodes"].items():
for aid, adata in ndata.get("arbitrators", {}).items():
if aid not in arbitrators:
arbitrators.append(aid)
arbitrators_name[aid] = adata["name"]
if len(arbitrators) == 0:
return
load_header("Arbitrators")
for aid in arbitrators:
line = [
colorize(" " + arbitrators_name[aid], color.BOLD),
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"|" if nodenames else "",
]
for nodename in nodenames:
status = data["monitor"]["nodes"].get(nodename, {}).get(
"arbitrators", {}).get(aid, {}).get("status", "undef")
if status != "up":
line[1] = colorize_status("warn", lpad=0)
status = colorize_status(status, lpad=0).replace(
status, unicons[status])
line.append(status)
out.append(line)
out.append([])
def load_nodes():
if "nodes" not in sections or not nodenames:
return
load_header("Nodes")
load_metrics()
load_node_compat()
load_node_version()
load_node_state()
out.append([])
def load_metrics():
load_score()
load_loadavg()
load_free_total("mem")
load_free_total("swap")
# init the services hash
slave_parents = {}
if "monitor" in data:
for _node in nodenames:
if _node not in data["monitor"]["nodes"]:
continue
try:
node_svc_status = data["monitor"]["nodes"][_node]["services"][
"status"]
except KeyError:
continue
for path, _data in node_svc_status.items():
if _data is None:
continue
if paths is not None and path not in paths:
continue
if path not in services:
services[path] = Storage({
"drp":
_data.get("drp", False),
"topology":
_data.get("topology", ""),
"orchestrate":
_data.get("orchestrate", "-"),
"flex_target":
_data.get("flex_target"),
"scale":
_data.get("scale"),
"avail":
"undef",
"overall":
"",
"nodes": {},
"slaves":
set(),
"n_up":
0,
"resources":
set(),
})
try:
services[path]["resources"] |= set(
_data["resources"].keys())
except KeyError:
pass
slaves = _data.get("slaves", [])
scale = _data.get("scale")
if scale:
name, _namespace, kind = split_path(path)
if _namespace:
pattern = "^%s/%s/[0-9]+\.%s$" % (_namespace, kind,
name)
else:
pattern = "^[0-9]+\.%s$" % name
for child in data["monitor"]["services"]:
if re.match(pattern, child) is None:
continue
slaves.append(child)
if node_svc_status.get(child, {}).get("avail") == "up":
services[path].n_up += 1
else:
if node_svc_status.get(path, {}).get("avail") == "up":
services[path].n_up += 1
for child in slaves:
if child not in slave_parents:
slave_parents[child] = set([path])
else:
slave_parents[child] |= set([path])
global_expect = _data["monitor"].get("global_expect")
if global_expect and "@" in global_expect:
global_expect = global_expect[:global_expect.index("@") +
1]
services[path].nodes[_node] = {
"avail": _data.get("avail", "undef"),
"overall": _data.get("overall", "undef"),
"frozen": _data.get("frozen", False),
"mon": _data["monitor"].get("status", ""),
"global_expect": global_expect,
"placement": _data["monitor"].get("placement", ""),
"provisioned": _data.get("provisioned"),
}
services[path].slaves |= set(slaves)
services[path]["wrapper"] = (services[path].resources == set()
and
services[path].slaves != set()
and scale is None)
try:
# hint we have missing instances
for path, cnf in data["monitor"]["nodes"][_node]["services"][
"config"].items():
if path not in services:
continue
for __node in cnf.get("scope", []):
if __node not in services[path].nodes:
services[path].nodes[__node] = None
except KeyError:
pass
for path, _data in data["monitor"]["services"].items():
if paths is not None and path not in paths:
continue
if path not in services:
services[path] = Storage({
"avail": "undef",
"overall": "",
"nodes": {}
})
services[path].avail = _data.get("avail", "n/a")
services[path].overall = _data.get("overall", "n/a")
services[path].placement = _data.get("placement", "n/a")
def load_services(selector, namespace=None):
if "services" not in sections:
return
selectors = []
context = os.environ.get("OSVC_CONTEXT", "")
if context:
selectors.append(context)
buff = format_path_selector(selector, namespace)
if len(buff) > 15:
buff = buff[:12] + "..."
selectors.append(buff)
load_header([
"/".join(selectors),
"",
"",
"since" if stats_data else "",
"tasks" if stats_data else "",
"usage" if stats_data else "",
"time" if stats_data else "",
"mem" if stats_data else "",
"blkrb" if stats_data else "",
"blkwb" if stats_data else "",
"blkrbps" if stats_data else "",
"blkwbps" if stats_data else "",
"",
])
for path in sorted(list(services.keys())):
load_svc(path)
# load data in lists
load_threads()
load_arbitrators()
load_nodes()
load_services(selector, namespace)
# print tabulated lists
return print_section(out)
def set_ns(path, parent_ns):
name, _namespace, kind = split_path(path)
if _namespace:
return path
else:
return fmt_path(name, parent_ns, kind)
def rbac_create_obj(self, path, cd, all_ns, thr=None, **kwargs):
errors = []
name, namespace, kind = split_path(path)
grants = thr.user_grants(all_ns | set([namespace]))
if namespace not in all_ns:
if namespace == "system":
errors.append(
"%s: create the new namespace system requires the root cluster role"
)
return errors
elif "squatter" not in grants:
errors.append(
"%s: create the new namespace %s requires the squatter cluster role"
% (path, namespace))
return errors
elif namespace not in grants["admin"]:
thr.usr.set_multi(["grant+=admin:%s" % namespace])
grants["admin"].add(namespace)
thr.rbac_requires(roles=["admin"],
namespaces=[namespace],
grants=grants,
**kwargs)
try:
orig_obj = factory(kind)(name,
namespace=namespace,
volatile=True,
node=shared.NODE)
except:
orig_obj = None
try:
obj = factory(kind)(name,
namespace=namespace,
volatile=True,
cd=cd,
node=shared.NODE)
except Exception as exc:
errors.append("%s: unbuildable config: %s" % (path, exc))
return errors
if kind == "vol":
errors.append("%s: volume create requires the root privilege" %
path)
elif kind == "ccfg":
errors.append(
"%s: cluster config create requires the root privilege" % path)
elif kind == "svc":
groups = ["disk", "fs", "app", "share", "sync"]
for r in obj.get_resources(groups):
if r.rid == "sync#i0":
continue
errors.append("%s: resource %s requires the root privilege" %
(path, r.rid))
for r in obj.get_resources("task"):
if r.type not in ("task.podman", "task.docker"):
errors.append(
"%s: resource %s type %s requires the root privilege" %
(path, r.rid, r.type))
for r in obj.get_resources("container"):
if r.type not in ("container.podman", "container.docker"):
errors.append(
"%s: resource %s type %s requires the root privilege" %
(path, r.rid, r.type))
for r in obj.get_resources("ip"):
if r.type not in ("ip.cni"):
errors.append(
"%s: resource %s type %s requires the root privilege" %
(path, r.rid, r.type))
for section, sdata in cd.items():
rtype = cd[section].get("type")
errors += thr.rbac_create_data_section(path,
section,
rtype,
sdata,
grants,
obj,
orig_obj,
all_ns,
thr=thr)
return errors
def end_action(self, path, action, begin, end, cron, alogfile):
err = 'ok'
res = None
res_err = None
pid = None
msg = None
name, namespace, kind = split_path(path)
with open(alogfile, 'r') as ofile:
lines = ofile.read()
try:
os.unlink(alogfile)
except Exception:
pass
pids = set()
"""Example logfile line:
2009-11-11 01:03:25,252;;DISK.VG;;INFO;;unxtstsvc01_data is already up;;10200;;EOL
"""
vars = ["svcname",
"action",
"hostname",
"pid",
"begin",
"end",
"status_log",
"status",
"cron"]
vals = []
last = []
for line in lines.split(";;EOL\n"):
if line.count(";;") != 4:
continue
if ";;status_history;;" in line:
continue
date = line.split(";;")[0]
res_err = "ok"
date, res, lvl, msg, pid = line.split(";;")
res = res.lower().replace(rcEnv.nodename+"."+kind+"."+name, "").replace(rcEnv.nodename, "").lstrip(".")
res_action = res + " " + action
res_action = res_action.strip()
date = date.split(",")[0]
# database overflow protection
trim_lim = 10000
trim_tag = " <trimmed> "
trim_head = trim_lim // 2
trim_tail = trim_head-len(trim_tag)
if len(msg) > trim_lim:
msg = msg[:trim_head]+" <trimmed> "+msg[-trim_tail:]
pids |= set([pid])
if lvl is None or lvl == "DEBUG":
continue
elif lvl == "ERROR":
err = "err"
res_err = "err"
elif lvl == "WARNING" and err != "err":
err = "warn"
elif lvl == "WARNING" and res_err != "err":
res_err = "warn"
try:
if last:
if last[3] == pid and last[1] == res_action and last[7] == res_err:
last[6] += "\n"+msg
continue
else:
vals.append(last)
except Exception as exc:
print(exc)
continue
last = [
path,
res_action,
rcEnv.nodename,
pid,
date,
"",
msg,
res_err,
"1" if cron else "0"
]
if last:
vals.append(last)
if len(vals) > 0:
args = [vars, vals]
args += [(self.node.collector_env.uuid, rcEnv.nodename)]
self.proxy.res_action_batch(*args)
"""Complete the wrap-up database entry
"""
""" If logfile is empty, default to current process pid
"""
if len(pids) == 0:
pids = set([os.getpid()])
duration = datetime.strptime(end, "%Y-%m-%d %H:%M:%S") - \
datetime.strptime(begin, "%Y-%m-%d %H:%M:%S")
args = [
['svcname',
'action',
'hostname',
'pid',
'begin',
'end',
'time',
'status',
'cron'],
[str(path),
str(action),
str(rcEnv.nodename),
','.join(map(str, pids)),
begin,
end,
str(duration.seconds),
str(err),
'1' if cron else '0']
]
args += [(self.node.collector_env.uuid, rcEnv.nodename)]
self.proxy.end_action(*args)
def action(self, nodename, thr=None, **kwargs):
options = self.parse_options(kwargs)
name, namespace, kind = split_path(options.path)
if thr.get_service(options.path) is None and options.action not in (
"create", "deploy"):
thr.log_request("service action (%s not installed)" % options.path,
nodename,
lvl="warning",
**kwargs)
raise HTTP(404, "%s not found" % options.path)
if not options.action and not options.cmd:
thr.log_request("service action (no action set)",
nodename,
lvl="error",
**kwargs)
raise HTTP(400, "action not set")
for opt in ("node", "daemon", "svcs", "service", "s", "parm_svcs",
"local", "id"):
if opt in options.options:
del options.options[opt]
for opt, ropt in (("jsonpath_filter", "filter"), ):
if opt in options.options:
options.options[ropt] = options.options[opt]
del options.options[opt]
options.options["local"] = True
pmod = __import__(kind + "mgr_parser")
popt = pmod.OPT
def find_opt(opt):
for k, o in popt.items():
if o.dest == opt:
return o
if o.dest == "parm_" + opt:
return o
if options.cmd:
cmd = [options.cmd]
else:
cmd = [options.action]
for opt, val in options.options.items():
po = find_opt(opt)
if po is None:
continue
if val == po.default:
continue
if val is None:
continue
opt = po._long_opts[0] if po._long_opts else po._short_opts[0]
if po.action == "append":
cmd += [opt + "=" + str(v) for v in val]
elif po.action == "store_true" and val:
cmd.append(opt)
elif po.action == "store_false" and not val:
cmd.append(opt)
elif po.type == "string":
opt += "=" + val
cmd.append(opt)
elif po.type == "integer":
opt += "=" + str(val)
cmd.append(opt)
fullcmd = rcEnv.python_cmd + [
os.path.join(rcEnv.paths.pathlib, kind + "mgr.py"), "-s",
options.path
] + cmd
thr.log_request("run '%s'" % " ".join(fullcmd), nodename, **kwargs)
if options.sync:
proc = Popen(fullcmd,
stdout=PIPE,
stderr=PIPE,
stdin=None,
close_fds=True)
out, err = proc.communicate()
try:
result = json.loads(out)
except Exception:
result = {
"status": 0,
"data": {
"out": bdecode(out),
"err": bdecode(err),
"ret": proc.returncode,
},
}
else:
proc = Popen(fullcmd, stdin=None, close_fds=True)
thr.push_proc(proc)
result = {
"status": 0,
"info": "started %s action %s" % (options.path, " ".join(cmd)),
}
return result
