def init(self): if self.username is not None and self.password is not None: return s = "array#" + self.head try: stype = self.node.oget(s, "type") except Exception: raise ex.excError("no array configuration for head %s" % self.head) if stype != "nexenta": raise ex.excError("array %s type is not nexanta" % self.head) try: self.username = self.node.oget(s, "username") except Exception: raise ex.excError("no username information for head %s" % self.head) try: self.password = self.node.oget(s, "password") except Exception: raise ex.excError("no password information for head %s" % self.head) self.port = self.node.oget(s, "port") try: secname, namespace, _ = split_path(self.password) self.password = factory("sec")( secname, namespace=namespace, volatile=True).decode_key("password") except Exception as exc: raise ex.excError("error decoding password: %s" % exc) self.url = 'https://%(head)s:%(port)d/rest/nms/ <https://%(head)s:%(port)d/rest/nms/>' % dict( head=self.head, port=self.port)
def get_service_slaves(self, path, slaves=None): """ Recursive lookup of object slaves. """ if slaves is None: slaves = set() _, namespace, _ = split_path(path) def set_ns(path, parent_ns): name, _namespace, kind = split_path(path) if _namespace: return path else: return fmt_path(name, parent_ns, kind) for nodename in shared.CLUSTER_DATA: try: data = shared.CLUSTER_DATA[nodename]["services"]["status"][ path] except KeyError: continue slaves.add(path) new_slaves = set(data.get("slaves", [])) | set( data.get("scaler_slaves", [])) new_slaves = set( [set_ns(slave, namespace) for slave in new_slaves]) new_slaves -= slaves for slave in new_slaves: slaves |= self.get_service_slaves(slave, slaves) return slaves
def ca(self): capath = self.oget("DEFAULT", "ca") name, namespace, kind = split_path(capath) return factory("sec")(name, namespace=namespace, volatile=True, node=self.node)
def on_create(self): changes = [] has_ca = False if not self.oget("DEFAULT", "cn"): if self.namespace == "system": changes.append("cn=%s" % self.name) else: changes.append("cn=%s" % self.fullname) if self.namespace != "system": try: self.conf_get("DEFAULT", "validity") except ex.OptNotFound: changes.append("validity=%s" % DEFAULT_SACC_CERT_VALIDITY) grant = "guest:" + self.namespace changes.append("grant=%s" % grant) if not self.oget("DEFAULT", "ca"): capath = self.node.oget("cluster", "ca") if capath is None: capath = "system/sec/ca-" + self.node.cluster_name name, namespace, kind = split_path(capath) casec = factory("sec")(name, namespace="system", volatile=True, log=self.log) if casec.exists(): has_ca = True changes.append("ca=%s" % capath) else: print("no cluster CA defined. skip certificate generation.") if changes: self.set_multi(changes) if has_ca and "certificate" not in self.data_keys( ) and "private_key" in casec.data_keys(): self.gen_cert()
def format_service(path, idata, mon_data=None, discard_disabled=False, nodename=None): name, namespace, kind = split_path(path) svc_notice = get_svc_notice(idata) tree = Forest( separator=" ", widths=( (14, None), None, 10, None, ), ) node_name = tree.add_node() node_name.add_column(strip_path(path, os.environ.get("OSVC_NAMESPACE")), color.BOLD) node_name.add_column() if "cluster" in idata: node_name.add_column(idata["cluster"].get("avail", "n/a"), STATUS_COLOR[idata["cluster"].get("avail", "n/a")]) else: node_name.add_column() node_name.add_column(svc_notice) node_instances = node_name.add_node() node_instances.add_column("instances") add_instances(node_instances, path, nodename, mon_data) if nodename in service_nodes(path, mon_data): add_node_node(node_instances, nodename, idata, mon_data, discard_disabled=discard_disabled) add_parents(node_name, idata, mon_data, namespace) add_children(node_name, idata, mon_data, namespace) add_scaler_slaves(node_name, idata, mon_data, namespace) add_slaves(node_name, idata, mon_data, namespace) tree.out()
def svc_ptr_record(self, qname): if not qname.endswith(PTR_SUFFIX): return [] names = [] ref = ".".join(reversed(qname[:-PTR_SUFFIX_LEN].split("."))) with shared.CLUSTER_DATA_LOCK: for nodename, node in shared.CLUSTER_DATA.items(): status = node.get("services", {}).get("status", {}) for path, svc in status.items(): name, namespace, kind = split_path(path) if kind != "svc": continue if not namespace: namespace = "root" for rid, resource in status[path].get("resources", {}).items(): addr = resource.get("info", {}).get("ipaddr") if addr is None: continue if addr != ref: continue try: hostname = resource.get( "info", {}).get("hostname").split(".")[0].lower() except Exception: hostname = None gen_name = "%s.%s.%s.%s." % (name, namespace, kind, self.cluster_name) gen_name = gen_name.lower() if hostname and hostname != name: names.append("%s.%s" % (hostname, gen_name)) else: names.append(gen_name) return names
def rbac(self, nodename, thr=None, **kwargs): options = self.parse_options(kwargs) name, namespace, kind = split_path(options.path) if kind == "cfg": role = "guest" else: # sec, usr role = "admin" thr.rbac_requires(roles=[role], namespaces=[namespace], **kwargs)
def rbac(self, nodename, thr=None, **kwargs): options = self.parse_options(kwargs) if options.path: name, namespace, kind = split_path(options.path) thr.rbac_requires(roles=["operator"], namespaces=[namespace], **kwargs) else: thr.rbac_requires(roles=["operator"], namespaces="ANY", **kwargs)
def a_records(self): data = self.get_cache("a") if data is not None: return data names = {} for nodename in self.cluster_nodes: try: node = shared.CLUSTER_DATA[nodename] except KeyError: continue status = node.get("services", {}).get("status", {}) for path, svc in status.items(): name, namespace, kind = split_path(path) if kind != "svc": continue if namespace: namespace = namespace.lower() else: namespace = "root" scaler_slave = svc.get("scaler_slave") if scaler_slave: _name = name[name.index(".") + 1:] else: _name = name zone = "%s.%s.%s." % (namespace, kind, self.cluster_name) qname = "%s.%s" % (_name, zone) if qname not in names: names[qname] = set() for rid, resource in status.get(path, {}).get("resources", {}).items(): addr = resource.get("info", {}).get("ipaddr") if addr is None: continue hostname = resource.get("info", {}).get("hostname") names[qname].add(addr) rname = self.unique_name(addr) + "." + qname if rname not in names: names[rname] = set() names[rname].add(addr) if hostname: name = hostname.split(".")[0] + "." + qname if name not in names: names[name] = set() names[name].add(addr) for i, ip in enumerate(shared.NODE.dns): try: dns = "%s.%s." % (shared.NODE.dnsnodes[i].split(".")[0], self.cluster_name) names[dns] = set([ip]) except IndexError: self.log.warning("dns (%s) and dnsnodes (%s) are not aligned" "" % (shared.NODE.dns, shared.NODE.dnsnodes)) break self.set_cache("a", names) return names
def __init__(self, objects=[], node=None): self.objects = objects self.filtering = len(objects) > 0 if node: self.node = node else: self.node = Node() done = [] for s in self.node.conf_sections(cat="array"): try: name = self.node.oget(s, "name") except Exception: name = None if not name: name = s.split("#", 1)[-1] if name in done: continue if self.filtering and name not in self.objects: continue try: stype = self.node.oget(s, "type") except: continue if stype != "hds": continue try: bin = self.node.oget(s, 'bin') jre_path = self.node.oget(s, 'jre_path') url = self.node.oget(s, 'url') username = self.node.oget(s, 'username') password = self.node.oget(s, 'password') except Exception as exc: print("error parsing section %s: %s" % (s, exc), file=sys.stderr) continue try: secname, namespace, _ = split_path(password) password = factory("sec")(secname, namespace=namespace, volatile=True).decode_key("password") except Exception as exc: print("error decoding password: %s", exc, file=sys.stderr) continue self.arrays.append( Array(name, url, username, password, bin=bin, jre_path=jre_path, node=self.node)) done.append(name)
def __init__(self, name): self.node_flag = os.path.join(rcEnv.paths.pathvar, "node", "frozen") if name == "node": self.flag = self.node_flag else: name, namespace, kind = split_path(name) if namespace: self.flag = os.path.join(rcEnv.paths.pathvar, "namespaces", namespace, kind, name, "frozen") else: self.flag = os.path.join(rcEnv.paths.pathvar, kind, name, "frozen")
def __init__(self, objects=[], node=None): self.objects = objects if len(objects) > 0: self.filtering = True else: self.filtering = False self.arrays = [] if node: self.node = node else: self.node = Node() done = [] for s in self.node.conf_sections(cat="array"): name = s.split("#", 1)[-1] if name in done: continue if self.filtering and name not in self.objects: continue try: stype = self.node.oget(s, "type") except: continue if stype != "centera": continue try: server = self.node.oget(s, "server") username = self.node.oget(s, "username") password = self.node.oget(s, "password") jcass_dir = self.node.oget(s, "jcass_dir") java_bin = self.node.oget(s, "java_bin") except: print("error parsing section", s, file=sys.stderr) try: secname, namespace, _ = split_path(password) password = factory("sec")(secname, namespace=namespace, volatile=True).decode_key("password") except Exception as exc: print("error decoding password: %s", exc, file=sys.stderr) continue self.arrays.append( Centera(name, server=server, username=username, password=password, java_bin=java_bin, jcass_dir=jcass_dir, node=self.node)) done.append(name)
def rbac(self, nodename, thr=None, **kwargs): options = self.parse_options(kwargs) name, namespace, kind = split_path(options.path) if options.action in GUEST_ACTIONS: role = "guest" elif options.action in OPERATOR_ACTIONS: role = "operator" elif options.action in ADMIN_ACTIONS: role = "admin" else: role = "root" if options.action == "set": # load current config try: cf = shared.SERVICES[options.path].print_config_data() except Exception as exc: cf = {} # purge unwanted sections try: del cf["metadata"] except Exception: pass # merge changes in current config for buff in options.options.get("kw", []): k, v = buff.split("=", 1) if k[-1] in ("+", "-"): k = k[:-1] k = k.strip() try: s, k = k.split(".", 1) except Exception: s = "DEFAULT" if s not in cf: cf[s] = {} cf[s][k] = v # apply object create rbac to the amended config payload = {options.path: cf} errors = self.rbac_create_data(payload, thr=thr, **kwargs) if errors: raise HTTP(403, errors) else: thr.rbac_requires(roles=[role], namespaces=[namespace], **kwargs) if options.cmd: # compat, requires root kwargs["roles"] = ["root"] thr.rbac_requires(**kwargs)
def __init__(self, objects=[], node=None): self.objects = objects if len(objects) > 0: self.filtering = True else: self.filtering = False self.arrays = [] if node: self.node = node else: self.node = Node() done = [] for s in self.node.conf_sections(cat="array"): name = s.split("#", 1)[-1] if name in done: continue if self.filtering and name not in self.objects: continue try: stype = self.node.oget(s, "type") except: continue if stype != "emcvnx": continue try: method = self.node.oget(s, "method") scope = self.node.oget(s, "scope") spa = self.node.oget(s, "spa") spb = self.node.oget(s, "spb") username = self.node.oget(s, "username") password = self.node.oget(s, "password") except Exception as exc: print("error parsing section %s: %s" % (s, exc), file=sys.stderr) continue if method == "credentials": if username is None or password is None: print("error parsing section %s: username and password are mandatory" % s, file=sys.stderr) continue try: secname, namespace, _ = split_path(password) password = factory("sec")(secname, namespace=namespace, volatile=True).decode_key("password") except Exception as exc: print("error decoding password: %s", exc, file=sys.stderr) continue self.arrays.append(EmcVnx(name, method, scope, spa, spb, username=username, password=password, node=self.node)) done.append(name)
def dispatch_svcs(paths): data = {} for path in paths: try: _, _, kind = split_path(path) except ValueError: continue try: validate_kind(kind) except ValueError as exc: raise ex.excError(str(exc)) try: data[kind].append(path) except KeyError: data[kind] = [path] return data
def postinstall(self, key=None): """ Refresh installed keys """ for path in self.node.svcs_selector("*/svc/*", namespace=self.namespace, local=True): name, _, _ = split_path(path) svc = factory("svc")(name, namespace=self.namespace, volatile=True, node=self.node, log=self.log) for vol in svc.get_resources("volume"): if vol.has_data(self.kind, self.path, key) and vol._status() == rcStatus.UP: vol._install_data(self.kind)
def __init__(self, objects=[], node=None): self.objects = objects self.filtering = len(objects) > 0 if node: self.node = node else: self.node = Node() done = [] for s in self.node.conf_sections(cat="array"): name = s.split("#", 1)[-1] if name in done: continue try: stype = self.node.oget(s, "type") except: continue if stype != "eva": continue try: manager = self.node.oget(s, 'manager') username = self.node.oget(s, 'username') password = self.node.oget(s, 'password') sssubin = self.node.oget(s, 'bin') except Exception as exc: print("error parsing section %s: %s" % (s, exc), file=sys.stderr) pass try: secname, namespace, _ = split_path(password) password = factory("sec")(secname, namespace=namespace, volatile=True).decode_key("password") except Exception as exc: print("error decoding password: %s", exc, file=sys.stderr) continue out, err = sssu('ls system', manager, username, password, sssubin=sssubin) _in = False for line in out.split('\n'): if 'Systems avail' in line: _in = True continue if not _in: continue name = line.strip() if self.filtering and name not in self.objects: continue self.arrays.append(Eva(name, manager, username, password, sssubin=sssubin)) done.append(name)
def __init__(self, objects=[], node=None): self.objects = objects self.filtering = len(objects) > 0 self.timeout = 10 if node: self.node = node else: self.node = Node() done = [] for s in self.node.conf_sections(cat="array"): try: name = self.node.oget(s, 'name') except Exception: name = None if not name: name = s.split("#", 1)[-1] if name in done: continue if self.filtering and name not in self.objects: continue try: stype = self.node.oget(s, "type") except: continue if stype != "dorado": continue timeout = self.node.oget(s, "timeout") try: username = self.node.oget(s, "username") password = self.node.oget(s, "password") api = self.node.oget(s, "api") except: print("error parsing section", s, file=sys.stderr) continue try: secname, namespace, _ = split_path(password) password = factory("sec")(secname, namespace=namespace, volatile=True).decode_key("password") except Exception as exc: print("error decoding password: %s", exc, file=sys.stderr) continue self.arrays.append( Dorado(name, api, username, password, timeout, node=self.node)) done.append(name)
def _data_status(self, kind): for data in self.data_data(kind): name, _, kind = split_path(data["obj"]) obj = factory(kind)(name, namespace=self.svc.namespace, volatile=True, node=self.svc.node) if not obj.exists(): self.status_log( "referenced %s %s does not exist: " "expected data %s can not be installed in the volume" % (kind, name, data["key"]), "warn") continue keys = obj.resolve_key(data["key"]) if not keys and not is_glob(data["key"]): self.status_log( "%s %s has no key %s. " "expected data can not be installed in the volume" % (kind, name, data["key"]), "warn")
def action(self, nodename, thr=None, **kwargs): """ Care with locks """ thr.log_request("shutdown daemon", nodename, **kwargs) with shared.THREADS_LOCK: shared.THREADS["scheduler"].stop() mon = shared.THREADS["monitor"] if thr.stopped() or shared.NMON_DATA.status == "shutting": thr.log.info("already shutting") # wait for service shutdown to finish before releasing the dup client while True: if mon._shutdown: break time.sleep(0.3) return {"status": 0} try: thr.set_nmon("shutting") mon.kill_procs() for path in shared.SMON_DATA: _, _, kind = split_path(path) if kind not in ("svc", "vol"): continue thr.set_smon(path, local_expect="shutdown") self.wait_shutdown() # send a last status to peers so they can takeover asap mon.update_hb_data() mon._shutdown = True shared.wake_monitor("services shutdown done") except Exception as exc: thr.log.exception(exc) thr.log.info("services are now shutdown") while True: with shared.THREADS_LOCK: if not shared.THREADS["monitor"].is_alive(): break time.sleep(0.3) shared.DAEMON_STOP.set() return {"status": 0}
def rbac(self, nodename, thr=None, **kwargs): options = self.parse_options(kwargs) name, namespace, kind = split_path(options.path) role = "admin" operator = ( # (local_expect, global_expect, reset_retries) (None, None, True), (None, "thawed", False), (None, "frozen", False), (None, "started", False), (None, "stopped", False), (None, "aborted", False), (None, "placed", False), (None, "shutdown", False), ) _global_expect = options.global_expect.split( "@")[0] if options.global_expect else options.global_expect if (options.local_expect, _global_expect, options.reset_retries) in operator: role = "operator" thr.rbac_requires(roles=[role], namespaces=[namespace], **kwargs)
def __init__(self, objects=[], node=None): self.objects = objects self.filtering = len(objects) > 0 self.arrays = [] if node: self.node = node else: self.node = Node() done = [] for s in self.node.conf_sections(cat="array"): name = s.split("#", 1)[-1] if name in done: continue if self.filtering and name not in self.objects: continue try: stype = self.node.oget(s, "type") except: continue if stype != "netapp": continue kwargs = {"node": self.node} for key in ("server", "username", "key"): try: kwargs[key] = self.node.oget(s, key) except: print("missing parameter: %s", s) if "server" not in kwargs or "username" not in kwargs or "key" not in kwargs: continue try: secname, namespace, _ = split_path(kwargs["password"]) kwargs["password"] = factory("sec")( secname, namespace=namespace, volatile=True).decode_key("password") except Exception as exc: print("error decoding password: %s", exc, file=sys.stderr) continue self.arrays.append(Netapp(s, **kwargs))
def _install_data(self, kind): for data in self.data_data(kind): name, _, kind = split_path(data["obj"]) obj = factory(kind)(name, namespace=self.svc.namespace, volatile=True, node=self.svc.node) if not obj.exists(): self.log.warning( "referenced %s %s does not exist: " "expected data %s can not be installed in the volume", kind, name, data["key"]) continue keys = obj.resolve_key(data["key"]) if not keys and "*" not in data["key"] and "?" not in data["key"]: self.log.warning( "%s %s has no key %s. " "expected data can not be installed in the volume", kind, name, data["key"]) continue for key in obj.resolve_key(data["key"]): obj._install(key, data["path"])
def configure_volume(self, volume, size=None, fmt=True, access="rwo", shared=False, nodes=None, env=None): if self.template is None: raise ex.excError("pool#%s.template is not set" % self.name) if not is_service(self.template): raise ex.excError("%s template volume not found" % self.template) name = self.default_disk_name(volume) tname, tnamespace, tkind = split_path(self.template) if tkind != "vol": raise ex.excError("%s template kind is not vol") svc = factory(tkind)(tname, tnamespace, volatile=True, node=self.node) config = svc.print_config_data() try: del config["DEFAULT"]["disable"] except KeyError: pass if "DEFAULT" not in config: config["DEFAULT"] = {} if "env" not in config: config["env"] = {} config["DEFAULT"]["pool"] = self.name config["DEFAULT"]["access"] = access if access in ("rox", "rwx"): config["DEFAULT"]["topology"] = "flex" config["DEFAULT"]["flex_min"] = 0 if nodes: config["DEFAULT"]["nodes"] = nodes config["env"]["size"] = size if env: config["env"].update(env) self.node.install_svc_conf_from_data(volume.name, volume.namespace, volume.kind, config)
def gen_cert(self): data = {} for key in ("cn", "c", "st", "l", "o", "ou", "email", "alt_names", "bits", "validity", "ca"): val = self.oget("DEFAULT", key) if val is not None: data[key] = val ca = data.get("ca") casec = None if ca is not None: casecname, canamespace, _ = split_path(ca) casec = factory("sec")(casecname, namespace=canamespace, log=self.log, volatile=True) if not casec.exists(): raise ex.excError("ca secret %s does not exist" % ca) for key in ("crt", "key", "csr"): data[key] = self.tempfilename() if "alt_names" in data: data["cnf"] = self.tempfilename() try: if casec: for key, kw in (("cacrt", "certificate"), ("cakey", "private_key")): if kw not in casec.data_keys(): continue data[key] = self.tempfilename() buff = bdecode(casec.decode_key(kw)) with open(data[key], "w") as ofile: ofile.write(buff) gen_cert(log=self.log, **data) self._add("private_key", value_from=data["key"]) if data.get("crt") is not None: self._add("certificate", value_from=data["crt"]) if data.get("csr") is not None: self._add("certificate_signing_request", value_from=data["csr"]) if data.get("cakey") is None: self._add("certificate_chain", value_from=data["crt"]) else: # merge cacrt and crt chain = self.tempfilename() try: with open(data["crt"], "r") as ofile: buff = ofile.read() with open(data["cacrt"], "r") as ofile: buff += ofile.read() with open(chain, "w") as ofile: ofile.write(buff) self._add("certificate_chain", value_from=chain) finally: try: os.unlink(chain) except Exception: pass self.add_key("fullpem", self._fullpem()) finally: for key in ("crt", "key", "cacrt", "cakey", "csr", "cnf"): if key not in data: continue try: os.unlink(data[key]) except Exception: pass
def format_cluster(paths=None, node=None, data=None, prev_stats_data=None, stats_data=None, sections=None, selector=None, namespace=None): if not data or data.get("status", 0) != 0: return if sections is None: sections = DEFAULT_SECTIONS out = [] nodenames = get_nodes(data) nodenames = sorted([n for n in nodenames if n in node]) show_nodenames = abbrev([n for n in nodenames if n in node]) services = {} def load_header(title=""): if isinstance(title, list): line = title else: line = [ title, "", "", "", "", "", "", "", "", "", "", "", "", ] for nodename in show_nodenames: line.append(colorize(nodename, color.BOLD)) out.append(line) def load_svc(path, prefix=""): if path not in services: return data = services[path] if path in slave_parents and prefix == "": return try: topology = services[path].topology except KeyError: topology = "" if services[path].get("drp", False): topology = "drp " + topology # status status = colorize_status(data["avail"], lpad=0) if data["overall"] == "warn": status += colorize("!", color.BROWN) if data["placement"] == "non-optimal": status += colorize("^", color.RED) # info info = { "topology": data.get("topology", ""), "orchestrate": data.get("orchestrate", "-"), "status": "%d/1" % data["n_up"] if data["n_up"] is not None else 0, } if data.get("scale") is not None: info["status"] = "%d/%d" % (data["n_up"], data.get("scale")) elif data.get("wrapper"): info = { "topology": "", "orchestrate": "", "status": "", } elif topology == "flex": info["status"] = "%d/%d" % (data["n_up"], data["flex_target"]) if data["avail"] == "n/a": info["status"] = "" info = "%(orchestrate)-5s %(status)-5s" % info line = [ " " + colorize( prefix + strip_path(path, os.environ.get("OSVC_NAMESPACE")), color.BOLD), status, info, fmt_svc_uptime(path, stats_data), fmt_svc_tasks(path, prev_stats_data), fmt_svc_cpu_usage(path, prev_stats_data, stats_data), fmt_svc_cpu_time(path, stats_data), fmt_svc_mem_total(path, stats_data), fmt_svc_blk_rb(path, stats_data), fmt_svc_blk_wb(path, stats_data), fmt_svc_blk_rbps(path, prev_stats_data, stats_data), fmt_svc_blk_wbps(path, prev_stats_data, stats_data), "|" if nodenames else "", ] for nodename in nodenames: if nodename not in data["nodes"]: line.append("") elif data["nodes"][nodename] is None: line.append(colorize("?", color.RED)) elif data["nodes"][nodename] is not None: val = [] # frozen unicon if data["nodes"][nodename]["frozen"]: frozen_icon = colorize(unicons["frozen"], color.BLUE) else: frozen_icon = "" # avail status unicon if data["wrapper"]: avail_icon = "" else: avail = data["nodes"][nodename]["avail"] if avail == "unknown": avail_icon = colorize("?", color.RED) else: avail_icon = colorize_status( avail, lpad=0, agg_status=data["avail"]).replace( avail, unicons[avail]) # overall status unicon if data["wrapper"]: overall_icon = "" else: overall = data["nodes"][nodename]["overall"] if overall == "warn": overall_icon = colorize_status( overall, lpad=0).replace(overall, unicons[overall]) else: overall_icon = "" # mon status smon = data["nodes"][nodename]["mon"] if smon == "idle": # don't display 'idle', as its to normal status and thus repeated as nauseam smon = "" else: smon = " " + smon # global expect if smon == "": global_expect = data["nodes"][nodename]["global_expect"] if global_expect: global_expect = colorize(" >" + str(global_expect), color.LIGHTBLUE) else: global_expect = "" else: global_expect = "" # leader if data["wrapper"]: leader = "" else: if data["nodes"][nodename]["placement"] == "leader": leader = colorize("^", color.LIGHTBLUE) else: leader = "" # provisioned if data["nodes"][nodename].get("provisioned") is False: provisioned = colorize("P", color.RED) else: provisioned = "" val.append(avail_icon) val.append(overall_icon) val.append(leader) val.append(frozen_icon) val.append(provisioned) val.append(smon) val.append(global_expect) line.append("".join(val)) out.append(line) for child in sorted(list(data.get("slaves", []))): load_svc(child, prefix=prefix + " ") def load_hb(key, _data): state = _data.get("state", "") if state == "running": state = colorize(state, color.GREEN) else: state = colorize(state, color.RED) if _data.get("alerts"): state += colorize("!", color.BROWN) cf = _data.get("config", {}) addr = cf.get("addr", "") port = cf.get("port", "") dev = cf.get("dev", "") relay = cf.get("relay", "") if addr and port: config = addr + ":" + str(port) elif dev: config = os.path.basename(dev) elif relay: config = relay else: config = "" line = [ " " + colorize(key, color.BOLD), state, config, fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", "|" if nodenames else "", ] peers = _data.get("peers", {}) for nodename in nodenames: beating = peers.get(nodename, {}).get("beating") if beating is None: status = "n/a" elif beating: status = "up" else: status = "down" status = colorize_status(status, lpad=0).replace(status, unicons[status]) line.append(status) out.append(line) def load_monitor(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) transitions = _data.get("transitions", 0) if transitions: status = "%d transition" % transitions else: status = "" out.append(( " " + colorize(key, color.BOLD), state, status, fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", )) def load_listener(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) out.append(( " " + colorize(key, color.BOLD), state, _data["config"]["addr"] + ":" + str(_data["config"]["port"]), fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", )) def load_scheduler(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) out.append(( " " + colorize(key, color.BOLD), state, "", fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", )) def load_daemon(): key = "daemon" state = colorize("running", color.GREEN) line = [ " " + colorize(key, color.BOLD), "%s" % state, "", str(data.get("pid", "")) if stats_data else "", "", fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), fmt_thr_mem_total(key, stats_data), "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: speaker = data["monitor"].get("nodes", {}).get(nodename, {}).get("speaker") if speaker: status = "up" status = colorize_status(status, lpad=0).replace( status, unicons[status]) else: status = "" line.append(status) out.append(line) def load_collector(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) line = [ " " + colorize(key, color.BOLD), state, "", fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: speaker = data["monitor"].get("nodes", {}).get(nodename, {}).get("speaker") if speaker: status = "up" status = colorize_status(status, lpad=0).replace( status, unicons[status]) else: status = "" line.append(status) out.append(line) def load_generic_thread(key, _data): if _data["state"] == "running": state = colorize(_data["state"], color.GREEN) else: state = colorize(_data["state"], color.RED) out.append(( " " + colorize(key, color.BOLD), state, "", fmt_tid(_data, stats_data), fmt_thr_tasks(key, stats_data), fmt_thr_cpu_usage(key, prev_stats_data, stats_data), fmt_thr_cpu_time(key, stats_data), "", "", "", )) def load_threads(): if "threads" not in sections: return load_header([ "Threads", "", "", "pid/tid" if stats_data else "", "thr/sub" if stats_data else "", "usage" if stats_data else "", "time" if stats_data else "", "rss" if stats_data else "", "", "", "", "", "", ]) load_daemon() for key in sorted([key for key in data if key != "cluster"]): if key.startswith("hb#"): load_hb(key, data[key]) elif key == "monitor": load_monitor(key, data[key]) elif key == "scheduler": load_scheduler(key, data[key]) elif key == "listener": load_listener(key, data[key]) elif key == "collector": load_collector(key, data[key]) else: try: load_generic_thread(key, data[key]) except Exception: pass out.append([]) def load_score(): if "monitor" not in data: return line = [ colorize(" score", color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: line.append( str(data["monitor"]["nodes"].get(nodename, {}).get("stats", {}).get("score", ""))) out.append(line) def load_loadavg(): if "monitor" not in data: return line = [ colorize(" load 15m", color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: line.append( str(data["monitor"]["nodes"].get(nodename, {}).get( "stats", {}).get("load_15m", ""))) out.append(line) def load_free_total(key): if "monitor" not in data: return line = [ colorize(" " + key, color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: total = data["monitor"]["nodes"].get(nodename, {}).get( "stats", {}).get(key + "_total") avail = data["monitor"]["nodes"].get(nodename, {}).get( "stats", {}).get(key + "_avail") limit = 100 - data["monitor"]["nodes"].get(nodename, {}).get( "min_avail_" + key, 0) if avail is None or total in (0, None): line.append("-") continue usage = 100 - avail total = print_size(total, unit="MB", compact=True) if limit: cell = "%d/%d%%:%s" % (usage, limit, total) else: cell = "%d%%:%s" % (usage, total) if usage > limit: cell = colorize(cell, color.RED) line.append(cell) out.append(line) def load_node_state(): if "monitor" not in data: return line = [ colorize(" state", color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: nmon_state = data["monitor"]["nodes"].get(nodename, {}).get( "monitor", {}).get("status", "") if nmon_state == "idle": nmon_state = "" if data["monitor"]["nodes"].get(nodename, {}).get("frozen", ""): frozen = frozen_icon = colorize(unicons["frozen"], color.BLUE) else: frozen = "" global_expect = data["monitor"]["nodes"].get(nodename, {}).get( "monitor", {}).get("global_expect") if global_expect: global_expect = colorize(" >" + str(global_expect), color.LIGHTBLUE) else: global_expect = "" line.append(str(nmon_state) + frozen + global_expect) out.append(line) def load_node_compat(): if "monitor" not in data: return if data["monitor"].get("compat") is True: # no need to clutter if the situation is normal return line = [ colorize(" compat", color.BOLD), colorize("warn", color.BROWN), "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: compat = data["monitor"]["nodes"].get(nodename, {}).get("compat", "") line.append(str(compat)) out.append(line) def load_node_version(): if "monitor" not in data: return line = [ colorize(" version", color.BOLD), colorize("warn", color.BROWN), "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] versions = [] for nodename in nodenames: agent = data["monitor"]["nodes"].get(nodename, {}).get("agent", "") line.append(str(agent)) if agent != "": versions.append(str(agent)) if len(set(versions)) > 1: out.append(line) def load_arbitrators(): if "arbitrators" not in sections: return arbitrators = [] arbitrators_name = {} for nodename, ndata in data["monitor"]["nodes"].items(): for aid, adata in ndata.get("arbitrators", {}).items(): if aid not in arbitrators: arbitrators.append(aid) arbitrators_name[aid] = adata["name"] if len(arbitrators) == 0: return load_header("Arbitrators") for aid in arbitrators: line = [ colorize(" " + arbitrators_name[aid], color.BOLD), "", "", "", "", "", "", "", "", "", "", "", "|" if nodenames else "", ] for nodename in nodenames: status = data["monitor"]["nodes"].get(nodename, {}).get( "arbitrators", {}).get(aid, {}).get("status", "undef") if status != "up": line[1] = colorize_status("warn", lpad=0) status = colorize_status(status, lpad=0).replace( status, unicons[status]) line.append(status) out.append(line) out.append([]) def load_nodes(): if "nodes" not in sections or not nodenames: return load_header("Nodes") load_metrics() load_node_compat() load_node_version() load_node_state() out.append([]) def load_metrics(): load_score() load_loadavg() load_free_total("mem") load_free_total("swap") # init the services hash slave_parents = {} if "monitor" in data: for _node in nodenames: if _node not in data["monitor"]["nodes"]: continue try: node_svc_status = data["monitor"]["nodes"][_node]["services"][ "status"] except KeyError: continue for path, _data in node_svc_status.items(): if _data is None: continue if paths is not None and path not in paths: continue if path not in services: services[path] = Storage({ "drp": _data.get("drp", False), "topology": _data.get("topology", ""), "orchestrate": _data.get("orchestrate", "-"), "flex_target": _data.get("flex_target"), "scale": _data.get("scale"), "avail": "undef", "overall": "", "nodes": {}, "slaves": set(), "n_up": 0, "resources": set(), }) try: services[path]["resources"] |= set( _data["resources"].keys()) except KeyError: pass slaves = _data.get("slaves", []) scale = _data.get("scale") if scale: name, _namespace, kind = split_path(path) if _namespace: pattern = "^%s/%s/[0-9]+\.%s$" % (_namespace, kind, name) else: pattern = "^[0-9]+\.%s$" % name for child in data["monitor"]["services"]: if re.match(pattern, child) is None: continue slaves.append(child) if node_svc_status.get(child, {}).get("avail") == "up": services[path].n_up += 1 else: if node_svc_status.get(path, {}).get("avail") == "up": services[path].n_up += 1 for child in slaves: if child not in slave_parents: slave_parents[child] = set([path]) else: slave_parents[child] |= set([path]) global_expect = _data["monitor"].get("global_expect") if global_expect and "@" in global_expect: global_expect = global_expect[:global_expect.index("@") + 1] services[path].nodes[_node] = { "avail": _data.get("avail", "undef"), "overall": _data.get("overall", "undef"), "frozen": _data.get("frozen", False), "mon": _data["monitor"].get("status", ""), "global_expect": global_expect, "placement": _data["monitor"].get("placement", ""), "provisioned": _data.get("provisioned"), } services[path].slaves |= set(slaves) services[path]["wrapper"] = (services[path].resources == set() and services[path].slaves != set() and scale is None) try: # hint we have missing instances for path, cnf in data["monitor"]["nodes"][_node]["services"][ "config"].items(): if path not in services: continue for __node in cnf.get("scope", []): if __node not in services[path].nodes: services[path].nodes[__node] = None except KeyError: pass for path, _data in data["monitor"]["services"].items(): if paths is not None and path not in paths: continue if path not in services: services[path] = Storage({ "avail": "undef", "overall": "", "nodes": {} }) services[path].avail = _data.get("avail", "n/a") services[path].overall = _data.get("overall", "n/a") services[path].placement = _data.get("placement", "n/a") def load_services(selector, namespace=None): if "services" not in sections: return selectors = [] context = os.environ.get("OSVC_CONTEXT", "") if context: selectors.append(context) buff = format_path_selector(selector, namespace) if len(buff) > 15: buff = buff[:12] + "..." selectors.append(buff) load_header([ "/".join(selectors), "", "", "since" if stats_data else "", "tasks" if stats_data else "", "usage" if stats_data else "", "time" if stats_data else "", "mem" if stats_data else "", "blkrb" if stats_data else "", "blkwb" if stats_data else "", "blkrbps" if stats_data else "", "blkwbps" if stats_data else "", "", ]) for path in sorted(list(services.keys())): load_svc(path) # load data in lists load_threads() load_arbitrators() load_nodes() load_services(selector, namespace) # print tabulated lists return print_section(out)
def set_ns(path, parent_ns): name, _namespace, kind = split_path(path) if _namespace: return path else: return fmt_path(name, parent_ns, kind)
def rbac_create_obj(self, path, cd, all_ns, thr=None, **kwargs): errors = [] name, namespace, kind = split_path(path) grants = thr.user_grants(all_ns | set([namespace])) if namespace not in all_ns: if namespace == "system": errors.append( "%s: create the new namespace system requires the root cluster role" ) return errors elif "squatter" not in grants: errors.append( "%s: create the new namespace %s requires the squatter cluster role" % (path, namespace)) return errors elif namespace not in grants["admin"]: thr.usr.set_multi(["grant+=admin:%s" % namespace]) grants["admin"].add(namespace) thr.rbac_requires(roles=["admin"], namespaces=[namespace], grants=grants, **kwargs) try: orig_obj = factory(kind)(name, namespace=namespace, volatile=True, node=shared.NODE) except: orig_obj = None try: obj = factory(kind)(name, namespace=namespace, volatile=True, cd=cd, node=shared.NODE) except Exception as exc: errors.append("%s: unbuildable config: %s" % (path, exc)) return errors if kind == "vol": errors.append("%s: volume create requires the root privilege" % path) elif kind == "ccfg": errors.append( "%s: cluster config create requires the root privilege" % path) elif kind == "svc": groups = ["disk", "fs", "app", "share", "sync"] for r in obj.get_resources(groups): if r.rid == "sync#i0": continue errors.append("%s: resource %s requires the root privilege" % (path, r.rid)) for r in obj.get_resources("task"): if r.type not in ("task.podman", "task.docker"): errors.append( "%s: resource %s type %s requires the root privilege" % (path, r.rid, r.type)) for r in obj.get_resources("container"): if r.type not in ("container.podman", "container.docker"): errors.append( "%s: resource %s type %s requires the root privilege" % (path, r.rid, r.type)) for r in obj.get_resources("ip"): if r.type not in ("ip.cni"): errors.append( "%s: resource %s type %s requires the root privilege" % (path, r.rid, r.type)) for section, sdata in cd.items(): rtype = cd[section].get("type") errors += thr.rbac_create_data_section(path, section, rtype, sdata, grants, obj, orig_obj, all_ns, thr=thr) return errors
def end_action(self, path, action, begin, end, cron, alogfile): err = 'ok' res = None res_err = None pid = None msg = None name, namespace, kind = split_path(path) with open(alogfile, 'r') as ofile: lines = ofile.read() try: os.unlink(alogfile) except Exception: pass pids = set() """Example logfile line: 2009-11-11 01:03:25,252;;DISK.VG;;INFO;;unxtstsvc01_data is already up;;10200;;EOL """ vars = ["svcname", "action", "hostname", "pid", "begin", "end", "status_log", "status", "cron"] vals = [] last = [] for line in lines.split(";;EOL\n"): if line.count(";;") != 4: continue if ";;status_history;;" in line: continue date = line.split(";;")[0] res_err = "ok" date, res, lvl, msg, pid = line.split(";;") res = res.lower().replace(rcEnv.nodename+"."+kind+"."+name, "").replace(rcEnv.nodename, "").lstrip(".") res_action = res + " " + action res_action = res_action.strip() date = date.split(",")[0] # database overflow protection trim_lim = 10000 trim_tag = " <trimmed> " trim_head = trim_lim // 2 trim_tail = trim_head-len(trim_tag) if len(msg) > trim_lim: msg = msg[:trim_head]+" <trimmed> "+msg[-trim_tail:] pids |= set([pid]) if lvl is None or lvl == "DEBUG": continue elif lvl == "ERROR": err = "err" res_err = "err" elif lvl == "WARNING" and err != "err": err = "warn" elif lvl == "WARNING" and res_err != "err": res_err = "warn" try: if last: if last[3] == pid and last[1] == res_action and last[7] == res_err: last[6] += "\n"+msg continue else: vals.append(last) except Exception as exc: print(exc) continue last = [ path, res_action, rcEnv.nodename, pid, date, "", msg, res_err, "1" if cron else "0" ] if last: vals.append(last) if len(vals) > 0: args = [vars, vals] args += [(self.node.collector_env.uuid, rcEnv.nodename)] self.proxy.res_action_batch(*args) """Complete the wrap-up database entry """ """ If logfile is empty, default to current process pid """ if len(pids) == 0: pids = set([os.getpid()]) duration = datetime.strptime(end, "%Y-%m-%d %H:%M:%S") - \ datetime.strptime(begin, "%Y-%m-%d %H:%M:%S") args = [ ['svcname', 'action', 'hostname', 'pid', 'begin', 'end', 'time', 'status', 'cron'], [str(path), str(action), str(rcEnv.nodename), ','.join(map(str, pids)), begin, end, str(duration.seconds), str(err), '1' if cron else '0'] ] args += [(self.node.collector_env.uuid, rcEnv.nodename)] self.proxy.end_action(*args)
def action(self, nodename, thr=None, **kwargs): options = self.parse_options(kwargs) name, namespace, kind = split_path(options.path) if thr.get_service(options.path) is None and options.action not in ( "create", "deploy"): thr.log_request("service action (%s not installed)" % options.path, nodename, lvl="warning", **kwargs) raise HTTP(404, "%s not found" % options.path) if not options.action and not options.cmd: thr.log_request("service action (no action set)", nodename, lvl="error", **kwargs) raise HTTP(400, "action not set") for opt in ("node", "daemon", "svcs", "service", "s", "parm_svcs", "local", "id"): if opt in options.options: del options.options[opt] for opt, ropt in (("jsonpath_filter", "filter"), ): if opt in options.options: options.options[ropt] = options.options[opt] del options.options[opt] options.options["local"] = True pmod = __import__(kind + "mgr_parser") popt = pmod.OPT def find_opt(opt): for k, o in popt.items(): if o.dest == opt: return o if o.dest == "parm_" + opt: return o if options.cmd: cmd = [options.cmd] else: cmd = [options.action] for opt, val in options.options.items(): po = find_opt(opt) if po is None: continue if val == po.default: continue if val is None: continue opt = po._long_opts[0] if po._long_opts else po._short_opts[0] if po.action == "append": cmd += [opt + "=" + str(v) for v in val] elif po.action == "store_true" and val: cmd.append(opt) elif po.action == "store_false" and not val: cmd.append(opt) elif po.type == "string": opt += "=" + val cmd.append(opt) elif po.type == "integer": opt += "=" + str(val) cmd.append(opt) fullcmd = rcEnv.python_cmd + [ os.path.join(rcEnv.paths.pathlib, kind + "mgr.py"), "-s", options.path ] + cmd thr.log_request("run '%s'" % " ".join(fullcmd), nodename, **kwargs) if options.sync: proc = Popen(fullcmd, stdout=PIPE, stderr=PIPE, stdin=None, close_fds=True) out, err = proc.communicate() try: result = json.loads(out) except Exception: result = { "status": 0, "data": { "out": bdecode(out), "err": bdecode(err), "ret": proc.returncode, }, } else: proc = Popen(fullcmd, stdin=None, close_fds=True) thr.push_proc(proc) result = { "status": 0, "info": "started %s action %s" % (options.path, " ".join(cmd)), } return result