def aggregate_status(service, check): """Aggregate the check on any services starting with service.""" if not OTHER_SHIELDS: intra_shield(verify=True) aggregates = [] if OTHER_SHIELDS: for server, services in all_other_services().items(): for _service, checks in services.items(): if _service.startswith(service) and check in checks: res = requests.get( "https://{}/_/{}/{}/?raw".format( server, _service, check, ), headers=intra_shield(), ) try: res.raise_for_status() except: continue else: aggregates.append(res.json()) for _service in all_services(): if _service.startswith(service): aggregates.append(do_check(_service, check, raw=True)) sum_total = {"label": check, "status": None, "color": ""} sum_health = {"ready": 0, "total": 0, "restarts": 0} for aggregate in aggregates: if isinstance(aggregate, dict): this_color = aggregate.get("color") if this_color and sum_total["color"] not in ("red", "yellow"): sum_total["color"] = this_color elif this_color == "red" and sum_total["color"] != "red": sum_total["color"] = this_color if check == "health": sum_health["ready"] += aggregate["ready"] sum_health["total"] += aggregate["total"] sum_health["restarts"] += aggregate["restarts"] else: if not sum_total["status"]: sum_total["status"] = aggregate.get("status", "") else: sum_total["status"] += aggregate.get("status", "") if check == "health": sum_total["status"] = "{}/{} ({} restart{})".format( sum_health["ready"], sum_health["total"], sum_health["restarts"], "s" * int(sum_health["restarts"] != 1)) sum_total["color"] = sum_total["color"] or ( "green" if sum_health["ready"] == sum_health["total"] else "red") return redirect(as_redirect_url(**sum_total))
def get_services_checks(service): """Returns a list of checks available for the service.""" intra_shield(verify=True) if service in all_services(): return as_json(services_checks(service)) else: abort(404)
def get_services_checks(service): """Returns a list of checks available for the service.""" if not OTHER_SHIELDS: intra_shield(verify=True) if service in all_services() + all_snowflakes(): return as_json(services_checks(service)) else: abort(404)
def get_all_statuses(): """Main page, lists all services/clusters if master server.""" if OTHER_SHIELDS: all_statuses = {"local": {}} # first do our own services for service in all_services(): all_statuses["local"][service] = {} for check in services_checks(service): all_statuses["local"][service][check] = check_to_redirect( service, check ) all_statuses["local"][service] = [ (k, all_statuses["local"][service][k]) for k in sorted(all_statuses["local"][service]) ] all_statuses["local"] = [(k, all_statuses["local"][k]) for k in sorted(all_statuses["local"])] # then all other services for server, services in all_other_services().items(): all_statuses[server] = {} for service, checks in services.items(): all_statuses[server][service] = {} for check in checks: res = requests.get( "https://{}/_/{}/{}/".format(server, service, check), headers=intra_shield(), ) try: res.raise_for_status() except: continue else: all_statuses[server][service][check] = res.url all_statuses[server][service] = [ (k, all_statuses[server][service][k]) for k in sorted(all_statuses[server][service]) ] all_statuses[server] = [(k, all_statuses[server][k]) for k in sorted(all_statuses[server])] all_statuses = [(k, all_statuses[k]) for k in sorted(all_statuses)] return render_template( "index.html", all_statuses=all_statuses, site_name=SITE_NAME, ) else: abort(404)
def get_all_statuses(): """Main page, lists all services/clusters if master server.""" if OTHER_SHIELDS: all_statuses = {"local": {}} # first do our own services for service in all_services(): all_statuses["local"][service] = {} for check in services_checks(service): all_statuses["local"][service][check] = check_to_redirect( service, check) all_statuses["local"][service] = [ (k, all_statuses["local"][service][k]) for k in sorted(all_statuses["local"][service]) ] all_statuses["local"] = [(k, all_statuses["local"][k]) for k in sorted(all_statuses["local"])] # then all other services for server, services in all_other_services().items(): all_statuses[server] = {} for service, checks in services.items(): all_statuses[server][service] = {} for check in checks: res = requests.get( "https://{}/_/{}/{}/".format(server, service, check), headers=intra_shield(), ) try: res.raise_for_status() except: continue else: all_statuses[server][service][check] = res.url all_statuses[server][service] = [ (k, all_statuses[server][service][k]) for k in sorted(all_statuses[server][service]) ] all_statuses[server] = [(k, all_statuses[server][k]) for k in sorted(all_statuses[server])] all_statuses = [(k, all_statuses[k]) for k in sorted(all_statuses)] return render_template( "index.html", all_statuses=all_statuses, site_name=SITE_NAME, ) else: abort(404)
def services_checks(service): """Determine the list of checks for a service.""" checks = [] if service in all_services(): # the service was found via kube, can provide automatic checks checks.append("health") try: mod = import_module("kube_shields.snowflakes.{}".format(service)) except ImportError: pass # no additional checks else: for func in [f for f in dir(mod) if not f.startswith("_")]: if getattr(getattr(mod, func), "__snowflake__", False) is True: checks.append(func) return checks
def service_status(service, check): """Checks the status of the check for the service. If this is the master server, the check result will be retrived from the kube_shield server in the service's cluster (if the service is not local). """ if not OTHER_SHIELDS: intra_shield(verify=True) raw = request.args.get("raw") is not None if service in all_services(): if raw: return as_json(do_check(service, check, True)) else: return redirect(check_to_redirect(service, check), code=302) elif OTHER_SHIELDS: # Get the result from the other service for server, services in all_other_services().items(): for _service, _ in services.items(): if service == _service: url = "https://{}/_/{}/{}/".format(server, service, check) if raw: url = "{}?raw".format(url) res = requests.get(url, headers=intra_shield()) try: res.raise_for_status() except requests.HTTPError as error: abort(error.code) else: if raw: return as_json(res.json()) else: return redirect(res.url, code=302) abort(404)
def get_services(): """Returns a list of all known services in our cluster.""" intra_shield(verify=True) return as_json(all_services())
def aggregate_status(service, check): """Aggregate the check on any services starting with service.""" if not OTHER_SHIELDS: intra_shield(verify=True) aggregates = [] if OTHER_SHIELDS: for server, services in all_other_services().items(): for _service, checks in services.items(): if _service.startswith(service) and check in checks: res = requests.get( "https://{}/_/{}/{}/?raw".format( server, _service, check, ), headers=intra_shield(), ) try: res.raise_for_status() except: continue else: aggregates.append(res.json()) for _service in all_services(): if _service.startswith(service): aggregates.append(do_check(_service, check, raw=True)) sum_total = {"label": check, "status": None, "color": ""} sum_health = {"ready": 0, "total": 0, "restarts": 0} for aggregate in aggregates: if isinstance(aggregate, dict): this_color = aggregate.get("color") if this_color and sum_total["color"] not in ("red", "yellow"): sum_total["color"] = this_color elif this_color == "red" and sum_total["color"] != "red": sum_total["color"] = this_color if check == "health": sum_health["ready"] += aggregate["ready"] sum_health["total"] += aggregate["total"] sum_health["restarts"] += aggregate["restarts"] else: if not sum_total["status"]: sum_total["status"] = aggregate.get("status", "") else: sum_total["status"] += aggregate.get("status", "") if check == "health": sum_total["status"] = "{}/{} ({} restart{})".format( sum_health["ready"], sum_health["total"], sum_health["restarts"], "s" * int(sum_health["restarts"] != 1) ) sum_total["color"] = sum_total["color"] or ( "green" if sum_health["ready"] == sum_health["total"] else "red" ) return redirect(as_redirect_url(**sum_total))
def all_services_and_snowflakes(): """Returns a sorted combined list of all services and snowflakes.""" return sorted(set(all_services() + all_snowflakes()))