def __init__(self): self.clusters, self.clients, self.active_cluster= cr.get_clients() # preload app and deployable information (for yaml) apps = amb.all_applications() self.apps = { app["metadata"]["name"] : app for app in apps} self.dpbs = {} for app in self.apps.values(): app_name = app["metadata"]["name"] app_ns = app["metadata"]["namespace"] app_cluster = app["metadata"]["cluster_name"] deployables = amb.application_deployables(cluster_name=app_cluster, namespace=app_ns, app_name=app_name) for dpb in deployables: dpb_name = dpb["metadata"]["name"] self.dpbs[dpb_name] = dpb
k8s_config.update_available_clusters() names = lambda ls: [l["metadata"]["name"] for l in ls] # find all Applications defined across your clusters apps = amb.all_applications() for app in apps: app_name = app["metadata"]["name"] app_ns = app["metadata"]["namespace"] app_cluster = app["metadata"]["cluster_name"] print("Application:", app_name) # find all Deployables that belong to this Application deployables = amb.application_deployables(cluster_name=app_cluster, namespace=app_ns, app_name=app_name) for dpb in deployables: dpb_name = dpb["metadata"]["name"] dpb_ns = dpb["metadata"]["namespace"] dpb_cluster = dpb["metadata"]["cluster_name"] print("\tDeployable:", dpb_name) # find the resource that belongs to this Deployable resource = amb.deployable_resource(cluster_name=dpb_cluster, namespace=dpb_ns, deployable_name=dpb_name) try: resource_name = resource["metadata"]["name"] except: resource_name = "Deployer name not found"
def start(mode): """ Retrieves and stores clusters+namespaces, and applications+deployables, and returns starting table :param mode: 'app' or 'cluster' :return: json response including path_names (List[str], list of names of resources in the path), path_rtypes (List[str], list of rtypes of resources in the path), path_uids (List[str], list of skipper uids of resources in the path), table_items (List[Dict], list of dictionaries for resources to be displayed), index (int, row to be selected), has_children (List[bool]), whether each resource in table has children), has_apps (bool, if MCM applications are found) """ cluster_rows = db.session.query(Resource).filter( Resource.rtype == "Cluster").all() if len(cluster_rows) == 0: # lazy load clusters clusters = k8s_config.all_cluster_names() mcm_clusters = cmb.mcm_clusters(clusters) for cluster in clusters: if cluster in mcm_clusters: mcm_cluster = mcm_clusters[cluster] mcm_cluster["yaml"] = yaml.dump(mcm_cluster, sort_keys=False) resource_data = { 'uid': cluster, "rtype": 'Cluster', "name": cluster, "cluster": cluster, "cluster_path": "/root/", "info": json.dumps(mcm_cluster) } else: resource_data = { "uid": cluster, "rtype": "Cluster", "name": cluster, "cluster": cluster, "cluster_path": "/root/" } edge_data = { 'start_uid': 'root', 'end_uid': cluster, 'relation': "Root<-Cluster" } requests.post('http://127.0.0.1:5000/resource/{}'.format(cluster), data=resource_data) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format( 'root', cluster), data=edge_data) # lazy load namespaces namespaces = cmb.cluster_namespaces(cluster) for ns in namespaces: ns_uid = cluster + "_" + ns.metadata.uid ns_name = ns.metadata.name created_at = ns.metadata.creation_timestamp resource_data = { "uid": ns_uid, "created_at": created_at, "rtype": "Namespace", "name": ns_name, "cluster": cluster, "namespace": ns_name, "cluster_path": "/root/{}/".format(cluster), "info": to_json(ns) } edge_data = { 'start_uid': cluster, 'end_uid': ns_uid, 'relation': "Cluster<-Namespace" } requests.post( 'http://127.0.0.1:5000/resource/{}'.format(ns_uid), data=resource_data) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format( cluster, ns_uid), data=edge_data) # lazy load applications app_rows = db.session.query(Resource).filter( Resource.rtype == "Application").all() has_apps = True if len(app_rows) == 0: apps = amb.all_applications() has_apps = True if len(apps) > 0 else False for app in apps: md = app["metadata"] app_name, app_cluster, app_ns, k8s_uid = md["name"], md[ "cluster_name"], md["namespace"], md["uid"] app_uid = app_cluster + "_" + k8s_uid created_at = parse(md["creationTimestamp"]) data = { "uid": app_uid, "created_at": created_at, "rtype": "Application", "name": app_name, "cluster": app_cluster, "namespace": app_ns, "application": app_name, "app_path": "/root/", "info": json.dumps(app) } requests.post('http://127.0.0.1:5000/resource/{}'.format(app_uid), data=data) edge_data = { 'start_uid': 'root', 'end_uid': app_uid, 'relation': "Root<-Application" } requests.post('http://127.0.0.1:5000/edge/{}/{}'.format( 'root', app_uid), data=edge_data) # lazy load deployables dpbs = amb.application_deployables(app_cluster, app_ns, app_name) for dpb in dpbs: md = dpb["metadata"] dpb_name, k8s_uid, cname, ns = md["name"], md["uid"], md[ "cluster_name"], md["namespace"] dpb_uid = cname + "_" + k8s_uid created_at = parse(dpb["metadata"]["creationTimestamp"]) resource_data = { "uid": dpb_uid, "created_at": created_at, "rtype": "Deployable", "name": dpb_name, "cluster": cname, "namespace": ns, "application": app_name, "app_path": "/root/{}/".format(app_uid), "info": json.dumps(dpb) } edge_data = { 'start_uid': app_uid, 'end_uid': dpb_uid, 'relation': "Application<-Deployable" } requests.post( 'http://127.0.0.1:5000/resource/{}'.format(dpb_uid), data=resource_data) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format( app_uid, dpb_uid), data=edge_data) # get the starting apps or clusters if mode == 'app': table = db.session.query(Resource).filter( Resource.rtype == "Application").all() elif mode == 'cluster': table = db.session.query(Resource).filter( Resource.rtype == "Cluster").all() table_dicts = [row_to_dict(table_item) for table_item in table] return jsonify(path_names=[], path_rtypes=[], path_uids=[], table_items=table_dicts, index=0, has_children=has_children(table), has_apps=has_apps)
def get_table_by_resource(mode, uid): """ Get the table and relevant info for navigating INTO a resource (aka the table lists resource's children) :param mode: 'app' or 'cluster' :param uid: skipper uid of resource :return: json response including path_names (List[str], list of names of resources in the path), path_rtypes (List[str], list of rtypes of resources in the path), path_uids (List[str], list of skipper uids of resources in the path), table_items (List[Dict], list of dictionaries for resources to be displayed), index (int, row to be selected), has_children (List[bool]), whether each resource in table has children) """ resource = db.session.query(Resource).filter_by(uid=uid).first() outgoing_edges = db.session.query(Edge).filter_by(start_uid=uid).all() if len(outgoing_edges) == 0: children = [] # lazy load depending on current resource type if resource.rtype == 'Cluster': namespaces = cmb.cluster_namespaces(resource.cluster) for ns in namespaces: cname = ns.metadata.cluster_name ns_uid = cname + "_" + ns.metadata.uid ns_name = ns.metadata.name created_at = ns.metadata.creation_timestamp resource_data = { "uid": ns_uid, "created_at": created_at, "rtype": "Namespace", "name": ns_name, "cluster": cname, "namespace": ns_name, "cluster_path": "/root/{}/".format(cname), "info": to_json(ns) } edge_data = { 'start_uid': cname, 'end_uid': ns_uid, 'relation': "Cluster<-Namespace" } requests.post( 'http://127.0.0.1:5000/resource/{}'.format(ns_uid), data=resource_data) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format( cname, ns_uid), data=edge_data) # for all other resources, storing(child, child_type) into the children list elif resource.rtype == 'Namespace': children.extend([(res, "Deployment") for res in cmb.namespace_deployments( resource.name, resource.cluster)]) children.extend([(res, "Service") for res in cmb.namespace_services( resource.name, resource.cluster)]) children.extend([(res, "StatefulSet") for res in cmb.namespace_stateful_sets( resource.name, resource.cluster)]) children.extend([(res, "DaemonSet") for res in cmb.namespace_daemon_sets( resource.name, resource.cluster)]) elif resource.rtype == 'Application': children.extend([ (res, "Deployable") for res in amb.application_deployables( resource.cluster, resource.namespace, resource.name) ]) elif resource.rtype == 'Deployable': deployer_dict = amb.deployable_resource(resource.cluster, resource.namespace, resource.name) if deployer_dict != {}: children.extend([(deployer_dict, deployer_dict["kind"])]) elif resource.rtype == 'Deployment': children.extend([(res, "Pod") for res in cmb.deployment_pods( resource.name, resource.namespace, resource.cluster)]) elif resource.rtype == 'Service': children.extend([(res, "Pod") for res in cmb.service_pods( resource.name, resource.namespace, resource.cluster)]) elif resource.rtype == 'StatefulSet': children.extend([(res, "Pod") for res in cmb.stateful_set_pods( resource.name, resource.namespace, resource.cluster)]) elif resource.rtype == 'DaemonSet': children.extend([(res, "Pod") for res in cmb.daemon_set_pods( resource.name, resource.namespace, resource.cluster)]) # loop through children and add to db for child, rtype in children: child_obj = child.to_dict() if not isinstance(child, dict) else child # get cluster, falls back to parent resource's cluster (maybe risky) cluster = child_obj["metadata"]["cluster_name"] if child_obj[ "metadata"]["cluster_name"] is not None else resource.cluster namespace = child_obj["metadata"]["namespace"] skipper_uid = cluster + "_" + child_obj["metadata"]["uid"] created_at = child_obj["metadata"][ "creation_timestamp"] if not child_obj["metadata"].get( "creationTimestamp") else child_obj["metadata"].get( "creationTimestamp") # build dict resource_data = {'uid': skipper_uid, "created_at": created_at, \ "rtype": rtype, "name" : child_obj["metadata"]["name"], \ "cluster" : cluster, "namespace" : namespace, "info": json.dumps(child_obj, default=str)} # fill in sev_measure and sev_reason fields if we are looking at a pod if rtype == "Pod": sm, sr = eb.pod_state(child) resource_data["sev_measure"] = sm resource_data["sev_reason"] = sr # update paths if resource.app_path != None: resource_data[ 'app_path'] = resource.app_path + resource.uid + "/" app_uid = resource.app_path.split("/")[2] resource_data["application"] = requests.get( 'http://127.0.0.1:5000/resource/{}'.format( app_uid)).json()['data']['name'] if resource.cluster_path != None: resource_data[ 'cluster_path'] = resource.cluster_path + resource.uid + "/" requests.post( 'http://127.0.0.1:5000/resource/{}'.format(skipper_uid), data=resource_data) edge_data = { 'start_uid': resource.uid, 'end_uid': skipper_uid, 'relation': resource.rtype + "<-" + rtype } requests.post('http://127.0.0.1:5000/edge/{}/{}'.format( resource.uid, skipper_uid), data=edge_data) table_items = db.session.query(Resource).join( Edge, Resource.uid == Edge.end_uid).filter(Edge.start_uid == uid).all() # get path info for frontend if mode == 'app': full_path = resource.app_path elif mode == 'cluster': full_path = resource.cluster_path full_path += "{}/".format(resource.uid) # convert path using uids to breadcrumbs of resource names and types path_uids = full_path.split("/")[2:-1] path_names = [] path_rtypes = [] for res_uid in path_uids: path_names.append( db.session.query( Resource.name).filter(Resource.uid == res_uid).first()[0]) path_rtypes.append( db.session.query( Resource.rtype).filter(Resource.uid == res_uid).first()[0]) return jsonify(path_names=path_names, path_rtypes=path_rtypes, path_uids=path_uids, table_items=[row_to_dict(t_item) for t_item in table_items], index=0, has_children=has_children(table_items))
def load_all() -> None: """ Loads all Resources and Edges into the database. """ def to_json(k8s_obj: k8s.client.models) -> str: to_s = lambda dt: dt.__str__() return json.dumps(k8s_obj.to_dict(), default=to_s) all_nss = [] # all namespaces all_deploys = [] # all deployments all_svcs = [] # all services all_dsets = [] # all daemonsets all_ssets = [] # all statefulsets all_pods = [] # all pods all_dpbs = [] # all deployables start = time.time() # time how long one round of load-all takes split_start = time.time() # time how long each portion takes print("\nStarting a round of load-all. Strap yourself in, astronaut.") # insert all clusters into the database stale_clusters = Resource.query.filter(Resource.rtype=="Cluster").all() stale_cdict = { c.name: c for c in stale_clusters } cluster_names = k8s_config.all_cluster_names() mcm_clusters = cmb.mcm_clusters(cluster_names) for cname in cluster_names: if cname in mcm_clusters: mcm_cluster = mcm_clusters[cname] mcm_cluster["yaml"] = yaml.dump(mcm_cluster, sort_keys=False) cluster_data = {'uid': cname, "rtype": 'Cluster', "name": cname , "cluster": cname , "cluster_path": "/root/", "created_at" : mcm_cluster["metadata"].get("creationTimestamp"), "info" : json.dumps(mcm_cluster)} else: cluster_data = {"uid": cname, "rtype": "Cluster", "name": cname, "cluster": cname, "cluster_path": "/root/"} # remove cluster from running list of stale clusters if cname in stale_cdict.keys(): stale_cdict.pop(cname) requests.post('http://127.0.0.1:5000/resource/{}'.format(cname), data=cluster_data) # find all namespaces under this cluster and add it # to the running list of namespaces (all_nss) nss = cmb.cluster_namespaces(cluster_name=cname) for ns in nss: ns.metadata.cluster_name = cname all_nss += nss # remove all stale clusters, descendants, and associated edges # i.e. clusters that were in the db but that we didn't find for cname in stale_cdict.keys(): requests.delete('http://127.0.0.1:5000/resource/{}'.format(cname)) # remove any apps that were defined in the cluster cluster_apps = Resource.query.filter(Resource.rtype=="Application" and Resource.uid.like(cname + "%")).all() for capp in cluster_apps: requests.delete('http://127.0.0.1:5000/resource/{}'.format(capp.uid)) print("Wrote %d clusters and found all child namespaces in %d seconds." % (len(cluster_names), time.time() - split_start)) # insert all applications into the database split_start = time.time() stale_apps = Resource.query.filter(Resource.rtype == "Application").all() stale_adict = { app.uid: app for app in stale_apps } apps = amb.all_applications() for app in apps: md = app["metadata"] name, cname, ns, k8s_uid = md["name"], md["cluster_name"], md["namespace"], md["uid"] app_uid = cname + "_" + k8s_uid created_at = parse(app["metadata"]["creationTimestamp"]) app_data = { "uid": app_uid, "created_at": created_at, "rtype": "Application", "name": name, "cluster": cname, "namespace": ns, "application": name, "app_path": "/root/", "info": json.dumps(app)} requests.post('http://127.0.0.1:5000/resource/{}'.format(app_uid), data=app_data) # find all deployables under this application and add it to the running # list of all deployables (all_dpbs) deployables = amb.application_deployables(cluster_name=cname, namespace=ns, app_name=name) for dpb in deployables: dpb["metadata"]["app_name"] = name dpb["metadata"]["app_uid"] = app_uid all_dpbs += deployables # remove this app from running list of stale apps if app_uid in stale_adict.keys(): stale_adict.pop(app_uid) # remove all stale apps, descendants, and associated edges for app_uid in stale_adict: requests.delete('http://127.0.0.1:5000/resource/{}'.format(app_uid)) print("Wrote %d applications and found all child deployables in %d seconds." % (len(apps), time.time() - split_start)) # insert all namespaces and corresponding edges into the database split_start = time.time() stale_nss = Resource.query.filter(Resource.rtype == "Namespace").all() stale_nsdict = { ns.uid: ns for ns in stale_nss } for ns in all_nss: cname = ns.metadata.cluster_name ns_uid = cname + "_" + ns.metadata.uid ns_name = ns.metadata.name created_at = ns.metadata.creation_timestamp ns_resource = {"uid": ns_uid, "created_at": created_at, "rtype": "Namespace", "name": ns_name, "cluster": cname, "namespace": ns_name, "cluster_path": "/root/{}/".format(cname), "info": to_json(ns)} ns_edge = {"start_uid": cname, "end_uid": ns_uid, "relation": "Cluster<-Namespace"} requests.post('http://127.0.0.1:5000/resource/{}'.format(ns_uid), data=ns_resource) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format(cname, ns_uid), data=ns_edge) # remove this ns from list of stale namespaces if ns_uid in stale_nsdict.keys(): stale_nsdict.pop(ns_uid) # find all deployments under this namespace and add it to the running # list of all deployments (all_deploys) deploys = cmb.namespace_deployments(cluster_name=cname, namespace=ns_name) for deploy in deploys: deploy.metadata.cluster_name = cname deploy.metadata.ns_uid = ns_uid all_deploys += deploys # find all services under this namespace and add it to the running # list of all services (all_svcs) svcs = cmb.namespace_services(cluster_name=cname, namespace=ns_name) for svc in svcs: svc.metadata.cluster_name = cname svc.metadata.ns_uid = ns_uid all_svcs += svcs # find all daemonsets under this namespace and add it to the running # list of all daemonsets (all_dsets) dsets = cmb.namespace_daemon_sets(cluster_name=cname, namespace=ns_name) for dset in dsets: dset.metadata.cluster_name = cname dset.metadata.ns_uid = ns_uid all_dsets += dsets # find all statefulsets under this namespace and add it to the running # list of all statefulsets (all_ssets) ssets = cmb.namespace_stateful_sets(cluster_name=cname, namespace=ns_name) for sset in ssets: sset.metadata.cluster_name = cname sset.metadata.ns_uid = ns_uid all_ssets += ssets # remove all stale namespaces, their descendants and associated edges for sns_uid in stale_nsdict.keys(): requests.delete('http://127.0.0.1:5000/resource/{}'.format(sns_uid)) print("Wrote %d namespaces and found child deployments, services, daemonsets and statefulsets in %d seconds." % (len(all_nss), time.time() - split_start)) # insert all deployables and corresponding edges into the database split_start = time.time() stale_dpbs = Resource.query.filter(Resource.rtype == "Deployable").all() stale_dpb_dict = { dpb.uid: dpb for dpb in stale_dpbs } for dpb in all_dpbs: md = dpb["metadata"] app_name, app_uid = md["app_name"], md["app_uid"] dpb_name, k8s_uid, cname, ns = md["name"], md["uid"], md["cluster_name"], md["namespace"] dpb_uid = cname + "_" + k8s_uid created_at = parse(dpb["metadata"]["creationTimestamp"]) dpb_resource = {"uid": dpb_uid, "created_at": created_at, "rtype": "Deployable", "name": dpb_name, "cluster": cname, "namespace": ns, "application": app_name, "app_path": "/root/{}/".format(app_uid), "info": json.dumps(dpb)} dpb_edge = {"start_uid": app_uid, "end_uid": dpb_uid, "relation": "Application<-Deployable"} requests.post('http://127.0.0.1:5000/resource/{}'.format(dpb_uid), data=dpb_resource) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format(app_uid, dpb_uid), data=dpb_edge) # remove this deployable from the list of stale deployables if dpb_uid in stale_dpb_dict.keys(): stale_dpb_dict.pop(dpb_uid) # remove all stale deployables, their descendants and associated edges for dpb_uid in stale_dpb_dict.keys(): requests.delete('http://127.0.0.1:5000/resource/{}'.format(dpb_uid)) print("Wrote %d deployables in %d seconds." % (len(all_dpbs), time.time() - split_start )) # insert all deployments and corresponding edges into the database split_start = time.time() stale_deploys = Resource.query.filter(Resource.rtype == "Deployment").all() stale_deploy_dict = { deploy.uid: deploy for deploy in stale_deploys } for deploy in all_deploys: cname = deploy.metadata.cluster_name ns_uid = deploy.metadata.ns_uid deploy_uid = cname + "_" + deploy.metadata.uid deploy_path = "/root/{}/{}/".format(cname, ns_uid) created_at = deploy.metadata.creation_timestamp deploy_resource = {"uid": deploy_uid, "created_at": created_at, "rtype": "Deployment", "name": deploy.metadata.name, "cluster": cname, "namespace": deploy.metadata.namespace, "cluster_path": deploy_path, "info": to_json(deploy)} deploy_edge = {"start_uid": ns_uid, "end_uid": deploy_uid, "relation": "Namespace<-Deployment"} requests.post('http://127.0.0.1:5000/resource/{}'.format(deploy_uid), data=deploy_resource) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format(ns_uid, deploy_uid), data=deploy_edge) # find all pods being managed by this deployment and add them to the list of all pods (all_pods) pods = cmb.deployment_pods(cluster_name=cname, namespace=deploy.metadata.namespace, deploy_name=deploy.metadata.name) for pod in pods: pod.metadata.cluster_name = cname pod.metadata.ns_uid = ns_uid pod.metadata.parent_uid = deploy_uid pod.metadata.parent_type = "Deployment" all_pods += pods # remove this deployment from the list of stale deployments if deploy_uid in stale_deploy_dict.keys(): stale_deploy_dict.pop(deploy_uid) # remove all stale deployments, their descendants and associated edges for deploy_uid in stale_deploy_dict.keys(): requests.delete('http://127.0.0.1:5000/resource/{}'.format(deploy_uid)) print("Wrote %d deployments and found managed pods in %d seconds." % (len(all_deploys), time.time() - split_start)) # insert all services and corresponding edges into the database split_start = time.time() stale_svcs = Resource.query.filter(Resource.rtype == "Service").all() stale_svc_dict = { svc.uid: svc for svc in stale_svcs } for svc in all_svcs: cname = svc.metadata.cluster_name ns_uid = svc.metadata.ns_uid svc_uid = cname + "_" + svc.metadata.uid svc_path = "/root/{}/{}/".format(cname, ns_uid) created_at = svc.metadata.creation_timestamp svc_resource = {"uid": svc_uid, "created_at": created_at, "rtype": "Service", "name": svc.metadata.name, "cluster": cname, "namespace": svc.metadata.namespace, "cluster_path": svc_path, "info": to_json(svc)} svc_edge = {"start_uid": ns_uid, "end_uid": svc_uid, "relation": "Namespace<-Service"} requests.post('http://127.0.0.1:5000/resource/{}'.format(svc_uid), data=svc_resource) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format(ns_uid, svc_uid), data=svc_edge) # find all pods selected by this service and add them to the list of all pods (all_pods) pods = cmb.service_pods(cluster_name=cname, namespace=svc.metadata.namespace, svc_name=svc.metadata.name) for pod in pods: pod.metadata.cluster_name = cname pod.metadata.ns_uid = ns_uid pod.metadata.parent_uid = svc_uid pod.metadata.parent_type = "Service" all_pods += pods # remove this service from the list of stale services if svc_uid in stale_svc_dict.keys(): stale_svc_dict.pop(svc_uid) # remove all stale services, their descendants and associated edges for svc_uid in stale_svc_dict.keys(): requests.delete('http://127.0.0.1:5000/resource/{}'.format(svc_uid)) print("Wrote %d services and found selected pods in %d seconds." % (len(all_svcs), time.time() - split_start)) # insert all daemonsets and corresponding edges into the database split_start = time.time() stale_dsets = Resource.query.filter(Resource.rtype == "DaemonSet").all() stale_dset_dict = { dset.uid: dset for dset in stale_dsets } for dset in all_dsets: cname = dset.metadata.cluster_name ns_uid = dset.metadata.ns_uid dset_uid = cname + "_" + dset.metadata.uid dset_path = "/root/{}/{}/".format(cname, ns_uid) created_at = dset.metadata.creation_timestamp dset_resource = {"uid": dset_uid, "created_at": created_at, "rtype": "DaemonSet", "name": dset.metadata.name, "cluster": cname, "namespace": dset.metadata.namespace, "cluster_path": dset_path, "info": to_json(dset)} dset_edge = {"start_uid": ns_uid, "end_uid": dset_uid, "relation": "Namespace<-DaemonSet"} requests.post('http://127.0.0.1:5000/resource/{}'.format(dset_uid), data=dset_resource) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format(ns_uid, dset_uid), data=dset_edge) # find all pods managed by this daemonset and add them to the list of all pods (all_pods) pods = cmb.daemon_set_pods(cluster_name=cname, namespace=dset.metadata.namespace, dset_name=dset.metadata.name) for pod in pods: pod.metadata.cluster_name = cname pod.metadata.ns_uid = ns_uid pod.metadata.parent_uid = dset_uid pod.metadata.parent_type = "DaemonSet" all_pods += pods # remove this daemonset from the list of stale daemonsets if dset_uid in stale_dset_dict.keys(): stale_dset_dict.pop(dset_uid) # remove all stale daemonsets, their descendants and associated edges for dset_uid in stale_dset_dict.keys(): requests.delete('http://127.0.0.1:5000/resource/{}'.format(dset_uid)) print("Wrote %d daemonsets and found managed pods in %d seconds." % (len(all_dsets), time.time() - split_start)) # insert all stateful sets and corresponding edges into the database split_start = time.time() stale_ssets = Resource.query.filter(Resource.rtype == "StatefulSet").all() stale_sset_dict = { sset.uid: sset for sset in stale_ssets } for sset in all_ssets: cname = sset.metadata.cluster_name ns_uid = sset.metadata.ns_uid sset_uid = cname + "_" + sset.metadata.uid sset_path = "/root/{}/{}/".format(cname, ns_uid) created_at = sset.metadata.creation_timestamp sset_resource = {"uid": sset_uid, "created_at": created_at, "rtype": "StatefulSet", "name": sset.metadata.name, "cluster": cname, "namespace": sset.metadata.namespace, "cluster_path": sset_path, "info": to_json(sset)} sset_edge = {"start_uid": ns_uid, "end_uid": sset_uid, "relation": "Namespace<-StatefulSet"} requests.post('http://127.0.0.1:5000/resource/{}'.format(sset_uid), data=sset_resource) requests.post('http://127.0.0.1:5000/edge/{}/{}'.format(ns_uid, sset_uid), data=sset_edge) # find all pods managed by this statefulset and add them to the list of all pods (all_pods) pods = cmb.stateful_set_pods(cluster_name=cname, namespace=sset.metadata.namespace, sset_name=sset.metadata.name) for pod in pods: pod.metadata.cluster_name = cname pod.metadata.ns_uid = ns_uid pod.metadata.parent_uid = sset_uid pod.metadata.parent_type = "StatefulSet" all_pods += pods # remove this stateful set from the list of stale statefulsets if sset_uid in stale_sset_dict.keys(): stale_sset_dict.pop(sset_uid) # remove all stale statefulsets, their descendants and associated edges for sset_uid in stale_sset_dict.keys(): requests.delete('http://127.0.0.1:5000/resource/{}'.format(sset_uid)) print("Wrote %d statefulsets and found managed pods in %d seconds." % (len(all_ssets), time.time() - split_start)) # insert all pods and corresponding edges into the database split_start = time.time() stale_pods = Resource.query.filter(Resource.rtype == "Pod").all() stale_pod_dict = { pod.uid: pod for pod in stale_pods } for pod in all_pods: cname = pod.metadata.cluster_name ns_uid = pod.metadata.ns_uid parent_uid = pod.metadata.parent_uid parent_type = pod.metadata.parent_type pod_uid = cname + "_" + pod.metadata.uid pod_path = "/root/{}/{}/{}/".format(cname, ns_uid, parent_uid) sev_measure, sev_reason = eb.pod_state(pod) created_at = pod.metadata.creation_timestamp # don't write this pod to db w/ a cluster_path that goes through a service if parent_type != "Service": pod_resource = {"uid": pod_uid, "created_at": created_at, "rtype": "Pod", "name": pod.metadata.name, "cluster": cname, "namespace": pod.metadata.namespace, "cluster_path": pod_path, "sev_measure": sev_measure, "sev_reason": sev_reason, "info": to_json(pod)} requests.post('http://127.0.0.1:5000/resource/{}'.format(pod_uid), data=pod_resource) pod_edge = {"start_uid": parent_uid, "end_uid": pod_uid, "relation": parent_type + "<-Pod"} requests.post('http://127.0.0.1:5000/edge/{}/{}'.format(parent_uid, pod_uid), data=pod_edge) # remove this pod from the list of stale pods if pod_uid in stale_pod_dict.keys(): stale_pod_dict.pop(pod_uid) # remove all stale pods and associated edges for pod_uid in stale_pod_dict.keys(): requests.delete('http://127.0.0.1:5000/resource/{}'.format(pod_uid)) print("Wrote %d pods in %d seconds." % (len(all_pods), time.time() - split_start)) # create edges between deployables and their resources split_start = time.time() for dpb in all_dpbs: cname = dpb["metadata"]["cluster_name"] ns_name = dpb["metadata"]["namespace"] app_name = dpb["metadata"]["app_name"] app_uid = dpb["metadata"]["app_uid"] dpb_name = dpb["metadata"]["name"] dpb_uid = cname + "_" + dpb["metadata"]["uid"] resource = amb.deployable_resource(cluster_name=cname, namespace=ns_name, deployable_name=dpb_name) # case: helm charts if resource == {}: continue resource_uid = resource["metadata"]["cluster_name"] + "_" + resource["metadata"]["uid"] resource_edge = {"start_uid": dpb_uid, "end_uid": resource_uid, "relation": "Deployable<-" + resource["kind"]} requests.post('http://127.0.0.1:5000/edge/{}/{}'.format(dpb_uid, resource_uid), data=resource_edge) # update app_path of resource update_info = {"uid": resource_uid, "app_path": "/root/{}/{}/".format(app_uid, dpb_uid), "application" : app_name} requests.post('http://127.0.0.1:5000/resource/{}'.format(resource_uid), data=update_info) # update app_path of the resource's pods pods = [] if resource["kind"] == "Deployment": pods = cmb.deployment_pods(cluster_name=resource["metadata"]["cluster_name"], namespace=resource["metadata"]["namespace"], deploy_name=resource["metadata"]["name"]) elif resource["kind"] == "DaemonSet": pods = cmb.daemon_set_pods(cluster_name=resource["metadata"]["cluster_name"], namespace=resource["metadata"]["namespace"], dset_name=resource["metadata"]["name"]) elif resource["kind"] == "StatefulSet": pods = cmb.stateful_set_pods(cluster_name=resource["metadata"]["cluster_name"], namespace=resource["metadata"]["namespace"], sset_name=resource["metadata"]["name"]) for pod in pods: pod_uid = resource["metadata"]["cluster_name"] + "_" + pod.metadata.uid update_info = {"uid": pod_uid, "application": app_name, "app_path": "/root/{}/{}/{}/".format(app_uid, dpb_uid, resource_uid)} requests.post('http://127.0.0.1:5000/resource/{}'.format(pod_uid), data=update_info) print("Updated app_paths and created app mode edges in %d seconds." % (time.time() - split_start)) print("Total time elapsed: {} secs".format(time.time()-start))