def query_cluster(cluster, executor, system_namespaces,
additional_cost_per_cluster, no_ingress_status, node_label):
logger.info(f"Querying cluster {cluster.id} ({cluster.api_server_url})..")
pods = {}
nodes = {}
namespaces = {}
for namespace in Namespace.objects(cluster.client):
email = namespace.annotations.get('email')
namespaces[namespace.name] = {
"status": namespace.obj['status']['phase'],
"email": email,
}
cluster_capacity = collections.defaultdict(float)
cluster_allocatable = collections.defaultdict(float)
cluster_requests = collections.defaultdict(float)
user_requests = collections.defaultdict(float)
node_count = collections.defaultdict(int)
cluster_cost = additional_cost_per_cluster
for _node in Node.objects(cluster.client):
node = _node.obj
nodes[_node.name] = node
node["capacity"] = {}
node["allocatable"] = {}
node["requests"] = new_resources()
node["usage"] = new_resources()
for k, v in node["status"].get("capacity", {}).items():
parsed = parse_resource(v)
node["capacity"][k] = parsed
cluster_capacity[k] += parsed
for k, v in node["status"].get("allocatable", {}).items():
parsed = parse_resource(v)
node["allocatable"][k] = parsed
cluster_allocatable[k] += parsed
role = _node.labels.get(NODE_LABEL_ROLE) or "worker"
node_count[role] += 1
region = _node.labels.get(NODE_LABEL_REGION, "unknown")
instance_type = _node.labels.get(NODE_LABEL_INSTANCE_TYPE, "unknown")
is_spot = _node.labels.get(NODE_LABEL_SPOT) == "true"
node["spot"] = is_spot
node["kubelet_version"] = (node["status"].get("nodeInfo", {}).get(
"kubeletVersion", ""))
node["role"] = role
node["instance_type"] = instance_type
node["cost"] = pricing.get_node_cost(region, instance_type, is_spot)
cluster_cost += node["cost"]
get_node_usage(cluster, nodes)
cluster_usage = collections.defaultdict(float)
for node in nodes.values():
for k, v in node['usage'].items():
cluster_usage[k] += v
cost_per_cpu = cluster_cost / cluster_allocatable["cpu"]
cost_per_memory = cluster_cost / cluster_allocatable["memory"]
for pod in Pod.objects(cluster.client, namespace=pykube.all):
if pod.obj["status"].get("phase") != "Running":
# ignore unschedulable/completed pods
continue
application = get_application_from_labels(pod.labels)
component = get_component_from_labels(pod.labels)
requests = collections.defaultdict(float)
ns = pod.namespace
container_images = []
for container in pod.obj["spec"]["containers"]:
# note that the "image" field is optional according to Kubernetes docs
image = container.get("image")
if image:
container_images.append(image)
for k, v in container["resources"].get("requests", {}).items():
pv = parse_resource(v)
requests[k] += pv
cluster_requests[k] += pv
if ns not in system_namespaces:
user_requests[k] += pv
if "nodeName" in pod.obj["spec"] and pod.obj["spec"][
"nodeName"] in nodes:
for k in ("cpu", "memory"):
nodes[pod.obj["spec"]
["nodeName"]]["requests"][k] += requests.get(k, 0)
cost = max(requests["cpu"] * cost_per_cpu,
requests["memory"] * cost_per_memory)
pods[(ns, pod.name)] = {
"requests": requests,
"application": application,
"component": component,
"container_images": container_images,
"cost": cost,
"usage": new_resources(),
}
hourly_cost = cluster_cost / HOURS_PER_MONTH
cluster_summary = {
"cluster":
cluster,
"nodes":
nodes,
"pods":
pods,
"namespaces":
namespaces,
"user_pods":
len([p for ns, p in pods if ns not in system_namespaces]),
"master_nodes":
node_count["master"],
"worker_nodes":
node_count[node_label],
"kubelet_versions":
set([
n["kubelet_version"] for n in nodes.values()
if n["role"] == node_label
]),
"worker_instance_types":
set([
n["instance_type"] for n in nodes.values()
if n["role"] == node_label
]),
"worker_instance_is_spot":
any([n["spot"] for n in nodes.values() if n["role"] == node_label]),
"capacity":
cluster_capacity,
"allocatable":
cluster_allocatable,
"requests":
cluster_requests,
"user_requests":
user_requests,
"usage":
cluster_usage,
"cost":
cluster_cost,
"cost_per_user_request_hour": {
"cpu":
0.5 * hourly_cost /
max(user_requests["cpu"], MIN_CPU_USER_REQUESTS),
"memory":
0.5 * hourly_cost /
max(user_requests["memory"] / ONE_GIBI, MIN_MEMORY_USER_REQUESTS),
},
"ingresses": [],
}
get_pod_usage(cluster, pods)
cluster_slack_cost = 0
for pod in pods.values():
usage_cost = max(
pod["usage"]["cpu"] * cost_per_cpu,
pod["usage"]["memory"] * cost_per_memory,
)
pod["slack_cost"] = pod["cost"] - usage_cost
cluster_slack_cost += pod["slack_cost"]
cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost)
with FuturesSession(max_workers=10, session=session) as futures_session:
futures_by_host = {} # hostname -> future
futures = collections.defaultdict(list) # future -> [ingress]
for _ingress in Ingress.objects(cluster.client, namespace=pykube.all):
application = get_application_from_labels(_ingress.labels)
for rule in _ingress.obj["spec"].get("rules", []):
host = rule.get('host', '')
if not application:
# find the application by getting labels from pods
backend_application = find_backend_application(
cluster.client, _ingress, rule)
else:
backend_application = None
ingress = [
_ingress.namespace, _ingress.name, application
or backend_application, host, 0
]
if host and not no_ingress_status:
try:
future = futures_by_host[host]
except KeyError:
future = futures_session.get(f"https://{host}/",
timeout=5)
futures_by_host[host] = future
futures[future].append(ingress)
cluster_summary["ingresses"].append(ingress)
if not no_ingress_status:
logger.info(
f'Waiting for ingress status for {cluster.id} ({cluster.api_server_url})..'
)
for future in concurrent.futures.as_completed(futures):
ingresses = futures[future]
try:
response = future.result()
status = response.status_code
except:
status = 999
for ingress in ingresses:
ingress[4] = status
return cluster_summary
def query_cluster(
cluster,
executor,
system_namespaces,
additional_cost_per_cluster,
alpha_ema,
prev_cluster_summaries,
no_ingress_status,
node_labels,
):
logger.info(f"Querying cluster {cluster.id} ({cluster.api_server_url})..")
pods = {}
nodes = {}
namespaces = {}
for namespace in Namespace.objects(cluster.client):
email = namespace.annotations.get("email")
namespaces[namespace.name] = {
"status": namespace.obj["status"]["phase"],
"email": email,
}
cluster_capacity = collections.defaultdict(float)
cluster_allocatable = collections.defaultdict(float)
cluster_requests = collections.defaultdict(float)
user_requests = collections.defaultdict(float)
cluster_cost = additional_cost_per_cluster
for _node in Node.objects(cluster.client):
node = map_node(_node)
nodes[_node.name] = node
for k, v in node["capacity"].items():
cluster_capacity[k] += v
for k, v in node["allocatable"].items():
cluster_allocatable[k] += v
cluster_cost += node["cost"]
metrics.get_node_usage(cluster, nodes,
prev_cluster_summaries.get("nodes", {}), alpha_ema)
cluster_usage = collections.defaultdict(float)
for node in nodes.values():
for k, v in node["usage"].items():
cluster_usage[k] += v
try:
vpas_by_namespace_label = get_vpas_by_match_labels(cluster.client)
except Exception as e:
logger.warning(f"Failed to query VPAs in cluster {cluster.id}: {e}")
vpas_by_namespace_label = collections.defaultdict(list)
cost_per_cpu = cluster_cost / cluster_allocatable["cpu"]
cost_per_memory = cluster_cost / cluster_allocatable["memory"]
for pod in Pod.objects(cluster.client, namespace=pykube.all):
# ignore unschedulable/completed pods
if not pod_active(pod):
continue
pod_ = map_pod(pod, cost_per_cpu, cost_per_memory)
for k, v in pod_["requests"].items():
cluster_requests[k] += v
if pod.namespace not in system_namespaces:
user_requests[k] += v
node_name = pod.obj["spec"].get("nodeName")
if node_name and node_name in nodes:
for k in ("cpu", "memory"):
nodes[node_name]["requests"][k] += pod_["requests"].get(k, 0)
found_vpa = False
for k, v in pod.labels.items():
vpas = vpas_by_namespace_label[(pod.namespace, k, v)]
for vpa in vpas:
if vpa.matches_pod(pod):
recommendation = new_resources()
container_names = set()
for container in pod.obj["spec"]["containers"]:
container_names.add(container["name"])
for container in vpa.container_recommendations:
# VPA might contain recommendations for containers which are no longer there!
if container["containerName"] in container_names:
for k in ("cpu", "memory"):
recommendation[k] += parse_resource(
container["target"][k])
pod_["recommendation"] = recommendation
found_vpa = True
break
if found_vpa:
break
pods[(pod.namespace, pod.name)] = pod_
hourly_cost = cluster_cost / HOURS_PER_MONTH
cluster_summary = {
"cluster":
cluster,
"nodes":
nodes,
"pods":
pods,
"namespaces":
namespaces,
"user_pods":
len([p for ns, p in pods if ns not in system_namespaces]),
"master_nodes":
len([n for n in nodes.values() if n["role"] == "master"]),
"worker_nodes":
len([n for n in nodes.values() if n["role"] in node_labels]),
"kubelet_versions":
set([
n["kubelet_version"] for n in nodes.values()
if n["role"] in node_labels
]),
"worker_instance_types":
set([
n["instance_type"] for n in nodes.values()
if n["role"] in node_labels
]),
"worker_instance_is_spot":
any([n["spot"] for n in nodes.values() if n["role"] in node_labels]),
"capacity":
cluster_capacity,
"allocatable":
cluster_allocatable,
"requests":
cluster_requests,
"user_requests":
user_requests,
"usage":
cluster_usage,
"cost":
cluster_cost,
"cost_per_user_request_hour": {
"cpu":
0.5 * hourly_cost /
max(user_requests["cpu"], MIN_CPU_USER_REQUESTS),
"memory":
0.5 * hourly_cost /
max(user_requests["memory"] / ONE_GIBI, MIN_MEMORY_USER_REQUESTS),
},
"ingresses": [],
}
metrics.get_pod_usage(cluster, pods,
prev_cluster_summaries.get("pods", {}), alpha_ema)
cluster_slack_cost = 0
for pod in pods.values():
usage_cost = max(
pod["usage"]["cpu"] * cost_per_cpu,
pod["usage"]["memory"] * cost_per_memory,
)
pod["slack_cost"] = pod["cost"] - usage_cost
cluster_slack_cost += pod["slack_cost"]
cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost)
with FuturesSession(max_workers=10, session=session) as futures_session:
futures_by_host = {} # hostname -> future
futures = collections.defaultdict(list) # future -> [ingress]
for _ingress in Ingress.objects(cluster.client, namespace=pykube.all):
application = get_application_from_labels(_ingress.labels)
for rule in _ingress.obj["spec"].get("rules", []):
host = rule.get("host", "")
if not application:
# find the application by getting labels from pods
backend_application = find_backend_application(
cluster.client, _ingress, rule)
else:
backend_application = None
ingress = [
_ingress.namespace,
_ingress.name,
application or backend_application,
host,
0,
]
if host and not no_ingress_status:
try:
future = futures_by_host[host]
except KeyError:
future = futures_session.get(f"https://{host}/",
timeout=5)
futures_by_host[host] = future
futures[future].append(ingress)
cluster_summary["ingresses"].append(ingress)
if not no_ingress_status:
logger.info(
f"Waiting for ingress status for {cluster.id} ({cluster.api_server_url}).."
)
for future in concurrent.futures.as_completed(futures):
ingresses = futures[future]
try:
response = future.result()
status = response.status_code
except Exception:
status = 999
for ingress in ingresses:
ingress[4] = status
return cluster_summary
def query_kubernetes_cluster(cluster):
cluster_id = cluster.id
api_server_url = cluster.api_server_url
nodes = {}
pods_by_namespace_name = {}
unassigned_pods = {}
for node in Node.objects(cluster.client):
obj = map_node(node.obj)
nodes[obj['name']] = obj
now = time.time()
for pod in Pod.objects(cluster.client, namespace=pykube.all):
obj = map_pod(pod.obj)
if 'deletionTimestamp' in pod.metadata:
obj['deleted'] = parse_time(pod.metadata['deletionTimestamp'])
for cont in pod.obj['spec']['containers']:
obj['containers'].append(map_container(cont, pod.obj))
if obj['phase'] in ('Succeeded', 'Failed'):
last_termination_time = 0
for container in obj['containers']:
termination_time = container.get('state',
{}).get('terminated',
{}).get('finishedAt')
if termination_time:
termination_time = parse_time(termination_time)
if termination_time > last_termination_time:
last_termination_time = termination_time
if (last_termination_time and
last_termination_time < now - 3600) or (obj.get('reason')
== 'Evicted'):
# the job/pod finished more than an hour ago or if it is evicted by cgroup limits
# => filter out
continue
pods_by_namespace_name[(pod.namespace, pod.name)] = obj
pod_key = f'{pod.namespace}/{pod.name}'
node_name = pod.obj['spec'].get('nodeName')
if node_name in nodes:
nodes[node_name]['pods'][pod_key] = obj
else:
unassigned_pods[pod_key] = obj
try:
for node_metrics in NodeMetrics.objects(cluster.client):
key = node_metrics.name
nodes[key]['usage'] = node_metrics.obj.get('usage', {})
except Exception as e:
logger.warning('Failed to query node metrics {}: {}'.format(
cluster.id, get_short_error_message(e)))
try:
for pod_metrics in PodMetrics.objects(cluster.client,
namespace=pykube.all):
key = (pod_metrics.namespace, pod_metrics.name)
pod = pods_by_namespace_name.get(key)
if pod:
for container in pod['containers']:
for container_metrics in pod_metrics.obj.get(
'containers', []):
if container['name'] == container_metrics['name']:
container['resources'][
'usage'] = container_metrics['usage']
except Exception as e:
logger.warning('Failed to query pod metrics for cluster {}: {}'.format(
cluster.id, get_short_error_message(e)))
return {
'id': cluster_id,
'api_server_url': api_server_url,
'nodes': nodes,
'unassigned_pods': unassigned_pods
}
def query_kubernetes_cluster(cluster):
cluster_id = cluster.id
api_server_url = cluster.api_server_url
nodes = {}
pods_by_namespace_name = {}
unassigned_pods = {}
for node in Node.objects(cluster.client):
obj = map_node(node.obj)
nodes[obj["name"]] = obj
now = time.time()
for pod in Pod.objects(cluster.client, namespace=pykube.all):
obj = map_pod(pod.obj)
if "deletionTimestamp" in pod.metadata:
obj["deleted"] = parse_time(pod.metadata["deletionTimestamp"])
for cont in pod.obj["spec"]["containers"]:
obj["containers"].append(map_container(cont, pod.obj))
if obj["phase"] in ("Succeeded", "Failed"):
last_termination_time = 0
for container in obj["containers"]:
termination_time = (container.get("state", {}).get(
"terminated", {}).get("finishedAt"))
if termination_time:
termination_time = parse_time(termination_time)
if termination_time > last_termination_time:
last_termination_time = termination_time
if (last_termination_time and
last_termination_time < now - 3600) or (obj.get("reason")
== "Evicted"):
# the job/pod finished more than an hour ago or if it is evicted by cgroup limits
# => filter out
continue
pods_by_namespace_name[(pod.namespace, pod.name)] = obj
pod_key = f"{pod.namespace}/{pod.name}"
node_name = pod.obj["spec"].get("nodeName")
if node_name in nodes:
nodes[node_name]["pods"][pod_key] = obj
else:
unassigned_pods[pod_key] = obj
try:
for node_metrics in NodeMetrics.objects(cluster.client):
key = node_metrics.name
nodes[key]["usage"] = node_metrics.obj.get("usage", {})
except Exception as e:
logger.warning("Failed to query node metrics {}: {}".format(
cluster.id, get_short_error_message(e)))
try:
for pod_metrics in PodMetrics.objects(cluster.client,
namespace=pykube.all):
key = (pod_metrics.namespace, pod_metrics.name)
pod = pods_by_namespace_name.get(key)
if pod:
for container in pod["containers"]:
for container_metrics in pod_metrics.obj.get(
"containers", []):
if container["name"] == container_metrics["name"]:
container["resources"][
"usage"] = container_metrics["usage"]
except Exception as e:
logger.warning("Failed to query pod metrics for cluster {}: {}".format(
cluster.id, get_short_error_message(e)))
return {
"id": cluster_id,
"api_server_url": api_server_url,
"nodes": nodes,
"unassigned_pods": unassigned_pods,
}
def _get_nodes(self):
return Node.objects(self.api).filter(selector=self.labels)