def maintain_games(api, games):
for object_type, creation_callback in (
(ReplicationController, create_game_rc),
(Service, create_game_service),
):
current_game_names = set()
for game in object_type.objects(api).filter(selector={'app': 'aimmo-game'}):
game_name = game.obj['metadata']['labels']['game']
current_game_names.add(game_name)
if game_name not in games:
LOGGER.info("Deleting game %s", game_name)
game.delete()
for game_id, game_config in games.items():
if game_id not in current_game_names:
LOGGER.info("Creating game %s", game_id)
creation_callback(api, game_id, game_config)
ingress = Ingress(
api,
{
'apiVersion': 'extensions/v1beta1',
'kind': 'Ingress',
'metadata': {
'name': 'game',
},
'spec': {
'rules': [
{
'host': 'staging.aimmo.codeforlife.education',
'http': {
'paths': [
{
'path': "/game/%s/*" % name,
'backend': {
'serviceName': "game-%s" % name,
'servicePort': 80,
},
} for name in games.keys()
],
},
},
],
},
},
)
if ingress.exists():
ingress.update()
else:
ingress.create()
def query_cluster(cluster, executor, system_namespaces,
additional_cost_per_cluster, no_ingress_status, node_label):
logger.info(f"Querying cluster {cluster.id} ({cluster.api_server_url})..")
pods = {}
nodes = {}
namespaces = {}
for namespace in Namespace.objects(cluster.client):
email = namespace.annotations.get('email')
namespaces[namespace.name] = {
"status": namespace.obj['status']['phase'],
"email": email,
}
cluster_capacity = collections.defaultdict(float)
cluster_allocatable = collections.defaultdict(float)
cluster_requests = collections.defaultdict(float)
user_requests = collections.defaultdict(float)
node_count = collections.defaultdict(int)
cluster_cost = additional_cost_per_cluster
for _node in Node.objects(cluster.client):
node = _node.obj
nodes[_node.name] = node
node["capacity"] = {}
node["allocatable"] = {}
node["requests"] = new_resources()
node["usage"] = new_resources()
for k, v in node["status"].get("capacity", {}).items():
parsed = parse_resource(v)
node["capacity"][k] = parsed
cluster_capacity[k] += parsed
for k, v in node["status"].get("allocatable", {}).items():
parsed = parse_resource(v)
node["allocatable"][k] = parsed
cluster_allocatable[k] += parsed
role = _node.labels.get(NODE_LABEL_ROLE) or "worker"
node_count[role] += 1
region = _node.labels.get(NODE_LABEL_REGION, "unknown")
instance_type = _node.labels.get(NODE_LABEL_INSTANCE_TYPE, "unknown")
is_spot = _node.labels.get(NODE_LABEL_SPOT) == "true"
node["spot"] = is_spot
node["kubelet_version"] = (node["status"].get("nodeInfo", {}).get(
"kubeletVersion", ""))
node["role"] = role
node["instance_type"] = instance_type
node["cost"] = pricing.get_node_cost(region, instance_type, is_spot)
cluster_cost += node["cost"]
get_node_usage(cluster, nodes)
cluster_usage = collections.defaultdict(float)
for node in nodes.values():
for k, v in node['usage'].items():
cluster_usage[k] += v
cost_per_cpu = cluster_cost / cluster_allocatable["cpu"]
cost_per_memory = cluster_cost / cluster_allocatable["memory"]
for pod in Pod.objects(cluster.client, namespace=pykube.all):
if pod.obj["status"].get("phase") != "Running":
# ignore unschedulable/completed pods
continue
application = get_application_from_labels(pod.labels)
component = get_component_from_labels(pod.labels)
requests = collections.defaultdict(float)
ns = pod.namespace
container_images = []
for container in pod.obj["spec"]["containers"]:
# note that the "image" field is optional according to Kubernetes docs
image = container.get("image")
if image:
container_images.append(image)
for k, v in container["resources"].get("requests", {}).items():
pv = parse_resource(v)
requests[k] += pv
cluster_requests[k] += pv
if ns not in system_namespaces:
user_requests[k] += pv
if "nodeName" in pod.obj["spec"] and pod.obj["spec"][
"nodeName"] in nodes:
for k in ("cpu", "memory"):
nodes[pod.obj["spec"]
["nodeName"]]["requests"][k] += requests.get(k, 0)
cost = max(requests["cpu"] * cost_per_cpu,
requests["memory"] * cost_per_memory)
pods[(ns, pod.name)] = {
"requests": requests,
"application": application,
"component": component,
"container_images": container_images,
"cost": cost,
"usage": new_resources(),
}
hourly_cost = cluster_cost / HOURS_PER_MONTH
cluster_summary = {
"cluster":
cluster,
"nodes":
nodes,
"pods":
pods,
"namespaces":
namespaces,
"user_pods":
len([p for ns, p in pods if ns not in system_namespaces]),
"master_nodes":
node_count["master"],
"worker_nodes":
node_count[node_label],
"kubelet_versions":
set([
n["kubelet_version"] for n in nodes.values()
if n["role"] == node_label
]),
"worker_instance_types":
set([
n["instance_type"] for n in nodes.values()
if n["role"] == node_label
]),
"worker_instance_is_spot":
any([n["spot"] for n in nodes.values() if n["role"] == node_label]),
"capacity":
cluster_capacity,
"allocatable":
cluster_allocatable,
"requests":
cluster_requests,
"user_requests":
user_requests,
"usage":
cluster_usage,
"cost":
cluster_cost,
"cost_per_user_request_hour": {
"cpu":
0.5 * hourly_cost /
max(user_requests["cpu"], MIN_CPU_USER_REQUESTS),
"memory":
0.5 * hourly_cost /
max(user_requests["memory"] / ONE_GIBI, MIN_MEMORY_USER_REQUESTS),
},
"ingresses": [],
}
get_pod_usage(cluster, pods)
cluster_slack_cost = 0
for pod in pods.values():
usage_cost = max(
pod["usage"]["cpu"] * cost_per_cpu,
pod["usage"]["memory"] * cost_per_memory,
)
pod["slack_cost"] = pod["cost"] - usage_cost
cluster_slack_cost += pod["slack_cost"]
cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost)
with FuturesSession(max_workers=10, session=session) as futures_session:
futures_by_host = {} # hostname -> future
futures = collections.defaultdict(list) # future -> [ingress]
for _ingress in Ingress.objects(cluster.client, namespace=pykube.all):
application = get_application_from_labels(_ingress.labels)
for rule in _ingress.obj["spec"].get("rules", []):
host = rule.get('host', '')
if not application:
# find the application by getting labels from pods
backend_application = find_backend_application(
cluster.client, _ingress, rule)
else:
backend_application = None
ingress = [
_ingress.namespace, _ingress.name, application
or backend_application, host, 0
]
if host and not no_ingress_status:
try:
future = futures_by_host[host]
except KeyError:
future = futures_session.get(f"https://{host}/",
timeout=5)
futures_by_host[host] = future
futures[future].append(ingress)
cluster_summary["ingresses"].append(ingress)
if not no_ingress_status:
logger.info(
f'Waiting for ingress status for {cluster.id} ({cluster.api_server_url})..'
)
for future in concurrent.futures.as_completed(futures):
ingresses = futures[future]
try:
response = future.result()
status = response.status_code
except:
status = 999
for ingress in ingresses:
ingress[4] = status
return cluster_summary
def query_cluster(
cluster,
executor,
system_namespaces,
additional_cost_per_cluster,
alpha_ema,
prev_cluster_summaries,
no_ingress_status,
node_labels,
):
logger.info(f"Querying cluster {cluster.id} ({cluster.api_server_url})..")
pods = {}
nodes = {}
namespaces = {}
for namespace in Namespace.objects(cluster.client):
email = namespace.annotations.get("email")
namespaces[namespace.name] = {
"status": namespace.obj["status"]["phase"],
"email": email,
}
cluster_capacity = collections.defaultdict(float)
cluster_allocatable = collections.defaultdict(float)
cluster_requests = collections.defaultdict(float)
user_requests = collections.defaultdict(float)
cluster_cost = additional_cost_per_cluster
for _node in Node.objects(cluster.client):
node = map_node(_node)
nodes[_node.name] = node
for k, v in node["capacity"].items():
cluster_capacity[k] += v
for k, v in node["allocatable"].items():
cluster_allocatable[k] += v
cluster_cost += node["cost"]
metrics.get_node_usage(cluster, nodes,
prev_cluster_summaries.get("nodes", {}), alpha_ema)
cluster_usage = collections.defaultdict(float)
for node in nodes.values():
for k, v in node["usage"].items():
cluster_usage[k] += v
try:
vpas_by_namespace_label = get_vpas_by_match_labels(cluster.client)
except Exception as e:
logger.warning(f"Failed to query VPAs in cluster {cluster.id}: {e}")
vpas_by_namespace_label = collections.defaultdict(list)
cost_per_cpu = cluster_cost / cluster_allocatable["cpu"]
cost_per_memory = cluster_cost / cluster_allocatable["memory"]
for pod in Pod.objects(cluster.client, namespace=pykube.all):
# ignore unschedulable/completed pods
if not pod_active(pod):
continue
pod_ = map_pod(pod, cost_per_cpu, cost_per_memory)
for k, v in pod_["requests"].items():
cluster_requests[k] += v
if pod.namespace not in system_namespaces:
user_requests[k] += v
node_name = pod.obj["spec"].get("nodeName")
if node_name and node_name in nodes:
for k in ("cpu", "memory"):
nodes[node_name]["requests"][k] += pod_["requests"].get(k, 0)
found_vpa = False
for k, v in pod.labels.items():
vpas = vpas_by_namespace_label[(pod.namespace, k, v)]
for vpa in vpas:
if vpa.matches_pod(pod):
recommendation = new_resources()
container_names = set()
for container in pod.obj["spec"]["containers"]:
container_names.add(container["name"])
for container in vpa.container_recommendations:
# VPA might contain recommendations for containers which are no longer there!
if container["containerName"] in container_names:
for k in ("cpu", "memory"):
recommendation[k] += parse_resource(
container["target"][k])
pod_["recommendation"] = recommendation
found_vpa = True
break
if found_vpa:
break
pods[(pod.namespace, pod.name)] = pod_
hourly_cost = cluster_cost / HOURS_PER_MONTH
cluster_summary = {
"cluster":
cluster,
"nodes":
nodes,
"pods":
pods,
"namespaces":
namespaces,
"user_pods":
len([p for ns, p in pods if ns not in system_namespaces]),
"master_nodes":
len([n for n in nodes.values() if n["role"] == "master"]),
"worker_nodes":
len([n for n in nodes.values() if n["role"] in node_labels]),
"kubelet_versions":
set([
n["kubelet_version"] for n in nodes.values()
if n["role"] in node_labels
]),
"worker_instance_types":
set([
n["instance_type"] for n in nodes.values()
if n["role"] in node_labels
]),
"worker_instance_is_spot":
any([n["spot"] for n in nodes.values() if n["role"] in node_labels]),
"capacity":
cluster_capacity,
"allocatable":
cluster_allocatable,
"requests":
cluster_requests,
"user_requests":
user_requests,
"usage":
cluster_usage,
"cost":
cluster_cost,
"cost_per_user_request_hour": {
"cpu":
0.5 * hourly_cost /
max(user_requests["cpu"], MIN_CPU_USER_REQUESTS),
"memory":
0.5 * hourly_cost /
max(user_requests["memory"] / ONE_GIBI, MIN_MEMORY_USER_REQUESTS),
},
"ingresses": [],
}
metrics.get_pod_usage(cluster, pods,
prev_cluster_summaries.get("pods", {}), alpha_ema)
cluster_slack_cost = 0
for pod in pods.values():
usage_cost = max(
pod["usage"]["cpu"] * cost_per_cpu,
pod["usage"]["memory"] * cost_per_memory,
)
pod["slack_cost"] = pod["cost"] - usage_cost
cluster_slack_cost += pod["slack_cost"]
cluster_summary["slack_cost"] = min(cluster_cost, cluster_slack_cost)
with FuturesSession(max_workers=10, session=session) as futures_session:
futures_by_host = {} # hostname -> future
futures = collections.defaultdict(list) # future -> [ingress]
for _ingress in Ingress.objects(cluster.client, namespace=pykube.all):
application = get_application_from_labels(_ingress.labels)
for rule in _ingress.obj["spec"].get("rules", []):
host = rule.get("host", "")
if not application:
# find the application by getting labels from pods
backend_application = find_backend_application(
cluster.client, _ingress, rule)
else:
backend_application = None
ingress = [
_ingress.namespace,
_ingress.name,
application or backend_application,
host,
0,
]
if host and not no_ingress_status:
try:
future = futures_by_host[host]
except KeyError:
future = futures_session.get(f"https://{host}/",
timeout=5)
futures_by_host[host] = future
futures[future].append(ingress)
cluster_summary["ingresses"].append(ingress)
if not no_ingress_status:
logger.info(
f"Waiting for ingress status for {cluster.id} ({cluster.api_server_url}).."
)
for future in concurrent.futures.as_completed(futures):
ingresses = futures[future]
try:
response = future.result()
status = response.status_code
except Exception:
status = 999
for ingress in ingresses:
ingress[4] = status
return cluster_summary