def top_node(self, node):
custom = self.custom_api
data = custom.get_cluster_custom_object("metrics.k8s.io", "v1beta1", "nodes", node)
node = data['metadata']['name']
cpu = parse_resource(data['usage']['cpu'])
memory = parse_resource(data['usage']['memory'])
return NodeMetric(node=node, cpu=cpu, memory=memory / ONE_GIBI)
def top_pods(self):
custom = self.custom_api
data = custom.list_cluster_custom_object("metrics.k8s.io", "v1beta1", "pods")
usage_by_pod = collections.defaultdict(list)
for pod_data in data['items']:
pod_name = pod_data['metadata']['name']
for container_data in pod_data['containers']:
usage_by_pod[pod_name].append(
{
'pod': container_data['name'],
'cpu': parse_resource(container_data['usage']['cpu']),
'memory': parse_resource(container_data['usage']['memory']) / ONE_MEBI,
}
)
return usage_by_pod
def aggregate_container_resource(pod):
values = {
'memory_limits': 0,
'cpu_limits': 0,
'memory_requests': 0,
'cpu_requests': 0,
}
for container in pod.spec.containers:
limits = getattr(container.resources, 'limits', None)
if limits:
values['memory_limits'] += round(parse_resource(limits.get('memory')) / ONE_GIBI, 1)
values['cpu_limits'] += parse_resource(limits.get('cpu'))
requests = getattr(container.resources, 'requests', None)
if requests:
values['memory_requests'] += round(parse_resource(requests.get('memory')) / ONE_GIBI, 1)
values['cpu_requests'] += parse_resource(requests.get('cpu'))
return values
def get_node(self):
api_instance = self.core_v1_api
nodes = api_instance.list_node()
result = []
for i in nodes.items:
node = dict()
for x in i.status.addresses:
node[x.type] = x.address
for s in i.status.conditions:
if s.type == "Ready":
node['status'] = "Ready" if s.status else "NotReady"
node['kernel'] = i.status.node_info.kernel_version
node['container_runtime'] = i.status.node_info.container_runtime_version
node['cpu'] = i.status.capacity['cpu']
node['memory'] = round(parse_resource(i.status.capacity['memory']) / ONE_GIBI)
result.append(node)
return {"desc": "node", "result": result}
def get_node(self):
api_instance = CoreV1Api(self.api_client)
nodes = api_instance.list_node()
result = []
for i in nodes.items:
node = dict()
node['name'] = i.metadata.name
for s in i.status.conditions:
if s.type == "Ready":
node['status'] = "Ready" if s.status else "NotReady"
node['kernel'] = i.status.node_info.kernel_version
node[
'container_runtime'] = i.status.node_info.container_runtime_version
node['cpu'] = i.status.capacity['cpu']
node['memory'] = round(
parse_resource(i.status.capacity['memory']) / ONE_GIBI)
result.append(node)
return {"desc": "node", "result": result}
def __get_checkout_for_tenant_and_partitions(self, objs):
for key in objs.keys():
cpu_request_total = parse_resource(
objs[key]['status']['hard']['requests.cpu'])
cpu_request_used = parse_resource(
objs[key]['status']['used']['requests.cpu'])
cpu_request_unused = cpu_request_total - cpu_request_used
cpu_request_status = True if cpu_request_used < cpu_request_total * 0.8 else False
mem_request_total = parse_resource(
objs[key]['status']['hard']['requests.memory'])
mem_request_used = parse_resource(
objs[key]['status']['used']['requests.memory'])
mem_request_unused = mem_request_total - mem_request_used
mem_request_status = True if mem_request_used < mem_request_total * 0.8 else False
cpu_limit_total = parse_resource(
objs[key]['status']['hard']['limits.cpu'])
cpu_limit_used = parse_resource(
objs[key]['status']['used']['limits.cpu'])
cpu_limit_unused = cpu_limit_total - cpu_limit_used
cpu_limit_status = True if cpu_limit_used < cpu_limit_total * 0.8 else False
mem_limit_total = parse_resource(
objs[key]['status']['hard']['limits.memory'])
mem_limit_used = parse_resource(
objs[key]['status']['used']['limits.memory'])
mem_limit_unused = mem_limit_total - mem_limit_used
mem_limit_status = True if mem_limit_used < mem_limit_total * 0.8 else False
data = dict()
data[key] = self.__get_resource_json(
cpu_request_total, cpu_request_used, cpu_request_unused,
cpu_request_status, cpu_limit_total, cpu_limit_used,
cpu_limit_unused, cpu_limit_status, mem_request_total,
mem_request_used, mem_request_unused, mem_request_status,
mem_limit_total, mem_limit_used, mem_limit_unused,
mem_limit_status)
return data
def check_clusters_quotas(self):
logger.info(f"check {self.cluster_name} cluster quota")
clusters = self.get_clusterquotas()
physical_cpu_total = parse_resource(
clusters['system']['status']['physical']['capacity']['cpu'])
physical_cpu_unused = parse_resource(
clusters['system']['status']['physical']['allocatable']['cpu'])
physical_cpu_used = physical_cpu_total - physical_cpu_unused
physical_cpu_status = True if physical_cpu_used < physical_cpu_total * 0.8 else False
physical_mem_total = parse_resource(
clusters['system']['status']['physical']['capacity']['memory'])
physical_mem_unused = parse_resource(
clusters['system']['status']['physical']['allocatable']['memory'])
physical_mem_used = physical_mem_total - physical_mem_unused
physical_mem_status = True if physical_mem_used < physical_mem_total * 0.8 else False
logical_cpu_request_total = parse_resource(
clusters['system']['status']['logical']['total']['requests.cpu'])
logical_cpu_request_used = parse_resource(
clusters['system']['status']['logical']['allocated']
['requests.cpu'])
logical_cpu_request_unused = logical_cpu_request_total - logical_cpu_request_used
logical_cpu_request_status = True if logical_cpu_request_used < logical_cpu_request_total * 0.8 else False
logical_cpu_limit_total = parse_resource(
clusters['system']['status']['logical']['total']['limits.cpu'])
logical_cpu_limit_used = parse_resource(
clusters['system']['status']['logical']['allocated']['limits.cpu'])
logical_cpu_limit_unused = logical_cpu_limit_total - logical_cpu_limit_used
logical_cpu_limit_status = True if logical_cpu_limit_used < logical_cpu_limit_total * 0.8 else False
logical_mem_request_total = parse_resource(
clusters['system']['status']['logical']['total']
['requests.memory'])
logical_mem_request_used = parse_resource(
clusters['system']['status']['logical']['allocated']
['requests.memory'])
logical_mem_request_unused = logical_mem_request_total - logical_mem_request_used
logical_mem_request_status = True if logical_mem_request_used < logical_mem_request_total * 0.8 else False
logical_mem_limit_total = parse_resource(
clusters['system']['status']['logical']['total']['limits.memory'])
logical_mem_limit_used = parse_resource(
clusters['system']['status']['logical']['allocated']
['limits.memory'])
logical_mem_limit_unused = logical_mem_limit_total - logical_mem_limit_used
logical_mem_limit_status = True if logical_mem_limit_used < logical_mem_limit_total * 0.8 else False
self.checkout[self.cluster_name]['cluster_quota'] = {
"physical": {
"cpu": {
"data": {
"total": physical_cpu_total,
"unused": physical_cpu_unused,
"used": physical_cpu_used
},
"status": physical_cpu_status
},
"mem": {
"data": {
"total": physical_mem_total,
"unused": physical_mem_unused,
"used": physical_mem_used
},
"status": physical_mem_status
}
},
"logical":
self.__get_resource_json(
logical_cpu_request_total, logical_cpu_request_used,
logical_cpu_request_unused, logical_cpu_request_status,
logical_cpu_limit_total, logical_cpu_limit_used,
logical_cpu_limit_unused, logical_cpu_limit_status,
logical_mem_request_total, logical_mem_request_used,
logical_mem_request_unused, logical_mem_request_status,
logical_mem_limit_total, logical_mem_limit_used,
logical_mem_limit_unused, logical_mem_limit_status)
}