def test_get_maas_stats_no_machines(self):
expected = {
"controllers": {"regionracks": 0, "regions": 0, "racks": 0},
"nodes": {"machines": 0, "devices": 0},
"machine_stats": {
"total_cpu": 0,
"total_mem": 0,
"total_storage": 0,
},
"machine_status": {
"new": 0,
"ready": 0,
"allocated": 0,
"deployed": 0,
"commissioning": 0,
"testing": 0,
"deploying": 0,
"failed_deployment": 0,
"failed_commissioning": 0,
"failed_testing": 0,
"broken": 0,
},
"network_stats": {
"spaces": 0,
"fabrics": 1,
"vlans": 1,
"subnets_v4": 0,
"subnets_v6": 0,
},
}
self.assertEqual(json.loads(get_maas_stats()), expected)
def test_get_maas_stats(self):
# Make one component of everything
factory.make_RegionRackController()
factory.make_RegionController()
factory.make_RackController()
factory.make_Machine(cpu_count=2, memory=200)
factory.make_Machine(cpu_count=3, memory=100)
factory.make_Device()
stats = get_maas_stats()
machine_stats = get_machine_stats()
# Due to floating point calculation subtleties, sometimes the value the
# database returns is off by one compared to the value Python
# calculates, so just get it directly from the database for the test.
total_storage = machine_stats['total_storage']
compare = {
"controllers": {
"regionracks": 1,
"regions": 1,
"racks": 1,
},
"nodes": {
"machines": 2,
"devices": 1,
},
"machine_stats": {
"total_cpu": 5,
"total_mem": 300,
"total_storage": total_storage,
},
}
self.assertEquals(stats, json.dumps(compare))
def test_get_maas_stats(self):
# Make one component of everything
factory.make_RegionRackController()
factory.make_RegionController()
factory.make_RackController()
factory.make_Machine(cpu_count=2, memory=200, status=NODE_STATUS.READY)
factory.make_Machine(status=NODE_STATUS.READY)
factory.make_Machine(status=NODE_STATUS.NEW)
for _ in range(4):
factory.make_Machine(status=NODE_STATUS.ALLOCATED)
factory.make_Machine(
cpu_count=3, memory=100, status=NODE_STATUS.FAILED_DEPLOYMENT
)
for _ in range(2):
factory.make_Machine(status=NODE_STATUS.DEPLOYED)
factory.make_Device()
factory.make_Device()
subnets = Subnet.objects.all()
v4 = [net for net in subnets if net.get_ip_version() == 4]
v6 = [net for net in subnets if net.get_ip_version() == 6]
stats = get_maas_stats()
machine_stats = get_machine_stats()
# Due to floating point calculation subtleties, sometimes the value the
# database returns is off by one compared to the value Python
# calculates, so just get it directly from the database for the test.
total_storage = machine_stats["total_storage"]
expected = {
"controllers": {"regionracks": 1, "regions": 1, "racks": 1},
"nodes": {"machines": 10, "devices": 2},
"machine_stats": {
"total_cpu": 5,
"total_mem": 300,
"total_storage": total_storage,
},
"machine_status": {
"new": 1,
"ready": 2,
"allocated": 4,
"deployed": 2,
"commissioning": 0,
"testing": 0,
"deploying": 0,
"failed_deployment": 1,
"failed_commissioning": 0,
"failed_testing": 0,
"broken": 0,
},
"network_stats": {
"spaces": Space.objects.count(),
"fabrics": Fabric.objects.count(),
"vlans": VLAN.objects.count(),
"subnets_v4": len(v4),
"subnets_v6": len(v6),
},
}
self.assertEqual(json.loads(stats), expected)
def test_get_request_params_returns_params(self):
factory.make_RegionRackController()
params = {
"data":
base64.b64encode(json.dumps(get_maas_stats()).encode()).decode()
}
self.assertEquals(params, get_request_params())
def get_stats_for_prometheus():
registry = prom_cli.CollectorRegistry()
stats = json.loads(get_maas_stats())
architectures = get_machines_by_architecture()
pods = get_kvm_pods_stats()
# Gather counter for machines per status
counter = prom_cli.Gauge("machine_status",
"Number of machines per status", ["status"],
registry=registry)
for status, machines in stats['machine_status'].items():
counter.labels(status).set(machines)
# Gather counter for number of nodes (controllers/machine/devices)
counter = prom_cli.Gauge(
"nodes",
"Number of nodes per type (e.g. racks, machines, etc).", ["type"],
registry=registry)
for ctype, number in stats['controllers'].items():
counter.labels(ctype).set(number)
for ctype, number in stats['nodes'].items():
counter.labels(ctype).set(number)
# Gather counter for networks
counter = prom_cli.Gauge("networks",
"General statistics for subnets.", ["type"],
registry=registry)
for stype, number in stats['network_stats'].items():
counter.labels(stype).set(number)
# Gather overall amount of machine resources
counter = prom_cli.Gauge("machine_resources",
"Amount of combined resources for all machines",
["resource"],
registry=registry)
for resource, value in stats['machine_stats'].items():
counter.labels(resource).set(value)
# Gather all stats for pods
counter = prom_cli.Gauge("kvm_pods",
"General stats for KVM pods", ["type"],
registry=registry)
for resource, value in pods.items():
if isinstance(value, dict):
for r, v in value.items():
counter.labels("%s_%s" % (resource, r)).set(v)
else:
counter.labels(resource).set(value)
# Gather statistics for architectures
if len(architectures.keys()) > 0:
counter = prom_cli.Gauge("machine_arches",
"Number of machines per architecture.",
["arches"],
registry=registry)
for arch, machines in architectures.items():
counter.labels(arch).set(machines)
return registry
def update_prometheus_stats(metrics: PrometheusMetrics):
"""Update metrics in a PrometheusMetrics based on database values."""
stats = json.loads(get_maas_stats())
architectures = get_machines_by_architecture()
pods = get_kvm_pods_stats()
# Gather counter for machines per status
for status, machines in stats['machine_status'].items():
metrics.update('machine_status',
'set',
value=machines,
labels={'status': status})
# Gather counter for number of nodes (controllers/machine/devices)
for ctype, number in stats['controllers'].items():
metrics.update('nodes', 'set', value=number, labels={'type': ctype})
for ctype, number in stats['nodes'].items():
metrics.update('nodes', 'set', value=number, labels={'type': ctype})
# Gather counter for networks
for stype, number in stats['network_stats'].items():
metrics.update('networks', 'set', value=number, labels={'type': stype})
# Gather overall amount of machine resources
for resource, value in stats['machine_stats'].items():
metrics.update('machine_resources',
'set',
value=value,
labels={'resource': resource})
# Gather all stats for pods
for resource, value in pods.items():
if isinstance(value, dict):
for r, v in value.items():
metrics.update('kvm_pods',
'set',
value=v,
labels={'type': '{}_{}'.format(resource, r)})
else:
metrics.update('kvm_pods',
'set',
value=value,
labels={'type': resource})
# Gather statistics for architectures
if len(architectures.keys()) > 0:
for arch, machines in architectures.items():
metrics.update('machine_arches',
'set',
value=machines,
labels={'arches': arch})
return metrics
def test_get_maas_stats(self):
# Make one component of everything
factory.make_RegionRackController()
factory.make_RegionController()
factory.make_RackController()
factory.make_Machine()
factory.make_Device()
stats = get_maas_stats()
compare = {
"controllers": {
"regionracks": 1,
"regions": 1,
"racks": 1,
},
"nodes": {
"machines": 1,
"devices": 1,
},
}
self.assertEquals(stats, json.dumps(compare))
def update_prometheus_stats(metrics: PrometheusMetrics):
"""Update metrics in a PrometheusMetrics based on database values."""
stats = json.loads(get_maas_stats())
architectures = get_machines_by_architecture()
pods = get_kvm_pods_stats()
# Gather counter for machines per status
for status, machines in stats["machine_status"].items():
metrics.update("maas_machines",
"set",
value=machines,
labels={"status": status})
# Gather counter for number of nodes (controllers/machine/devices)
for ctype, number in stats["controllers"].items():
metrics.update("maas_nodes",
"set",
value=number,
labels={"type": ctype})
for ctype, number in stats["nodes"].items():
metrics.update("maas_nodes",
"set",
value=number,
labels={"type": ctype})
# Gather counter for networks
for stype, number in stats["network_stats"].items():
metrics.update("maas_net_{}".format(stype), "set", value=number)
# Gather overall amount of machine resources
for resource, value in stats["machine_stats"].items():
metrics.update("maas_machines_{}".format(resource), "set", value=value)
# Gather all stats for pods
metrics.update("maas_kvm_pods", "set", value=pods["kvm_pods"])
metrics.update("maas_kvm_machines", "set", value=pods["kvm_machines"])
for metric in ("cores", "memory", "storage"):
metrics.update(
"maas_kvm_{}".format(metric),
"set",
value=pods["kvm_available_resources"][metric],
labels={"status": "available"},
)
metrics.update(
"maas_kvm_{}".format(metric),
"set",
value=pods["kvm_utilized_resources"][metric],
labels={"status": "used"},
)
metrics.update(
"maas_kvm_overcommit_cores",
"set",
value=pods["kvm_available_resources"]["over_cores"],
)
metrics.update(
"maas_kvm_overcommit_memory",
"set",
value=pods["kvm_available_resources"]["over_memory"],
)
# Gather statistics for architectures
if len(architectures.keys()) > 0:
for arch, machines in architectures.items():
metrics.update(
"maas_machine_arches",
"set",
value=machines,
labels={"arches": arch},
)
# Update metrics for subnets
for cidr, stats in get_subnets_utilisation_stats().items():
for status in ("available", "unavailable"):
metrics.update(
"maas_net_subnet_ip_count",
"set",
value=stats[status],
labels={
"cidr": cidr,
"status": status
},
)
metrics.update(
"maas_net_subnet_ip_static",
"set",
value=stats["static"],
labels={"cidr": cidr},
)
for addr_type in ("dynamic", "reserved"):
metric_name = "maas_net_subnet_ip_{}".format(addr_type)
for status in ("available", "used"):
metrics.update(
metric_name,
"set",
value=stats["{}_{}".format(addr_type, status)],
labels={
"cidr": cidr,
"status": status
},
)
return metrics
def test_get_maas_stats(self):
# Make one component of everything
factory.make_RegionRackController()
factory.make_RegionController()
factory.make_RackController()
factory.make_Machine(cpu_count=2, memory=200, status=4)
factory.make_Machine(cpu_count=3, memory=100, status=11)
factory.make_Device()
subnets = Subnet.objects.all()
v4 = [net for net in subnets if net.get_ip_version() == 4]
v6 = [net for net in subnets if net.get_ip_version() == 6]
stats = get_maas_stats()
machine_stats = get_machine_stats()
# Due to floating point calculation subtleties, sometimes the value the
# database returns is off by one compared to the value Python
# calculates, so just get it directly from the database for the test.
total_storage = machine_stats['total_storage']
node_status = Node.objects.values_list('status', flat=True)
node_status = Counter(node_status)
compare = {
"controllers": {
"regionracks": 1,
"regions": 1,
"racks": 1,
},
"nodes": {
"machines": 2,
"devices": 1,
},
"machine_stats": {
"total_cpu": 5,
"total_mem": 300,
"total_storage": total_storage,
},
"machine_status": {
"new":
node_status.get(NODE_STATUS.NEW, 0),
"ready":
node_status.get(NODE_STATUS.READY, 0),
"allocated":
node_status.get(NODE_STATUS.ALLOCATED, 0),
"deployed":
node_status.get(NODE_STATUS.DEPLOYED, 0),
"commissioning":
node_status.get(NODE_STATUS.COMMISSIONING, 0),
"testing":
node_status.get(NODE_STATUS.TESTING, 0),
"deploying":
node_status.get(NODE_STATUS.DEPLOYING, 0),
"failed_deployment":
node_status.get(NODE_STATUS.FAILED_DEPLOYMENT, 0),
"failed_commissioning":
node_status.get(NODE_STATUS.COMMISSIONING, 0),
"failed_testing":
node_status.get(NODE_STATUS.FAILED_TESTING, 0),
"broken":
node_status.get(NODE_STATUS.BROKEN, 0),
},
"network_stats": {
"spaces": Space.objects.count(),
"fabrics": Fabric.objects.count(),
"vlans": VLAN.objects.count(),
"subnets_v4": len(v4),
"subnets_v6": len(v6),
},
}
self.assertEquals(stats, json.dumps(compare))
def update_prometheus_stats(metrics: PrometheusMetrics):
"""Update metrics in a PrometheusMetrics based on database values."""
stats = json.loads(get_maas_stats())
architectures = get_machines_by_architecture()
pods = get_kvm_pods_stats()
# Gather counter for machines per status
for status, machines in stats['machine_status'].items():
metrics.update(
'maas_machines', 'set', value=machines,
labels={'status': status})
# Gather counter for number of nodes (controllers/machine/devices)
for ctype, number in stats['controllers'].items():
metrics.update(
'maas_nodes', 'set', value=number, labels={'type': ctype})
for ctype, number in stats['nodes'].items():
metrics.update(
'maas_nodes', 'set', value=number, labels={'type': ctype})
# Gather counter for networks
for stype, number in stats['network_stats'].items():
metrics.update('maas_net_{}'.format(stype), 'set', value=number)
# Gather overall amount of machine resources
for resource, value in stats['machine_stats'].items():
metrics.update('maas_machines_{}'.format(resource), 'set', value=value)
# Gather all stats for pods
metrics.update('maas_kvm_pods', 'set', value=pods['kvm_pods'])
metrics.update('maas_kvm_machines', 'set', value=pods['kvm_machines'])
for metric in ('cores', 'memory', 'storage'):
metrics.update(
'maas_kvm_{}'.format(metric), 'set',
value=pods['kvm_available_resources'][metric],
labels={'status': 'available'})
metrics.update(
'maas_kvm_{}'.format(metric), 'set',
value=pods['kvm_utilized_resources'][metric],
labels={'status': 'used'})
metrics.update(
'maas_kvm_overcommit_cores', 'set',
value=pods['kvm_available_resources']['over_cores'])
metrics.update(
'maas_kvm_overcommit_memory', 'set',
value=pods['kvm_available_resources']['over_memory'])
# Gather statistics for architectures
if len(architectures.keys()) > 0:
for arch, machines in architectures.items():
metrics.update(
'maas_machine_arches', 'set', value=machines,
labels={'arches': arch})
# Update metrics for subnets
for cidr, stats in get_subnets_utilisation_stats().items():
for status in ('available', 'unavailable'):
metrics.update(
'maas_net_subnet_ip_count', 'set',
value=stats[status],
labels={'cidr': cidr, 'status': status})
metrics.update(
'maas_net_subnet_ip_static', 'set',
value=stats['static'], labels={'cidr': cidr})
for addr_type in ('dynamic', 'reserved'):
metric_name = 'maas_net_subnet_ip_{}'.format(addr_type)
for status in ('available', 'used'):
metrics.update(
metric_name, 'set',
value=stats['{}_{}'.format(addr_type, status)],
labels={'cidr': cidr, 'status': status})
return metrics
def test_get_maas_stats_no_machines(self):
expected = {
"controllers": {
"regionracks": 0,
"regions": 0,
"racks": 0
},
"nodes": {
"machines": 0,
"devices": 0
},
"machine_stats": {
"total_cpu": 0,
"total_mem": 0,
"total_storage": 0,
},
"machine_status": {
"new": 0,
"ready": 0,
"allocated": 0,
"deployed": 0,
"commissioning": 0,
"testing": 0,
"deploying": 0,
"failed_deployment": 0,
"failed_commissioning": 0,
"failed_testing": 0,
"broken": 0,
},
"network_stats": {
"spaces": 0,
"fabrics": Fabric.objects.count(),
"vlans": VLAN.objects.count(),
"subnets_v4": 0,
"subnets_v6": 0,
},
"vm_hosts": {
"lxd": {
"vm_hosts": 0,
"vms": 0,
"available_resources": {
"cores": 0,
"memory": 0,
"over_cores": 0.0,
"over_memory": 0.0,
"storage": 0,
},
"utilized_resources": {
"cores": 0,
"memory": 0,
"storage": 0,
},
},
"virsh": {
"vm_hosts": 0,
"vms": 0,
"available_resources": {
"cores": 0,
"memory": 0,
"over_cores": 0.0,
"over_memory": 0.0,
"storage": 0,
},
"utilized_resources": {
"cores": 0,
"memory": 0,
"storage": 0,
},
},
},
"workload_annotations": {
"annotated_machines": 0,
"total_annotations": 0,
"unique_keys": 0,
"unique_values": 0,
},
}
self.assertEqual(get_maas_stats(), expected)
def test_get_maas_stats(self):
# Make one component of everything
factory.make_RegionRackController()
factory.make_RegionController()
factory.make_RackController()
factory.make_Machine(cpu_count=2, memory=200, status=NODE_STATUS.READY)
factory.make_Machine(status=NODE_STATUS.READY)
factory.make_Machine(status=NODE_STATUS.NEW)
for _ in range(4):
factory.make_Machine(status=NODE_STATUS.ALLOCATED)
factory.make_Machine(cpu_count=3,
memory=100,
status=NODE_STATUS.FAILED_DEPLOYMENT)
factory.make_Machine(status=NODE_STATUS.DEPLOYED)
deployed_machine = factory.make_Machine(status=NODE_STATUS.DEPLOYED)
OwnerData.objects.set_owner_data(deployed_machine, {"foo": "bar"})
factory.make_Device()
factory.make_Device()
self.make_pod(cpu=10, mem=100, pod_type="lxd")
self.make_pod(cpu=20, mem=200, pod_type="virsh")
subnets = Subnet.objects.all()
v4 = [net for net in subnets if net.get_ip_version() == 4]
v6 = [net for net in subnets if net.get_ip_version() == 6]
stats = get_maas_stats()
machine_stats = get_machine_stats()
# Due to floating point calculation subtleties, sometimes the value the
# database returns is off by one compared to the value Python
# calculates, so just get it directly from the database for the test.
total_storage = machine_stats["total_storage"]
expected = {
"controllers": {
"regionracks": 1,
"regions": 1,
"racks": 1
},
"nodes": {
"machines": 10,
"devices": 2
},
"machine_stats": {
"total_cpu": 5,
"total_mem": 300,
"total_storage": total_storage,
},
"machine_status": {
"new": 1,
"ready": 2,
"allocated": 4,
"deployed": 2,
"commissioning": 0,
"testing": 0,
"deploying": 0,
"failed_deployment": 1,
"failed_commissioning": 0,
"failed_testing": 0,
"broken": 0,
},
"network_stats": {
"spaces": Space.objects.count(),
"fabrics": Fabric.objects.count(),
"vlans": VLAN.objects.count(),
"subnets_v4": len(v4),
"subnets_v6": len(v6),
},
"vm_hosts": {
"lxd": {
"vm_hosts": 1,
"vms": 0,
"available_resources": {
"cores": 10,
"memory": 100,
"over_cores": 10.0,
"over_memory": 100.0,
"storage": 0,
},
"utilized_resources": {
"cores": 0,
"memory": 0,
"storage": 0,
},
},
"virsh": {
"vm_hosts": 1,
"vms": 0,
"available_resources": {
"cores": 20,
"memory": 200,
"over_cores": 20.0,
"over_memory": 200.0,
"storage": 0,
},
"utilized_resources": {
"cores": 0,
"memory": 0,
"storage": 0,
},
},
},
"workload_annotations": {
"annotated_machines": 1,
"total_annotations": 1,
"unique_keys": 1,
"unique_values": 1,
},
}
self.assertEqual(stats, expected)
