def test_get_maas_stats_no_machines(self): expected = { "controllers": {"regionracks": 0, "regions": 0, "racks": 0}, "nodes": {"machines": 0, "devices": 0}, "machine_stats": { "total_cpu": 0, "total_mem": 0, "total_storage": 0, }, "machine_status": { "new": 0, "ready": 0, "allocated": 0, "deployed": 0, "commissioning": 0, "testing": 0, "deploying": 0, "failed_deployment": 0, "failed_commissioning": 0, "failed_testing": 0, "broken": 0, }, "network_stats": { "spaces": 0, "fabrics": 1, "vlans": 1, "subnets_v4": 0, "subnets_v6": 0, }, } self.assertEqual(json.loads(get_maas_stats()), expected)
def test_get_maas_stats(self): # Make one component of everything factory.make_RegionRackController() factory.make_RegionController() factory.make_RackController() factory.make_Machine(cpu_count=2, memory=200) factory.make_Machine(cpu_count=3, memory=100) factory.make_Device() stats = get_maas_stats() machine_stats = get_machine_stats() # Due to floating point calculation subtleties, sometimes the value the # database returns is off by one compared to the value Python # calculates, so just get it directly from the database for the test. total_storage = machine_stats['total_storage'] compare = { "controllers": { "regionracks": 1, "regions": 1, "racks": 1, }, "nodes": { "machines": 2, "devices": 1, }, "machine_stats": { "total_cpu": 5, "total_mem": 300, "total_storage": total_storage, }, } self.assertEquals(stats, json.dumps(compare))
def test_get_maas_stats(self): # Make one component of everything factory.make_RegionRackController() factory.make_RegionController() factory.make_RackController() factory.make_Machine(cpu_count=2, memory=200, status=NODE_STATUS.READY) factory.make_Machine(status=NODE_STATUS.READY) factory.make_Machine(status=NODE_STATUS.NEW) for _ in range(4): factory.make_Machine(status=NODE_STATUS.ALLOCATED) factory.make_Machine( cpu_count=3, memory=100, status=NODE_STATUS.FAILED_DEPLOYMENT ) for _ in range(2): factory.make_Machine(status=NODE_STATUS.DEPLOYED) factory.make_Device() factory.make_Device() subnets = Subnet.objects.all() v4 = [net for net in subnets if net.get_ip_version() == 4] v6 = [net for net in subnets if net.get_ip_version() == 6] stats = get_maas_stats() machine_stats = get_machine_stats() # Due to floating point calculation subtleties, sometimes the value the # database returns is off by one compared to the value Python # calculates, so just get it directly from the database for the test. total_storage = machine_stats["total_storage"] expected = { "controllers": {"regionracks": 1, "regions": 1, "racks": 1}, "nodes": {"machines": 10, "devices": 2}, "machine_stats": { "total_cpu": 5, "total_mem": 300, "total_storage": total_storage, }, "machine_status": { "new": 1, "ready": 2, "allocated": 4, "deployed": 2, "commissioning": 0, "testing": 0, "deploying": 0, "failed_deployment": 1, "failed_commissioning": 0, "failed_testing": 0, "broken": 0, }, "network_stats": { "spaces": Space.objects.count(), "fabrics": Fabric.objects.count(), "vlans": VLAN.objects.count(), "subnets_v4": len(v4), "subnets_v6": len(v6), }, } self.assertEqual(json.loads(stats), expected)
def test_get_request_params_returns_params(self): factory.make_RegionRackController() params = { "data": base64.b64encode(json.dumps(get_maas_stats()).encode()).decode() } self.assertEquals(params, get_request_params())
def get_stats_for_prometheus(): registry = prom_cli.CollectorRegistry() stats = json.loads(get_maas_stats()) architectures = get_machines_by_architecture() pods = get_kvm_pods_stats() # Gather counter for machines per status counter = prom_cli.Gauge("machine_status", "Number of machines per status", ["status"], registry=registry) for status, machines in stats['machine_status'].items(): counter.labels(status).set(machines) # Gather counter for number of nodes (controllers/machine/devices) counter = prom_cli.Gauge( "nodes", "Number of nodes per type (e.g. racks, machines, etc).", ["type"], registry=registry) for ctype, number in stats['controllers'].items(): counter.labels(ctype).set(number) for ctype, number in stats['nodes'].items(): counter.labels(ctype).set(number) # Gather counter for networks counter = prom_cli.Gauge("networks", "General statistics for subnets.", ["type"], registry=registry) for stype, number in stats['network_stats'].items(): counter.labels(stype).set(number) # Gather overall amount of machine resources counter = prom_cli.Gauge("machine_resources", "Amount of combined resources for all machines", ["resource"], registry=registry) for resource, value in stats['machine_stats'].items(): counter.labels(resource).set(value) # Gather all stats for pods counter = prom_cli.Gauge("kvm_pods", "General stats for KVM pods", ["type"], registry=registry) for resource, value in pods.items(): if isinstance(value, dict): for r, v in value.items(): counter.labels("%s_%s" % (resource, r)).set(v) else: counter.labels(resource).set(value) # Gather statistics for architectures if len(architectures.keys()) > 0: counter = prom_cli.Gauge("machine_arches", "Number of machines per architecture.", ["arches"], registry=registry) for arch, machines in architectures.items(): counter.labels(arch).set(machines) return registry
def update_prometheus_stats(metrics: PrometheusMetrics): """Update metrics in a PrometheusMetrics based on database values.""" stats = json.loads(get_maas_stats()) architectures = get_machines_by_architecture() pods = get_kvm_pods_stats() # Gather counter for machines per status for status, machines in stats['machine_status'].items(): metrics.update('machine_status', 'set', value=machines, labels={'status': status}) # Gather counter for number of nodes (controllers/machine/devices) for ctype, number in stats['controllers'].items(): metrics.update('nodes', 'set', value=number, labels={'type': ctype}) for ctype, number in stats['nodes'].items(): metrics.update('nodes', 'set', value=number, labels={'type': ctype}) # Gather counter for networks for stype, number in stats['network_stats'].items(): metrics.update('networks', 'set', value=number, labels={'type': stype}) # Gather overall amount of machine resources for resource, value in stats['machine_stats'].items(): metrics.update('machine_resources', 'set', value=value, labels={'resource': resource}) # Gather all stats for pods for resource, value in pods.items(): if isinstance(value, dict): for r, v in value.items(): metrics.update('kvm_pods', 'set', value=v, labels={'type': '{}_{}'.format(resource, r)}) else: metrics.update('kvm_pods', 'set', value=value, labels={'type': resource}) # Gather statistics for architectures if len(architectures.keys()) > 0: for arch, machines in architectures.items(): metrics.update('machine_arches', 'set', value=machines, labels={'arches': arch}) return metrics
def test_get_maas_stats(self): # Make one component of everything factory.make_RegionRackController() factory.make_RegionController() factory.make_RackController() factory.make_Machine() factory.make_Device() stats = get_maas_stats() compare = { "controllers": { "regionracks": 1, "regions": 1, "racks": 1, }, "nodes": { "machines": 1, "devices": 1, }, } self.assertEquals(stats, json.dumps(compare))
def update_prometheus_stats(metrics: PrometheusMetrics): """Update metrics in a PrometheusMetrics based on database values.""" stats = json.loads(get_maas_stats()) architectures = get_machines_by_architecture() pods = get_kvm_pods_stats() # Gather counter for machines per status for status, machines in stats["machine_status"].items(): metrics.update("maas_machines", "set", value=machines, labels={"status": status}) # Gather counter for number of nodes (controllers/machine/devices) for ctype, number in stats["controllers"].items(): metrics.update("maas_nodes", "set", value=number, labels={"type": ctype}) for ctype, number in stats["nodes"].items(): metrics.update("maas_nodes", "set", value=number, labels={"type": ctype}) # Gather counter for networks for stype, number in stats["network_stats"].items(): metrics.update("maas_net_{}".format(stype), "set", value=number) # Gather overall amount of machine resources for resource, value in stats["machine_stats"].items(): metrics.update("maas_machines_{}".format(resource), "set", value=value) # Gather all stats for pods metrics.update("maas_kvm_pods", "set", value=pods["kvm_pods"]) metrics.update("maas_kvm_machines", "set", value=pods["kvm_machines"]) for metric in ("cores", "memory", "storage"): metrics.update( "maas_kvm_{}".format(metric), "set", value=pods["kvm_available_resources"][metric], labels={"status": "available"}, ) metrics.update( "maas_kvm_{}".format(metric), "set", value=pods["kvm_utilized_resources"][metric], labels={"status": "used"}, ) metrics.update( "maas_kvm_overcommit_cores", "set", value=pods["kvm_available_resources"]["over_cores"], ) metrics.update( "maas_kvm_overcommit_memory", "set", value=pods["kvm_available_resources"]["over_memory"], ) # Gather statistics for architectures if len(architectures.keys()) > 0: for arch, machines in architectures.items(): metrics.update( "maas_machine_arches", "set", value=machines, labels={"arches": arch}, ) # Update metrics for subnets for cidr, stats in get_subnets_utilisation_stats().items(): for status in ("available", "unavailable"): metrics.update( "maas_net_subnet_ip_count", "set", value=stats[status], labels={ "cidr": cidr, "status": status }, ) metrics.update( "maas_net_subnet_ip_static", "set", value=stats["static"], labels={"cidr": cidr}, ) for addr_type in ("dynamic", "reserved"): metric_name = "maas_net_subnet_ip_{}".format(addr_type) for status in ("available", "used"): metrics.update( metric_name, "set", value=stats["{}_{}".format(addr_type, status)], labels={ "cidr": cidr, "status": status }, ) return metrics
def test_get_maas_stats(self): # Make one component of everything factory.make_RegionRackController() factory.make_RegionController() factory.make_RackController() factory.make_Machine(cpu_count=2, memory=200, status=4) factory.make_Machine(cpu_count=3, memory=100, status=11) factory.make_Device() subnets = Subnet.objects.all() v4 = [net for net in subnets if net.get_ip_version() == 4] v6 = [net for net in subnets if net.get_ip_version() == 6] stats = get_maas_stats() machine_stats = get_machine_stats() # Due to floating point calculation subtleties, sometimes the value the # database returns is off by one compared to the value Python # calculates, so just get it directly from the database for the test. total_storage = machine_stats['total_storage'] node_status = Node.objects.values_list('status', flat=True) node_status = Counter(node_status) compare = { "controllers": { "regionracks": 1, "regions": 1, "racks": 1, }, "nodes": { "machines": 2, "devices": 1, }, "machine_stats": { "total_cpu": 5, "total_mem": 300, "total_storage": total_storage, }, "machine_status": { "new": node_status.get(NODE_STATUS.NEW, 0), "ready": node_status.get(NODE_STATUS.READY, 0), "allocated": node_status.get(NODE_STATUS.ALLOCATED, 0), "deployed": node_status.get(NODE_STATUS.DEPLOYED, 0), "commissioning": node_status.get(NODE_STATUS.COMMISSIONING, 0), "testing": node_status.get(NODE_STATUS.TESTING, 0), "deploying": node_status.get(NODE_STATUS.DEPLOYING, 0), "failed_deployment": node_status.get(NODE_STATUS.FAILED_DEPLOYMENT, 0), "failed_commissioning": node_status.get(NODE_STATUS.COMMISSIONING, 0), "failed_testing": node_status.get(NODE_STATUS.FAILED_TESTING, 0), "broken": node_status.get(NODE_STATUS.BROKEN, 0), }, "network_stats": { "spaces": Space.objects.count(), "fabrics": Fabric.objects.count(), "vlans": VLAN.objects.count(), "subnets_v4": len(v4), "subnets_v6": len(v6), }, } self.assertEquals(stats, json.dumps(compare))
def update_prometheus_stats(metrics: PrometheusMetrics): """Update metrics in a PrometheusMetrics based on database values.""" stats = json.loads(get_maas_stats()) architectures = get_machines_by_architecture() pods = get_kvm_pods_stats() # Gather counter for machines per status for status, machines in stats['machine_status'].items(): metrics.update( 'maas_machines', 'set', value=machines, labels={'status': status}) # Gather counter for number of nodes (controllers/machine/devices) for ctype, number in stats['controllers'].items(): metrics.update( 'maas_nodes', 'set', value=number, labels={'type': ctype}) for ctype, number in stats['nodes'].items(): metrics.update( 'maas_nodes', 'set', value=number, labels={'type': ctype}) # Gather counter for networks for stype, number in stats['network_stats'].items(): metrics.update('maas_net_{}'.format(stype), 'set', value=number) # Gather overall amount of machine resources for resource, value in stats['machine_stats'].items(): metrics.update('maas_machines_{}'.format(resource), 'set', value=value) # Gather all stats for pods metrics.update('maas_kvm_pods', 'set', value=pods['kvm_pods']) metrics.update('maas_kvm_machines', 'set', value=pods['kvm_machines']) for metric in ('cores', 'memory', 'storage'): metrics.update( 'maas_kvm_{}'.format(metric), 'set', value=pods['kvm_available_resources'][metric], labels={'status': 'available'}) metrics.update( 'maas_kvm_{}'.format(metric), 'set', value=pods['kvm_utilized_resources'][metric], labels={'status': 'used'}) metrics.update( 'maas_kvm_overcommit_cores', 'set', value=pods['kvm_available_resources']['over_cores']) metrics.update( 'maas_kvm_overcommit_memory', 'set', value=pods['kvm_available_resources']['over_memory']) # Gather statistics for architectures if len(architectures.keys()) > 0: for arch, machines in architectures.items(): metrics.update( 'maas_machine_arches', 'set', value=machines, labels={'arches': arch}) # Update metrics for subnets for cidr, stats in get_subnets_utilisation_stats().items(): for status in ('available', 'unavailable'): metrics.update( 'maas_net_subnet_ip_count', 'set', value=stats[status], labels={'cidr': cidr, 'status': status}) metrics.update( 'maas_net_subnet_ip_static', 'set', value=stats['static'], labels={'cidr': cidr}) for addr_type in ('dynamic', 'reserved'): metric_name = 'maas_net_subnet_ip_{}'.format(addr_type) for status in ('available', 'used'): metrics.update( metric_name, 'set', value=stats['{}_{}'.format(addr_type, status)], labels={'cidr': cidr, 'status': status}) return metrics
def test_get_maas_stats_no_machines(self): expected = { "controllers": { "regionracks": 0, "regions": 0, "racks": 0 }, "nodes": { "machines": 0, "devices": 0 }, "machine_stats": { "total_cpu": 0, "total_mem": 0, "total_storage": 0, }, "machine_status": { "new": 0, "ready": 0, "allocated": 0, "deployed": 0, "commissioning": 0, "testing": 0, "deploying": 0, "failed_deployment": 0, "failed_commissioning": 0, "failed_testing": 0, "broken": 0, }, "network_stats": { "spaces": 0, "fabrics": Fabric.objects.count(), "vlans": VLAN.objects.count(), "subnets_v4": 0, "subnets_v6": 0, }, "vm_hosts": { "lxd": { "vm_hosts": 0, "vms": 0, "available_resources": { "cores": 0, "memory": 0, "over_cores": 0.0, "over_memory": 0.0, "storage": 0, }, "utilized_resources": { "cores": 0, "memory": 0, "storage": 0, }, }, "virsh": { "vm_hosts": 0, "vms": 0, "available_resources": { "cores": 0, "memory": 0, "over_cores": 0.0, "over_memory": 0.0, "storage": 0, }, "utilized_resources": { "cores": 0, "memory": 0, "storage": 0, }, }, }, "workload_annotations": { "annotated_machines": 0, "total_annotations": 0, "unique_keys": 0, "unique_values": 0, }, } self.assertEqual(get_maas_stats(), expected)
def test_get_maas_stats(self): # Make one component of everything factory.make_RegionRackController() factory.make_RegionController() factory.make_RackController() factory.make_Machine(cpu_count=2, memory=200, status=NODE_STATUS.READY) factory.make_Machine(status=NODE_STATUS.READY) factory.make_Machine(status=NODE_STATUS.NEW) for _ in range(4): factory.make_Machine(status=NODE_STATUS.ALLOCATED) factory.make_Machine(cpu_count=3, memory=100, status=NODE_STATUS.FAILED_DEPLOYMENT) factory.make_Machine(status=NODE_STATUS.DEPLOYED) deployed_machine = factory.make_Machine(status=NODE_STATUS.DEPLOYED) OwnerData.objects.set_owner_data(deployed_machine, {"foo": "bar"}) factory.make_Device() factory.make_Device() self.make_pod(cpu=10, mem=100, pod_type="lxd") self.make_pod(cpu=20, mem=200, pod_type="virsh") subnets = Subnet.objects.all() v4 = [net for net in subnets if net.get_ip_version() == 4] v6 = [net for net in subnets if net.get_ip_version() == 6] stats = get_maas_stats() machine_stats = get_machine_stats() # Due to floating point calculation subtleties, sometimes the value the # database returns is off by one compared to the value Python # calculates, so just get it directly from the database for the test. total_storage = machine_stats["total_storage"] expected = { "controllers": { "regionracks": 1, "regions": 1, "racks": 1 }, "nodes": { "machines": 10, "devices": 2 }, "machine_stats": { "total_cpu": 5, "total_mem": 300, "total_storage": total_storage, }, "machine_status": { "new": 1, "ready": 2, "allocated": 4, "deployed": 2, "commissioning": 0, "testing": 0, "deploying": 0, "failed_deployment": 1, "failed_commissioning": 0, "failed_testing": 0, "broken": 0, }, "network_stats": { "spaces": Space.objects.count(), "fabrics": Fabric.objects.count(), "vlans": VLAN.objects.count(), "subnets_v4": len(v4), "subnets_v6": len(v6), }, "vm_hosts": { "lxd": { "vm_hosts": 1, "vms": 0, "available_resources": { "cores": 10, "memory": 100, "over_cores": 10.0, "over_memory": 100.0, "storage": 0, }, "utilized_resources": { "cores": 0, "memory": 0, "storage": 0, }, }, "virsh": { "vm_hosts": 1, "vms": 0, "available_resources": { "cores": 20, "memory": 200, "over_cores": 20.0, "over_memory": 200.0, "storage": 0, }, "utilized_resources": { "cores": 0, "memory": 0, "storage": 0, }, }, }, "workload_annotations": { "annotated_machines": 1, "total_annotations": 1, "unique_keys": 1, "unique_values": 1, }, } self.assertEqual(stats, expected)