def test_record_call_latency_async(self):
definitions = [
MetricDefinition('Histogram', 'histo', 'An histogram',
['foo', 'bar'])
]
prometheus_metrics = PrometheusMetrics(
definitions=definitions,
registry=prometheus_client.CollectorRegistry())
label_call_args = []
def get_labels(*args, **kwargs):
label_call_args.append((args, kwargs))
return {'foo': 'FOO', 'bar': 'BAR'}
@prometheus_metrics.record_call_latency('histo', get_labels=get_labels)
@inlineCallbacks
def func(param1, param2=None):
yield
returnValue(param1)
obj = object()
result = yield func(obj, param2='baz')
self.assertIs(result, obj)
# the get_labels function is called with the same args as the function
self.assertEqual(label_call_args, [((obj, ), {'param2': 'baz'})])
self.assertIn('histo_count{bar="BAR",foo="FOO"} 1.0',
prometheus_metrics.generate_latest().decode('ascii'))
def test_record_call_latency_sync(self):
definitions = [
MetricDefinition("Histogram", "histo", "An histogram",
["foo", "bar"])
]
prometheus_metrics = PrometheusMetrics(
definitions=definitions,
registry=prometheus_client.CollectorRegistry(),
)
label_call_args = []
def get_labels(*args, **kwargs):
label_call_args.append((args, kwargs))
return {"foo": "FOO", "bar": "BAR"}
@prometheus_metrics.record_call_latency("histo", get_labels=get_labels)
def func(param1, param2=None):
return param1
obj = object()
result = func(obj, param2="baz")
self.assertIs(result, obj)
# the get_labels function is called with the same args as the function
self.assertEqual(label_call_args, [((obj, ), {"param2": "baz"})])
self.assertIn(
'histo_count{bar="BAR",foo="FOO"} 1.0',
prometheus_metrics.generate_latest().decode("ascii"),
)
def test_update_call_value_class(self):
definitions = [MetricDefinition('Counter', 'a_counter', 'A Counter')]
prometheus_metrics = PrometheusMetrics(
definitions=definitions,
registry=prometheus_client.CollectorRegistry())
prometheus_metrics.update('a_counter', 'set', value=22)
self.assertIn('a_counter 22.0',
prometheus_metrics.generate_latest().decode('ascii'))
def test_with_update_handlers(self):
def update_gauge(metrics):
metrics.update('a_gauge', 'set', value=33)
prometheus_metrics = PrometheusMetrics(
definitions=[MetricDefinition('Gauge', 'a_gauge', 'A Gauge', [])],
update_handlers=[update_gauge],
registry=prometheus_client.CollectorRegistry())
self.assertIn('a_gauge 33.0',
prometheus_metrics.generate_latest().decode('ascii'))
def test_update_call_value_class(self):
definitions = [MetricDefinition("Counter", "a_counter", "A Counter")]
prometheus_metrics = PrometheusMetrics(
definitions=definitions,
registry=prometheus_client.CollectorRegistry(),
)
prometheus_metrics.update("a_counter", "set", value=22)
self.assertIn(
"a_counter 22.0",
prometheus_metrics.generate_latest().decode("ascii"),
)
def update_memory_metrics(prometheus_metrics: PrometheusMetrics, path=None):
"""Update memory-related metrics."""
from provisioningserver.prometheus.metrics import GLOBAL_LABELS
metric_values = _collect_memory_values(path=path)
for field in MEMINFO_FIELDS:
value = metric_values.get(field)
if value is not None:
prometheus_metrics.update(
'maas_node_mem_{}'.format(field),
'set',
value=value,
labels={'service_type': GLOBAL_LABELS['service_type']})
def test_with_update_handlers(self):
def update_gauge(metrics):
metrics.update("a_gauge", "set", value=33)
prometheus_metrics = PrometheusMetrics(
definitions=[MetricDefinition("Gauge", "a_gauge", "A Gauge", [])],
update_handlers=[update_gauge],
registry=prometheus_client.CollectorRegistry(),
)
self.assertIn(
"a_gauge 33.0",
prometheus_metrics.generate_latest().decode("ascii"),
)
def test_update(self):
registry = prometheus_client.CollectorRegistry()
metric = prometheus_client.Gauge('a_gauge',
'A Gauge', ['foo', 'bar'],
registry=registry)
prometheus_metrics = PrometheusMetrics(registry=registry,
metrics={'a_gauge': metric})
prometheus_metrics.update('a_gauge',
'set',
value=22,
labels={
'foo': 'FOO',
'bar': 'BAR'
})
self.assertIn('a_gauge{bar="BAR",foo="FOO"} 22.0',
prometheus_metrics.generate_latest().decode('ascii'))
def test_update(self):
definitions = [
MetricDefinition('Gauge', 'a_gauge', 'A Gauge', ['foo', 'bar'])
]
prometheus_metrics = PrometheusMetrics(
definitions=definitions,
registry=prometheus_client.CollectorRegistry())
prometheus_metrics.update('a_gauge',
'set',
value=22,
labels={
'foo': 'FOO',
'bar': 'BAR'
})
self.assertIn('a_gauge{bar="BAR",foo="FOO"} 22.0',
prometheus_metrics.generate_latest().decode('ascii'))
def update_cpu_metrics(prometheus_metrics: PrometheusMetrics, path=None):
"""Update memory-related metrics."""
from provisioningserver.prometheus.metrics import GLOBAL_LABELS
cpu_values = _collect_cpu_values(path=path)
for field in CPU_TIME_FIELDS:
value = cpu_values.get(field)
if value is not None:
prometheus_metrics.update('maas_node_cpu_time',
'set',
value=value / USER_HZ,
labels={
'service_type':
GLOBAL_LABELS['service_type'],
'state':
field
})
def test_no_register_atexit_custom_registry(self):
mock_register = self.patch(atexit, 'register')
definitions = [
MetricDefinition('Gauge', 'a_gauge', 'A Gauge', ['foo', 'bar'])
]
PrometheusMetrics(definitions=definitions,
registry=prometheus_client.CollectorRegistry())
mock_register.assert_not_called()
def test_register_atexit_global_registry(self):
mock_register = self.patch(atexit, 'register')
definitions = [
MetricDefinition('Gauge', 'a_gauge', 'A Gauge', ['foo', 'bar'])
]
prometheus_metrics = PrometheusMetrics(definitions=definitions)
mock_register.assert_called_once_with(
prometheus_metrics._cleanup_metric_files)
def test_update(self):
definitions = [
MetricDefinition("Gauge", "a_gauge", "A Gauge", ["foo", "bar"])
]
prometheus_metrics = PrometheusMetrics(
definitions=definitions,
registry=prometheus_client.CollectorRegistry(),
)
prometheus_metrics.update("a_gauge",
"set",
value=22,
labels={
"foo": "FOO",
"bar": "BAR"
})
self.assertIn(
'a_gauge{bar="BAR",foo="FOO"} 22.0',
prometheus_metrics.generate_latest().decode("ascii"),
)
def update_prometheus_stats(metrics: PrometheusMetrics):
"""Update metrics in a PrometheusMetrics based on database values."""
stats = json.loads(get_maas_stats())
architectures = get_machines_by_architecture()
pods = get_kvm_pods_stats()
# Gather counter for machines per status
for status, machines in stats["machine_status"].items():
metrics.update("maas_machines",
"set",
value=machines,
labels={"status": status})
# Gather counter for number of nodes (controllers/machine/devices)
for ctype, number in stats["controllers"].items():
metrics.update("maas_nodes",
"set",
value=number,
labels={"type": ctype})
for ctype, number in stats["nodes"].items():
metrics.update("maas_nodes",
"set",
value=number,
labels={"type": ctype})
# Gather counter for networks
for stype, number in stats["network_stats"].items():
metrics.update("maas_net_{}".format(stype), "set", value=number)
# Gather overall amount of machine resources
for resource, value in stats["machine_stats"].items():
metrics.update("maas_machines_{}".format(resource), "set", value=value)
# Gather all stats for pods
metrics.update("maas_kvm_pods", "set", value=pods["kvm_pods"])
metrics.update("maas_kvm_machines", "set", value=pods["kvm_machines"])
for metric in ("cores", "memory", "storage"):
metrics.update(
"maas_kvm_{}".format(metric),
"set",
value=pods["kvm_available_resources"][metric],
labels={"status": "available"},
)
metrics.update(
"maas_kvm_{}".format(metric),
"set",
value=pods["kvm_utilized_resources"][metric],
labels={"status": "used"},
)
metrics.update(
"maas_kvm_overcommit_cores",
"set",
value=pods["kvm_available_resources"]["over_cores"],
)
metrics.update(
"maas_kvm_overcommit_memory",
"set",
value=pods["kvm_available_resources"]["over_memory"],
)
# Gather statistics for architectures
if len(architectures.keys()) > 0:
for arch, machines in architectures.items():
metrics.update(
"maas_machine_arches",
"set",
value=machines,
labels={"arches": arch},
)
# Update metrics for subnets
for cidr, stats in get_subnets_utilisation_stats().items():
for status in ("available", "unavailable"):
metrics.update(
"maas_net_subnet_ip_count",
"set",
value=stats[status],
labels={
"cidr": cidr,
"status": status
},
)
metrics.update(
"maas_net_subnet_ip_static",
"set",
value=stats["static"],
labels={"cidr": cidr},
)
for addr_type in ("dynamic", "reserved"):
metric_name = "maas_net_subnet_ip_{}".format(addr_type)
for status in ("available", "used"):
metrics.update(
metric_name,
"set",
value=stats["{}_{}".format(addr_type, status)],
labels={
"cidr": cidr,
"status": status
},
)
return metrics
def test_update_empty(self):
prometheus_metrics = PrometheusMetrics()
prometheus_metrics.update('some_metric', 'inc')
self.assertIsNone(prometheus_metrics.generate_latest())
def test_empty(self):
prometheus_metrics = PrometheusMetrics()
self.assertEqual(prometheus_metrics.available_metrics, [])
self.assertIsNone(prometheus_metrics.generate_latest())
def update_prometheus_stats(metrics: PrometheusMetrics):
"""Update metrics in a PrometheusMetrics based on database values."""
stats = json.loads(get_maas_stats())
architectures = get_machines_by_architecture()
pods = get_kvm_pods_stats()
# Gather counter for machines per status
for status, machines in stats['machine_status'].items():
metrics.update(
'maas_machines', 'set', value=machines,
labels={'status': status})
# Gather counter for number of nodes (controllers/machine/devices)
for ctype, number in stats['controllers'].items():
metrics.update(
'maas_nodes', 'set', value=number, labels={'type': ctype})
for ctype, number in stats['nodes'].items():
metrics.update(
'maas_nodes', 'set', value=number, labels={'type': ctype})
# Gather counter for networks
for stype, number in stats['network_stats'].items():
metrics.update('maas_net_{}'.format(stype), 'set', value=number)
# Gather overall amount of machine resources
for resource, value in stats['machine_stats'].items():
metrics.update('maas_machines_{}'.format(resource), 'set', value=value)
# Gather all stats for pods
metrics.update('maas_kvm_pods', 'set', value=pods['kvm_pods'])
metrics.update('maas_kvm_machines', 'set', value=pods['kvm_machines'])
for metric in ('cores', 'memory', 'storage'):
metrics.update(
'maas_kvm_{}'.format(metric), 'set',
value=pods['kvm_available_resources'][metric],
labels={'status': 'available'})
metrics.update(
'maas_kvm_{}'.format(metric), 'set',
value=pods['kvm_utilized_resources'][metric],
labels={'status': 'used'})
metrics.update(
'maas_kvm_overcommit_cores', 'set',
value=pods['kvm_available_resources']['over_cores'])
metrics.update(
'maas_kvm_overcommit_memory', 'set',
value=pods['kvm_available_resources']['over_memory'])
# Gather statistics for architectures
if len(architectures.keys()) > 0:
for arch, machines in architectures.items():
metrics.update(
'maas_machine_arches', 'set', value=machines,
labels={'arches': arch})
# Update metrics for subnets
for cidr, stats in get_subnets_utilisation_stats().items():
for status in ('available', 'unavailable'):
metrics.update(
'maas_net_subnet_ip_count', 'set',
value=stats[status],
labels={'cidr': cidr, 'status': status})
metrics.update(
'maas_net_subnet_ip_static', 'set',
value=stats['static'], labels={'cidr': cidr})
for addr_type in ('dynamic', 'reserved'):
metric_name = 'maas_net_subnet_ip_{}'.format(addr_type)
for status in ('available', 'used'):
metrics.update(
metric_name, 'set',
value=stats['{}_{}'.format(addr_type, status)],
labels={'cidr': cidr, 'status': status})
return metrics
def update_prometheus_stats(metrics: PrometheusMetrics):
"""Update metrics in a PrometheusMetrics based on database values."""
stats = json.loads(get_maas_stats())
architectures = get_machines_by_architecture()
pods = get_kvm_pods_stats()
# Gather counter for machines per status
for status, machines in stats['machine_status'].items():
metrics.update(
'machine_status', 'set', value=machines, labels={'status': status})
# Gather counter for number of nodes (controllers/machine/devices)
for ctype, number in stats['controllers'].items():
metrics.update(
'nodes', 'set', value=number, labels={'type': ctype})
for ctype, number in stats['nodes'].items():
metrics.update(
'nodes', 'set', value=number, labels={'type': ctype})
# Gather counter for networks
for stype, number in stats['network_stats'].items():
metrics.update(
'networks', 'set', value=number, labels={'type': stype})
# Gather overall amount of machine resources
for resource, value in stats['machine_stats'].items():
metrics.update(
'machine_resources', 'set', value=value,
labels={'resource': resource})
# Gather all stats for pods
for resource, value in pods.items():
if isinstance(value, dict):
for r, v in value.items():
metrics.update(
'kvm_pods', 'set', value=v,
labels={'type': '{}_{}'.format(resource, r)})
else:
metrics.update(
'kvm_pods', 'set', value=value,
labels={'type': resource})
# Gather statistics for architectures
if len(architectures.keys()) > 0:
for arch, machines in architectures.items():
metrics.update(
'machine_arches', 'set', value=machines,
labels={'arches': arch})
return metrics
