def test_update_metrics(self):
self.patch(metrics, "GLOBAL_LABELS", {"service_type": "rack"})
tempdir = self.useFixture(TempDirectory())
meminfo = Path(tempdir.path) / "meminfo"
meminfo.write_text(
dedent(
"""\
MemTotal: 123 Kb
SwapCached: 456 Kb
VmallocUsed: 789 Kb
HugePages_Total: 321
"""
)
)
prometheus_metrics = create_metrics(
node_metrics_definitions(),
registry=prometheus_client.CollectorRegistry(),
)
update_memory_metrics(prometheus_metrics, path=meminfo)
output = prometheus_metrics.generate_latest().decode("ascii")
self.assertIn(
'maas_node_mem_MemTotal{service_type="rack"} 123.0', output
)
self.assertIn(
'maas_node_mem_SwapCached{service_type="rack"} 456.0', output
)
self.assertIn(
'maas_node_mem_VmallocUsed{service_type="rack"} 789.0', output
)
self.assertIn(
'maas_node_mem_HugePages_Total{service_type="rack"} 321.0', output
)
def test_metrics_prometheus_not_availble(self):
self.patch(utils, "PROMETHEUS_SUPPORTED", False)
prometheus_metrics = create_metrics(
self.metrics_definitions,
registry=prometheus_client.CollectorRegistry(),
)
self.assertEqual(prometheus_metrics.available_metrics, [])
def test_update_metrics(self):
prometheus_metrics = create_metrics(
METRICS_DEFINITIONS,
registry=prometheus_client.CollectorRegistry())
middleware = PrometheusRequestMetricsMiddleware(
self.get_response, prometheus_metrics=prometheus_metrics)
middleware(factory.make_fake_request("/MAAS/accounts/login/"))
middleware(factory.make_fake_request("/MAAS/accounts/login/"))
middleware(factory.make_fake_request("/MAAS/other/path"))
middleware(
factory.make_fake_request("/MAAS/other/path",
data={'op': 'do-foo'}))
middleware(
factory.make_fake_request("/MAAS/other/path",
method='POST',
data={'op': 'do-bar'}))
metrics_text = prometheus_metrics.generate_latest().decode('ascii')
self.assertIn(
'maas_http_request_latency_count{method="GET",op="",'
'path="/MAAS/accounts/login/",status="200"} 2.0', metrics_text)
self.assertIn(
'maas_http_request_latency_count{method="GET",op="do-foo",'
'path="/MAAS/other/path",status="404"} 1.0', metrics_text)
self.assertIn(
'maas_http_request_latency_count{method="POST",op="do-bar",'
'path="/MAAS/other/path",status="404"} 1.0', metrics_text)
def test_metrics(self):
prometheus_metrics = create_metrics(
self.metrics_definitions,
registry=prometheus_client.CollectorRegistry())
self.assertIsInstance(prometheus_metrics, PrometheusMetrics)
self.assertCountEqual(prometheus_metrics.available_metrics,
['sample_counter', 'sample_histogram'])
def push_stats_to_prometheus(maas_name, push_gateway):
metrics = create_metrics(STATS_DEFINITIONS,
registry=prom_cli.CollectorRegistry())
update_prometheus_stats(metrics)
prom_cli.push_to_gateway(push_gateway,
job="stats_for_%s" % maas_name,
registry=metrics.registry)
def test_track_tftp_latency(self):
class Thing:
did_something = False
def do_something(self):
self.did_something = True
return True
thing = Thing()
start_time = time.time()
prometheus_metrics = create_metrics(
METRICS_DEFINITIONS,
registry=prometheus_client.CollectorRegistry())
thing.do_something = track_tftp_latency(
thing.do_something,
start_time=start_time,
filename='myfile.txt',
prometheus_metrics=prometheus_metrics)
time_mock = self.patch(tftp_module, 'time')
time_mock.return_value = start_time + 0.5
result = thing.do_something()
self.assertTrue(result)
self.assertTrue(thing.did_something)
metrics = prometheus_metrics.generate_latest().decode('ascii')
self.assertIn(
'maas_tftp_file_transfer_latency_count{filename="myfile.txt"} 1.0',
metrics)
self.assertIn(
'maas_tftp_file_transfer_latency_bucket'
'{filename="myfile.txt",le="0.5"} 1.0', metrics)
self.assertIn(
'maas_tftp_file_transfer_latency_bucket'
'{filename="myfile.txt",le="0.25"} 0.0', metrics)
def test_metrics(self):
prometheus_metrics = create_metrics(self.metrics_definitions)
resource = http.PrometheusMetricsResource(prometheus_metrics)
request = Request(DummyChannel(), False)
content = resource.render_GET(request).decode('utf-8')
self.assertIn('TYPE sample_histogram histogram', content)
self.assertIn('TYPE sample_counter counter', content)
def test_metrics_disabled(self):
prometheus_metrics = create_metrics(
None, registry=prometheus_client.CollectorRegistry())
resource = http.PrometheusMetricsResource(prometheus_metrics)
request = Request(DummyChannel(), False)
content = resource.render_GET(request).decode('utf-8')
self.assertEqual(request.code, 404)
self.assertEqual(content, '')
def prometheus_stats_handler(request):
if not Config.objects.get_config('prometheus_enabled'):
return HttpResponseNotFound()
metrics = create_metrics(STATS_DEFINITIONS)
update_prometheus_stats(metrics)
return HttpResponse(content=metrics.generate_latest(),
content_type="text/plain")
def test_update_metrics(self):
self.patch(metrics, "GLOBAL_LABELS", {"service_type": "rack"})
tempdir = self.useFixture(TempDirectory())
stat = Path(tempdir.path) / "stat"
stat.write_text(
dedent("""\
cpu 111 222 333 444 555 666 7 888 9 11
cpu0 222 333 444 555 666 777 8 999 1 22
cpu1 222 333 444 555 666 777 8 999 1 22
other line
other line
"""))
prometheus_metrics = create_metrics(
node_metrics_definitions(),
registry=prometheus_client.CollectorRegistry(),
)
update_cpu_metrics(prometheus_metrics, path=stat)
output = prometheus_metrics.generate_latest().decode("ascii")
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="user"} 1.11',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="nice"} 2.22',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="system"} 3.33',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="idle"} 4.44',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="iowait"} 5.55',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="irq"} 6.66',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="softirq"} 0.07',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="steal"} 8.88',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="guest"} 0.09',
output,
)
self.assertIn(
'maas_node_cpu_time_total{service_type="rack",state="guest_nice"} 0.11',
output,
)
def test_metrics(self):
prometheus_metrics = create_metrics(
self.metrics_definitions,
registry=prometheus_client.CollectorRegistry(),
)
resource = http.PrometheusMetricsResource(prometheus_metrics)
request = Request(DummyChannel(), False)
content = resource.render_GET(request).decode("utf-8")
self.assertEqual(request.code, 200)
self.assertIn("TYPE sample_histogram histogram", content)
self.assertIn("TYPE sample_counter counter", content)
def test_extra_labels(self):
prometheus_metrics = create_metrics(
self.metrics_definitions,
extra_labels={
'foo': 'FOO',
'bar': 'BAR'
},
registry=prometheus_client.CollectorRegistry())
prometheus_metrics.update('sample_counter', 'inc')
content = prometheus_metrics.generate_latest().decode('ascii')
self.assertIn('sample_counter{bar="BAR",foo="FOO"} 1.0', content)
def test_extra_labels_callable(self):
values = ['a', 'b']
prometheus_metrics = create_metrics(
self.metrics_definitions,
extra_labels={'foo': values.pop},
registry=prometheus_client.CollectorRegistry())
prometheus_metrics.update('sample_counter', 'inc')
prometheus_metrics.update('sample_counter', 'inc')
content = prometheus_metrics.generate_latest().decode('ascii')
self.assertIn('sample_counter{foo="a"} 1.0', content)
self.assertIn('sample_counter{foo="b"} 1.0', content)
def test_extra_labels_callable(self):
values = ["a", "b"]
prometheus_metrics = create_metrics(
self.metrics_definitions,
extra_labels={"foo": values.pop},
registry=prometheus_client.CollectorRegistry(),
)
prometheus_metrics.update("sample_counter", "inc")
prometheus_metrics.update("sample_counter", "inc")
content = prometheus_metrics.generate_latest().decode("ascii")
self.assertIn('sample_counter{foo="a"} 1.0', content)
self.assertIn('sample_counter{foo="b"} 1.0', content)
def test_extra_labels(self):
prometheus_metrics = create_metrics(
self.metrics_definitions,
extra_labels={
"foo": "FOO",
"bar": "BAR"
},
registry=prometheus_client.CollectorRegistry(),
)
prometheus_metrics.update("sample_counter", "inc")
content = prometheus_metrics.generate_latest().decode("ascii")
self.assertIn('sample_counter{bar="BAR",foo="FOO"} 1.0', content)
def test_track_time(self):
prometheus_metrics = create_metrics(METRICS_DEFINITIONS)
session = TransferTimeTrackingSession(
'file.txt', BytesReader(b'some data'), _clock=Clock(),
prometheus_metrics=prometheus_metrics)
session.transport = Mock()
session.startProtocol()
session.cancel()
metrics = prometheus_metrics.generate_latest().decode('ascii')
self.assertIn(
'maas_tftp_file_transfer_latency_count{filename="file.txt"} 1.0',
metrics)
def prometheus_stats_handler(request):
configs = Config.objects.get_configs(['prometheus_enabled', 'uuid'])
have_prometheus = PROMETHEUS_SUPPORTED and configs['prometheus_enabled']
if not have_prometheus:
return HttpResponseNotFound()
metrics = create_metrics(
STATS_DEFINITIONS,
extra_labels={'maas_id': configs['uuid']},
update_handlers=[update_prometheus_stats],
registry=prom_cli.CollectorRegistry())
return HttpResponse(
content=metrics.generate_latest(), content_type="text/plain")
def prometheus_stats_handler(request):
have_prometheus = (
PROMETHEUS_SUPPORTED and
Config.objects.get_config('prometheus_enabled'))
if not have_prometheus:
return HttpResponseNotFound()
metrics = create_metrics(
STATS_DEFINITIONS,
registry=prom_cli.CollectorRegistry())
update_prometheus_stats(metrics)
return HttpResponse(
content=metrics.generate_latest(), content_type="text/plain")
def test_update_metrics(self):
prometheus_metrics = create_metrics(METRICS_DEFINITIONS)
middleware = PrometheusRequestMetricsMiddleware(
self.get_response, prometheus_metrics=prometheus_metrics)
middleware(factory.make_fake_request("/MAAS/accounts/login/"))
middleware(factory.make_fake_request("/MAAS/accounts/login/"))
middleware(factory.make_fake_request("/MAAS/other/path"))
metrics_text = prometheus_metrics.generate_latest().decode('ascii')
self.assertIn(
'maas_http_request_latency_count{method="GET",'
'path="/MAAS/accounts/login/",status="200"} 2.0', metrics_text)
self.assertIn(
'maas_http_request_latency_count{method="GET",'
'path="/MAAS/other/path",status="404"} 1.0', metrics_text)
def test_subnet_stats(self):
subnet = factory.make_Subnet(cidr='1.2.0.0/16', gateway_ip='1.2.0.254')
factory.make_IPRange(subnet=subnet,
start_ip='1.2.0.11',
end_ip='1.2.0.20',
alloc_type=IPRANGE_TYPE.DYNAMIC)
factory.make_IPRange(subnet=subnet,
start_ip='1.2.0.51',
end_ip='1.2.0.70',
alloc_type=IPRANGE_TYPE.RESERVED)
factory.make_StaticIPAddress(ip='1.2.0.12',
alloc_type=IPADDRESS_TYPE.DHCP,
subnet=subnet)
for n in (60, 61):
factory.make_StaticIPAddress(
ip='1.2.0.{}'.format(n),
alloc_type=IPADDRESS_TYPE.USER_RESERVED,
subnet=subnet)
for n in (80, 90, 100):
factory.make_StaticIPAddress(ip='1.2.0.{}'.format(n),
alloc_type=IPADDRESS_TYPE.STICKY,
subnet=subnet)
metrics = create_metrics(
STATS_DEFINITIONS, registry=prometheus_client.CollectorRegistry())
update_prometheus_stats(metrics)
output = metrics.generate_latest().decode('ascii')
self.assertIn(
'maas_net_subnet_ip_count'
'{cidr="1.2.0.0/16",status="available"} 65500.0', output)
self.assertIn(
'maas_net_subnet_ip_count'
'{cidr="1.2.0.0/16",status="unavailable"} 34.0', output)
self.assertIn(
'maas_net_subnet_ip_dynamic'
'{cidr="1.2.0.0/16",status="available"} 9.0', output)
self.assertIn(
'maas_net_subnet_ip_dynamic{cidr="1.2.0.0/16",status="used"} 1.0',
output)
self.assertIn(
'maas_net_subnet_ip_reserved'
'{cidr="1.2.0.0/16",status="available"} 18.0', output)
self.assertIn(
'maas_net_subnet_ip_reserved{cidr="1.2.0.0/16",status="used"} 2.0',
output)
self.assertIn('maas_net_subnet_ip_static{cidr="1.2.0.0/16"} 3.0',
output)
def prometheus_stats_handler(request):
configs = Config.objects.get_configs(["prometheus_enabled", "uuid"])
have_prometheus = PROMETHEUS_SUPPORTED and configs["prometheus_enabled"]
if not have_prometheus:
return HttpResponseNotFound()
global _METRICS
if not _METRICS:
_METRICS = create_metrics(
STATS_DEFINITIONS,
extra_labels={"maas_id": configs["uuid"]},
update_handlers=[update_prometheus_stats],
registry=prom_cli.CollectorRegistry(),
)
return HttpResponse(
content=_METRICS.generate_latest(), content_type="text/plain"
)
def test_update_prometheus_stats(self):
self.patch(stats, 'prom_cli')
# general values
values = {
"machine_status": {
"random_status": 0,
},
"controllers": {
"regions": 0,
},
"nodes": {
"machines": 0,
},
"network_stats": {
"spaces": 0,
},
"machine_stats": {
"total_cpus": 0,
},
}
mock = self.patch(stats, "get_maas_stats")
mock.return_value = json.dumps(values)
# architecture
arches = {
"amd64": 0,
"i386": 0,
}
mock_arches = self.patch(stats, "get_machines_by_architecture")
mock_arches.return_value = arches
# pods
pods = {
"kvm_pods": 0,
"kvm_machines": 0,
}
mock_pods = self.patch(stats, "get_kvm_pods_stats")
mock_pods.return_value = pods
metrics = create_metrics(
STATS_DEFINITIONS, registry=prometheus_client.CollectorRegistry())
update_prometheus_stats(metrics)
self.assertThat(mock, MockCalledOnce())
self.assertThat(mock_arches, MockCalledOnce())
self.assertThat(mock_pods, MockCalledOnce())
def test_wb_start_session(self):
prometheus_metrics = create_metrics(
METRICS_DEFINITIONS,
registry=prometheus_client.CollectorRegistry())
stream_session = FakeStreamSession()
session = FakeSession(stream_session)
tftp_mock = self.patch(tftp.protocol.TFTP, '_startSession')
tftp_mock.return_value = succeed(session)
tracking_tftp = TransferTimeTrackingTFTP(sentinel.backend)
datagram = RQDatagram(b'file.txt', b'octet', {})
result = yield tracking_tftp._startSession(
datagram,
'192.168.1.1',
'read',
prometheus_metrics=prometheus_metrics)
result.session.cancel()
metrics = prometheus_metrics.generate_latest().decode('ascii')
self.assertIs(result, session)
self.assertTrue(stream_session.cancelled)
self.assertIn(
'maas_tftp_file_transfer_latency_count{filename="file.txt"} 1.0',
metrics)
def test_metrics(self):
prometheus_metrics = create_metrics(self.metrics_definitions)
self.assertIsInstance(prometheus_metrics, PrometheusMetrics)
self.assertCountEqual(prometheus_metrics.available_metrics,
['sample_counter', 'sample_histogram'])
# Copyright 2019 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Prometheus metrics."""
from provisioningserver.prometheus.utils import (
create_metrics,
MetricDefinition,
)
METRICS_DEFINITIONS = [
MetricDefinition('Histogram', 'maas_http_request_latency',
'HTTP request latency',
['method', 'path', 'status', 'op']),
MetricDefinition('Histogram', 'maas_websocket_call_latency',
'Latency of a Websocket handler call', ['call']),
]
PROMETHEUS_METRICS = create_metrics(METRICS_DEFINITIONS)
def test_metrics_prometheus_not_availble(self):
self.patch(utils, 'PROMETHEUS_SUPPORTED', False)
prometheus_metrics = create_metrics(self.metrics_definitions)
self.assertEqual(prometheus_metrics.available_metrics, [])
def test_update_prometheus_stats(self):
self.patch(stats, "prom_cli")
# general values
values = {
"machine_status": {
"random_status": 0
},
"controllers": {
"regions": 0
},
"nodes": {
"machines": 0
},
"network_stats": {
"spaces": 0
},
"machine_stats": {
"total_cpu": 0
},
}
mock = self.patch(stats, "get_maas_stats")
mock.return_value = json.dumps(values)
# architecture
arches = {"amd64": 0, "i386": 0}
mock_arches = self.patch(stats, "get_machines_by_architecture")
mock_arches.return_value = arches
# pods
pods = {
"kvm_pods": 0,
"kvm_machines": 0,
"kvm_available_resources": {
"cores": 10,
"memory": 20,
"storage": 30,
"over_cores": 100,
"over_memory": 200,
},
"kvm_utilized_resources": {
"cores": 5,
"memory": 10,
"storage": 15,
},
}
mock_pods = self.patch(stats, "get_kvm_pods_stats")
mock_pods.return_value = pods
subnet_stats = {
"1.2.0.0/16": {
"available": 2**16 - 3,
"dynamic_available": 0,
"dynamic_used": 0,
"reserved_available": 0,
"reserved_used": 0,
"static": 0,
"unavailable": 1,
},
"::1/128": {
"available": 1,
"dynamic_available": 0,
"dynamic_used": 0,
"reserved_available": 0,
"reserved_used": 0,
"static": 0,
"unavailable": 0,
},
}
mock_subnet_stats = self.patch(stats, "get_subnets_utilisation_stats")
mock_subnet_stats.return_value = subnet_stats
metrics = create_metrics(
STATS_DEFINITIONS, registry=prometheus_client.CollectorRegistry())
update_prometheus_stats(metrics)
self.assertThat(mock, MockCalledOnce())
self.assertThat(mock_arches, MockCalledOnce())
self.assertThat(mock_pods, MockCalledOnce())
self.assertThat(mock_subnet_stats, MockCalledOnce())
'Latency of a Websocket handler call', ['call']),
# Common metrics
*node_metrics_definitions()
]
# Global for tracking global values for metrics label. These are set
# differently from rackd and regiond code, but this is defined here so the
# logic using it can be generic.
GLOBAL_LABELS = {
# The MAAS installation UUID, the same for all regions/racks within a
# deployment
'maas_uuid': None,
# The type of service (region/rack) exporting the metrics.
'service_type': None
}
def set_global_labels(**labels):
"""Update global labels for metrics."""
global GLOBAL_LABELS
GLOBAL_LABELS.update(labels)
PROMETHEUS_METRICS = create_metrics(
METRICS_DEFINITIONS,
extra_labels={
'host': get_machine_default_gateway_ip,
'maas_id': lambda: GLOBAL_LABELS['maas_uuid'],
},
update_handlers=[update_cpu_metrics, update_memory_metrics])
def push_stats_to_prometheus(maas_name, push_gateway):
metrics = create_metrics(STATS_DEFINITIONS)
update_prometheus_stats(metrics)
prom_cli.push_to_gateway(push_gateway,
job='stats_for_%s' % maas_name,
registry=metrics.registry)