def test_update_metrics(self): self.patch(metrics, "GLOBAL_LABELS", {"service_type": "rack"}) tempdir = self.useFixture(TempDirectory()) meminfo = Path(tempdir.path) / "meminfo" meminfo.write_text( dedent( """\ MemTotal: 123 Kb SwapCached: 456 Kb VmallocUsed: 789 Kb HugePages_Total: 321 """ ) ) prometheus_metrics = create_metrics( node_metrics_definitions(), registry=prometheus_client.CollectorRegistry(), ) update_memory_metrics(prometheus_metrics, path=meminfo) output = prometheus_metrics.generate_latest().decode("ascii") self.assertIn( 'maas_node_mem_MemTotal{service_type="rack"} 123.0', output ) self.assertIn( 'maas_node_mem_SwapCached{service_type="rack"} 456.0', output ) self.assertIn( 'maas_node_mem_VmallocUsed{service_type="rack"} 789.0', output ) self.assertIn( 'maas_node_mem_HugePages_Total{service_type="rack"} 321.0', output )
def test_metrics_prometheus_not_availble(self): self.patch(utils, "PROMETHEUS_SUPPORTED", False) prometheus_metrics = create_metrics( self.metrics_definitions, registry=prometheus_client.CollectorRegistry(), ) self.assertEqual(prometheus_metrics.available_metrics, [])
def test_update_metrics(self): prometheus_metrics = create_metrics( METRICS_DEFINITIONS, registry=prometheus_client.CollectorRegistry()) middleware = PrometheusRequestMetricsMiddleware( self.get_response, prometheus_metrics=prometheus_metrics) middleware(factory.make_fake_request("/MAAS/accounts/login/")) middleware(factory.make_fake_request("/MAAS/accounts/login/")) middleware(factory.make_fake_request("/MAAS/other/path")) middleware( factory.make_fake_request("/MAAS/other/path", data={'op': 'do-foo'})) middleware( factory.make_fake_request("/MAAS/other/path", method='POST', data={'op': 'do-bar'})) metrics_text = prometheus_metrics.generate_latest().decode('ascii') self.assertIn( 'maas_http_request_latency_count{method="GET",op="",' 'path="/MAAS/accounts/login/",status="200"} 2.0', metrics_text) self.assertIn( 'maas_http_request_latency_count{method="GET",op="do-foo",' 'path="/MAAS/other/path",status="404"} 1.0', metrics_text) self.assertIn( 'maas_http_request_latency_count{method="POST",op="do-bar",' 'path="/MAAS/other/path",status="404"} 1.0', metrics_text)
def test_metrics(self): prometheus_metrics = create_metrics( self.metrics_definitions, registry=prometheus_client.CollectorRegistry()) self.assertIsInstance(prometheus_metrics, PrometheusMetrics) self.assertCountEqual(prometheus_metrics.available_metrics, ['sample_counter', 'sample_histogram'])
def push_stats_to_prometheus(maas_name, push_gateway): metrics = create_metrics(STATS_DEFINITIONS, registry=prom_cli.CollectorRegistry()) update_prometheus_stats(metrics) prom_cli.push_to_gateway(push_gateway, job="stats_for_%s" % maas_name, registry=metrics.registry)
def test_track_tftp_latency(self): class Thing: did_something = False def do_something(self): self.did_something = True return True thing = Thing() start_time = time.time() prometheus_metrics = create_metrics( METRICS_DEFINITIONS, registry=prometheus_client.CollectorRegistry()) thing.do_something = track_tftp_latency( thing.do_something, start_time=start_time, filename='myfile.txt', prometheus_metrics=prometheus_metrics) time_mock = self.patch(tftp_module, 'time') time_mock.return_value = start_time + 0.5 result = thing.do_something() self.assertTrue(result) self.assertTrue(thing.did_something) metrics = prometheus_metrics.generate_latest().decode('ascii') self.assertIn( 'maas_tftp_file_transfer_latency_count{filename="myfile.txt"} 1.0', metrics) self.assertIn( 'maas_tftp_file_transfer_latency_bucket' '{filename="myfile.txt",le="0.5"} 1.0', metrics) self.assertIn( 'maas_tftp_file_transfer_latency_bucket' '{filename="myfile.txt",le="0.25"} 0.0', metrics)
def test_metrics(self): prometheus_metrics = create_metrics(self.metrics_definitions) resource = http.PrometheusMetricsResource(prometheus_metrics) request = Request(DummyChannel(), False) content = resource.render_GET(request).decode('utf-8') self.assertIn('TYPE sample_histogram histogram', content) self.assertIn('TYPE sample_counter counter', content)
def test_metrics_disabled(self): prometheus_metrics = create_metrics( None, registry=prometheus_client.CollectorRegistry()) resource = http.PrometheusMetricsResource(prometheus_metrics) request = Request(DummyChannel(), False) content = resource.render_GET(request).decode('utf-8') self.assertEqual(request.code, 404) self.assertEqual(content, '')
def prometheus_stats_handler(request): if not Config.objects.get_config('prometheus_enabled'): return HttpResponseNotFound() metrics = create_metrics(STATS_DEFINITIONS) update_prometheus_stats(metrics) return HttpResponse(content=metrics.generate_latest(), content_type="text/plain")
def test_update_metrics(self): self.patch(metrics, "GLOBAL_LABELS", {"service_type": "rack"}) tempdir = self.useFixture(TempDirectory()) stat = Path(tempdir.path) / "stat" stat.write_text( dedent("""\ cpu 111 222 333 444 555 666 7 888 9 11 cpu0 222 333 444 555 666 777 8 999 1 22 cpu1 222 333 444 555 666 777 8 999 1 22 other line other line """)) prometheus_metrics = create_metrics( node_metrics_definitions(), registry=prometheus_client.CollectorRegistry(), ) update_cpu_metrics(prometheus_metrics, path=stat) output = prometheus_metrics.generate_latest().decode("ascii") self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="user"} 1.11', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="nice"} 2.22', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="system"} 3.33', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="idle"} 4.44', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="iowait"} 5.55', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="irq"} 6.66', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="softirq"} 0.07', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="steal"} 8.88', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="guest"} 0.09', output, ) self.assertIn( 'maas_node_cpu_time_total{service_type="rack",state="guest_nice"} 0.11', output, )
def test_metrics(self): prometheus_metrics = create_metrics( self.metrics_definitions, registry=prometheus_client.CollectorRegistry(), ) resource = http.PrometheusMetricsResource(prometheus_metrics) request = Request(DummyChannel(), False) content = resource.render_GET(request).decode("utf-8") self.assertEqual(request.code, 200) self.assertIn("TYPE sample_histogram histogram", content) self.assertIn("TYPE sample_counter counter", content)
def test_extra_labels(self): prometheus_metrics = create_metrics( self.metrics_definitions, extra_labels={ 'foo': 'FOO', 'bar': 'BAR' }, registry=prometheus_client.CollectorRegistry()) prometheus_metrics.update('sample_counter', 'inc') content = prometheus_metrics.generate_latest().decode('ascii') self.assertIn('sample_counter{bar="BAR",foo="FOO"} 1.0', content)
def test_extra_labels_callable(self): values = ['a', 'b'] prometheus_metrics = create_metrics( self.metrics_definitions, extra_labels={'foo': values.pop}, registry=prometheus_client.CollectorRegistry()) prometheus_metrics.update('sample_counter', 'inc') prometheus_metrics.update('sample_counter', 'inc') content = prometheus_metrics.generate_latest().decode('ascii') self.assertIn('sample_counter{foo="a"} 1.0', content) self.assertIn('sample_counter{foo="b"} 1.0', content)
def test_extra_labels_callable(self): values = ["a", "b"] prometheus_metrics = create_metrics( self.metrics_definitions, extra_labels={"foo": values.pop}, registry=prometheus_client.CollectorRegistry(), ) prometheus_metrics.update("sample_counter", "inc") prometheus_metrics.update("sample_counter", "inc") content = prometheus_metrics.generate_latest().decode("ascii") self.assertIn('sample_counter{foo="a"} 1.0', content) self.assertIn('sample_counter{foo="b"} 1.0', content)
def test_extra_labels(self): prometheus_metrics = create_metrics( self.metrics_definitions, extra_labels={ "foo": "FOO", "bar": "BAR" }, registry=prometheus_client.CollectorRegistry(), ) prometheus_metrics.update("sample_counter", "inc") content = prometheus_metrics.generate_latest().decode("ascii") self.assertIn('sample_counter{bar="BAR",foo="FOO"} 1.0', content)
def test_track_time(self): prometheus_metrics = create_metrics(METRICS_DEFINITIONS) session = TransferTimeTrackingSession( 'file.txt', BytesReader(b'some data'), _clock=Clock(), prometheus_metrics=prometheus_metrics) session.transport = Mock() session.startProtocol() session.cancel() metrics = prometheus_metrics.generate_latest().decode('ascii') self.assertIn( 'maas_tftp_file_transfer_latency_count{filename="file.txt"} 1.0', metrics)
def prometheus_stats_handler(request): configs = Config.objects.get_configs(['prometheus_enabled', 'uuid']) have_prometheus = PROMETHEUS_SUPPORTED and configs['prometheus_enabled'] if not have_prometheus: return HttpResponseNotFound() metrics = create_metrics( STATS_DEFINITIONS, extra_labels={'maas_id': configs['uuid']}, update_handlers=[update_prometheus_stats], registry=prom_cli.CollectorRegistry()) return HttpResponse( content=metrics.generate_latest(), content_type="text/plain")
def prometheus_stats_handler(request): have_prometheus = ( PROMETHEUS_SUPPORTED and Config.objects.get_config('prometheus_enabled')) if not have_prometheus: return HttpResponseNotFound() metrics = create_metrics( STATS_DEFINITIONS, registry=prom_cli.CollectorRegistry()) update_prometheus_stats(metrics) return HttpResponse( content=metrics.generate_latest(), content_type="text/plain")
def test_update_metrics(self): prometheus_metrics = create_metrics(METRICS_DEFINITIONS) middleware = PrometheusRequestMetricsMiddleware( self.get_response, prometheus_metrics=prometheus_metrics) middleware(factory.make_fake_request("/MAAS/accounts/login/")) middleware(factory.make_fake_request("/MAAS/accounts/login/")) middleware(factory.make_fake_request("/MAAS/other/path")) metrics_text = prometheus_metrics.generate_latest().decode('ascii') self.assertIn( 'maas_http_request_latency_count{method="GET",' 'path="/MAAS/accounts/login/",status="200"} 2.0', metrics_text) self.assertIn( 'maas_http_request_latency_count{method="GET",' 'path="/MAAS/other/path",status="404"} 1.0', metrics_text)
def test_subnet_stats(self): subnet = factory.make_Subnet(cidr='1.2.0.0/16', gateway_ip='1.2.0.254') factory.make_IPRange(subnet=subnet, start_ip='1.2.0.11', end_ip='1.2.0.20', alloc_type=IPRANGE_TYPE.DYNAMIC) factory.make_IPRange(subnet=subnet, start_ip='1.2.0.51', end_ip='1.2.0.70', alloc_type=IPRANGE_TYPE.RESERVED) factory.make_StaticIPAddress(ip='1.2.0.12', alloc_type=IPADDRESS_TYPE.DHCP, subnet=subnet) for n in (60, 61): factory.make_StaticIPAddress( ip='1.2.0.{}'.format(n), alloc_type=IPADDRESS_TYPE.USER_RESERVED, subnet=subnet) for n in (80, 90, 100): factory.make_StaticIPAddress(ip='1.2.0.{}'.format(n), alloc_type=IPADDRESS_TYPE.STICKY, subnet=subnet) metrics = create_metrics( STATS_DEFINITIONS, registry=prometheus_client.CollectorRegistry()) update_prometheus_stats(metrics) output = metrics.generate_latest().decode('ascii') self.assertIn( 'maas_net_subnet_ip_count' '{cidr="1.2.0.0/16",status="available"} 65500.0', output) self.assertIn( 'maas_net_subnet_ip_count' '{cidr="1.2.0.0/16",status="unavailable"} 34.0', output) self.assertIn( 'maas_net_subnet_ip_dynamic' '{cidr="1.2.0.0/16",status="available"} 9.0', output) self.assertIn( 'maas_net_subnet_ip_dynamic{cidr="1.2.0.0/16",status="used"} 1.0', output) self.assertIn( 'maas_net_subnet_ip_reserved' '{cidr="1.2.0.0/16",status="available"} 18.0', output) self.assertIn( 'maas_net_subnet_ip_reserved{cidr="1.2.0.0/16",status="used"} 2.0', output) self.assertIn('maas_net_subnet_ip_static{cidr="1.2.0.0/16"} 3.0', output)
def prometheus_stats_handler(request): configs = Config.objects.get_configs(["prometheus_enabled", "uuid"]) have_prometheus = PROMETHEUS_SUPPORTED and configs["prometheus_enabled"] if not have_prometheus: return HttpResponseNotFound() global _METRICS if not _METRICS: _METRICS = create_metrics( STATS_DEFINITIONS, extra_labels={"maas_id": configs["uuid"]}, update_handlers=[update_prometheus_stats], registry=prom_cli.CollectorRegistry(), ) return HttpResponse( content=_METRICS.generate_latest(), content_type="text/plain" )
def test_update_prometheus_stats(self): self.patch(stats, 'prom_cli') # general values values = { "machine_status": { "random_status": 0, }, "controllers": { "regions": 0, }, "nodes": { "machines": 0, }, "network_stats": { "spaces": 0, }, "machine_stats": { "total_cpus": 0, }, } mock = self.patch(stats, "get_maas_stats") mock.return_value = json.dumps(values) # architecture arches = { "amd64": 0, "i386": 0, } mock_arches = self.patch(stats, "get_machines_by_architecture") mock_arches.return_value = arches # pods pods = { "kvm_pods": 0, "kvm_machines": 0, } mock_pods = self.patch(stats, "get_kvm_pods_stats") mock_pods.return_value = pods metrics = create_metrics( STATS_DEFINITIONS, registry=prometheus_client.CollectorRegistry()) update_prometheus_stats(metrics) self.assertThat(mock, MockCalledOnce()) self.assertThat(mock_arches, MockCalledOnce()) self.assertThat(mock_pods, MockCalledOnce())
def test_wb_start_session(self): prometheus_metrics = create_metrics( METRICS_DEFINITIONS, registry=prometheus_client.CollectorRegistry()) stream_session = FakeStreamSession() session = FakeSession(stream_session) tftp_mock = self.patch(tftp.protocol.TFTP, '_startSession') tftp_mock.return_value = succeed(session) tracking_tftp = TransferTimeTrackingTFTP(sentinel.backend) datagram = RQDatagram(b'file.txt', b'octet', {}) result = yield tracking_tftp._startSession( datagram, '192.168.1.1', 'read', prometheus_metrics=prometheus_metrics) result.session.cancel() metrics = prometheus_metrics.generate_latest().decode('ascii') self.assertIs(result, session) self.assertTrue(stream_session.cancelled) self.assertIn( 'maas_tftp_file_transfer_latency_count{filename="file.txt"} 1.0', metrics)
def test_metrics(self): prometheus_metrics = create_metrics(self.metrics_definitions) self.assertIsInstance(prometheus_metrics, PrometheusMetrics) self.assertCountEqual(prometheus_metrics.available_metrics, ['sample_counter', 'sample_histogram'])
# Copyright 2019 Canonical Ltd. This software is licensed under the # GNU Affero General Public License version 3 (see the file LICENSE). """Prometheus metrics.""" from provisioningserver.prometheus.utils import ( create_metrics, MetricDefinition, ) METRICS_DEFINITIONS = [ MetricDefinition('Histogram', 'maas_http_request_latency', 'HTTP request latency', ['method', 'path', 'status', 'op']), MetricDefinition('Histogram', 'maas_websocket_call_latency', 'Latency of a Websocket handler call', ['call']), ] PROMETHEUS_METRICS = create_metrics(METRICS_DEFINITIONS)
def test_metrics_prometheus_not_availble(self): self.patch(utils, 'PROMETHEUS_SUPPORTED', False) prometheus_metrics = create_metrics(self.metrics_definitions) self.assertEqual(prometheus_metrics.available_metrics, [])
def test_update_prometheus_stats(self): self.patch(stats, "prom_cli") # general values values = { "machine_status": { "random_status": 0 }, "controllers": { "regions": 0 }, "nodes": { "machines": 0 }, "network_stats": { "spaces": 0 }, "machine_stats": { "total_cpu": 0 }, } mock = self.patch(stats, "get_maas_stats") mock.return_value = json.dumps(values) # architecture arches = {"amd64": 0, "i386": 0} mock_arches = self.patch(stats, "get_machines_by_architecture") mock_arches.return_value = arches # pods pods = { "kvm_pods": 0, "kvm_machines": 0, "kvm_available_resources": { "cores": 10, "memory": 20, "storage": 30, "over_cores": 100, "over_memory": 200, }, "kvm_utilized_resources": { "cores": 5, "memory": 10, "storage": 15, }, } mock_pods = self.patch(stats, "get_kvm_pods_stats") mock_pods.return_value = pods subnet_stats = { "1.2.0.0/16": { "available": 2**16 - 3, "dynamic_available": 0, "dynamic_used": 0, "reserved_available": 0, "reserved_used": 0, "static": 0, "unavailable": 1, }, "::1/128": { "available": 1, "dynamic_available": 0, "dynamic_used": 0, "reserved_available": 0, "reserved_used": 0, "static": 0, "unavailable": 0, }, } mock_subnet_stats = self.patch(stats, "get_subnets_utilisation_stats") mock_subnet_stats.return_value = subnet_stats metrics = create_metrics( STATS_DEFINITIONS, registry=prometheus_client.CollectorRegistry()) update_prometheus_stats(metrics) self.assertThat(mock, MockCalledOnce()) self.assertThat(mock_arches, MockCalledOnce()) self.assertThat(mock_pods, MockCalledOnce()) self.assertThat(mock_subnet_stats, MockCalledOnce())
'Latency of a Websocket handler call', ['call']), # Common metrics *node_metrics_definitions() ] # Global for tracking global values for metrics label. These are set # differently from rackd and regiond code, but this is defined here so the # logic using it can be generic. GLOBAL_LABELS = { # The MAAS installation UUID, the same for all regions/racks within a # deployment 'maas_uuid': None, # The type of service (region/rack) exporting the metrics. 'service_type': None } def set_global_labels(**labels): """Update global labels for metrics.""" global GLOBAL_LABELS GLOBAL_LABELS.update(labels) PROMETHEUS_METRICS = create_metrics( METRICS_DEFINITIONS, extra_labels={ 'host': get_machine_default_gateway_ip, 'maas_id': lambda: GLOBAL_LABELS['maas_uuid'], }, update_handlers=[update_cpu_metrics, update_memory_metrics])
def push_stats_to_prometheus(maas_name, push_gateway): metrics = create_metrics(STATS_DEFINITIONS) update_prometheus_stats(metrics) prom_cli.push_to_gateway(push_gateway, job='stats_for_%s' % maas_name, registry=metrics.registry)