class TestCounter(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() def test_initialize(self): c = Counter('test_value', 'Testing roller', registry=self.registry) r = CounterRoller(c, registry=self.registry) self.assertEqual(r.name, 'test_value_sum_rolled') def test_initialize_errors(self): # Raise error because tried to use wrong type of item def wrong_type_exception(): h = Histogram('test_value', 'Testing roller', registry=self.registry) roller = CounterRoller(h, registry=self.registry) self.assertRaises(ValueError, wrong_type_exception) # Update seconds must be > 0 def update_seconds_lt_1_exception(): c = Counter('test_value', 'Testing roller', registry=self.registry) roller = CounterRoller(c, registry=self.registry, options={ 'update_seconds': 0 }) self.assertRaises(ValueError, update_seconds_lt_1_exception) # Update seconds must be a multiple of 1 def update_seconds_not_divisible_by_1_exception(): c = Counter('test_value', 'Testing roller', registry=self.registry) roller = CounterRoller(c, registry=self.registry, options={ 'update_seconds': 2.5 }) self.assertRaises(ValueError, update_seconds_not_divisible_by_1_exception) def test_collect(self): c = Counter('test_value', 'Testing roller', registry=self.registry) r = CounterRoller(c, registry=self.registry) r.collect() nchecks = 0 for m in self.registry.collect(): if m.name.endswith('sum_rolled'): for name, labels, val in m.samples: self.assertEqual(val, 0.0) nchecks += 1 self.assertTrue(nchecks > 0) c.inc() c.inc(1.5) r.collect() nchecks = 0 for m in self.registry.collect(): if m.name.endswith('sum_rolled'): for name, labels, val in m.samples: self.assertEqual(val, 2.5) nchecks += 1 self.assertTrue(nchecks > 0)
def get_metrics(registry: CollectorRegistry = REGISTRY, verbose: bool = False): """ Collects timeseries samples from prometheus metric collector registry adds a common timestamp, and encodes them to protobuf Arguments: registry: a prometheus CollectorRegistry instance verbose: whether to optimize for bandwidth and ignore metric name/help Returns: a prometheus MetricFamily protobuf stream """ timestamp_ms = int(time.time() * 1000) for metric_family in registry.collect(): if metric_family.type in ('counter', 'gauge'): family_proto = encode_counter_gauge(metric_family, timestamp_ms) elif metric_family.type == 'summary': family_proto = encode_summary(metric_family, timestamp_ms) elif metric_family.type == 'histogram': family_proto = encode_histogram(metric_family, timestamp_ms) family_proto.name = metric_family.name if verbose: family_proto.help = metric_family.documentation yield family_proto
async def test_exception( self, client: TestClient, client_name: str, namespace_prefix: str, current_registry: prometheus_client.CollectorRegistry, ): with pytest.raises(TypeError): response = await client.post("/200", data=TestClient) await response.json() current_frozen_registry: List[prometheus_client.Metric] = list( current_registry.collect()) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_requests_exceptions", f"{namespace_prefix}aiohttp_client_requests_exceptions_total", 1.0, labels={ "client_name": client_name, "method": "POST", "scheme": "http", "remote": "127.0.0.1", "exception_name": "TypeError", }, )
async def test_parallel_connection( self, client: TestClient, client_name: str, namespace_prefix: str, current_registry: prometheus_client.CollectorRegistry, ): results = await asyncio.gather(client.get("/200"), client.get("/200")) await asyncio.gather(*(r.json() for r in results)) current_frozen_registry: List[prometheus_client.Metric] = list( current_registry.collect()) assert_metric_exists( current_frozen_registry, f"{namespace_prefix}aiohttp_client_connection_create_seconds", f"{namespace_prefix}aiohttp_client_connection_create_seconds_bucket", labels={ "client_name": client_name, }, ) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_connection_reuseconn", f"{namespace_prefix}aiohttp_client_connection_reuseconn_total", 1.0, labels={ "client_name": client_name, }, )
def test_old_labels_are_removed_from_registry_after_rebalance( self, monitor: PrometheusMonitor, metrics: FaustMetrics, registry: CollectorRegistry, stream: StreamT, event: EventT, app: AppT, ) -> None: self._handle_event( monitor=monitor, topic_partition=TP1, stream=stream, event=event, offset=10, ) monitor.on_rebalance_start(app) monitor.on_rebalance_end(app, state={"time_start": monitor.time()}) self._handle_event( monitor=monitor, topic_partition=TP2, stream=stream, event=event, offset=11, ) collected_topics = frozenset( sample.labels["topic"] for metric in registry.collect() if metric.name == "test_messages_received_per_topic" for sample in metric.samples ) assert collected_topics == frozenset([TP2.topic]) collected_partitions = frozenset( (sample.labels["topic"], sample.labels["partition"]) for metric in registry.collect() if metric.name == "test_messages_received_per_topics_partition" for sample in metric.samples ) assert collected_partitions == frozenset([(TP2.topic, str(TP2.partition))])
async def test_redirect( self, client: TestClient, client_name: str, namespace_prefix: str, current_registry: prometheus_client.CollectorRegistry, ): response = await client.get("/redirect") await response.json() current_frozen_registry: List[prometheus_client.Metric] = list( current_registry.collect()) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_requests_redirect", f"{namespace_prefix}aiohttp_client_requests_redirect_total", 1.0, labels={ "client_name": client_name, "method": "GET", "scheme": "http", "remote": "127.0.0.1", "status_code": "302", }, ) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_requests", f"{namespace_prefix}aiohttp_client_requests_total", 1.0, labels={ "client_name": client_name, "method": "GET", "scheme": "http", "remote": "127.0.0.1", "status_code": "200", }, ) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_requests", f"{namespace_prefix}aiohttp_client_requests_total", 1.0, labels={ "client_name": client_name, "method": "GET", "scheme": "http", "remote": "127.0.0.1", "status_code": "302", }, )
class TestGCCollector(unittest.TestCase): def setUp(self): gc.disable() gc.collect() self.registry = CollectorRegistry() def test_working(self): GCCollector(registry=self.registry) self.registry.collect() before = self.registry.get_sample_value( 'python_gc_objects_collected_total', labels={"generation": "0"}) # add targets for gc a = [] a.append(a) del a b = [] b.append(b) del b gc.collect(0) self.registry.collect() after = self.registry.get_sample_value( 'python_gc_objects_collected_total', labels={"generation": "0"}) self.assertEqual(2, after - before) self.assertEqual( 0, self.registry.get_sample_value( 'python_gc_objects_uncollectable_total', labels={"generation": "0"})) def test_empty(self): GCCollector(registry=self.registry) self.registry.collect() before = self.registry.get_sample_value( 'python_gc_objects_collected_total', labels={"generation": "0"}) gc.collect(0) self.registry.collect() after = self.registry.get_sample_value( 'python_gc_objects_collected_total', labels={"generation": "0"}) self.assertEqual(0, after - before) def tearDown(self): gc.enable()
def collect(view_id, creds_path, metric_prefix, dry_run=False): """Submit data to Push Gateway.""" registry = CollectorRegistry() try: analytics = initialize_analyticsreporting(creds_path, SCOPES) # Get historical, all-time total counters. response = get_report(analytics, view_id, '2010-01-01') set_gauges(registry, response, metric_prefix) except Exception: # pylint: disable=broad-except logging.exception('Error collecting metrics') finally: if not dry_run: # If set_gauges bombed out for any reason, just upload blank data. # Make sure no proxy is used. for var in ('http_proxy', 'http_proxy'): for envvar in (var.lower(), var.upper()): os.environ.pop(envvar, None) util.push2gateway(metric_prefix, registry) else: # Debugging enabled. pprint.pprint([(x.name, x.samples) for x in registry.collect()])
class TestPlatformCollector(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() self.platform = _MockPlatform() def test_python_info(self): PlatformCollector(registry=self.registry, platform=self.platform) self.assertLabels( "python_info", { "version": "python_version", "implementation": "python_implementation", "major": "pvt_major", "minor": "pvt_minor", "patchlevel": "pvt_patchlevel" }) def test_system_info_java(self): self.platform._system = "Java" PlatformCollector(registry=self.registry, platform=self.platform) self.assertLabels( "python_info", { "version": "python_version", "implementation": "python_implementation", "major": "pvt_major", "minor": "pvt_minor", "patchlevel": "pvt_patchlevel", "jvm_version": "jv_release", "jvm_release": "vm_release", "jvm_vendor": "vm_vendor", "jvm_name": "vm_name" }) def assertLabels(self, name, labels): for metric in self.registry.collect(): for s in metric.samples: if s.name == name: assert s.labels == labels return assert False
async def test_google( self, registry, namespace: str, namespace_prefix: str, client_name: str, current_registry: prometheus_client.CollectorRegistry, ): params = {} if registry: params["registry"] = registry if namespace is not None: params["namespace"] = namespace if client_name is not None: params["client_name"] = client_name connector = aiohttp.TCPConnector(ttl_dns_cache=300, force_close=True) async with aiohttp.ClientSession( trace_configs=[PrometheusTraceConfig(**params)], connector=connector) as session: async with session.get("http://www.google.com/") as resp: assert resp.status async with session.get("http://www.google.com/") as resp: assert resp.status async with aiohttp.ClientSession( trace_configs=[PrometheusTraceConfig(**params)]) as session: async with session.get("http://www.google.com/") as resp: assert resp.status current_frozen_registry: List[prometheus_client.Metric] = list( current_registry.collect()) assert_metric_exists( current_frozen_registry, f"{namespace_prefix}aiohttp_client_dns_resolvehost_seconds", f"{namespace_prefix}aiohttp_client_dns_resolvehost_seconds_bucket", labels={ "client_name": client_name, "host": "www.google.com", }, ) assert_metric_exists( current_frozen_registry, f"{namespace_prefix}aiohttp_client_dns_cache_miss", f"{namespace_prefix}aiohttp_client_dns_cache_miss_total", labels={ "client_name": client_name, "host": "www.google.com", }, ) assert_metric_exists( current_frozen_registry, f"{namespace_prefix}aiohttp_client_dns_cache_hit", f"{namespace_prefix}aiohttp_client_dns_cache_hit_total", labels={ "client_name": client_name, "host": "www.google.com", }, )
async def test_ok( self, client: TestClient, client_name: str, namespace_prefix: str, current_registry: prometheus_client.CollectorRegistry, ): response = await client.get("/200") await response.json() current_frozen_registry: List[prometheus_client.Metric] = list( current_registry.collect()) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_requests", f"{namespace_prefix}aiohttp_client_requests_total", 1.0, labels={ "client_name": client_name, "method": "GET", "scheme": "http", "remote": "127.0.0.1", "status_code": "200", }, ) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_requests_in_progress", f"{namespace_prefix}aiohttp_client_requests_in_progress", 0.0, labels={ "client_name": client_name, "method": "GET", "scheme": "http", "remote": "127.0.0.1", }, ) assert_metric_exists( current_frozen_registry, f"{namespace_prefix}aiohttp_client_request_duration_seconds", f"{namespace_prefix}aiohttp_client_request_duration_seconds_bucket", labels={ "client_name": client_name, "method": "GET", "scheme": "http", "remote": "127.0.0.1", }, ) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_chunks_sent_bytes", f"{namespace_prefix}aiohttp_client_chunks_sent_bytes_total", 0.0, labels={ "client_name": client_name, }, ) assert_metric_value( current_frozen_registry, f"{namespace_prefix}aiohttp_client_chunks_received_bytes", f"{namespace_prefix}aiohttp_client_chunks_received_bytes_total", 26.0, labels={ "client_name": client_name, }, ) assert_metric_exists( current_frozen_registry, f"{namespace_prefix}aiohttp_client_connection_create_seconds", f"{namespace_prefix}aiohttp_client_connection_create_seconds_bucket", labels={ "client_name": client_name, }, )
class TestHistogram(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry() def get_rolled_samples(self): """Get all 'rolled' type gauges in the current registry """ for m in self.registry.collect(): if m.name.endswith('_rolled'): for name, labels, val in m.samples: yield name, labels, val def get_hist_samples(self): """Get all histogram buckets in the current registry """ for m in self.registry.collect(): if m.name == 'test_value': for name, labels, val in m.samples: if name.endswith('_bucket'): yield name, labels, val def test_initialize(self): h = Histogram('test_value', 'Testing roller', registry=self.registry) roller = HistogramRoller(h, registry=self.registry) n_buckets = 0 for name, _, _ in self.get_hist_samples(): if name.endswith('_bucket'): n_buckets += 1 n_created_guages = 0 for _, _, _ in self.get_rolled_samples(): n_created_guages += 1 # Check that roller gauges don't exist until values are added self.assertTrue(n_buckets > 0) self.assertTrue(n_created_guages == 0) self.assertEqual(roller.name, 'test_value_sum_rolled') def test_initialize_errors(self): # Raise error because tried to use wrong type of item def wrong_type_exception(): c = Counter('test_value', 'Testing roller', registry=self.registry) roller = HistogramRoller(c, registry=self.registry) self.assertRaises(ValueError, wrong_type_exception) # Update seconds must be > 0 def update_seconds_lt_1_exception(): h = Histogram('test_value', 'Testing roller', registry=self.registry) roller = HistogramRoller(h, registry=self.registry, options={ 'update_seconds': 0 }) self.assertRaises(ValueError, update_seconds_lt_1_exception) # Update seconds must be a multiple of 1 def update_seconds_not_divisible_by_1_exception(): h = Histogram('test_value', 'Testing roller', registry=self.registry) roller = HistogramRoller(h, registry=self.registry, options={ 'update_seconds': 2.5 }) self.assertRaises(ValueError, update_seconds_not_divisible_by_1_exception) def test_collect(self): h = Histogram('test_value', 'Testing roller', registry=self.registry) roller = HistogramRoller(h, registry=self.registry) # Get values roller.collect() n_buckets = 0 for _, _, _ in self.get_hist_samples(): n_buckets += 1 n_created_guages = 0 for _, _, _ in self.get_rolled_samples(): n_created_guages += 1 self.assertTrue(n_buckets > 0) self.assertTrue(n_created_guages > 0) self.assertEqual(n_buckets, n_created_guages) # Check that roller values are still 0.0 after initial collection for name, labels, value in self.get_rolled_samples(): self.assertEqual(value, 0.0) # Add some samples for i in range(100): h.observe(pow(2, i/10 - 2)) # Collect hisogram values hist_values = dict() for name, labels, value in self.get_hist_samples(): hist_values[labels['le']] = value # Make sure they are still equal after collection for name, labels, value in self.get_rolled_samples(): self.assertEqual(value, 0.0) roller.collect() for name, labels, value in self.get_rolled_samples(): self.assertEqual(value, hist_values[labels['le']]) def test_customize_reducer(self): h = Histogram('test_value', 'Testing roller', registry=self.registry) roller_max = HistogramRoller(h, registry=self.registry, options={ 'reducer': 'max' }) roller_min = HistogramRoller(h, registry=self.registry, options={ 'reducer': 'sum' }) def always_one(*args, **kwargs): return 1 roller_one = HistogramRoller(h, registry=self.registry, options={ 'reducer': always_one }) for state in [2.6, 4.7, 3.8, 2.8]: h.observe(state) roller_max.collect() roller_min.collect() roller_one.collect() # Deltas = 1, 1, 1 nchecks = 0 for m in self.registry.collect(): if m.name.endswith('max_rolled'): for name, labels, val in m.samples: if labels['le'] == '5.0': nchecks += 1 self.assertEqual(val, 1.0) self.assertTrue(nchecks > 0) nchecks = 0 for m in self.registry.collect(): if m.name.endswith('sum_rolled'): for name, labels, val in m.samples: if labels['le'] == '5.0': self.assertEqual(val, 3.0) nchecks += 1 self.assertTrue(nchecks > 0) nchecks = 0 for m in self.registry.collect(): if m.name.endswith('always_one_rolled'): for name, labels, val in m.samples: if labels['le'] == '5.0': self.assertEqual(val, 1.0) nchecks += 1 self.assertTrue(nchecks > 0)