def configure_prometheus_metrics_exporter(app: Starlette): app.add_middleware(MetricsMiddleware, webapp=app) app.registry = Registry() const_labels = { "host": socket.gethostname(), "name": "service1", "version": "1" } app.counter_gauge = Gauge("counter", "Current count.", const_labels=const_labels) app.registry.register(app.counter_gauge) app.svc_requests_total = Counter("svc_requests_total", "Count of service HTTP requests", const_labels=const_labels) app.registry.register(app.svc_requests_total) app.svc_responses_total = Counter("svc_responses_total", "Count of service HTTP responses", const_labels=const_labels) app.registry.register(app.svc_responses_total) app.svc_internal_error_total = Counter( "svc_internal_error_total", "Histogram of internal errors by method, path and type of error", const_labels=const_labels) app.registry.register(app.svc_internal_error_total)
def __init__(self, bot: commands.Bot): self.bot = bot self.registry = Registry() self.service = Service(self.registry) self.events = Counter("events", "Discord API event counts.") self.registry.register(self.events) self.latency = Histogram("latency", "Discord API latency.") self.registry.register(self.latency) self.gc_started: typing.Optional[float] = None self.gc_latency = Histogram( "gc_latency", "CPython garbage collector execution times." ) self.registry.register(self.gc_latency) self.gc_stats = Counter("gc_stats", "CPython garbage collector stats.") self.registry.register(self.gc_stats) self.process = psutil.Process(os.getpid()) self.resources = Gauge("resources", "Process resource usage gauges.") self.registry.register(self.resources) self.hook_gc() self.update_gc_and_resource_stats.start() # pylint: disable=no-member self.serve.start() # pylint: disable=no-member self.update_latency.start() # pylint: disable=no-member
async def test_grouping_key_with_empty_value(self): # See https://github.com/prometheus/pushgateway/blob/master/README.md#url # for encoding rules. job_name = "example" p = pusher.Pusher( job_name, self.server.url, grouping_key={ "first": "", "second": "foo" }, ) registry = Registry() c = Counter("example_total", "Total examples", {}) registry.register(c) c.inc({}) # Push to the pushgateway resp = await p.replace(registry) self.assertEqual(resp.status, 200) self.assertEqual( "/metrics/job/example/first@base64/=/second/foo", self.server.test_results["path"], )
async def test_no_accept_header(self): """ check default format is used when no accept header is defined """ # Add some metrics data = (({"data": 1}, 100), ) c = Counter("test_counter", "Test Counter.", {"test": "test_counter"}) self.server.register(c) for i in data: c.set(i[0], i[1]) expected_data = """# HELP test_counter Test Counter. # TYPE test_counter counter test_counter{data="1",test="test_counter"} 100 """ async with aiohttp.ClientSession() as session: # Fetch without explicit accept type async with session.get(self.metrics_url) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(text.TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) self.assertEqual(expected_data, content.decode())
def test_register_deregister(self): """ check registering and deregistering metrics """ c = Counter("test_counter", "Test Counter.", {"test": "test_counter"}) self.server.register(c) # Check registering a collector with same name raises an exception c2 = Counter("test_counter", "Another Test Counter.") with self.assertRaises(ValueError) as cm: self.server.register(c2) self.assertIn("is already registered", str(cm.exception)) self.server.deregister("test_counter") # Check deregistering a non-existant collector raises an exception with self.assertRaises(KeyError) as cm: self.server.deregister("test_counter")
async def test_push_add(self): job_name = "my-job" p = pusher.Pusher(job_name, self.server.url) registry = Registry() counter = Counter("counter_test", "A counter.", {"type": "counter"}) registry.register(counter) counter_data = (({"c_sample": "1", "c_subsample": "b"}, 400), ) [counter.set(c[0], c[1]) for c in counter_data] # TextFormatter expected result valid_result = ( b"# HELP counter_test A counter.\n" b"# TYPE counter_test counter\n" b'counter_test{c_sample="1",c_subsample="b",type="counter"} 400\n') # BinaryFormatter expected result # valid_result = (b'[\n\x0ccounter_test\x12\nA counter.\x18\x00"=\n\r' # b'\n\x08c_sample\x12\x011\n\x10\n\x0bc_subsample\x12' # b'\x01b\n\x0f\n\x04type\x12\x07counter\x1a\t\t\x00' # b'\x00\x00\x00\x00\x00y@') # Push to the pushgateway resp = await p.add(registry) self.assertEqual(resp.status, 200) self.assertEqual(expected_job_path(job_name), self.server.test_results["path"]) self.assertEqual("POST", self.server.test_results["method"]) self.assertEqual(valid_result, self.server.test_results["body"])
async def test_count_exceptions(self): m = Counter("metric_label", "metric help") # decorator should work methods as well as functions @count_exceptions(m, {"kind": "function"}) async def a(raise_exc=False): if raise_exc: raise Exception("dummy exception") return await a() # metric should not exist until an exception occurs with self.assertRaises(KeyError): m.get({"kind": "function"}) with self.assertRaises(Exception) as cm: await a(True) self.assertIn("dummy exception", str(cm.exception)) m_function_value = m.get({"kind": "function"}) self.assertEqual(m_function_value, 1) # decorator should work methods as well as functions class B(object): @count_exceptions(m, {"kind": "method"}) async def b(self, arg1, arg2=None, raise_exc=False): if raise_exc: raise Exception("dummy exception") return arg1 == "b_arg", arg2 == "arg_2" b = B() results = await b.b("b_arg", arg2="arg_2") self.assertTrue(all(results)) # metric should not exist until an exception occurs with self.assertRaises(KeyError): m.get({"kind": "method"}) with self.assertRaises(Exception) as cm: await b.b("b_arg", arg2="arg_2", raise_exc=True) self.assertIn("dummy exception", str(cm.exception)) m_method_value = m.get({"kind": "method"}) self.assertEqual(m_method_value, 1) # Only Gauge metric type can be used with @timer, others should # raise an exception. with self.assertRaises(Exception) as cm: m = Histogram("metric_label", "metric help") @count_exceptions(m) async def c(): return self.assertIn( "count_exceptions decorator expects a Counter metric but got:", str(cm.exception), )
def setUp(self): self.data = { 'name': "logged_users_total", 'doc': "Logged users in the application", 'const_labels': {"app": "my_app"}, } self.c = Counter(**self.data)
async def test_counter(self): """ check counter metric export """ # Add some metrics data = ( ({ "data": 1 }, 100), ({ "data": "2" }, 200), ({ "data": 3 }, 300), ({ "data": 1 }, 400), ) c = Counter("test_counter", "Test Counter.", {"test": "test_counter"}) self.server.register(c) for i in data: c.set(i[0], i[1]) expected_data = """# HELP test_counter Test Counter. # TYPE test_counter counter test_counter{data="1",test="test_counter"} 400 test_counter{data="2",test="test_counter"} 200 test_counter{data="3",test="test_counter"} 300 """ async with aiohttp.ClientSession() as session: # Fetch as text async with session.get(self.metrics_url, headers={ACCEPT: text.TEXT_CONTENT_TYPE}) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(text.TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) self.assertEqual(expected_data, content.decode()) # Fetch as binary async with session.get(self.metrics_url, headers={ ACCEPT: binary.BINARY_CONTENT_TYPE }) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(binary.BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) metrics = pmp.decode(content) self.assertEqual(len(metrics), 1) mf = metrics[0] self.assertIsInstance(mf, pmp.MetricFamily) self.assertEqual(mf.type, pmp.COUNTER) self.assertEqual(len(mf.metric), 3)
def setUp(self): self.data = { "name": "logged_users_total", "doc": "Logged users in the application", "const_labels": { "app": "my_app" }, } self.c = Counter(**self.data)
def __init__(self, bot): self.bot = bot self.msvr = Service() if platform.system() == "Linux": self.platform = platform self.pid = os.path.join("/proc", "self") self.pagesize = resource.getpagesize() self.ticks = os.sysconf("SC_CLK_TCK") self.btime = 0 with open(os.path.join("/proc", "stat"), "rb") as stat: for line in stat: if line.startswith(b"btime "): self.btime = float(line.split()[1]) break self.vmem = Gauge("process_virtual_memory_bytes", "Virtual memory size in bytes.") self.rss = Gauge("process_resident_memory_bytes", "Resident memory size in bytes.") self.start_time = Gauge( "process_start_time_seconds", "Start time of the process since unix epoch in seconds.") self.cpu = Counter("process_cpu_seconds", "Total user and system CPU time spent in seconds.") self.fds = Gauge("process_open_fds", "Number of open file descriptors.") self.info = Gauge("python_info", "Python platform information.") self.collected = Counter("python_gc_objects_collected", "Objects collected during GC.") self.uncollectable = Counter("python_gc_objects_uncollectable", "Uncollectable objects found during GC.") self.collections = Counter( "python_gc_collections", "Number of times this generation was collected.") self.http = Counter("modmail_http_requests", "The number of http requests sent to Discord.") self.commands = Counter( "modmail_commands", "The total number of commands used on the bot.") self.tickets = Counter( "modmail_tickets", "The total number of tickets created by the bot.") self.tickets_message = Counter( "modmail_tickets_message", "The total number of messages sent in tickets.")
async def test_push_job_ping(self): job_name = "my-job" p = pusher.Pusher(job_name, self.server.url, loop=self.loop) registry = Registry() c = Counter("total_requests", "Total requests.", {}) registry.register(c) c.inc({"url": "/p/user"}) # Push to the pushgateway resp = await p.replace(registry) self.assertEqual(resp.status, 200) self.assertEqual(expected_job_path(job_name), self.server.test_results["path"])
def test_counter_format_with_const_labels(self): data = { 'name': "logged_users_total", 'doc': "Logged users in the application", 'const_labels': { "app": "my_app" }, } c = Counter(**data) counter_data = ( ({ 'country': "sp", "device": "desktop" }, 520), ({ 'country': "us", "device": "mobile" }, 654), ({ 'country': "uk", "device": "desktop" }, 1001), ({ 'country': "de", "device": "desktop" }, 995), ({ 'country': "zh", "device": "desktop" }, 520), ) # Construct the result to compare valid_result = self._create_protobuf_object(data, counter_data, pmp.COUNTER, data['const_labels']) # Add data to the collector for i in counter_data: c.set_value(i[0], i[1]) f = BinaryFormatter() result = f.marshall_collector(c) self.assertTrue(self._protobuf_metric_equal(valid_result, result))
async def main(svr: Service) -> None: events_counter = Counter("events", "Number of events.", const_labels={"host": socket.gethostname()}) svr.register(events_counter) await svr.start(addr="", port=5000) print(f"Serving prometheus metrics on: {svr.metrics_url}") # Now start another coroutine to periodically update a metric to # simulate the application making some progress. async def updater(c: Counter): while True: c.inc({"kind": "timer_expiry"}) await asyncio.sleep(1.0) await updater(events_counter)
def test_no_metric_instances_present_binary(self): """ Check marshalling a collector with no metrics instances present """ c = Counter( name=self.counter_metric_name, doc=self.counter_metric_help, const_labels=self.const_labels, ) f = binary.BinaryFormatter() result = f.marshall_collector(c) self.assertIsInstance(result, pmp.MetricFamily) # Construct the result expected to receive when the counter # collector is marshalled. expected_result = pmp.create_counter(self.counter_metric_name, self.counter_metric_help, []) self.assertEqual(result, expected_result)
def test_counter_format_with_timestamp(self): self.data = { "name": "logged_users_total", "doc": "Logged users in the application", "const_labels": {}, } c = Counter(**self.data) counter_data = ({"country": "ch", "device": "mobile"}, 654) c.set_value(counter_data[0], counter_data[1]) result_regex = r"""# HELP logged_users_total Logged users in the application # TYPE logged_users_total counter logged_users_total{country="ch",device="mobile"} 654 \d*(?:.\d*)?$""" f_with_ts = text.TextFormatter(True) result = f_with_ts.marshall_collector(c) self.assertTrue(re.match(result_regex, result))
def test_registry_marshall_counter(self): counter_data = (({"c_sample": "1", "c_subsample": "b"}, 400), ) counter = Counter("counter_test", "A counter.", const_labels={"type": "counter"}) for labels, value in counter_data: counter.set(labels, value) registry = Registry() registry.register(counter) valid_result = (b'[\n\x0ccounter_test\x12\nA counter.\x18\x00"=\n\r' b"\n\x08c_sample\x12\x011\n\x10\n\x0bc_subsample\x12" b"\x01b\n\x0f\n\x04type\x12\x07counter\x1a\t\t\x00\x00" b"\x00\x00\x00\x00y@") f = binary.BinaryFormatter() self.assertEqual(valid_result, f.marshall(registry))
async def test_counter(self): # Add some metrics data = ( ({ 'data': 1 }, 100), ({ 'data': "2" }, 200), ({ 'data': 3 }, 300), ({ 'data': 1 }, 400), ) c = Counter("test_counter", "Test Counter.", {'test': "test_counter"}) self.registry.register(c) for i in data: c.set(i[0], i[1]) expected_data = """# HELP test_counter Test Counter. # TYPE test_counter counter test_counter{data="1",test="test_counter"} 400 test_counter{data="2",test="test_counter"} 200 test_counter{data="3",test="test_counter"} 300 """ with aiohttp.ClientSession(loop=self.loop) as session: headers = {ACCEPT: 'text/plain; version=0.0.4'} async with session.get(self.metrics_url, headers=headers) as resp: assert resp.status == 200 content = await resp.read() self.assertEqual("text/plain; version=0.0.4; charset=utf-8", resp.headers.get(CONTENT_TYPE)) self.assertEqual(200, resp.status) self.assertEqual(expected_data, content.decode())
async def test_inprogress(self): m = Gauge("metric_label", "metric help") # decorator should work methods as well as functions @inprogress(m, {"kind": "function"}) async def a(): return await a() m_function_value = m.get({"kind": "function"}) self.assertEqual(m_function_value, 0) # decorator should work methods as well as functions class B(object): @inprogress(m, {"kind": "method"}) async def b(self, arg1, arg2=None): return arg1 == "b_arg", arg2 == "arg_2" b = B() results = await b.b("b_arg", arg2="arg_2") self.assertTrue(all(results)) m_method_value = m.get({"kind": "method"}) self.assertEqual(m_method_value, 0) # Only Gauge metric type can be used with @timer, others should # raise an exception. with self.assertRaises(Exception) as cm: m = Counter("metric_label", "metric help") @inprogress(m) async def c(): return self.assertIn( "inprogess decorator expects a Gauge metric but got:", str(cm.exception) )
async def test_timer(self): m = Summary("metric_label", "metric help") # decorator should work methods as well as functions @timer(m, {"kind": "function"}) async def a(): return await a() m_function = m.get({"kind": "function"}) self.assertEqual(m_function["count"], 1) # decorator should work methods as well as functions class B(object): @timer(m, {"kind": "method"}) async def b(self, arg1, arg2=None): return arg1 == "b_arg", arg2 == "arg_2" b = B() results = await b.b("b_arg", arg2="arg_2") self.assertTrue(all(results)) m_method = m.get({"kind": "method"}) self.assertEqual(m_method["count"], 1) # Only Summary metric type can be used with @timer, others should # raise an exception. with self.assertRaises(Exception) as cm: m = Counter("metric_label", "metric help") @timer(m) async def c(): return self.assertIn( "timer decorator expects a Summary metric but got:", str(cm.exception) )
async def test_push_add(self): job_name = "my-job" p = Pusher(job_name, TEST_URL) registry = Registry() counter = Counter("counter_test", "A counter.", {'type': "counter"}) registry.register(counter) counter_data = (({'c_sample': '1', 'c_subsample': 'b'}, 400), ) [counter.set(c[0], c[1]) for c in counter_data] valid_result = (b'[\n\x0ccounter_test\x12\nA counter.\x18\x00"=\n\r' b'\n\x08c_sample\x12\x011\n\x10\n\x0bc_subsample\x12' b'\x01b\n\x0f\n\x04type\x12\x07counter\x1a\t\t\x00' b'\x00\x00\x00\x00\x00y@') # Push to the pushgateway resp = await p.add(registry) self.assertEqual(resp.status, 200) self.assertEqual(expected_job_path(job_name), self.server.test_results['path']) self.assertEqual("POST", self.server.test_results['method']) self.assertEqual(valid_result, self.server.test_results['body'])
def test_registry_marshall_counter(self): format_times = 10 counter_data = (({'c_sample': '1', 'c_subsample': 'b'}, 400), ) registry = Registry() counter = Counter("counter_test", "A counter.", {'type': "counter"}) # Add data [counter.set(c[0], c[1]) for c in counter_data] registry.register(counter) valid_result = (b'[\n\x0ccounter_test\x12\nA counter.\x18\x00"=\n\r' b'\n\x08c_sample\x12\x011\n\x10\n\x0bc_subsample\x12' b'\x01b\n\x0f\n\x04type\x12\x07counter\x1a\t\t\x00\x00' b'\x00\x00\x00\x00y@') f = BinaryFormatter() # Check multiple times to ensure multiple marshalling requests for i in range(format_times): self.assertEqual(valid_result, f.marshall(registry))
async def test_grouping_key(self): # See https://github.com/prometheus/pushgateway/blob/master/README.md#url # for encoding rules. job_name = "my-job" p = pusher.Pusher( job_name, self.server.url, grouping_key={"instance": "127.0.0.1:1234"}, ) registry = Registry() c = Counter("total_requests", "Total requests.", {}) registry.register(c) c.inc({}) # Push to the pushgateway resp = await p.replace(registry) self.assertEqual(resp.status, 200) self.assertEqual( "/metrics/job/my-job/instance/127.0.0.1:1234", self.server.test_results["path"], )
def test_single_counter_format_text(self): name = "prometheus_dns_sd_lookups_total" doc = "The number of DNS-SD lookups." valid_result = """# HELP prometheus_dns_sd_lookups_total The number of DNS-SD lookups. # TYPE prometheus_dns_sd_lookups_total counter prometheus_dns_sd_lookups_total 10""" data = ((None, 10), ) # Create the counter c = Counter(name=name, doc=doc, const_labels={}) for i in data: c.set_value(i[0], i[1]) # Select format f = text.TextFormatter() result = f.marshall_collector(c) self.assertEqual(valid_result, result)
async def test_timer(self): m = Summary('metric_label', 'metric help') # decorator should work methods as well as functions @timer(m, {'kind': 'function'}) async def a(): return await a() m_function = m.get({'kind': 'function'}) self.assertEqual(m_function['count'], 1) # decorator should work methods as well as functions class B(object): @timer(m, {'kind': 'method'}) async def b(self, arg1, arg2=None): return arg1 == 'b_arg', arg2 == 'arg_2' b = B() results = await b.b('b_arg', arg2='arg_2') self.assertTrue(all(results)) m_method = m.get({'kind': 'method'}) self.assertEqual(m_method['count'], 1) # Only Summary metric type can be used with @timer, others should # raise an exception. with self.assertRaises(Exception) as cm: m = Counter('metric_label', 'metric help') @timer(m) async def c(): return self.assertIn("timer decorator expects a Summary metric but got:", str(cm.exception))
async def test_grouping_key_with_value_containing_slash(self): # See https://github.com/prometheus/pushgateway/blob/master/README.md#url # for encoding rules. job_name = "directory_cleaner" p = pusher.Pusher( job_name, self.server.url, grouping_key={"path": "/var/tmp"}, ) registry = Registry() c = Counter("exec_total", "Total executions", {}) registry.register(c) c.inc({}) # Push to the pushgateway resp = await p.replace(registry) self.assertEqual(resp.status, 200) # Generated base64 content include '=' as padding. self.assertEqual( "/metrics/job/directory_cleaner/path@base64/L3Zhci90bXA=", self.server.test_results["path"], )
$ curl :8000/metrics # HELP request_handler_exceptions Number of exceptions in requests # TYPE request_handler_exceptions counter request_handler_exceptions{route="/"} 3 You may need to Ctrl+C twice to exit the example script. ''' import asyncio import random from aioprometheus import Service, Counter, count_exceptions # Create a metric to track requests currently in progress. REQUESTS = Counter('request_handler_exceptions', 'Number of exceptions in requests') # Decorate function with metric. @count_exceptions(REQUESTS, {'route': '/'}) async def handle_request(duration): ''' A dummy function that occasionally raises an exception ''' if duration < 0.3: raise Exception('Ooops') await asyncio.sleep(duration) async def handle_requests(): # Start up the server to expose the metrics. await svr.start(port=8000) # Generate some requests.
def __init__( self, metrics_host="127.0.0.1", metrics_port: int = 5000, loop: BaseEventLoop = None, ): self.metrics_host = metrics_host self.metrics_port = metrics_port self.loop = loop or asyncio.get_event_loop() self.timer = None # type: asyncio.Handle ###################################################################### # Create application metrics and metrics service # Create a metrics server. The server will create a metrics collector # registry if one is not specifically created and passed in. self.msvr = Service() # Define some constant labels that need to be added to all metrics const_labels = { "host": socket.gethostname(), "app": f"{self.__class__.__name__}-{uuid.uuid4().hex}", } # Create metrics collectors # Create a counter metric to track requests self.requests_metric = Counter( "requests", "Number of requests.", const_labels=const_labels ) # Collectors must be registered with the registry before they # get exposed. self.msvr.register(self.requests_metric) # Create a gauge metrics to track memory usage. self.ram_metric = Gauge( "memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels ) self.msvr.register(self.ram_metric) # Create a gauge metrics to track CPU. self.cpu_metric = Gauge( "cpu_usage_percent", "CPU usage percent.", const_labels=const_labels ) self.msvr.register(self.cpu_metric) self.payload_metric = Summary( "request_payload_size_bytes", "Request payload size in bytes.", const_labels=const_labels, invariants=[(0.50, 0.05), (0.99, 0.001)], ) self.msvr.register(self.payload_metric) self.latency_metric = Histogram( "request_latency_seconds", "Request latency in seconds", const_labels=const_labels, buckets=[0.1, 0.5, 1.0, 5.0], ) self.msvr.register(self.latency_metric)
def test_register_counter(self): """ check registering a counter collector """ r = CollectorRegistry() r.register(Counter(**self.data)) self.assertEqual(1, len(r.collectors))
Sometimes you want to expose Prometheus metrics from within an existing web service and don't want to start a separate Prometheus metrics server. This example uses the aioprometheus package to add Prometheus instrumentation to a Vibora application. In this example a registry and a counter metric is instantiated. A '/metrics' route is added to the application and the render function from aioprometheus is called to format the metrics into the appropriate format. """ from aioprometheus import render, Counter, Registry from vibora import Vibora, Request, Response app = Vibora(__name__) app.registry = Registry() app.events_counter = Counter("events", "Number of events.") app.registry.register(app.events_counter) @app.route("/") async def hello(request: Request): app.events_counter.inc({"path": "/"}) return Response(b"hello") @app.route("/metrics") async def handle_metrics(request: Request): """ Negotiate a response format by inspecting the ACCEPTS headers and selecting the most efficient format. Render metrics in the registry into the chosen format and return a response.