def configure_prometheus_metrics_exporter(app: Starlette):
app.add_middleware(MetricsMiddleware, webapp=app)
app.registry = Registry()
const_labels = {
"host": socket.gethostname(),
"name": "service1",
"version": "1"
}
app.counter_gauge = Gauge("counter",
"Current count.",
const_labels=const_labels)
app.registry.register(app.counter_gauge)
app.svc_requests_total = Counter("svc_requests_total",
"Count of service HTTP requests",
const_labels=const_labels)
app.registry.register(app.svc_requests_total)
app.svc_responses_total = Counter("svc_responses_total",
"Count of service HTTP responses",
const_labels=const_labels)
app.registry.register(app.svc_responses_total)
app.svc_internal_error_total = Counter(
"svc_internal_error_total",
"Histogram of internal errors by method, path and type of error",
const_labels=const_labels)
app.registry.register(app.svc_internal_error_total)
def __init__(self, bot: commands.Bot):
self.bot = bot
self.registry = Registry()
self.service = Service(self.registry)
self.events = Counter("events", "Discord API event counts.")
self.registry.register(self.events)
self.latency = Histogram("latency", "Discord API latency.")
self.registry.register(self.latency)
self.gc_started: typing.Optional[float] = None
self.gc_latency = Histogram(
"gc_latency", "CPython garbage collector execution times."
)
self.registry.register(self.gc_latency)
self.gc_stats = Counter("gc_stats", "CPython garbage collector stats.")
self.registry.register(self.gc_stats)
self.process = psutil.Process(os.getpid())
self.resources = Gauge("resources", "Process resource usage gauges.")
self.registry.register(self.resources)
self.hook_gc()
self.update_gc_and_resource_stats.start() # pylint: disable=no-member
self.serve.start() # pylint: disable=no-member
self.update_latency.start() # pylint: disable=no-member
async def test_grouping_key_with_empty_value(self):
# See https://github.com/prometheus/pushgateway/blob/master/README.md#url
# for encoding rules.
job_name = "example"
p = pusher.Pusher(
job_name,
self.server.url,
grouping_key={
"first": "",
"second": "foo"
},
)
registry = Registry()
c = Counter("example_total", "Total examples", {})
registry.register(c)
c.inc({})
# Push to the pushgateway
resp = await p.replace(registry)
self.assertEqual(resp.status, 200)
self.assertEqual(
"/metrics/job/example/first@base64/=/second/foo",
self.server.test_results["path"],
)
async def test_no_accept_header(self):
""" check default format is used when no accept header is defined """
# Add some metrics
data = (({"data": 1}, 100), )
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
for i in data:
c.set(i[0], i[1])
expected_data = """# HELP test_counter Test Counter.
# TYPE test_counter counter
test_counter{data="1",test="test_counter"} 100
"""
async with aiohttp.ClientSession() as session:
# Fetch without explicit accept type
async with session.get(self.metrics_url) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
def test_register_deregister(self):
""" check registering and deregistering metrics """
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
# Check registering a collector with same name raises an exception
c2 = Counter("test_counter", "Another Test Counter.")
with self.assertRaises(ValueError) as cm:
self.server.register(c2)
self.assertIn("is already registered", str(cm.exception))
self.server.deregister("test_counter")
# Check deregistering a non-existant collector raises an exception
with self.assertRaises(KeyError) as cm:
self.server.deregister("test_counter")
async def test_push_add(self):
job_name = "my-job"
p = pusher.Pusher(job_name, self.server.url)
registry = Registry()
counter = Counter("counter_test", "A counter.", {"type": "counter"})
registry.register(counter)
counter_data = (({"c_sample": "1", "c_subsample": "b"}, 400), )
[counter.set(c[0], c[1]) for c in counter_data]
# TextFormatter expected result
valid_result = (
b"# HELP counter_test A counter.\n"
b"# TYPE counter_test counter\n"
b'counter_test{c_sample="1",c_subsample="b",type="counter"} 400\n')
# BinaryFormatter expected result
# valid_result = (b'[\n\x0ccounter_test\x12\nA counter.\x18\x00"=\n\r'
# b'\n\x08c_sample\x12\x011\n\x10\n\x0bc_subsample\x12'
# b'\x01b\n\x0f\n\x04type\x12\x07counter\x1a\t\t\x00'
# b'\x00\x00\x00\x00\x00y@')
# Push to the pushgateway
resp = await p.add(registry)
self.assertEqual(resp.status, 200)
self.assertEqual(expected_job_path(job_name),
self.server.test_results["path"])
self.assertEqual("POST", self.server.test_results["method"])
self.assertEqual(valid_result, self.server.test_results["body"])
async def test_count_exceptions(self):
m = Counter("metric_label", "metric help")
# decorator should work methods as well as functions
@count_exceptions(m, {"kind": "function"})
async def a(raise_exc=False):
if raise_exc:
raise Exception("dummy exception")
return
await a()
# metric should not exist until an exception occurs
with self.assertRaises(KeyError):
m.get({"kind": "function"})
with self.assertRaises(Exception) as cm:
await a(True)
self.assertIn("dummy exception", str(cm.exception))
m_function_value = m.get({"kind": "function"})
self.assertEqual(m_function_value, 1)
# decorator should work methods as well as functions
class B(object):
@count_exceptions(m, {"kind": "method"})
async def b(self, arg1, arg2=None, raise_exc=False):
if raise_exc:
raise Exception("dummy exception")
return arg1 == "b_arg", arg2 == "arg_2"
b = B()
results = await b.b("b_arg", arg2="arg_2")
self.assertTrue(all(results))
# metric should not exist until an exception occurs
with self.assertRaises(KeyError):
m.get({"kind": "method"})
with self.assertRaises(Exception) as cm:
await b.b("b_arg", arg2="arg_2", raise_exc=True)
self.assertIn("dummy exception", str(cm.exception))
m_method_value = m.get({"kind": "method"})
self.assertEqual(m_method_value, 1)
# Only Gauge metric type can be used with @timer, others should
# raise an exception.
with self.assertRaises(Exception) as cm:
m = Histogram("metric_label", "metric help")
@count_exceptions(m)
async def c():
return
self.assertIn(
"count_exceptions decorator expects a Counter metric but got:",
str(cm.exception),
)
def setUp(self):
self.data = {
'name': "logged_users_total",
'doc': "Logged users in the application",
'const_labels': {"app": "my_app"},
}
self.c = Counter(**self.data)
async def test_counter(self):
""" check counter metric export """
# Add some metrics
data = (
({
"data": 1
}, 100),
({
"data": "2"
}, 200),
({
"data": 3
}, 300),
({
"data": 1
}, 400),
)
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
for i in data:
c.set(i[0], i[1])
expected_data = """# HELP test_counter Test Counter.
# TYPE test_counter counter
test_counter{data="1",test="test_counter"} 400
test_counter{data="2",test="test_counter"} 200
test_counter{data="3",test="test_counter"} 300
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(self.metrics_url,
headers={ACCEPT:
text.TEXT_CONTENT_TYPE}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(self.metrics_url,
headers={
ACCEPT: binary.BINARY_CONTENT_TYPE
}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(binary.BINARY_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.COUNTER)
self.assertEqual(len(mf.metric), 3)
def setUp(self):
self.data = {
"name": "logged_users_total",
"doc": "Logged users in the application",
"const_labels": {
"app": "my_app"
},
}
self.c = Counter(**self.data)
def __init__(self, bot):
self.bot = bot
self.msvr = Service()
if platform.system() == "Linux":
self.platform = platform
self.pid = os.path.join("/proc", "self")
self.pagesize = resource.getpagesize()
self.ticks = os.sysconf("SC_CLK_TCK")
self.btime = 0
with open(os.path.join("/proc", "stat"), "rb") as stat:
for line in stat:
if line.startswith(b"btime "):
self.btime = float(line.split()[1])
break
self.vmem = Gauge("process_virtual_memory_bytes",
"Virtual memory size in bytes.")
self.rss = Gauge("process_resident_memory_bytes",
"Resident memory size in bytes.")
self.start_time = Gauge(
"process_start_time_seconds",
"Start time of the process since unix epoch in seconds.")
self.cpu = Counter("process_cpu_seconds",
"Total user and system CPU time spent in seconds.")
self.fds = Gauge("process_open_fds",
"Number of open file descriptors.")
self.info = Gauge("python_info", "Python platform information.")
self.collected = Counter("python_gc_objects_collected",
"Objects collected during GC.")
self.uncollectable = Counter("python_gc_objects_uncollectable",
"Uncollectable objects found during GC.")
self.collections = Counter(
"python_gc_collections",
"Number of times this generation was collected.")
self.http = Counter("modmail_http_requests",
"The number of http requests sent to Discord.")
self.commands = Counter(
"modmail_commands",
"The total number of commands used on the bot.")
self.tickets = Counter(
"modmail_tickets",
"The total number of tickets created by the bot.")
self.tickets_message = Counter(
"modmail_tickets_message",
"The total number of messages sent in tickets.")
async def test_push_job_ping(self):
job_name = "my-job"
p = pusher.Pusher(job_name, self.server.url, loop=self.loop)
registry = Registry()
c = Counter("total_requests", "Total requests.", {})
registry.register(c)
c.inc({"url": "/p/user"})
# Push to the pushgateway
resp = await p.replace(registry)
self.assertEqual(resp.status, 200)
self.assertEqual(expected_job_path(job_name),
self.server.test_results["path"])
def test_counter_format_with_const_labels(self):
data = {
'name': "logged_users_total",
'doc': "Logged users in the application",
'const_labels': {
"app": "my_app"
},
}
c = Counter(**data)
counter_data = (
({
'country': "sp",
"device": "desktop"
}, 520),
({
'country': "us",
"device": "mobile"
}, 654),
({
'country': "uk",
"device": "desktop"
}, 1001),
({
'country': "de",
"device": "desktop"
}, 995),
({
'country': "zh",
"device": "desktop"
}, 520),
)
# Construct the result to compare
valid_result = self._create_protobuf_object(data, counter_data,
pmp.COUNTER,
data['const_labels'])
# Add data to the collector
for i in counter_data:
c.set_value(i[0], i[1])
f = BinaryFormatter()
result = f.marshall_collector(c)
self.assertTrue(self._protobuf_metric_equal(valid_result, result))
async def main(svr: Service) -> None:
events_counter = Counter("events",
"Number of events.",
const_labels={"host": socket.gethostname()})
svr.register(events_counter)
await svr.start(addr="", port=5000)
print(f"Serving prometheus metrics on: {svr.metrics_url}")
# Now start another coroutine to periodically update a metric to
# simulate the application making some progress.
async def updater(c: Counter):
while True:
c.inc({"kind": "timer_expiry"})
await asyncio.sleep(1.0)
await updater(events_counter)
def test_no_metric_instances_present_binary(self):
""" Check marshalling a collector with no metrics instances present """
c = Counter(
name=self.counter_metric_name,
doc=self.counter_metric_help,
const_labels=self.const_labels,
)
f = binary.BinaryFormatter()
result = f.marshall_collector(c)
self.assertIsInstance(result, pmp.MetricFamily)
# Construct the result expected to receive when the counter
# collector is marshalled.
expected_result = pmp.create_counter(self.counter_metric_name,
self.counter_metric_help, [])
self.assertEqual(result, expected_result)
def test_counter_format_with_timestamp(self):
self.data = {
"name": "logged_users_total",
"doc": "Logged users in the application",
"const_labels": {},
}
c = Counter(**self.data)
counter_data = ({"country": "ch", "device": "mobile"}, 654)
c.set_value(counter_data[0], counter_data[1])
result_regex = r"""# HELP logged_users_total Logged users in the application
# TYPE logged_users_total counter
logged_users_total{country="ch",device="mobile"} 654 \d*(?:.\d*)?$"""
f_with_ts = text.TextFormatter(True)
result = f_with_ts.marshall_collector(c)
self.assertTrue(re.match(result_regex, result))
def test_registry_marshall_counter(self):
counter_data = (({"c_sample": "1", "c_subsample": "b"}, 400), )
counter = Counter("counter_test",
"A counter.",
const_labels={"type": "counter"})
for labels, value in counter_data:
counter.set(labels, value)
registry = Registry()
registry.register(counter)
valid_result = (b'[\n\x0ccounter_test\x12\nA counter.\x18\x00"=\n\r'
b"\n\x08c_sample\x12\x011\n\x10\n\x0bc_subsample\x12"
b"\x01b\n\x0f\n\x04type\x12\x07counter\x1a\t\t\x00\x00"
b"\x00\x00\x00\x00y@")
f = binary.BinaryFormatter()
self.assertEqual(valid_result, f.marshall(registry))
async def test_counter(self):
# Add some metrics
data = (
({
'data': 1
}, 100),
({
'data': "2"
}, 200),
({
'data': 3
}, 300),
({
'data': 1
}, 400),
)
c = Counter("test_counter", "Test Counter.", {'test': "test_counter"})
self.registry.register(c)
for i in data:
c.set(i[0], i[1])
expected_data = """# HELP test_counter Test Counter.
# TYPE test_counter counter
test_counter{data="1",test="test_counter"} 400
test_counter{data="2",test="test_counter"} 200
test_counter{data="3",test="test_counter"} 300
"""
with aiohttp.ClientSession(loop=self.loop) as session:
headers = {ACCEPT: 'text/plain; version=0.0.4'}
async with session.get(self.metrics_url, headers=headers) as resp:
assert resp.status == 200
content = await resp.read()
self.assertEqual("text/plain; version=0.0.4; charset=utf-8",
resp.headers.get(CONTENT_TYPE))
self.assertEqual(200, resp.status)
self.assertEqual(expected_data, content.decode())
async def test_inprogress(self):
m = Gauge("metric_label", "metric help")
# decorator should work methods as well as functions
@inprogress(m, {"kind": "function"})
async def a():
return
await a()
m_function_value = m.get({"kind": "function"})
self.assertEqual(m_function_value, 0)
# decorator should work methods as well as functions
class B(object):
@inprogress(m, {"kind": "method"})
async def b(self, arg1, arg2=None):
return arg1 == "b_arg", arg2 == "arg_2"
b = B()
results = await b.b("b_arg", arg2="arg_2")
self.assertTrue(all(results))
m_method_value = m.get({"kind": "method"})
self.assertEqual(m_method_value, 0)
# Only Gauge metric type can be used with @timer, others should
# raise an exception.
with self.assertRaises(Exception) as cm:
m = Counter("metric_label", "metric help")
@inprogress(m)
async def c():
return
self.assertIn(
"inprogess decorator expects a Gauge metric but got:", str(cm.exception)
)
async def test_timer(self):
m = Summary("metric_label", "metric help")
# decorator should work methods as well as functions
@timer(m, {"kind": "function"})
async def a():
return
await a()
m_function = m.get({"kind": "function"})
self.assertEqual(m_function["count"], 1)
# decorator should work methods as well as functions
class B(object):
@timer(m, {"kind": "method"})
async def b(self, arg1, arg2=None):
return arg1 == "b_arg", arg2 == "arg_2"
b = B()
results = await b.b("b_arg", arg2="arg_2")
self.assertTrue(all(results))
m_method = m.get({"kind": "method"})
self.assertEqual(m_method["count"], 1)
# Only Summary metric type can be used with @timer, others should
# raise an exception.
with self.assertRaises(Exception) as cm:
m = Counter("metric_label", "metric help")
@timer(m)
async def c():
return
self.assertIn(
"timer decorator expects a Summary metric but got:", str(cm.exception)
)
async def test_push_add(self):
job_name = "my-job"
p = Pusher(job_name, TEST_URL)
registry = Registry()
counter = Counter("counter_test", "A counter.", {'type': "counter"})
registry.register(counter)
counter_data = (({'c_sample': '1', 'c_subsample': 'b'}, 400), )
[counter.set(c[0], c[1]) for c in counter_data]
valid_result = (b'[\n\x0ccounter_test\x12\nA counter.\x18\x00"=\n\r'
b'\n\x08c_sample\x12\x011\n\x10\n\x0bc_subsample\x12'
b'\x01b\n\x0f\n\x04type\x12\x07counter\x1a\t\t\x00'
b'\x00\x00\x00\x00\x00y@')
# Push to the pushgateway
resp = await p.add(registry)
self.assertEqual(resp.status, 200)
self.assertEqual(expected_job_path(job_name),
self.server.test_results['path'])
self.assertEqual("POST", self.server.test_results['method'])
self.assertEqual(valid_result, self.server.test_results['body'])
def test_registry_marshall_counter(self):
format_times = 10
counter_data = (({'c_sample': '1', 'c_subsample': 'b'}, 400), )
registry = Registry()
counter = Counter("counter_test", "A counter.", {'type': "counter"})
# Add data
[counter.set(c[0], c[1]) for c in counter_data]
registry.register(counter)
valid_result = (b'[\n\x0ccounter_test\x12\nA counter.\x18\x00"=\n\r'
b'\n\x08c_sample\x12\x011\n\x10\n\x0bc_subsample\x12'
b'\x01b\n\x0f\n\x04type\x12\x07counter\x1a\t\t\x00\x00'
b'\x00\x00\x00\x00y@')
f = BinaryFormatter()
# Check multiple times to ensure multiple marshalling requests
for i in range(format_times):
self.assertEqual(valid_result, f.marshall(registry))
async def test_grouping_key(self):
# See https://github.com/prometheus/pushgateway/blob/master/README.md#url
# for encoding rules.
job_name = "my-job"
p = pusher.Pusher(
job_name,
self.server.url,
grouping_key={"instance": "127.0.0.1:1234"},
)
registry = Registry()
c = Counter("total_requests", "Total requests.", {})
registry.register(c)
c.inc({})
# Push to the pushgateway
resp = await p.replace(registry)
self.assertEqual(resp.status, 200)
self.assertEqual(
"/metrics/job/my-job/instance/127.0.0.1:1234",
self.server.test_results["path"],
)
def test_single_counter_format_text(self):
name = "prometheus_dns_sd_lookups_total"
doc = "The number of DNS-SD lookups."
valid_result = """# HELP prometheus_dns_sd_lookups_total The number of DNS-SD lookups.
# TYPE prometheus_dns_sd_lookups_total counter
prometheus_dns_sd_lookups_total 10"""
data = ((None, 10), )
# Create the counter
c = Counter(name=name, doc=doc, const_labels={})
for i in data:
c.set_value(i[0], i[1])
# Select format
f = text.TextFormatter()
result = f.marshall_collector(c)
self.assertEqual(valid_result, result)
async def test_timer(self):
m = Summary('metric_label', 'metric help')
# decorator should work methods as well as functions
@timer(m, {'kind': 'function'})
async def a():
return
await a()
m_function = m.get({'kind': 'function'})
self.assertEqual(m_function['count'], 1)
# decorator should work methods as well as functions
class B(object):
@timer(m, {'kind': 'method'})
async def b(self, arg1, arg2=None):
return arg1 == 'b_arg', arg2 == 'arg_2'
b = B()
results = await b.b('b_arg', arg2='arg_2')
self.assertTrue(all(results))
m_method = m.get({'kind': 'method'})
self.assertEqual(m_method['count'], 1)
# Only Summary metric type can be used with @timer, others should
# raise an exception.
with self.assertRaises(Exception) as cm:
m = Counter('metric_label', 'metric help')
@timer(m)
async def c():
return
self.assertIn("timer decorator expects a Summary metric but got:",
str(cm.exception))
async def test_grouping_key_with_value_containing_slash(self):
# See https://github.com/prometheus/pushgateway/blob/master/README.md#url
# for encoding rules.
job_name = "directory_cleaner"
p = pusher.Pusher(
job_name,
self.server.url,
grouping_key={"path": "/var/tmp"},
)
registry = Registry()
c = Counter("exec_total", "Total executions", {})
registry.register(c)
c.inc({})
# Push to the pushgateway
resp = await p.replace(registry)
self.assertEqual(resp.status, 200)
# Generated base64 content include '=' as padding.
self.assertEqual(
"/metrics/job/directory_cleaner/path@base64/L3Zhci90bXA=",
self.server.test_results["path"],
)
$ curl :8000/metrics
# HELP request_handler_exceptions Number of exceptions in requests
# TYPE request_handler_exceptions counter
request_handler_exceptions{route="/"} 3
You may need to Ctrl+C twice to exit the example script.
'''
import asyncio
import random
from aioprometheus import Service, Counter, count_exceptions
# Create a metric to track requests currently in progress.
REQUESTS = Counter('request_handler_exceptions',
'Number of exceptions in requests')
# Decorate function with metric.
@count_exceptions(REQUESTS, {'route': '/'})
async def handle_request(duration):
''' A dummy function that occasionally raises an exception '''
if duration < 0.3:
raise Exception('Ooops')
await asyncio.sleep(duration)
async def handle_requests():
# Start up the server to expose the metrics.
await svr.start(port=8000)
# Generate some requests.
def __init__(
self,
metrics_host="127.0.0.1",
metrics_port: int = 5000,
loop: BaseEventLoop = None,
):
self.metrics_host = metrics_host
self.metrics_port = metrics_port
self.loop = loop or asyncio.get_event_loop()
self.timer = None # type: asyncio.Handle
######################################################################
# Create application metrics and metrics service
# Create a metrics server. The server will create a metrics collector
# registry if one is not specifically created and passed in.
self.msvr = Service()
# Define some constant labels that need to be added to all metrics
const_labels = {
"host": socket.gethostname(),
"app": f"{self.__class__.__name__}-{uuid.uuid4().hex}",
}
# Create metrics collectors
# Create a counter metric to track requests
self.requests_metric = Counter(
"requests", "Number of requests.", const_labels=const_labels
)
# Collectors must be registered with the registry before they
# get exposed.
self.msvr.register(self.requests_metric)
# Create a gauge metrics to track memory usage.
self.ram_metric = Gauge(
"memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels
)
self.msvr.register(self.ram_metric)
# Create a gauge metrics to track CPU.
self.cpu_metric = Gauge(
"cpu_usage_percent", "CPU usage percent.", const_labels=const_labels
)
self.msvr.register(self.cpu_metric)
self.payload_metric = Summary(
"request_payload_size_bytes",
"Request payload size in bytes.",
const_labels=const_labels,
invariants=[(0.50, 0.05), (0.99, 0.001)],
)
self.msvr.register(self.payload_metric)
self.latency_metric = Histogram(
"request_latency_seconds",
"Request latency in seconds",
const_labels=const_labels,
buckets=[0.1, 0.5, 1.0, 5.0],
)
self.msvr.register(self.latency_metric)
def test_register_counter(self):
""" check registering a counter collector """
r = CollectorRegistry()
r.register(Counter(**self.data))
self.assertEqual(1, len(r.collectors))
Sometimes you want to expose Prometheus metrics from within an existing web
service and don't want to start a separate Prometheus metrics server.
This example uses the aioprometheus package to add Prometheus instrumentation
to a Vibora application. In this example a registry and a counter metric is
instantiated. A '/metrics' route is added to the application and the render
function from aioprometheus is called to format the metrics into the
appropriate format.
"""
from aioprometheus import render, Counter, Registry
from vibora import Vibora, Request, Response
app = Vibora(__name__)
app.registry = Registry()
app.events_counter = Counter("events", "Number of events.")
app.registry.register(app.events_counter)
@app.route("/")
async def hello(request: Request):
app.events_counter.inc({"path": "/"})
return Response(b"hello")
@app.route("/metrics")
async def handle_metrics(request: Request):
"""
Negotiate a response format by inspecting the ACCEPTS headers and selecting
the most efficient format. Render metrics in the registry into the chosen
format and return a response.