def __init__(self):
self.logger = logging.getLogger(
"cobald.runtime.tardis.plugins.prometheusmonitoring")
self.logger.setLevel(logging.DEBUG)
config = Configuration().Plugins.PrometheusMonitoring
self._port = config.port
self._addr = config.addr
self._svr_started = False
self._drones = {}
self._svr = Service()
self._gauges = {
ResourceStatus.Booting: Gauge("booting", "Booting drones"),
ResourceStatus.Running: Gauge("running", "Running drones"),
ResourceStatus.Stopped: Gauge("stopped", "Stopped drones"),
ResourceStatus.Deleted: Gauge("deleted", "Deleted drones"),
ResourceStatus.Error: Gauge("error", "Drones in error state"),
}
for gauge in self._gauges.values():
self._svr.register(gauge)
gauge.set({}, 0)
def __init__(self, bot: commands.Bot):
self.bot = bot
self.registry = Registry()
self.service = Service(self.registry)
self.events = Counter("events", "Discord API event counts.")
self.registry.register(self.events)
self.latency = Histogram("latency", "Discord API latency.")
self.registry.register(self.latency)
self.gc_started: typing.Optional[float] = None
self.gc_latency = Histogram(
"gc_latency", "CPython garbage collector execution times."
)
self.registry.register(self.gc_latency)
self.gc_stats = Counter("gc_stats", "CPython garbage collector stats.")
self.registry.register(self.gc_stats)
self.process = psutil.Process(os.getpid())
self.resources = Gauge("resources", "Process resource usage gauges.")
self.registry.register(self.resources)
self.hook_gc()
self.update_gc_and_resource_stats.start() # pylint: disable=no-member
self.serve.start() # pylint: disable=no-member
self.update_latency.start() # pylint: disable=no-member
def setUp(self):
self.data = {
'name': "hdd_disk_used",
'doc': "Disk space used",
'const_labels': {"server": "1.db.production.my-app"},
}
self.g = Gauge(**self.data)
async def test_gauge(self):
""" check gauge metric export """
# Add some metrics
data = (
({
"data": 1
}, 100),
({
"data": "2"
}, 200),
({
"data": 3
}, 300),
({
"data": 1
}, 400),
)
g = Gauge("test_gauge", "Test Gauge.", {"test": "test_gauge"})
self.server.register(g)
for i in data:
g.set(i[0], i[1])
expected_data = """# HELP test_gauge Test Gauge.
# TYPE test_gauge gauge
test_gauge{data="1",test="test_gauge"} 400
test_gauge{data="2",test="test_gauge"} 200
test_gauge{data="3",test="test_gauge"} 300
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(self.metrics_url,
headers={ACCEPT:
text.TEXT_CONTENT_TYPE}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(self.metrics_url,
headers={
ACCEPT: binary.BINARY_CONTENT_TYPE
}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(binary.BINARY_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.GAUGE)
self.assertEqual(len(mf.metric), 3)
def setUp(self):
self.data = {
"name": "hdd_disk_used",
"doc": "Disk space used",
"const_labels": {
"server": "1.db.production.my-app"
},
}
self.g = Gauge(**self.data)
def test_gauge_format_with_const_labels(self):
self.data = {
"name": "logged_users_total",
"doc": "Logged users in the application",
"const_labels": {"app": "my_app"},
}
g = Gauge(**self.data)
counter_data = (
({"country": "sp", "device": "desktop"}, 520),
({"country": "us", "device": "mobile"}, 654),
({"country": "uk", "device": "desktop"}, 1001),
({"country": "de", "device": "desktop"}, 995),
({"country": "zh", "device": "desktop"}, 520),
({"country": "ch", "device": "mobile"}, 654),
({"country": "ca", "device": "desktop"}, 1001),
({"country": "jp", "device": "desktop"}, 995),
({"country": "au", "device": "desktop"}, 520),
({"country": "py", "device": "mobile"}, 654),
({"country": "ar", "device": "desktop"}, 1001),
({"country": "pt", "device": "desktop"}, 995),
)
valid_result = (
"# HELP logged_users_total Logged users in the application",
"# TYPE logged_users_total gauge",
'logged_users_total{app="my_app",country="ch",device="mobile"} 654',
'logged_users_total{app="my_app",country="zh",device="desktop"} 520',
'logged_users_total{app="my_app",country="jp",device="desktop"} 995',
'logged_users_total{app="my_app",country="de",device="desktop"} 995',
'logged_users_total{app="my_app",country="pt",device="desktop"} 995',
'logged_users_total{app="my_app",country="ca",device="desktop"} 1001',
'logged_users_total{app="my_app",country="sp",device="desktop"} 520',
'logged_users_total{app="my_app",country="au",device="desktop"} 520',
'logged_users_total{app="my_app",country="uk",device="desktop"} 1001',
'logged_users_total{app="my_app",country="py",device="mobile"} 654',
'logged_users_total{app="my_app",country="us",device="mobile"} 654',
'logged_users_total{app="my_app",country="ar",device="desktop"} 1001',
)
# Add data to the collector
for i in counter_data:
g.set_value(i[0], i[1])
# Select format
f = TextFormatter()
result = f.marshall_lines(g)
result = sorted(result)
valid_result = sorted(valid_result)
self.assertEqual(valid_result, result)
def __init__(self, bot):
self.bot = bot
self.msvr = Service()
if platform.system() == "Linux":
self.platform = platform
self.pid = os.path.join("/proc", "self")
self.pagesize = resource.getpagesize()
self.ticks = os.sysconf("SC_CLK_TCK")
self.btime = 0
with open(os.path.join("/proc", "stat"), "rb") as stat:
for line in stat:
if line.startswith(b"btime "):
self.btime = float(line.split()[1])
break
self.vmem = Gauge("process_virtual_memory_bytes",
"Virtual memory size in bytes.")
self.rss = Gauge("process_resident_memory_bytes",
"Resident memory size in bytes.")
self.start_time = Gauge(
"process_start_time_seconds",
"Start time of the process since unix epoch in seconds.")
self.cpu = Counter("process_cpu_seconds",
"Total user and system CPU time spent in seconds.")
self.fds = Gauge("process_open_fds",
"Number of open file descriptors.")
self.info = Gauge("python_info", "Python platform information.")
self.collected = Counter("python_gc_objects_collected",
"Objects collected during GC.")
self.uncollectable = Counter("python_gc_objects_uncollectable",
"Uncollectable objects found during GC.")
self.collections = Counter(
"python_gc_collections",
"Number of times this generation was collected.")
self.http = Counter("modmail_http_requests",
"The number of http requests sent to Discord.")
self.commands = Counter(
"modmail_commands",
"The total number of commands used on the bot.")
self.tickets = Counter(
"modmail_tickets",
"The total number of tickets created by the bot.")
self.tickets_message = Counter(
"modmail_tickets_message",
"The total number of messages sent in tickets.")
def get_or_create_gauge(registry, name, description=""):
if name in registry.collectors:
return registry.collectors[name]
else:
gauge = Gauge(name, description)
registry.register(gauge)
return gauge
def configure_prometheus_metrics_exporter(app: Starlette):
app.add_middleware(MetricsMiddleware, webapp=app)
app.registry = Registry()
const_labels = {
"host": socket.gethostname(),
"name": "service1",
"version": "1"
}
app.counter_gauge = Gauge("counter",
"Current count.",
const_labels=const_labels)
app.registry.register(app.counter_gauge)
app.svc_requests_total = Counter("svc_requests_total",
"Count of service HTTP requests",
const_labels=const_labels)
app.registry.register(app.svc_requests_total)
app.svc_responses_total = Counter("svc_responses_total",
"Count of service HTTP responses",
const_labels=const_labels)
app.registry.register(app.svc_responses_total)
app.svc_internal_error_total = Counter(
"svc_internal_error_total",
"Histogram of internal errors by method, path and type of error",
const_labels=const_labels)
app.registry.register(app.svc_internal_error_total)
async def test_gauge(self):
""" check gauge metric export """
# Add some metrics
data = (
({"data": 1}, 100),
({"data": "2"}, 200),
({"data": 3}, 300),
({"data": 1}, 400),
)
g = Gauge("test_gauge", "Test Gauge.", {"test": "test_gauge"})
self.server.register(g)
for i in data:
g.set(i[0], i[1])
expected_data = """# HELP test_gauge Test Gauge.
# TYPE test_gauge gauge
test_gauge{data="1",test="test_gauge"} 400
test_gauge{data="2",test="test_gauge"} 200
test_gauge{data="3",test="test_gauge"} 300
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(
self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(
self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.GAUGE)
self.assertEqual(len(mf.metric), 3)
def test_gauge_format_with_const_labels(self):
data = {
'name': "logged_users_total",
'doc': "Logged users in the application",
'const_labels': {
"app": "my_app"
},
}
g = Gauge(**data)
gauge_data = (
({
'country': "sp",
"device": "desktop"
}, 520),
({
'country': "us",
"device": "mobile"
}, 654),
({
'country': "uk",
"device": "desktop"
}, 1001),
({
'country': "de",
"device": "desktop"
}, 995),
({
'country': "zh",
"device": "desktop"
}, 520),
)
# Construct the result to compare
valid_result = self._create_protobuf_object(data, gauge_data,
pmp.GAUGE,
data['const_labels'])
# Add data to the collector
for i in gauge_data:
g.set_value(i[0], i[1])
f = BinaryFormatter()
result = f.marshall_collector(g)
self.assertTrue(self._protobuf_metric_equal(valid_result, result))
def setUp(self):
self.data = {
"name": "hdd_disk_used",
"doc": "Disk space used",
"const_labels": {"server": "1.db.production.my-app"},
}
self.g = Gauge(**self.data)
def test_gauge_format_with_timestamp(self):
self.data = {
"name": "logged_users_total",
"doc": "Logged users in the application",
"const_labels": {},
}
g = Gauge(**self.data)
counter_data = ({"country": "ch", "device": "mobile"}, 654)
g.set_value(counter_data[0], counter_data[1])
result_regex = r"""# HELP logged_users_total Logged users in the application
# TYPE logged_users_total gauge
logged_users_total{country="ch",device="mobile"} 654 \d*(?:.\d*)?$"""
f_with_ts = TextFormatter(True)
result = f_with_ts.marshall_collector(g)
self.assertTrue(re.match(result_regex, result))
def test_gauge_format_with_timestamp(self):
self.data = {
"name": "logged_users_total",
"doc": "Logged users in the application",
"const_labels": {},
}
g = Gauge(**self.data)
counter_data = ({"country": "ch", "device": "mobile"}, 654)
g.set_value(counter_data[0], counter_data[1])
result_regex = r"""# HELP logged_users_total Logged users in the application
# TYPE logged_users_total gauge
logged_users_total{country="ch",device="mobile"} 654 \d*(?:.\d*)?$"""
f_with_ts = text.TextFormatter(True)
result = f_with_ts.marshall_collector(g)
self.assertTrue(re.match(result_regex, result))
def test_registry_marshall_gauge(self):
gauge_data = (({"g_sample": "1", "g_subsample": "b"}, 800), )
gauge = Gauge("gauge_test", "A gauge.", const_labels={"type": "gauge"})
for labels, value in gauge_data:
gauge.set(labels, value)
registry = Registry()
registry.register(gauge)
valid_result = (b'U\n\ngauge_test\x12\x08A gauge.\x18\x01";'
b"\n\r\n\x08g_sample\x12\x011\n\x10\n\x0bg_subsample"
b"\x12\x01b\n\r\n\x04type\x12\x05gauge\x12\t\t\x00"
b"\x00\x00\x00\x00\x00\x89@")
f = binary.BinaryFormatter()
self.assertEqual(valid_result, f.marshall(registry))
def __init__(self):
config = Configuration().Plugins.PrometheusMonitoring
self._port = config.port
self._addr = config.addr
self._svr_started = False
self._drones = {}
self._svr = Service()
self._gauges = {
ResourceStatus.Booting: Gauge("booting", "Booting drones"),
ResourceStatus.Running: Gauge("running", "Running drones"),
ResourceStatus.Stopped: Gauge("stopped", "Stopped drones"),
ResourceStatus.Deleted: Gauge("deleted", "Deleted drones"),
ResourceStatus.Error: Gauge("error", "Drones in error state"),
}
for gauge in self._gauges.values():
gauge.set({}, 0)
async def test_inprogress(self):
m = Gauge("metric_label", "metric help")
# decorator should work methods as well as functions
@inprogress(m, {"kind": "function"})
async def a():
return
await a()
m_function_value = m.get({"kind": "function"})
self.assertEqual(m_function_value, 0)
# decorator should work methods as well as functions
class B(object):
@inprogress(m, {"kind": "method"})
async def b(self, arg1, arg2=None):
return arg1 == "b_arg", arg2 == "arg_2"
b = B()
results = await b.b("b_arg", arg2="arg_2")
self.assertTrue(all(results))
m_method_value = m.get({"kind": "method"})
self.assertEqual(m_method_value, 0)
# Only Gauge metric type can be used with @timer, others should
# raise an exception.
with self.assertRaises(Exception) as cm:
m = Counter("metric_label", "metric help")
@inprogress(m)
async def c():
return
self.assertIn(
"inprogess decorator expects a Gauge metric but got:", str(cm.exception)
)
async def test_gauge(self):
# Add some metrics
data = (
({
'data': 1
}, 100),
({
'data': "2"
}, 200),
({
'data': 3
}, 300),
({
'data': 1
}, 400),
)
g = Gauge("test_gauge", "Test Gauge.", {'test': "test_gauge"})
self.registry.register(g)
for i in data:
g.set(i[0], i[1])
expected_data = """# HELP test_gauge Test Gauge.
# TYPE test_gauge gauge
test_gauge{data="1",test="test_gauge"} 400
test_gauge{data="2",test="test_gauge"} 200
test_gauge{data="3",test="test_gauge"} 300
"""
with aiohttp.ClientSession(loop=self.loop) as session:
headers = {ACCEPT: 'text/plain; version=0.0.4'}
async with session.get(self.metrics_url, headers=headers) as resp:
assert resp.status == 200
content = await resp.read()
self.assertEqual("text/plain; version=0.0.4; charset=utf-8",
resp.headers.get(CONTENT_TYPE))
self.assertEqual(200, resp.status)
self.assertEqual(expected_data, content.decode())
async def test_inprogress(self):
m = Gauge('metric_label', 'metric help')
# decorator should work methods as well as functions
@inprogress(m, {'kind': 'function'})
async def a():
return
await a()
m_function_value = m.get({'kind': 'function'})
self.assertEqual(m_function_value, 0)
# decorator should work methods as well as functions
class B(object):
@inprogress(m, {'kind': 'method'})
async def b(self, arg1, arg2=None):
return arg1 == 'b_arg', arg2 == 'arg_2'
b = B()
results = await b.b('b_arg', arg2='arg_2')
self.assertTrue(all(results))
m_method_value = m.get({'kind': 'method'})
self.assertEqual(m_method_value, 0)
# Only Gauge metric type can be used with @timer, others should
# raise an exception.
with self.assertRaises(Exception) as cm:
m = Counter('metric_label', 'metric help')
@inprogress(m)
async def c():
return
self.assertIn(
"inprogess decorator expects a Gauge metric but got:",
str(cm.exception))
def test_registry_marshall_gauge(self):
format_times = 10
gauge_data = (({'g_sample': '1', 'g_subsample': 'b'}, 800), )
registry = Registry()
gauge = Gauge("gauge_test", "A gauge.", {'type': "gauge"})
# Add data
[gauge.set(g[0], g[1]) for g in gauge_data]
registry.register(gauge)
valid_result = (b'U\n\ngauge_test\x12\x08A gauge.\x18\x01";'
b'\n\r\n\x08g_sample\x12\x011\n\x10\n\x0bg_subsample'
b'\x12\x01b\n\r\n\x04type\x12\x05gauge\x12\t\t\x00'
b'\x00\x00\x00\x00\x00\x89@')
f = BinaryFormatter()
# Check multiple times to ensure multiple marshalling requests
for i in range(format_times):
self.assertEqual(valid_result, f.marshall(registry))
def test_single_gauge_format_text(self):
name = "prometheus_local_storage_indexing_queue_capacity"
doc = "The capacity of the indexing queue."
valid_result = """# HELP prometheus_local_storage_indexing_queue_capacity The capacity of the indexing queue.
# TYPE prometheus_local_storage_indexing_queue_capacity gauge
prometheus_local_storage_indexing_queue_capacity 16384"""
data = ((None, 16384),)
# Create the counter
g = Gauge(name=name, doc=doc, const_labels={})
for i in data:
g.set_value(i[0], i[1])
# Select format
f = TextFormatter()
result = f.marshall_collector(g)
self.assertEqual(valid_result, result)
def test_single_gauge_format_text(self):
name = "prometheus_local_storage_indexing_queue_capacity"
doc = "The capacity of the indexing queue."
valid_result = """# HELP prometheus_local_storage_indexing_queue_capacity The capacity of the indexing queue.
# TYPE prometheus_local_storage_indexing_queue_capacity gauge
prometheus_local_storage_indexing_queue_capacity 16384"""
data = ((None, 16384), )
# Create the counter
g = Gauge(name=name, doc=doc, const_labels={})
for i in data:
g.set_value(i[0], i[1])
# Select format
f = text.TextFormatter()
result = f.marshall_collector(g)
self.assertEqual(valid_result, result)
class TestGauge(unittest.TestCase):
def setUp(self):
self.data = {
"name": "hdd_disk_used",
"doc": "Disk space used",
"const_labels": {"server": "1.db.production.my-app"},
}
self.g = Gauge(**self.data)
def test_set(self):
data = (
{"labels": {"max": "500G", "dev": "sda"}, "values": range(0, 500, 50)},
{"labels": {"max": "1T", "dev": "sdb"}, "values": range(0, 1000, 100)},
{"labels": {"max": "10T", "dev": "sdc"}, "values": range(0, 10000, 1000)},
)
for i in data:
for j in i["values"]:
self.g.set(i["labels"], j)
self.assertEqual(len(data), len(self.g.values))
def test_get(self):
data = (
{"labels": {"max": "500G", "dev": "sda"}, "values": range(0, 500, 50)},
{"labels": {"max": "1T", "dev": "sdb"}, "values": range(0, 1000, 100)},
{"labels": {"max": "10T", "dev": "sdc"}, "values": range(0, 10000, 1000)},
)
for i in data:
for j in i["values"]:
self.g.set(i["labels"], j)
self.assertEqual(j, self.g.get(i["labels"]))
for i in data:
self.assertEqual(max(i["values"]), self.g.get(i["labels"]))
def test_set_get_without_labels(self):
data = {"labels": {}, "values": range(100)}
for i in data["values"]:
self.g.set(data["labels"], i)
self.assertEqual(1, len(self.g.values))
self.assertEqual(max(data["values"]), self.g.get(data["labels"]))
def test_inc(self):
iterations = 100
labels = {"max": "10T", "dev": "sdc"}
for i in range(iterations):
self.g.inc(labels)
self.assertEqual(i + 1, self.g.get(labels))
self.assertEqual(iterations, self.g.get(labels))
def test_dec(self):
iterations = 100
labels = {"max": "10T", "dev": "sdc"}
self.g.set(labels, iterations)
for i in range(iterations):
self.g.dec(labels)
self.assertEqual(iterations - (i + 1), self.g.get(labels))
self.assertEqual(0, self.g.get(labels))
def test_add(self):
iterations = 100
labels = {"max": "10T", "dev": "sdc"}
for i in range(iterations):
self.g.add(labels, i)
self.assertEqual(sum(range(iterations)), self.g.get(labels))
def test_add_negative(self):
iterations = 100
labels = {"max": "10T", "dev": "sdc"}
for i in range(iterations):
self.g.add(labels, -i)
self.assertEqual(sum(map(lambda x: -x, range(iterations))), self.g.get(labels))
def test_sub(self):
iterations = 100
labels = {"max": "10T", "dev": "sdc"}
for i in range(iterations):
self.g.sub(labels, i)
self.assertEqual(sum(map(lambda x: -x, range(iterations))), self.g.get(labels))
def test_sub_positive(self):
iterations = 100
labels = {"max": "10T", "dev": "sdc"}
for i in range(iterations):
self.g.sub(labels, -i)
self.assertEqual(sum(range(iterations)), self.g.get(labels))
if __name__ == "__main__":
LISTEN_PORT = getenv("PENDLESSH_PORT", default=2222)
LISTEN_ADDRESS = getenv("PENDLESSH_ADDRESS", default="0.0.0.0")
MESSAGE_MAX_DELAY = int(getenv("PENDLESSH_MAX_DELAY", default=30))
PROMETHEUS_HOSTNAME = getenv("PENDLESSH_PROMETHEUS_HOSTNAME",
default=gethostname())
log.info(f"Starting server on {LISTEN_ADDRESS}:{LISTEN_PORT} "
f"with {MESSAGE_MAX_DELAY}s max delay")
const_labels = {
"host": PROMETHEUS_HOSTNAME,
"app": f"{sys.argv[0].split('/')[-1].replace('.py', '')}",
}
PromConnectionCounter = Counter(
"pendlessh_connections",
"Number of connections received",
const_labels=const_labels,
)
PromActiveConnectionGauge = Gauge(
"pendlessh_active_connections",
"Number of currently active connections",
const_labels=const_labels,
)
PromServer = Service()
PromServer.register(PromConnectionCounter)
PromServer.register(PromActiveConnectionGauge)
asyncio.run(main(PromServer))
def test_gauge_format_binary(self):
g = Gauge(name=self.gauge_metric_name, doc=self.gauge_metric_help)
# Add data to the collector
for labels, values in self.gauge_metric_data:
g.set_value(labels, values)
f = binary.BinaryFormatter()
result = f.marshall_collector(g)
self.assertIsInstance(result, pmp.MetricFamily)
# Construct the result to expected to receive when the gauge
# collector is marshalled.
expected_result = pmp.create_gauge(self.gauge_metric_name,
self.gauge_metric_help,
self.gauge_metric_data)
self.assertEqual(result, expected_result)
######################################################################
# Check metric with constant labels
g = Gauge(
name=self.gauge_metric_name,
doc=self.gauge_metric_help,
const_labels=self.const_labels,
)
# Add data to the collector
for labels, values in self.gauge_metric_data:
g.set_value(labels, values)
f = binary.BinaryFormatter()
result = f.marshall_collector(g)
self.assertIsInstance(result, pmp.MetricFamily)
# Construct the result to expected to receive when the gauge
# collector is marshalled.
expected_result = pmp.create_gauge(
self.gauge_metric_name,
self.gauge_metric_help,
self.gauge_metric_data,
const_labels=self.const_labels,
)
self.assertEqual(result, expected_result)
######################################################################
# Check metric with timestamps
with unittest.mock.patch.object(pmp.utils,
"_timestamp_ms",
return_value=TEST_TIMESTAMP):
g = Gauge(name=self.gauge_metric_name, doc=self.gauge_metric_help)
# Add data to the collector
for labels, values in self.gauge_metric_data:
g.set_value(labels, values)
f = binary.BinaryFormatter(timestamp=True)
result = f.marshall_collector(g)
self.assertIsInstance(result, pmp.MetricFamily)
# Construct the result to expected to receive when the gauge
# collector is marshalled.
expected_result = pmp.create_gauge(
self.gauge_metric_name,
self.gauge_metric_help,
self.gauge_metric_data,
timestamp=True,
)
self.assertEqual(result, expected_result)
def __init__(
self,
metrics_host="127.0.0.1",
metrics_port: int = 5000,
loop: BaseEventLoop = None,
):
self.metrics_host = metrics_host
self.metrics_port = metrics_port
self.loop = loop or asyncio.get_event_loop()
self.timer = None # type: asyncio.Handle
######################################################################
# Create application metrics and metrics service
# Create a metrics server. The server will create a metrics collector
# registry if one is not specifically created and passed in.
self.msvr = Service()
# Define some constant labels that need to be added to all metrics
const_labels = {
"host": socket.gethostname(),
"app": f"{self.__class__.__name__}-{uuid.uuid4().hex}",
}
# Create metrics collectors
# Create a counter metric to track requests
self.requests_metric = Counter(
"requests", "Number of requests.", const_labels=const_labels
)
# Collectors must be registered with the registry before they
# get exposed.
self.msvr.register(self.requests_metric)
# Create a gauge metrics to track memory usage.
self.ram_metric = Gauge(
"memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels
)
self.msvr.register(self.ram_metric)
# Create a gauge metrics to track CPU.
self.cpu_metric = Gauge(
"cpu_usage_percent", "CPU usage percent.", const_labels=const_labels
)
self.msvr.register(self.cpu_metric)
self.payload_metric = Summary(
"request_payload_size_bytes",
"Request payload size in bytes.",
const_labels=const_labels,
invariants=[(0.50, 0.05), (0.99, 0.001)],
)
self.msvr.register(self.payload_metric)
self.latency_metric = Histogram(
"request_latency_seconds",
"Request latency in seconds",
const_labels=const_labels,
buckets=[0.1, 0.5, 1.0, 5.0],
)
self.msvr.register(self.latency_metric)
def test_registry_marshall(self):
format_times = 3
counter_data = (
({"c_sample": "1"}, 100),
({"c_sample": "2"}, 200),
({"c_sample": "3"}, 300),
({"c_sample": "1", "c_subsample": "b"}, 400),
)
gauge_data = (
({"g_sample": "1"}, 500),
({"g_sample": "2"}, 600),
({"g_sample": "3"}, 700),
({"g_sample": "1", "g_subsample": "b"}, 800),
)
summary_data = (
({"s_sample": "1"}, range(1000, 2000, 4)),
({"s_sample": "2"}, range(2000, 3000, 20)),
({"s_sample": "3"}, range(3000, 4000, 13)),
({"s_sample": "1", "s_subsample": "b"}, range(4000, 5000, 47)),
)
registry = Registry()
counter = Counter("counter_test", "A counter.", {"type": "counter"})
gauge = Gauge("gauge_test", "A gauge.", {"type": "gauge"})
summary = Summary("summary_test", "A summary.", {"type": "summary"})
# Add data
[counter.set(c[0], c[1]) for c in counter_data]
[gauge.set(g[0], g[1]) for g in gauge_data]
[summary.add(i[0], s) for i in summary_data for s in i[1]]
registry.register(counter)
registry.register(gauge)
registry.register(summary)
valid_regex = r"""# HELP counter_test A counter.
# TYPE counter_test counter
counter_test{c_sample="1",type="counter"} 100
counter_test{c_sample="2",type="counter"} 200
counter_test{c_sample="3",type="counter"} 300
counter_test{c_sample="1",c_subsample="b",type="counter"} 400
# HELP gauge_test A gauge.
# TYPE gauge_test gauge
gauge_test{g_sample="1",type="gauge"} 500
gauge_test{g_sample="2",type="gauge"} 600
gauge_test{g_sample="3",type="gauge"} 700
gauge_test{g_sample="1",g_subsample="b",type="gauge"} 800
# HELP summary_test A summary.
# TYPE summary_test summary
summary_test{quantile="0.5",s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.9",s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.99",s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test_count{s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test_sum{s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.5",s_sample="2",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.9",s_sample="2",type="summary"} 2\d*(?:.\d*)?
summary_test{quantile="0.99",s_sample="2",type="summary"} \d*(?:.\d*)?
summary_test_count{s_sample="2",type="summary"} \d*(?:.\d*)?
summary_test_sum{s_sample="2",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.5",s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.9",s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.99",s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test_count{s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test_sum{s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.5",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.9",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.99",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
summary_test_count{s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
summary_test_sum{s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
"""
f = TextFormatter()
self.maxDiff = None
# Check multiple times to ensure multiple calls to marshalling
# produce the same results
for i in range(format_times):
self.assertTrue(re.match(valid_regex, f.marshall(registry).decode()))
.. code-block:: console
$ curl :8000/metrics
# HELP request_in_progress Number of requests in progress
# TYPE request_in_progress gauge
request_in_progress{route="/"} 1
'''
import asyncio
import random
from aioprometheus import Service, Gauge, inprogress
# Create a metric to track requests currently in progress.
REQUESTS = Gauge('request_in_progress', 'Number of requests in progress')
# Decorate function with metric.
@inprogress(REQUESTS, {'route': '/'})
async def handle_request(duration):
''' A dummy function that takes some time '''
await asyncio.sleep(duration)
async def handle_requests():
# Start up the server to expose the metrics.
await svr.start(port=8000)
# Generate some requests.
while True:
await handle_request(random.random())
class ExampleApp(object):
"""
An example application that demonstrates how ``aioprometheus`` can be
integrated and used within a Python application built upon asyncio.
This application attempts to simulate a long running distributed system
process, say a socket relay or some kind of message adapter. It is
intentionally not hosting an existing web service in the application.
In this case the aioprometheus.Service object is used to provide a
new HTTP endpoint that can be used to expose Prometheus metrics on.
If this application was a web service (i.e. already had an existing web
interface) then the aioprometheus.Service object could be used as before
to add another web interface or a different approach could be used that
provides a metrics handler function for use with the existing web service.
"""
def __init__(
self,
metrics_host="127.0.0.1",
metrics_port: int = 5000,
loop: BaseEventLoop = None,
):
self.metrics_host = metrics_host
self.metrics_port = metrics_port
self.loop = loop or asyncio.get_event_loop()
self.timer = None # type: asyncio.Handle
######################################################################
# Create application metrics and metrics service
# Create a metrics server. The server will create a metrics collector
# registry if one is not specifically created and passed in.
self.msvr = Service()
# Define some constant labels that need to be added to all metrics
const_labels = {
"host": socket.gethostname(),
"app": f"{self.__class__.__name__}-{uuid.uuid4().hex}",
}
# Create metrics collectors
# Create a counter metric to track requests
self.requests_metric = Counter(
"requests", "Number of requests.", const_labels=const_labels
)
# Collectors must be registered with the registry before they
# get exposed.
self.msvr.register(self.requests_metric)
# Create a gauge metrics to track memory usage.
self.ram_metric = Gauge(
"memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels
)
self.msvr.register(self.ram_metric)
# Create a gauge metrics to track CPU.
self.cpu_metric = Gauge(
"cpu_usage_percent", "CPU usage percent.", const_labels=const_labels
)
self.msvr.register(self.cpu_metric)
self.payload_metric = Summary(
"request_payload_size_bytes",
"Request payload size in bytes.",
const_labels=const_labels,
invariants=[(0.50, 0.05), (0.99, 0.001)],
)
self.msvr.register(self.payload_metric)
self.latency_metric = Histogram(
"request_latency_seconds",
"Request latency in seconds",
const_labels=const_labels,
buckets=[0.1, 0.5, 1.0, 5.0],
)
self.msvr.register(self.latency_metric)
async def start(self):
""" Start the application """
await self.msvr.start(addr=self.metrics_host, port=self.metrics_port)
logger.debug("Serving prometheus metrics on: %s", self.msvr.metrics_url)
# Schedule a timer to update internal metrics. In a realistic
# application metrics would be updated as needed. In this example
# application a simple timer is used to emulate things happening,
# which conveniently allows all metrics to be updated at once.
self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
async def stop(self):
""" Stop the application """
await self.msvr.stop()
if self.timer:
self.timer.cancel()
self.timer = None
def on_timer_expiry(self):
""" Update application to simulate work """
# Update memory metrics
self.ram_metric.set({"type": "virtual"}, psutil.virtual_memory().used)
self.ram_metric.set({"type": "swap"}, psutil.swap_memory().used)
# Update cpu metrics
for c, p in enumerate(psutil.cpu_percent(interval=1, percpu=True)):
self.cpu_metric.set({"core": c}, p)
# Incrementing a requests counter to emulate webserver app
self.requests_metric.inc({"path": "/"})
# Monitor request payload data to emulate webserver app
self.payload_metric.add({"path": "/data"}, random.random() * 2 ** 10)
# Monitor request latency to emulate webserver app
self.latency_metric.add({"path": "/data"}, random.random() * 5)
# re-schedule another metrics update
self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
def __init__(
self,
metrics_host="127.0.0.1",
metrics_port: int = 5000,
loop: BaseEventLoop = None,
):
self.metrics_host = metrics_host
self.metrics_port = metrics_port
self.loop = loop or asyncio.get_event_loop()
self.timer = None # type: asyncio.Handle
######################################################################
# Create application metrics and metrics service
# Create a metrics server. The server will create a metrics collector
# registry if one is not specifically created and passed in.
self.msvr = Service()
# Define some constant labels that need to be added to all metrics
const_labels = {
"host": socket.gethostname(),
"app": f"{self.__class__.__name__}-{uuid.uuid4().hex}",
}
# Create metrics collectors
# Create a counter metric to track requests
self.requests_metric = Counter(
"requests", "Number of requests.", const_labels=const_labels
)
# Collectors must be registered with the registry before they
# get exposed.
self.msvr.register(self.requests_metric)
# Create a gauge metrics to track memory usage.
self.ram_metric = Gauge(
"memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels
)
self.msvr.register(self.ram_metric)
# Create a gauge metrics to track CPU.
self.cpu_metric = Gauge(
"cpu_usage_percent", "CPU usage percent.", const_labels=const_labels
)
self.msvr.register(self.cpu_metric)
self.payload_metric = Summary(
"request_payload_size_bytes",
"Request payload size in bytes.",
const_labels=const_labels,
invariants=[(0.50, 0.05), (0.99, 0.001)],
)
self.msvr.register(self.payload_metric)
self.latency_metric = Histogram(
"request_latency_seconds",
"Request latency in seconds",
const_labels=const_labels,
buckets=[0.1, 0.5, 1.0, 5.0],
)
self.msvr.register(self.latency_metric)
def __init__(self, namespace=""):
if namespace:
ns = namespace + '_'
else:
ns = ""
self._ticks = 100.0
try:
self._ticks = os.sysconf('SC_CLK_TCK')
except (ValueError, TypeError, AttributeError):
pass
# This is used to test if we can access /proc.
self._btime = 0
try:
self._btime = self._boot_time()
except IOError:
pass
major, minor, patchlevel = platform.python_version_tuple()
info = {
"version": platform.python_version(),
"implementation": platform.python_implementation(),
"major": major,
"minor": minor,
"patchlevel": patchlevel
}
self.process_metrics = {
"info":
Gauge(ns + "python_info", "Python platform information."),
"vmem":
Gauge(ns + "process_virtual_memory_bytes",
"Virtual memory size in bytes."),
"rss":
Gauge(ns + "process_resident_memory_bytes",
"Resident memory size in bytes."),
"start_time":
Gauge(ns + "process_start_time_seconds",
"Start time of the process since unix epoch in seconds."),
"cpu":
Counter(ns + "process_cpu_seconds_total",
"Total user and system CPU time spent in seconds."),
"open_fds":
Gauge(ns + "process_open_fds", "Number of open file descriptors."),
"max_fds":
Gauge(ns + "process_max_fds",
"Maximum number of open file descriptors.")
}
# Only include these metrics if CPython and gc supports get_stats
if hasattr(
gc,
'get_stats') and platform.python_implementation() == 'CPython':
self.process_metrics["collected"] = Counter(
ns + "python_gc_objects_collected",
"Objects collected during gc.")
self.process_metrics["uncollectable"] = Counter(
ns + "python_gc_objects_uncollectable",
"Uncollectable object found during GC.")
self.process_metrics["collections"] = Counter(
ns + "python_gc_collections",
"Number of times this generation was collected.")
self.process_metrics["info"].set(info, 1.0)
class TestGauge(unittest.TestCase):
def setUp(self):
self.data = {
'name': "hdd_disk_used",
'doc': "Disk space used",
'const_labels': {"server": "1.db.production.my-app"},
}
self.g = Gauge(**self.data)
def test_set(self):
data = (
{
'labels': {'max': "500G", 'dev': "sda"},
'values': range(0, 500, 50)
},
{
'labels': {'max': "1T", 'dev': "sdb"},
'values': range(0, 1000, 100)
},
{
'labels': {'max': "10T", 'dev': "sdc"},
'values': range(0, 10000, 1000)
}
)
for i in data:
for j in i['values']:
self.g.set(i['labels'], j)
self.assertEqual(len(data), len(self.g.values))
def test_get(self):
data = (
{
'labels': {'max': "500G", 'dev': "sda"},
'values': range(0, 500, 50)
},
{
'labels': {'max': "1T", 'dev': "sdb"},
'values': range(0, 1000, 100)
},
{
'labels': {'max': "10T", 'dev': "sdc"},
'values': range(0, 10000, 1000)
}
)
for i in data:
for j in i['values']:
self.g.set(i['labels'], j)
self.assertEqual(j, self.g.get(i['labels']))
for i in data:
self.assertEqual(max(i['values']), self.g.get(i['labels']))
def test_set_get_without_labels(self):
data = {
'labels': {},
'values': range(100)
}
for i in data['values']:
self.g.set(data['labels'], i)
self.assertEqual(1, len(self.g.values))
self.assertEqual(max(data['values']), self.g.get(data['labels']))
def test_inc(self):
iterations = 100
labels = {'max': "10T", 'dev': "sdc"}
for i in range(iterations):
self.g.inc(labels)
self.assertEqual(i + 1, self.g.get(labels))
self.assertEqual(iterations, self.g.get(labels))
def test_dec(self):
iterations = 100
labels = {'max': "10T", 'dev': "sdc"}
self.g.set(labels, iterations)
for i in range(iterations):
self.g.dec(labels)
self.assertEqual(iterations - (i + 1), self.g.get(labels))
self.assertEqual(0, self.g.get(labels))
def test_add(self):
iterations = 100
labels = {'max': "10T", 'dev': "sdc"}
for i in range(iterations):
self.g.add(labels, i)
self.assertEqual(sum(range(iterations)), self.g.get(labels))
def test_add_negative(self):
iterations = 100
labels = {'max': "10T", 'dev': "sdc"}
for i in range(iterations):
self.g.add(labels, -i)
self.assertEqual(sum(map(lambda x: -x, range(iterations))),
self.g.get(labels))
def test_sub(self):
iterations = 100
labels = {'max': "10T", 'dev': "sdc"}
for i in range(iterations):
self.g.sub(labels, i)
self.assertEqual(sum(map(lambda x: -x, range(iterations))),
self.g.get(labels))
def test_sub_positive(self):
iterations = 100
labels = {'max': "10T", 'dev': "sdc"}
for i in range(iterations):
self.g.sub(labels, -i)
self.assertEqual(sum(range(iterations)), self.g.get(labels))
class Ctx:
def __init__(self):
self.tracer = None
self.tracer_current_span = None
self.profile_grpc = None
self.profile_http = None
ctx = Ctx()
const_labels = {
'pod': socket.gethostname() # corresponds to pod name
}
ctx.profile_grpc = (Gauge('grpc_requests_in_flight',
'Number of gRPC requests being served.',
const_labels=const_labels),
Histogram(
'grpc_requests',
'Histogram of gRPC request latencies in milliseconds',
const_labels=const_labels,
buckets=[30, 50, 100, 200],
),
Counter('grpc_requests_failed',
'Number of failed gRPC requests',
const_labels=const_labels))
ctx.profile_http = (Gauge('http_requests_in_flight',
'Number of HTTP requests being served.',
const_labels=const_labels),
Histogram(
'http_requests',
def test_registry_marshall(self):
format_times = 3
counter_data = (
({
"c_sample": "1"
}, 100),
({
"c_sample": "2"
}, 200),
({
"c_sample": "3"
}, 300),
({
"c_sample": "1",
"c_subsample": "b"
}, 400),
)
gauge_data = (
({
"g_sample": "1"
}, 500),
({
"g_sample": "2"
}, 600),
({
"g_sample": "3"
}, 700),
({
"g_sample": "1",
"g_subsample": "b"
}, 800),
)
summary_data = (
({
"s_sample": "1"
}, range(1000, 2000, 4)),
({
"s_sample": "2"
}, range(2000, 3000, 20)),
({
"s_sample": "3"
}, range(3000, 4000, 13)),
({
"s_sample": "1",
"s_subsample": "b"
}, range(4000, 5000, 47)),
)
registry = Registry()
counter = Counter("counter_test", "A counter.", {"type": "counter"})
gauge = Gauge("gauge_test", "A gauge.", {"type": "gauge"})
summary = Summary("summary_test", "A summary.", {"type": "summary"})
# Add data
[counter.set(c[0], c[1]) for c in counter_data]
[gauge.set(g[0], g[1]) for g in gauge_data]
[summary.add(i[0], s) for i in summary_data for s in i[1]]
registry.register(counter)
registry.register(gauge)
registry.register(summary)
valid_regex = r"""# HELP counter_test A counter.
# TYPE counter_test counter
counter_test{c_sample="1",type="counter"} 100
counter_test{c_sample="2",type="counter"} 200
counter_test{c_sample="3",type="counter"} 300
counter_test{c_sample="1",c_subsample="b",type="counter"} 400
# HELP gauge_test A gauge.
# TYPE gauge_test gauge
gauge_test{g_sample="1",type="gauge"} 500
gauge_test{g_sample="2",type="gauge"} 600
gauge_test{g_sample="3",type="gauge"} 700
gauge_test{g_sample="1",g_subsample="b",type="gauge"} 800
# HELP summary_test A summary.
# TYPE summary_test summary
summary_test{quantile="0.5",s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.9",s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.99",s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test_count{s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test_sum{s_sample="1",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.5",s_sample="2",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.9",s_sample="2",type="summary"} 2\d*(?:.\d*)?
summary_test{quantile="0.99",s_sample="2",type="summary"} \d*(?:.\d*)?
summary_test_count{s_sample="2",type="summary"} \d*(?:.\d*)?
summary_test_sum{s_sample="2",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.5",s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.9",s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.99",s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test_count{s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test_sum{s_sample="3",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.5",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.9",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
summary_test{quantile="0.99",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
summary_test_count{s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
summary_test_sum{s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)?
"""
f = text.TextFormatter()
self.maxDiff = None
# Check multiple times to ensure multiple calls to marshalling
# produce the same results
for i in range(format_times):
self.assertTrue(
re.match(valid_regex,
f.marshall(registry).decode()))
def _create_gauge_metric(self, label, doc):
gauge = Gauge('dump1090_{}'.format(label), doc)
self.svr.registry.register(gauge)
return gauge
def test_gauge_format_text(self):
name = "container_memory_max_usage_bytes"
doc = "Maximum memory usage ever recorded in bytes."
valid_result = """# HELP container_memory_max_usage_bytes Maximum memory usage ever recorded in bytes.
# TYPE container_memory_max_usage_bytes gauge
container_memory_max_usage_bytes{id="4f70875bb57986783064fe958f694c9e225643b0d18e9cde6bdee56d47b7ce76",name="prometheus"} 0
container_memory_max_usage_bytes{id="89042838f24f0ec0aa2a6c93ff44fd3f3e43057d35cfd32de89558112ecb92a0",name="calendall_web_run_3"} 0
container_memory_max_usage_bytes{id="d11c6bc95459822e995fac4d4ae527f6cac442a1896a771dbb307ba276beceb9",name="db"} 0
container_memory_max_usage_bytes{id="e4260cc9dca3e4e50ad2bffb0ec7432442197f135023ab629fe3576485cc65dd",name="container-extractor"} 0
container_memory_max_usage_bytes{id="f30d1caaa142b1688a0684ed744fcae6d202a36877617b985e20a5d33801b311",name="calendall_db_1"} 0
container_memory_max_usage_bytes{id="f835d921ffaf332f8d88ef5231ba149e389a2f37276f081878d6f982ef89a981",name="cocky_fermat"} 0"""
data = (
(
{
"id": "4f70875bb57986783064fe958f694c9e225643b0d18e9cde6bdee56d47b7ce76",
"name": "prometheus",
},
0,
),
(
{
"id": "89042838f24f0ec0aa2a6c93ff44fd3f3e43057d35cfd32de89558112ecb92a0",
"name": "calendall_web_run_3",
},
0,
),
(
{
"id": "d11c6bc95459822e995fac4d4ae527f6cac442a1896a771dbb307ba276beceb9",
"name": "db",
},
0,
),
(
{
"id": "e4260cc9dca3e4e50ad2bffb0ec7432442197f135023ab629fe3576485cc65dd",
"name": "container-extractor",
},
0,
),
(
{
"id": "f30d1caaa142b1688a0684ed744fcae6d202a36877617b985e20a5d33801b311",
"name": "calendall_db_1",
},
0,
),
(
{
"id": "f835d921ffaf332f8d88ef5231ba149e389a2f37276f081878d6f982ef89a981",
"name": "cocky_fermat",
},
0,
),
)
# Create the counter
g = Gauge(name=name, doc=doc, const_labels={})
for i in data:
g.set_value(i[0], i[1])
# Select format
f = TextFormatter()
result = f.marshall_collector(g)
self.assertEqual(valid_result, result)
class ExampleApp(object):
"""
An example application that demonstrates how ``aioprometheus`` can be
integrated and used within a Python application built upon asyncio.
This application attempts to simulate a long running distributed system
process, say a socket relay or some kind of message adapter. It is
intentionally not hosting an existing web service in the application.
In this case the aioprometheus.Service object is used to provide a
new HTTP endpoint that can be used to expose Prometheus metrics on.
If this application was a web service (i.e. already had an existing web
interface) then the aioprometheus.Service object could be used as before
to add another web interface or a different approach could be used that
provides a metrics handler function for use with the existing web service.
"""
def __init__(
self,
metrics_host="127.0.0.1",
metrics_port: int = 5000,
loop: BaseEventLoop = None,
):
self.metrics_host = metrics_host
self.metrics_port = metrics_port
self.loop = loop or asyncio.get_event_loop()
self.timer = None # type: asyncio.Handle
######################################################################
# Create application metrics and metrics service
# Create a metrics server. The server will create a metrics collector
# registry if one is not specifically created and passed in.
self.msvr = Service()
# Define some constant labels that need to be added to all metrics
const_labels = {
"host": socket.gethostname(),
"app": f"{self.__class__.__name__}-{uuid.uuid4().hex}",
}
# Create metrics collectors
# Create a counter metric to track requests
self.requests_metric = Counter(
"requests", "Number of requests.", const_labels=const_labels
)
# Collectors must be registered with the registry before they
# get exposed.
self.msvr.register(self.requests_metric)
# Create a gauge metrics to track memory usage.
self.ram_metric = Gauge(
"memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels
)
self.msvr.register(self.ram_metric)
# Create a gauge metrics to track CPU.
self.cpu_metric = Gauge(
"cpu_usage_percent", "CPU usage percent.", const_labels=const_labels
)
self.msvr.register(self.cpu_metric)
self.payload_metric = Summary(
"request_payload_size_bytes",
"Request payload size in bytes.",
const_labels=const_labels,
invariants=[(0.50, 0.05), (0.99, 0.001)],
)
self.msvr.register(self.payload_metric)
self.latency_metric = Histogram(
"request_latency_seconds",
"Request latency in seconds",
const_labels=const_labels,
buckets=[0.1, 0.5, 1.0, 5.0],
)
self.msvr.register(self.latency_metric)
async def start(self):
""" Start the application """
await self.msvr.start(addr=self.metrics_host, port=self.metrics_port)
logger.debug("Serving prometheus metrics on: %s", self.msvr.metrics_url)
# Schedule a timer to update internal metrics. In a realistic
# application metrics would be updated as needed. In this example
# application a simple timer is used to emulate things happening,
# which conveniently allows all metrics to be updated at once.
self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
async def stop(self):
""" Stop the application """
await self.msvr.stop()
if self.timer:
self.timer.cancel()
self.timer = None
def on_timer_expiry(self):
""" Update application to simulate work """
# Update memory metrics
self.ram_metric.set({"type": "virtual"}, psutil.virtual_memory().used)
self.ram_metric.set({"type": "swap"}, psutil.swap_memory().used)
# Update cpu metrics
for c, p in enumerate(psutil.cpu_percent(interval=1, percpu=True)):
self.cpu_metric.set({"core": c}, p)
# Incrementing a requests counter to emulate webserver app
self.requests_metric.inc({"path": "/"})
# Monitor request payload data to emulate webserver app
self.payload_metric.add({"path": "/data"}, random.random() * 2 ** 10)
# Monitor request latency to emulate webserver app
self.latency_metric.add({"path": "/data"}, random.random() * 5)
# re-schedule another metrics update
self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
async def test_all(self):
counter_data = (
({"c_sample": "1"}, 100),
({"c_sample": "2"}, 200),
({"c_sample": "3"}, 300),
({"c_sample": "1", "c_subsample": "b"}, 400),
)
gauge_data = (
({"g_sample": "1"}, 500),
({"g_sample": "2"}, 600),
({"g_sample": "3"}, 700),
({"g_sample": "1", "g_subsample": "b"}, 800),
)
summary_data = (
({"s_sample": "1"}, range(1000, 2000, 4)),
({"s_sample": "2"}, range(2000, 3000, 20)),
({"s_sample": "3"}, range(3000, 4000, 13)),
({"s_sample": "1", "s_subsample": "b"}, range(4000, 5000, 47)),
)
histogram_data = (
({"h_sample": "1"}, [3, 14]),
({"h_sample": "2"}, range(1, 20, 2)),
({"h_sample": "3"}, range(1, 20, 2)),
({"h_sample": "1", "h_subsample": "b"}, range(1, 20, 2)),
)
counter = Counter("counter_test", "A counter.", {"type": "counter"})
gauge = Gauge("gauge_test", "A gauge.", {"type": "gauge"})
summary = Summary("summary_test", "A summary.", {"type": "summary"})
histogram = Histogram(
"histogram_test",
"A histogram.",
{"type": "histogram"},
buckets=[5.0, 10.0, 15.0],
)
self.server.register(counter)
self.server.register(gauge)
self.server.register(summary)
self.server.register(histogram)
# Add data
[counter.set(c[0], c[1]) for c in counter_data]
[gauge.set(g[0], g[1]) for g in gauge_data]
[summary.add(i[0], s) for i in summary_data for s in i[1]]
[histogram.observe(i[0], h) for i in histogram_data for h in i[1]]
expected_data = """# HELP counter_test A counter.
# TYPE counter_test counter
counter_test{c_sample="1",type="counter"} 100
counter_test{c_sample="2",type="counter"} 200
counter_test{c_sample="3",type="counter"} 300
counter_test{c_sample="1",c_subsample="b",type="counter"} 400
# HELP gauge_test A gauge.
# TYPE gauge_test gauge
gauge_test{g_sample="1",type="gauge"} 500
gauge_test{g_sample="2",type="gauge"} 600
gauge_test{g_sample="3",type="gauge"} 700
gauge_test{g_sample="1",g_subsample="b",type="gauge"} 800
# HELP histogram_test A histogram.
# TYPE histogram_test histogram
histogram_test_bucket{h_sample="1",le="5.0",type="histogram"} 1.0
histogram_test_bucket{h_sample="1",le="10.0",type="histogram"} 1.0
histogram_test_bucket{h_sample="1",le="15.0",type="histogram"} 2.0
histogram_test_bucket{h_sample="1",le="+Inf",type="histogram"} 2.0
histogram_test_count{h_sample="1",type="histogram"} 2.0
histogram_test_sum{h_sample="1",type="histogram"} 17.0
histogram_test_bucket{h_sample="2",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="2",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="2",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="2",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="2",type="histogram"} 10.0
histogram_test_sum{h_sample="2",type="histogram"} 100.0
histogram_test_bucket{h_sample="3",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="3",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="3",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="3",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="3",type="histogram"} 10.0
histogram_test_sum{h_sample="3",type="histogram"} 100.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="1",h_subsample="b",type="histogram"} 10.0
histogram_test_sum{h_sample="1",h_subsample="b",type="histogram"} 100.0
# HELP summary_test A summary.
# TYPE summary_test summary
summary_test{quantile="0.5",s_sample="1",type="summary"} 1272.0
summary_test{quantile="0.9",s_sample="1",type="summary"} 1452.0
summary_test{quantile="0.99",s_sample="1",type="summary"} 1496.0
summary_test_count{s_sample="1",type="summary"} 250
summary_test_sum{s_sample="1",type="summary"} 374500.0
summary_test{quantile="0.5",s_sample="2",type="summary"} 2260.0
summary_test{quantile="0.9",s_sample="2",type="summary"} 2440.0
summary_test{quantile="0.99",s_sample="2",type="summary"} 2500.0
summary_test_count{s_sample="2",type="summary"} 50
summary_test_sum{s_sample="2",type="summary"} 124500.0
summary_test{quantile="0.5",s_sample="3",type="summary"} 3260.0
summary_test{quantile="0.9",s_sample="3",type="summary"} 3442.0
summary_test{quantile="0.99",s_sample="3",type="summary"} 3494.0
summary_test_count{s_sample="3",type="summary"} 77
summary_test_sum{s_sample="3",type="summary"} 269038.0
summary_test{quantile="0.5",s_sample="1",s_subsample="b",type="summary"} 4235.0
summary_test{quantile="0.9",s_sample="1",s_subsample="b",type="summary"} 4470.0
summary_test{quantile="0.99",s_sample="1",s_subsample="b",type="summary"} 4517.0
summary_test_count{s_sample="1",s_subsample="b",type="summary"} 22
summary_test_sum{s_sample="1",s_subsample="b",type="summary"} 98857.0
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(
self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(
self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 4)
for mf in metrics:
self.assertIsInstance(mf, pmp.MetricFamily)
if mf.type == pmp.COUNTER:
self.assertEqual(len(mf.metric), 4)
elif mf.type == pmp.GAUGE:
self.assertEqual(len(mf.metric), 4)
elif mf.type == pmp.SUMMARY:
self.assertEqual(len(mf.metric), 4)
self.assertEqual(len(mf.metric[0].summary.quantile), 3)
elif mf.type == pmp.HISTOGRAM:
self.assertEqual(len(mf.metric), 4)
self.assertEqual(len(mf.metric[0].histogram.bucket), 4)
def test_register_gauge(self):
""" check registering a gauge collector """
r = CollectorRegistry()
r.register(Gauge(**self.data))
self.assertEqual(1, len(r.collectors))
def test_gauge_format_with_const_labels(self):
self.data = {
"name": "logged_users_total",
"doc": "Logged users in the application",
"const_labels": {
"app": "my_app"
},
}
g = Gauge(**self.data)
counter_data = (
({
"country": "sp",
"device": "desktop"
}, 520),
({
"country": "us",
"device": "mobile"
}, 654),
({
"country": "uk",
"device": "desktop"
}, 1001),
({
"country": "de",
"device": "desktop"
}, 995),
({
"country": "zh",
"device": "desktop"
}, 520),
({
"country": "ch",
"device": "mobile"
}, 654),
({
"country": "ca",
"device": "desktop"
}, 1001),
({
"country": "jp",
"device": "desktop"
}, 995),
({
"country": "au",
"device": "desktop"
}, 520),
({
"country": "py",
"device": "mobile"
}, 654),
({
"country": "ar",
"device": "desktop"
}, 1001),
({
"country": "pt",
"device": "desktop"
}, 995),
)
valid_result = (
"# HELP logged_users_total Logged users in the application",
"# TYPE logged_users_total gauge",
'logged_users_total{app="my_app",country="ch",device="mobile"} 654',
'logged_users_total{app="my_app",country="zh",device="desktop"} 520',
'logged_users_total{app="my_app",country="jp",device="desktop"} 995',
'logged_users_total{app="my_app",country="de",device="desktop"} 995',
'logged_users_total{app="my_app",country="pt",device="desktop"} 995',
'logged_users_total{app="my_app",country="ca",device="desktop"} 1001',
'logged_users_total{app="my_app",country="sp",device="desktop"} 520',
'logged_users_total{app="my_app",country="au",device="desktop"} 520',
'logged_users_total{app="my_app",country="uk",device="desktop"} 1001',
'logged_users_total{app="my_app",country="py",device="mobile"} 654',
'logged_users_total{app="my_app",country="us",device="mobile"} 654',
'logged_users_total{app="my_app",country="ar",device="desktop"} 1001',
)
# Add data to the collector
for i in counter_data:
g.set_value(i[0], i[1])
# Select format
f = text.TextFormatter()
result = f.marshall_lines(g)
result = sorted(result)
valid_result = sorted(valid_result)
self.assertEqual(valid_result, result)
class Prometheus:
def __init__(self, bot):
self.bot = bot
self.msvr = Service()
if platform.system() == "Linux":
self.platform = platform
self.pid = os.path.join("/proc", "self")
self.pagesize = resource.getpagesize()
self.ticks = os.sysconf("SC_CLK_TCK")
self.btime = 0
with open(os.path.join("/proc", "stat"), "rb") as stat:
for line in stat:
if line.startswith(b"btime "):
self.btime = float(line.split()[1])
break
self.vmem = Gauge("process_virtual_memory_bytes", "Virtual memory size in bytes.")
self.rss = Gauge("process_resident_memory_bytes", "Resident memory size in bytes.")
self.start_time = Gauge("process_start_time_seconds", "Start time of the process since unix epoch in seconds.")
self.cpu = Counter("process_cpu_seconds", "Total user and system CPU time spent in seconds.")
self.fds = Gauge("process_open_fds", "Number of open file descriptors.")
self.info = Gauge("python_info", "Python platform information.")
self.collected = Counter("python_gc_objects_collected", "Objects collected during GC.")
self.uncollectable = Counter("python_gc_objects_uncollectable", "Uncollectable objects found during GC.")
self.collections = Counter("python_gc_collections", "Number of times this generation was collected.")
self.latency = Gauge("modmail_latency", "The average latency for shards on this cluster")
self.events = Counter("modmail_discord_events", "The total number of processed events.")
self.dispatch = Counter("modmail_dispatch_events", "The total number of dispatched events.")
self.http = Counter("modmail_http_requests", "The number of http requests sent to Discord.")
self.guilds_join = Counter("modmail_guilds_join", "The number of guilds ModMail is added to.")
self.guilds_leave = Counter("modmail_guilds_leave", "The number of guilds ModMail is removed from.")
self.shards = Gauge("modmail_shards", "The total number of shards on this cluster.")
self.guilds = Gauge("modmail_guilds", "The total number of guilds on this cluster.")
self.users = Gauge("modmail_users", "The total number of users on this cluster.")
self.commands = Counter("modmail_commands", "The total number of commands used on the bot.")
self.tickets = Counter("modmail_tickets", "The total number of tickets created by the bot.")
self.tickets_message = Counter("modmail_tickets_message", "The total number of messages sent in tickets.")
async def start(self):
for name, value in vars(self).items():
if issubclass(type(value), Collector):
self.msvr.register(getattr(self, name))
await self.msvr.start(addr="127.0.0.1", port=6000 + self.bot.cluster)
self.msvr._runner._server._kwargs["access_log"] = None
self.bot.loop.create_task(self.update_bot_stats())
if platform.system() == "Linux":
self.bot.loop.create_task(self.update_process_stats())
self.bot.loop.create_task(self.update_platform_stats())
async def update_bot_stats(self):
while True:
await self.bot.wait_until_ready()
await asyncio.sleep(60)
self.shards.set({}, len(self.bot.shards))
self.guilds.set({}, len(self.bot.guilds))
self.users.set({}, len(self.bot.users))
self.latency.set({}, self.bot.latency)
await asyncio.sleep(10)
async def update_process_stats(self):
while True:
with open(os.path.join(self.pid, "stat"), "rb") as stat:
parts = stat.read().split(b")")[-1].split()
self.vmem.set({}, float(parts[20]))
self.rss.set({}, float(parts[21]) * self.pagesize)
self.start_time.set({}, float(parts[19]) / self.ticks + self.btime)
self.cpu.set({}, float(parts[11]) / self.ticks + float(parts[12]) / self.ticks)
self.fds.set({}, len(os.listdir(os.path.join(self.pid, "fd"))))
await asyncio.sleep(5)
async def update_platform_stats(self):
while True:
self.info.set(
{
"version": self.platform.python_version(),
"implementation": self.platform.python_implementation(),
"major": platform.python_version_tuple()[0],
"minor": platform.python_version_tuple()[1],
"patchlevel": platform.python_version_tuple()[2],
},
1,
)
for gen, stat in enumerate(gc.get_stats()):
self.collected.set({"generation": str(gen)}, stat["collected"])
self.uncollectable.set({"generation": str(gen)}, stat["uncollectable"])
self.collections.set({"generation": str(gen)}, stat["collections"])
await asyncio.sleep(5)
def test_gauge_format_text(self):
name = "container_memory_max_usage_bytes"
doc = "Maximum memory usage ever recorded in bytes."
valid_result = """# HELP container_memory_max_usage_bytes Maximum memory usage ever recorded in bytes.
# TYPE container_memory_max_usage_bytes gauge
container_memory_max_usage_bytes{id="4f70875bb57986783064fe958f694c9e225643b0d18e9cde6bdee56d47b7ce76",name="prometheus"} 0
container_memory_max_usage_bytes{id="89042838f24f0ec0aa2a6c93ff44fd3f3e43057d35cfd32de89558112ecb92a0",name="calendall_web_run_3"} 0
container_memory_max_usage_bytes{id="d11c6bc95459822e995fac4d4ae527f6cac442a1896a771dbb307ba276beceb9",name="db"} 0
container_memory_max_usage_bytes{id="e4260cc9dca3e4e50ad2bffb0ec7432442197f135023ab629fe3576485cc65dd",name="container-extractor"} 0
container_memory_max_usage_bytes{id="f30d1caaa142b1688a0684ed744fcae6d202a36877617b985e20a5d33801b311",name="calendall_db_1"} 0
container_memory_max_usage_bytes{id="f835d921ffaf332f8d88ef5231ba149e389a2f37276f081878d6f982ef89a981",name="cocky_fermat"} 0"""
data = (
(
{
"id":
"4f70875bb57986783064fe958f694c9e225643b0d18e9cde6bdee56d47b7ce76",
"name": "prometheus",
},
0,
),
(
{
"id":
"89042838f24f0ec0aa2a6c93ff44fd3f3e43057d35cfd32de89558112ecb92a0",
"name": "calendall_web_run_3",
},
0,
),
(
{
"id":
"d11c6bc95459822e995fac4d4ae527f6cac442a1896a771dbb307ba276beceb9",
"name": "db",
},
0,
),
(
{
"id":
"e4260cc9dca3e4e50ad2bffb0ec7432442197f135023ab629fe3576485cc65dd",
"name": "container-extractor",
},
0,
),
(
{
"id":
"f30d1caaa142b1688a0684ed744fcae6d202a36877617b985e20a5d33801b311",
"name": "calendall_db_1",
},
0,
),
(
{
"id":
"f835d921ffaf332f8d88ef5231ba149e389a2f37276f081878d6f982ef89a981",
"name": "cocky_fermat",
},
0,
),
)
# Create the counter
g = Gauge(name=name, doc=doc, const_labels={})
for i in data:
g.set_value(i[0], i[1])
# Select format
f = text.TextFormatter()
result = f.marshall_collector(g)
self.assertEqual(valid_result, result)
