async def run_server(host: str = "127.0.0.1",
port: str = "9999",
sleeptime: int = 5,
urls: Optional[str] = None) -> None:
"""Starts the metrics server"""
logger = logging.getLogger(__name__)
prom_service = Service()
for metric in (url_request_times, url_health_metric,
url_requests_in_progress):
prom_service.register(metric)
urls = parsed_urls(urls)
logger.info(f"Urls to check: {urls}")
try:
logger.info('Starting metrics server')
await prom_service.start(addr=host, port=port)
url_tasks = [
url_checker(url_health_metric, url, sleeptime=sleeptime)
for url in urls if urls
]
tasks = [
*url_tasks
# add other tasks if needed
]
await asyncio.gather(*tasks)
finally:
await prom_service.stop()
async def start_webapp(port):
prometheus_service = Service()
prometheus_service.register(PROM_CLOUDFOUNDRY_EVENT)
app = web.Application()
app.add_routes([
web.get('/check', lambda _: web.Response(text='OK')),
web.get('/metrics', prometheus_service.handle_metrics)
])
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, '0.0.0.0', port)
await site.start()
class MetricsServer(object):
def __init__(self):
self.msvr = Service()
async def start(self, host: str, port: int):
metrics = []
metrics += list(ctx.profile_http)
metrics += list(ctx.profile_grpc)
# attach metrics
for m in metrics:
self.msvr.register(m)
await self.msvr.start(addr=host, port=port)
async def stop(self):
await self.msvr.stop()
async def main(svr: Service) -> None:
events_counter = Counter("events",
"Number of events.",
const_labels={"host": socket.gethostname()})
svr.register(events_counter)
await svr.start(addr="", port=5000)
print(f"Serving prometheus metrics on: {svr.metrics_url}")
# Now start another coroutine to periodically update a metric to
# simulate the application making some progress.
async def updater(c: Counter):
while True:
c.inc({"kind": "timer_expiry"})
await asyncio.sleep(1.0)
await updater(events_counter)
async def main(svr: Service) -> None:
events_counter = Counter(
"events", "Number of events.", const_labels={"host": socket.gethostname()}
)
svr.register(events_counter)
await svr.start(addr="127.0.0.1", port=5000)
print(f"Serving prometheus metrics on: {svr.metrics_url}")
# Now start another coroutine to periodically update a metric to
# simulate the application making some progress.
async def updater(c: Counter):
while True:
c.inc({"kind": "timer_expiry"})
await asyncio.sleep(1.0)
await updater(events_counter)
async def start_webapp(port):
prometheus_service = Service()
prometheus_service.register(PROM_GET_METRICS_TIME)
prometheus_service.register(PROM_AUTOSCALER_CHECK_TIME)
prometheus_service.register(PROM_AUTOSCALING_ENABLED)
prometheus_service.register(PROM_INSUFFICIENT_DATA)
prometheus_service.register(PROM_SCALING_ACTIONS)
app = web.Application()
app.add_routes([
web.get('/check', lambda _: web.Response(text='OK')),
web.get('/metrics', prometheus_service.handle_metrics)
])
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, '0.0.0.0', port)
await site.start()
async def server(svc: Service, metrics_list: list, addr: str, port: int,
collector: Callable) -> None:
"""
This function runs the metrics updater task, that runs forever.
:param svc:
:param metrics_list:
:param addr:
:param port:
:param collector:
:return:
"""
[svc.register(my_metric.metric) for my_metric in metrics_list]
await svc.start(addr=addr, port=port)
print(f"Serving prometheus metrics on: {svc.metrics_url}")
await collector(metrics_list)
class PrometheusMonitoring(Plugin):
"""
The :py:class:`~.PrometheusMonitoring`
implements an interface to monitor the state of the Drones using Prometheus.
"""
def __init__(self):
config = Configuration().Plugins.PrometheusMonitoring
self._port = config.port
self._addr = config.addr
self._svr_started = False
self._drones = {}
self._svr = Service()
self._gauges = {
ResourceStatus.Booting: Gauge("booting", "Booting drones"),
ResourceStatus.Running: Gauge("running", "Running drones"),
ResourceStatus.Stopped: Gauge("stopped", "Stopped drones"),
ResourceStatus.Deleted: Gauge("deleted", "Deleted drones"),
ResourceStatus.Error: Gauge("error", "Drones in error state"),
}
for gauge in self._gauges.values():
self._svr.register(gauge)
gauge.set({}, 0)
async def start(self):
await self._svr.start(addr=self._addr, port=self._port)
logger.debug(f"Serving Prometheus metrics on {self._svr.metrics_url}")
self._svr_started = True
async def notify(self, state: State,
resource_attributes: AttributeDict) -> None:
"""
Update Prometheus metrics at every state change
:param state: New state of the Drone
:type state: State
:param resource_attributes: Contains all meta-data of the Drone (created and
updated timestamps, dns name, unique id, site_name, machine_type, etc.)
:type resource_attributes: AttributeDict
:return: None
"""
if not self._svr_started:
await self.start()
logger.debug(
f"Drone: {str(resource_attributes)} has changed state to {state}")
if resource_attributes.drone_uuid in self._drones:
old_status = self._drones[resource_attributes.drone_uuid]
self._gauges[old_status].dec({})
new_status = resource_attributes.resource_status
self._drones[resource_attributes.drone_uuid] = new_status
self._gauges[new_status].inc({})
if new_status == ResourceStatus.Deleted:
self._drones.pop(resource_attributes.drone_uuid, None)
class Dump1090Exporter(object):
"""
This class is responsible for fetching, parsing and exporting dump1090
metrics to Prometheus.
"""
def __init__(
self,
resource_path: str,
host: str = None,
port: int = 9105,
aircraft_interval: int = 10,
stats_interval: int = 60,
time_periods: Sequence[str] = ("last1min",),
origin: PositionType = None,
fetch_timeout: float = 2.0,
loop: AbstractEventLoop = None,
) -> None:
"""
:param resource_path: The base dump1090 resource address. This can be
a web address or a directory path.
:param host: The host to expose Prometheus metrics on. Defaults
to listen on all interfaces.
:param port: The port to expose Prometheus metrics on. Defaults to
port 9105.
:param aircraft_interval: number of seconds between processing the
dump1090 aircraft data. Defaults to 10 seconds.
:param stats_interval: number of seconds between processing the
dump1090 stats data. Defaults to 60 seconds as the data only
seems to be updated at 60 second intervals.
:param time_periods: A list of time period keys to extract from the
statistics data. By default this is just the 'last1min' time
period as Prometheus can provide the historical access.
:param origin: a tuple of (lat, lon) representing the receiver
location. The origin is used for distance calculations with
aircraft data. If it is not provided then range calculations
can not be performed and the maximum range metric will always
be zero.
:param fetch_timeout: The number of seconds to wait for a response
from dump1090.
:param loop: the event loop.
"""
self.resources = build_resources(resource_path)
self.loop = loop or asyncio.get_event_loop()
self.host = host
self.port = port
self.aircraft_interval = datetime.timedelta(seconds=aircraft_interval)
self.stats_interval = datetime.timedelta(seconds=stats_interval)
self.stats_time_periods = time_periods
self.origin = Position(*origin) if origin else None
self.fetch_timeout = fetch_timeout
self.svr = Service()
self.stats_task = None # type: Union[asyncio.Task, None]
self.aircraft_task = None # type: Union[asyncio.Task, None]
self.initialise_metrics()
logger.info(f"Monitoring dump1090 resources at: {self.resources.base}")
logger.info(
f"Refresh rates: aircraft={self.aircraft_interval}, statstics={self.stats_interval}"
)
logger.info(f"Origin: {self.origin}")
async def start(self) -> None:
""" Start the monitor """
await self.svr.start(addr=self.host, port=self.port)
logger.info(f"serving dump1090 prometheus metrics on: {self.svr.metrics_url}")
# Attempt to retrieve the optional lat and lon position from
# the dump1090 receiver data. If present this data will override
# command line configuration.
try:
receiver = await self._fetch(self.resources.receiver)
if receiver:
if "lat" in receiver and "lon" in receiver:
self.origin = Position(receiver["lat"], receiver["lon"])
logger.info(
f"Origin successfully extracted from receiver data: {self.origin}"
)
except Exception as exc:
logger.error(f"Error fetching dump1090 receiver data: {exc}")
# fmt: off
self.stats_task = asyncio.ensure_future(self.updater_stats()) # type: ignore
self.aircraft_task = asyncio.ensure_future(self.updater_aircraft()) # type: ignore
# fmt: on
async def stop(self) -> None:
""" Stop the monitor """
if self.stats_task:
self.stats_task.cancel()
try:
await self.stats_task
except asyncio.CancelledError:
pass
self.stats_task = None
if self.aircraft_task:
self.aircraft_task.cancel()
try:
await self.aircraft_task
except asyncio.CancelledError:
pass
self.aircraft_task = None
await self.svr.stop()
def initialise_metrics(self) -> None:
"""Create metrics
This method initialises a dict as the metrics attribute.
The metrics dict has two str keys; one is `aircraft` and the other
is `stats`.
The `aircraft` key stores aircraft summary metrics using a value
of Dict[str, Gauge].
The `stats` key stores metrics under group keys. It has a value
of Dict[str, Dict[str, Gauge]]
"""
self.metrics = {"aircraft": {}, "stats": {}} # type: ignore
# aircraft
d = self.metrics["aircraft"]
for (name, label, doc) in Specs["aircraft"]: # type: ignore
d[name] = self._create_gauge_metric(label, doc)
# statistics
for group, metrics_specs in Specs["stats"].items(): # type: ignore
d = self.metrics["stats"].setdefault(group, {})
for name, label, doc in metrics_specs:
d[name] = self._create_gauge_metric(label, doc)
def _create_gauge_metric(self, label, doc):
gauge = Gauge("dump1090_{}".format(label), doc)
self.svr.register(gauge)
return gauge
async def _fetch(
self,
resource: str,
) -> Dict[Any, Any]:
""" Fetch JSON data from a web or file resource and return a dict """
logger.debug(f"fetching {resource}")
if resource.startswith("http"):
try:
async with aiohttp.ClientSession() as session:
async with session.get(
resource, timeout=self.fetch_timeout
) as resp:
if not resp.status == 200:
raise Exception(f"Fetch failed {resp.status}: {resource}")
data = await resp.json()
except asyncio.TimeoutError:
raise Exception(f"Request timed out to {resource}") from None
except aiohttp.ClientError as exc:
raise Exception(f"Client error {exc}, {resource}") from None
else:
with open(resource, "rt") as f:
data = json.loads(f.read())
return data
async def updater_stats(self) -> None:
"""
This long running coroutine task is responsible for fetching current
statistics from dump1090 and then updating internal metrics.
"""
while True:
start = datetime.datetime.now()
try:
stats = await self._fetch(self.resources.stats)
self.process_stats(stats, time_periods=self.stats_time_periods)
except Exception as exc:
logger.error(f"Error fetching dump1090 stats data: {exc}")
# wait until next collection time
end = datetime.datetime.now()
wait_seconds = (start + self.stats_interval - end).total_seconds()
await asyncio.sleep(wait_seconds)
async def updater_aircraft(self) -> None:
"""
This long running coroutine task is responsible for fetching current
statistics from dump1090 and then updating internal metrics.
"""
while True:
start = datetime.datetime.now()
try:
aircraft = await self._fetch(self.resources.aircraft)
self.process_aircraft(aircraft)
except Exception as exc:
logger.error(f"Error fetching dump1090 aircraft data: {exc}")
# wait until next collection time
end = datetime.datetime.now()
wait_seconds = (start + self.aircraft_interval - end).total_seconds()
await asyncio.sleep(wait_seconds)
def process_stats(
self, stats: dict, time_periods: Sequence[str] = ("last1min",)
) -> None:
"""Process dump1090 statistics into exported metrics.
:param stats: a dict containing dump1090 statistics data.
"""
metrics = self.metrics["stats"] # type: Dict[str, Dict[str, Gauge]]
for time_period in time_periods:
try:
tp_stats = stats[time_period]
except KeyError:
logger.exception(f"Problem extracting time period: {time_period}")
continue
labels = dict(time_period=time_period)
for key in metrics:
d = tp_stats[key] if key else tp_stats
for name, metric in metrics[key].items():
try:
value = d[name]
# 'accepted' values are in a list
if isinstance(value, list):
value = value[0]
except KeyError:
# 'signal' and 'peak_signal' are not present if
# there are no aircraft.
if name not in ["peak_signal", "signal"]:
key_str = " {} ".format(key) if key else " "
logger.warning(
f"Problem extracting{key_str}item '{name}' from: {d}"
)
value = math.nan
metric.set(labels, value)
def process_aircraft(self, aircraft: dict, threshold: int = 15) -> None:
"""Process aircraft statistics into exported metrics.
:param aircraft: a dict containing aircraft data.
:param threshold: only let aircraft seen within this threshold to
contribute to the metrics.
"""
# Ensure aircraft dict always contains all keys, as optional
# items are not always present.
for entry in aircraft["aircraft"]:
for key in AircraftKeys:
entry.setdefault(key, None)
messages = aircraft["messages"]
# 'seen' shows how long ago (in seconds before "now") a message
# was last received from an aircraft.
# 'seen_pos' shows how long ago (in seconds before "now") the
# position was last updated
aircraft_observed = 0
aircraft_with_pos = 0
aircraft_with_mlat = 0
aircraft_max_range = 0.0
# Filter aircraft to only those that have been seen within the
# last n seconds to minimise contributions from aged obsevations.
for a in aircraft["aircraft"]:
if a["seen"] < threshold:
aircraft_observed += 1
if a["seen_pos"] and a["seen_pos"] < threshold:
aircraft_with_pos += 1
if self.origin:
distance = haversine_distance(
self.origin, Position(a["lat"], a["lon"])
)
if distance > aircraft_max_range:
aircraft_max_range = distance
if a["mlat"] and "lat" in a["mlat"]:
aircraft_with_mlat += 1
# Add any current data into the 'latest' time_period bucket
labels = dict(time_period="latest")
d = self.metrics["aircraft"]
d["observed"].set(labels, aircraft_observed)
d["observed_with_pos"].set(labels, aircraft_with_pos)
d["observed_with_mlat"].set(labels, aircraft_with_mlat)
d["max_range"].set(labels, aircraft_max_range)
d["messages_total"].set(labels, messages)
logger.debug(
f"aircraft: observed={aircraft_observed}, "
f"with_pos={aircraft_with_pos}, with_mlat={aircraft_with_mlat}, "
f"max_range={aircraft_max_range}, messages={messages}"
)
if __name__ == "__main__":
LISTEN_PORT = getenv("PENDLESSH_PORT", default=2222)
LISTEN_ADDRESS = getenv("PENDLESSH_ADDRESS", default="0.0.0.0")
MESSAGE_MAX_DELAY = int(getenv("PENDLESSH_MAX_DELAY", default=30))
PROMETHEUS_HOSTNAME = getenv("PENDLESSH_PROMETHEUS_HOSTNAME",
default=gethostname())
log.info(f"Starting server on {LISTEN_ADDRESS}:{LISTEN_PORT} "
f"with {MESSAGE_MAX_DELAY}s max delay")
const_labels = {
"host": PROMETHEUS_HOSTNAME,
"app": f"{sys.argv[0].split('/')[-1].replace('.py', '')}",
}
PromConnectionCounter = Counter(
"pendlessh_connections",
"Number of connections received",
const_labels=const_labels,
)
PromActiveConnectionGauge = Gauge(
"pendlessh_active_connections",
"Number of currently active connections",
const_labels=const_labels,
)
PromServer = Service()
PromServer.register(PromConnectionCounter)
PromServer.register(PromActiveConnectionGauge)
asyncio.run(main(PromServer))
import asyncio
import socket
from aioprometheus import Counter, Service
if __name__ == "__main__":
loop = asyncio.get_event_loop()
svr = Service()
events_counter = Counter(
"events", "Number of events.", const_labels={"host": socket.gethostname()}
)
svr.register(events_counter)
loop.run_until_complete(svr.start(addr="127.0.0.1"))
print(f"Serving prometheus metrics on: {svr.metrics_url}")
async def updater(m: Counter):
# Periodically update the metric to simulate some progress
# happening in a real application.
while True:
m.inc({"kind": "timer_expiry"})
await asyncio.sleep(1.0)
try:
loop.run_until_complete(updater(events_counter))
except KeyboardInterrupt:
pass
class TestTextExporter(asynctest.TestCase):
async def setUp(self):
self.registry = Registry()
self.server = Service(registry=self.registry)
await self.server.start(addr="127.0.0.1")
self.metrics_url = self.server.metrics_url
self.root_url = self.server.root_url
async def tearDown(self):
await self.server.stop()
async def test_invalid_registry(self):
""" check only valid registry can be provided """
for invalid_registry in ["nope", dict(), list()]:
with self.assertRaises(Exception) as cm:
Service(registry=invalid_registry)
self.assertIn("registry must be a Registry, got:", str(cm.exception))
Service(registry=Registry())
def test_fetch_url_before_starting_server(self):
""" check accessing a URL property raises expection if not available """
s = Service()
with self.assertRaises(Exception) as cm:
_ = s.root_url
self.assertIn(
"No URL available, Prometheus metrics server is not running",
str(cm.exception),
)
with self.assertRaises(Exception) as cm:
_ = s.metrics_url
self.assertIn(
"No URL available, Prometheus metrics server is not running",
str(cm.exception),
)
def test_register_deregister(self):
""" check registering and deregistering metrics """
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
# Check registering a collector with same name raises an exception
c2 = Counter("test_counter", "Another Test Counter.")
with self.assertRaises(ValueError) as cm:
self.server.register(c2)
self.assertIn("is already registered", str(cm.exception))
self.server.deregister("test_counter")
# Check deregistering a non-existant collector raises an exception
with self.assertRaises(KeyError) as cm:
self.server.deregister("test_counter")
async def test_start_started_server(self):
""" check starting a started server """
with unittest.mock.patch.object(
aioprometheus.service.logger, "warning"
) as mock_warn:
await self.server.start(addr="127.0.0.1")
self.assertEqual(mock_warn.call_count, 1)
mock_warn.assert_called_once_with(
"Prometheus metrics server is already running"
)
async def test_stop_stopped_server(self):
""" check stopping a stopped server """
s = Service(registry=self.registry)
await s.start(addr="127.0.0.1")
await s.stop()
with unittest.mock.patch.object(
aioprometheus.service.logger, "warning"
) as mock_warn:
await s.stop()
self.assertEqual(mock_warn.call_count, 1)
mock_warn.assert_called_once_with(
"Prometheus metrics server is already stopped"
)
async def test_counter(self):
""" check counter metric export """
# Add some metrics
data = (
({"data": 1}, 100),
({"data": "2"}, 200),
({"data": 3}, 300),
({"data": 1}, 400),
)
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
for i in data:
c.set(i[0], i[1])
expected_data = """# HELP test_counter Test Counter.
# TYPE test_counter counter
test_counter{data="1",test="test_counter"} 400
test_counter{data="2",test="test_counter"} 200
test_counter{data="3",test="test_counter"} 300
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(
self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(
self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.COUNTER)
self.assertEqual(len(mf.metric), 3)
async def test_gauge(self):
""" check gauge metric export """
# Add some metrics
data = (
({"data": 1}, 100),
({"data": "2"}, 200),
({"data": 3}, 300),
({"data": 1}, 400),
)
g = Gauge("test_gauge", "Test Gauge.", {"test": "test_gauge"})
self.server.register(g)
for i in data:
g.set(i[0], i[1])
expected_data = """# HELP test_gauge Test Gauge.
# TYPE test_gauge gauge
test_gauge{data="1",test="test_gauge"} 400
test_gauge{data="2",test="test_gauge"} 200
test_gauge{data="3",test="test_gauge"} 300
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(
self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(
self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.GAUGE)
self.assertEqual(len(mf.metric), 3)
async def test_summary(self):
""" check summary metric export """
# Add some metrics
data = [3, 5.2, 13, 4]
label = {"data": 1}
s = Summary("test_summary", "Test Summary.", {"test": "test_summary"})
self.server.register(s)
for i in data:
s.add(label, i)
expected_data = """# HELP test_summary Test Summary.
# TYPE test_summary summary
test_summary{data="1",quantile="0.5",test="test_summary"} 4.0
test_summary{data="1",quantile="0.9",test="test_summary"} 5.2
test_summary{data="1",quantile="0.99",test="test_summary"} 5.2
test_summary_count{data="1",test="test_summary"} 4
test_summary_sum{data="1",test="test_summary"} 25.2
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(
self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(
self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.SUMMARY)
self.assertEqual(len(mf.metric), 1)
self.assertEqual(len(mf.metric[0].summary.quantile), 3)
async def test_histogram(self):
""" check histogram metric export """
# Add some metrics
data = [3, 5.2, 13, 4]
label = {"data": 1}
h = Histogram(
"histogram_test",
"Test Histogram.",
{"type": "test_histogram"},
buckets=[5.0, 10.0, 15.0],
)
self.server.register(h)
for i in data:
h.add(label, i)
expected_data = """# HELP histogram_test Test Histogram.
# TYPE histogram_test histogram
histogram_test_bucket{data="1",le="5.0",type="test_histogram"} 2.0
histogram_test_bucket{data="1",le="10.0",type="test_histogram"} 3.0
histogram_test_bucket{data="1",le="15.0",type="test_histogram"} 4.0
histogram_test_bucket{data="1",le="+Inf",type="test_histogram"} 4.0
histogram_test_count{data="1",type="test_histogram"} 4.0
histogram_test_sum{data="1",type="test_histogram"} 25.2
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(
self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(
self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.HISTOGRAM)
self.assertEqual(len(mf.metric), 1)
self.assertEqual(len(mf.metric[0].histogram.bucket), 4)
async def test_all(self):
counter_data = (
({"c_sample": "1"}, 100),
({"c_sample": "2"}, 200),
({"c_sample": "3"}, 300),
({"c_sample": "1", "c_subsample": "b"}, 400),
)
gauge_data = (
({"g_sample": "1"}, 500),
({"g_sample": "2"}, 600),
({"g_sample": "3"}, 700),
({"g_sample": "1", "g_subsample": "b"}, 800),
)
summary_data = (
({"s_sample": "1"}, range(1000, 2000, 4)),
({"s_sample": "2"}, range(2000, 3000, 20)),
({"s_sample": "3"}, range(3000, 4000, 13)),
({"s_sample": "1", "s_subsample": "b"}, range(4000, 5000, 47)),
)
histogram_data = (
({"h_sample": "1"}, [3, 14]),
({"h_sample": "2"}, range(1, 20, 2)),
({"h_sample": "3"}, range(1, 20, 2)),
({"h_sample": "1", "h_subsample": "b"}, range(1, 20, 2)),
)
counter = Counter("counter_test", "A counter.", {"type": "counter"})
gauge = Gauge("gauge_test", "A gauge.", {"type": "gauge"})
summary = Summary("summary_test", "A summary.", {"type": "summary"})
histogram = Histogram(
"histogram_test",
"A histogram.",
{"type": "histogram"},
buckets=[5.0, 10.0, 15.0],
)
self.server.register(counter)
self.server.register(gauge)
self.server.register(summary)
self.server.register(histogram)
# Add data
[counter.set(c[0], c[1]) for c in counter_data]
[gauge.set(g[0], g[1]) for g in gauge_data]
[summary.add(i[0], s) for i in summary_data for s in i[1]]
[histogram.observe(i[0], h) for i in histogram_data for h in i[1]]
expected_data = """# HELP counter_test A counter.
# TYPE counter_test counter
counter_test{c_sample="1",type="counter"} 100
counter_test{c_sample="2",type="counter"} 200
counter_test{c_sample="3",type="counter"} 300
counter_test{c_sample="1",c_subsample="b",type="counter"} 400
# HELP gauge_test A gauge.
# TYPE gauge_test gauge
gauge_test{g_sample="1",type="gauge"} 500
gauge_test{g_sample="2",type="gauge"} 600
gauge_test{g_sample="3",type="gauge"} 700
gauge_test{g_sample="1",g_subsample="b",type="gauge"} 800
# HELP histogram_test A histogram.
# TYPE histogram_test histogram
histogram_test_bucket{h_sample="1",le="5.0",type="histogram"} 1.0
histogram_test_bucket{h_sample="1",le="10.0",type="histogram"} 1.0
histogram_test_bucket{h_sample="1",le="15.0",type="histogram"} 2.0
histogram_test_bucket{h_sample="1",le="+Inf",type="histogram"} 2.0
histogram_test_count{h_sample="1",type="histogram"} 2.0
histogram_test_sum{h_sample="1",type="histogram"} 17.0
histogram_test_bucket{h_sample="2",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="2",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="2",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="2",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="2",type="histogram"} 10.0
histogram_test_sum{h_sample="2",type="histogram"} 100.0
histogram_test_bucket{h_sample="3",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="3",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="3",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="3",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="3",type="histogram"} 10.0
histogram_test_sum{h_sample="3",type="histogram"} 100.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="1",h_subsample="b",type="histogram"} 10.0
histogram_test_sum{h_sample="1",h_subsample="b",type="histogram"} 100.0
# HELP summary_test A summary.
# TYPE summary_test summary
summary_test{quantile="0.5",s_sample="1",type="summary"} 1272.0
summary_test{quantile="0.9",s_sample="1",type="summary"} 1452.0
summary_test{quantile="0.99",s_sample="1",type="summary"} 1496.0
summary_test_count{s_sample="1",type="summary"} 250
summary_test_sum{s_sample="1",type="summary"} 374500.0
summary_test{quantile="0.5",s_sample="2",type="summary"} 2260.0
summary_test{quantile="0.9",s_sample="2",type="summary"} 2440.0
summary_test{quantile="0.99",s_sample="2",type="summary"} 2500.0
summary_test_count{s_sample="2",type="summary"} 50
summary_test_sum{s_sample="2",type="summary"} 124500.0
summary_test{quantile="0.5",s_sample="3",type="summary"} 3260.0
summary_test{quantile="0.9",s_sample="3",type="summary"} 3442.0
summary_test{quantile="0.99",s_sample="3",type="summary"} 3494.0
summary_test_count{s_sample="3",type="summary"} 77
summary_test_sum{s_sample="3",type="summary"} 269038.0
summary_test{quantile="0.5",s_sample="1",s_subsample="b",type="summary"} 4235.0
summary_test{quantile="0.9",s_sample="1",s_subsample="b",type="summary"} 4470.0
summary_test{quantile="0.99",s_sample="1",s_subsample="b",type="summary"} 4517.0
summary_test_count{s_sample="1",s_subsample="b",type="summary"} 22
summary_test_sum{s_sample="1",s_subsample="b",type="summary"} 98857.0
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(
self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(
self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE}
) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 4)
for mf in metrics:
self.assertIsInstance(mf, pmp.MetricFamily)
if mf.type == pmp.COUNTER:
self.assertEqual(len(mf.metric), 4)
elif mf.type == pmp.GAUGE:
self.assertEqual(len(mf.metric), 4)
elif mf.type == pmp.SUMMARY:
self.assertEqual(len(mf.metric), 4)
self.assertEqual(len(mf.metric[0].summary.quantile), 3)
elif mf.type == pmp.HISTOGRAM:
self.assertEqual(len(mf.metric), 4)
self.assertEqual(len(mf.metric[0].histogram.bucket), 4)
async def test_no_accept_header(self):
""" check default format is used when no accept header is defined """
# Add some metrics
data = (({"data": 1}, 100),)
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
for i in data:
c.set(i[0], i[1])
expected_data = """# HELP test_counter Test Counter.
# TYPE test_counter counter
test_counter{data="1",test="test_counter"} 100
"""
async with aiohttp.ClientSession() as session:
# Fetch without explicit accept type
async with session.get(self.metrics_url) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# TODO: Add another test here that includes the ACCEPT header
# but with no value set. I have not worked out how to do this
# yet as aiohttp expects headers to be a dict and a value of None
# is not permitted.
async def test_root_route(self):
""" check root route returns content """
async with aiohttp.ClientSession() as session:
async with session.get(self.root_url) as resp:
self.assertEqual(resp.status, 200)
self.assertIn("text/html", resp.headers.get(CONTENT_TYPE))
async def test_robots_route(self):
""" check robots route returns content """
async with aiohttp.ClientSession() as session:
async with session.get(f"{self.root_url}robots.txt") as resp:
self.assertEqual(resp.status, 200)
self.assertIn("text/plain", resp.headers.get(CONTENT_TYPE))
class ExampleApp(object):
"""
An example application that demonstrates how ``aioprometheus`` can be
integrated and used within a Python application built upon asyncio.
This application attempts to simulate a long running distributed system
process, say a socket relay or some kind of message adapter. It is
intentionally not hosting an existing web service in the application.
In this case the aioprometheus.Service object is used to provide a
new HTTP endpoint that can be used to expose Prometheus metrics on.
If this application was a web service (i.e. already had an existing web
interface) then the aioprometheus.Service object could be used as before
to add another web interface or a different approach could be used that
provides a metrics handler function for use with the existing web service.
"""
def __init__(
self,
metrics_host="127.0.0.1",
metrics_port: int = 5000,
loop: BaseEventLoop = None,
):
self.metrics_host = metrics_host
self.metrics_port = metrics_port
self.loop = loop or asyncio.get_event_loop()
self.timer = None # type: asyncio.Handle
######################################################################
# Create application metrics and metrics service
# Create a metrics server. The server will create a metrics collector
# registry if one is not specifically created and passed in.
self.msvr = Service()
# Define some constant labels that need to be added to all metrics
const_labels = {
"host": socket.gethostname(),
"app": f"{self.__class__.__name__}-{uuid.uuid4().hex}",
}
# Create metrics collectors
# Create a counter metric to track requests
self.requests_metric = Counter(
"requests", "Number of requests.", const_labels=const_labels
)
# Collectors must be registered with the registry before they
# get exposed.
self.msvr.register(self.requests_metric)
# Create a gauge metrics to track memory usage.
self.ram_metric = Gauge(
"memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels
)
self.msvr.register(self.ram_metric)
# Create a gauge metrics to track CPU.
self.cpu_metric = Gauge(
"cpu_usage_percent", "CPU usage percent.", const_labels=const_labels
)
self.msvr.register(self.cpu_metric)
self.payload_metric = Summary(
"request_payload_size_bytes",
"Request payload size in bytes.",
const_labels=const_labels,
invariants=[(0.50, 0.05), (0.99, 0.001)],
)
self.msvr.register(self.payload_metric)
self.latency_metric = Histogram(
"request_latency_seconds",
"Request latency in seconds",
const_labels=const_labels,
buckets=[0.1, 0.5, 1.0, 5.0],
)
self.msvr.register(self.latency_metric)
async def start(self):
""" Start the application """
await self.msvr.start(addr=self.metrics_host, port=self.metrics_port)
logger.debug("Serving prometheus metrics on: %s", self.msvr.metrics_url)
# Schedule a timer to update internal metrics. In a realistic
# application metrics would be updated as needed. In this example
# application a simple timer is used to emulate things happening,
# which conveniently allows all metrics to be updated at once.
self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
async def stop(self):
""" Stop the application """
await self.msvr.stop()
if self.timer:
self.timer.cancel()
self.timer = None
def on_timer_expiry(self):
""" Update application to simulate work """
# Update memory metrics
self.ram_metric.set({"type": "virtual"}, psutil.virtual_memory().used)
self.ram_metric.set({"type": "swap"}, psutil.swap_memory().used)
# Update cpu metrics
for c, p in enumerate(psutil.cpu_percent(interval=1, percpu=True)):
self.cpu_metric.set({"core": c}, p)
# Incrementing a requests counter to emulate webserver app
self.requests_metric.inc({"path": "/"})
# Monitor request payload data to emulate webserver app
self.payload_metric.add({"path": "/data"}, random.random() * 2 ** 10)
# Monitor request latency to emulate webserver app
self.latency_metric.add({"path": "/data"}, random.random() * 5)
# re-schedule another metrics update
self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
class ExampleApp(object):
"""
An example application that demonstrates how ``aioprometheus`` can be
integrated and used within a Python application built upon asyncio.
This application attempts to simulate a long running distributed system
process, say a socket relay or some kind of message adapter. It is
intentionally not hosting an existing web service in the application.
In this case the aioprometheus.Service object is used to provide a
new HTTP endpoint that can be used to expose Prometheus metrics on.
If this application was a web service (i.e. already had an existing web
interface) then the aioprometheus.Service object could be used as before
to add another web interface or a different approach could be used that
provides a metrics handler function for use with the existing web service.
"""
def __init__(
self,
metrics_host="127.0.0.1",
metrics_port: int = 5000,
loop: BaseEventLoop = None,
):
self.metrics_host = metrics_host
self.metrics_port = metrics_port
self.loop = loop or asyncio.get_event_loop()
self.timer = None # type: asyncio.Handle
######################################################################
# Create application metrics and metrics service
# Create a metrics server. The server will create a metrics collector
# registry if one is not specifically created and passed in.
self.msvr = Service()
# Define some constant labels that need to be added to all metrics
const_labels = {
"host": socket.gethostname(),
"app": f"{self.__class__.__name__}-{uuid.uuid4().hex}",
}
# Create metrics collectors
# Create a counter metric to track requests
self.requests_metric = Counter(
"requests", "Number of requests.", const_labels=const_labels
)
# Collectors must be registered with the registry before they
# get exposed.
self.msvr.register(self.requests_metric)
# Create a gauge metrics to track memory usage.
self.ram_metric = Gauge(
"memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels
)
self.msvr.register(self.ram_metric)
# Create a gauge metrics to track CPU.
self.cpu_metric = Gauge(
"cpu_usage_percent", "CPU usage percent.", const_labels=const_labels
)
self.msvr.register(self.cpu_metric)
self.payload_metric = Summary(
"request_payload_size_bytes",
"Request payload size in bytes.",
const_labels=const_labels,
invariants=[(0.50, 0.05), (0.99, 0.001)],
)
self.msvr.register(self.payload_metric)
self.latency_metric = Histogram(
"request_latency_seconds",
"Request latency in seconds",
const_labels=const_labels,
buckets=[0.1, 0.5, 1.0, 5.0],
)
self.msvr.register(self.latency_metric)
async def start(self):
""" Start the application """
await self.msvr.start(addr=self.metrics_host, port=self.metrics_port)
logger.debug("Serving prometheus metrics on: %s", self.msvr.metrics_url)
# Schedule a timer to update internal metrics. In a realistic
# application metrics would be updated as needed. In this example
# application a simple timer is used to emulate things happening,
# which conveniently allows all metrics to be updated at once.
self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
async def stop(self):
""" Stop the application """
await self.msvr.stop()
if self.timer:
self.timer.cancel()
self.timer = None
def on_timer_expiry(self):
""" Update application to simulate work """
# Update memory metrics
self.ram_metric.set({"type": "virtual"}, psutil.virtual_memory().used)
self.ram_metric.set({"type": "swap"}, psutil.swap_memory().used)
# Update cpu metrics
for c, p in enumerate(psutil.cpu_percent(interval=1, percpu=True)):
self.cpu_metric.set({"core": c}, p)
# Incrementing a requests counter to emulate webserver app
self.requests_metric.inc({"path": "/"})
# Monitor request payload data to emulate webserver app
self.payload_metric.add({"path": "/data"}, random.random() * 2 ** 10)
# Monitor request latency to emulate webserver app
self.latency_metric.add({"path": "/data"}, random.random() * 5)
# re-schedule another metrics update
self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
raise Exception("Ooops")
await asyncio.sleep(duration)
async def handle_requests():
# Start up the server to expose the metrics.
await svr.start(port=8000)
# Generate some requests.
while True:
try:
await handle_request(random.random())
except Exception:
pass # keep handling
if __name__ == "__main__":
loop = asyncio.get_event_loop()
svr = Service(loop=loop)
svr.register(REQUESTS)
try:
loop.run_until_complete(handle_requests())
except KeyboardInterrupt:
pass
finally:
loop.run_until_complete(svr.stop())
loop.stop()
loop.close()
class TestTextExporter(asynctest.TestCase):
async def setUp(self):
self.registry = Registry()
self.server = Service(registry=self.registry)
await self.server.start(addr="127.0.0.1")
self.metrics_url = self.server.metrics_url
self.root_url = self.server.root_url
async def tearDown(self):
await self.server.stop()
async def test_invalid_registry(self):
""" check only valid registry can be provided """
for invalid_registry in ["nope", dict(), list()]:
with self.assertRaises(Exception) as cm:
Service(registry=invalid_registry)
self.assertIn("registry must be a Registry, got:",
str(cm.exception))
Service(registry=Registry())
def test_fetch_url_before_starting_server(self):
""" check accessing a URL property raises expection if not available """
s = Service()
with self.assertRaises(Exception) as cm:
_ = s.root_url
self.assertIn(
"No URL available, Prometheus metrics server is not running",
str(cm.exception),
)
with self.assertRaises(Exception) as cm:
_ = s.metrics_url
self.assertIn(
"No URL available, Prometheus metrics server is not running",
str(cm.exception),
)
def test_register_deregister(self):
""" check registering and deregistering metrics """
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
# Check registering a collector with same name raises an exception
c2 = Counter("test_counter", "Another Test Counter.")
with self.assertRaises(ValueError) as cm:
self.server.register(c2)
self.assertIn("is already registered", str(cm.exception))
self.server.deregister("test_counter")
# Check deregistering a non-existant collector raises an exception
with self.assertRaises(KeyError) as cm:
self.server.deregister("test_counter")
async def test_start_started_server(self):
""" check starting a started server """
with unittest.mock.patch.object(aioprometheus.service.logger,
"warning") as mock_warn:
await self.server.start(addr="127.0.0.1")
self.assertEqual(mock_warn.call_count, 1)
mock_warn.assert_called_once_with(
"Prometheus metrics server is already running")
async def test_stop_stopped_server(self):
""" check stopping a stopped server """
s = Service(registry=self.registry)
await s.start(addr="127.0.0.1")
await s.stop()
with unittest.mock.patch.object(aioprometheus.service.logger,
"warning") as mock_warn:
await s.stop()
self.assertEqual(mock_warn.call_count, 1)
mock_warn.assert_called_once_with(
"Prometheus metrics server is already stopped")
async def test_counter(self):
""" check counter metric export """
# Add some metrics
data = (
({
"data": 1
}, 100),
({
"data": "2"
}, 200),
({
"data": 3
}, 300),
({
"data": 1
}, 400),
)
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
for i in data:
c.set(i[0], i[1])
expected_data = """# HELP test_counter Test Counter.
# TYPE test_counter counter
test_counter{data="1",test="test_counter"} 400
test_counter{data="2",test="test_counter"} 200
test_counter{data="3",test="test_counter"} 300
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(self.metrics_url,
headers={ACCEPT:
text.TEXT_CONTENT_TYPE}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(self.metrics_url,
headers={
ACCEPT: binary.BINARY_CONTENT_TYPE
}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(binary.BINARY_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.COUNTER)
self.assertEqual(len(mf.metric), 3)
async def test_gauge(self):
""" check gauge metric export """
# Add some metrics
data = (
({
"data": 1
}, 100),
({
"data": "2"
}, 200),
({
"data": 3
}, 300),
({
"data": 1
}, 400),
)
g = Gauge("test_gauge", "Test Gauge.", {"test": "test_gauge"})
self.server.register(g)
for i in data:
g.set(i[0], i[1])
expected_data = """# HELP test_gauge Test Gauge.
# TYPE test_gauge gauge
test_gauge{data="1",test="test_gauge"} 400
test_gauge{data="2",test="test_gauge"} 200
test_gauge{data="3",test="test_gauge"} 300
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(self.metrics_url,
headers={ACCEPT:
text.TEXT_CONTENT_TYPE}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(self.metrics_url,
headers={
ACCEPT: binary.BINARY_CONTENT_TYPE
}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(binary.BINARY_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.GAUGE)
self.assertEqual(len(mf.metric), 3)
async def test_summary(self):
""" check summary metric export """
# Add some metrics
data = [3, 5.2, 13, 4]
label = {"data": 1}
s = Summary("test_summary", "Test Summary.", {"test": "test_summary"})
self.server.register(s)
for i in data:
s.add(label, i)
expected_data = """# HELP test_summary Test Summary.
# TYPE test_summary summary
test_summary{data="1",quantile="0.5",test="test_summary"} 4.0
test_summary{data="1",quantile="0.9",test="test_summary"} 5.2
test_summary{data="1",quantile="0.99",test="test_summary"} 5.2
test_summary_count{data="1",test="test_summary"} 4
test_summary_sum{data="1",test="test_summary"} 25.2
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(self.metrics_url,
headers={ACCEPT:
text.TEXT_CONTENT_TYPE}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(self.metrics_url,
headers={
ACCEPT: binary.BINARY_CONTENT_TYPE
}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(binary.BINARY_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.SUMMARY)
self.assertEqual(len(mf.metric), 1)
self.assertEqual(len(mf.metric[0].summary.quantile), 3)
async def test_histogram(self):
""" check histogram metric export """
# Add some metrics
data = [3, 5.2, 13, 4]
label = {"data": 1}
h = Histogram(
"histogram_test",
"Test Histogram.",
{"type": "test_histogram"},
buckets=[5.0, 10.0, 15.0],
)
self.server.register(h)
for i in data:
h.add(label, i)
expected_data = """# HELP histogram_test Test Histogram.
# TYPE histogram_test histogram
histogram_test_bucket{data="1",le="5.0",type="test_histogram"} 2.0
histogram_test_bucket{data="1",le="10.0",type="test_histogram"} 3.0
histogram_test_bucket{data="1",le="15.0",type="test_histogram"} 4.0
histogram_test_bucket{data="1",le="+Inf",type="test_histogram"} 4.0
histogram_test_count{data="1",type="test_histogram"} 4.0
histogram_test_sum{data="1",type="test_histogram"} 25.2
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(self.metrics_url,
headers={ACCEPT:
text.TEXT_CONTENT_TYPE}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(self.metrics_url,
headers={
ACCEPT: binary.BINARY_CONTENT_TYPE
}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(binary.BINARY_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 1)
mf = metrics[0]
self.assertIsInstance(mf, pmp.MetricFamily)
self.assertEqual(mf.type, pmp.HISTOGRAM)
self.assertEqual(len(mf.metric), 1)
self.assertEqual(len(mf.metric[0].histogram.bucket), 4)
async def test_all(self):
counter_data = (
({
"c_sample": "1"
}, 100),
({
"c_sample": "2"
}, 200),
({
"c_sample": "3"
}, 300),
({
"c_sample": "1",
"c_subsample": "b"
}, 400),
)
gauge_data = (
({
"g_sample": "1"
}, 500),
({
"g_sample": "2"
}, 600),
({
"g_sample": "3"
}, 700),
({
"g_sample": "1",
"g_subsample": "b"
}, 800),
)
summary_data = (
({
"s_sample": "1"
}, range(1000, 2000, 4)),
({
"s_sample": "2"
}, range(2000, 3000, 20)),
({
"s_sample": "3"
}, range(3000, 4000, 13)),
({
"s_sample": "1",
"s_subsample": "b"
}, range(4000, 5000, 47)),
)
histogram_data = (
({
"h_sample": "1"
}, [3, 14]),
({
"h_sample": "2"
}, range(1, 20, 2)),
({
"h_sample": "3"
}, range(1, 20, 2)),
({
"h_sample": "1",
"h_subsample": "b"
}, range(1, 20, 2)),
)
counter = Counter("counter_test", "A counter.", {"type": "counter"})
gauge = Gauge("gauge_test", "A gauge.", {"type": "gauge"})
summary = Summary("summary_test", "A summary.", {"type": "summary"})
histogram = Histogram(
"histogram_test",
"A histogram.",
{"type": "histogram"},
buckets=[5.0, 10.0, 15.0],
)
self.server.register(counter)
self.server.register(gauge)
self.server.register(summary)
self.server.register(histogram)
# Add data
[counter.set(c[0], c[1]) for c in counter_data]
[gauge.set(g[0], g[1]) for g in gauge_data]
[summary.add(i[0], s) for i in summary_data for s in i[1]]
[histogram.observe(i[0], h) for i in histogram_data for h in i[1]]
expected_data = """# HELP counter_test A counter.
# TYPE counter_test counter
counter_test{c_sample="1",type="counter"} 100
counter_test{c_sample="2",type="counter"} 200
counter_test{c_sample="3",type="counter"} 300
counter_test{c_sample="1",c_subsample="b",type="counter"} 400
# HELP gauge_test A gauge.
# TYPE gauge_test gauge
gauge_test{g_sample="1",type="gauge"} 500
gauge_test{g_sample="2",type="gauge"} 600
gauge_test{g_sample="3",type="gauge"} 700
gauge_test{g_sample="1",g_subsample="b",type="gauge"} 800
# HELP histogram_test A histogram.
# TYPE histogram_test histogram
histogram_test_bucket{h_sample="1",le="5.0",type="histogram"} 1.0
histogram_test_bucket{h_sample="1",le="10.0",type="histogram"} 1.0
histogram_test_bucket{h_sample="1",le="15.0",type="histogram"} 2.0
histogram_test_bucket{h_sample="1",le="+Inf",type="histogram"} 2.0
histogram_test_count{h_sample="1",type="histogram"} 2.0
histogram_test_sum{h_sample="1",type="histogram"} 17.0
histogram_test_bucket{h_sample="2",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="2",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="2",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="2",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="2",type="histogram"} 10.0
histogram_test_sum{h_sample="2",type="histogram"} 100.0
histogram_test_bucket{h_sample="3",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="3",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="3",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="3",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="3",type="histogram"} 10.0
histogram_test_sum{h_sample="3",type="histogram"} 100.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="5.0",type="histogram"} 3.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="10.0",type="histogram"} 5.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="15.0",type="histogram"} 8.0
histogram_test_bucket{h_sample="1",h_subsample="b",le="+Inf",type="histogram"} 10.0
histogram_test_count{h_sample="1",h_subsample="b",type="histogram"} 10.0
histogram_test_sum{h_sample="1",h_subsample="b",type="histogram"} 100.0
# HELP summary_test A summary.
# TYPE summary_test summary
summary_test{quantile="0.5",s_sample="1",type="summary"} 1272.0
summary_test{quantile="0.9",s_sample="1",type="summary"} 1452.0
summary_test{quantile="0.99",s_sample="1",type="summary"} 1496.0
summary_test_count{s_sample="1",type="summary"} 250
summary_test_sum{s_sample="1",type="summary"} 374500.0
summary_test{quantile="0.5",s_sample="2",type="summary"} 2260.0
summary_test{quantile="0.9",s_sample="2",type="summary"} 2440.0
summary_test{quantile="0.99",s_sample="2",type="summary"} 2500.0
summary_test_count{s_sample="2",type="summary"} 50
summary_test_sum{s_sample="2",type="summary"} 124500.0
summary_test{quantile="0.5",s_sample="3",type="summary"} 3260.0
summary_test{quantile="0.9",s_sample="3",type="summary"} 3442.0
summary_test{quantile="0.99",s_sample="3",type="summary"} 3494.0
summary_test_count{s_sample="3",type="summary"} 77
summary_test_sum{s_sample="3",type="summary"} 269038.0
summary_test{quantile="0.5",s_sample="1",s_subsample="b",type="summary"} 4235.0
summary_test{quantile="0.9",s_sample="1",s_subsample="b",type="summary"} 4470.0
summary_test{quantile="0.99",s_sample="1",s_subsample="b",type="summary"} 4517.0
summary_test_count{s_sample="1",s_subsample="b",type="summary"} 22
summary_test_sum{s_sample="1",s_subsample="b",type="summary"} 98857.0
"""
async with aiohttp.ClientSession() as session:
# Fetch as text
async with session.get(self.metrics_url,
headers={ACCEPT:
text.TEXT_CONTENT_TYPE}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# Fetch as binary
async with session.get(self.metrics_url,
headers={
ACCEPT: binary.BINARY_CONTENT_TYPE
}) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(binary.BINARY_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
metrics = pmp.decode(content)
self.assertEqual(len(metrics), 4)
for mf in metrics:
self.assertIsInstance(mf, pmp.MetricFamily)
if mf.type == pmp.COUNTER:
self.assertEqual(len(mf.metric), 4)
elif mf.type == pmp.GAUGE:
self.assertEqual(len(mf.metric), 4)
elif mf.type == pmp.SUMMARY:
self.assertEqual(len(mf.metric), 4)
self.assertEqual(len(mf.metric[0].summary.quantile), 3)
elif mf.type == pmp.HISTOGRAM:
self.assertEqual(len(mf.metric), 4)
self.assertEqual(len(mf.metric[0].histogram.bucket), 4)
async def test_no_accept_header(self):
""" check default format is used when no accept header is defined """
# Add some metrics
data = (({"data": 1}, 100), )
c = Counter("test_counter", "Test Counter.", {"test": "test_counter"})
self.server.register(c)
for i in data:
c.set(i[0], i[1])
expected_data = """# HELP test_counter Test Counter.
# TYPE test_counter counter
test_counter{data="1",test="test_counter"} 100
"""
async with aiohttp.ClientSession() as session:
# Fetch without explicit accept type
async with session.get(self.metrics_url) as resp:
self.assertEqual(resp.status, 200)
content = await resp.read()
self.assertEqual(text.TEXT_CONTENT_TYPE,
resp.headers.get(CONTENT_TYPE))
self.assertEqual(expected_data, content.decode())
# TODO: Add another test here that includes the ACCEPT header
# but with no value set. I have not worked out how to do this
# yet as aiohttp expects headers to be a dict and a value of None
# is not permitted.
async def test_root_route(self):
""" check root route returns content """
async with aiohttp.ClientSession() as session:
async with session.get(self.root_url) as resp:
self.assertEqual(resp.status, 200)
self.assertIn("text/html", resp.headers.get(CONTENT_TYPE))
async def test_robots_route(self):
""" check robots route returns content """
async with aiohttp.ClientSession() as session:
async with session.get(f"{self.root_url}robots.txt") as resp:
self.assertEqual(resp.status, 200)
self.assertIn("text/plain", resp.headers.get(CONTENT_TYPE))
# Decorate function with metric.
@timer(REQUEST_TIME)
async def handle_request(duration):
""" A dummy function that takes some time """
await asyncio.sleep(duration)
async def handle_requests():
# Start up the server to expose the metrics.
await svr.start(port=8000)
# Generate some requests.
while True:
await handle_request(random.random())
if __name__ == "__main__":
loop = asyncio.get_event_loop()
svr = Service(loop=loop)
svr.register(REQUEST_TIME)
try:
loop.run_until_complete(handle_requests())
except KeyboardInterrupt:
pass
finally:
loop.run_until_complete(svr.stop())
loop.stop()
loop.close()
class Prometheus:
def __init__(self, bot):
self.bot = bot
self.msvr = Service()
if platform.system() == "Linux":
self.platform = platform
self.pid = os.path.join("/proc", "self")
self.pagesize = resource.getpagesize()
self.ticks = os.sysconf("SC_CLK_TCK")
self.btime = 0
with open(os.path.join("/proc", "stat"), "rb") as stat:
for line in stat:
if line.startswith(b"btime "):
self.btime = float(line.split()[1])
break
self.vmem = Gauge("process_virtual_memory_bytes", "Virtual memory size in bytes.")
self.rss = Gauge("process_resident_memory_bytes", "Resident memory size in bytes.")
self.start_time = Gauge("process_start_time_seconds", "Start time of the process since unix epoch in seconds.")
self.cpu = Counter("process_cpu_seconds", "Total user and system CPU time spent in seconds.")
self.fds = Gauge("process_open_fds", "Number of open file descriptors.")
self.info = Gauge("python_info", "Python platform information.")
self.collected = Counter("python_gc_objects_collected", "Objects collected during GC.")
self.uncollectable = Counter("python_gc_objects_uncollectable", "Uncollectable objects found during GC.")
self.collections = Counter("python_gc_collections", "Number of times this generation was collected.")
self.latency = Gauge("modmail_latency", "The average latency for shards on this cluster")
self.events = Counter("modmail_discord_events", "The total number of processed events.")
self.dispatch = Counter("modmail_dispatch_events", "The total number of dispatched events.")
self.http = Counter("modmail_http_requests", "The number of http requests sent to Discord.")
self.guilds_join = Counter("modmail_guilds_join", "The number of guilds ModMail is added to.")
self.guilds_leave = Counter("modmail_guilds_leave", "The number of guilds ModMail is removed from.")
self.shards = Gauge("modmail_shards", "The total number of shards on this cluster.")
self.guilds = Gauge("modmail_guilds", "The total number of guilds on this cluster.")
self.users = Gauge("modmail_users", "The total number of users on this cluster.")
self.commands = Counter("modmail_commands", "The total number of commands used on the bot.")
self.tickets = Counter("modmail_tickets", "The total number of tickets created by the bot.")
self.tickets_message = Counter("modmail_tickets_message", "The total number of messages sent in tickets.")
async def start(self):
for name, value in vars(self).items():
if issubclass(type(value), Collector):
self.msvr.register(getattr(self, name))
await self.msvr.start(addr="127.0.0.1", port=6000 + self.bot.cluster)
self.msvr._runner._server._kwargs["access_log"] = None
self.bot.loop.create_task(self.update_bot_stats())
if platform.system() == "Linux":
self.bot.loop.create_task(self.update_process_stats())
self.bot.loop.create_task(self.update_platform_stats())
async def update_bot_stats(self):
while True:
await self.bot.wait_until_ready()
await asyncio.sleep(60)
self.shards.set({}, len(self.bot.shards))
self.guilds.set({}, len(self.bot.guilds))
self.users.set({}, len(self.bot.users))
self.latency.set({}, self.bot.latency)
await asyncio.sleep(10)
async def update_process_stats(self):
while True:
with open(os.path.join(self.pid, "stat"), "rb") as stat:
parts = stat.read().split(b")")[-1].split()
self.vmem.set({}, float(parts[20]))
self.rss.set({}, float(parts[21]) * self.pagesize)
self.start_time.set({}, float(parts[19]) / self.ticks + self.btime)
self.cpu.set({}, float(parts[11]) / self.ticks + float(parts[12]) / self.ticks)
self.fds.set({}, len(os.listdir(os.path.join(self.pid, "fd"))))
await asyncio.sleep(5)
async def update_platform_stats(self):
while True:
self.info.set(
{
"version": self.platform.python_version(),
"implementation": self.platform.python_implementation(),
"major": platform.python_version_tuple()[0],
"minor": platform.python_version_tuple()[1],
"patchlevel": platform.python_version_tuple()[2],
},
1,
)
for gen, stat in enumerate(gc.get_stats()):
self.collected.set({"generation": str(gen)}, stat["collected"])
self.uncollectable.set({"generation": str(gen)}, stat["uncollectable"])
self.collections.set({"generation": str(gen)}, stat["collections"])
await asyncio.sleep(5)
# Decorate function with metric.
@timer(REQUEST_TIME)
async def handle_request(duration):
""" A dummy function that takes some time """
await asyncio.sleep(duration)
async def handle_requests():
# Start up the server to expose the metrics.
await svr.start(port=8000)
# Generate some requests.
while True:
await handle_request(random.random())
if __name__ == "__main__":
loop = asyncio.get_event_loop()
svr = Service(loop=loop)
svr.register(REQUEST_TIME)
try:
loop.run_until_complete(handle_requests())
except KeyboardInterrupt:
pass
finally:
loop.run_until_complete(svr.stop())
loop.stop()
loop.close()