def __init__(self): self.logger = logging.getLogger( "cobald.runtime.tardis.plugins.prometheusmonitoring") self.logger.setLevel(logging.DEBUG) config = Configuration().Plugins.PrometheusMonitoring self._port = config.port self._addr = config.addr self._svr_started = False self._drones = {} self._svr = Service() self._gauges = { ResourceStatus.Booting: Gauge("booting", "Booting drones"), ResourceStatus.Running: Gauge("running", "Running drones"), ResourceStatus.Stopped: Gauge("stopped", "Stopped drones"), ResourceStatus.Deleted: Gauge("deleted", "Deleted drones"), ResourceStatus.Error: Gauge("error", "Drones in error state"), } for gauge in self._gauges.values(): self._svr.register(gauge) gauge.set({}, 0)
def __init__(self, bot: commands.Bot): self.bot = bot self.registry = Registry() self.service = Service(self.registry) self.events = Counter("events", "Discord API event counts.") self.registry.register(self.events) self.latency = Histogram("latency", "Discord API latency.") self.registry.register(self.latency) self.gc_started: typing.Optional[float] = None self.gc_latency = Histogram( "gc_latency", "CPython garbage collector execution times." ) self.registry.register(self.gc_latency) self.gc_stats = Counter("gc_stats", "CPython garbage collector stats.") self.registry.register(self.gc_stats) self.process = psutil.Process(os.getpid()) self.resources = Gauge("resources", "Process resource usage gauges.") self.registry.register(self.resources) self.hook_gc() self.update_gc_and_resource_stats.start() # pylint: disable=no-member self.serve.start() # pylint: disable=no-member self.update_latency.start() # pylint: disable=no-member
def setUp(self): self.data = { 'name': "hdd_disk_used", 'doc': "Disk space used", 'const_labels': {"server": "1.db.production.my-app"}, } self.g = Gauge(**self.data)
async def test_gauge(self): """ check gauge metric export """ # Add some metrics data = ( ({ "data": 1 }, 100), ({ "data": "2" }, 200), ({ "data": 3 }, 300), ({ "data": 1 }, 400), ) g = Gauge("test_gauge", "Test Gauge.", {"test": "test_gauge"}) self.server.register(g) for i in data: g.set(i[0], i[1]) expected_data = """# HELP test_gauge Test Gauge. # TYPE test_gauge gauge test_gauge{data="1",test="test_gauge"} 400 test_gauge{data="2",test="test_gauge"} 200 test_gauge{data="3",test="test_gauge"} 300 """ async with aiohttp.ClientSession() as session: # Fetch as text async with session.get(self.metrics_url, headers={ACCEPT: text.TEXT_CONTENT_TYPE}) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(text.TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) self.assertEqual(expected_data, content.decode()) # Fetch as binary async with session.get(self.metrics_url, headers={ ACCEPT: binary.BINARY_CONTENT_TYPE }) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(binary.BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) metrics = pmp.decode(content) self.assertEqual(len(metrics), 1) mf = metrics[0] self.assertIsInstance(mf, pmp.MetricFamily) self.assertEqual(mf.type, pmp.GAUGE) self.assertEqual(len(mf.metric), 3)
def setUp(self): self.data = { "name": "hdd_disk_used", "doc": "Disk space used", "const_labels": { "server": "1.db.production.my-app" }, } self.g = Gauge(**self.data)
def test_gauge_format_with_const_labels(self): self.data = { "name": "logged_users_total", "doc": "Logged users in the application", "const_labels": {"app": "my_app"}, } g = Gauge(**self.data) counter_data = ( ({"country": "sp", "device": "desktop"}, 520), ({"country": "us", "device": "mobile"}, 654), ({"country": "uk", "device": "desktop"}, 1001), ({"country": "de", "device": "desktop"}, 995), ({"country": "zh", "device": "desktop"}, 520), ({"country": "ch", "device": "mobile"}, 654), ({"country": "ca", "device": "desktop"}, 1001), ({"country": "jp", "device": "desktop"}, 995), ({"country": "au", "device": "desktop"}, 520), ({"country": "py", "device": "mobile"}, 654), ({"country": "ar", "device": "desktop"}, 1001), ({"country": "pt", "device": "desktop"}, 995), ) valid_result = ( "# HELP logged_users_total Logged users in the application", "# TYPE logged_users_total gauge", 'logged_users_total{app="my_app",country="ch",device="mobile"} 654', 'logged_users_total{app="my_app",country="zh",device="desktop"} 520', 'logged_users_total{app="my_app",country="jp",device="desktop"} 995', 'logged_users_total{app="my_app",country="de",device="desktop"} 995', 'logged_users_total{app="my_app",country="pt",device="desktop"} 995', 'logged_users_total{app="my_app",country="ca",device="desktop"} 1001', 'logged_users_total{app="my_app",country="sp",device="desktop"} 520', 'logged_users_total{app="my_app",country="au",device="desktop"} 520', 'logged_users_total{app="my_app",country="uk",device="desktop"} 1001', 'logged_users_total{app="my_app",country="py",device="mobile"} 654', 'logged_users_total{app="my_app",country="us",device="mobile"} 654', 'logged_users_total{app="my_app",country="ar",device="desktop"} 1001', ) # Add data to the collector for i in counter_data: g.set_value(i[0], i[1]) # Select format f = TextFormatter() result = f.marshall_lines(g) result = sorted(result) valid_result = sorted(valid_result) self.assertEqual(valid_result, result)
def __init__(self, bot): self.bot = bot self.msvr = Service() if platform.system() == "Linux": self.platform = platform self.pid = os.path.join("/proc", "self") self.pagesize = resource.getpagesize() self.ticks = os.sysconf("SC_CLK_TCK") self.btime = 0 with open(os.path.join("/proc", "stat"), "rb") as stat: for line in stat: if line.startswith(b"btime "): self.btime = float(line.split()[1]) break self.vmem = Gauge("process_virtual_memory_bytes", "Virtual memory size in bytes.") self.rss = Gauge("process_resident_memory_bytes", "Resident memory size in bytes.") self.start_time = Gauge( "process_start_time_seconds", "Start time of the process since unix epoch in seconds.") self.cpu = Counter("process_cpu_seconds", "Total user and system CPU time spent in seconds.") self.fds = Gauge("process_open_fds", "Number of open file descriptors.") self.info = Gauge("python_info", "Python platform information.") self.collected = Counter("python_gc_objects_collected", "Objects collected during GC.") self.uncollectable = Counter("python_gc_objects_uncollectable", "Uncollectable objects found during GC.") self.collections = Counter( "python_gc_collections", "Number of times this generation was collected.") self.http = Counter("modmail_http_requests", "The number of http requests sent to Discord.") self.commands = Counter( "modmail_commands", "The total number of commands used on the bot.") self.tickets = Counter( "modmail_tickets", "The total number of tickets created by the bot.") self.tickets_message = Counter( "modmail_tickets_message", "The total number of messages sent in tickets.")
def get_or_create_gauge(registry, name, description=""): if name in registry.collectors: return registry.collectors[name] else: gauge = Gauge(name, description) registry.register(gauge) return gauge
def configure_prometheus_metrics_exporter(app: Starlette): app.add_middleware(MetricsMiddleware, webapp=app) app.registry = Registry() const_labels = { "host": socket.gethostname(), "name": "service1", "version": "1" } app.counter_gauge = Gauge("counter", "Current count.", const_labels=const_labels) app.registry.register(app.counter_gauge) app.svc_requests_total = Counter("svc_requests_total", "Count of service HTTP requests", const_labels=const_labels) app.registry.register(app.svc_requests_total) app.svc_responses_total = Counter("svc_responses_total", "Count of service HTTP responses", const_labels=const_labels) app.registry.register(app.svc_responses_total) app.svc_internal_error_total = Counter( "svc_internal_error_total", "Histogram of internal errors by method, path and type of error", const_labels=const_labels) app.registry.register(app.svc_internal_error_total)
async def test_gauge(self): """ check gauge metric export """ # Add some metrics data = ( ({"data": 1}, 100), ({"data": "2"}, 200), ({"data": 3}, 300), ({"data": 1}, 400), ) g = Gauge("test_gauge", "Test Gauge.", {"test": "test_gauge"}) self.server.register(g) for i in data: g.set(i[0], i[1]) expected_data = """# HELP test_gauge Test Gauge. # TYPE test_gauge gauge test_gauge{data="1",test="test_gauge"} 400 test_gauge{data="2",test="test_gauge"} 200 test_gauge{data="3",test="test_gauge"} 300 """ async with aiohttp.ClientSession() as session: # Fetch as text async with session.get( self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE} ) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) self.assertEqual(expected_data, content.decode()) # Fetch as binary async with session.get( self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE} ) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) metrics = pmp.decode(content) self.assertEqual(len(metrics), 1) mf = metrics[0] self.assertIsInstance(mf, pmp.MetricFamily) self.assertEqual(mf.type, pmp.GAUGE) self.assertEqual(len(mf.metric), 3)
def test_gauge_format_with_const_labels(self): data = { 'name': "logged_users_total", 'doc': "Logged users in the application", 'const_labels': { "app": "my_app" }, } g = Gauge(**data) gauge_data = ( ({ 'country': "sp", "device": "desktop" }, 520), ({ 'country': "us", "device": "mobile" }, 654), ({ 'country': "uk", "device": "desktop" }, 1001), ({ 'country': "de", "device": "desktop" }, 995), ({ 'country': "zh", "device": "desktop" }, 520), ) # Construct the result to compare valid_result = self._create_protobuf_object(data, gauge_data, pmp.GAUGE, data['const_labels']) # Add data to the collector for i in gauge_data: g.set_value(i[0], i[1]) f = BinaryFormatter() result = f.marshall_collector(g) self.assertTrue(self._protobuf_metric_equal(valid_result, result))
def setUp(self): self.data = { "name": "hdd_disk_used", "doc": "Disk space used", "const_labels": {"server": "1.db.production.my-app"}, } self.g = Gauge(**self.data)
def test_gauge_format_with_timestamp(self): self.data = { "name": "logged_users_total", "doc": "Logged users in the application", "const_labels": {}, } g = Gauge(**self.data) counter_data = ({"country": "ch", "device": "mobile"}, 654) g.set_value(counter_data[0], counter_data[1]) result_regex = r"""# HELP logged_users_total Logged users in the application # TYPE logged_users_total gauge logged_users_total{country="ch",device="mobile"} 654 \d*(?:.\d*)?$""" f_with_ts = TextFormatter(True) result = f_with_ts.marshall_collector(g) self.assertTrue(re.match(result_regex, result))
def test_gauge_format_with_timestamp(self): self.data = { "name": "logged_users_total", "doc": "Logged users in the application", "const_labels": {}, } g = Gauge(**self.data) counter_data = ({"country": "ch", "device": "mobile"}, 654) g.set_value(counter_data[0], counter_data[1]) result_regex = r"""# HELP logged_users_total Logged users in the application # TYPE logged_users_total gauge logged_users_total{country="ch",device="mobile"} 654 \d*(?:.\d*)?$""" f_with_ts = text.TextFormatter(True) result = f_with_ts.marshall_collector(g) self.assertTrue(re.match(result_regex, result))
def test_registry_marshall_gauge(self): gauge_data = (({"g_sample": "1", "g_subsample": "b"}, 800), ) gauge = Gauge("gauge_test", "A gauge.", const_labels={"type": "gauge"}) for labels, value in gauge_data: gauge.set(labels, value) registry = Registry() registry.register(gauge) valid_result = (b'U\n\ngauge_test\x12\x08A gauge.\x18\x01";' b"\n\r\n\x08g_sample\x12\x011\n\x10\n\x0bg_subsample" b"\x12\x01b\n\r\n\x04type\x12\x05gauge\x12\t\t\x00" b"\x00\x00\x00\x00\x00\x89@") f = binary.BinaryFormatter() self.assertEqual(valid_result, f.marshall(registry))
def __init__(self): config = Configuration().Plugins.PrometheusMonitoring self._port = config.port self._addr = config.addr self._svr_started = False self._drones = {} self._svr = Service() self._gauges = { ResourceStatus.Booting: Gauge("booting", "Booting drones"), ResourceStatus.Running: Gauge("running", "Running drones"), ResourceStatus.Stopped: Gauge("stopped", "Stopped drones"), ResourceStatus.Deleted: Gauge("deleted", "Deleted drones"), ResourceStatus.Error: Gauge("error", "Drones in error state"), } for gauge in self._gauges.values(): gauge.set({}, 0)
async def test_inprogress(self): m = Gauge("metric_label", "metric help") # decorator should work methods as well as functions @inprogress(m, {"kind": "function"}) async def a(): return await a() m_function_value = m.get({"kind": "function"}) self.assertEqual(m_function_value, 0) # decorator should work methods as well as functions class B(object): @inprogress(m, {"kind": "method"}) async def b(self, arg1, arg2=None): return arg1 == "b_arg", arg2 == "arg_2" b = B() results = await b.b("b_arg", arg2="arg_2") self.assertTrue(all(results)) m_method_value = m.get({"kind": "method"}) self.assertEqual(m_method_value, 0) # Only Gauge metric type can be used with @timer, others should # raise an exception. with self.assertRaises(Exception) as cm: m = Counter("metric_label", "metric help") @inprogress(m) async def c(): return self.assertIn( "inprogess decorator expects a Gauge metric but got:", str(cm.exception) )
async def test_gauge(self): # Add some metrics data = ( ({ 'data': 1 }, 100), ({ 'data': "2" }, 200), ({ 'data': 3 }, 300), ({ 'data': 1 }, 400), ) g = Gauge("test_gauge", "Test Gauge.", {'test': "test_gauge"}) self.registry.register(g) for i in data: g.set(i[0], i[1]) expected_data = """# HELP test_gauge Test Gauge. # TYPE test_gauge gauge test_gauge{data="1",test="test_gauge"} 400 test_gauge{data="2",test="test_gauge"} 200 test_gauge{data="3",test="test_gauge"} 300 """ with aiohttp.ClientSession(loop=self.loop) as session: headers = {ACCEPT: 'text/plain; version=0.0.4'} async with session.get(self.metrics_url, headers=headers) as resp: assert resp.status == 200 content = await resp.read() self.assertEqual("text/plain; version=0.0.4; charset=utf-8", resp.headers.get(CONTENT_TYPE)) self.assertEqual(200, resp.status) self.assertEqual(expected_data, content.decode())
async def test_inprogress(self): m = Gauge('metric_label', 'metric help') # decorator should work methods as well as functions @inprogress(m, {'kind': 'function'}) async def a(): return await a() m_function_value = m.get({'kind': 'function'}) self.assertEqual(m_function_value, 0) # decorator should work methods as well as functions class B(object): @inprogress(m, {'kind': 'method'}) async def b(self, arg1, arg2=None): return arg1 == 'b_arg', arg2 == 'arg_2' b = B() results = await b.b('b_arg', arg2='arg_2') self.assertTrue(all(results)) m_method_value = m.get({'kind': 'method'}) self.assertEqual(m_method_value, 0) # Only Gauge metric type can be used with @timer, others should # raise an exception. with self.assertRaises(Exception) as cm: m = Counter('metric_label', 'metric help') @inprogress(m) async def c(): return self.assertIn( "inprogess decorator expects a Gauge metric but got:", str(cm.exception))
def test_registry_marshall_gauge(self): format_times = 10 gauge_data = (({'g_sample': '1', 'g_subsample': 'b'}, 800), ) registry = Registry() gauge = Gauge("gauge_test", "A gauge.", {'type': "gauge"}) # Add data [gauge.set(g[0], g[1]) for g in gauge_data] registry.register(gauge) valid_result = (b'U\n\ngauge_test\x12\x08A gauge.\x18\x01";' b'\n\r\n\x08g_sample\x12\x011\n\x10\n\x0bg_subsample' b'\x12\x01b\n\r\n\x04type\x12\x05gauge\x12\t\t\x00' b'\x00\x00\x00\x00\x00\x89@') f = BinaryFormatter() # Check multiple times to ensure multiple marshalling requests for i in range(format_times): self.assertEqual(valid_result, f.marshall(registry))
def test_single_gauge_format_text(self): name = "prometheus_local_storage_indexing_queue_capacity" doc = "The capacity of the indexing queue." valid_result = """# HELP prometheus_local_storage_indexing_queue_capacity The capacity of the indexing queue. # TYPE prometheus_local_storage_indexing_queue_capacity gauge prometheus_local_storage_indexing_queue_capacity 16384""" data = ((None, 16384),) # Create the counter g = Gauge(name=name, doc=doc, const_labels={}) for i in data: g.set_value(i[0], i[1]) # Select format f = TextFormatter() result = f.marshall_collector(g) self.assertEqual(valid_result, result)
def test_single_gauge_format_text(self): name = "prometheus_local_storage_indexing_queue_capacity" doc = "The capacity of the indexing queue." valid_result = """# HELP prometheus_local_storage_indexing_queue_capacity The capacity of the indexing queue. # TYPE prometheus_local_storage_indexing_queue_capacity gauge prometheus_local_storage_indexing_queue_capacity 16384""" data = ((None, 16384), ) # Create the counter g = Gauge(name=name, doc=doc, const_labels={}) for i in data: g.set_value(i[0], i[1]) # Select format f = text.TextFormatter() result = f.marshall_collector(g) self.assertEqual(valid_result, result)
class TestGauge(unittest.TestCase): def setUp(self): self.data = { "name": "hdd_disk_used", "doc": "Disk space used", "const_labels": {"server": "1.db.production.my-app"}, } self.g = Gauge(**self.data) def test_set(self): data = ( {"labels": {"max": "500G", "dev": "sda"}, "values": range(0, 500, 50)}, {"labels": {"max": "1T", "dev": "sdb"}, "values": range(0, 1000, 100)}, {"labels": {"max": "10T", "dev": "sdc"}, "values": range(0, 10000, 1000)}, ) for i in data: for j in i["values"]: self.g.set(i["labels"], j) self.assertEqual(len(data), len(self.g.values)) def test_get(self): data = ( {"labels": {"max": "500G", "dev": "sda"}, "values": range(0, 500, 50)}, {"labels": {"max": "1T", "dev": "sdb"}, "values": range(0, 1000, 100)}, {"labels": {"max": "10T", "dev": "sdc"}, "values": range(0, 10000, 1000)}, ) for i in data: for j in i["values"]: self.g.set(i["labels"], j) self.assertEqual(j, self.g.get(i["labels"])) for i in data: self.assertEqual(max(i["values"]), self.g.get(i["labels"])) def test_set_get_without_labels(self): data = {"labels": {}, "values": range(100)} for i in data["values"]: self.g.set(data["labels"], i) self.assertEqual(1, len(self.g.values)) self.assertEqual(max(data["values"]), self.g.get(data["labels"])) def test_inc(self): iterations = 100 labels = {"max": "10T", "dev": "sdc"} for i in range(iterations): self.g.inc(labels) self.assertEqual(i + 1, self.g.get(labels)) self.assertEqual(iterations, self.g.get(labels)) def test_dec(self): iterations = 100 labels = {"max": "10T", "dev": "sdc"} self.g.set(labels, iterations) for i in range(iterations): self.g.dec(labels) self.assertEqual(iterations - (i + 1), self.g.get(labels)) self.assertEqual(0, self.g.get(labels)) def test_add(self): iterations = 100 labels = {"max": "10T", "dev": "sdc"} for i in range(iterations): self.g.add(labels, i) self.assertEqual(sum(range(iterations)), self.g.get(labels)) def test_add_negative(self): iterations = 100 labels = {"max": "10T", "dev": "sdc"} for i in range(iterations): self.g.add(labels, -i) self.assertEqual(sum(map(lambda x: -x, range(iterations))), self.g.get(labels)) def test_sub(self): iterations = 100 labels = {"max": "10T", "dev": "sdc"} for i in range(iterations): self.g.sub(labels, i) self.assertEqual(sum(map(lambda x: -x, range(iterations))), self.g.get(labels)) def test_sub_positive(self): iterations = 100 labels = {"max": "10T", "dev": "sdc"} for i in range(iterations): self.g.sub(labels, -i) self.assertEqual(sum(range(iterations)), self.g.get(labels))
if __name__ == "__main__": LISTEN_PORT = getenv("PENDLESSH_PORT", default=2222) LISTEN_ADDRESS = getenv("PENDLESSH_ADDRESS", default="0.0.0.0") MESSAGE_MAX_DELAY = int(getenv("PENDLESSH_MAX_DELAY", default=30)) PROMETHEUS_HOSTNAME = getenv("PENDLESSH_PROMETHEUS_HOSTNAME", default=gethostname()) log.info(f"Starting server on {LISTEN_ADDRESS}:{LISTEN_PORT} " f"with {MESSAGE_MAX_DELAY}s max delay") const_labels = { "host": PROMETHEUS_HOSTNAME, "app": f"{sys.argv[0].split('/')[-1].replace('.py', '')}", } PromConnectionCounter = Counter( "pendlessh_connections", "Number of connections received", const_labels=const_labels, ) PromActiveConnectionGauge = Gauge( "pendlessh_active_connections", "Number of currently active connections", const_labels=const_labels, ) PromServer = Service() PromServer.register(PromConnectionCounter) PromServer.register(PromActiveConnectionGauge) asyncio.run(main(PromServer))
def test_gauge_format_binary(self): g = Gauge(name=self.gauge_metric_name, doc=self.gauge_metric_help) # Add data to the collector for labels, values in self.gauge_metric_data: g.set_value(labels, values) f = binary.BinaryFormatter() result = f.marshall_collector(g) self.assertIsInstance(result, pmp.MetricFamily) # Construct the result to expected to receive when the gauge # collector is marshalled. expected_result = pmp.create_gauge(self.gauge_metric_name, self.gauge_metric_help, self.gauge_metric_data) self.assertEqual(result, expected_result) ###################################################################### # Check metric with constant labels g = Gauge( name=self.gauge_metric_name, doc=self.gauge_metric_help, const_labels=self.const_labels, ) # Add data to the collector for labels, values in self.gauge_metric_data: g.set_value(labels, values) f = binary.BinaryFormatter() result = f.marshall_collector(g) self.assertIsInstance(result, pmp.MetricFamily) # Construct the result to expected to receive when the gauge # collector is marshalled. expected_result = pmp.create_gauge( self.gauge_metric_name, self.gauge_metric_help, self.gauge_metric_data, const_labels=self.const_labels, ) self.assertEqual(result, expected_result) ###################################################################### # Check metric with timestamps with unittest.mock.patch.object(pmp.utils, "_timestamp_ms", return_value=TEST_TIMESTAMP): g = Gauge(name=self.gauge_metric_name, doc=self.gauge_metric_help) # Add data to the collector for labels, values in self.gauge_metric_data: g.set_value(labels, values) f = binary.BinaryFormatter(timestamp=True) result = f.marshall_collector(g) self.assertIsInstance(result, pmp.MetricFamily) # Construct the result to expected to receive when the gauge # collector is marshalled. expected_result = pmp.create_gauge( self.gauge_metric_name, self.gauge_metric_help, self.gauge_metric_data, timestamp=True, ) self.assertEqual(result, expected_result)
def __init__( self, metrics_host="127.0.0.1", metrics_port: int = 5000, loop: BaseEventLoop = None, ): self.metrics_host = metrics_host self.metrics_port = metrics_port self.loop = loop or asyncio.get_event_loop() self.timer = None # type: asyncio.Handle ###################################################################### # Create application metrics and metrics service # Create a metrics server. The server will create a metrics collector # registry if one is not specifically created and passed in. self.msvr = Service() # Define some constant labels that need to be added to all metrics const_labels = { "host": socket.gethostname(), "app": f"{self.__class__.__name__}-{uuid.uuid4().hex}", } # Create metrics collectors # Create a counter metric to track requests self.requests_metric = Counter( "requests", "Number of requests.", const_labels=const_labels ) # Collectors must be registered with the registry before they # get exposed. self.msvr.register(self.requests_metric) # Create a gauge metrics to track memory usage. self.ram_metric = Gauge( "memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels ) self.msvr.register(self.ram_metric) # Create a gauge metrics to track CPU. self.cpu_metric = Gauge( "cpu_usage_percent", "CPU usage percent.", const_labels=const_labels ) self.msvr.register(self.cpu_metric) self.payload_metric = Summary( "request_payload_size_bytes", "Request payload size in bytes.", const_labels=const_labels, invariants=[(0.50, 0.05), (0.99, 0.001)], ) self.msvr.register(self.payload_metric) self.latency_metric = Histogram( "request_latency_seconds", "Request latency in seconds", const_labels=const_labels, buckets=[0.1, 0.5, 1.0, 5.0], ) self.msvr.register(self.latency_metric)
def test_registry_marshall(self): format_times = 3 counter_data = ( ({"c_sample": "1"}, 100), ({"c_sample": "2"}, 200), ({"c_sample": "3"}, 300), ({"c_sample": "1", "c_subsample": "b"}, 400), ) gauge_data = ( ({"g_sample": "1"}, 500), ({"g_sample": "2"}, 600), ({"g_sample": "3"}, 700), ({"g_sample": "1", "g_subsample": "b"}, 800), ) summary_data = ( ({"s_sample": "1"}, range(1000, 2000, 4)), ({"s_sample": "2"}, range(2000, 3000, 20)), ({"s_sample": "3"}, range(3000, 4000, 13)), ({"s_sample": "1", "s_subsample": "b"}, range(4000, 5000, 47)), ) registry = Registry() counter = Counter("counter_test", "A counter.", {"type": "counter"}) gauge = Gauge("gauge_test", "A gauge.", {"type": "gauge"}) summary = Summary("summary_test", "A summary.", {"type": "summary"}) # Add data [counter.set(c[0], c[1]) for c in counter_data] [gauge.set(g[0], g[1]) for g in gauge_data] [summary.add(i[0], s) for i in summary_data for s in i[1]] registry.register(counter) registry.register(gauge) registry.register(summary) valid_regex = r"""# HELP counter_test A counter. # TYPE counter_test counter counter_test{c_sample="1",type="counter"} 100 counter_test{c_sample="2",type="counter"} 200 counter_test{c_sample="3",type="counter"} 300 counter_test{c_sample="1",c_subsample="b",type="counter"} 400 # HELP gauge_test A gauge. # TYPE gauge_test gauge gauge_test{g_sample="1",type="gauge"} 500 gauge_test{g_sample="2",type="gauge"} 600 gauge_test{g_sample="3",type="gauge"} 700 gauge_test{g_sample="1",g_subsample="b",type="gauge"} 800 # HELP summary_test A summary. # TYPE summary_test summary summary_test{quantile="0.5",s_sample="1",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.9",s_sample="1",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.99",s_sample="1",type="summary"} \d*(?:.\d*)? summary_test_count{s_sample="1",type="summary"} \d*(?:.\d*)? summary_test_sum{s_sample="1",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.5",s_sample="2",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.9",s_sample="2",type="summary"} 2\d*(?:.\d*)? summary_test{quantile="0.99",s_sample="2",type="summary"} \d*(?:.\d*)? summary_test_count{s_sample="2",type="summary"} \d*(?:.\d*)? summary_test_sum{s_sample="2",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.5",s_sample="3",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.9",s_sample="3",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.99",s_sample="3",type="summary"} \d*(?:.\d*)? summary_test_count{s_sample="3",type="summary"} \d*(?:.\d*)? summary_test_sum{s_sample="3",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.5",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.9",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.99",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? summary_test_count{s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? summary_test_sum{s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? """ f = TextFormatter() self.maxDiff = None # Check multiple times to ensure multiple calls to marshalling # produce the same results for i in range(format_times): self.assertTrue(re.match(valid_regex, f.marshall(registry).decode()))
.. code-block:: console $ curl :8000/metrics # HELP request_in_progress Number of requests in progress # TYPE request_in_progress gauge request_in_progress{route="/"} 1 ''' import asyncio import random from aioprometheus import Service, Gauge, inprogress # Create a metric to track requests currently in progress. REQUESTS = Gauge('request_in_progress', 'Number of requests in progress') # Decorate function with metric. @inprogress(REQUESTS, {'route': '/'}) async def handle_request(duration): ''' A dummy function that takes some time ''' await asyncio.sleep(duration) async def handle_requests(): # Start up the server to expose the metrics. await svr.start(port=8000) # Generate some requests. while True: await handle_request(random.random())
class ExampleApp(object): """ An example application that demonstrates how ``aioprometheus`` can be integrated and used within a Python application built upon asyncio. This application attempts to simulate a long running distributed system process, say a socket relay or some kind of message adapter. It is intentionally not hosting an existing web service in the application. In this case the aioprometheus.Service object is used to provide a new HTTP endpoint that can be used to expose Prometheus metrics on. If this application was a web service (i.e. already had an existing web interface) then the aioprometheus.Service object could be used as before to add another web interface or a different approach could be used that provides a metrics handler function for use with the existing web service. """ def __init__( self, metrics_host="127.0.0.1", metrics_port: int = 5000, loop: BaseEventLoop = None, ): self.metrics_host = metrics_host self.metrics_port = metrics_port self.loop = loop or asyncio.get_event_loop() self.timer = None # type: asyncio.Handle ###################################################################### # Create application metrics and metrics service # Create a metrics server. The server will create a metrics collector # registry if one is not specifically created and passed in. self.msvr = Service() # Define some constant labels that need to be added to all metrics const_labels = { "host": socket.gethostname(), "app": f"{self.__class__.__name__}-{uuid.uuid4().hex}", } # Create metrics collectors # Create a counter metric to track requests self.requests_metric = Counter( "requests", "Number of requests.", const_labels=const_labels ) # Collectors must be registered with the registry before they # get exposed. self.msvr.register(self.requests_metric) # Create a gauge metrics to track memory usage. self.ram_metric = Gauge( "memory_usage_bytes", "Memory usage in bytes.", const_labels=const_labels ) self.msvr.register(self.ram_metric) # Create a gauge metrics to track CPU. self.cpu_metric = Gauge( "cpu_usage_percent", "CPU usage percent.", const_labels=const_labels ) self.msvr.register(self.cpu_metric) self.payload_metric = Summary( "request_payload_size_bytes", "Request payload size in bytes.", const_labels=const_labels, invariants=[(0.50, 0.05), (0.99, 0.001)], ) self.msvr.register(self.payload_metric) self.latency_metric = Histogram( "request_latency_seconds", "Request latency in seconds", const_labels=const_labels, buckets=[0.1, 0.5, 1.0, 5.0], ) self.msvr.register(self.latency_metric) async def start(self): """ Start the application """ await self.msvr.start(addr=self.metrics_host, port=self.metrics_port) logger.debug("Serving prometheus metrics on: %s", self.msvr.metrics_url) # Schedule a timer to update internal metrics. In a realistic # application metrics would be updated as needed. In this example # application a simple timer is used to emulate things happening, # which conveniently allows all metrics to be updated at once. self.timer = self.loop.call_later(1.0, self.on_timer_expiry) async def stop(self): """ Stop the application """ await self.msvr.stop() if self.timer: self.timer.cancel() self.timer = None def on_timer_expiry(self): """ Update application to simulate work """ # Update memory metrics self.ram_metric.set({"type": "virtual"}, psutil.virtual_memory().used) self.ram_metric.set({"type": "swap"}, psutil.swap_memory().used) # Update cpu metrics for c, p in enumerate(psutil.cpu_percent(interval=1, percpu=True)): self.cpu_metric.set({"core": c}, p) # Incrementing a requests counter to emulate webserver app self.requests_metric.inc({"path": "/"}) # Monitor request payload data to emulate webserver app self.payload_metric.add({"path": "/data"}, random.random() * 2 ** 10) # Monitor request latency to emulate webserver app self.latency_metric.add({"path": "/data"}, random.random() * 5) # re-schedule another metrics update self.timer = self.loop.call_later(1.0, self.on_timer_expiry)
def __init__(self, namespace=""): if namespace: ns = namespace + '_' else: ns = "" self._ticks = 100.0 try: self._ticks = os.sysconf('SC_CLK_TCK') except (ValueError, TypeError, AttributeError): pass # This is used to test if we can access /proc. self._btime = 0 try: self._btime = self._boot_time() except IOError: pass major, minor, patchlevel = platform.python_version_tuple() info = { "version": platform.python_version(), "implementation": platform.python_implementation(), "major": major, "minor": minor, "patchlevel": patchlevel } self.process_metrics = { "info": Gauge(ns + "python_info", "Python platform information."), "vmem": Gauge(ns + "process_virtual_memory_bytes", "Virtual memory size in bytes."), "rss": Gauge(ns + "process_resident_memory_bytes", "Resident memory size in bytes."), "start_time": Gauge(ns + "process_start_time_seconds", "Start time of the process since unix epoch in seconds."), "cpu": Counter(ns + "process_cpu_seconds_total", "Total user and system CPU time spent in seconds."), "open_fds": Gauge(ns + "process_open_fds", "Number of open file descriptors."), "max_fds": Gauge(ns + "process_max_fds", "Maximum number of open file descriptors.") } # Only include these metrics if CPython and gc supports get_stats if hasattr( gc, 'get_stats') and platform.python_implementation() == 'CPython': self.process_metrics["collected"] = Counter( ns + "python_gc_objects_collected", "Objects collected during gc.") self.process_metrics["uncollectable"] = Counter( ns + "python_gc_objects_uncollectable", "Uncollectable object found during GC.") self.process_metrics["collections"] = Counter( ns + "python_gc_collections", "Number of times this generation was collected.") self.process_metrics["info"].set(info, 1.0)
class TestGauge(unittest.TestCase): def setUp(self): self.data = { 'name': "hdd_disk_used", 'doc': "Disk space used", 'const_labels': {"server": "1.db.production.my-app"}, } self.g = Gauge(**self.data) def test_set(self): data = ( { 'labels': {'max': "500G", 'dev': "sda"}, 'values': range(0, 500, 50) }, { 'labels': {'max': "1T", 'dev': "sdb"}, 'values': range(0, 1000, 100) }, { 'labels': {'max': "10T", 'dev': "sdc"}, 'values': range(0, 10000, 1000) } ) for i in data: for j in i['values']: self.g.set(i['labels'], j) self.assertEqual(len(data), len(self.g.values)) def test_get(self): data = ( { 'labels': {'max': "500G", 'dev': "sda"}, 'values': range(0, 500, 50) }, { 'labels': {'max': "1T", 'dev': "sdb"}, 'values': range(0, 1000, 100) }, { 'labels': {'max': "10T", 'dev': "sdc"}, 'values': range(0, 10000, 1000) } ) for i in data: for j in i['values']: self.g.set(i['labels'], j) self.assertEqual(j, self.g.get(i['labels'])) for i in data: self.assertEqual(max(i['values']), self.g.get(i['labels'])) def test_set_get_without_labels(self): data = { 'labels': {}, 'values': range(100) } for i in data['values']: self.g.set(data['labels'], i) self.assertEqual(1, len(self.g.values)) self.assertEqual(max(data['values']), self.g.get(data['labels'])) def test_inc(self): iterations = 100 labels = {'max': "10T", 'dev': "sdc"} for i in range(iterations): self.g.inc(labels) self.assertEqual(i + 1, self.g.get(labels)) self.assertEqual(iterations, self.g.get(labels)) def test_dec(self): iterations = 100 labels = {'max': "10T", 'dev': "sdc"} self.g.set(labels, iterations) for i in range(iterations): self.g.dec(labels) self.assertEqual(iterations - (i + 1), self.g.get(labels)) self.assertEqual(0, self.g.get(labels)) def test_add(self): iterations = 100 labels = {'max': "10T", 'dev': "sdc"} for i in range(iterations): self.g.add(labels, i) self.assertEqual(sum(range(iterations)), self.g.get(labels)) def test_add_negative(self): iterations = 100 labels = {'max': "10T", 'dev': "sdc"} for i in range(iterations): self.g.add(labels, -i) self.assertEqual(sum(map(lambda x: -x, range(iterations))), self.g.get(labels)) def test_sub(self): iterations = 100 labels = {'max': "10T", 'dev': "sdc"} for i in range(iterations): self.g.sub(labels, i) self.assertEqual(sum(map(lambda x: -x, range(iterations))), self.g.get(labels)) def test_sub_positive(self): iterations = 100 labels = {'max': "10T", 'dev': "sdc"} for i in range(iterations): self.g.sub(labels, -i) self.assertEqual(sum(range(iterations)), self.g.get(labels))
class Ctx: def __init__(self): self.tracer = None self.tracer_current_span = None self.profile_grpc = None self.profile_http = None ctx = Ctx() const_labels = { 'pod': socket.gethostname() # corresponds to pod name } ctx.profile_grpc = (Gauge('grpc_requests_in_flight', 'Number of gRPC requests being served.', const_labels=const_labels), Histogram( 'grpc_requests', 'Histogram of gRPC request latencies in milliseconds', const_labels=const_labels, buckets=[30, 50, 100, 200], ), Counter('grpc_requests_failed', 'Number of failed gRPC requests', const_labels=const_labels)) ctx.profile_http = (Gauge('http_requests_in_flight', 'Number of HTTP requests being served.', const_labels=const_labels), Histogram( 'http_requests',
def test_registry_marshall(self): format_times = 3 counter_data = ( ({ "c_sample": "1" }, 100), ({ "c_sample": "2" }, 200), ({ "c_sample": "3" }, 300), ({ "c_sample": "1", "c_subsample": "b" }, 400), ) gauge_data = ( ({ "g_sample": "1" }, 500), ({ "g_sample": "2" }, 600), ({ "g_sample": "3" }, 700), ({ "g_sample": "1", "g_subsample": "b" }, 800), ) summary_data = ( ({ "s_sample": "1" }, range(1000, 2000, 4)), ({ "s_sample": "2" }, range(2000, 3000, 20)), ({ "s_sample": "3" }, range(3000, 4000, 13)), ({ "s_sample": "1", "s_subsample": "b" }, range(4000, 5000, 47)), ) registry = Registry() counter = Counter("counter_test", "A counter.", {"type": "counter"}) gauge = Gauge("gauge_test", "A gauge.", {"type": "gauge"}) summary = Summary("summary_test", "A summary.", {"type": "summary"}) # Add data [counter.set(c[0], c[1]) for c in counter_data] [gauge.set(g[0], g[1]) for g in gauge_data] [summary.add(i[0], s) for i in summary_data for s in i[1]] registry.register(counter) registry.register(gauge) registry.register(summary) valid_regex = r"""# HELP counter_test A counter. # TYPE counter_test counter counter_test{c_sample="1",type="counter"} 100 counter_test{c_sample="2",type="counter"} 200 counter_test{c_sample="3",type="counter"} 300 counter_test{c_sample="1",c_subsample="b",type="counter"} 400 # HELP gauge_test A gauge. # TYPE gauge_test gauge gauge_test{g_sample="1",type="gauge"} 500 gauge_test{g_sample="2",type="gauge"} 600 gauge_test{g_sample="3",type="gauge"} 700 gauge_test{g_sample="1",g_subsample="b",type="gauge"} 800 # HELP summary_test A summary. # TYPE summary_test summary summary_test{quantile="0.5",s_sample="1",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.9",s_sample="1",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.99",s_sample="1",type="summary"} \d*(?:.\d*)? summary_test_count{s_sample="1",type="summary"} \d*(?:.\d*)? summary_test_sum{s_sample="1",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.5",s_sample="2",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.9",s_sample="2",type="summary"} 2\d*(?:.\d*)? summary_test{quantile="0.99",s_sample="2",type="summary"} \d*(?:.\d*)? summary_test_count{s_sample="2",type="summary"} \d*(?:.\d*)? summary_test_sum{s_sample="2",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.5",s_sample="3",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.9",s_sample="3",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.99",s_sample="3",type="summary"} \d*(?:.\d*)? summary_test_count{s_sample="3",type="summary"} \d*(?:.\d*)? summary_test_sum{s_sample="3",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.5",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.9",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? summary_test{quantile="0.99",s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? summary_test_count{s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? summary_test_sum{s_sample="1",s_subsample="b",type="summary"} \d*(?:.\d*)? """ f = text.TextFormatter() self.maxDiff = None # Check multiple times to ensure multiple calls to marshalling # produce the same results for i in range(format_times): self.assertTrue( re.match(valid_regex, f.marshall(registry).decode()))
def _create_gauge_metric(self, label, doc): gauge = Gauge('dump1090_{}'.format(label), doc) self.svr.registry.register(gauge) return gauge
def test_gauge_format_text(self): name = "container_memory_max_usage_bytes" doc = "Maximum memory usage ever recorded in bytes." valid_result = """# HELP container_memory_max_usage_bytes Maximum memory usage ever recorded in bytes. # TYPE container_memory_max_usage_bytes gauge container_memory_max_usage_bytes{id="4f70875bb57986783064fe958f694c9e225643b0d18e9cde6bdee56d47b7ce76",name="prometheus"} 0 container_memory_max_usage_bytes{id="89042838f24f0ec0aa2a6c93ff44fd3f3e43057d35cfd32de89558112ecb92a0",name="calendall_web_run_3"} 0 container_memory_max_usage_bytes{id="d11c6bc95459822e995fac4d4ae527f6cac442a1896a771dbb307ba276beceb9",name="db"} 0 container_memory_max_usage_bytes{id="e4260cc9dca3e4e50ad2bffb0ec7432442197f135023ab629fe3576485cc65dd",name="container-extractor"} 0 container_memory_max_usage_bytes{id="f30d1caaa142b1688a0684ed744fcae6d202a36877617b985e20a5d33801b311",name="calendall_db_1"} 0 container_memory_max_usage_bytes{id="f835d921ffaf332f8d88ef5231ba149e389a2f37276f081878d6f982ef89a981",name="cocky_fermat"} 0""" data = ( ( { "id": "4f70875bb57986783064fe958f694c9e225643b0d18e9cde6bdee56d47b7ce76", "name": "prometheus", }, 0, ), ( { "id": "89042838f24f0ec0aa2a6c93ff44fd3f3e43057d35cfd32de89558112ecb92a0", "name": "calendall_web_run_3", }, 0, ), ( { "id": "d11c6bc95459822e995fac4d4ae527f6cac442a1896a771dbb307ba276beceb9", "name": "db", }, 0, ), ( { "id": "e4260cc9dca3e4e50ad2bffb0ec7432442197f135023ab629fe3576485cc65dd", "name": "container-extractor", }, 0, ), ( { "id": "f30d1caaa142b1688a0684ed744fcae6d202a36877617b985e20a5d33801b311", "name": "calendall_db_1", }, 0, ), ( { "id": "f835d921ffaf332f8d88ef5231ba149e389a2f37276f081878d6f982ef89a981", "name": "cocky_fermat", }, 0, ), ) # Create the counter g = Gauge(name=name, doc=doc, const_labels={}) for i in data: g.set_value(i[0], i[1]) # Select format f = TextFormatter() result = f.marshall_collector(g) self.assertEqual(valid_result, result)
async def test_all(self): counter_data = ( ({"c_sample": "1"}, 100), ({"c_sample": "2"}, 200), ({"c_sample": "3"}, 300), ({"c_sample": "1", "c_subsample": "b"}, 400), ) gauge_data = ( ({"g_sample": "1"}, 500), ({"g_sample": "2"}, 600), ({"g_sample": "3"}, 700), ({"g_sample": "1", "g_subsample": "b"}, 800), ) summary_data = ( ({"s_sample": "1"}, range(1000, 2000, 4)), ({"s_sample": "2"}, range(2000, 3000, 20)), ({"s_sample": "3"}, range(3000, 4000, 13)), ({"s_sample": "1", "s_subsample": "b"}, range(4000, 5000, 47)), ) histogram_data = ( ({"h_sample": "1"}, [3, 14]), ({"h_sample": "2"}, range(1, 20, 2)), ({"h_sample": "3"}, range(1, 20, 2)), ({"h_sample": "1", "h_subsample": "b"}, range(1, 20, 2)), ) counter = Counter("counter_test", "A counter.", {"type": "counter"}) gauge = Gauge("gauge_test", "A gauge.", {"type": "gauge"}) summary = Summary("summary_test", "A summary.", {"type": "summary"}) histogram = Histogram( "histogram_test", "A histogram.", {"type": "histogram"}, buckets=[5.0, 10.0, 15.0], ) self.server.register(counter) self.server.register(gauge) self.server.register(summary) self.server.register(histogram) # Add data [counter.set(c[0], c[1]) for c in counter_data] [gauge.set(g[0], g[1]) for g in gauge_data] [summary.add(i[0], s) for i in summary_data for s in i[1]] [histogram.observe(i[0], h) for i in histogram_data for h in i[1]] expected_data = """# HELP counter_test A counter. # TYPE counter_test counter counter_test{c_sample="1",type="counter"} 100 counter_test{c_sample="2",type="counter"} 200 counter_test{c_sample="3",type="counter"} 300 counter_test{c_sample="1",c_subsample="b",type="counter"} 400 # HELP gauge_test A gauge. # TYPE gauge_test gauge gauge_test{g_sample="1",type="gauge"} 500 gauge_test{g_sample="2",type="gauge"} 600 gauge_test{g_sample="3",type="gauge"} 700 gauge_test{g_sample="1",g_subsample="b",type="gauge"} 800 # HELP histogram_test A histogram. # TYPE histogram_test histogram histogram_test_bucket{h_sample="1",le="5.0",type="histogram"} 1.0 histogram_test_bucket{h_sample="1",le="10.0",type="histogram"} 1.0 histogram_test_bucket{h_sample="1",le="15.0",type="histogram"} 2.0 histogram_test_bucket{h_sample="1",le="+Inf",type="histogram"} 2.0 histogram_test_count{h_sample="1",type="histogram"} 2.0 histogram_test_sum{h_sample="1",type="histogram"} 17.0 histogram_test_bucket{h_sample="2",le="5.0",type="histogram"} 3.0 histogram_test_bucket{h_sample="2",le="10.0",type="histogram"} 5.0 histogram_test_bucket{h_sample="2",le="15.0",type="histogram"} 8.0 histogram_test_bucket{h_sample="2",le="+Inf",type="histogram"} 10.0 histogram_test_count{h_sample="2",type="histogram"} 10.0 histogram_test_sum{h_sample="2",type="histogram"} 100.0 histogram_test_bucket{h_sample="3",le="5.0",type="histogram"} 3.0 histogram_test_bucket{h_sample="3",le="10.0",type="histogram"} 5.0 histogram_test_bucket{h_sample="3",le="15.0",type="histogram"} 8.0 histogram_test_bucket{h_sample="3",le="+Inf",type="histogram"} 10.0 histogram_test_count{h_sample="3",type="histogram"} 10.0 histogram_test_sum{h_sample="3",type="histogram"} 100.0 histogram_test_bucket{h_sample="1",h_subsample="b",le="5.0",type="histogram"} 3.0 histogram_test_bucket{h_sample="1",h_subsample="b",le="10.0",type="histogram"} 5.0 histogram_test_bucket{h_sample="1",h_subsample="b",le="15.0",type="histogram"} 8.0 histogram_test_bucket{h_sample="1",h_subsample="b",le="+Inf",type="histogram"} 10.0 histogram_test_count{h_sample="1",h_subsample="b",type="histogram"} 10.0 histogram_test_sum{h_sample="1",h_subsample="b",type="histogram"} 100.0 # HELP summary_test A summary. # TYPE summary_test summary summary_test{quantile="0.5",s_sample="1",type="summary"} 1272.0 summary_test{quantile="0.9",s_sample="1",type="summary"} 1452.0 summary_test{quantile="0.99",s_sample="1",type="summary"} 1496.0 summary_test_count{s_sample="1",type="summary"} 250 summary_test_sum{s_sample="1",type="summary"} 374500.0 summary_test{quantile="0.5",s_sample="2",type="summary"} 2260.0 summary_test{quantile="0.9",s_sample="2",type="summary"} 2440.0 summary_test{quantile="0.99",s_sample="2",type="summary"} 2500.0 summary_test_count{s_sample="2",type="summary"} 50 summary_test_sum{s_sample="2",type="summary"} 124500.0 summary_test{quantile="0.5",s_sample="3",type="summary"} 3260.0 summary_test{quantile="0.9",s_sample="3",type="summary"} 3442.0 summary_test{quantile="0.99",s_sample="3",type="summary"} 3494.0 summary_test_count{s_sample="3",type="summary"} 77 summary_test_sum{s_sample="3",type="summary"} 269038.0 summary_test{quantile="0.5",s_sample="1",s_subsample="b",type="summary"} 4235.0 summary_test{quantile="0.9",s_sample="1",s_subsample="b",type="summary"} 4470.0 summary_test{quantile="0.99",s_sample="1",s_subsample="b",type="summary"} 4517.0 summary_test_count{s_sample="1",s_subsample="b",type="summary"} 22 summary_test_sum{s_sample="1",s_subsample="b",type="summary"} 98857.0 """ async with aiohttp.ClientSession() as session: # Fetch as text async with session.get( self.metrics_url, headers={ACCEPT: TEXT_CONTENT_TYPE} ) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(TEXT_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) self.assertEqual(expected_data, content.decode()) # Fetch as binary async with session.get( self.metrics_url, headers={ACCEPT: BINARY_CONTENT_TYPE} ) as resp: self.assertEqual(resp.status, 200) content = await resp.read() self.assertEqual(BINARY_CONTENT_TYPE, resp.headers.get(CONTENT_TYPE)) metrics = pmp.decode(content) self.assertEqual(len(metrics), 4) for mf in metrics: self.assertIsInstance(mf, pmp.MetricFamily) if mf.type == pmp.COUNTER: self.assertEqual(len(mf.metric), 4) elif mf.type == pmp.GAUGE: self.assertEqual(len(mf.metric), 4) elif mf.type == pmp.SUMMARY: self.assertEqual(len(mf.metric), 4) self.assertEqual(len(mf.metric[0].summary.quantile), 3) elif mf.type == pmp.HISTOGRAM: self.assertEqual(len(mf.metric), 4) self.assertEqual(len(mf.metric[0].histogram.bucket), 4)
def test_register_gauge(self): """ check registering a gauge collector """ r = CollectorRegistry() r.register(Gauge(**self.data)) self.assertEqual(1, len(r.collectors))
def test_gauge_format_with_const_labels(self): self.data = { "name": "logged_users_total", "doc": "Logged users in the application", "const_labels": { "app": "my_app" }, } g = Gauge(**self.data) counter_data = ( ({ "country": "sp", "device": "desktop" }, 520), ({ "country": "us", "device": "mobile" }, 654), ({ "country": "uk", "device": "desktop" }, 1001), ({ "country": "de", "device": "desktop" }, 995), ({ "country": "zh", "device": "desktop" }, 520), ({ "country": "ch", "device": "mobile" }, 654), ({ "country": "ca", "device": "desktop" }, 1001), ({ "country": "jp", "device": "desktop" }, 995), ({ "country": "au", "device": "desktop" }, 520), ({ "country": "py", "device": "mobile" }, 654), ({ "country": "ar", "device": "desktop" }, 1001), ({ "country": "pt", "device": "desktop" }, 995), ) valid_result = ( "# HELP logged_users_total Logged users in the application", "# TYPE logged_users_total gauge", 'logged_users_total{app="my_app",country="ch",device="mobile"} 654', 'logged_users_total{app="my_app",country="zh",device="desktop"} 520', 'logged_users_total{app="my_app",country="jp",device="desktop"} 995', 'logged_users_total{app="my_app",country="de",device="desktop"} 995', 'logged_users_total{app="my_app",country="pt",device="desktop"} 995', 'logged_users_total{app="my_app",country="ca",device="desktop"} 1001', 'logged_users_total{app="my_app",country="sp",device="desktop"} 520', 'logged_users_total{app="my_app",country="au",device="desktop"} 520', 'logged_users_total{app="my_app",country="uk",device="desktop"} 1001', 'logged_users_total{app="my_app",country="py",device="mobile"} 654', 'logged_users_total{app="my_app",country="us",device="mobile"} 654', 'logged_users_total{app="my_app",country="ar",device="desktop"} 1001', ) # Add data to the collector for i in counter_data: g.set_value(i[0], i[1]) # Select format f = text.TextFormatter() result = f.marshall_lines(g) result = sorted(result) valid_result = sorted(valid_result) self.assertEqual(valid_result, result)
class Prometheus: def __init__(self, bot): self.bot = bot self.msvr = Service() if platform.system() == "Linux": self.platform = platform self.pid = os.path.join("/proc", "self") self.pagesize = resource.getpagesize() self.ticks = os.sysconf("SC_CLK_TCK") self.btime = 0 with open(os.path.join("/proc", "stat"), "rb") as stat: for line in stat: if line.startswith(b"btime "): self.btime = float(line.split()[1]) break self.vmem = Gauge("process_virtual_memory_bytes", "Virtual memory size in bytes.") self.rss = Gauge("process_resident_memory_bytes", "Resident memory size in bytes.") self.start_time = Gauge("process_start_time_seconds", "Start time of the process since unix epoch in seconds.") self.cpu = Counter("process_cpu_seconds", "Total user and system CPU time spent in seconds.") self.fds = Gauge("process_open_fds", "Number of open file descriptors.") self.info = Gauge("python_info", "Python platform information.") self.collected = Counter("python_gc_objects_collected", "Objects collected during GC.") self.uncollectable = Counter("python_gc_objects_uncollectable", "Uncollectable objects found during GC.") self.collections = Counter("python_gc_collections", "Number of times this generation was collected.") self.latency = Gauge("modmail_latency", "The average latency for shards on this cluster") self.events = Counter("modmail_discord_events", "The total number of processed events.") self.dispatch = Counter("modmail_dispatch_events", "The total number of dispatched events.") self.http = Counter("modmail_http_requests", "The number of http requests sent to Discord.") self.guilds_join = Counter("modmail_guilds_join", "The number of guilds ModMail is added to.") self.guilds_leave = Counter("modmail_guilds_leave", "The number of guilds ModMail is removed from.") self.shards = Gauge("modmail_shards", "The total number of shards on this cluster.") self.guilds = Gauge("modmail_guilds", "The total number of guilds on this cluster.") self.users = Gauge("modmail_users", "The total number of users on this cluster.") self.commands = Counter("modmail_commands", "The total number of commands used on the bot.") self.tickets = Counter("modmail_tickets", "The total number of tickets created by the bot.") self.tickets_message = Counter("modmail_tickets_message", "The total number of messages sent in tickets.") async def start(self): for name, value in vars(self).items(): if issubclass(type(value), Collector): self.msvr.register(getattr(self, name)) await self.msvr.start(addr="127.0.0.1", port=6000 + self.bot.cluster) self.msvr._runner._server._kwargs["access_log"] = None self.bot.loop.create_task(self.update_bot_stats()) if platform.system() == "Linux": self.bot.loop.create_task(self.update_process_stats()) self.bot.loop.create_task(self.update_platform_stats()) async def update_bot_stats(self): while True: await self.bot.wait_until_ready() await asyncio.sleep(60) self.shards.set({}, len(self.bot.shards)) self.guilds.set({}, len(self.bot.guilds)) self.users.set({}, len(self.bot.users)) self.latency.set({}, self.bot.latency) await asyncio.sleep(10) async def update_process_stats(self): while True: with open(os.path.join(self.pid, "stat"), "rb") as stat: parts = stat.read().split(b")")[-1].split() self.vmem.set({}, float(parts[20])) self.rss.set({}, float(parts[21]) * self.pagesize) self.start_time.set({}, float(parts[19]) / self.ticks + self.btime) self.cpu.set({}, float(parts[11]) / self.ticks + float(parts[12]) / self.ticks) self.fds.set({}, len(os.listdir(os.path.join(self.pid, "fd")))) await asyncio.sleep(5) async def update_platform_stats(self): while True: self.info.set( { "version": self.platform.python_version(), "implementation": self.platform.python_implementation(), "major": platform.python_version_tuple()[0], "minor": platform.python_version_tuple()[1], "patchlevel": platform.python_version_tuple()[2], }, 1, ) for gen, stat in enumerate(gc.get_stats()): self.collected.set({"generation": str(gen)}, stat["collected"]) self.uncollectable.set({"generation": str(gen)}, stat["uncollectable"]) self.collections.set({"generation": str(gen)}, stat["collections"]) await asyncio.sleep(5)
def test_gauge_format_text(self): name = "container_memory_max_usage_bytes" doc = "Maximum memory usage ever recorded in bytes." valid_result = """# HELP container_memory_max_usage_bytes Maximum memory usage ever recorded in bytes. # TYPE container_memory_max_usage_bytes gauge container_memory_max_usage_bytes{id="4f70875bb57986783064fe958f694c9e225643b0d18e9cde6bdee56d47b7ce76",name="prometheus"} 0 container_memory_max_usage_bytes{id="89042838f24f0ec0aa2a6c93ff44fd3f3e43057d35cfd32de89558112ecb92a0",name="calendall_web_run_3"} 0 container_memory_max_usage_bytes{id="d11c6bc95459822e995fac4d4ae527f6cac442a1896a771dbb307ba276beceb9",name="db"} 0 container_memory_max_usage_bytes{id="e4260cc9dca3e4e50ad2bffb0ec7432442197f135023ab629fe3576485cc65dd",name="container-extractor"} 0 container_memory_max_usage_bytes{id="f30d1caaa142b1688a0684ed744fcae6d202a36877617b985e20a5d33801b311",name="calendall_db_1"} 0 container_memory_max_usage_bytes{id="f835d921ffaf332f8d88ef5231ba149e389a2f37276f081878d6f982ef89a981",name="cocky_fermat"} 0""" data = ( ( { "id": "4f70875bb57986783064fe958f694c9e225643b0d18e9cde6bdee56d47b7ce76", "name": "prometheus", }, 0, ), ( { "id": "89042838f24f0ec0aa2a6c93ff44fd3f3e43057d35cfd32de89558112ecb92a0", "name": "calendall_web_run_3", }, 0, ), ( { "id": "d11c6bc95459822e995fac4d4ae527f6cac442a1896a771dbb307ba276beceb9", "name": "db", }, 0, ), ( { "id": "e4260cc9dca3e4e50ad2bffb0ec7432442197f135023ab629fe3576485cc65dd", "name": "container-extractor", }, 0, ), ( { "id": "f30d1caaa142b1688a0684ed744fcae6d202a36877617b985e20a5d33801b311", "name": "calendall_db_1", }, 0, ), ( { "id": "f835d921ffaf332f8d88ef5231ba149e389a2f37276f081878d6f982ef89a981", "name": "cocky_fermat", }, 0, ), ) # Create the counter g = Gauge(name=name, doc=doc, const_labels={}) for i in data: g.set_value(i[0], i[1]) # Select format f = text.TextFormatter() result = f.marshall_collector(g) self.assertEqual(valid_result, result)