title="Configs Latency",
expressions=[
("99th centile",
'histogram_quantile(0.99, sum(rate(cortex_configs_request_duration_seconds_bucket{job="cortex/ruler"}[2m])) by (le)) * 1e3'
),
("50th centile",
'histogram_quantile(0.50, sum(rate(cortex_configs_request_duration_seconds_bucket{job="cortex/ruler"}[2m])) by (le)) * 1e3'
),
("Mean",
'sum(rate(cortex_configs_request_duration_seconds_sum{job="cortex/ruler"}[2m])) / sum(rate(cortex_configs_request_duration_seconds_count{job="cortex/ruler"}[2m])) * 1e3'
),
],
yAxes=common.LATENCY_AXES,
),
]),
common.REDRow('cortex', 'Ruler service', 'cortex/ruler',
collapse=True),
G.Row([
common.PromGraph(
title="Group Evaluations per Second",
expressions=
[("Groups per second",
'sum(rate(cortex_group_evaluation_duration_seconds_count{job="cortex/ruler"}[1m]))'
),
("Groups per second",
'sum(rate(cortex_prometheus_rule_group_duration_seconds_count{job="cortex/ruler"}[1m]))'
)],
yAxes=common.OPS_AXIS,
),
common.PromGraph(
title="Group Evaluation Durations",
expressions=[
# -*- mode: python; python-indent-offset: 2 -*-
import grafanalib.core as G
import sys, os
sys.path.append(os.path.dirname(__file__))
import common
dashboard = common.Dashboard(
uid='reads',
title="Cortex > Services (Reads)",
rows=[
common.REDRow(
'cortex',
'Query Frontend read',
'cortex/query-frontend',
rule_root="job_route:",
extra_conditions=",route=\"api_prom_api_v1_query_range\""),
common.REDRow('cortex', 'Querier read', 'cortex/querier'),
G.Row(
title="Ingester",
panels=[
common.PromGraph(
title="Ingester read QPS",
expressions=[
('{{route}}: {{status_code}}',
'sum(rate(cortex_request_duration_seconds_count{job="cortex/ingester", route!="/cortex.Ingester/Push"}[1m])) by (route, status_code)'
),
],
yAxes=common.OPS_AXIS,
),
title="Configs Latency",
expressions=[
("99th centile",
'histogram_quantile(0.99, sum(rate(cortex_configs_request_duration_seconds_bucket{job="cortex/alertmanager"}[2m])) by (le)) * 1e3'
),
("50th centile",
'histogram_quantile(0.50, sum(rate(cortex_configs_request_duration_seconds_bucket{job="cortex/alertmanager"}[2m])) by (le)) * 1e3'
),
("Mean",
'sum(rate(cortex_configs_request_duration_seconds_sum{job="cortex/alertmanager"}[2m])) / sum(rate(cortex_configs_request_duration_seconds_count{job="cortex/alertmanager"}[2m])) * 1e3'
),
],
yAxes=common.LATENCY_AXES,
),
]),
common.REDRow('cortex', 'Alertmanager', 'cortex/alertmanager'),
G.Row([
common.PromGraph(
title="Known Configurations",
expressions=[
("{{instance}}",
'cortex_alertmanager_configs_total{job="cortex/alertmanager"}'
),
],
),
common.PromGraph(
title="Cluster Members",
expressions=[
("{{instance}}",
'sum(alertmanager_cluster_members{job="cortex/alertmanager"}) by (instance)'
),