('99% {{name}}',
'histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{job=~"cortex/querier|cortex/store-gateway",operation="getmulti"}[2m])) by (le, name))'
),
('Mean',
'sum(rate(thanos_memcached_operation_duration_seconds_sum{job=~"cortex/querier|cortex/store-gateway",operation="getmulti"}[2m])) / sum(rate(thanos_memcached_operation_duration_seconds_count{job=~"cortex/querier|cortex/store-gateway",operation="getmulti"}[2m]))'
),
],
yAxes=G.single_y_axis(format=G.SECONDS_FORMAT),
),
],
),
G.Row(
title="Memcache (chunks)",
panels=[
common.StatusQPSGraph(
common.PROMETHEUS, "Memcache read QPS (chunks)",
'sum by (job,status_code)(rate(cortex_memcache_request_duration_seconds_count{method="Memcache.GetMulti", job=~"cortex/querier|cortex/query-frontend"}[1m]))'
),
common.PromGraph(
title="Memcache read latency (chunks)",
expressions=[
('99% {{name}}',
'histogram_quantile(0.99, sum(rate(cortex_memcache_request_duration_seconds_bucket{job=~"cortex/querier|cortex/query-frontend",method="Memcache.GetMulti"}[2m])) by (le, name))'
),
('Mean',
'sum(rate(cortex_memcache_request_duration_seconds_sum{job=~"cortex/querier|cortex/query-frontend",method="Memcache.GetMulti"}[2m])) / sum(rate(cortex_memcache_request_duration_seconds_count{job=~"cortex/querier|cortex/query-frontend",method="Memcache.GetMulti"}[2m]))'
),
],
yAxes=G.single_y_axis(format=G.SECONDS_FORMAT),
),
],
),
'sum(prometheus_remote_storage_pending_samples) by (url)'
),
('{{url}}: lag',
'max(time()-prometheus_remote_storage_queue_highest_sent_timestamp_seconds) by (url)'
),
('{{url}}: shards',
'max(prometheus_remote_storage_shards) by (url)'),
],
),
],
),
G.Row(
title="Distributor",
panels=[
common.StatusQPSGraph(
common.PROMETHEUS, "Distributor write QPS",
'rate(cortex_request_duration_seconds_count{job="cortex/distributor"}[1m])'
),
common.LatencyGraph("cortex", "Distributor Write",
"cortex/distributor"),
],
),
G.Row(
title="Distributor breakdown",
collapse=True,
panels=[
common.PromGraph(
title="Distributor Error Rate",
expressions=[
('{{instance}}',
'sum by (instance)(rate(cortex_request_duration_seconds_count{job="cortex/distributor", status_code =~ "5.."}[1m]))'
),
),
("50th centile",
'histogram_quantile(0.50, sum(rate(cortex_distributor_query_duration_seconds_bucket{job="cortex/ruler"}[2m])) by (le)) * 1e3'
),
("Mean",
'sum(rate(cortex_distributor_query_duration_seconds_sum{job="cortex/ruler"}[2m])) / sum(rate(cortex_distributor_query_duration_seconds_count{job="cortex/ruler"}[2m])) * 1e3'
),
],
yAxes=common.LATENCY_AXES,
),
]),
G.Row(
title="Ingester Push",
panels=[
common.StatusQPSGraph(
common.PROMETHEUS, "Ingester Push",
'rate(cortex_ingester_client_request_duration_seconds_count{job="cortex/ruler",operation="/cortex.Ingester/Push"}[1m])'
),
common.PromGraph(
title="Ingester Push Latency",
expressions=[
("99.7th centile",
'histogram_quantile(0.997, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job="cortex/ruler",operation="/cortex.Ingester/Push"}[2m])) by (le)) * 1e3'
),
("50th centile",
'histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job="cortex/ruler",operation="/cortex.Ingester/Push"}[2m])) by (le)) * 1e3'
),
("Mean",
'sum(rate(cortex_ingester_client_request_duration_seconds_sum{job="cortex/ruler",operation="/cortex.Ingester/Push"}[2m])) / sum(rate(cortex_ingester_client_request_duration_seconds_count{job="cortex/ruler",operation="/cortex.Ingester/Push"}[2m])) * 1e3'
),
],
yAxes=common.LATENCY_AXES,