def create_slo_panel(metric="apiserver:apiserver_request_latency:histogram_quantile"):
return [
api_call_latency(
title="Read-only API call latency (scope=resource, threshold=1s)",
metric=metric,
verb="GET",
scope="resource",
threshold=1,
),
api_call_latency(
title="Read-only API call latency (scope=namespace, threshold=5s)",
metric=metric,
verb="LIST",
scope="namespace",
threshold=5,
),
api_call_latency(
title="Read-only API call latency (scope=cluster, threshold=30s)",
metric=metric,
verb="LIST",
scope="cluster",
threshold=30,
),
api_call_latency(
title="Mutating API call latency (threshold=1s)",
metric=metric,
verb=d.any_of("CREATE", "DELETE", "PATCH", "POST", "PUT"),
scope=d.any_of("namespace", "cluster"),
threshold=1,
),
]
def api_call_latency_panel(expression):
def api_call_latency(title, verb, scope, threshold):
return d.Graph(
title=title,
targets=[
g.Target(expr=str(threshold), legendFormat="threshold"),
g.Target(
expr=d.one_line(expression % {
"verb": verb,
"scope": scope
}),
# TODO(github.com/grafana/grafana/issues/19410): uncomment once fixed
# legendFormat="{{verb}} {{scope}}/{{resource}}",
),
],
yAxes=g.single_y_axis(format=g.SECONDS_FORMAT),
)
return [
api_call_latency(
title=
"Read-only API call latency (percentaile=99, scope=resource, threshold=1s)",
verb="GET",
scope="namespace",
threshold=1,
),
api_call_latency(
title=
"Read-only API call latency (percentaile=99, scope=namespace, threshold=5s)",
verb="LIST",
scope="namespace",
threshold=5,
),
api_call_latency(
title=
"Read-only API call latency (percentaile=99, scope=cluster, threshold=30s)",
verb="LIST",
scope="cluster",
threshold=30,
),
api_call_latency(
title="Mutating API call latency (threshold=1s)",
verb=d.any_of("CREATE", "DELETE", "PATCH", "POST", "PUT"),
scope=d.any_of("namespace", "cluster"),
threshold=1,
),
]
title=
"Read-only API call latency (percentaile=99, scope=namespace, threshold=5s)",
verb="LIST",
scope="namespace",
threshold=5,
),
api_call_latency(
title=
"Read-only API call latency (percentaile=99, scope=cluster, threshold=30s)",
verb="LIST",
scope="cluster",
threshold=30,
),
api_call_latency(
title="Mutating API call latency (threshold=1s)",
verb=d.any_of("CREATE", "DELETE", "PATCH", "POST", "PUT"),
scope=d.any_of("namespace", "cluster"),
threshold=1,
),
]
HEALTH_PANELS = [
d.simple_graph(
"Unhealthy nodes",
"sum(node_collector_unhealthy_nodes_in_zone) by (zone)",
legend="{{zone}}",
),
d.simple_graph(
"Pod creations",
'sum(irate(apiserver_request_total{verb="POST", resource="pods", subresource=""}[1m])) by (client)',
yAxes=g.single_y_axis(format=g.OPS_FORMAT),