def api_call_latency(title, verb, scope, threshold):
return d.Graph(
title=title,
targets=[
g.Target(expr=str(threshold), legendFormat="threshold"),
g.Target(
expr=d.one_line(expression % {
"verb": verb,
"scope": scope
}),
# TODO(github.com/grafana/grafana/issues/19410): uncomment once fixed
# legendFormat="{{verb}} {{scope}}/{{resource}}",
),
],
yAxes=g.single_y_axis(format=g.SECONDS_FORMAT),
)
def api_call_latency(title, verb, scope, threshold):
return d.Graph(
title=title,
targets=[
g.Target(expr=str(threshold), legendFormat="threshold"),
g.Target(
expr=d.one_line("""
apiserver:apiserver_request_latency_1m:histogram_quantile{
quantile="0.99",
verb=~"%(verb)s",
scope=~"%(scope)s",
resource=~"${resource:regex}s*",
}""" % {
"verb": verb,
"scope": scope
}),
# TODO(github.com/grafana/grafana/issues/19410): uncomment once fixed
# legendFormat="{{verb}} {{scope}}/{{resource}}",
),
],
yAxes=g.single_y_axis(format=g.SECONDS_FORMAT),
)
d.simple_graph("etcd leader",
"etcd_server_is_leader",
legend="{{instance}}"),
d.simple_graph(
"etcd bytes sent",
"rate(etcd_network_client_grpc_sent_bytes_total[1m])",
yAxes=g.single_y_axis(format=g.BYTES_PER_SEC_FORMAT),
legend="{{instance}}",
),
d.simple_graph(
"etcd operations rate",
d.one_line("""
sum(
rate(
etcd_request_duration_seconds_count{
operation=~"${etcd_operation:regex}",
type=~".*(${etcd_type:pipe})"
}[1m]
)
) by (operation, type)
"""),
yAxes=g.single_y_axis(format=g.OPS_FORMAT),
legend="{{operation}} {{type}}",
),
d.simple_graph(
"etcd get latency by type (99th percentile)",
d.one_line("""
histogram_quantile(
0.99,
sum(
rate(
etcd_request_duration_seconds_bucket{
