def test_summary(self):
"""Test that we can track summaries in Service303"""
# Add a summary with a label to the regisry
c = Summary('process_max_fds', 'A summary', ['result'], registry=self.registry)
c.labels('success').observe(1.23)
c.labels('failure').observe(2.34)
# Build proto outputs
summary1 = metrics_pb2.Summary(sample_count=1, sample_sum=1.23)
summary2 = metrics_pb2.Summary(sample_count=1, sample_sum=2.34)
metric1 = metrics_pb2.Metric(summary=summary1,
timestamp_ms=1234000)
metric2 = metrics_pb2.Metric(summary=summary2,
timestamp_ms=1234000)
family = metrics_pb2.MetricFamily(
name=str(metricsd_pb2.process_max_fds),
type=metrics_pb2.SUMMARY)
metric1.label.add(
name=str(metricsd_pb2.result),
value='success')
metric2.label.add(
name=str(metricsd_pb2.result),
value='failure')
family.metric.extend([metric1, metric2])
with unittest.mock.patch('time.time') as mock_time:
mock_time.side_effect = lambda: 1234
self.assertCountEqual(list(metrics_export.get_metrics(self.registry))[0].metric,
family.metric)
def test_histogram(self):
"""Test that we can track histogram in Service303"""
# Add a histogram with a label to the regisry
c = Histogram('process_max_fds',
'A summary', ['result'],
registry=self.registry,
buckets=[0, 2, float('inf')])
c.labels('success').observe(1.23)
c.labels('failure').observe(2.34)
# Build proto outputs
histogram1 = metrics_pb2.Histogram(sample_count=1, sample_sum=1.23)
histogram1.bucket.add(upper_bound=0, cumulative_count=0)
histogram1.bucket.add(upper_bound=2, cumulative_count=1)
histogram1.bucket.add(upper_bound=float('inf'), cumulative_count=1)
histogram2 = metrics_pb2.Histogram(sample_count=1, sample_sum=2.34)
histogram2.bucket.add(upper_bound=0, cumulative_count=0)
histogram2.bucket.add(upper_bound=2, cumulative_count=0)
histogram2.bucket.add(upper_bound=float('inf'), cumulative_count=1)
metric1 = metrics_pb2.Metric(histogram=histogram1,
timestamp_ms=1234000)
metric2 = metrics_pb2.Metric(histogram=histogram2,
timestamp_ms=1234000)
family = metrics_pb2.MetricFamily(name=str(
metricsd_pb2.process_max_fds),
type=metrics_pb2.HISTOGRAM)
metric1.label.add(name=str(metricsd_pb2.result), value='success')
metric2.label.add(name=str(metricsd_pb2.result), value='failure')
family.metric.extend([metric1, metric2])
with unittest.mock.patch('time.time') as mock_time:
mock_time.side_effect = lambda: 1234
self.assertCountEqual(
list(metrics_export.get_metrics(self.registry))[0].metric,
family.metric)
def test_gauge(self):
"""Test that we can track gauges in Service303"""
# Add a gauge with a label to the regisry
c = Gauge('process_max_fds', 'A gauge', ['result'],
registry=self.registry)
# Create two series for value1 and value2
c.labels('success').inc(1.23)
c.labels('failure').inc(2.34)
# Build proto outputs
gauge1 = metrics_pb2.Gauge(value=1.23)
gauge2 = metrics_pb2.Gauge(value=2.34)
metric1 = metrics_pb2.Metric(gauge=gauge1,
timestamp_ms=1234000)
metric2 = metrics_pb2.Metric(gauge=gauge2,
timestamp_ms=1234000)
family = metrics_pb2.MetricFamily(
name=str(metricsd_pb2.process_max_fds),
type=metrics_pb2.GAUGE)
metric1.label.add(
name=str(metricsd_pb2.result),
value='success')
metric2.label.add(
name=str(metricsd_pb2.result),
value='failure')
family.metric.extend([metric1, metric2])
with unittest.mock.patch('time.time') as mock_time:
mock_time.side_effect = lambda: 1234
self.assertCountEqual(list(metrics_export.get_metrics(self.registry))[0].metric,
family.metric)
def encode_counter_gauge(family, timestamp_ms):
"""
Takes a Counter/Gauge family which is a collection of timeseries
samples that share a name (uniquely identified by labels) and yields
equivalent protobufs.
Each timeseries corresponds to a single sample tuple of the format:
(NAME, LABELS, VALUE)
Arguments:
family: a prometheus gauge metric family
timestamp_ms: the timestamp to attach to the samples
Returns:
A Counter or Gauge prometheus MetricFamily protobuf
"""
family_proto = metrics_pb2.MetricFamily()
family_proto.type = \
metrics_pb2.MetricType.Value(family.type.upper())
for sample in family.samples:
metric_proto = metrics_pb2.Metric()
if family_proto.type == metrics_pb2.COUNTER:
metric_proto.counter.value = sample[2]
elif family_proto.type == metrics_pb2.GAUGE:
metric_proto.gauge.value = sample[2]
# Add meta-data to the timeseries
metric_proto.timestamp_ms = timestamp_ms
metric_proto.label.extend(
_convert_labels_to_pb_pairs(sample[1].items()))
# Append metric sample to family
family_proto.metric.extend([metric_proto])
return family_proto
def test_counter(self):
"""Test that we can track counters in Service303"""
# Add a counter with a label to the regisry
process_max_metric_name = 'process_max_fds'
c = Counter(
process_max_metric_name,
'A counter',
['result'],
registry=self.registry,
)
# Create two series for value1 and value2
c.labels('success').inc(1.23)
c.labels('failure').inc(2.34)
# Build proto outputs
counter1 = metrics_pb2.Counter(value=1.23)
counter2 = metrics_pb2.Counter(value=2.34)
metric1 = metrics_pb2.Metric(
counter=counter1,
timestamp_ms=1234000,
)
metric2 = metrics_pb2.Metric(
counter=counter2,
timestamp_ms=1234000,
)
family = metrics_pb2.MetricFamily(
name=process_max_metric_name,
type=metrics_pb2.COUNTER,
)
metric1.label.add(
name='result',
value='success',
)
metric2.label.add(
name='result',
value='failure',
)
family.metric.extend([metric1, metric2])
with unittest.mock.patch('time.time') as mock_time:
mock_time.side_effect = lambda: 1234
self.assertCountEqual(
list(metrics_export.get_metrics(self.registry))[0].metric,
family.metric,
)
def _gauge_to_proto(
metric: prometheus_client.core.Metric, ) -> metrics_pb2.MetricFamily:
ret = metrics_pb2.MetricFamily(name=metric.name, type=metrics_pb2.GAUGE)
for sample in metric.samples:
(_, labels, value, *_) = sample
met = metrics_pb2.Metric(gauge=metrics_pb2.Gauge(value=value))
for key in labels:
met.label.add(name=key, value=labels[key])
ret.metric.extend([met])
return ret
def _counter_to_proto(
metric: prometheus_client.core.Metric, ) -> metrics_pb2.MetricFamily:
ret = metrics_pb2.MetricFamily(name=metric.name, type=metrics_pb2.COUNTER)
for sample in metric.samples:
counter = metrics_pb2.Counter(value=sample[2])
met = metrics_pb2.Metric(counter=counter)
for key in sample[1]:
met.label.add(name=key, value=sample[1][key])
ret.metric.extend([met])
return ret
def _untyped_to_proto(
metric: prometheus_client.core.Metric, ) -> metrics_pb2.MetricFamily:
ret = metrics_pb2.MetricFamily(name=metric.name, type=metrics_pb2.UNTYPED)
for sample in metric.samples:
(_, labels, value, *_) = sample
new_untyped = metrics_pb2.Untyped(value=value)
met = metrics_pb2.Metric(untyped=new_untyped)
for key in labels:
met.label.add(name=key, value=labels[key])
ret.metric.extend([met])
return ret
def _get_collect_success_metric(service_name, gw_up):
"""
Get a the service_metrics_collected metric for a service which is either 0
if the collection was unsuccessful or 1 if it was successful
"""
family_proto = metrics_pb2.MetricFamily(
type=metrics_pb2.GAUGE,
name=str(metricsd_pb2.service_metrics_collected),
)
metric_proto = metrics_pb2.Metric(timestamp_ms=int(time.time() * 1000))
metric_proto.gauge.value = 1 if gw_up else 0
metric_proto.label.add(name="service", value=service_name)
family_proto.metric.extend([metric_proto])
return family_proto
def _get_uptime_metric(service_name, start_time):
"""
Returns a metric for service uptime using the prometheus exposed
process_start_time_seconds
"""
# Metrics collection should never fail, so only log exceptions
curr_time = calendar.timegm(time.gmtime())
uptime = curr_time - start_time
family_proto = metrics_pb2.MetricFamily(
type=metrics_pb2.GAUGE,
name=str(metricsd_pb2.process_uptime_seconds),
)
metric_proto = metrics_pb2.Metric(timestamp_ms=int(time.time() * 1000))
metric_proto.gauge.value = uptime
metric_proto.label.add(name="service", value=service_name)
family_proto.metric.extend([metric_proto])
return family_proto
def _summary_to_proto(
metric: prometheus_client.core.Metric,
) -> [metrics_pb2.MetricFamily]:
"""
1. Get metrics by unique labelset ignoring quantile
2. convert to proto separately for each one
"""
family_by_labelset = {}
for sample in metric.samples:
(name, labels, value, *_) = sample
# get real family by checking labels (ignoring quantile)
distinct_labels = frozenset(_remove_label(labels, 'quantile').items())
if distinct_labels not in family_by_labelset:
fam = metrics_pb2.MetricFamily(
name=metric.name,
type=metrics_pb2.SUMMARY,
)
summ = metrics_pb2.Summary(sample_count=0, sample_sum=0)
fam.metric.extend([metrics_pb2.Metric(summary=summ)])
family_by_labelset[distinct_labels] = fam
unique_family = family_by_labelset[distinct_labels]
if str.endswith(name, "_sum"):
unique_family.metric[0].summary.sample_sum = value
elif str.endswith(name, "_count"):
unique_family.metric[0].summary.sample_count = int(value)
elif 'quantile' in labels:
unique_family.metric[0].summary.quantile.extend([
metrics_pb2.Quantile(
quantile=float(labels['quantile']),
value=value,
),
])
# Add non-quantile labels to all metrics
for labelset in family_by_labelset.keys():
for label in labelset:
family_by_labelset[labelset].metric[0].label.add(
name=label[0],
value=label[1],
)
return list(family_by_labelset.values())
def encode_summary(family, timestamp_ms):
"""
Takes a Summary Metric family which is a collection of timeseries
samples that share a name (uniquely identified by labels) and yields
equivalent protobufs.
Each summary timeseries consists of sample tuples for the count, sum,
and quantiles in the format (NAME,LABELS,VALUE). The NAME is suffixed
with either _count, _sum to indicate count and sum respectively.
Quantile samples will be of the same NAME with quantile label.
Arguments:
family: a prometheus summary metric family
timestamp_ms: the timestamp to attach to the samples
Raises:
ValueError if metric name is not defined in MetricNames protobuf
Returns:
a Summary prometheus MetricFamily protobuf
"""
family_proto = metrics_pb2.MetricFamily()
family_proto.type = metrics_pb2.SUMMARY
metric_protos = {}
# Build a map of each of the summary timeseries from the samples
for sample in family.samples:
quantile = sample[1].pop('quantile', None) # Remove from label set
# Each time series identified by label set excluding the quantile
metric_proto = \
metric_protos.setdefault(
frozenset(sample[1].items()),
metrics_pb2.Metric(),
)
if sample[0].endswith('_count'):
metric_proto.summary.sample_count = int(sample[2])
elif sample[0].endswith('_sum'):
metric_proto.summary.sample_sum = sample[2]
elif quantile:
quantile = metric_proto.summary.quantile.add()
quantile.value = sample[2]
quantile.quantile = _goStringToFloat(quantile)
# Go back and add meta-data to the timeseries
for labels, metric_proto in metric_protos.items():
metric_proto.timestamp_ms = timestamp_ms
metric_proto.label.extend(_convert_labels_to_enums(labels))
# Add it to the family
family_proto.metric.extend([metric_proto])
return family_proto
def _histogram_to_proto(
metric: prometheus_client.core.Metric,
) -> [metrics_pb2.MetricFamily]:
"""
1. Get metrics by unique labelset ignoring quantile
2. convert to proto separately for each one
"""
family_by_labelset = {}
for sample in metric.samples:
(name, labels, value, *_) = sample
# get real family by checking labels (ignoring le)
distinct_labels = frozenset(_remove_label(labels, 'le').items())
if distinct_labels not in family_by_labelset:
fam = metrics_pb2.MetricFamily(
name=metric.name,
type=metrics_pb2.HISTOGRAM,
)
hist = metrics_pb2.Histogram(sample_count=0, sample_sum=0)
fam.metric.extend([metrics_pb2.Metric(histogram=hist)])
family_by_labelset[distinct_labels] = fam
unique_family = family_by_labelset[distinct_labels]
if str.endswith(name, "_sum"):
unique_family.metric[0].histogram.sample_sum = value
elif str.endswith(name, "_count"):
unique_family.metric[0].histogram.sample_count = int(value)
elif 'le' in labels:
unique_family.metric[0].histogram.bucket.extend([
metrics_pb2.Bucket(
upper_bound=float(labels['le']),
cumulative_count=value,
),
])
# Add non-quantile labels to all metrics
for labelset in family_by_labelset.keys():
for label in labelset:
family_by_labelset[labelset].metric[0].label.add(
name=str(label[0]), value=str(label[1]),
)
return list(family_by_labelset.values())
def encode_histogram(family, timestamp_ms):
"""
Takes a Histogram Metric family which is a collection of timeseries
samples that share a name (uniquely identified by labels) and yields
equivalent protobufs.
Each summary timeseries consists of sample tuples for the count, sum,
and quantiles in the format (NAME,LABELS,VALUE). The NAME is suffixed
with either _count, _sum, _buckets to indicate count, sum and buckets
respectively. Bucket samples will also contain a le to indicate its
upper bound.
Arguments:
family: a prometheus histogram metric family
timestamp_ms: the timestamp to attach to the samples
Raises:
ValueError if metric name is not defined in MetricNames protobuf
Returns:
a Histogram prometheus MetricFamily protobuf
"""
family_proto = metrics_pb2.MetricFamily()
family_proto.type = metrics_pb2.HISTOGRAM
metric_protos = {}
for sample in family.samples:
upper_bound = sample[1].pop('le', None) # Remove from label set
metric_proto = \
metric_protos.setdefault(
frozenset(sample[1].items()),
metrics_pb2.Metric(),
)
if sample[0].endswith('_count'):
metric_proto.histogram.sample_count = int(sample[2])
elif sample[0].endswith('_sum'):
metric_proto.histogram.sample_sum = sample[2]
elif sample[0].endswith('_bucket'):
quantile = metric_proto.histogram.bucket.add()
quantile.cumulative_count = int(sample[2])
quantile.upper_bound = _goStringToFloat(upper_bound)
# Go back and add meta-data to the timeseries
for labels, metric_proto in metric_protos.items():
metric_proto.timestamp_ms = timestamp_ms
metric_proto.label.extend(_convert_labels_to_enums(labels))
# Add it to the family
family_proto.metric.extend([metric_proto])
return family_proto