def test_summary_labels(self):
cmf = SummaryMetricFamily('s', 'help', labels=['a'])
cmf.add_metric(['b'], count_value=1, sum_value=2)
self.custom_collector(cmf)
self.assertEqual(1,
self.registry.get_sample_value('s_count', {'a': 'b'}))
self.assertEqual(2,
self.registry.get_sample_value('s_sum', {'a': 'b'}))
def test_summary_quantiles(self):
families = text_string_to_metric_families("""# TYPE a summary
# HELP a help
a_count 1
a_sum 2
a{quantile="0.5"} 0.7
""")
# The Python client doesn't support quantiles, but we
# still need to be able to parse them.
metric_family = SummaryMetricFamily("a", "help", count_value=1, sum_value=2)
metric_family.add_sample("a", {"quantile": "0.5"}, 0.7)
self.assertEqual([metric_family], list(families))
def test_simple_summary(self):
families = text_string_to_metric_families("""# TYPE a summary
# HELP a help
a_count 1
a_sum 2
""")
summary = SummaryMetricFamily("a", "help", count_value=1, sum_value=2)
self.assertEqual([summary], list(families))
def test_labelnames(self):
cmf = UntypedMetricFamily('u', 'help', labels=iter(['a']))
self.assertEqual(('a', ), cmf._labelnames)
cmf = CounterMetricFamily('c_total', 'help', labels=iter(['a']))
self.assertEqual(('a', ), cmf._labelnames)
gmf = GaugeMetricFamily('g', 'help', labels=iter(['a']))
self.assertEqual(('a', ), gmf._labelnames)
smf = SummaryMetricFamily('s', 'help', labels=iter(['a']))
self.assertEqual(('a', ), smf._labelnames)
hmf = HistogramMetricFamily('h', 'help', labels=iter(['a']))
self.assertEqual(('a', ), hmf._labelnames)
def _translate_to_prometheus(self, export_record: ExportRecord):
prometheus_metric = None
label_values = []
label_keys = []
for label_tuple in export_record.labels:
label_keys.append(self._sanitize(label_tuple[0]))
label_values.append(label_tuple[1])
metric_name = ""
if self._prefix != "":
metric_name = self._prefix + "_"
metric_name += self._sanitize(export_record.instrument.name)
description = getattr(export_record.instrument, "description", "")
if isinstance(export_record.instrument, Counter):
prometheus_metric = CounterMetricFamily(name=metric_name,
documentation=description,
labels=label_keys)
prometheus_metric.add_metric(
labels=label_values, value=export_record.aggregator.checkpoint)
elif isinstance(export_record.instrument, Observer):
prometheus_metric = GaugeMetricFamily(name=metric_name,
documentation=description,
labels=label_keys)
prometheus_metric.add_metric(
labels=label_values,
value=export_record.aggregator.checkpoint.last)
# TODO: Add support for histograms when supported in OT
elif isinstance(export_record.instrument, ValueRecorder):
value = export_record.aggregator.checkpoint
if isinstance(export_record.aggregator, MinMaxSumCountAggregator):
prometheus_metric = SummaryMetricFamily(
name=metric_name,
documentation=description,
labels=label_keys,
)
prometheus_metric.add_metric(
labels=label_values,
count_value=value.count,
sum_value=value.sum,
)
else:
prometheus_metric = UnknownMetricFamily(
name=metric_name,
documentation=description,
labels=label_keys,
)
prometheus_metric.add_metric(labels=label_values, value=value)
else:
logger.warning("Unsupported metric type. %s",
type(export_record.instrument))
return prometheus_metric
def collect(self):
response = requests.get(args.uri)
json = response.json()
metrics = json['metrics']
# iterate over all metrics
for k in metrics:
underscores = re.sub('\.|-|\s', '_', k).lower()
if metrics[k]['type'] == 'timer':
# we have a timer, expose as a Prometheus Summary
underscores = underscores + '_' + metrics[k]['duration_unit']
summary = SummaryMetricFamily(underscores, 'libmedida metric type: ' + metrics[k]['type'], count_value=metrics[k]['count'], sum_value=(metrics[k]['mean'] * metrics[k]['count']))
# add stellar-core calculated quantiles to our summary
summary.add_sample(underscores, labels={'quantile':'0.75'}, value=metrics[k]['75%'])
summary.add_sample(underscores, labels={'quantile':'0.99'}, value=metrics[k]['99%'])
yield summary
elif metrics[k]['type'] == 'counter':
# we have a counter, this is a Prometheus Gauge
yield GaugeMetricFamily(underscores, 'libmedida metric type: ' + metrics[k]['type'], value=metrics[k]['count'])
elif metrics[k]['type'] == 'meter':
# we have a meter, this is a Prometheus Counter
yield CounterMetricFamily(underscores, 'libmedida metric type: ' + metrics[k]['type'], value=metrics[k]['count'])
def collect(self):
out = urlopen("http://localhost:8500/v1/agent/metrics").read()
metrics = json.loads(out.decode("utf-8"))
for g in metrics["Gauges"]:
yield GaugeMetricFamily(sanitise_name(g["Name"]),
"Consul metric " + g["Name"], g["Value"])
for c in metrics["Counters"]:
yield CounterMetricFamily(sanitise_name(c["Name"]) + "_total",
"Consul metric " + c["Name"], c["Count"])
for s in metrics["Samples"]:
yield SummaryMetricFamily(sanitise_name(s["Name"]) + "_seconds",
"Consul metric " + s["Name"],
count_value=c["Count"], sum_value=s["Sum"] / 1000)
def collect(self):
yield SummaryMetricFamily('summary',
'This is simple summary',
labels={'name': 'horizon.stellar.org'})
log.info('current_data.items(): %s' % current_data.items())
for k, v in current_data.items():
yield CounterMetricFamily(k,
'stellar base metric values',
value=float(v))
log.info('current_payment_detail.items(): %s' %
current_payment_detail.items())
for asset, asset_data in current_payment_detail.items():
summ = CounterMetricFamily('sum_payment',
'stellar payment metric values',
labels=['sum_payment'])
summ.add_metric(asset, asset_data['sum'])
yield summ
yield CounterMetricFamily('nb_payment',
'stellar payment metric values',
value=float(asset_data['nm']))
metric = GaugeMetricFamily(
'large_native_payment_detail',
'large native stellar payment metric values',
value=7)
for from_addr, amount_by_dest in current_large_native_payment_detail.items(
):
for to_addr, amount in amount_by_dest.items():
metric.add_sample('sum_large_native_payment',
value=amount,
labels={
'from_addr': from_addr,
'to_addr': to_addr
})
yield metric
def test_summary(self):
self.custom_collector(
SummaryMetricFamily('s', 'help', count_value=1, sum_value=2))
self.assertEqual(1, self.registry.get_sample_value('s_count', {}))
self.assertEqual(2, self.registry.get_sample_value('s_sum', {}))
def collect(self):
# TODO handle missing labels, probably return 500?
labels = self.get_labels()
labels_p75 = labels.copy() # Work on copy of labels variable to avoid other metrics getting quantile label
labels_p75.update({'quantile': '0.75'})
labels_p99 = labels.copy() # Work on copy of labels variable to avoid other metrics getting quantile label
labels_p99.update({'quantile': '0.99'})
response = requests.get(args.uri)
metrics = response.json()['metrics']
# iterate over all metrics
for k in metrics:
metric_name = re.sub('\.|-|\s', '_', k).lower()
metric_name = 'stellar_core_' + metric_name
if metrics[k]['type'] == 'timer':
# we have a timer, expose as a Prometheus Summary
# we convert stellar-core time units to seconds, as per Prometheus best practices
metric_name = metric_name + '_seconds'
if 'sum' in metrics[k]:
# use libmedida sum value
total_duration = metrics[k]['sum']
else:
# compute sum value
total_duration = (metrics[k]['mean'] * metrics[k]['count'])
summary = SummaryMetricFamily(metric_name, 'libmedida metric type: ' + metrics[k]['type'], labels=labels.keys())
summary.add_metric(labels.values(), count_value=metrics[k]['count'],
sum_value=(duration_to_seconds(total_duration, metrics[k]['duration_unit'])))
# add stellar-core calculated quantiles to our summary
summary.add_sample(metric_name, labels=labels_p75,
value=(duration_to_seconds(metrics[k]['75%'], metrics[k]['duration_unit'])))
summary.add_sample(metric_name, labels=labels_p99,
value=(duration_to_seconds(metrics[k]['99%'], metrics[k]['duration_unit'])))
yield summary
elif metrics[k]['type'] == 'counter':
# we have a counter, this is a Prometheus Gauge
g = GaugeMetricFamily(metric_name, 'libmedida metric type: ' + metrics[k]['type'], labels=labels.keys())
g.add_metric(labels.values(), metrics[k]['count'])
yield g
elif metrics[k]['type'] == 'meter':
# we have a meter, this is a Prometheus Counter
c = CounterMetricFamily(metric_name, 'libmedida metric type: ' + metrics[k]['type'], labels=labels.keys())
c.add_metric(labels.values(), metrics[k]['count'])
yield c
# Export metrics from the info endpoint
response = requests.get(args.info_uri)
info = response.json()['info']
if not all([i in info for i in self.info_keys]):
print('WARNING: info endpoint did not return all required fields')
return
# Ledger metrics
for core_name, prom_name in self.ledger_metrics.items():
g = GaugeMetricFamily('stellar_core_ledger_{}'.format(prom_name),
'Stellar core ledger metric name: {}'.format(core_name),
labels=labels.keys())
g.add_metric(labels.values(), info['ledger'][core_name])
yield g
# Quorum metrics are reported under dynamic name for example:
# "quorum" : {
# "758110" : {
# "agree" : 3,
tmp = info['quorum'].values()[0]
for metric in self.quorum_metrics:
g = GaugeMetricFamily('stellar_core_quorum_{}'.format(metric),
'Stellar core quorum metric: {}'.format(metric),
labels=labels.keys())
g.add_metric(labels.values(), tmp[metric])
yield g
# Peers metrics
g = GaugeMetricFamily('stellar_core_peers_authenticated_count',
'Stellar core authenticated_count count',
labels=labels.keys())
g.add_metric(labels.values(), info['peers']['authenticated_count'])
yield g
g = GaugeMetricFamily('stellar_core_peers_pending_count',
'Stellar core pending_count count',
labels=labels.keys())
g.add_metric(labels.values(), info['peers']['pending_count'])
yield g
g = GaugeMetricFamily('stellar_core_protocol_version',
'Stellar core protocol_version',
labels=labels.keys())
g.add_metric(labels.values(), info['protocol_version'])
yield g
g = GaugeMetricFamily('stellar_core_synced', 'Stellar core sync status', labels=labels.keys())
if info['state'] == 'Synced!':
g.add_metric(labels.values(), 1)
else:
g.add_metric(labels.values(), 0)
yield g
g = GaugeMetricFamily('stellar_core_started_on', 'Stellar core start time in epoch', labels=labels.keys())
date = datetime.strptime(info['startedOn'], "%Y-%m-%dT%H:%M:%SZ")
g.add_metric(labels.values(), int(date.strftime('%s')))
yield g
def collect(self):
config = self._config
info_request = urllib2.Request(config['json_data_url'])
if config['basic_auth_user'] != "":
info_request.add_header(
"Authorization", "Basic %s" %
base64.standard_b64encode('%s:%s' %
(config['basic_auth_user'],
config['basic_auth_password'])))
if config['host'] != "":
info_request.add_header("Host", config['host'])
result = json.loads(urllib2.urlopen(info_request).read())
result_tree = Tree(result)
for metric_config in config['metrics']:
metric_path = metric_config['path']
metric_type = metric_config['type']
values = result_tree.execute(metric_path)
if metric_type == "gauge":
for current_gauge_metric_attribute in tuple(values):
current_gauge_metric_attribute_value = result_tree.execute(
metric_path + ".*['" + current_gauge_metric_attribute +
"'].value")
if isinstance(current_gauge_metric_attribute_value, float):
metric_name = "{}_{}".format(
config['metric_name_prefix'],
current_gauge_metric_attribute.replace(
'.', '_').replace('-', '_').lower())
metric = GaugeMetricFamily(
metric_name,
'',
value=current_gauge_metric_attribute_value)
yield metric
if metric_type == "meters":
exception_count_map = {}
for current_meters_metric_attribute in tuple(values):
count_value = result_tree.execute(
metric_path + ".*['" +
current_meters_metric_attribute + "'].count")
exception_count_map[
current_meters_metric_attribute] = count_value
for key in exception_count_map.keys():
suffix = key.replace('.', '_').replace('-', '_').lower()
label = "type"
formatted_key = key
if ".exception" in key:
suffix = "exceptions"
label = "exceptionType"
formatted_key = key.replace(".exception", "")
elif "metrics." in key:
suffix = "log"
label = "level"
formatted_key = key.replace("metrics.", "")
elif "-responses" in key:
suffix = "response"
label = "status"
formatted_key = key.replace("-responses", "").replace(
"org.eclipse.jetty.webapp.WebAppContext.", "")
metric_name = "{}_{}".format(config['metric_name_prefix'],
suffix)
metric = SummaryMetricFamily(metric_name,
'',
labels=[label])
metric.add_metric([formatted_key],
count_value=exception_count_map.get(key),
sum_value=exception_count_map.get(key))
yield metric
