def test_summary_labels(self): cmf = SummaryMetricFamily('s', 'help', labels=['a']) cmf.add_metric(['b'], count_value=1, sum_value=2) self.custom_collector(cmf) self.assertEqual(1, self.registry.get_sample_value('s_count', {'a': 'b'})) self.assertEqual(2, self.registry.get_sample_value('s_sum', {'a': 'b'}))
def test_summary_quantiles(self): families = text_string_to_metric_families("""# TYPE a summary # HELP a help a_count 1 a_sum 2 a{quantile="0.5"} 0.7 """) # The Python client doesn't support quantiles, but we # still need to be able to parse them. metric_family = SummaryMetricFamily("a", "help", count_value=1, sum_value=2) metric_family.add_sample("a", {"quantile": "0.5"}, 0.7) self.assertEqual([metric_family], list(families))
def test_simple_summary(self): families = text_string_to_metric_families("""# TYPE a summary # HELP a help a_count 1 a_sum 2 """) summary = SummaryMetricFamily("a", "help", count_value=1, sum_value=2) self.assertEqual([summary], list(families))
def test_labelnames(self): cmf = UntypedMetricFamily('u', 'help', labels=iter(['a'])) self.assertEqual(('a', ), cmf._labelnames) cmf = CounterMetricFamily('c_total', 'help', labels=iter(['a'])) self.assertEqual(('a', ), cmf._labelnames) gmf = GaugeMetricFamily('g', 'help', labels=iter(['a'])) self.assertEqual(('a', ), gmf._labelnames) smf = SummaryMetricFamily('s', 'help', labels=iter(['a'])) self.assertEqual(('a', ), smf._labelnames) hmf = HistogramMetricFamily('h', 'help', labels=iter(['a'])) self.assertEqual(('a', ), hmf._labelnames)
def _translate_to_prometheus(self, export_record: ExportRecord): prometheus_metric = None label_values = [] label_keys = [] for label_tuple in export_record.labels: label_keys.append(self._sanitize(label_tuple[0])) label_values.append(label_tuple[1]) metric_name = "" if self._prefix != "": metric_name = self._prefix + "_" metric_name += self._sanitize(export_record.instrument.name) description = getattr(export_record.instrument, "description", "") if isinstance(export_record.instrument, Counter): prometheus_metric = CounterMetricFamily(name=metric_name, documentation=description, labels=label_keys) prometheus_metric.add_metric( labels=label_values, value=export_record.aggregator.checkpoint) elif isinstance(export_record.instrument, Observer): prometheus_metric = GaugeMetricFamily(name=metric_name, documentation=description, labels=label_keys) prometheus_metric.add_metric( labels=label_values, value=export_record.aggregator.checkpoint.last) # TODO: Add support for histograms when supported in OT elif isinstance(export_record.instrument, ValueRecorder): value = export_record.aggregator.checkpoint if isinstance(export_record.aggregator, MinMaxSumCountAggregator): prometheus_metric = SummaryMetricFamily( name=metric_name, documentation=description, labels=label_keys, ) prometheus_metric.add_metric( labels=label_values, count_value=value.count, sum_value=value.sum, ) else: prometheus_metric = UnknownMetricFamily( name=metric_name, documentation=description, labels=label_keys, ) prometheus_metric.add_metric(labels=label_values, value=value) else: logger.warning("Unsupported metric type. %s", type(export_record.instrument)) return prometheus_metric
def collect(self): response = requests.get(args.uri) json = response.json() metrics = json['metrics'] # iterate over all metrics for k in metrics: underscores = re.sub('\.|-|\s', '_', k).lower() if metrics[k]['type'] == 'timer': # we have a timer, expose as a Prometheus Summary underscores = underscores + '_' + metrics[k]['duration_unit'] summary = SummaryMetricFamily(underscores, 'libmedida metric type: ' + metrics[k]['type'], count_value=metrics[k]['count'], sum_value=(metrics[k]['mean'] * metrics[k]['count'])) # add stellar-core calculated quantiles to our summary summary.add_sample(underscores, labels={'quantile':'0.75'}, value=metrics[k]['75%']) summary.add_sample(underscores, labels={'quantile':'0.99'}, value=metrics[k]['99%']) yield summary elif metrics[k]['type'] == 'counter': # we have a counter, this is a Prometheus Gauge yield GaugeMetricFamily(underscores, 'libmedida metric type: ' + metrics[k]['type'], value=metrics[k]['count']) elif metrics[k]['type'] == 'meter': # we have a meter, this is a Prometheus Counter yield CounterMetricFamily(underscores, 'libmedida metric type: ' + metrics[k]['type'], value=metrics[k]['count'])
def collect(self): out = urlopen("http://localhost:8500/v1/agent/metrics").read() metrics = json.loads(out.decode("utf-8")) for g in metrics["Gauges"]: yield GaugeMetricFamily(sanitise_name(g["Name"]), "Consul metric " + g["Name"], g["Value"]) for c in metrics["Counters"]: yield CounterMetricFamily(sanitise_name(c["Name"]) + "_total", "Consul metric " + c["Name"], c["Count"]) for s in metrics["Samples"]: yield SummaryMetricFamily(sanitise_name(s["Name"]) + "_seconds", "Consul metric " + s["Name"], count_value=c["Count"], sum_value=s["Sum"] / 1000)
def collect(self): yield SummaryMetricFamily('summary', 'This is simple summary', labels={'name': 'horizon.stellar.org'}) log.info('current_data.items(): %s' % current_data.items()) for k, v in current_data.items(): yield CounterMetricFamily(k, 'stellar base metric values', value=float(v)) log.info('current_payment_detail.items(): %s' % current_payment_detail.items()) for asset, asset_data in current_payment_detail.items(): summ = CounterMetricFamily('sum_payment', 'stellar payment metric values', labels=['sum_payment']) summ.add_metric(asset, asset_data['sum']) yield summ yield CounterMetricFamily('nb_payment', 'stellar payment metric values', value=float(asset_data['nm'])) metric = GaugeMetricFamily( 'large_native_payment_detail', 'large native stellar payment metric values', value=7) for from_addr, amount_by_dest in current_large_native_payment_detail.items( ): for to_addr, amount in amount_by_dest.items(): metric.add_sample('sum_large_native_payment', value=amount, labels={ 'from_addr': from_addr, 'to_addr': to_addr }) yield metric
def test_summary(self): self.custom_collector( SummaryMetricFamily('s', 'help', count_value=1, sum_value=2)) self.assertEqual(1, self.registry.get_sample_value('s_count', {})) self.assertEqual(2, self.registry.get_sample_value('s_sum', {}))
def collect(self): # TODO handle missing labels, probably return 500? labels = self.get_labels() labels_p75 = labels.copy() # Work on copy of labels variable to avoid other metrics getting quantile label labels_p75.update({'quantile': '0.75'}) labels_p99 = labels.copy() # Work on copy of labels variable to avoid other metrics getting quantile label labels_p99.update({'quantile': '0.99'}) response = requests.get(args.uri) metrics = response.json()['metrics'] # iterate over all metrics for k in metrics: metric_name = re.sub('\.|-|\s', '_', k).lower() metric_name = 'stellar_core_' + metric_name if metrics[k]['type'] == 'timer': # we have a timer, expose as a Prometheus Summary # we convert stellar-core time units to seconds, as per Prometheus best practices metric_name = metric_name + '_seconds' if 'sum' in metrics[k]: # use libmedida sum value total_duration = metrics[k]['sum'] else: # compute sum value total_duration = (metrics[k]['mean'] * metrics[k]['count']) summary = SummaryMetricFamily(metric_name, 'libmedida metric type: ' + metrics[k]['type'], labels=labels.keys()) summary.add_metric(labels.values(), count_value=metrics[k]['count'], sum_value=(duration_to_seconds(total_duration, metrics[k]['duration_unit']))) # add stellar-core calculated quantiles to our summary summary.add_sample(metric_name, labels=labels_p75, value=(duration_to_seconds(metrics[k]['75%'], metrics[k]['duration_unit']))) summary.add_sample(metric_name, labels=labels_p99, value=(duration_to_seconds(metrics[k]['99%'], metrics[k]['duration_unit']))) yield summary elif metrics[k]['type'] == 'counter': # we have a counter, this is a Prometheus Gauge g = GaugeMetricFamily(metric_name, 'libmedida metric type: ' + metrics[k]['type'], labels=labels.keys()) g.add_metric(labels.values(), metrics[k]['count']) yield g elif metrics[k]['type'] == 'meter': # we have a meter, this is a Prometheus Counter c = CounterMetricFamily(metric_name, 'libmedida metric type: ' + metrics[k]['type'], labels=labels.keys()) c.add_metric(labels.values(), metrics[k]['count']) yield c # Export metrics from the info endpoint response = requests.get(args.info_uri) info = response.json()['info'] if not all([i in info for i in self.info_keys]): print('WARNING: info endpoint did not return all required fields') return # Ledger metrics for core_name, prom_name in self.ledger_metrics.items(): g = GaugeMetricFamily('stellar_core_ledger_{}'.format(prom_name), 'Stellar core ledger metric name: {}'.format(core_name), labels=labels.keys()) g.add_metric(labels.values(), info['ledger'][core_name]) yield g # Quorum metrics are reported under dynamic name for example: # "quorum" : { # "758110" : { # "agree" : 3, tmp = info['quorum'].values()[0] for metric in self.quorum_metrics: g = GaugeMetricFamily('stellar_core_quorum_{}'.format(metric), 'Stellar core quorum metric: {}'.format(metric), labels=labels.keys()) g.add_metric(labels.values(), tmp[metric]) yield g # Peers metrics g = GaugeMetricFamily('stellar_core_peers_authenticated_count', 'Stellar core authenticated_count count', labels=labels.keys()) g.add_metric(labels.values(), info['peers']['authenticated_count']) yield g g = GaugeMetricFamily('stellar_core_peers_pending_count', 'Stellar core pending_count count', labels=labels.keys()) g.add_metric(labels.values(), info['peers']['pending_count']) yield g g = GaugeMetricFamily('stellar_core_protocol_version', 'Stellar core protocol_version', labels=labels.keys()) g.add_metric(labels.values(), info['protocol_version']) yield g g = GaugeMetricFamily('stellar_core_synced', 'Stellar core sync status', labels=labels.keys()) if info['state'] == 'Synced!': g.add_metric(labels.values(), 1) else: g.add_metric(labels.values(), 0) yield g g = GaugeMetricFamily('stellar_core_started_on', 'Stellar core start time in epoch', labels=labels.keys()) date = datetime.strptime(info['startedOn'], "%Y-%m-%dT%H:%M:%SZ") g.add_metric(labels.values(), int(date.strftime('%s'))) yield g
def collect(self): config = self._config info_request = urllib2.Request(config['json_data_url']) if config['basic_auth_user'] != "": info_request.add_header( "Authorization", "Basic %s" % base64.standard_b64encode('%s:%s' % (config['basic_auth_user'], config['basic_auth_password']))) if config['host'] != "": info_request.add_header("Host", config['host']) result = json.loads(urllib2.urlopen(info_request).read()) result_tree = Tree(result) for metric_config in config['metrics']: metric_path = metric_config['path'] metric_type = metric_config['type'] values = result_tree.execute(metric_path) if metric_type == "gauge": for current_gauge_metric_attribute in tuple(values): current_gauge_metric_attribute_value = result_tree.execute( metric_path + ".*['" + current_gauge_metric_attribute + "'].value") if isinstance(current_gauge_metric_attribute_value, float): metric_name = "{}_{}".format( config['metric_name_prefix'], current_gauge_metric_attribute.replace( '.', '_').replace('-', '_').lower()) metric = GaugeMetricFamily( metric_name, '', value=current_gauge_metric_attribute_value) yield metric if metric_type == "meters": exception_count_map = {} for current_meters_metric_attribute in tuple(values): count_value = result_tree.execute( metric_path + ".*['" + current_meters_metric_attribute + "'].count") exception_count_map[ current_meters_metric_attribute] = count_value for key in exception_count_map.keys(): suffix = key.replace('.', '_').replace('-', '_').lower() label = "type" formatted_key = key if ".exception" in key: suffix = "exceptions" label = "exceptionType" formatted_key = key.replace(".exception", "") elif "metrics." in key: suffix = "log" label = "level" formatted_key = key.replace("metrics.", "") elif "-responses" in key: suffix = "response" label = "status" formatted_key = key.replace("-responses", "").replace( "org.eclipse.jetty.webapp.WebAppContext.", "") metric_name = "{}_{}".format(config['metric_name_prefix'], suffix) metric = SummaryMetricFamily(metric_name, '', labels=[label]) metric.add_metric([formatted_key], count_value=exception_count_map.get(key), sum_value=exception_count_map.get(key)) yield metric