def write_metrics(metrics, histogram_file, counter_file):
from prometheus_client import core
try:
key_func = core._mmap_key
except AttributeError:
# pre 0.4 key format
def key_func(metric_name, name, labelnames, labelvalues):
return json.dumps(
(metric_name, name, tuple(labels), tuple(labels.values())))
histograms = core._MmapedDict(histogram_file)
counters = core._MmapedDict(counter_file)
try:
for metric in metrics:
if metric.type == 'histogram':
sink = histograms
elif metric.type == 'counter':
sink = counters
else:
continue
for sample in metric.samples:
# prometheus_client 0.4+ adds extra fields
name, labels, value = sample[:3]
key = key_func(
metric.name,
name,
tuple(labels),
tuple(labels.values()),
)
sink.write_value(key, value)
finally:
histograms.close()
counters.close()
def write_metrics(metrics, histogram_file, counter_file):
from prometheus_client.core import _MmapedDict
histograms = _MmapedDict(histogram_file)
counters = _MmapedDict(counter_file)
try:
for metric in metrics:
if metric.type == 'histogram':
sink = histograms
elif metric.type == 'counter':
sink = counters
for name, labels, value in metric.samples:
key = json.dumps(
(metric.name, name, tuple(labels), tuple(labels.values())))
sink.write_value(key, value)
finally:
histograms.close()
counters.close()
def write_metrics(metrics, histogram_file, counter_file):
from prometheus_client import core
try:
key_func = core._mmap_key
except AttributeError:
# pre 0.4 key format
def key_func(metric_name, name, labelnames, labelvalues):
return json.dumps(
(metric_name, name, tuple(labels), tuple(labels.values()))
)
histograms = core._MmapedDict(histogram_file)
counters = core._MmapedDict(counter_file)
try:
for metric in metrics:
if metric.type == 'histogram':
sink = histograms
elif metric.type == 'counter':
sink = counters
else:
continue
for sample in metric.samples:
# prometheus_client 0.4+ adds extra fields
name, labels, value = sample[:3]
key = key_func(
metric.name,
name,
tuple(labels),
tuple(labels.values()),
)
sink.write_value(key, value)
finally:
histograms.close()
counters.close()
def collect(files):
"""This almost verbatim from MultiProcessCollector.collect().
The original collects all results in a format designed to be scraped. We
instead need to collect limited results, in a format that can be written
back to disk. To facilitate this, this version of collect() preserves label
ordering, and does not aggregate the histograms.
Specifically, it differs from the original:
1. it takes its files as an argument, rather than hardcoding '*.db'
2. it does not accumulate histograms
3. it preserves label order, to facilitate being inserted back into an mmap
file.
It needs to be kept up to date with changes to prometheus_client as much as
possible, or until changes are landed upstream to allow reuse of collect().
"""
from prometheus_client import core
metrics = {}
for f in files:
if not os.path.exists(f):
continue
# verbatim from here...
parts = os.path.basename(f).split('_')
typ = parts[0]
d = core._MmapedDict(f, read_mode=True)
for key, value in d.read_all_values():
metric_name, name, labelnames, labelvalues = json.loads(key)
metric = metrics.get(metric_name)
if metric is None:
metric = core.Metric(metric_name, 'Multiprocess metric', typ)
metrics[metric_name] = metric
if typ == 'gauge':
pid = parts[2][:-3]
metric._multiprocess_mode = parts[1]
metric.add_sample(
name,
tuple(zip(labelnames, labelvalues)) + (('pid', pid), ),
value,
)
else:
# The duplicates and labels are fixed in the next for.
metric.add_sample(
name,
tuple(zip(labelnames, labelvalues)),
value,
)
d.close()
for metric in metrics.values():
samples = defaultdict(float)
buckets = {}
for name, labels, value in metric.samples:
if metric.type == 'gauge':
without_pid = tuple(l for l in labels if l[0] != 'pid')
if metric._multiprocess_mode == 'min':
current = samples.setdefault((name, without_pid), value)
if value < current:
samples[(name, without_pid)] = value
elif metric._multiprocess_mode == 'max':
current = samples.setdefault((name, without_pid), value)
if value > current:
samples[(name, without_pid)] = value
elif metric._multiprocess_mode == 'livesum':
samples[(name, without_pid)] += value
else: # all/liveall
samples[(name, labels)] = value
elif metric.type == 'histogram':
bucket = tuple(float(l[1]) for l in labels if l[0] == 'le')
if bucket:
# _bucket
without_le = tuple(l for l in labels if l[0] != 'le')
buckets.setdefault(without_le, {})
buckets[without_le].setdefault(bucket[0], 0.0)
buckets[without_le][bucket[0]] += value
else:
# _sum/_count
samples[(name, labels)] += value
else:
# Counter and Summary.
samples[(name, labels)] += value
# end of verbatim copy
# modified to remove accumulation
if metric.type == 'histogram':
for labels, values in buckets.items():
for bucket, value in sorted(values.items()):
key = (
metric.name + '_bucket',
labels + (('le', core._floatToGoString(bucket)), ),
)
samples[key] = value
# Convert to correct sample format.
metric.samples = [
# OrderedDict used instead of dict
(name, OrderedDict(labels), value)
for (name, labels), value in samples.items()
]
return metrics.values()
def test_process_restart(self):
self.d.write_value('abc', 123.0)
self.d.close()
self.d = core._MmapedDict(self.tempfile)
self.assertEqual(123, self.d.read_value('abc'))
self.assertEqual([('abc', 123.0)], list(self.d.read_all_values()))
def setUp(self):
fd, self.tempfile = tempfile.mkstemp()
os.close(fd)
self.d = core._MmapedDict(self.tempfile)
def legacy_collect(files):
"""This almost verbatim from MultiProcessCollector.collect(), pre 0.4.0
The original collects all results in a format designed to be scraped. We
instead need to collect limited results, in a format that can be written
back to disk. To facilitate this, this version of collect() preserves label
ordering, and does not aggregate the histograms.
Specifically, it differs from the original:
1. it takes its files as an argument, rather than hardcoding '*.db'
2. it does not accumulate histograms
3. it preserves label order, to facilitate being inserted back into an mmap
file.
It needs to be kept up to date with changes to prometheus_client as much as
possible, or until changes are landed upstream to allow reuse of collect().
"""
from prometheus_client import core
metrics = {}
for f in files:
if not os.path.exists(f):
continue
# verbatim from here...
parts = os.path.basename(f).split('_')
typ = parts[0]
d = core._MmapedDict(f, read_mode=True)
for key, value in d.read_all_values():
# Note: key format changed in 0.4+
metric_name, name, labelnames, labelvalues = json.loads(key)
metric = metrics.get(metric_name)
if metric is None:
metric = core.Metric(metric_name, 'Multiprocess metric', typ)
metrics[metric_name] = metric
if typ == 'gauge':
pid = parts[2][:-3]
metric._multiprocess_mode = parts[1]
metric.add_sample(
name,
tuple(zip(labelnames, labelvalues)) + (('pid', pid), ),
value,
)
else:
# The duplicates and labels are fixed in the next for.
metric.add_sample(
name,
tuple(zip(labelnames, labelvalues)),
value,
)
d.close()
for metric in metrics.values():
samples = defaultdict(float)
buckets = {}
for name, labels, value in metric.samples:
if metric.type == 'gauge':
without_pid = tuple(l for l in labels if l[0] != 'pid')
if metric._multiprocess_mode == 'min':
current = samples.setdefault((name, without_pid), value)
if value < current:
samples[(name, without_pid)] = value
elif metric._multiprocess_mode == 'max':
current = samples.setdefault((name, without_pid), value)
if value > current:
samples[(name, without_pid)] = value
elif metric._multiprocess_mode == 'livesum':
samples[(name, without_pid)] += value
else: # all/liveall
samples[(name, labels)] = value
elif metric.type == 'histogram':
bucket = tuple(float(l[1]) for l in labels if l[0] == 'le')
if bucket:
# _bucket
without_le = tuple(l for l in labels if l[0] != 'le')
buckets.setdefault(without_le, {})
buckets[without_le].setdefault(bucket[0], 0.0)
buckets[without_le][bucket[0]] += value
else:
# _sum/_count
samples[(name, labels)] += value
else:
# Counter and Summary.
samples[(name, labels)] += value
# end of verbatim copy
# modified to remove accumulation
if metric.type == 'histogram':
for labels, values in buckets.items():
for bucket, value in sorted(values.items()):
key = (
metric.name + '_bucket',
labels + (('le', core._floatToGoString(bucket)),),
)
samples[key] = value
# Convert to correct sample format.
metric.samples = [
# OrderedDict used instead of dict
(name, OrderedDict(labels), value)
for (name, labels), value in samples.items()
]
return metrics.values()