def test_prometheus_metric(client):
response = client.get('/_status/test/prometheus',
environ_overrides={'REMOTE_ADDR': b'127.0.0.1'})
assert response.status_code == 200
response = client.get('/_status/metrics',
environ_overrides={'REMOTE_ADDR': b'127.0.0.1'})
assert response.status_code == 200
output = response.data.decode('utf8')
name = counter_name('test_total')
assert '# HELP {} Multiprocess metric\n'.format(name) in output
assert '# TYPE {} counter'.format(name) in output
assert '{} 1.0'.format(name) in output
def test_prometheus_metric():
client = get_client()
response = client.get('/_status/test/prometheus',
environ_overrides={'REMOTE_ADDR': b'127.0.0.1'})
assert response.status_code == 200
response = client.get('/_status/metrics',
environ_overrides={'REMOTE_ADDR': b'127.0.0.1'})
assert response.status_code == 200
output = response.data.decode('utf8')
name = counter_name('test_total')
assert '# HELP {} Multiprocess metric\n'.format(name) in output
assert '# TYPE {} counter'.format(name) in output
assert '{} 1.0'.format(name) in output
def test_gunicorn_prometheus_cleanup(caplog):
caplog.set_level(logging.INFO)
app = __name__ + ':counter_app'
workers = 8
server = GunicornProcess(
app, args=['--worker-class=sync', '-w', str(workers)])
def increment(n):
for i in range(n):
requests.get(server.url('/_status/test/prometheus'))
def files(pid):
pid = str(pid)
pid_files = set()
archives = set()
for path in os.listdir(os.environ['prometheus_multiproc_dir']):
# ignore master pids
if pid in path:
continue
if '_archive.db' in path:
archives.add(path)
else:
pid_files.add(path)
return archives, pid_files
def stats():
return requests.get(server.url('/_status/metrics')).text
name = counter_name('test_total')
valid_archives = set(['counter_archive.db', 'histogram_archive.db'])
sleep_factor = 1
if os.environ.get('CI') == 'true':
# travis is slow
sleep_factor = 20
with server:
# forking can be really slow on travis, so make sure *all* the workers
# have had time to spin up before running the test
time.sleep(1 * sleep_factor)
increment(2000)
archives, pid_files_1 = files(server.ps.pid)
assert len(archives) == 0
# different number of files depending on prometheus_client version
# so we assert against 1x or 2x workers
assert len(pid_files_1) in (workers, 2 * workers)
assert name + ' 2000.0' in stats()
os.kill(server.ps.pid, signal.SIGHUP)
time.sleep(2 * sleep_factor)
archives, pid_files_2 = files(server.ps.pid)
assert archives == valid_archives
assert pid_files_1.isdisjoint(pid_files_2)
assert name + ' 2000.0' in stats()
increment(2000)
assert name + ' 4000.0' in stats()
archives, pid_files_3 = files(server.ps.pid)
assert archives == valid_archives
assert len(pid_files_3) in (workers, 2 * workers)
os.kill(server.ps.pid, signal.SIGHUP)
time.sleep(2 * sleep_factor)
archives, pid_files_4 = files(server.ps.pid)
assert archives == valid_archives
assert pid_files_3.isdisjoint(pid_files_4)
assert name + ' 4000.0' in stats()
def test_prometheus_cleanup(registry): # NOQA
pid = 1
def getpid():
return pid
talisker.testing.reset_prometheus(getpid)
histogram = talisker.metrics.Histogram(
name='histogram',
documentation='test histogram',
labelnames=['foo', 'bar', 'baz'],
statsd='{name}.{label}',
registry=registry,
)
counter = talisker.metrics.Counter(
name='counter',
documentation='test counter',
labelnames=['foo', 'bar', 'baz'],
statsd='{name}.{label}',
registry=registry,
)
from prometheus_client.multiprocess import MultiProcessCollector
collector = MultiProcessCollector(registry)
labels = {'foo': 'foo', 'bar': 'bar', 'baz': 'baz'}
def collect():
return {m.name: m for m in collector.collect()}
def files():
return list(sorted(os.listdir(os.environ['prometheus_multiproc_dir'])))
counter.inc(1, **labels)
histogram.observe(0.5, **labels)
histogram.observe(2.5, **labels)
assert files() == [
'counter_1.db',
'histogram_1.db',
]
before = collect()
prometheus.prometheus_cleanup_worker(pid)
after = collect()
assert files() == [
'counter_archive.db',
'histogram_archive.db',
]
assert before == after
# magic!
pid += 1
# new worker, create some new metrics, check they are all combined
counter.inc(2, **labels)
histogram.observe(0.5, **labels)
histogram.observe(2.5, **labels)
later = collect()
assert files() == [
'counter_2.db',
'counter_archive.db',
'histogram_2.db',
'histogram_archive.db',
]
# check counter is correct
assert later['counter'].samples == [
Sample(counter_name('counter_total'), labels, 3.0),
]
expected_histogram = [
Sample('histogram_bucket', dict(le='0.005', **labels), 0.0),
Sample('histogram_bucket', dict(le='0.01', **labels), 0.0),
Sample('histogram_bucket', dict(le='0.025', **labels), 0.0),
Sample('histogram_bucket', dict(le='0.05', **labels), 0.0),
Sample('histogram_bucket', dict(le='0.075', **labels), 0.0),
Sample('histogram_bucket', dict(le='0.1', **labels), 0.0),
Sample('histogram_bucket', dict(le='0.25', **labels), 0.0),
Sample('histogram_bucket', dict(le='0.5', **labels), 2.0),
Sample('histogram_bucket', dict(le='0.75', **labels), 2.0),
Sample('histogram_bucket', dict(le='1.0', **labels), 2.0),
Sample('histogram_bucket', dict(le='2.5', **labels), 4.0),
Sample('histogram_bucket', dict(le='5.0', **labels), 4.0),
Sample('histogram_bucket', dict(le='7.5', **labels), 4.0),
Sample('histogram_bucket', dict(le='10.0', **labels), 4.0),
Sample('histogram_bucket', dict(le='+Inf', **labels), 4.0),
Sample('histogram_count', labels, 4.0),
Sample('histogram_sum', labels, 6.0),
]
# check histogram is correct
later['histogram'].samples.sort(key=histogram_sorter)
assert later['histogram'].samples == expected_histogram
# check the final files produce the correct numbers
prometheus.prometheus_cleanup_worker(pid)
final = collect()
assert files() == [
'counter_archive.db',
'histogram_archive.db',
]
final['histogram'].samples.sort(key=histogram_sorter)
assert later == final