def run_gunicorn():
config = get_config()
# configure prometheus_client early as possible
if pkg_is_installed('prometheus-client'):
# Early throw-away parsing of gunicorn config, as we need to decide
# whether to enable prometheus multiprocess before we start importing
from gunicorn.app.wsgiapp import WSGIApplication
g_cfg = WSGIApplication().cfg
if g_cfg.workers > 1 or 'prometheus_multiproc_dir' in os.environ:
from talisker.prometheus import setup_prometheus_multiproc
async_workers = ('gevent', 'eventlet')
# must be done before prometheus_client is imported *anywhere*
setup_prometheus_multiproc(
any(n in g_cfg.worker_class_str for n in async_workers)
)
initialise()
import talisker.gunicorn
if pkg_is_installed('celery'):
import talisker.celery
talisker.celery.enable_signals()
app = talisker.gunicorn.TaliskerApplication(
"%(prog)s [OPTIONS] [APP_MODULE]", config.devel, config.debuglog)
clear_contexts()
return app.run()
def __init__(self, app, namespace='_status'):
self.app = app
self.namespace = namespace
self.prefix = '/' + namespace
# Publish /metrics only if prometheus_client is available
if pkg_is_installed('prometheus-client'):
self.urlmap['/metrics'] = 'metrics'
self.urlmap['/test/prometheus'] = 'test_prometheus'
if pkg_is_installed('logging-tree'):
self.urlmap['/debug/logtree'] = 'debug_logtree'
def __init__(self, app, namespace='_status'):
self.app = app
self.namespace = namespace
self.prefix = '/' + namespace
# Publish /metrics only if prometheus_client is available
if pkg_is_installed('prometheus-client'):
self.urlmap['/metrics'] = 'metrics'
self.urlmap['/test/prometheus'] = 'test_prometheus'
if pkg_is_installed('logging-tree'):
self.urlmap['/info/logtree'] = 'logtree'
if pkg_is_installed('psutil'):
self.urlmap['/info/workers'] = 'workers'
if pkg_is_installed('objgraph'):
self.urlmap['/info/objgraph'] = 'objgraph'
def __init__(self, app, namespace='_status'):
self.app = app
self.namespace = namespace
self.prefix = '/' + namespace
# Publish /metrics only if prometheus_client is available
if pkg_is_installed('prometheus-client'):
self.urlmap['/metrics'] = 'metrics'
self.urlmap['/test/prometheus'] = 'test_prometheus'
if pkg_is_installed('logging-tree'):
self.urlmap['/info/logtree'] = 'logtree'
if pkg_is_installed('psutil'):
self.urlmap['/info/workers'] = 'workers'
if pkg_is_installed('objgraph'):
self.urlmap['/info/objgraph'] = 'objgraph'
def test_prometheus(self, request):
"""Increment prometheus metric for testing"""
if not pkg_is_installed('prometheus-client'):
return Response('Not Supported', status=501)
test_counter().inc()
return Response('Incremented test counter')
def metrics(self, request):
"""Endpoint exposing Prometheus metrics"""
if not pkg_is_installed('prometheus-client'):
return Response('Not Supported', status=501)
# Importing this too early would break multiprocess metrics
from prometheus_client import (
CONTENT_TYPE_LATEST,
CollectorRegistry,
REGISTRY,
generate_latest,
multiprocess,
)
if 'prometheus_multiproc_dir' in os.environ:
# prometheus_client is running in multiprocess mode.
# Use a custom registry, as the global one includes custom
# collectors which are not supported in this mode
registry = CollectorRegistry()
multiprocess.MultiProcessCollector(registry)
else:
if request.environ.get('wsgi.multiprocess', False):
return Response(
'Not Supported: running in multiprocess mode but '
'`prometheus_multiproc_dir` envvar not set',
status=501)
# prometheus_client is running in single process mode.
# Use the global registry (includes CPU and RAM collectors)
registry = REGISTRY
with prometheus_lock:
data = generate_latest(registry)
return Response(data, status=200, mimetype=CONTENT_TYPE_LATEST)
def test_prometheus(self, request):
"""Increment prometheus metric for testing"""
if not pkg_is_installed('prometheus-client'):
return Response('Not Supported', status=501)
test_counter().inc()
return Response('Incremented test counter')
def init(self, parser, opts, args):
"""Provide talisker specific default config for gunicorn.
These are just defaults, and can be overridden in cli/config,
but it is helpful to set them here.
"""
cfg = super(TaliskerApplication, self).init(parser, opts, args)
if cfg is None:
cfg = {}
cfg['logger_class'] = GunicornLogger
cfg['pre_request'] = gunicorn_pre_request
if pkg_is_installed('prometheus-client'):
cfg['on_starting'] = gunicorn_on_starting
cfg['child_exit'] = gunicorn_child_exit
# development config
if self._devel:
logger = logging.getLogger(__name__)
logger.debug('devel mode: setting gunicorn devel default config',
extra=DEVEL_SETTINGS)
cfg.update(DEVEL_SETTINGS)
return cfg
def init(self, parser, opts, args):
"""Provide talisker specific default config for gunicorn.
These are just defaults, and can be overridden in cli/config,
but it is helpful to set them here.
"""
cfg = super(TaliskerApplication, self).init(parser, opts, args)
if cfg is None:
cfg = {}
cfg['logger_class'] = GunicornLogger
cfg['pre_request'] = gunicorn_pre_request
# Use pip to find out if prometheus_client is available, as
# importing it here would break multiprocess metrics
if pkg_is_installed('prometheus-client'):
cfg['worker_exit'] = prometheus_multiprocess_worker_exit
# development config
if self._devel:
logger = logging.getLogger(__name__)
logger.debug('devel mode: setting gunicorn devel default config',
extra=DEVEL_SETTINGS)
cfg.update(DEVEL_SETTINGS)
return cfg
def setup_multiproc_dir():
global prometheus_lock
if 'prometheus_multiproc_dir' not in os.environ:
if pkg_is_installed('prometheus-client'):
tmp = tempfile.mkdtemp(prefix='prometheus_multiproc')
os.environ['prometheus_multiproc_dir'] = tmp
if prometheus_lock is None:
initialise_prometheus_lock()
def run_gunicorn():
config = get_config()
# Early throw-away parsing of gunicorn config, as we need to decide
# whether to enable prometheus multiprocess before we start importing
from gunicorn.app.wsgiapp import WSGIApplication
g_cfg = WSGIApplication().cfg
# configure prometheus_client early as possible
if pkg_is_installed('prometheus-client'):
if g_cfg.workers > 1 or 'prometheus_multiproc_dir' in os.environ:
from talisker.prometheus import setup_prometheus_multiproc
async_workers = ('gevent', 'eventlet')
# must be done before prometheus_client is imported *anywhere*
setup_prometheus_multiproc(
any(n in g_cfg.worker_class_str for n in async_workers))
try:
from gunicorn.workers.ggevent import GeventWorker
from talisker.context import enable_gevent_context
except Exception:
pass
else:
if g_cfg.worker_class == GeventWorker:
enable_gevent_context()
try:
from gunicorn.workers.geventlet import EventletWorker
from talisker.context import enable_eventlet_context
except Exception:
pass
else:
if g_cfg.worker_class == EventletWorker:
enable_eventlet_context()
initialise()
import talisker.gunicorn
if pkg_is_installed('celery'):
import talisker.celery
talisker.celery.enable_signals()
app = talisker.gunicorn.TaliskerApplication(
"%(prog)s [OPTIONS] [APP_MODULE]", config.devel, config.debuglog)
clear_context()
return app.run()
def test_prometheus(self, request):
"""Increment prometheus metric for testing"""
if not pkg_is_installed('prometheus-client'):
return Response('Not Supported', status=501)
if not hasattr(self, 'test_counter'):
import prometheus_client
self.test_counter = prometheus_client.Counter('test', 'test')
self.test_counter.inc()
return Response('Incremented test counter')
def _patch_gevent_contextvars():
# gunicorn will attempt to patch contextvars for gevent workers, via
# gevent.monkey.patch_all(). There's a bug in gevent 1.5 that raises when
# on <py3.7 and the backported contextvars module is installed. Workaround
# this by overriding gevent's decision to not patch contextvars.
# Hopefully a temporary abomination.
try:
import gevent.contextvars
except ImportError:
return
if pkg_is_installed('contextvars'):
gevent.contextvars.__implements__ = gevent.contextvars.__all__
def load_config(self):
super(TaliskerApplication, self).load_config()
logger = logging.getLogger(__name__)
# override and warn
if self.cfg.errorlog != '-':
logger.warning(
'ignoring gunicorn errorlog config, talisker logs to stderr',
extra={'errorlog': self.cfg.errorlog})
self.cfg.set('errorlog', '-')
if self.cfg.loglevel.lower() == 'debug' and self._devel:
# user has configured debug level logging
self.cfg.set('loglevel', 'DEBUG')
talisker.logs.enable_debug_log_stderr()
# ensure gunicorn sends debug level messages when needed
if self._debuglog:
self.cfg.set('loglevel', 'DEBUG')
# override and warn
if self.cfg.statsd_host or self.cfg.statsd_prefix:
logger.warning(
'ignoring gunicorn statsd config, as has no effect when '
'using talisker, as it uses STATS_DSN env var',
extra={
'statsd_host': self.cfg.statsd_host,
'statsd_prefix': self.cfg.statsd_prefix
})
self.cfg.set('statsd_host', None)
self.cfg.set('statsd_prefix', None)
# trust but warn
if self.cfg.logger_class is not GunicornLogger:
logger.warning(
'using custom gunicorn logger class - this may break '
'Talisker\'s logging configuration',
extra={'logger_class': self.cfg.logger_class})
# Use pip to find out if prometheus_client is available, as
# importing it here would break multiprocess metrics
if (pkg_is_installed('prometheus-client')
and (self.cfg.workers or 1) > 1):
if 'prometheus_multiproc_dir' not in os.environ:
logger.info('running in multiprocess mode but '
'`prometheus_multiproc_dir` envvar not set')
tmpdir = tempfile.mkdtemp()
os.environ['prometheus_multiproc_dir'] = tmpdir
logger.info('using `%s` for multiprocess prometheus metrics',
os.environ['prometheus_multiproc_dir'])
def run_gunicorn():
# set this early so any imports of prometheus client will be imported
# correctly
if 'prometheus_multiproc_dir' not in os.environ:
if pkg_is_installed('prometheus-client'):
tmp = tempfile.mkdtemp(prefix='prometheus_multiproc')
os.environ['prometheus_multiproc_dir'] = tmp
config = initialise()
import talisker.celery
import talisker.gunicorn
talisker.celery.enable_signals()
app = talisker.gunicorn.TaliskerApplication(
"%(prog)s [OPTIONS] [APP_MODULE]", config['devel'], config['debuglog'])
return app.run()
def test_context_asyncio():
if sys.version_info < (3, 7):
if sys.version_info < (3, 5, 3):
pytest.skip('aiocontextvars does not work in python {}'.format(
sys.version))
elif not pkg_is_installed('aiocontextvars'):
pytest.skip('aiocontextvars not installed')
async def r1():
Context.new()
Context.logging.push(a=1)
Context.track('test', 1.0)
assert Context.logging.flat == {'a': 1}
assert Context.current().tracking['test'].count == 1
await sub()
# changes made by sub should be visible
assert Context.logging.flat == {'a': 2}
assert Context.current().tracking['test'].count == 2
async def sub():
# should be same context as r1
assert Context.logging.flat == {'a': 1}
Context.logging.push(a=2)
Context.track('test', 1.0)
assert Context.logging.flat == {'a': 2}
assert Context.current().tracking['test'].count == 2
async def r2():
# should be a separate context from r1
Context.new()
Context.logging.push(a=3)
Context.track('test', 1.0)
assert Context.logging.flat == {'a': 3}
assert Context.current().tracking['test'].count == 1
# ensure we have no context
loop = asyncio.get_event_loop()
Context.clear()
t1 = loop.create_task(r1())
t2 = loop.create_task(r2())
loop.run_until_complete(asyncio.gather(t1, t2))
def metrics(self, request):
"""Endpoint exposing Prometheus metrics"""
if not pkg_is_installed('prometheus-client'):
return Response('Not Supported', status=501)
from prometheus_client import CONTENT_TYPE_LATEST
from talisker.prometheus import (
collect_metrics,
PrometheusLockTimeout,
)
try:
data = collect_metrics()
except PrometheusLockTimeout:
msg = 'Failed to acquire prometheus lock to collect metrics'
logger.exception(msg)
return Response([msg], status=500, mimetype='text/plain')
return Response(data, status=200, mimetype=CONTENT_TYPE_LATEST)
def metrics(self, request):
"""Endpoint exposing Prometheus metrics"""
if not pkg_is_installed('prometheus-client'):
return Response('Not Supported', status=501)
from prometheus_client import CONTENT_TYPE_LATEST
from talisker.prometheus import (
collect_metrics,
PrometheusLockTimeout,
)
try:
data = collect_metrics()
except PrometheusLockTimeout:
msg = 'Failed to acquire prometheus lock to collect metrics'
logger.exception(msg)
return Response([msg], status=500, mimetype='text/plain')
return Response(data, status=200, mimetype=CONTENT_TYPE_LATEST)
def init(self, parser, opts, args):
"""Provide talisker specific default config for gunicorn.
These are just defaults, and can be overridden in cli/config,
but it is helpful to set them here.
"""
cfg = super(TaliskerApplication, self).init(parser, opts, args)
if cfg is None:
cfg = {}
cfg['logger_class'] = GunicornLogger
cfg['pre_request'] = gunicorn_pre_request
if pkg_is_installed('prometheus-client'):
# only available in gunicorn 19.7+
if hasattr(gunicorn.config, 'ChildExit'):
# we can do the full cleanup
cleanup = talisker.metrics.prometheus_cleanup_worker
server_hook = 'child_exit'
else:
# weaksauce cleanup
from prometheus_client import multiprocess
cleanup = multiprocess.mark_process_dead
server_hook = 'worker_exit'
def prometheus_multiprocess_worker_exit(server, worker):
"Worker cleanup function for multiprocess prometheus_client."
logging.getLogger(__name__).info(
'Performing multiprocess prometheus_client cleanup')
cleanup(worker.pid)
cfg[server_hook] = prometheus_multiprocess_worker_exit
# development config
if self._devel:
logger = logging.getLogger(__name__)
logger.debug('devel mode: setting gunicorn devel default config',
extra=DEVEL_SETTINGS)
cfg.update(DEVEL_SETTINGS)
return cfg
from builtins import * # noqa
from collections import defaultdict, OrderedDict
from contextlib import contextmanager
from multiprocessing import Lock
import errno
import json
import logging
import os
import tempfile
import time
import talisker
from talisker.util import pkg_is_installed, TaliskerVersionException
prometheus_installed = pkg_is_installed('prometheus_client')
if prometheus_installed and prometheus_installed.version in ('0.4.0', '0.4.1'):
raise TaliskerVersionException(
'prometheus_client {} has a critical bug in multiprocess mode, '
'and is not supported in Talisker. '
'https://github.com/prometheus/client_python/issues/322'.format(
prometheus_installed.version,
)
)
_lock = None
histogram_archive = 'histogram_archive.db'
counter_archive = 'counter_archive.db'
__all__ = ['Context']
# Global storage for contexts by id. We use a process global, so that we can
# provide best effort logging of outstanding requests when a process is killed,
# e.g. worker killed by master whilst having inflight requests
CONTEXT_MAP = {}
if future.utils.PY3:
import contextvars
# enable asyncio aware contextvars in 3.5.3+/3.6
if pkg_is_installed('aiocontextvars'):
import asyncio
# aiocontextvars only supports python 3.5.3+
if hasattr(asyncio, '_get_running_loop'):
import aiocontextvars # NOQA
else:
early_log(
__name__,
'warning',
'aiocontextvars is installed, but it does not function with '
'python {}. Please use python >= 3.5.3 if you wish to use '
'talisker with asyncio.'.format(
'.'.join(str(v) for v in sys.version_info[:3])
)
)
from contextlib import contextmanager
from multiprocessing import Lock
import errno
import logging
import os
import tempfile
import time
import talisker
from talisker.util import (
early_log,
pkg_is_installed,
TaliskerVersionException,
)
prometheus_installed = pkg_is_installed('prometheus_client')
if prometheus_installed and prometheus_installed.version in ('0.4.0', '0.4.1'):
raise TaliskerVersionException(
'prometheus_client {} has a critical bug in multiprocess mode, '
'and is not supported in Talisker. '
'https://github.com/prometheus/client_python/issues/322'.format(
prometheus_installed.version, ))
_lock = None
histogram_archive = 'histogram_archive.db'
counter_archive = 'counter_archive.db'
class PrometheusLockTimeout(Exception):
pass
def setup_multiproc_dir():
if 'prometheus_multiproc_dir' not in os.environ:
if pkg_is_installed('prometheus-client'):
tmp = tempfile.mkdtemp(prefix='prometheus_multiproc')
os.environ['prometheus_multiproc_dir'] = tmp
