def run_gunicorn(): config = get_config() # configure prometheus_client early as possible if pkg_is_installed('prometheus-client'): # Early throw-away parsing of gunicorn config, as we need to decide # whether to enable prometheus multiprocess before we start importing from gunicorn.app.wsgiapp import WSGIApplication g_cfg = WSGIApplication().cfg if g_cfg.workers > 1 or 'prometheus_multiproc_dir' in os.environ: from talisker.prometheus import setup_prometheus_multiproc async_workers = ('gevent', 'eventlet') # must be done before prometheus_client is imported *anywhere* setup_prometheus_multiproc( any(n in g_cfg.worker_class_str for n in async_workers) ) initialise() import talisker.gunicorn if pkg_is_installed('celery'): import talisker.celery talisker.celery.enable_signals() app = talisker.gunicorn.TaliskerApplication( "%(prog)s [OPTIONS] [APP_MODULE]", config.devel, config.debuglog) clear_contexts() return app.run()
def __init__(self, app, namespace='_status'): self.app = app self.namespace = namespace self.prefix = '/' + namespace # Publish /metrics only if prometheus_client is available if pkg_is_installed('prometheus-client'): self.urlmap['/metrics'] = 'metrics' self.urlmap['/test/prometheus'] = 'test_prometheus' if pkg_is_installed('logging-tree'): self.urlmap['/debug/logtree'] = 'debug_logtree'
def __init__(self, app, namespace='_status'): self.app = app self.namespace = namespace self.prefix = '/' + namespace # Publish /metrics only if prometheus_client is available if pkg_is_installed('prometheus-client'): self.urlmap['/metrics'] = 'metrics' self.urlmap['/test/prometheus'] = 'test_prometheus' if pkg_is_installed('logging-tree'): self.urlmap['/info/logtree'] = 'logtree' if pkg_is_installed('psutil'): self.urlmap['/info/workers'] = 'workers' if pkg_is_installed('objgraph'): self.urlmap['/info/objgraph'] = 'objgraph'
def test_prometheus(self, request): """Increment prometheus metric for testing""" if not pkg_is_installed('prometheus-client'): return Response('Not Supported', status=501) test_counter().inc() return Response('Incremented test counter')
def metrics(self, request): """Endpoint exposing Prometheus metrics""" if not pkg_is_installed('prometheus-client'): return Response('Not Supported', status=501) # Importing this too early would break multiprocess metrics from prometheus_client import ( CONTENT_TYPE_LATEST, CollectorRegistry, REGISTRY, generate_latest, multiprocess, ) if 'prometheus_multiproc_dir' in os.environ: # prometheus_client is running in multiprocess mode. # Use a custom registry, as the global one includes custom # collectors which are not supported in this mode registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) else: if request.environ.get('wsgi.multiprocess', False): return Response( 'Not Supported: running in multiprocess mode but ' '`prometheus_multiproc_dir` envvar not set', status=501) # prometheus_client is running in single process mode. # Use the global registry (includes CPU and RAM collectors) registry = REGISTRY with prometheus_lock: data = generate_latest(registry) return Response(data, status=200, mimetype=CONTENT_TYPE_LATEST)
def init(self, parser, opts, args): """Provide talisker specific default config for gunicorn. These are just defaults, and can be overridden in cli/config, but it is helpful to set them here. """ cfg = super(TaliskerApplication, self).init(parser, opts, args) if cfg is None: cfg = {} cfg['logger_class'] = GunicornLogger cfg['pre_request'] = gunicorn_pre_request if pkg_is_installed('prometheus-client'): cfg['on_starting'] = gunicorn_on_starting cfg['child_exit'] = gunicorn_child_exit # development config if self._devel: logger = logging.getLogger(__name__) logger.debug('devel mode: setting gunicorn devel default config', extra=DEVEL_SETTINGS) cfg.update(DEVEL_SETTINGS) return cfg
def init(self, parser, opts, args): """Provide talisker specific default config for gunicorn. These are just defaults, and can be overridden in cli/config, but it is helpful to set them here. """ cfg = super(TaliskerApplication, self).init(parser, opts, args) if cfg is None: cfg = {} cfg['logger_class'] = GunicornLogger cfg['pre_request'] = gunicorn_pre_request # Use pip to find out if prometheus_client is available, as # importing it here would break multiprocess metrics if pkg_is_installed('prometheus-client'): cfg['worker_exit'] = prometheus_multiprocess_worker_exit # development config if self._devel: logger = logging.getLogger(__name__) logger.debug('devel mode: setting gunicorn devel default config', extra=DEVEL_SETTINGS) cfg.update(DEVEL_SETTINGS) return cfg
def setup_multiproc_dir(): global prometheus_lock if 'prometheus_multiproc_dir' not in os.environ: if pkg_is_installed('prometheus-client'): tmp = tempfile.mkdtemp(prefix='prometheus_multiproc') os.environ['prometheus_multiproc_dir'] = tmp if prometheus_lock is None: initialise_prometheus_lock()
def run_gunicorn(): config = get_config() # Early throw-away parsing of gunicorn config, as we need to decide # whether to enable prometheus multiprocess before we start importing from gunicorn.app.wsgiapp import WSGIApplication g_cfg = WSGIApplication().cfg # configure prometheus_client early as possible if pkg_is_installed('prometheus-client'): if g_cfg.workers > 1 or 'prometheus_multiproc_dir' in os.environ: from talisker.prometheus import setup_prometheus_multiproc async_workers = ('gevent', 'eventlet') # must be done before prometheus_client is imported *anywhere* setup_prometheus_multiproc( any(n in g_cfg.worker_class_str for n in async_workers)) try: from gunicorn.workers.ggevent import GeventWorker from talisker.context import enable_gevent_context except Exception: pass else: if g_cfg.worker_class == GeventWorker: enable_gevent_context() try: from gunicorn.workers.geventlet import EventletWorker from talisker.context import enable_eventlet_context except Exception: pass else: if g_cfg.worker_class == EventletWorker: enable_eventlet_context() initialise() import talisker.gunicorn if pkg_is_installed('celery'): import talisker.celery talisker.celery.enable_signals() app = talisker.gunicorn.TaliskerApplication( "%(prog)s [OPTIONS] [APP_MODULE]", config.devel, config.debuglog) clear_context() return app.run()
def test_prometheus(self, request): """Increment prometheus metric for testing""" if not pkg_is_installed('prometheus-client'): return Response('Not Supported', status=501) if not hasattr(self, 'test_counter'): import prometheus_client self.test_counter = prometheus_client.Counter('test', 'test') self.test_counter.inc() return Response('Incremented test counter')
def _patch_gevent_contextvars(): # gunicorn will attempt to patch contextvars for gevent workers, via # gevent.monkey.patch_all(). There's a bug in gevent 1.5 that raises when # on <py3.7 and the backported contextvars module is installed. Workaround # this by overriding gevent's decision to not patch contextvars. # Hopefully a temporary abomination. try: import gevent.contextvars except ImportError: return if pkg_is_installed('contextvars'): gevent.contextvars.__implements__ = gevent.contextvars.__all__
def load_config(self): super(TaliskerApplication, self).load_config() logger = logging.getLogger(__name__) # override and warn if self.cfg.errorlog != '-': logger.warning( 'ignoring gunicorn errorlog config, talisker logs to stderr', extra={'errorlog': self.cfg.errorlog}) self.cfg.set('errorlog', '-') if self.cfg.loglevel.lower() == 'debug' and self._devel: # user has configured debug level logging self.cfg.set('loglevel', 'DEBUG') talisker.logs.enable_debug_log_stderr() # ensure gunicorn sends debug level messages when needed if self._debuglog: self.cfg.set('loglevel', 'DEBUG') # override and warn if self.cfg.statsd_host or self.cfg.statsd_prefix: logger.warning( 'ignoring gunicorn statsd config, as has no effect when ' 'using talisker, as it uses STATS_DSN env var', extra={ 'statsd_host': self.cfg.statsd_host, 'statsd_prefix': self.cfg.statsd_prefix }) self.cfg.set('statsd_host', None) self.cfg.set('statsd_prefix', None) # trust but warn if self.cfg.logger_class is not GunicornLogger: logger.warning( 'using custom gunicorn logger class - this may break ' 'Talisker\'s logging configuration', extra={'logger_class': self.cfg.logger_class}) # Use pip to find out if prometheus_client is available, as # importing it here would break multiprocess metrics if (pkg_is_installed('prometheus-client') and (self.cfg.workers or 1) > 1): if 'prometheus_multiproc_dir' not in os.environ: logger.info('running in multiprocess mode but ' '`prometheus_multiproc_dir` envvar not set') tmpdir = tempfile.mkdtemp() os.environ['prometheus_multiproc_dir'] = tmpdir logger.info('using `%s` for multiprocess prometheus metrics', os.environ['prometheus_multiproc_dir'])
def run_gunicorn(): # set this early so any imports of prometheus client will be imported # correctly if 'prometheus_multiproc_dir' not in os.environ: if pkg_is_installed('prometheus-client'): tmp = tempfile.mkdtemp(prefix='prometheus_multiproc') os.environ['prometheus_multiproc_dir'] = tmp config = initialise() import talisker.celery import talisker.gunicorn talisker.celery.enable_signals() app = talisker.gunicorn.TaliskerApplication( "%(prog)s [OPTIONS] [APP_MODULE]", config['devel'], config['debuglog']) return app.run()
def test_context_asyncio(): if sys.version_info < (3, 7): if sys.version_info < (3, 5, 3): pytest.skip('aiocontextvars does not work in python {}'.format( sys.version)) elif not pkg_is_installed('aiocontextvars'): pytest.skip('aiocontextvars not installed') async def r1(): Context.new() Context.logging.push(a=1) Context.track('test', 1.0) assert Context.logging.flat == {'a': 1} assert Context.current().tracking['test'].count == 1 await sub() # changes made by sub should be visible assert Context.logging.flat == {'a': 2} assert Context.current().tracking['test'].count == 2 async def sub(): # should be same context as r1 assert Context.logging.flat == {'a': 1} Context.logging.push(a=2) Context.track('test', 1.0) assert Context.logging.flat == {'a': 2} assert Context.current().tracking['test'].count == 2 async def r2(): # should be a separate context from r1 Context.new() Context.logging.push(a=3) Context.track('test', 1.0) assert Context.logging.flat == {'a': 3} assert Context.current().tracking['test'].count == 1 # ensure we have no context loop = asyncio.get_event_loop() Context.clear() t1 = loop.create_task(r1()) t2 = loop.create_task(r2()) loop.run_until_complete(asyncio.gather(t1, t2))
def metrics(self, request): """Endpoint exposing Prometheus metrics""" if not pkg_is_installed('prometheus-client'): return Response('Not Supported', status=501) from prometheus_client import CONTENT_TYPE_LATEST from talisker.prometheus import ( collect_metrics, PrometheusLockTimeout, ) try: data = collect_metrics() except PrometheusLockTimeout: msg = 'Failed to acquire prometheus lock to collect metrics' logger.exception(msg) return Response([msg], status=500, mimetype='text/plain') return Response(data, status=200, mimetype=CONTENT_TYPE_LATEST)
def init(self, parser, opts, args): """Provide talisker specific default config for gunicorn. These are just defaults, and can be overridden in cli/config, but it is helpful to set them here. """ cfg = super(TaliskerApplication, self).init(parser, opts, args) if cfg is None: cfg = {} cfg['logger_class'] = GunicornLogger cfg['pre_request'] = gunicorn_pre_request if pkg_is_installed('prometheus-client'): # only available in gunicorn 19.7+ if hasattr(gunicorn.config, 'ChildExit'): # we can do the full cleanup cleanup = talisker.metrics.prometheus_cleanup_worker server_hook = 'child_exit' else: # weaksauce cleanup from prometheus_client import multiprocess cleanup = multiprocess.mark_process_dead server_hook = 'worker_exit' def prometheus_multiprocess_worker_exit(server, worker): "Worker cleanup function for multiprocess prometheus_client." logging.getLogger(__name__).info( 'Performing multiprocess prometheus_client cleanup') cleanup(worker.pid) cfg[server_hook] = prometheus_multiprocess_worker_exit # development config if self._devel: logger = logging.getLogger(__name__) logger.debug('devel mode: setting gunicorn devel default config', extra=DEVEL_SETTINGS) cfg.update(DEVEL_SETTINGS) return cfg
from builtins import * # noqa from collections import defaultdict, OrderedDict from contextlib import contextmanager from multiprocessing import Lock import errno import json import logging import os import tempfile import time import talisker from talisker.util import pkg_is_installed, TaliskerVersionException prometheus_installed = pkg_is_installed('prometheus_client') if prometheus_installed and prometheus_installed.version in ('0.4.0', '0.4.1'): raise TaliskerVersionException( 'prometheus_client {} has a critical bug in multiprocess mode, ' 'and is not supported in Talisker. ' 'https://github.com/prometheus/client_python/issues/322'.format( prometheus_installed.version, ) ) _lock = None histogram_archive = 'histogram_archive.db' counter_archive = 'counter_archive.db'
__all__ = ['Context'] # Global storage for contexts by id. We use a process global, so that we can # provide best effort logging of outstanding requests when a process is killed, # e.g. worker killed by master whilst having inflight requests CONTEXT_MAP = {} if future.utils.PY3: import contextvars # enable asyncio aware contextvars in 3.5.3+/3.6 if pkg_is_installed('aiocontextvars'): import asyncio # aiocontextvars only supports python 3.5.3+ if hasattr(asyncio, '_get_running_loop'): import aiocontextvars # NOQA else: early_log( __name__, 'warning', 'aiocontextvars is installed, but it does not function with ' 'python {}. Please use python >= 3.5.3 if you wish to use ' 'talisker with asyncio.'.format( '.'.join(str(v) for v in sys.version_info[:3]) ) )
from contextlib import contextmanager from multiprocessing import Lock import errno import logging import os import tempfile import time import talisker from talisker.util import ( early_log, pkg_is_installed, TaliskerVersionException, ) prometheus_installed = pkg_is_installed('prometheus_client') if prometheus_installed and prometheus_installed.version in ('0.4.0', '0.4.1'): raise TaliskerVersionException( 'prometheus_client {} has a critical bug in multiprocess mode, ' 'and is not supported in Talisker. ' 'https://github.com/prometheus/client_python/issues/322'.format( prometheus_installed.version, )) _lock = None histogram_archive = 'histogram_archive.db' counter_archive = 'counter_archive.db' class PrometheusLockTimeout(Exception): pass
def setup_multiproc_dir(): if 'prometheus_multiproc_dir' not in os.environ: if pkg_is_installed('prometheus-client'): tmp = tempfile.mkdtemp(prefix='prometheus_multiproc') os.environ['prometheus_multiproc_dir'] = tmp