def ensure_talisker_config(kwargs): # ensure default processors # this is provided as a list from settings, but we need a set # to ensure we don't duplicate config = talisker.get_config() processors = set(kwargs.get('processors') or []) kwargs['processors'] = list(default_processors | processors) if 'transport' not in kwargs: kwargs['transport'] = TaliskerRequestsTransport # note: style clash - sentry client api is 'sanitize_keys' sanitise_keys = kwargs.get('sanitize_keys', []) if sanitise_keys is None: # flask integration explicitly sets None sanitise_keys = [] kwargs['sanitize_keys'] = (set(sanitise_keys) | config.DEFAULT_SANITISE_KEYS | config.sanitise_keys) # override it or it interferes with talisker logging if kwargs.get('install_logging_hook'): logging.getLogger(__name__).info( 'ignoring install_logging_hook=True in sentry config ' '- talisker manages this') kwargs['install_logging_hook'] = False # flask integration explictly sets options as None if kwargs.get('release') is None: kwargs['release'] = talisker.get_config().revision_id # don't hook libraries by default if kwargs.get('hook_libraries') is None: kwargs['hook_libraries'] = [] tags = kwargs.get('tags', {}) # set from the environment if config.unit is not None: tags['unit'] = config.unit if config.environment is not None: tags['environment'] = config.environment if config.domain is not None: tags['domain'] = config.domain kwargs['tags'] = tags dsn = kwargs.get('dsn', None) if not dsn: kwargs['dsn'] = config.sentry_dsn
def workers(self, request): """Information about workers resource usage.""" import psutil arbiter = psutil.Process(os.getppid()) workers = arbiter.children() workers.sort(key=lambda p: p.pid) rows = [format_psutil_row('Gunicorn Master', arbiter)] for i, worker in enumerate(workers): rows.append(format_psutil_row('Worker {}'.format(i), worker)) master = arbiter.as_dict(MASTER_FIELDS) master['cmdline'] = ' '.join(master['cmdline']) environ = master.pop('environ') config = talisker.get_config() clean_environ = [(k, v) for k, v in sorted(environ.items()) if k in config.METADATA] sorted_master = [(k, master[k]) for k in MASTER_FIELDS if k in master] return info_response( request.environ, 'Workers', Content('Workers', 'h2'), Table(rows, headers=HEADERS, id='workers'), Content('Process Information', 'h2'), Table(sorted_master, id='process_info'), Content('Process Environment (whitelist)', 'h2'), Table(clean_environ, id='process_env'), )
def config(self, request): config = talisker.get_config() rows = [] for name, meta in config.metadata().items(): if meta.default is None: is_default = '' else: is_default = meta.default == meta.value rows.append(( name, meta.value, '' if meta.raw is None else repr(meta.raw), is_default, )) return info_response( request.environ, 'Config', Content('Config', 'h2'), Table( rows, headers=['Name', 'Value', 'Raw Value', 'Is Default'], id='config', ) )
def workers(self, request): """Information about workers resource usage.""" import psutil arbiter = psutil.Process(os.getppid()) workers = arbiter.children() workers.sort(key=lambda p: p.pid) rows = [format_psutil_row('Gunicorn Master', arbiter)] for i, worker in enumerate(workers): rows.append(format_psutil_row('Worker {}'.format(i), worker)) master = arbiter.as_dict(MASTER_FIELDS) master['cmdline'] = ' '.join(master['cmdline']) environ = master.pop('environ') config = talisker.get_config() clean_environ = [ (k, v) for k, v in sorted(environ.items()) if k in config.METADATA ] sorted_master = [(k, master[k]) for k in MASTER_FIELDS if k in master] return info_response( request.environ, 'Workers', Content('Workers', 'h2'), Table(rows, headers=HEADERS, id='workers'), Content('Process Information', 'h2'), Table(sorted_master, id='process_info'), Content('Process Environment (whitelist)', 'h2'), Table(clean_environ, id='process_env'), )
def access(self, resp, req, environ, request_time): if not (self.cfg.accesslog or self.cfg.logconfig or self.cfg.syslog): return status_url = environ.get('PATH_INFO', '').startswith('/_status/') if status_url and not talisker.get_config()['logstatus']: return status = self.get_response_status(resp) msg, extra = self.get_extra(resp, req, environ, request_time, status) try: self.access_log.info(msg, extra=extra) except Exception: self.exception() if not status_url: labels = { 'view': extra.get('view', 'unknown'), 'method': extra['method'], 'status': str(status), } GunicornMetric.count.inc(**labels) if status >= 500: GunicornMetric.errors.inc(**labels) GunicornMetric.latency.observe(extra['duration_ms'], **labels)
def log_client(client): """Safely log client creation at INFO level.""" if not client.is_enabled(): # raven already logs a *disabled* client at INFO level return # base_url shouldn't have secrets in, but just in case, clean it public_dsn = client.remote.get_public_dsn() scheme = parse_url(client.remote.base_url).scheme url = scheme + ':' + public_dsn clean_url = sanitize_url(url) msg = 'configured raven DSN' extra = {'dsn': clean_url} config = talisker.get_config() env_cfg = config.raw.get('SENTRY_DSN') if env_cfg: # make a full url look like a public dsn clean_env = sanitize_url(re.sub(r'://(.*):.*@', r'://\1@', env_cfg)) if clean_env == clean_url: msg += ' from SENTRY_DSN config' extra['from_env'] = True else: msg += ' overriding SENTRY_DSN config' extra['SENTRY_DSN'] = clean_env logging.getLogger(__name__).info(msg, extra=extra)
def ensure_talisker_config(kwargs): # ensure default processors # this is provided as a list from settings, but we need a set # to ensure we don't duplicate config = talisker.get_config() processors = set(kwargs.get('processors') or []) kwargs['processors'] = list(default_processors | processors) # note: style clash - sentry client api is 'sanitize_keys' sanitise_keys = kwargs.get('sanitize_keys', []) if sanitise_keys is None: # flask integration explicitly sets None sanitise_keys = [] kwargs['sanitize_keys'] = ( set(sanitise_keys) | config.DEFAULT_SANITISE_KEYS | config.sanitise_keys ) # override it or it interferes with talisker logging if kwargs.get('install_logging_hook'): logging.getLogger(__name__).info( 'ignoring install_logging_hook=True in sentry config ' '- talisker manages this') kwargs['install_logging_hook'] = False # flask integration explictly sets options as None if kwargs.get('release') is None: kwargs['release'] = talisker.get_config().revision_id # don't hook libraries by default if kwargs.get('hook_libraries') is None: kwargs['hook_libraries'] = [] tags = kwargs.get('tags', {}) # set from the environment if config.unit is not None: tags['unit'] = config.unit if config.environment is not None: tags['environment'] = config.environment if config.domain is not None: tags['domain'] = config.domain kwargs['tags'] = tags dsn = kwargs.get('dsn', None) if not dsn: kwargs['dsn'] = config.sentry_dsn
def get_client(): client = None dsn = talisker.get_config().statsd_dsn if dsn is None: client = DummyClient() else: if not dsn.startswith('udp'): raise Exception('Talisker only supports udp stastd client') client = StatsClient(*parse_statsd_dsn(dsn)) return client
def process_request(self, request, start_response): config = talisker.get_config() if config.status_interface: iface, port = request.environ['gunicorn.socket'].getsockname() if iface != config.status_interface: return None path = request.path[len(self.prefix):].rstrip('/') try: funcname = self.urlmap.get(path, None) func = getattr(self, funcname) except (KeyError, AttributeError, TypeError): return None else: return func(request)
def wrapper(self, request): config = talisker.get_config() if not request.access_route: # this means something probably bugged in werkzeug, but let's fail # gracefully return Response('no client ip provided', status='403 Forbidden') ip_str = force_unicode(request.access_route[-1]) ip = ip_address(ip_str) if ip.is_loopback or any(ip in network for network in config.networks): return f(self, request) else: msg = PRIVATE_BODY_RESPONSE_TEMPLATE.format( ip_str, force_unicode(request.remote_addr), request.headers.get('x-forwarded-for')) return Response(msg, status='403 Forbidden')
def wrapper(self, request): config = talisker.get_config() # talisker middleware provides this ip_str = request.environ.get('CLIENT_ADDR') if ip_str is None: # fallback to werkzeug's handling ip_str = force_unicode(request.access_route[-1]) if ip_str is None: return Response( 'no client ip provided', status='403 Forbidden', ) if config.is_trusted_addr(ip_str): return f(self, request) else: msg = PRIVATE_BODY_RESPONSE_TEMPLATE.format( ip_str, force_unicode(request.remote_addr), request.headers.get('x-forwarded-for')) return Response(msg, status='403 Forbidden')
def send_wrapper(func): """Sets header and records exception details.""" config = talisker.get_config() @functools.wraps(func) def send(request, **kwargs): rid = Context.request_id if rid and config.id_header not in request.headers: request.headers[config.id_header] = rid if Context.current.deadline: deadline = datetime.utcfromtimestamp(Context.current.deadline) formatted = deadline.isoformat() + 'Z' request.headers[config.deadline_header] = formatted if Context.debug: request.headers[DEBUG_HEADER] = '1' try: return func(request, **kwargs) except Exception as e: record_request(request, None, e) raise send._send_wrapper = True return send
def config(self, request): config = talisker.get_config() rows = [] for name, meta in config.metadata().items(): if meta.default is None: is_default = '' else: is_default = meta.default == meta.value rows.append(( name, meta.value, '' if meta.raw is None else repr(meta.raw), is_default, )) return info_response( request.environ, 'Config', Content('Config', 'h2'), Table( rows, headers=['Name', 'Value', 'Raw Value', 'Is Default'], id='config', ))
def assert_config(env, **expected): cfg = talisker.get_config(env) for k, v in expected.items(): assert cfg[k] == v
def query_threshold(self): if self._threshold is None: self._threshold = talisker.get_config()['slowquery_threshold'] return self._threshold
def query_threshold(self): if self._threshold is None: self._threshold = talisker.get_config().slowquery_threshold return self._threshold
def ok_response(): return Response(talisker.get_config().revision_id + '\n')
def explain_breadcrumbs(self): if self._explain is None: self._explain = talisker.get_config().explain_sql return self._explain