def ensure_talisker_config(kwargs):
# ensure default processors
# this is provided as a list from settings, but we need a set
# to ensure we don't duplicate
config = talisker.get_config()
processors = set(kwargs.get('processors') or [])
kwargs['processors'] = list(default_processors | processors)
if 'transport' not in kwargs:
kwargs['transport'] = TaliskerRequestsTransport
# note: style clash - sentry client api is 'sanitize_keys'
sanitise_keys = kwargs.get('sanitize_keys', [])
if sanitise_keys is None: # flask integration explicitly sets None
sanitise_keys = []
kwargs['sanitize_keys'] = (set(sanitise_keys)
| config.DEFAULT_SANITISE_KEYS
| config.sanitise_keys)
# override it or it interferes with talisker logging
if kwargs.get('install_logging_hook'):
logging.getLogger(__name__).info(
'ignoring install_logging_hook=True in sentry config '
'- talisker manages this')
kwargs['install_logging_hook'] = False
# flask integration explictly sets options as None
if kwargs.get('release') is None:
kwargs['release'] = talisker.get_config().revision_id
# don't hook libraries by default
if kwargs.get('hook_libraries') is None:
kwargs['hook_libraries'] = []
tags = kwargs.get('tags', {})
# set from the environment
if config.unit is not None:
tags['unit'] = config.unit
if config.environment is not None:
tags['environment'] = config.environment
if config.domain is not None:
tags['domain'] = config.domain
kwargs['tags'] = tags
dsn = kwargs.get('dsn', None)
if not dsn:
kwargs['dsn'] = config.sentry_dsn
def workers(self, request):
"""Information about workers resource usage."""
import psutil
arbiter = psutil.Process(os.getppid())
workers = arbiter.children()
workers.sort(key=lambda p: p.pid)
rows = [format_psutil_row('Gunicorn Master', arbiter)]
for i, worker in enumerate(workers):
rows.append(format_psutil_row('Worker {}'.format(i), worker))
master = arbiter.as_dict(MASTER_FIELDS)
master['cmdline'] = ' '.join(master['cmdline'])
environ = master.pop('environ')
config = talisker.get_config()
clean_environ = [(k, v) for k, v in sorted(environ.items())
if k in config.METADATA]
sorted_master = [(k, master[k]) for k in MASTER_FIELDS if k in master]
return info_response(
request.environ,
'Workers',
Content('Workers', 'h2'),
Table(rows, headers=HEADERS, id='workers'),
Content('Process Information', 'h2'),
Table(sorted_master, id='process_info'),
Content('Process Environment (whitelist)', 'h2'),
Table(clean_environ, id='process_env'),
)
def config(self, request):
config = talisker.get_config()
rows = []
for name, meta in config.metadata().items():
if meta.default is None:
is_default = ''
else:
is_default = meta.default == meta.value
rows.append((
name,
meta.value,
'' if meta.raw is None else repr(meta.raw),
is_default,
))
return info_response(
request.environ,
'Config',
Content('Config', 'h2'),
Table(
rows,
headers=['Name', 'Value', 'Raw Value', 'Is Default'],
id='config',
)
)
def workers(self, request):
"""Information about workers resource usage."""
import psutil
arbiter = psutil.Process(os.getppid())
workers = arbiter.children()
workers.sort(key=lambda p: p.pid)
rows = [format_psutil_row('Gunicorn Master', arbiter)]
for i, worker in enumerate(workers):
rows.append(format_psutil_row('Worker {}'.format(i), worker))
master = arbiter.as_dict(MASTER_FIELDS)
master['cmdline'] = ' '.join(master['cmdline'])
environ = master.pop('environ')
config = talisker.get_config()
clean_environ = [
(k, v) for k, v in sorted(environ.items())
if k in config.METADATA
]
sorted_master = [(k, master[k]) for k in MASTER_FIELDS if k in master]
return info_response(
request.environ,
'Workers',
Content('Workers', 'h2'),
Table(rows, headers=HEADERS, id='workers'),
Content('Process Information', 'h2'),
Table(sorted_master, id='process_info'),
Content('Process Environment (whitelist)', 'h2'),
Table(clean_environ, id='process_env'),
)
def access(self, resp, req, environ, request_time):
if not (self.cfg.accesslog or self.cfg.logconfig or self.cfg.syslog):
return
status_url = environ.get('PATH_INFO', '').startswith('/_status/')
if status_url and not talisker.get_config()['logstatus']:
return
status = self.get_response_status(resp)
msg, extra = self.get_extra(resp, req, environ, request_time, status)
try:
self.access_log.info(msg, extra=extra)
except Exception:
self.exception()
if not status_url:
labels = {
'view': extra.get('view', 'unknown'),
'method': extra['method'],
'status': str(status),
}
GunicornMetric.count.inc(**labels)
if status >= 500:
GunicornMetric.errors.inc(**labels)
GunicornMetric.latency.observe(extra['duration_ms'], **labels)
def log_client(client):
"""Safely log client creation at INFO level."""
if not client.is_enabled():
# raven already logs a *disabled* client at INFO level
return
# base_url shouldn't have secrets in, but just in case, clean it
public_dsn = client.remote.get_public_dsn()
scheme = parse_url(client.remote.base_url).scheme
url = scheme + ':' + public_dsn
clean_url = sanitize_url(url)
msg = 'configured raven DSN'
extra = {'dsn': clean_url}
config = talisker.get_config()
env_cfg = config.raw.get('SENTRY_DSN')
if env_cfg:
# make a full url look like a public dsn
clean_env = sanitize_url(re.sub(r'://(.*):.*@', r'://\1@', env_cfg))
if clean_env == clean_url:
msg += ' from SENTRY_DSN config'
extra['from_env'] = True
else:
msg += ' overriding SENTRY_DSN config'
extra['SENTRY_DSN'] = clean_env
logging.getLogger(__name__).info(msg, extra=extra)
def log_client(client):
"""Safely log client creation at INFO level."""
if not client.is_enabled():
# raven already logs a *disabled* client at INFO level
return
# base_url shouldn't have secrets in, but just in case, clean it
public_dsn = client.remote.get_public_dsn()
scheme = parse_url(client.remote.base_url).scheme
url = scheme + ':' + public_dsn
clean_url = sanitize_url(url)
msg = 'configured raven DSN'
extra = {'dsn': clean_url}
config = talisker.get_config()
env_cfg = config.raw.get('SENTRY_DSN')
if env_cfg:
# make a full url look like a public dsn
clean_env = sanitize_url(re.sub(r'://(.*):.*@', r'://\1@', env_cfg))
if clean_env == clean_url:
msg += ' from SENTRY_DSN config'
extra['from_env'] = True
else:
msg += ' overriding SENTRY_DSN config'
extra['SENTRY_DSN'] = clean_env
logging.getLogger(__name__).info(msg, extra=extra)
def ensure_talisker_config(kwargs):
# ensure default processors
# this is provided as a list from settings, but we need a set
# to ensure we don't duplicate
config = talisker.get_config()
processors = set(kwargs.get('processors') or [])
kwargs['processors'] = list(default_processors | processors)
# note: style clash - sentry client api is 'sanitize_keys'
sanitise_keys = kwargs.get('sanitize_keys', [])
if sanitise_keys is None: # flask integration explicitly sets None
sanitise_keys = []
kwargs['sanitize_keys'] = (
set(sanitise_keys)
| config.DEFAULT_SANITISE_KEYS
| config.sanitise_keys
)
# override it or it interferes with talisker logging
if kwargs.get('install_logging_hook'):
logging.getLogger(__name__).info(
'ignoring install_logging_hook=True in sentry config '
'- talisker manages this')
kwargs['install_logging_hook'] = False
# flask integration explictly sets options as None
if kwargs.get('release') is None:
kwargs['release'] = talisker.get_config().revision_id
# don't hook libraries by default
if kwargs.get('hook_libraries') is None:
kwargs['hook_libraries'] = []
tags = kwargs.get('tags', {})
# set from the environment
if config.unit is not None:
tags['unit'] = config.unit
if config.environment is not None:
tags['environment'] = config.environment
if config.domain is not None:
tags['domain'] = config.domain
kwargs['tags'] = tags
dsn = kwargs.get('dsn', None)
if not dsn:
kwargs['dsn'] = config.sentry_dsn
def get_client():
client = None
dsn = talisker.get_config().statsd_dsn
if dsn is None:
client = DummyClient()
else:
if not dsn.startswith('udp'):
raise Exception('Talisker only supports udp stastd client')
client = StatsClient(*parse_statsd_dsn(dsn))
return client
def get_client():
client = None
dsn = talisker.get_config().statsd_dsn
if dsn is None:
client = DummyClient()
else:
if not dsn.startswith('udp'):
raise Exception('Talisker only supports udp stastd client')
client = StatsClient(*parse_statsd_dsn(dsn))
return client
def process_request(self, request, start_response):
config = talisker.get_config()
if config.status_interface:
iface, port = request.environ['gunicorn.socket'].getsockname()
if iface != config.status_interface:
return None
path = request.path[len(self.prefix):].rstrip('/')
try:
funcname = self.urlmap.get(path, None)
func = getattr(self, funcname)
except (KeyError, AttributeError, TypeError):
return None
else:
return func(request)
def wrapper(self, request):
config = talisker.get_config()
if not request.access_route:
# this means something probably bugged in werkzeug, but let's fail
# gracefully
return Response('no client ip provided', status='403 Forbidden')
ip_str = force_unicode(request.access_route[-1])
ip = ip_address(ip_str)
if ip.is_loopback or any(ip in network for network in config.networks):
return f(self, request)
else:
msg = PRIVATE_BODY_RESPONSE_TEMPLATE.format(
ip_str,
force_unicode(request.remote_addr),
request.headers.get('x-forwarded-for'))
return Response(msg, status='403 Forbidden')
def wrapper(self, request):
config = talisker.get_config()
# talisker middleware provides this
ip_str = request.environ.get('CLIENT_ADDR')
if ip_str is None:
# fallback to werkzeug's handling
ip_str = force_unicode(request.access_route[-1])
if ip_str is None:
return Response(
'no client ip provided',
status='403 Forbidden',
)
if config.is_trusted_addr(ip_str):
return f(self, request)
else:
msg = PRIVATE_BODY_RESPONSE_TEMPLATE.format(
ip_str, force_unicode(request.remote_addr),
request.headers.get('x-forwarded-for'))
return Response(msg, status='403 Forbidden')
def send_wrapper(func):
"""Sets header and records exception details."""
config = talisker.get_config()
@functools.wraps(func)
def send(request, **kwargs):
rid = Context.request_id
if rid and config.id_header not in request.headers:
request.headers[config.id_header] = rid
if Context.current.deadline:
deadline = datetime.utcfromtimestamp(Context.current.deadline)
formatted = deadline.isoformat() + 'Z'
request.headers[config.deadline_header] = formatted
if Context.debug:
request.headers[DEBUG_HEADER] = '1'
try:
return func(request, **kwargs)
except Exception as e:
record_request(request, None, e)
raise
send._send_wrapper = True
return send
def config(self, request):
config = talisker.get_config()
rows = []
for name, meta in config.metadata().items():
if meta.default is None:
is_default = ''
else:
is_default = meta.default == meta.value
rows.append((
name,
meta.value,
'' if meta.raw is None else repr(meta.raw),
is_default,
))
return info_response(
request.environ, 'Config', Content('Config', 'h2'),
Table(
rows,
headers=['Name', 'Value', 'Raw Value', 'Is Default'],
id='config',
))
def assert_config(env, **expected):
cfg = talisker.get_config(env)
for k, v in expected.items():
assert cfg[k] == v
def query_threshold(self):
if self._threshold is None:
self._threshold = talisker.get_config()['slowquery_threshold']
return self._threshold
def query_threshold(self):
if self._threshold is None:
self._threshold = talisker.get_config().slowquery_threshold
return self._threshold
def ok_response():
return Response(talisker.get_config().revision_id + '\n')
def ok_response():
return Response(talisker.get_config().revision_id + '\n')
def explain_breadcrumbs(self):
if self._explain is None:
self._explain = talisker.get_config().explain_sql
return self._explain
