def handle_slow_request(self, environ, dt):
headers = dict(get_headers(environ))
if 'Authorization' in headers:
headers['Authorization'] = 'redacted'
if 'Cookie' in headers:
headers['Cookie'] = 'redacted'
cak = environ.get('controller_action_key', None) or environ.get('PATH_INFO', "NOPATH").strip('/').replace('/', '.')
event_id = self.client.captureMessage(
"SLOREQ: %s" % cak,
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
'headers': headers,
'env': dict(get_environ(environ)),
}
},
extra={
'request_id': environ.get('request_id', 'Unknown'),
'request_duration_millis': dt * 1000
},
level="warning",
tags={
'type': 'sloreq',
'action_key': cak
}
)
# Galaxy: store event_id in environment so we can show it to the user
environ['sentry_event_id'] = event_id
return event_id
def handle_slow_request(self, environ, dt):
headers = dict(get_headers(environ))
if 'Authorization' in headers:
headers['Authorization'] = 'redacted'
if 'Cookie' in headers:
headers['Cookie'] = 'redacted'
cak = environ.get('controller_action_key', None) or environ.get('PATH_INFO', "NOPATH").strip('/').replace('/', '.')
event_id = self.client.captureMessage(
"SLOREQ: %s" % cak,
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
'headers': headers,
'env': dict(get_environ(environ)),
}
},
extra={
'request_id': environ.get('request_id', 'Unknown'),
'request_duration_millis': dt * 1000
},
level="warning",
tags={
'type': 'sloreq',
'action_key': cak
}
)
# Galaxy: store event_id in environment so we can show it to the user
environ['sentry_event_id'] = event_id
return event_id
def get_http_context(self, environ):
return {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ))
}
def handle_exception(self, exc_info, environ):
event_id = self.client.create_from_exception(
exc_info=exc_info,
url=get_current_url(environ, strip_querystring=True),
data={
'META': environ,
},
)
return event_id
def handle_exception(self, exc_info, environ):
event_id = self.client.create_from_exception(
exc_info=exc_info,
url=get_current_url(environ, strip_querystring=True),
data={
'META': environ,
},
)
return event_id
def get_http_context(self, environ):
return {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
def get_http_context(self, environ):
return {
"method": environ.get("REQUEST_METHOD"),
"url": get_current_url(environ, strip_querystring=True),
"query_string": environ.get("QUERY_STRING"),
# TODO
# 'data': environ.get('wsgi.input'),
"headers": dict(get_headers(environ)),
"env": dict(get_environ(environ)),
}
def handle_exception(self, environ):
event_id = self.client.captureException(
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
'headers': self.cleanup_headers(dict(get_headers(environ))),
'env': dict(get_environ(environ)),
}
},
)
return event_id
def handle_exception(self, environ):
event_id = self.client.captureException(
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
}, )
return event_id
def handle_exception(self, environ):
event_id = self.client.captureException(
data={
"sentry.interfaces.Http": {
"method": environ.get("REQUEST_METHOD"),
"url": get_current_url(environ, strip_querystring=True),
"query_string": environ.get("QUERY_STRING"),
# TODO
# 'data': environ.get('wsgi.input'),
"headers": dict(get_headers(environ)),
"env": dict(get_environ(environ)),
}
}
)
return event_id
def handle_exception(self, exc_info, environ):
event_id = self.client.capture('Exception',
exc_info=exc_info,
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
},
)
return event_id
def handle_exception(self, exc_info, environ):
request = webob.Request(environ)
event_id = self.client.create_from_exception(
exc_info=exc_info,
url=get_current_url(environ, strip_querystring=True),
data={
'sentry.interfaces.Http': {
'method': request.method,
'url': request.path_url,
'query_string': request.query_string,
'data': request.params.mixed(),
'cookies': dict(request.cookies),
'headers': dict(request.headers),
'env': request.environ,
}
},
)
return event_id
def handle_exception(self, environ):
event_id = self.client.captureException(
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
},
# Galaxy: add request id from environment if available
extra={'request_id': environ.get('request_id', 'Unknown')})
# Galaxy: store event_id in environment so we can show it to the user
environ['sentry_event_id'] = event_id[0]
return event_id
def get_http_context(self, environ):
from raven.utils.wsgi import get_current_url, get_headers, get_environ
data = {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
for k in environ.keys():
try:
if k not in data['env'] and environ.get(k):
data['env'][k.upper()] = str(environ.get(k))
except:
pass
return data
def get_http_context(self, environ):
from raven.utils.wsgi import get_current_url, get_headers, get_environ
data = {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
for k in environ.keys():
try:
if k not in data['env'] and environ.get(k):
data['env'][k.upper()] = str(environ.get(k))
except:
pass
return data
def handle_exception(self, environ):
headers = dict(get_headers(environ))
# Authorization header for REMOTE_USER sites consists of a base64() of
# their plaintext password. It is a security issue for this password to
# be exposed to a third party system which may or may not be under
# control of the same administrators as the local Authentication
# system. E.g. university LDAP systems.
if 'Authorization' in headers:
# Redact so the administrator knows that a value is indeed present.
headers['Authorization'] = 'redacted'
# Passing cookies allows for impersonation of users (depending on
# remote service) and can be considered a security risk as well. For
# multiple services running alongside Galaxy on the same host, this
# could allow a sentry user with access to logs to impersonate a user
# on another service. In the case of services like Jupyter, this can be
# a serious concern as that would allow for terminal access. Furthermore,
# very little debugging information can be gained as a result of having
# access to all of the users cookies (including Galaxy cookies)
if 'Cookie' in headers:
headers['Cookie'] = 'redacted'
event_id = self.client.captureException(
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': headers,
'env': dict(get_environ(environ)),
}
},
# Galaxy: add request id from environment if available
extra={
'request_id': environ.get('request_id', 'Unknown')
}
)
# Galaxy: store event_id in environment so we can show it to the user
environ['sentry_event_id'] = event_id
return event_id
def handle_exception(self, environ):
headers = dict(get_headers(environ))
# Authorization header for REMOTE_USER sites consists of a base64() of
# their plaintext password. It is a security issue for this password to
# be exposed to a third party system which may or may not be under
# control of the same administrators as the local Authentication
# system. E.g. university LDAP systems.
if 'Authorization' in headers:
# Redact so the administrator knows that a value is indeed present.
headers['Authorization'] = 'redacted'
# Passing cookies allows for impersonation of users (depending on
# remote service) and can be considered a security risk as well. For
# multiple services running alongside Galaxy on the same host, this
# could allow a sentry user with access to logs to impersonate a user
# on another service. In the case of services like IPython, this can be
# a serious concern as that would allow for terminal access. Furthermore,
# very little debugging information can be gained as a result of having
# access to all of the users cookies (including Galaxy cookies)
if 'Cookie' in headers:
headers['Cookie'] = 'redacted'
event_id = self.client.captureException(
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': headers,
'env': dict(get_environ(environ)),
}
},
# Galaxy: add request id from environment if available
extra={
'request_id': environ.get( 'request_id', 'Unknown' )
}
)
# Galaxy: store event_id in environment so we can show it to the user
environ['sentry_event_id'] = event_id[0]
return event_id
def send_sentry(self, metadata, msg=None, data=None, **kwargs):
from raven.utils.wsgi import get_current_url, get_environ, get_headers
if data is None:
data = {}
if 'SENTRY_ID' in self.environ:
data['event_id'] = self.environ['SENTRY_ID']
# sentry displays these specific fields in a different way
http_context = {
'url': get_current_url(self.environ),
# we don't use the sanitized version from metadata, we want the
# real query string
'query_string': self.environ.get('QUERY_STRING'),
'method': metadata['method'],
'headers': dict(get_headers(self.environ)),
'env': dict(get_environ(self.environ)),
}
talisker.sentry.report_wsgi(http_context,
exc_info=self.exc_info,
msg=msg,
data=data,
**kwargs)
def __call__(self, environ, start_response):
try:
return self._handle_request(environ, start_response)
except (KeyboardInterrupt, SystemExit):
raise
except:
if self.error_reporter:
self.error_reporter.http_context({
'method':
environ.get('REQUEST_METHOD'),
'url':
get_current_url(environ, strip_querystring=True),
'query_string':
environ.get('QUERY_STRING'),
'headers':
dict(get_headers(environ)),
'env':
dict(get_environ(environ)),
})
self.error_reporter.captureException()
self.error_reporter.context.clear()
raise
def report(self, traceback):
environ = traceback.context.get('environ', {})
data = {
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
}
is_backlash_event = getattr(traceback.exc_value, 'backlash_event', False)
if is_backlash_event:
# Just a Stack Dump request from backlash
self.client.captureMessage(traceback.exception, data=data,
stack=traceback.frames)
else:
# This is a real crash
self.client.captureException(exc_info=traceback.exc_info, data=data)
def handle_exception(self, environ):
event_id = self.client.captureException(
data={
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
},
# Galaxy: add request id from environment if available
extra={
'request_id': environ.get( 'request_id', 'Unknown' )
}
)
# Galaxy: store event_id in environment so we can show it to the user
environ['sentry_event_id'] = event_id[0]
return event_id
def get_data(self, data, **kwargs):
if 'sentry.interfaces.Http' in data or not has_request_context():
return
environ = request.environ
headers = dict(request.headers)
guru_key = getattr(request, 'guru_id_header', None)
if guru_key and guru_key not in headers:
headers[guru_key] = getattr(request, 'guru_id', None)
request_data = {
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': headers,
'env': dict(get_environ(environ)),
'form': dict(request.form),
}
}
data.update(request_data)
def report(self, traceback):
environ = traceback.context.get('environ', {})
data = {
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': dict(get_headers(environ)),
'env': dict(get_environ(environ)),
}
}
is_backlash_event = getattr(traceback.exc_value, 'backlash_event', False)
if is_backlash_event:
# Just a Stack Dump request from backlash
self.client.captureMessage(traceback.exception, data=data,
stack=traceback.frames)
else:
# This is a real crash
self.client.captureException(data=data)
def get_data(self, data, **kwargs):
if 'sentry.interfaces.Http' in data or not has_request_context():
return
environ = request.environ
headers = dict(request.headers)
guru_key = getattr(request, 'guru_id_header', None)
if guru_key and guru_key not in headers:
headers[guru_key] = getattr(request, 'guru_id', None)
request_data = {
'sentry.interfaces.Http': {
'method': environ.get('REQUEST_METHOD'),
'url': get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
# TODO
# 'data': environ.get('wsgi.input'),
'headers': headers,
'env': dict(get_environ(environ)),
'form': dict(request.form),
}
}
data.update(request_data)
def logErrorAndEmail(cfg, e_type, e_value, e_tb, location, info_dict,
prefix='mint-error-', smallStream=None, doEmail=True):
timeStamp = time.ctime(time.time())
realHostName = socket.getfqdn()
if 'mint.authToken' in info_dict:
info_dict['mint.authToken'] = repr(info_dict['mint.authToken'])
if 'HTTP_AUTHORIZATION' in info_dict:
info_dict['HTTP_AUTHORIZATION'] = info_dict['HTTP_AUTHORIZATION'].split()[0] + ' ******'
# Format large traceback to file
(tbFd, tbPath) = tempfile.mkstemp('.txt', prefix)
large = os.fdopen(tbFd, 'w+')
print >> large, ('Unhandled exception from %s on %s:'
% (location, realHostName))
print >> large, 'Time of occurrence: %s' % timeStamp
print >> large, 'See also: %s' % tbPath
print >> large
conary_util.formatTrace(e_type, e_value, e_tb, stream=large,
withLocals=False)
print >> large
print >> large, 'Full stack:'
print >> large
try:
conary_util.formatTrace(e_type, e_value, e_tb, stream=large,
withLocals=True)
except:
# The extended traceback formatter can crash when uncooperative
# objects do something special when marshalled. For example,
# derivatives of Thread may abort before Thread itself is
# initialized, which causes __repr__ to crash, and
# ServerProxy derivatives may crash when __safe_str__ is
# improperly sent as a remote procedure call instead of
# throwing an AttributeError.
#
# None of these conditions should prevent the original
# exception from being logged and emailed, if only in their
# short form.
handlerErrorType, handlerErrorValue, handlerErrorTB = sys.exc_info()
print >> large
print >> large, '*** Traceback formatter crashed! ***'
print >> large, 'Formatter crash follows:'
conary_util.formatTrace(handlerErrorType, handlerErrorValue,
handlerErrorTB, stream=large, withLocals=False)
print >> large, '*** End formatter crash log ***'
print >> large
print >> large, 'Environment:'
for key, val in sorted(info_dict.items()):
try:
print >> large, '%s: %s' % (key, val)
except:
pass
# Format small traceback
small = conary_util.BoundedStringIO()
print >> small, ('Unhandled exception from %s on %s:'
% (location, realHostName))
conary_util.formatTrace(e_type, e_value, e_tb, stream=small,
withLocals=False)
print >> small, 'Extended traceback at %s' % tbPath
if cfg and cfg.sentryDSN:
from raven import Client
from raven.utils import wsgi
client = Client(cfg.sentryDSN)
data = {}
extra = {
'environ': info_dict,
'reference': '%s:%s' % (realHostName, tbPath),
}
if 'REQUEST_METHOD' in info_dict:
# Looks like WSGI
environ = info_dict
data['sentry.interfaces.Http'] = {
'method': environ.get('REQUEST_METHOD'),
'url': wsgi.get_current_url(environ, strip_querystring=True),
'query_string': environ.get('QUERY_STRING'),
'headers': dict(wsgi.get_headers(environ)),
'env': dict(wsgi.get_environ(environ)),
}
sentry_id = client.captureException(exc_info=(e_type, e_value, e_tb),
data=data, extra=extra)
print >> large, "Sentry event ID: ", sentry_id[0]
print >> small, "Sentry event ID: ", sentry_id[0]
small.seek(0)
log.error(small.read())
# send email
if cfg and doEmail:
base_exception = traceback.format_exception_only(
e_type, e_value)[-1].strip()
if cfg.rBuilderOnline:
subject = '%s: %s' % (realHostName, base_exception)
else:
extra = {'hostname': realHostName}
subject = cfg.bugsEmailSubject % extra
if cfg.bugsEmail:
large.seek(0)
maillib.sendMailWithChecks(cfg.bugsEmail, cfg.bugsEmailName,
cfg.bugsEmail, subject, large.read())
if cfg.smallBugsEmail:
small.seek(0)
maillib.sendMailWithChecks(cfg.bugsEmail, cfg.bugsEmailName,
cfg.smallBugsEmail, subject, small.read())
large.close()
