def process_response(self, request, response):
"""
Take over CommonMiddleware's error reporting for broken links.
"""
if response.status_code == 404:
if FUSIONBOX_SEND_BROKEN_LINK_EMAILS and not settings.DEBUG:
# If the referrer was from an internal link or a non-search-engine site,
# send a note to the managers.
domain = request.get_host()
referer = request.META.get('HTTP_REFERER', None)
is_internal = bool(_is_internal_request(domain, referer))
path = request.get_full_path()
if referer and not _is_ignorable_404(path) and (is_internal or '?' not in referer):
error_hash = hash_args(domain, referer, is_internal, path)
try:
Logged404.objects.get(hash=error_hash)
created = False
except Logged404.DoesNotExist:
Logged404.objects.create(
domain=domain,
referer=referer,
is_internal=is_internal,
path=path,
)
created = True
if (is_internal and FUSIONBOX_INTERNAL_ALWAYS_NOTIFY) or created:
ua = request.META.get('HTTP_USER_AGENT', '<none>')
ip = request.META.get('REMOTE_ADDR', '<none>')
mail_managers("Broken %slink on %s" % ((is_internal and 'INTERNAL ' or ''), domain),
"Referrer: %s\nRequested URL: %s\nUser agent: %s\nIP address: %s\n" \
% (referer, request.get_full_path(), ua, ip),
fail_silently=True)
return response
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(request.get_full_path()):
return response
message_id = get_client().create_from_text('Http 404', request=request, level=logging.INFO, logger='http404')
request.sentry = {
'id': message_id,
}
return response
def process_response(self, request, response):
"Send broken link emails and calculate the Etag, if needed."
if response.status_code == 404:
if settings.SEND_BROKEN_LINK_EMAILS and not settings.DEBUG:
# If the referrer was from an internal link or a
# non-search-engine site, send a note to the managers.
domain = request.get_host()
referer = request.META.get('HTTP_REFERER')
is_internal = _is_internal_request(domain, referer)
path = request.get_full_path()
if (referer
and not _is_ignorable_404(path)
and (is_internal or '?' not in referer)):
ua = request.META.get('HTTP_USER_AGENT', '<none>')
ip = request.META.get('REMOTE_ADDR', '<none>')
user = None
if request.user and hasattr(request.user, 'email'):
user = request.user.email
content = (
"Referrer: %s\n"
"Requested URL: %s\n"
"User agent: %s\n"
"IP address: %s\n"
"User: %s\n"
) % (referer, request.get_full_path(), ua, ip, user)
internal = is_internal and 'INTERNAL ' or ''
mail_managers(
"Broken %slink on %s" % (internal, domain),
content,
fail_silently=True,
)
return response
# Use ETags, if requested.
if settings.USE_ETAGS:
if response.has_header('ETag'):
etag = response['ETag']
elif response.streaming:
etag = None
else:
etag = '"%s"' % hashlib.md5(response.content).hexdigest()
if etag is not None:
if (200 <= response.status_code < 300
and request.META.get('HTTP_IF_NONE_MATCH') == etag):
cookies = response.cookies
response = http.HttpResponseNotModified()
response.cookies = cookies
else:
response['ETag'] = etag
return response
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(
request.get_full_path()):
return response
client = get_client()
result = client.create_from_text('Http 404',
request=request,
level=logging.INFO,
logger='http404')
request.sentry = {
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(request.get_full_path()):
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data)
request.sentry = {
'project_id': data.get('project', client.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(request.get_full_path()):
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.capture('Message', param_message={'message':'Page Not Found: %s','params':[request.build_absolute_uri()]}, data=data)
request.opbeat = {
'app_id': data.get('app_id', client.app_id),
'id': client.get_ident(result),
}
return response
def create_log(self, request):
if request.user and request.session:
stamp = datetime.datetime.now()
url = request.get_full_path()
if _is_ignorable_404(url):
return None
# Truncate in case it doesn't fit
url = url[:self.model._meta.get_field_by_name('url')[0].max_length]
log = self.model(user=request.user,
session=request.session.session_key,
url=url,
stamp=stamp)
log.save()
return log
else:
return None
def process_response(self, request, response):
if response.status_code == 404 and \
not _is_ignorable_404(request.get_full_path()):
l, _ = Logger404.objects.get_or_create(
url=request.build_absolute_uri(),
defaults={
'host': request.META.get('HTTP_HOST', ''),
'referer': request.META.get('HTTP_REFERER', ''),
'user_agent': request.META.get('HTTP_USER_AGENT', ''),
'remote_addr': request.META.get('REMOTE_ADDR', ''),
'remote_host': request.META.get('REMOTE_HOST', ''),
'method': request.META.get('REQUEST_METHOD', '')
})
l.hit = l.hit + 1
l.save()
return response
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(
request.get_full_path()):
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' %
request.build_absolute_uri(),
data=data)
request.sentry = {
'project_id': data.get('project', client.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
"""
Take over CommonMiddleware's error reporting for broken links.
"""
if response.status_code == 404:
if FUSIONBOX_SEND_BROKEN_LINK_EMAILS and not settings.DEBUG:
# If the referrer was from an internal link or a non-search-engine site,
# send a note to the managers.
domain = request.get_host()
referer = request.META.get('HTTP_REFERER', None)
is_internal = bool(_is_internal_request(domain, referer))
path = request.get_full_path()
if referer and not _is_ignorable_404(path) and (
is_internal or '?' not in referer):
error_hash = hash_args(domain, referer, is_internal, path)
try:
Logged404.objects.get(hash=error_hash)
created = False
except Logged404.DoesNotExist:
Logged404.objects.create(
domain=domain,
referer=referer,
is_internal=is_internal,
path=path,
)
created = True
if (is_internal
and FUSIONBOX_INTERNAL_ALWAYS_NOTIFY) or created:
ua = request.META.get('HTTP_USER_AGENT', '<none>')
ip = request.META.get('REMOTE_ADDR', '<none>')
mail_managers("Broken %slink on %s" % ((is_internal and 'INTERNAL ' or ''), domain),
"Referrer: %s\nRequested URL: %s\nUser agent: %s\nIP address: %s\n" \
% (referer, request.get_full_path(), ua, ip),
fail_silently=True)
return response
def __init__(self, request, status, **kw):
# first off, these items can just be ignored, we
# really don't care about them too much
path = request.get_full_path()
if _is_ignorable_404(path):
return
# if you've set INTERNAL_IPS, we'll respect that and
# ignore any requests, we suggest settings this so your
# unit tests don't blast the server
if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
return
exc_info = sys.exc_info()
items = ['HOME', 'HTTP_ACCEPT', 'HTTP_ACCEPT_ENCODING', 'HTTP_REFERER', \
'HTTP_ACCEPT_LANGUAGE', 'HTTP_CONNECTION', 'HTTP_HOST', 'LANG', \
'PATH_INFO', 'QUERY_STRING', 'REQUEST_METHOD', 'SCRIPT_NAME', \
'SERVER_NAME', 'SERVER_PORT', 'SERVER_PROTOCOL', 'SERVER_SOFTWARE']
data = [ "%s: %s" % (k, request.META[k]) for k in items if request.META.get(k)]
if request.method.lower() == "post":
data.append("POST and FILES Variables:")
data.extend([" %s: %s" % self.filter_post_var(k, v)
for k, v in request.POST.items()
if not self.exclude_post_var(k) ])
data.extend([" %s: %s" % self.filter_file(k, v)
for k, v in request.FILES.items()
if not self.exclude_file(k) ])
# build out data to send to Arecibo some fields (like timestamp)
# are automatically added
self.data = {
"account": getattr(settings, 'ARECIBO_PUBLIC_ACCOUNT_NUMBER', ''),
"url": request.build_absolute_uri(),
"ip": request.META.get('REMOTE_ADDR'),
"traceback": u"\n".join(traceback.format_tb(exc_info[2])).encode("utf-8"),
# Replace any chars that can't be represented in UTF-8 with the
# Unicode replacement char:
"request": "\n".join(smart_unicode(d, errors='replace') for d in data),
"type": exc_info[0],
"msg": str(exc_info[1]),
"status": status,
"uid": uuid.uuid4(),
"user_agent": request.META.get('HTTP_USER_AGENT'),
}
# we might have a traceback, but it's not required
try:
if self.data["type"]:
self.data["type"] = str(self.data["type"].__name__)
except AttributeError:
pass
self.data.update(kw)
# it could be the site does not have the standard django auth
# setup and hence no request.user
try:
self.data["username"] = request.user.username,
# this will be "" for Anonymous
except Exception:
pass
# a 404 has some specific formatting of the error that can be useful
if status == 404:
msg = ""
for m in exc_info[1]:
if isinstance(m, dict):
tried = "\n".join(map(self.get_pattern, m["tried"]))
msg = "Failed to find %s, tried: \n%s" % (m["path"], tried)
else:
msg += m
self.data["msg"] = msg
# if we don't get a priority, lets create one
if not self.data.get("priority"):
if status == 500: self.data["priority"] = 1
else: self.data["priority"] = 5
# populate my arecibo object
self.err = error()
for key, value in self.data.items():
self.err.set(key, value)
def post(request, status, **kw):
# first off, these items can just be ignored, we
# really don't care about them too much
path = request.get_full_path()
if _is_ignorable_404(path):
return
# if you've set INTERNAL_IPS, we'll respect that and
# ignore any requests, we suggest settings this so your
# unit tests don't blast the server
if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
return
exc_info = sys.exc_info()
items = ['HOME', 'HTTP_ACCEPT', 'HTTP_ACCEPT_ENCODING', 'HTTP_REFERER', \
'HTTP_ACCEPT_LANGUAGE', 'HTTP_CONNECTION', 'HTTP_HOST', 'LANG', \
'PATH_INFO', 'QUERY_STRING', 'REQUEST_METHOD', 'SCRIPT_NAME', \
'SERVER_NAME', 'SERVER_PORT', 'SERVER_PROTOCOL', 'SERVER_SOFTWARE']
data = [ "%s: %s" % (k, request.META[k]) for k in items if request.META.get(k)]
if request.method.lower() == "post":
data.append("POST and FILES Variables:")
data.extend( [ " %s: %s" % (k, v) for k, v in request.POST.items() ])
data.extend( [ " %s: %s" % (k, v) for k, v in request.FILES.items() ])
# build out data to send to Arecibo some fields (like timestamp)
# are automatically added
data = {
"account": settings.ARECIBO_PUBLIC_ACCOUNT_NUMBER,
"url": request.build_absolute_uri(),
"ip": request.META.get('REMOTE_ADDR'),
"traceback": "\n".join(traceback.format_tb(exc_info[2])),
"request": "\n".join(data).encode("utf-8"),
"type": exc_info[0],
"msg": str(exc_info[1]),
"status": status,
"uid": time.time(),
"user_agent": request.META.get('HTTP_USER_AGENT'),
}
# we might have a traceback, but it's not required
try:
if data["type"]:
data["type"] = str(data["type"].__name__)
except AttributeError:
pass
data.update(kw)
# it could be the site does not have the standard django auth
# setup and hence no reques.user
try:
data["username"] = request.user.username,
# this will be "" for Anonymous
except AttributeError:
pass
# a 404 has some specific formatting of the error that can be useful
if status == 404:
msg = ""
for m in exc_info[1]:
if isinstance(m, dict):
tried = "\n".join(m["tried"])
msg = "Failed to find %s, tried: \n%s" % (m["path"], tried)
else:
msg += m
data["msg"] = msg
# if we don't get a priority, lets create one
if not data.get("priority"):
if status == 500: data["priority"] = 1
else: data["priority"] = 5
# populate my arecibo object
err = error()
for key, value in data.items():
err.set(key, value)
try:
if getattr(settings, "ARECIBO_TRANSPORT", "") == "smtp":
# use Djangos builtin mail
send_mail("Error", err.as_json(),
settings.DEFAULT_FROM_EMAIL,
[settings.ARECIBO_SERVER_EMAIL,])
else:
err.server(url=settings.ARECIBO_SERVER_URL)
err.send()
except:
# if you want this to be an explicit fail swap
# change the comments on the next two lines around
raise
#pass
return data["uid"]
def post(request, status, **kw):
exc_info = sys.exc_info()
path = request.get_full_path()
if _is_ignorable_404(path):
return
if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
return
data = {
"account": settings.ARECIBO_PUBLIC_ACCOUNT_NUMBER,
"url": getpath(request),
"ip": request.META.get('REMOTE_ADDR'),
"traceback": "\n".join(traceback.format_tb(exc_info[2])),
"type": str(exc_info[0].__name__),
"msg": str(exc_info[1]),
"status": status,
"uid": time.time(),
"user_agent": request.META.get('HTTP_USER_AGENT'),
"server": "Google App Engine"
}
# it could be the site does not have the standard django auth
# setup and hence no reques.user
try:
data["username"] = request.user.username,
# this will be "" for Anonymous
except AttributeError:
pass
data.update(kw)
# a 404 has some specific formatting of the error that
# can be useful
if status == 404:
msg = ""
for m in exc_info[1]:
tried = "\n".join(m["tried"])
msg = "Failed to find %s, tried: \n\t%s" % (m["path"], tried)
data["msg"] = msg
# if we don't get a priority, make create one
if not data.get("priority"):
if status == 500:
data["priority"] = 1
else:
data["priority"] = 5
try:
if getattr(settings, "ARECIBO_TRANSPORT", None):
try:
sender = settings.ARECIBO_EMAIL_SENDER_ADDRESS
except AttributeError:
raise ValueError, "We must have a valid ARECIBO_EMAIL_SENDER_ADDRESS in the settings."
mail.send_mail(sender=sender,
to=settings.ARECIBO_SERVER_EMAIL,
subject="Error",
body=simplejson.dumps(data))
else:
udata = urlencode(data)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
"Accept": "text/plain"
}
url = list(urlparse(settings.ARECIBO_SERVER_URL))
if url[2] != "/v/1/": url[2] = "/v/1/"
result = urlfetch.fetch(url=urlunparse(url),
payload=udata,
method=urlfetch.POST,
headers=headers)
except:
raise
# write to the standard google app engine log
# http://code.google.com/appengine/docs/python/logging.html
exctype, value = sys.exc_info()[:2]
msg = "There was an error posting that error to Arecibo via smtp %s, %s" % (
exctype, value)
logging.error("Arecibo: %s", msg)
return data["uid"]
def post(request, status, **kw):
exc_info = sys.exc_info()
path = request.get_full_path()
if _is_ignorable_404(path):
return
if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
return
data = {
"account": settings.ARECIBO_PUBLIC_ACCOUNT_NUMBER,
"url": getpath(request),
"ip": request.META.get('REMOTE_ADDR'),
"traceback": "\n".join(traceback.format_tb(exc_info[2])),
"type": str(exc_info[0].__name__),
"msg": str(exc_info[1]),
"status": status,
"uid": time.time(),
"user_agent": request.META.get('HTTP_USER_AGENT'),
"server": "Google App Engine"
}
# it could be the site does not have the standard django auth
# setup and hence no reques.user
try:
data["username"] = request.user.username,
# this will be "" for Anonymous
except AttributeError:
pass
data.update(kw)
# a 404 has some specific formatting of the error that
# can be useful
if status == 404:
msg = ""
for m in exc_info[1]:
tried = "\n".join(m["tried"])
msg = "Failed to find %s, tried: \n\t%s" % (m["path"], tried)
data["msg"] = msg
# if we don't get a priority, make create one
if not data.get("priority"):
if status == 500:
data["priority"] = 1
else:
data["priority"] = 5
try:
if getattr(settings, "ARECIBO_TRANSPORT", None):
try:
sender = settings.ARECIBO_EMAIL_SENDER_ADDRESS
except AttributeError:
raise ValueError, "We must have a valid ARECIBO_EMAIL_SENDER_ADDRESS in the settings."
mail.send_mail(
sender=sender,
to=settings.ARECIBO_SERVER_EMAIL,
subject="Error",
body=simplejson.dumps(data))
else:
udata = urlencode(data)
headers = {'Content-Type': 'application/x-www-form-urlencoded', "Accept": "text/plain"}
url = list(urlparse(settings.ARECIBO_SERVER_URL))
if url[2] != "/v/1/": url[2] = "/v/1/"
result = urlfetch.fetch(
url=urlunparse(url),
payload=udata,
method=urlfetch.POST,
headers=headers)
except:
raise
# write to the standard google app engine log
# http://code.google.com/appengine/docs/python/logging.html
exctype, value = sys.exc_info()[:2]
msg = "There was an error posting that error to Arecibo via smtp %s, %s" % (exctype, value)
logging.error("Arecibo: %s", msg)
return data["uid"]
def __init__(self, request, status, **kw):
# first off, these items can just be ignored, we
# really don't care about them too much
path = request.get_full_path()
if _is_ignorable_404(path):
return
# if you've set INTERNAL_IPS, we'll respect that and
# ignore any requests, we suggest settings this so your
# unit tests don't blast the server
if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
return
exc_info = sys.exc_info()
items = ['HOME', 'HTTP_ACCEPT', 'HTTP_ACCEPT_ENCODING', 'HTTP_REFERER', \
'HTTP_ACCEPT_LANGUAGE', 'HTTP_CONNECTION', 'HTTP_HOST', 'LANG', \
'PATH_INFO', 'QUERY_STRING', 'REQUEST_METHOD', 'SCRIPT_NAME', \
'SERVER_NAME', 'SERVER_PORT', 'SERVER_PROTOCOL', 'SERVER_SOFTWARE']
data = [
"%s: %s" % (k, request.META[k]) for k in items
if request.META.get(k)
]
if request.method.lower() == "post":
data.append("POST and FILES Variables:")
data.extend([
" %s: %s" % self.filter_post_var(k, v)
for k, v in request.POST.items()
if not self.exclude_post_var(k)
])
data.extend([
" %s: %s" % self.filter_file(k, v)
for k, v in request.FILES.items() if not self.exclude_file(k)
])
# build out data to send to Arecibo some fields (like timestamp)
# are automatically added
self.data = {
"account":
getattr(settings, 'ARECIBO_PUBLIC_ACCOUNT_NUMBER', ''),
"url":
request.build_absolute_uri(),
"ip":
request.META.get('REMOTE_ADDR'),
"traceback":
u"\n".join(traceback.format_tb(exc_info[2])).encode("utf-8"),
# Replace any chars that can't be represented in UTF-8 with the
# Unicode replacement char:
"request":
"\n".join(smart_unicode(d, errors='replace') for d in data),
"type":
exc_info[0],
"msg":
str(exc_info[1]),
"status":
status,
"uid":
uuid.uuid4(),
"user_agent":
request.META.get('HTTP_USER_AGENT'),
}
# we might have a traceback, but it's not required
try:
if self.data["type"]:
self.data["type"] = str(self.data["type"].__name__)
except AttributeError:
pass
self.data.update(kw)
# it could be the site does not have the standard django auth
# setup and hence no request.user
try:
self.data["username"] = request.user.username,
# this will be "" for Anonymous
except Exception:
pass
# a 404 has some specific formatting of the error that can be useful
if status == 404:
msg = ""
for m in exc_info[1]:
if isinstance(m, dict):
tried = "\n".join(map(self.get_pattern, m["tried"]))
msg = "Failed to find %s, tried: \n%s" % (m["path"], tried)
else:
msg += m
self.data["msg"] = msg
# if we don't get a priority, lets create one
if not self.data.get("priority"):
if status == 500: self.data["priority"] = 1
else: self.data["priority"] = 5
# populate my arecibo object
self.err = error()
for key, value in self.data.items():
self.err.set(key, value)
def post(request, status, **kw):
# first off, these items can just be ignored, we
# really don't care about them too much
path = request.get_full_path()
if _is_ignorable_404(path):
return
# if you've set INTERNAL_IPS, we'll respect that and
# ignore any requests, we suggest settings this so your
# unit tests don't blast the server
if request.META.get('REMOTE_ADDR') in settings.INTERNAL_IPS:
return
exc_info = sys.exc_info()
items = ['HOME', 'HTTP_ACCEPT', 'HTTP_ACCEPT_ENCODING', 'HTTP_REFERER', \
'HTTP_ACCEPT_LANGUAGE', 'HTTP_CONNECTION', 'HTTP_HOST', 'LANG', \
'PATH_INFO', 'QUERY_STRING', 'REQUEST_METHOD', 'SCRIPT_NAME', \
'SERVER_NAME', 'SERVER_PORT', 'SERVER_PROTOCOL', 'SERVER_SOFTWARE']
data = [
"%s: %s" % (k, request.META[k]) for k in items if request.META.get(k)
]
if request.method.lower() == "post":
data.append("POST and FILES Variables:")
data.extend([" %s: %s" % (k, v) for k, v in request.POST.items()])
data.extend([" %s: %s" % (k, v) for k, v in request.FILES.items()])
# build out data to send to Arecibo some fields (like timestamp)
# are automatically added
data = {
"account": settings.ARECIBO_PUBLIC_ACCOUNT_NUMBER,
"url": request.build_absolute_uri(),
"ip": request.META.get('REMOTE_ADDR'),
"traceback": "\n".join(traceback.format_tb(exc_info[2])),
"request": "\n".join(data).encode("utf-8"),
"type": exc_info[0],
"msg": str(exc_info[1]),
"status": status,
"uid": time.time(),
"user_agent": request.META.get('HTTP_USER_AGENT'),
}
# we might have a traceback, but it's not required
try:
if data["type"]:
data["type"] = str(data["type"].__name__)
except AttributeError:
pass
data.update(kw)
# it could be the site does not have the standard django auth
# setup and hence no reques.user
try:
data["username"] = request.user.username,
# this will be "" for Anonymous
except AttributeError:
pass
# a 404 has some specific formatting of the error that can be useful
if status == 404:
msg = ""
for m in exc_info[1]:
if isinstance(m, dict):
tried = "\n".join(m["tried"])
msg = "Failed to find %s, tried: \n%s" % (m["path"], tried)
else:
msg += m
data["msg"] = msg
# if we don't get a priority, lets create one
if not data.get("priority"):
if status == 500: data["priority"] = 1
else: data["priority"] = 5
# populate my arecibo object
err = error()
for key, value in data.items():
err.set(key, value)
try:
if getattr(settings, "ARECIBO_TRANSPORT", "") == "smtp":
# use Djangos builtin mail
send_mail("Error", err.as_json(), settings.DEFAULT_FROM_EMAIL, [
settings.ARECIBO_SERVER_EMAIL,
])
else:
err.server(url=settings.ARECIBO_SERVER_URL)
err.send()
except:
# if you want this to be an explicit fail swap
# change the comments on the next two lines around
raise
#pass
return data["uid"]
