def process_response(self, request, response):
if response.status_code != 404:
return response
if is_ignorable_404(request.get_full_path()):
return response
from raven.contrib.django.models import client
if not client.is_enabled():
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' %
request.build_absolute_uri(),
data=data)
if not result:
return
request.sentry = {
'project_id': data.get('project', client.remote.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code != 404:
return response
if is_ignorable_404(request.get_full_path()):
return response
from raven.contrib.django.models import client
if not client.is_enabled():
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data)
if not result:
return
request.sentry = {
'project_id': data.get('project', client.remote.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
from raven.contrib.django.models import client
if response.status_code < 400 or response.status_code > 450 or is_ignorable_404(
request.get_full_path()) or not client.is_enabled():
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http{0}'.format(str(response.status_code)),
})
result = client.captureMessage(message='{0}: {1}'.format(
getattr(response, 'status_text', 'Page Not Found'),
request.build_absolute_uri()),
data=data)
if not result:
return
request.sentry = {
'project_id': data.get('project', client.remote.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code != 400:
return response
from raven.contrib.django.models import client
if not client.is_enabled():
return response
data = client.get_data_from_request(request)
# Ignore `django.security.DisallowedHost` errors
if re.search(r'\d+\.\d+\.\d+\.\d+',
data.get('request', {}).get('url', '')):
return response
data.update({
'level': logging.DEBUG,
'logger': 'manual_sentry_logger',
})
result = client.captureMessage(
message='400 response',
data=data,
extra={'response': self._parse_response(response)},
)
if not result:
return response
request.sentry = {
'project_id': data.get('project', client.remote.project),
'id': client.get_ident(result),
}
return response
def check_reporting(app_configs, **kwargs):
errors = []
current_user = pwd.getpwuid(os.getuid()).pw_name
# Get all configs if we don't get a specific set
if not app_configs:
app_configs = apps.apps.app_configs.values()
raven_installed = bool(list(filter(lambda app: app.name == "raven.contrib.django.raven_compat", app_configs)))
if not raven_installed:
errors.append(W001)
else:
raven_config = getattr(settings, 'RAVEN_CONFIG', None)
if raven_config is None or not isinstance(raven_config, dict):
errors.append(E001)
try:
# noinspection PyPackageRequirements,PyUnresolvedReferences
from raven.contrib.django.models import client
# So we are duplicating code from manage.py raven test here...
if not all([client.servers, client.project, client.public_key, client.secret_key]):
errors.append(E002)
if not client.is_enabled():
errors.append(E003)
else:
# Test sending
data = {
'culprit': 'preflight.checks.check_reporting',
'logger': 'raven.test',
'request': {
'method': 'GET',
'url': 'http://example.com',
}
}
ident = client.get_ident(client.captureMessage(
message='This is a test message generated using ``raven test``',
data=data,
level=logging.INFO,
stack=True,
tags={},
extra={
'user': current_user,
'loadavg': os.getloadavg(),
},
))
if client.state.did_fail():
errors.append(E004)
except ImportError:
errors.append(E005)
# TODO: check if we actually log something
return errors
def debug_log(request):
if request.method != 'POST' or not settings.HAVE_RAVEN or not settings.HAVE_SENTRY:
raise Http404('Method not supported')
form = forms.DebugLogForm(request.POST)
if not form.is_valid():
raise krest.BadRequestError(_form_errors(form))
client = LocalRavenClient([])
message = form.cleaned_data['message']
ident = client.get_ident(client.create_from_text(message))
client.send()
return {'log_id': ident}
def debug_log(request):
if request.method != 'POST' or not settings.HAVE_RAVEN or not settings.HAVE_SENTRY:
raise Http404('Method not supported')
form = forms.DebugLogForm(request.POST)
if not form.is_valid():
raise kbapi.BadRequestError(_form_errors(form))
client = LocalRavenClient([])
message = form.cleaned_data['message']
ident = client.get_ident(client.create_from_text(message))
client.send()
return {'log_id': ident}
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(request.get_full_path()):
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data)
request.sentry = {
'project_id': data.get('project', client.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(
request.get_full_path()):
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' %
request.build_absolute_uri(),
data=data)
request.sentry = {
'project_id': data.get('project', client.project),
'id': client.get_ident(result),
}
return response
def update_status(self):
"""
Update URL fields. This method uses socket to get IP of url and python
requests to get request info.
PRIVATE IPs will not be checked by default. To check PRIVATE IPs set
URLCHECK_ALLOW_PRIVATE_IPS to True in settings.python
RESERVED, SPECIALPURPOSE and LOOPBACK IPs will not be checked aniway.
"""
try:
logger.debug(u"Checking url %s for user %s.",self.url,self.user.username)
url = urlparse.urlparse(self.url)
self.hostname = url.hostname
if self.hostname:
self.site_ip = socket.gethostbyname(self.hostname)
self.site_fqdn = socket.getfqdn(self.hostname)
if self.get_ip_type() in ["RESERVED","SPECIALPURPOSE","LOOPBACK"]:
logger.warn(u"%s is a %s IP and will not be checked.",self.site_ip,
self.get_ip_type())
return
if not getattr(settings,"URLCHECK_ALLOW_PRIVATE_IPS",False) and \
self.get_ip_type() == "PRIVATE":
logger.warn(u"%s is a %s IP and will not be checked. Set "\
"URLCHECK_ALLOW_PRIVATE_IPS in settings.py to check this IPs.",
self.site_ip,self.get_ip_type())
return
try:
request = requests.get(self.url)
self.web_server_name = request.headers.get('server','')
self.html_status_code = request.status_code
except requests.exceptions.Timeout:
self.html_status_code = request.codes.timeout
self.last_time_checked = timezone.now()
old = URL.objects.get(pk=self.pk)
# some field is different, update
if self.site_ip != old.site_ip or \
self.site_fqdn != old.site_fqdn or \
self.web_server_name != old.web_server_name or \
self.html_status_code != old.html_status_code:
self.update_time = self.last_time_checked
# create a new history entry
self.history.create(
update_time = self.update_status,
site_ip = self.site_ip,
site_fqdn = self.site_fqdn,
web_server_name = self.web_server_name,
html_status_code = self.html_status_code
)
logger.info(u"Updated url %s for user %s.",self.url,self.user.username)
self.save()
except Exception as err:
sentry_id = client.get_ident(client.captureException())
logger.error(u"Error on update url status [Sentry ID '%s']",
sentry_id)
# remove key from cache so next request to status cache response again
cache_key = "%s:%s" % (self.user.username,self.url)
cache.delete(cache_key)
def update_url_info(url,timeout=10):
""" Update url status from URLCHECKER_SERVER.
It update URLStatusHistory table and URLCHECKER_SERVER
"""
server = getattr(settings,'URLCHECKER_SERVER')
key = getattr(settings,'URLCHECKER_KEY')
# it already is checked on __init__.py
if not server or not key:
raise ImproperlyConfigured(u"URLCHECKER: Configure URLCHECKER_SERVER "\
"and URLCHECKER_KEY on settings.py.")
try:
URLValidator()(url)
except ValidationError:
raise ValidationError(u"'%s' is one invalid URL. Update aborted.")
try:
url_request = requests.post(server,data={'key':key,'url':url},
timeout=timeout)
except request.TIMEOUT as err:
# capture exception to raven and try again
sentry_id = client.get_ident(client.captureException())
logger.error(u"Timeout processing url %s [Sentry ID '%s']",
url,sentry_id)
else:
if url_request.status_code in (200,304):
# request without errors
urldata = url_request.json['data']
print urldata
urlstatus, created = URLStatusHistory.objects.get_or_create(url=url)
urlstatus.update_time = datetime.datetime.now(pytz.utc)
urlstatus.site_ip = urldata['site_ip']
urlstatus.site_fqdn = urldata['site_fqdn']
urlstatus.web_server_name = urldata['web_server_name']
urlstatus.html_status_code = urldata['html_status_code']
urlstatus.save()
cache_key = 'urlchecker.client_%s' % md5(url).hexdigest()
cache.set(cache_key,urlstatus,300)
else:
try:
url_request.raise_for_status()
except requests.HTTPError, err:
# something happened, create a error in sentry if error is
# unknow
send_to_sentry=True
if url_request.status_code == 412 and \
url_request.json.has_key('msg') and \
"is a PRIVATE IP and" in url_request.json['msg']:
# know error, is a internal IP, we just get url info
update_internal_url_info(url)
send_to_sentry=False
if send_to_sentry:
import ipdb
ipdb.set_trace()
extra = {
'url': url,
'urlobj_id': getattr(urlobj,'id',None),
'code': url_request.status_code,
'method': 'post',
'json': url_request.json,
'text': url_request.text,
'request_dict': url_request.__dict__
}
sentry_id = client.get_ident(client.captureException(extra=extra))
logger.error(u"HTTP error %s on update_url_info. Msg was '%s'",
url_request.status_code,url_request.json.get('msg'))
def check_reporting(app_configs, **kwargs):
errors = []
current_user = pwd.getpwuid(os.getuid()).pw_name
# Get all configs if we don't get a specific set
if not app_configs:
app_configs = apps.apps.app_configs.values()
raven_installed = bool(
list(
filter(lambda app: app.name == "raven.contrib.django.raven_compat",
app_configs)))
if not raven_installed:
errors.append(W001)
else:
raven_config = getattr(settings, 'RAVEN_CONFIG', None)
if raven_config is None or not isinstance(raven_config, dict):
errors.append(E001)
try:
# noinspection PyPackageRequirements,PyUnresolvedReferences
from raven.contrib.django.models import client
# So we are duplicating code from manage.py raven test here...
if not all([
client.servers, client.project, client.public_key,
client.secret_key
]):
errors.append(E002)
if not client.is_enabled():
errors.append(E003)
else:
# Test sending
data = {
'culprit': 'preflight.checks.check_reporting',
'logger': 'raven.test',
'request': {
'method': 'GET',
'url': 'http://example.com',
}
}
ident = client.get_ident(
client.captureMessage(
message=
'This is a test message generated using ``raven test``',
data=data,
level=logging.INFO,
stack=True,
tags={},
extra={
'user': current_user,
'loadavg': os.getloadavg(),
},
))
if client.state.did_fail():
errors.append(E004)
except ImportError:
errors.append(E005)
# TODO: check if we actually log something
return errors
def query_url(request):
"""
This view returns status for url.
This view returns appropriate responses for each possible status of a query.
Remarkably 304 and 200 for success responses and 400, 403 and 500 for errors.
You would like to read it: http://en.wikipedia.org/wiki/List_of_HTTP_status_codes
"""
messages = {
'key_not_send': u"You need to send a key in POST data to be valid "\
"request. The key should be requested to system administrator.",
'key_invalid': u"Your key is invalid or not found. Contact system "\
"administrator for more information.",
'url_invalid_parameter': u"You need to send a url in POST data to be tested. "\
"Use 'url' field for that.",
'url_malformed': u"You sent a malformed url in POST. Check and try again.",
}
if not request.method == 'POST':
if not request.is_ajax():
return HttpResponseBadRequest('This url should be requested via'\
' Ajax POST requests ')
return HttpResponseNotAllowed(['POST'])
# TODO: if you have more views that use key put it in a decorator
request.key = None
key = request.POST.get('key',None)
if not key:
return HttpResponseForbidden(simplejson.dumps(
{'code':403,'msg':messages['key_not_send']}),
mimetype="application/json")
else:
try:
keyobj = Key.objects.get(key=key,active=True,
valid_until__gte=timezone.now())
except Key.DoesNotExist:
return HttpResponseForbidden(simplejson.dumps(
{'code':403,'msg':messages['key_invalid']}),
mimetype="application/json")
else:
request.user = keyobj.user
request.key = keyobj
if request.key:
if 'url' not in request.POST:
return HttpResponseBadRequest(simplejson.dumps(
{'code':400,'msg':messages['url_invalid_parameter']}),
mimetype="application/json")
url = request.POST['url']
url_validator = URLValidator()
try:
url_validator(url)
except ValidationError:
return HttpResponseBadRequest(simplejson.dumps(
{'code':400,'msg':messages['url_malformed']}),
mimetype='application/json')
# try to get from cache
try:
return response_from_cache(url,request.user.username)
except CacheKeyError:
pass
urlobj, created = URL.objects.get_or_create(user=request.user,url=url)
if not urlobj.last_time_checked:
try:
urlobj.update_status()
except Exception, err:
from .tasks import update_urls
update_urls.delay(urlobj.url)
return cache_and_send(urlobj,
{'code':500,
'msg':'Server error. Task was sent to queue. '\
'Try again in some minutes.',
'error':unicode(err)},500)
if urlobj.get_ip_type() in ["RESERVED","SPECIALPURPOSE","LOOPBACK"]:
msg = u"%s is a %s IP and will not be checked." % (
urlobj.site_ip, urlobj.get_ip_type())
return cache_and_send(urlobj,{'code':412,'msg':msg},412)
if not getattr(settings,"URLCHECK_ALLOW_PRIVATE_IPS",False) and \
urlobj.get_ip_type() == "PRIVATE":
msg = u"%s is a %s IP and will not be checked. Set "\
"URLCHECK_ALLOW_PRIVATE_IPS in settings.py to check this IPs." % (
urlobj.site_ip,urlobj.get_ip_type())
return cache_and_send(urlobj,{'code':412,'msg':msg},412)
try:
url_data = urlobj.get_status()
if not created and urlobj.update_time < urlobj.last_time_checked:
return cache_and_send(urlobj,{'code': 304,
'msg': 'OK', 'data': url_data},304)
else:
return cache_and_send(urlobj,{'code': 200, 'msg': 'OK',
'data': url_data},200)
except Exception, err:
sentry_id = client.get_ident(client.captureException())
logger.error(u"Error processing url %s [Sentry ID '%s']",
url,sentry_id)
return cache_and_send(urlobj,{'code':500,'msg':'Server error',
'error':unicode(err)},500)
