def process_response(self, request, response): if response.status_code != 404: return response if is_ignorable_404(request.get_full_path()): return response from raven.contrib.django.models import client if not client.is_enabled(): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http404', }) result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data) if not result: return request.sentry = { 'project_id': data.get('project', client.remote.project), 'id': client.get_ident(result), } return response
def process_response(self, request, response): from raven.contrib.django.models import client if response.status_code < 400 or response.status_code > 450 or is_ignorable_404( request.get_full_path()) or not client.is_enabled(): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http{0}'.format(str(response.status_code)), }) result = client.captureMessage(message='{0}: {1}'.format( getattr(response, 'status_text', 'Page Not Found'), request.build_absolute_uri()), data=data) if not result: return request.sentry = { 'project_id': data.get('project', client.remote.project), 'id': client.get_ident(result), } return response
def process_response(self, request, response): if response.status_code != 400: return response from raven.contrib.django.models import client if not client.is_enabled(): return response data = client.get_data_from_request(request) # Ignore `django.security.DisallowedHost` errors if re.search(r'\d+\.\d+\.\d+\.\d+', data.get('request', {}).get('url', '')): return response data.update({ 'level': logging.DEBUG, 'logger': 'manual_sentry_logger', }) result = client.captureMessage( message='400 response', data=data, extra={'response': self._parse_response(response)}, ) if not result: return response request.sentry = { 'project_id': data.get('project', client.remote.project), 'id': client.get_ident(result), } return response
def check_reporting(app_configs, **kwargs): errors = [] current_user = pwd.getpwuid(os.getuid()).pw_name # Get all configs if we don't get a specific set if not app_configs: app_configs = apps.apps.app_configs.values() raven_installed = bool(list(filter(lambda app: app.name == "raven.contrib.django.raven_compat", app_configs))) if not raven_installed: errors.append(W001) else: raven_config = getattr(settings, 'RAVEN_CONFIG', None) if raven_config is None or not isinstance(raven_config, dict): errors.append(E001) try: # noinspection PyPackageRequirements,PyUnresolvedReferences from raven.contrib.django.models import client # So we are duplicating code from manage.py raven test here... if not all([client.servers, client.project, client.public_key, client.secret_key]): errors.append(E002) if not client.is_enabled(): errors.append(E003) else: # Test sending data = { 'culprit': 'preflight.checks.check_reporting', 'logger': 'raven.test', 'request': { 'method': 'GET', 'url': 'http://example.com', } } ident = client.get_ident(client.captureMessage( message='This is a test message generated using ``raven test``', data=data, level=logging.INFO, stack=True, tags={}, extra={ 'user': current_user, 'loadavg': os.getloadavg(), }, )) if client.state.did_fail(): errors.append(E004) except ImportError: errors.append(E005) # TODO: check if we actually log something return errors
def debug_log(request): if request.method != 'POST' or not settings.HAVE_RAVEN or not settings.HAVE_SENTRY: raise Http404('Method not supported') form = forms.DebugLogForm(request.POST) if not form.is_valid(): raise krest.BadRequestError(_form_errors(form)) client = LocalRavenClient([]) message = form.cleaned_data['message'] ident = client.get_ident(client.create_from_text(message)) client.send() return {'log_id': ident}
def debug_log(request): if request.method != 'POST' or not settings.HAVE_RAVEN or not settings.HAVE_SENTRY: raise Http404('Method not supported') form = forms.DebugLogForm(request.POST) if not form.is_valid(): raise kbapi.BadRequestError(_form_errors(form)) client = LocalRavenClient([]) message = form.cleaned_data['message'] ident = client.get_ident(client.create_from_text(message)) client.send() return {'log_id': ident}
def process_response(self, request, response): if response.status_code != 404 or _is_ignorable_404(request.get_full_path()): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http404', }) result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data) request.sentry = { 'project_id': data.get('project', client.project), 'id': client.get_ident(result), } return response
def process_response(self, request, response): if response.status_code != 404 or _is_ignorable_404( request.get_full_path()): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http404', }) result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data) request.sentry = { 'project_id': data.get('project', client.project), 'id': client.get_ident(result), } return response
def update_status(self): """ Update URL fields. This method uses socket to get IP of url and python requests to get request info. PRIVATE IPs will not be checked by default. To check PRIVATE IPs set URLCHECK_ALLOW_PRIVATE_IPS to True in settings.python RESERVED, SPECIALPURPOSE and LOOPBACK IPs will not be checked aniway. """ try: logger.debug(u"Checking url %s for user %s.",self.url,self.user.username) url = urlparse.urlparse(self.url) self.hostname = url.hostname if self.hostname: self.site_ip = socket.gethostbyname(self.hostname) self.site_fqdn = socket.getfqdn(self.hostname) if self.get_ip_type() in ["RESERVED","SPECIALPURPOSE","LOOPBACK"]: logger.warn(u"%s is a %s IP and will not be checked.",self.site_ip, self.get_ip_type()) return if not getattr(settings,"URLCHECK_ALLOW_PRIVATE_IPS",False) and \ self.get_ip_type() == "PRIVATE": logger.warn(u"%s is a %s IP and will not be checked. Set "\ "URLCHECK_ALLOW_PRIVATE_IPS in settings.py to check this IPs.", self.site_ip,self.get_ip_type()) return try: request = requests.get(self.url) self.web_server_name = request.headers.get('server','') self.html_status_code = request.status_code except requests.exceptions.Timeout: self.html_status_code = request.codes.timeout self.last_time_checked = timezone.now() old = URL.objects.get(pk=self.pk) # some field is different, update if self.site_ip != old.site_ip or \ self.site_fqdn != old.site_fqdn or \ self.web_server_name != old.web_server_name or \ self.html_status_code != old.html_status_code: self.update_time = self.last_time_checked # create a new history entry self.history.create( update_time = self.update_status, site_ip = self.site_ip, site_fqdn = self.site_fqdn, web_server_name = self.web_server_name, html_status_code = self.html_status_code ) logger.info(u"Updated url %s for user %s.",self.url,self.user.username) self.save() except Exception as err: sentry_id = client.get_ident(client.captureException()) logger.error(u"Error on update url status [Sentry ID '%s']", sentry_id) # remove key from cache so next request to status cache response again cache_key = "%s:%s" % (self.user.username,self.url) cache.delete(cache_key)
def update_url_info(url,timeout=10): """ Update url status from URLCHECKER_SERVER. It update URLStatusHistory table and URLCHECKER_SERVER """ server = getattr(settings,'URLCHECKER_SERVER') key = getattr(settings,'URLCHECKER_KEY') # it already is checked on __init__.py if not server or not key: raise ImproperlyConfigured(u"URLCHECKER: Configure URLCHECKER_SERVER "\ "and URLCHECKER_KEY on settings.py.") try: URLValidator()(url) except ValidationError: raise ValidationError(u"'%s' is one invalid URL. Update aborted.") try: url_request = requests.post(server,data={'key':key,'url':url}, timeout=timeout) except request.TIMEOUT as err: # capture exception to raven and try again sentry_id = client.get_ident(client.captureException()) logger.error(u"Timeout processing url %s [Sentry ID '%s']", url,sentry_id) else: if url_request.status_code in (200,304): # request without errors urldata = url_request.json['data'] print urldata urlstatus, created = URLStatusHistory.objects.get_or_create(url=url) urlstatus.update_time = datetime.datetime.now(pytz.utc) urlstatus.site_ip = urldata['site_ip'] urlstatus.site_fqdn = urldata['site_fqdn'] urlstatus.web_server_name = urldata['web_server_name'] urlstatus.html_status_code = urldata['html_status_code'] urlstatus.save() cache_key = 'urlchecker.client_%s' % md5(url).hexdigest() cache.set(cache_key,urlstatus,300) else: try: url_request.raise_for_status() except requests.HTTPError, err: # something happened, create a error in sentry if error is # unknow send_to_sentry=True if url_request.status_code == 412 and \ url_request.json.has_key('msg') and \ "is a PRIVATE IP and" in url_request.json['msg']: # know error, is a internal IP, we just get url info update_internal_url_info(url) send_to_sentry=False if send_to_sentry: import ipdb ipdb.set_trace() extra = { 'url': url, 'urlobj_id': getattr(urlobj,'id',None), 'code': url_request.status_code, 'method': 'post', 'json': url_request.json, 'text': url_request.text, 'request_dict': url_request.__dict__ } sentry_id = client.get_ident(client.captureException(extra=extra)) logger.error(u"HTTP error %s on update_url_info. Msg was '%s'", url_request.status_code,url_request.json.get('msg'))
def check_reporting(app_configs, **kwargs): errors = [] current_user = pwd.getpwuid(os.getuid()).pw_name # Get all configs if we don't get a specific set if not app_configs: app_configs = apps.apps.app_configs.values() raven_installed = bool( list( filter(lambda app: app.name == "raven.contrib.django.raven_compat", app_configs))) if not raven_installed: errors.append(W001) else: raven_config = getattr(settings, 'RAVEN_CONFIG', None) if raven_config is None or not isinstance(raven_config, dict): errors.append(E001) try: # noinspection PyPackageRequirements,PyUnresolvedReferences from raven.contrib.django.models import client # So we are duplicating code from manage.py raven test here... if not all([ client.servers, client.project, client.public_key, client.secret_key ]): errors.append(E002) if not client.is_enabled(): errors.append(E003) else: # Test sending data = { 'culprit': 'preflight.checks.check_reporting', 'logger': 'raven.test', 'request': { 'method': 'GET', 'url': 'http://example.com', } } ident = client.get_ident( client.captureMessage( message= 'This is a test message generated using ``raven test``', data=data, level=logging.INFO, stack=True, tags={}, extra={ 'user': current_user, 'loadavg': os.getloadavg(), }, )) if client.state.did_fail(): errors.append(E004) except ImportError: errors.append(E005) # TODO: check if we actually log something return errors
def query_url(request): """ This view returns status for url. This view returns appropriate responses for each possible status of a query. Remarkably 304 and 200 for success responses and 400, 403 and 500 for errors. You would like to read it: http://en.wikipedia.org/wiki/List_of_HTTP_status_codes """ messages = { 'key_not_send': u"You need to send a key in POST data to be valid "\ "request. The key should be requested to system administrator.", 'key_invalid': u"Your key is invalid or not found. Contact system "\ "administrator for more information.", 'url_invalid_parameter': u"You need to send a url in POST data to be tested. "\ "Use 'url' field for that.", 'url_malformed': u"You sent a malformed url in POST. Check and try again.", } if not request.method == 'POST': if not request.is_ajax(): return HttpResponseBadRequest('This url should be requested via'\ ' Ajax POST requests ') return HttpResponseNotAllowed(['POST']) # TODO: if you have more views that use key put it in a decorator request.key = None key = request.POST.get('key',None) if not key: return HttpResponseForbidden(simplejson.dumps( {'code':403,'msg':messages['key_not_send']}), mimetype="application/json") else: try: keyobj = Key.objects.get(key=key,active=True, valid_until__gte=timezone.now()) except Key.DoesNotExist: return HttpResponseForbidden(simplejson.dumps( {'code':403,'msg':messages['key_invalid']}), mimetype="application/json") else: request.user = keyobj.user request.key = keyobj if request.key: if 'url' not in request.POST: return HttpResponseBadRequest(simplejson.dumps( {'code':400,'msg':messages['url_invalid_parameter']}), mimetype="application/json") url = request.POST['url'] url_validator = URLValidator() try: url_validator(url) except ValidationError: return HttpResponseBadRequest(simplejson.dumps( {'code':400,'msg':messages['url_malformed']}), mimetype='application/json') # try to get from cache try: return response_from_cache(url,request.user.username) except CacheKeyError: pass urlobj, created = URL.objects.get_or_create(user=request.user,url=url) if not urlobj.last_time_checked: try: urlobj.update_status() except Exception, err: from .tasks import update_urls update_urls.delay(urlobj.url) return cache_and_send(urlobj, {'code':500, 'msg':'Server error. Task was sent to queue. '\ 'Try again in some minutes.', 'error':unicode(err)},500) if urlobj.get_ip_type() in ["RESERVED","SPECIALPURPOSE","LOOPBACK"]: msg = u"%s is a %s IP and will not be checked." % ( urlobj.site_ip, urlobj.get_ip_type()) return cache_and_send(urlobj,{'code':412,'msg':msg},412) if not getattr(settings,"URLCHECK_ALLOW_PRIVATE_IPS",False) and \ urlobj.get_ip_type() == "PRIVATE": msg = u"%s is a %s IP and will not be checked. Set "\ "URLCHECK_ALLOW_PRIVATE_IPS in settings.py to check this IPs." % ( urlobj.site_ip,urlobj.get_ip_type()) return cache_and_send(urlobj,{'code':412,'msg':msg},412) try: url_data = urlobj.get_status() if not created and urlobj.update_time < urlobj.last_time_checked: return cache_and_send(urlobj,{'code': 304, 'msg': 'OK', 'data': url_data},304) else: return cache_and_send(urlobj,{'code': 200, 'msg': 'OK', 'data': url_data},200) except Exception, err: sentry_id = client.get_ident(client.captureException()) logger.error(u"Error processing url %s [Sentry ID '%s']", url,sentry_id) return cache_and_send(urlobj,{'code':500,'msg':'Server error', 'error':unicode(err)},500)