def csp_violation_capture(request): """ Capture CSP violation reports, forward to Sentry. HT @glogiotatidis https://github.com/mozmeao/lumbergh/pull/180 HT @pmac, @jgmize https://github.com/mozilla/bedrock/pull/4335 """ if not settings.CSP_REPORT_ENABLE: # mitigation option for a flood of violation reports return HttpResponse() data = client.get_data_from_request(request) data.update({"level": logging.INFO, "logger": "CSP"}) try: csp_data = json.loads(request.body) except ValueError: # Cannot decode CSP violation data, ignore return HttpResponseBadRequest("Invalid CSP Report") try: blocked_uri = csp_data["csp-report"]["blocked-uri"] except KeyError: # Incomplete CSP report return HttpResponseBadRequest("Incomplete CSP Report") client.captureMessage(message="CSP Violation: {}".format(blocked_uri), data=data) return HttpResponse("Captured CSP violation, thanks for reporting.")
def process_response(self, request, response): if response.status_code < 400 or response.status_code == 404: return response from raven.contrib.django.models import client if not client.is_enabled(): return response # format error message message = ("{status_code} code returned for URL: {url}" "with message: {message}").format( status_code=response.status_code, url=request.build_absolute_uri(), message=str(response.content), ) # create data representation data = client.get_data_from_request(request) data.update({ "level": logging.WARN, "logger": "BadRequestMiddleware", }) client.captureMessage(message=message, data=data) return response
def update_video_feed(video_feed_id): try: video_feed = VideoFeed.objects.get(pk=video_feed_id) video_feed.update() except VideoFeed: msg = '**update_video_feed**. VideoFeed does not exist. ID: %s' % video_feed_id client.captureMessage(msg)
def csp_violation_capture(request): # HT @glogiotatidis https://github.com/mozmar/lumbergh/pull/180/ if not settings.CSP_REPORT_ENABLE: # mitigation option for a flood of violation reports return HttpResponse() data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'CSP', }) try: csp_data = json.loads(request.body.decode()) except ValueError: # Cannot decode CSP violation data, ignore return HttpResponseBadRequest('Invalid CSP Report') try: blocked_uri = csp_data['csp-report']['blocked-uri'] except KeyError: # Incomplete CSP report return HttpResponseBadRequest('Incomplete CSP Report') client.captureMessage(message='CSP Violation: {}'.format(blocked_uri), data=data) return HttpResponse('Captured CSP violation, thanks for reporting.')
def sentry_message(message, **kwargs): """ Log a message to sentry, including the stack trace and extra variables Log level can be set by adding a ``log_level`` key to :data:`kwargs`. Default is ``info`` Example: .. code-block::python sentry_message('IMPORT: file received', extra={ 'args': kwargs, 'path': path, 'zipped': self.zipped }) :param message: message text :type message: string :param kwargs: context to be logged :type kwargs: dict :return: None """ from raven.contrib.django.models import client kwargs['stack'] = True kwargs['data'] = {'level': kwargs.get('log_level', 'info')} client.captureMessage(message, **kwargs)
def csp_violation_capture(request): # HT @glogiotatidis https://github.com/mozmeao/lumbergh/pull/180/ if not settings.CSP_REPORT_ENABLE: # mitigation option for a flood of violation reports return HttpResponse() data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'CSP', }) try: csp_data = json.loads(request.body) except ValueError: # Cannot decode CSP violation data, ignore return HttpResponseBadRequest('Invalid CSP Report') try: blocked_uri = csp_data['csp-report']['blocked-uri'] except KeyError: # Incomplete CSP report return HttpResponseBadRequest('Incomplete CSP Report') client.captureMessage(message='CSP Violation: {}'.format(blocked_uri), data=data) return HttpResponse('Captured CSP violation, thanks for reporting.')
def test_simple(self): assert client.__class__ is SentryInternalClient with self.tasks(): client.captureMessage('internal client test') event = Event.objects.get() assert event.message == 'internal client test'
def test_simple(self): assert client.__class__ is SentryInternalClient with self.settings(CELERY_ALWAYS_EAGER=True): client.captureMessage('internal client test') event = Event.objects.get() assert event.message == 'internal client test'
def test_simple(self, send): assert client.__class__ is SentryInternalClient with self.tasks(): client.captureMessage("internal client test") event = Event.objects.get() assert event.data["sentry.interfaces.Message"]["message"] == "internal client test" assert send.call_count == 0
def test_simple(self, send): assert client.__class__ is SentryInternalClient with self.tasks(): client.captureMessage('internal client test') event = Event.objects.get() assert event.data['sentry.interfaces.Message']['message'] == \ 'internal client test' assert send.call_count == 0
def test_simple(self, send): assert client.__class__ is SentryInternalClient with self.tasks(): client.captureMessage('internal client test') event = Event.objects.get() assert event.data['sentry.interfaces.Message']['message'] == \ 'internal client test' assert send.call_count == 0
def process_exception(self, request, exception): from raven.contrib.django.models import client if not client.is_enabled(): return data = client.get_data_from_request(request) data.update( {"level": logging.WARN, "logger": "BadRequestMiddleware",} ) client.captureMessage(message=str(exception), data=data)
def test_upstream(self, send): with self.dsn("http://*****:*****@example.com/1"): with self.options({"sentry:install-id": "abc123"}): with self.tasks(): client.captureMessage("internal client test") event = Event.objects.get() assert event.data["sentry.interfaces.Message"]["message"] == "internal client test" # Make sure that the event also got sent upstream assert send.call_count == 1 _, kwargs = send.call_args # and got tagged properly assert kwargs["tags"]["install-id"] == "abc123"
def test_upstream(self, send): with self.dsn('http://*****:*****@example.com/1'): with self.options({'sentry:install-id': 'abc123'}): with self.tasks(): client.captureMessage('internal client test') event = Event.objects.get() assert event.message == 'internal client test' # Make sure that the event also got sent upstream assert send.call_count == 1 _, kwargs = send.call_args # and got tagged properly assert kwargs['tags']['install-id'] == 'abc123'
def process_response(self, request, response): from raven.contrib.django.models import client if response.status_code < 400 or response.status_code > 450 or is_ignorable_404( request.get_full_path()) or not client.is_enabled(): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http{0}'.format(str(response.status_code)), }) result = client.captureMessage(message='{0}: {1}'.format( getattr(response, 'status_text', 'Page Not Found'), request.build_absolute_uri()), data=data) if not result: return request.sentry = { 'project_id': data.get('project', client.remote.project), 'id': client.get_ident(result), } return response
def process_response(self, request, response): if response.status_code < 400 or response.status_code == 404: return response from raven.contrib.django.models import client if not client.is_enabled(): return response data = client.get_data_from_request(request) data.update({ "level": logging.WARN, "logger": "BadRequestMiddleware", }) message_template = "{status_code} code returned for URL: {url} {content}" content = "" try: content = "with message: {}".format(str(response.content)) except: pass message = message_template.format(status_code=response.status_code, url=request.build_absolute_uri(), content=content) result = client.captureMessage(message=message, data=data) return response
def process_response(self, request, response): if response.status_code != 404: return response if is_ignorable_404(request.get_full_path()): return response from raven.contrib.django.models import client if not client.is_enabled(): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http404', }) result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data) if not result: return request.sentry = { 'project_id': data.get('project', client.remote.project), 'id': client.get_ident(result), } return response
def process_response(self, request, response): if response.status_code != 404: return response if is_ignorable_404(request.get_full_path()): return response from raven.contrib.django.models import client if not client.is_enabled(): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http404', }) result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data) if not result: return request.sentry = { 'project_id': data.get('project', client.remote.project), 'id': client.get_ident(result), } return response
def process_response(self, request, response): if response.status_code != 400: return response from raven.contrib.django.models import client if not client.is_enabled(): return response data = client.get_data_from_request(request) # Ignore `django.security.DisallowedHost` errors if re.search(r'\d+\.\d+\.\d+\.\d+', data.get('request', {}).get('url', '')): return response data.update({ 'level': logging.DEBUG, 'logger': 'manual_sentry_logger', }) result = client.captureMessage( message='400 response', data=data, extra={'response': self._parse_response(response)}, ) if not result: return response request.sentry = { 'project_id': data.get('project', client.remote.project), 'id': client.get_ident(result), } return response
def check_reporting(app_configs, **kwargs): errors = [] current_user = pwd.getpwuid(os.getuid()).pw_name # Get all configs if we don't get a specific set if not app_configs: app_configs = apps.apps.app_configs.values() raven_installed = bool(list(filter(lambda app: app.name == "raven.contrib.django.raven_compat", app_configs))) if not raven_installed: errors.append(W001) else: raven_config = getattr(settings, 'RAVEN_CONFIG', None) if raven_config is None or not isinstance(raven_config, dict): errors.append(E001) try: # noinspection PyPackageRequirements,PyUnresolvedReferences from raven.contrib.django.models import client # So we are duplicating code from manage.py raven test here... if not all([client.servers, client.project, client.public_key, client.secret_key]): errors.append(E002) if not client.is_enabled(): errors.append(E003) else: # Test sending data = { 'culprit': 'preflight.checks.check_reporting', 'logger': 'raven.test', 'request': { 'method': 'GET', 'url': 'http://example.com', } } ident = client.get_ident(client.captureMessage( message='This is a test message generated using ``raven test``', data=data, level=logging.INFO, stack=True, tags={}, extra={ 'user': current_user, 'loadavg': os.getloadavg(), }, )) if client.state.did_fail(): errors.append(E004) except ImportError: errors.append(E005) # TODO: check if we actually log something return errors
def test_simple(self, send): assert client.__class__ is SentryInternalClient with self.tasks(): event_id = client.captureMessage('internal client test') event = Event.objects.get() assert event.project_id == settings.SENTRY_PROJECT assert event.event_id == event_id assert event.data['sentry.interfaces.Message']['message'] == \ 'internal client test' assert send.call_count == 0
def capture_csp_violation(request): data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'CSP', }) try: csp_data = json.loads(request.body) except ValueError: # Cannot decode CSP violation data, ignore return HttpResponseBadRequest('Invalid CSP Report') try: blocked_uri = csp_data['csp-report']['blocked-uri'] except KeyError: # Incomplete CSP report return HttpResponseBadRequest('Incomplete CSP Report') client.captureMessage(message='CSP Violation: {}'.format(blocked_uri), data=data) return HttpResponse('Captured CSP violation, thanks for reporting.')
def csp_violation_capture(request): data = sentry_client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'CSP', }) try: csp_data = json.loads(request.body) except ValueError: # Cannot decode CSP violation data, ignore return HttpResponseBadRequest('Invalid CSP Report') try: blocked_uri = csp_data['csp-report']['blocked-uri'] except KeyError: # Incomplete CSP report return HttpResponseBadRequest('Incomplete CSP Report') sentry_client.captureMessage( message='CSP Violation: {}'.format(blocked_uri), data=data) return HttpResponse('Captured CSP violation, thanks for reporting.')
def execute_search_query(query): """ Executes an Elasticsearch query using the globally configured request timeout. (A warning is also logged if the query takes longer than a set threshold.) """ response = query.params( request_timeout=settings.ES_SEARCH_REQUEST_TIMEOUT).execute() if response.took >= settings.ES_SEARCH_REQUEST_WARNING_THRESHOLD * 1000: logger.warning( f'Elasticsearch query took a long time ({response.took/1000:.2f} seconds)' ) client.captureMessage( 'Elasticsearch query took a long time', extra={ 'query': query.to_dict(), 'took': response.took, 'timed_out': response.timed_out, }, ) return response
def process_response(self, request, response): if response.status_code != 404 or _is_ignorable_404(request.get_full_path()): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http404', }) result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data) request.sentry = { 'project_id': data.get('project', client.project), 'id': client.get_ident(result), } return response
def process_response(self, request, response): if response.status_code != 404 or _is_ignorable_404( request.get_full_path()): return response data = client.get_data_from_request(request) data.update({ 'level': logging.INFO, 'logger': 'http404', }) result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data) request.sentry = { 'project_id': data.get('project', client.project), 'id': client.get_ident(result), } return response
def test_upstream(self, send): with self.dsn('http://*****:*****@example.com/1'): with self.options({'sentry:install-id': 'abc123'}): with self.tasks(): event_id = client.captureMessage('internal client test') event = Event.objects.get() assert event.project_id == settings.SENTRY_PROJECT assert event.event_id == event_id assert event.data['sentry.interfaces.Message']['message'] == \ 'internal client test' # Make sure that the event also got sent upstream assert send.call_count == 1 _, kwargs = send.call_args # and got tagged properly assert kwargs['tags']['install-id'] == 'abc123'
def handle(self, *args, **options): jobs_added = 0 jobs_updated = 0 jobs_removed = 0 job_ids = [] response = requests.get(GREENHOUSE_URL.format(settings.GREENHOUSE_BOARD_TOKEN)) response.raise_for_status() data = response.json() for job in data['jobs']: # Maybe GH sometimes includes jobs with the same ID multiple times # in the json. Capture the event in Sentry and look the other way. if job['id'] in job_ids: client.captureMessage( message='[GH Sync] Job {} twice in the same json'.format(job['id']), data={'extra': {'jobs': data['jobs']}}) continue job_ids.append(job['id']) job_object, created = (Position.objects .get_or_create(job_id=job['id'], source='gh')) departments = job.get('departments', '') if departments: department = departments[0]['name'] or '' else: department = '' offices = job.get('offices', '') if offices: location = ','.join([office['name'] for office in offices]) else: location = '' description = H.unescape(job.get('content', '')) description = cleaner.clean(description) # Remove empty paragraphs and h4s and paragraphs with \xa0 # (no-brake space). I ♥ regex description = re.sub(r'<(p|h4)>([ ]*|(\xa0)+)</(p|h4)>', '', description) for metadata in job.get('metadata', []): if metadata.get('name', '') == 'Employment Type': position_type = metadata['value'] or '' break else: position_type = '' object_data = { 'title': job['title'], 'department': department, 'location': location, 'description': description, 'position_type': position_type, 'apply_url': job['absolute_url'], } changed = False for key, value in object_data.items(): if getattr(job_object, key, None) != value: changed = True setattr(job_object, key, value) if changed: if created: jobs_added += 1 else: jobs_updated += 1 job_object.save() positions_to_be_removed = Position.objects.exclude(job_id__in=job_ids, source='gh') jobs_removed = positions_to_be_removed.count() positions_to_be_removed.delete() self.stdout.write( 'Jobs added: {added} updated: {updated} ' 'removed: {removed}'.format(added=jobs_added, updated=jobs_updated, removed=jobs_removed))
def test_basic(self, captureMessage): client.captureMessage(message='foo') captureMessage.assert_called_once_with(message='foo')
def handle(self, *args, **options): jobs_added = 0 jobs_updated = 0 jobs_removed = 0 job_ids = [] response = requests.get( GREENHOUSE_URL.format(settings.GREENHOUSE_BOARD_TOKEN)) response.raise_for_status() data = response.json() for job in data['jobs']: # Maybe GH sometimes includes jobs with the same ID multiple times # in the json. Capture the event in Sentry and look the other way. if job['id'] in job_ids: client.captureMessage( message='[GH Sync] Job {} twice in the same json'.format( job['id']), data={'extra': { 'jobs': data['jobs'] }}) continue job_ids.append(job['id']) job_object, created = (Position.objects.get_or_create( job_id=job['id'], source='gh')) departments = job.get('departments', '') if departments: department = departments[0]['name'] or '' else: department = '' offices = job.get('offices', '') if offices: location = ','.join([office['name'] for office in offices]) else: location = '' description = H.unescape(job.get('content', '')) description = cleaner.clean(description) # Remove empty paragraphs and h4s and paragraphs with \xa0 # (no-brake space). I ♥ regex description = re.sub(r'<(p|h4)>([ ]*|(\xa0)+)</(p|h4)>', '', description) for metadata in job.get('metadata', []): if metadata.get('name', '') == 'Employment Type': position_type = metadata['value'] or '' break else: position_type = '' object_data = { 'title': job['title'], 'department': department, 'location': location, 'description': description, 'position_type': position_type, 'apply_url': job['absolute_url'], } changed = False for key, value in object_data.items(): if getattr(job_object, key, None) != value: changed = True setattr(job_object, key, value) if changed: if created: jobs_added += 1 else: jobs_updated += 1 job_object.save() positions_to_be_removed = Position.objects.exclude(job_id__in=job_ids, source='gh') jobs_removed = positions_to_be_removed.count() positions_to_be_removed.delete() self.stdout.write('Jobs added: {added} updated: {updated} ' 'removed: {removed}'.format(added=jobs_added, updated=jobs_updated, removed=jobs_removed))
def check_reporting(app_configs, **kwargs): errors = [] current_user = pwd.getpwuid(os.getuid()).pw_name # Get all configs if we don't get a specific set if not app_configs: app_configs = apps.apps.app_configs.values() raven_installed = bool( list( filter(lambda app: app.name == "raven.contrib.django.raven_compat", app_configs))) if not raven_installed: errors.append(W001) else: raven_config = getattr(settings, 'RAVEN_CONFIG', None) if raven_config is None or not isinstance(raven_config, dict): errors.append(E001) try: # noinspection PyPackageRequirements,PyUnresolvedReferences from raven.contrib.django.models import client # So we are duplicating code from manage.py raven test here... if not all([ client.servers, client.project, client.public_key, client.secret_key ]): errors.append(E002) if not client.is_enabled(): errors.append(E003) else: # Test sending data = { 'culprit': 'preflight.checks.check_reporting', 'logger': 'raven.test', 'request': { 'method': 'GET', 'url': 'http://example.com', } } ident = client.get_ident( client.captureMessage( message= 'This is a test message generated using ``raven test``', data=data, level=logging.INFO, stack=True, tags={}, extra={ 'user': current_user, 'loadavg': os.getloadavg(), }, )) if client.state.did_fail(): errors.append(E004) except ImportError: errors.append(E005) # TODO: check if we actually log something return errors
def test_basic(self, captureMessage): client.captureMessage(message='foo') captureMessage.assert_called_once_with(message='foo')
def capture_event(request): client.captureMessage('test') return HttpResponse('')