def csp_violation_capture(request):
"""
Capture CSP violation reports, forward to Sentry.
HT @glogiotatidis https://github.com/mozmeao/lumbergh/pull/180
HT @pmac, @jgmize https://github.com/mozilla/bedrock/pull/4335
"""
if not settings.CSP_REPORT_ENABLE:
# mitigation option for a flood of violation reports
return HttpResponse()
data = client.get_data_from_request(request)
data.update({"level": logging.INFO, "logger": "CSP"})
try:
csp_data = json.loads(request.body)
except ValueError:
# Cannot decode CSP violation data, ignore
return HttpResponseBadRequest("Invalid CSP Report")
try:
blocked_uri = csp_data["csp-report"]["blocked-uri"]
except KeyError:
# Incomplete CSP report
return HttpResponseBadRequest("Incomplete CSP Report")
client.captureMessage(message="CSP Violation: {}".format(blocked_uri),
data=data)
return HttpResponse("Captured CSP violation, thanks for reporting.")
def process_response(self, request, response):
if response.status_code < 400 or response.status_code == 404:
return response
from raven.contrib.django.models import client
if not client.is_enabled():
return response
# format error message
message = ("{status_code} code returned for URL: {url}"
"with message: {message}").format(
status_code=response.status_code,
url=request.build_absolute_uri(),
message=str(response.content),
)
# create data representation
data = client.get_data_from_request(request)
data.update({
"level": logging.WARN,
"logger": "BadRequestMiddleware",
})
client.captureMessage(message=message, data=data)
return response
def update_video_feed(video_feed_id):
try:
video_feed = VideoFeed.objects.get(pk=video_feed_id)
video_feed.update()
except VideoFeed:
msg = '**update_video_feed**. VideoFeed does not exist. ID: %s' % video_feed_id
client.captureMessage(msg)
def csp_violation_capture(request):
# HT @glogiotatidis https://github.com/mozmar/lumbergh/pull/180/
if not settings.CSP_REPORT_ENABLE:
# mitigation option for a flood of violation reports
return HttpResponse()
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'CSP',
})
try:
csp_data = json.loads(request.body.decode())
except ValueError:
# Cannot decode CSP violation data, ignore
return HttpResponseBadRequest('Invalid CSP Report')
try:
blocked_uri = csp_data['csp-report']['blocked-uri']
except KeyError:
# Incomplete CSP report
return HttpResponseBadRequest('Incomplete CSP Report')
client.captureMessage(message='CSP Violation: {}'.format(blocked_uri),
data=data)
return HttpResponse('Captured CSP violation, thanks for reporting.')
def sentry_message(message, **kwargs):
"""
Log a message to sentry, including the stack trace and extra variables
Log level can be set by adding a ``log_level`` key to :data:`kwargs`.
Default is ``info``
Example:
.. code-block::python
sentry_message('IMPORT: file received', extra={
'args': kwargs, 'path': path, 'zipped': self.zipped
})
:param message: message text
:type message: string
:param kwargs: context to be logged
:type kwargs: dict
:return: None
"""
from raven.contrib.django.models import client
kwargs['stack'] = True
kwargs['data'] = {'level': kwargs.get('log_level', 'info')}
client.captureMessage(message, **kwargs)
def csp_violation_capture(request):
# HT @glogiotatidis https://github.com/mozmeao/lumbergh/pull/180/
if not settings.CSP_REPORT_ENABLE:
# mitigation option for a flood of violation reports
return HttpResponse()
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'CSP',
})
try:
csp_data = json.loads(request.body)
except ValueError:
# Cannot decode CSP violation data, ignore
return HttpResponseBadRequest('Invalid CSP Report')
try:
blocked_uri = csp_data['csp-report']['blocked-uri']
except KeyError:
# Incomplete CSP report
return HttpResponseBadRequest('Incomplete CSP Report')
client.captureMessage(message='CSP Violation: {}'.format(blocked_uri),
data=data)
return HttpResponse('Captured CSP violation, thanks for reporting.')
def test_simple(self):
assert client.__class__ is SentryInternalClient
with self.tasks():
client.captureMessage('internal client test')
event = Event.objects.get()
assert event.message == 'internal client test'
def test_simple(self):
assert client.__class__ is SentryInternalClient
with self.settings(CELERY_ALWAYS_EAGER=True):
client.captureMessage('internal client test')
event = Event.objects.get()
assert event.message == 'internal client test'
def test_simple(self, send):
assert client.__class__ is SentryInternalClient
with self.tasks():
client.captureMessage("internal client test")
event = Event.objects.get()
assert event.data["sentry.interfaces.Message"]["message"] == "internal client test"
assert send.call_count == 0
def test_simple(self, send):
assert client.__class__ is SentryInternalClient
with self.tasks():
client.captureMessage('internal client test')
event = Event.objects.get()
assert event.data['sentry.interfaces.Message']['message'] == \
'internal client test'
assert send.call_count == 0
def test_simple(self, send):
assert client.__class__ is SentryInternalClient
with self.tasks():
client.captureMessage('internal client test')
event = Event.objects.get()
assert event.data['sentry.interfaces.Message']['message'] == \
'internal client test'
assert send.call_count == 0
def process_exception(self, request, exception):
from raven.contrib.django.models import client
if not client.is_enabled():
return
data = client.get_data_from_request(request)
data.update(
{"level": logging.WARN, "logger": "BadRequestMiddleware",}
)
client.captureMessage(message=str(exception), data=data)
def test_upstream(self, send):
with self.dsn("http://*****:*****@example.com/1"):
with self.options({"sentry:install-id": "abc123"}):
with self.tasks():
client.captureMessage("internal client test")
event = Event.objects.get()
assert event.data["sentry.interfaces.Message"]["message"] == "internal client test"
# Make sure that the event also got sent upstream
assert send.call_count == 1
_, kwargs = send.call_args
# and got tagged properly
assert kwargs["tags"]["install-id"] == "abc123"
def test_upstream(self, send):
with self.dsn('http://*****:*****@example.com/1'):
with self.options({'sentry:install-id': 'abc123'}):
with self.tasks():
client.captureMessage('internal client test')
event = Event.objects.get()
assert event.message == 'internal client test'
# Make sure that the event also got sent upstream
assert send.call_count == 1
_, kwargs = send.call_args
# and got tagged properly
assert kwargs['tags']['install-id'] == 'abc123'
def process_response(self, request, response):
from raven.contrib.django.models import client
if response.status_code < 400 or response.status_code > 450 or is_ignorable_404(
request.get_full_path()) or not client.is_enabled():
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http{0}'.format(str(response.status_code)),
})
result = client.captureMessage(message='{0}: {1}'.format(
getattr(response, 'status_text', 'Page Not Found'),
request.build_absolute_uri()),
data=data)
if not result:
return
request.sentry = {
'project_id': data.get('project', client.remote.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code < 400 or response.status_code == 404:
return response
from raven.contrib.django.models import client
if not client.is_enabled():
return response
data = client.get_data_from_request(request)
data.update({
"level": logging.WARN,
"logger": "BadRequestMiddleware",
})
message_template = "{status_code} code returned for URL: {url} {content}"
content = ""
try:
content = "with message: {}".format(str(response.content))
except:
pass
message = message_template.format(status_code=response.status_code,
url=request.build_absolute_uri(),
content=content)
result = client.captureMessage(message=message, data=data)
return response
def process_response(self, request, response):
if response.status_code != 404:
return response
if is_ignorable_404(request.get_full_path()):
return response
from raven.contrib.django.models import client
if not client.is_enabled():
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' %
request.build_absolute_uri(),
data=data)
if not result:
return
request.sentry = {
'project_id': data.get('project', client.remote.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code != 404:
return response
if is_ignorable_404(request.get_full_path()):
return response
from raven.contrib.django.models import client
if not client.is_enabled():
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data)
if not result:
return
request.sentry = {
'project_id': data.get('project', client.remote.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code != 400:
return response
from raven.contrib.django.models import client
if not client.is_enabled():
return response
data = client.get_data_from_request(request)
# Ignore `django.security.DisallowedHost` errors
if re.search(r'\d+\.\d+\.\d+\.\d+',
data.get('request', {}).get('url', '')):
return response
data.update({
'level': logging.DEBUG,
'logger': 'manual_sentry_logger',
})
result = client.captureMessage(
message='400 response',
data=data,
extra={'response': self._parse_response(response)},
)
if not result:
return response
request.sentry = {
'project_id': data.get('project', client.remote.project),
'id': client.get_ident(result),
}
return response
def check_reporting(app_configs, **kwargs):
errors = []
current_user = pwd.getpwuid(os.getuid()).pw_name
# Get all configs if we don't get a specific set
if not app_configs:
app_configs = apps.apps.app_configs.values()
raven_installed = bool(list(filter(lambda app: app.name == "raven.contrib.django.raven_compat", app_configs)))
if not raven_installed:
errors.append(W001)
else:
raven_config = getattr(settings, 'RAVEN_CONFIG', None)
if raven_config is None or not isinstance(raven_config, dict):
errors.append(E001)
try:
# noinspection PyPackageRequirements,PyUnresolvedReferences
from raven.contrib.django.models import client
# So we are duplicating code from manage.py raven test here...
if not all([client.servers, client.project, client.public_key, client.secret_key]):
errors.append(E002)
if not client.is_enabled():
errors.append(E003)
else:
# Test sending
data = {
'culprit': 'preflight.checks.check_reporting',
'logger': 'raven.test',
'request': {
'method': 'GET',
'url': 'http://example.com',
}
}
ident = client.get_ident(client.captureMessage(
message='This is a test message generated using ``raven test``',
data=data,
level=logging.INFO,
stack=True,
tags={},
extra={
'user': current_user,
'loadavg': os.getloadavg(),
},
))
if client.state.did_fail():
errors.append(E004)
except ImportError:
errors.append(E005)
# TODO: check if we actually log something
return errors
def test_simple(self, send):
assert client.__class__ is SentryInternalClient
with self.tasks():
event_id = client.captureMessage('internal client test')
event = Event.objects.get()
assert event.project_id == settings.SENTRY_PROJECT
assert event.event_id == event_id
assert event.data['sentry.interfaces.Message']['message'] == \
'internal client test'
assert send.call_count == 0
def capture_csp_violation(request):
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'CSP',
})
try:
csp_data = json.loads(request.body)
except ValueError:
# Cannot decode CSP violation data, ignore
return HttpResponseBadRequest('Invalid CSP Report')
try:
blocked_uri = csp_data['csp-report']['blocked-uri']
except KeyError:
# Incomplete CSP report
return HttpResponseBadRequest('Incomplete CSP Report')
client.captureMessage(message='CSP Violation: {}'.format(blocked_uri),
data=data)
return HttpResponse('Captured CSP violation, thanks for reporting.')
def csp_violation_capture(request):
data = sentry_client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'CSP',
})
try:
csp_data = json.loads(request.body)
except ValueError:
# Cannot decode CSP violation data, ignore
return HttpResponseBadRequest('Invalid CSP Report')
try:
blocked_uri = csp_data['csp-report']['blocked-uri']
except KeyError:
# Incomplete CSP report
return HttpResponseBadRequest('Incomplete CSP Report')
sentry_client.captureMessage(
message='CSP Violation: {}'.format(blocked_uri),
data=data)
return HttpResponse('Captured CSP violation, thanks for reporting.')
def execute_search_query(query):
"""
Executes an Elasticsearch query using the globally configured request timeout.
(A warning is also logged if the query takes longer than a set threshold.)
"""
response = query.params(
request_timeout=settings.ES_SEARCH_REQUEST_TIMEOUT).execute()
if response.took >= settings.ES_SEARCH_REQUEST_WARNING_THRESHOLD * 1000:
logger.warning(
f'Elasticsearch query took a long time ({response.took/1000:.2f} seconds)'
)
client.captureMessage(
'Elasticsearch query took a long time',
extra={
'query': query.to_dict(),
'took': response.took,
'timed_out': response.timed_out,
},
)
return response
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(request.get_full_path()):
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' % request.build_absolute_uri(), data=data)
request.sentry = {
'project_id': data.get('project', client.project),
'id': client.get_ident(result),
}
return response
def process_response(self, request, response):
if response.status_code != 404 or _is_ignorable_404(
request.get_full_path()):
return response
data = client.get_data_from_request(request)
data.update({
'level': logging.INFO,
'logger': 'http404',
})
result = client.captureMessage(message='Page Not Found: %s' %
request.build_absolute_uri(),
data=data)
request.sentry = {
'project_id': data.get('project', client.project),
'id': client.get_ident(result),
}
return response
def test_upstream(self, send):
with self.dsn('http://*****:*****@example.com/1'):
with self.options({'sentry:install-id': 'abc123'}):
with self.tasks():
event_id = client.captureMessage('internal client test')
event = Event.objects.get()
assert event.project_id == settings.SENTRY_PROJECT
assert event.event_id == event_id
assert event.data['sentry.interfaces.Message']['message'] == \
'internal client test'
# Make sure that the event also got sent upstream
assert send.call_count == 1
_, kwargs = send.call_args
# and got tagged properly
assert kwargs['tags']['install-id'] == 'abc123'
def handle(self, *args, **options):
jobs_added = 0
jobs_updated = 0
jobs_removed = 0
job_ids = []
response = requests.get(GREENHOUSE_URL.format(settings.GREENHOUSE_BOARD_TOKEN))
response.raise_for_status()
data = response.json()
for job in data['jobs']:
# Maybe GH sometimes includes jobs with the same ID multiple times
# in the json. Capture the event in Sentry and look the other way.
if job['id'] in job_ids:
client.captureMessage(
message='[GH Sync] Job {} twice in the same json'.format(job['id']),
data={'extra': {'jobs': data['jobs']}})
continue
job_ids.append(job['id'])
job_object, created = (Position.objects
.get_or_create(job_id=job['id'], source='gh'))
departments = job.get('departments', '')
if departments:
department = departments[0]['name'] or ''
else:
department = ''
offices = job.get('offices', '')
if offices:
location = ','.join([office['name'] for office in offices])
else:
location = ''
description = H.unescape(job.get('content', ''))
description = cleaner.clean(description)
# Remove empty paragraphs and h4s and paragraphs with \xa0
# (no-brake space). I ♥ regex
description = re.sub(r'<(p|h4)>([ ]*|(\xa0)+)</(p|h4)>', '', description)
for metadata in job.get('metadata', []):
if metadata.get('name', '') == 'Employment Type':
position_type = metadata['value'] or ''
break
else:
position_type = ''
object_data = {
'title': job['title'],
'department': department,
'location': location,
'description': description,
'position_type': position_type,
'apply_url': job['absolute_url'],
}
changed = False
for key, value in object_data.items():
if getattr(job_object, key, None) != value:
changed = True
setattr(job_object, key, value)
if changed:
if created:
jobs_added += 1
else:
jobs_updated += 1
job_object.save()
positions_to_be_removed = Position.objects.exclude(job_id__in=job_ids, source='gh')
jobs_removed = positions_to_be_removed.count()
positions_to_be_removed.delete()
self.stdout.write(
'Jobs added: {added} updated: {updated} '
'removed: {removed}'.format(added=jobs_added,
updated=jobs_updated,
removed=jobs_removed))
def test_basic(self, captureMessage):
client.captureMessage(message='foo')
captureMessage.assert_called_once_with(message='foo')
def handle(self, *args, **options):
jobs_added = 0
jobs_updated = 0
jobs_removed = 0
job_ids = []
response = requests.get(
GREENHOUSE_URL.format(settings.GREENHOUSE_BOARD_TOKEN))
response.raise_for_status()
data = response.json()
for job in data['jobs']:
# Maybe GH sometimes includes jobs with the same ID multiple times
# in the json. Capture the event in Sentry and look the other way.
if job['id'] in job_ids:
client.captureMessage(
message='[GH Sync] Job {} twice in the same json'.format(
job['id']),
data={'extra': {
'jobs': data['jobs']
}})
continue
job_ids.append(job['id'])
job_object, created = (Position.objects.get_or_create(
job_id=job['id'], source='gh'))
departments = job.get('departments', '')
if departments:
department = departments[0]['name'] or ''
else:
department = ''
offices = job.get('offices', '')
if offices:
location = ','.join([office['name'] for office in offices])
else:
location = ''
description = H.unescape(job.get('content', ''))
description = cleaner.clean(description)
# Remove empty paragraphs and h4s and paragraphs with \xa0
# (no-brake space). I ♥ regex
description = re.sub(r'<(p|h4)>([ ]*|(\xa0)+)</(p|h4)>', '',
description)
for metadata in job.get('metadata', []):
if metadata.get('name', '') == 'Employment Type':
position_type = metadata['value'] or ''
break
else:
position_type = ''
object_data = {
'title': job['title'],
'department': department,
'location': location,
'description': description,
'position_type': position_type,
'apply_url': job['absolute_url'],
}
changed = False
for key, value in object_data.items():
if getattr(job_object, key, None) != value:
changed = True
setattr(job_object, key, value)
if changed:
if created:
jobs_added += 1
else:
jobs_updated += 1
job_object.save()
positions_to_be_removed = Position.objects.exclude(job_id__in=job_ids,
source='gh')
jobs_removed = positions_to_be_removed.count()
positions_to_be_removed.delete()
self.stdout.write('Jobs added: {added} updated: {updated} '
'removed: {removed}'.format(added=jobs_added,
updated=jobs_updated,
removed=jobs_removed))
def check_reporting(app_configs, **kwargs):
errors = []
current_user = pwd.getpwuid(os.getuid()).pw_name
# Get all configs if we don't get a specific set
if not app_configs:
app_configs = apps.apps.app_configs.values()
raven_installed = bool(
list(
filter(lambda app: app.name == "raven.contrib.django.raven_compat",
app_configs)))
if not raven_installed:
errors.append(W001)
else:
raven_config = getattr(settings, 'RAVEN_CONFIG', None)
if raven_config is None or not isinstance(raven_config, dict):
errors.append(E001)
try:
# noinspection PyPackageRequirements,PyUnresolvedReferences
from raven.contrib.django.models import client
# So we are duplicating code from manage.py raven test here...
if not all([
client.servers, client.project, client.public_key,
client.secret_key
]):
errors.append(E002)
if not client.is_enabled():
errors.append(E003)
else:
# Test sending
data = {
'culprit': 'preflight.checks.check_reporting',
'logger': 'raven.test',
'request': {
'method': 'GET',
'url': 'http://example.com',
}
}
ident = client.get_ident(
client.captureMessage(
message=
'This is a test message generated using ``raven test``',
data=data,
level=logging.INFO,
stack=True,
tags={},
extra={
'user': current_user,
'loadavg': os.getloadavg(),
},
))
if client.state.did_fail():
errors.append(E004)
except ImportError:
errors.append(E005)
# TODO: check if we actually log something
return errors
def test_basic(self, captureMessage):
client.captureMessage(message='foo')
captureMessage.assert_called_once_with(message='foo')
def capture_event(request):
client.captureMessage('test')
return HttpResponse('')
