def post(self, request, project, auth, helper, **kwargs):
data = helper.safely_load_json_string(request.body)
# Do origin check based on the `document-uri` key as explained
# in `_dispatch`.
try:
report = data['csp-report']
except KeyError:
raise APIError('Missing csp-report')
origin = report.get('document-uri')
# No idea, but this is garbage
if origin == 'about:blank':
raise APIForbidden('Invalid document-uri')
if not is_valid_origin(origin, project):
raise APIForbidden('Invalid document-uri')
response_or_event_id = self.process(request,
project=project,
auth=auth,
helper=helper,
data=report,
**kwargs)
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
return HttpResponse(status=201)
def post(self, request, project, helper, **kwargs):
data = helper.safely_load_json_string(request.body)
# Do origin check based on the `document-uri` key as explained
# in `_dispatch`.
try:
report = data['csp-report']
except KeyError:
raise APIError('Missing csp-report')
origin = report.get('document-uri')
# No idea, but this is garbage
if origin == 'about:blank':
raise APIForbidden('Invalid document-uri')
if not is_valid_origin(origin, project):
if project:
tsdb.incr(tsdb.models.project_total_received_cors, project.id)
raise APIForbidden('Invalid document-uri')
# Attach on collected meta data. This data obviously isn't a part
# of the spec, but we need to append to the report sentry specific things.
report['_meta'] = {
'release': request.GET.get('sentry_release'),
}
response_or_event_id = self.process(request,
project=project,
helper=helper,
data=report,
**kwargs)
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
return HttpResponse(status=201)
def post(self, request, project, helper, key, **kwargs):
json_body = safely_load_json_string(request.body)
report_type = self.security_report_type(json_body)
if report_type is None:
raise APIError('Unrecognized security report type')
interface = get_interface(report_type)
try:
instance = interface.from_raw(json_body)
except jsonschema.ValidationError as e:
raise APIError('Invalid security report: %s' %
str(e).splitlines()[0])
# Do origin check based on the `document-uri` key as explained in `_dispatch`.
origin = instance.get_origin()
if not is_valid_origin(origin, project):
if project:
track_outcome(project.organization_id, project.id, key.id,
'invalid', FilterStatKeys.CORS)
raise APIForbidden('Invalid origin')
data = {
'interface': interface.path,
'report': instance,
'release': request.GET.get('sentry_release'),
'environment': request.GET.get('sentry_environment'),
}
self.process(request,
project=project,
helper=helper,
data=data,
key=key,
**kwargs)
return HttpResponse(content_type='application/javascript', status=201)
def _dispatch(self, request, helper, project_config, origin=None, *args, **kwargs):
request.user = AnonymousUser()
project = project_config.project
config = project_config.config
allowed = config.get('allowed_domains')
if origin is not None:
if not is_valid_origin(origin, allowed=allowed):
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
FilterStatKeys.CORS)
raise APIForbidden('Invalid origin: %s' % (origin,))
auth = self._parse_header(request, project_config)
key = helper.project_key_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if six.text_type(key.project_id) != six.text_type(project_config.project_id):
raise APIError('Two different projects were specified')
helper.context.bind_auth(auth)
response = super(APIView, self).dispatch(
request=request, project=project, auth=auth, helper=helper, key=key,
project_config=project_config, **kwargs
)
return response
def post(self, request, project, event_id, project_config, **kwargs):
if not features.has("organizations:event-attachments",
project.organization,
actor=request.user):
raise APIForbidden(
"Event attachments are not enabled for this organization.")
project_id = project_config.project_id
if len(request.FILES) == 0:
return HttpResponse(status=400)
for name, uploaded_file in six.iteritems(request.FILES):
file = File.objects.create(
name=uploaded_file.name,
type="event.attachment",
headers={"Content-Type": uploaded_file.content_type},
)
file.putfile(uploaded_file)
EventAttachment.objects.create(project_id=project_id,
event_id=event_id,
name=uploaded_file.name,
file=file)
return HttpResponse(status=201)
def post(self, request, project, helper, **kwargs):
json_body = safely_load_json_string(request.body)
report_type = self.security_report_type(json_body)
if report_type is None:
raise APIError('Unrecognized security report type')
interface = get_interface(report_type)
try:
instance = interface.from_raw(json_body)
except jsonschema.ValidationError as e:
raise APIError('Invalid security report: %s' %
str(e).splitlines()[0])
# Do origin check based on the `document-uri` key as explained in `_dispatch`.
origin = instance.get_origin()
if not is_valid_origin(origin, project):
if project:
tsdb.incr(tsdb.models.project_total_received_cors, project.id)
raise APIForbidden('Invalid origin')
data = {
'interface': interface.path,
'report': instance,
'release': request.GET.get('sentry_release'),
'environment': request.GET.get('sentry_environment'),
}
response_or_event_id = self.process(request,
project=project,
helper=helper,
data=data,
**kwargs)
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
return HttpResponse(content_type='application/javascript', status=201)
def process(self, request, project, key, auth, helper, data, attachments=None, **kwargs):
metrics.incr('events.total', skip_internal=False)
if not data:
raise APIError('No JSON data was found')
remote_addr = request.META['REMOTE_ADDR']
event_manager = EventManager(
data,
project=project,
key=key,
auth=auth,
client_ip=remote_addr,
user_agent=helper.context.agent,
version=auth.version,
content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''),
)
del data
self.pre_normalize(event_manager, helper)
event_manager.normalize()
data = event_manager.get_data()
dict_data = dict(data)
data_size = len(json.dumps(dict_data))
if data_size > 10000000:
metrics.timing('events.size.rejected', data_size)
raise APIForbidden("Event size exceeded 10MB after normalization.")
return process_event(event_manager, project,
key, remote_addr, helper, attachments)
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
Raven.tags_context(helper.context.get_tags_context())
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
tsdb.incr(tsdb.models.project_total_received_cors,
project.id)
raise APIForbidden('Invalid origin: %s' % (origin, ))
# XXX: It seems that the OPTIONS call does not always include custom headers
if request.method == 'OPTIONS':
response = self.options(request, project)
else:
auth = self._parse_header(request, helper, project)
key = helper.project_key_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
project = Project.objects.get_from_cache(id=key.project_id)
helper.context.bind_project(project)
elif key.project_id != project.id:
raise APIError('Two different projects were specified')
helper.context.bind_auth(auth)
Raven.tags_context(helper.context.get_tags_context())
# Explicitly bind Organization so we don't implicitly query it later
# this just allows us to comfortably assure that `project.organization` is safe.
# This also allows us to pull the object from cache, instead of being
# implicitly fetched from database.
project.organization = Organization.objects.get_from_cache(
id=project.organization_id)
response = super(APIView, self).dispatch(
request=request, project=project, auth=auth, helper=helper, key=key, **kwargs
)
if origin:
if origin == 'null':
# If an Origin is `null`, but we got this far, that means
# we've gotten past our CORS check for some reason. But the
# problem is that we can't return "null" as a valid response
# to `Access-Control-Allow-Origin` and we don't have another
# value to work with, so just allow '*' since they've gotten
# this far.
response['Access-Control-Allow-Origin'] = '*'
else:
response['Access-Control-Allow-Origin'] = origin
return response
def process(self,
request,
project,
key,
auth,
helper,
data,
project_config,
attachments=None,
**kwargs):
disable_transaction_events()
metrics.incr("events.total", skip_internal=False)
project_id = project_config.project_id
organization_id = project_config.organization_id
if not data:
track_outcome(organization_id, project_id, key.id, Outcome.INVALID,
"no_data")
raise APIError("No JSON data was found")
remote_addr = request.META["REMOTE_ADDR"]
event_manager = EventManager(
data,
project=project,
key=key,
auth=auth,
client_ip=remote_addr,
user_agent=helper.context.agent,
version=auth.version,
content_encoding=request.META.get("HTTP_CONTENT_ENCODING", ""),
project_config=project_config,
)
del data
self.pre_normalize(event_manager, helper)
event_manager.normalize()
data = event_manager.get_data()
dict_data = dict(data)
data_size = len(json.dumps(dict_data))
if data_size > 10000000:
metrics.timing("events.size.rejected", data_size)
track_outcome(
organization_id,
project_id,
key.id,
Outcome.INVALID,
"too_large",
event_id=dict_data.get("event_id"),
)
raise APIForbidden("Event size exceeded 10MB after normalization.")
metrics.timing("events.size.data.post_storeendpoint", data_size)
return process_event(event_manager, project, key, remote_addr, helper,
attachments, project_config)
def post(self, request, project, helper, key, project_config, **kwargs):
json_body = safely_load_json_string(request.body)
report_type = self.security_report_type(json_body)
if report_type is None:
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
"security_report_type",
)
raise APIError("Unrecognized security report type")
interface = get_interface(report_type)
try:
instance = interface.from_raw(json_body)
except jsonschema.ValidationError as e:
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
"security_report",
)
raise APIError("Invalid security report: %s" % str(e).splitlines()[0])
# Do origin check based on the `document-uri` key as explained in `_dispatch`.
origin = instance.get_origin()
if not is_valid_origin(origin, project):
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
FilterStatKeys.CORS,
)
raise APIForbidden("Invalid origin")
data = {
"interface": interface.path,
"report": instance,
"release": request.GET.get("sentry_release"),
"environment": request.GET.get("sentry_environment"),
}
self.process(
request,
project=project,
helper=helper,
data=data,
key=key,
project_config=project_config,
**kwargs
)
return HttpResponse(content_type="application/javascript", status=201)
def process(self, request, project, auth, data, **kwargs):
event_received.send_robust(ip=request.META['REMOTE_ADDR'],
sender=type(self))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited, project=project)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
if rate_limit is not None and rate_limit.is_limited:
raise APIRateLimited(rate_limit.retry_after)
result = plugins.first('has_perm', request.user, 'create_event',
project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '')
if content_encoding == 'gzip':
data = decompress_gzip(data)
elif content_encoding == 'deflate':
data = decompress_deflate(data)
elif not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData as e:
raise APIError(u'Invalid data: %s (%s)' %
(six.text_type(e), type(e)))
# mutates data
manager = EventManager(data)
data = manager.normalize()
# insert IP address if not available
if auth.is_public:
ensure_has_ip(data, request.META['REMOTE_ADDR'])
event_id = data['event_id']
# We filter data immediately before it ever gets into the queue
inst = SensitiveDataFilter()
inst.apply(data)
# mutates data (strips a lot of context if not queued)
insert_data_to_database(data)
logger.debug('New event from project %s/%s (id=%s)', project.team.slug,
project.slug, event_id)
return event_id
def post(self, request, project, auth, helper, **kwargs):
data = helper.safely_load_json_string(request.body)
# Do origin check based on the `document-uri` key as explained
# in `_dispatch`.
try:
report = data['csp-report']
except KeyError:
raise APIError('Missing csp-report')
origin = report.get('document-uri')
# No idea, but this is garbage
if origin == 'about:blank':
raise APIForbidden('Invalid document-uri')
if not is_valid_origin(origin, project):
raise APIForbidden('Invalid document-uri')
# An invalid CSP report must go against quota
if not is_valid_csp_report(report, project):
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_blacklisted, project.id),
(app.tsdb.models.organization_total_received, project.organization_id),
(app.tsdb.models.organization_total_blacklisted, project.organization_id),
])
metrics.incr('events.blacklisted')
raise APIForbidden('Rejected CSP report')
response_or_event_id = self.process(
request,
project=project,
auth=auth,
helper=helper,
data=report,
**kwargs
)
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
return HttpResponse(status=201)
def _dispatch(self,
request,
helper,
project_id=None,
origin=None,
*args,
**kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
track_outcome(project.organization_id, project.id, None,
Outcome.INVALID, FilterStatKeys.CORS)
raise APIForbidden('Invalid origin: %s' % (origin, ))
auth = self._parse_header(request, helper, project)
key = helper.project_key_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
project = Project.objects.get_from_cache(id=key.project_id)
helper.context.bind_project(project)
elif key.project_id != project.id:
raise APIError('Two different projects were specified')
helper.context.bind_auth(auth)
# Explicitly bind Organization so we don't implicitly query it later
# this just allows us to comfortably assure that `project.organization` is safe.
# This also allows us to pull the object from cache, instead of being
# implicitly fetched from database.
project.organization = Organization.objects.get_from_cache(
id=project.organization_id)
response = super(APIView, self).dispatch(request=request,
project=project,
auth=auth,
helper=helper,
key=key,
**kwargs)
return response
def process(self, request, project, auth, data, **kwargs):
result = plugins.first('has_perm', request.user, 'create_event',
project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
if not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
validate_data(project, data, auth.client)
except InvalidData, e:
raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e)))
def process_csp_report(self):
"""Only called from the CSP report endpoint."""
data = self._data
try:
interface = get_interface(data.pop('interface'))
report = data.pop('report')
except KeyError:
raise APIForbidden('No report or interface data')
# To support testing, we can either accept a built interface instance, or the raw data in
# which case we build the instance ourselves
try:
instance = (
report if isinstance(report, interface) else interface.from_raw(report)
)
except jsonschema.ValidationError as e:
raise APIError('Invalid security report: %s' % str(e).splitlines()[0])
def clean(d):
return dict(filter(lambda x: x[1], d.items()))
data.update(
{
'logger': 'csp',
'message': instance.get_message(),
'culprit': instance.get_culprit(),
instance.path: instance.to_json(),
'tags': instance.get_tags(),
'errors': [],
'user': {'ip_address': self._client_ip},
# Construct a faux Http interface based on the little information we have
# This is a bit weird, since we don't have nearly enough
# information to create an Http interface, but
# this automatically will pick up tags for the User-Agent
# which is actually important here for CSP
'request': {
'url': instance.get_origin(),
'headers': clean(
{
'User-Agent': self._user_agent,
'Referer': instance.get_referrer(),
}
),
},
}
)
self._data = data
def process_csp_report(self):
"""Only called from the CSP report endpoint."""
data = self._data
try:
interface = get_interface(data.pop("interface"))
report = data.pop("report")
except KeyError:
raise APIForbidden("No report or interface data")
# To support testing, we can either accept a built interface instance, or the raw data in
# which case we build the instance ourselves
try:
instance = report if isinstance(
report, interface) else interface.from_raw(report)
except jsonschema.ValidationError as e:
raise APIError("Invalid security report: %s" %
str(e).splitlines()[0])
def clean(d):
return dict([x for x in d.items() if x[1]])
data.update({
"logger": "csp",
"message": instance.get_message(),
"culprit": instance.get_culprit(),
instance.path: instance.to_json(),
"tags": instance.get_tags(),
"errors": [],
"user": {
"ip_address": self._client_ip
},
# Construct a faux Http interface based on the little information we have
# This is a bit weird, since we don't have nearly enough
# information to create an Http interface, but
# this automatically will pick up tags for the User-Agent
# which is actually important here for CSP
"request": {
"url":
instance.get_origin(),
"headers":
clean({
"User-Agent": self._user_agent,
"Referer": instance.get_referrer()
}),
},
})
self._data = data
def post(self, request, project, event_id, relay_config, **kwargs):
if not features.has('organizations:event-attachments',
project.organization,
actor=request.user):
raise APIForbidden(
"Event attachments are not enabled for this organization.")
project_id = relay_config.project_id
if len(request.FILES) == 0:
return HttpResponse(status=400)
for name, uploaded_file in six.iteritems(request.FILES):
file = File.objects.create(
name=uploaded_file.name,
type='event.attachment',
headers={'Content-Type': uploaded_file.content_type},
)
file.putfile(uploaded_file)
# To avoid a race with EventManager which tries to set the group_id on attachments received before
# the event, first insert the attachment, then lookup for the event for its group.
event_attachment = EventAttachment.objects.create(
project_id=project_id,
event_id=event_id,
name=uploaded_file.name,
file=file,
)
try:
event = Event.objects.get(
project_id=project_id,
event_id=event_id,
)
except Event.DoesNotExist:
pass
else:
# If event was created but the group not defined, EventManager will take care of setting the
# group to all dangling attachments
if event.group_id is not None:
EventAttachment.objects.filter(
id=event_attachment.id, ).update(
group_id=event.group_id, )
return HttpResponse(status=201)
def process(self, request, project, auth, data, **kwargs):
event_received.send(ip=request.META['REMOTE_ADDR'], sender=type(self))
is_rate_limited = safe_execute(app.quotas.is_rate_limited,
project=project)
for plugin in plugins.all():
if safe_execute(plugin.is_rate_limited, project=project):
is_rate_limited = True
if is_rate_limited:
raise APIRateLimited
result = plugins.first('has_perm', request.user, 'create_event',
project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
if not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData as e:
raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e)))
# mutates data
Group.objects.normalize_event_data(data)
# insert IP address if not available
if auth.is_public:
ensure_has_ip(data, request.META['REMOTE_ADDR'])
event_id = data['event_id']
# mutates data (strips a lot of context if not queued)
insert_data_to_database(data)
logger.debug('New event from project %s/%s (id=%s)', project.team.slug,
project.slug, event_id)
return event_id
def handle_sentry(data, address):
from sentry.coreapi import (project_from_auth_vars,
decode_and_decompress_data,
safely_load_json_string, validate_data,
insert_data_to_database, APIError,
APIForbidden)
from sentry.models import Group
from sentry.exceptions import InvalidData
from sentry.plugins import plugins
from sentry.utils.auth import parse_auth_header
try:
try:
auth_header, data = data.split('\n\n', 1)
except ValueError:
raise APIError('missing auth header')
try:
auth_vars = parse_auth_header(auth_header)
except (ValueError, IndexError):
raise APIError('invalid auth header')
project, user = project_from_auth_vars(auth_vars)
result = plugins.first('has_perm', user, 'create_event', project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
client = auth_vars.get('sentry_client')
if not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
validate_data(project, data, client)
except InvalidData, e:
raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e)))
Group.objects.normalize_event_data(data)
return insert_data_to_database(data)
def process(self, request, project, auth, data, **kwargs):
if safe_execute(app.quotas.is_rate_limited, project=project):
raise APIRateLimited
for plugin in plugins.all():
if safe_execute(plugin.is_rate_limited, project=project):
raise APIRateLimited
result = plugins.first('has_perm', request.user, 'create_event',
project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
if not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData, e:
raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e)))
def process(self, request, project, auth, data, **kwargs):
for plugin in plugins.all():
if safe_execute(plugin.is_rate_limited, project=project):
return HttpResponse(
'Creation of this event was denied due to rate limiting.',
content_type='text/plain',
status=405)
result = plugins.first('has_perm', request.user, 'create_event',
project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
if not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData, e:
raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e)))
def process(self, request, project, auth, helper, data, **kwargs):
metrics.incr('events.total')
if not data:
raise APIError('No JSON data was found')
remote_addr = request.META['REMOTE_ADDR']
data = LazyData(
data=data,
content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''),
helper=helper,
project=project,
auth=auth,
client_ip=remote_addr,
)
event_received.send_robust(
ip=remote_addr,
project=project,
sender=type(self),
)
if helper.should_filter(project, data, ip_address=remote_addr):
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_blacklisted, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
(app.tsdb.models.organization_total_blacklisted,
project.organization_id),
])
metrics.incr('events.blacklisted')
event_filtered.send_robust(
ip=remote_addr,
project=project,
sender=type(self),
)
raise APIForbidden('Event dropped due to filter')
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited,
project=project,
_with_transaction=False)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
# XXX(dcramer): when the rate limiter fails we drop events to ensure
# it cannot cascade
if rate_limit is None or rate_limit.is_limited:
if rate_limit is None:
helper.log.debug(
'Dropped event due to error with rate limiter')
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_rejected, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
(app.tsdb.models.organization_total_rejected,
project.organization_id),
])
metrics.incr(
'events.dropped',
tags={
'reason':
rate_limit.reason_code if rate_limit else 'unknown',
})
event_dropped.send_robust(
ip=remote_addr,
project=project,
sender=type(self),
reason_code=rate_limit.reason_code if rate_limit else None,
)
if rate_limit is not None:
raise APIRateLimited(rate_limit.retry_after)
else:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
])
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
if org_options.get('sentry:require_scrub_ip_address', False):
scrub_ip_address = True
else:
scrub_ip_address = project.get_option('sentry:scrub_ip_address',
False)
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (
project.id,
event_id,
)
if cache.get(cache_key) is not None:
raise APIForbidden(
'An event with the same ID already exists (%s)' % (event_id, ))
if org_options.get('sentry:require_scrub_data', False):
scrub_data = True
else:
scrub_data = project.get_option('sentry:scrub_data', True)
if scrub_data:
# We filter data immediately before it ever gets into the queue
sensitive_fields_key = 'sentry:sensitive_fields'
sensitive_fields = (org_options.get(sensitive_fields_key, []) +
project.get_option(sensitive_fields_key, []))
exclude_fields_key = 'sentry:safe_fields'
exclude_fields = (org_options.get(exclude_fields_key, []) +
project.get_option(exclude_fields_key, []))
if org_options.get('sentry:require_scrub_defaults', False):
scrub_defaults = True
else:
scrub_defaults = project.get_option('sentry:scrub_defaults',
True)
inst = SensitiveDataFilter(
fields=sensitive_fields,
include_defaults=scrub_defaults,
exclude_fields=exclude_fields,
)
inst.apply(data)
if scrub_ip_address:
# We filter data immediately before it ever gets into the queue
helper.ensure_does_not_have_ip(data)
# mutates data (strips a lot of context if not queued)
helper.insert_data_to_database(data)
cache.set(cache_key, '', 60 * 5)
helper.log.debug('New event received (%s)', event_id)
event_accepted.send_robust(
ip=remote_addr,
data=data,
project=project,
sender=type(self),
)
return event_id
def process_event(event_manager, project, key, remote_addr, helper, attachments, project_config):
event_received.send_robust(ip=remote_addr, project=project, sender=process_event)
start_time = time()
data = event_manager.get_data()
should_filter, filter_reason = event_manager.should_filter()
del event_manager
event_id = data["event_id"]
if should_filter:
signals_in_consumer = decide_signals_in_consumer()
if not signals_in_consumer:
# Mark that the event_filtered signal is sent. Do this before emitting
# the outcome to avoid a potential race between OutcomesConsumer and
# `event_filtered.send_robust` below.
mark_signal_sent(project_config.project_id, event_id)
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.FILTERED,
filter_reason,
event_id=event_id,
)
metrics.incr("events.blacklisted", tags={"reason": filter_reason}, skip_internal=False)
if not signals_in_consumer:
event_filtered.send_robust(ip=remote_addr, project=project, sender=process_event)
# relay will no longer be able to provide information about filter
# status so to see the impact we're adding a way to turn on relay
# like behavior here.
if options.get("store.lie-about-filter-status"):
return event_id
raise APIForbidden("Event dropped due to filter: %s" % (filter_reason,))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(
quotas.is_rate_limited, project=project, key=key, _with_transaction=False
)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
# XXX(dcramer): when the rate limiter fails we drop events to ensure
# it cannot cascade
if rate_limit is None or rate_limit.is_limited:
if rate_limit is None:
api_logger.debug("Dropped event due to error with rate limiter")
signals_in_consumer = decide_signals_in_consumer()
if not signals_in_consumer:
# Mark that the event_dropped signal is sent. Do this before emitting
# the outcome to avoid a potential race between OutcomesConsumer and
# `event_dropped.send_robust` below.
mark_signal_sent(project_config.project_id, event_id)
reason = rate_limit.reason_code if rate_limit else None
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.RATE_LIMITED,
reason,
event_id=event_id,
)
metrics.incr("events.dropped", tags={"reason": reason or "unknown"}, skip_internal=False)
if not signals_in_consumer:
event_dropped.send_robust(
ip=remote_addr, project=project, reason_code=reason, sender=process_event
)
if rate_limit is not None:
raise APIRateLimited(rate_limit.retry_after)
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = "ev:%s:%s" % (project_config.project_id, event_id)
if cache.get(cache_key) is not None:
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
"duplicate",
event_id=event_id,
)
raise APIForbidden("An event with the same ID already exists (%s)" % (event_id,))
config = project_config.config
datascrubbing_settings = config.get("datascrubbingSettings") or {}
data = _scrub_event_data(data, datascrubbing_settings)
# mutates data (strips a lot of context if not queued)
helper.insert_data_to_database(data, start_time=start_time, attachments=attachments)
cache.set(cache_key, "", 60 * 60) # Cache for 1 hour
api_logger.debug("New event received (%s)", event_id)
event_accepted.send_robust(ip=remote_addr, data=data, project=project, sender=process_event)
return event_id
def apierror(message="Invalid data"):
from sentry.coreapi import APIForbidden
raise APIForbidden(message)
def _dispatch(self, request, helper, project_id=None, origin=None,
*args, **kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
Raven.tags_context(helper.context.get_tags_context())
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
raise APIForbidden('Invalid origin: %s' % (origin,))
# XXX: It seems that the OPTIONS call does not always include custom headers
if request.method == 'OPTIONS':
response = self.options(request, project)
else:
auth = self._parse_header(request, helper, project)
project_ = helper.project_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
if not project_:
raise APIError('Unable to identify project')
project = project_
elif project_ != project:
raise APIError('Two different project were specified')
helper.context.bind_auth(auth)
Raven.tags_context(helper.context.get_tags_context())
if auth.version != '2.0':
if request.method == 'GET':
# GET only requires an Origin/Referer check
# If an Origin isn't passed, it's possible that the project allows no origin,
# so we need to explicitly check for that here. If Origin is not None,
# it can be safely assumed that it was checked previously and it's ok.
if origin is None and not is_valid_origin(origin, project):
# Special case an error message for a None origin when None wasn't allowed
raise APIForbidden('Missing required Origin or Referer header')
else:
# Version 3 enforces secret key for server side requests
if not auth.secret_key:
raise APIForbidden('Missing required attribute in authentication header: sentry_secret')
response = super(APIView, self).dispatch(
request=request,
project=project,
auth=auth,
helper=helper,
**kwargs
)
if origin:
response['Access-Control-Allow-Origin'] = origin
return response
def _dispatch(self,
request,
helper,
project_id=None,
origin=None,
*args,
**kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
Raven.tags_context(helper.context.get_tags_context())
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
raise APIForbidden('Invalid origin: %s' % (origin, ))
# XXX: It seems that the OPTIONS call does not always include custom headers
if request.method == 'OPTIONS':
response = self.options(request, project)
else:
auth = self._parse_header(request, helper, project)
project_id = helper.project_id_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
project = Project.objects.get_from_cache(id=project_id)
helper.context.bind_project(project)
elif project_id != project.id:
raise APIError('Two different projects were specified')
helper.context.bind_auth(auth)
Raven.tags_context(helper.context.get_tags_context())
# Explicitly bind Organization so we don't implicitly query it later
# this just allows us to comfortably assure that `project.organization` is safe.
# This also allows us to pull the object from cache, instead of being
# implicitly fetched from database.
project.organization = Organization.objects.get_from_cache(
id=project.organization_id)
if auth.version != '2.0':
if not auth.secret_key:
# If we're missing a secret_key, check if we are allowed
# to do a CORS request.
# If we're missing an Origin/Referrer header entirely,
# we only want to support this on GET requests. By allowing
# un-authenticated CORS checks for POST, we basially
# are obsoleting our need for a secret key entirely.
if origin is None and request.method != 'GET':
raise APIForbidden(
'Missing required attribute in authentication header: sentry_secret'
)
if not is_valid_origin(origin, project):
raise APIForbidden(
'Missing required Origin or Referer header')
response = super(APIView, self).dispatch(request=request,
project=project,
auth=auth,
helper=helper,
**kwargs)
if origin:
if origin == 'null':
# If an Origin is `null`, but we got this far, that means
# we've gotten past our CORS check for some reason. But the
# problem is that we can't return "null" as a valid response
# to `Access-Control-Allow-Origin` and we don't have another
# value to work with, so just allow '*' since they've gotten
# this far.
response['Access-Control-Allow-Origin'] = '*'
else:
response['Access-Control-Allow-Origin'] = origin
return response
def process_event(event_manager, project, key, remote_addr, helper,
attachments):
event_received.send_robust(ip=remote_addr,
project=project,
sender=process_event)
start_time = time()
tsdb_start_time = to_datetime(start_time)
should_filter, filter_reason = event_manager.should_filter()
if should_filter:
increment_list = [
(tsdb.models.project_total_received, project.id),
(tsdb.models.project_total_blacklisted, project.id),
(tsdb.models.organization_total_received, project.organization_id),
(tsdb.models.organization_total_blacklisted,
project.organization_id),
(tsdb.models.key_total_received, key.id),
(tsdb.models.key_total_blacklisted, key.id),
]
try:
increment_list.append(
(FILTER_STAT_KEYS_TO_VALUES[filter_reason], project.id))
# should error when filter_reason does not match a key in FILTER_STAT_KEYS_TO_VALUES
except KeyError:
pass
tsdb.incr_multi(
increment_list,
timestamp=tsdb_start_time,
)
metrics.incr('events.blacklisted',
tags={'reason': filter_reason},
skip_internal=False)
event_filtered.send_robust(
ip=remote_addr,
project=project,
sender=process_event,
)
raise APIForbidden('Event dropped due to filter: %s' %
(filter_reason, ))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(quotas.is_rate_limited,
project=project,
key=key,
_with_transaction=False)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
# XXX(dcramer): when the rate limiter fails we drop events to ensure
# it cannot cascade
if rate_limit is None or rate_limit.is_limited:
if rate_limit is None:
api_logger.debug('Dropped event due to error with rate limiter')
tsdb.incr_multi(
[
(tsdb.models.project_total_received, project.id),
(tsdb.models.project_total_rejected, project.id),
(tsdb.models.organization_total_received,
project.organization_id),
(tsdb.models.organization_total_rejected,
project.organization_id),
(tsdb.models.key_total_received, key.id),
(tsdb.models.key_total_rejected, key.id),
],
timestamp=tsdb_start_time,
)
metrics.incr(
'events.dropped',
tags={
'reason': rate_limit.reason_code if rate_limit else 'unknown',
},
skip_internal=False,
)
event_dropped.send_robust(
ip=remote_addr,
project=project,
reason_code=rate_limit.reason_code if rate_limit else None,
sender=process_event,
)
if rate_limit is not None:
raise APIRateLimited(rate_limit.retry_after)
else:
tsdb.incr_multi(
[
(tsdb.models.project_total_received, project.id),
(tsdb.models.organization_total_received,
project.organization_id),
(tsdb.models.key_total_received, key.id),
],
timestamp=tsdb_start_time,
)
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
data = event_manager.get_data()
del event_manager
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (
project.id,
event_id,
)
if cache.get(cache_key) is not None:
raise APIForbidden('An event with the same ID already exists (%s)' %
(event_id, ))
scrub_ip_address = (
org_options.get('sentry:require_scrub_ip_address', False)
or project.get_option('sentry:scrub_ip_address', False))
scrub_data = (org_options.get('sentry:require_scrub_data', False)
or project.get_option('sentry:scrub_data', True))
if scrub_data:
# We filter data immediately before it ever gets into the queue
sensitive_fields_key = 'sentry:sensitive_fields'
sensitive_fields = (org_options.get(sensitive_fields_key, []) +
project.get_option(sensitive_fields_key, []))
exclude_fields_key = 'sentry:safe_fields'
exclude_fields = (org_options.get(exclude_fields_key, []) +
project.get_option(exclude_fields_key, []))
scrub_defaults = (org_options.get('sentry:require_scrub_defaults',
False)
or project.get_option('sentry:scrub_defaults', True))
SensitiveDataFilter(
fields=sensitive_fields,
include_defaults=scrub_defaults,
exclude_fields=exclude_fields,
).apply(data)
if scrub_ip_address:
# We filter data immediately before it ever gets into the queue
helper.ensure_does_not_have_ip(data)
# mutates data (strips a lot of context if not queued)
helper.insert_data_to_database(data,
start_time=start_time,
attachments=attachments)
cache.set(cache_key, '', 60 * 5)
api_logger.debug('New event received (%s)', event_id)
event_accepted.send_robust(
ip=remote_addr,
data=data,
project=project,
sender=process_event,
)
return event_id
def process(self, request, project, auth, data, **kwargs):
event_received.send_robust(ip=request.META['REMOTE_ADDR'],
sender=type(self))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited,
project=project,
_with_transaction=False)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
if rate_limit is not None and rate_limit.is_limited:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_rejected, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
(app.tsdb.models.organization_total_rejected,
project.organization_id),
])
raise APIRateLimited(rate_limit.retry_after)
else:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
])
result = plugins.first('has_perm',
request.user,
'create_event',
project,
version=1)
if result is False:
raise APIForbidden('Creation of this event was blocked')
content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '')
if content_encoding == 'gzip':
data = decompress_gzip(data)
elif content_encoding == 'deflate':
data = decompress_deflate(data)
elif not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData as e:
raise APIError(u'Invalid data: %s (%s)' %
(six.text_type(e), type(e)))
# mutates data
manager = EventManager(data, version=auth.version)
data = manager.normalize()
scrub_ip_address = project.get_option('sentry:scrub_ip_address', False)
# insert IP address if not available
if auth.is_public and not scrub_ip_address:
ensure_has_ip(data, request.META['REMOTE_ADDR'])
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (
project.id,
event_id,
)
if cache.get(cache_key) is not None:
logger.warning(
'Discarded recent duplicate event from project %s/%s (id=%s)',
project.organization.slug, project.slug, event_id)
raise InvalidRequest('An event with the same ID already exists.')
if project.get_option('sentry:scrub_data', True):
# We filter data immediately before it ever gets into the queue
inst = SensitiveDataFilter()
inst.apply(data)
if scrub_ip_address:
# We filter data immediately before it ever gets into the queue
ensure_does_not_have_ip(data)
# mutates data (strips a lot of context if not queued)
insert_data_to_database(data)
cache.set(cache_key, '', 60 * 5)
logger.debug('New event from project %s/%s (id=%s)',
project.organization.slug, project.slug, event_id)
return event_id
def _dispatch(self,
request,
helper,
project_id=None,
origin=None,
*args,
**kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
Raven.tags_context(helper.context.get_tags_context())
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
raise APIForbidden('Invalid origin: %s' % (origin, ))
# XXX: It seems that the OPTIONS call does not always include custom headers
if request.method == 'OPTIONS':
response = self.options(request, project)
else:
auth = self._parse_header(request, helper, project)
project_ = helper.project_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
if not project_:
raise APIError('Unable to identify project')
project = project_
helper.context.bind_project(project)
elif project_ != project:
raise APIError('Two different project were specified')
helper.context.bind_auth(auth)
Raven.tags_context(helper.context.get_tags_context())
if auth.version != '2.0':
if not auth.secret_key:
# If we're missing a secret_key, check if we are allowed
# to do a CORS request.
# If we're missing an Origin/Referrer header entirely,
# we only want to support this on GET requests. By allowing
# un-authenticated CORS checks for POST, we basially
# are obsoleting our need for a secret key entirely.
if origin is None and request.method != 'GET':
raise APIForbidden(
'Missing required attribute in authentication header: sentry_secret'
)
if not is_valid_origin(origin, project):
raise APIForbidden(
'Missing required Origin or Referer header')
response = super(APIView, self).dispatch(request=request,
project=project,
auth=auth,
helper=helper,
**kwargs)
if origin:
response['Access-Control-Allow-Origin'] = origin
return response
def process(self, request, project, auth, helper, data, **kwargs):
metrics.incr('events.total')
remote_addr = request.META['REMOTE_ADDR']
event_received.send_robust(ip=remote_addr, sender=type(self))
if not is_valid_ip(remote_addr, project):
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_blacklisted, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
(app.tsdb.models.organization_total_blacklisted,
project.organization_id),
])
metrics.incr('events.blacklisted')
raise APIForbidden('Blacklisted IP address: %s' % (remote_addr, ))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited,
project=project,
_with_transaction=False)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
# XXX(dcramer): when the rate limiter fails we drop events to ensure
# it cannot cascade
if rate_limit is None or rate_limit.is_limited:
if rate_limit is None:
helper.log.debug(
'Dropped event due to error with rate limiter')
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_rejected, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
(app.tsdb.models.organization_total_rejected,
project.organization_id),
])
metrics.incr('events.dropped')
if rate_limit is not None:
raise APIRateLimited(rate_limit.retry_after)
else:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
])
content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '')
if isinstance(data, basestring):
if content_encoding == 'gzip':
data = helper.decompress_gzip(data)
elif content_encoding == 'deflate':
data = helper.decompress_deflate(data)
elif not data.startswith('{'):
data = helper.decode_and_decompress_data(data)
data = helper.safely_load_json_string(data)
# mutates data
data = helper.validate_data(project, data)
# mutates data
manager = EventManager(data, version=auth.version)
data = manager.normalize()
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
if org_options.get('sentry:require_scrub_ip_address', False):
scrub_ip_address = True
else:
scrub_ip_address = project.get_option('sentry:scrub_ip_address',
False)
# insert IP address if not available
if auth.is_public and not scrub_ip_address:
helper.ensure_has_ip(data, remote_addr)
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (
project.id,
event_id,
)
if cache.get(cache_key) is not None:
raise APIForbidden(
'An event with the same ID already exists (%s)' % (event_id, ))
if org_options.get('sentry:require_scrub_data', False):
scrub_data = True
else:
scrub_data = project.get_option('sentry:scrub_data', True)
if scrub_data:
# We filter data immediately before it ever gets into the queue
sensitive_fields_key = 'sentry:sensitive_fields'
sensitive_fields = (org_options.get(sensitive_fields_key, []) +
project.get_option(sensitive_fields_key, []))
if org_options.get('sentry:require_scrub_defaults', False):
scrub_defaults = True
else:
scrub_defaults = project.get_option('sentry:scrub_defaults',
True)
inst = SensitiveDataFilter(
fields=sensitive_fields,
include_defaults=scrub_defaults,
)
inst.apply(data)
if scrub_ip_address:
# We filter data immediately before it ever gets into the queue
helper.ensure_does_not_have_ip(data)
# mutates data (strips a lot of context if not queued)
helper.insert_data_to_database(data)
cache.set(cache_key, '', 60 * 5)
helper.log.debug('New event received (%s)', event_id)
return event_id
