def _dispatch(self,
request,
helper,
project_id=None,
origin=None,
*args,
**kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
tsdb.incr(tsdb.models.project_total_received_cors, project.id)
raise APIForbidden('Invalid origin: %s' % (origin, ))
# XXX: It seems that the OPTIONS call does not always include custom headers
if request.method == 'OPTIONS':
response = self.options(request, project)
else:
auth = self._parse_header(request, helper, project)
key = helper.project_key_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
project = Project.objects.get_from_cache(id=key.project_id)
helper.context.bind_project(project)
elif key.project_id != project.id:
raise APIError('Two different projects were specified')
helper.context.bind_auth(auth)
# Explicitly bind Organization so we don't implicitly query it later
# this just allows us to comfortably assure that `project.organization` is safe.
# This also allows us to pull the object from cache, instead of being
# implicitly fetched from database.
project.organization = Organization.objects.get_from_cache(
id=project.organization_id)
response = super(APIView, self).dispatch(request=request,
project=project,
auth=auth,
helper=helper,
key=key,
**kwargs)
if origin:
if origin == 'null':
# If an Origin is `null`, but we got this far, that means
# we've gotten past our CORS check for some reason. But the
# problem is that we can't return "null" as a valid response
# to `Access-Control-Allow-Origin` and we don't have another
# value to work with, so just allow '*' since they've gotten
# this far.
response['Access-Control-Allow-Origin'] = '*'
else:
response['Access-Control-Allow-Origin'] = origin
return response
def post(self, request, project, relay_config, **kwargs):
attachments_enabled = features.has('organizations:event-attachments',
project.organization,
actor=request.user)
is_apple_crash_report = False
attachments = []
event = {'event_id': uuid.uuid4().hex}
try:
unreal = process_unreal_crash(request.body,
request.GET.get('UserID'),
request.GET.get('AppEnvironment'),
event)
apple_crash_report = unreal.get_apple_crash_report()
if apple_crash_report:
merge_apple_crash_report(apple_crash_report, event)
is_apple_crash_report = True
except (ProcessMinidumpError, Unreal4Error) as e:
minidumps_logger.exception(e)
track_outcome(relay_config.organization_id,
relay_config.project_id, None, Outcome.INVALID,
"process_minidump_unreal")
raise APIError(e.message.split('\n', 1)[0])
try:
unreal_context = unreal.get_context()
if unreal_context is not None:
merge_unreal_context_event(unreal_context, event, project)
except Unreal4Error as e:
# we'll continue without the context data
minidumps_logger.exception(e)
try:
unreal_logs = unreal.get_logs()
if unreal_logs is not None:
merge_unreal_logs_event(unreal_logs, event)
except Unreal4Error as e:
# we'll continue without the breadcrumbs
minidumps_logger.exception(e)
is_minidump = False
for file in unreal.files():
# Known attachment: msgpack event
if file.name == "__sentry-event":
merge_attached_event(file.open_stream(), event)
continue
if file.name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file.open_stream(), event)
continue
if file.type == "minidump" and not is_apple_crash_report:
is_minidump = True
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
if file.type == "minidump" or attachments_enabled:
attachments.append(
CachedAttachment(
name=file.name,
data=file.open_stream().read(),
type=unreal_attachment_type(file),
))
if is_minidump:
write_minidump_placeholder(event)
event_id = self.process(request,
attachments=attachments,
data=event,
project=project,
relay_config=relay_config,
**kwargs)
# The return here is only useful for consistency
# because the UE4 crash reporter doesn't care about it.
return HttpResponse(six.text_type(uuid.UUID(event_id)),
content_type='text/plain')
def _dispatch(self,
request,
helper,
project_id=None,
origin=None,
*args,
**kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
Raven.tags_context(helper.context.get_tags_context())
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
raise APIForbidden('Invalid origin: %s' % (origin, ))
# XXX: It seems that the OPTIONS call does not always include custom headers
if request.method == 'OPTIONS':
response = self.options(request, project)
else:
auth = self._parse_header(request, helper, project)
project_ = helper.project_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
if not project_:
raise APIError('Unable to identify project')
project = project_
helper.context.bind_project(project)
elif project_ != project:
raise APIError('Two different project were specified')
helper.context.bind_auth(auth)
Raven.tags_context(helper.context.get_tags_context())
if auth.version != '2.0':
if not auth.secret_key:
# If we're missing a secret_key, check if we are allowed
# to do a CORS request.
# If we're missing an Origin/Referrer header entirely,
# we only want to support this on GET requests. By allowing
# un-authenticated CORS checks for POST, we basially
# are obsoleting our need for a secret key entirely.
if origin is None and request.method != 'GET':
raise APIForbidden(
'Missing required attribute in authentication header: sentry_secret'
)
if not is_valid_origin(origin, project):
raise APIForbidden(
'Missing required Origin or Referer header')
response = super(APIView, self).dispatch(request=request,
project=project,
auth=auth,
helper=helper,
**kwargs)
if origin:
response['Access-Control-Allow-Origin'] = origin
return response
def process(self, request, project, key, auth, helper, data, **kwargs):
metrics.incr('events.total')
if not data:
raise APIError('No JSON data was found')
remote_addr = request.META['REMOTE_ADDR']
data = LazyData(
data=data,
content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''),
helper=helper,
project=project,
key=key,
auth=auth,
client_ip=remote_addr,
)
event_received.send_robust(
ip=remote_addr,
project=project,
sender=type(self),
)
start_time = time()
tsdb_start_time = to_datetime(start_time)
should_filter, filter_reason = helper.should_filter(
project, data, ip_address=remote_addr)
if should_filter:
increment_list = [
(tsdb.models.project_total_received, project.id),
(tsdb.models.project_total_blacklisted, project.id),
(tsdb.models.organization_total_received,
project.organization_id),
(tsdb.models.organization_total_blacklisted,
project.organization_id),
(tsdb.models.key_total_received, key.id),
(tsdb.models.key_total_blacklisted, key.id),
]
try:
increment_list.append(
(FILTER_STAT_KEYS_TO_VALUES[filter_reason], project.id))
# should error when filter_reason does not match a key in FILTER_STAT_KEYS_TO_VALUES
except KeyError:
pass
tsdb.incr_multi(
increment_list,
timestamp=tsdb_start_time,
)
metrics.incr('events.blacklisted', tags={
'reason': filter_reason})
event_filtered.send_robust(
ip=remote_addr,
project=project,
sender=type(self),
)
raise APIForbidden('Event dropped due to filter: %s' % (filter_reason,))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(
quotas.is_rate_limited, project=project, key=key, _with_transaction=False
)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
# XXX(dcramer): when the rate limiter fails we drop events to ensure
# it cannot cascade
if rate_limit is None or rate_limit.is_limited:
if rate_limit is None:
helper.log.debug(
'Dropped event due to error with rate limiter')
tsdb.incr_multi(
[
(tsdb.models.project_total_received, project.id),
(tsdb.models.project_total_rejected, project.id),
(tsdb.models.organization_total_received,
project.organization_id),
(tsdb.models.organization_total_rejected,
project.organization_id),
(tsdb.models.key_total_received, key.id),
(tsdb.models.key_total_rejected, key.id),
],
timestamp=tsdb_start_time,
)
metrics.incr(
'events.dropped',
tags={
'reason': rate_limit.reason_code if rate_limit else 'unknown',
}
)
event_dropped.send_robust(
ip=remote_addr,
project=project,
sender=type(self),
reason_code=rate_limit.reason_code if rate_limit else None,
)
if rate_limit is not None:
raise APIRateLimited(rate_limit.retry_after)
else:
tsdb.incr_multi(
[
(tsdb.models.project_total_received, project.id),
(tsdb.models.organization_total_received,
project.organization_id),
(tsdb.models.key_total_received, key.id),
],
timestamp=tsdb_start_time,
)
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (project.id, event_id, )
if cache.get(cache_key) is not None:
raise APIForbidden(
'An event with the same ID already exists (%s)' % (event_id, ))
scrub_ip_address = (org_options.get('sentry:require_scrub_ip_address', False) or
project.get_option('sentry:scrub_ip_address', False))
scrub_data = (org_options.get('sentry:require_scrub_data', False) or
project.get_option('sentry:scrub_data', True))
if scrub_data:
# We filter data immediately before it ever gets into the queue
sensitive_fields_key = 'sentry:sensitive_fields'
sensitive_fields = (
org_options.get(sensitive_fields_key, []) +
project.get_option(sensitive_fields_key, [])
)
exclude_fields_key = 'sentry:safe_fields'
exclude_fields = (
org_options.get(exclude_fields_key, []) +
project.get_option(exclude_fields_key, [])
)
scrub_defaults = (org_options.get('sentry:require_scrub_defaults', False) or
project.get_option('sentry:scrub_defaults', True))
SensitiveDataFilter(
fields=sensitive_fields,
include_defaults=scrub_defaults,
exclude_fields=exclude_fields,
).apply(data)
if scrub_ip_address:
# We filter data immediately before it ever gets into the queue
helper.ensure_does_not_have_ip(data)
# mutates data (strips a lot of context if not queued)
helper.insert_data_to_database(data, start_time=start_time)
cache.set(cache_key, '', 60 * 5)
helper.log.debug('New event received (%s)', event_id)
event_accepted.send_robust(
ip=remote_addr,
data=data,
project=project,
sender=type(self),
)
return event_id
def _dispatch(self, request, helper, project_id=None, origin=None,
*args, **kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
Raven.tags_context(helper.context.get_tags_context())
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
raise APIForbidden('Invalid origin: %s' % (origin,))
# XXX: It seems that the OPTIONS call does not always include custom headers
if request.method == 'OPTIONS':
response = self.options(request, project)
else:
auth = self._parse_header(request, helper, project)
project_ = helper.project_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
if not project_:
raise APIError('Unable to identify project')
project = project_
helper.context.bind_project(project)
elif project_ != project:
raise APIError('Two different project were specified')
helper.context.bind_auth(auth)
Raven.tags_context(helper.context.get_tags_context())
if auth.version != '2.0':
if request.method == 'GET':
# GET only requires an Origin/Referer check
# If an Origin isn't passed, it's possible that the project allows no origin,
# so we need to explicitly check for that here. If Origin is not None,
# it can be safely assumed that it was checked previously and it's ok.
if origin is None and not is_valid_origin(origin, project):
# Special case an error message for a None origin when None wasn't allowed
raise APIForbidden('Missing required Origin or Referer header')
else:
# Version 3 enforces secret key for server side requests
if not auth.secret_key:
raise APIForbidden('Missing required attribute in authentication header: sentry_secret')
response = super(APIView, self).dispatch(
request=request,
project=project,
auth=auth,
helper=helper,
**kwargs
)
if origin:
response['Access-Control-Allow-Origin'] = origin
return response
def post(self, request, project, **kwargs):
attachments_enabled = features.has('organizations:event-attachments',
project.organization,
actor=request.user)
event_id = uuid.uuid4().hex
data = {
'event_id': event_id,
'environment': request.GET.get('AppEnvironment'),
}
user_id = request.GET.get('UserID')
if user_id:
data['user'] = {'id': user_id}
attachments = []
try:
unreal = process_unreal_crash(request.body)
process_state = unreal.process_minidump()
except (ProcessMinidumpError, Unreal4Error) as e:
minidumps_logger.exception(e)
raise APIError(e.message.split('\n', 1)[0])
if process_state:
merge_process_state_event(data, process_state)
else:
raise APIError("missing minidump in unreal crash report")
try:
unreal_context = unreal.get_context()
if unreal_context is not None:
merge_unreal_context_event(unreal_context, data, project)
except Unreal4Error as e:
# we'll continue without the context data
minidumps_logger.exception(e)
try:
unreal_logs = unreal.get_logs()
if unreal_logs is not None:
merge_unreal_logs_event(unreal_logs, data)
except Unreal4Error as e:
# we'll continue without the breadcrumbs
minidumps_logger.exception(e)
for file in unreal.files():
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
if file.type == "minidump" or attachments_enabled:
attachments.append(
CachedAttachment(
name=file.name,
data=file.open_stream().read(),
type=unreal_attachment_type(file),
))
response_or_event_id = self.process(request,
attachments=attachments,
data=data,
project=project,
**kwargs)
# The return here is only useful for consistency
# because the UE4 crash reporter doesn't care about it.
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
return HttpResponse(six.text_type(uuid.UUID(response_or_event_id)),
content_type='text/plain')
def post(self, request, project, project_config, **kwargs):
attachments_enabled = features.has(
"organizations:event-attachments", project.organization, actor=request.user
)
attachments = []
event = {"event_id": uuid.uuid4().hex, "environment": request.GET.get("AppEnvironment")}
user_id = request.GET.get("UserID")
if user_id:
merge_unreal_user(event, user_id)
try:
unreal = Unreal4Crash.from_bytes(request.body)
except (ProcessMinidumpError, Unreal4Error) as e:
minidumps_logger.exception(e)
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"process_unreal",
)
raise APIError(e.message.split("\n", 1)[0])
try:
unreal_context = unreal.get_context()
except Unreal4Error as e:
# we'll continue without the context data
unreal_context = None
minidumps_logger.exception(e)
else:
if unreal_context is not None:
merge_unreal_context_event(unreal_context, event, project)
try:
unreal_logs = unreal.get_logs()
except Unreal4Error as e:
# we'll continue without the breadcrumbs
minidumps_logger.exception(e)
else:
if unreal_logs is not None:
merge_unreal_logs_event(unreal_logs, event)
is_minidump = False
is_applecrashreport = False
for file in unreal.files():
# Known attachment: msgpack event
if file.name == "__sentry-event":
merge_attached_event(file.open_stream(), event)
continue
if file.name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file.open_stream(), event)
continue
if file.type == "minidump":
is_minidump = True
if file.type == "applecrashreport":
is_applecrashreport = True
# Always store attachments that can be processed, regardless of the
# event-attachments feature.
if file.type in self.required_attachments or attachments_enabled:
attachments.append(
CachedAttachment(
name=file.name,
data=file.open_stream().read(),
type=unreal_attachment_type(file),
)
)
if is_minidump:
write_minidump_placeholder(event)
elif is_applecrashreport:
write_applecrashreport_placeholder(event)
event_id = self.process(
request,
attachments=attachments,
data=event,
project=project,
project_config=project_config,
**kwargs
)
# The return here is only useful for consistency
# because the UE4 crash reporter doesn't care about it.
return HttpResponse(six.text_type(uuid.UUID(event_id)), content_type="text/plain")
def store(request, project=None):
"""
The primary endpoint for storing new events.
This will validate the client's authentication and data, and if
successfull pass on the payload to the internal database handler.
Authentication works in three flavors:
1. Explicit signed requests
These are implemented using the documented signed request protocol, and
require an authentication header which is signed using with the project
member's secret key.
2. CORS Secured Requests
Generally used for communications with client-side platforms (such as
JavaScript in the browser), they require a standard header, excluding
the signature and timestamp requirements, and must be listed in the
origins for the given project (or the global origins).
3. Implicit trusted requests
Used by the Sentry core, they are only available from same-domain requests
and do not require any authentication information. They only require that
the user be authenticated, and a project_id be sent in the GET variables.
"""
logger.debug('Inbound %r request from %r', request.method,
request.META['REMOTE_ADDR'])
client = '<unknown client>'
response = HttpResponse()
if request.method == 'POST':
try:
auth_vars = extract_auth_vars(request)
data = request.raw_post_data
if auth_vars:
server_version = auth_vars.get('sentry_version', '1.0')
client = auth_vars.get('sentry_client')
else:
server_version = request.GET.get('version', '1.0')
client = request.META.get('HTTP_USER_AGENT',
request.GET.get('client'))
if server_version not in ('1.0', '2.0'):
raise APIError(
'Client/server version mismatch: Unsupported version: %r' %
server_version)
if server_version != '1.0' and not client:
raise APIError(
'Client request error: Missing client version identifier.')
referrer = request.META.get('HTTP_REFERER')
if auth_vars:
# We only require a signature if a referrer was not set
# (this is restricted via the CORS headers)
project_ = project_from_auth_vars(auth_vars,
data,
require_signature=False)
if not project:
project = project_
elif project_ != project:
raise APIError('Project ID mismatch')
elif request.user.is_authenticated() and is_same_domain(
request.build_absolute_uri(), referrer):
# authenticated users are simply trusted to provide the right id
project_ = project_from_id(request)
if not project:
project = project_
elif project_ != project:
raise APIError('Project ID mismatch')
else:
raise APIUnauthorized()
if not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
validate_data(project, data, client)
except InvalidData, e:
raise APIError(u'Invalid data: %s' % unicode(e))
insert_data_to_database(data)
except APIError, error:
logger.error('Client %r raised API error: %s',
client,
error,
extra={
'request': request,
},
exc_info=True)
response = HttpResponse(unicode(error.msg),
status=error.http_status)
def _dispatch(self,
request,
helper,
project_id=None,
origin=None,
*args,
**kwargs):
request.user = AnonymousUser()
project = self._get_project_from_id(project_id)
if project:
helper.context.bind_project(project)
Raven.tags_context(helper.context.get_tags_context())
if origin is not None:
# This check is specific for clients who need CORS support
if not project:
raise APIError('Client must be upgraded for CORS support')
if not is_valid_origin(origin, project):
raise APIForbidden('Invalid origin: %s' % (origin, ))
# XXX: It seems that the OPTIONS call does not always include custom headers
if request.method == 'OPTIONS':
response = self.options(request, project)
else:
auth = self._parse_header(request, helper, project)
project_id = helper.project_id_from_auth(auth)
# Legacy API was /api/store/ and the project ID was only available elsewhere
if not project:
project = Project.objects.get_from_cache(id=project_id)
helper.context.bind_project(project)
elif project_id != project.id:
raise APIError('Two different projects were specified')
helper.context.bind_auth(auth)
Raven.tags_context(helper.context.get_tags_context())
# Explicitly bind Organization so we don't implicitly query it later
# this just allows us to comfortably assure that `project.organization` is safe.
# This also allows us to pull the object from cache, instead of being
# implicitly fetched from database.
project.organization = Organization.objects.get_from_cache(
id=project.organization_id)
if auth.version != '2.0':
if not auth.secret_key:
# If we're missing a secret_key, check if we are allowed
# to do a CORS request.
# If we're missing an Origin/Referrer header entirely,
# we only want to support this on GET requests. By allowing
# un-authenticated CORS checks for POST, we basially
# are obsoleting our need for a secret key entirely.
if origin is None and request.method != 'GET':
raise APIForbidden(
'Missing required attribute in authentication header: sentry_secret'
)
if not is_valid_origin(origin, project):
raise APIForbidden(
'Missing required Origin or Referer header')
response = super(APIView, self).dispatch(request=request,
project=project,
auth=auth,
helper=helper,
**kwargs)
if origin:
if origin == 'null':
# If an Origin is `null`, but we got this far, that means
# we've gotten past our CORS check for some reason. But the
# problem is that we can't return "null" as a valid response
# to `Access-Control-Allow-Origin` and we don't have another
# value to work with, so just allow '*' since they've gotten
# this far.
response['Access-Control-Allow-Origin'] = '*'
else:
response['Access-Control-Allow-Origin'] = origin
return response
def process(self,
request,
project,
key,
auth,
helper,
data,
project_config,
attachments=None,
**kwargs):
disable_transaction_events()
metrics.incr("events.total", skip_internal=False)
project_id = project_config.project_id
organization_id = project_config.organization_id
if not data:
track_outcome(organization_id, project_id, key.id, Outcome.INVALID,
"no_data")
raise APIError("No JSON data was found")
remote_addr = request.META["REMOTE_ADDR"]
event_manager = EventManager(
data,
project=project,
key=key,
auth=auth,
client_ip=remote_addr,
user_agent=helper.context.agent,
version=auth.version,
content_encoding=request.META.get("HTTP_CONTENT_ENCODING", ""),
project_config=project_config,
)
del data
self.pre_normalize(event_manager, helper)
try:
event_manager.normalize()
except ProcessingErrorInvalidTransaction as e:
track_outcome(
organization_id,
project_id,
key.id,
Outcome.INVALID,
"invalid_transaction",
category=DataCategory.TRANSACTION,
)
raise APIError(six.text_type(e).split("\n", 1)[0])
data = event_manager.get_data()
dict_data = dict(data)
data_size = len(json.dumps(dict_data))
if data_size > 10000000:
metrics.timing("events.size.rejected", data_size)
track_outcome(
organization_id,
project_id,
key.id,
Outcome.INVALID,
"too_large",
event_id=dict_data.get("event_id"),
category=DataCategory.from_event_type(dict_data.get("type")),
)
raise APIForbidden("Event size exceeded 10MB after normalization.")
metrics.timing("events.size.data.post_storeendpoint", data_size)
return process_event(event_manager, project, key, remote_addr, helper,
attachments, project_config)
def process(self, request, project, auth, helper, data, **kwargs):
metrics.incr('events.total')
if not data:
raise APIError('No JSON data was found')
data = LazyData(
data=data,
content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''),
helper=helper,
)
remote_addr = request.META['REMOTE_ADDR']
event_received.send_robust(
ip=remote_addr,
sender=type(self),
)
if helper.should_filter(project, data, ip_address=remote_addr):
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_blacklisted, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
(app.tsdb.models.organization_total_blacklisted,
project.organization_id),
])
metrics.incr('events.blacklisted')
raise APIForbidden('Event dropped due to filter')
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited,
project=project,
_with_transaction=False)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
# XXX(dcramer): when the rate limiter fails we drop events to ensure
# it cannot cascade
if rate_limit is None or rate_limit.is_limited:
if rate_limit is None:
helper.log.debug(
'Dropped event due to error with rate limiter')
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_rejected, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
(app.tsdb.models.organization_total_rejected,
project.organization_id),
])
metrics.incr('events.dropped')
if rate_limit is not None:
raise APIRateLimited(rate_limit.retry_after)
else:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.organization_total_received,
project.organization_id),
])
# mutates data
data = helper.validate_data(project, data)
if 'sdk' not in data:
sdk = helper.parse_client_as_sdk(auth.client)
if sdk:
data['sdk'] = sdk
else:
data['sdk'] = {}
data['sdk']['client_ip'] = remote_addr
# mutates data
manager = EventManager(data, version=auth.version)
manager.normalize()
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
if org_options.get('sentry:require_scrub_ip_address', False):
scrub_ip_address = True
else:
scrub_ip_address = project.get_option('sentry:scrub_ip_address',
False)
# insert IP address if not available and wanted
if not scrub_ip_address:
helper.ensure_has_ip(
data,
remote_addr,
set_if_missing=auth.is_public
or data.get('platform') in ('javascript', 'cocoa', 'objc'))
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (
project.id,
event_id,
)
if cache.get(cache_key) is not None:
raise APIForbidden(
'An event with the same ID already exists (%s)' % (event_id, ))
if org_options.get('sentry:require_scrub_data', False):
scrub_data = True
else:
scrub_data = project.get_option('sentry:scrub_data', True)
if scrub_data:
# We filter data immediately before it ever gets into the queue
sensitive_fields_key = 'sentry:sensitive_fields'
sensitive_fields = (org_options.get(sensitive_fields_key, []) +
project.get_option(sensitive_fields_key, []))
if org_options.get('sentry:require_scrub_defaults', False):
scrub_defaults = True
else:
scrub_defaults = project.get_option('sentry:scrub_defaults',
True)
inst = SensitiveDataFilter(
fields=sensitive_fields,
include_defaults=scrub_defaults,
)
inst.apply(data)
if scrub_ip_address:
# We filter data immediately before it ever gets into the queue
helper.ensure_does_not_have_ip(data)
# mutates data (strips a lot of context if not queued)
helper.insert_data_to_database(data)
cache.set(cache_key, '', 60 * 5)
helper.log.debug('New event received (%s)', event_id)
event_accepted.send_robust(
ip=remote_addr,
data=data,
project=project,
sender=type(self),
)
return event_id
def post(self, request, project, helper, key, project_config, **kwargs):
# This endpoint only accepts security reports.
data_category = DataCategory.SECURITY
json_body = safely_load_json_string(request.body)
report_type = self.security_report_type(json_body)
if report_type is None:
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
"security_report_type",
category=data_category,
)
raise APIError("Unrecognized security report type")
interface = get_interface(report_type)
try:
instance = interface.from_raw(json_body)
except jsonschema.ValidationError as e:
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
"security_report",
category=data_category,
)
raise APIError("Invalid security report: %s" %
str(e).splitlines()[0])
# Do origin check based on the `document-uri` key as explained in `_dispatch`.
origin = instance.get_origin()
if not is_valid_origin(origin, project):
track_outcome(
project_config.organization_id,
project_config.project_id,
key.id,
Outcome.INVALID,
FilterStatKeys.CORS,
category=data_category,
)
raise APIForbidden("Invalid origin")
data = {
"interface": interface.path,
"report": instance,
"release": request.GET.get("sentry_release"),
"environment": request.GET.get("sentry_environment"),
}
self.process(request,
project=project,
helper=helper,
data=data,
key=key,
project_config=project_config,
**kwargs)
return HttpResponse(content_type="application/javascript", status=201)
def _update_scopes(self):
if self.sentry_app.status == SentryAppStatus.PUBLISHED:
raise APIError('Cannot update scopes on published App.')
self.sentry_app.scope_list = self.scopes
def _update_verify_install(self):
if self.sentry_app.is_internal:
raise APIError(
u'Internal integrations do not require installation verification.',
)
self.sentry_app.verify_install = self.verify_install
def process(self,
request,
project,
key,
auth,
helper,
data,
attachments=None,
**kwargs):
metrics.incr('events.total', skip_internal=False)
if not data:
raise APIError('No JSON data was found')
remote_addr = request.META['REMOTE_ADDR']
event_manager = EventManager(
data,
project=project,
key=key,
auth=auth,
client_ip=remote_addr,
user_agent=helper.context.agent,
version=auth.version,
content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''),
)
del data
self.pre_normalize(event_manager, helper)
event_manager.normalize()
agent = request.META.get('HTTP_USER_AGENT')
# TODO: Some form of coordination between the Kafka consumer
# and this method (the 'relay') to decide whether a 429 should
# be returned here.
# Everything before this will eventually be done in the relay.
if (kafka_publisher is not None and not attachments
and random.random() < options.get('store.kafka-sample-rate')):
process_in_kafka = options.get('store.process-in-kafka')
try:
kafka_publisher.publish(
channel=getattr(settings, 'KAFKA_EVENTS_PUBLISHER_TOPIC',
'store-events'),
# Relay will (eventually) need to produce a Kafka message
# with this JSON format.
value=json.dumps({
'data': event_manager.get_data(),
'project_id': project.id,
'auth': {
'sentry_client': auth.client,
'sentry_version': auth.version,
'sentry_secret': auth.secret_key,
'sentry_key': auth.public_key,
'is_public': auth.is_public,
},
'remote_addr': remote_addr,
'agent': agent,
# Whether or not the Kafka consumer is in charge
# of actually processing this event.
'should_process': process_in_kafka,
}))
except Exception as e:
logger.exception("Cannot publish event to Kafka: {}".format(
e.message))
else:
if process_in_kafka:
# This event will be processed by the Kafka consumer, so we
# shouldn't double process it here.
return event_manager.get_data()['event_id']
# Everything after this will eventually be done in a Kafka consumer.
return process_event(event_manager, project, key, remote_addr, helper,
attachments)
def get_project_config(project_id, full_config=True, for_store=False):
"""
Constructs the ProjectConfig information.
:param project_id: the project id as int or string
:param full_config: True if only the full config is required, False
if only the restricted (for external relays) is required
(default True, i.e. full configuration)
:param for_store: If set to true, this omits all parameters that are not
needed for store normalization. This is a temporary flag that should
be removed once store has been moved to Relay. Most importantly, this
avoids database accesses.
:return: a ProjectConfig object for the given project
"""
project = _get_project_from_id(six.text_type(project_id))
if project is None:
raise APIError("Invalid project id:{}".format(project_id))
with configure_scope() as scope:
scope.set_tag("project", project.id)
if for_store:
project_keys = []
else:
project_keys = ProjectKey.objects.filter(project=project).all()
public_keys = {}
for project_key in project_keys:
public_keys[project_key.public_key] = project_key.status == 0
now = datetime.utcnow().replace(tzinfo=utc)
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
cfg = {
"disabled": project.status > 0,
"slug": project.slug,
"lastFetch": now,
"lastChange": project.get_option("sentry:relay-rev-lastchange", now),
"rev": project.get_option("sentry:relay-rev",
uuid.uuid4().hex),
"publicKeys": public_keys,
"config": {
"allowedDomains":
project.get_option("sentry:origins", ["*"]),
"trustedRelays":
org_options.get("sentry:trusted-relays", []),
"piiConfig":
_get_pii_config(project),
"datascrubbingSettings":
_get_datascrubbing_settings(project, org_options),
},
"project_id": project.id,
}
if not full_config:
# This is all we need for external Relay processors
return ProjectConfig(project, **cfg)
# The organization id is only required for reporting when processing events
# internally. Do not expose it to external Relays.
cfg["organization_id"] = project.organization_id
# Explicitly bind Organization so we don't implicitly query it later
# this just allows us to comfortably assure that `project.organization` is safe.
# This also allows us to pull the object from cache, instead of being
# implicitly fetched from database.
project.organization = Organization.objects.get_from_cache(
id=project.organization_id)
if project.organization is not None:
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
else:
org_options = {}
project_cfg = cfg["config"]
# get the filter settings for this project
filter_settings = {}
project_cfg["filter_settings"] = filter_settings
for flt in get_all_filters():
filter_id = get_filter_key(flt)
settings = _load_filter_settings(flt, project)
filter_settings[filter_id] = settings
invalid_releases = project.get_option(u"sentry:{}".format(
FilterTypes.RELEASES))
if invalid_releases:
filter_settings[FilterTypes.RELEASES] = {"releases": invalid_releases}
blacklisted_ips = project.get_option("sentry:blacklisted_ips")
if blacklisted_ips:
filter_settings["client_ips"] = {"blacklisted_ips": blacklisted_ips}
error_messages = project.get_option(u"sentry:{}".format(
FilterTypes.ERROR_MESSAGES))
if error_messages:
filter_settings[FilterTypes.ERROR_MESSAGES] = {
"patterns": error_messages
}
csp_disallowed_sources = []
if bool(project.get_option("sentry:csp_ignored_sources_defaults", True)):
csp_disallowed_sources += DEFAULT_DISALLOWED_SOURCES
csp_disallowed_sources += project.get_option("sentry:csp_ignored_sources",
[])
if csp_disallowed_sources:
filter_settings["csp"] = {"disallowed_sources": csp_disallowed_sources}
scrub_ip_address = org_options.get("sentry:require_scrub_ip_address",
False) or project.get_option(
"sentry:scrub_ip_address", False)
project_cfg["scrub_ip_addresses"] = scrub_ip_address
project_cfg["grouping_config"] = get_grouping_config_dict_for_project(
project)
project_cfg["allowed_domains"] = list(get_origins(project))
return ProjectConfig(project, **cfg)
def post(self, request, project, **kwargs):
# Minidump request payloads do not have the same structure as
# usual events from other SDKs. Most notably, the event needs
# to be transfered in the `sentry` form field. All other form
# fields are assumed "extra" information. The only exception
# to this is `upload_file_minidump`, which contains the minidump.
if any(key.startswith('sentry[') for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop('sentry', {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST
json_data = extra.pop('sentry', None)
data = json.loads(json_data[0]) if json_data else {}
# Merge additional form fields from the request with `extra`
# data from the event payload and set defaults for processing.
extra.update(data.get('extra', {}))
data['extra'] = extra
# Assign our own UUID so we can track this minidump. We cannot trust the
# uploaded filename, and if reading the minidump fails there is no way
# we can ever retrieve the original UUID from the minidump.
event_id = data.get('event_id') or uuid.uuid4().hex
data['event_id'] = event_id
# At this point, we only extract the bare minimum information
# needed to continue processing. This requires to process the
# minidump without symbols and CFI to obtain an initial stack
# trace (most likely via stack scanning). If all validations
# pass, the event will be inserted into the database.
try:
minidump = request.FILES['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
# Breakpad on linux sometimes stores the entire HTTP request body as
# dump file instead of just the minidump. The Electron SDK then for
# example uploads a multipart formdata body inside the minidump file.
# It needs to be re-parsed, to extract the actual minidump before
# continuing.
minidump.seek(0)
if minidump.read(2) == b'--':
# The remaining bytes of the first line are the form boundary. We
# have already read two bytes, the remainder is the form boundary
# (excluding the initial '--').
boundary = minidump.readline().rstrip()
minidump.seek(0)
# Next, we have to fake a HTTP request by specifying the form
# boundary and the content length, or otherwise Django will not try
# to parse our form body. Also, we need to supply new upload
# handlers since they cannot be reused from the current request.
meta = {
'CONTENT_TYPE': b'multipart/form-data; boundary=%s' % boundary,
'CONTENT_LENGTH': minidump.size,
}
handlers = [
uploadhandler.load_handler(handler, request)
for handler in settings.FILE_UPLOAD_HANDLERS
]
_, files = MultiPartParser(meta, minidump, handlers).parse()
try:
minidump = files['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
if minidump.size == 0:
raise APIError('Empty minidump upload received')
if settings.SENTRY_MINIDUMP_CACHE:
if not os.path.exists(settings.SENTRY_MINIDUMP_PATH):
os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744)
with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id),
'wb') as out:
for chunk in minidump.chunks():
out.write(chunk)
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
attachments = []
minidump.seek(0)
attachments.append(
CachedAttachment.from_upload(minidump,
type=MINIDUMP_ATTACHMENT_TYPE))
# Append all other files as generic attachments. We can skip this if the
# feature is disabled since they won't be saved.
if features.has('organizations:event-attachments',
project.organization,
actor=request.user):
for name, file in six.iteritems(request.FILES):
if name != 'upload_file_minidump':
attachments.append(CachedAttachment.from_upload(file))
try:
state = process_minidump(minidump)
merge_process_state_event(data, state)
except ProcessMinidumpError as e:
minidumps_logger.exception(e)
raise APIError(e.message.split('\n', 1)[0])
response_or_event_id = self.process(request,
attachments=attachments,
data=data,
project=project,
**kwargs)
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(six.text_type(uuid.UUID(response_or_event_id)),
content_type='text/plain')
def process(self, request, project, auth, data, **kwargs):
metrics.incr('events.total', 1)
event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited, project=project,
_with_transaction=False)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
if rate_limit is not None and rate_limit.is_limited:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.project_total_rejected, project.id),
(app.tsdb.models.organization_total_received, project.organization_id),
(app.tsdb.models.organization_total_rejected, project.organization_id),
])
metrics.incr('events.dropped', 1)
raise APIRateLimited(rate_limit.retry_after)
else:
app.tsdb.incr_multi([
(app.tsdb.models.project_total_received, project.id),
(app.tsdb.models.organization_total_received, project.organization_id),
])
result = plugins.first('has_perm', request.user, 'create_event', project,
version=1)
if result is False:
metrics.incr('events.dropped', 1)
raise APIForbidden('Creation of this event was blocked')
content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '')
if content_encoding == 'gzip':
data = decompress_gzip(data)
elif content_encoding == 'deflate':
data = decompress_deflate(data)
elif not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData as e:
raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e)))
# mutates data
manager = EventManager(data, version=auth.version)
data = manager.normalize()
scrub_ip_address = project.get_option('sentry:scrub_ip_address', False)
# insert IP address if not available
if auth.is_public and not scrub_ip_address:
ensure_has_ip(data, request.META['REMOTE_ADDR'])
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (project.id, event_id,)
if cache.get(cache_key) is not None:
logger.warning('Discarded recent duplicate event from project %s/%s (id=%s)', project.organization.slug, project.slug, event_id)
raise InvalidRequest('An event with the same ID already exists.')
if project.get_option('sentry:scrub_data', True):
# We filter data immediately before it ever gets into the queue
inst = SensitiveDataFilter(project.get_option('sentry:sensitive_fields', None))
inst.apply(data)
if scrub_ip_address:
# We filter data immediately before it ever gets into the queue
ensure_does_not_have_ip(data)
# mutates data (strips a lot of context if not queued)
insert_data_to_database(data)
cache.set(cache_key, '', 60 * 5)
logger.debug('New event from project %s/%s (id=%s)', project.organization.slug, project.slug, event_id)
return event_id
def post(self, request, project, project_config, **kwargs):
# Minidump request payloads do not have the same structure as usual
# events from other SDKs. The minidump can either be transmitted as
# request body, or as `upload_file_minidump` in a multipart formdata
# request. Optionally, an event payload can be sent in the `sentry` form
# field, either as JSON or as nested form data.
request_files = request.FILES or {}
content_type = request.META.get("CONTENT_TYPE")
if content_type in self.dump_types:
minidump = io.BytesIO(request.body)
minidump_name = "Minidump"
data = {}
else:
minidump = request_files.get("upload_file_minidump")
minidump_name = minidump and minidump.name or None
if any(key.startswith("sentry[") for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop("sentry", {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST.dict()
json_data = extra.pop("sentry", None)
try:
data = json.loads(json_data) if json_data else {}
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
# Merge additional form fields from the request with `extra` data
# from the event payload and set defaults for processing. This is
# sent by clients like Breakpad or Crashpad.
extra.update(data.get("extra") or ())
data["extra"] = extra
if not minidump:
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"missing_minidump_upload",
)
raise APIError("Missing minidump upload")
# Breakpad on linux sometimes stores the entire HTTP request body as
# dump file instead of just the minidump. The Electron SDK then for
# example uploads a multipart formdata body inside the minidump file.
# It needs to be re-parsed, to extract the actual minidump before
# continuing.
minidump.seek(0)
if minidump.read(2) == b"--":
# The remaining bytes of the first line are the form boundary. We
# have already read two bytes, the remainder is the form boundary
# (excluding the initial '--').
boundary = minidump.readline().rstrip()
minidump.seek(0)
# Next, we have to fake a HTTP request by specifying the form
# boundary and the content length, or otherwise Django will not try
# to parse our form body. Also, we need to supply new upload
# handlers since they cannot be reused from the current request.
meta = {
"CONTENT_TYPE": b"multipart/form-data; boundary=%s" % boundary,
"CONTENT_LENGTH": minidump.size,
}
handlers = [
uploadhandler.load_handler(handler, request)
for handler in settings.FILE_UPLOAD_HANDLERS
]
_, inner_files = MultiPartParser(meta, minidump, handlers).parse()
try:
minidump = inner_files["upload_file_minidump"]
minidump_name = minidump.name
except KeyError:
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"missing_minidump_upload",
)
raise APIError("Missing minidump upload")
minidump.seek(0)
if minidump.read(4) != "MDMP":
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"invalid_minidump",
)
raise APIError("Uploaded file was not a minidump")
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This is
# required to process the minidump with debug information.
attachments = []
# The minidump attachment is special. It has its own attachment type to
# distinguish it from regular attachments for processing. Also, it might
# not be part of `request_files` if it has been uploaded as raw request
# body instead of a multipart formdata request.
minidump.seek(0)
attachments.append(
CachedAttachment(
name=minidump_name,
content_type="application/octet-stream",
data=minidump.read(),
type=MINIDUMP_ATTACHMENT_TYPE,
)
)
# Append all other files as generic attachments.
# RaduW 4 Jun 2019 always sent attachments for minidump (does not use
# event-attachments feature)
for name, file in six.iteritems(request_files):
if name == "upload_file_minidump":
continue
# Known attachment: msgpack event
if name == "__sentry-event":
merge_attached_event(file, data)
continue
if name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file, data)
continue
# Add any other file as attachment
attachments.append(CachedAttachment.from_upload(file))
# Assign our own UUID so we can track this minidump. We cannot trust
# the uploaded filename, and if reading the minidump fails there is
# no way we can ever retrieve the original UUID from the minidump.
event_id = data.get("event_id") or uuid.uuid4().hex
data["event_id"] = event_id
# Write a minimal event payload that is required to kick off native
# event processing. It is also used as fallback if processing of the
# minidump fails.
# NB: This occurs after merging attachments to overwrite potentially
# contradicting payloads transmitted in __sentry_event.
write_minidump_placeholder(data)
event_id = self.process(
request,
attachments=attachments,
data=data,
project=project,
project_config=project_config,
**kwargs
)
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(six.text_type(uuid.UUID(event_id)), content_type="text/plain")
def _update_verify_install(self):
if self.sentry_app.is_internal and self.verify_install:
raise APIError(
u"Internal integrations cannot have verify_install=True.")
self.sentry_app.verify_install = self.verify_install
def get_project_config(project_id, full_config=True, for_store=False):
"""
Constructs the ProjectConfig information.
:param project_id: the project id as int or string
:param full_config: True if only the full config is required, False
if only the restricted (for external relays) is required
(default True, i.e. full configuration)
:param for_store: If set to true, this omits all parameters that are not
needed for store normalization. This is a temporary flag that should
be removed once store has been moved to Relay. Most importantly, this
avoids database accesses.
:return: a ProjectConfig object for the given project
"""
project = _get_project_from_id(six.text_type(project_id))
if project is None:
raise APIError("Invalid project id:{}".format(project_id))
with configure_scope() as scope:
scope.set_tag("project", project.id)
if for_store:
project_keys = []
else:
project_keys = ProjectKey.objects \
.filter(project=project) \
.all()
public_keys = {}
for project_key in project_keys:
public_keys[project_key.public_key] = project_key.status == 0
now = datetime.utcnow().replace(tzinfo=utc)
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
cfg = {
'disabled': project.status > 0,
'slug': project.slug,
'lastFetch': now,
'lastChange': project.get_option('sentry:relay-rev-lastchange', now),
'rev': project.get_option('sentry:relay-rev', uuid.uuid4().hex),
'publicKeys': public_keys,
'config': {
'allowedDomains': project.get_option('sentry:origins', ['*']),
'trustedRelays': org_options.get('sentry:trusted-relays', []),
'piiConfig': _get_pii_config(project, org_options),
},
'project_id': project.id,
}
if not full_config:
# This is all we need for external Relay processors
return ProjectConfig(project, **cfg)
# The organization id is only required for reporting when processing events
# internally. Do not expose it to external Relays.
cfg['organization_id'] = project.organization_id
# Explicitly bind Organization so we don't implicitly query it later
# this just allows us to comfortably assure that `project.organization` is safe.
# This also allows us to pull the object from cache, instead of being
# implicitly fetched from database.
project.organization = Organization.objects.get_from_cache(
id=project.organization_id)
if project.organization is not None:
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
else:
org_options = {}
project_cfg = cfg['config']
# get the filter settings for this project
filter_settings = {}
project_cfg['filter_settings'] = filter_settings
for flt in get_all_filters():
filter_id = get_filter_key(flt)
settings = _load_filter_settings(flt, project)
filter_settings[filter_id] = settings
invalid_releases = project.get_option(u'sentry:{}'.format(FilterTypes.RELEASES))
if invalid_releases:
filter_settings[FilterTypes.RELEASES] = {'releases': invalid_releases}
blacklisted_ips = project.get_option('sentry:blacklisted_ips')
if blacklisted_ips:
filter_settings['client_ips'] = {'blacklisted_ips': blacklisted_ips}
error_messages = project.get_option(u'sentry:{}'.format(FilterTypes.ERROR_MESSAGES))
if error_messages:
filter_settings[FilterTypes.ERROR_MESSAGES] = {'patterns': error_messages}
csp_disallowed_sources = []
if bool(project.get_option('sentry:csp_ignored_sources_defaults', True)):
csp_disallowed_sources += DEFAULT_DISALLOWED_SOURCES
csp_disallowed_sources += project.get_option('sentry:csp_ignored_sources', [])
if csp_disallowed_sources:
filter_settings['csp'] = {'disallowed_sources': csp_disallowed_sources}
scrub_ip_address = (org_options.get('sentry:require_scrub_ip_address', False) or
project.get_option('sentry:scrub_ip_address', False))
project_cfg['scrub_ip_addresses'] = scrub_ip_address
scrub_data = (org_options.get('sentry:require_scrub_data', False) or
project.get_option('sentry:scrub_data', True))
project_cfg['scrub_data'] = scrub_data
project_cfg['grouping_config'] = get_grouping_config_dict_for_project(project)
project_cfg['allowed_domains'] = list(get_origins(project))
if scrub_data:
# We filter data immediately before it ever gets into the queue
sensitive_fields_key = 'sentry:sensitive_fields'
sensitive_fields = (
org_options.get(sensitive_fields_key, []) +
project.get_option(sensitive_fields_key, [])
)
project_cfg['sensitive_fields'] = sensitive_fields
exclude_fields_key = 'sentry:safe_fields'
exclude_fields = (
org_options.get(exclude_fields_key, []) +
project.get_option(exclude_fields_key, [])
)
project_cfg['exclude_fields'] = exclude_fields
scrub_defaults = (org_options.get('sentry:require_scrub_defaults', False) or
project.get_option('sentry:scrub_defaults', True))
project_cfg['scrub_defaults'] = scrub_defaults
return ProjectConfig(project, **cfg)
def get_full_relay_config(project_id):
"""
Constructs the internal (big) RelayConfig
:param project_id: the project id as int or string
:return: FullRelayConfig the relay configuration
"""
cfg = {}
project = _get_project_from_id(six.text_type(project_id))
if project is None:
raise APIError("Invalid project id:{}".format(project_id))
cfg['project_id'] = project.id
cfg['organization_id'] = project.organization_id
# Explicitly bind Organization so we don't implicitly query it later
# this just allows us to comfortably assure that `project.organization` is safe.
# This also allows us to pull the object from cache, instead of being
# implicitly fetched from database.
project.organization = Organization.objects.get_from_cache(
id=project.organization_id)
if project.organization is not None:
org_options = OrganizationOption.objects.get_all_values(
project.organization_id)
else:
org_options = {}
# get the project options
project_cfg = {}
cfg['config'] = project_cfg
# getting kafka info
try:
project_cfg['kafka_max_event_size'] = options.get(
'kafka-publisher.max-event-size')
project_cfg['kafka_raw_event_sample_rate'] = options.get(
'kafka-publisher.raw-event-sample-rate')
except Exception:
pass # should we log ?
invalid_releases = project.get_option(u'sentry:{}'.format(
FilterTypes.RELEASES))
if invalid_releases is not None:
project_cfg['invalid_releases'] = invalid_releases
# get the filters enabled for the current project
enabled_filters = [
filter_class.id for filter_class in filters.all()
if filter_class(project).is_enabled()
]
project_cfg['enabled_filters'] = enabled_filters
scrub_ip_address = (
org_options.get('sentry:require_scrub_ip_address', False)
or project.get_option('sentry:scrub_ip_address', False))
project_cfg['scrub_ip_addresses'] = scrub_ip_address
scrub_data = (org_options.get('sentry:require_scrub_data', False)
or project.get_option('sentry:scrub_data', True))
project_cfg['scrub_data'] = scrub_data
project_cfg['grouping_config'] = get_grouping_config_dict_for_project(
project)
project_cfg['allowed_domains'] = list(get_origins(project))
if scrub_data:
# We filter data immediately before it ever gets into the queue
sensitive_fields_key = 'sentry:sensitive_fields'
sensitive_fields = (org_options.get(sensitive_fields_key, []) +
project.get_option(sensitive_fields_key, []))
project_cfg['sensitive_fields'] = sensitive_fields
exclude_fields_key = 'sentry:safe_fields'
exclude_fields = (org_options.get(exclude_fields_key, []) +
project.get_option(exclude_fields_key, []))
project_cfg['exclude_fields'] = exclude_fields
scrub_defaults = (org_options.get('sentry:require_scrub_defaults',
False)
or project.get_option('sentry:scrub_defaults', True))
project_cfg['scrub_defaults'] = scrub_defaults
return FullRelayConfig(project, **cfg)
def post(self, request, **kwargs):
# Minidump request payloads do not have the same structure as
# usual events from other SDKs. Most notably, the event needs
# to be transfered in the `sentry` form field. All other form
# fields are assumed "extra" information. The only exception
# to this is `upload_file_minidump`, which contains the minidump.
if any(key.startswith('sentry[') for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop('sentry', {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST
json_data = extra.pop('sentry', None)
data = json.loads(json_data[0]) if json_data else {}
# Merge additional form fields from the request with `extra`
# data from the event payload and set defaults for processing.
extra.update(data.get('extra', {}))
data['extra'] = extra
# Assign our own UUID so we can track this minidump. We cannot trust the
# uploaded filename, and if reading the minidump fails there is no way
# we can ever retrieve the original UUID from the minidump.
event_id = data.get('event_id') or uuid.uuid4().hex
data['event_id'] = event_id
# At this point, we only extract the bare minimum information
# needed to continue processing. This requires to process the
# minidump without symbols and CFI to obtain an initial stack
# trace (most likely via stack scanning). If all validations
# pass, the event will be inserted into the database.
try:
minidump = request.FILES['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
if settings.SENTRY_MINIDUMP_CACHE:
if not os.path.exists(settings.SENTRY_MINIDUMP_PATH):
os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744)
with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id), 'wb') as out:
for chunk in minidump.chunks():
out.write(chunk)
merge_minidump_event(data, minidump)
response_or_event_id = self.process(request, data=data, **kwargs)
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(
six.text_type(uuid.UUID(response_or_event_id)),
content_type='text/plain'
)
def _update_scopes(self):
if (self.sentry_app.status == SentryAppStatus.PUBLISHED
and self.sentry_app.scope_list != self.scopes):
raise APIError(
"Cannot update permissions on a published integration.")
self.sentry_app.scope_list = self.scopes
