def post(self, request, project, project_config, **kwargs):
# Minidump request payloads do not have the same structure as usual
# events from other SDKs. The minidump can either be transmitted as
# request body, or as `upload_file_minidump` in a multipart formdata
# request. Optionally, an event payload can be sent in the `sentry` form
# field, either as JSON or as nested form data.
request_files = request.FILES or {}
content_type = request.META.get("CONTENT_TYPE")
if content_type in self.dump_types:
minidump = io.BytesIO(request.body)
minidump_name = "Minidump"
data = {}
else:
minidump = request_files.get("upload_file_minidump")
minidump_name = minidump and minidump.name or None
if any(key.startswith("sentry[") for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop("sentry", {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST.dict()
json_data = extra.pop("sentry", None)
try:
data = json.loads(json_data) if json_data else {}
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
# Merge additional form fields from the request with `extra` data
# from the event payload and set defaults for processing. This is
# sent by clients like Breakpad or Crashpad.
extra.update(data.get("extra", {}))
data["extra"] = extra
if not minidump:
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"missing_minidump_upload",
)
raise APIError("Missing minidump upload")
# Breakpad on linux sometimes stores the entire HTTP request body as
# dump file instead of just the minidump. The Electron SDK then for
# example uploads a multipart formdata body inside the minidump file.
# It needs to be re-parsed, to extract the actual minidump before
# continuing.
minidump.seek(0)
if minidump.read(2) == b"--":
# The remaining bytes of the first line are the form boundary. We
# have already read two bytes, the remainder is the form boundary
# (excluding the initial '--').
boundary = minidump.readline().rstrip()
minidump.seek(0)
# Next, we have to fake a HTTP request by specifying the form
# boundary and the content length, or otherwise Django will not try
# to parse our form body. Also, we need to supply new upload
# handlers since they cannot be reused from the current request.
meta = {
"CONTENT_TYPE": b"multipart/form-data; boundary=%s" % boundary,
"CONTENT_LENGTH": minidump.size,
}
handlers = [
uploadhandler.load_handler(handler, request)
for handler in settings.FILE_UPLOAD_HANDLERS
]
_, inner_files = MultiPartParser(meta, minidump, handlers).parse()
try:
minidump = inner_files["upload_file_minidump"]
minidump_name = minidump.name
except KeyError:
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"missing_minidump_upload",
)
raise APIError("Missing minidump upload")
minidump.seek(0)
if minidump.read(4) != "MDMP":
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"invalid_minidump",
)
raise APIError("Uploaded file was not a minidump")
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This is
# required to process the minidump with debug information.
attachments = []
# The minidump attachment is special. It has its own attachment type to
# distinguish it from regular attachments for processing. Also, it might
# not be part of `request_files` if it has been uploaded as raw request
# body instead of a multipart formdata request.
minidump.seek(0)
attachments.append(
CachedAttachment(
name=minidump_name,
content_type="application/octet-stream",
data=minidump.read(),
type=MINIDUMP_ATTACHMENT_TYPE,
)
)
# Append all other files as generic attachments.
# RaduW 4 Jun 2019 always sent attachments for minidump (does not use
# event-attachments feature)
for name, file in six.iteritems(request_files):
if name == "upload_file_minidump":
continue
# Known attachment: msgpack event
if name == "__sentry-event":
merge_attached_event(file, data)
continue
if name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file, data)
continue
# Add any other file as attachment
attachments.append(CachedAttachment.from_upload(file))
# Assign our own UUID so we can track this minidump. We cannot trust
# the uploaded filename, and if reading the minidump fails there is
# no way we can ever retrieve the original UUID from the minidump.
event_id = data.get("event_id") or uuid.uuid4().hex
data["event_id"] = event_id
# Write a minimal event payload that is required to kick off native
# event processing. It is also used as fallback if processing of the
# minidump fails.
# NB: This occurs after merging attachments to overwrite potentially
# contradicting payloads transmitted in __sentry_event.
write_minidump_placeholder(data)
event_id = self.process(
request,
attachments=attachments,
data=data,
project=project,
project_config=project_config,
**kwargs
)
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(six.text_type(uuid.UUID(event_id)), content_type="text/plain")
def post(self, request, project, project_config, **kwargs):
attachments_enabled = features.has(
"organizations:event-attachments", project.organization, actor=request.user
)
attachments = []
event = {"event_id": uuid.uuid4().hex, "environment": request.GET.get("AppEnvironment")}
user_id = request.GET.get("UserID")
if user_id:
merge_unreal_user(event, user_id)
try:
unreal = Unreal4Crash.from_bytes(request.body)
except (ProcessMinidumpError, Unreal4Error) as e:
minidumps_logger.exception(e)
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"process_unreal",
)
raise APIError(e.message.split("\n", 1)[0])
try:
unreal_context = unreal.get_context()
except Unreal4Error as e:
# we'll continue without the context data
unreal_context = None
minidumps_logger.exception(e)
else:
if unreal_context is not None:
merge_unreal_context_event(unreal_context, event, project)
try:
unreal_logs = unreal.get_logs()
except Unreal4Error as e:
# we'll continue without the breadcrumbs
minidumps_logger.exception(e)
else:
if unreal_logs is not None:
merge_unreal_logs_event(unreal_logs, event)
is_minidump = False
is_applecrashreport = False
for file in unreal.files():
# Known attachment: msgpack event
if file.name == "__sentry-event":
merge_attached_event(file.open_stream(), event)
continue
if file.name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file.open_stream(), event)
continue
if file.type == "minidump":
is_minidump = True
if file.type == "applecrashreport":
is_applecrashreport = True
# Always store attachments that can be processed, regardless of the
# event-attachments feature.
if file.type in self.required_attachments or attachments_enabled:
attachments.append(
CachedAttachment(
name=file.name,
data=file.open_stream().read(),
type=unreal_attachment_type(file),
)
)
if is_minidump:
write_minidump_placeholder(event)
elif is_applecrashreport:
write_applecrashreport_placeholder(event)
event_id = self.process(
request,
attachments=attachments,
data=event,
project=project,
project_config=project_config,
**kwargs
)
# The return here is only useful for consistency
# because the UE4 crash reporter doesn't care about it.
return HttpResponse(six.text_type(uuid.UUID(event_id)), content_type="text/plain")
def test_merge_attached_event_invalid_file():
event = {}
merge_attached_event(MockFile(b"\xde"), event)
assert not event
def post(self, request, project, relay_config, **kwargs):
attachments_enabled = features.has('organizations:event-attachments',
project.organization,
actor=request.user)
is_apple_crash_report = False
attachments = []
event = {'event_id': uuid.uuid4().hex}
try:
unreal = process_unreal_crash(request.body,
request.GET.get('UserID'),
request.GET.get('AppEnvironment'),
event)
apple_crash_report = unreal.get_apple_crash_report()
if apple_crash_report:
merge_apple_crash_report(apple_crash_report, event)
is_apple_crash_report = True
except (ProcessMinidumpError, Unreal4Error) as e:
minidumps_logger.exception(e)
track_outcome(relay_config.organization_id,
relay_config.project_id, None, Outcome.INVALID,
"process_minidump_unreal")
raise APIError(e.message.split('\n', 1)[0])
try:
unreal_context = unreal.get_context()
if unreal_context is not None:
merge_unreal_context_event(unreal_context, event, project)
except Unreal4Error as e:
# we'll continue without the context data
minidumps_logger.exception(e)
try:
unreal_logs = unreal.get_logs()
if unreal_logs is not None:
merge_unreal_logs_event(unreal_logs, event)
except Unreal4Error as e:
# we'll continue without the breadcrumbs
minidumps_logger.exception(e)
is_minidump = False
for file in unreal.files():
# Known attachment: msgpack event
if file.name == "__sentry-event":
merge_attached_event(file.open_stream(), event)
continue
if file.name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file.open_stream(), event)
continue
if file.type == "minidump" and not is_apple_crash_report:
is_minidump = True
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
if file.type == "minidump" or attachments_enabled:
attachments.append(
CachedAttachment(
name=file.name,
data=file.open_stream().read(),
type=unreal_attachment_type(file),
))
if is_minidump:
write_minidump_placeholder(event)
event_id = self.process(request,
attachments=attachments,
data=event,
project=project,
relay_config=relay_config,
**kwargs)
# The return here is only useful for consistency
# because the UE4 crash reporter doesn't care about it.
return HttpResponse(six.text_type(uuid.UUID(event_id)),
content_type='text/plain')
def test_merge_attached_event_arbitrary_key():
mpack_event = msgpack.packb({"key": "value"})
event = {}
merge_attached_event(MockFile(mpack_event), event)
assert event["key"] == "value"
def test_merge_attached_event_empty_file():
event = {}
merge_attached_event(MockFile(b""), event)
assert not event
def test_merge_attached_event_empty():
mpack_event = msgpack.packb({})
event = {}
merge_attached_event(MockFile(mpack_event), event)
assert not event
def test_merge_attached_event_too_large_empty():
mpack_event = msgpack.packb({"a": "a" * 100000})
event = {}
merge_attached_event(MockFile(mpack_event), event)
assert not event
def post(self, request, project, **kwargs):
attachments_enabled = features.has('organizations:event-attachments',
project.organization, actor=request.user)
attachments = []
event = {'event_id': uuid.uuid4().hex}
try:
unreal = process_unreal_crash(request.body, request.GET.get(
'UserID'), request.GET.get('AppEnvironment'), event)
process_state = unreal.process_minidump()
if process_state:
merge_process_state_event(event, process_state)
else:
apple_crash_report = unreal.get_apple_crash_report()
if apple_crash_report:
merge_apple_crash_report(apple_crash_report, event)
else:
track_outcome(project.organization_id, project.id, None,
Outcome.INVALID, "missing_minidump_unreal")
raise APIError("missing minidump in unreal crash report")
except (ProcessMinidumpError, Unreal4Error) as e:
minidumps_logger.exception(e)
track_outcome(
project.organization_id,
project.id,
None,
Outcome.INVALID,
"process_minidump_unreal")
raise APIError(e.message.split('\n', 1)[0])
try:
unreal_context = unreal.get_context()
if unreal_context is not None:
merge_unreal_context_event(unreal_context, event, project)
except Unreal4Error as e:
# we'll continue without the context data
minidumps_logger.exception(e)
try:
unreal_logs = unreal.get_logs()
if unreal_logs is not None:
merge_unreal_logs_event(unreal_logs, event)
except Unreal4Error as e:
# we'll continue without the breadcrumbs
minidumps_logger.exception(e)
for file in unreal.files():
# Known attachment: msgpack event
if file.name == "__sentry-event":
merge_attached_event(file.open_stream(), event)
continue
if file.name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file.open_stream(), event)
continue
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
if file.type == "minidump" or attachments_enabled:
attachments.append(CachedAttachment(
name=file.name,
data=file.open_stream().read(),
type=unreal_attachment_type(file),
))
event_id = self.process(
request,
attachments=attachments,
data=event,
project=project,
**kwargs)
# The return here is only useful for consistency
# because the UE4 crash reporter doesn't care about it.
return HttpResponse(
six.text_type(uuid.UUID(event_id)),
content_type='text/plain'
)
def post(self, request, project, **kwargs):
# Minidump request payloads do not have the same structure as
# usual events from other SDKs. Most notably, the event needs
# to be transfered in the `sentry` form field. All other form
# fields are assumed "extra" information. The only exception
# to this is `upload_file_minidump`, which contains the minidump.
if any(key.startswith('sentry[') for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop('sentry', {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST
json_data = extra.pop('sentry', None)
data = json.loads(json_data[0]) if json_data else {}
# Merge additional form fields from the request with `extra`
# data from the event payload and set defaults for processing.
extra.update(data.get('extra', {}))
data['extra'] = extra
# Assign our own UUID so we can track this minidump. We cannot trust the
# uploaded filename, and if reading the minidump fails there is no way
# we can ever retrieve the original UUID from the minidump.
event_id = data.get('event_id') or uuid.uuid4().hex
data['event_id'] = event_id
# At this point, we only extract the bare minimum information
# needed to continue processing. This requires to process the
# minidump without symbols and CFI to obtain an initial stack
# trace (most likely via stack scanning). If all validations
# pass, the event will be inserted into the database.
try:
minidump = request.FILES['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
# Breakpad on linux sometimes stores the entire HTTP request body as
# dump file instead of just the minidump. The Electron SDK then for
# example uploads a multipart formdata body inside the minidump file.
# It needs to be re-parsed, to extract the actual minidump before
# continuing.
minidump.seek(0)
if minidump.read(2) == b'--':
# The remaining bytes of the first line are the form boundary. We
# have already read two bytes, the remainder is the form boundary
# (excluding the initial '--').
boundary = minidump.readline().rstrip()
minidump.seek(0)
# Next, we have to fake a HTTP request by specifying the form
# boundary and the content length, or otherwise Django will not try
# to parse our form body. Also, we need to supply new upload
# handlers since they cannot be reused from the current request.
meta = {
'CONTENT_TYPE': b'multipart/form-data; boundary=%s' % boundary,
'CONTENT_LENGTH': minidump.size,
}
handlers = [
uploadhandler.load_handler(handler, request)
for handler in settings.FILE_UPLOAD_HANDLERS
]
_, files = MultiPartParser(meta, minidump, handlers).parse()
try:
minidump = files['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
if minidump.size == 0:
raise APIError('Empty minidump upload received')
if settings.SENTRY_MINIDUMP_CACHE:
if not os.path.exists(settings.SENTRY_MINIDUMP_PATH):
os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744)
with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id), 'wb') as out:
for chunk in minidump.chunks():
out.write(chunk)
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
attachments = []
minidump.seek(0)
attachments.append(CachedAttachment.from_upload(minidump, type=MINIDUMP_ATTACHMENT_TYPE))
# Append all other files as generic attachments. We can skip this if the
# feature is disabled since they won't be saved.
if features.has('organizations:event-attachments',
project.organization, actor=request.user):
for name, file in six.iteritems(request.FILES):
if name == 'upload_file_minidump':
continue
# Known attachment: msgpack event
if name == "__sentry-event":
merge_attached_event(file, data)
continue
if name == "__sentry-breadcrumb1" or name == "__sentry-breadcrumb2":
merge_attached_breadcrumbs(file, data)
continue
# Add any other file as attachment
attachments.append(CachedAttachment.from_upload(file))
try:
state = process_minidump(minidump)
merge_process_state_event(data, state)
except ProcessMinidumpError as e:
minidumps_logger.exception(e)
raise APIError(e.message.split('\n', 1)[0])
event_id = self.process(
request,
attachments=attachments,
data=data,
project=project,
**kwargs)
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(
six.text_type(uuid.UUID(event_id)),
content_type='text/plain'
)
def post(self, request, project, **kwargs):
# Minidump request payloads do not have the same structure as
# usual events from other SDKs. Most notably, the event needs
# to be transfered in the `sentry` form field. All other form
# fields are assumed "extra" information. The only exception
# to this is `upload_file_minidump`, which contains the minidump.
if any(key.startswith('sentry[') for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop('sentry', {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST
json_data = extra.pop('sentry', None)
data = json.loads(json_data[0]) if json_data else {}
# Merge additional form fields from the request with `extra`
# data from the event payload and set defaults for processing.
extra.update(data.get('extra', {}))
data['extra'] = extra
# Assign our own UUID so we can track this minidump. We cannot trust the
# uploaded filename, and if reading the minidump fails there is no way
# we can ever retrieve the original UUID from the minidump.
event_id = data.get('event_id') or uuid.uuid4().hex
data['event_id'] = event_id
# At this point, we only extract the bare minimum information
# needed to continue processing. This requires to process the
# minidump without symbols and CFI to obtain an initial stack
# trace (most likely via stack scanning). If all validations
# pass, the event will be inserted into the database.
try:
minidump = request.FILES['upload_file_minidump']
except KeyError:
track_outcome(
project.organization_id,
project.id,
None,
Outcome.INVALID,
"missing_minidump_upload")
raise APIError('Missing minidump upload')
# Breakpad on linux sometimes stores the entire HTTP request body as
# dump file instead of just the minidump. The Electron SDK then for
# example uploads a multipart formdata body inside the minidump file.
# It needs to be re-parsed, to extract the actual minidump before
# continuing.
minidump.seek(0)
if minidump.read(2) == b'--':
# The remaining bytes of the first line are the form boundary. We
# have already read two bytes, the remainder is the form boundary
# (excluding the initial '--').
boundary = minidump.readline().rstrip()
minidump.seek(0)
# Next, we have to fake a HTTP request by specifying the form
# boundary and the content length, or otherwise Django will not try
# to parse our form body. Also, we need to supply new upload
# handlers since they cannot be reused from the current request.
meta = {
'CONTENT_TYPE': b'multipart/form-data; boundary=%s' % boundary,
'CONTENT_LENGTH': minidump.size,
}
handlers = [
uploadhandler.load_handler(handler, request)
for handler in settings.FILE_UPLOAD_HANDLERS
]
_, files = MultiPartParser(meta, minidump, handlers).parse()
try:
minidump = files['upload_file_minidump']
except KeyError:
track_outcome(
project.organization_id,
project.id,
None,
Outcome.INVALID,
"missing_minidump_upload")
raise APIError('Missing minidump upload')
if minidump.size == 0:
track_outcome(
project.organization_id,
project.id,
None,
Outcome.INVALID,
"empty_minidump")
raise APIError('Empty minidump upload received')
if settings.SENTRY_MINIDUMP_CACHE:
if not os.path.exists(settings.SENTRY_MINIDUMP_PATH):
os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744)
with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id), 'wb') as out:
for chunk in minidump.chunks():
out.write(chunk)
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
attachments = []
minidump.seek(0)
attachments.append(CachedAttachment.from_upload(minidump, type=MINIDUMP_ATTACHMENT_TYPE))
has_event_attachments = features.has('organizations:event-attachments',
project.organization, actor=request.user)
# Append all other files as generic attachments. We can skip this if the
# feature is disabled since they won't be saved.
for name, file in six.iteritems(request.FILES):
if name == 'upload_file_minidump':
continue
# Known attachment: msgpack event
if name == "__sentry-event":
merge_attached_event(file, data)
continue
if name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file, data)
continue
# Add any other file as attachment
if has_event_attachments:
attachments.append(CachedAttachment.from_upload(file))
try:
state = process_minidump(minidump)
merge_process_state_event(data, state)
except ProcessMinidumpError as e:
minidumps_logger.exception(e)
track_outcome(
project.organization_id,
project.id,
None,
Outcome.INVALID,
"process_minidump")
raise APIError(e.message.split('\n', 1)[0])
event_id = self.process(
request,
attachments=attachments,
data=data,
project=project,
**kwargs)
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(
six.text_type(uuid.UUID(event_id)),
content_type='text/plain'
)
def test_merge_attached_event_arbitrary_key():
mpack_event = msgpack.packb({'key': 'value'})
event = {}
merge_attached_event(MockFile(mpack_event), event)
assert event['key'] == 'value'
def test_merge_attached_event_arbitrary_key():
mpack_event = msgpack.packb({'key': 'value'})
event = {}
merge_attached_event(MockFile(mpack_event), event)
assert event['key'] == 'value'
def test_merge_attached_event_too_large_empty():
mpack_event = msgpack.packb({'a': 'a' * 100000})
event = {}
merge_attached_event(MockFile(mpack_event), event)
assert not event
def test_merge_attached_event_empty():
mpack_event = msgpack.packb({})
event = {}
merge_attached_event(MockFile(mpack_event), event)
assert not event