def test_progressbarhandler(self):
progress_id = '1234'
# fake request, just to get one, don't use post data
request = self.factory.post('/?X-Progress-ID=%s' % progress_id)
# instanciate progress bar upload handler with request
h = load_handler(
"progressbarupload.uploadhandler.ProgressBarUploadHandler",
request)
h.handle_raw_input('', h.request.META, 2 ** 24, 'bOuNdArY')
self.assertTrue(h.cache_key in cache)
self.assertTrue(h.progress_id == '1234')
self.assertTrue(h.cache_key == '127.0.0.1_1234')
h.receive_data_chunk('a' * 65536, 1)
# test if the cache is well filled
self.assertTrue(cache.get(h.cache_key) ==
{'uploaded': 65536, 'length': 16777216})
h.upload_complete()
# test if cache is cleared for the cache_key
self.assertFalse(h.cache_key in cache)
def parse_file_upload(self):
""" Put files that were attached to self.FILES using
standard Django upload handlers. """
upload_handlers = [uploadhandler.load_handler(handler, self)
for handler in settings.FILE_UPLOAD_HANDLERS]
if not self.message.is_multipart():
return []
for payload in self.message.get_payload():
if payload.get_filename() is None:
continue
field_name = "--"
file_name = payload.get_filename()
content_type = payload.get_content_type()
charset = payload.get_content_charset()
content = payload.get_payload(decode=True)
content_length = len(content)
#import pdb; pdb.set_trace()
try:
for handler in upload_handlers:
result = handler.handle_raw_input(content, {},
content_length, None, charset)
if result is not None:
file_obj = handler.file_complete(content_length)
self.FILES.appendlist(field_name, file_obj)
continue
try:
handler.new_file(field_name, file_name,
content_type, content_length,
charset)
except uploadhandler.StopFutureHandlers:
break
for handler in upload_handlers:
try:
chunk = handler.receive_data_chunk(content, 0)
if chunk is None:
file_obj = handler.file_complete(content_length)
self.FILES.appendlist(field_name, file_obj)
break
except:
raise
except uploadhandler.SkipFile, e:
raise
def test_progressbarhandler(self):
progress_id = '1234'
# fake request, just to get one, don't use post data
request = self.factory.post('/?X-Progress-ID=%s' % progress_id)
# instantiate progress bar upload handler with request
h = load_handler(
"progressbarupload.uploadhandler.ProgressBarUploadHandler",
request)
h.file_name = 'some_file.jpg'
h.handle_raw_input('', h.request.META, 2 ** 24, 'bOuNdArY')
self.assertIn(h.cache_key, cache)
self.assertEqual(h.progress_id, '1234')
self.assertEqual(h.cache_key, '127.0.0.1_1234')
h.receive_data_chunk('a' * 65536, 1)
# test if the cache is well filled
self.assertEqual(cache.get(h.cache_key),
{'received': 65536, 'size': 16777216})
h.upload_complete()
# test if cache is cleared for the cache_key
sleep(31)
self.assertNotIn(h.cache_key, cache)
def _initialize_handlers(self):
self._upload_handlers = [uploadhandler.load_handler(handler, self)
for handler in settings.FILE_UPLOAD_HANDLERS]
def _initialize_handlers(self, file_upload_handlers=('django.core.files.uploadhandler.MemoryFileUploadHandler', 'django.core.files.uploadhandler.TemporaryFileUploadHandler',)):
self._upload_handlers = [uploadhandler.load_handler(handler, self)
for handler in file_upload_handlers]
def _initialize_handlers(self):
self._upload_handlers = [uploadhandler.load_handler(handler, self)
for handler in settings.FILE_UPLOAD_HANDLERS]
def post(self, request, project, **kwargs):
# Minidump request payloads do not have the same structure as
# usual events from other SDKs. Most notably, the event needs
# to be transfered in the `sentry` form field. All other form
# fields are assumed "extra" information. The only exception
# to this is `upload_file_minidump`, which contains the minidump.
if any(key.startswith('sentry[') for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop('sentry', {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST
json_data = extra.pop('sentry', None)
data = json.loads(json_data[0]) if json_data else {}
# Merge additional form fields from the request with `extra`
# data from the event payload and set defaults for processing.
extra.update(data.get('extra', {}))
data['extra'] = extra
# Assign our own UUID so we can track this minidump. We cannot trust the
# uploaded filename, and if reading the minidump fails there is no way
# we can ever retrieve the original UUID from the minidump.
event_id = data.get('event_id') or uuid.uuid4().hex
data['event_id'] = event_id
# At this point, we only extract the bare minimum information
# needed to continue processing. This requires to process the
# minidump without symbols and CFI to obtain an initial stack
# trace (most likely via stack scanning). If all validations
# pass, the event will be inserted into the database.
try:
minidump = request.FILES['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
# Breakpad on linux sometimes stores the entire HTTP request body as
# dump file instead of just the minidump. The Electron SDK then for
# example uploads a multipart formdata body inside the minidump file.
# It needs to be re-parsed, to extract the actual minidump before
# continuing.
minidump.seek(0)
if minidump.read(2) == b'--':
# The remaining bytes of the first line are the form boundary. We
# have already read two bytes, the remainder is the form boundary
# (excluding the initial '--').
boundary = minidump.readline().rstrip()
minidump.seek(0)
# Next, we have to fake a HTTP request by specifying the form
# boundary and the content length, or otherwise Django will not try
# to parse our form body. Also, we need to supply new upload
# handlers since they cannot be reused from the current request.
meta = {
'CONTENT_TYPE': b'multipart/form-data; boundary=%s' % boundary,
'CONTENT_LENGTH': minidump.size,
}
handlers = [
uploadhandler.load_handler(handler, request)
for handler in settings.FILE_UPLOAD_HANDLERS
]
_, files = MultiPartParser(meta, minidump, handlers).parse()
try:
minidump = files['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
if minidump.size == 0:
raise APIError('Empty minidump upload received')
if settings.SENTRY_MINIDUMP_CACHE:
if not os.path.exists(settings.SENTRY_MINIDUMP_PATH):
os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744)
with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id), 'wb') as out:
for chunk in minidump.chunks():
out.write(chunk)
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
attachments = []
minidump.seek(0)
attachments.append(CachedAttachment.from_upload(minidump, type='event.minidump'))
# Append all other files as generic attachments. We can skip this if the
# feature is disabled since they won't be saved.
if features.has('organizations:event-attachments',
project.organization, actor=request.user):
for name, file in six.iteritems(request.FILES):
if name != 'upload_file_minidump':
attachments.append(CachedAttachment.from_upload(file))
try:
merge_minidump_event(data, minidump)
except ProcessMinidumpError as e:
logger.exception(e)
raise APIError(e.message.split('\n', 1)[0])
response_or_event_id = self.process(
request,
attachments=attachments,
data=data,
project=project,
**kwargs)
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(
six.text_type(uuid.UUID(response_or_event_id)),
content_type='text/plain'
)
def post(self, request, project, project_config, **kwargs):
# Minidump request payloads do not have the same structure as usual
# events from other SDKs. The minidump can either be transmitted as
# request body, or as `upload_file_minidump` in a multipart formdata
# request. Optionally, an event payload can be sent in the `sentry` form
# field, either as JSON or as nested form data.
request_files = request.FILES or {}
content_type = request.META.get("CONTENT_TYPE")
if content_type in self.dump_types:
minidump = io.BytesIO(request.body)
minidump_name = "Minidump"
data = {}
else:
minidump = request_files.get("upload_file_minidump")
minidump_name = minidump and minidump.name or None
if any(key.startswith("sentry[") for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop("sentry", {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST.dict()
json_data = extra.pop("sentry", None)
try:
data = json.loads(json_data) if json_data else {}
except ValueError:
data = {}
if not isinstance(data, dict):
data = {}
# Merge additional form fields from the request with `extra` data
# from the event payload and set defaults for processing. This is
# sent by clients like Breakpad or Crashpad.
extra.update(data.get("extra") or ())
data["extra"] = extra
if not minidump:
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"missing_minidump_upload",
)
raise APIError("Missing minidump upload")
# Breakpad on linux sometimes stores the entire HTTP request body as
# dump file instead of just the minidump. The Electron SDK then for
# example uploads a multipart formdata body inside the minidump file.
# It needs to be re-parsed, to extract the actual minidump before
# continuing.
minidump.seek(0)
if minidump.read(2) == b"--":
# The remaining bytes of the first line are the form boundary. We
# have already read two bytes, the remainder is the form boundary
# (excluding the initial '--').
boundary = minidump.readline().rstrip()
minidump.seek(0)
# Next, we have to fake a HTTP request by specifying the form
# boundary and the content length, or otherwise Django will not try
# to parse our form body. Also, we need to supply new upload
# handlers since they cannot be reused from the current request.
meta = {
"CONTENT_TYPE": b"multipart/form-data; boundary=%s" % boundary,
"CONTENT_LENGTH": minidump.size,
}
handlers = [
uploadhandler.load_handler(handler, request)
for handler in settings.FILE_UPLOAD_HANDLERS
]
_, inner_files = MultiPartParser(meta, minidump, handlers).parse()
try:
minidump = inner_files["upload_file_minidump"]
minidump_name = minidump.name
except KeyError:
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"missing_minidump_upload",
)
raise APIError("Missing minidump upload")
minidump.seek(0)
if minidump.read(4) != "MDMP":
track_outcome(
project_config.organization_id,
project_config.project_id,
None,
Outcome.INVALID,
"invalid_minidump",
)
raise APIError("Uploaded file was not a minidump")
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This is
# required to process the minidump with debug information.
attachments = []
# The minidump attachment is special. It has its own attachment type to
# distinguish it from regular attachments for processing. Also, it might
# not be part of `request_files` if it has been uploaded as raw request
# body instead of a multipart formdata request.
minidump.seek(0)
attachments.append(
CachedAttachment(
name=minidump_name,
content_type="application/octet-stream",
data=minidump.read(),
type=MINIDUMP_ATTACHMENT_TYPE,
)
)
# Append all other files as generic attachments.
# RaduW 4 Jun 2019 always sent attachments for minidump (does not use
# event-attachments feature)
for name, file in six.iteritems(request_files):
if name == "upload_file_minidump":
continue
# Known attachment: msgpack event
if name == "__sentry-event":
merge_attached_event(file, data)
continue
if name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
merge_attached_breadcrumbs(file, data)
continue
# Add any other file as attachment
attachments.append(CachedAttachment.from_upload(file))
# Assign our own UUID so we can track this minidump. We cannot trust
# the uploaded filename, and if reading the minidump fails there is
# no way we can ever retrieve the original UUID from the minidump.
event_id = data.get("event_id") or uuid.uuid4().hex
data["event_id"] = event_id
# Write a minimal event payload that is required to kick off native
# event processing. It is also used as fallback if processing of the
# minidump fails.
# NB: This occurs after merging attachments to overwrite potentially
# contradicting payloads transmitted in __sentry_event.
write_minidump_placeholder(data)
event_id = self.process(
request,
attachments=attachments,
data=data,
project=project,
project_config=project_config,
**kwargs
)
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(six.text_type(uuid.UUID(event_id)), content_type="text/plain")
def post(self, request, project, **kwargs):
# Minidump request payloads do not have the same structure as
# usual events from other SDKs. Most notably, the event needs
# to be transfered in the `sentry` form field. All other form
# fields are assumed "extra" information. The only exception
# to this is `upload_file_minidump`, which contains the minidump.
if any(key.startswith('sentry[') for key in request.POST):
# First, try to parse the nested form syntax `sentry[key][key]`
# This is required for the Breakpad client library, which only
# supports string values of up to 64 characters.
extra = parser.parse(request.POST.urlencode())
data = extra.pop('sentry', {})
else:
# Custom clients can submit longer payloads and should JSON
# encode event data into the optional `sentry` field.
extra = request.POST
json_data = extra.pop('sentry', None)
data = json.loads(json_data[0]) if json_data else {}
# Merge additional form fields from the request with `extra`
# data from the event payload and set defaults for processing.
extra.update(data.get('extra', {}))
data['extra'] = extra
# Assign our own UUID so we can track this minidump. We cannot trust the
# uploaded filename, and if reading the minidump fails there is no way
# we can ever retrieve the original UUID from the minidump.
event_id = data.get('event_id') or uuid.uuid4().hex
data['event_id'] = event_id
# At this point, we only extract the bare minimum information
# needed to continue processing. This requires to process the
# minidump without symbols and CFI to obtain an initial stack
# trace (most likely via stack scanning). If all validations
# pass, the event will be inserted into the database.
try:
minidump = request.FILES['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
# Breakpad on linux sometimes stores the entire HTTP request body as
# dump file instead of just the minidump. The Electron SDK then for
# example uploads a multipart formdata body inside the minidump file.
# It needs to be re-parsed, to extract the actual minidump before
# continuing.
minidump.seek(0)
if minidump.read(2) == b'--':
# The remaining bytes of the first line are the form boundary. We
# have already read two bytes, the remainder is the form boundary
# (excluding the initial '--').
boundary = minidump.readline().rstrip()
minidump.seek(0)
# Next, we have to fake a HTTP request by specifying the form
# boundary and the content length, or otherwise Django will not try
# to parse our form body. Also, we need to supply new upload
# handlers since they cannot be reused from the current request.
meta = {
'CONTENT_TYPE': b'multipart/form-data; boundary=%s' % boundary,
'CONTENT_LENGTH': minidump.size,
}
handlers = [
uploadhandler.load_handler(handler, request)
for handler in settings.FILE_UPLOAD_HANDLERS
]
_, files = MultiPartParser(meta, minidump, handlers).parse()
try:
minidump = files['upload_file_minidump']
except KeyError:
raise APIError('Missing minidump upload')
if minidump.size == 0:
raise APIError('Empty minidump upload received')
if settings.SENTRY_MINIDUMP_CACHE:
if not os.path.exists(settings.SENTRY_MINIDUMP_PATH):
os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744)
with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id),
'wb') as out:
for chunk in minidump.chunks():
out.write(chunk)
# Always store the minidump in attachments so we can access it during
# processing, regardless of the event-attachments feature. This will
# allow us to stack walk again with CFI once symbols are loaded.
attachments = []
minidump.seek(0)
attachments.append(
CachedAttachment.from_upload(minidump,
type=MINIDUMP_ATTACHMENT_TYPE))
# Append all other files as generic attachments. We can skip this if the
# feature is disabled since they won't be saved.
if features.has('organizations:event-attachments',
project.organization,
actor=request.user):
for name, file in six.iteritems(request.FILES):
if name != 'upload_file_minidump':
attachments.append(CachedAttachment.from_upload(file))
try:
state = process_minidump(minidump)
merge_process_state_event(data, state)
except ProcessMinidumpError as e:
minidumps_logger.exception(e)
raise APIError(e.message.split('\n', 1)[0])
response_or_event_id = self.process(request,
attachments=attachments,
data=data,
project=project,
**kwargs)
if isinstance(response_or_event_id, HttpResponse):
return response_or_event_id
# Return the formatted UUID of the generated event. This is
# expected by the Electron http uploader on Linux and doesn't
# break the default Breakpad client library.
return HttpResponse(six.text_type(uuid.UUID(response_or_event_id)),
content_type='text/plain')
def upload_handlers(self):
self._upload_handlers = [uploadhandler.load_handler(handler, self)
for handler in settings.FILE_UPLOAD_HANDLERS]
return self._upload_handlers
from __future__ import unicode_literals