コード例 #1
0
 def test_progressbarhandler(self):
     progress_id = '1234'
     # fake request, just to get one, don't use post data
     request = self.factory.post('/?X-Progress-ID=%s' % progress_id)
     # instanciate progress bar upload handler with request
     h = load_handler(
         "progressbarupload.uploadhandler.ProgressBarUploadHandler",
         request)
     h.handle_raw_input('', h.request.META, 2 ** 24, 'bOuNdArY')
     self.assertTrue(h.cache_key in cache)
     self.assertTrue(h.progress_id == '1234')
     self.assertTrue(h.cache_key == '127.0.0.1_1234')
     h.receive_data_chunk('a' * 65536, 1)
     # test if the cache is well filled
     self.assertTrue(cache.get(h.cache_key) ==
         {'uploaded': 65536, 'length': 16777216})
     h.upload_complete()
     # test if cache is cleared for the cache_key
     self.assertFalse(h.cache_key in cache)
コード例 #2
0
 def parse_file_upload(self):
     """ Put files that were attached to self.FILES using
         standard Django upload handlers. """
     upload_handlers = [uploadhandler.load_handler(handler, self)
                        for handler in settings.FILE_UPLOAD_HANDLERS]
     if not self.message.is_multipart():
         return []
     for payload in self.message.get_payload():
         if payload.get_filename() is None:
             continue
         field_name = "--"
         file_name = payload.get_filename()
         content_type = payload.get_content_type()
         charset = payload.get_content_charset()
         content = payload.get_payload(decode=True)
         content_length = len(content)
         #import pdb; pdb.set_trace()
         try:
             for handler in upload_handlers:
                 result = handler.handle_raw_input(content, {},
                     content_length, None, charset)
                 if result is not None:
                     file_obj = handler.file_complete(content_length)
                     self.FILES.appendlist(field_name, file_obj)
                     continue
                 try:
                     handler.new_file(field_name, file_name,
                                      content_type, content_length,
                                      charset)
                 except uploadhandler.StopFutureHandlers:
                     break
             for handler in upload_handlers:
                 try:
                     chunk = handler.receive_data_chunk(content, 0)
                     if chunk is None:
                         file_obj = handler.file_complete(content_length)
                         self.FILES.appendlist(field_name, file_obj)
                         break
                 except:
                     raise
         except uploadhandler.SkipFile, e:
             raise
コード例 #3
0
 def test_progressbarhandler(self):
     progress_id = '1234'
     # fake request, just to get one, don't use post data
     request = self.factory.post('/?X-Progress-ID=%s' % progress_id)
     # instantiate progress bar upload handler with request
     h = load_handler(
         "progressbarupload.uploadhandler.ProgressBarUploadHandler",
         request)
     h.file_name = 'some_file.jpg'
     h.handle_raw_input('', h.request.META, 2 ** 24, 'bOuNdArY')
     self.assertIn(h.cache_key, cache)
     self.assertEqual(h.progress_id, '1234')
     self.assertEqual(h.cache_key, '127.0.0.1_1234')
     h.receive_data_chunk('a' * 65536, 1)
     # test if the cache is well filled
     self.assertEqual(cache.get(h.cache_key),
                      {'received': 65536, 'size': 16777216})
     h.upload_complete()
     # test if cache is cleared for the cache_key
     sleep(31)
     self.assertNotIn(h.cache_key, cache)
コード例 #4
0
ファイル: __init__.py プロジェクト: MarcusT0/django
 def _initialize_handlers(self):
     self._upload_handlers = [uploadhandler.load_handler(handler, self)
                              for handler in settings.FILE_UPLOAD_HANDLERS]
コード例 #5
0
ファイル: request.py プロジェクト: SlashRoot/django
 def _initialize_handlers(self, file_upload_handlers=('django.core.files.uploadhandler.MemoryFileUploadHandler', 'django.core.files.uploadhandler.TemporaryFileUploadHandler',)):
     self._upload_handlers = [uploadhandler.load_handler(handler, self)
                              for handler in file_upload_handlers]
コード例 #6
0
 def _initialize_handlers(self):
     self._upload_handlers = [uploadhandler.load_handler(handler, self)
                              for handler in settings.FILE_UPLOAD_HANDLERS]
コード例 #7
0
ファイル: api.py プロジェクト: mjumbewu/sentry
    def post(self, request, project, **kwargs):
        # Minidump request payloads do not have the same structure as
        # usual events from other SDKs. Most notably, the event needs
        # to be transfered in the `sentry` form field. All other form
        # fields are assumed "extra" information. The only exception
        # to this is `upload_file_minidump`, which contains the minidump.

        if any(key.startswith('sentry[') for key in request.POST):
            # First, try to parse the nested form syntax `sentry[key][key]`
            # This is required for the Breakpad client library, which only
            # supports string values of up to 64 characters.
            extra = parser.parse(request.POST.urlencode())
            data = extra.pop('sentry', {})
        else:
            # Custom clients can submit longer payloads and should JSON
            # encode event data into the optional `sentry` field.
            extra = request.POST
            json_data = extra.pop('sentry', None)
            data = json.loads(json_data[0]) if json_data else {}

        # Merge additional form fields from the request with `extra`
        # data from the event payload and set defaults for processing.
        extra.update(data.get('extra', {}))
        data['extra'] = extra

        # Assign our own UUID so we can track this minidump. We cannot trust the
        # uploaded filename, and if reading the minidump fails there is no way
        # we can ever retrieve the original UUID from the minidump.
        event_id = data.get('event_id') or uuid.uuid4().hex
        data['event_id'] = event_id

        # At this point, we only extract the bare minimum information
        # needed to continue processing. This requires to process the
        # minidump without symbols and CFI to obtain an initial stack
        # trace (most likely via stack scanning). If all validations
        # pass, the event will be inserted into the database.
        try:
            minidump = request.FILES['upload_file_minidump']
        except KeyError:
            raise APIError('Missing minidump upload')

        # Breakpad on linux sometimes stores the entire HTTP request body as
        # dump file instead of just the minidump. The Electron SDK then for
        # example uploads a multipart formdata body inside the minidump file.
        # It needs to be re-parsed, to extract the actual minidump before
        # continuing.
        minidump.seek(0)
        if minidump.read(2) == b'--':
            # The remaining bytes of the first line are the form boundary. We
            # have already read two bytes, the remainder is the form boundary
            # (excluding the initial '--').
            boundary = minidump.readline().rstrip()
            minidump.seek(0)

            # Next, we have to fake a HTTP request by specifying the form
            # boundary and the content length, or otherwise Django will not try
            # to parse our form body. Also, we need to supply new upload
            # handlers since they cannot be reused from the current request.
            meta = {
                'CONTENT_TYPE': b'multipart/form-data; boundary=%s' % boundary,
                'CONTENT_LENGTH': minidump.size,
            }
            handlers = [
                uploadhandler.load_handler(handler, request)
                for handler in settings.FILE_UPLOAD_HANDLERS
            ]

            _, files = MultiPartParser(meta, minidump, handlers).parse()
            try:
                minidump = files['upload_file_minidump']
            except KeyError:
                raise APIError('Missing minidump upload')

        if minidump.size == 0:
            raise APIError('Empty minidump upload received')

        if settings.SENTRY_MINIDUMP_CACHE:
            if not os.path.exists(settings.SENTRY_MINIDUMP_PATH):
                os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744)

            with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id), 'wb') as out:
                for chunk in minidump.chunks():
                    out.write(chunk)

        # Always store the minidump in attachments so we can access it during
        # processing, regardless of the event-attachments feature. This will
        # allow us to stack walk again with CFI once symbols are loaded.
        attachments = []
        minidump.seek(0)
        attachments.append(CachedAttachment.from_upload(minidump, type='event.minidump'))

        # Append all other files as generic attachments. We can skip this if the
        # feature is disabled since they won't be saved.
        if features.has('organizations:event-attachments',
                        project.organization, actor=request.user):
            for name, file in six.iteritems(request.FILES):
                if name != 'upload_file_minidump':
                    attachments.append(CachedAttachment.from_upload(file))

        try:
            merge_minidump_event(data, minidump)
        except ProcessMinidumpError as e:
            logger.exception(e)
            raise APIError(e.message.split('\n', 1)[0])

        response_or_event_id = self.process(
            request,
            attachments=attachments,
            data=data,
            project=project,
            **kwargs)

        if isinstance(response_or_event_id, HttpResponse):
            return response_or_event_id

        # Return the formatted UUID of the generated event. This is
        # expected by the Electron http uploader on Linux and doesn't
        # break the default Breakpad client library.
        return HttpResponse(
            six.text_type(uuid.UUID(response_or_event_id)),
            content_type='text/plain'
        )
コード例 #8
0
    def post(self, request, project, project_config, **kwargs):
        # Minidump request payloads do not have the same structure as usual
        # events from other SDKs. The minidump can either be transmitted as
        # request body, or as `upload_file_minidump` in a multipart formdata
        # request. Optionally, an event payload can be sent in the `sentry` form
        # field, either as JSON or as nested form data.

        request_files = request.FILES or {}
        content_type = request.META.get("CONTENT_TYPE")

        if content_type in self.dump_types:
            minidump = io.BytesIO(request.body)
            minidump_name = "Minidump"
            data = {}
        else:
            minidump = request_files.get("upload_file_minidump")
            minidump_name = minidump and minidump.name or None

            if any(key.startswith("sentry[") for key in request.POST):
                # First, try to parse the nested form syntax `sentry[key][key]`
                # This is required for the Breakpad client library, which only
                # supports string values of up to 64 characters.
                extra = parser.parse(request.POST.urlencode())
                data = extra.pop("sentry", {})
            else:
                # Custom clients can submit longer payloads and should JSON
                # encode event data into the optional `sentry` field.
                extra = request.POST.dict()
                json_data = extra.pop("sentry", None)
                try:
                    data = json.loads(json_data) if json_data else {}
                except ValueError:
                    data = {}

            if not isinstance(data, dict):
                data = {}

            # Merge additional form fields from the request with `extra` data
            # from the event payload and set defaults for processing. This is
            # sent by clients like Breakpad or Crashpad.
            extra.update(data.get("extra") or ())
            data["extra"] = extra

        if not minidump:
            track_outcome(
                project_config.organization_id,
                project_config.project_id,
                None,
                Outcome.INVALID,
                "missing_minidump_upload",
            )
            raise APIError("Missing minidump upload")

        # Breakpad on linux sometimes stores the entire HTTP request body as
        # dump file instead of just the minidump. The Electron SDK then for
        # example uploads a multipart formdata body inside the minidump file.
        # It needs to be re-parsed, to extract the actual minidump before
        # continuing.
        minidump.seek(0)
        if minidump.read(2) == b"--":
            # The remaining bytes of the first line are the form boundary. We
            # have already read two bytes, the remainder is the form boundary
            # (excluding the initial '--').
            boundary = minidump.readline().rstrip()
            minidump.seek(0)

            # Next, we have to fake a HTTP request by specifying the form
            # boundary and the content length, or otherwise Django will not try
            # to parse our form body. Also, we need to supply new upload
            # handlers since they cannot be reused from the current request.
            meta = {
                "CONTENT_TYPE": b"multipart/form-data; boundary=%s" % boundary,
                "CONTENT_LENGTH": minidump.size,
            }
            handlers = [
                uploadhandler.load_handler(handler, request)
                for handler in settings.FILE_UPLOAD_HANDLERS
            ]

            _, inner_files = MultiPartParser(meta, minidump, handlers).parse()
            try:
                minidump = inner_files["upload_file_minidump"]
                minidump_name = minidump.name
            except KeyError:
                track_outcome(
                    project_config.organization_id,
                    project_config.project_id,
                    None,
                    Outcome.INVALID,
                    "missing_minidump_upload",
                )
                raise APIError("Missing minidump upload")

        minidump.seek(0)
        if minidump.read(4) != "MDMP":
            track_outcome(
                project_config.organization_id,
                project_config.project_id,
                None,
                Outcome.INVALID,
                "invalid_minidump",
            )
            raise APIError("Uploaded file was not a minidump")

        # Always store the minidump in attachments so we can access it during
        # processing, regardless of the event-attachments feature. This is
        # required to process the minidump with debug information.
        attachments = []

        # The minidump attachment is special. It has its own attachment type to
        # distinguish it from regular attachments for processing. Also, it might
        # not be part of `request_files` if it has been uploaded as raw request
        # body instead of a multipart formdata request.
        minidump.seek(0)
        attachments.append(
            CachedAttachment(
                name=minidump_name,
                content_type="application/octet-stream",
                data=minidump.read(),
                type=MINIDUMP_ATTACHMENT_TYPE,
            )
        )

        # Append all other files as generic attachments.
        # RaduW 4 Jun 2019 always sent attachments for minidump (does not use
        # event-attachments feature)
        for name, file in six.iteritems(request_files):
            if name == "upload_file_minidump":
                continue

            # Known attachment: msgpack event
            if name == "__sentry-event":
                merge_attached_event(file, data)
                continue
            if name in ("__sentry-breadcrumb1", "__sentry-breadcrumb2"):
                merge_attached_breadcrumbs(file, data)
                continue

            # Add any other file as attachment
            attachments.append(CachedAttachment.from_upload(file))

        # Assign our own UUID so we can track this minidump. We cannot trust
        # the uploaded filename, and if reading the minidump fails there is
        # no way we can ever retrieve the original UUID from the minidump.
        event_id = data.get("event_id") or uuid.uuid4().hex
        data["event_id"] = event_id

        # Write a minimal event payload that is required to kick off native
        # event processing. It is also used as fallback if processing of the
        # minidump fails.
        # NB: This occurs after merging attachments to overwrite potentially
        # contradicting payloads transmitted in __sentry_event.
        write_minidump_placeholder(data)

        event_id = self.process(
            request,
            attachments=attachments,
            data=data,
            project=project,
            project_config=project_config,
            **kwargs
        )

        # Return the formatted UUID of the generated event. This is
        # expected by the Electron http uploader on Linux and doesn't
        # break the default Breakpad client library.
        return HttpResponse(six.text_type(uuid.UUID(event_id)), content_type="text/plain")
コード例 #9
0
    def post(self, request, project, **kwargs):
        # Minidump request payloads do not have the same structure as
        # usual events from other SDKs. Most notably, the event needs
        # to be transfered in the `sentry` form field. All other form
        # fields are assumed "extra" information. The only exception
        # to this is `upload_file_minidump`, which contains the minidump.

        if any(key.startswith('sentry[') for key in request.POST):
            # First, try to parse the nested form syntax `sentry[key][key]`
            # This is required for the Breakpad client library, which only
            # supports string values of up to 64 characters.
            extra = parser.parse(request.POST.urlencode())
            data = extra.pop('sentry', {})
        else:
            # Custom clients can submit longer payloads and should JSON
            # encode event data into the optional `sentry` field.
            extra = request.POST
            json_data = extra.pop('sentry', None)
            data = json.loads(json_data[0]) if json_data else {}

        # Merge additional form fields from the request with `extra`
        # data from the event payload and set defaults for processing.
        extra.update(data.get('extra', {}))
        data['extra'] = extra

        # Assign our own UUID so we can track this minidump. We cannot trust the
        # uploaded filename, and if reading the minidump fails there is no way
        # we can ever retrieve the original UUID from the minidump.
        event_id = data.get('event_id') or uuid.uuid4().hex
        data['event_id'] = event_id

        # At this point, we only extract the bare minimum information
        # needed to continue processing. This requires to process the
        # minidump without symbols and CFI to obtain an initial stack
        # trace (most likely via stack scanning). If all validations
        # pass, the event will be inserted into the database.
        try:
            minidump = request.FILES['upload_file_minidump']
        except KeyError:
            raise APIError('Missing minidump upload')

        # Breakpad on linux sometimes stores the entire HTTP request body as
        # dump file instead of just the minidump. The Electron SDK then for
        # example uploads a multipart formdata body inside the minidump file.
        # It needs to be re-parsed, to extract the actual minidump before
        # continuing.
        minidump.seek(0)
        if minidump.read(2) == b'--':
            # The remaining bytes of the first line are the form boundary. We
            # have already read two bytes, the remainder is the form boundary
            # (excluding the initial '--').
            boundary = minidump.readline().rstrip()
            minidump.seek(0)

            # Next, we have to fake a HTTP request by specifying the form
            # boundary and the content length, or otherwise Django will not try
            # to parse our form body. Also, we need to supply new upload
            # handlers since they cannot be reused from the current request.
            meta = {
                'CONTENT_TYPE': b'multipart/form-data; boundary=%s' % boundary,
                'CONTENT_LENGTH': minidump.size,
            }
            handlers = [
                uploadhandler.load_handler(handler, request)
                for handler in settings.FILE_UPLOAD_HANDLERS
            ]

            _, files = MultiPartParser(meta, minidump, handlers).parse()
            try:
                minidump = files['upload_file_minidump']
            except KeyError:
                raise APIError('Missing minidump upload')

        if minidump.size == 0:
            raise APIError('Empty minidump upload received')

        if settings.SENTRY_MINIDUMP_CACHE:
            if not os.path.exists(settings.SENTRY_MINIDUMP_PATH):
                os.mkdir(settings.SENTRY_MINIDUMP_PATH, 0o744)

            with open('%s/%s.dmp' % (settings.SENTRY_MINIDUMP_PATH, event_id),
                      'wb') as out:
                for chunk in minidump.chunks():
                    out.write(chunk)

        # Always store the minidump in attachments so we can access it during
        # processing, regardless of the event-attachments feature. This will
        # allow us to stack walk again with CFI once symbols are loaded.
        attachments = []
        minidump.seek(0)
        attachments.append(
            CachedAttachment.from_upload(minidump,
                                         type=MINIDUMP_ATTACHMENT_TYPE))

        # Append all other files as generic attachments. We can skip this if the
        # feature is disabled since they won't be saved.
        if features.has('organizations:event-attachments',
                        project.organization,
                        actor=request.user):
            for name, file in six.iteritems(request.FILES):
                if name != 'upload_file_minidump':
                    attachments.append(CachedAttachment.from_upload(file))

        try:
            state = process_minidump(minidump)
            merge_process_state_event(data, state)
        except ProcessMinidumpError as e:
            minidumps_logger.exception(e)
            raise APIError(e.message.split('\n', 1)[0])

        response_or_event_id = self.process(request,
                                            attachments=attachments,
                                            data=data,
                                            project=project,
                                            **kwargs)

        if isinstance(response_or_event_id, HttpResponse):
            return response_or_event_id

        # Return the formatted UUID of the generated event. This is
        # expected by the Electron http uploader on Linux and doesn't
        # break the default Breakpad client library.
        return HttpResponse(six.text_type(uuid.UUID(response_or_event_id)),
                            content_type='text/plain')
コード例 #10
0
ファイル: __init__.py プロジェクト: waylybaye/kiss
 def upload_handlers(self):
     self._upload_handlers = [uploadhandler.load_handler(handler, self)
                  for handler in settings.FILE_UPLOAD_HANDLERS]
     return self._upload_handlers
コード例 #11
0
from __future__ import unicode_literals