def test_assemble_duplicate_blobs(self): files = [] file_checksum = sha1() blob = os.urandom(1024 * 1024 * 8) hash = sha1(blob).hexdigest() for _ in range(8): file_checksum.update(blob) files.append((io.BytesIO(blob), hash)) # upload all blobs FileBlob.from_files(files, organization=self.organization) # find all blobs for reference, checksum in files: blob = FileBlob.objects.get(checksum=checksum) ref_bytes = reference.getvalue() assert blob.getfile().read(len(ref_bytes)) == ref_bytes FileBlobOwner.objects.filter( blob=blob, organization_id=self.organization.id).get() rv = assemble_file( AssembleTask.DIF, self.project, "testfile", file_checksum.hexdigest(), [x[1] for x in files], "dummy.type", ) assert rv is not None f, tmp = rv assert f.checksum == file_checksum.hexdigest() assert f.type == "dummy.type"
def post(self, request, organization): """ Upload chunks and store them as FileBlobs ````````````````````````````````````````` :pparam file file: The filename should be sha1 hash of the content. Also not you can add up to MAX_CHUNKS_PER_REQUEST files in this request. :auth: required """ # Create a unique instance so our logger can be decoupled from the request # and used in threads. logger = logging.getLogger('sentry.files') logger.info('chunkupload.start') files = request.data.getlist('file') files += [GzipChunk(chunk) for chunk in request.data.getlist('file_gzip')] if len(files) == 0: # No files uploaded is ok logger.info('chunkupload.end', extra={'status': status.HTTP_200_OK}) return Response(status=status.HTTP_200_OK) logger.info('chunkupload.post.files', extra={'len': len(files)}) # Validate file size checksums = [] size = 0 for chunk in files: size += chunk.size if chunk.size > CHUNK_UPLOAD_BLOB_SIZE: logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST}) return Response({'error': 'Chunk size too large'}, status=status.HTTP_400_BAD_REQUEST) checksums.append(chunk.name) if size > MAX_REQUEST_SIZE: logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST}) return Response({'error': 'Request too large'}, status=status.HTTP_400_BAD_REQUEST) if len(files) > MAX_CHUNKS_PER_REQUEST: logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST}) return Response({'error': 'Too many chunks'}, status=status.HTTP_400_BAD_REQUEST) try: FileBlob.from_files(izip(files, checksums), organization=organization, logger=logger) except IOError as err: logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST}) return Response({'error': six.text_type(err)}, status=status.HTTP_400_BAD_REQUEST) logger.info('chunkupload.end', extra={'status': status.HTTP_200_OK}) return Response(status=status.HTTP_200_OK)
def post(self, request, organization): """ Upload chunks and store them as FileBlobs ````````````````````````````````````````` :pparam file file: The filename should be sha1 hash of the content. Also not you can add up to MAX_CHUNKS_PER_REQUEST files in this request. :auth: required """ # Create a unique instance so our logger can be decoupled from the request # and used in threads. logger = logging.getLogger('sentry.files') logger.info('chunkupload.start') files = request.FILES.getlist('file') files += [GzipChunk(chunk) for chunk in request.FILES.getlist('file_gzip')] if len(files) == 0: # No files uploaded is ok logger.info('chunkupload.end', extra={'status': status.HTTP_200_OK}) return Response(status=status.HTTP_200_OK) logger.info('chunkupload.post.files', extra={'len': len(files)}) # Validate file size checksums = [] size = 0 for chunk in files: size += chunk.size if chunk.size > CHUNK_UPLOAD_BLOB_SIZE: logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST}) return Response({'error': 'Chunk size too large'}, status=status.HTTP_400_BAD_REQUEST) checksums.append(chunk.name) if size > MAX_REQUEST_SIZE: logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST}) return Response({'error': 'Request too large'}, status=status.HTTP_400_BAD_REQUEST) if len(files) > MAX_CHUNKS_PER_REQUEST: logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST}) return Response({'error': 'Too many chunks'}, status=status.HTTP_400_BAD_REQUEST) try: FileBlob.from_files(izip(files, checksums), organization=organization, logger=logger) except IOError as err: logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST}) return Response({'error': six.text_type(err)}, status=status.HTTP_400_BAD_REQUEST) logger.info('chunkupload.end', extra={'status': status.HTTP_200_OK}) return Response(status=status.HTTP_200_OK)
def post(self, request, organization): """ Upload chunks and store them as FileBlobs ````````````````````````````````````````` :pparam file file: The filename should be sha1 hash of the content. Also not you can add up to MAX_CHUNKS_PER_REQUEST files in this request. :auth: required """ # Create a unique instance so our logger can be decoupled from the request # and used in threads. logger = logging.getLogger("sentry.files") logger.info("chunkupload.start") files = [] if request.data: files = request.data.getlist("file") files += [GzipChunk(chunk) for chunk in request.data.getlist("file_gzip")] if len(files) == 0: # No files uploaded is ok logger.info("chunkupload.end", extra={"status": status.HTTP_200_OK}) return Response(status=status.HTTP_200_OK) logger.info("chunkupload.post.files", extra={"len": len(files)}) # Validate file size checksums = [] size = 0 for chunk in files: size += chunk.size if chunk.size > settings.SENTRY_CHUNK_UPLOAD_BLOB_SIZE: logger.info("chunkupload.end", extra={"status": status.HTTP_400_BAD_REQUEST}) return Response( {"error": "Chunk size too large"}, status=status.HTTP_400_BAD_REQUEST ) checksums.append(chunk.name) if size > MAX_REQUEST_SIZE: logger.info("chunkupload.end", extra={"status": status.HTTP_400_BAD_REQUEST}) return Response({"error": "Request too large"}, status=status.HTTP_400_BAD_REQUEST) if len(files) > MAX_CHUNKS_PER_REQUEST: logger.info("chunkupload.end", extra={"status": status.HTTP_400_BAD_REQUEST}) return Response({"error": "Too many chunks"}, status=status.HTTP_400_BAD_REQUEST) try: FileBlob.from_files(zip(files, checksums), organization=organization, logger=logger) except OSError as err: logger.info("chunkupload.end", extra={"status": status.HTTP_400_BAD_REQUEST}) return Response({"error": str(err)}, status=status.HTTP_400_BAD_REQUEST) logger.info("chunkupload.end", extra={"status": status.HTTP_200_OK}) return Response(status=status.HTTP_200_OK)
def post(self, request, organization): """ Upload chunks and store them as FileBlobs ````````````````````````````````````````` :pparam file file: The filename should be sha1 hash of the content. Also not you can add up to MAX_CHUNKS_PER_REQUEST files in this request. :auth: required """ files = request.FILES.getlist('file') files += [ GzipChunk(chunk) for chunk in request.FILES.getlist('file_gzip') ] if len(files) == 0: # No files uploaded is ok return Response(status=status.HTTP_200_OK) # Validate file size checksums = [] size = 0 for chunk in files: size += chunk.size if chunk.size > CHUNK_UPLOAD_BLOB_SIZE: return Response({'error': 'Chunk size too large'}, status=status.HTTP_400_BAD_REQUEST) checksums.append(chunk.name) if size > MAX_REQUEST_SIZE: return Response({'error': 'Request too large'}, status=status.HTTP_400_BAD_REQUEST) if len(files) > MAX_CHUNKS_PER_REQUEST: return Response({'error': 'Too many chunks'}, status=status.HTTP_400_BAD_REQUEST) try: FileBlob.from_files(izip(files, checksums), organization=organization) except IOError as err: return Response({'error': six.text_type(err)}, status=status.HTTP_400_BAD_REQUEST) return Response(status=status.HTTP_200_OK)
def test_assemble_from_files(self): files = [] file_checksum = sha1() for _ in xrange(8): blob = os.urandom(1024 * 1024 * 8) hash = sha1(blob).hexdigest() file_checksum.update(blob) files.append((io.BytesIO(blob), hash)) # upload all blobs FileBlob.from_files(files, organization=self.organization) # find all blobs for reference, checksum in files: blob = FileBlob.objects.get(checksum=checksum) ref_bytes = reference.getvalue() assert blob.getfile().read(len(ref_bytes)) == ref_bytes FileBlobOwner.objects.filter( blob=blob, organization=self.organization ).get() rv = assemble_file(AssembleTask.DIF, self.project, 'testfile', file_checksum.hexdigest(), [x[1] for x in files], 'dummy.type') assert rv is not None f, tmp = rv assert f.checksum == file_checksum.hexdigest() assert f.type == 'dummy.type' # upload all blobs a second time for f, _ in files: f.seek(0) FileBlob.from_files(files, organization=self.organization) # assemble a second time f = assemble_file(AssembleTask.DIF, self.project, 'testfile', file_checksum.hexdigest(), [x[1] for x in files], 'dummy.type')[0] assert f.checksum == file_checksum.hexdigest()
def test_assemble_from_files(self): files = [] file_checksum = sha1() for _ in xrange(8): blob = os.urandom(1024 * 1024 * 8) hash = sha1(blob).hexdigest() file_checksum.update(blob) files.append((io.BytesIO(blob), hash)) # upload all blobs FileBlob.from_files(files, organization=self.organization) # find all blobs for reference, checksum in files: blob = FileBlob.objects.get(checksum=checksum) ref_bytes = reference.getvalue() assert blob.getfile().read(len(ref_bytes)) == ref_bytes FileBlobOwner.objects.filter( blob=blob, organization=self.organization ).get() rv = assemble_file( self.project, 'testfile', file_checksum.hexdigest(), [x[1] for x in files], 'dummy.type') assert rv is not None f, tmp = rv assert f.checksum == file_checksum.hexdigest() assert f.type == 'dummy.type' # upload all blobs a second time for f, _ in files: f.seek(0) FileBlob.from_files(files, organization=self.organization) # assemble a second time f = assemble_file( self.project, 'testfile', file_checksum.hexdigest(), [x[1] for x in files], 'dummy.type')[0] assert f.checksum == file_checksum.hexdigest()