def test_wrong_dif(self): content1 = 'foo'.encode('utf-8') fileobj1 = ContentFile(content1) content2 = 'bar'.encode('utf-8') fileobj2 = ContentFile(content2) content3 = 'baz'.encode('utf-8') fileobj3 = ContentFile(content3) total_checksum = sha1(content2 + content1 + content3).hexdigest() # The order here is on purpose because we check for the order of checksums blob1 = FileBlob.from_file(fileobj1) blob3 = FileBlob.from_file(fileobj3) blob2 = FileBlob.from_file(fileobj2) chunks = [blob2.checksum, blob1.checksum, blob3.checksum] assemble_dif( project_id=self.project.id, name='foo.sym', checksum=total_checksum, chunks=chunks, ) assert get_assemble_status(self.project, total_checksum)[0] == ChunkFileState.ERROR
def test_from_file(self): fileobj = ContentFile('foo bar'.encode('utf-8')) my_file1 = FileBlob.from_file(fileobj) assert my_file1.path my_file2 = FileBlob.from_file(fileobj) # deep check assert my_file1.id == my_file2.id assert my_file1.checksum == my_file2.checksum assert my_file1.path == my_file2.path
def test_from_file(self): fileobj = ContentFile("foo bar") my_file1 = FileBlob.from_file(fileobj) assert my_file1.path assert my_file1.storage == settings.SENTRY_FILESTORE my_file2 = FileBlob.from_file(fileobj) # deep check assert my_file1.id == my_file2.id assert my_file1.checksum == my_file2.checksum assert my_file1.path == my_file2.path assert my_file1.storage == my_file2.storage assert my_file1.storage_options == my_file2.storage_options
def test_dif_response(self): sym_file = self.load_fixture('crash.sym') blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() chunks = [blob1.checksum] assemble_dif( project_id=self.project.id, name='crash.sym', checksum=total_checksum, chunks=chunks, ) response = self.client.post( self.url, data={ total_checksum: { 'name': 'test.sym', 'chunks': chunks, } }, HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) ) assert response.status_code == 200, response.content assert response.data[total_checksum]['state'] == ChunkFileState.OK assert response.data[total_checksum]['dif']['cpuName'] == 'x86_64' assert response.data[total_checksum]['dif']['uuid'] == '67e9247c-814e-392b-a027-dbde6748fcbf'
def test_dif_error_response(self): sym_file = 'fail' blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() chunks = [blob1.checksum] assemble_dif( project_id=self.project.id, name='test.sym', checksum=total_checksum, chunks=chunks, ) response = self.client.post( self.url, data={ total_checksum: { 'name': 'test.sym', 'chunks': [], } }, HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) ) assert response.status_code == 200, response.content assert response.data[total_checksum]['state'] == ChunkFileState.ERROR assert response.data[total_checksum]['detail'].startswith('Invalid debug information file')
def test_legacy_blob(self): fileobj = ContentFile("foo bar") blob = FileBlob.from_file(fileobj) file1 = File.objects.create( name='baz.js', type='default', size=7, blob=blob, ) with file1.getfile() as fp: assert fp.read() == 'foo bar'
def post(self, request, organization): """ Upload chunks and store them as FileBlobs ````````````````````````````````````````` :pparam file file: The filename should be sha1 hash of the content. Also not you can add up to MAX_CHUNKS_PER_REQUEST files in this request. :auth: required """ files = request.FILES.getlist('file') files += [GzipChunk(chunk) for chunk in request.FILES.getlist('file_gzip')] if len(files) == 0: # No files uploaded is ok return Response(status=status.HTTP_200_OK) # Validate file size checksums = [] size = 0 for chunk in files: size += chunk.size if chunk.size > DEFAULT_BLOB_SIZE: return Response({'error': 'Chunk size too large'}, status=status.HTTP_400_BAD_REQUEST) checksums.append(chunk.name) if size > MAX_REQUEST_SIZE: return Response({'error': 'Request too large'}, status=status.HTTP_400_BAD_REQUEST) if len(files) > MAX_CHUNKS_PER_REQUEST: return Response({'error': 'Too many chunks'}, status=status.HTTP_400_BAD_REQUEST) for checksum, chunk in izip(checksums, files): # Here we create the actual blob blob = FileBlob.from_file(chunk) # Add ownership to the blob here try: with transaction.atomic(): FileBlobOwner.objects.create( organization=organization, blob=blob ) except IntegrityError: pass if blob.checksum != checksum: # We do not clean up here since we have a cleanup job return Response({'error': 'Checksum missmatch'}, status=status.HTTP_400_BAD_REQUEST) return Response(status=status.HTTP_200_OK)
def test_artifacts_invalid_org(self): bundle_file = self.create_artifact_bundle(org="invalid") blob1 = FileBlob.from_file(ContentFile(bundle_file)) total_checksum = sha1(bundle_file).hexdigest() assemble_artifacts( org_id=self.organization.id, version=self.release.version, checksum=total_checksum, chunks=[blob1.checksum], ) status, details = get_assemble_status(AssembleTask.ARTIFACTS, self.organization.id, total_checksum) assert status == ChunkFileState.ERROR
def test_dif(self): sym_file = self.load_fixture('crash.sym') blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() assemble_dif( project_id=self.project.id, name='crash.sym', checksum=total_checksum, chunks=[blob1.checksum], ) dif = ProjectDebugFile.objects.filter( project=self.project, file__checksum=total_checksum, ).get() assert dif.file.headers == {'Content-Type': 'text/x-breakpad'}
def test_dif(self): sym_file = self.load_fixture('crash.sym') blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() assemble_dif( project_id=self.project.id, name='crash.sym', checksum=total_checksum, chunks=[blob1.checksum], ) dif = ProjectDSymFile.objects.filter( project=self.project, file__checksum=total_checksum, ).get() assert dif.file.headers == {'Content-Type': 'text/x-breakpad'}
def test_dif(self): sym_file = self.load_fixture("crash.sym") blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() assemble_dif( project_id=self.project.id, name="crash.sym", checksum=total_checksum, chunks=[blob1.checksum], ) status, _ = get_assemble_status(AssembleTask.DIF, self.project.id, total_checksum) assert status == ChunkFileState.OK dif = ProjectDebugFile.objects.filter(project=self.project, checksum=total_checksum).get() assert dif.file.headers == {"Content-Type": "text/x-breakpad"}
def test_dif_error_response(self): sym_file = b"fail" blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() chunks = [blob1.checksum] assemble_dif( project_id=self.project.id, name="test.sym", checksum=total_checksum, chunks=chunks ) response = self.client.post( self.url, data={total_checksum: {"name": "test.sym", "chunks": []}}, HTTP_AUTHORIZATION=f"Bearer {self.token.token}", ) assert response.status_code == 200, response.content assert response.data[total_checksum]["state"] == ChunkFileState.ERROR assert "unsupported object file format" in response.data[total_checksum]["detail"]
def test_artifacts(self): bundle_file = self.create_artifact_bundle() blob1 = FileBlob.from_file(ContentFile(bundle_file)) total_checksum = sha1(bundle_file).hexdigest() for has_release_archives in (True, False): with self.options({ "processing.save-release-archives": has_release_archives, "processing.release-archive-min-files": 1, }): assemble_artifacts( org_id=self.organization.id, version=self.release.version, checksum=total_checksum, chunks=[blob1.checksum], ) status, details = get_assemble_status(AssembleTask.ARTIFACTS, self.organization.id, total_checksum) assert status == ChunkFileState.OK assert details is None release_file = ReleaseFile.objects.get( organization=self.organization, release=self.release, name="release-artifacts.zip" if has_release_archives else "~/index.js", dist=None, ) assert release_file if has_release_archives: assert release_file.file.headers == {} # Artifact is the same as original bundle assert release_file.file.size == len(bundle_file) else: assert release_file.file.headers == { "Sourcemap": "index.js.map" }
def test_assemble_response(self): bundle_file = self.create_artifact_bundle() total_checksum = sha1(bundle_file).hexdigest() blob1 = FileBlob.from_file(ContentFile(bundle_file)) assemble_artifacts( org_id=self.organization.id, version=self.release.version, checksum=total_checksum, chunks=[blob1.checksum], ) response = self.client.post( self.url, data={"checksum": total_checksum, "chunks": [blob1.checksum]}, HTTP_AUTHORIZATION=u"Bearer {}".format(self.token.token), ) assert response.status_code == 200, response.content assert response.data["state"] == ChunkFileState.OK
def create_release_file(self, project, release, path, content_type=None, contents=None): from sentry.models import File, FileBlob, ReleaseFile if content_type is None: content_type = mimetypes.guess_type(path)[0] or 'text/plain' if content_type.startswith('text/'): content_type += '; encoding=utf-8' f = File.objects.create( name=path.rsplit('/', 1)[-1], type='release.file', headers={ 'Content-Type': content_type }, blob=FileBlob.from_file(StringIO(contents or '')), ) return ReleaseFile.objects.create( project=project, release=release, file=f, name=path )
def test_failing_update(self, _): bundle_file = self.create_artifact_bundle() blob1 = FileBlob.from_file(ContentFile(bundle_file)) total_checksum = sha1(bundle_file).hexdigest() with self.options({ "processing.save-release-archives": True, "processing.release-archive-min-files": 1, }): assemble_artifacts( org_id=self.organization.id, version=self.release.version, checksum=total_checksum, chunks=[blob1.checksum], ) # Status is still OK: status, details = get_assemble_status(AssembleTask.ARTIFACTS, self.organization.id, total_checksum) assert status == ChunkFileState.OK
def test_assemble_debug_id_override(self): sym_file = self.load_fixture("crash.sym") blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() assemble_dif( project_id=self.project.id, name="crash.sym", checksum=total_checksum, chunks=[blob1.checksum], debug_id="67e9247c-814e-392b-a027-dbde6748fcbf-beef", ) status, _ = get_assemble_status(AssembleTask.DIF, self.project.id, total_checksum) assert status == ChunkFileState.OK dif = ProjectDebugFile.objects.filter( project=self.project, file__checksum=total_checksum).get() assert dif.file.headers == {"Content-Type": "text/x-breakpad"} assert dif.debug_id == "67e9247c-814e-392b-a027-dbde6748fcbf-beef"
def test_dif_response(self): sym_file = self.load_fixture("crash.sym") blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() chunks = [blob1.checksum] assemble_dif( project_id=self.project.id, name="crash.sym", checksum=total_checksum, chunks=chunks ) response = self.client.post( self.url, data={total_checksum: {"name": "test.sym", "chunks": chunks}}, HTTP_AUTHORIZATION=f"Bearer {self.token.token}", ) assert response.status_code == 200, response.content assert response.data[total_checksum]["state"] == ChunkFileState.OK assert response.data[total_checksum]["dif"]["cpuName"] == "x86_64" assert ( response.data[total_checksum]["dif"]["uuid"] == "67e9247c-814e-392b-a027-dbde6748fcbf" )
def store_export_chunk_as_blob(data_export, bytes_written, fileobj, blob_size=DEFAULT_BLOB_SIZE): # adapted from `putfile` in `src/sentry/models/file.py` bytes_offset = 0 while True: contents = fileobj.read(blob_size) if not contents: return bytes_offset blob_fileobj = ContentFile(contents) blob = FileBlob.from_file(blob_fileobj, logger=logger) ExportedDataBlob.objects.create( data_export=data_export, blob=blob, offset=bytes_written + bytes_offset ) bytes_offset += blob.size # there is a maximum file size allowed, so we need to make sure we don't exceed it # NOTE: there seems to be issues with downloading files larger than 1 GB on slower # networks, limit the export to 1 GB for now to improve reliability if bytes_written + bytes_offset >= min(MAX_FILE_SIZE, 2 ** 30): transaction.set_rollback(True) return 0
def test_dif_error_response(self): bundle_file = b'invalid' total_checksum = sha1(bundle_file).hexdigest() blob1 = FileBlob.from_file(ContentFile(bundle_file)) assemble_artifacts( org_id=self.organization.id, version=self.release.version, checksum=total_checksum, chunks=[blob1.checksum], ) response = self.client.post( self.url, data={ 'checksum': total_checksum, 'chunks': [blob1.checksum], }, HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) ) assert response.status_code == 200, response.content assert response.data['state'] == ChunkFileState.ERROR
def store_export_chunk_as_blob(data_export, bytes_written, fileobj, blob_size=DEFAULT_BLOB_SIZE): # adapted from `putfile` in `src/sentry/models/file.py` bytes_offset = 0 while True: contents = fileobj.read(blob_size) if not contents: return bytes_offset blob_fileobj = ContentFile(contents) blob = FileBlob.from_file(blob_fileobj, logger=logger) ExportedDataBlob.objects.create(data_export=data_export, blob=blob, offset=bytes_written + bytes_offset) bytes_offset += blob.size # there is a maximum file size allowed, so we need to make sure we don't exceed it if bytes_written + bytes_offset >= MAX_FILE_SIZE: transaction.set_rollback(True) return 0
def test_dif(self): sym_file = self.load_fixture('crash.sym') blob1 = FileBlob.from_file(ContentFile(sym_file)) total_checksum = sha1(sym_file).hexdigest() file = File.objects.create( name='test.sym', checksum=total_checksum, type='chunked', headers={CHUNK_STATE_HEADER: ChunkFileState.CREATED}) file_blob_id_order = [blob1.id] assemble_dif( project_id=self.project.id, file_id=file.id, file_blob_ids=file_blob_id_order, checksum=total_checksum, ) file = File.objects.filter(checksum=total_checksum, ).get() assert file.headers == {'Content-Type': 'text/x-breakpad'}
def test_artifacts(self): bundle_file = self.create_artifact_bundle() blob1 = FileBlob.from_file(ContentFile(bundle_file)) total_checksum = sha1(bundle_file).hexdigest() assemble_artifacts( org_id=self.organization.id, version=self.release.version, checksum=total_checksum, chunks=[blob1.checksum], ) status, details = get_assemble_status( AssembleTask.ARTIFACTS, self.organization.id, total_checksum ) assert status == ChunkFileState.OK assert details is None release_file = ReleaseFile.objects.get( organization=self.organization, release=self.release, name="~/index.js", dist=None ) assert release_file assert release_file.file.headers == {"Sourcemap": "index.js.map"}
def post(self, request, project, version): """ Upload a New File ````````````````` Upload a new file for the given release. Unlike other API requests, files must be uploaded using the traditional multipart/form-data content-type. The optional 'name' attribute should reflect the absolute path that this file will be referenced as. For example, in the case of JavaScript you might specify the full web URI. :pparam string organization_slug: the slug of the organization the release belongs to. :pparam string project_slug: the slug of the project to change the release of. :pparam string version: the version identifier of the release. :param string name: the name (full path) of the file. :param file file: the multipart encoded file. :param string header: this parameter can be supplied multiple times to attach headers to the file. Each header is a string in the format ``key:value``. For instance it can be used to define a content type. :auth: required """ try: release = Release.objects.get( project=project, version=version, ) except Release.DoesNotExist: raise ResourceDoesNotExist if 'file' not in request.FILES: return Response({'detail': 'Missing uploaded file'}, status=400) fileobj = request.FILES['file'] full_name = request.DATA.get('name', fileobj.name) name = full_name.rsplit('/', 1)[-1] headers = { 'Content-Type': fileobj.content_type, } for headerval in request.DATA.getlist('header') or (): try: k, v = headerval.split(':', 1) except ValueError: return Response( {'detail': 'header value was not formatted correctly'}, status=400) else: headers[k] = v.strip() blob = FileBlob.from_file(fileobj) file = File.objects.create( name=name, type='release.file', headers=headers, blob=blob, ) try: with transaction.atomic(): releasefile = ReleaseFile.objects.create( project=release.project, release=release, file=file, name=full_name, ) except IntegrityError: file.delete() return Response(status=409) return Response(serialize(releasefile, request.user), status=201)
def post(self, request, project, version): """ Upload a New File ````````````````` Upload a new file for the given release. Unlike other API requests, files must be uploaded using the traditional multipart/form-data content-type. The optional 'name' attribute should reflect the absolute path that this file will be referenced as. For example, in the case of JavaScript you might specify the full web URI. :pparam string organization_slug: the slug of the organization the release belongs to. :pparam string project_slug: the slug of the project to change the release of. :pparam string version: the version identifier of the release. :param string name: the name (full path) of the file. :param file file: the multipart encoded file. :param string header: this parameter can be supplied multiple times to attach headers to the file. Each header is a string in the format ``key:value``. For instance it can be used to define a content type. :auth: required """ version = unquote(version) try: release = Release.objects.get( project=project, version=version, ) except Release.DoesNotExist: raise ResourceDoesNotExist if 'file' not in request.FILES: return Response({'detail': 'Missing uploaded file'}, status=400) fileobj = request.FILES['file'] full_name = request.DATA.get('name', fileobj.name) name = full_name.rsplit('/', 1)[-1] headers = { 'Content-Type': fileobj.content_type, } for headerval in request.DATA.getlist('header') or (): try: k, v = headerval.split(':', 1) except ValueError: return Response({'detail': 'header value was not formatted correctly'}, status=400) else: headers[k] = v.strip() blob = FileBlob.from_file(fileobj) file = File.objects.create( name=name, type='release.file', headers=headers, blob=blob, ) try: with transaction.atomic(): releasefile = ReleaseFile.objects.create( project=release.project, release=release, file=file, name=full_name, ) except IntegrityError: file.delete() return Response(status=409) return Response(serialize(releasefile, request.user), status=201)
def test_expansion_via_release_artifacts(self): project = self.project release = Release.objects.create( project=project, version='abc', ) f1 = File.objects.create(name='file.min.js', type='release.file', headers={'Content-Type': 'application/json'}, blob=FileBlob.from_file( open(get_fixture_path('file.min.js'), 'rb'))) ReleaseFile.objects.create( name='http://example.com/{}'.format(f1.name), release=release, project=project, file=f1, ) f2 = File.objects.create(name='file1.js', type='release.file', headers={'Content-Type': 'application/json'}, blob=FileBlob.from_file( open(get_fixture_path('file1.js'), 'rb'))) ReleaseFile.objects.create( name='http://example.com/{}'.format(f2.name), release=release, project=project, file=f2, ) f3 = File.objects.create(name='file2.js', type='release.file', headers={'Content-Type': 'application/json'}, blob=FileBlob.from_file( open(get_fixture_path('file2.js'), 'rb'))) ReleaseFile.objects.create( name='http://example.com/{}'.format(f3.name), release=release, project=project, file=f3, ) f4 = File.objects.create(name='file.sourcemap.js', type='release.file', headers={'Content-Type': 'application/json'}, blob=FileBlob.from_file( open( get_fixture_path('file.sourcemap.js'), 'rb'))) ReleaseFile.objects.create( name='http://example.com/{}'.format(f4.name), release=release, project=project, file=f4, ) data = { 'message': 'hello', 'platform': 'javascript', 'release': 'abc', 'sentry.interfaces.Exception': { 'values': [{ 'type': 'Error', 'stacktrace': { 'frames': [ { 'abs_path': 'http://example.com/file.min.js', 'filename': 'file.min.js', 'lineno': 1, 'colno': 39, }, ], }, }], } } resp = self._postWithHeader(data) assert resp.status_code, 200 event = Event.objects.get() assert not event.data['errors'] exception = event.interfaces['sentry.interfaces.Exception'] frame_list = exception.values[0].stacktrace.frames frame = frame_list[0] assert frame.pre_context == [ 'function add(a, b) {', '\t"use strict";', ] assert frame.context_line == '\treturn a + b;' assert frame.post_context == ['}']
def test_expansion_via_release_artifacts(self): project = self.project release = Release.objects.create( project=project, version='abc', ) f1 = File.objects.create( name='file.min.js', type='release.file', headers={'Content-Type': 'application/json'}, blob=FileBlob.from_file(open(get_fixture_path('file.min.js'), 'rb')) ) ReleaseFile.objects.create( name='http://example.com/{}'.format(f1.name), release=release, project=project, file=f1, ) f2 = File.objects.create( name='file1.js', type='release.file', headers={'Content-Type': 'application/json'}, blob=FileBlob.from_file(open(get_fixture_path('file1.js'), 'rb')) ) ReleaseFile.objects.create( name='http://example.com/{}'.format(f2.name), release=release, project=project, file=f2, ) f3 = File.objects.create( name='file2.js', type='release.file', headers={'Content-Type': 'application/json'}, blob=FileBlob.from_file(open(get_fixture_path('file2.js'), 'rb')) ) ReleaseFile.objects.create( name='http://example.com/{}'.format(f3.name), release=release, project=project, file=f3, ) f4 = File.objects.create( name='file.sourcemap.js', type='release.file', headers={'Content-Type': 'application/json'}, blob=FileBlob.from_file(open(get_fixture_path('file.sourcemap.js'), 'rb')) ) ReleaseFile.objects.create( name='http://example.com/{}'.format(f4.name), release=release, project=project, file=f4, ) data = { 'message': 'hello', 'platform': 'javascript', 'release': 'abc', 'sentry.interfaces.Exception': { 'values': [{ 'type': 'Error', 'stacktrace': { 'frames': [ { 'abs_path': 'http://example.com/file.min.js', 'filename': 'file.min.js', 'lineno': 1, 'colno': 39, }, ], }, }], } } resp = self._postWithHeader(data) assert resp.status_code, 200 event = Event.objects.get() assert not event.data['errors'] exception = event.interfaces['sentry.interfaces.Exception'] frame_list = exception.values[0].stacktrace.frames frame = frame_list[0] assert frame.pre_context == [ 'function add(a, b) {', '\t"use strict";', ] assert frame.context_line == '\treturn a + b;' assert frame.post_context == ['}']
def test_assemble(self, mock_assemble_dif): content1 = 'foo'.encode('utf-8') fileobj1 = ContentFile(content1) checksum1 = sha1(content1).hexdigest() content2 = 'bar'.encode('utf-8') fileobj2 = ContentFile(content2) checksum2 = sha1(content2).hexdigest() content3 = 'baz'.encode('utf-8') fileobj3 = ContentFile(content3) checksum3 = sha1(content3).hexdigest() total_checksum = sha1(content2 + content1 + content3).hexdigest() # The order here is on purpose because we check for the order of checksums blob1 = FileBlob.from_file(fileobj1) FileBlobOwner.objects.get_or_create( organization=self.organization, blob=blob1 ) blob3 = FileBlob.from_file(fileobj3) FileBlobOwner.objects.get_or_create( organization=self.organization, blob=blob3 ) blob2 = FileBlob.from_file(fileobj2) # we make a request now but we are missing ownership for chunk 2 response = self.client.post( self.url, data={ total_checksum: { 'name': 'test', 'chunks': [ checksum2, checksum1, checksum3 ] } }, HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) ) assert response.status_code == 200, response.content assert response.data[total_checksum]['state'] == ChunkFileState.NOT_FOUND assert response.data[total_checksum]['missingChunks'] == [checksum2] # we add ownership to chunk 2 FileBlobOwner.objects.get_or_create( organization=self.organization, blob=blob2 ) # new request, ownership for all chunks is there but file does not exist yet response = self.client.post( self.url, data={ total_checksum: { 'name': 'test', 'chunks': [ checksum2, checksum1, checksum3 ], } }, HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token) ) assert response.status_code == 200, response.content assert response.data[total_checksum]['state'] == ChunkFileState.CREATED assert response.data[total_checksum]['missingChunks'] == [] chunks = [checksum2, checksum1, checksum3] mock_assemble_dif.apply_async.assert_called_once_with( kwargs={ 'project_id': self.project.id, 'name': 'test', 'chunks': chunks, 'checksum': total_checksum, } ) file = assemble_file(self.project, 'test', total_checksum, chunks, 'project.dsym')[0] assert get_assemble_status(self.project, total_checksum)[0] != ChunkFileState.ERROR assert file.checksum == total_checksum file_blob_index = FileBlobIndex.objects.all() assert len(file_blob_index) == 3
def test_assemble(self, mock_assemble_dif): content1 = b"foo" fileobj1 = ContentFile(content1) checksum1 = sha1(content1).hexdigest() content2 = b"bar" fileobj2 = ContentFile(content2) checksum2 = sha1(content2).hexdigest() content3 = b"baz" fileobj3 = ContentFile(content3) checksum3 = sha1(content3).hexdigest() total_checksum = sha1(content2 + content1 + content3).hexdigest() # The order here is on purpose because we check for the order of checksums blob1 = FileBlob.from_file(fileobj1) FileBlobOwner.objects.get_or_create( organization_id=self.organization.id, blob=blob1) blob3 = FileBlob.from_file(fileobj3) FileBlobOwner.objects.get_or_create( organization_id=self.organization.id, blob=blob3) blob2 = FileBlob.from_file(fileobj2) # we make a request now but we are missing ownership for chunk 2 response = self.client.post( self.url, data={ total_checksum: { "name": "test", "chunks": [checksum2, checksum1, checksum3] } }, HTTP_AUTHORIZATION=f"Bearer {self.token.token}", ) assert response.status_code == 200, response.content assert response.data[total_checksum][ "state"] == ChunkFileState.NOT_FOUND assert response.data[total_checksum]["missingChunks"] == [checksum2] # we add ownership to chunk 2 FileBlobOwner.objects.get_or_create( organization_id=self.organization.id, blob=blob2) # new request, ownership for all chunks is there but file does not exist yet response = self.client.post( self.url, data={ total_checksum: { "name": "test", "chunks": [checksum2, checksum1, checksum3] } }, HTTP_AUTHORIZATION=f"Bearer {self.token.token}", ) assert response.status_code == 200, response.content assert response.data[total_checksum]["state"] == ChunkFileState.CREATED assert response.data[total_checksum]["missingChunks"] == [] chunks = [checksum2, checksum1, checksum3] mock_assemble_dif.apply_async.assert_called_once_with( kwargs={ "project_id": self.project.id, "name": "test", "chunks": chunks, "checksum": total_checksum, "debug_id": None, }) file = assemble_file(AssembleTask.DIF, self.project, "test", total_checksum, chunks, "project.dif")[0] status, _ = get_assemble_status(AssembleTask.DIF, self.project.id, total_checksum) assert status != ChunkFileState.ERROR assert file.checksum == total_checksum file_blob_index = FileBlobIndex.objects.all() assert len(file_blob_index) == 3
def test_assemble(self, mock_assemble_dif): content1 = 'foo'.encode('utf-8') fileobj1 = ContentFile(content1) checksum1 = sha1(content1).hexdigest() content2 = 'bar'.encode('utf-8') fileobj2 = ContentFile(content2) checksum2 = sha1(content2).hexdigest() content3 = 'baz'.encode('utf-8') fileobj3 = ContentFile(content3) checksum3 = sha1(content3).hexdigest() total_checksum = sha1(content2 + content1 + content3).hexdigest() # The order here is on purpose because we check for the order of checksums blob1 = FileBlob.from_file(fileobj1) FileBlobOwner.objects.get_or_create( organization=self.organization, blob=blob1 ) blob3 = FileBlob.from_file(fileobj3) FileBlobOwner.objects.get_or_create( organization=self.organization, blob=blob3 ) blob2 = FileBlob.from_file(fileobj2) # we make a request now but we are missing ownership for chunk 2 response = self.client.post( self.url, data={ total_checksum: { 'name': 'test', 'chunks': [ checksum2, checksum1, checksum3 ] } }, HTTP_AUTHORIZATION='Bearer {}'.format(self.token.token) ) assert response.status_code == 200, response.content assert response.data[total_checksum]['state'] == ChunkFileState.NOT_FOUND assert response.data[total_checksum]['missingChunks'] == [checksum2] # we add ownership to chunk 2 FileBlobOwner.objects.get_or_create( organization=self.organization, blob=blob2 ) # new request, ownership for all chunks is there but file does not exist yet response = self.client.post( self.url, data={ total_checksum: { 'name': 'test', 'chunks': [ checksum2, checksum1, checksum3 ], } }, HTTP_AUTHORIZATION='Bearer {}'.format(self.token.token) ) assert response.status_code == 200, response.content assert response.data[total_checksum]['state'] == ChunkFileState.CREATED assert response.data[total_checksum]['missingChunks'] == [] chunks = [checksum2, checksum1, checksum3] mock_assemble_dif.apply_async.assert_called_once_with( kwargs={ 'project_id': self.project.id, 'name': 'test', 'chunks': chunks, 'checksum': total_checksum, } ) file = assemble_file(self.project, 'test', total_checksum, chunks, 'project.dsym')[0] assert get_assemble_status(self.project, total_checksum)[0] != ChunkFileState.ERROR assert file.checksum == total_checksum file_blob_index = FileBlobIndex.objects.all() assert len(file_blob_index) == 3