def post(self, request, project): """ Assemble one or multiple chunks (FileBlob) into debug files ```````````````````````````````````````````````````````````` :auth: required """ schema = { "type": "object", "patternProperties": { "^[0-9a-f]{40}$": { "type": "object", "required": ["name", "chunks"], "properties": { "name": { "type": "string" }, "debug_id": { "type": "string" }, "chunks": { "type": "array", "items": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, }, }, "additionalProperties": True, } }, "additionalProperties": False, } try: files = json.loads(request.body) jsonschema.validate(files, schema) except jsonschema.ValidationError as e: return Response({"error": str(e).splitlines()[0]}, status=400) except BaseException: return Response({"error": "Invalid json body"}, status=400) file_response = {} for checksum, file_to_assemble in six.iteritems(files): name = file_to_assemble.get("name", None) debug_id = file_to_assemble.get("debug_id", None) chunks = file_to_assemble.get("chunks", []) # First, check the cached assemble status. During assembling, a # ProjectDebugFile will be created and we need to prevent a race # condition. state, detail = get_assemble_status(AssembleTask.DIF, project.id, checksum) if state == ChunkFileState.OK: file_response[checksum] = { "state": state, "detail": None, "missingChunks": [], "dif": detail, } continue elif state is not None: file_response[checksum] = { "state": state, "detail": detail, "missingChunks": [] } continue # Next, check if this project already owns the ProjectDebugFile. # This can under rare circumstances yield more than one file # which is why we use first() here instead of get(). dif = (ProjectDebugFile.objects.filter( project=project, file__checksum=checksum).select_related( "file").order_by("-id").first()) if dif is not None: file_response[checksum] = { "state": ChunkFileState.OK, "detail": None, "missingChunks": [], "dif": serialize(dif), } continue # There is neither a known file nor a cached state, so we will # have to create a new file. Assure that there are checksums. # If not, we assume this is a poll and report NOT_FOUND if not chunks: file_response[checksum] = { "state": ChunkFileState.NOT_FOUND, "missingChunks": [] } continue # Check if all requested chunks have been uploaded. missing_chunks = find_missing_chunks(project.organization, chunks) if missing_chunks: file_response[checksum] = { "state": ChunkFileState.NOT_FOUND, "missingChunks": missing_chunks, } continue # We don't have a state yet, this means we can now start # an assemble job in the background. set_assemble_status(AssembleTask.DIF, project.id, checksum, ChunkFileState.CREATED) from sentry.tasks.assemble import assemble_dif assemble_dif.apply_async( kwargs={ "project_id": project.id, "name": name, "debug_id": debug_id, "checksum": checksum, "chunks": chunks, }) file_response[checksum] = { "state": ChunkFileState.CREATED, "missingChunks": [] } return Response(file_response, status=200)
def test_assemble_check(self): content = 'foo bar'.encode('utf-8') fileobj = ContentFile(content) file1 = File.objects.create( name='baz.dSYM', type='default', size=7, ) file1.putfile(fileobj, 3) checksum = sha1(content).hexdigest() blobs = FileBlob.objects.all() checksums = [] for blob in blobs: checksums.append(blob.checksum) # Request to see of file is there # file exists but we have no overship for the chunks response = self.client.post( self.url, data={checksum: { 'name': 'dif', 'chunks': checksums, }}, HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token)) assert response.status_code == 200, response.content assert response.data[checksum]['state'] == ChunkFileState.NOT_FOUND assert set(response.data[checksum]['missingChunks']) == set(checksums) # Now we add ownership to the blob blobs = FileBlob.objects.all() for blob in blobs: FileBlobOwner.objects.create(blob=blob, organization=self.organization) # The request will start the job to assemble the file response = self.client.post( self.url, data={checksum: { 'name': 'dif', 'chunks': checksums, }}, HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token)) assert response.status_code == 200, response.content assert response.data[checksum]['state'] == ChunkFileState.CREATED assert response.data[checksum]['missingChunks'] == [] # Finally, we simulate a successful job ProjectDebugFile.objects.create( file=file1, object_name='baz.dSYM', cpu_name='x86_64', project=self.project, debug_id='df449af8-0dcd-4320-9943-ec192134d593', code_id='DF449AF80DCD43209943EC192134D593', ) set_assemble_status(AssembleTask.DIF, self.project.id, checksum, None) # Request now tells us that everything is alright response = self.client.post( self.url, data={checksum: { 'name': 'dif', 'chunks': checksums, }}, HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token)) assert response.status_code == 200, response.content assert response.data[checksum]['state'] == ChunkFileState.OK assert response.data[checksum]['missingChunks'] == [] not_found_checksum = sha1('1').hexdigest() response = self.client.post(self.url, data={ not_found_checksum: { 'name': 'dif', 'chunks': [not_found_checksum], } }, HTTP_AUTHORIZATION=u'Bearer {}'.format( self.token.token)) assert response.status_code == 200, response.content assert response.data[not_found_checksum][ 'state'] == ChunkFileState.NOT_FOUND assert set(response.data[not_found_checksum]['missingChunks']) == set( [not_found_checksum])
def test_assemble_check(self): content = b"foo bar" fileobj = ContentFile(content) file1 = File.objects.create(name="baz.dSYM", type="default", size=7) file1.putfile(fileobj, 3) checksum = sha1(content).hexdigest() blobs = FileBlob.objects.all() checksums = [] for blob in blobs: checksums.append(blob.checksum) # Request to see of file is there # file exists but we have no overship for the chunks response = self.client.post( self.url, data={checksum: { "name": "dif", "chunks": checksums }}, HTTP_AUTHORIZATION=f"Bearer {self.token.token}", ) assert response.status_code == 200, response.content assert response.data[checksum]["state"] == ChunkFileState.NOT_FOUND assert set(response.data[checksum]["missingChunks"]) == set(checksums) # Now we add ownership to the blob blobs = FileBlob.objects.all() for blob in blobs: FileBlobOwner.objects.create(blob=blob, organization_id=self.organization.id) # The request will start the job to assemble the file response = self.client.post( self.url, data={checksum: { "name": "dif", "chunks": checksums }}, HTTP_AUTHORIZATION=f"Bearer {self.token.token}", ) assert response.status_code == 200, response.content assert response.data[checksum]["state"] == ChunkFileState.CREATED assert response.data[checksum]["missingChunks"] == [] # Finally, we simulate a successful job ProjectDebugFile.objects.create( file=file1, checksum=file1.checksum, object_name="baz.dSYM", cpu_name="x86_64", project_id=self.project.id, debug_id="df449af8-0dcd-4320-9943-ec192134d593", code_id="DF449AF80DCD43209943EC192134D593", ) set_assemble_status(AssembleTask.DIF, self.project.id, checksum, None) # Request now tells us that everything is alright response = self.client.post( self.url, data={checksum: { "name": "dif", "chunks": checksums }}, HTTP_AUTHORIZATION=f"Bearer {self.token.token}", ) assert response.status_code == 200, response.content assert response.data[checksum]["state"] == ChunkFileState.OK assert response.data[checksum]["missingChunks"] == [] not_found_checksum = sha1(b"1").hexdigest() response = self.client.post( self.url, data={ not_found_checksum: { "name": "dif", "chunks": [not_found_checksum] } }, HTTP_AUTHORIZATION=f"Bearer {self.token.token}", ) assert response.status_code == 200, response.content assert response.data[not_found_checksum][ "state"] == ChunkFileState.NOT_FOUND assert set(response.data[not_found_checksum]["missingChunks"]) == { not_found_checksum }
def post(self, request, organization, version): """ Handle an artifact bundle and merge it into the release ``````````````````````````````````````````````````````` :auth: required """ try: release = Release.objects.get(organization_id=organization.id, version=version) except Release.DoesNotExist: raise ResourceDoesNotExist if not self.has_release_permission(request, organization, release): raise ResourceDoesNotExist schema = { "type": "object", "properties": { "checksum": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, "chunks": { "type": "array", "items": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, }, }, "required": ["checksum", "chunks"], "additionalProperties": False, } try: data = json.loads(request.body) jsonschema.validate(data, schema) except jsonschema.ValidationError as e: return Response({"error": str(e).splitlines()[0]}, status=400) except BaseException: return Response({"error": "Invalid json body"}, status=400) checksum = data.get("checksum", None) chunks = data.get("chunks", []) state, detail = get_assemble_status(AssembleTask.ARTIFACTS, organization.id, checksum) if state == ChunkFileState.OK: return Response({"state": state, "detail": None, "missingChunks": []}, status=200) elif state is not None: return Response({"state": state, "detail": detail, "missingChunks": []}) # There is neither a known file nor a cached state, so we will # have to create a new file. Assure that there are checksums. # If not, we assume this is a poll and report NOT_FOUND if not chunks: return Response({"state": ChunkFileState.NOT_FOUND, "missingChunks": []}, status=200) set_assemble_status( AssembleTask.ARTIFACTS, organization.id, checksum, ChunkFileState.CREATED ) from sentry.tasks.assemble import assemble_artifacts assemble_artifacts.apply_async( kwargs={ "org_id": organization.id, "version": version, "checksum": checksum, "chunks": chunks, } ) return Response({"state": ChunkFileState.CREATED, "missingChunks": []}, status=200)