def post(self, request, project):
"""
Assemble one or multiple chunks (FileBlob) into debug files
````````````````````````````````````````````````````````````
:auth: required
"""
schema = {
"type": "object",
"patternProperties": {
"^[0-9a-f]{40}$": {
"type": "object",
"required": ["name", "chunks"],
"properties": {
"name": {
"type": "string"
},
"debug_id": {
"type": "string"
},
"chunks": {
"type": "array",
"items": {
"type": "string",
"pattern": "^[0-9a-f]{40}$"
},
},
},
"additionalProperties": True,
}
},
"additionalProperties": False,
}
try:
files = json.loads(request.body)
jsonschema.validate(files, schema)
except jsonschema.ValidationError as e:
return Response({"error": str(e).splitlines()[0]}, status=400)
except BaseException:
return Response({"error": "Invalid json body"}, status=400)
file_response = {}
for checksum, file_to_assemble in six.iteritems(files):
name = file_to_assemble.get("name", None)
debug_id = file_to_assemble.get("debug_id", None)
chunks = file_to_assemble.get("chunks", [])
# First, check the cached assemble status. During assembling, a
# ProjectDebugFile will be created and we need to prevent a race
# condition.
state, detail = get_assemble_status(AssembleTask.DIF, project.id,
checksum)
if state == ChunkFileState.OK:
file_response[checksum] = {
"state": state,
"detail": None,
"missingChunks": [],
"dif": detail,
}
continue
elif state is not None:
file_response[checksum] = {
"state": state,
"detail": detail,
"missingChunks": []
}
continue
# Next, check if this project already owns the ProjectDebugFile.
# This can under rare circumstances yield more than one file
# which is why we use first() here instead of get().
dif = (ProjectDebugFile.objects.filter(
project=project, file__checksum=checksum).select_related(
"file").order_by("-id").first())
if dif is not None:
file_response[checksum] = {
"state": ChunkFileState.OK,
"detail": None,
"missingChunks": [],
"dif": serialize(dif),
}
continue
# There is neither a known file nor a cached state, so we will
# have to create a new file. Assure that there are checksums.
# If not, we assume this is a poll and report NOT_FOUND
if not chunks:
file_response[checksum] = {
"state": ChunkFileState.NOT_FOUND,
"missingChunks": []
}
continue
# Check if all requested chunks have been uploaded.
missing_chunks = find_missing_chunks(project.organization, chunks)
if missing_chunks:
file_response[checksum] = {
"state": ChunkFileState.NOT_FOUND,
"missingChunks": missing_chunks,
}
continue
# We don't have a state yet, this means we can now start
# an assemble job in the background.
set_assemble_status(AssembleTask.DIF, project.id, checksum,
ChunkFileState.CREATED)
from sentry.tasks.assemble import assemble_dif
assemble_dif.apply_async(
kwargs={
"project_id": project.id,
"name": name,
"debug_id": debug_id,
"checksum": checksum,
"chunks": chunks,
})
file_response[checksum] = {
"state": ChunkFileState.CREATED,
"missingChunks": []
}
return Response(file_response, status=200)
def test_assemble_check(self):
content = 'foo bar'.encode('utf-8')
fileobj = ContentFile(content)
file1 = File.objects.create(
name='baz.dSYM',
type='default',
size=7,
)
file1.putfile(fileobj, 3)
checksum = sha1(content).hexdigest()
blobs = FileBlob.objects.all()
checksums = []
for blob in blobs:
checksums.append(blob.checksum)
# Request to see of file is there
# file exists but we have no overship for the chunks
response = self.client.post(
self.url,
data={checksum: {
'name': 'dif',
'chunks': checksums,
}},
HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token))
assert response.status_code == 200, response.content
assert response.data[checksum]['state'] == ChunkFileState.NOT_FOUND
assert set(response.data[checksum]['missingChunks']) == set(checksums)
# Now we add ownership to the blob
blobs = FileBlob.objects.all()
for blob in blobs:
FileBlobOwner.objects.create(blob=blob,
organization=self.organization)
# The request will start the job to assemble the file
response = self.client.post(
self.url,
data={checksum: {
'name': 'dif',
'chunks': checksums,
}},
HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token))
assert response.status_code == 200, response.content
assert response.data[checksum]['state'] == ChunkFileState.CREATED
assert response.data[checksum]['missingChunks'] == []
# Finally, we simulate a successful job
ProjectDebugFile.objects.create(
file=file1,
object_name='baz.dSYM',
cpu_name='x86_64',
project=self.project,
debug_id='df449af8-0dcd-4320-9943-ec192134d593',
code_id='DF449AF80DCD43209943EC192134D593',
)
set_assemble_status(AssembleTask.DIF, self.project.id, checksum, None)
# Request now tells us that everything is alright
response = self.client.post(
self.url,
data={checksum: {
'name': 'dif',
'chunks': checksums,
}},
HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token))
assert response.status_code == 200, response.content
assert response.data[checksum]['state'] == ChunkFileState.OK
assert response.data[checksum]['missingChunks'] == []
not_found_checksum = sha1('1').hexdigest()
response = self.client.post(self.url,
data={
not_found_checksum: {
'name': 'dif',
'chunks': [not_found_checksum],
}
},
HTTP_AUTHORIZATION=u'Bearer {}'.format(
self.token.token))
assert response.status_code == 200, response.content
assert response.data[not_found_checksum][
'state'] == ChunkFileState.NOT_FOUND
assert set(response.data[not_found_checksum]['missingChunks']) == set(
[not_found_checksum])
def test_assemble_check(self):
content = b"foo bar"
fileobj = ContentFile(content)
file1 = File.objects.create(name="baz.dSYM", type="default", size=7)
file1.putfile(fileobj, 3)
checksum = sha1(content).hexdigest()
blobs = FileBlob.objects.all()
checksums = []
for blob in blobs:
checksums.append(blob.checksum)
# Request to see of file is there
# file exists but we have no overship for the chunks
response = self.client.post(
self.url,
data={checksum: {
"name": "dif",
"chunks": checksums
}},
HTTP_AUTHORIZATION=f"Bearer {self.token.token}",
)
assert response.status_code == 200, response.content
assert response.data[checksum]["state"] == ChunkFileState.NOT_FOUND
assert set(response.data[checksum]["missingChunks"]) == set(checksums)
# Now we add ownership to the blob
blobs = FileBlob.objects.all()
for blob in blobs:
FileBlobOwner.objects.create(blob=blob,
organization_id=self.organization.id)
# The request will start the job to assemble the file
response = self.client.post(
self.url,
data={checksum: {
"name": "dif",
"chunks": checksums
}},
HTTP_AUTHORIZATION=f"Bearer {self.token.token}",
)
assert response.status_code == 200, response.content
assert response.data[checksum]["state"] == ChunkFileState.CREATED
assert response.data[checksum]["missingChunks"] == []
# Finally, we simulate a successful job
ProjectDebugFile.objects.create(
file=file1,
checksum=file1.checksum,
object_name="baz.dSYM",
cpu_name="x86_64",
project_id=self.project.id,
debug_id="df449af8-0dcd-4320-9943-ec192134d593",
code_id="DF449AF80DCD43209943EC192134D593",
)
set_assemble_status(AssembleTask.DIF, self.project.id, checksum, None)
# Request now tells us that everything is alright
response = self.client.post(
self.url,
data={checksum: {
"name": "dif",
"chunks": checksums
}},
HTTP_AUTHORIZATION=f"Bearer {self.token.token}",
)
assert response.status_code == 200, response.content
assert response.data[checksum]["state"] == ChunkFileState.OK
assert response.data[checksum]["missingChunks"] == []
not_found_checksum = sha1(b"1").hexdigest()
response = self.client.post(
self.url,
data={
not_found_checksum: {
"name": "dif",
"chunks": [not_found_checksum]
}
},
HTTP_AUTHORIZATION=f"Bearer {self.token.token}",
)
assert response.status_code == 200, response.content
assert response.data[not_found_checksum][
"state"] == ChunkFileState.NOT_FOUND
assert set(response.data[not_found_checksum]["missingChunks"]) == {
not_found_checksum
}
def post(self, request, organization, version):
"""
Handle an artifact bundle and merge it into the release
```````````````````````````````````````````````````````
:auth: required
"""
try:
release = Release.objects.get(organization_id=organization.id, version=version)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if not self.has_release_permission(request, organization, release):
raise ResourceDoesNotExist
schema = {
"type": "object",
"properties": {
"checksum": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
"chunks": {
"type": "array",
"items": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
},
},
"required": ["checksum", "chunks"],
"additionalProperties": False,
}
try:
data = json.loads(request.body)
jsonschema.validate(data, schema)
except jsonschema.ValidationError as e:
return Response({"error": str(e).splitlines()[0]}, status=400)
except BaseException:
return Response({"error": "Invalid json body"}, status=400)
checksum = data.get("checksum", None)
chunks = data.get("chunks", [])
state, detail = get_assemble_status(AssembleTask.ARTIFACTS, organization.id, checksum)
if state == ChunkFileState.OK:
return Response({"state": state, "detail": None, "missingChunks": []}, status=200)
elif state is not None:
return Response({"state": state, "detail": detail, "missingChunks": []})
# There is neither a known file nor a cached state, so we will
# have to create a new file. Assure that there are checksums.
# If not, we assume this is a poll and report NOT_FOUND
if not chunks:
return Response({"state": ChunkFileState.NOT_FOUND, "missingChunks": []}, status=200)
set_assemble_status(
AssembleTask.ARTIFACTS, organization.id, checksum, ChunkFileState.CREATED
)
from sentry.tasks.assemble import assemble_artifacts
assemble_artifacts.apply_async(
kwargs={
"org_id": organization.id,
"version": version,
"checksum": checksum,
"chunks": chunks,
}
)
return Response({"state": ChunkFileState.CREATED, "missingChunks": []}, status=200)
