def test_wrong_dif(self):
content1 = 'foo'.encode('utf-8')
fileobj1 = ContentFile(content1)
content2 = 'bar'.encode('utf-8')
fileobj2 = ContentFile(content2)
content3 = 'baz'.encode('utf-8')
fileobj3 = ContentFile(content3)
total_checksum = sha1(content2 + content1 + content3).hexdigest()
# The order here is on purpose because we check for the order of checksums
blob1 = FileBlob.from_file(fileobj1)
blob3 = FileBlob.from_file(fileobj3)
blob2 = FileBlob.from_file(fileobj2)
chunks = [blob2.checksum, blob1.checksum, blob3.checksum]
assemble_dif(
project_id=self.project.id,
name='foo.sym',
checksum=total_checksum,
chunks=chunks,
)
assert get_assemble_status(self.project,
total_checksum)[0] == ChunkFileState.ERROR
def test_wrong_dif(self):
content1 = 'foo'.encode('utf-8')
fileobj1 = ContentFile(content1)
content2 = 'bar'.encode('utf-8')
fileobj2 = ContentFile(content2)
content3 = 'baz'.encode('utf-8')
fileobj3 = ContentFile(content3)
total_checksum = sha1(content2 + content1 + content3).hexdigest()
# The order here is on purpose because we check for the order of checksums
blob1 = FileBlob.from_file(fileobj1)
blob3 = FileBlob.from_file(fileobj3)
blob2 = FileBlob.from_file(fileobj2)
chunks = [blob2.checksum, blob1.checksum, blob3.checksum]
assemble_dif(
project_id=self.project.id,
name='foo.sym',
checksum=total_checksum,
chunks=chunks,
)
assert get_assemble_status(self.project, total_checksum)[0] == ChunkFileState.ERROR
def test_assemble_duplicate_blobs(self):
files = []
file_checksum = sha1()
blob = os.urandom(1024 * 1024 * 8)
hash = sha1(blob).hexdigest()
for _ in range(8):
file_checksum.update(blob)
files.append((io.BytesIO(blob), hash))
# upload all blobs
FileBlob.from_files(files, organization=self.organization)
# find all blobs
for reference, checksum in files:
blob = FileBlob.objects.get(checksum=checksum)
ref_bytes = reference.getvalue()
assert blob.getfile().read(len(ref_bytes)) == ref_bytes
FileBlobOwner.objects.filter(
blob=blob, organization_id=self.organization.id).get()
rv = assemble_file(
AssembleTask.DIF,
self.project,
"testfile",
file_checksum.hexdigest(),
[x[1] for x in files],
"dummy.type",
)
assert rv is not None
f, tmp = rv
assert f.checksum == file_checksum.hexdigest()
assert f.type == "dummy.type"
def test_wrong_dif(self):
content1 = b"foo"
fileobj1 = ContentFile(content1)
content2 = b"bar"
fileobj2 = ContentFile(content2)
content3 = b"baz"
fileobj3 = ContentFile(content3)
total_checksum = sha1(content2 + content1 + content3).hexdigest()
# The order here is on purpose because we check for the order of checksums
blob1 = FileBlob.from_file(fileobj1)
blob3 = FileBlob.from_file(fileobj3)
blob2 = FileBlob.from_file(fileobj2)
chunks = [blob2.checksum, blob1.checksum, blob3.checksum]
assemble_dif(project_id=self.project.id,
name="foo.sym",
checksum=total_checksum,
chunks=chunks)
status, _ = get_assemble_status(AssembleTask.DIF, self.project.id,
total_checksum)
assert status == ChunkFileState.ERROR
def post(self, request, organization):
"""
Upload chunks and store them as FileBlobs
`````````````````````````````````````````
:pparam file file: The filename should be sha1 hash of the content.
Also not you can add up to MAX_CHUNKS_PER_REQUEST files
in this request.
:auth: required
"""
# Create a unique instance so our logger can be decoupled from the request
# and used in threads.
logger = logging.getLogger('sentry.files')
logger.info('chunkupload.start')
files = request.FILES.getlist('file')
files += [GzipChunk(chunk) for chunk in request.FILES.getlist('file_gzip')]
if len(files) == 0:
# No files uploaded is ok
logger.info('chunkupload.end', extra={'status': status.HTTP_200_OK})
return Response(status=status.HTTP_200_OK)
logger.info('chunkupload.post.files', extra={'len': len(files)})
# Validate file size
checksums = []
size = 0
for chunk in files:
size += chunk.size
if chunk.size > CHUNK_UPLOAD_BLOB_SIZE:
logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST})
return Response({'error': 'Chunk size too large'},
status=status.HTTP_400_BAD_REQUEST)
checksums.append(chunk.name)
if size > MAX_REQUEST_SIZE:
logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST})
return Response({'error': 'Request too large'},
status=status.HTTP_400_BAD_REQUEST)
if len(files) > MAX_CHUNKS_PER_REQUEST:
logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST})
return Response({'error': 'Too many chunks'},
status=status.HTTP_400_BAD_REQUEST)
try:
FileBlob.from_files(izip(files, checksums),
organization=organization,
logger=logger)
except IOError as err:
logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST})
return Response({'error': six.text_type(err)},
status=status.HTTP_400_BAD_REQUEST)
logger.info('chunkupload.end', extra={'status': status.HTTP_200_OK})
return Response(status=status.HTTP_200_OK)
def post(self, request, organization):
"""
Upload chunks and store them as FileBlobs
`````````````````````````````````````````
:pparam file file: The filename should be sha1 hash of the content.
Also not you can add up to MAX_CHUNKS_PER_REQUEST files
in this request.
:auth: required
"""
# Create a unique instance so our logger can be decoupled from the request
# and used in threads.
logger = logging.getLogger('sentry.files')
logger.info('chunkupload.start')
files = request.data.getlist('file')
files += [GzipChunk(chunk) for chunk in request.data.getlist('file_gzip')]
if len(files) == 0:
# No files uploaded is ok
logger.info('chunkupload.end', extra={'status': status.HTTP_200_OK})
return Response(status=status.HTTP_200_OK)
logger.info('chunkupload.post.files', extra={'len': len(files)})
# Validate file size
checksums = []
size = 0
for chunk in files:
size += chunk.size
if chunk.size > CHUNK_UPLOAD_BLOB_SIZE:
logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST})
return Response({'error': 'Chunk size too large'},
status=status.HTTP_400_BAD_REQUEST)
checksums.append(chunk.name)
if size > MAX_REQUEST_SIZE:
logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST})
return Response({'error': 'Request too large'},
status=status.HTTP_400_BAD_REQUEST)
if len(files) > MAX_CHUNKS_PER_REQUEST:
logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST})
return Response({'error': 'Too many chunks'},
status=status.HTTP_400_BAD_REQUEST)
try:
FileBlob.from_files(izip(files, checksums),
organization=organization,
logger=logger)
except IOError as err:
logger.info('chunkupload.end', extra={'status': status.HTTP_400_BAD_REQUEST})
return Response({'error': six.text_type(err)},
status=status.HTTP_400_BAD_REQUEST)
logger.info('chunkupload.end', extra={'status': status.HTTP_200_OK})
return Response(status=status.HTTP_200_OK)
def post(self, request, organization):
"""
Upload chunks and store them as FileBlobs
`````````````````````````````````````````
:pparam file file: The filename should be sha1 hash of the content.
Also not you can add up to MAX_CHUNKS_PER_REQUEST files
in this request.
:auth: required
"""
# Create a unique instance so our logger can be decoupled from the request
# and used in threads.
logger = logging.getLogger("sentry.files")
logger.info("chunkupload.start")
files = []
if request.data:
files = request.data.getlist("file")
files += [GzipChunk(chunk) for chunk in request.data.getlist("file_gzip")]
if len(files) == 0:
# No files uploaded is ok
logger.info("chunkupload.end", extra={"status": status.HTTP_200_OK})
return Response(status=status.HTTP_200_OK)
logger.info("chunkupload.post.files", extra={"len": len(files)})
# Validate file size
checksums = []
size = 0
for chunk in files:
size += chunk.size
if chunk.size > settings.SENTRY_CHUNK_UPLOAD_BLOB_SIZE:
logger.info("chunkupload.end", extra={"status": status.HTTP_400_BAD_REQUEST})
return Response(
{"error": "Chunk size too large"}, status=status.HTTP_400_BAD_REQUEST
)
checksums.append(chunk.name)
if size > MAX_REQUEST_SIZE:
logger.info("chunkupload.end", extra={"status": status.HTTP_400_BAD_REQUEST})
return Response({"error": "Request too large"}, status=status.HTTP_400_BAD_REQUEST)
if len(files) > MAX_CHUNKS_PER_REQUEST:
logger.info("chunkupload.end", extra={"status": status.HTTP_400_BAD_REQUEST})
return Response({"error": "Too many chunks"}, status=status.HTTP_400_BAD_REQUEST)
try:
FileBlob.from_files(zip(files, checksums), organization=organization, logger=logger)
except OSError as err:
logger.info("chunkupload.end", extra={"status": status.HTTP_400_BAD_REQUEST})
return Response({"error": str(err)}, status=status.HTTP_400_BAD_REQUEST)
logger.info("chunkupload.end", extra={"status": status.HTTP_200_OK})
return Response(status=status.HTTP_200_OK)
def test_generate_unique_path(self):
path = FileBlob.generate_unique_path()
assert path
parts = path.split('/')
assert len(parts) == 3
assert map(len, parts) == [2, 4, 26]
# Check uniqueness
path2 = FileBlob.generate_unique_path()
assert path != path2
def test_from_file(self):
fileobj = ContentFile('foo bar'.encode('utf-8'))
my_file1 = FileBlob.from_file(fileobj)
assert my_file1.path
my_file2 = FileBlob.from_file(fileobj)
# deep check
assert my_file1.id == my_file2.id
assert my_file1.checksum == my_file2.checksum
assert my_file1.path == my_file2.path
def test_from_file(self):
fileobj = ContentFile("foo bar".encode("utf-8"))
my_file1 = FileBlob.from_file(fileobj)
assert my_file1.path
fileobj.seek(0)
my_file2 = FileBlob.from_file(fileobj)
# deep check
assert my_file1.id == my_file2.id
assert my_file1.checksum == my_file2.checksum
assert my_file1.path == my_file2.path
def test_from_file(self):
fileobj = ContentFile("foo bar")
my_file1 = FileBlob.from_file(fileobj)
assert my_file1.path
assert my_file1.storage == settings.SENTRY_FILESTORE
my_file2 = FileBlob.from_file(fileobj)
# deep check
assert my_file1.id == my_file2.id
assert my_file1.checksum == my_file2.checksum
assert my_file1.path == my_file2.path
assert my_file1.storage == my_file2.storage
assert my_file1.storage_options == my_file2.storage_options
def test_dif_response(self):
sym_file = self.load_fixture('crash.sym')
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
chunks = [blob1.checksum]
assemble_dif(
project_id=self.project.id,
name='crash.sym',
checksum=total_checksum,
chunks=chunks,
)
response = self.client.post(
self.url,
data={
total_checksum: {
'name': 'test.sym',
'chunks': chunks,
}
},
HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token)
)
assert response.status_code == 200, response.content
assert response.data[total_checksum]['state'] == ChunkFileState.OK
assert response.data[total_checksum]['dif']['cpuName'] == 'x86_64'
assert response.data[total_checksum]['dif']['uuid'] == '67e9247c-814e-392b-a027-dbde6748fcbf'
def post(self, request):
files = request.FILES.getlist('file')
if len(files) > MAX_CHUNKS_PER_REQUEST:
return Response({'error': 'Too many chunks'},
status=status.HTTP_400_BAD_REQUEST)
elif len(files) == 0:
# No files uploaded is ok
return Response(status=status.HTTP_200_OK)
# Validate file size
checksum_list = []
for chunk in files:
if chunk._size > DEFAULT_BLOB_SIZE:
return Response({'error': 'Chunk size too large'},
status=status.HTTP_400_BAD_REQUEST)
checksum_list.append(chunk._name)
for chunk in files:
# Here we create the actual file
blob = FileBlob.from_file(chunk)
if blob.checksum not in checksum_list:
# We do not clean up here since we have a cleanup job
return Response({'error': 'Checksum missmatch'},
status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_200_OK)
def test_dif_error_response(self):
sym_file = 'fail'
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
chunks = [blob1.checksum]
assemble_dif(
project_id=self.project.id,
name='test.sym',
checksum=total_checksum,
chunks=chunks,
)
response = self.client.post(
self.url,
data={
total_checksum: {
'name': 'test.sym',
'chunks': [],
}
},
HTTP_AUTHORIZATION='Bearer {}'.format(self.token.token)
)
assert response.status_code == 200, response.content
assert response.data[total_checksum]['state'] == ChunkFileState.ERROR
assert response.data[total_checksum]['detail'].startswith('Invalid debug information file')
def test_assemble(self, mock_assemble_artifacts):
bundle_file = self.create_artifact_bundle()
total_checksum = sha1(bundle_file).hexdigest()
blob1 = FileBlob.from_file(ContentFile(bundle_file))
FileBlobOwner.objects.get_or_create(
organization=self.organization,
blob=blob1
)
response = self.client.post(
self.url,
data={
'checksum': total_checksum,
'chunks': [blob1.checksum],
},
HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token)
)
assert response.status_code == 200, response.content
assert response.data['state'] == ChunkFileState.CREATED
assert set(response.data['missingChunks']) == set([])
mock_assemble_artifacts.apply_async.assert_called_once_with(
kwargs={
'org_id': self.organization.id,
'version': self.release.version,
'chunks': [blob1.checksum],
'checksum': total_checksum,
}
)
def store_export_chunk_as_blob(data_export,
bytes_written,
fileobj,
blob_size=DEFAULT_BLOB_SIZE):
# adapted from `putfile` in `src/sentry/models/file.py`
bytes_offset = 0
while True:
contents = fileobj.read(blob_size)
if not contents:
return bytes_offset
blob_fileobj = ContentFile(contents)
blob = FileBlob.from_file(blob_fileobj, logger=logger)
ExportedDataBlob.objects.create(data_export=data_export,
blob=blob,
offset=bytes_written + bytes_offset)
bytes_offset += blob.size
# there is a maximum file size allowed, so we need to make sure we don't exceed it
# NOTE: there seems to be issues with downloading files larger than 1 GB on slower
# networks, limit the export to 1 GB for now to improve reliability
if bytes_written + bytes_offset >= min(MAX_FILE_SIZE, 2**30):
transaction.set_rollback(True)
return 0
def test_dif_response(self):
sym_file = self.load_fixture("crash.sym")
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
chunks = [blob1.checksum]
assemble_dif(project_id=self.project.id,
name="crash.sym",
checksum=total_checksum,
chunks=chunks)
response = self.client.post(
self.url,
data={total_checksum: {
"name": "test.sym",
"chunks": chunks
}},
HTTP_AUTHORIZATION=f"Bearer {self.token.token}",
)
assert response.status_code == 200, response.content
assert response.data[total_checksum]["state"] == ChunkFileState.OK
assert response.data[total_checksum]["dif"]["cpuName"] == "x86_64"
assert (response.data[total_checksum]["dif"]["uuid"] ==
"67e9247c-814e-392b-a027-dbde6748fcbf")
def test_assemble(self, mock_assemble_artifacts):
bundle_file = self.create_artifact_bundle()
total_checksum = sha1(bundle_file).hexdigest()
blob1 = FileBlob.from_file(ContentFile(bundle_file))
FileBlobOwner.objects.get_or_create(organization=self.organization,
blob=blob1)
response = self.client.post(
self.url,
data={
"checksum": total_checksum,
"chunks": [blob1.checksum]
},
HTTP_AUTHORIZATION=f"Bearer {self.token.token}",
)
assert response.status_code == 200, response.content
assert response.data["state"] == ChunkFileState.CREATED
assert set(response.data["missingChunks"]) == set()
mock_assemble_artifacts.apply_async.assert_called_once_with(
kwargs={
"org_id": self.organization.id,
"version": self.release.version,
"chunks": [blob1.checksum],
"checksum": total_checksum,
})
def test_artifacts(self):
bundle_file = self.create_artifact_bundle()
blob1 = FileBlob.from_file(ContentFile(bundle_file))
total_checksum = sha1(bundle_file).hexdigest()
assemble_artifacts(
org_id=self.organization.id,
version=self.release.version,
checksum=total_checksum,
chunks=[blob1.checksum],
)
status, details = get_assemble_status(AssembleTask.ARTIFACTS,
self.organization.id,
total_checksum)
assert status == ChunkFileState.OK
assert details is None
release_file = ReleaseFile.objects.get(organization=self.organization,
release=self.release,
name="~/index.js",
dist=None)
assert release_file
assert release_file.file.headers == {"Sourcemap": "index.js.map"}
def test_dif_response(self):
sym_file = self.load_fixture('crash.sym')
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
chunks = [blob1.checksum]
assemble_dif(
project_id=self.project.id,
name='crash.sym',
checksum=total_checksum,
chunks=chunks,
)
response = self.client.post(
self.url,
data={
total_checksum: {
'name': 'test.sym',
'chunks': chunks,
}
},
HTTP_AUTHORIZATION='Bearer {}'.format(self.token.token)
)
assert response.status_code == 200, response.content
assert response.data[total_checksum]['state'] == ChunkFileState.OK
assert response.data[total_checksum]['dif']['cpuName'] == 'x86_64'
assert response.data[total_checksum]['dif']['uuid'] == '67e9247c-814e-392b-a027-dbde6748fcbf'
def test_dif_error_response(self):
sym_file = 'fail'
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
chunks = [blob1.checksum]
assemble_dif(
project_id=self.project.id,
name='test.sym',
checksum=total_checksum,
chunks=chunks,
)
response = self.client.post(
self.url,
data={
total_checksum: {
'name': 'test.sym',
'chunks': [],
}
},
HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token)
)
assert response.status_code == 200, response.content
assert response.data[total_checksum]['state'] == ChunkFileState.ERROR
assert response.data[total_checksum]['detail'].startswith('Invalid debug information file')
def test_dif_error_reponse(self):
sym_file = 'fail'
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
file = File.objects.create(
name='test.sym',
checksum=total_checksum,
type='chunked',
headers={CHUNK_STATE_HEADER: ChunkFileState.CREATED})
file_blob_id_order = [blob1.id]
assemble_dif(
project_id=self.project.id,
file_id=file.id,
file_blob_ids=file_blob_id_order,
checksum=total_checksum,
)
response = self.client.post(self.url,
data={
total_checksum: {
'name': 'test.sym',
'chunks': [blob1.checksum]
}
},
HTTP_AUTHORIZATION='Bearer {}'.format(
self.token.token))
assert response.status_code == 200, response.content
assert response.data[total_checksum]['error'] == 'Invalid object file'
def post(self, request, organization):
"""
Upload chunks and store them as FileBlobs
`````````````````````````````````````````
:pparam file file: The filename should be sha1 hash of the content.
Also not you can add up to MAX_CHUNKS_PER_REQUEST files
in this request.
:auth: required
"""
files = request.FILES.getlist('file')
files += [
GzipChunk(chunk) for chunk in request.FILES.getlist('file_gzip')
]
if len(files) == 0:
# No files uploaded is ok
return Response(status=status.HTTP_200_OK)
# Validate file size
checksums = []
size = 0
for chunk in files:
size += chunk.size
if chunk.size > CHUNK_UPLOAD_BLOB_SIZE:
return Response({'error': 'Chunk size too large'},
status=status.HTTP_400_BAD_REQUEST)
checksums.append(chunk.name)
if size > MAX_REQUEST_SIZE:
return Response({'error': 'Request too large'},
status=status.HTTP_400_BAD_REQUEST)
if len(files) > MAX_CHUNKS_PER_REQUEST:
return Response({'error': 'Too many chunks'},
status=status.HTTP_400_BAD_REQUEST)
try:
FileBlob.from_files(izip(files, checksums),
organization=organization)
except IOError as err:
return Response({'error': six.text_type(err)},
status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_200_OK)
def test_legacy_blob(self):
fileobj = ContentFile("foo bar")
blob = FileBlob.from_file(fileobj)
file1 = File.objects.create(
name='baz.js',
type='default',
size=7,
blob=blob,
)
with file1.getfile() as fp:
assert fp.read() == 'foo bar'
def test_assemble_from_files(self):
files = []
file_checksum = sha1()
for _ in xrange(8):
blob = os.urandom(1024 * 1024 * 8)
hash = sha1(blob).hexdigest()
file_checksum.update(blob)
files.append((io.BytesIO(blob), hash))
# upload all blobs
FileBlob.from_files(files, organization=self.organization)
# find all blobs
for reference, checksum in files:
blob = FileBlob.objects.get(checksum=checksum)
ref_bytes = reference.getvalue()
assert blob.getfile().read(len(ref_bytes)) == ref_bytes
FileBlobOwner.objects.filter(
blob=blob,
organization=self.organization
).get()
rv = assemble_file(AssembleTask.DIF,
self.project, 'testfile', file_checksum.hexdigest(),
[x[1] for x in files], 'dummy.type')
assert rv is not None
f, tmp = rv
assert f.checksum == file_checksum.hexdigest()
assert f.type == 'dummy.type'
# upload all blobs a second time
for f, _ in files:
f.seek(0)
FileBlob.from_files(files, organization=self.organization)
# assemble a second time
f = assemble_file(AssembleTask.DIF,
self.project, 'testfile', file_checksum.hexdigest(),
[x[1] for x in files], 'dummy.type')[0]
assert f.checksum == file_checksum.hexdigest()
def test_assemble_from_files(self):
files = []
file_checksum = sha1()
for _ in xrange(8):
blob = os.urandom(1024 * 1024 * 8)
hash = sha1(blob).hexdigest()
file_checksum.update(blob)
files.append((io.BytesIO(blob), hash))
# upload all blobs
FileBlob.from_files(files, organization=self.organization)
# find all blobs
for reference, checksum in files:
blob = FileBlob.objects.get(checksum=checksum)
ref_bytes = reference.getvalue()
assert blob.getfile().read(len(ref_bytes)) == ref_bytes
FileBlobOwner.objects.filter(
blob=blob,
organization=self.organization
).get()
rv = assemble_file(
self.project, 'testfile', file_checksum.hexdigest(),
[x[1] for x in files], 'dummy.type')
assert rv is not None
f, tmp = rv
assert f.checksum == file_checksum.hexdigest()
assert f.type == 'dummy.type'
# upload all blobs a second time
for f, _ in files:
f.seek(0)
FileBlob.from_files(files, organization=self.organization)
# assemble a second time
f = assemble_file(
self.project, 'testfile', file_checksum.hexdigest(),
[x[1] for x in files], 'dummy.type')[0]
assert f.checksum == file_checksum.hexdigest()
def test_wrong_dif(self):
content1 = 'foo'.encode('utf-8')
fileobj1 = ContentFile(content1)
content2 = 'bar'.encode('utf-8')
fileobj2 = ContentFile(content2)
content3 = 'baz'.encode('utf-8')
fileobj3 = ContentFile(content3)
total_checksum = sha1(content2 + content1 + content3).hexdigest()
# The order here is on purpose because we check for the order of checksums
blob1 = FileBlob.from_file(fileobj1)
bolb3 = FileBlob.from_file(fileobj3)
bolb2 = FileBlob.from_file(fileobj2)
file = File.objects.create(
name='test',
checksum=total_checksum,
type='chunked',
headers={CHUNK_STATE_HEADER: ChunkFileState.CREATED})
file_blob_id_order = [bolb2.id, blob1.id, bolb3.id]
file = File.objects.create(
name='test',
checksum=total_checksum,
type='chunked',
headers={CHUNK_STATE_HEADER: ChunkFileState.CREATED})
assemble_dif(
project_id=self.project.id,
file_id=file.id,
file_blob_ids=file_blob_id_order,
checksum=total_checksum,
)
file = File.objects.filter(id=file.id, ).get()
assert file.headers.get(CHUNK_STATE_HEADER) == ChunkFileState.ERROR
def post(self, request, organization):
"""
Upload chunks and store them as FileBlobs
`````````````````````````````````````````
:pparam file file: The filename should be sha1 hash of the content.
Also not you can add up to MAX_CHUNKS_PER_REQUEST files
in this request.
:auth: required
"""
files = request.FILES.getlist('file')
files += [GzipChunk(chunk) for chunk in request.FILES.getlist('file_gzip')]
if len(files) == 0:
# No files uploaded is ok
return Response(status=status.HTTP_200_OK)
# Validate file size
checksums = []
size = 0
for chunk in files:
size += chunk.size
if chunk.size > DEFAULT_BLOB_SIZE:
return Response({'error': 'Chunk size too large'},
status=status.HTTP_400_BAD_REQUEST)
checksums.append(chunk.name)
if size > MAX_REQUEST_SIZE:
return Response({'error': 'Request too large'},
status=status.HTTP_400_BAD_REQUEST)
if len(files) > MAX_CHUNKS_PER_REQUEST:
return Response({'error': 'Too many chunks'},
status=status.HTTP_400_BAD_REQUEST)
for checksum, chunk in izip(checksums, files):
# Here we create the actual blob
blob = FileBlob.from_file(chunk)
# Add ownership to the blob here
try:
with transaction.atomic():
FileBlobOwner.objects.create(
organization=organization,
blob=blob
)
except IntegrityError:
pass
if blob.checksum != checksum:
# We do not clean up here since we have a cleanup job
return Response({'error': 'Checksum missmatch'},
status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_200_OK)
def test_artifacts(self):
bundle_file = self.create_artifact_bundle()
blob1 = FileBlob.from_file(ContentFile(bundle_file))
total_checksum = sha1(bundle_file).hexdigest()
for min_files in (10, 1):
with self.options({
"processing.release-archive-min-files":
min_files,
}):
ReleaseFile.objects.filter(release_id=self.release.id).delete()
assert self.release.count_artifacts() == 0
assemble_artifacts(
org_id=self.organization.id,
version=self.release.version,
checksum=total_checksum,
chunks=[blob1.checksum],
)
assert self.release.count_artifacts() == 2
status, details = get_assemble_status(AssembleTask.ARTIFACTS,
self.organization.id,
total_checksum)
assert status == ChunkFileState.OK
assert details is None
if min_files == 1:
# An archive was saved
index = read_artifact_index(self.release, dist=None)
archive_ident = index["files"]["~/index.js"][
"archive_ident"]
releasefile = ReleaseFile.objects.get(
release_id=self.release.id, ident=archive_ident)
# Artifact is the same as original bundle
assert releasefile.file.size == len(bundle_file)
else:
# Individual files were saved
release_file = ReleaseFile.objects.get(
organization_id=self.organization.id,
release_id=self.release.id,
name="~/index.js",
dist_id=None,
)
assert release_file.file.headers == {
"Sourcemap": "index.js.map"
}
def test_artifacts_invalid_zip(self):
bundle_file = b''
blob1 = FileBlob.from_file(ContentFile(bundle_file))
total_checksum = sha1(bundle_file).hexdigest()
assemble_artifacts(
org_id=self.organization.id,
version=self.release.version,
checksum=total_checksum,
chunks=[blob1.checksum],
)
status, details = get_assemble_status(AssembleTask.ARTIFACTS, self.organization.id,
total_checksum)
assert status == ChunkFileState.ERROR
def test_dif(self):
sym_file = self.load_fixture("crash.sym")
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
assemble_dif(
project_id=self.project.id,
name="crash.sym",
checksum=total_checksum,
chunks=[blob1.checksum],
)
status, _ = get_assemble_status(AssembleTask.DIF, self.project.id, total_checksum)
assert status == ChunkFileState.OK
dif = ProjectDebugFile.objects.filter(project=self.project, checksum=total_checksum).get()
assert dif.file.headers == {"Content-Type": "text/x-breakpad"}
def test_dif(self):
sym_file = self.load_fixture('crash.sym')
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
assemble_dif(
project_id=self.project.id,
name='crash.sym',
checksum=total_checksum,
chunks=[blob1.checksum],
)
dif = ProjectDSymFile.objects.filter(
project=self.project,
file__checksum=total_checksum,
).get()
assert dif.file.headers == {'Content-Type': 'text/x-breakpad'}
def test_dif(self):
sym_file = self.load_fixture('crash.sym')
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
assemble_dif(
project_id=self.project.id,
name='crash.sym',
checksum=total_checksum,
chunks=[blob1.checksum],
)
dif = ProjectDebugFile.objects.filter(
project=self.project,
file__checksum=total_checksum,
).get()
assert dif.file.headers == {'Content-Type': 'text/x-breakpad'}
def test_dif_error_response(self):
sym_file = b"fail"
blob1 = FileBlob.from_file(ContentFile(sym_file))
total_checksum = sha1(sym_file).hexdigest()
chunks = [blob1.checksum]
assemble_dif(
project_id=self.project.id, name="test.sym", checksum=total_checksum, chunks=chunks
)
response = self.client.post(
self.url,
data={total_checksum: {"name": "test.sym", "chunks": []}},
HTTP_AUTHORIZATION=f"Bearer {self.token.token}",
)
assert response.status_code == 200, response.content
assert response.data[total_checksum]["state"] == ChunkFileState.ERROR
assert "unsupported object file format" in response.data[total_checksum]["detail"]
def test_artifacts(self):
bundle_file = self.create_artifact_bundle()
blob1 = FileBlob.from_file(ContentFile(bundle_file))
total_checksum = sha1(bundle_file).hexdigest()
for has_release_archives in (True, False):
with self.options({
"processing.save-release-archives": has_release_archives,
"processing.release-archive-min-files": 1,
}):
assemble_artifacts(
org_id=self.organization.id,
version=self.release.version,
checksum=total_checksum,
chunks=[blob1.checksum],
)
status, details = get_assemble_status(AssembleTask.ARTIFACTS,
self.organization.id,
total_checksum)
assert status == ChunkFileState.OK
assert details is None
release_file = ReleaseFile.objects.get(
organization=self.organization,
release=self.release,
name="release-artifacts.zip"
if has_release_archives else "~/index.js",
dist=None,
)
assert release_file
if has_release_archives:
assert release_file.file.headers == {}
# Artifact is the same as original bundle
assert release_file.file.size == len(bundle_file)
else:
assert release_file.file.headers == {
"Sourcemap": "index.js.map"
}
def test_assemble_response(self):
bundle_file = self.create_artifact_bundle()
total_checksum = sha1(bundle_file).hexdigest()
blob1 = FileBlob.from_file(ContentFile(bundle_file))
assemble_artifacts(
org_id=self.organization.id,
version=self.release.version,
checksum=total_checksum,
chunks=[blob1.checksum],
)
response = self.client.post(
self.url,
data={"checksum": total_checksum, "chunks": [blob1.checksum]},
HTTP_AUTHORIZATION=u"Bearer {}".format(self.token.token),
)
assert response.status_code == 200, response.content
assert response.data["state"] == ChunkFileState.OK
def create_release_file(self, project, release, path,
content_type=None, contents=None):
from sentry.models import File, FileBlob, ReleaseFile
if content_type is None:
content_type = mimetypes.guess_type(path)[0] or 'text/plain'
if content_type.startswith('text/'):
content_type += '; encoding=utf-8'
f = File.objects.create(
name=path.rsplit('/', 1)[-1],
type='release.file',
headers={
'Content-Type': content_type
},
blob=FileBlob.from_file(StringIO(contents or '')),
)
return ReleaseFile.objects.create(
project=project,
release=release,
file=f,
name=path
)
def test_expansion_via_release_artifacts(self):
project = self.project
release = Release.objects.create(
project=project,
version='abc',
)
f1 = File.objects.create(
name='file.min.js',
type='release.file',
headers={'Content-Type': 'application/json'},
blob=FileBlob.from_file(open(get_fixture_path('file.min.js'), 'rb'))
)
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f1.name),
release=release,
project=project,
file=f1,
)
f2 = File.objects.create(
name='file1.js',
type='release.file',
headers={'Content-Type': 'application/json'},
blob=FileBlob.from_file(open(get_fixture_path('file1.js'), 'rb'))
)
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f2.name),
release=release,
project=project,
file=f2,
)
f3 = File.objects.create(
name='file2.js',
type='release.file',
headers={'Content-Type': 'application/json'},
blob=FileBlob.from_file(open(get_fixture_path('file2.js'), 'rb'))
)
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f3.name),
release=release,
project=project,
file=f3,
)
f4 = File.objects.create(
name='file.sourcemap.js',
type='release.file',
headers={'Content-Type': 'application/json'},
blob=FileBlob.from_file(open(get_fixture_path('file.sourcemap.js'), 'rb'))
)
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f4.name),
release=release,
project=project,
file=f4,
)
data = {
'message': 'hello',
'platform': 'javascript',
'release': 'abc',
'sentry.interfaces.Exception': {
'values': [{
'type': 'Error',
'stacktrace': {
'frames': [
{
'abs_path': 'http://example.com/file.min.js',
'filename': 'file.min.js',
'lineno': 1,
'colno': 39,
},
],
},
}],
}
}
resp = self._postWithHeader(data)
assert resp.status_code, 200
event = Event.objects.get()
assert not event.data['errors']
exception = event.interfaces['sentry.interfaces.Exception']
frame_list = exception.values[0].stacktrace.frames
frame = frame_list[0]
assert frame.pre_context == [
'function add(a, b) {',
'\t"use strict";',
]
assert frame.context_line == '\treturn a + b;'
assert frame.post_context == ['}']
def post(self, request, project, version):
"""
Upload a New File
`````````````````
Upload a new file for the given release.
Unlike other API requests, files must be uploaded using the
traditional multipart/form-data content-type.
The optional 'name' attribute should reflect the absolute path
that this file will be referenced as. For example, in the case of
JavaScript you might specify the full web URI.
:pparam string organization_slug: the slug of the organization the
release belongs to.
:pparam string project_slug: the slug of the project to change the
release of.
:pparam string version: the version identifier of the release.
:param string name: the name (full path) of the file.
:param file file: the multipart encoded file.
:param string header: this parameter can be supplied multiple times
to attach headers to the file. Each header
is a string in the format ``key:value``. For
instance it can be used to define a content
type.
:auth: required
"""
version = unquote(version)
try:
release = Release.objects.get(
project=project,
version=version,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if 'file' not in request.FILES:
return Response({'detail': 'Missing uploaded file'}, status=400)
fileobj = request.FILES['file']
full_name = request.DATA.get('name', fileobj.name)
name = full_name.rsplit('/', 1)[-1]
headers = {
'Content-Type': fileobj.content_type,
}
for headerval in request.DATA.getlist('header') or ():
try:
k, v = headerval.split(':', 1)
except ValueError:
return Response({'detail': 'header value was not formatted correctly'}, status=400)
else:
headers[k] = v.strip()
blob = FileBlob.from_file(fileobj)
file = File.objects.create(
name=name,
type='release.file',
headers=headers,
blob=blob,
)
try:
with transaction.atomic():
releasefile = ReleaseFile.objects.create(
project=release.project,
release=release,
file=file,
name=full_name,
)
except IntegrityError:
file.delete()
return Response(status=409)
return Response(serialize(releasefile, request.user), status=201)
def test_assemble(self, mock_assemble_dif):
content1 = 'foo'.encode('utf-8')
fileobj1 = ContentFile(content1)
checksum1 = sha1(content1).hexdigest()
content2 = 'bar'.encode('utf-8')
fileobj2 = ContentFile(content2)
checksum2 = sha1(content2).hexdigest()
content3 = 'baz'.encode('utf-8')
fileobj3 = ContentFile(content3)
checksum3 = sha1(content3).hexdigest()
total_checksum = sha1(content2 + content1 + content3).hexdigest()
# The order here is on purpose because we check for the order of checksums
blob1 = FileBlob.from_file(fileobj1)
FileBlobOwner.objects.get_or_create(
organization=self.organization,
blob=blob1
)
blob3 = FileBlob.from_file(fileobj3)
FileBlobOwner.objects.get_or_create(
organization=self.organization,
blob=blob3
)
blob2 = FileBlob.from_file(fileobj2)
# we make a request now but we are missing ownership for chunk 2
response = self.client.post(
self.url,
data={
total_checksum: {
'name': 'test',
'chunks': [
checksum2, checksum1, checksum3
]
}
},
HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token)
)
assert response.status_code == 200, response.content
assert response.data[total_checksum]['state'] == ChunkFileState.NOT_FOUND
assert response.data[total_checksum]['missingChunks'] == [checksum2]
# we add ownership to chunk 2
FileBlobOwner.objects.get_or_create(
organization=self.organization,
blob=blob2
)
# new request, ownership for all chunks is there but file does not exist yet
response = self.client.post(
self.url,
data={
total_checksum: {
'name': 'test',
'chunks': [
checksum2, checksum1, checksum3
],
}
},
HTTP_AUTHORIZATION=u'Bearer {}'.format(self.token.token)
)
assert response.status_code == 200, response.content
assert response.data[total_checksum]['state'] == ChunkFileState.CREATED
assert response.data[total_checksum]['missingChunks'] == []
chunks = [checksum2, checksum1, checksum3]
mock_assemble_dif.apply_async.assert_called_once_with(
kwargs={
'project_id': self.project.id,
'name': 'test',
'chunks': chunks,
'checksum': total_checksum,
}
)
file = assemble_file(self.project, 'test', total_checksum, chunks, 'project.dsym')[0]
assert get_assemble_status(self.project, total_checksum)[0] != ChunkFileState.ERROR
assert file.checksum == total_checksum
file_blob_index = FileBlobIndex.objects.all()
assert len(file_blob_index) == 3
