def _upsert_release_file(file: File, archive: ReleaseArchive, update_fn,
key_fields, additional_fields) -> bool:
success = False
release_file = None
# Release files must have unique names within their release
# and dist. If a matching file already exists, replace its
# file with the new one; otherwise create it.
try:
release_file = ReleaseFile.objects.get(**key_fields)
except ReleaseFile.DoesNotExist:
try:
with atomic_transaction(using=router.db_for_write(ReleaseFile)):
release_file = ReleaseFile.objects.create(
file=file, **dict(key_fields, **additional_fields))
except IntegrityError:
# NB: This indicates a race, where another assemble task or
# file upload job has just created a conflicting file. Since
# we're upserting here anyway, yield to the faster actor and
# do not try again.
file.delete()
else:
success = True
else:
success = update_fn(release_file, file, archive, additional_fields)
return success
def _merge_archives(release_file: ReleaseFile, new_file: File,
new_archive: ReleaseArchive):
max_attempts = RELEASE_ARCHIVE_MAX_MERGE_ATTEMPTS
success = False
for attempt in range(max_attempts):
old_file = release_file.file
with ReleaseArchive(old_file.getfile().file) as old_archive:
buffer = BytesIO()
merge_release_archives(old_archive, new_archive, buffer)
replacement = File.objects.create(name=old_file.name,
type=old_file.type)
buffer.seek(0)
replacement.putfile(buffer)
with transaction.atomic():
release_file.refresh_from_db()
if release_file.file == old_file:
# Nothing has changed. It is safe to update
release_file.update(file=replacement)
success = True
break
else:
metrics.incr("tasks.assemble.merge_archives_retry",
instance=str(attempt))
else:
logger.error("Failed to merge archive in %s attempts, giving up.",
max_attempts)
if success:
old_file.delete()
new_file.delete()
def _merge_archives(release_file: ReleaseFile, new_file: File,
new_archive: ReleaseArchive):
lock_key = f"assemble:merge_archives:{release_file.id}"
lock = app.locks.get(lock_key, duration=60)
try:
with lock.blocking_acquire(RELEASE_ARCHIVE_MERGE_INITIAL_DELAY,
RELEASE_ARCHIVE_MERGE_TIMEOUT):
old_file = release_file.file
old_file_contents = ReleaseFile.cache.getfile(release_file)
buffer = BytesIO()
with metrics.timer("tasks.assemble.merge_archives_pure"):
did_merge = merge_release_archives(old_file_contents,
new_archive, buffer)
if did_merge:
replacement = File.objects.create(name=old_file.name,
type=old_file.type)
buffer.seek(0)
replacement.putfile(buffer)
release_file.update(file=replacement)
old_file.delete()
except UnableToAcquireLock as error:
logger.error("merge_archives.fail", extra={"error": error})
new_file.delete()
def create_release_file(self, project, release, path, content_type=None, contents=None):
from sentry.models import File, ReleaseFile
if content_type is None:
content_type = mimetypes.guess_type(path)[0] or "text/plain"
if content_type.startswith("text/"):
content_type += "; encoding=utf-8"
f = File(name=path.rsplit("/", 1)[-1], type="release.file", headers={"Content-Type": content_type})
f.putfile(StringIO(contents or ""))
return ReleaseFile.objects.create(project=project, release=release, file=f, name=path)
def post(self, request, project, version):
"""
Upload a new file
Upload a new file for the given release.
{method} {path}
name=http%3A%2F%2Fexample.com%2Fapplication.js
# ...
Unlike other API requests, files must be uploaded using the traditional
multipart/form-data content-type.
The optional 'name' attribute should reflect the absolute path that this
file will be referenced as. For example, in the case of JavaScript you
might specify the full web URI.
"""
release = Release.objects.get(
project=project,
version=version,
)
if 'file' not in request.FILES:
return Response(status=400)
fileobj = request.FILES['file']
full_name = request.DATA.get('name', fileobj.name)
name = full_name.rsplit('/', 1)[-1]
# TODO(dcramer): File's are unique on (name, checksum) so we need to
# ensure that this file does not already exist for other purposes
file = File(
name=name,
type='release.file',
headers={
'Content-Type': fileobj.content_type,
}
)
file.putfile(fileobj)
try:
with transaction.atomic():
releasefile = ReleaseFile.objects.create(
project=release.project,
release=release,
file=file,
name=full_name,
)
except IntegrityError:
file.delete()
return Response(status=409)
return Response(serialize(releasefile, request.user), status=201)
def create_release_file(self, project, release, path,
content_type=None, contents=None):
from sentry.models import File, ReleaseFile
if content_type is None:
content_type = mimetypes.guess_type(path)[0] or 'text/plain'
if content_type.startswith('text/'):
content_type += '; encoding=utf-8'
f = File(name=path.rsplit('/', 1)[-1], type='release.file', headers={
'Content-Type': content_type
})
f.putfile(StringIO(contents or ''))
return ReleaseFile.objects.create(
project=project,
release=release,
file=f,
name=path
)
def test_putfile(self):
fileobj = ContentFile("foo bar")
my_file = File(name='app.dsym', type='release.artifact')
my_file.putfile(fileobj, commit=False)
my_file.save()
assert my_file.path
assert my_file.storage == settings.SENTRY_FILESTORE
with self.assertRaises(Exception):
my_file.putfile(fileobj, commit=False)
def _upsert_release_file(file: File, archive: ReleaseArchive, update_fn,
**kwargs):
release_file = None
# Release files must have unique names within their release
# and dist. If a matching file already exists, replace its
# file with the new one; otherwise create it.
try:
release_file = ReleaseFile.objects.get(**kwargs)
except ReleaseFile.DoesNotExist:
try:
with transaction.atomic():
release_file = ReleaseFile.objects.create(file=file, **kwargs)
except IntegrityError:
# NB: This indicates a race, where another assemble task or
# file upload job has just created a conflicting file. Since
# we're upserting here anyway, yield to the faster actor and
# do not try again.
file.delete()
else:
update_fn(release_file, file, archive)
def pseudo_releasefile(url, info, dist):
"""Create a pseudo-ReleaseFile from an ArtifactIndex entry"""
return ReleaseFile(
name=url,
file=File(
headers=info.get("headers", {}),
size=info["size"],
timestamp=info["date_created"],
checksum=info["sha1"],
),
dist_id=dist.id if dist else dist,
)
def test_putfile(self):
fileobj = ContentFile("foo bar")
my_file = File(name="app.dsym", type="release.artifact")
my_file.putfile(fileobj, commit=False)
my_file.save()
assert my_file.path
assert my_file.storage == settings.SENTRY_FILESTORE
with self.assertRaises(Exception):
my_file.putfile(fileobj, commit=False)
def post(self, request, project, version):
"""
Upload a new file
Upload a new file for the given release.
{method} {path}
name=http%3A%2F%2Fexample.com%2Fapplication.js
# ...
Unlike other API requests, files must be uploaded using the traditional
multipart/form-data content-type.
The optional 'name' attribute should reflect the absolute path that this
file will be referenced as. For example, in the case of JavaScript you
might specify the full web URI.
"""
release = Release.objects.get(
project=project,
version=version,
)
if 'file' not in request.FILES:
return Response(status=400)
fileobj = request.FILES['file']
full_name = request.DATA.get('name', fileobj.name)
name = full_name.rsplit('/', 1)[-1]
# TODO(dcramer): File's are unique on (name, checksum) so we need to
# ensure that this file does not already exist for other purposes
file = File(name=name,
type='release.file',
headers={
'Content-Type': fileobj.content_type,
})
file.putfile(fileobj)
try:
with transaction.atomic():
releasefile = ReleaseFile.objects.create(
project=release.project,
release=release,
file=file,
name=full_name,
)
except IntegrityError:
file.delete()
return Response(status=409)
return Response(serialize(releasefile, request.user), status=201)
def test_expansion_via_release_artifacts(self):
project = self.project
release = Release.objects.create(
project=project,
version='abc',
)
f1 = File(
name='file.min.js',
type='release.file',
headers={'Content-Type': 'application/json'},
)
f1.putfile(open(get_fixture_path(f1.name), 'rb'))
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f1.name),
release=release,
project=project,
file=f1,
)
f2 = File(
name='file1.js',
type='release.file',
headers={'Content-Type': 'application/json'},
)
f2.putfile(open(get_fixture_path(f2.name), 'rb'))
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f2.name),
release=release,
project=project,
file=f2,
)
f3 = File(
name='file2.js',
type='release.file',
headers={'Content-Type': 'application/json'},
)
f3.putfile(open(get_fixture_path(f3.name), 'rb'))
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f3.name),
release=release,
project=project,
file=f3,
)
f4 = File(
name='file.sourcemap.js',
type='release.file',
headers={'Content-Type': 'application/json'},
)
f4.putfile(open(get_fixture_path(f4.name), 'rb'))
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f4.name),
release=release,
project=project,
file=f4,
)
data = {
'message': 'hello',
'platform': 'javascript',
'release': 'abc',
'sentry.interfaces.Exception': {
'values': [{
'type': 'Error',
'stacktrace': {
'frames': [
{
'abs_path': 'http://example.com/file.min.js',
'filename': 'file.min.js',
'lineno': 1,
'colno': 39,
},
],
},
}],
}
}
resp = self._postWithHeader(data)
assert resp.status_code, 200
event = Event.objects.get()
exception = event.interfaces['sentry.interfaces.Exception']
frame_list = exception.values[0].stacktrace.frames
frame = frame_list[0]
assert not frame.errors
assert frame.pre_context == [
'function add(a, b) {',
'\t"use strict";',
]
assert frame.context_line == '\treturn a + b;'
assert frame.post_context == ['}']
def post(self, request, project, version):
"""
Upload a new file
Upload a new file for the given release.
{method} {path}
name=http%3A%2F%2Fexample.com%2Fapplication.js
&header=X-SourceMap%3A%20http%3A%2F%2Fexample.com%2Fapplication.js.map
# ...
Unlike other API requests, files must be uploaded using the traditional
multipart/form-data content-type.
The optional 'name' attribute should reflect the absolute path that this
file will be referenced as. For example, in the case of JavaScript you
might specify the full web URI.
"""
try:
release = Release.objects.get(
project=project,
version=version,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if 'file' not in request.FILES:
return Response({'detail': 'Missing uploaded file'}, status=400)
fileobj = request.FILES['file']
full_name = request.DATA.get('name', fileobj.name)
name = full_name.rsplit('/', 1)[-1]
headers = {
'Content-Type': fileobj.content_type,
}
for headerval in request.DATA.getlist('header') or ():
try:
k, v = headerval.split(':', 1)
except ValueError:
return Response({'detail': 'header value was not formatted correctly'}, status=400)
else:
headers[k] = v.strip()
# TODO(dcramer): File's are unique on (name, checksum) so we need to
# ensure that this file does not already exist for other purposes
file = File(
name=name,
type='release.file',
headers=headers,
)
file.putfile(fileobj)
try:
with transaction.atomic():
releasefile = ReleaseFile.objects.create(
project=release.project,
release=release,
file=file,
name=full_name,
)
except IntegrityError:
file.delete()
return Response(status=409)
return Response(serialize(releasefile, request.user), status=201)
def test_expansion_via_release_artifacts(self):
project = self.project
release = Release.objects.create(
project=project,
version='abc',
)
f1 = File(
name='file.min.js',
type='release.file',
headers={'Content-Type': 'application/json'},
)
f1.putfile(open(get_fixture_path(f1.name), 'rb'))
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f1.name),
release=release,
project=project,
file=f1,
)
f2 = File(
name='file1.js',
type='release.file',
headers={'Content-Type': 'application/json'},
)
f2.putfile(open(get_fixture_path(f2.name), 'rb'))
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f2.name),
release=release,
project=project,
file=f2,
)
f3 = File(
name='file2.js',
type='release.file',
headers={'Content-Type': 'application/json'},
)
f3.putfile(open(get_fixture_path(f3.name), 'rb'))
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f3.name),
release=release,
project=project,
file=f3,
)
f4 = File(
name='file.sourcemap.js',
type='release.file',
headers={'Content-Type': 'application/json'},
)
f4.putfile(open(get_fixture_path(f4.name), 'rb'))
ReleaseFile.objects.create(
name='http://example.com/{}'.format(f4.name),
release=release,
project=project,
file=f4,
)
data = {
'message': 'hello',
'platform': 'javascript',
'release': 'abc',
'sentry.interfaces.Exception': {
'values': [{
'type': 'Error',
'stacktrace': {
'frames': [
{
'abs_path': 'http://example.com/file.min.js',
'filename': 'file.min.js',
'lineno': 1,
'colno': 39,
},
],
},
}],
}
}
resp = self._postWithHeader(data)
assert resp.status_code, 200
event = Event.objects.get()
assert not event.data['errors']
exception = event.interfaces['sentry.interfaces.Exception']
frame_list = exception.values[0].stacktrace.frames
frame = frame_list[0]
assert frame.pre_context == [
'function add(a, b) {',
'\t"use strict";',
]
assert frame.context_line == '\treturn a + b;'
assert frame.post_context == ['}']
def post(self, request, project, version):
"""
Upload a New File
`````````````````
Upload a new file for the given release.
Unlike other API requests, files must be uploaded using the
traditional multipart/form-data content-type.
The optional 'name' attribute should reflect the absolute path
that this file will be referenced as. For example, in the case of
JavaScript you might specify the full web URI.
:pparam string organization_slug: the slug of the organization the
release belongs to.
:pparam string project_slug: the slug of the project to change the
release of.
:pparam string version: the version identifier of the release.
:param string name: the name (full path) of the file.
:param file file: the multipart encoded file.
:param string header: this parameter can be supplied multiple times
to attach headers to the file. Each header
is a string in the format ``key:value``. For
instance it can be used to define a content
type.
:auth: required
"""
try:
release = Release.objects.get(
project=project,
version=version,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if 'file' not in request.FILES:
return Response({'detail': 'Missing uploaded file'}, status=400)
fileobj = request.FILES['file']
full_name = request.DATA.get('name', fileobj.name)
name = full_name.rsplit('/', 1)[-1]
headers = {
'Content-Type': fileobj.content_type,
}
for headerval in request.DATA.getlist('header') or ():
try:
k, v = headerval.split(':', 1)
except ValueError:
return Response({'detail': 'header value was not formatted correctly'}, status=400)
else:
headers[k] = v.strip()
# TODO(dcramer): File's are unique on (name, checksum) so we need to
# ensure that this file does not already exist for other purposes
file = File(
name=name,
type='release.file',
headers=headers,
)
file.putfile(fileobj)
try:
with transaction.atomic():
releasefile = ReleaseFile.objects.create(
project=release.project,
release=release,
file=file,
name=full_name,
)
except IntegrityError:
file.delete()
return Response(status=409)
return Response(serialize(releasefile, request.user), status=201)
def post(self, request, project, version):
"""
Upload a New File
`````````````````
Upload a new file for the given release.
Unlike other API requests, files must be uploaded using the
traditional multipart/form-data content-type.
The optional 'name' attribute should reflect the absolute path
that this file will be referenced as. For example, in the case of
JavaScript you might specify the full web URI.
:pparam string organization_slug: the slug of the organization the
release belongs to.
:pparam string project_slug: the slug of the project to change the
release of.
:pparam string version: the version identifier of the release.
:param string name: the name (full path) of the file.
:param file file: the multipart encoded file.
:param string header: this parameter can be supplied multiple times
to attach headers to the file. Each header
is a string in the format ``key:value``. For
instance it can be used to define a content
type.
:auth: required
"""
try:
release = Release.objects.get(
project=project,
version=version,
)
except Release.DoesNotExist:
raise ResourceDoesNotExist
if 'file' not in request.FILES:
return Response({'detail': 'Missing uploaded file'}, status=400)
fileobj = request.FILES['file']
full_name = request.DATA.get('name', fileobj.name)
name = full_name.rsplit('/', 1)[-1]
headers = {
'Content-Type': fileobj.content_type,
}
for headerval in request.DATA.getlist('header') or ():
try:
k, v = headerval.split(':', 1)
except ValueError:
return Response(
{'detail': 'header value was not formatted correctly'},
status=400)
else:
headers[k] = v.strip()
# TODO(dcramer): File's are unique on (name, checksum) so we need to
# ensure that this file does not already exist for other purposes
file = File(
name=name,
type='release.file',
headers=headers,
)
file.putfile(fileobj)
try:
with transaction.atomic():
releasefile = ReleaseFile.objects.create(
project=release.project,
release=release,
file=file,
name=full_name,
)
except IntegrityError:
file.delete()
return Response(status=409)
return Response(serialize(releasefile, request.user), status=201)