def test_non_url_with_release_archive(self):
compressed = BytesIO()
with zipfile.ZipFile(compressed, mode="w") as zip_file:
zip_file.writestr("example.js", b"foo")
zip_file.writestr(
"manifest.json",
json.dumps({
"files": {
"example.js": {
"url": "/example.js",
"headers": {
"content-type": "application/json"
},
}
}
}),
)
release = Release.objects.create(
version="1", organization_id=self.project.organization_id)
release.add_project(self.project)
compressed.seek(0)
file_ = File.objects.create(name="foo", type="release.bundle")
file_.putfile(compressed)
update_artifact_index(release, None, file_)
# Attempt to fetch nonexisting
with pytest.raises(http.BadSource):
fetch_file("does-not-exist.js", release=release)
# Attempt to fetch nonexsting again (to check if cache works)
with pytest.raises(http.BadSource):
result = fetch_file("does-not-exist.js", release=release)
result = fetch_file("/example.js", release=release)
assert result.url == "/example.js"
assert result.body == b"foo"
assert isinstance(result.body, bytes)
assert result.headers == {"content-type": "application/json"}
assert result.encoding == "utf-8"
# Make sure cache loading works:
result2 = fetch_file("/example.js", release=release)
assert result2 == result
def create_release_archive(cls,
org,
release: str,
project=None,
dist=None):
bundle = cls.create_artifact_bundle(org, release, project)
file_ = File.objects.create(name="release-artifacts.zip")
file_.putfile(ContentFile(bundle))
release = Release.objects.get(organization__slug=org, version=release)
return update_artifact_index(release, dist, file_)
def create_archive(self, fields, files, dist=None):
manifest = dict(fields,
files={
filename: {
"url": f"fake://{filename}"
}
for filename in files
})
buffer = BytesIO()
with ZipFile(buffer, mode="w") as zf:
zf.writestr("manifest.json", json.dumps(manifest))
for filename, content in files.items():
zf.writestr(filename, content)
buffer.seek(0)
file_ = File.objects.create(name=str(hash(tuple(files.items()))))
file_.putfile(buffer)
file_.update(timestamp=datetime(
2021, 6, 11, 9, 13, 1, 317902, tzinfo=timezone.utc))
return update_artifact_index(self.release, dist, file_)
def assemble_artifacts(org_id, version, checksum, chunks, **kwargs):
"""
Creates release files from an uploaded artifact bundle.
"""
try:
organization = Organization.objects.get_from_cache(pk=org_id)
bind_organization_context(organization)
set_assemble_status(AssembleTask.ARTIFACTS, org_id, checksum,
ChunkFileState.ASSEMBLING)
archive_filename = f"release-artifacts-{uuid.uuid4().hex}.zip"
# Assemble the chunks into a temporary file
rv = assemble_file(
AssembleTask.ARTIFACTS,
organization,
archive_filename,
checksum,
chunks,
file_type="release.bundle",
)
# If not file has been created this means that the file failed to
# assemble because of bad input data. In this case, assemble_file
# has set the assemble status already.
if rv is None:
return
bundle, temp_file = rv
try:
archive = ReleaseArchive(temp_file)
except Exception:
raise AssembleArtifactsError("failed to open release manifest")
with archive:
manifest = archive.manifest
org_slug = manifest.get("org")
if organization.slug != org_slug:
raise AssembleArtifactsError(
"organization does not match uploaded bundle")
release_name = manifest.get("release")
if release_name != version:
raise AssembleArtifactsError(
"release does not match uploaded bundle")
try:
release = Release.objects.get(organization_id=organization.id,
version=release_name)
except Release.DoesNotExist:
raise AssembleArtifactsError("release does not exist")
dist_name = manifest.get("dist")
dist = None
if dist_name:
dist = release.add_dist(dist_name)
num_files = len(manifest.get("files", {}))
meta = { # Required for release file creation
"organization_id": organization.id,
"release_id": release.id,
"dist_id": dist.id if dist else dist,
}
saved_as_archive = False
min_size = options.get("processing.release-archive-min-files")
if num_files >= min_size:
try:
update_artifact_index(release, dist, bundle)
saved_as_archive = True
except Exception as exc:
logger.error("Unable to update artifact index",
exc_info=exc)
if not saved_as_archive:
_store_single_files(archive, meta, True)
# Count files extracted, to compare them to release files endpoint
metrics.incr("tasks.assemble.extracted_files", amount=num_files)
except AssembleArtifactsError as e:
set_assemble_status(AssembleTask.ARTIFACTS,
org_id,
checksum,
ChunkFileState.ERROR,
detail=str(e))
except Exception:
logger.error("failed to assemble release bundle", exc_info=True)
set_assemble_status(
AssembleTask.ARTIFACTS,
org_id,
checksum,
ChunkFileState.ERROR,
detail="internal server error",
)
else:
set_assemble_status(AssembleTask.ARTIFACTS, org_id, checksum,
ChunkFileState.OK)