def _get_manifest_local_blobs(self,
manifest,
repo_id,
include_placements=False,
by_manifest=False):
parsed = manifest.get_parsed_manifest()
if parsed is None:
return None
local_blob_digests = list(set(parsed.local_blob_digests))
if not len(local_blob_digests):
return []
blob_query = self._lookup_repo_storages_by_content_checksum(
repo_id, local_blob_digests, by_manifest=by_manifest)
blobs = []
for image_storage in blob_query:
placements = None
if include_placements:
placements = list(
model.storage.get_storage_locations(image_storage.uuid))
blob = Blob.for_image_storage(
image_storage,
storage_path=model.storage.get_layer_path(image_storage),
placements=placements,
)
blobs.append(blob)
return blobs
def get_repo_blob_by_digest(self,
repository_ref,
blob_digest,
include_placements=False):
"""
Returns the blob in the repository with the given digest, if any or None if none. Note that
there may be multiple records in the same repository for the same blob digest, so the return
value of this function may change.
"""
image_storage = self._get_shared_storage(blob_digest)
if image_storage is None:
try:
image_storage = model.blob.get_repository_blob_by_digest(
repository_ref._db_id, blob_digest)
except model.BlobDoesNotExist:
return None
assert image_storage.cas_path is not None
placements = None
if include_placements:
placements = list(
model.storage.get_storage_locations(image_storage.uuid))
return Blob.for_image_storage(
image_storage,
storage_path=model.storage.get_layer_path(image_storage),
placements=placements)
def commit_blob_upload(self, blob_upload, blob_digest_str,
blob_expiration_seconds):
"""
Commits the blob upload into a blob and sets an expiration before that blob will be GCed.
"""
upload_record = model.blob.get_blob_upload_by_uuid(
blob_upload.upload_id)
if upload_record is None:
return None
repository_id = upload_record.repository_id
# Create the blob and temporarily tag it.
location_obj = model.storage.get_image_location_for_name(
blob_upload.location_name)
blob_record = model.blob.store_blob_record_and_temp_link_in_repo(
repository_id,
blob_digest_str,
location_obj.id,
blob_upload.byte_count,
blob_expiration_seconds,
blob_upload.uncompressed_byte_count,
)
# Delete the blob upload.
upload_record.delete_instance()
return Blob.for_image_storage(
blob_record,
storage_path=model.storage.get_layer_path(blob_record))
def get_legacy_image(self,
repository_ref,
docker_image_id,
include_parents=False,
include_blob=False):
"""
Returns the matching LegacyImages under the matching repository, if any. If none,
returns None.
"""
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
image = model.image.get_image(repository_ref._db_id, docker_image_id)
if image is None:
return None
parent_images_map = None
if include_parents:
parent_images = model.image.get_parent_images(
repo.namespace_user.username, repo.name, image)
parent_images_map = {image.id: image for image in parent_images}
blob = None
if include_blob:
placements = list(
model.storage.get_storage_locations(image.storage.uuid))
blob = Blob.for_image_storage(
image.storage,
storage_path=model.storage.get_layer_path(image.storage),
placements=placements)
return LegacyImage.for_image(image,
images_map=parent_images_map,
blob=blob)
def get_cached_repo_blob(self, model_cache, namespace_name, repo_name,
blob_digest):
"""
Returns the blob in the repository with the given digest if any or None if none.
Caches the result in the caching system.
"""
def load_blob():
repository_ref = self.lookup_repository(namespace_name, repo_name)
if repository_ref is None:
return None
blob_found = self.get_repo_blob_by_digest(repository_ref,
blob_digest,
include_placements=True)
if blob_found is None:
return None
return blob_found.asdict()
blob_cache_key = cache_key.for_repository_blob(namespace_name,
repo_name, blob_digest,
2)
blob_dict = model_cache.retrieve(blob_cache_key, load_blob)
try:
return Blob.from_dict(blob_dict) if blob_dict is not None else None
except FromDictionaryException:
# The data was stale in some way. Simply reload.
repository_ref = self.lookup_repository(namespace_name, repo_name)
if repository_ref is None:
return None
return self.get_repo_blob_by_digest(repository_ref,
blob_digest,
include_placements=True)
def _list_manifest_layers(self,
repo_id,
parsed,
storage,
include_placements=False,
by_manifest=False):
""" Returns an *ordered list* of the layers found in the manifest, starting at the base and
working towards the leaf, including the associated Blob and its placements (if specified).
Returns None if the manifest could not be parsed and validated.
"""
assert not parsed.is_manifest_list
retriever = RepositoryContentRetriever(repo_id, storage)
requires_empty_blob = parsed.get_requires_empty_layer_blob(retriever)
storage_map = {}
blob_digests = list(parsed.local_blob_digests)
if requires_empty_blob:
blob_digests.append(EMPTY_LAYER_BLOB_DIGEST)
if blob_digests:
blob_query = self._lookup_repo_storages_by_content_checksum(
repo_id, blob_digests, by_manifest=by_manifest)
storage_map = {blob.content_checksum: blob for blob in blob_query}
layers = parsed.get_layers(retriever)
if layers is None:
logger.error("Could not load layers for manifest `%s`",
parsed.digest)
return None
manifest_layers = []
for layer in layers:
if layer.is_remote:
manifest_layers.append(ManifestLayer(layer, None))
continue
digest_str = str(layer.blob_digest)
if digest_str not in storage_map:
logger.error("Missing digest `%s` for manifest `%s`",
layer.blob_digest, parsed.digest)
return None
image_storage = storage_map[digest_str]
assert image_storage.cas_path is not None
assert image_storage.image_size is not None
placements = None
if include_placements:
placements = list(
model.storage.get_storage_locations(image_storage.uuid))
blob = Blob.for_image_storage(
image_storage,
storage_path=model.storage.get_layer_path(image_storage),
placements=placements,
)
manifest_layers.append(ManifestLayer(layer, blob))
return manifest_layers
def _build_derived(self, derived, verb, varying_metadata, include_placements):
if derived is None:
return None
derived_storage = derived.derivative
placements = None
if include_placements:
placements = list(model.storage.get_storage_locations(derived_storage.uuid))
blob = Blob.for_image_storage(
derived_storage,
storage_path=model.storage.get_layer_path(derived_storage),
placements=placements,
)
return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)