def delete_manifests(): ManifestLegacyImage.delete().execute() ManifestBlob.delete().execute() Manifest.delete().execute() TagManifestToManifest.delete().execute() TagManifest.delete().execute() return "OK"
def populate_manifest(repository, manifest, legacy_image, storage_ids): """ Populates the rows for the manifest, including its blobs and legacy image. """ media_type = Manifest.media_type.get_id(manifest.media_type) # Check for an existing manifest. If present, return it. try: return Manifest.get(repository=repository, digest=manifest.digest) except Manifest.DoesNotExist: pass with db_transaction(): try: manifest_row = Manifest.create( digest=manifest.digest, repository=repository, manifest_bytes=manifest.bytes.as_encoded_str(), media_type=media_type, ) except IntegrityError as ie: logger.debug( "Got integrity error when trying to write manifest: %s", ie) return Manifest.get(repository=repository, digest=manifest.digest) ManifestLegacyImage.create(manifest=manifest_row, repository=repository, image=legacy_image) blobs_to_insert = [ dict(manifest=manifest_row, repository=repository, blob=storage_id) for storage_id in storage_ids ] if blobs_to_insert: ManifestBlob.insert_many(blobs_to_insert).execute() return manifest_row
def _check_image_used(legacy_image_id): assert legacy_image_id is not None with db_transaction(): # Check if the image is referenced by a manifest. try: ManifestLegacyImage.select().where(ManifestLegacyImage.image == legacy_image_id).get() return True except ManifestLegacyImage.DoesNotExist: pass # Check if the image is referenced by a tag. try: RepositoryTag.select().where(RepositoryTag.image == legacy_image_id).get() return True except RepositoryTag.DoesNotExist: pass # Check if the image is referenced by another image. try: Image.select().where(Image.parent == legacy_image_id).get() return True except Image.DoesNotExist: pass return False
def clear_rows(initialized_db): # Remove all new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute()
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will attempt to have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository("devtable", "complex") tag_v50 = model.tag.get_active_tag("devtable", "gargantuan", "v5.0") # Add a mislinked manifest, by having its layer point to an invalid blob but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder("devtable", "gargantuan", "sometag") builder.add_layer("sha256:deadbeef", '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) broken_manifest = TagManifest.create( json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50 ) # Backfill the manifest and ensure it is marked as broken. assert _backfill_manifest(broken_manifest) map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 0
def test_store_tag_manifest(get_storages, initialized_db): # Create a manifest with some layers. builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'sometag') storages = get_storages() assert storages repo = model.repository.get_repository('devtable', 'simple') storage_id_map = {} for index, storage in enumerate(storages): image_id = 'someimage%s' % index builder.add_layer(storage.content_checksum, json.dumps({'id': image_id})) find_create_or_link_image(image_id, repo, 'devtable', {}, 'local_us') storage_id_map[storage.content_checksum] = storage.id manifest = builder.build(docker_v2_signing_key) tag_manifest, _ = store_tag_manifest_for_testing('devtable', 'simple', 'sometag', manifest, manifest.leaf_layer_v1_image_id, storage_id_map) # Ensure we have the new-model expected rows. mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert mapping_row.manifest is not None assert mapping_row.manifest.manifest_bytes == manifest.bytes.as_encoded_str() assert mapping_row.manifest.digest == str(manifest.digest) blob_rows = {m.blob_id for m in ManifestBlob.select().where(ManifestBlob.manifest == mapping_row.manifest)} assert blob_rows == {s.id for s in storages} assert ManifestLegacyImage.get(manifest=mapping_row.manifest).image == tag_manifest.tag.image
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db): """ Tests that a manifest whose image is mislinked will have its storages relinked properly. """ # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() repo = model.repository.get_repository('devtable', 'complex') tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0') tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') # Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image # be the v5.0 image. builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}') manifest = builder.build(docker_v2_signing_key) mislinked_manifest = TagManifest.create(json_data=manifest.bytes.as_encoded_str(), digest=manifest.digest, tag=tag_v50) # Backfill the manifest and ensure its proper content checksum was linked. assert _backfill_manifest(mislinked_manifest) map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest) assert not map_row.broken manifest_row = map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert legacy_image == tag_v50.image manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) assert len(manifest_blobs) == 1 assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
def get_legacy_image_for_manifest(manifest_id): """ Returns the legacy image associated with the given manifest, if any, or None if none. """ try: query = (ManifestLegacyImage.select( ManifestLegacyImage, Image).join(Image).where( ManifestLegacyImage.manifest == manifest_id)) return query.get().image except ManifestLegacyImage.DoesNotExist: return None
def get_manifest_for_legacy_image(image_id): """ Returns a manifest that is associated with the given image, if any, or None if none. """ try: query = (ManifestLegacyImage.select( ManifestLegacyImage, Manifest).join(Manifest).where( ManifestLegacyImage.image == image_id)) return query.get().manifest except ManifestLegacyImage.DoesNotExist: return None
def populate_legacy_images_for_testing(manifest, manifest_interface_instance, storage): """ Populates the legacy image rows for the given manifest. """ # NOTE: This method is only kept around for use by legacy tests that still require # legacy images. As a result, we make sure we're in testing mode before we run. assert os.getenv("TEST") == "true" repository_id = manifest.repository_id retriever = RepositoryContentRetriever.for_repository( repository_id, storage) blob_map = _build_blob_map(repository_id, manifest_interface_instance, storage, True, require_empty_layer=True) if blob_map is None: return None # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy # image. legacy_image = None if manifest_interface_instance.has_legacy_image: try: legacy_image_id = _populate_legacy_image( repository_id, manifest_interface_instance, blob_map, retriever, True) except ManifestException as me: raise CreateManifestException( "Attempt to create an invalid manifest: %s. Please report this issue." % me) if legacy_image_id is None: return None legacy_image = get_image(repository_id, legacy_image_id) if legacy_image is None: return None # Set the legacy image (if applicable). if legacy_image is not None: ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest)
def verify_backfill(namespace_name): logger.info('Checking namespace %s', namespace_name) namespace_user = model.user.get_namespace_user(namespace_name) assert namespace_user repo_tags = (RepositoryTag .select() .join(Repository) .where(Repository.namespace_user == namespace_user) .where(RepositoryTag.hidden == False)) repo_tags = list(repo_tags) logger.info('Found %s tags', len(repo_tags)) for index, repo_tag in enumerate(repo_tags): logger.info('Checking tag %s under repository %s (%s/%s)', repo_tag.name, repo_tag.repository.name, index + 1, len(repo_tags)) tag = TagToRepositoryTag.get(repository_tag=repo_tag).tag assert not tag.hidden assert tag.repository == repo_tag.repository assert tag.name == repo_tag.name, _vs(tag.name, repo_tag.name) assert tag.repository == repo_tag.repository, _vs(tag.repository_id, repo_tag.repository_id) assert tag.reversion == repo_tag.reversion, _vs(tag.reversion, repo_tag.reversion) start_check = int(tag.lifetime_start_ms / 1000) == repo_tag.lifetime_start_ts assert start_check, _vs(tag.lifetime_start_ms, repo_tag.lifetime_start_ts) if repo_tag.lifetime_end_ts is not None: end_check = int(tag.lifetime_end_ms / 1000) == repo_tag.lifetime_end_ts assert end_check, _vs(tag.lifetime_end_ms, repo_tag.lifetime_end_ts) else: assert tag.lifetime_end_ms is None try: tag_manifest = tag.manifest repo_tag_manifest = TagManifest.get(tag=repo_tag) digest_check = tag_manifest.digest == repo_tag_manifest.digest assert digest_check, _vs(tag_manifest.digest, repo_tag_manifest.digest) bytes_check = tag_manifest.manifest_bytes == repo_tag_manifest.json_data assert bytes_check, _vs(tag_manifest.manifest_bytes, repo_tag_manifest.json_data) except TagManifest.DoesNotExist: logger.info('No tag manifest found for repository tag %s', repo_tag.id) mli = ManifestLegacyImage.get(manifest=tag_manifest) assert mli.repository == repo_tag.repository manifest_legacy_image = mli.image assert manifest_legacy_image == repo_tag.image, _vs(manifest_legacy_image.id, repo_tag.image_id)
def get_legacy_images_for_tags(tags): """ Returns a map from tag ID to the legacy image for the tag. """ if not tags: return {} query = (ManifestLegacyImage.select( ManifestLegacyImage, Image, ImageStorage).join(Image).join(ImageStorage).where( ManifestLegacyImage.manifest << [tag.manifest_id for tag in tags])) by_manifest = {mli.manifest_id: mli.image for mli in query} return { tag.id: by_manifest[tag.manifest_id] for tag in tags if tag.manifest_id in by_manifest }
def compute_layer_id(layer): """ Returns the ID for the layer in the security scanner. """ assert isinstance(layer, ManifestDataType) manifest = Manifest.get(id=layer._db_id) try: layer = ManifestLegacyImage.get(manifest=manifest).image except ManifestLegacyImage.DoesNotExist: return None assert layer.docker_image_id assert layer.storage.uuid return "%s.%s" % (layer.docker_image_id, layer.storage.uuid)
def test_load_security_information(indexed_v2, indexed_v4, expected_status, initialized_db): secscan_model.configure(app, instance_keys, storage) repository_ref = registry_model.lookup_repository("devtable", "simple") tag = registry_model.find_matching_tag(repository_ref, ["latest"]) manifest = registry_model.get_manifest_for_tag(tag) assert manifest registry_model.populate_legacy_images_for_testing(manifest, storage) image = shared.get_legacy_image_for_manifest(manifest._db_id) if indexed_v2: image.security_indexed = False image.security_indexed_engine = 3 image.save() else: ManifestLegacyImage.delete().where( ManifestLegacyImage.manifest == manifest._db_id).execute() if indexed_v4: ManifestSecurityStatus.create( manifest=manifest._db_id, repository=repository_ref._db_id, error_json={}, index_status=IndexStatus.MANIFEST_UNSUPPORTED, indexer_hash="abc", indexer_version=IndexerVersion.V4, metadata_json={}, ) result = secscan_model.load_security_information(manifest, True) assert isinstance(result, SecurityInformationLookupResult) assert result.status == expected_status
def compute_layer_id(layer): """ Returns the ID for the layer in the security scanner. """ # NOTE: this is temporary until we switch to Clair V3. if isinstance(layer, ManifestDataType): if layer._is_tag_manifest: layer = TagManifest.get(id=layer._db_id).tag.image else: manifest = Manifest.get(id=layer._db_id) try: layer = ManifestLegacyImage.get(manifest=manifest).image except ManifestLegacyImage.DoesNotExist: return None elif isinstance(layer, LegacyImage): layer = Image.get(id=layer._db_id) assert layer.docker_image_id assert layer.storage.uuid return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db): # Delete existing tag manifest so we can reuse the tag. TagManifestLabel.delete().execute() TagManifest.delete().execute() # Add a broken manifest. broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar', tag=RepositoryTag.get()) # Ensure the backfill works. assert _backfill_manifest(broken_manifest) # Ensure the mapping is marked as broken. map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) assert map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == broken_manifest.json_data assert manifest_row.digest == broken_manifest.digest assert manifest_row.repository == broken_manifest.tag.repository legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert broken_manifest.tag.image == legacy_image
def test_list_alive_tags(initialized_db): found = False for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())): tags = list_alive_tags(tag.repository) assert tag in tags with assert_query_count(1): legacy_images = get_legacy_images_for_tags(tags) for tag in tags: assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id] found = True assert found # Ensure hidden tags cannot be listed. tag = Tag.get() tag.hidden = True tag.save() tags = list_alive_tags(tag.repository) assert tag not in tags
def test_tagbackfillworker(clear_all_rows, initialized_db): # Remove the new-style rows so we can backfill. TagToRepositoryTag.delete().execute() Tag.delete().execute() if clear_all_rows: TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() ManifestLegacyImage.delete().execute() TagManifestToManifest.delete().execute() Manifest.delete().execute() found_dead_tag = False for repository_tag in list(RepositoryTag.select()): # Backfill the tag. assert backfill_tag(repository_tag) # Ensure if we try again, the backfill is skipped. assert not backfill_tag(repository_tag) # Ensure that we now have the expected tag rows. tag_to_repo_tag = TagToRepositoryTag.get(repository_tag=repository_tag) tag = tag_to_repo_tag.tag assert tag.name == repository_tag.name assert tag.repository == repository_tag.repository assert not tag.hidden assert tag.reversion == repository_tag.reversion if repository_tag.lifetime_start_ts is None: assert tag.lifetime_start_ms is None else: assert tag.lifetime_start_ms == (repository_tag.lifetime_start_ts * 1000) if repository_tag.lifetime_end_ts is None: assert tag.lifetime_end_ms is None else: assert tag.lifetime_end_ms == (repository_tag.lifetime_end_ts * 1000) found_dead_tag = True assert tag.manifest # Ensure that we now have the expected manifest rows. try: tag_manifest = TagManifest.get(tag=repository_tag) except TagManifest.DoesNotExist: continue map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert not map_row.broken manifest_row = map_row.manifest assert manifest_row.manifest_bytes == tag_manifest.json_data assert manifest_row.digest == tag_manifest.digest assert manifest_row.repository == tag_manifest.tag.repository assert tag.manifest == map_row.manifest legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image assert tag_manifest.tag.image == legacy_image expected_storages = {tag_manifest.tag.image.storage.id} for parent_image_id in tag_manifest.tag.image.ancestor_id_list(): expected_storages.add(Image.get(id=parent_image_id).storage_id) found_storages = { manifest_blob.blob_id for manifest_blob in ManifestBlob.select().where( ManifestBlob.manifest == manifest_row) } assert expected_storages == found_storages # Ensure the labels were copied over. tmls = list(TagManifestLabel.select().where( TagManifestLabel.annotated == tag_manifest)) expected_labels = {tml.label_id for tml in tmls} found_labels = { m.label_id for m in ManifestLabel.select().where( ManifestLabel.manifest == manifest_row) } assert found_labels == expected_labels # Verify at the repository level. for repository in list(Repository.select()): tags = RepositoryTag.select().where( RepositoryTag.repository == repository, RepositoryTag.hidden == False) oci_tags = Tag.select().where(Tag.repository == repository) assert len(tags) == len(oci_tags) assert {t.name for t in tags} == {t.name for t in oci_tags} for tag in tags: tag_manifest = TagManifest.get(tag=tag) ttr = TagToRepositoryTag.get(repository_tag=tag) manifest = ttr.tag.manifest assert tag_manifest.json_data == manifest.manifest_bytes assert tag_manifest.digest == manifest.digest assert tag.image == ManifestLegacyImage.get( manifest=manifest).image assert tag.lifetime_start_ts == (ttr.tag.lifetime_start_ms / 1000) if tag.lifetime_end_ts: assert tag.lifetime_end_ts == (ttr.tag.lifetime_end_ms / 1000) else: assert ttr.tag.lifetime_end_ms is None assert found_dead_tag
def _garbage_collect_manifest(manifest_id, context): assert manifest_id is not None # Make sure the manifest isn't referenced. if _check_manifest_used(manifest_id): return False # Add the manifest's blobs to the context to be GCed. for manifest_blob in ManifestBlob.select().where(ManifestBlob.manifest == manifest_id): context.add_blob_id(manifest_blob.blob_id) # Retrieve the manifest's associated image, if any. try: legacy_image_id = ManifestLegacyImage.get(manifest=manifest_id).image_id context.add_legacy_image_id(legacy_image_id) except ManifestLegacyImage.DoesNotExist: legacy_image_id = None # Add child manifests to be GCed. for connector in ManifestChild.select().where(ManifestChild.manifest == manifest_id): context.add_manifest_id(connector.child_manifest_id) # Add the labels to be GCed. for manifest_label in ManifestLabel.select().where(ManifestLabel.manifest == manifest_id): context.add_label_id(manifest_label.label_id) # Delete the manifest. with db_transaction(): try: manifest = Manifest.select().where(Manifest.id == manifest_id).get() except Manifest.DoesNotExist: return False assert manifest.id == manifest_id assert manifest.repository_id == context.repository.id if _check_manifest_used(manifest_id): return False # Delete any label mappings. (TagManifestLabelMap .delete() .where(TagManifestLabelMap.manifest == manifest_id) .execute()) # Delete any mapping rows for the manifest. TagManifestToManifest.delete().where(TagManifestToManifest.manifest == manifest_id).execute() # Delete any label rows. ManifestLabel.delete().where(ManifestLabel.manifest == manifest_id, ManifestLabel.repository == context.repository).execute() # Delete any child manifest rows. ManifestChild.delete().where(ManifestChild.manifest == manifest_id, ManifestChild.repository == context.repository).execute() # Delete the manifest blobs for the manifest. ManifestBlob.delete().where(ManifestBlob.manifest == manifest_id, ManifestBlob.repository == context.repository).execute() # Delete the manifest legacy image row. if legacy_image_id: (ManifestLegacyImage .delete() .where(ManifestLegacyImage.manifest == manifest_id, ManifestLegacyImage.repository == context.repository) .execute()) # Delete the manifest. manifest.delete_instance() context.mark_manifest_removed(manifest) return True
def _get_legacy_image(namespace, repo, tag, include_storage=True): repo_ref = registry_model.lookup_repository(namespace, repo) repo_tag = registry_model.get_repo_tag(repo_ref, tag) manifest = registry_model.get_manifest_for_tag(repo_tag) return ManifestLegacyImage.get(manifest_id=manifest._db_id).image
def _garbage_collect_manifest(manifest_id, context): assert manifest_id is not None # Make sure the manifest isn't referenced. if _check_manifest_used(manifest_id): return False # Add the manifest's blobs to the context to be GCed. for manifest_blob in ManifestBlob.select().where( ManifestBlob.manifest == manifest_id): context.add_blob_id(manifest_blob.blob_id) # Retrieve the manifest's associated image, if any. try: legacy_image_id = ManifestLegacyImage.get( manifest=manifest_id).image_id context.add_legacy_image_id(legacy_image_id) except ManifestLegacyImage.DoesNotExist: legacy_image_id = None # Add child manifests to be GCed. for connector in ManifestChild.select().where( ManifestChild.manifest == manifest_id): context.add_manifest_id(connector.child_manifest_id) # Add the labels to be GCed. for manifest_label in ManifestLabel.select().where( ManifestLabel.manifest == manifest_id): context.add_label_id(manifest_label.label_id) # Delete the manifest. with db_transaction(): try: manifest = Manifest.select().where( Manifest.id == manifest_id).get() except Manifest.DoesNotExist: return False assert manifest.id == manifest_id assert manifest.repository_id == context.repository.id if _check_manifest_used(manifest_id): return False # Delete any label mappings. deleted_tag_manifest_label_map = (TagManifestLabelMap.delete().where( TagManifestLabelMap.manifest == manifest_id).execute()) # Delete any mapping rows for the manifest. deleted_tag_manifest_to_manifest = ( TagManifestToManifest.delete().where( TagManifestToManifest.manifest == manifest_id).execute()) # Delete any label rows. deleted_manifest_label = (ManifestLabel.delete().where( ManifestLabel.manifest == manifest_id, ManifestLabel.repository == context.repository, ).execute()) # Delete any child manifest rows. deleted_manifest_child = (ManifestChild.delete().where( ManifestChild.manifest == manifest_id, ManifestChild.repository == context.repository, ).execute()) # Delete the manifest blobs for the manifest. deleted_manifest_blob = (ManifestBlob.delete().where( ManifestBlob.manifest == manifest_id, ManifestBlob.repository == context.repository).execute()) # Delete the security status for the manifest deleted_manifest_security = (ManifestSecurityStatus.delete().where( ManifestSecurityStatus.manifest == manifest_id, ManifestSecurityStatus.repository == context.repository, ).execute()) # Delete the manifest legacy image row. deleted_manifest_legacy_image = 0 if legacy_image_id: deleted_manifest_legacy_image = ( ManifestLegacyImage.delete().where( ManifestLegacyImage.manifest == manifest_id, ManifestLegacyImage.repository == context.repository, ).execute()) # Delete the manifest. manifest.delete_instance() context.mark_manifest_removed(manifest) gc_table_rows_deleted.labels( table="TagManifestLabelMap").inc(deleted_tag_manifest_label_map) gc_table_rows_deleted.labels( table="TagManifestToManifest").inc(deleted_tag_manifest_to_manifest) gc_table_rows_deleted.labels( table="ManifestLabel").inc(deleted_manifest_label) gc_table_rows_deleted.labels( table="ManifestChild").inc(deleted_manifest_child) gc_table_rows_deleted.labels( table="ManifestBlob").inc(deleted_manifest_blob) gc_table_rows_deleted.labels( table="ManifestSecurityStatus").inc(deleted_manifest_security) if deleted_manifest_legacy_image: gc_table_rows_deleted.labels( table="ManifestLegacyImage").inc(deleted_manifest_legacy_image) gc_table_rows_deleted.labels(table="Manifest").inc() return True
def _create_manifest( repository_id, manifest_interface_instance, storage, temp_tag_expiration_sec=TEMP_TAG_EXPIRATION_SEC, for_tagging=False, raise_on_error=False, retriever=None, ): # Validate the manifest. retriever = retriever or RepositoryContentRetriever.for_repository( repository_id, storage) try: manifest_interface_instance.validate(retriever) except (ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist, IOError) as ex: logger.exception("Could not validate manifest `%s`", manifest_interface_instance.digest) if raise_on_error: raise CreateManifestException(str(ex)) return None # Load, parse and get/create the child manifests, if any. child_manifest_refs = manifest_interface_instance.child_manifests( retriever) child_manifest_rows = {} child_manifest_label_dicts = [] if child_manifest_refs is not None: for child_manifest_ref in child_manifest_refs: # Load and parse the child manifest. try: child_manifest = child_manifest_ref.manifest_obj except ( ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist, IOError, ) as ex: logger.exception( "Could not load manifest list for manifest `%s`", manifest_interface_instance.digest, ) if raise_on_error: raise CreateManifestException(str(ex)) return None # Retrieve its labels. labels = child_manifest.get_manifest_labels(retriever) if labels is None: if raise_on_error: raise CreateManifestException( "Unable to retrieve manifest labels") logger.exception( "Could not load manifest labels for child manifest") return None # Get/create the child manifest in the database. child_manifest_info = get_or_create_manifest( repository_id, child_manifest, storage, raise_on_error=raise_on_error) if child_manifest_info is None: if raise_on_error: raise CreateManifestException( "Unable to retrieve child manifest") logger.error("Could not get/create child manifest") return None child_manifest_rows[child_manifest_info.manifest. digest] = child_manifest_info.manifest child_manifest_label_dicts.append(labels) # Ensure all the blobs in the manifest exist. digests = set(manifest_interface_instance.local_blob_digests) blob_map = {} # If the special empty layer is required, simply load it directly. This is much faster # than trying to load it on a per repository basis, and that is unnecessary anyway since # this layer is predefined. if EMPTY_LAYER_BLOB_DIGEST in digests: digests.remove(EMPTY_LAYER_BLOB_DIGEST) blob_map[EMPTY_LAYER_BLOB_DIGEST] = get_shared_blob( EMPTY_LAYER_BLOB_DIGEST) if not blob_map[EMPTY_LAYER_BLOB_DIGEST]: if raise_on_error: raise CreateManifestException( "Unable to retrieve specialized empty blob") logger.warning("Could not find the special empty blob in storage") return None if digests: query = lookup_repo_storages_by_content_checksum( repository_id, digests) blob_map.update({s.content_checksum: s for s in query}) for digest_str in digests: if digest_str not in blob_map: logger.warning( "Unknown blob `%s` under manifest `%s` for repository `%s`", digest_str, manifest_interface_instance.digest, repository_id, ) if raise_on_error: raise CreateManifestException("Unknown blob `%s`" % digest_str) return None # Special check: If the empty layer blob is needed for this manifest, add it to the # blob map. This is necessary because Docker decided to elide sending of this special # empty layer in schema version 2, but we need to have it referenced for GC and schema version 1. if EMPTY_LAYER_BLOB_DIGEST not in blob_map: try: requires_empty_layer = manifest_interface_instance.get_requires_empty_layer_blob( retriever) except ManifestException as ex: if raise_on_error: raise CreateManifestException(str(ex)) return None if requires_empty_layer is None: if raise_on_error: raise CreateManifestException( "Could not load configuration blob") return None if requires_empty_layer: shared_blob = get_or_create_shared_blob(EMPTY_LAYER_BLOB_DIGEST, EMPTY_LAYER_BYTES, storage) assert not shared_blob.uploading assert shared_blob.content_checksum == EMPTY_LAYER_BLOB_DIGEST blob_map[EMPTY_LAYER_BLOB_DIGEST] = shared_blob # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy # image. legacy_image = None if manifest_interface_instance.has_legacy_image: legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map, retriever, raise_on_error) if legacy_image_id is None: return None legacy_image = get_image(repository_id, legacy_image_id) if legacy_image is None: return None # Create the manifest and its blobs. media_type = Manifest.media_type.get_id( manifest_interface_instance.media_type) storage_ids = {storage.id for storage in blob_map.values()} with db_transaction(): # Check for the manifest. This is necessary because Postgres doesn't handle IntegrityErrors # well under transactions. try: manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest) return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None) except Manifest.DoesNotExist: pass # Create the manifest. try: manifest = Manifest.create( repository=repository_id, digest=manifest_interface_instance.digest, media_type=media_type, manifest_bytes=manifest_interface_instance.bytes. as_encoded_str(), ) except IntegrityError as ie: try: manifest = Manifest.get( repository=repository_id, digest=manifest_interface_instance.digest) except Manifest.DoesNotExist: logger.error( "Got integrity error when trying to create manifest: %s", ie) if raise_on_error: raise CreateManifestException( "Attempt to create an invalid manifest. Please report this issue." ) return None return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None) # Insert the blobs. blobs_to_insert = [ dict(manifest=manifest, repository=repository_id, blob=storage_id) for storage_id in storage_ids ] if blobs_to_insert: ManifestBlob.insert_many(blobs_to_insert).execute() # Set the legacy image (if applicable). if legacy_image is not None: ManifestLegacyImage.create(repository=repository_id, image=legacy_image, manifest=manifest) # Insert the manifest child rows (if applicable). if child_manifest_rows: children_to_insert = [ dict(manifest=manifest, child_manifest=child_manifest, repository=repository_id) for child_manifest in child_manifest_rows.values() ] ManifestChild.insert_many(children_to_insert).execute() # If this manifest is being created not for immediate tagging, add a temporary tag to the # manifest to ensure it isn't being GCed. If the manifest *is* for tagging, then since we're # creating a new one here, it cannot be GCed (since it isn't referenced by anything yet), so # its safe to elide the temp tag operation. If we ever change GC code to collect *all* manifests # in a repository for GC, then we will have to reevaluate this optimization at that time. if not for_tagging: create_temporary_tag_if_necessary(manifest, temp_tag_expiration_sec) # Define the labels for the manifest (if any). # TODO: Once the old data model is gone, turn this into a batch operation and make the label # application to the manifest occur under the transaction. labels = manifest_interface_instance.get_manifest_labels(retriever) if labels: for key, value in labels.iteritems(): # NOTE: There can technically be empty label keys via Dockerfile's. We ignore any # such `labels`, as they don't really mean anything. if not key: continue media_type = "application/json" if is_json(value) else "text/plain" create_manifest_label(manifest, key, value, "manifest", media_type) # Return the dictionary of labels to apply (i.e. those labels that cause an action to be taken # on the manifest or its resulting tags). We only return those labels either defined on # the manifest or shared amongst all the child manifests. We intersect amongst all child manifests # to ensure that any action performed is defined in all manifests. labels_to_apply = labels or {} if child_manifest_label_dicts: labels_to_apply = child_manifest_label_dicts[0].viewitems() for child_manifest_label_dict in child_manifest_label_dicts[1:]: # Intersect the key+values of the labels to ensure we get the exact same result # for all the child manifests. labels_to_apply = labels_to_apply & child_manifest_label_dict.viewitems( ) labels_to_apply = dict(labels_to_apply) return CreatedManifest(manifest=manifest, newly_created=True, labels_to_apply=labels_to_apply)