def test_store_tag_manifest(get_storages, initialized_db): # Create a manifest with some layers. builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'sometag') storages = get_storages() assert storages repo = model.repository.get_repository('devtable', 'simple') storage_id_map = {} for index, storage in enumerate(storages): image_id = 'someimage%s' % index builder.add_layer(storage.content_checksum, json.dumps({'id': image_id})) find_create_or_link_image(image_id, repo, 'devtable', {}, 'local_us') storage_id_map[storage.content_checksum] = storage.id manifest = builder.build(docker_v2_signing_key) tag_manifest, _ = store_tag_manifest_for_testing('devtable', 'simple', 'sometag', manifest, manifest.leaf_layer_v1_image_id, storage_id_map) # Ensure we have the new-model expected rows. mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest) assert mapping_row.manifest is not None assert mapping_row.manifest.manifest_bytes == manifest.bytes.as_encoded_str() assert mapping_row.manifest.digest == str(manifest.digest) blob_rows = {m.blob_id for m in ManifestBlob.select().where(ManifestBlob.manifest == mapping_row.manifest)} assert blob_rows == {s.id for s in storages} assert ManifestLegacyImage.get(manifest=mapping_row.manifest).image == tag_manifest.tag.image
def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db): repository = create_repository('devtable', 'somenewrepo', None) image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') manifest = Manifest.get() footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id, oci_manifest=manifest) expiration_date = None if expiration_offset is not None: expiration_date = datetime.utcnow() + convert_to_timedelta(expiration_offset) assert change_tag_expiration(footag, expiration_date) # Lookup the tag again. footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo') oci_tag = _get_oci_tag(footag_updated) if expected_offset is None: assert footag_updated.lifetime_end_ts is None assert oci_tag.lifetime_end_ms is None else: start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts) end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts) expected_end_date = start_date + convert_to_timedelta(expected_offset) assert (expected_end_date - end_date).total_seconds() < 5 # variance in test assert oci_tag.lifetime_end_ms == (footag_updated.lifetime_end_ts * 1000)
def test_create_reversion_tag(initialized_db): repository = create_repository('devtable', 'somenewrepo', None) manifest = Manifest.get() image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id, oci_manifest=manifest, reversion=True) assert footag.reversion oci_tag = _get_oci_tag(footag) assert oci_tag.name == footag.name assert not oci_tag.hidden assert oci_tag.reversion == footag.reversion
def test_remove_obsolete_tags(initialized_db): """ As part of the mirror, the set of tags on the remote repository is compared to the local existing tags. Those not present on the remote are removed locally. """ mirror, repository = create_mirror_repo_robot(["updated", "created"], repo_name="removed") manifest = Manifest.get() image = find_create_or_link_image("removed", repository, None, {}, "local_us") tag = create_or_update_tag_for_repo( repository, "oldtag", image.docker_image_id, oci_manifest=manifest, reversion=True ) incoming_tags = ["one", "two"] deleted_tags = delete_obsolete_tags(mirror, incoming_tags) assert [tag.name for tag in deleted_tags] == [tag.name]
def test_list_active_tags(initialized_db): # Create a new repository. repository = create_repository('devtable', 'somenewrepo', None) manifest = Manifest.get() # Create some images. image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') image2 = find_create_or_link_image('foobarimage2', repository, None, {}, 'local_us') # Make sure its tags list is empty. assert_tags(repository) # Add some new tags. footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id, oci_manifest=manifest) bartag = create_or_update_tag_for_repo(repository, 'bar', image1.docker_image_id, oci_manifest=manifest) # Since timestamps are stored on a second-granularity, we need to make the tags "start" # before "now", so when we recreate them below, they don't conflict. footag.lifetime_start_ts -= 5 footag.save() bartag.lifetime_start_ts -= 5 bartag.save() footag_oci = _get_oci_tag(footag) footag_oci.lifetime_start_ms -= 5000 footag_oci.save() bartag_oci = _get_oci_tag(bartag) bartag_oci.lifetime_start_ms -= 5000 bartag_oci.save() # Make sure they are returned. assert_tags(repository, 'foo', 'bar') # Set the expirations to be explicitly empty. set_tag_end_ts(footag, None) set_tag_end_ts(bartag, None) # Make sure they are returned. assert_tags(repository, 'foo', 'bar') # Mark as a tag as expiring in the far future, and make sure it is still returned. set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000) # Make sure they are returned. assert_tags(repository, 'foo', 'bar') # Delete a tag and make sure it isn't returned. footag = delete_tag('devtable', 'somenewrepo', 'foo') set_tag_end_ts(footag, footag.lifetime_end_ts - 4) assert_tags(repository, 'bar') # Add a new foo again. footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id, oci_manifest=manifest) footag.lifetime_start_ts -= 3 footag.save() footag_oci = _get_oci_tag(footag) footag_oci.lifetime_start_ms -= 3000 footag_oci.save() assert_tags(repository, 'foo', 'bar') # Mark as a tag as expiring in the far future, and make sure it is still returned. set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000) # Make sure they are returned. assert_tags(repository, 'foo', 'bar') # "Move" foo by updating it and make sure we don't get duplicates. create_or_update_tag_for_repo(repository, 'foo', image2.docker_image_id, oci_manifest=manifest) assert_tags(repository, 'foo', 'bar')
def test_get_or_create_manifest_list_duplicate_child_manifest(initialized_db): repository = create_repository("devtable", "newrepo", None) expected_labels = { "Foo": "Bar", "Baz": "Meh", } layer_json = json.dumps({ "id": "somelegacyid", "config": { "Labels": expected_labels, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Create a legacy image. find_create_or_link_image("somelegacyid", repository, "devtable", {}, "local_us") # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = "hello world" _, random_digest = _populate_blob(random_data) # Build the manifest. v2_builder = DockerSchema2ManifestBuilder() v2_builder.set_config_digest(config_digest, len(layer_json.encode("utf-8"))) v2_builder.add_layer(random_digest, len(random_data.encode("utf-8"))) v2_manifest = v2_builder.build() # Write the manifest. v2_created = get_or_create_manifest(repository, v2_manifest, storage) assert v2_created assert v2_created.manifest.digest == v2_manifest.digest # Build the manifest list, with the child manifest repeated. list_builder = DockerSchema2ManifestListBuilder() list_builder.add_manifest(v2_manifest, "amd64", "linux") list_builder.add_manifest(v2_manifest, "amd32", "linux") manifest_list = list_builder.build() # Write the manifest list, which should also write the manifests themselves. created_tuple = get_or_create_manifest(repository, manifest_list, storage) assert created_tuple is not None created_list = created_tuple.manifest assert created_list assert created_list.media_type.name == manifest_list.media_type assert created_list.digest == manifest_list.digest # Ensure the child manifest links exist. child_manifests = { cm.child_manifest.digest: cm.child_manifest for cm in ManifestChild.select().where( ManifestChild.manifest == created_list) } assert len(child_manifests) == 1 assert v2_manifest.digest in child_manifests assert child_manifests[ v2_manifest.digest].media_type.name == v2_manifest.media_type # Try to create again and ensure we get back the same manifest list. created2_tuple = get_or_create_manifest(repository, manifest_list, storage) assert created2_tuple is not None assert created2_tuple.manifest == created_list
def test_get_or_create_manifest(schema_version, initialized_db): repository = create_repository("devtable", "newrepo", None) expected_labels = { "Foo": "Bar", "Baz": "Meh", } layer_json = json.dumps({ "id": "somelegacyid", "config": { "Labels": expected_labels, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Create a legacy image. find_create_or_link_image("somelegacyid", repository, "devtable", {}, "local_us") # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = "hello world" _, random_digest = _populate_blob(random_data) # Build the manifest. if schema_version == 1: builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag") builder.add_layer(random_digest, layer_json) sample_manifest_instance = builder.build(docker_v2_signing_key) elif schema_version == 2: builder = DockerSchema2ManifestBuilder() builder.set_config_digest(config_digest, len(layer_json.encode("utf-8"))) builder.add_layer(random_digest, len(random_data.encode("utf-8"))) sample_manifest_instance = builder.build() assert sample_manifest_instance.layers_compressed_size is not None # Create a new manifest. created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage) created = created_manifest.manifest newly_created = created_manifest.newly_created assert newly_created assert created is not None assert created.media_type.name == sample_manifest_instance.media_type assert created.digest == sample_manifest_instance.digest assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str( ) assert created_manifest.labels_to_apply == expected_labels assert created.config_media_type == sample_manifest_instance.config_media_type assert created.layers_compressed_size == sample_manifest_instance.layers_compressed_size # Lookup the manifest and verify. found = lookup_manifest(repository, created.digest, allow_dead=True) assert found.digest == created.digest assert found.config_media_type == created.config_media_type assert found.layers_compressed_size == created.layers_compressed_size # Verify it has a temporary tag pointing to it. assert Tag.get(manifest=created, hidden=True).lifetime_end_ms # Verify the linked blobs. blob_digests = [ mb.blob.content_checksum for mb in ManifestBlob.select().where(ManifestBlob.manifest == created) ] assert random_digest in blob_digests if schema_version == 2: assert config_digest in blob_digests # Retrieve it again and ensure it is the same manifest. created_manifest2 = get_or_create_manifest(repository, sample_manifest_instance, storage) created2 = created_manifest2.manifest newly_created2 = created_manifest2.newly_created assert not newly_created2 assert created2 == created # Ensure it again has a temporary tag. assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms # Ensure the labels were added. labels = list(list_manifest_labels(created)) assert len(labels) == 2 labels_dict = {label.key: label.value for label in labels} assert labels_dict == expected_labels
def test_get_or_create_manifest_list(initialized_db): repository = create_repository('devtable', 'newrepo', None) expected_labels = { 'Foo': 'Bar', 'Baz': 'Meh', } layer_json = json.dumps({ 'id': 'somelegacyid', 'config': { 'Labels': expected_labels, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Create a legacy image. find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us') # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = 'hello world' _, random_digest = _populate_blob(random_data) # Build the manifests. v1_builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag') v1_builder.add_layer(random_digest, layer_json) v1_manifest = v1_builder.build(docker_v2_signing_key).unsigned() v2_builder = DockerSchema2ManifestBuilder() v2_builder.set_config_digest(config_digest, len(layer_json)) v2_builder.add_layer(random_digest, len(random_data)) v2_manifest = v2_builder.build() # Write the manifests. v1_created = get_or_create_manifest(repository, v1_manifest, storage) assert v1_created assert v1_created.manifest.digest == v1_manifest.digest v2_created = get_or_create_manifest(repository, v2_manifest, storage) assert v2_created assert v2_created.manifest.digest == v2_manifest.digest # Build the manifest list. list_builder = DockerSchema2ManifestListBuilder() list_builder.add_manifest(v1_manifest, 'amd64', 'linux') list_builder.add_manifest(v2_manifest, 'amd32', 'linux') manifest_list = list_builder.build() # Write the manifest list, which should also write the manifests themselves. created_tuple = get_or_create_manifest(repository, manifest_list, storage) assert created_tuple is not None created_list = created_tuple.manifest assert created_list assert created_list.media_type.name == manifest_list.media_type assert created_list.digest == manifest_list.digest # Ensure the child manifest links exist. child_manifests = { cm.child_manifest.digest: cm.child_manifest for cm in ManifestChild.select().where( ManifestChild.manifest == created_list) } assert len(child_manifests) == 2 assert v1_manifest.digest in child_manifests assert v2_manifest.digest in child_manifests assert child_manifests[ v1_manifest.digest].media_type.name == v1_manifest.media_type assert child_manifests[ v2_manifest.digest].media_type.name == v2_manifest.media_type
def test_get_or_create_manifest(schema_version, initialized_db): repository = create_repository('devtable', 'newrepo', None) expected_labels = { 'Foo': 'Bar', 'Baz': 'Meh', } layer_json = json.dumps({ 'id': 'somelegacyid', 'config': { 'Labels': expected_labels, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Create a legacy image. find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us') # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = 'hello world' _, random_digest = _populate_blob(random_data) # Build the manifest. if schema_version == 1: builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag') builder.add_layer(random_digest, layer_json) sample_manifest_instance = builder.build(docker_v2_signing_key) elif schema_version == 2: builder = DockerSchema2ManifestBuilder() builder.set_config_digest(config_digest, len(layer_json)) builder.add_layer(random_digest, len(random_data)) sample_manifest_instance = builder.build() # Create a new manifest. created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage) created = created_manifest.manifest newly_created = created_manifest.newly_created assert newly_created assert created is not None assert created.media_type.name == sample_manifest_instance.media_type assert created.digest == sample_manifest_instance.digest assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str( ) assert created_manifest.labels_to_apply == expected_labels # Verify it has a temporary tag pointing to it. assert Tag.get(manifest=created, hidden=True).lifetime_end_ms # Verify the legacy image. legacy_image = get_legacy_image_for_manifest(created) assert legacy_image is not None assert legacy_image.storage.content_checksum == random_digest # Verify the linked blobs. blob_digests = [ mb.blob.content_checksum for mb in ManifestBlob.select().where(ManifestBlob.manifest == created) ] assert random_digest in blob_digests if schema_version == 2: assert config_digest in blob_digests # Retrieve it again and ensure it is the same manifest. created_manifest2 = get_or_create_manifest(repository, sample_manifest_instance, storage) created2 = created_manifest2.manifest newly_created2 = created_manifest2.newly_created assert not newly_created2 assert created2 == created # Ensure it again has a temporary tag. assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms # Ensure the labels were added. labels = list(list_manifest_labels(created)) assert len(labels) == 2 labels_dict = {label.key: label.value for label in labels} assert labels_dict == expected_labels