def test_does_not_bump_tag_expiration_when_manifest_is_cached_and_upstream_errors( self, create_repo, proxy_manifest_response): repo_ref = create_repo(self.orgname, self.upstream_repository, self.user) proxy_mock = proxy_manifest_response( UBI8_8_4_DIGEST, UBI8_8_4_MANIFEST_SCHEMA2, DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest = proxy_model.lookup_manifest_by_digest( repo_ref, UBI8_8_4_DIGEST) assert manifest is not None first_tag = Tag.get(manifest_id=manifest.id) proxy_mock = proxy_manifest_response("not-existing-ref", "", "") with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest = proxy_model.lookup_manifest_by_digest( repo_ref, UBI8_8_4_DIGEST) assert manifest is not None tag = Tag.get(manifest_id=manifest.id) assert tag.id == first_tag.id assert tag.lifetime_end_ms == first_tag.lifetime_end_ms
def test_set_tag_expiration_for_manifest(initialized_db): tag = Tag.get() manifest = tag.manifest assert manifest is not None set_tag_expiration_for_manifest(manifest, datetime.utcnow() + timedelta(weeks=1)) updated_tag = Tag.get(id=tag.id) assert updated_tag.lifetime_end_ms is not None
def test_create_temporary_tag_if_necessary(initialized_db): tag = Tag.get() manifest = tag.manifest assert manifest is not None # Ensure no tag is created, since an existing one is present. created = create_temporary_tag_if_necessary(manifest, 60) assert created is None # Mark the tag as deleted. tag.lifetime_end_ms = 1 tag.save() # Now create a temp tag. created = create_temporary_tag_if_necessary(manifest, 60) assert created is not None assert created.hidden assert created.name.startswith("$temp-") assert created.manifest == manifest assert created.lifetime_end_ms is not None assert created.lifetime_end_ms == (created.lifetime_start_ms + 60000) # Try again and ensure it is not created. created = create_temporary_tag_if_necessary(manifest, 30) assert created is None
def test_lookup_manifest_child_tag(initialized_db): repository = create_repository("devtable", "newrepo", None) manifest, manifest_impl = create_manifest_for_testing(repository) # Mark the hidden tag as dead. hidden_tag = Tag.get(manifest=manifest, hidden=True) hidden_tag.lifetime_end_ms = hidden_tag.lifetime_start_ms hidden_tag.save() # Ensure the manifest cannot currently be looked up, as it is not pointed to by an alive tag. assert lookup_manifest(repository, manifest.digest) is None assert lookup_manifest(repository, manifest.digest, allow_dead=True) is not None # Populate a manifest list. list_builder = DockerSchema2ManifestListBuilder() list_builder.add_manifest(manifest_impl, "amd64", "linux") manifest_list = list_builder.build() # Write the manifest list, which should also write the manifests themselves. created_tuple = get_or_create_manifest(repository, manifest_list, storage) assert created_tuple is not None # Since the manifests are not yet referenced by a tag, they cannot be found. assert lookup_manifest(repository, manifest.digest) is None assert lookup_manifest(repository, manifest_list.digest) is None # Unless we ask for "dead" manifests. assert lookup_manifest(repository, manifest.digest, allow_dead=True) is not None assert lookup_manifest(repository, manifest_list.digest, allow_dead=True) is not None
def change_tag_expiration(tag_id, expiration_datetime): """ Changes the expiration of the specified tag to the given expiration datetime. If the expiration datetime is None, then the tag is marked as not expiring. Returns a tuple of the previous expiration timestamp in seconds (if any), and whether the operation succeeded. """ try: tag = Tag.get(id=tag_id) except Tag.DoesNotExist: return (None, False) new_end_ms = None min_expire_sec = convert_to_timedelta( config.app_config.get("LABELED_EXPIRATION_MINIMUM", "1h")) max_expire_sec = convert_to_timedelta( config.app_config.get("LABELED_EXPIRATION_MAXIMUM", "104w")) if expiration_datetime is not None: lifetime_start_ts = int(tag.lifetime_start_ms // 1000) offset = timegm(expiration_datetime.utctimetuple()) - lifetime_start_ts offset = min(max(offset, min_expire_sec.total_seconds()), max_expire_sec.total_seconds()) new_end_ms = tag.lifetime_start_ms + (offset * 1000) if new_end_ms == tag.lifetime_end_ms: return (None, True) return set_tag_end_ms(tag, new_end_ms)
def test_change_tag_expiration(timedelta, expected_timedelta, initialized_db): now = datetime.utcnow() now_ms = timegm(now.utctimetuple()) * 1000 tag = Tag.get() tag.lifetime_start_ms = now_ms tag.save() original_end_ms, okay = change_tag_expiration(tag, now + timedelta) assert okay assert original_end_ms == tag.lifetime_end_ms updated_tag = Tag.get(id=tag.id) offset = expected_timedelta.total_seconds() * 1000 expected_ms = updated_tag.lifetime_start_ms + offset assert updated_tag.lifetime_end_ms == expected_ms original_end_ms, okay = change_tag_expiration(tag, None) assert okay assert original_end_ms == expected_ms updated_tag = Tag.get(id=tag.id) assert updated_tag.lifetime_end_ms is None
def test_lookup_alive_tags_shallow(initialized_db): found = False for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())): tags = lookup_alive_tags_shallow(tag.repository) found = True assert tag in tags assert found # Ensure hidden tags cannot be listed. tag = Tag.get() tag.hidden = True tag.save() tags = lookup_alive_tags_shallow(tag.repository) assert tag not in tags
def test_renew_tag_when_cache_is_expired_and_manifest_is_up_to_date_with_upstream( self, create_repo, proxy_manifest_response): repo_ref = create_repo(self.orgname, self.upstream_repository, self.user) proxy_mock = proxy_manifest_response( UBI8_8_4_DIGEST, UBI8_8_4_MANIFEST_SCHEMA2, DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE) with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest = proxy_model.lookup_manifest_by_digest( repo_ref, UBI8_8_4_DIGEST) assert manifest is not None before_ms = get_epoch_timestamp_ms() - timedelta( hours=24).total_seconds() * 1000 Tag.update( lifetime_start_ms=before_ms, lifetime_end_ms=before_ms + 5, ).where(Tag.manifest == manifest.id).execute() with patch("data.registry_model.registry_proxy_model.Proxy", MagicMock(return_value=proxy_mock)): proxy_model = ProxyModel( self.orgname, self.upstream_repository, self.user, ) manifest = proxy_model.lookup_manifest_by_digest( repo_ref, UBI8_8_4_DIGEST) assert manifest is not None tag = Tag.get(manifest_id=manifest.id) now_ms = get_epoch_timestamp_ms() assert tag.lifetime_end_ms > now_ms
def test_list_alive_tags(initialized_db): found = False for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())): tags = list_alive_tags(tag.repository) assert tag in tags with assert_query_count(1): legacy_images = get_legacy_images_for_tags(tags) for tag in tags: assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id] found = True assert found # Ensure hidden tags cannot be listed. tag = Tag.get() tag.hidden = True tag.save() tags = list_alive_tags(tag.repository) assert tag not in tags
def test_get_or_create_manifest(schema_version, initialized_db): repository = create_repository("devtable", "newrepo", None) expected_labels = { "Foo": "Bar", "Baz": "Meh", } layer_json = json.dumps({ "id": "somelegacyid", "config": { "Labels": expected_labels, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Create a legacy image. find_create_or_link_image("somelegacyid", repository, "devtable", {}, "local_us") # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = "hello world" _, random_digest = _populate_blob(random_data) # Build the manifest. if schema_version == 1: builder = DockerSchema1ManifestBuilder("devtable", "simple", "anothertag") builder.add_layer(random_digest, layer_json) sample_manifest_instance = builder.build(docker_v2_signing_key) elif schema_version == 2: builder = DockerSchema2ManifestBuilder() builder.set_config_digest(config_digest, len(layer_json.encode("utf-8"))) builder.add_layer(random_digest, len(random_data.encode("utf-8"))) sample_manifest_instance = builder.build() assert sample_manifest_instance.layers_compressed_size is not None # Create a new manifest. created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage) created = created_manifest.manifest newly_created = created_manifest.newly_created assert newly_created assert created is not None assert created.media_type.name == sample_manifest_instance.media_type assert created.digest == sample_manifest_instance.digest assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str( ) assert created_manifest.labels_to_apply == expected_labels assert created.config_media_type == sample_manifest_instance.config_media_type assert created.layers_compressed_size == sample_manifest_instance.layers_compressed_size # Lookup the manifest and verify. found = lookup_manifest(repository, created.digest, allow_dead=True) assert found.digest == created.digest assert found.config_media_type == created.config_media_type assert found.layers_compressed_size == created.layers_compressed_size # Verify it has a temporary tag pointing to it. assert Tag.get(manifest=created, hidden=True).lifetime_end_ms # Verify the linked blobs. blob_digests = [ mb.blob.content_checksum for mb in ManifestBlob.select().where(ManifestBlob.manifest == created) ] assert random_digest in blob_digests if schema_version == 2: assert config_digest in blob_digests # Retrieve it again and ensure it is the same manifest. created_manifest2 = get_or_create_manifest(repository, sample_manifest_instance, storage) created2 = created_manifest2.manifest newly_created2 = created_manifest2.newly_created assert not newly_created2 assert created2 == created # Ensure it again has a temporary tag. assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms # Ensure the labels were added. labels = list(list_manifest_labels(created)) assert len(labels) == 2 labels_dict = {label.key: label.value for label in labels} assert labels_dict == expected_labels
def test_get_or_create_manifest(schema_version, initialized_db): repository = create_repository('devtable', 'newrepo', None) expected_labels = { 'Foo': 'Bar', 'Baz': 'Meh', } layer_json = json.dumps({ 'id': 'somelegacyid', 'config': { 'Labels': expected_labels, }, "rootfs": { "type": "layers", "diff_ids": [] }, "history": [ { "created": "2018-04-03T18:37:09.284840891Z", "created_by": "do something", }, ], }) # Create a legacy image. find_create_or_link_image('somelegacyid', repository, 'devtable', {}, 'local_us') # Add a blob containing the config. _, config_digest = _populate_blob(layer_json) # Add a blob of random data. random_data = 'hello world' _, random_digest = _populate_blob(random_data) # Build the manifest. if schema_version == 1: builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag') builder.add_layer(random_digest, layer_json) sample_manifest_instance = builder.build(docker_v2_signing_key) elif schema_version == 2: builder = DockerSchema2ManifestBuilder() builder.set_config_digest(config_digest, len(layer_json)) builder.add_layer(random_digest, len(random_data)) sample_manifest_instance = builder.build() # Create a new manifest. created_manifest = get_or_create_manifest(repository, sample_manifest_instance, storage) created = created_manifest.manifest newly_created = created_manifest.newly_created assert newly_created assert created is not None assert created.media_type.name == sample_manifest_instance.media_type assert created.digest == sample_manifest_instance.digest assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str( ) assert created_manifest.labels_to_apply == expected_labels # Verify it has a temporary tag pointing to it. assert Tag.get(manifest=created, hidden=True).lifetime_end_ms # Verify the legacy image. legacy_image = get_legacy_image_for_manifest(created) assert legacy_image is not None assert legacy_image.storage.content_checksum == random_digest # Verify the linked blobs. blob_digests = [ mb.blob.content_checksum for mb in ManifestBlob.select().where(ManifestBlob.manifest == created) ] assert random_digest in blob_digests if schema_version == 2: assert config_digest in blob_digests # Retrieve it again and ensure it is the same manifest. created_manifest2 = get_or_create_manifest(repository, sample_manifest_instance, storage) created2 = created_manifest2.manifest newly_created2 = created_manifest2.newly_created assert not newly_created2 assert created2 == created # Ensure it again has a temporary tag. assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms # Ensure the labels were added. labels = list(list_manifest_labels(created)) assert len(labels) == 2 labels_dict = {label.key: label.value for label in labels} assert labels_dict == expected_labels