def test_does_not_bump_tag_expiration_when_manifest_is_cached_and_upstream_errors(
self, create_repo, proxy_manifest_response):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
proxy_mock = proxy_manifest_response(
UBI8_8_4_DIGEST, UBI8_8_4_MANIFEST_SCHEMA2,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_8_4_DIGEST)
assert manifest is not None
first_tag = Tag.get(manifest_id=manifest.id)
proxy_mock = proxy_manifest_response("not-existing-ref", "", "")
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_8_4_DIGEST)
assert manifest is not None
tag = Tag.get(manifest_id=manifest.id)
assert tag.id == first_tag.id
assert tag.lifetime_end_ms == first_tag.lifetime_end_ms
def test_set_tag_expiration_for_manifest(initialized_db):
tag = Tag.get()
manifest = tag.manifest
assert manifest is not None
set_tag_expiration_for_manifest(manifest, datetime.utcnow() + timedelta(weeks=1))
updated_tag = Tag.get(id=tag.id)
assert updated_tag.lifetime_end_ms is not None
def test_create_temporary_tag_if_necessary(initialized_db):
tag = Tag.get()
manifest = tag.manifest
assert manifest is not None
# Ensure no tag is created, since an existing one is present.
created = create_temporary_tag_if_necessary(manifest, 60)
assert created is None
# Mark the tag as deleted.
tag.lifetime_end_ms = 1
tag.save()
# Now create a temp tag.
created = create_temporary_tag_if_necessary(manifest, 60)
assert created is not None
assert created.hidden
assert created.name.startswith("$temp-")
assert created.manifest == manifest
assert created.lifetime_end_ms is not None
assert created.lifetime_end_ms == (created.lifetime_start_ms + 60000)
# Try again and ensure it is not created.
created = create_temporary_tag_if_necessary(manifest, 30)
assert created is None
def test_lookup_manifest_child_tag(initialized_db):
repository = create_repository("devtable", "newrepo", None)
manifest, manifest_impl = create_manifest_for_testing(repository)
# Mark the hidden tag as dead.
hidden_tag = Tag.get(manifest=manifest, hidden=True)
hidden_tag.lifetime_end_ms = hidden_tag.lifetime_start_ms
hidden_tag.save()
# Ensure the manifest cannot currently be looked up, as it is not pointed to by an alive tag.
assert lookup_manifest(repository, manifest.digest) is None
assert lookup_manifest(repository, manifest.digest,
allow_dead=True) is not None
# Populate a manifest list.
list_builder = DockerSchema2ManifestListBuilder()
list_builder.add_manifest(manifest_impl, "amd64", "linux")
manifest_list = list_builder.build()
# Write the manifest list, which should also write the manifests themselves.
created_tuple = get_or_create_manifest(repository, manifest_list, storage)
assert created_tuple is not None
# Since the manifests are not yet referenced by a tag, they cannot be found.
assert lookup_manifest(repository, manifest.digest) is None
assert lookup_manifest(repository, manifest_list.digest) is None
# Unless we ask for "dead" manifests.
assert lookup_manifest(repository, manifest.digest,
allow_dead=True) is not None
assert lookup_manifest(repository, manifest_list.digest,
allow_dead=True) is not None
def change_tag_expiration(tag_id, expiration_datetime):
"""
Changes the expiration of the specified tag to the given expiration datetime.
If the expiration datetime is None, then the tag is marked as not expiring. Returns a tuple of
the previous expiration timestamp in seconds (if any), and whether the operation succeeded.
"""
try:
tag = Tag.get(id=tag_id)
except Tag.DoesNotExist:
return (None, False)
new_end_ms = None
min_expire_sec = convert_to_timedelta(
config.app_config.get("LABELED_EXPIRATION_MINIMUM", "1h"))
max_expire_sec = convert_to_timedelta(
config.app_config.get("LABELED_EXPIRATION_MAXIMUM", "104w"))
if expiration_datetime is not None:
lifetime_start_ts = int(tag.lifetime_start_ms // 1000)
offset = timegm(expiration_datetime.utctimetuple()) - lifetime_start_ts
offset = min(max(offset, min_expire_sec.total_seconds()),
max_expire_sec.total_seconds())
new_end_ms = tag.lifetime_start_ms + (offset * 1000)
if new_end_ms == tag.lifetime_end_ms:
return (None, True)
return set_tag_end_ms(tag, new_end_ms)
def test_change_tag_expiration(timedelta, expected_timedelta, initialized_db):
now = datetime.utcnow()
now_ms = timegm(now.utctimetuple()) * 1000
tag = Tag.get()
tag.lifetime_start_ms = now_ms
tag.save()
original_end_ms, okay = change_tag_expiration(tag, now + timedelta)
assert okay
assert original_end_ms == tag.lifetime_end_ms
updated_tag = Tag.get(id=tag.id)
offset = expected_timedelta.total_seconds() * 1000
expected_ms = updated_tag.lifetime_start_ms + offset
assert updated_tag.lifetime_end_ms == expected_ms
original_end_ms, okay = change_tag_expiration(tag, None)
assert okay
assert original_end_ms == expected_ms
updated_tag = Tag.get(id=tag.id)
assert updated_tag.lifetime_end_ms is None
def test_lookup_alive_tags_shallow(initialized_db):
found = False
for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
tags = lookup_alive_tags_shallow(tag.repository)
found = True
assert tag in tags
assert found
# Ensure hidden tags cannot be listed.
tag = Tag.get()
tag.hidden = True
tag.save()
tags = lookup_alive_tags_shallow(tag.repository)
assert tag not in tags
def test_renew_tag_when_cache_is_expired_and_manifest_is_up_to_date_with_upstream(
self, create_repo, proxy_manifest_response):
repo_ref = create_repo(self.orgname, self.upstream_repository,
self.user)
proxy_mock = proxy_manifest_response(
UBI8_8_4_DIGEST, UBI8_8_4_MANIFEST_SCHEMA2,
DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE)
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_8_4_DIGEST)
assert manifest is not None
before_ms = get_epoch_timestamp_ms() - timedelta(
hours=24).total_seconds() * 1000
Tag.update(
lifetime_start_ms=before_ms,
lifetime_end_ms=before_ms + 5,
).where(Tag.manifest == manifest.id).execute()
with patch("data.registry_model.registry_proxy_model.Proxy",
MagicMock(return_value=proxy_mock)):
proxy_model = ProxyModel(
self.orgname,
self.upstream_repository,
self.user,
)
manifest = proxy_model.lookup_manifest_by_digest(
repo_ref, UBI8_8_4_DIGEST)
assert manifest is not None
tag = Tag.get(manifest_id=manifest.id)
now_ms = get_epoch_timestamp_ms()
assert tag.lifetime_end_ms > now_ms
def test_list_alive_tags(initialized_db):
found = False
for tag in filter_to_visible_tags(filter_to_alive_tags(Tag.select())):
tags = list_alive_tags(tag.repository)
assert tag in tags
with assert_query_count(1):
legacy_images = get_legacy_images_for_tags(tags)
for tag in tags:
assert ManifestLegacyImage.get(manifest=tag.manifest).image == legacy_images[tag.id]
found = True
assert found
# Ensure hidden tags cannot be listed.
tag = Tag.get()
tag.hidden = True
tag.save()
tags = list_alive_tags(tag.repository)
assert tag not in tags
def test_get_or_create_manifest(schema_version, initialized_db):
repository = create_repository("devtable", "newrepo", None)
expected_labels = {
"Foo": "Bar",
"Baz": "Meh",
}
layer_json = json.dumps({
"id":
"somelegacyid",
"config": {
"Labels": expected_labels,
},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
# Create a legacy image.
find_create_or_link_image("somelegacyid", repository, "devtable", {},
"local_us")
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
# Add a blob of random data.
random_data = "hello world"
_, random_digest = _populate_blob(random_data)
# Build the manifest.
if schema_version == 1:
builder = DockerSchema1ManifestBuilder("devtable", "simple",
"anothertag")
builder.add_layer(random_digest, layer_json)
sample_manifest_instance = builder.build(docker_v2_signing_key)
elif schema_version == 2:
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest,
len(layer_json.encode("utf-8")))
builder.add_layer(random_digest, len(random_data.encode("utf-8")))
sample_manifest_instance = builder.build()
assert sample_manifest_instance.layers_compressed_size is not None
# Create a new manifest.
created_manifest = get_or_create_manifest(repository,
sample_manifest_instance,
storage)
created = created_manifest.manifest
newly_created = created_manifest.newly_created
assert newly_created
assert created is not None
assert created.media_type.name == sample_manifest_instance.media_type
assert created.digest == sample_manifest_instance.digest
assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str(
)
assert created_manifest.labels_to_apply == expected_labels
assert created.config_media_type == sample_manifest_instance.config_media_type
assert created.layers_compressed_size == sample_manifest_instance.layers_compressed_size
# Lookup the manifest and verify.
found = lookup_manifest(repository, created.digest, allow_dead=True)
assert found.digest == created.digest
assert found.config_media_type == created.config_media_type
assert found.layers_compressed_size == created.layers_compressed_size
# Verify it has a temporary tag pointing to it.
assert Tag.get(manifest=created, hidden=True).lifetime_end_ms
# Verify the linked blobs.
blob_digests = [
mb.blob.content_checksum
for mb in ManifestBlob.select().where(ManifestBlob.manifest == created)
]
assert random_digest in blob_digests
if schema_version == 2:
assert config_digest in blob_digests
# Retrieve it again and ensure it is the same manifest.
created_manifest2 = get_or_create_manifest(repository,
sample_manifest_instance,
storage)
created2 = created_manifest2.manifest
newly_created2 = created_manifest2.newly_created
assert not newly_created2
assert created2 == created
# Ensure it again has a temporary tag.
assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms
# Ensure the labels were added.
labels = list(list_manifest_labels(created))
assert len(labels) == 2
labels_dict = {label.key: label.value for label in labels}
assert labels_dict == expected_labels
def test_get_or_create_manifest(schema_version, initialized_db):
repository = create_repository('devtable', 'newrepo', None)
expected_labels = {
'Foo': 'Bar',
'Baz': 'Meh',
}
layer_json = json.dumps({
'id':
'somelegacyid',
'config': {
'Labels': expected_labels,
},
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [
{
"created": "2018-04-03T18:37:09.284840891Z",
"created_by": "do something",
},
],
})
# Create a legacy image.
find_create_or_link_image('somelegacyid', repository, 'devtable', {},
'local_us')
# Add a blob containing the config.
_, config_digest = _populate_blob(layer_json)
# Add a blob of random data.
random_data = 'hello world'
_, random_digest = _populate_blob(random_data)
# Build the manifest.
if schema_version == 1:
builder = DockerSchema1ManifestBuilder('devtable', 'simple',
'anothertag')
builder.add_layer(random_digest, layer_json)
sample_manifest_instance = builder.build(docker_v2_signing_key)
elif schema_version == 2:
builder = DockerSchema2ManifestBuilder()
builder.set_config_digest(config_digest, len(layer_json))
builder.add_layer(random_digest, len(random_data))
sample_manifest_instance = builder.build()
# Create a new manifest.
created_manifest = get_or_create_manifest(repository,
sample_manifest_instance,
storage)
created = created_manifest.manifest
newly_created = created_manifest.newly_created
assert newly_created
assert created is not None
assert created.media_type.name == sample_manifest_instance.media_type
assert created.digest == sample_manifest_instance.digest
assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str(
)
assert created_manifest.labels_to_apply == expected_labels
# Verify it has a temporary tag pointing to it.
assert Tag.get(manifest=created, hidden=True).lifetime_end_ms
# Verify the legacy image.
legacy_image = get_legacy_image_for_manifest(created)
assert legacy_image is not None
assert legacy_image.storage.content_checksum == random_digest
# Verify the linked blobs.
blob_digests = [
mb.blob.content_checksum
for mb in ManifestBlob.select().where(ManifestBlob.manifest == created)
]
assert random_digest in blob_digests
if schema_version == 2:
assert config_digest in blob_digests
# Retrieve it again and ensure it is the same manifest.
created_manifest2 = get_or_create_manifest(repository,
sample_manifest_instance,
storage)
created2 = created_manifest2.manifest
newly_created2 = created_manifest2.newly_created
assert not newly_created2
assert created2 == created
# Ensure it again has a temporary tag.
assert Tag.get(manifest=created2, hidden=True).lifetime_end_ms
# Ensure the labels were added.
labels = list(list_manifest_labels(created))
assert len(labels) == 2
labels_dict = {label.key: label.value for label in labels}
assert labels_dict == expected_labels
