def test_images_shared_cas(default_tag_policy, initialized_db):
"""
A repository, each two tags, pointing to the same image, which has image storage with the same
*CAS path*, but *distinct records*.
Deleting the first tag should delete the first image, and its storage, but not the file in
storage, as it shares its CAS path.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create two image storage records with the same content checksum.
content = "hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
is2 = database.ImageStorage.create(content_checksum=digest, uploading=False)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=is1)
database.ImageStoragePlacement.create(location=location, storage=is2)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
# Create two images in the repository, and two tags, each pointing to one of the storages.
first_image = Image.create(
docker_image_id="i1", repository=repository, storage=is1, ancestors="/"
)
second_image = Image.create(
docker_image_id="i2", repository=repository, storage=is2, ancestors="/"
)
store_tag_manifest(
repository.namespace_user.username,
repository.name,
"first",
first_image.docker_image_id,
)
store_tag_manifest(
repository.namespace_user.username,
repository.name,
"second",
second_image.docker_image_id,
)
assert_not_deleted(repository, "i1", "i2")
# Delete the first tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
assert_not_deleted(repository, "i2")
# Ensure the CAS path still exists.
assert storage.exists({preferred}, storage.blob_path(digest))
def _populate_blob(content):
digest = str(sha256_digest(content))
location = ImageStorageLocation.get(name="local_us")
blob = store_blob_record_and_temp_link("devtable", "newrepo", digest,
location, len(content), 120)
storage.put_content(["local_us"], get_layer_path(blob), content)
return blob, digest
def _populate_blob(repo, content):
digest = str(sha256_digest(content))
location = ImageStorageLocation.get(name="local_us")
storage.put_content(["local_us"], storage.blob_path(digest), "somecontent")
blob = model.blob.store_blob_record_and_temp_link_in_repo(
repo, digest, location, len(content), 120)
return blob, digest
def _populate_blob(repo, content):
assert isinstance(content, bytes)
digest = sha256_digest(content)
location = ImageStorageLocation.get(name="local_us")
storage.put_content(["local_us"], storage.blob_path(digest), content)
blob = model.blob.store_blob_record_and_temp_link_in_repo(
repo, digest, location, len(content), 120)
return blob, digest
def test_image_with_cas(default_tag_policy, initialized_db):
"""
A repository with a tag pointing to an image backed by CAS.
Deleting and GCing the tag should result in the storage and its CAS data being removed.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create an image storage record under CAS.
content = b"hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
image_storage = database.ImageStorage.create(content_checksum=digest)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=image_storage)
# Temp link so its available.
model.blob.store_blob_record_and_temp_link_in_repo(
repository, digest, location, len(content), 120
)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
# Store a manifest pointing to that path.
builder = DockerSchema1ManifestBuilder(
repository.namespace_user.username, repository.name, "first"
)
builder.insert_layer(
digest,
json.dumps(
{
"id": "i1",
}
),
)
# Store the manifest.
manifest = builder.build(docker_v2_signing_key)
repo_ref = RepositoryReference.for_repo_obj(repository)
registry_model.create_manifest_and_retarget_tag(
repo_ref, manifest, "first", storage, raise_on_error=True
)
# Delete the temp reference.
_delete_temp_links(repository)
# Delete the tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
# Ensure the CAS path is gone.
assert not storage.exists({preferred}, storage.blob_path(digest))
def populate_storage_for_gc():
"""
Populate FakeStorage with dummy data for each ImageStorage row.
"""
preferred = storage.preferred_locations[0]
for storage_row in ImageStorage.select():
content = b"hello world"
storage.put_content({preferred}, storage.blob_path(storage_row.content_checksum), content)
assert storage.exists({preferred}, storage.blob_path(storage_row.content_checksum))
yield
def get_blob(layer):
content = Bytes.for_string_or_unicode(layer).as_encoded_str()
digest = str(sha256_digest(content))
blob = store_blob_record_and_temp_link(
self.orgname,
self.upstream_repository,
digest,
ImageStorageLocation.get(name="local_us"),
len(content),
120,
)
storage.put_content(["local_us"], get_layer_path(blob), content)
return blob, digest
def test_images_shared_cas_with_new_blob_table(default_tag_policy,
initialized_db):
""" A repository with a tag and image that shares its CAS path with a record in the new Blob
table. Deleting the first tag should delete the first image, and its storage, but not the
file in storage, as it shares its CAS path with the blob row.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create two image storage records with the same content checksum.
content = "hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
media_type = database.MediaType.get(name="text/plain")
is1 = database.ImageStorage.create(content_checksum=digest,
uploading=False)
database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=is1)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
# Create the image in the repository, and the tag.
first_image = Image.create(docker_image_id="i1",
repository=repository,
storage=is1,
ancestors="/")
store_tag_manifest(
repository.namespace_user.username,
repository.name,
"first",
first_image.docker_image_id,
)
assert_not_deleted(repository, "i1")
# Delete the tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
# Ensure the CAS path still exists, as it is referenced by the Blob table
assert storage.exists({preferred}, storage.blob_path(digest))
def test_image_with_cas(default_tag_policy, initialized_db):
""" A repository with a tag pointing to an image backed by CAS. Deleting and GCing the tag
should result in the storage and its CAS data being removed.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create an image storage record under CAS.
content = "hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
image_storage = database.ImageStorage.create(content_checksum=digest,
uploading=False)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location,
storage=image_storage)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
# Create the image and the tag.
first_image = Image.create(docker_image_id="i1",
repository=repository,
storage=image_storage,
ancestors="/")
store_tag_manifest(
repository.namespace_user.username,
repository.name,
"first",
first_image.docker_image_id,
)
assert_not_deleted(repository, "i1")
# Delete the tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
# Ensure the CAS path is gone.
assert not storage.exists({preferred}, storage.blob_path(digest))
def test_analyze_layer_nodirectdownload_success(self):
"""
Tests analyzing a layer when direct download is disabled.
"""
# Disable direct download in fake storage.
storage.put_content(["local_us"], "supports_direct_download", "false")
try:
app.register_blueprint(v2_bp, url_prefix="/v2")
except:
# Already registered.
pass
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER,
SIMPLE_REPO,
"latest",
include_storage=True)
self.assertFalse(layer.security_indexed)
self.assertEquals(-1, layer.security_indexed_engine)
# Ensure that the download is a registry+JWT download.
uri, auth_header = self.api._get_image_url_and_auth(layer)
self.assertIsNotNone(uri)
self.assertIsNotNone(auth_header)
# Ensure the download doesn't work without the header.
rv = self.app.head(uri)
self.assertEquals(rv.status_code, 401)
# Ensure the download works with the header. Note we use a HEAD here, as GET causes DB
# access which messes with the test runner's rollback.
rv = self.app.head(uri, headers=[("authorization", auth_header)])
self.assertEquals(rv.status_code, 200)
# Ensure the code works when called via analyze.
with fake_security_scanner() as security_scanner:
analyzer = LayerAnalyzer(app.config, self.api)
analyzer.analyze_recursively(layer)
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO,
"latest")
self.assertAnalyzed(layer, security_scanner, True, 1)
def test_export_logs_failure(initialized_db):
# Make all uploads fail.
test_storage.put_content("local_us", "except_upload", b"true")
repo = model.repository.get_repository("devtable", "simple")
user = model.user.get_user("devtable")
worker = ExportActionLogsWorker(None)
called = [{}]
@urlmatch(netloc=r"testcallback")
def handle_request(url, request):
called[0] = json.loads(request.body)
return {"status_code": 200, "content": "{}"}
def format_date(datetime):
return datetime.strftime("%m/%d/%Y")
now = datetime.now()
with HTTMock(handle_request):
with pytest.raises(IOError):
worker._process_queue_item(
{
"export_id": "someid",
"repository_id": repo.id,
"namespace_id": repo.namespace_user.id,
"namespace_name": "devtable",
"repository_name": "simple",
"start_time": format_date(now + timedelta(days=-10)),
"end_time": format_date(now + timedelta(days=10)),
"callback_url": "http://testcallback/",
"callback_email": None,
},
test_storage,
)
test_storage.remove("local_us", "except_upload")
assert called[0]
assert called[0]["export_id"] == "someid"
assert called[0]["status"] == "failed"
def test_export_logs_failure(initialized_db):
# Make all uploads fail.
test_storage.put_content('local_us', 'except_upload', 'true')
repo = model.repository.get_repository('devtable', 'simple')
user = model.user.get_user('devtable')
worker = ExportActionLogsWorker(None)
called = [{}]
@urlmatch(netloc=r'testcallback')
def handle_request(url, request):
called[0] = json.loads(request.body)
return {'status_code': 200, 'content': '{}'}
def format_date(datetime):
return datetime.strftime("%m/%d/%Y")
now = datetime.now()
with HTTMock(handle_request):
with pytest.raises(IOError):
worker._process_queue_item(
{
'export_id': 'someid',
'repository_id': repo.id,
'namespace_id': repo.namespace_user.id,
'namespace_name': 'devtable',
'repository_name': 'simple',
'start_time': format_date(now + timedelta(days=-10)),
'end_time': format_date(now + timedelta(days=10)),
'callback_url': 'http://testcallback/',
'callback_email': None,
}, test_storage)
test_storage.remove('local_us', 'except_upload')
assert called[0]
assert called[0][u'export_id'] == 'someid'
assert called[0][u'status'] == 'failed'
def setUp(self):
# Enable direct download in fake storage.
storage.put_content(['local_us'], 'supports_direct_download', 'true')
# Have fake storage say all files exist for the duration of the test.
storage.put_content(['local_us'], 'all_files_exist', 'true')
# Setup the database with fake storage.
setup_database_for_testing(self)
self.app = app.test_client()
self.ctx = app.test_request_context()
self.ctx.__enter__()
instance_keys = InstanceKeys(app)
self.api = SecurityScannerAPI(app.config,
storage,
app.config['SERVER_HOSTNAME'],
app.config['HTTPCLIENT'],
uri_creator=get_blob_download_uri_getter(
app.test_request_context('/'),
url_scheme_and_hostname),
instance_keys=instance_keys)
def test_images_shared_cas(default_tag_policy, initialized_db):
"""
A repository, each two tags, pointing to the same image, which has image storage with the same
*CAS path*, but *distinct records*.
Deleting the first tag should delete the first image, and its storage, but not the file in
storage, as it shares its CAS path.
"""
with assert_gc_integrity(expect_storage_removed=True):
repository = create_repository()
# Create two image storage records with the same content checksum.
content = b"hello world"
digest = "sha256:" + hashlib.sha256(content).hexdigest()
preferred = storage.preferred_locations[0]
storage.put_content({preferred}, storage.blob_path(digest), content)
is1 = database.ImageStorage.create(content_checksum=digest)
is2 = database.ImageStorage.create(content_checksum=digest)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=is1)
database.ImageStoragePlacement.create(location=location, storage=is2)
# Temp link so its available.
model.blob.store_blob_record_and_temp_link_in_repo(
repository, digest, location, len(content), 120)
# Ensure the CAS path exists.
assert storage.exists({preferred}, storage.blob_path(digest))
repo_ref = RepositoryReference.for_repo_obj(repository)
# Store a manifest pointing to that path as `first`.
builder = DockerSchema1ManifestBuilder(
repository.namespace_user.username, repository.name, "first")
builder.insert_layer(
digest,
json.dumps({
"id": "i1",
}),
)
manifest = builder.build(docker_v2_signing_key)
registry_model.create_manifest_and_retarget_tag(repo_ref,
manifest,
"first",
storage,
raise_on_error=True)
tag_ref = registry_model.get_repo_tag(repo_ref, "first")
manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
registry_model.populate_legacy_images_for_testing(
manifest_ref, storage)
# Store another as `second`.
builder = DockerSchema1ManifestBuilder(
repository.namespace_user.username, repository.name, "second")
builder.insert_layer(
digest,
json.dumps({
"id": "i2",
}),
)
manifest = builder.build(docker_v2_signing_key)
created, _ = registry_model.create_manifest_and_retarget_tag(
repo_ref, manifest, "second", storage, raise_on_error=True)
tag_ref = registry_model.get_repo_tag(repo_ref, "second")
manifest_ref = registry_model.get_manifest_for_tag(tag_ref)
registry_model.populate_legacy_images_for_testing(
manifest_ref, storage)
# Manually retarget the second manifest's blob to the second row.
try:
second_blob = ManifestBlob.get(manifest=created._db_id, blob=is1)
second_blob.blob = is2
second_blob.save()
except ManifestBlob.DoesNotExist:
second_blob = ManifestBlob.get(manifest=created._db_id, blob=is2)
second_blob.blob = is1
second_blob.save()
# Delete the temp reference.
_delete_temp_links(repository)
# Ensure the legacy images exist.
assert_not_deleted(repository, "i1", "i2")
# Delete the first tag.
delete_tag(repository, "first")
assert_deleted(repository, "i1")
assert_not_deleted(repository, "i2")
# Ensure the CAS path still exists.
assert storage.exists({preferred}, storage.blob_path(digest))
def set_supports_direct_download(enabled):
storage.put_content(["local_us"], "supports_direct_download",
b"true" if enabled else b"false")
return "OK"
def set_supports_direct_download(enabled): storage.put_content(['local_us'], 'supports_direct_download', 'true' if enabled else 'false') return 'OK'
def __create_subtree(with_storage, repo, structure, creator_username, parent,
tag_map):
num_nodes, subtrees, last_node_tags = structure
# create the nodes
for model_num in range(num_nodes):
image_num = next(global_image_num)
docker_image_id = __gen_image_id(repo, image_num)
logger.debug("new docker id: %s", docker_image_id)
checksum = __gen_checksum(docker_image_id)
new_image = model.image.find_create_or_link_image(
docker_image_id, repo, None, {}, "local_us")
new_image.storage.uuid = __gen_image_uuid(repo, image_num)
new_image.storage.uploading = False
new_image.storage.save()
# Write out a fake torrentinfo
model.storage.save_torrent_info(new_image.storage, 1, "deadbeef")
# Write some data for the storage.
if with_storage or os.environ.get("WRITE_STORAGE_FILES"):
storage_paths = StoragePaths()
paths = [storage_paths.v1_image_layer_path]
for path_builder in paths:
path = path_builder(new_image.storage.uuid)
store.put_content("local_us", path, checksum)
new_image.security_indexed = False
new_image.security_indexed_engine = -1
new_image.save()
creation_time = REFERENCE_DATE + timedelta(
weeks=image_num) + timedelta(days=model_num)
command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
command = json.dumps(command_list) if command_list else None
v1_metadata = {
"id": docker_image_id,
}
if parent is not None:
v1_metadata["parent"] = parent.docker_image_id
new_image = model.image.set_image_metadata(
docker_image_id,
repo.namespace_user.username,
repo.name,
str(creation_time),
"no comment",
command,
json.dumps(v1_metadata),
parent,
)
new_image.storage.content_checksum = checksum
new_image.storage.save()
compressed_size = random.randrange(1, 1024 * 1024 * 1024)
model.storage.set_image_storage_metadata(
docker_image_id,
repo.namespace_user.username,
repo.name,
compressed_size,
int(compressed_size * 1.4),
)
parent = new_image
if last_node_tags:
if not isinstance(last_node_tags, list):
last_node_tags = [last_node_tags]
repo_ref = registry_model.lookup_repository(
repo.namespace_user.username, repo.name)
for tag_name in last_node_tags:
adjusted_tag_name = tag_name
now_ms = None
if tag_name[0] == "#":
adjusted_tag_name = tag_name[1:]
now_ms = get_epoch_timestamp_ms() - 1000
new_tag = model.tag.create_or_update_tag(
repo.namespace_user.username,
repo.name,
adjusted_tag_name,
new_image.docker_image_id,
now_ms=now_ms,
)
derived = model.image.find_or_create_derived_storage(
new_tag, "squash", "local_us")
model.storage.find_or_create_storage_signature(derived, "gpg2")
tag = pre_oci_model.get_repo_tag(repo_ref, adjusted_tag_name)
assert tag._db_id == new_tag.id
assert pre_oci_model.backfill_manifest_for_tag(tag)
tag_map[tag_name] = new_tag
for subtree in subtrees:
__create_subtree(with_storage, repo, subtree, creator_username,
new_image, tag_map)
