def pull_config(self, image, arch="amd64"):
"""
Nydusify converts oci to nydus format and push the nydus image manifest to registry,
which belongs to a manifest index.
"""
skopeo = utils.Skopeo()
nydus_manifest, digest = skopeo.inspect(image,
tls_verify=False,
image_arch=arch)
import requests
# Currently, we can handle auth
# OCI distribution spec: /v2/<name>/manifests/<digest>
reader = requests.get(
f"http://{self.registry_url}/v2/{self.original_repo}/manifests/{digest}",
stream=True,
)
manifest = json.load(reader.raw)
config_digest = manifest["config"]["digest"]
reader = requests.get(
f"http://{self.registry_url}/v2/{self.original_repo}/blobs/{config_digest}",
stream=True,
)
config = json.load(reader.raw)
return config
def find_nydus_image(self, image, arch):
skopeo = utils.Skopeo()
nydus_manifest, digest = skopeo.inspect(
image,
tls_verify=False,
image_arch=arch,
features="nydus.remoteimage.v1")
assert nydus_manifest is not None
def get_build_cache_records(self, ref):
skopeo = utils.Skopeo()
build_cache_records, _ = skopeo.inspect(ref, tls_verify=False)
c = json.dumps(build_cache_records, indent=4, sort_keys=False)
logging.info("build cache: %s", c)
records = build_cache_records["layers"]
return records
def extract_converted_layers_names(self, arch="amd64"):
skopeo = utils.Skopeo()
manifest, _ = skopeo.inspect(
self.target_ref,
tls_verify=False,
features="nydus.remoteimage.v1",
image_arch=arch,
)
layers = [l["digest"] for l in manifest["layers"]]
layers.reverse()
return layers
def extract_source_layers_names_and_download(self, arch="amd64"):
skopeo = utils.Skopeo()
manifest, digest = skopeo.inspect(self.__source, image_arch=arch)
layers = [l["digest"] for l in manifest["layers"]]
# trimmed_layers = [os.path.join(self.work_dir, self.__source, l) for l in layers]
# trimmed_layers.reverse()
layers.reverse()
skopeo.copy_to_local(
self.__source,
layers,
os.path.join(self.work_dir, self.__source),
resource_digest=digest,
)
return layers, os.path.join(self.work_dir, self.__source)
def pull_bootstrap(self, downloaded_dir, bootstrap_name, arch="amd64"):
"""
Nydusify converts oci to nydus format and push the nydus image manifest to registry,
which belongs to a manifest index.
"""
skopeo = utils.Skopeo()
nydus_manifest, _ = skopeo.inspect(
self.target_ref,
tls_verify=False,
features="nydus.remoteimage.v1",
image_arch=arch,
)
layers = nydus_manifest["layers"]
for l in layers:
if l["mediaType"] == "application/vnd.docker.image.rootfs.diff.tar.gzip":
bootstrap_digest = l["digest"]
import requests
# Currently, we can not handle auth
# OCI distribution spec: /v2/<name>/blobs/<digest>
os.makedirs(downloaded_dir, exist_ok=True)
reader = requests.get(
f"http://{self.registry_url}/v2/{self.anchor.registry_namespace}/{self.original_repo}/blobs/{bootstrap_digest}",
stream=True,
)
with utils.pushd(downloaded_dir):
with open("image.gzip", "wb") as w:
shutil.copyfileobj(reader.raw, w)
with tarfile.open("image.gzip", "r:gz") as tar_gz:
tar_gz.extractall()
os.rename("image/image.boot", bootstrap_name)
os.remove("image.gzip")
return os.path.join(downloaded_dir, bootstrap_name)
def test_cross_platform_multiplatform(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
source,
arch,
enable_multiplatform,
local_registry,
nydusify_converter,
):
"""
- copy the entire repo from source registry to target registry
- One image coresponds to manifest list while the other one to single manifest
- Use cloned source rather than the one from original registry
- Push the converted images to the original source
- Also test multiplatform here
- ? Seems with flag --multiplatform to nydusify, it still just push single manifest
- converted manifest index has one more image than origin.
"""
# Copy the entire repo for multiplatform
skopeo = utils.Skopeo()
source_name_tagged = posixpath.basename(source)
target_image = f"localhost:5000/{source_name_tagged}"
cloned_source = f"localhost:5000/{source_name_tagged}"
skopeo.copy_all_to_registry(source, target_image)
origin_manifest_index = skopeo.manifest_list(cloned_source)
utils.Skopeo.pretty_print(origin_manifest_index)
converter = Nydusify(nydus_anchor)
converter.docker_v2(
).build_cache_ref("localhost:5000/build_cache:000").platform(
f"linux/{arch}").enable_multiplatfrom(enable_multiplatform).convert(
cloned_source, target_ref=target_image)
# TODO: configure registry backend from `local_registry` rather than anchor
rafs_conf.set_rafs_backend(Backend.REGISTRY,
repo=posixpath.basename(source).split(":")[0])
rafs_conf.enable_fs_prefetch()
rafs_conf.enable_rafs_blobcache()
pulled_bootstrap = converter.pull_bootstrap(
tempfile.TemporaryDirectory(dir=nydus_anchor.workspace,
suffix="bootstrap").name,
"pulled_bootstrap",
arch,
)
# Skopeo does not support media type: "application/vnd.oci.image.layer.nydus.blob.v1",
# So can't download build cache like a oci image.
layers, base = converter.extract_source_layers_names_and_download(
arch=arch)
nydus_anchor.mount_overlayfs(layers, base)
converted_layers = converter.extract_converted_layers_names(arch=arch)
converted_layers.sort()
converted_manifest_index = skopeo.manifest_list(cloned_source)
utils.Skopeo.pretty_print(converted_manifest_index)
assert (len(converted_manifest_index["manifests"]) -
len(origin_manifest_index["manifests"]) == 1)
# `inspect` will succeed if image to arch can be found.
skopeo.inspect(target_image, image_arch=arch)
converter.find_nydus_image(target_image, arch)
target_image_config = converter.pull_config(target_image, arch=arch)
assert target_image_config["architecture"] == arch
records = converter.get_build_cache_records(
"localhost:5000/build_cache:000")
assert len(records) != 0
cached_layers = [c["digest"] for c in records]
cached_layers.sort()
# > assert cached_layers == converted_layers
# E AssertionError: assert None == ['sha256:3f18...af3234b4c257']
# E +None
# E -['sha256:3f18b27a912188108c8590684206bd9da7d81bbfd0e8325f3ef0af3234b4c257']
for r in converted_layers:
assert r in cached_layers
# Use `nydus-image inspect` to compare blob table in bootstrap and manifest
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_anchor.overlayfs)
# No need to locate where bootstrap is as we can directly pull it from registry
rafs = RafsMount(nydus_anchor, None, rafs_conf)
rafs.thread_num(6).bootstrap(pulled_bootstrap).prefetch_files("/").mount()
assert workload_gen.verify_entire_fs()
workload_gen.setup_workload_generator()
workload_gen.torture_read(8, 12, verify=True)
workload_gen.finish_torture_read()