def test_records_readahead(nydus_anchor, nydus_image): nydus_image.set_backend(Backend.BACKEND_PROXY).create_image() rafs_conf = RafsConf(nydus_anchor, nydus_image) rafs_conf.enable_records_readahead(interval=1).set_rafs_backend( Backend.LOCALFS, image=nydus_image) rafs = RafsMount(nydus_anchor, nydus_image, rafs_conf) rafs.mount() wg = WorkloadGen(nydus_anchor.mount_point, nydus_image.rootfs()) # TODO: Run several parallel read workers against the mount_point wg.setup_workload_generator() wg.torture_read(8, 5) wg.finish_torture_read() rafs.umount() utils.clean_pagecache() rafs.mount() wg.torture_read(8, 5) wg.finish_torture_read()
def test_blob_prefetch(nydus_anchor: NydusAnchor, nydus_scratch_image: RafsImage, readahead_policy): """ description: For rafs, there are two types of prefetching. 1. Prefetch files from fs-layer, which means each file is prefetched one by one. 2. Prefetch directly from backend/blob layer, which means a range will be fetched from blob """ # Try to delete any access log since if it present, bootstrap blob prefetch won't work. utils.execute("rm -rf *.access", shell=True) dist = Distributor(nydus_scratch_image.rootfs(), 8, 2) dist.generate_tree() dist.put_directories(20) dist.put_multiple_files(100, Size(64, Unit.KB)) dist.put_symlinks(30) dist.put_hardlinks(20) dist.put_multiple_files(40, Size(64, Unit.KB)) utils.clean_pagecache() hint_files = dist.files[-40:] hint_files.extend(dist.symlinks[-20:]) hint_files = [os.path.join("/", p) for p in hint_files] hint_files = "\n".join(hint_files) nydus_scratch_image.set_backend(Backend.LOCALFS).create_image( readahead_policy=readahead_policy, readahead_files=hint_files.encode(), ) rafs_conf = RafsConf(nydus_anchor, nydus_scratch_image) rafs_conf.set_rafs_backend(Backend.LOCALFS, image=nydus_scratch_image) rafs_conf.enable_records_readahead(interval=1) rafs_conf.dump_rafs_conf() rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf) with utils.timer("Mount elapse"): rafs.thread_num(7).mount() assert rafs.is_mounted() wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs()) # TODO: Run several parallel read workers against the mount_point wg.setup_workload_generator() wg.torture_read(5, 5) wg.finish_torture_read() utils.clean_pagecache()
def test_pseudo_fs(nydus_anchor, nydus_image, rafs_conf: RafsConf): nydus_image.set_backend(Backend.BACKEND_PROXY).create_image() rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY) rafs = RafsMount(nydus_anchor, None, rafs_conf) rafs.mount() time.sleep(1) nc = NydusAPIClient(rafs.get_apisock()) try: shutil.rmtree("pseudo_fs_scratch") except FileNotFoundError: pass scratch_rootfs = shutil.copytree(nydus_image.rootfs(), "pseudo_fs_scratch", symlinks=True) dist = Distributor(scratch_rootfs, 5, 5) dist.generate_tree() dist.put_multiple_files(20, Size(8, Unit.KB)) ### suffix = "1" image = RafsImage( nydus_anchor, scratch_rootfs, "bs" + suffix, "blob" + suffix, ) conf = RafsConf(nydus_anchor) conf.enable_fs_prefetch() conf.enable_validation() conf.set_rafs_backend(Backend.BACKEND_PROXY) conf.dump_rafs_conf() image.set_backend(Backend.BACKEND_PROXY).create_image() nc.pseudo_fs_mount(image.bootstrap_path, f"/pseudo{suffix}", conf.path(), None) ### suffix = "2" image = RafsImage( nydus_anchor, scratch_rootfs, "bs" + suffix, "blob" + suffix, ) conf = RafsConf(nydus_anchor) conf.enable_rafs_blobcache() conf.enable_validation() conf.enable_records_readahead() conf.set_rafs_backend(Backend.BACKEND_PROXY) conf.dump_rafs_conf() dist.put_multiple_files(20, Size(8, Unit.KB)) image.set_backend(Backend.BACKEND_PROXY).create_image() nc.pseudo_fs_mount(image.bootstrap_path, f"/pseudo{suffix}", conf.path(), None) ### suffix = "3" image = RafsImage( nydus_anchor, scratch_rootfs, "bs" + suffix, "blob" + suffix, ) conf = RafsConf(nydus_anchor) conf.enable_rafs_blobcache() conf.enable_records_readahead() conf.set_rafs_backend(Backend.BACKEND_PROXY) conf.dump_rafs_conf() dist.put_multiple_files(20, Size(8, Unit.KB)) image.set_backend(Backend.BACKEND_PROXY).create_image() nc.pseudo_fs_mount(image.bootstrap_path, f"/pseudo{suffix}", conf.path(), None) wg1 = WorkloadGen(os.path.join(nydus_anchor.mount_point, "pseudo1"), scratch_rootfs) wg2 = WorkloadGen(os.path.join(nydus_anchor.mount_point, "pseudo2"), scratch_rootfs) wg3 = WorkloadGen(os.path.join(nydus_anchor.mount_point, "pseudo3"), scratch_rootfs) time.sleep(2) wg1.setup_workload_generator() wg2.setup_workload_generator() wg3.setup_workload_generator() wg1.torture_read(4, 8) wg2.torture_read(4, 8) wg3.torture_read(4, 8) wg1.finish_torture_read() wg2.finish_torture_read() wg3.finish_torture_read() # TODO: Temporarily disable the verification as hard to select `verify dir` # assert wg1.verify_entire_fs() # assert wg2.verify_entire_fs() # assert wg3.verify_entire_fs() nc.umount_rafs("/pseudo1") nc.umount_rafs("/pseudo2") nc.umount_rafs("/pseudo3")