def test_prefetch_with_cache(
nydus_anchor: NydusAnchor,
nydus_scratch_image: RafsImage,
nydus_scratch_parent_image: RafsImage,
rafs_conf: RafsConf,
):
parent_rootfs = nydus_scratch_parent_image.rootfs()
upper_rootfs = nydus_scratch_image.rootfs()
rafs_conf.enable_validation()
rafs_conf.set_rafs_backend(Backend.OSS)
rafs_conf.enable_rafs_blobcache()
rafs_conf.enable_fs_prefetch(threads_count=4, merging_size=512 * 1024)
rafs_conf.dump_rafs_conf()
dist_parent = Distributor(parent_rootfs, 6, 4)
dist_parent.generate_tree()
dist_parent.put_directories(20)
dist_parent.put_multiple_files(100, Size(64, Unit.KB))
dist_parent.put_symlinks(30)
dist_parent.put_hardlinks(20)
dist_upper = Distributor(upper_rootfs, 3, 8)
dist_upper.generate_tree()
dist_upper.put_multiple_files(27, Size(3, Unit.MB))
dist_upper.put_symlinks(5)
# hint_files_parent = dist_parent.put_multiple_files(1000, Size(8, Unit.KB))
# hint_files_parent = [os.path.join(parent_rootfs, p) for p in hint_files_parent]
# hint_files_parent = "\n".join(hint_files_parent)
nydus_scratch_parent_image.set_backend(Backend.OSS).create_image(
readahead_policy="fs", readahead_files="/".encode())
hint_files = dist_upper.put_multiple_files(1000, Size(8, Unit.KB))
hint_files.extend(dist_upper.put_multiple_empty_files(200))
hint_files = [os.path.join("/", p) for p in hint_files]
hint_files = "\n".join(hint_files)
nydus_scratch_image.set_backend(Backend.OSS).create_image(
parent_image=nydus_scratch_parent_image,
readahead_policy="fs",
readahead_files=hint_files.encode(),
)
nydus_anchor.mount_overlayfs(
[nydus_scratch_image.rootfs(),
nydus_scratch_parent_image.rootfs()])
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.thread_num(5).mount()
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_anchor.overlayfs)
workload_gen.setup_workload_generator()
assert workload_gen.verify_entire_fs()
workload_gen.torture_read(5, 20)
workload_gen.finish_torture_read()
def test_build_image(nydus_anchor, nydus_scratch_image: RafsImage,
rafs_conf: RafsConf):
"""
title: Build nydus image
description: Build nydus image from rootfs generating proper bootstrap and
blob
pass_criteria:
- Image can successfully builded and mounted
- Rafs can be unmounted and do a small account of read io and attr
operation
- Try let image builder upload blob itself.
"""
dist = Distributor(nydus_scratch_image.rootfs(), 80, 1)
dist.generate_tree()
dist.put_directories(100)
dist.put_hardlinks(90)
dist.put_symlinks(200)
dist.put_multiple_files(random.randint(20, 28), Size(10, Unit.MB))
dist.put_multiple_chinese_files(random.randint(20, 28), Size(20, Unit.KB))
Whiteout().whiteout_one_file(nydus_scratch_image.rootfs(),
"i/am/troublemaker/foo")
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs_conf.set_rafs_backend(backend_type=Backend.BACKEND_PROXY)
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
rafs.mount()
assert wg.verify_entire_fs()
rafs.umount()
def test_large_file(nydus_anchor, compressor, backend, amplified_size):
_tmp_dir = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
large_file_dir = _tmp_dir.name
dist = Distributor(large_file_dir, 3, 3)
dist.generate_tree()
dist.put_single_file(Size(20, Unit.MB))
dist.put_single_file(Size(10891, Unit.KB))
dist.put_multiple_files(10, Size(2, Unit.MB))
dist.put_multiple_files(10, Size(4, Unit.MB))
image = RafsImage(nydus_anchor, large_file_dir, "bs_large", "blob_large")
image.set_backend(backend).create_image(compressor=compressor)
rafs_conf = (RafsConf(nydus_anchor,
image).enable_rafs_blobcache().amplify_io(
amplified_size).set_rafs_backend(backend,
image=image))
rafs = RafsMount(nydus_anchor, image, rafs_conf)
rafs.thread_num(4).mount()
workload_gen = WorkloadGen(nydus_anchor.mount_point, large_file_dir)
workload_gen.setup_workload_generator()
workload_gen.torture_read(8, 5)
workload_gen.finish_torture_read()
assert not workload_gen.io_error
rafs.umount()
image.clean_up()
def test_specified_prefetch(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
nydus_scratch_image: RafsImage,
backend,
):
"""
description:
Nydusd can have a list including files and directories input when started.
Then it can prefetch files from backend per as to the list.
"""
rafs_conf.set_rafs_backend(backend)
rafs_conf.enable_fs_prefetch(prefetch_all=True)
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 8, 2)
dist.generate_tree()
dirs = dist.put_directories(20)
dist.put_multiple_files(100, Size(64, Unit.KB))
dist.put_symlinks(30)
dist.put_hardlinks(20)
dist.put_multiple_files(40, Size(64, Unit.KB))
dist.put_single_file(Size(3, Unit.MB), name="test")
nydus_scratch_image.set_backend(backend).create_image()
prefetching_files = dirs
prefetching_files += dist.files[:-10]
prefetching_files += dist.dirs[:-5]
prefetching_files += dist.symlinks[:-10]
# Fuzz
prefetching_files.append("/a/b/c/d")
prefetching_files.append(os.path.join("/", "f/g/h/"))
specified_dirs = " ".join(
[os.path.join("/", d) for d in prefetching_files])
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.prefetch_files(specified_dirs).mount()
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
nc = NydusAPIClient(rafs.get_apisock())
wg.setup_workload_generator()
blobcache_metrics = nc.get_blobcache_metrics()
wg.torture_read(5, 10)
while blobcache_metrics["prefetch_workers"] != 0:
time.sleep(0.5)
blobcache_metrics = nc.get_blobcache_metrics()
begin = nc.get_backend_metrics()["read_amount_total"]
time.sleep(1)
end = nc.get_backend_metrics()["read_amount_total"]
assert end == begin
wg.finish_torture_read()
def test_prefetch_with_cache(
nydus_anchor,
nydus_scratch_image: RafsImage,
rafs_conf: RafsConf,
thread_cnt,
compressor,
is_cache_compressed,
):
"""
title: Prefetch from various backend
description:
- Enable rafs backend blob cache, as it is disabled by default
pass_criteria:
- Rafs can be mounted.
- Rafs can be unmounted.
"""
rafs_conf.enable_rafs_blobcache(is_compressed=is_cache_compressed)
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY, prefix="object_prefix/")
rafs_conf.enable_fs_prefetch(threads_count=4,
bandwidth_rate=Size(40, Unit.MB).B)
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 4, 4)
dist.generate_tree()
dist.put_directories(20)
dist.put_multiple_files(40, Size(3, Unit.MB))
dist.put_hardlinks(6)
dist.put_multiple_chinese_files(random.randint(20, 28), Size(20, Unit.KB))
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY,
prefix="object_prefix/").create_image(
compressor=compressor,
readahead_policy="fs",
readahead_files="/".encode(),
)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.thread_num(4).mount()
nc = NydusAPIClient(rafs.get_apisock())
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_scratch_image.rootfs())
m = nc.get_blobcache_metrics()
time.sleep(0.3)
assert m["prefetch_data_amount"] != 0
workload_gen.setup_workload_generator()
workload_gen.torture_read(thread_cnt, 10)
assert NydusAnchor.check_nydusd_health()
workload_gen.finish_torture_read()
assert not workload_gen.io_error
# In this way, we can check if nydusd is crashed.
assert rafs.is_mounted()
rafs.umount()
def test_file_tail(nydus_anchor: NydusAnchor, nydus_scratch_image: RafsImage,
backend):
"""
description: Read data from file tail
- Create several files of different sizes
- Punch hole to each file of which some should have hole tail
- Create rafs image from test scratch directory.
- Mount rafs
- Do some test.
"""
file_size_list = [
Size(1, Unit.KB),
Size(6, Unit.KB),
Size(2, Unit.MB),
Size(10034, Unit.KB),
]
file_list = []
dist = Distributor(nydus_anchor.scratch_dir, 2, 2)
dist.generate_tree()
for f_s in file_size_list:
f_name = dist.put_single_file(f_s)
file_list.append(f_name)
# Punch hole
with utils.pushd(nydus_anchor.scratch_dir):
with open(f_name, "a+b") as f:
fallocate(
f,
f_s.B - 500,
1000,
mode=FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
)
nydus_scratch_image.set_backend(backend).create_image()
rafs_conf = RafsConf(nydus_anchor, nydus_scratch_image)
rafs_conf.set_rafs_backend(backend, image=nydus_scratch_image)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
with utils.pushd(nydus_anchor.mount_point):
for name in file_list:
with open(name, "rb") as f:
size = os.stat(name).st_size
f.seek(size - 300)
buf = f.read(1000)
assert len(buf) == 300
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
for f in file_list:
wg.verify_single_file(os.path.join(nydus_anchor.mount_point, f))
assert wg.io_error == False
def __init__(self, **kwargs):
self.__NGCNo = kwargs['NGCNo']
self.__RA_2000 = kwargs['RA_2000']
self.__DEC_2000 = kwargs['DEC_2000']
self.__Const = kwargs['Const']
self.__ObjectType = kwargs['ObjectType']
self.__Size = kwargs['Size']
self.__Bmag = kwargs['Bmag']
self.__Vmag = kwargs['Vmag']
self.__AlsoCatalogedAs = kwargs['AlsoCatalogedAs']
self.__PA = kwargs['PA']
self.ra = EquatorialCoordinate(self.__RA_2000)
self.dec = EquatorialCoordinate(self.__DEC_2000)
# parse the size
size_match = ngc_size_re.match(self.__Size)
if size_match:
major = size_match.groups()[0]
minor = size_match.groups()[2]
self.size = Size(major=float(major),
minor=(float(minor) if minor is not None else 0))
else:
self.size = Size(major=0, minor=0)
# Some objects don't have a Vmag... I'm not sure why.
try:
self.magnitude = float(self.__Vmag)
except ValueError as e:
try:
self.magnitude = float(self.__Bmag)
except Exception as e:
self.magnitude = 20
try:
self.angle = float(self.__PA)
except ValueError:
self.angle = None
self.identifier = self.__NGCNo
self.catalog = 'NGC'
# Lookup the NGC type and map to one of our types
self.type = ngc_object_types[self.__ObjectType]
# The HCNGC Catalog gives us some aliases. Add them.
# This is imperfect because the catalog doesn't use a consistent
# seperator between catalog and identifier.
aliases = [
re.split('[ -]', a.strip(), 1)
for a in self.__AlsoCatalogedAs.split(',')
]
for pair in aliases:
self.add_alias(*reversed(pair))
def __call__(self, data, label, gt):
#-----------------------------------------------------------------------
# Check whether to sample or not
#-----------------------------------------------------------------------
if not self.sample:
return data, label, gt
#-----------------------------------------------------------------------
# Retry sampling a couple of times
#-----------------------------------------------------------------------
source_boxes = anchors2array(gt.boxes, gt.imgsize)
box = None
box_arr = None
for _ in range(self.max_trials):
#-------------------------------------------------------------------
# Sample a bounding box
#-------------------------------------------------------------------
scale = random.uniform(self.min_scale, self.max_scale)
aspect_ratio = random.uniform(self.min_aspect_ratio,
self.max_aspect_ratio)
# make sure width and height will not be larger than 1
aspect_ratio = max(aspect_ratio, scale**2)
aspect_ratio = min(aspect_ratio, 1 / (scale**2))
width = scale * sqrt(aspect_ratio)
height = scale / sqrt(aspect_ratio)
cx = 0.5 * width + random.uniform(0, 1 - width)
cy = 0.5 * height + random.uniform(0, 1 - height)
center = Point(cx, cy)
size = Size(width, height)
#-------------------------------------------------------------------
# Check if the box satisfies the jaccard overlap constraint
#-------------------------------------------------------------------
box_arr = np.array(prop2abs(center, size, gt.imgsize))
overlap = compute_overlap(box_arr, source_boxes, 0)
if overlap.best and overlap.best.score >= self.min_jaccard_overlap:
box = Box(None, None, center, size)
break
if box is None:
return None
#-----------------------------------------------------------------------
# Crop the box and adjust the ground truth
#-----------------------------------------------------------------------
new_size = Size(box_arr[1] - box_arr[0], box_arr[3] - box_arr[2])
w_off = -box_arr[0]
h_off = -box_arr[2]
data = data[box_arr[2]:box_arr[3], box_arr[0]:box_arr[1]]
gt = transform_gt(gt, new_size, h_off, w_off)
return data, label, gt
def test_blob_prefetch(nydus_anchor: NydusAnchor,
nydus_scratch_image: RafsImage, readahead_policy):
"""
description:
For rafs, there are two types of prefetching.
1. Prefetch files from fs-layer, which means each file is prefetched one by one.
2. Prefetch directly from backend/blob layer, which means a range will be fetched from blob
"""
# Try to delete any access log since if it present, bootstrap blob prefetch won't work.
utils.execute("rm -rf *.access", shell=True)
dist = Distributor(nydus_scratch_image.rootfs(), 8, 2)
dist.generate_tree()
dist.put_directories(20)
dist.put_multiple_files(100, Size(64, Unit.KB))
dist.put_symlinks(30)
dist.put_hardlinks(20)
dist.put_multiple_files(40, Size(64, Unit.KB))
utils.clean_pagecache()
hint_files = dist.files[-40:]
hint_files.extend(dist.symlinks[-20:])
hint_files = [os.path.join("/", p) for p in hint_files]
hint_files = "\n".join(hint_files)
nydus_scratch_image.set_backend(Backend.LOCALFS).create_image(
readahead_policy=readahead_policy,
readahead_files=hint_files.encode(),
)
rafs_conf = RafsConf(nydus_anchor, nydus_scratch_image)
rafs_conf.set_rafs_backend(Backend.LOCALFS, image=nydus_scratch_image)
rafs_conf.enable_records_readahead(interval=1)
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
with utils.timer("Mount elapse"):
rafs.thread_num(7).mount()
assert rafs.is_mounted()
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
# TODO: Run several parallel read workers against the mount_point
wg.setup_workload_generator()
wg.torture_read(5, 5)
wg.finish_torture_read()
utils.clean_pagecache()
def test_blobcache_recovery(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
nydus_scratch_image: RafsImage,
):
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY)
rafs_conf.enable_fs_prefetch()
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 8, 2)
dist.generate_tree()
dirs = dist.put_directories(20)
dist.put_multiple_files(100, Size(64, Unit.KB))
dist.put_symlinks(30)
dist.put_hardlinks(20)
dist.put_multiple_files(40, Size(64, Unit.KB))
dist.put_single_file(Size(3, Unit.MB), name="test")
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.prefetch_files("/").mount()
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
wg.setup_workload_generator()
wg.torture_read(4, 4)
# Hopefully, prefetch can be done in 5 secondes.
time.sleep(5)
wg.finish_torture_read()
rafs.umount()
rafs2 = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs2.mount()
wg.torture_read(4, 4)
time.sleep(0.5)
nc = NydusAPIClient(rafs2.get_apisock())
begin = nc.get_backend_metrics()["read_amount_total"]
time.sleep(1)
end = nc.get_backend_metrics()["read_amount_total"]
assert end == begin == 0
wg.finish_torture_read()
def test_prefetch_without_cache(nydus_anchor: NydusAnchor,
nydus_scratch_image: RafsImage,
rafs_conf: RafsConf):
"""Files prefetch test
1. relative hinted prefetch files
2. absolute hinted prefetch files
3. source rootfs root dir.
"""
rafs_conf.enable_fs_prefetch().set_rafs_backend(Backend.BACKEND_PROXY)
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 4, 4)
dist.generate_tree()
dist.put_directories(20)
dist.put_multiple_files(40, Size(8, Unit.KB))
dist.put_hardlinks(6)
dist.put_multiple_chinese_files(random.randint(20, 28), Size(20, Unit.KB))
hint_files = ["/"]
hint_files.extend(dist.files)
hint_files.extend(dist.dirs)
hint_files.extend(dist.symlinks)
hint_files.extend(dist.hardlinks)
hint_files = [os.path.join("/", p) for p in hint_files]
hint_files = "\n".join(hint_files)
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image(
readahead_policy="fs", readahead_files=hint_files.encode())
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
assert rafs.is_mounted()
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_scratch_image.rootfs())
# TODO: Run several parallel read workers against the mount_point
workload_gen.setup_workload_generator()
workload_gen.torture_read(8, 5)
workload_gen.finish_torture_read()
assert NydusAnchor.check_nydusd_health()
assert not workload_gen.io_error
assert rafs.is_mounted()
rafs.umount()
def test_syscalls(
nydus_anchor: NydusAnchor,
nydus_scratch_image: RafsImage,
rafs_conf: RafsConf,
):
syscall_helper = "framework/test_syscalls"
ret, _ = utils.execute(
["gcc", "framework/test_syscalls.c", "-o", syscall_helper],
shell=False,
print_output=True,
)
assert ret
dist = Distributor(nydus_scratch_image.rootfs(), 2, 2)
dist.generate_tree()
dist.put_single_file(Size(8, Unit.KB),
pos=nydus_scratch_image.rootfs(),
name="xattr_no_kv")
dist.put_single_file_with_xattr(
Size(8, Unit.KB),
("trusted.nydus.key", ""),
pos=nydus_scratch_image.rootfs(),
name="xattr_empty_value",
)
dist.put_single_file_with_xattr(
Size(8, Unit.KB),
("trusted.nydus.key", "1234567890"),
pos=nydus_scratch_image.rootfs(),
name="xattr_insufficient_buffer",
)
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs_conf.enable_xattr().set_rafs_backend(Backend.BACKEND_PROXY)
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
for no in [58]:
ret, _ = utils.execute(
[syscall_helper, nydus_anchor.mount_point,
str(no)],
shell=False,
print_output=True,
)
assert ret
def add_detections(self, filename, boxes):
fileid = os.path.basename(filename)
fileid = ''.join(fileid.split('.')[:-1])
img = cv2.imread(filename)
img_size = Size(img.shape[1], img.shape[0])
for conf, box in boxes:
xmin, xmax, ymin, ymax = prop2abs(box.center, box.size, img_size)
if xmin < 0:
xmin = 0
if xmin >= img_size.w:
xmin = img_size.w - 1
if xmax < 0:
xmax = 0
if xmax >= img_size.w:
xmax = img_size.w - 1
if ymin < 0:
ymin = 0
if ymin >= img_size.h:
ymin = img_size.h - 1
if ymax < 0:
ymax = 0
if ymax >= img_size.h:
ymax = img_size.h - 1
det = Detection(fileid, conf, float(xmin + 1), float(ymin + 1),
float(xmax + 1), float(ymax + 1))
self.boxes[box.label].append(det)
def scratch(self, scratch_dir):
"""Put various kinds of xattr value into.
1. Very long value
2. a common short value
3. Nothing resides in value field
4. Single file, multiple pairs.
5. /n
6. whitespace
7. 中文
8. Binary
9. Only key?
"""
self.dist.put_symlinks(100)
files_cnt = 20
self.dist.put_multiple_files(files_cnt, Size(9, Unit.KB))
self.scratch_dir = os.path.abspath(scratch_dir)
self.source_files = {}
self.source_xattrs = {}
self.source_dirs = {}
self.source_dirs_xattrs = {}
self.encoding = "gb2312"
self.xattr_pairs = 50 if utils.get_fs_type(
os.getcwd()) == "xfs" else 20
# TODO: Only key without values?
with pushd(self.scratch_dir):
for f in self.dist.files[-files_cnt:]:
relative_path = os.path.relpath(f, start=self.scratch_dir)
self.source_xattrs[relative_path] = {}
for idx in range(0, self.xattr_pairs):
# TODO: Random this Key
k = f"trusted.nydus.{Distributor.generate_random_name(20, chinese=True)}"
v = f"_{Distributor.generate_random_length_name(20, chinese=True)}"
xattr.setxattr(f, k.encode(self.encoding),
v.encode(self.encoding))
# Use relative or canonicalized names as key to locate
# path in source rootfs directory. So we verify if image is
# packed correctly.
self.source_files[relative_path] = os.path.abspath(f)
self.source_xattrs[relative_path][k] = v
dir_cnt = 20
self.dist.put_directories(dir_cnt)
# Add xattr key-value paris to directories.
with pushd(self.scratch_dir):
for d in self.dist.dirs[-dir_cnt:]:
relative_path = os.path.relpath(d, start=self.scratch_dir)
self.source_dirs_xattrs[relative_path] = {}
for idx in range(0, self.xattr_pairs):
# TODO: Random this Key
k = f"trusted.{Distributor.generate_random_name(20)}"
v = f"{Distributor.generate_random_length_name(50)}"
xattr.setxattr(d, k, v.encode())
# Use relative or canonicalized names as key to locate
# path in source rootfs directory. So we verify if image is
# packed correctly.
self.source_dirs[relative_path] = os.path.abspath(d)
self.source_dirs_xattrs[relative_path][k] = v
def test_limited_mem(nydus_anchor, rafs_conf, nydus_image):
"""
description: Run nydusd in a memory limited environment.
- Use `ulimit` to limit virtual memory nydusd can use.
- Mount rafs
- Torture rafs
"""
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_image, rafs_conf)
rafs.mount(limited_mem=Size(3, Unit.GB))
wg = WorkloadGen(nydus_anchor.mount_point, nydus_image.rootfs())
wg.setup_workload_generator()
wg.torture_read(8, 10)
nydus_anchor.start_stats_checker()
wg.finish_torture_read()
nydus_anchor.stop_stats_checker()
assert wg.io_error == False
assert nydus_anchor.check_nydusd_health()
def put_files(dist: Distributor, f_type, count, size):
"""Example:
depth: 4
width: 4
layers:
- layer1:
- size: 10KB
type: regular
count: 2000
- size: 12MB
type: regular
count: 10
- size: 90MB
type: regular
count: 1
- type: symlink
count: 100
"""
logging.info("putting %s, count %d", f_type, count)
if f_type == "regular":
size_in_bytes = utils.parse_size(size)
dist.put_multiple_files(count, Size(size_in_bytes))
elif f_type == "dir":
dist.put_directories(count)
elif f_type == "symlink":
dist.put_symlinks(count)
elif f_type == "hardlink":
dist.put_hardlinks(count)
def initialize(self):
self.anchors = get_anchors_for_preset(self.preset)
self.vheight = len(self.anchors)
self.vwidth = self.num_classes + 5 # background class + location offsets
self.img_size = Size(1000, 1000)
self.anchors_arr = anchors2array(self.anchors, self.img_size)
self.initialized = True
def __build_sample_list(self, root, annot_files, dataset_name):
image_root = root + '/JPEGImages/'
samples = []
for fn in tqdm(annot_files, desc=dataset_name, unit='samples'):
with open(fn, 'r') as f:
doc = lxml.etree.parse(f)
filename = image_root + doc.xpath(
'/annotation/filename')[0].text
if not os.path.exists(filename):
continue
img = cv2.imread(filename)
imgsize = Size(img.shape[1], img.shape[0])
boxes = []
objects = doc.xpath('/annotation/object')
for obj in objects:
label = obj.xpath('name')[0].text
if label == myObject:
xmin = int(float(obj.xpath('bndbox/xmin')[0].text))
xmax = int(float(obj.xpath('bndbox/xmax')[0].text))
ymin = int(float(obj.xpath('bndbox/ymin')[0].text))
ymax = int(float(obj.xpath('bndbox/ymax')[0].text))
center, size = abs2prop(xmin, xmax, ymin, ymax,
imgsize)
box = Box(label, self.lname2id[label], center, size)
boxes.append(box)
if not boxes:
continue
sample = Sample(filename, boxes, imgsize)
samples.append(sample)
return samples
def __init__(self, **kwargs):
self.__StarID = kwargs['StarID']
self.__HIP = kwargs['HIP']
self.__HD = kwargs['HD']
self.__HR = kwargs['HR']
self.__BayerFlamsteed = kwargs['BayerFlamsteed']
self.__ProperName = kwargs['ProperName']
self.__RA = kwargs['RA']
self.__Dec = kwargs['Dec']
self.__Mag = kwargs['Mag']
self.__AbsMag = kwargs['AbsMag']
self.__Spectrum = kwargs['Spectrum']
self.__ColorIndex = kwargs['ColorIndex']
# The HYG catalog contains HIP, HD, and HR identifiers the
# `catalog` property will corrospond to the one we prefer for
# the `identifier`.
self.identifier = self.__HIP
self.catalog = 'HIP'
self.type = 0
self.size = Size(-1, -1)
# NOTE: HYG RA is in degrees.
self.ra = EquatorialCoordinate(self.__RA, hours=True)
self.dec = EquatorialCoordinate(self.__Dec, degrees=True)
self.magnitude = float(self.__Mag)
# HYG gives us a lot of aliases. Add them.
self.add_alias(self.__HD, 'HD')
self.add_alias(self.__HR, 'HR')
self.add_alias(self.__ProperName)
def __call__(self, data, label, gt):
#-----------------------------------------------------------------------
# Calculate sizes and offsets
#-----------------------------------------------------------------------
ratio = random.uniform(1, self.max_ratio) # 2
orig_size = gt.imgsize # 400,300
new_size = Size(int(orig_size.w * ratio),
int(orig_size.h * ratio)) # 800, 600
h_off = random.randint(0, new_size.h -
orig_size.h) # 0 ~ (800-400) => 100
w_off = random.randint(0,
new_size.w - orig_size.w) # 0 ~ (600-300) => 50
#-----------------------------------------------------------------------
# Create the new image and place the input image in it
#-----------------------------------------------------------------------
imsi = Image.new('RGB', (new_size.w, new_size.h), self.mean_value)
imsi.paste(data,
(w_off, h_off, w_off + orig_size.w, h_off + orig_size.h))
#-----------------------------------------------------------------------
# Transform the ground truth
#-----------------------------------------------------------------------
gt = transform_gt(gt, new_size, h_off, w_off)
return imsi, label, gt
def mount(self, limited_mem=False, wait_mount=True, dump_config=True):
"""
:limited_mem: Unit is KB, limit nydusd process virtual memory usage thus to
inject some faults.
"""
cmd = str(self).split()
self.anchor.checker_sock = self.get_apisock()
if dump_config:
self.conf.dump_rafs_conf()
if isinstance(limited_mem, Size):
limit_kb = limited_mem.B // Size(1, Unit.KB).B
cmd = f"ulimit -v {limit_kb};" + cmd
_, p = utils.run(
cmd,
False,
shell=False,
stdout=self.anchor.logging_file,
stderr=self.anchor.logging_file,
)
self.p = p
if wait_mount:
self._wait_for_mount()
return self
def calc_file_md5(path):
md5 = hashlib.md5()
with open(path, "rb") as f:
for block in iter(lambda: f.read(Size(128, Unit.KB).B), b""):
md5.update(block)
return md5.digest()
def test_different_partitions(nydus_anchor: NydusAnchor, rafs_conf):
loop_file_1 = tempfile.NamedTemporaryFile(suffix="loop")
loop_file_2 = tempfile.NamedTemporaryFile(suffix="loop")
loop_mnt_1 = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
loop_mnt_2 = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
os.posix_fallocate(loop_file_1.fileno(), 0, Size(400, Unit.MB).B)
os.posix_fallocate(loop_file_2.fileno(), 0, Size(400, Unit.MB).B)
utils.execute(["mkfs.ext4", "-F", loop_file_1.name])
utils.execute(["mkfs.ext4", "-F", loop_file_2.name])
utils.execute(["mount", loop_file_1.name, loop_mnt_1.name])
utils.execute(["mount", loop_file_2.name, loop_mnt_2.name])
# TODO: Put more special files into
dist1 = Distributor(loop_mnt_1.name, 5, 7)
dist1.generate_tree()
dist1.put_multiple_files(100, Size(12, Unit.KB))
dist2 = Distributor(loop_mnt_2.name, 5, 7)
dist2.generate_tree()
dist2.put_symlinks(20)
dist2.put_multiple_files(50, Size(12, Unit.KB))
Whiteout.mirror_files(dist2.files[:20], loop_mnt_2.name, loop_mnt_1.name)
parent_image = (RafsImage(nydus_anchor, loop_mnt_1.name).set_backend(
Backend.OSS).create_image())
image = RafsImage(nydus_anchor, loop_mnt_2.name)
image.set_backend(Backend.OSS).create_image(parent_image=parent_image)
rafs_conf.set_rafs_backend(Backend.OSS)
rafs = RafsMount(nydus_anchor, image, rafs_conf)
rafs.mount()
nydus_anchor.mount_overlayfs([image.rootfs(), parent_image.rootfs()])
wg = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
wg.setup_workload_generator()
wg.torture_read(5, 5)
wg.finish_torture_read()
utils.execute(["umount", loop_mnt_1.name])
utils.execute(["umount", loop_mnt_2.name])
nydus_anchor.umount_overlayfs()
def decode_location(box, anchor):
box[box > 100] = 100 # only happens early training
x = box[0] / 10 * anchor.size.w + anchor.center.x
y = box[1] / 10 * anchor.size.h + anchor.center.y
w = exp(box[2] / 5) * anchor.size.w
h = exp(box[3] / 5) * anchor.size.h
return Point(x, y), Size(w, h)
def test_layered_rebuild(
nydus_anchor,
nydus_scratch_image: RafsImage,
nydus_scratch_parent_image: RafsImage,
rafs_conf: RafsConf,
backend,
):
"""
title: Layered image rebuild
description:
- Parent and upper have files whose contents are exactly the same.
- Use files stats to check if file is overlayed.
- Files with the same name but different modes.
- Files with xattr in parent should be shadowed.
pass_criteria:
- Mount successfully.
- No data corruption.
"""
rafs_conf.set_rafs_backend(backend)
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()
parent_rootfs = nydus_scratch_parent_image.rootfs()
upper_rootfs = nydus_scratch_image.rootfs()
nydus_anchor.mount_overlayfs(
[nydus_scratch_image.rootfs(),
nydus_scratch_parent_image.rootfs()])
shared_files = []
dist_parent = Distributor(parent_rootfs, 6, 4)
dist_parent.generate_tree()
shared_files.extend(dist_parent.put_multiple_files(100, Size(64, Unit.KB)))
shared_files.extend(dist_parent.put_symlinks(30))
shared_files.extend(dist_parent.put_hardlinks(30))
xattr_verifier = XattrVerifier(parent_rootfs, dist_parent)
Whiteout.mirror_files(shared_files, parent_rootfs, upper_rootfs)
xattr_verifier.scratch(parent_rootfs)
nydus_scratch_parent_image.set_backend(backend).create_image()
nydus_scratch_image.set_backend(backend).create_image(
parent_image=nydus_scratch_parent_image)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_anchor.overlayfs)
workload_gen.setup_workload_generator()
xattr_verifier.verify(nydus_anchor.mount_point)
assert workload_gen.verify_entire_fs()
workload_gen.torture_read(5, 4)
workload_gen.finish_torture_read()
def __init__(self):
self.bg = pygame.image.load('game_1/game_1.jpg')
self.body_frame = None
self.body_index_frame = None
self.solved = True
self.time_angles = None
self.threshold = 5 # in degrees
self.player_size = Size(SCREEN_SIZE.W / 3, SCREEN_SIZE.H / 3)
KINECT_EVENT_STREAM.subscribe(self.event_handler)
def _put_single_file(
self,
parent_dir,
file_size: Size,
specified_name=None,
letters=False,
chinese=False,
name_len=32,
):
if specified_name is None:
name = Distributor.generate_random_length_name(
name_len, suffix="regular", chinese=chinese
)
else:
name = specified_name
this_path = os.path.join(parent_dir, name)
with pushd(parent_dir):
if chinese:
fd = os.open(name.encode("gb2312"), os.O_CREAT | os.O_RDWR)
else:
fd = os.open(name.encode("ascii"), os.O_CREAT | os.O_RDWR)
if file_size.B != 0:
left = file_size.B
logging.debug("Putting file %s", this_path)
while left:
length = Size(1, Unit.MB).B if Size(1, Unit.MB).B < left else left
if not letters:
left -= os.write(fd, os.urandom(length))
else:
picked_list = "".join(
random.choices(string.ascii_lowercase[1:4], k=length)
)
left -= os.write(fd, picked_list.encode())
os.close(fd)
self.files.append(self._relative_path_to_top(this_path))
return name, this_path
def __build_sample_list(self, root, annot_files):
"""
Build a list of samples for the VOC dataset (either trainval or test)
"""
image_root = os.path.join(root, 'rgb-images')
samples = []
#-----------------------------------------------------------------------
# Process each annotated sample
#-----------------------------------------------------------------------
for fn in tqdm(annot_files, desc='ucf_24_frame', unit='samples'):
act = fn.split('/')[4]
video = fn.split('/')[5]
frame_id = fn.split('/')[-1][:-4]
image_path = os.path.join(image_root, act, video,
'{}.jpg'.format(frame_id))
#---------------------------------------------------------------
# Get the file dimensions
#---------------------------------------------------------------
if not os.path.exists(image_path):
continue
img = cv2.imread(image_path)
imgsize = Size(img.shape[1], img.shape[0])
#---------------------------------------------------------------
# Get boxes for all the objects
#---------------------------------------------------------------
boxes = []
with open(fn, 'r') as fin:
objects = fin.readlines()
for line in objects:
line = line[:-1]
#-----------------------------------------------------------
# Get the properties of the box and convert them to the
# proportional terms
#-----------------------------------------------------------
obj = line.split(' ')
label = int(obj[0]) - 1
xmin = int(float(obj[1]))
ymin = int(float(obj[2]))
xmax = int(float(obj[3]))
ymax = int(float(obj[4]))
center, size = abs2prop(xmin, xmax, ymin, ymax, imgsize)
box = Box(self.lid2name[label], label, center, size)
boxes.append(box)
if not boxes:
continue
sample = Sample(image_path, boxes, imgsize)
samples.append(sample)
return samples
def _build_sample_list(self, root, annot_files):
image_root = root + 'VOC_Jpeg/'
image_seg_root = root + 'VOC_Segmentation/'
samples = []
for fn in tqdm(annot_files, unit='samples'):
with open(fn, 'r') as f:
doc = lxml.etree.parse(f)
filename = image_root + doc.xpath(
'/annotation/filename')[0].text
with open(fn, 'r') as f1:
doc1 = lxml.etree.parse(f1)
seg_gt = image_seg_root + doc1.xpath(
'/annotation/filename')[0].text
seg_gt = seg_gt.replace('jpg', 'png')
seg_gt_to_compare = seg_gt
#---------------------------------------------------------------
# Get the file dimensions
#---------------------------------------------------------------
if not os.path.exists(filename):
continue
img = cv2.imread(filename)
img_seg_gt = cv2.imread(seg_gt)
imgsize = Size(img.shape[1], img.shape[0])
#---------------------------------------------------------------
# Get boxes for all the objects
#---------------------------------------------------------------
boxes = []
objects = doc.xpath('/annotation/object')
for obj in objects:
#-----------------------------------------------------------
# Get the properties of the box and convert them to the
# proportional terms
#-----------------------------------------------------------
label = obj.xpath('name')[0].text
xmin = int(float(obj.xpath('bndbox/xmin')[0].text))
xmax = int(float(obj.xpath('bndbox/xmax')[0].text))
ymin = int(float(obj.xpath('bndbox/ymin')[0].text))
ymax = int(float(obj.xpath('bndbox/ymax')[0].text))
center, size = abs2prop(xmin, xmax, ymin, ymax, imgsize)
box = Box(label, self.lname2id[label], center, size)
boxes.append(box)
if not boxes:
continue
sample = Sample(filename, boxes, imgsize, seg_gt,
seg_gt_to_compare)
samples.append(sample)
return samples
def __build_sample_list(self, root, dataset_name):
"""
Build a list of samples for the VOC dataset (either trainval or test)
"""
image_root = root + '/JPEGImages/'
annot_root = root + '/Annotations/'
annot_files = glob(annot_root + '/*xml')
samples = []
#-----------------------------------------------------------------------
# Process each annotated sample
#-----------------------------------------------------------------------
for fn in tqdm(annot_files, desc=dataset_name, unit='samples'):
with open(fn, 'r') as f:
doc = lxml.etree.parse(f)
filename = image_root + doc.xpath(
'/annotation/filename')[0].text
#---------------------------------------------------------------
# Get the file dimensions
#---------------------------------------------------------------
if not os.path.exists(filename):
continue
img = cv2.imread(filename)
imgsize = Size(img.shape[1], img.shape[0])
#---------------------------------------------------------------
# Get boxes for all the objects
#---------------------------------------------------------------
boxes = []
objects = doc.xpath('/annotation/object')
for obj in objects:
#-----------------------------------------------------------
# Get the properties of the box and convert them to the
# proportional terms
#-----------------------------------------------------------
label = obj.xpath('name')[0].text
xmin = int(float(obj.xpath('bndbox/xmin')[0].text))
xmax = int(float(obj.xpath('bndbox/xmax')[0].text))
ymin = int(float(obj.xpath('bndbox/ymin')[0].text))
ymax = int(float(obj.xpath('bndbox/ymax')[0].text))
center, size = abs2prop(xmin, xmax, ymin, ymax, imgsize)
box = Box(label, self.lname2id[label], center, size)
boxes.append(box)
if not boxes:
continue
sample = Sample(filename, boxes, imgsize)
samples.append(sample)
return samples
