def test_meta(nydus_anchor: NydusAnchor, rafs_conf: RafsConf,
nydus_scratch_image: RafsImage):
anchor = nydus_anchor
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY).enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 8, 5)
dist.generate_tree()
xattr_verifier = verifier.XattrVerifier(anchor.mount_point, dist)
xattr_verifier.scratch(nydus_scratch_image.rootfs())
symlink_verifier = verifier.SymlinkVerifier(anchor.mount_point, dist)
symlink_verifier.scratch()
# Do some meta operations on scratch dir before creating rafs image file.
# Use scratch dir as image source dir as we just prepared test meta into it.
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs = RafsMount(anchor, nydus_scratch_image, rafs_conf)
rafs.thread_num(4).mount()
assert rafs.is_mounted()
xattr_verifier.verify(anchor.mount_point)
symlink_verifier.verify(anchor.mount_point, nydus_scratch_image.rootfs())
workload_gen = WorkloadGen(anchor.mount_point,
nydus_scratch_image.rootfs())
workload_gen.setup_workload_generator()
workload_gen.torture_read(10, 3)
workload_gen.finish_torture_read()
assert workload_gen.io_error == False
assert anchor.check_nydusd_health()
def test_large_file(nydus_anchor, compressor, backend, amplified_size):
_tmp_dir = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
large_file_dir = _tmp_dir.name
dist = Distributor(large_file_dir, 3, 3)
dist.generate_tree()
dist.put_single_file(Size(20, Unit.MB))
dist.put_single_file(Size(10891, Unit.KB))
dist.put_multiple_files(10, Size(2, Unit.MB))
dist.put_multiple_files(10, Size(4, Unit.MB))
image = RafsImage(nydus_anchor, large_file_dir, "bs_large", "blob_large")
image.set_backend(backend).create_image(compressor=compressor)
rafs_conf = (RafsConf(nydus_anchor,
image).enable_rafs_blobcache().amplify_io(
amplified_size).set_rafs_backend(backend,
image=image))
rafs = RafsMount(nydus_anchor, image, rafs_conf)
rafs.thread_num(4).mount()
workload_gen = WorkloadGen(nydus_anchor.mount_point, large_file_dir)
workload_gen.setup_workload_generator()
workload_gen.torture_read(8, 5)
workload_gen.finish_torture_read()
assert not workload_gen.io_error
rafs.umount()
image.clean_up()
class TestRunner:
def __init__(self, heuristics, max_task, max_time, max_cores):
import multiprocessing
from distributor import Distributor
from test import Test
self.heuristics = heuristics
self.max_task = max_task
self.max_time = max_time
self.q = multiprocessing.Queue()
self.populate_queue()
test = Test()
self.d = Distributor(test.run, self.q, self.max_time, max_cores)
def populate_queue(self):
for heuristic in self.heuristics:
for problem_tuple in sorted(__problems__, key=lambda p: -p[1]):
problem = problem_tuple[0]
for task in range(1, min(maxtask(problem) + 1, self.max_task + 1)):
resultfilename = "results/{0}_{1}.res".format(problem, heuristic)
open(resultfilename, "w").close()
self.q.put((problem, task, heuristic))
def start(self):
self.d.start()
def run(self):
print("==== Checking hosts on ====")
# Here we can block by online workers (only run if haver 4 workers),
# but we can work with less then 4 workers, and in my test, I have only 1.
num_hosts = self._get_hosts_on(1)
print("Hosts ON: %d" % num_hosts)
print("==== Creating random matrices ====")
self.matrix_a = self._get_random_matrix(self.a_n, self.a_m)
self.matrix_b = self._get_random_matrix(self.b_n, self.b_m)
print("==== Created Matrices: ====")
self._print_two_matrices(self.matrix_a, self.matrix_b)
self.matrix_divide()
distributor = Distributor("*", "50010")
collector = Collector("*", "50012", 5000)
distributor.send_jobs(self._create_jobs())
# For test, check services in rasp's
self._check_services(1)
results = collector.collect(4)
if 'err' in results:
print("Error in some RasPI: %s" % results['err'])
exit()
print("==== Appending matrices ====")
C1 = self._matrix_sum(results['A1B1'], results['A2B2'])
C2 = self._matrix_sum(results['A3B1'], results['A4B2'])
C = C1 + C2
print("==== Final result: ====")
self._print_matrix(C)
def main(**kwargs):
project = Project()
chain_name = kwargs['chain']
distributor_address = kwargs['distributor']
distributor_tx = kwargs['distributor_tx']
auction_address = kwargs['auction']
auction_tx = kwargs['auction_tx']
claims = kwargs['claims']
with open('build/contracts.json') as json_data:
abis = json.load(json_data)
with project.get_chain(chain_name) as chain:
web3 = chain.web3
auction_abi = abis['DutchAuction']['abi']
distributor_abi = abis['Distributor']['abi']
# Load Populus contract proxy classes
auction = web3.eth.contract(abi=auction_abi, address=auction_address)
distributor = web3.eth.contract(abi=distributor_abi,
address=distributor_address)
print("Web3 provider is", web3.currentProvider)
distrib = Distributor(web3, auction, auction_tx, auction_abi,
distributor, distributor_tx, distributor_abi,
claims)
distrib.distribute()
def test_file_tail(nydus_anchor: NydusAnchor, nydus_scratch_image: RafsImage,
backend):
"""
description: Read data from file tail
- Create several files of different sizes
- Punch hole to each file of which some should have hole tail
- Create rafs image from test scratch directory.
- Mount rafs
- Do some test.
"""
file_size_list = [
Size(1, Unit.KB),
Size(6, Unit.KB),
Size(2, Unit.MB),
Size(10034, Unit.KB),
]
file_list = []
dist = Distributor(nydus_anchor.scratch_dir, 2, 2)
dist.generate_tree()
for f_s in file_size_list:
f_name = dist.put_single_file(f_s)
file_list.append(f_name)
# Punch hole
with utils.pushd(nydus_anchor.scratch_dir):
with open(f_name, "a+b") as f:
fallocate(
f,
f_s.B - 500,
1000,
mode=FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
)
nydus_scratch_image.set_backend(backend).create_image()
rafs_conf = RafsConf(nydus_anchor, nydus_scratch_image)
rafs_conf.set_rafs_backend(backend, image=nydus_scratch_image)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
with utils.pushd(nydus_anchor.mount_point):
for name in file_list:
with open(name, "rb") as f:
size = os.stat(name).st_size
f.seek(size - 300)
buf = f.read(1000)
assert len(buf) == 300
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
for f in file_list:
wg.verify_single_file(os.path.join(nydus_anchor.mount_point, f))
assert wg.io_error == False
def distributor_process(self, listen_port, run_time):
if not self.printing:
sys.stdout = open(os.devnull, 'w')
d = Distributor(port=listen_port,
life_time=run_time,
speedup=self.speedup,
printing=self.printing)
print("Created distributor process " + str(listen_port))
# let the distributor listen to all incoming messages until lifetime is up
d.read_ports()
print("Distributor closed on port " + str(listen_port))
def put_files(dist: Distributor, f_type, count, size):
"""Example:
depth: 4
width: 4
layers:
- layer1:
- size: 10KB
type: regular
count: 2000
- size: 12MB
type: regular
count: 10
- size: 90MB
type: regular
count: 1
- type: symlink
count: 100
"""
logging.info("putting %s, count %d", f_type, count)
if f_type == "regular":
size_in_bytes = utils.parse_size(size)
dist.put_multiple_files(count, Size(size_in_bytes))
elif f_type == "dir":
dist.put_directories(count)
elif f_type == "symlink":
dist.put_symlinks(count)
elif f_type == "hardlink":
dist.put_hardlinks(count)
def test_syscalls(
nydus_anchor: NydusAnchor,
nydus_scratch_image: RafsImage,
rafs_conf: RafsConf,
):
syscall_helper = "framework/test_syscalls"
ret, _ = utils.execute(
["gcc", "framework/test_syscalls.c", "-o", syscall_helper],
shell=False,
print_output=True,
)
assert ret
dist = Distributor(nydus_scratch_image.rootfs(), 2, 2)
dist.generate_tree()
dist.put_single_file(Size(8, Unit.KB),
pos=nydus_scratch_image.rootfs(),
name="xattr_no_kv")
dist.put_single_file_with_xattr(
Size(8, Unit.KB),
("trusted.nydus.key", ""),
pos=nydus_scratch_image.rootfs(),
name="xattr_empty_value",
)
dist.put_single_file_with_xattr(
Size(8, Unit.KB),
("trusted.nydus.key", "1234567890"),
pos=nydus_scratch_image.rootfs(),
name="xattr_insufficient_buffer",
)
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs_conf.enable_xattr().set_rafs_backend(Backend.BACKEND_PROXY)
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
for no in [58]:
ret, _ = utils.execute(
[syscall_helper, nydus_anchor.mount_point,
str(no)],
shell=False,
print_output=True,
)
assert ret
def start(cls):
cls.init()
# 开启任务分发线程
Distributor(FETCH_MID_FROM, FETCH_MID_TO + 1).start()
# 开启爬虫线程
for i in range(0, THREADS_NUM):
Worker(f'Worker-{i}').start()
def test_layered_rebuild(
nydus_anchor,
nydus_scratch_image: RafsImage,
nydus_scratch_parent_image: RafsImage,
rafs_conf: RafsConf,
backend,
):
"""
title: Layered image rebuild
description:
- Parent and upper have files whose contents are exactly the same.
- Use files stats to check if file is overlayed.
- Files with the same name but different modes.
- Files with xattr in parent should be shadowed.
pass_criteria:
- Mount successfully.
- No data corruption.
"""
rafs_conf.set_rafs_backend(backend)
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()
parent_rootfs = nydus_scratch_parent_image.rootfs()
upper_rootfs = nydus_scratch_image.rootfs()
nydus_anchor.mount_overlayfs(
[nydus_scratch_image.rootfs(),
nydus_scratch_parent_image.rootfs()])
shared_files = []
dist_parent = Distributor(parent_rootfs, 6, 4)
dist_parent.generate_tree()
shared_files.extend(dist_parent.put_multiple_files(100, Size(64, Unit.KB)))
shared_files.extend(dist_parent.put_symlinks(30))
shared_files.extend(dist_parent.put_hardlinks(30))
xattr_verifier = XattrVerifier(parent_rootfs, dist_parent)
Whiteout.mirror_files(shared_files, parent_rootfs, upper_rootfs)
xattr_verifier.scratch(parent_rootfs)
nydus_scratch_parent_image.set_backend(backend).create_image()
nydus_scratch_image.set_backend(backend).create_image(
parent_image=nydus_scratch_parent_image)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_anchor.overlayfs)
workload_gen.setup_workload_generator()
xattr_verifier.verify(nydus_anchor.mount_point)
assert workload_gen.verify_entire_fs()
workload_gen.torture_read(5, 4)
workload_gen.finish_torture_read()
def test_deep_directory(nydus_anchor, rafs_conf: RafsConf,
nydus_scratch_image: RafsImage):
dist = Distributor(nydus_anchor.scratch_dir, 100, 1)
dist.generate_tree()
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY)
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
wg.setup_workload_generator()
wg.torture_read(8, 5)
wg.finish_torture_read()
assert wg.verify_entire_fs()
def __init__(self, heuristics, max_task, max_time, max_cores):
import multiprocessing
from distributor import Distributor
from test import Test
self.heuristics = heuristics
self.max_task = max_task
self.max_time = max_time
self.q = multiprocessing.Queue()
self.populate_queue()
test = Test()
self.d = Distributor(test.run, self.q, self.max_time, max_cores)
def test_hardlink(nydus_anchor: NydusAnchor, nydus_scratch_image,
rafs_conf: RafsConf):
dist = Distributor(nydus_scratch_image.rootfs(), 8, 6)
dist.generate_tree()
hardlink_verifier = verifier.HardlinkVerifier(nydus_scratch_image.rootfs(),
dist)
hardlink_verifier.scratch()
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
hardlink_verifier.verify(nydus_anchor.mount_point,
nydus_scratch_image.rootfs())
wg.setup_workload_generator()
wg.io_read(3)
nydus_anchor.check_nydusd_health()
assert wg.io_error == False
def __init__(self, UPLOAD=True, USE_GUI=True, USE_FILESTREAM=False):
self._socket_address = 'src/client/data_stream.sock'
self._receiver = Receiver(self._socket_address)
self._geo_listen = Geometry()
self._frequency = 15
if USE_FILESTREAM:
self._stream = FileStream(
"gen_data/downtown-east2_only_turn_sigs_speed_lims.json",
self._socket_address)
self._send_frequency = 1
self._speed_limit = None
self._speed_time = None
self._gui = None
self._forgot_signals_event = threading.Event()
self._speeding_event = threading.Event()
self._aggressive_event = threading.Event()
self._green_event = threading.Event()
self._green_event.direction = None
self._speeding_event.speed_limit = None
self._last_turn_forget = None
self._turn_analyzer = TurnSignalAnalyzer(self._forgot_signals_event)
self._speeding_analyzer = SpeedingAnalyzer(self._speeding_event)
self._green = Green(1, self._green_event)
self._aggressive_analyze = AggressiveAnalyzer(self._aggressive_event)
self._evaluatebox_last_time = None
if USE_GUI:
self._gui = GUI(self._receiver.is_alive)
t = threading.Thread(target=self._mainloop, daemon=True)
t.daemon = True
t.start()
if UPLOAD:
self._distributor = Distributor(
'/src/client/upload_stream.sock', self._send_frequency)
s = threading.Thread(target=self._sender, daemon=True)
s.start()
if USE_GUI:
self._gui.mainloop()
else:
self._receiver.join() # keep main from exiting
def test_blobcache(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
nydus_image: RafsImage,
nydus_scratch_parent_image: RafsImage,
thread_cnt,
io_duration,
):
dist_parent = Distributor(nydus_scratch_parent_image.rootfs(), 6, 4)
dist_parent.generate_tree()
dist_parent.put_multiple_files(20, Size(4, Unit.KB))
hint_files_parent = [os.path.join("/", p) for p in dist_parent.files[-20:]]
hint_files_parent = "\n".join(hint_files_parent[-1:])
nydus_scratch_parent_image.set_backend(Backend.OSS).create_image()
# shutil.rmtree(nydus_scratch_parent_image.rootfs())
nydus_image.set_backend(Backend.OSS).create_image(
readahead_policy="fs",
parent_image=nydus_scratch_parent_image,
readahead_files=hint_files_parent.encode(),
)
nydus_anchor.mount_overlayfs(
[nydus_image.rootfs(),
nydus_scratch_parent_image.rootfs()])
rafs_conf.enable_rafs_blobcache().set_rafs_backend(Backend.OSS)
rafs_conf.enable_fs_prefetch()
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_image, rafs_conf)
rafs.thread_num(4).mount()
nc = NydusAPIClient(rafs.get_apisock())
m = nc.get_blobcache_metrics()
# TODO: Open this check when prefetch is fixed.
time.sleep(1)
assert m["prefetch_data_amount"] != 0
wg = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
wg.setup_workload_generator()
wg.torture_read(thread_cnt, io_duration)
wg.finish_torture_read()
def test_signal_handling(nydus_anchor: NydusAnchor,
nydus_scratch_image: RafsImage, rafs_conf: RafsConf,
sig):
dist = Distributor(nydus_scratch_image.rootfs(), 2, 2)
dist.generate_tree()
dist.put_multiple_files(5, Size(2, Unit.KB))
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
victim = os.path.join(nydus_anchor.mount_point, dist.files[-1])
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
fd = os.open(victim, os.O_RDONLY)
assert rafs.is_mounted()
os.kill(rafs.p.pid, sig)
time.sleep(3)
assert not os.path.ismount(nydus_anchor.mount_point)
rafs.p.wait()
def test_backend_swap(nydus_anchor, nydus_scratch_image: RafsImage,
rafs_conf: RafsConf):
dist = Distributor(nydus_scratch_image.rootfs(), 5, 4)
dist.generate_tree()
dist.put_multiple_files(100, Size(2, Unit.MB))
nydus_scratch_image.set_backend(Backend.OSS).create_image(
readahead_policy="fs", readahead_files="/".encode())
rafs_conf.set_rafs_backend(
Backend.OSS).enable_rafs_blobcache().enable_fs_prefetch(
threads_count=7, bandwidth_rate=Size(2, Unit.MB).B)
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, None, rafs_conf, with_defaults=False)
rafs.thread_num(4).set_mountpoint(
nydus_anchor.mount_point).apisock("api_sock").mount()
nc = NydusAPIClient(rafs.get_apisock())
nc.pseudo_fs_mount(nydus_scratch_image.bootstrap_path, "/",
rafs_conf.path(), None)
nc.umount_rafs("/")
assert len(os.listdir(nydus_anchor.mount_point)) == 0
mp = "/pseudo1"
nc.pseudo_fs_mount(nydus_scratch_image.bootstrap_path, mp,
rafs_conf.path(), None)
rafs_conf_2nd = RafsConf(nydus_anchor, nydus_scratch_image)
rafs_conf_2nd.set_rafs_backend(
Backend.LOCALFS,
image=nydus_scratch_image).enable_rafs_blobcache().enable_fs_prefetch(
threads_count=3, bandwidth_rate=Size(1, Unit.MB).B)
rafs_conf_2nd.dump_rafs_conf()
new_image = (RafsImage(
nydus_anchor, nydus_scratch_image.rootfs()).set_backend(
Backend.LOCALFS).create_image(readahead_policy="fs",
readahead_files="/".encode()))
# TODO: Once upon a time, more than one fd are open. Check why this happens.
wg = WorkloadGen(
os.path.join(nydus_anchor.mount_point, mp.strip("/")),
nydus_scratch_image.rootfs(),
)
wg.setup_workload_generator()
wg.torture_read(8, 8)
for i in range(1, 50):
logging.debug("swap for the %dth time", i)
nc.swap_backend(mp, new_image.bootstrap_name, rafs_conf_2nd.path())
# assert nc.get_blobcache_metrics(mp)["prefetch_workers"] == 3
time.sleep(0.2)
nc.swap_backend(mp, nydus_scratch_image.bootstrap_name,
rafs_conf.path())
utils.clean_pagecache()
wg.finish_torture_read()
assert wg.io_error == False
nc.umount_rafs(mp)
utils.clean_pagecache()
def test_blobcache_recovery(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
nydus_scratch_image: RafsImage,
):
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY)
rafs_conf.enable_fs_prefetch()
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 8, 2)
dist.generate_tree()
dirs = dist.put_directories(20)
dist.put_multiple_files(100, Size(64, Unit.KB))
dist.put_symlinks(30)
dist.put_hardlinks(20)
dist.put_multiple_files(40, Size(64, Unit.KB))
dist.put_single_file(Size(3, Unit.MB), name="test")
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.prefetch_files("/").mount()
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
wg.setup_workload_generator()
wg.torture_read(4, 4)
# Hopefully, prefetch can be done in 5 secondes.
time.sleep(5)
wg.finish_torture_read()
rafs.umount()
rafs2 = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs2.mount()
wg.torture_read(4, 4)
time.sleep(0.5)
nc = NydusAPIClient(rafs2.get_apisock())
begin = nc.get_backend_metrics()["read_amount_total"]
time.sleep(1)
end = nc.get_backend_metrics()["read_amount_total"]
assert end == begin == 0
wg.finish_torture_read()
def test_stargz(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
nydus_scratch_image: RafsImage,
):
"""
Example command:
stargzify file:`pwd`/foo.tar.gz foo.stargz
"""
intermediator = "tmp.tar.gz"
stargz_image = "tmp.stargz"
dist = Distributor(nydus_scratch_image.rootfs(), 4, 4)
dist.generate_tree()
dirs = dist.put_directories(20)
dist.put_multiple_files(100, Size(64, Unit.KB))
dist.put_symlinks(30)
dist.put_multiple_files(10, Size(4, Unit.MB))
dist.put_hardlinks(20)
dist.put_single_file(Size(3, Unit.MB), name="test")
try:
shutil.rmtree("origin")
except Exception:
pass
shutil.copytree(nydus_scratch_image.rootfs(), "origin", symlinks=True)
utils.write_tar_gz(nydus_scratch_image.rootfs(), intermediator)
cmd = ["framework/bin/stargzify", f"file:{intermediator}", stargz_image]
utils.execute(cmd)
toc = utils.parse_stargz(stargz_image)
image = RafsImage(
nydus_anchor,
toc,
"bootstrap_scratched",
"blob_scratched",
clear_from_oss=True,
)
# This is a trick since blob name is usually a temp file created when RafsImage instantiated.
# framework will upload stargz to oss.
image.blob_abs_path = stargz_image
image.set_backend(Backend.OSS).set_param(
"blob-id", uuid.uuid4()).create_image(from_stargz=True)
rafs_conf.set_rafs_backend(Backend.OSS)
rafs_conf.enable_rafs_blobcache(is_compressed=True)
rafs = RafsMount(nydus_anchor, image, rafs_conf)
rafs.mount()
wg = WorkloadGen(nydus_anchor.mount_point, "origin")
wg.verify_entire_fs()
wg.setup_workload_generator()
wg.torture_read(4, 4)
wg.finish_torture_read()
assert not wg.io_error
def test_specified_prefetch(
nydus_anchor: NydusAnchor,
rafs_conf: RafsConf,
nydus_scratch_image: RafsImage,
backend,
):
"""
description:
Nydusd can have a list including files and directories input when started.
Then it can prefetch files from backend per as to the list.
"""
rafs_conf.set_rafs_backend(backend)
rafs_conf.enable_fs_prefetch(prefetch_all=True)
rafs_conf.enable_rafs_blobcache()
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 8, 2)
dist.generate_tree()
dirs = dist.put_directories(20)
dist.put_multiple_files(100, Size(64, Unit.KB))
dist.put_symlinks(30)
dist.put_hardlinks(20)
dist.put_multiple_files(40, Size(64, Unit.KB))
dist.put_single_file(Size(3, Unit.MB), name="test")
nydus_scratch_image.set_backend(backend).create_image()
prefetching_files = dirs
prefetching_files += dist.files[:-10]
prefetching_files += dist.dirs[:-5]
prefetching_files += dist.symlinks[:-10]
# Fuzz
prefetching_files.append("/a/b/c/d")
prefetching_files.append(os.path.join("/", "f/g/h/"))
specified_dirs = " ".join(
[os.path.join("/", d) for d in prefetching_files])
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.prefetch_files(specified_dirs).mount()
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
nc = NydusAPIClient(rafs.get_apisock())
wg.setup_workload_generator()
blobcache_metrics = nc.get_blobcache_metrics()
wg.torture_read(5, 10)
while blobcache_metrics["prefetch_workers"] != 0:
time.sleep(0.5)
blobcache_metrics = nc.get_blobcache_metrics()
begin = nc.get_backend_metrics()["read_amount_total"]
time.sleep(1)
end = nc.get_backend_metrics()["read_amount_total"]
assert end == begin
wg.finish_torture_read()
def test_prefetch_without_cache(nydus_anchor: NydusAnchor,
nydus_scratch_image: RafsImage,
rafs_conf: RafsConf):
"""Files prefetch test
1. relative hinted prefetch files
2. absolute hinted prefetch files
3. source rootfs root dir.
"""
rafs_conf.enable_fs_prefetch().set_rafs_backend(Backend.BACKEND_PROXY)
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 4, 4)
dist.generate_tree()
dist.put_directories(20)
dist.put_multiple_files(40, Size(8, Unit.KB))
dist.put_hardlinks(6)
dist.put_multiple_chinese_files(random.randint(20, 28), Size(20, Unit.KB))
hint_files = ["/"]
hint_files.extend(dist.files)
hint_files.extend(dist.dirs)
hint_files.extend(dist.symlinks)
hint_files.extend(dist.hardlinks)
hint_files = [os.path.join("/", p) for p in hint_files]
hint_files = "\n".join(hint_files)
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image(
readahead_policy="fs", readahead_files=hint_files.encode())
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.mount()
assert rafs.is_mounted()
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_scratch_image.rootfs())
# TODO: Run several parallel read workers against the mount_point
workload_gen.setup_workload_generator()
workload_gen.torture_read(8, 5)
workload_gen.finish_torture_read()
assert NydusAnchor.check_nydusd_health()
assert not workload_gen.io_error
assert rafs.is_mounted()
rafs.umount()
def test_different_partitions(nydus_anchor: NydusAnchor, rafs_conf):
loop_file_1 = tempfile.NamedTemporaryFile(suffix="loop")
loop_file_2 = tempfile.NamedTemporaryFile(suffix="loop")
loop_mnt_1 = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
loop_mnt_2 = tempfile.TemporaryDirectory(dir=nydus_anchor.workspace)
os.posix_fallocate(loop_file_1.fileno(), 0, Size(400, Unit.MB).B)
os.posix_fallocate(loop_file_2.fileno(), 0, Size(400, Unit.MB).B)
utils.execute(["mkfs.ext4", "-F", loop_file_1.name])
utils.execute(["mkfs.ext4", "-F", loop_file_2.name])
utils.execute(["mount", loop_file_1.name, loop_mnt_1.name])
utils.execute(["mount", loop_file_2.name, loop_mnt_2.name])
# TODO: Put more special files into
dist1 = Distributor(loop_mnt_1.name, 5, 7)
dist1.generate_tree()
dist1.put_multiple_files(100, Size(12, Unit.KB))
dist2 = Distributor(loop_mnt_2.name, 5, 7)
dist2.generate_tree()
dist2.put_symlinks(20)
dist2.put_multiple_files(50, Size(12, Unit.KB))
Whiteout.mirror_files(dist2.files[:20], loop_mnt_2.name, loop_mnt_1.name)
parent_image = (RafsImage(nydus_anchor, loop_mnt_1.name).set_backend(
Backend.OSS).create_image())
image = RafsImage(nydus_anchor, loop_mnt_2.name)
image.set_backend(Backend.OSS).create_image(parent_image=parent_image)
rafs_conf.set_rafs_backend(Backend.OSS)
rafs = RafsMount(nydus_anchor, image, rafs_conf)
rafs.mount()
nydus_anchor.mount_overlayfs([image.rootfs(), parent_image.rootfs()])
wg = WorkloadGen(nydus_anchor.mount_point, nydus_anchor.overlayfs)
wg.setup_workload_generator()
wg.torture_read(5, 5)
wg.finish_torture_read()
utils.execute(["umount", loop_mnt_1.name])
utils.execute(["umount", loop_mnt_2.name])
nydus_anchor.umount_overlayfs()
from geneticalg import GeneticAlgorithm
from fitness import TestFitness
from distributor import Distributor
from tournament import Tournament
from elitism import Elitism
from singlemut import SingleMutator
from multmut import MultipleMutator
from annealer import Annealer
from onepointcross import OnePointCrossover
from datetime import datetime
import sys
# Search using a genetic algorithm
evaluator = Distributor(25)
parent_selector = Tournament(4)
survival_selector = Elitism()
breeder = OnePointCrossover()
mutator = Annealer(18, 3)
searcher = GeneticAlgorithm(25, 500000, evaluator, parent_selector, breeder, 4,
mutator, 1, survival_selector)
searcher.search()
#this file can bei copy pasted into a python consol started in the distributor directory.
#It generates a distributor, a monitor and the tasksets. it also adds the taskst to the distributor.
# After the code in this file the distributor can be configured with its function (set_max_machine_value(20) etc.
import sys
sys.path.append('../')
from distributor import Distributor
from taskgen.setgenerator import SetGenerator
from monitors.sql_monitor import SQLMonitor
dis = Distributor()
gen = SetGenerator()
sqlmon = SQLMonitor()
#creates the generator with Tasksets consiting of 1 up to 5 Tasks
#adds them with add_job to the job_queue of the distributor
for a in range(1, 6):
list1 = gen.specialSet(a)
for l in list1:
setvar = l.variants()
print(len(list(setvar))) #prints the length of the Taskset
dis.add_job(l, sqlmon)
def test_prefetch_with_cache(
nydus_anchor,
nydus_scratch_image: RafsImage,
rafs_conf: RafsConf,
thread_cnt,
compressor,
is_cache_compressed,
converter,
items,
):
"""
title: Prefetch from various backend
description:
- Enable rafs backend blob cache, as it is disabled by default
pass_criteria:
- Rafs can be mounted.
- Rafs can be unmounted.
"""
dist = Distributor(nydus_scratch_image.rootfs(), 4, 4)
dist.generate_tree()
dist.put_directories(20)
dist.put_multiple_files(40, Size(3, Unit.MB))
dist.put_multiple_files(10, Size(5, Unit.MB))
dist.put_hardlinks(6)
dist.put_multiple_chinese_files(random.randint(20, 28), Size(20, Unit.KB))
nydus_scratch_image.set_backend(Backend.LOCALFS).create_image(
image_bin=converter,
compressor=compressor,
readahead_policy="fs",
readahead_files="/".encode(),
)
rafs_conf.enable_rafs_blobcache(
is_compressed=is_cache_compressed).enable_fs_prefetch()
rafs_conf.set_rafs_backend(Backend.LOCALFS, image=nydus_scratch_image)
if len(items) > 0:
for i in items:
item = RafsConf.__dict__[i]
item(rafs_conf)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.thread_num(6).mount()
nc = NydusAPIClient(rafs.get_apisock())
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_scratch_image.rootfs())
m = nc.get_blobcache_metrics()
time.sleep(0.3)
assert m["prefetch_data_amount"] != 0
workload_gen.verify_entire_fs()
workload_gen.setup_workload_generator()
workload_gen.torture_read(thread_cnt, 6)
assert NydusAnchor.check_nydusd_health()
workload_gen.finish_torture_read()
assert not workload_gen.io_error
def test_build_image(nydus_anchor, nydus_scratch_image: RafsImage,
rafs_conf: RafsConf):
"""
title: Build nydus image
description: Build nydus image from rootfs generating proper bootstrap and
blob
pass_criteria:
- Image can successfully builded and mounted
- Rafs can be unmounted and do a small account of read io and attr
operation
- Try let image builder upload blob itself.
"""
dist = Distributor(nydus_scratch_image.rootfs(), 80, 1)
dist.generate_tree()
dist.put_directories(100)
dist.put_hardlinks(90)
dist.put_symlinks(200)
dist.put_multiple_files(random.randint(20, 28), Size(10, Unit.MB))
dist.put_multiple_chinese_files(random.randint(20, 28), Size(20, Unit.KB))
Whiteout().whiteout_one_file(nydus_scratch_image.rootfs(),
"i/am/troublemaker/foo")
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs_conf.set_rafs_backend(backend_type=Backend.BACKEND_PROXY)
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
rafs.mount()
assert wg.verify_entire_fs()
rafs.umount()
class Main:
def __init__(self, UPLOAD=True, USE_GUI=True, USE_FILESTREAM=False):
self._socket_address = 'src/client/data_stream.sock'
self._receiver = Receiver(self._socket_address)
self._geo_listen = Geometry()
self._frequency = 15
if USE_FILESTREAM:
self._stream = FileStream(
"gen_data/downtown-east2_only_turn_sigs_speed_lims.json",
self._socket_address)
self._send_frequency = 1
self._speed_limit = None
self._speed_time = None
self._gui = None
self._forgot_signals_event = threading.Event()
self._speeding_event = threading.Event()
self._aggressive_event = threading.Event()
self._green_event = threading.Event()
self._green_event.direction = None
self._speeding_event.speed_limit = None
self._last_turn_forget = None
self._turn_analyzer = TurnSignalAnalyzer(self._forgot_signals_event)
self._speeding_analyzer = SpeedingAnalyzer(self._speeding_event)
self._green = Green(1, self._green_event)
self._aggressive_analyze = AggressiveAnalyzer(self._aggressive_event)
self._evaluatebox_last_time = None
if USE_GUI:
self._gui = GUI(self._receiver.is_alive)
t = threading.Thread(target=self._mainloop, daemon=True)
t.daemon = True
t.start()
if UPLOAD:
self._distributor = Distributor(
'/src/client/upload_stream.sock', self._send_frequency)
s = threading.Thread(target=self._sender, daemon=True)
s.start()
if USE_GUI:
self._gui.mainloop()
else:
self._receiver.join() # keep main from exiting
def _mainloop(self):
while True:
sleep(1 / self._frequency)
if self._evaluatebox_last_time is None or self._evaluatebox_last_time - time.time() > 5:
self._evaluatebox_last_time = time.time()
self._gui._evaluate_box.set_value(EvaluateBox.GOOD)
if self._gui is None:
continue
if Geometry._pos is not None:
if Geometry._inter_pos:
self._gui.set_coords(*Geometry._inter_pos)
else:
self._gui.set_coords(*Geometry._pos)
if Geometry._marker is not None:
self._gui.set_marker(Geometry._marker)
if self._forgot_signals_event.is_set():
self._last_turn_forget = time.time()
self._gui._turn_signal_sym.set_vibrate(10)
self._forgot_signals_event.clear()
if self._last_turn_forget is not None and time.time() - self._last_turn_forget > 3:
self._gui._turn_signal_sym.set_vibrate(0)
if self._evaluatebox_last_time is None or time.time() - \
self._evaluatebox_last_time > 3:
if self._aggressive_event.is_set():
self._gui._evaluate_box.set_value(EvaluateBox.BAD)
self._evaluatebox_last_time = time.time()
self._aggressive_event.clear()
elif self._green_event.is_set():
if self._green_event.direction == 'up':
self._gui._evaluate_box.set_value(EvaluateBox.GEAR_UP)
elif self._green_event.direction == 'down':
self._gui._evaluate_box.set_value(
EvaluateBox.GEAR_DOWN)
self._evaluatebox_last_time = time.time()
self._green_event.clear()
else:
self._gui._evaluate_box.set_value(EvaluateBox.GOOD)
if self._speed_limit != self._speeding_event.speed_limit:
self._speed_limit = self._speeding_event.speed_limit
self._gui.set_speed_limit(self._speed_limit)
if self._speeding_event.is_set():
offset = 3
self._gui._speed_limit_sym.set_vibrate(
offset + (10 - offset) * self._speeding_event.speeding_percentage)
self._speeding_event.clear()
self._speed_time = time.time()
elif self._speed_time is not None and time.time() - self._speed_time > 2:
self._gui._speed_limit_sym.set_vibrate(0)
def _sender(self):
while True:
self._distributor.send()
sleep(1 / self._send_frequency)
def test_prefetch_with_cache(
nydus_anchor,
nydus_scratch_image: RafsImage,
rafs_conf: RafsConf,
thread_cnt,
compressor,
is_cache_compressed,
):
"""
title: Prefetch from various backend
description:
- Enable rafs backend blob cache, as it is disabled by default
pass_criteria:
- Rafs can be mounted.
- Rafs can be unmounted.
"""
rafs_conf.enable_rafs_blobcache(is_compressed=is_cache_compressed)
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY, prefix="object_prefix/")
rafs_conf.enable_fs_prefetch(threads_count=4,
bandwidth_rate=Size(40, Unit.MB).B)
rafs_conf.dump_rafs_conf()
dist = Distributor(nydus_scratch_image.rootfs(), 4, 4)
dist.generate_tree()
dist.put_directories(20)
dist.put_multiple_files(40, Size(3, Unit.MB))
dist.put_hardlinks(6)
dist.put_multiple_chinese_files(random.randint(20, 28), Size(20, Unit.KB))
nydus_scratch_image.set_backend(Backend.BACKEND_PROXY,
prefix="object_prefix/").create_image(
compressor=compressor,
readahead_policy="fs",
readahead_files="/".encode(),
)
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
rafs.thread_num(4).mount()
nc = NydusAPIClient(rafs.get_apisock())
workload_gen = WorkloadGen(nydus_anchor.mount_point,
nydus_scratch_image.rootfs())
m = nc.get_blobcache_metrics()
time.sleep(0.3)
assert m["prefetch_data_amount"] != 0
workload_gen.setup_workload_generator()
workload_gen.torture_read(thread_cnt, 10)
assert NydusAnchor.check_nydusd_health()
workload_gen.finish_torture_read()
assert not workload_gen.io_error
# In this way, we can check if nydusd is crashed.
assert rafs.is_mounted()
rafs.umount()
try:
shutil.rmtree(generated_parent_rootfs)
except FileNotFoundError:
pass
os.mkdir(generated_rootfs)
os.mkdir(generated_parent_rootfs)
fs_dist = define_fs_structure(dist_define)
depth = fs_dist["depth"]
width = fs_dist["width"]
layers_desc = fs_dist["layers"]
dist = Distributor(generated_rootfs, depth, width)
dist.generate_tree()
for ld in layers_desc:
with utils.timer("Generating test layer"):
for d in ld.values():
for f in d:
try:
size = f["size"]
except KeyError:
size = None
put_files(dist, f["type"], f["count"], size)
parent_dist = Distributor(generated_parent_rootfs, depth, width)
parent_dist.generate_tree()
def setUp(self):
self.pwd = dirname(realpath(__file__))
self.D = Distributor(join(self.pwd, "configs"),
join(self.pwd, "config_t.ini"))
def test_pseudo_fs(nydus_anchor, nydus_image, rafs_conf: RafsConf):
nydus_image.set_backend(Backend.BACKEND_PROXY).create_image()
rafs_conf.set_rafs_backend(Backend.BACKEND_PROXY)
rafs = RafsMount(nydus_anchor, None, rafs_conf)
rafs.mount()
time.sleep(1)
nc = NydusAPIClient(rafs.get_apisock())
try:
shutil.rmtree("pseudo_fs_scratch")
except FileNotFoundError:
pass
scratch_rootfs = shutil.copytree(nydus_image.rootfs(),
"pseudo_fs_scratch",
symlinks=True)
dist = Distributor(scratch_rootfs, 5, 5)
dist.generate_tree()
dist.put_multiple_files(20, Size(8, Unit.KB))
###
suffix = "1"
image = RafsImage(
nydus_anchor,
scratch_rootfs,
"bs" + suffix,
"blob" + suffix,
)
conf = RafsConf(nydus_anchor)
conf.enable_fs_prefetch()
conf.enable_validation()
conf.set_rafs_backend(Backend.BACKEND_PROXY)
conf.dump_rafs_conf()
image.set_backend(Backend.BACKEND_PROXY).create_image()
nc.pseudo_fs_mount(image.bootstrap_path, f"/pseudo{suffix}", conf.path(),
None)
###
suffix = "2"
image = RafsImage(
nydus_anchor,
scratch_rootfs,
"bs" + suffix,
"blob" + suffix,
)
conf = RafsConf(nydus_anchor)
conf.enable_rafs_blobcache()
conf.enable_validation()
conf.enable_records_readahead()
conf.set_rafs_backend(Backend.BACKEND_PROXY)
conf.dump_rafs_conf()
dist.put_multiple_files(20, Size(8, Unit.KB))
image.set_backend(Backend.BACKEND_PROXY).create_image()
nc.pseudo_fs_mount(image.bootstrap_path, f"/pseudo{suffix}", conf.path(),
None)
###
suffix = "3"
image = RafsImage(
nydus_anchor,
scratch_rootfs,
"bs" + suffix,
"blob" + suffix,
)
conf = RafsConf(nydus_anchor)
conf.enable_rafs_blobcache()
conf.enable_records_readahead()
conf.set_rafs_backend(Backend.BACKEND_PROXY)
conf.dump_rafs_conf()
dist.put_multiple_files(20, Size(8, Unit.KB))
image.set_backend(Backend.BACKEND_PROXY).create_image()
nc.pseudo_fs_mount(image.bootstrap_path, f"/pseudo{suffix}", conf.path(),
None)
wg1 = WorkloadGen(os.path.join(nydus_anchor.mount_point, "pseudo1"),
scratch_rootfs)
wg2 = WorkloadGen(os.path.join(nydus_anchor.mount_point, "pseudo2"),
scratch_rootfs)
wg3 = WorkloadGen(os.path.join(nydus_anchor.mount_point, "pseudo3"),
scratch_rootfs)
time.sleep(2)
wg1.setup_workload_generator()
wg2.setup_workload_generator()
wg3.setup_workload_generator()
wg1.torture_read(4, 8)
wg2.torture_read(4, 8)
wg3.torture_read(4, 8)
wg1.finish_torture_read()
wg2.finish_torture_read()
wg3.finish_torture_read()
# TODO: Temporarily disable the verification as hard to select `verify dir`
# assert wg1.verify_entire_fs()
# assert wg2.verify_entire_fs()
# assert wg3.verify_entire_fs()
nc.umount_rafs("/pseudo1")
nc.umount_rafs("/pseudo2")
nc.umount_rafs("/pseudo3")
class TestDistributor(TestCase):
def setUp(self):
self.pwd = dirname(realpath(__file__))
self.D = Distributor(join(self.pwd, "configs"),
join(self.pwd, "config_t.ini"))
def test_config(self):
with self.assertRaises(SystemExit):
try:
with self.assertLogs():
Distributor(join(self.pwd, "configs"),
join(self.pwd, "config_t_bad.ini"))
except AttributeError:
Distributor(join(self.pwd, "configs"),
join(self.pwd, "config_t_bad.ini"))
def test_parse_nginx(self):
self.D.parse_nginx(join(self.pwd, "configs", "nginx.test_server.all"))
self.assertSetEqual({'promo', 'stream', 'web'},
set(self.D.services.keys()))
self.assertSetEqual({'example.net', 'example.net:443'},
set(self.D.services['web'].keys()))
self.assertListEqual(
[{'2.2.2.2'}, {'2.2.2.2:443'}],
sorted((s['test_server']
for _, s in self.D.services['web'].items()),
key=lambda k: next(iter(k)))
)
self.assertSetEqual({'example.com'},
set(self.D.services['promo'].keys()))
self.assertSetEqual(
{'1.1.1.1'},
self.D.services['promo']['example.com']['test_server']
)
self.assertSetEqual({'git.example.net', 'rdp.example.com'},
set(self.D.services['stream'].keys()))
self.assertListEqual(
[{'1.1.1.1:443'}, {'2.2.2.2:22'}],
sorted((s['test_server']
for _, s in self.D.services['stream'].items()),
key=lambda k: next(iter(k)))
)
self.assertSetEqual({'web'}, set(self.D._api.keys()))
self.assertSetEqual({'example.com', 'example.net', 'example.net:443'},
set(self.D._api['web']['servers'].keys()))
self.assertSetEqual({'long', 'spdy_long'},
set(self.D._api['web']['log_format'].keys()))
def test_parse_haproxy(self):
self.D.parse_haproxy(join(self.pwd, "configs",
"haproxy.test_server.all"))
self.assertSetEqual({'http', 'ssh'}, set(self.D.services.keys()))
self.assertSetEqual({'https.example.org'},
set(self.D.services['http'].keys()))
self.assertEqual(
{'3.3.3.3:443'},
self.D.services['http']['https.example.org']['test_server']
)
self.assertSetEqual({'ssh.example.org'},
set(self.D.services['ssh'].keys()))
self.assertEqual(
{'3.3.3.3:22'},
self.D.services['ssh']['ssh.example.org']['test_server']
)
def test_index(self):
index = self.D.index()
self.assertIn("<html", index)
self.assertNotIn("{%", index)
self.assertIn("example.com", index)
self.assertIn("example.net", index)
def test_get_cats(self):
self.assertListEqual(sorted(self.D.services.keys()), self.D.get_cats())
def test_write_nginx(self):
self.D.parse_nginx(join(self.pwd, "configs", "nginx.test_server.all"))
self.D.parse_nginx(join(self.pwd, "configs", "nginx.front1.all"))
web_table = self.D.write("web")
self.assertIn("ch-test_server", web_table)
self.assertIn("ch-author", web_table)
self.assertIn("th:nth-child", web_table)
self.assertIn("<table", web_table)
self.assertIn("</table>", web_table)
self.assertIn("<th>front</th>", web_table)
self.assertIn('class="zone', web_table)
self.assertIn('id="web-table"', web_table)
self.assertIn('https://example.net', web_table)
promo_table = self.D.write("promo")
self.assertIn("<table", promo_table)
self.assertIn("</table>", promo_table)
self.assertNotIn('class="zone', promo_table)
self.assertIn('id="promo-table"', promo_table)
self.assertIn('example.com', promo_table)
stream_table = self.D.write("stream")
self.assertIn("<table", stream_table)
self.assertIn("</table>", stream_table)
self.assertIn('class="zone', stream_table)
self.assertIn('id="stream-table"', stream_table)
self.assertIn('rdp.example.com', stream_table)
self.assertIn('git.example.net', stream_table)
def test_write_haproxy(self):
self.D.parse_haproxy(
join(self.pwd, "configs", "haproxy.test_server.all")
)
http_table = self.D.write("http")
self.assertIn("<table", http_table)
self.assertIn("</table>", http_table)
self.assertIn('class="zone', http_table)
self.assertIn('id="http-table"', http_table)
self.assertIn('https.example.org', http_table)
ssh_table = self.D.write("ssh")
self.assertIn("<table", ssh_table)
self.assertIn("</table>", ssh_table)
self.assertIn('class="zone', ssh_table)
self.assertIn('id="ssh-table"', ssh_table)
self.assertIn('ssh.example.org', ssh_table)
def test_blob_prefetch(nydus_anchor: NydusAnchor,
nydus_scratch_image: RafsImage, readahead_policy):
"""
description:
For rafs, there are two types of prefetching.
1. Prefetch files from fs-layer, which means each file is prefetched one by one.
2. Prefetch directly from backend/blob layer, which means a range will be fetched from blob
"""
# Try to delete any access log since if it present, bootstrap blob prefetch won't work.
utils.execute("rm -rf *.access", shell=True)
dist = Distributor(nydus_scratch_image.rootfs(), 8, 2)
dist.generate_tree()
dist.put_directories(20)
dist.put_multiple_files(100, Size(64, Unit.KB))
dist.put_symlinks(30)
dist.put_hardlinks(20)
dist.put_multiple_files(40, Size(64, Unit.KB))
utils.clean_pagecache()
hint_files = dist.files[-40:]
hint_files.extend(dist.symlinks[-20:])
hint_files = [os.path.join("/", p) for p in hint_files]
hint_files = "\n".join(hint_files)
nydus_scratch_image.set_backend(Backend.LOCALFS).create_image(
readahead_policy=readahead_policy,
readahead_files=hint_files.encode(),
)
rafs_conf = RafsConf(nydus_anchor, nydus_scratch_image)
rafs_conf.set_rafs_backend(Backend.LOCALFS, image=nydus_scratch_image)
rafs_conf.enable_records_readahead(interval=1)
rafs_conf.dump_rafs_conf()
rafs = RafsMount(nydus_anchor, nydus_scratch_image, rafs_conf)
with utils.timer("Mount elapse"):
rafs.thread_num(7).mount()
assert rafs.is_mounted()
wg = WorkloadGen(nydus_anchor.mount_point, nydus_scratch_image.rootfs())
# TODO: Run several parallel read workers against the mount_point
wg.setup_workload_generator()
wg.torture_read(5, 5)
wg.finish_torture_read()
utils.clean_pagecache()
class Spawner(object):
roles = {
'harvester': Harvester(),
'builder': Builder(),
'distributor': Distributor(),
'upgrader': Upgrader(),
'defender': Defender(),
'remoteHarvester': RemoteHarvester('E41N41', 'E42N42'),
'reserver': Reserver('E41N41', 'E42N42')
}
targetCreeps = {
'harvester': 4,
'distributor': 2,
'remoteHarvester': 3,
}
def __init__(self):
pass
def run(self, spawn: StructureSpawn):
if spawn.spawning:
return
creepRoles = [
creep.memory.role for creep in Object.values(Game.creeps)
]
roleAssignments = dict(_.countBy(creepRoles))
for role in Object.keys(Spawner.roles):
if not role in roleAssignments:
roleAssignments[role] = 0
remoteRoom = Game.getObjectById(Game.rooms['E42N41'])
storedEnergy = sum([
struct.store.getUsedCapacity(RESOURCE_ENERGY)
for room in Object.values(Game.rooms)
for struct in spawn.room.find(FIND_STRUCTURES)
if struct.structureType == STRUCTURE_STORAGE
])
desiredUpgraders = 2 + math.floor(storedEnergy / 30000)
constructionSites = [
site for room in Object.values(Game.rooms)
for site in room.find(FIND_CONSTRUCTION_SITES)
]
desiredBuilders = math.ceil(
sum([
site.progressTotal - site.progress
for site in constructionSites
]) / 10000)
halfHpStructures = [
struct for room in Object.values(Game.rooms)
for struct in room.find(FIND_STRUCTURES)
if struct.hits < struct.hitsMax / 2 and struct.structureType not in
[STRUCTURE_WALL, STRUCTURE_RAMPART]
]
desiredBuilders = max(desiredBuilders,
math.ceil(len(halfHpStructures) / 10))
if roleAssignments['harvesters'] == 0:
self.spawn(spawn, 'harvester')
elif roleAssignments['distributor'] == 0:
self.spawn(spawn, 'distributor')
elif roleAssignments['defender'] == 0:
self.spawn(spawn, 'defender')
# Spawn harvesters if we don't have enough
elif roleAssignments['harvester'] < Spawner.targetCreeps['harvester']:
self.spawn(spawn, 'harvester')
# Spawn additional harvester if any are close to dying
elif roleAssignments['harvester'] == Spawner.targetCreeps[
'harvester'] and any([
creep.ticksToLive < 500
for creep in Object.values(Game.creeps)
if creep.memory.role == 'harvester'
]):
self.spawn(spawn, 'harvester')
# Spawn distributor if we don't have enough
elif roleAssignments['distributor'] < Spawner.targetCreeps[
'distributor']:
self.spawn(spawn, 'distributor')
# Spawn additional distributor if any are close to dying
elif roleAssignments['distributor'] == Spawner.targetCreeps[
'distributor'] and any([
creep.ticksToLive < 500
for creep in Object.values(Game.creeps)
if creep.memory.role == 'distributor'
]):
self.spawn(spawn, 'distributor')
# Spawn builders up to some fraction of the remaining progress on all construction sites (allProgressRemaining / 2000)
elif roleAssignments['builder'] < desiredBuilders:
self.spawn(spawn, 'builder')
# Spawn reserver if we don't have one and don't have vision on the remoteRoom (remoteRoom == None) or if the reservation will end in < 1000 ticks
elif roleAssignments['reserver'] == 0 and (
remoteRoom == None
or remoteRoom.controller.reservation.ticksToEnd < 1000):
self.spawn(spawn, 'reserver')
# Spawn remote harvesters if we don't have enough
elif roleAssignments['remoteHarvester'] < Spawner.targetCreeps[
'remoteHarvester']:
self.spawn(spawn, 'remoteHarvester')
# Spawn upgrader if the storage / 10000 > numUpgraders
elif roleAssignments['upgrader'] < desiredUpgraders:
self.spawn(spawn, 'upgrader')
else:
pass
#print("No desired creeps to spawn")
def spawn(self, spawn: StructureSpawn, roleName: str):
bodyParts = Spawner.roles[roleName].getBodyParts(spawn)
cost = sum([BODYPART_COST[part] for part in bodyParts])
if cost <= spawn.room.energyAvailable:
spawn.spawnCreep(bodyParts, roleName + "_" + Game.time,
{'memory': {
'role': roleName
}})
else:
textStyle = {
'color': '#ffffff',
'font': '10px',
'stroke': '#000000',
'strokeWidth': .15
}
spawn.room.visual.text(roleName, spawn.pos, textStyle)
def end_all(self):
distributor = Distributor("*", "50010")
distributor.send_jobs([{'id': 0}])
