def run_ros2_tests(qemu: boot_cheribsd.CheriBSDInstance,
_: argparse.Namespace) -> bool:
boot_cheribsd.info("Running ROS2 tests")
boot_cheribsd.set_ld_library_path_with_sysroot(qemu)
qemu.checked_run("cd /source && sh -xe ./run-ros2-tests.sh",
timeout=240 * 60)
return True
def flush_thread(f, qemu: boot_cheribsd.CheriBSDInstance,
should_exit_event: threading.Event):
while not should_exit_event.wait(timeout=0.1):
if f:
f.flush()
if should_exit_event.is_set():
break
# keep reading line-by-line to output any QEMU trap messages:
i = qemu.expect(
[pexpect.TIMEOUT, "KDB: enter:", pexpect.EOF, qemu.crlf],
timeout=qemu.flush_interval)
if boot_cheribsd.PRETEND:
time.sleep(1)
elif i == 1:
boot_cheribsd.failure("GOT KERNEL PANIC!", exit=False)
boot_cheribsd.debug_kernel_panic(qemu)
global KERNEL_PANIC
KERNEL_PANIC = True # TODO: tell lit to abort now....
elif i == 2:
boot_cheribsd.failure("GOT QEMU EOF!", exit=False)
# QEMU exited?
break
# One final expect to flush the buffer:
qemu.expect([pexpect.TIMEOUT, pexpect.EOF], timeout=1)
boot_cheribsd.success("QEMU output flushing thread terminated.")
def run_juliet_tests(qemu: boot_cheribsd.CheriBSDInstance, args: argparse.Namespace) -> bool:
# args.ld_preload_path should be a path on the host
if args.ld_preload_path:
# hack until libcaprevoke is always present in cheribsd and can be added to the disk image via METALOG:
# copy it into the runtime linker's search path from the sysroot
qemu.checked_run("cp /sysroot/usr/libcheri/libcheri_caprevoke* /usr/libcheri")
try:
shutil.copy2(args.ld_preload_path, args.build_dir)
except Exception as e:
boot_cheribsd.failure("could not copy shared library for preload: ", e)
return False
preload_path = Path(args.ld_preload_path)
run_command = "/build/juliet-run.sh {} {}".format(args.testcase_timeout, "/build/" + preload_path.name)
else:
run_command = "/build/juliet-run.sh {}".format(args.testcase_timeout)
build_dir = Path(args.build_dir)
qemu.checked_run(run_command, ignore_cheri_trap=True, timeout=60000)
xml = junitparser.JUnitXml()
output_to_junit_suite(xml, build_dir / "bin" / "good.run", "good", True)
output_to_junit_suite(xml, build_dir / "bin" / "bad.run", "bad", False)
xml.write(build_dir / "results.xml")
return True
def run_qtbase_tests(qemu: boot_cheribsd.CheriBSDInstance, _: argparse.Namespace):
print("Running qtbase tests")
boot_cheribsd.set_ld_library_path_with_sysroot(qemu)
boot_cheribsd.prepend_ld_library_path(qemu, "/build/lib")
qemu.run("/build/tests/auto/corelib/tools/qarraydata/tst_qarraydata")
qemu.run("/build/tests/auto/corelib/global/qtendian/tst_qtendian")
# TODO: -o /path/to/file,xunitxml
return True
def run_simple_test(qemu: boot_cheribsd.CheriBSDInstance, args: argparse.Namespace) -> bool:
boot_cheribsd.info("Running tests")
# TODO: copy over the logfile and enable coredumps?
# Run tests with a two hour timeout:
qemu.checked_run("cd '{}'".format(qemu.smb_dirs[0].in_target), timeout=10)
qemu.checked_run(args.test_command, timeout=args.test_timeout, pretend_result=0,
ignore_cheri_trap=args.ignore_cheri_trap)
return True
def setup_qtbase_tests(qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace):
if args.junit_xml is None:
args.junit_xml = Path(
args.build_dir,
("junit-results-" +
datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") + ".xml"))
else:
args.junit_xml = Path(args.junit_xml)
assert args.junit_xml.parent.exists(), args.junit_xml
boot_cheribsd.set_ld_library_path_with_sysroot(qemu)
boot_cheribsd.prepend_ld_library_path(qemu, "/build/lib")
qemu.run("export QT_PLUGIN_PATH=/build/plugins")
def run_qtbase_tests(qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace):
# TODO: also run the non-corelib tests
xml = junitparser.JUnitXml()
failed_tests = []
successful_tests = []
build_dir = Path(args.build_dir)
all_tests_starttime = datetime.datetime.utcnow()
test_subset = Path(args.test_subset)
tests_root = Path(build_dir, "tests/auto")
assert Path(tests_root,
test_subset).is_relative_to(tests_root), "Invalid path " + str(
tests_root / test_subset)
boot_cheribsd.info("Running qtbase tests for ", test_subset)
# Start with some basic smoketests:
qemu.checked_run(
"/build/tests/auto/corelib/tools/qarraydata/tst_qarraydata")
qemu.checked_run("/build/tests/auto/corelib/global/qtendian/tst_qtendian")
run_subdir(qemu,
Path(tests_root, test_subset),
xml,
build_dir=build_dir,
successful_tests=successful_tests,
failed_tests=failed_tests)
xml.time = (datetime.datetime.utcnow() -
all_tests_starttime).total_seconds()
xml.update_statistics()
boot_cheribsd.info("JUnit results:", xml)
boot_cheribsd.info("Ran " + str(len(successful_tests) + len(failed_tests)),
" tests in ",
(datetime.datetime.utcnow() - all_tests_starttime))
if failed_tests:
boot_cheribsd.failure("The following ",
len(failed_tests),
" tests failed:\n\t",
"\n\t".join(x.name for x in failed_tests),
exit=False)
# Finally, write the Junit XML file:
if not boot_cheribsd.PRETEND:
xml.write(args.junit_xml, pretty=True)
boot_cheribsd.info("Wrote Junit results to ", args.junit_xml)
return not failed_tests
def run_subdir(qemu: boot_cheribsd.CheriBSDInstance, subdir: Path,
xml: junitparser.JUnitXml, successful_tests: list,
failed_tests: list, build_dir: Path):
tests = []
for root, dirs, files in os.walk(str(subdir), topdown=True):
for name in files:
if not name.startswith("tst_") or name.endswith(".core"):
continue
tests.append(Path(root, name))
# Ignore .moc and .obj directories:
dirs[:] = [d for d in dirs if not d.startswith(".")]
# Ensure that we run the tests in a reproducible order
for f in sorted(tests):
starttime = datetime.datetime.utcnow()
try:
# TODO: -o /path/to/file -junitxml
qemu.checked_run(
"rm -f /build/test.xml && {} -o /build/test.xml,junitxml -o -,txt -v1"
.format(f),
timeout=10)
endtime = datetime.datetime.utcnow()
successful_tests.append(f)
qemu.checked_run("fsync /build/test.xml")
test_xml = build_dir / "test.xml"
qt_test = junitparser.JUnitXml.fromfile(str(test_xml))
if not isinstance(qt_test, junitparser.TestSuite):
raise ValueError(
"Got unexpected parse result loading JUnit Xml: " +
qt_test.tostring())
if qt_test.name.lower() != f.name:
raise ValueError(
"Got unexpected test suite name: '{}' instead of '{}'".
format(qt_test.name, f.name))
if not qt_test.time:
qt_test.time = (endtime - starttime).total_seconds()
boot_cheribsd.info("Results for ", f.name, ": ", qt_test)
xml.add_testsuite(qt_test)
except Exception as e:
if isinstance(e, boot_cheribsd.CheriBSDCommandFailed):
boot_cheribsd.failure("Failed to run ",
f.name,
": ",
str(e),
exit=False)
else:
boot_cheribsd.failure("Error loading JUnit result for",
f.name,
": ",
str(e),
exit=False)
failed_tests.append(f)
add_junit_failure(xml, f, str(e), starttime)
# Kill the process that timed out:
qemu.sendintr()
qemu.expect_prompt(timeout=60)
def run_bodiagsuite(qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace) -> bool:
boot_cheribsd.info("Running BODiagSuite")
assert not args.use_valgrind, "Not support for CheriBSD"
if not args.junit_xml_only:
qemu.checked_run("rm -rf {}/run".format(LONG_NAME_FOR_BUILDDIR))
qemu.checked_run(
"cd {} && mkdir -p run".format(LONG_NAME_FOR_BUILDDIR))
# Don't log all the CHERI traps while running (should speed up the tests a bit and produce shorter logfiles)
qemu.run("sysctl machdep.log_user_cheri_exceptions=0 || true")
qemu.checked_run("{} -r -f {}/Makefile.bsd-run all".format(
args.bmake_path, LONG_NAME_FOR_BUILDDIR),
timeout=120 * 60,
ignore_cheri_trap=True)
# restore old behaviour
qemu.run("sysctl machdep.log_user_cheri_exceptions=1 || true")
if not create_junit_xml(Path(args.build_dir), args.junit_testsuite_name,
args.tools):
return False
return True
def run_qtwebkit_tests(qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace) -> bool:
boot_cheribsd.info("Running QtWebkit tests")
try:
# Check that jsc + dumprendertree work
qemu.checked_run("/tmp/jsc --help", timeout=1200)
# Run a simple javascript loop
qemu.checked_run(
"/tmp/jsc -e 'for (i = 0; i < 10; i++) print(1 + i);'",
timeout=1200)
qemu.checked_run("/tmp/DumpRenderTree -v /tmp/helloworld.html",
timeout=1800)
qemu.checked_run(
"/tmp/DumpRenderTree -p --stdout /build/hello.png /tmp/helloworld.html",
timeout=1800)
if not args.smoketest:
qemu.checked_run(
"/source/Tools/Scripts/run-layout-jsc -j /tmp/jsc -t "
"/source/LayoutTests -r /build/results -x /build/results.xml",
timeout=None)
return True
finally:
tests_xml_path = Path(args.build_dir, 'results.xml')
try:
if not args.smoketest and tests_xml_path.exists():
# Process junit xml file with junitparser to update the number of tests, failures, total time, etc.
xml = junitparser.JUnitXml.fromfile(str(tests_xml_path))
xml.update_statistics()
xml.write()
except Exception as e:
boot_cheribsd.failure("Could not update JUnit XML",
tests_xml_path,
": ",
e,
exit=False)
return False
def setup_libunwind_env(qemu: boot_cheribsd.CheriBSDInstance,
_: argparse.Namespace):
# We also need libdl and libcxxrt from the sysroot:
libdir = "libcheri" if qemu.xtarget.is_cheri_purecap() else "lib64"
qemu.checked_run("ln -sfv /build/lib/libunwind.so* /usr/{libdir}/".format(
libdir=libdir))
qemu.checked_run(
"ln -sfv /sysroot/usr/{libdir}/libcxxrt.so* /sysroot/usr/{libdir}/libdl.so* /usr/{libdir}/"
.format(libdir=libdir))
# Add a fake libgcc_s link to libunwind (this works now that we build libunwind with version info)
qemu.checked_run(
"ln -sfv /usr/{libdir}/libunwind.so /usr/{libdir}/libgcc_s.so.1".
format(libdir=libdir))
def run_libcxxrt_tests(qemu: boot_cheribsd.CheriBSDInstance,
_: argparse.Namespace) -> bool:
boot_cheribsd.info("Running libcxxrt tests")
boot_cheribsd.set_ld_library_path_with_sysroot(qemu)
qemu.run("export LIBUNWIND_PRINT_UNWINDING=1", timeout=2)
qemu.run("export LIBUNWIND_PRINT_APIS=1", timeout=2)
qemu.run("export LIBUNWIND_PRINT_DWARF=1", timeout=2)
# Copy the libunwind library to both MIPS and CHERI library dirs so that it is picked up
qemu.checked_run("ln -sfv /libunwind/lib/libunwind.so* /usr/lib/")
qemu.checked_run("ln -sfv /libunwind/lib/libunwind.so* /usr/libcheri/")
qemu.checked_run("'/build/bin/cxxrt-test-static' -v")
qemu.checked_run("'/build/bin/cxxrt-test-foreign-exceptions' -v")
qemu.checked_run("'/build/bin/cxxrt-test-shared' -v")
# Check the test binaries linked against libunwind
qemu.checked_run("'/build/bin/cxxrt-test-libunwind-static' -v")
qemu.checked_run("'/build/bin/cxxrt-test-libunwind-allstatic' -v")
qemu.checked_run("'/build/bin/cxxrt-test-libunwind-shared' -v")
return True
def setup_qtwebkit_test_environment(qemu: boot_cheribsd.CheriBSDInstance,
_: argparse.Namespace):
boot_cheribsd.set_ld_library_path_with_sysroot(qemu)
qemu.run("export ICU_DATA=/sysroot/usr/local/share/icu/60.0.1")
qemu.run("export LANG=en_US.UTF-8")
qemu.run("echo '<h1>Hello World!</h1>' > /tmp/helloworld.html")
# mime database
qemu.run("mkdir -p /usr/share/mime/packages")
# old directory names:
qemu.run("mkdir -p /usr/local/Qt-cheri/lib/fonts")
qemu.run("ln -sf /usr/local/Qt-cheri /usr/local/Qt-mips")
# New directory names:
qemu.checked_run("ln -sf /usr/local/Qt-cheri /usr/local/mips")
qemu.checked_run("ln -sf /usr/local/Qt-cheri /usr/local/cheri")
qemu.checked_run(
"cp /source/LayoutTests/resources/Ahem.ttf /usr/local/Qt-cheri/lib/fonts"
)
qemu.checked_run(
"cp /source/LayoutTests/fast/writing-mode/resources/DroidSansFallback-reduced.ttf "
"/usr/local/Qt-cheri/lib/fonts")
qemu.checked_run("cp /build/mime.cache /usr/share/mime")
qemu.checked_run(
"cp /build/freedesktop.org.xml /usr/share/mime/packages/freedesktop.org.xml"
)
boot_cheribsd.success(
"To debug crashes run: `sysctl kern.corefile=/build/%N.%P.core; sysctl kern.coredump=1`"
" and then run CHERI gdb on the host system.")
# copy the smaller files to /tmp to avoid the smbfs overhead
qemu.checked_run("cp /build/bin/jsc.stripped /tmp/jsc")
qemu.checked_run(
"cp /build/bin/DumpRenderTree.stripped /tmp/DumpRenderTree")
def run_cheribsd_test(qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace):
boot_cheribsd.success("Booted successfully")
qemu.checked_run("kenv")
# unchecked since mount_smbfs returns non-zero for --help:
qemu.run("mount_smbfs --help", cheri_trap_fatal=True)
# same for ld-cheri-elf.so (but do check for CHERI traps):
qemu.run("/libexec/ld-cheri-elf.so.1 -h", cheri_trap_fatal=True)
tests_successful = True
host_has_kyua = shutil.which("kyua") is not None
try:
# potentially bootstrap kyua for later testing
if args.bootstrap_kyua or args.kyua_tests_files:
qemu.checked_run("/sbin/prepare-testsuite.sh", timeout=30 * 60)
qemu.checked_run("kyua help", timeout=60)
for i, tests_file in enumerate(args.kyua_tests_files):
# TODO: is the results file too big for tmpfs? No should be fine, only a few megabytes
qemu.checked_run("rm -f /tmp/results.db")
# Allow up to 24 hours to run the full testsuite
# Not a checked run since it might return false if some tests fail
test_start = datetime.datetime.now()
qemu.run("kyua test --results-file=/tmp/results.db -k {}".format(
shlex.quote(tests_file)),
ignore_cheri_trap=True,
cheri_trap_fatal=False,
timeout=24 * 60 * 60)
if i == 0:
results_db = Path("/kyua-results/test-results.db")
else:
results_db = Path("/kyua-results/test-results-{}.db".format(i))
results_xml = results_db.with_suffix(".xml")
assert shlex.quote(str(results_db)) == str(
results_db), "Should not contain any special chars"
qemu.checked_run("cp -v /tmp/results.db {}".format(results_db))
qemu.checked_run("fsync " + str(results_db))
boot_cheribsd.success("Running tests for ", tests_file, " took: ",
datetime.datetime.now() - test_start)
# run: kyua report-junit --results-file=test-results.db | vis -os > ${CPU}-${TEST_NAME}-test-results.xml
# Not sure how much we gain by running it on the host instead.
# Converting the full test suite to xml can take over an hour (probably a lot faster without the vis -os
# pipe)
# TODO: should escape the XML file but that's probably faster on the host
if host_has_kyua:
boot_cheribsd.info(
"KYUA installed on the host, no need to do slow conversion in QEMU"
)
else:
xml_conversion_start = datetime.datetime.now()
qemu.checked_run(
"kyua report-junit --results-file=/tmp/results.db > /tmp/results.xml",
timeout=200 * 60)
qemu.checked_run(
"cp -v /tmp/results.xml {}".format(results_xml))
qemu.checked_run("fsync " + str(results_xml))
boot_cheribsd.success(
"Creating JUnit XML ", results_xml, " took: ",
datetime.datetime.now() - xml_conversion_start)
except boot_cheribsd.CheriBSDCommandTimeout as e:
boot_cheribsd.failure("Timeout running tests: " + str(e), exit=False)
qemu.sendintr()
qemu.sendintr()
# Try to cancel the running command and get back to having a sensible prompt
qemu.checked_run("pwd")
time.sleep(10)
tests_successful = False
except boot_cheribsd.CheriBSDCommandFailed as e:
boot_cheribsd.failure("Failed to run: " + str(e), exit=False)
boot_cheribsd.info("Trying to shut down cleanly")
tests_successful = False
# Update the JUnit stats in the XML file
if args.kyua_tests_files:
if not boot_cheribsd.PRETEND:
time.sleep(2) # sleep two seconds to ensure the files exist
junit_dir = Path(args.kyua_tests_output)
try:
if host_has_kyua:
boot_cheribsd.info(
"Converting kyua databases to JUNitXML in output directory ",
junit_dir)
for host_kyua_db_path in junit_dir.glob("*.db"):
convert_kyua_db_to_junit_xml(
host_kyua_db_path,
host_kyua_db_path.with_suffix(".xml"))
else:
boot_cheribsd.info(
"Updating statistics in JUnit output directory ",
junit_dir)
for host_xml_path in junit_dir.glob("*.xml"):
fixup_kyua_generated_junit_xml(host_xml_path)
except Exception as e:
boot_cheribsd.failure("Could not update stats in ",
junit_dir,
": ",
e,
exit=False)
tests_successful = False
if args.interact or args.skip_poweroff:
boot_cheribsd.info(
"Skipping poweroff step since --interact/--skip-poweroff was passed."
)
return tests_successful
poweroff_start = datetime.datetime.now()
qemu.sendline("poweroff")
i = qemu.expect(["Uptime:", pexpect.TIMEOUT, pexpect.EOF] +
boot_cheribsd.FATAL_ERROR_MESSAGES,
timeout=360)
if i != 0:
boot_cheribsd.failure("Poweroff " +
("timed out" if i == 1 else "failed"))
return False
# 240 secs since it takes a lot longer on a full image (it took 44 seconds after installing kyua, so on a really
# busy jenkins slave it might be a lot slower)
i = qemu.expect([pexpect.TIMEOUT, pexpect.EOF], timeout=240)
if i == 0:
boot_cheribsd.failure("QEMU didn't exit after shutdown!")
return False
boot_cheribsd.success("Poweroff took: ",
datetime.datetime.now() - poweroff_start)
return tests_successful
def run_tests(qemu: boot_cheribsd.CheriBSDInstance, args: argparse.Namespace) -> bool:
boot_cheribsd.info("Running Python tests")
# Need the library path for libpython.so
boot_cheribsd.prepend_ld_library_path(qemu, "/build")
# When running the full test suite we want all python files in tmpfs:
if args.full_test:
# copy python libs from smb to tmpfs:
install_prefix = Path(args.install_prefix)
qemu.checked_run("time cp -a '{pfx}' '{pfx}.tmpfs'".format(pfx=install_prefix))
qemu.checked_run("umount '{pfx}'".format(pfx=install_prefix))
qemu.checked_run("rmdir '{pfx}' && mv '{pfx}.tmpfs' '{pfx}'".format(pfx=install_prefix))
# run basic sanity check:
build_python_exe = "python" + args.buildexe_suffix
qemu.checked_run("/build/{} --version".format(build_python_exe))
qemu.checked_run("/build/{} -E -c 'import sys; sys.exit(0)'".format(build_python_exe))
if args.full_test:
# Run the full test suite:
qemu.checked_run("cd /build && ./{} -m test -v --junit-xml=python-tests.xml".format(build_python_exe))
return True
def run_cheritest(qemu: boot_cheribsd.CheriBSDInstance, binary_name,
args: argparse.Namespace) -> bool:
try:
qemu.checked_run("rm -f /tmp/{}.xml".format(binary_name))
# Run it once with textual output (for debugging)
# qemu.run("/bin/{} -a".format(binary_name, binary_name),
# ignore_cheri_trap=True, cheri_trap_fatal=False, timeout=5 * 60)
# Generate JUnit XML:
qemu.run("/bin/{} -a -x > /tmp/{}.xml".format(binary_name,
binary_name),
ignore_cheri_trap=True,
cheri_trap_fatal=False,
timeout=5 * 60)
qemu.sendline("echo EXITCODE=$?")
qemu.expect(["EXITCODE=(\\d+)\r"], timeout=5, pretend_result=0)
if boot_cheribsd.PRETEND:
exit_code = 0
else:
print(qemu.match.groups())
exit_code = int(qemu.match.group(1))
qemu.expect_prompt()
if qemu.smb_failed:
boot_cheribsd.info("SMB mount has failed, performing normal scp")
host_path = Path(args.test_output_dir, binary_name + ".xml")
qemu.scp_from_guest("/tmp/{}.xml".format(binary_name), host_path)
else:
qemu.checked_run("mv -f /tmp/{}.xml /test-results/{}.xml".format(
binary_name, binary_name))
qemu.run("fsync /test-results/{}.xml".format(binary_name))
return exit_code == 0
except boot_cheribsd.CheriBSDCommandTimeout as e:
boot_cheribsd.failure("Timeout running cheritest: " + str(e),
exit=False)
qemu.sendintr()
qemu.sendintr()
# Try to cancel the running command and get back to having a sensible prompt
qemu.checked_run("pwd")
time.sleep(10)
return False
except boot_cheribsd.CheriBSDCommandFailed as e:
if "command not found" in e.args:
boot_cheribsd.failure("Cannot find cheritest binary ",
binary_name,
": " + str(e),
exit=False)
else:
boot_cheribsd.failure("Failed to run: " + str(e), exit=False)
return False
def run_cheribsd_test(qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace):
boot_cheribsd.success("Booted successfully")
qemu.checked_run("kenv")
# unchecked since mount_smbfs returns non-zero for --help:
qemu.run("mount_smbfs --help", cheri_trap_fatal=True)
# same for ld-cheri-elf.so (but do check for CHERI traps):
if qemu.xtarget.is_cheri_hybrid():
qemu.run("/libexec/ld-cheri-elf.so.1 -h", cheri_trap_fatal=True)
qemu.run("/libexec/ld-elf.so.1 -h", cheri_trap_fatal=True)
tests_successful = True
# Check that we can connect to QEMU using SSH. This catches regressions that break SSHD.
if not qemu.check_ssh_connection():
tests_successful = False
host_has_kyua = shutil.which("kyua") is not None
# Run the various cheritest binaries
if args.run_cheritest:
# Disable trap dumps while running cheritest (handle both old and new sysctl names until dev is merged):
qemu.run(
"sysctl machdep.log_user_cheri_exceptions=0 || sysctl machdep.log_cheri_exceptions=0"
)
# The minimal disk image only has the statically linked variants:
test_binaries = ["cheritest", "cheriabitest"]
if not args.minimal_image:
test_binaries.extend([
"cheriabitest-dynamic", "cheriabitest-dynamic-mt",
"cheriabitest-mt", "cheritest-dynamic", "cheritest-dynamic-mt",
"cheritest-mt"
])
for test in test_binaries:
if not run_cheritest(qemu, test, args):
tests_successful = False
boot_cheribsd.failure("At least one test failure in",
test,
exit=False)
qemu.run(
"sysctl machdep.log_user_cheri_exceptions=1 || sysctl machdep.log_cheri_exceptions=1"
)
# Run kyua tests
try:
if args.kyua_tests_files:
qemu.checked_run("kyua help", timeout=60)
# Try to load the pf module for the pfctl test
qemu.run(
"kldstat -m pf || kldload pf || echo 'failed to load pf module'"
)
for i, tests_file in enumerate(args.kyua_tests_files):
# TODO: is the results file too big for tmpfs? No should be fine, only a few megabytes
qemu.checked_run("rm -f /tmp/results.db")
# Allow up to 24 hours to run the full testsuite
# Not a checked run since it might return false if some tests fail
test_start = datetime.datetime.now()
qemu.run("kyua test --results-file=/tmp/results.db -k {}".format(
shlex.quote(tests_file)),
ignore_cheri_trap=True,
cheri_trap_fatal=False,
timeout=24 * 60 * 60)
if i == 0:
result_name = "test-results.db"
else:
result_name = "test-results-{}.db".format(i)
results_db = Path("/test-results/{}".format(result_name))
results_xml = results_db.with_suffix(".xml")
assert shlex.quote(str(results_db)) == str(
results_db), "Should not contain any special chars"
if qemu.smb_failed:
boot_cheribsd.info(
"SMB mount has failed, performing normal scp")
qemu.scp_from_guest(
"/tmp/results.db",
Path(args.test_output_dir, results_db.name))
else:
qemu.checked_run("cp -v /tmp/results.db {}".format(results_db))
qemu.checked_run("fsync " + str(results_db))
boot_cheribsd.success("Running tests for ", tests_file, " took: ",
datetime.datetime.now() - test_start)
# run: kyua report-junit --results-file=test-results.db | vis -os > ${CPU}-${TEST_NAME}-test-results.xml
# Not sure how much we gain by running it on the host instead.
# Converting the full test suite to xml can take over an hour (probably a lot faster without the vis -os
# pipe)
# TODO: should escape the XML file but that's probably faster on the host
if host_has_kyua:
boot_cheribsd.info(
"KYUA installed on the host, no need to do slow conversion in QEMU"
)
else:
xml_conversion_start = datetime.datetime.now()
qemu.checked_run(
"kyua report-junit --results-file=/tmp/results.db > /tmp/results.xml",
timeout=200 * 60)
if qemu.smb_failed:
boot_cheribsd.info(
"SMB mount has failed, performing normal scp")
qemu.scp_from_guest(
"/tmp/results.xml",
Path(args.test_output_dir, results_xml.name))
else:
qemu.checked_run(
"cp -v /tmp/results.xml {}".format(results_xml))
qemu.checked_run("fsync " + str(results_xml))
boot_cheribsd.success(
"Creating JUnit XML ", results_xml, " took: ",
datetime.datetime.now() - xml_conversion_start)
except boot_cheribsd.CheriBSDCommandTimeout as e:
boot_cheribsd.failure("Timeout running tests: " + str(e), exit=False)
qemu.sendintr()
qemu.sendintr()
# Try to cancel the running command and get back to having a sensible prompt
qemu.checked_run("pwd")
time.sleep(10)
tests_successful = False
except boot_cheribsd.CheriBSDCommandFailed as e:
boot_cheribsd.failure("Failed to run: " + str(e), exit=False)
boot_cheribsd.info("Trying to shut down cleanly")
tests_successful = False
# Update the JUnit stats in the XML files (both kyua and cheritest):
if args.kyua_tests_files or args.run_cheritest:
if not boot_cheribsd.PRETEND:
time.sleep(2) # sleep two seconds to ensure the files exist
junit_dir = Path(args.test_output_dir)
if host_has_kyua:
try:
boot_cheribsd.info(
"Converting kyua databases to JUNitXML in output directory ",
junit_dir)
for host_kyua_db_path in junit_dir.glob("*.db"):
convert_kyua_db_to_junit_xml(
host_kyua_db_path,
host_kyua_db_path.with_suffix(".xml"))
except Exception as e:
boot_cheribsd.failure("Could not convert kyua database in ",
junit_dir,
": ",
e,
exit=False)
tests_successful = False
boot_cheribsd.info("Updating statistics in JUnit output directory ",
junit_dir)
for host_xml_path in junit_dir.glob("*.xml"):
try:
fixup_kyua_generated_junit_xml(
host_xml_path) # Despite the name also works for cheritest
except Exception as e:
boot_cheribsd.failure("Could not update stats in ",
junit_dir,
": ",
e,
exit=False)
tests_successful = False
if args.interact or args.skip_poweroff:
boot_cheribsd.info(
"Skipping poweroff step since --interact/--skip-poweroff was passed."
)
return tests_successful
poweroff_start = datetime.datetime.now()
qemu.sendline("poweroff")
i = qemu.expect(["Uptime: ", pexpect.TIMEOUT, pexpect.EOF], timeout=360)
if i != 0:
boot_cheribsd.failure("Poweroff " +
("timed out" if i == 1 else "failed"))
return False
# 240 secs since it takes a lot longer on a full image (it took 44 seconds after installing kyua, so on a really
# busy jenkins slave it might be a lot slower)
i = qemu.expect(
[pexpect.TIMEOUT, "Please press any key to reboot.", pexpect.EOF],
timeout=240)
if i == 0:
boot_cheribsd.failure("QEMU didn't exit after shutdown!")
return False
boot_cheribsd.success("Poweroff took: ",
datetime.datetime.now() - poweroff_start)
if tests_successful and qemu.smb_failed:
boot_cheribsd.info(
"Tests succeeded, but SMB mount failed -> marking tests as failed."
)
tests_successful = False
return tests_successful
def run_remote_lit_tests_impl(testsuite: str,
qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace,
tempdir: str,
mp_q: multiprocessing.Queue = None,
barrier: multiprocessing.Barrier = None,
llvm_lit_path: str = None,
lit_extra_args: list = None) -> bool:
qemu.EXIT_ON_KERNEL_PANIC = False # since we run multiple threads we shouldn't use sys.exit()
boot_cheribsd.info("PID of QEMU: ", qemu.pid)
if args.pretend and os.getenv(
"FAIL_TIMEOUT_BOOT") and args.internal_shard == 2:
time.sleep(10)
if mp_q:
assert barrier is not None
notify_main_process(args,
MultiprocessStages.TESTING_SSH_CONNECTION,
mp_q,
barrier=barrier)
if args.pretend and os.getenv(
"FAIL_RAISE_EXCEPTION") and args.internal_shard == 1:
raise RuntimeError("SOMETHING WENT WRONG!")
qemu.checked_run("cat /root/.ssh/authorized_keys", timeout=20)
port = args.ssh_port
user = "******" # TODO: run these tests as non-root!
test_build_dir = Path(args.build_dir)
# TODO: move this to boot_cheribsd.py
config_contents = """
Host cheribsd-test-instance
User {user}
HostName localhost
Port {port}
IdentityFile {ssh_key}
# avoid errors due to changed host key:
UserKnownHostsFile /dev/null
StrictHostKeyChecking no
NoHostAuthenticationForLocalhost yes
# faster connection by reusing the existing one:
ControlPath {home}/.ssh/controlmasters/%r@%h:%p
# ConnectTimeout 20
# ConnectionAttempts 2
ControlMaster auto
""".format(user=user,
port=port,
ssh_key=Path(args.ssh_key).with_suffix(""),
home=Path.home())
config_contents += " ControlPersist {control_persist}\n"
# print("Writing ssh config: ", config_contents)
with Path(tempdir, "config").open("w") as c:
# Keep socket open for 10 min (600) or indefinitely (yes)
c.write(config_contents.format(control_persist="yes"))
Path(Path.home(), ".ssh/controlmasters").mkdir(exist_ok=True)
boot_cheribsd.run_host_command(["cat", str(Path(tempdir, "config"))])
# Check that the config file works:
def check_ssh_connection(prefix):
connection_test_start = datetime.datetime.utcnow()
boot_cheribsd.run_host_command([
"ssh", "-F",
str(Path(tempdir, "config")), "cheribsd-test-instance", "-p",
str(port), "--", "echo", "connection successful"
],
cwd=str(test_build_dir))
connection_time = (datetime.datetime.utcnow() -
connection_test_start).total_seconds()
boot_cheribsd.success(prefix, " successful after ", connection_time,
" seconds")
check_ssh_connection("First SSH connection")
controlmaster_running = False
try:
# Check that controlmaster worked by running ssh -O check
boot_cheribsd.info("Checking if SSH control master is working.")
boot_cheribsd.run_host_command([
"ssh", "-F",
str(Path(tempdir, "config")), "cheribsd-test-instance", "-p",
str(port), "-O", "check"
],
cwd=str(test_build_dir))
check_ssh_connection("Second SSH connection (with controlmaster)")
controlmaster_running = True
except subprocess.CalledProcessError:
boot_cheribsd.failure(
"WARNING: Could not connect to ControlMaster SSH connection. Running tests will be slower",
exit=False)
with Path(tempdir, "config").open("w") as c:
c.write(config_contents.format(control_persist="no"))
check_ssh_connection("Second SSH connection (without controlmaster)")
if args.pretend:
time.sleep(2.5)
extra_ssh_args = commandline_to_str(
("-n", "-4", "-F", "{tempdir}/config".format(tempdir=tempdir)))
extra_scp_args = commandline_to_str(
("-F", "{tempdir}/config".format(tempdir=tempdir)))
ssh_executor_args = [
args.ssh_executor_script, "--host", "cheribsd-test-instance",
"--extra-ssh-args=" + extra_ssh_args
]
if args.use_shared_mount_for_tests:
# If we have a shared directory use that to massively speed up running tests
tmpdir_name = args.shared_tmpdir_local.name
ssh_executor_args.append("--shared-mount-local-path=" +
str(args.shared_tmpdir_local))
ssh_executor_args.append("--shared-mount-remote-path=/build/" +
tmpdir_name)
else:
# slow executor using scp:
ssh_executor_args.append("--extra-scp-args=" + extra_scp_args)
executor = commandline_to_str(ssh_executor_args)
# TODO: I was previously passing -t -t to ssh. Is this actually needed?
boot_cheribsd.success("Running", testsuite, "tests with executor",
executor)
notify_main_process(args, MultiprocessStages.RUNNING_TESTS, mp_q)
# have to use -j1 since otherwise CheriBSD might wedge
if llvm_lit_path is None:
llvm_lit_path = str(test_build_dir / "bin/llvm-lit")
# Note: we require python 3 since otherwise it seems to deadlock in Jenkins
lit_cmd = [
sys.executable, llvm_lit_path, "-j1", "-vv", "-Dexecutor=" + executor,
"test"
]
if lit_extra_args:
lit_cmd.extend(lit_extra_args)
if args.lit_debug_output:
lit_cmd.append("--debug")
# This does not work since it doesn't handle running ssh commands....
lit_cmd.append(
"--timeout=120"
) # 2 minutes max per test (in case there is an infinite loop)
xunit_file = None # type: typing.Optional[Path]
if args.xunit_output:
lit_cmd.append("--xunit-xml-output")
xunit_file = Path(args.xunit_output).absolute()
if args.internal_shard:
xunit_file = xunit_file.with_name("shard-" +
str(args.internal_shard) + "-" +
xunit_file.name)
lit_cmd.append(str(xunit_file))
qemu_logfile = qemu.logfile
if args.internal_shard:
assert args.internal_num_shards, "Invalid call!"
lit_cmd.append("--num-shards=" + str(args.internal_num_shards))
lit_cmd.append("--run-shard=" + str(args.internal_shard))
if xunit_file:
assert qemu_logfile is not None, "Should have a valid logfile when running multiple shards"
boot_cheribsd.success("Writing QEMU output to ", qemu_logfile)
# Fixme starting lit at the same time does not work!
# TODO: add the polling to the main thread instead of having another thread?
# start the qemu output flushing thread so that we can see the kernel panic
qemu.flush_interval = 15 # flush the logfile every 15 seconds
should_exit_event = threading.Event()
t = threading.Thread(target=flush_thread,
args=(qemu_logfile, qemu, should_exit_event))
t.daemon = True
t.start()
shard_prefix = "SHARD" + str(
args.internal_shard) + ": " if args.internal_shard else ""
try:
boot_cheribsd.success("Starting llvm-lit: cd ", test_build_dir, " && ",
" ".join(lit_cmd))
boot_cheribsd.run_host_command(lit_cmd, cwd=str(test_build_dir))
# lit_proc = pexpect.spawnu(lit_cmd[0], lit_cmd[1:], echo=True, timeout=60, cwd=str(test_build_dir))
# TODO: get stderr!!
# while lit_proc.isalive():
# lit_proc = None
# while False:
# line = lit_proc.readline()
# if shard_prefix:
# line = shard_prefix + line
# print(line)
# global KERNEL_PANIC
# # Abort once we detect a kernel panic
# if KERNEL_PANIC:
# lit_proc.sendintr()
# print(shard_prefix + lit_proc.read())
# print("Lit finished.")
# if lit_proc and lit_proc.exitstatus == 1:
# boot_cheribsd.failure(shard_prefix + "SOME TESTS FAILED", exit=False)
except subprocess.CalledProcessError as e:
boot_cheribsd.failure(shard_prefix + "SOME TESTS FAILED: ",
e,
exit=False)
# Should only ever return 1 (otherwise something else went wrong!)
if e.returncode == 1:
return False
else:
raise
finally:
if qemu_logfile:
qemu_logfile.flush()
if controlmaster_running:
boot_cheribsd.info("Terminating SSH controlmaster")
try:
boot_cheribsd.run_host_command([
"ssh", "-F",
str(Path(tempdir, "config")), "cheribsd-test-instance",
"-p",
str(port), "-O", "exit"
],
cwd=str(test_build_dir))
except subprocess.CalledProcessError:
boot_cheribsd.failure(
"Could not close SSH controlmaster connection.",
exit=False)
qemu.flush_interval = 0.1
should_exit_event.set()
t.join(timeout=30)
if t.is_alive():
boot_cheribsd.failure(
"Failed to kill flush thread. Interacting with CheriBSD will not work!",
exit=True)
return False
if not qemu.isalive():
boot_cheribsd.failure("QEMU died while running tests! ",
qemu,
exit=True)
return True