def cheribsd_setup_args(args: argparse.Namespace):
if args.run_cheritest is None:
# Only hybrid and purecap images have cheritest
assert isinstance(args.xtarget, CrossCompileTarget)
args.run_cheritest = args.xtarget.is_hybrid_or_purecap_cheri()
if args.kyua_tests_files:
# flatten the list (https://stackoverflow.com/a/45323085/894271):
args.kyua_tests_files = functools.reduce(operator.iconcat,
args.kyua_tests_files, [])
print(args.kyua_tests_files)
for file in args.kyua_tests_files:
if not Path(file).name == "Kyuafile":
boot_cheribsd.failure(
"Expected a path to a Kyuafile but got: ", file)
# Make sure we mount the output directory if we are running kyua and/or cheritest
if args.kyua_tests_files or args.run_cheritest:
test_output_dir = Path(
os.path.expandvars(os.path.expanduser(args.test_output_dir)))
if not test_output_dir.is_dir():
boot_cheribsd.failure("Output directory does not exist: ",
test_output_dir)
# Create a timestamped directory:
if args.no_timestamped_test_subdir:
real_output_dir = test_output_dir.absolute()
else:
args.timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
real_output_dir = (test_output_dir / args.timestamp).absolute()
args.test_output_dir = str(real_output_dir)
boot_cheribsd.run_host_command(["mkdir", "-p", str(real_output_dir)])
if not boot_cheribsd.PRETEND:
(real_output_dir / "cmdline").write_text(str(sys.argv))
args.smb_mount_directories.append(
boot_cheribsd.SmbMount(real_output_dir,
readonly=False,
in_target="/test-results"))
def cheribsd_setup_args(args: argparse.Namespace):
args.use_smb_instead_of_ssh = True # skip the ssh setup
args.skip_ssh_setup = True
if args.kyua_tests_files:
# flatten the list (https://stackoverflow.com/a/45323085/894271):
args.kyua_tests_files = functools.reduce(operator.iconcat,
args.kyua_tests_files, [])
print(args.kyua_tests_files)
for file in args.kyua_tests_files:
if not Path(file).name == "Kyuafile":
boot_cheribsd.failure(
"Expected a path to a Kyuafile but got: ", file)
test_output_dir = Path(
os.path.expandvars(os.path.expanduser(args.kyua_tests_output)))
if not test_output_dir.is_dir():
boot_cheribsd.failure("Output directory does not exist: ",
test_output_dir)
# Create a timestamped directory:
if args.kyua_tests_output_no_timestamped_subdir:
real_output_dir = test_output_dir.absolute()
else:
args.timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
real_output_dir = (test_output_dir / args.timestamp).absolute()
args.kyua_tests_output = str(real_output_dir)
boot_cheribsd.run_host_command(["mkdir", "-p", str(real_output_dir)])
if not boot_cheribsd.PRETEND:
(real_output_dir / "cmdline").write_text(str(sys.argv))
args.smb_mount_directories.append(
boot_cheribsd.SmbMount(real_output_dir,
readonly=False,
in_target="/kyua-results"))
def main():
if "--junit-xml-only" in sys.argv or "--test-native" in sys.argv:
parser = argparse.ArgumentParser()
add_args(parser)
parser.add_argument("--test-native", action="store_true")
parser.add_argument("--build-dir")
args, remaining = parser.parse_known_args()
if args.test_native and not args.junit_xml_only:
cmd = [
args.bmake_path, "-r", "-f",
args.build_dir + "/Makefile.bsd-run", "all"
]
if args.jobs > 1:
cmd += ["-j", str(args.jobs)]
if args.use_valgrind:
cmd.append("-DUSE_VALGRIND")
boot_cheribsd.run_host_command(cmd, cwd=args.build_dir)
if not create_junit_xml(Path(args.build_dir),
args.junit_testsuite_name, args.tools):
sys.exit("Failed to create JUnit xml")
sys.exit()
# we don't need ssh running to execute the tests
run_tests_main(test_function=run_bodiagsuite,
need_ssh=False,
should_mount_builddir=True,
argparse_setup_callback=add_args,
build_dir_in_target=LONG_NAME_FOR_BUILDDIR)
def fixup_kyua_generated_junit_xml(xml_file: Path, prefix: str = None):
boot_cheribsd.info("Updating statistics in JUnit file ", xml_file)
# Process junit xml file with junitparser to update the number of tests, failures, total time, etc.
orig_xml_str = xml_file.read_text("utf-8", errors='backslashreplace')
xml_str = orig_xml_str
for i in range(32):
if chr(i) not in ("\n", "\t"):
# Can't reference NULL character -> backslashescape instead
# xml_str = xml_str.replace(chr(i), "&#" + str(i) + ";")
xml_str = xml_str.replace(chr(i), "\\x" + format(i, '02x') + ";")
with tempfile.NamedTemporaryFile("wb") as tf:
# create a temporary file first to avoid clobbering the original one if we fail to parse it
tf.write(xml_str.encode("ascii", errors="xmlcharrefreplace"))
tf.flush()
xml = junitparser.JUnitXml.fromfile(tf.name)
xml.update_statistics()
if prefix is not None:
if isinstance(xml, junitparser.TestSuite):
xml.name = prefix if xml.name is None else prefix + "-" + xml.name
else:
for suite in xml:
suite.name = prefix if suite.name is None else prefix + "-" + suite.name
# Now we can overwrite the input file
xml.write(str(xml_file))
boot_cheribsd.run_host_command(["grep", "<testsuite", str(xml_file)])
def convert_kyua_db_to_junit_xml(db_file: Path, output_file: Path):
assert output_file.resolve() != db_file.resolve()
with output_file.open("w") as output_stream:
command = ["kyua", "report-junit", "--results-file=" + str(db_file)]
boot_cheribsd.run_host_command(command, stdout=output_stream)
# TODO: xml escape the file?
if not boot_cheribsd.PRETEND:
fixup_kyua_generated_junit_xml(output_file)
def create_junit_xml(builddir, name, tools):
_create_junit_xml(builddir, name, tools)
test_output = Path(builddir, "test-results.xml")
if not test_output.exists():
boot_cheribsd.failure("Failed to create the JUnit XML file")
return False
# boot_cheribsd.run_host_command(["head", "-n2", str(test_output)])
boot_cheribsd.run_host_command(["grep", "<testsuite", str(test_output)])
return True
def check_ssh_connection(prefix):
connection_test_start = datetime.datetime.utcnow()
boot_cheribsd.run_host_command([
"ssh", "-F",
str(Path(tempdir, "config")), "cheribsd-test-instance", "-p",
str(port), "--", "echo", "connection successful"
],
cwd=str(test_build_dir))
connection_time = (datetime.datetime.utcnow() -
connection_test_start).total_seconds()
boot_cheribsd.success(prefix, " successful after ", connection_time,
" seconds")
def run_remote_lit_tests_impl(testsuite: str,
qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace,
tempdir: str,
mp_q: multiprocessing.Queue = None,
barrier: multiprocessing.Barrier = None,
llvm_lit_path: str = None,
lit_extra_args: list = None) -> bool:
qemu.EXIT_ON_KERNEL_PANIC = False # since we run multiple threads we shouldn't use sys.exit()
boot_cheribsd.info("PID of QEMU: ", qemu.pid)
if args.pretend and os.getenv(
"FAIL_TIMEOUT_BOOT") and args.internal_shard == 2:
time.sleep(10)
if mp_q:
assert barrier is not None
notify_main_process(args,
MultiprocessStages.TESTING_SSH_CONNECTION,
mp_q,
barrier=barrier)
if args.pretend and os.getenv(
"FAIL_RAISE_EXCEPTION") and args.internal_shard == 1:
raise RuntimeError("SOMETHING WENT WRONG!")
qemu.checked_run("cat /root/.ssh/authorized_keys", timeout=20)
port = args.ssh_port
user = "******" # TODO: run these tests as non-root!
test_build_dir = Path(args.build_dir)
# TODO: move this to boot_cheribsd.py
config_contents = """
Host cheribsd-test-instance
User {user}
HostName localhost
Port {port}
IdentityFile {ssh_key}
# avoid errors due to changed host key:
UserKnownHostsFile /dev/null
StrictHostKeyChecking no
NoHostAuthenticationForLocalhost yes
# faster connection by reusing the existing one:
ControlPath {home}/.ssh/controlmasters/%r@%h:%p
# ConnectTimeout 20
# ConnectionAttempts 2
ControlMaster auto
""".format(user=user,
port=port,
ssh_key=Path(args.ssh_key).with_suffix(""),
home=Path.home())
config_contents += " ControlPersist {control_persist}\n"
# print("Writing ssh config: ", config_contents)
with Path(tempdir, "config").open("w") as c:
# Keep socket open for 10 min (600) or indefinitely (yes)
c.write(config_contents.format(control_persist="yes"))
Path(Path.home(), ".ssh/controlmasters").mkdir(exist_ok=True)
boot_cheribsd.run_host_command(["cat", str(Path(tempdir, "config"))])
# Check that the config file works:
def check_ssh_connection(prefix):
connection_test_start = datetime.datetime.utcnow()
boot_cheribsd.run_host_command([
"ssh", "-F",
str(Path(tempdir, "config")), "cheribsd-test-instance", "-p",
str(port), "--", "echo", "connection successful"
],
cwd=str(test_build_dir))
connection_time = (datetime.datetime.utcnow() -
connection_test_start).total_seconds()
boot_cheribsd.success(prefix, " successful after ", connection_time,
" seconds")
check_ssh_connection("First SSH connection")
controlmaster_running = False
try:
# Check that controlmaster worked by running ssh -O check
boot_cheribsd.info("Checking if SSH control master is working.")
boot_cheribsd.run_host_command([
"ssh", "-F",
str(Path(tempdir, "config")), "cheribsd-test-instance", "-p",
str(port), "-O", "check"
],
cwd=str(test_build_dir))
check_ssh_connection("Second SSH connection (with controlmaster)")
controlmaster_running = True
except subprocess.CalledProcessError:
boot_cheribsd.failure(
"WARNING: Could not connect to ControlMaster SSH connection. Running tests will be slower",
exit=False)
with Path(tempdir, "config").open("w") as c:
c.write(config_contents.format(control_persist="no"))
check_ssh_connection("Second SSH connection (without controlmaster)")
if args.pretend:
time.sleep(2.5)
extra_ssh_args = commandline_to_str(
("-n", "-4", "-F", "{tempdir}/config".format(tempdir=tempdir)))
extra_scp_args = commandline_to_str(
("-F", "{tempdir}/config".format(tempdir=tempdir)))
ssh_executor_args = [
args.ssh_executor_script, "--host", "cheribsd-test-instance",
"--extra-ssh-args=" + extra_ssh_args
]
if args.use_shared_mount_for_tests:
# If we have a shared directory use that to massively speed up running tests
tmpdir_name = args.shared_tmpdir_local.name
ssh_executor_args.append("--shared-mount-local-path=" +
str(args.shared_tmpdir_local))
ssh_executor_args.append("--shared-mount-remote-path=/build/" +
tmpdir_name)
else:
# slow executor using scp:
ssh_executor_args.append("--extra-scp-args=" + extra_scp_args)
executor = commandline_to_str(ssh_executor_args)
# TODO: I was previously passing -t -t to ssh. Is this actually needed?
boot_cheribsd.success("Running", testsuite, "tests with executor",
executor)
notify_main_process(args, MultiprocessStages.RUNNING_TESTS, mp_q)
# have to use -j1 since otherwise CheriBSD might wedge
if llvm_lit_path is None:
llvm_lit_path = str(test_build_dir / "bin/llvm-lit")
# Note: we require python 3 since otherwise it seems to deadlock in Jenkins
lit_cmd = [
sys.executable, llvm_lit_path, "-j1", "-vv", "-Dexecutor=" + executor,
"test"
]
if lit_extra_args:
lit_cmd.extend(lit_extra_args)
if args.lit_debug_output:
lit_cmd.append("--debug")
# This does not work since it doesn't handle running ssh commands....
lit_cmd.append(
"--timeout=120"
) # 2 minutes max per test (in case there is an infinite loop)
xunit_file = None # type: typing.Optional[Path]
if args.xunit_output:
lit_cmd.append("--xunit-xml-output")
xunit_file = Path(args.xunit_output).absolute()
if args.internal_shard:
xunit_file = xunit_file.with_name("shard-" +
str(args.internal_shard) + "-" +
xunit_file.name)
lit_cmd.append(str(xunit_file))
qemu_logfile = qemu.logfile
if args.internal_shard:
assert args.internal_num_shards, "Invalid call!"
lit_cmd.append("--num-shards=" + str(args.internal_num_shards))
lit_cmd.append("--run-shard=" + str(args.internal_shard))
if xunit_file:
assert qemu_logfile is not None, "Should have a valid logfile when running multiple shards"
boot_cheribsd.success("Writing QEMU output to ", qemu_logfile)
# Fixme starting lit at the same time does not work!
# TODO: add the polling to the main thread instead of having another thread?
# start the qemu output flushing thread so that we can see the kernel panic
qemu.flush_interval = 15 # flush the logfile every 15 seconds
should_exit_event = threading.Event()
t = threading.Thread(target=flush_thread,
args=(qemu_logfile, qemu, should_exit_event))
t.daemon = True
t.start()
shard_prefix = "SHARD" + str(
args.internal_shard) + ": " if args.internal_shard else ""
try:
boot_cheribsd.success("Starting llvm-lit: cd ", test_build_dir, " && ",
" ".join(lit_cmd))
boot_cheribsd.run_host_command(lit_cmd, cwd=str(test_build_dir))
# lit_proc = pexpect.spawnu(lit_cmd[0], lit_cmd[1:], echo=True, timeout=60, cwd=str(test_build_dir))
# TODO: get stderr!!
# while lit_proc.isalive():
# lit_proc = None
# while False:
# line = lit_proc.readline()
# if shard_prefix:
# line = shard_prefix + line
# print(line)
# global KERNEL_PANIC
# # Abort once we detect a kernel panic
# if KERNEL_PANIC:
# lit_proc.sendintr()
# print(shard_prefix + lit_proc.read())
# print("Lit finished.")
# if lit_proc and lit_proc.exitstatus == 1:
# boot_cheribsd.failure(shard_prefix + "SOME TESTS FAILED", exit=False)
except subprocess.CalledProcessError as e:
boot_cheribsd.failure(shard_prefix + "SOME TESTS FAILED: ",
e,
exit=False)
# Should only ever return 1 (otherwise something else went wrong!)
if e.returncode == 1:
return False
else:
raise
finally:
if qemu_logfile:
qemu_logfile.flush()
if controlmaster_running:
boot_cheribsd.info("Terminating SSH controlmaster")
try:
boot_cheribsd.run_host_command([
"ssh", "-F",
str(Path(tempdir, "config")), "cheribsd-test-instance",
"-p",
str(port), "-O", "exit"
],
cwd=str(test_build_dir))
except subprocess.CalledProcessError:
boot_cheribsd.failure(
"Could not close SSH controlmaster connection.",
exit=False)
qemu.flush_interval = 0.1
should_exit_event.set()
t.join(timeout=30)
if t.is_alive():
boot_cheribsd.failure(
"Failed to kill flush thread. Interacting with CheriBSD will not work!",
exit=True)
return False
if not qemu.isalive():
boot_cheribsd.failure("QEMU died while running tests! ",
qemu,
exit=True)
return True
