def cheribsd_setup_args(args: argparse.Namespace): if args.run_cheritest is None: # Only hybrid and purecap images have cheritest assert isinstance(args.xtarget, CrossCompileTarget) args.run_cheritest = args.xtarget.is_hybrid_or_purecap_cheri() if args.kyua_tests_files: # flatten the list (https://stackoverflow.com/a/45323085/894271): args.kyua_tests_files = functools.reduce(operator.iconcat, args.kyua_tests_files, []) print(args.kyua_tests_files) for file in args.kyua_tests_files: if not Path(file).name == "Kyuafile": boot_cheribsd.failure( "Expected a path to a Kyuafile but got: ", file) # Make sure we mount the output directory if we are running kyua and/or cheritest if args.kyua_tests_files or args.run_cheritest: test_output_dir = Path( os.path.expandvars(os.path.expanduser(args.test_output_dir))) if not test_output_dir.is_dir(): boot_cheribsd.failure("Output directory does not exist: ", test_output_dir) # Create a timestamped directory: if args.no_timestamped_test_subdir: real_output_dir = test_output_dir.absolute() else: args.timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") real_output_dir = (test_output_dir / args.timestamp).absolute() args.test_output_dir = str(real_output_dir) boot_cheribsd.run_host_command(["mkdir", "-p", str(real_output_dir)]) if not boot_cheribsd.PRETEND: (real_output_dir / "cmdline").write_text(str(sys.argv)) args.smb_mount_directories.append( boot_cheribsd.SmbMount(real_output_dir, readonly=False, in_target="/test-results"))
def cheribsd_setup_args(args: argparse.Namespace): args.use_smb_instead_of_ssh = True # skip the ssh setup args.skip_ssh_setup = True if args.kyua_tests_files: # flatten the list (https://stackoverflow.com/a/45323085/894271): args.kyua_tests_files = functools.reduce(operator.iconcat, args.kyua_tests_files, []) print(args.kyua_tests_files) for file in args.kyua_tests_files: if not Path(file).name == "Kyuafile": boot_cheribsd.failure( "Expected a path to a Kyuafile but got: ", file) test_output_dir = Path( os.path.expandvars(os.path.expanduser(args.kyua_tests_output))) if not test_output_dir.is_dir(): boot_cheribsd.failure("Output directory does not exist: ", test_output_dir) # Create a timestamped directory: if args.kyua_tests_output_no_timestamped_subdir: real_output_dir = test_output_dir.absolute() else: args.timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") real_output_dir = (test_output_dir / args.timestamp).absolute() args.kyua_tests_output = str(real_output_dir) boot_cheribsd.run_host_command(["mkdir", "-p", str(real_output_dir)]) if not boot_cheribsd.PRETEND: (real_output_dir / "cmdline").write_text(str(sys.argv)) args.smb_mount_directories.append( boot_cheribsd.SmbMount(real_output_dir, readonly=False, in_target="/kyua-results"))
def main(): if "--junit-xml-only" in sys.argv or "--test-native" in sys.argv: parser = argparse.ArgumentParser() add_args(parser) parser.add_argument("--test-native", action="store_true") parser.add_argument("--build-dir") args, remaining = parser.parse_known_args() if args.test_native and not args.junit_xml_only: cmd = [ args.bmake_path, "-r", "-f", args.build_dir + "/Makefile.bsd-run", "all" ] if args.jobs > 1: cmd += ["-j", str(args.jobs)] if args.use_valgrind: cmd.append("-DUSE_VALGRIND") boot_cheribsd.run_host_command(cmd, cwd=args.build_dir) if not create_junit_xml(Path(args.build_dir), args.junit_testsuite_name, args.tools): sys.exit("Failed to create JUnit xml") sys.exit() # we don't need ssh running to execute the tests run_tests_main(test_function=run_bodiagsuite, need_ssh=False, should_mount_builddir=True, argparse_setup_callback=add_args, build_dir_in_target=LONG_NAME_FOR_BUILDDIR)
def fixup_kyua_generated_junit_xml(xml_file: Path, prefix: str = None): boot_cheribsd.info("Updating statistics in JUnit file ", xml_file) # Process junit xml file with junitparser to update the number of tests, failures, total time, etc. orig_xml_str = xml_file.read_text("utf-8", errors='backslashreplace') xml_str = orig_xml_str for i in range(32): if chr(i) not in ("\n", "\t"): # Can't reference NULL character -> backslashescape instead # xml_str = xml_str.replace(chr(i), "&#" + str(i) + ";") xml_str = xml_str.replace(chr(i), "\\x" + format(i, '02x') + ";") with tempfile.NamedTemporaryFile("wb") as tf: # create a temporary file first to avoid clobbering the original one if we fail to parse it tf.write(xml_str.encode("ascii", errors="xmlcharrefreplace")) tf.flush() xml = junitparser.JUnitXml.fromfile(tf.name) xml.update_statistics() if prefix is not None: if isinstance(xml, junitparser.TestSuite): xml.name = prefix if xml.name is None else prefix + "-" + xml.name else: for suite in xml: suite.name = prefix if suite.name is None else prefix + "-" + suite.name # Now we can overwrite the input file xml.write(str(xml_file)) boot_cheribsd.run_host_command(["grep", "<testsuite", str(xml_file)])
def convert_kyua_db_to_junit_xml(db_file: Path, output_file: Path): assert output_file.resolve() != db_file.resolve() with output_file.open("w") as output_stream: command = ["kyua", "report-junit", "--results-file=" + str(db_file)] boot_cheribsd.run_host_command(command, stdout=output_stream) # TODO: xml escape the file? if not boot_cheribsd.PRETEND: fixup_kyua_generated_junit_xml(output_file)
def create_junit_xml(builddir, name, tools): _create_junit_xml(builddir, name, tools) test_output = Path(builddir, "test-results.xml") if not test_output.exists(): boot_cheribsd.failure("Failed to create the JUnit XML file") return False # boot_cheribsd.run_host_command(["head", "-n2", str(test_output)]) boot_cheribsd.run_host_command(["grep", "<testsuite", str(test_output)]) return True
def check_ssh_connection(prefix): connection_test_start = datetime.datetime.utcnow() boot_cheribsd.run_host_command([ "ssh", "-F", str(Path(tempdir, "config")), "cheribsd-test-instance", "-p", str(port), "--", "echo", "connection successful" ], cwd=str(test_build_dir)) connection_time = (datetime.datetime.utcnow() - connection_test_start).total_seconds() boot_cheribsd.success(prefix, " successful after ", connection_time, " seconds")
def run_remote_lit_tests_impl(testsuite: str, qemu: boot_cheribsd.CheriBSDInstance, args: argparse.Namespace, tempdir: str, mp_q: multiprocessing.Queue = None, barrier: multiprocessing.Barrier = None, llvm_lit_path: str = None, lit_extra_args: list = None) -> bool: qemu.EXIT_ON_KERNEL_PANIC = False # since we run multiple threads we shouldn't use sys.exit() boot_cheribsd.info("PID of QEMU: ", qemu.pid) if args.pretend and os.getenv( "FAIL_TIMEOUT_BOOT") and args.internal_shard == 2: time.sleep(10) if mp_q: assert barrier is not None notify_main_process(args, MultiprocessStages.TESTING_SSH_CONNECTION, mp_q, barrier=barrier) if args.pretend and os.getenv( "FAIL_RAISE_EXCEPTION") and args.internal_shard == 1: raise RuntimeError("SOMETHING WENT WRONG!") qemu.checked_run("cat /root/.ssh/authorized_keys", timeout=20) port = args.ssh_port user = "******" # TODO: run these tests as non-root! test_build_dir = Path(args.build_dir) # TODO: move this to boot_cheribsd.py config_contents = """ Host cheribsd-test-instance User {user} HostName localhost Port {port} IdentityFile {ssh_key} # avoid errors due to changed host key: UserKnownHostsFile /dev/null StrictHostKeyChecking no NoHostAuthenticationForLocalhost yes # faster connection by reusing the existing one: ControlPath {home}/.ssh/controlmasters/%r@%h:%p # ConnectTimeout 20 # ConnectionAttempts 2 ControlMaster auto """.format(user=user, port=port, ssh_key=Path(args.ssh_key).with_suffix(""), home=Path.home()) config_contents += " ControlPersist {control_persist}\n" # print("Writing ssh config: ", config_contents) with Path(tempdir, "config").open("w") as c: # Keep socket open for 10 min (600) or indefinitely (yes) c.write(config_contents.format(control_persist="yes")) Path(Path.home(), ".ssh/controlmasters").mkdir(exist_ok=True) boot_cheribsd.run_host_command(["cat", str(Path(tempdir, "config"))]) # Check that the config file works: def check_ssh_connection(prefix): connection_test_start = datetime.datetime.utcnow() boot_cheribsd.run_host_command([ "ssh", "-F", str(Path(tempdir, "config")), "cheribsd-test-instance", "-p", str(port), "--", "echo", "connection successful" ], cwd=str(test_build_dir)) connection_time = (datetime.datetime.utcnow() - connection_test_start).total_seconds() boot_cheribsd.success(prefix, " successful after ", connection_time, " seconds") check_ssh_connection("First SSH connection") controlmaster_running = False try: # Check that controlmaster worked by running ssh -O check boot_cheribsd.info("Checking if SSH control master is working.") boot_cheribsd.run_host_command([ "ssh", "-F", str(Path(tempdir, "config")), "cheribsd-test-instance", "-p", str(port), "-O", "check" ], cwd=str(test_build_dir)) check_ssh_connection("Second SSH connection (with controlmaster)") controlmaster_running = True except subprocess.CalledProcessError: boot_cheribsd.failure( "WARNING: Could not connect to ControlMaster SSH connection. Running tests will be slower", exit=False) with Path(tempdir, "config").open("w") as c: c.write(config_contents.format(control_persist="no")) check_ssh_connection("Second SSH connection (without controlmaster)") if args.pretend: time.sleep(2.5) extra_ssh_args = commandline_to_str( ("-n", "-4", "-F", "{tempdir}/config".format(tempdir=tempdir))) extra_scp_args = commandline_to_str( ("-F", "{tempdir}/config".format(tempdir=tempdir))) ssh_executor_args = [ args.ssh_executor_script, "--host", "cheribsd-test-instance", "--extra-ssh-args=" + extra_ssh_args ] if args.use_shared_mount_for_tests: # If we have a shared directory use that to massively speed up running tests tmpdir_name = args.shared_tmpdir_local.name ssh_executor_args.append("--shared-mount-local-path=" + str(args.shared_tmpdir_local)) ssh_executor_args.append("--shared-mount-remote-path=/build/" + tmpdir_name) else: # slow executor using scp: ssh_executor_args.append("--extra-scp-args=" + extra_scp_args) executor = commandline_to_str(ssh_executor_args) # TODO: I was previously passing -t -t to ssh. Is this actually needed? boot_cheribsd.success("Running", testsuite, "tests with executor", executor) notify_main_process(args, MultiprocessStages.RUNNING_TESTS, mp_q) # have to use -j1 since otherwise CheriBSD might wedge if llvm_lit_path is None: llvm_lit_path = str(test_build_dir / "bin/llvm-lit") # Note: we require python 3 since otherwise it seems to deadlock in Jenkins lit_cmd = [ sys.executable, llvm_lit_path, "-j1", "-vv", "-Dexecutor=" + executor, "test" ] if lit_extra_args: lit_cmd.extend(lit_extra_args) if args.lit_debug_output: lit_cmd.append("--debug") # This does not work since it doesn't handle running ssh commands.... lit_cmd.append( "--timeout=120" ) # 2 minutes max per test (in case there is an infinite loop) xunit_file = None # type: typing.Optional[Path] if args.xunit_output: lit_cmd.append("--xunit-xml-output") xunit_file = Path(args.xunit_output).absolute() if args.internal_shard: xunit_file = xunit_file.with_name("shard-" + str(args.internal_shard) + "-" + xunit_file.name) lit_cmd.append(str(xunit_file)) qemu_logfile = qemu.logfile if args.internal_shard: assert args.internal_num_shards, "Invalid call!" lit_cmd.append("--num-shards=" + str(args.internal_num_shards)) lit_cmd.append("--run-shard=" + str(args.internal_shard)) if xunit_file: assert qemu_logfile is not None, "Should have a valid logfile when running multiple shards" boot_cheribsd.success("Writing QEMU output to ", qemu_logfile) # Fixme starting lit at the same time does not work! # TODO: add the polling to the main thread instead of having another thread? # start the qemu output flushing thread so that we can see the kernel panic qemu.flush_interval = 15 # flush the logfile every 15 seconds should_exit_event = threading.Event() t = threading.Thread(target=flush_thread, args=(qemu_logfile, qemu, should_exit_event)) t.daemon = True t.start() shard_prefix = "SHARD" + str( args.internal_shard) + ": " if args.internal_shard else "" try: boot_cheribsd.success("Starting llvm-lit: cd ", test_build_dir, " && ", " ".join(lit_cmd)) boot_cheribsd.run_host_command(lit_cmd, cwd=str(test_build_dir)) # lit_proc = pexpect.spawnu(lit_cmd[0], lit_cmd[1:], echo=True, timeout=60, cwd=str(test_build_dir)) # TODO: get stderr!! # while lit_proc.isalive(): # lit_proc = None # while False: # line = lit_proc.readline() # if shard_prefix: # line = shard_prefix + line # print(line) # global KERNEL_PANIC # # Abort once we detect a kernel panic # if KERNEL_PANIC: # lit_proc.sendintr() # print(shard_prefix + lit_proc.read()) # print("Lit finished.") # if lit_proc and lit_proc.exitstatus == 1: # boot_cheribsd.failure(shard_prefix + "SOME TESTS FAILED", exit=False) except subprocess.CalledProcessError as e: boot_cheribsd.failure(shard_prefix + "SOME TESTS FAILED: ", e, exit=False) # Should only ever return 1 (otherwise something else went wrong!) if e.returncode == 1: return False else: raise finally: if qemu_logfile: qemu_logfile.flush() if controlmaster_running: boot_cheribsd.info("Terminating SSH controlmaster") try: boot_cheribsd.run_host_command([ "ssh", "-F", str(Path(tempdir, "config")), "cheribsd-test-instance", "-p", str(port), "-O", "exit" ], cwd=str(test_build_dir)) except subprocess.CalledProcessError: boot_cheribsd.failure( "Could not close SSH controlmaster connection.", exit=False) qemu.flush_interval = 0.1 should_exit_event.set() t.join(timeout=30) if t.is_alive(): boot_cheribsd.failure( "Failed to kill flush thread. Interacting with CheriBSD will not work!", exit=True) return False if not qemu.isalive(): boot_cheribsd.failure("QEMU died while running tests! ", qemu, exit=True) return True