def check_all_cases_parsed(self):
for missing_test in self.expected_test_names:
self.error("Could not find output file for test: ", missing_test)
testcase = junitparser.TestCase(name=missing_test)
testcase.result = junitparser.Error(
message="Could not find output for test " + missing_test)
self.error_suite.add_testcase(testcase)
def output_to_junit_suite(xml, output_path, suite_name, good=True):
suite = junitparser.TestSuite(suite_name)
with open(output_path, "r") as output_file:
next(output_file) # skip first header
for line in output_file:
if line[0] == "=": # stop on next header
break
split = line.split()
case = junitparser.TestCase(split[0])
exit_code = int(split[1])
if exit_code == 124:
# timeout
case.result = junitparser.Error(split[1]) # TODO error on timeout?
elif good and exit_code != 0:
# good run had bad exit code
case.result = junitparser.Failure(split[1])
elif not good and exit_code == 0: # TODO we usually? expect a cheri exception
# bad run had good exit code
case.result = junitparser.Failure(split[1])
def run_parallel(args: argparse.Namespace):
if args.pretend:
boot_cheribsd.PRETEND = True
boot_cheribsd.MESSAGE_PREFIX = "\033[0;35m" + "main process: \033[0m"
if args.parallel_jobs < 1:
boot_cheribsd.failure("Invalid number of parallel jobs: ",
args.parallel_jobs,
exit=True)
boot_cheribsd.success("Running ", args.parallel_jobs, " parallel jobs")
# to ensure that all threads have started lit
mp_barrier = Barrier(parties=args.parallel_jobs + 1, timeout=4 * 60 * 60)
mp_q = Queue()
ssh_port_queue = Queue()
processes = []
# Extract the kernel + disk image in the main process to avoid race condition:
kernel_path = boot_cheribsd.maybe_decompress(Path(args.kernel), True, True,
args) if args.kernel else None
disk_image_path = boot_cheribsd.maybe_decompress(Path(
args.disk_image), True, True, args) if args.disk_image else None
for i in range(args.parallel_jobs):
shard_num = i + 1
boot_cheribsd.info(args)
p = Process(target=run_shard,
args=(mp_q, mp_barrier, shard_num, args.parallel_jobs,
ssh_port_queue, kernel_path, disk_image_path,
args.build_dir))
p.stage = run_remote_lit_test.MultiprocessStages.FINDING_SSH_PORT
p.daemon = True # kill process on parent exit
p.name = "<LIBCXX test shard " + str(shard_num) + ">"
p.start()
processes.append(p)
atexit.register(p.terminate)
dump_processes(processes)
try:
return run_parallel_impl(args, processes, mp_q, mp_barrier,
ssh_port_queue)
except BaseException as e:
boot_cheribsd.info("Got error while running run_parallel_impl (",
type(e), "): ", e)
raise
finally:
wait_or_terminate_all_shards(processes, max_time=5, timed_out=False)
# merge junit xml files
if args.xunit_output:
boot_cheribsd.success("Merging JUnit XML outputs")
result = junitparser.JUnitXml()
xunit_file = Path(args.xunit_output).absolute()
dump_processes(processes)
for i in range(args.parallel_jobs):
shard_num = i + 1
shard_file = xunit_file.with_name("shard-" + str(shard_num) +
"-" + xunit_file.name)
mp_debug(args, processes[i], processes[i].stage)
if shard_file.exists():
result += junitparser.JUnitXml.fromfile(str(shard_file))
else:
error_msg = "ERROR: could not find JUnit XML " + str(
shard_file) + " for shard " + str(shard_num)
boot_cheribsd.failure(error_msg, exit=False)
error_suite = junitparser.TestSuite(name="failed-shard-" +
str(shard_num))
error_case = junitparser.TestCase(name="cannot-find-file")
error_case.classname = "failed-shard-" + str(shard_num)
error_case.result = junitparser.Error(message=error_msg)
error_suite.add_testcase(error_case)
result.add_testsuite(error_suite)
if processes[
i].stage != run_remote_lit_test.MultiprocessStages.EXITED:
error_msg = "ERROR: shard " + str(
shard_num
) + " did not exit cleanly! Was in stage: " + processes[
i].stage.value
if hasattr(processes[i], "error_message"):
error_msg += "\nError message:\n" + processes[
i].error_message
error_suite = junitparser.TestSuite(
name="bad-exit-shard-" + str(shard_num))
error_case = junitparser.TestCase(name="bad-exit-status")
error_case.result = junitparser.Error(message=error_msg)
error_suite.add_testcase(error_case)
result.add_testsuite(error_suite)
result.update_statistics()
result.write(str(xunit_file))
if args.pretend:
print(xunit_file.read_text())
boot_cheribsd.success("Done merging JUnit XML outputs into ",
xunit_file)
print("Duration: ", result.time)
print("Tests: ", result.tests)
print("Failures: ", result.failures)
print("Errors: ", result.errors)
print("Skipped: ", result.skipped)
def handle_testcase(self, o: Path, tools: list):
stem = o.stem
assert stem.startswith(self.test_prefix), stem
exit_code_str = o.read_text(encoding="utf-8",
errors="replace").rstrip()
testcase = junitparser.TestCase(name=stem)
try:
index = self.expected_test_names.index(stem)
except ValueError:
self.error("Found output for unknown test: ", o)
testcase.result = junitparser.Error(
message="UNEXPECTED TEST NAME: " + o.name)
testcase.system_out = exit_code_str
self.error_suite.add_testcase(testcase)
return
# test has been handled -> remove from expected list
del self.expected_test_names[index]
if o.with_suffix(".stderr").exists():
stderr = o.with_suffix(
".stderr").read_bytes().rstrip() # type: bytes
stderr = stderr.replace(b"\x00", b"\\0")
testcase.system_err = stderr.decode("utf-8", errors="replace")
try:
exit_code = int(exit_code_str)
except ValueError:
self.error("Malformed output for test: ", o)
testcase.result = junitparser.Error(
message="INVALID OUTPUT FILE CONTENTS: " + o.name)
testcase.system_out = exit_code_str
self.error_suite.add_testcase(testcase)
return
signaled = os.WIFSIGNALED(exit_code)
exited = os.WIFEXITED(exit_code)
testcase.system_out = "WIFSIGNALED={} WIFEXITED={}, WTERMSIG={}, WEXITSTATUS={} WCOREDUMP={}".format(
signaled, exited, os.WTERMSIG(exit_code),
os.WEXITSTATUS(exit_code), os.WCOREDUMP(exit_code))
# -ok testcases are expected to run succesfully -> exit code zero
if stem.endswith("-ok"):
if not exited or os.WEXITSTATUS(exit_code) != 0:
# This is not just a failure, it means something is seriously wrong if the good case fails
self.error("One of the good test cases failed: ", o)
testcase.result = junitparser.Error(
message="Expected exit code 0 but got " + exit_code_str)
self.ok_suite.add_testcase(testcase)
else:
# all others should crash
if stem.endswith("-min"):
suite = self.min_suite
elif stem.endswith("-med"):
suite = self.med_suite
elif stem.endswith("-large"):
suite = self.large_suite
else:
self.error("Malformed output for test: ", o)
testcase.result = junitparser.Error(
message="INVALID OUTPUT FILE FOUND: " + o.name)
self.error_suite.add_testcase(testcase)
return
if exit_code == 1 and testcase.system_err and testcase.system_err.startswith(
"This test needs a CWD with length"):
testcase.result = junitparser.Skipped(
message="This test needs a large working directory")
# Handle tool-specific exit codes:
if "effectivesan" in tools:
# We do not instruct EffectiveSan to terminate on first error:
if "BOUNDS ERROR:\n" not in testcase.system_err:
testcase.result = junitparser.Failure(
message=
"EffectiveSan did not detect a bounds error. Exit code "
+ exit_code_str)
elif "softboundcets" in tools:
# We do not instruct EffectiveSan to terminate on first error:
if "Softboundcets: Memory safety violation detected" not in testcase.system_err:
testcase.result = junitparser.Failure(
message=
"SoftBoundCETS did not detect a bounds error. Exit code "
+ exit_code_str)
else:
# Otherwise we assume that the test must be killed by a signal
if not signaled:
# test should fail with a signal: (162 for CHERI)
# TODO: for CHERI check that it was signal 34?
testcase.result = junitparser.Failure(
message=
"Expected test to be killed by a SIGNAL but got exit code "
+ exit_code_str)
suite.add_testcase(testcase)