def run_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
run_start = time.time()
config_request = KunitConfigRequest(request.build_dir,
request.make_options)
config_result = config_tests(linux, config_request)
if config_result.status != KunitStatus.SUCCESS:
return config_result
build_request = KunitBuildRequest(request.jobs, request.build_dir,
request.alltests, request.make_options)
build_result = build_tests(linux, build_request)
if build_result.status != KunitStatus.SUCCESS:
return build_result
exec_request = KunitExecRequest(request.timeout, request.build_dir,
request.alltests, request.filter_glob)
exec_result = exec_tests(linux, exec_request)
if exec_result.status != KunitStatus.SUCCESS:
return exec_result
parse_request = KunitParseRequest(request.raw_output, exec_result.result,
request.build_dir, request.json)
parse_result = parse_tests(parse_request)
run_end = time.time()
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
'building, %.3fs running\n') %
(run_end - run_start, config_result.elapsed_time,
build_result.elapsed_time, exec_result.elapsed_time))
return parse_result
def config_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitConfigRequest) -> KunitResult:
kunit_parser.print_with_timestamp('Configuring KUnit Kernel ...')
config_start = time.time()
success = linux.build_reconfig(request.build_dir, request.make_options)
config_end = time.time()
if not success:
return KunitResult(KunitStatus.CONFIG_FAILURE,
config_end - config_start)
return KunitResult(KunitStatus.SUCCESS, config_end - config_start)
def exec_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitExecRequest) -> KunitResult:
kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
test_start = time.time()
result = linux.run_kernel(
timeout=None if request.alltests else request.timeout,
build_dir=request.build_dir)
test_end = time.time()
return KunitResult(KunitStatus.SUCCESS, result, test_end - test_start)
def get_json_result(test: Test, def_config: str, build_dir: Optional[str],
json_path: str) -> str:
test_group = _get_group_json(test, def_config, build_dir)
test_group["name"] = "KUnit Test Group"
json_obj = json.dumps(test_group, indent=4)
if json_path != 'stdout':
with open(json_path, 'w') as result_path:
result_path.write(json_obj)
root = __file__.split('tools/testing/kunit/')[0]
kunit_parser.print_with_timestamp("Test results stored in %s" %
os.path.join(root, result_path.name))
return json_obj
def exec_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitExecRequest) -> KunitResult:
filter_globs = [request.filter_glob]
if request.run_isolated:
tests = _list_tests(linux, request)
if request.run_isolated == 'test':
filter_globs = tests
if request.run_isolated == 'suite':
filter_globs = _suites_from_test_list(tests)
# Apply the test-part of the user's glob, if present.
if '.' in request.filter_glob:
test_glob = request.filter_glob.split('.', maxsplit=2)[1]
filter_globs = [g + '.' + test_glob for g in filter_globs]
metadata = kunit_json.Metadata(arch=linux.arch(),
build_dir=request.build_dir,
def_config='kunit_defconfig')
test_counts = kunit_parser.TestCounts()
exec_time = 0.0
for i, filter_glob in enumerate(filter_globs):
kunit_parser.print_with_timestamp(
'Starting KUnit Kernel ({}/{})...'.format(i + 1,
len(filter_globs)))
test_start = time.time()
run_result = linux.run_kernel(
args=request.kernel_args,
timeout=None if request.alltests else request.timeout,
filter_glob=filter_glob,
build_dir=request.build_dir)
_, test_result = parse_tests(request, metadata, run_result)
# run_kernel() doesn't block on the kernel exiting.
# That only happens after we get the last line of output from `run_result`.
# So exec_time here actually contains parsing + execution time, which is fine.
test_end = time.time()
exec_time += test_end - test_start
test_counts.add_subtest_counts(test_result.counts)
if len(filter_globs) == 1 and test_counts.crashed > 0:
bd = request.build_dir
print(
'The kernel seems to have crashed; you can decode the stack traces with:'
)
print(
'$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} | tee {}/decoded.log | {} parse'
.format(bd, bd, kunit_kernel.get_outfile_path(bd), bd,
sys.argv[0]))
kunit_status = _map_to_overall_status(test_counts.get_status())
return KunitResult(status=kunit_status, elapsed_time=exec_time)
def build_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitBuildRequest) -> KunitResult:
kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
build_start = time.time()
success = linux.build_kernel(request.alltests, request.jobs,
request.build_dir, request.make_options)
build_end = time.time()
if not success:
return KunitResult(KunitStatus.BUILD_FAILURE, build_end - build_start)
if not success:
return KunitResult(KunitStatus.BUILD_FAILURE, build_end - build_start)
return KunitResult(KunitStatus.SUCCESS, build_end - build_start)
def parse_tests(
request: KunitParseRequest, metadata: kunit_json.Metadata,
input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]:
parse_start = time.time()
test_result = kunit_parser.Test()
if request.raw_output:
# Treat unparsed results as one passing test.
test_result.status = kunit_parser.TestStatus.SUCCESS
test_result.counts.passed = 1
output: Iterable[str] = input_data
if request.raw_output == 'all':
pass
elif request.raw_output == 'kunit':
output = kunit_parser.extract_tap_lines(output)
for line in output:
print(line.rstrip())
else:
test_result = kunit_parser.parse_run_tests(input_data)
parse_end = time.time()
if request.json:
json_str = kunit_json.get_json_result(test=test_result,
metadata=metadata)
if request.json == 'stdout':
print(json_str)
else:
with open(request.json, 'w') as f:
f.write(json_str)
kunit_parser.print_with_timestamp("Test results stored in %s" %
os.path.abspath(request.json))
if test_result.status != kunit_parser.TestStatus.SUCCESS:
return KunitResult(KunitStatus.TEST_FAILURE,
parse_end - parse_start), test_result
return KunitResult(KunitStatus.SUCCESS,
parse_end - parse_start), test_result
def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest,
parse_request: KunitParseRequest) -> KunitResult:
filter_globs = [request.filter_glob]
if request.run_isolated:
tests = _list_tests(linux, request)
if request.run_isolated == 'test':
filter_globs = tests
if request.run_isolated == 'suite':
filter_globs = _suites_from_test_list(tests)
# Apply the test-part of the user's glob, if present.
if '.' in request.filter_glob:
test_glob = request.filter_glob.split('.', maxsplit=2)[1]
filter_globs = [g + '.' + test_glob for g in filter_globs]
test_counts = kunit_parser.TestCounts()
exec_time = 0.0
for i, filter_glob in enumerate(filter_globs):
kunit_parser.print_with_timestamp(
'Starting KUnit Kernel ({}/{})...'.format(i + 1,
len(filter_globs)))
test_start = time.time()
run_result = linux.run_kernel(
args=request.kernel_args,
timeout=None if request.alltests else request.timeout,
filter_glob=filter_glob,
build_dir=request.build_dir)
result = parse_tests(parse_request, run_result)
# run_kernel() doesn't block on the kernel exiting.
# That only happens after we get the last line of output from `run_result`.
# So exec_time here actually contains parsing + execution time, which is fine.
test_end = time.time()
exec_time += test_end - test_start
test_counts.add_subtest_counts(result.result.test.counts)
kunit_status = _map_to_overall_status(test_counts.get_status())
return KunitResult(status=kunit_status,
result=result.result,
elapsed_time=exec_time)
def run_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
run_start = time.time()
config_result = config_tests(linux, request)
if config_result.status != KunitStatus.SUCCESS:
return config_result
build_result = build_tests(linux, request)
if build_result.status != KunitStatus.SUCCESS:
return build_result
exec_result = exec_tests(linux, request)
run_end = time.time()
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
'building, %.3fs running\n') %
(run_end - run_start, config_result.elapsed_time,
build_result.elapsed_time, exec_result.elapsed_time))
return exec_result
def build_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
config_start = time.time()
success = linux.build_reconfig(request.build_dir)
config_end = time.time()
if not success:
return KunitResult(KunitStatus.CONFIG_FAILURE,
'could not configure kernel')
kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
build_start = time.time()
success = linux.build_um_kernel(request.jobs, request.build_dir)
build_end = time.time()
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs configuring, %.3fs ' + 'building\n') %
(config_end - config_start, build_end - build_start))
if not success:
return KunitResult(KunitStatus.BUILD_FAILURE, 'could not build kernel')
else:
return KunitResult(KunitStatus.SUCCESS, '')
def run_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
test_start = time.time()
test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, [],
'Tests not Parsed.')
if request.raw_output:
kunit_parser.raw_output(
linux.run_kernel(timeout=request.timeout,
build_dir=request.build_dir))
else:
kunit_output = linux.run_kernel(timeout=request.timeout,
build_dir=request.build_dir)
test_result = kunit_parser.parse_run_tests(kunit_output)
test_end = time.time()
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs running\n') % (test_end - test_start))
if test_result.status != kunit_parser.TestStatus.SUCCESS:
return KunitResult(KunitStatus.TEST_FAILURE, test_result)
else:
return KunitResult(KunitStatus.SUCCESS, test_result)
def make_allyesconfig(self):
kunit_parser.print_with_timestamp('Enabling all CONFIGs for UML...')
process = subprocess.Popen(['make', 'ARCH=um', 'allyesconfig'],
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
process.wait()
kunit_parser.print_with_timestamp(
'Disabling broken configs to run KUnit tests...')
with ExitStack() as es:
config = open(KCONFIG_PATH, 'a')
disable = open(BROKEN_ALLCONFIG_PATH, 'r').read()
config.write(disable)
kunit_parser.print_with_timestamp(
'Starting Kernel with all configs takes a few minutes...')
def make_allyesconfig(self, build_dir: str, make_options) -> None:
kunit_parser.print_with_timestamp('Enabling all CONFIGs for UML...')
command = ['make', 'ARCH=um', 'O=' + build_dir, 'allyesconfig']
if make_options:
command.extend(make_options)
process = subprocess.Popen(command,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
process.wait()
kunit_parser.print_with_timestamp(
'Disabling broken configs to run KUnit tests...')
with open(get_kconfig_path(build_dir), 'a') as config:
with open(BROKEN_ALLCONFIG_PATH, 'r') as disable:
config.write(disable.read())
kunit_parser.print_with_timestamp(
'Starting Kernel with all configs takes a few minutes...')
def run_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
if request.defconfig:
create_default_kunitconfig()
config_start = time.time()
success = linux.build_reconfig(request.build_dir)
config_end = time.time()
if not success:
return KunitResult(KunitStatus.CONFIG_FAILURE,
'could not configure kernel')
kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
build_start = time.time()
success = linux.build_um_kernel(request.jobs, request.build_dir)
build_end = time.time()
if not success:
return KunitResult(KunitStatus.BUILD_FAILURE, 'could not build kernel')
kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
test_start = time.time()
test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, [],
'Tests not Parsed.')
if request.raw_output:
kunit_parser.raw_output(
linux.run_kernel(timeout=request.timeout,
build_dir=request.build_dir))
else:
kunit_output = linux.run_kernel(timeout=request.timeout,
build_dir=request.build_dir)
test_result = kunit_parser.parse_run_tests(kunit_output)
test_end = time.time()
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
'building, %.3fs running\n') %
(test_end - config_start, config_end - config_start,
build_end - build_start, test_end - test_start))
if test_result.status != kunit_parser.TestStatus.SUCCESS:
return KunitResult(KunitStatus.TEST_FAILURE, test_result)
else:
return KunitResult(KunitStatus.SUCCESS, test_result)
def run_tests(linux: kunit_kernel.LinuxSourceTree,
request: KunitRequest) -> KunitResult:
config_start = time.time()
success = linux.build_reconfig(request.build_dir, request.make_options)
config_end = time.time()
if not success:
return KunitResult(KunitStatus.CONFIG_FAILURE,
'could not configure kernel')
kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
build_start = time.time()
success = linux.build_um_kernel(request.alltests, request.jobs,
request.build_dir, request.make_options)
build_end = time.time()
if not success:
return KunitResult(KunitStatus.BUILD_FAILURE, 'could not build kernel')
kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
test_start = time.time()
kunit_output = linux.run_kernel(
timeout=None if request.alltests else request.timeout,
build_dir=request.build_dir)
if request.raw_output:
raw_output = kunit_parser.raw_output(kunit_output)
isolated = list(kunit_parser.isolate_kunit_output(raw_output))
test_result = kunit_parser.parse_test_result(isolated)
else:
test_result = kunit_parser.parse_run_tests(kunit_output)
test_end = time.time()
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
'building, %.3fs running\n') %
(test_end - config_start, config_end - config_start,
build_end - build_start, test_end - test_start))
if test_result.status != kunit_parser.TestStatus.SUCCESS:
return KunitResult(KunitStatus.TEST_FAILURE, test_result)
else:
return KunitResult(KunitStatus.SUCCESS, test_result)
parser.add_argument('--timeout', help='maximum number of seconds to allow for '
'all tests to run. This does not include time taken to '
'build the tests.', type=int, default=300,
metavar='timeout')
cli_args = parser.parse_args()
linux = kunit_kernel.LinuxSourceTree()
config_start = time.time()
success = linux.build_reconfig()
config_end = time.time()
if not success:
quit()
kunit_parser.print_with_timestamp('Building KUnit Kernel ...')
build_start = time.time()
success = linux.build_um_kernel()
build_end = time.time()
if not success:
quit()
kunit_parser.print_with_timestamp('Starting KUnit Kernel ...')
test_start = time.time()
if cli_args.raw_output:
kunit_parser.raw_output(linux.run_kernel(timeout=cli_args.timeout))
else:
kunit_parser.parse_run_tests(linux.run_kernel(timeout=cli_args.timeout))
def main(argv, linux=None):
parser = argparse.ArgumentParser(
description='Helps writing and running KUnit tests.')
subparser = parser.add_subparsers(dest='subcommand')
# The 'run' command will config, build, exec, and parse in one go.
run_parser = subparser.add_parser('run', help='Runs KUnit tests.')
add_common_opts(run_parser)
add_build_opts(run_parser)
add_exec_opts(run_parser)
add_parse_opts(run_parser)
config_parser = subparser.add_parser(
'config',
help='Ensures that .config contains all of '
'the options in .kunitconfig')
add_common_opts(config_parser)
build_parser = subparser.add_parser(
'build', help='Builds a kernel with KUnit tests')
add_common_opts(build_parser)
add_build_opts(build_parser)
exec_parser = subparser.add_parser('exec',
help='Run a kernel with KUnit tests')
add_common_opts(exec_parser)
add_exec_opts(exec_parser)
add_parse_opts(exec_parser)
# The 'parse' option is special, as it doesn't need the kernel source
# (therefore there is no need for a build_dir, hence no add_common_opts)
# and the '--file' argument is not relevant to 'run', so isn't in
# add_parse_opts()
parse_parser = subparser.add_parser(
'parse',
help='Parses KUnit results from a file, '
'and parses formatted results.')
add_parse_opts(parse_parser)
parse_parser.add_argument('file',
help='Specifies the file to read results from.',
type=str,
nargs='?',
metavar='input_file')
cli_args = parser.parse_args(argv)
if get_kernel_root_path():
os.chdir(get_kernel_root_path())
if cli_args.subcommand == 'run':
if not os.path.exists(cli_args.build_dir):
os.mkdir(cli_args.build_dir)
if not linux:
linux = kunit_kernel.LinuxSourceTree(
cli_args.build_dir,
kunitconfig_path=cli_args.kunitconfig,
arch=cli_args.arch,
cross_compile=cli_args.cross_compile,
qemu_config_path=cli_args.qemu_config)
request = KunitRequest(cli_args.raw_output, cli_args.timeout,
cli_args.jobs, cli_args.build_dir,
cli_args.alltests, cli_args.filter_glob,
cli_args.json, cli_args.make_options)
result = run_tests(linux, request)
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
elif cli_args.subcommand == 'config':
if cli_args.build_dir and (not os.path.exists(cli_args.build_dir)):
os.mkdir(cli_args.build_dir)
if not linux:
linux = kunit_kernel.LinuxSourceTree(
cli_args.build_dir,
kunitconfig_path=cli_args.kunitconfig,
arch=cli_args.arch,
cross_compile=cli_args.cross_compile,
qemu_config_path=cli_args.qemu_config)
request = KunitConfigRequest(cli_args.build_dir, cli_args.make_options)
result = config_tests(linux, request)
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs\n') % (result.elapsed_time))
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
elif cli_args.subcommand == 'build':
if not linux:
linux = kunit_kernel.LinuxSourceTree(
cli_args.build_dir,
kunitconfig_path=cli_args.kunitconfig,
arch=cli_args.arch,
cross_compile=cli_args.cross_compile,
qemu_config_path=cli_args.qemu_config)
request = KunitBuildRequest(cli_args.jobs, cli_args.build_dir,
cli_args.alltests, cli_args.make_options)
result = build_tests(linux, request)
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs\n') % (result.elapsed_time))
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
elif cli_args.subcommand == 'exec':
if not linux:
linux = kunit_kernel.LinuxSourceTree(
cli_args.build_dir,
kunitconfig_path=cli_args.kunitconfig,
arch=cli_args.arch,
cross_compile=cli_args.cross_compile,
qemu_config_path=cli_args.qemu_config)
exec_request = KunitExecRequest(cli_args.timeout, cli_args.build_dir,
cli_args.alltests,
cli_args.filter_glob)
exec_result = exec_tests(linux, exec_request)
parse_request = KunitParseRequest(cli_args.raw_output,
exec_result.result,
cli_args.build_dir, cli_args.json)
result = parse_tests(parse_request)
kunit_parser.print_with_timestamp(
('Elapsed time: %.3fs\n') % (exec_result.elapsed_time))
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
elif cli_args.subcommand == 'parse':
if cli_args.file == None:
kunit_output = sys.stdin
else:
with open(cli_args.file, 'r') as f:
kunit_output = f.read().splitlines()
request = KunitParseRequest(cli_args.raw_output, kunit_output, None,
cli_args.json)
result = parse_tests(request)
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
else:
parser.print_help()
# TODO: Add attachments attribute in test_case with detailed
# failure message, see https://api.kernelci.org/schema-test-case.html#get
for case in test_suite.cases:
test_case = {"name": case.name, "status": "FAIL"}
if case.status == TestStatus.SUCCESS:
test_case["status"] = "PASS"
elif case.status == TestStatus.TEST_CRASHED:
test_case["status"] = "ERROR"
test_cases.append(test_case)
sub_group["test_cases"] = test_cases
sub_groups.append(sub_group)
test_group = {
"name": "KUnit Test Group",
"arch": "UM",
"defconfig": def_config,
"build_environment": build_dir,
"sub_groups": sub_groups,
"lab_name": None,
"kernel": None,
"job": None,
"git_branch": "kselftest",
}
json_obj = json.dumps(test_group, indent=4)
if json_path != 'stdout':
with open(json_path, 'w') as result_path:
result_path.write(json_obj)
root = __file__.split('tools/testing/kunit/')[0]
kunit_parser.print_with_timestamp("Test results stored in %s" %
os.path.join(root, result_path.name))
return json_obj
