def main():
if "--junit-xml-only" in sys.argv or "--test-native" in sys.argv:
parser = argparse.ArgumentParser()
add_args(parser)
parser.add_argument("--test-native", action="store_true")
parser.add_argument("--build-dir")
args, remaining = parser.parse_known_args()
if args.test_native and not args.junit_xml_only:
cmd = [
args.bmake_path, "-r", "-f",
args.build_dir + "/Makefile.bsd-run", "all"
]
if args.jobs > 1:
cmd += ["-j", str(args.jobs)]
if args.use_valgrind:
cmd.append("-DUSE_VALGRIND")
boot_cheribsd.run_host_command(cmd, cwd=args.build_dir)
if not create_junit_xml(Path(args.build_dir),
args.junit_testsuite_name, args.tools):
sys.exit("Failed to create JUnit xml")
sys.exit()
# we don't need ssh running to execute the tests
run_tests_main(test_function=run_bodiagsuite,
need_ssh=False,
should_mount_builddir=True,
argparse_setup_callback=add_args,
build_dir_in_target=LONG_NAME_FOR_BUILDDIR)
def libcxx_main(barrier: Barrier = None,
mp_queue: Queue = None,
ssh_port_queue: Queue = None,
shard_num: int = None):
def set_cmdline_args(args: argparse.Namespace):
boot_cheribsd.info("Setting args:", args)
if mp_queue:
# check that we don't get a conflict
mp_debug(args, "Syncing shard ", shard_num,
" with main process. Stage: assign SSH port")
ssh_port_queue.put(
(args.ssh_port,
shard_num)) # check that we don't get a conflict
run_remote_lit_test.notify_main_process(
args, run_remote_lit_test.MultiprocessStages.BOOTING_CHERIBSD,
mp_queue, barrier)
if args.interact and (shard_num is not None or args.internal_num_shards
or args.parallel_jobs):
boot_cheribsd.failure("Cannot use --interact with multiple shards")
sys.exit()
run_remote_lit_test.adjust_common_cmdline_args(args)
def run_libcxx_tests(qemu: boot_cheribsd.CheriBSDInstance,
args: argparse.Namespace) -> bool:
with tempfile.TemporaryDirectory(
prefix="cheribuild-libcxx-tests-") as tempdir:
# TODO: do we need lit_extra_args=["-Denable_filesystem=False"]?
# Some of the tests might fail on a SMBFS directory.
return run_remote_lit_test.run_remote_lit_tests("libcxx",
qemu,
args,
tempdir,
mp_q=mp_queue,
barrier=barrier)
try:
run_tests_main(
test_function=run_libcxx_tests,
need_ssh=True, # we need ssh running to execute the tests
should_mount_builddir=True,
argparse_setup_callback=add_cmdline_args,
argparse_adjust_args_callback=set_cmdline_args)
except Exception as e:
if mp_queue:
boot_cheribsd.failure("GOT EXCEPTION in shard ",
shard_num,
": ",
sys.exc_info(),
exit=False)
# print(sys.exc_info()[2])
boot_cheribsd.info("".join(traceback.format_tb(sys.exc_info()[2])))
mp_queue.put((run_remote_lit_test.FAILURE, shard_num,
str(type(e)) + ": " + str(e)))
raise
finally:
boot_cheribsd.info("Finished running ", " ".join(sys.argv))
def libcxx_main():
from run_tests_common import run_tests_main
try:
run_tests_main(
test_function=run_libcxx_tests,
need_ssh=True, # we need ssh running to execute the tests
argparse_setup_callback=add_cmdline_args,
argparse_adjust_args_callback=set_cmdline_args)
finally:
print("Finished running ", " ".join(sys.argv))
default=default_test_output,
help="Directory for the test outputs (it will be mounted with SMB)")
parser.add_argument(
"--no-timestamped-test-subdir",
action="store_true",
help="Don't create a timestamped subdirectory in the test output dir ")
parser.add_argument("--run-cheritest",
dest="run_cheritest",
action="store_true",
default=None,
help="Run cheritest and cheriabitest")
parser.add_argument(
"--minimal-image",
action="store_true",
help=
"Set this if tests are being run on the minimal disk image rather than the full one"
)
parser.add_argument("--no-run-cheritest",
dest="run_cheritest",
action="store_false",
help="Do not run cheritest and cheriabitest")
if __name__ == '__main__':
# we set need_ssh to True here to test that SSH connections work.
run_tests_main(test_function=run_cheribsd_test,
argparse_setup_callback=add_args,
should_mount_builddir=False,
argparse_adjust_args_callback=cheribsd_setup_args,
need_ssh=True)
# The cheribuild infrastructure expects the first entry in sys.path to be
# $CHERIBUILD/test-scripts, as if this script exists in that directory. This
# script should work in such an environment, but we typically fake it by setting
# PYTHONPATH (in `.buildbot.sh`), then dropping this script's actual directory
# from sys.path.
#
# This avoids the need to write outside the buildbot test directory.
test_scripts_dir = str(
Path(
importlib.util.find_spec("run_tests_common").origin).parent.absolute())
sys.path = sys.path[sys.path.index(test_scripts_dir):]
from run_tests_common import boot_cheribsd, run_tests_main
def run_cheri_examples_tests(qemu: boot_cheribsd.QemuCheriBSDInstance,
args: argparse.Namespace) -> bool:
if args.sysroot_dir is not None:
boot_cheribsd.set_ld_library_path_with_sysroot(qemu)
boot_cheribsd.info("Running tests for cheri-examples")
return os.system("./tests/run_tests.sh") == 0
if __name__ == '__main__':
run_tests_main(test_function=run_cheri_examples_tests,
need_ssh=True,
should_mount_builddir=False)
qemu.checked_run("'/build/bin/cxxrt-test-shared' -v")
# Check the test binaries linked against libunwind
qemu.checked_run("'/build/bin/cxxrt-test-libunwind-static' -v")
qemu.checked_run("'/build/bin/cxxrt-test-libunwind-allstatic' -v")
qemu.checked_run("'/build/bin/cxxrt-test-libunwind-shared' -v")
return True
def add_args(parser: argparse.ArgumentParser):
parser.add_argument("--libunwind-build-dir", required=True)
def adjust_args(args: argparse.Namespace):
args.build_dir = os.path.abspath(
os.path.expandvars(os.path.expanduser(args.build_dir)))
args.smb_mount_directories.append(
boot_cheribsd.SmbMount(args.libunwind_build_dir,
readonly=True,
in_target="/libunwind"))
if __name__ == '__main__':
# we don't need ssh running to execute the tests
run_tests_main(test_function=run_libcxxrt_tests,
need_ssh=False,
argparse_setup_callback=add_args,
argparse_adjust_args_callback=adjust_args,
should_mount_builddir=True,
should_mount_sysroot=True)
def add_args(parser: argparse.ArgumentParser):
parser.add_argument(
"--smoketest",
action="store_true",
required=False,
default=True,
help=
"Don't run full jsc layout tests, only check that jsc and DumpRenderTree work"
)
parser.add_argument(
"--full-test",
action="store_false",
required=False,
dest="smoketest",
help=
"Don't run full jsc layout tests, only check that jsc and DumpRenderTree work"
)
if __name__ == '__main__':
# we don't need ssh running to execute the tests, but we need both host and source dir mounted
run_tests_main(test_function=run_qtwebkit_tests,
test_setup_function=setup_qtwebkit_test_environment,
argparse_setup_callback=add_args,
need_ssh=False,
should_mount_builddir=True,
should_mount_srcdir=True,
should_mount_sysroot=True)
timeout=args.test_timeout,
pretend_result=0,
ignore_cheri_trap=args.ignore_cheri_trap)
return True
def add_args(parser: argparse.ArgumentParser):
parser.add_argument(
"--ignore-cheri-trap",
action="store_true",
required=False,
default=True,
help=
"Don't fail the tests when a CHERI trap happens (useful for BODiagSuite or similar)"
)
def adjust_args(args: argparse.Namespace):
# We don't support parallel jobs but are reusing libcxx infrastructure -> set the expected vars
if not args.test_command:
boot_cheribsd.failure("--test-command must be set!", exit=True)
if __name__ == '__main__':
# we don't need ssh running to execute the tests
run_tests_main(test_function=run_simple_test,
need_ssh=False,
should_mount_builddir=True,
argparse_setup_callback=add_args,
argparse_adjust_args_callback=adjust_args)
" tests failed:\n\t",
"\n\t".join(x.name for x in failed_tests),
exit=False)
# Finally, write the Junit XML file:
if not boot_cheribsd.PRETEND:
xml.write(args.junit_xml, pretty=True)
boot_cheribsd.info("Wrote Junit results to ", args.junit_xml)
return not failed_tests
def add_args(parser: argparse.ArgumentParser):
parser.add_argument(
"--test-subset",
required=False,
default="corelib/tools",
help="Subset of tests to run (set to '.' to run all tests)")
parser.add_argument("--junit-xml",
required=False,
help="Output file name for the JUnit XML results")
if __name__ == '__main__':
# we don't need ssh running to execute the tests, but we do need the sysroot for libexecinfo+libelf
run_tests_main(test_function=run_qtbase_tests,
test_setup_function=setup_qtbase_tests,
argparse_setup_callback=add_args,
need_ssh=False,
should_mount_sysroot=True,
should_mount_srcdir=True)
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
import argparse
from run_tests_common import boot_cheribsd, run_tests_main
def run_qtbase_tests(qemu: boot_cheribsd.CheriBSDInstance, _: argparse.Namespace):
print("Running qtbase tests")
boot_cheribsd.set_ld_library_path_with_sysroot(qemu)
boot_cheribsd.prepend_ld_library_path(qemu, "/build/lib")
qemu.run("/build/tests/auto/corelib/tools/qarraydata/tst_qarraydata")
qemu.run("/build/tests/auto/corelib/global/qtendian/tst_qtendian")
# TODO: -o /path/to/file,xunitxml
return True
if __name__ == '__main__':
# we don't need ssh running to execute the tests, but we do need the sysroot for libexecinfo+libelf
run_tests_main(test_function=run_qtbase_tests, need_ssh=False, should_mount_sysroot=True)
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
import argparse
import pexpect
def run_qtbase_tests(qemu: pexpect.spawn, args: argparse.Namespace):
print("Running qtbase tests")
import boot_cheribsd
boot_cheribsd.run_cheribsd_command(
qemu, "/mnt/tests/auto/corelib/global/qtendian/tst_qtendian")
# TODO: -o /path/to/file,xunitxml
return True
if __name__ == '__main__':
from run_tests_common import run_tests_main
run_tests_main(
test_function=run_qtbase_tests,
need_ssh=False) # we don't need ssh running to execute the tests
if args.full_test:
# copy python libs from smb to tmpfs:
install_prefix = Path(args.install_prefix)
qemu.checked_run("time cp -a '{pfx}' '{pfx}.tmpfs'".format(pfx=install_prefix))
qemu.checked_run("umount '{pfx}'".format(pfx=install_prefix))
qemu.checked_run("rmdir '{pfx}' && mv '{pfx}.tmpfs' '{pfx}'".format(pfx=install_prefix))
# run basic sanity check:
build_python_exe = "python" + args.buildexe_suffix
qemu.checked_run("/build/{} --version".format(build_python_exe))
qemu.checked_run("/build/{} -E -c 'import sys; sys.exit(0)'".format(build_python_exe))
if args.full_test:
# Run the full test suite:
qemu.checked_run("cd /build && ./{} -m test -v --junit-xml=python-tests.xml".format(build_python_exe))
return True
def add_args(parser: argparse.ArgumentParser):
parser.add_argument("--buildexe-suffix", required=False, default="", help="Suffix for build executables")
parser.add_argument("--full-test", action="store_true", required=False, dest="full_test",
help="Run the full python test suite")
parser.add_argument("--smoketest", action="store_false", required=False, dest="full_test",
help="Don't run full python test suite, only check that a basic program works")
if __name__ == '__main__':
# we don't need ssh running to execute the tests
run_tests_main(test_function=run_tests, need_ssh=False, should_mount_builddir=True, should_mount_srcdir=True,
argparse_setup_callback=add_args)
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
import argparse
from run_tests_common import boot_cheribsd, run_tests_main
def run_ros2_tests(qemu: boot_cheribsd.CheriBSDInstance,
_: argparse.Namespace) -> bool:
boot_cheribsd.info("Running ROS2 tests")
boot_cheribsd.set_ld_library_path_with_sysroot(qemu)
qemu.checked_run("cd /source && sh -xe ./run-ros2-tests.sh",
timeout=240 * 60)
return True
if __name__ == '__main__':
# we don't need ssh running to execute the tests
run_tests_main(test_function=run_ros2_tests,
need_ssh=False,
should_mount_builddir=False,
should_mount_srcdir=True,
should_mount_sysroot=True)
return static_libunwind_success and static_everything_success and shared_success
def add_cmdline_args(parser: argparse.ArgumentParser):
# Only 10 tests, don't do the multiprocessing here
run_remote_lit_test.add_common_cmdline_args(
parser,
default_xunit_output="qemu-libunwind-test-results.xml",
allow_multiprocessing=False)
def adjust_cmdline_args(args: argparse.Namespace):
# We don't support parallel jobs but are reusing libcxx infrastructure -> set the expected vars
args.internal_shard = None
args.parallel_jobs = None
run_remote_lit_test.adjust_common_cmdline_args(args)
if __name__ == '__main__':
try:
run_tests_main(
test_function=run_libunwind_tests,
need_ssh=True, # we need ssh running to execute the tests
argparse_setup_callback=add_cmdline_args,
argparse_adjust_args_callback=adjust_cmdline_args,
should_mount_sysroot=True,
should_mount_builddir=True,
test_setup_function=setup_libunwind_env)
finally:
print("Finished running ", " ".join(sys.argv))
