def main():
parser = argparse.ArgumentParser(description="Run tests")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("directories", metavar="DIRECTORY", nargs="+",
help="a testsuite directory, a TESTLIST file, or a list of test directories")
testsuite.add_arguments(parser)
runner.add_arguments(parser)
post.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmrunner")
logger.info("Options:")
logger.info(" directories: %s", args.directories)
logger.info(" verbose: %s", args.verbose)
testsuite.log_arguments(logger, args)
runner.log_arguments(logger, args)
post.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args,
log_level=logutil.INFO)
if not tests:
logger.error("test or testsuite directory invalid: %s", args.directories)
return 1
test_stats = stats.Tests()
result_stats = stats.Results()
try:
exit_code = 0
logger.info("run started at %s", timing.START_TIME)
runner.run_tests(logger, args, tests, test_stats, result_stats)
except KeyboardInterrupt:
logger.exception("**** interrupted ****")
exit_code = 1
test_stats.log_details(args.verbose and logger.info or logger.debug,
header="final stat details:", prefix=" ")
result_stats.log_details(logger.info, header="final test details:", prefix=" ")
test_stats.log_summary(logger.info, header="final test stats:", prefix=" ")
result_stats.log_summary(logger.info, header="final test results:", prefix=" ")
end_time = datetime.now()
logger.info("run finished at %s after %s", end_time, end_time - timing.START_TIME)
return exit_code
def main():
# If SIGUSR1, backtrace all threads; hopefully this is early
# enough.
faulthandler.register(signal.SIGUSR1)
parser = argparse.ArgumentParser(
description="Run tests", epilog="SIGUSR1 will dump all thread stacks")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--pid-file",
default="",
help="file to store process id of KVMRUNNER")
parser.add_argument(
"directories",
metavar="DIRECTORY",
nargs="+",
help=
"a testsuite directory, a TESTLIST file, or a list of test directories"
)
testsuite.add_arguments(parser)
runner.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stdout)
logger = logutil.getLogger("kvmrunner")
logger.info("Options:")
logger.info(" directories: %s", args.directories)
logger.info(" verbose: %s", args.verbose)
logger.info(" pid-file: %s", args.pid_file)
testsuite.log_arguments(logger, args)
runner.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
if args.pid_file:
pid = os.getpid()
logger.info("writing pid %d to '%s'", pid, args.pid_file)
with open(args.pid_file, "wt") as pidfile:
pidfile.write("%d\n" % os.getpid())
tests = testsuite.load_testsuite_or_tests(logger,
args.directories,
args,
log_level=logutil.INFO)
if not tests:
logger.error("test or testsuite directory invalid: %s",
args.directories)
return 1
if len(tests) == 1 and args.run_post_mortem is None:
logger.warning(
"skipping post-mortem.sh as only one test; use --run-post-mortem true to override this"
)
args.run_post_mortem = False
test_stats = stats.Tests()
result_stats = stats.Results()
try:
exit_code = 0
logger.info("run started at %s", timing.START_TIME)
runner.run_tests(logger, args, tests, test_stats, result_stats)
except KeyboardInterrupt:
logger.exception("**** interrupted ****")
exit_code = 1
test_stats.log_details(args.verbose and logger.info or logger.debug,
header="final stat details:",
prefix=" ")
result_stats.log_details(logger.info,
header="final test details:",
prefix=" ")
test_stats.log_summary(logger.info,
header="final test stats:",
prefix=" ")
result_stats.log_summary(logger.info,
header="final test results:",
prefix=" ")
stop_time = datetime.now()
logger.info("run finished at %s after %s", stop_time,
stop_time - timing.START_TIME)
return exit_code
def main():
parser = argparse.ArgumentParser(description="Run tests")
# This argument's behaviour is overloaded; the shorter word "try"
# is a python word.
parser.add_argument("--retry", type=int, metavar="COUNT", default=1,
help="which previously run tests should be retried: 0 selects not-started tests; 1 selects not-started+failed tests; -1 selects not-started+failed+passed tests (default is %(default)s)")
parser.add_argument("--attempts", type=int, default=1,
help="number of times to attempt a test before giving up; default %(default)s")
parser.add_argument("--dry-run", "-n", action="store_true")
parser.add_argument("--verbose", "-v", action="count", default=0)
# Default to BACKUP under the current directory. Name is
# arbitrary, chosen for its hopefully unique first letter
# (avoiding Makefile, OBJ, README, ... :-).
parser.add_argument("--backup-directory", metavar="DIRECTORY",
default=os.path.join("BACKUP", time.strftime("%Y%m%d%H%M%S", time.localtime())),
help="backup existing <test>/OUTPUT to %(metavar)s/<test> (default: %(default)s)")
parser.add_argument("directories", metavar="DIRECTORY", nargs="+",
help="either a testsuite directory or a list of test directories")
testsuite.add_arguments(parser)
runner.add_arguments(parser)
post.add_arguments(parser)
logutil.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmrunner")
logger.info("Options:")
logger.info(" retry: %s", args.retry)
logger.info(" attempts: %s", args.attempts)
logger.info(" dry-run: %s", args.dry_run)
logger.info(" backup-directory: %s", args.backup_directory)
logger.info(" directories: %s", args.directories)
testsuite.log_arguments(logger, args)
runner.log_arguments(logger, args)
post.log_arguments(logger, args)
logutil.log_arguments(logger, args)
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args,
log_level=logutil.INFO)
if not tests:
logger.error("test or testsuite directory invalid: %s", args.directories)
return 1
test_stats = stats.Tests()
result_stats = stats.Results()
start_time = datetime.now()
try:
logger.info("run started at %s", start_time)
test_count = 0
for test in tests:
test_stats.add("total", test)
test_count += 1
# Would the number of tests to be [re]run be better?
test_prefix = "****** %s (test %d of %d)" % (test.name, test_count, len(tests))
ignore = testsuite.ignore(test, args)
if ignore:
result_stats.add_ignore(test, ignore)
test_stats.add("ignored", test)
# No need to log all the ignored tests when an
# explicit sub-set of tests is being run. For
# instance, when running just one test.
if not args.test_name:
logger.info("%s: ignore (%s)", test_prefix, ignore)
continue
# Implement "--retry" as described above: if retry is -ve,
# the test is always run; if there's no result, the test
# is always run; skip passed tests; else things get a
# little wierd.
# Be lazy with gathering the results, don't run the
# sanitizer or diff.
old_result = post.mortem(test, args, skip_diff=True, skip_sanitize=True)
if args.retry >= 0:
if old_result:
if old_result.passed:
logger.info("%s: passed", test_prefix)
test_stats.add("skipped", test)
result_stats.add_skip(old_result)
continue
if args.retry == 0:
logger.info("%s: %s (delete '%s' to re-test)", test_prefix,
result, test.output_directory)
test_stats.add("skipped", test)
result_stats.add_skip(old_result)
continue
test_stats.add("retry", test)
logger.info("%s: starting ...", test_prefix)
test_stats.add("tests", test)
# Move the contents of the existing OUTPUT directory to
# BACKUP_DIRECTORY. Do it file-by-file so that, at no
# point, the directory is empty.
#
# By moving each test just before it is started a trail of
# what tests were attempted at each run is left.
#
# XXX: During boot, swan-transmogrify runs "chcon -R
# testing/pluto". Of course this means that each time a
# test is added and/or a test is run (adding files under
# <test>/OUTPUT), the boot process (and consequently the
# time taken to run a test) keeps increasing.
#
# Always moving the directory contents to the
# BACKUP_DIRECTORY mitigates this some.
saved_output_directory = None
if os.path.exists(test.output_directory):
saved_output_directory = os.path.join(args.backup_directory, test.name)
logger.info("moving contents of '%s' to '%s'",
test.output_directory, saved_output_directory)
# Copy "empty" OUTPUT directories too.
args.dry_run or os.makedirs(saved_output_directory, exist_ok=True)
for name in os.listdir(test.output_directory):
src = os.path.join(test.output_directory, name)
dst = os.path.join(saved_output_directory, name)
logger.debug("moving '%s' to '%s'", src, dst)
args.dry_run or os.replace(src, dst)
debugfile = None
result = None
# At least one iteration; above will have filtered out
# skips and ignores
for attempt in range(args.attempts):
test_stats.add("attempts", test)
# Create the OUTPUT directory.
try:
if not args.dry_run:
os.mkdir(test.output_directory)
elif os.exists(test.output_directory):
raise FileExistsError()
except FileExistsError:
# On first attempt, the OUTPUT directory will
# be empty (see above) so no need to save.
if attempt > 0:
saved_output_directory = os.path.join(test.output_directory, str(attempt))
logger.info("moving contents of '%s' to '%s'",
test.output_directory, saved_output_directory)
args.dry_run or os.makedirs(saved_output_directory, exist_ok=True)
for name in os.listdir(test.output_directory):
if os.path.isfile(src):
src = os.path.join(test.output_directory, name)
dst = os.path.join(saved_output_directory, name)
logger.debug("moving '%s' to '%s'", src, dst)
args.dry_run or os.replace(src, dst)
# Start a debug log in the OUTPUT directory; include
# timing for this specific test attempt.
with logutil.TIMER, logutil.Debug(logger, os.path.join(test.output_directory, "debug.log")):
logger.info("****** test %s attempt %d of %d started at %s ******",
test.name, attempt+1, args.attempts, datetime.now())
if saved_output_directory:
logger.info("contents of '%s' moved to '%s'",
test.output_directory, saved_output_directory)
saved_output_directory = None
ending = "undefined"
try:
if not args.dry_run:
runner.run_test(test, max_workers=args.workers)
ending = "finished"
result = post.mortem(test, args, update=(not args.dry_run))
if not args.dry_run:
# Store enough to fool the script
# pluto-testlist-scan.sh.
logger.info("storing result in '%s'", test.result_file)
with open(test.result_file, "w") as f:
f.write('"result": "%s"\n' % result)
except pexpect.TIMEOUT as e:
ending = "timeout"
logger.exception("**** test %s timed out ****", test.name)
result = post.mortem(test, args, update=(not args.dry_run))
# Since the OUTPUT directory exists, all paths to
# here should have a non-null RESULT.
test_stats.add("attempts(%s:%s)" % (ending, result), test)
if result.errors:
logger.info("****** test %s %s %s ******", test.name, result, result.errors)
else:
logger.info("****** test %s %s ******", test.name, result)
if result.passed:
break
# Above will have set RESULT. During a control-c or crash
# the below will not be executed.
test_stats.add("tests(%s)" % result, test)
result_stats.add_result(result, old_result)
test_stats.log_summary(logger.info, header="updated stats:",
prefix=" ")
result_stats.log_summary(logger.info, header="updated results:",
prefix=" ")
except KeyboardInterrupt:
logger.exception("**** test %s interrupted ****", test.name)
return 1
level = args.verbose and logger.info or logger.debug
test_stats.log_details(level, header="stat details:", prefix=" ")
result_stats.log_details(logger.info, header="result details:", prefix=" ")
test_stats.log_summary(logger.info, header="stat summary:", prefix=" ")
result_stats.log_summary(logger.info, header="result summary:", prefix=" ")
end_time = datetime.now()
logger.info("run finished at %s after %s", end_time, end_time - start_time)
return 0
def main():
# If SIGUSR1, backtrace all threads; hopefully this is early
# enough.
faulthandler.register(signal.SIGUSR1)
parser = argparse.ArgumentParser(description="Run tests",
epilog="SIGUSR1 will dump all thread stacks")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("directories", metavar="DIRECTORY", nargs="+",
help="a testsuite directory, a TESTLIST file, or a list of test directories")
testsuite.add_arguments(parser)
runner.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stdout)
logger = logutil.getLogger("kvmrunner")
logger.info("Options:")
logger.info(" directories: %s", args.directories)
logger.info(" verbose: %s", args.verbose)
testsuite.log_arguments(logger, args)
runner.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args,
log_level=logutil.INFO)
if not tests:
logger.error("test or testsuite directory invalid: %s", args.directories)
return 1
test_stats = stats.Tests()
result_stats = stats.Results()
try:
exit_code = 0
logger.info("run started at %s", timing.START_TIME)
runner.run_tests(logger, args, tests, test_stats, result_stats)
except KeyboardInterrupt:
logger.exception("**** interrupted ****")
exit_code = 1
test_stats.log_details(args.verbose and logger.info or logger.debug,
header="final stat details:", prefix=" ")
result_stats.log_details(logger.info, header="final test details:", prefix=" ")
test_stats.log_summary(logger.info, header="final test stats:", prefix=" ")
result_stats.log_summary(logger.info, header="final test results:", prefix=" ")
stop_time = datetime.now()
logger.info("run finished at %s after %s", stop_time, stop_time - timing.START_TIME)
return exit_code