def main():
# If SIGUSR1, backtrace all threads; hopefully this is early
# enough.
faulthandler.register(signal.SIGUSR1)
parser = argparse.ArgumentParser(description="list test results",
epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). SIGUSR1 will dump all thread stacks")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--quick", action="store_true",
help=("Use the previously generated '.console.txt' and '.console.diff' files"))
parser.add_argument("--update", action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--dump-args", action="store_true")
# how to parse --print directory,saved-directory,...?
parser.add_argument("--print", action="store",
default=printer.Print(printer.Print.path, printer.Print.result, printer.Print.issues),
type=printer.Print, metavar=str(printer.Print),
help="comman separate list of attributes to print for each test; default: '%(default)s'")
parser.add_argument("--stats", action="store", default=Stats.summary, type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"");
baseline_metavar = "BASELINE-DIRECTORY"
baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed"
parser.add_argument("--baseline", "-b",
metavar=baseline_metavar, help=baseline_help)
parser.add_argument("--json", action="store_true",
help="output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list")
parser.add_argument("directories", metavar="DIRECTORY-OR-FILE", nargs="+",
help="a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'")
# Note: this argument serves as documentation only. The RESULT
# argument should consumes all remaining parameters.
parser.add_argument("baseline_ignored", nargs="?",
metavar=baseline_metavar, help=baseline_help)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
# XXX: while checking for an UNTESTED test should be very cheap
# (does OUTPUT/ exist?) it isn't. Currently it triggers a full
# post-mortem analysis.
skip.add_arguments(parser, skip.skip.untested)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stderr)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Baseline: %s", args.baseline)
logger.info(" Json: %s", args.json)
logger.info(" Quick: %s", args.quick)
logger.info(" Update: %s", args.update)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
# Perhaps the baseline just contains output, magic up the
# corresponding testsuite directory.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log, header="Details:", prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log, header="Summary:", prefix=" ")
publish.json_results(logger, args)
publish.json_summary(logger, args)
return 0
def main():
parser = argparse.ArgumentParser(
description="list test results",
epilog=
"By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately)."
)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"--quick",
action="store_true",
help=
("Use the previously generated '.console.txt' and '.console.diff' files"
))
parser.add_argument(
"--update",
action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--dump-args", action="store_true")
# how to parse --print directory,saved-directory,...?
parser.add_argument(
"--print",
action="store",
default=printer.Print(printer.Print.path, printer.Print.result,
printer.Print.issues),
type=printer.Print,
metavar=str(printer.Print),
help=
"comman separate list of attributes to print for each test; default: '%(default)s'"
)
parser.add_argument(
"--stats",
action="store",
default=Stats.summary,
type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"")
baseline_metavar = "BASELINE-DIRECTORY"
baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed"
parser.add_argument("--baseline",
"-b",
metavar=baseline_metavar,
help=baseline_help)
parser.add_argument(
"--json",
action="store_true",
help=
"output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list"
)
parser.add_argument(
"directories",
metavar="DIRECTORY-OR-FILE",
nargs="+",
help=
"a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'"
)
# Note: this argument serves as documentation only. The RESULT
# argument should consumes all remaining parameters.
parser.add_argument("baseline_ignored",
nargs="?",
metavar=baseline_metavar,
help=baseline_help)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
# XXX: while checking for an UNTESTED test should be very cheap
# (does OUTPUT/ exist?) it isn't. Currently it triggers a full
# post-mortem analysis.
skip.add_arguments(parser, skip.skip.untested)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stderr)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Baseline: %s", args.baseline)
logger.info(" Json: %s", args.json)
logger.info(" Quick: %s", args.quick)
logger.info(" Update: %s", args.update)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
# Perhaps the baseline just contains output, magic up the
# corresponding testsuite directory.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'",
args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log,
header="Details:",
prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log,
header="Summary:",
prefix=" ")
publish.json_results(logger, args)
publish.json_summary(logger, args)
return 0
def results(logger, tests, baseline, args, result_stats):
failures = 0
unresolved = 0
passed = 0
nr = 0
for test in tests:
nr = nr + 1
publish.json_status(logger, args,
"rebuilding %s (test %d of %d)" % (test.name, nr, len(tests)))
# If debug logging is enabled this will provide fine grained
# per-test timing.
with logger.debug_time("processing test %s", test.name):
# Filter out tests that are being ignored?
ignored, details = ignore.test(logger, args, test)
if ignored:
result_stats.add_ignored(test, ignored)
continue
# Filter out test results that are being skipped.
#
# XXX: In the default case (skip=[UNTESTED]) this should
# be cheap (does OUTPUT/ exist?). It isn't, instead a
# full post-mortem analysis is performed.
#
# This is noticeable when printing static test value such
# as the test's name takes far longer than one would
# expect.
result = post.mortem(test, args,
baseline=baseline,
output_directory=test.saved_output_directory,
quick=args.quick)
if args.update:
result.save()
if args.skip:
if printer.Print.RESULT in args.print \
and skip.result(logger, args, result):
result_stats.add_skipped(result)
continue
result_stats.add_result(result)
if result.resolution in [post.Resolution.PASSED,
post.Resolution.UNTESTED,
post.Resolution.UNSUPPORTED]:
passed = passed + 1
elif result.resolution in [post.Resolution.UNRESOLVED]:
unresolved = unresolved + 1
else:
failures = failures + 1
publish.test_files(logger, args, result)
publish.test_output_files(logger, args, result)
publish.json_result(logger, args, result)
if baseline and test in baseline \
and not result.issues.crashed():
# Since there is a baseline and the test didn't crash
# limit what is printed to just those where the
# baseline's result is different.
#
# Note that, this skips baseline-different - where the
# baseline failed for a different reason.
if {post.Issues.BASELINE_FAILED, post.Issues.BASELINE_PASSED}.isdisjoint(result.issues):
continue
b = args.json and printer.JsonBuilder(sys.stdout) or printer.TextBuilder(sys.stdout)
printer.build_result(logger, result, baseline, args, args.print, b)
publish.json_results(logger, args)
publish.json_summary(logger, args)
publish.json_status(logger, args, "finished")
# exit code
if args.exit_ok:
return 0
elif unresolved:
return 125 # 'git bisect' magic for don't know
elif failures:
return 1
else:
return 0
