def main():
parser = argparse.ArgumentParser(description="Run tests")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("directories", metavar="DIRECTORY", nargs="+",
help="a testsuite directory, a TESTLIST file, or a list of test directories")
testsuite.add_arguments(parser)
runner.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stdout)
logger = logutil.getLogger("kvmrunner")
logger.info("Options:")
logger.info(" directories: %s", args.directories)
logger.info(" verbose: %s", args.verbose)
testsuite.log_arguments(logger, args)
runner.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args,
log_level=logutil.INFO)
if not tests:
logger.error("test or testsuite directory invalid: %s", args.directories)
return 1
test_stats = stats.Tests()
result_stats = stats.Results()
try:
exit_code = 0
logger.info("run started at %s", timing.START_TIME)
runner.run_tests(logger, args, tests, test_stats, result_stats)
except KeyboardInterrupt:
logger.exception("**** interrupted ****")
exit_code = 1
test_stats.log_details(args.verbose and logger.info or logger.debug,
header="final stat details:", prefix=" ")
result_stats.log_details(logger.info, header="final test details:", prefix=" ")
test_stats.log_summary(logger.info, header="final test stats:", prefix=" ")
result_stats.log_summary(logger.info, header="final test results:", prefix=" ")
end_time = datetime.now()
logger.info("run finished at %s after %s", end_time, end_time - timing.START_TIME)
return exit_code
def main():
parser = argparse.ArgumentParser(
description="list test results",
epilog=
"By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately)."
)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"--quick",
action="store_true",
help=
("Use the previously generated '.console.txt' and '.console.diff' files"
))
parser.add_argument(
"--update",
action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--dump-args", action="store_true")
# how to parse --print directory,saved-directory,...?
parser.add_argument(
"--print",
action="store",
default=printer.Print(printer.Print.path, printer.Print.result,
printer.Print.issues),
type=printer.Print,
metavar=str(printer.Print),
help=
"comman separate list of attributes to print for each test; default: '%(default)s'"
)
parser.add_argument(
"--stats",
action="store",
default=Stats.summary,
type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"")
baseline_metavar = "BASELINE-DIRECTORY"
baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed"
parser.add_argument("--baseline",
"-b",
metavar=baseline_metavar,
help=baseline_help)
parser.add_argument(
"--json",
action="store_true",
help=
"output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list"
)
parser.add_argument(
"directories",
metavar="DIRECTORY-OR-FILE",
nargs="+",
help=
"a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'"
)
# Note: this argument serves as documentation only. The RESULT
# argument should consumes all remaining parameters.
parser.add_argument("baseline_ignored",
nargs="?",
metavar=baseline_metavar,
help=baseline_help)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
# XXX: while checking for an UNTESTED test should be very cheap
# (does OUTPUT/ exist?) it isn't. Currently it triggers a full
# post-mortem analysis.
skip.add_arguments(parser, skip.skip.untested)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stderr)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Baseline: %s", args.baseline)
logger.info(" Json: %s", args.json)
logger.info(" Quick: %s", args.quick)
logger.info(" Update: %s", args.update)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
# Perhaps the baseline just contains output, magic up the
# corresponding testsuite directory.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'",
args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log,
header="Details:",
prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log,
header="Summary:",
prefix=" ")
publish.json_results(logger, args)
publish.json_summary(logger, args)
return 0
def main():
# If SIGUSR1, backtrace all threads; hopefully this is early
# enough.
faulthandler.register(signal.SIGUSR1)
parser = argparse.ArgumentParser(
description="Run tests", epilog="SIGUSR1 will dump all thread stacks")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--pid-file",
default="",
help="file to store process id of KVMRUNNER")
parser.add_argument(
"directories",
metavar="DIRECTORY",
nargs="+",
help=
"a testsuite directory, a TESTLIST file, or a list of test directories"
)
testsuite.add_arguments(parser)
runner.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stdout)
logger = logutil.getLogger("kvmrunner")
logger.info("Options:")
logger.info(" directories: %s", args.directories)
logger.info(" verbose: %s", args.verbose)
logger.info(" pid-file: %s", args.pid_file)
testsuite.log_arguments(logger, args)
runner.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
if args.pid_file:
pid = os.getpid()
logger.info("writing pid %d to '%s'", pid, args.pid_file)
with open(args.pid_file, "wt") as pidfile:
pidfile.write("%d\n" % os.getpid())
tests = testsuite.load_testsuite_or_tests(logger,
args.directories,
args,
log_level=logutil.INFO)
if not tests:
logger.error("test or testsuite directory invalid: %s",
args.directories)
return 1
if len(tests) == 1 and args.run_post_mortem is None:
logger.warning(
"skipping post-mortem.sh as only one test; use --run-post-mortem true to override this"
)
args.run_post_mortem = False
test_stats = stats.Tests()
result_stats = stats.Results()
try:
exit_code = 0
logger.info("run started at %s", timing.START_TIME)
runner.run_tests(logger, args, tests, test_stats, result_stats)
except KeyboardInterrupt:
logger.exception("**** interrupted ****")
exit_code = 1
test_stats.log_details(args.verbose and logger.info or logger.debug,
header="final stat details:",
prefix=" ")
result_stats.log_details(logger.info,
header="final test details:",
prefix=" ")
test_stats.log_summary(logger.info,
header="final test stats:",
prefix=" ")
result_stats.log_summary(logger.info,
header="final test results:",
prefix=" ")
stop_time = datetime.now()
logger.info("run finished at %s after %s", stop_time,
stop_time - timing.START_TIME)
return exit_code
def main():
parser = argparse.ArgumentParser(description="Run tests")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"directories",
metavar="DIRECTORY",
nargs="+",
help=
"a testsuite directory, a TESTLIST file, or a list of test directories"
)
testsuite.add_arguments(parser)
runner.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stdout)
logger = logutil.getLogger("kvmrunner")
logger.info("Options:")
logger.info(" directories: %s", args.directories)
logger.info(" verbose: %s", args.verbose)
testsuite.log_arguments(logger, args)
runner.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
tests = testsuite.load_testsuite_or_tests(logger,
args.directories,
args,
log_level=logutil.INFO)
if not tests:
logger.error("test or testsuite directory invalid: %s",
args.directories)
return 1
test_stats = stats.Tests()
result_stats = stats.Results()
try:
exit_code = 0
logger.info("run started at %s", timing.START_TIME)
runner.run_tests(logger, args, tests, test_stats, result_stats)
except KeyboardInterrupt:
logger.exception("**** interrupted ****")
exit_code = 1
test_stats.log_details(args.verbose and logger.info or logger.debug,
header="final stat details:",
prefix=" ")
result_stats.log_details(logger.info,
header="final test details:",
prefix=" ")
test_stats.log_summary(logger.info,
header="final test stats:",
prefix=" ")
result_stats.log_summary(logger.info,
header="final test results:",
prefix=" ")
end_time = datetime.now()
logger.info("run finished at %s after %s", end_time,
end_time - timing.START_TIME)
return exit_code
def main():
# If SIGUSR1, backtrace all threads; hopefully this is early
# enough.
faulthandler.register(signal.SIGUSR1)
parser = argparse.ArgumentParser(description="list test results",
epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). SIGUSR1 will dump all thread stacks")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--quick", action="store_true",
help=("Use the previously generated '.console.txt' and '.console.diff' files"))
parser.add_argument("--update", action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--dump-args", action="store_true")
# how to parse --print directory,saved-directory,...?
parser.add_argument("--print", action="store",
default=printer.Print(printer.Print.path, printer.Print.result, printer.Print.issues),
type=printer.Print, metavar=str(printer.Print),
help="comman separate list of attributes to print for each test; default: '%(default)s'")
parser.add_argument("--stats", action="store", default=Stats.summary, type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"");
baseline_metavar = "BASELINE-DIRECTORY"
baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed"
parser.add_argument("--baseline", "-b",
metavar=baseline_metavar, help=baseline_help)
parser.add_argument("--json", action="store_true",
help="output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list")
parser.add_argument("directories", metavar="DIRECTORY-OR-FILE", nargs="+",
help="a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'")
# Note: this argument serves as documentation only. The RESULT
# argument should consumes all remaining parameters.
parser.add_argument("baseline_ignored", nargs="?",
metavar=baseline_metavar, help=baseline_help)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
# XXX: while checking for an UNTESTED test should be very cheap
# (does OUTPUT/ exist?) it isn't. Currently it triggers a full
# post-mortem analysis.
skip.add_arguments(parser, skip.skip.untested)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stderr)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Baseline: %s", args.baseline)
logger.info(" Json: %s", args.json)
logger.info(" Quick: %s", args.quick)
logger.info(" Update: %s", args.update)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
# Perhaps the baseline just contains output, magic up the
# corresponding testsuite directory.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log, header="Details:", prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log, header="Summary:", prefix=" ")
publish.json_results(logger, args)
publish.json_summary(logger, args)
return 0
