def main():
parser = argparse.ArgumentParser(
description="list test results",
epilog=
"By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately)."
)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"--quick",
action="store_true",
help=
("Use the previously generated '.console.txt' and '.console.diff' files"
))
parser.add_argument(
"--update",
action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--dump-args", action="store_true")
# how to parse --print directory,saved-directory,...?
parser.add_argument(
"--print",
action="store",
default=Print(Print.path, Print.result, Print.issues),
type=Print,
metavar=str(Print),
help=
"comman separate list of attributes to print for each test; default: '%(default)s'"
)
parser.add_argument(
"--stats",
action="store",
default=Stats.summary,
type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"")
baseline_metavar = "BASELINE-DIRECTORY"
baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed"
parser.add_argument("--baseline",
"-b",
metavar=baseline_metavar,
help=baseline_help)
parser.add_argument(
"--json",
action="store_true",
help=
"output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list"
)
parser.add_argument(
"directories",
metavar="DIRECTORY-OR-FILE",
nargs="+",
help=
"a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'"
)
# Note: this argument serves as documentation only. The RESULT
# argument should consumes all remaining parameters.
parser.add_argument("baseline_ignored",
nargs="?",
metavar=baseline_metavar,
help=baseline_help)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stderr)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Baseline: %s", args.baseline)
logger.info(" Json: %s", args.json)
logger.info(" Quick: %s", args.quick)
logger.info(" Update: %s", args.update)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
# Perhaps the baseline just contains output, magic up the
# corresponding testsuite directory.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'",
args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log,
header="Details:",
prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log,
header="Summary:",
prefix=" ")
return 0
def main():
parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]",
epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--quick", action="store_true",
help=("Use the previously generated '.console.txt' and '.console.diff' files"))
parser.add_argument("--quick-sanitize", action="store_true",
help=("Use the previously generated '.console.txt' file"))
parser.add_argument("--quick-diff", action="store_true",
help=("Use the previously generated '.console.diff' file"))
parser.add_argument("--update", action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--update-sanitize", action="store_true",
help=("Update the '.console.txt' file"))
parser.add_argument("--update-diff", action="store_true",
help=("Update the '.console.diff' file"))
parser.add_argument("--dump-args", action="store_true")
parser.add_argument("--prefix", action="store", type=Prefix,
choices=[p for p in Prefix],
help="prefix to display with each test")
# how to parse --print directory,saved-directory,...?
parser.add_argument("--print", action="append", default=[],
choices=[p for p in Print], type=Print,
help="what information to display about each test")
parser.add_argument("--stats", action="store", default=Stats.summary, type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"");
parser.add_argument("--baseline", metavar="DIRECTORY",
help="a %(metavar)s containing baseline testsuite output")
parser.add_argument("directories", metavar="DIRECTORY", nargs="+",
help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), a TESTLIST file, test output, or testsuite output")
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consumes all remaining arguments.
parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?",
help="an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run")
post.add_arguments(parser)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# default to printing results
if not args.print:
args.print = [Print.result]
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Prefix: %s", args.prefix)
post.log_arguments(logger, args)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger, args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
# Perhaps the baseline just contains output, magic up the
# corresponding testsuite directory.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(logger, args,
testsuite_directory=baseline_directory,
saved_testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log, header="Details:", prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log, header="Summary:", prefix=" ")
return 0
def main():
parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]",
epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--quick", action="store_true",
help=("Use the previously generated '.console.txt' and '.console.diff' files"))
parser.add_argument("--quick-sanitize", action="store_true",
help=("Use the previously generated '.console.txt' file"))
parser.add_argument("--quick-diff", action="store_true",
help=("Use the previously generated '.console.diff' file"))
parser.add_argument("--update", action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--update-sanitize", action="store_true",
help=("Update the '.console.txt' file"))
parser.add_argument("--update-diff", action="store_true",
help=("Update the '.console.diff' file"))
parser.add_argument("--print-directory", action="store_true")
parser.add_argument("--print-name", action="store_true")
# parser.add_argument("--print-result", action="store_true")
parser.add_argument("--print-diff", action="store_true")
parser.add_argument("--print-args", action="store_true")
parser.add_argument("--print-output-directory", action="store_true")
parser.add_argument("--list-ignored", action="store_true",
help="include ignored tests in the list")
parser.add_argument("--list-untested", action="store_true",
help="include untested tests in the list")
parser.add_argument("directories", metavar="TEST-DIRECTORY", nargs="+",
help=("Either a testsuite (only one) or test directory"))
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consume all remaining parameters.
parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?",
help=("An optional testsuite directory containing"
" results from a previous test run"))
post.add_arguments(parser)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
args.print_directory = args.print_directory or args.verbose > v
args.print_name = args.print_name or args.verbose > v
v += 1
args.print_output_directory = args.print_output_directory or args.verbose > v
v += 1
args.list_untested = args.list_untested or args.verbose > v ; v += 1
args.list_ignored = args.list_ignored or args.verbose > v ; v += 1
v += 1
args.print_args = args.print_args or args.verbose > v
if args.print_args:
post.log_arguments(logger, args)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
return 1
# Is the last argument some sort of baseline? If it is, pre-load
# it.
#
# XXX: Should also support something like --baseline-testsuite and
# --baseline-output parameters.
baseline = None
if len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger, args.directories[-1], args,
error_level=logutil.DEBUG)
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
# When an explicit list of directories was specified always print
# all of them (otherwise, tests seem to get lost).
if isinstance(tests, list):
args.list_untested = True
for test in tests:
# Produce separate runtimes for each test.
with logutil.TIMER:
logger.debug("start processing test %s", test.name)
# Filter out tests that are being ignored?
ignore = testsuite.ignore(test, args)
if ignore and not args.list_ignored:
continue
# Filter out tests that have not been run?
result = None
if not ignore:
result = post.mortem(test, args, baseline=baseline,
output_directory=test.saved_output_directory,
skip_sanitize=args.quick or args.quick_sanitize,
skip_diff=args.quick or args.quick_diff,
update=args.update,
update_sanitize=args.update_sanitize,
update_diff=args.update_diff)
if not result and not args.list_untested:
continue
sep = ""
# Print the test's name/path
if not args.print_directory and not args.print_name and not args.print_output_directory:
# By default: when the path given on the command line
# explicitly specifies a test's output directory
# (found in TEST.SAVED_OUTPUT_DIRECTORY), print that;
# otherwise print the path to the test's directory.
print(sep, end="")
print((test.saved_output_directory
and test.saved_output_directory
or test.directory), end="")
sep = " "
else:
# Print the test name/path per command line
if args.print_name:
print(sep, end="")
print(test.name, end="")
sep = " "
if args.print_directory:
print(sep, end="")
print(test.directory, end="")
sep = " "
if args.print_output_directory:
print(sep, end="")
print((test.saved_output_directory
and test.saved_output_directory
or test.output_directory), end="")
sep = " "
if ignore:
print(sep, end="")
print("ignored", ignore, end="")
sep = " "
print(sep, end="")
if result.errors:
print(result, result.errors, end="")
else:
print(result, end="")
sep = " "
print()
if args.print_diff and result:
for domain in result.diffs:
for line in result.diffs[domain]:
if line:
print(line)
sys.stdout.flush()
logger.debug("stop processing test %s", test.name)
return 0
def main():
# If SIGUSR1, backtrace all threads; hopefully this is early
# enough.
faulthandler.register(signal.SIGUSR1)
parser = argparse.ArgumentParser(
description="list test results",
epilog=
"By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). SIGUSR1 will dump all thread stacks"
)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"--exit-ok",
action="store_true",
help=
("return a zero exit status; normally, when there are failures, a non-zero exit status is returned"
))
parser.add_argument(
"--quick",
action="store_true",
help=
("Use the previously generated '.console.txt' and '.console.diff' files"
))
parser.add_argument(
"--update",
action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--dump-args", action="store_true")
# how to parse --print directory,saved-directory,...?
parser.add_argument(
"--print",
action="store",
default=printer.Print(printer.Print.PATH, printer.Print.RESULT,
printer.Print.ISSUES),
type=printer.Print,
metavar=str(printer.Print),
help=
"comman separate list of attributes to print for each test; default: '%(default)s'"
)
parser.add_argument(
"--stats",
action="store",
default=Stats.summary,
type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"")
baseline_metavar = "BASELINE-DIRECTORY"
baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed"
parser.add_argument("--baseline",
"-b",
metavar=baseline_metavar,
help=baseline_help)
parser.add_argument(
"--json",
action="store_true",
help=
"output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list"
)
parser.add_argument(
"directories",
metavar="DIRECTORY-OR-FILE",
nargs="+",
help=
"a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'"
)
# Note: this argument serves as documentation only. The RESULT
# argument should consumes all remaining parameters.
parser.add_argument("baseline_ignored",
nargs="?",
metavar=baseline_metavar,
help=baseline_help)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
# XXX: while checking for an UNTESTED test should be very cheap
# (does OUTPUT/ exist?) it isn't. Currently it triggers a full
# post-mortem analysis.
skip.add_arguments(parser, skip.Skip.UNTESTED)
ignore.add_arguments(parser)
publish.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stderr)
logger = logutil.getLogger("kvmresults")
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Baseline: %s", args.baseline)
logger.info(" Json: %s", args.json)
logger.info(" Quick: %s", args.quick)
logger.info(" Update: %s", args.update)
logger.info(" Exit OK: %s", args.exit_ok)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
publish.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline and os.path.isdir(args.baseline):
# Perhaps, AKA BACKUP/YYYY-MM-DDD-..., the baseline
# directory only contains a copy of the output. Magic up
# a baseline by combining the output with the tests in
# ARGS.TESTING_DIRECTORY.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'",
args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
exit_code = 125 # assume a 'git bisect' barf
try:
exit_code = results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log,
header="Details:",
prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log,
header="Summary:",
prefix=" ")
publish.json_results(logger, args)
publish.json_summary(logger, args)
return exit_code
def main():
parser = argparse.ArgumentParser(
description=
"list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]",
epilog=
"By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified."
)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"--quick",
action="store_true",
help=
("Use the previously generated '.console.txt' and '.console.diff' files"
))
parser.add_argument(
"--quick-sanitize",
action="store_true",
help=("Use the previously generated '.console.txt' file"))
parser.add_argument(
"--quick-diff",
action="store_true",
help=("Use the previously generated '.console.diff' file"))
parser.add_argument(
"--update",
action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--update-sanitize",
action="store_true",
help=("Update the '.console.txt' file"))
parser.add_argument("--update-diff",
action="store_true",
help=("Update the '.console.diff' file"))
parser.add_argument("--dump-args", action="store_true")
parser.add_argument("--prefix",
action="store",
type=Prefix,
choices=[p for p in Prefix],
help="prefix to display with each test")
# how to parse --print directory,saved-directory,...?
parser.add_argument("--print",
action="append",
default=[],
choices=[p for p in Print],
type=Print,
help="what information to display about each test")
parser.add_argument(
"--stats",
action="store",
default=Stats.summary,
type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"")
parser.add_argument(
"--baseline",
metavar="DIRECTORY",
help="a %(metavar)s containing baseline testsuite output")
parser.add_argument(
"directories",
metavar="DIRECTORY",
nargs="+",
help=
"%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), a TESTLIST file, test output, or testsuite output"
)
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consumes all remaining arguments.
parser.add_argument(
"baseline",
metavar="BASELINE-DIRECTORY",
nargs="?",
help=
"an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run"
)
post.add_arguments(parser)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# default to printing results
if not args.print:
args.print = [Print.result]
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Prefix: %s", args.prefix)
post.log_arguments(logger, args)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger,
args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
# Perhaps the baseline just contains output, magic up the
# corresponding testsuite directory.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(
logger,
args,
testsuite_directory=baseline_directory,
saved_testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger,
logutil.DEBUG,
args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'",
args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log,
header="Details:",
prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log,
header="Summary:",
prefix=" ")
return 0
def main():
parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--print-directory", action="store_true")
parser.add_argument("--print-name", action="store_true")
parser.add_argument("--print-result", action="store_true")
parser.add_argument("--list-ignored", action="store_true",
help="include ignored tests in the list")
parser.add_argument("--list-untested", action="store_true",
help="include untested tests in the list")
parser.add_argument("directories", metavar="TEST-DIRECTORY", nargs="+",
help=("Either a testsuite (only one) or test directory"))
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consume all remaining parameters.
parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?",
help=("An optional testsuite directory containing"
" results from a previous test run"))
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
args.print_directory = args.print_directory or args.verbose > v
args.print_name = args.print_name or args.verbose > v
v += 1
args.list_untested = args.list_untested or args.verbose > v ; v += 1
args.list_ignored = args.list_ignored or args.verbose > v ; v += 1
# By default print the relative directory path.
if not args.print_directory and not args.print_name:
args.print_directory = True
# If there is more than one directory then the last might be the
# baseline. Try loading it as a testsuite (baselines are
# testsuites) to see if that is the case.
basetests = None
tests = None
if len(args.directories) > 1:
# Perhaps the last argument is the baseline?
basetests = testsuite.load(logger, args.directories[-1])
if basetests:
logger.debug("basetests loaded from '%s'", basetests.directory)
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories)
logger.debug("basetests=%s", basetests)
logger.debug("tests=%s", tests)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
# When an explicit list of directories was specified always print
# all of them (otherwise, tests seem to get lost).
if isinstance(tests, list):
args.list_untested = True
# Preload the baseline. This avoids re-scanning the TESTLIST and,
# when errors, printing those repeatedly. Also, passing the full
# baseline to Test.results() lets that function differentiate
# between a baseline missing results or being entirely absent.
baseline = None
if basetests:
baseline = {}
for test in basetests:
baseline[test.name] = test
for test in tests:
# Filter out tests that are being ignored?
ignore = testsuite.ignore(test, args)
if ignore and not args.list_ignored:
continue
# Filter out tests that have not been run?
result = None
if not ignore:
result = test.result(baseline)
if not result and not args.list_untested:
continue
sep = ""
if args.print_name:
print(sep, end="")
print(test.name, end="")
sep = " "
if args.print_directory:
print(sep, end="")
print(test.directory, end="")
sep = " "
if ignore:
print(sep, end="")
print("ignored", ignore, end="")
sep = " "
if result:
print(sep, end="")
print(result, end="")
sep = " "
print()
sys.stdout.flush()
return 0
def main():
parser = argparse.ArgumentParser(description="list test results",
epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately).")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--quick", action="store_true",
help=("Use the previously generated '.console.txt' and '.console.diff' files"))
parser.add_argument("--update", action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--dump-args", action="store_true")
# how to parse --print directory,saved-directory,...?
parser.add_argument("--print", action="store",
default=Print(Print.path, Print.result, Print.issues),
type=Print, metavar=str(Print),
help="comman separate list of attributes to print for each test; default: '%(default)s'")
parser.add_argument("--stats", action="store", default=Stats.summary, type=Stats,
choices=[c for c in Stats],
help="provide overview statistics; default: \"%(default)s\"");
baseline_metavar = "BASELINE-DIRECTORY"
baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed"
parser.add_argument("--baseline", "-b",
metavar=baseline_metavar, help=baseline_help)
parser.add_argument("--json", action="store_true",
help="output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list")
parser.add_argument("directories", metavar="DIRECTORY-OR-FILE", nargs="+",
help="a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'")
# Note: this argument serves as documentation only. The RESULT
# argument should consumes all remaining parameters.
parser.add_argument("baseline_ignored", nargs="?",
metavar=baseline_metavar, help=baseline_help)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
skip.add_arguments(parser)
ignore.add_arguments(parser)
# These three calls go together
args = parser.parse_args()
logutil.config(args, sys.stderr)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
if args.dump_args:
logger.info("Arguments:")
logger.info(" Stats: %s", args.stats)
logger.info(" Print: %s", args.print)
logger.info(" Baseline: %s", args.baseline)
logger.info(" Json: %s", args.json)
logger.info(" Quick: %s", args.quick)
logger.info(" Update: %s", args.update)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
skip.log_arguments(logger, args)
ignore.log_arguments(logger, args)
return 0
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving in how
# it is loaded.
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
# Perhaps the baseline just contains output, magic up the
# corresponding testsuite directory.
baseline_directory = os.path.join(args.testing_directory, "pluto")
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger, logutil.DEBUG, args,
testsuite_directory=args.directories[-1])
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats is Stats.details:
result_stats.log_details(stderr_log, header="Details:", prefix=" ")
if args.stats in [Stats.details, Stats.summary]:
result_stats.log_summary(stderr_log, header="Summary:", prefix=" ")
return 0
def main():
parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]",
epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--quick", action="store_true",
help=("Use the previously generated '.console.txt' and '.console.diff' files"))
parser.add_argument("--quick-sanitize", action="store_true",
help=("Use the previously generated '.console.txt' file"))
parser.add_argument("--quick-diff", action="store_true",
help=("Use the previously generated '.console.diff' file"))
parser.add_argument("--update", action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--update-sanitize", action="store_true",
help=("Update the '.console.txt' file"))
parser.add_argument("--update-diff", action="store_true",
help=("Update the '.console.diff' file"))
parser.add_argument("--prefix-directory", action="store_true")
parser.add_argument("--prefix-name", action="store_true")
parser.add_argument("--prefix-output-directory", action="store_true")
parser.add_argument("--print-result", action="store_true")
parser.add_argument("--print-diff", action="store_true")
parser.add_argument("--print-args", action="store_true")
parser.add_argument("--print-scripts", action="store_true")
parser.add_argument("--print-domains", action="store_true")
parser.add_argument("--print-initiators", action="store_true")
parser.add_argument("--stats", action="store", default="summary",
choices=["details", "summary", "none"],
help="provide overview statistics; default: \"%(default)s\"");
parser.add_argument("--list-ignored", action="store_true",
help="include ignored tests in the list")
parser.add_argument("--list-untested", action="store_true",
help="include untested tests in the list")
parser.add_argument("--baseline", metavar="DIRECTORY",
help="a %(metavar)s containing baseline testsuite output")
parser.add_argument("directories", metavar="DIRECTORY", nargs="+",
help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), test output, or testsuite output")
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consumes all remaining arguments.
parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?",
help="an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run")
post.add_arguments(parser)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# default to printing results
if not args.print_scripts \
and not args.print_result \
and not args.print_diff \
and not args.print_initiators \
and not args.print_domains:
args.print_result = True
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
args.prefix_directory = args.prefix_directory or args.verbose > v
args.prefix_name = args.prefix_name or args.verbose > v
args.print_result = args.print_result or args.verbose > v
v += 1
args.prefix_output_directory = args.prefix_output_directory or args.verbose > v
v += 1
args.list_untested = args.list_untested or args.verbose > v
args.list_ignored = args.list_ignored or args.verbose > v
v += 1
args.print_scripts = args.print_scripts or args.verbose > v
v += 1
args.print_args = args.print_args or args.verbose > v
if args.print_args:
post.log_arguments(logger, args)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
return 1
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving.
baseline = testsuite.load(logger, args,
testsuite_directory=args.baseline,
testsuite_output_directory=None,
error_level=logutil.DEBUG)
if not baseline:
# Assume that it is baseline output only.
if args.testing_directory:
baseline_directory = os.path.join(args.testing_directory, "pluto")
else:
baseline_directory = utils.directory("..", "pluto")
baseline = testsuite.load(logger, args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger, args,
testsuite_directory=args.directories[-1],
testsuite_output_directory=None,
error_level=logutil.DEBUG)
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
# When an explicit list of directories was specified always print
# all of them (otherwise, tests seem to get lost).
if isinstance(tests, list):
args.list_untested = True
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
if args.stats == "details":
result_stats.log_details(stderr_log, header="Details:", prefix=" ")
if args.stats in ["details", "summary"]:
result_stats.log_summary(stderr_log, header="Summary:", prefix=" ")
return 0
def main():
parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]",
epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--quick", action="store_true",
help=("Use the previously generated '.console.txt' and '.console.diff' files"))
parser.add_argument("--quick-sanitize", action="store_true",
help=("Use the previously generated '.console.txt' file"))
parser.add_argument("--quick-diff", action="store_true",
help=("Use the previously generated '.console.diff' file"))
parser.add_argument("--update", action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--update-sanitize", action="store_true",
help=("Update the '.console.txt' file"))
parser.add_argument("--update-diff", action="store_true",
help=("Update the '.console.diff' file"))
parser.add_argument("--prefix-directory", action="store_true")
parser.add_argument("--prefix-name", action="store_true")
parser.add_argument("--prefix-output-directory", action="store_true")
parser.add_argument("--print-result", action="store_true")
parser.add_argument("--print-diff", action="store_true")
parser.add_argument("--print-args", action="store_true")
parser.add_argument("--print-scripts", action="store_true")
parser.add_argument("--print-domains", action="store_true")
parser.add_argument("--print-initiators", action="store_true")
parser.add_argument("--list-ignored", action="store_true",
help="include ignored tests in the list")
parser.add_argument("--list-untested", action="store_true",
help="include untested tests in the list")
parser.add_argument("--baseline", metavar="DIRECTORY",
help="a %(metavar)s containing baseline testsuite output")
parser.add_argument("directories", metavar="DIRECTORY", nargs="+",
help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), test output, or testsuite output")
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consumes all remaining arguments.
parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?",
help="an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run")
post.add_arguments(parser)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# default to printing results
if not args.print_scripts \
and not args.print_result \
and not args.print_diff \
and not args.print_initiators \
and not args.print_domains:
args.print_result = True
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
args.prefix_directory = args.prefix_directory or args.verbose > v
args.prefix_name = args.prefix_name or args.verbose > v
args.print_result = args.print_result or args.verbose > v
v += 1
args.prefix_output_directory = args.prefix_output_directory or args.verbose > v
v += 1
args.list_untested = args.list_untested or args.verbose > v
args.list_ignored = args.list_ignored or args.verbose > v
v += 1
args.print_scripts = args.print_scripts or args.verbose > v
v += 1
args.print_args = args.print_args or args.verbose > v
if args.print_args:
post.log_arguments(logger, args)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
return 1
# Try to find a baseline. If present, pre-load it.
baseline = None
if args.baseline:
# An explict baseline testsuite, can be more forgiving.
baseline = testsuite.load(logger, args,
testsuite_directory=args.baseline,
testsuite_output_directory=None,
error_level=logutil.DEBUG)
if not baseline:
# Assume that it is baseline output only.
if args.testing_directory:
baseline_directory = os.path.join(args.testing_directory, "pluto")
else:
baseline_directory = utils.directory("..", "pluto")
baseline = testsuite.load(logger, args,
testsuite_directory=baseline_directory,
testsuite_output_directory=args.baseline,
error_level=logutil.DEBUG)
if not baseline:
logger.info("'%s' is not a baseline", args.baseline)
return 1
elif len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger, args,
testsuite_directory=args.directories[-1],
testsuite_output_directory=None,
error_level=logutil.DEBUG)
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
# When an explicit list of directories was specified always print
# all of them (otherwise, tests seem to get lost).
if isinstance(tests, list):
args.list_untested = True
result_stats = stats.Results()
try:
results(logger, tests, baseline, args, result_stats)
finally:
result_stats.log_summary(stderr_log, prefix=" ")
return 0
def main():
parser = argparse.ArgumentParser(
description=
"list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]",
epilog=
"By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified."
)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"--quick",
action="store_true",
help=
("Use the previously generated '.console.txt' and '.console.diff' files"
))
parser.add_argument(
"--quick-sanitize",
action="store_true",
help=("Use the previously generated '.console.txt' file"))
parser.add_argument(
"--quick-diff",
action="store_true",
help=("Use the previously generated '.console.diff' file"))
parser.add_argument(
"--update",
action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--update-sanitize",
action="store_true",
help=("Update the '.console.txt' file"))
parser.add_argument("--update-diff",
action="store_true",
help=("Update the '.console.diff' file"))
parser.add_argument("--print-directory", action="store_true")
parser.add_argument("--print-name", action="store_true")
# parser.add_argument("--print-result", action="store_true")
parser.add_argument("--print-diff", action="store_true")
parser.add_argument("--print-args", action="store_true")
parser.add_argument("--print-output-directory", action="store_true")
parser.add_argument("--list-ignored",
action="store_true",
help="include ignored tests in the list")
parser.add_argument("--list-untested",
action="store_true",
help="include untested tests in the list")
parser.add_argument(
"directories",
metavar="TEST-DIRECTORY",
nargs="+",
help=("Either a testsuite (only one) or test directory"))
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consume all remaining parameters.
parser.add_argument("baseline",
metavar="BASELINE-DIRECTORY",
nargs="?",
help=("An optional testsuite directory containing"
" results from a previous test run"))
post.add_arguments(parser)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
args.print_directory = args.print_directory or args.verbose > v
args.print_name = args.print_name or args.verbose > v
v += 1
args.print_output_directory = args.print_output_directory or args.verbose > v
v += 1
args.list_untested = args.list_untested or args.verbose > v
v += 1
args.list_ignored = args.list_ignored or args.verbose > v
v += 1
v += 1
args.print_args = args.print_args or args.verbose > v
if args.print_args:
post.log_arguments(logger, args)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
return 1
# Is the last argument some sort of baseline? If it is, pre-load
# it.
#
# XXX: Should also support something like --baseline-testsuite and
# --baseline-output parameters.
baseline = None
if len(args.directories) > 1:
# If there is more than one directory then, perhaps, the last
# one is a baseline. A baseline might be: a complete
# testsuite snapshot; or just output saved as
# testing/pluto/OUTPUT/TESTDIR.
baseline = testsuite.load(logger,
args.directories[-1],
args,
error_level=logutil.DEBUG)
if baseline:
# discard the last argument as consumed above.
logger.debug("discarding baseline testsuite argument '%s'",
args.directories[-1])
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories, args)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
# When an explicit list of directories was specified always print
# all of them (otherwise, tests seem to get lost).
if isinstance(tests, list):
args.list_untested = True
for test in tests:
# Produce separate runtimes for each test.
with logutil.TIMER:
logger.debug("start processing test %s", test.name)
# Filter out tests that are being ignored?
ignore = testsuite.ignore(test, args)
if ignore and not args.list_ignored:
continue
# Filter out tests that have not been run?
result = None
if not ignore:
result = post.mortem(
test,
args,
baseline=baseline,
output_directory=test.saved_output_directory,
skip_sanitize=args.quick or args.quick_sanitize,
skip_diff=args.quick or args.quick_diff,
update=args.update,
update_sanitize=args.update_sanitize,
update_diff=args.update_diff)
if not result and not args.list_untested:
continue
sep = ""
# Print the test's name/path
if not args.print_directory and not args.print_name and not args.print_output_directory:
# By default: when the path given on the command line
# explicitly specifies a test's output directory
# (found in TEST.SAVED_OUTPUT_DIRECTORY), print that;
# otherwise print the path to the test's directory.
print(sep, end="")
print((test.saved_output_directory
and test.saved_output_directory or test.directory),
end="")
sep = " "
else:
# Print the test name/path per command line
if args.print_name:
print(sep, end="")
print(test.name, end="")
sep = " "
if args.print_directory:
print(sep, end="")
print(test.directory, end="")
sep = " "
if args.print_output_directory:
print(sep, end="")
print((test.saved_output_directory
and test.saved_output_directory
or test.output_directory),
end="")
sep = " "
if ignore:
print(sep, end="")
print("ignored", ignore, end="")
sep = " "
print(sep, end="")
if result.errors:
print(result, result.errors, end="")
else:
print(result, end="")
sep = " "
print()
if args.print_diff and result:
for domain in result.diffs:
for line in result.diffs[domain]:
if line:
print(line)
sys.stdout.flush()
logger.debug("stop processing test %s", test.name)
return 0
def main():
parser = argparse.ArgumentParser(
description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]",
epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately).",
)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")
)
parser.add_argument(
"--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")
)
parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file"))
parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file"))
parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file"))
parser.add_argument("--print-directory", action="store_true")
parser.add_argument("--print-name", action="store_true")
parser.add_argument("--print-result", action="store_true")
parser.add_argument("--print-diff", action="store_true")
parser.add_argument("--print-args", action="store_true")
parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list")
parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list")
parser.add_argument(
"directories", metavar="TEST-DIRECTORY", nargs="+", help=("Either a testsuite (only one) or test directory")
)
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consume all remaining parameters.
parser.add_argument(
"baseline",
metavar="BASELINE-DIRECTORY",
nargs="?",
help=("An optional testsuite directory containing" " results from a previous test run"),
)
post.add_arguments(parser)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
args.print_directory = args.print_directory or args.verbose > v
args.print_name = args.print_name or args.verbose > v
v += 1
args.list_untested = args.list_untested or args.verbose > v
v += 1
args.list_ignored = args.list_ignored or args.verbose > v
v += 1
v += 1
args.print_args = args.print_args or args.verbose > v
# By default print the relative directory path.
if not args.print_directory and not args.print_name:
args.print_directory = True
if args.print_args:
post.log_arguments(logger, args)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
return 1
# If there is more than one directory then the last might be the
# baseline. Try loading it as a testsuite (baselines are
# testsuites) to see if that is the case.
basetests = None
tests = None
if len(args.directories) > 1:
# Perhaps the last argument is the baseline? Suppress any
# nasty errors.
basetests = testsuite.load(logger, args.directories[-1], error_level=logutil.DEBUG)
if basetests:
logger.debug("basetests loaded from '%s'", basetests.directory)
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories)
logger.debug("basetests=%s", basetests)
logger.debug("tests=%s", tests)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
# When an explicit list of directories was specified always print
# all of them (otherwise, tests seem to get lost).
if isinstance(tests, list):
args.list_untested = True
# Preload the baseline. This avoids re-scanning the TESTLIST.
# Also, passing the full baseline to Test.results() lets that
# function differentiate between a baseline missing results or
# being entirely absent.
baseline = None
if basetests:
baseline = {}
for test in basetests:
baseline[test.name] = test
for test in tests:
# Produce separate runtimes for each test.
with logutil.TIMER:
logger.debug("start processing test %s", test.name)
# Filter out tests that are being ignored?
ignore = testsuite.ignore(test, args)
if ignore and not args.list_ignored:
continue
# Filter out tests that have not been run?
result = None
if not ignore:
result = post.mortem(
test,
args,
baseline=baseline,
output_directory=test.old_output_directory,
skip_sanitize=args.quick or args.quick_sanitize,
skip_diff=args.quick or args.quick_diff,
update=args.update,
update_sanitize=args.update_sanitize,
update_diff=args.update_diff,
)
if not result and not args.list_untested:
continue
sep = ""
if args.print_name:
print(sep, end="")
print(test.name, end="")
sep = " "
if args.print_directory:
print(sep, end="")
print(test.directory, end="")
sep = " "
if ignore:
print(sep, end="")
print("ignored", ignore, end="")
sep = " "
if result:
print(sep, end="")
print(result, end="")
sep = " "
print()
if args.print_diff and result:
for domain in result.diffs:
for line in result.diffs[domain]:
if line:
print(line)
sys.stdout.flush()
logger.debug("stop processing test %s", test.name)
return 0
def main():
parser = argparse.ArgumentParser(
description=
"list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]",
epilog=
"By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately)."
)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument(
"--quick",
action="store_true",
help=
("Use the previously generated '.console.txt' and '.console.diff' files"
))
parser.add_argument(
"--quick-sanitize",
action="store_true",
help=("Use the previously generated '.console.txt' file"))
parser.add_argument(
"--quick-diff",
action="store_true",
help=("Use the previously generated '.console.diff' file"))
parser.add_argument(
"--update",
action="store_true",
help=("Update the '.console.txt' and '.console.diff' files"))
parser.add_argument("--update-sanitize",
action="store_true",
help=("Update the '.console.txt' file"))
parser.add_argument("--update-diff",
action="store_true",
help=("Update the '.console.diff' file"))
parser.add_argument("--print-directory", action="store_true")
parser.add_argument("--print-name", action="store_true")
parser.add_argument("--print-result", action="store_true")
parser.add_argument("--print-diff", action="store_true")
parser.add_argument("--print-args", action="store_true")
parser.add_argument("--list-ignored",
action="store_true",
help="include ignored tests in the list")
parser.add_argument("--list-untested",
action="store_true",
help="include untested tests in the list")
parser.add_argument(
"directories",
metavar="TEST-DIRECTORY",
nargs="+",
help=("Either a testsuite (only one) or test directory"))
# Note: this argument serves as documentation only. The
# TEST-DIRECTORY argument always consume all remaining parameters.
parser.add_argument("baseline",
metavar="BASELINE-DIRECTORY",
nargs="?",
help=("An optional testsuite directory containing"
" results from a previous test run"))
post.add_arguments(parser)
testsuite.add_arguments(parser)
logutil.add_arguments(parser)
args = parser.parse_args()
logutil.config(args)
logger = logutil.getLogger("kvmresults")
# The option -vvvvvvv is a short circuit for these; make
# re-ordering easy by using V as a counter.
v = 0
args.print_directory = args.print_directory or args.verbose > v
args.print_name = args.print_name or args.verbose > v
v += 1
args.list_untested = args.list_untested or args.verbose > v
v += 1
args.list_ignored = args.list_ignored or args.verbose > v
v += 1
v += 1
args.print_args = args.print_args or args.verbose > v
# By default print the relative directory path.
if not args.print_directory and not args.print_name:
args.print_directory = True
if args.print_args:
post.log_arguments(logger, args)
testsuite.log_arguments(logger, args)
logutil.log_arguments(logger, args)
return 1
# If there is more than one directory then the last might be the
# baseline. Try loading it as a testsuite (baselines are
# testsuites) to see if that is the case.
basetests = None
tests = None
if len(args.directories) > 1:
# Perhaps the last argument is the baseline? Suppress any
# nasty errors.
basetests = testsuite.load(logger,
args.directories[-1],
error_level=logutil.DEBUG)
if basetests:
logger.debug("basetests loaded from '%s'", basetests.directory)
args.directories.pop()
tests = testsuite.load_testsuite_or_tests(logger, args.directories)
logger.debug("basetests=%s", basetests)
logger.debug("tests=%s", tests)
# And check
if not tests:
logger.error("Invalid testsuite or test directories")
return 1
# When an explicit list of directories was specified always print
# all of them (otherwise, tests seem to get lost).
if isinstance(tests, list):
args.list_untested = True
# Preload the baseline. This avoids re-scanning the TESTLIST.
# Also, passing the full baseline to Test.results() lets that
# function differentiate between a baseline missing results or
# being entirely absent.
baseline = None
if basetests:
baseline = {}
for test in basetests:
baseline[test.name] = test
for test in tests:
# Produce separate runtimes for each test.
with logutil.TIMER:
logger.debug("start processing test %s", test.name)
# Filter out tests that are being ignored?
ignore = testsuite.ignore(test, args)
if ignore and not args.list_ignored:
continue
# Filter out tests that have not been run?
result = None
if not ignore:
result = post.mortem(
test,
args,
baseline=baseline,
output_directory=test.old_output_directory,
skip_sanitize=args.quick or args.quick_sanitize,
skip_diff=args.quick or args.quick_diff,
update=args.update,
update_sanitize=args.update_sanitize,
update_diff=args.update_diff)
if not result and not args.list_untested:
continue
sep = ""
if args.print_name:
print(sep, end="")
print(test.name, end="")
sep = " "
if args.print_directory:
print(sep, end="")
print(test.directory, end="")
sep = " "
if ignore:
print(sep, end="")
print("ignored", ignore, end="")
sep = " "
if result:
print(sep, end="")
print(result, end="")
sep = " "
print()
if args.print_diff and result:
for domain in result.diffs:
for line in result.diffs[domain]:
if line:
print(line)
sys.stdout.flush()
logger.debug("stop processing test %s", test.name)
return 0
