def main(): parser = argparse.ArgumentParser( description="list test results", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately)." ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--dump-args", action="store_true") # how to parse --print directory,saved-directory,...? parser.add_argument( "--print", action="store", default=Print(Print.path, Print.result, Print.issues), type=Print, metavar=str(Print), help= "comman separate list of attributes to print for each test; default: '%(default)s'" ) parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") baseline_metavar = "BASELINE-DIRECTORY" baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed" parser.add_argument("--baseline", "-b", metavar=baseline_metavar, help=baseline_help) parser.add_argument( "--json", action="store_true", help= "output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list" ) parser.add_argument( "directories", metavar="DIRECTORY-OR-FILE", nargs="+", help= "a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'" ) # Note: this argument serves as documentation only. The RESULT # argument should consumes all remaining parameters. parser.add_argument("baseline_ignored", nargs="?", metavar=baseline_metavar, help=baseline_help) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stderr) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Baseline: %s", args.baseline) logger.info(" Json: %s", args.json) logger.info(" Quick: %s", args.quick) logger.info(" Update: %s", args.update) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--dump-args", action="store_true") parser.add_argument("--prefix", action="store", type=Prefix, choices=[p for p in Prefix], help="prefix to display with each test") # how to parse --print directory,saved-directory,...? parser.add_argument("--print", action="append", default=[], choices=[p for p in Print], type=Print, help="what information to display about each test") parser.add_argument("--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\""); parser.add_argument("--baseline", metavar="DIRECTORY", help="a %(metavar)s containing baseline testsuite output") parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), a TESTLIST file, test output, or testsuite output") # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consumes all remaining arguments. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help="an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run") post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # default to printing results if not args.print: args.print = [Print.result] # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Prefix: %s", args.prefix) post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, args, testsuite_directory=baseline_directory, saved_testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--print-directory", action="store_true") parser.add_argument("--print-name", action="store_true") # parser.add_argument("--print-result", action="store_true") parser.add_argument("--print-diff", action="store_true") parser.add_argument("--print-args", action="store_true") parser.add_argument("--print-output-directory", action="store_true") parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list") parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list") parser.add_argument("directories", metavar="TEST-DIRECTORY", nargs="+", help=("Either a testsuite (only one) or test directory")) # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consume all remaining parameters. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help=("An optional testsuite directory containing" " results from a previous test run")) post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 args.print_directory = args.print_directory or args.verbose > v args.print_name = args.print_name or args.verbose > v v += 1 args.print_output_directory = args.print_output_directory or args.verbose > v v += 1 args.list_untested = args.list_untested or args.verbose > v ; v += 1 args.list_ignored = args.list_ignored or args.verbose > v ; v += 1 v += 1 args.print_args = args.print_args or args.verbose > v if args.print_args: post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) return 1 # Is the last argument some sort of baseline? If it is, pre-load # it. # # XXX: Should also support something like --baseline-testsuite and # --baseline-output parameters. baseline = None if len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, args.directories[-1], args, error_level=logutil.DEBUG) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 # When an explicit list of directories was specified always print # all of them (otherwise, tests seem to get lost). if isinstance(tests, list): args.list_untested = True for test in tests: # Produce separate runtimes for each test. with logutil.TIMER: logger.debug("start processing test %s", test.name) # Filter out tests that are being ignored? ignore = testsuite.ignore(test, args) if ignore and not args.list_ignored: continue # Filter out tests that have not been run? result = None if not ignore: result = post.mortem(test, args, baseline=baseline, output_directory=test.saved_output_directory, skip_sanitize=args.quick or args.quick_sanitize, skip_diff=args.quick or args.quick_diff, update=args.update, update_sanitize=args.update_sanitize, update_diff=args.update_diff) if not result and not args.list_untested: continue sep = "" # Print the test's name/path if not args.print_directory and not args.print_name and not args.print_output_directory: # By default: when the path given on the command line # explicitly specifies a test's output directory # (found in TEST.SAVED_OUTPUT_DIRECTORY), print that; # otherwise print the path to the test's directory. print(sep, end="") print((test.saved_output_directory and test.saved_output_directory or test.directory), end="") sep = " " else: # Print the test name/path per command line if args.print_name: print(sep, end="") print(test.name, end="") sep = " " if args.print_directory: print(sep, end="") print(test.directory, end="") sep = " " if args.print_output_directory: print(sep, end="") print((test.saved_output_directory and test.saved_output_directory or test.output_directory), end="") sep = " " if ignore: print(sep, end="") print("ignored", ignore, end="") sep = " " print(sep, end="") if result.errors: print(result, result.errors, end="") else: print(result, end="") sep = " " print() if args.print_diff and result: for domain in result.diffs: for line in result.diffs[domain]: if line: print(line) sys.stdout.flush() logger.debug("stop processing test %s", test.name) return 0
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser( description="list test results", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). SIGUSR1 will dump all thread stacks" ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--exit-ok", action="store_true", help= ("return a zero exit status; normally, when there are failures, a non-zero exit status is returned" )) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--dump-args", action="store_true") # how to parse --print directory,saved-directory,...? parser.add_argument( "--print", action="store", default=printer.Print(printer.Print.PATH, printer.Print.RESULT, printer.Print.ISSUES), type=printer.Print, metavar=str(printer.Print), help= "comman separate list of attributes to print for each test; default: '%(default)s'" ) parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") baseline_metavar = "BASELINE-DIRECTORY" baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed" parser.add_argument("--baseline", "-b", metavar=baseline_metavar, help=baseline_help) parser.add_argument( "--json", action="store_true", help= "output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list" ) parser.add_argument( "directories", metavar="DIRECTORY-OR-FILE", nargs="+", help= "a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'" ) # Note: this argument serves as documentation only. The RESULT # argument should consumes all remaining parameters. parser.add_argument("baseline_ignored", nargs="?", metavar=baseline_metavar, help=baseline_help) testsuite.add_arguments(parser) logutil.add_arguments(parser) # XXX: while checking for an UNTESTED test should be very cheap # (does OUTPUT/ exist?) it isn't. Currently it triggers a full # post-mortem analysis. skip.add_arguments(parser, skip.Skip.UNTESTED) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stderr) logger = logutil.getLogger("kvmresults") if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Baseline: %s", args.baseline) logger.info(" Json: %s", args.json) logger.info(" Quick: %s", args.quick) logger.info(" Update: %s", args.update) logger.info(" Exit OK: %s", args.exit_ok) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline and os.path.isdir(args.baseline): # Perhaps, AKA BACKUP/YYYY-MM-DDD-..., the baseline # directory only contains a copy of the output. Magic up # a baseline by combining the output with the tests in # ARGS.TESTING_DIRECTORY. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() exit_code = 125 # assume a 'git bisect' barf try: exit_code = results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") publish.json_results(logger, args) publish.json_summary(logger, args) return exit_code
def main(): parser = argparse.ArgumentParser( description= "list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified." ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument( "--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--dump-args", action="store_true") parser.add_argument("--prefix", action="store", type=Prefix, choices=[p for p in Prefix], help="prefix to display with each test") # how to parse --print directory,saved-directory,...? parser.add_argument("--print", action="append", default=[], choices=[p for p in Print], type=Print, help="what information to display about each test") parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") parser.add_argument( "--baseline", metavar="DIRECTORY", help="a %(metavar)s containing baseline testsuite output") parser.add_argument( "directories", metavar="DIRECTORY", nargs="+", help= "%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), a TESTLIST file, test output, or testsuite output" ) # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consumes all remaining arguments. parser.add_argument( "baseline", metavar="BASELINE-DIRECTORY", nargs="?", help= "an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run" ) post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # default to printing results if not args.print: args.print = [Print.result] # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Prefix: %s", args.prefix) post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load( logger, args, testsuite_directory=baseline_directory, saved_testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--print-directory", action="store_true") parser.add_argument("--print-name", action="store_true") parser.add_argument("--print-result", action="store_true") parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list") parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list") parser.add_argument("directories", metavar="TEST-DIRECTORY", nargs="+", help=("Either a testsuite (only one) or test directory")) # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consume all remaining parameters. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help=("An optional testsuite directory containing" " results from a previous test run")) testsuite.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 args.print_directory = args.print_directory or args.verbose > v args.print_name = args.print_name or args.verbose > v v += 1 args.list_untested = args.list_untested or args.verbose > v ; v += 1 args.list_ignored = args.list_ignored or args.verbose > v ; v += 1 # By default print the relative directory path. if not args.print_directory and not args.print_name: args.print_directory = True # If there is more than one directory then the last might be the # baseline. Try loading it as a testsuite (baselines are # testsuites) to see if that is the case. basetests = None tests = None if len(args.directories) > 1: # Perhaps the last argument is the baseline? basetests = testsuite.load(logger, args.directories[-1]) if basetests: logger.debug("basetests loaded from '%s'", basetests.directory) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories) logger.debug("basetests=%s", basetests) logger.debug("tests=%s", tests) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 # When an explicit list of directories was specified always print # all of them (otherwise, tests seem to get lost). if isinstance(tests, list): args.list_untested = True # Preload the baseline. This avoids re-scanning the TESTLIST and, # when errors, printing those repeatedly. Also, passing the full # baseline to Test.results() lets that function differentiate # between a baseline missing results or being entirely absent. baseline = None if basetests: baseline = {} for test in basetests: baseline[test.name] = test for test in tests: # Filter out tests that are being ignored? ignore = testsuite.ignore(test, args) if ignore and not args.list_ignored: continue # Filter out tests that have not been run? result = None if not ignore: result = test.result(baseline) if not result and not args.list_untested: continue sep = "" if args.print_name: print(sep, end="") print(test.name, end="") sep = " " if args.print_directory: print(sep, end="") print(test.directory, end="") sep = " " if ignore: print(sep, end="") print("ignored", ignore, end="") sep = " " if result: print(sep, end="") print(result, end="") sep = " " print() sys.stdout.flush() return 0
def main(): parser = argparse.ArgumentParser(description="list test results", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately).") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--dump-args", action="store_true") # how to parse --print directory,saved-directory,...? parser.add_argument("--print", action="store", default=Print(Print.path, Print.result, Print.issues), type=Print, metavar=str(Print), help="comman separate list of attributes to print for each test; default: '%(default)s'") parser.add_argument("--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\""); baseline_metavar = "BASELINE-DIRECTORY" baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed" parser.add_argument("--baseline", "-b", metavar=baseline_metavar, help=baseline_help) parser.add_argument("--json", action="store_true", help="output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list") parser.add_argument("directories", metavar="DIRECTORY-OR-FILE", nargs="+", help="a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'") # Note: this argument serves as documentation only. The RESULT # argument should consumes all remaining parameters. parser.add_argument("baseline_ignored", nargs="?", metavar=baseline_metavar, help=baseline_help) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stderr) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Baseline: %s", args.baseline) logger.info(" Json: %s", args.json) logger.info(" Quick: %s", args.quick) logger.info(" Update: %s", args.update) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--prefix-directory", action="store_true") parser.add_argument("--prefix-name", action="store_true") parser.add_argument("--prefix-output-directory", action="store_true") parser.add_argument("--print-result", action="store_true") parser.add_argument("--print-diff", action="store_true") parser.add_argument("--print-args", action="store_true") parser.add_argument("--print-scripts", action="store_true") parser.add_argument("--print-domains", action="store_true") parser.add_argument("--print-initiators", action="store_true") parser.add_argument("--stats", action="store", default="summary", choices=["details", "summary", "none"], help="provide overview statistics; default: \"%(default)s\""); parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list") parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list") parser.add_argument("--baseline", metavar="DIRECTORY", help="a %(metavar)s containing baseline testsuite output") parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), test output, or testsuite output") # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consumes all remaining arguments. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help="an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run") post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # default to printing results if not args.print_scripts \ and not args.print_result \ and not args.print_diff \ and not args.print_initiators \ and not args.print_domains: args.print_result = True # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 args.prefix_directory = args.prefix_directory or args.verbose > v args.prefix_name = args.prefix_name or args.verbose > v args.print_result = args.print_result or args.verbose > v v += 1 args.prefix_output_directory = args.prefix_output_directory or args.verbose > v v += 1 args.list_untested = args.list_untested or args.verbose > v args.list_ignored = args.list_ignored or args.verbose > v v += 1 args.print_scripts = args.print_scripts or args.verbose > v v += 1 args.print_args = args.print_args or args.verbose > v if args.print_args: post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) return 1 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving. baseline = testsuite.load(logger, args, testsuite_directory=args.baseline, testsuite_output_directory=None, error_level=logutil.DEBUG) if not baseline: # Assume that it is baseline output only. if args.testing_directory: baseline_directory = os.path.join(args.testing_directory, "pluto") else: baseline_directory = utils.directory("..", "pluto") baseline = testsuite.load(logger, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, args, testsuite_directory=args.directories[-1], testsuite_output_directory=None, error_level=logutil.DEBUG) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 # When an explicit list of directories was specified always print # all of them (otherwise, tests seem to get lost). if isinstance(tests, list): args.list_untested = True result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats == "details": result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in ["details", "summary"]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--prefix-directory", action="store_true") parser.add_argument("--prefix-name", action="store_true") parser.add_argument("--prefix-output-directory", action="store_true") parser.add_argument("--print-result", action="store_true") parser.add_argument("--print-diff", action="store_true") parser.add_argument("--print-args", action="store_true") parser.add_argument("--print-scripts", action="store_true") parser.add_argument("--print-domains", action="store_true") parser.add_argument("--print-initiators", action="store_true") parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list") parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list") parser.add_argument("--baseline", metavar="DIRECTORY", help="a %(metavar)s containing baseline testsuite output") parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), test output, or testsuite output") # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consumes all remaining arguments. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help="an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run") post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # default to printing results if not args.print_scripts \ and not args.print_result \ and not args.print_diff \ and not args.print_initiators \ and not args.print_domains: args.print_result = True # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 args.prefix_directory = args.prefix_directory or args.verbose > v args.prefix_name = args.prefix_name or args.verbose > v args.print_result = args.print_result or args.verbose > v v += 1 args.prefix_output_directory = args.prefix_output_directory or args.verbose > v v += 1 args.list_untested = args.list_untested or args.verbose > v args.list_ignored = args.list_ignored or args.verbose > v v += 1 args.print_scripts = args.print_scripts or args.verbose > v v += 1 args.print_args = args.print_args or args.verbose > v if args.print_args: post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) return 1 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving. baseline = testsuite.load(logger, args, testsuite_directory=args.baseline, testsuite_output_directory=None, error_level=logutil.DEBUG) if not baseline: # Assume that it is baseline output only. if args.testing_directory: baseline_directory = os.path.join(args.testing_directory, "pluto") else: baseline_directory = utils.directory("..", "pluto") baseline = testsuite.load(logger, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, args, testsuite_directory=args.directories[-1], testsuite_output_directory=None, error_level=logutil.DEBUG) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 # When an explicit list of directories was specified always print # all of them (otherwise, tests seem to get lost). if isinstance(tests, list): args.list_untested = True result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: result_stats.log_summary(stderr_log, prefix=" ") return 0
def main(): parser = argparse.ArgumentParser( description= "list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified." ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument( "--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--print-directory", action="store_true") parser.add_argument("--print-name", action="store_true") # parser.add_argument("--print-result", action="store_true") parser.add_argument("--print-diff", action="store_true") parser.add_argument("--print-args", action="store_true") parser.add_argument("--print-output-directory", action="store_true") parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list") parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list") parser.add_argument( "directories", metavar="TEST-DIRECTORY", nargs="+", help=("Either a testsuite (only one) or test directory")) # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consume all remaining parameters. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help=("An optional testsuite directory containing" " results from a previous test run")) post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 args.print_directory = args.print_directory or args.verbose > v args.print_name = args.print_name or args.verbose > v v += 1 args.print_output_directory = args.print_output_directory or args.verbose > v v += 1 args.list_untested = args.list_untested or args.verbose > v v += 1 args.list_ignored = args.list_ignored or args.verbose > v v += 1 v += 1 args.print_args = args.print_args or args.verbose > v if args.print_args: post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) return 1 # Is the last argument some sort of baseline? If it is, pre-load # it. # # XXX: Should also support something like --baseline-testsuite and # --baseline-output parameters. baseline = None if len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, args.directories[-1], args, error_level=logutil.DEBUG) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 # When an explicit list of directories was specified always print # all of them (otherwise, tests seem to get lost). if isinstance(tests, list): args.list_untested = True for test in tests: # Produce separate runtimes for each test. with logutil.TIMER: logger.debug("start processing test %s", test.name) # Filter out tests that are being ignored? ignore = testsuite.ignore(test, args) if ignore and not args.list_ignored: continue # Filter out tests that have not been run? result = None if not ignore: result = post.mortem( test, args, baseline=baseline, output_directory=test.saved_output_directory, skip_sanitize=args.quick or args.quick_sanitize, skip_diff=args.quick or args.quick_diff, update=args.update, update_sanitize=args.update_sanitize, update_diff=args.update_diff) if not result and not args.list_untested: continue sep = "" # Print the test's name/path if not args.print_directory and not args.print_name and not args.print_output_directory: # By default: when the path given on the command line # explicitly specifies a test's output directory # (found in TEST.SAVED_OUTPUT_DIRECTORY), print that; # otherwise print the path to the test's directory. print(sep, end="") print((test.saved_output_directory and test.saved_output_directory or test.directory), end="") sep = " " else: # Print the test name/path per command line if args.print_name: print(sep, end="") print(test.name, end="") sep = " " if args.print_directory: print(sep, end="") print(test.directory, end="") sep = " " if args.print_output_directory: print(sep, end="") print((test.saved_output_directory and test.saved_output_directory or test.output_directory), end="") sep = " " if ignore: print(sep, end="") print("ignored", ignore, end="") sep = " " print(sep, end="") if result.errors: print(result, result.errors, end="") else: print(result, end="") sep = " " print() if args.print_diff and result: for domain in result.diffs: for line in result.diffs[domain]: if line: print(line) sys.stdout.flush() logger.debug("stop processing test %s", test.name) return 0
def main(): parser = argparse.ArgumentParser( description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately).", ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files") ) parser.add_argument( "--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file") ) parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--print-directory", action="store_true") parser.add_argument("--print-name", action="store_true") parser.add_argument("--print-result", action="store_true") parser.add_argument("--print-diff", action="store_true") parser.add_argument("--print-args", action="store_true") parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list") parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list") parser.add_argument( "directories", metavar="TEST-DIRECTORY", nargs="+", help=("Either a testsuite (only one) or test directory") ) # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consume all remaining parameters. parser.add_argument( "baseline", metavar="BASELINE-DIRECTORY", nargs="?", help=("An optional testsuite directory containing" " results from a previous test run"), ) post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 args.print_directory = args.print_directory or args.verbose > v args.print_name = args.print_name or args.verbose > v v += 1 args.list_untested = args.list_untested or args.verbose > v v += 1 args.list_ignored = args.list_ignored or args.verbose > v v += 1 v += 1 args.print_args = args.print_args or args.verbose > v # By default print the relative directory path. if not args.print_directory and not args.print_name: args.print_directory = True if args.print_args: post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) return 1 # If there is more than one directory then the last might be the # baseline. Try loading it as a testsuite (baselines are # testsuites) to see if that is the case. basetests = None tests = None if len(args.directories) > 1: # Perhaps the last argument is the baseline? Suppress any # nasty errors. basetests = testsuite.load(logger, args.directories[-1], error_level=logutil.DEBUG) if basetests: logger.debug("basetests loaded from '%s'", basetests.directory) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories) logger.debug("basetests=%s", basetests) logger.debug("tests=%s", tests) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 # When an explicit list of directories was specified always print # all of them (otherwise, tests seem to get lost). if isinstance(tests, list): args.list_untested = True # Preload the baseline. This avoids re-scanning the TESTLIST. # Also, passing the full baseline to Test.results() lets that # function differentiate between a baseline missing results or # being entirely absent. baseline = None if basetests: baseline = {} for test in basetests: baseline[test.name] = test for test in tests: # Produce separate runtimes for each test. with logutil.TIMER: logger.debug("start processing test %s", test.name) # Filter out tests that are being ignored? ignore = testsuite.ignore(test, args) if ignore and not args.list_ignored: continue # Filter out tests that have not been run? result = None if not ignore: result = post.mortem( test, args, baseline=baseline, output_directory=test.old_output_directory, skip_sanitize=args.quick or args.quick_sanitize, skip_diff=args.quick or args.quick_diff, update=args.update, update_sanitize=args.update_sanitize, update_diff=args.update_diff, ) if not result and not args.list_untested: continue sep = "" if args.print_name: print(sep, end="") print(test.name, end="") sep = " " if args.print_directory: print(sep, end="") print(test.directory, end="") sep = " " if ignore: print(sep, end="") print("ignored", ignore, end="") sep = " " if result: print(sep, end="") print(result, end="") sep = " " print() if args.print_diff and result: for domain in result.diffs: for line in result.diffs[domain]: if line: print(line) sys.stdout.flush() logger.debug("stop processing test %s", test.name) return 0
def main(): parser = argparse.ArgumentParser( description= "list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately)." ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument( "--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--print-directory", action="store_true") parser.add_argument("--print-name", action="store_true") parser.add_argument("--print-result", action="store_true") parser.add_argument("--print-diff", action="store_true") parser.add_argument("--print-args", action="store_true") parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list") parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list") parser.add_argument( "directories", metavar="TEST-DIRECTORY", nargs="+", help=("Either a testsuite (only one) or test directory")) # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consume all remaining parameters. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help=("An optional testsuite directory containing" " results from a previous test run")) post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 args.print_directory = args.print_directory or args.verbose > v args.print_name = args.print_name or args.verbose > v v += 1 args.list_untested = args.list_untested or args.verbose > v v += 1 args.list_ignored = args.list_ignored or args.verbose > v v += 1 v += 1 args.print_args = args.print_args or args.verbose > v # By default print the relative directory path. if not args.print_directory and not args.print_name: args.print_directory = True if args.print_args: post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) return 1 # If there is more than one directory then the last might be the # baseline. Try loading it as a testsuite (baselines are # testsuites) to see if that is the case. basetests = None tests = None if len(args.directories) > 1: # Perhaps the last argument is the baseline? Suppress any # nasty errors. basetests = testsuite.load(logger, args.directories[-1], error_level=logutil.DEBUG) if basetests: logger.debug("basetests loaded from '%s'", basetests.directory) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories) logger.debug("basetests=%s", basetests) logger.debug("tests=%s", tests) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 # When an explicit list of directories was specified always print # all of them (otherwise, tests seem to get lost). if isinstance(tests, list): args.list_untested = True # Preload the baseline. This avoids re-scanning the TESTLIST. # Also, passing the full baseline to Test.results() lets that # function differentiate between a baseline missing results or # being entirely absent. baseline = None if basetests: baseline = {} for test in basetests: baseline[test.name] = test for test in tests: # Produce separate runtimes for each test. with logutil.TIMER: logger.debug("start processing test %s", test.name) # Filter out tests that are being ignored? ignore = testsuite.ignore(test, args) if ignore and not args.list_ignored: continue # Filter out tests that have not been run? result = None if not ignore: result = post.mortem( test, args, baseline=baseline, output_directory=test.old_output_directory, skip_sanitize=args.quick or args.quick_sanitize, skip_diff=args.quick or args.quick_diff, update=args.update, update_sanitize=args.update_sanitize, update_diff=args.update_diff) if not result and not args.list_untested: continue sep = "" if args.print_name: print(sep, end="") print(test.name, end="") sep = " " if args.print_directory: print(sep, end="") print(test.directory, end="") sep = " " if ignore: print(sep, end="") print("ignored", ignore, end="") sep = " " if result: print(sep, end="") print(result, end="") sep = " " print() if args.print_diff and result: for domain in result.diffs: for line in result.diffs[domain]: if line: print(line) sys.stdout.flush() logger.debug("stop processing test %s", test.name) return 0