def main(): parser = argparse.ArgumentParser(description="Run tests") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="a testsuite directory, a TESTLIST file, or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stdout) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") end_time = datetime.now() logger.info("run finished at %s after %s", end_time, end_time - timing.START_TIME) return exit_code
def main(): parser = argparse.ArgumentParser(description="Run tests") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="a testsuite directory, a TESTLIST file, or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) post.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) post.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") end_time = datetime.now() logger.info("run finished at %s after %s", end_time, end_time - timing.START_TIME) return exit_code
def main(): parser = argparse.ArgumentParser( description="list test results", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately)." ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--dump-args", action="store_true") # how to parse --print directory,saved-directory,...? parser.add_argument( "--print", action="store", default=Print(Print.path, Print.result, Print.issues), type=Print, metavar=str(Print), help= "comman separate list of attributes to print for each test; default: '%(default)s'" ) parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") baseline_metavar = "BASELINE-DIRECTORY" baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed" parser.add_argument("--baseline", "-b", metavar=baseline_metavar, help=baseline_help) parser.add_argument( "--json", action="store_true", help= "output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list" ) parser.add_argument( "directories", metavar="DIRECTORY-OR-FILE", nargs="+", help= "a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'" ) # Note: this argument serves as documentation only. The RESULT # argument should consumes all remaining parameters. parser.add_argument("baseline_ignored", nargs="?", metavar=baseline_metavar, help=baseline_help) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stderr) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Baseline: %s", args.baseline) logger.info(" Json: %s", args.json) logger.info(" Quick: %s", args.quick) logger.info(" Update: %s", args.update) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser( description="Run tests", epilog="SIGUSR1 will dump all thread stacks") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--pid-file", default="", help="file to store process id of KVMRUNNER") parser.add_argument( "directories", metavar="DIRECTORY", nargs="+", help= "a testsuite directory, a TESTLIST file, or a list of test directories" ) testsuite.add_arguments(parser) runner.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stdout) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) logger.info(" pid-file: %s", args.pid_file) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) if args.pid_file: pid = os.getpid() logger.info("writing pid %d to '%s'", pid, args.pid_file) with open(args.pid_file, "wt") as pidfile: pidfile.write("%d\n" % os.getpid()) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 if len(tests) == 1 and args.run_post_mortem is None: logger.warning( "skipping post-mortem.sh as only one test; use --run-post-mortem true to override this" ) args.run_post_mortem = False test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") stop_time = datetime.now() logger.info("run finished at %s after %s", stop_time, stop_time - timing.START_TIME) return exit_code
def main(): parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--dump-args", action="store_true") parser.add_argument("--prefix", action="store", type=Prefix, choices=[p for p in Prefix], help="prefix to display with each test") # how to parse --print directory,saved-directory,...? parser.add_argument("--print", action="append", default=[], choices=[p for p in Print], type=Print, help="what information to display about each test") parser.add_argument("--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\""); parser.add_argument("--baseline", metavar="DIRECTORY", help="a %(metavar)s containing baseline testsuite output") parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), a TESTLIST file, test output, or testsuite output") # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consumes all remaining arguments. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help="an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run") post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # default to printing results if not args.print: args.print = [Print.result] # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Prefix: %s", args.prefix) post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, args, testsuite_directory=baseline_directory, saved_testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): parser = argparse.ArgumentParser( description= "list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified." ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument( "--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--dump-args", action="store_true") parser.add_argument("--prefix", action="store", type=Prefix, choices=[p for p in Prefix], help="prefix to display with each test") # how to parse --print directory,saved-directory,...? parser.add_argument("--print", action="append", default=[], choices=[p for p in Print], type=Print, help="what information to display about each test") parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") parser.add_argument( "--baseline", metavar="DIRECTORY", help="a %(metavar)s containing baseline testsuite output") parser.add_argument( "directories", metavar="DIRECTORY", nargs="+", help= "%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), a TESTLIST file, test output, or testsuite output" ) # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consumes all remaining arguments. parser.add_argument( "baseline", metavar="BASELINE-DIRECTORY", nargs="?", help= "an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run" ) post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # default to printing results if not args.print: args.print = [Print.result] # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Prefix: %s", args.prefix) post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load( logger, args, testsuite_directory=baseline_directory, saved_testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser( description="list test results", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). SIGUSR1 will dump all thread stacks" ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--exit-ok", action="store_true", help= ("return a zero exit status; normally, when there are failures, a non-zero exit status is returned" )) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--dump-args", action="store_true") # how to parse --print directory,saved-directory,...? parser.add_argument( "--print", action="store", default=printer.Print(printer.Print.PATH, printer.Print.RESULT, printer.Print.ISSUES), type=printer.Print, metavar=str(printer.Print), help= "comman separate list of attributes to print for each test; default: '%(default)s'" ) parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") baseline_metavar = "BASELINE-DIRECTORY" baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed" parser.add_argument("--baseline", "-b", metavar=baseline_metavar, help=baseline_help) parser.add_argument( "--json", action="store_true", help= "output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list" ) parser.add_argument( "directories", metavar="DIRECTORY-OR-FILE", nargs="+", help= "a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'" ) # Note: this argument serves as documentation only. The RESULT # argument should consumes all remaining parameters. parser.add_argument("baseline_ignored", nargs="?", metavar=baseline_metavar, help=baseline_help) testsuite.add_arguments(parser) logutil.add_arguments(parser) # XXX: while checking for an UNTESTED test should be very cheap # (does OUTPUT/ exist?) it isn't. Currently it triggers a full # post-mortem analysis. skip.add_arguments(parser, skip.Skip.UNTESTED) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stderr) logger = logutil.getLogger("kvmresults") if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Baseline: %s", args.baseline) logger.info(" Json: %s", args.json) logger.info(" Quick: %s", args.quick) logger.info(" Update: %s", args.update) logger.info(" Exit OK: %s", args.exit_ok) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline and os.path.isdir(args.baseline): # Perhaps, AKA BACKUP/YYYY-MM-DDD-..., the baseline # directory only contains a copy of the output. Magic up # a baseline by combining the output with the tests in # ARGS.TESTING_DIRECTORY. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() exit_code = 125 # assume a 'git bisect' barf try: exit_code = results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") publish.json_results(logger, args) publish.json_summary(logger, args) return exit_code
def main(): parser = argparse.ArgumentParser(description="list test results", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately).") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--dump-args", action="store_true") # how to parse --print directory,saved-directory,...? parser.add_argument("--print", action="store", default=Print(Print.path, Print.result, Print.issues), type=Print, metavar=str(Print), help="comman separate list of attributes to print for each test; default: '%(default)s'") parser.add_argument("--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\""); baseline_metavar = "BASELINE-DIRECTORY" baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed" parser.add_argument("--baseline", "-b", metavar=baseline_metavar, help=baseline_help) parser.add_argument("--json", action="store_true", help="output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list") parser.add_argument("directories", metavar="DIRECTORY-OR-FILE", nargs="+", help="a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'") # Note: this argument serves as documentation only. The RESULT # argument should consumes all remaining parameters. parser.add_argument("baseline_ignored", nargs="?", metavar=baseline_metavar, help=baseline_help) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stderr) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Baseline: %s", args.baseline) logger.info(" Json: %s", args.json) logger.info(" Quick: %s", args.quick) logger.info(" Update: %s", args.update) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser(description="Run tests", epilog="SIGUSR1 will dump all thread stacks") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="a testsuite directory, a TESTLIST file, or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stdout) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") stop_time = datetime.now() logger.info("run finished at %s after %s", stop_time, stop_time - timing.START_TIME) return exit_code
def main(): parser = argparse.ArgumentParser(description="write 'table.json' to standard output") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), a TESTLIST file, test output, or testsuite output") post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 columns = [ "Test", "Expected", "Result", "Run time", "Responder", "..." ] rows = [] for test in tests: sys.stderr.write("%s\n" % test.name) # Filter out tests that are being ignored? ignored, include_ignored, details = ignore.test(logger, args, test) if ignored: if not include_ignored: continue # Filter out tests that have not been run result = None if not ignored: result = post.mortem(test, args, baseline=None, output_directory=test.saved_output_directory, test_finished=None, skip_sanitize=args.quick or args.quick_sanitize, skip_diff=args.quick or args.quick_diff, update=args.update, update_sanitize=args.update_sanitize, update_diff=args.update_diff) if skip.result(logger, args, result): continue row = [ test.name, test.expected_result, str(result), "run-time" ] for host in sorted(test.host_names): errors = result.errors.errors if host in errors: row.append("%s %s" % (host, " ".join(sorted(errors[host])))) else: row.append("%s passed" % (host)) rows.append(row) summary = { "Total": 0, "passed": 0, "failed": 0, "abort": 0, "missing baseline": 0, "missing console output": 0, "missing OUTPUT": 0, "missing RESULT": 0, "ASSERT": 0, "CORE": 0, "EXPECT": 0, "GPFAULT": 0, "SEGFAULT": 0, "date": "0000-00-00", "dir": "testing/pluto", "runtime": 0, "runtime_str": "00:00:00", } table = { "suffix": "/OUTPUT", "summary": summary, "columns": columns, "rows": rows, "runDir": "???", } print(json.dumps(table, indent=2)) return 0