def main(): parser = argparse.ArgumentParser(description="Run tests") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="a testsuite directory, a TESTLIST file, or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) post.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) post.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") end_time = datetime.now() logger.info("run finished at %s after %s", end_time, end_time - timing.START_TIME) return exit_code
def main(): parser = argparse.ArgumentParser( description="list test results", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately)." ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--dump-args", action="store_true") # how to parse --print directory,saved-directory,...? parser.add_argument( "--print", action="store", default=Print(Print.path, Print.result, Print.issues), type=Print, metavar=str(Print), help= "comman separate list of attributes to print for each test; default: '%(default)s'" ) parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") baseline_metavar = "BASELINE-DIRECTORY" baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed" parser.add_argument("--baseline", "-b", metavar=baseline_metavar, help=baseline_help) parser.add_argument( "--json", action="store_true", help= "output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list" ) parser.add_argument( "directories", metavar="DIRECTORY-OR-FILE", nargs="+", help= "a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'" ) # Note: this argument serves as documentation only. The RESULT # argument should consumes all remaining parameters. parser.add_argument("baseline_ignored", nargs="?", metavar=baseline_metavar, help=baseline_help) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stderr) logger = logutil.getLogger("kvmresults") # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Baseline: %s", args.baseline) logger.info(" Json: %s", args.json) logger.info(" Quick: %s", args.quick) logger.info(" Update: %s", args.update) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser( description="Run tests", epilog="SIGUSR1 will dump all thread stacks") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--pid-file", default="", help="file to store process id of KVMRUNNER") parser.add_argument( "directories", metavar="DIRECTORY", nargs="+", help= "a testsuite directory, a TESTLIST file, or a list of test directories" ) testsuite.add_arguments(parser) runner.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stdout) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) logger.info(" pid-file: %s", args.pid_file) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) if args.pid_file: pid = os.getpid() logger.info("writing pid %d to '%s'", pid, args.pid_file) with open(args.pid_file, "wt") as pidfile: pidfile.write("%d\n" % os.getpid()) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 if len(tests) == 1 and args.run_post_mortem is None: logger.warning( "skipping post-mortem.sh as only one test; use --run-post-mortem true to override this" ) args.run_post_mortem = False test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") stop_time = datetime.now() logger.info("run finished at %s after %s", stop_time, stop_time - timing.START_TIME) return exit_code
def main(): parser = argparse.ArgumentParser(description="Run tests") # This argument's behaviour is overloaded; the shorter word "try" # is a python word. parser.add_argument("--retry", type=int, metavar="COUNT", default=1, help="which previously run tests should be retried: 0 selects not-started tests; 1 selects not-started+failed tests; -1 selects not-started+failed+passed tests (default is %(default)s)") parser.add_argument("--attempts", type=int, default=1, help="number of times to attempt a test before giving up; default %(default)s") parser.add_argument("--dry-run", "-n", action="store_true") parser.add_argument("--verbose", "-v", action="count", default=0) # Default to BACKUP under the current directory. Name is # arbitrary, chosen for its hopefully unique first letter # (avoiding Makefile, OBJ, README, ... :-). parser.add_argument("--backup-directory", metavar="DIRECTORY", default=os.path.join("BACKUP", time.strftime("%Y%m%d%H%M%S", time.localtime())), help="backup existing <test>/OUTPUT to %(metavar)s/<test> (default: %(default)s)") parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="either a testsuite directory or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) post.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" retry: %s", args.retry) logger.info(" attempts: %s", args.attempts) logger.info(" dry-run: %s", args.dry_run) logger.info(" backup-directory: %s", args.backup_directory) logger.info(" directories: %s", args.directories) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) post.log_arguments(logger, args) logutil.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() start_time = datetime.now() try: logger.info("run started at %s", start_time) test_count = 0 for test in tests: test_stats.add("total", test) test_count += 1 # Would the number of tests to be [re]run be better? test_prefix = "****** %s (test %d of %d)" % (test.name, test_count, len(tests)) ignore = testsuite.ignore(test, args) if ignore: result_stats.add_ignore(test, ignore) test_stats.add("ignored", test) # No need to log all the ignored tests when an # explicit sub-set of tests is being run. For # instance, when running just one test. if not args.test_name: logger.info("%s: ignore (%s)", test_prefix, ignore) continue # Implement "--retry" as described above: if retry is -ve, # the test is always run; if there's no result, the test # is always run; skip passed tests; else things get a # little wierd. # Be lazy with gathering the results, don't run the # sanitizer or diff. old_result = post.mortem(test, args, skip_diff=True, skip_sanitize=True) if args.retry >= 0: if old_result: if old_result.passed: logger.info("%s: passed", test_prefix) test_stats.add("skipped", test) result_stats.add_skip(old_result) continue if args.retry == 0: logger.info("%s: %s (delete '%s' to re-test)", test_prefix, result, test.output_directory) test_stats.add("skipped", test) result_stats.add_skip(old_result) continue test_stats.add("retry", test) logger.info("%s: starting ...", test_prefix) test_stats.add("tests", test) # Move the contents of the existing OUTPUT directory to # BACKUP_DIRECTORY. Do it file-by-file so that, at no # point, the directory is empty. # # By moving each test just before it is started a trail of # what tests were attempted at each run is left. # # XXX: During boot, swan-transmogrify runs "chcon -R # testing/pluto". Of course this means that each time a # test is added and/or a test is run (adding files under # <test>/OUTPUT), the boot process (and consequently the # time taken to run a test) keeps increasing. # # Always moving the directory contents to the # BACKUP_DIRECTORY mitigates this some. saved_output_directory = None if os.path.exists(test.output_directory): saved_output_directory = os.path.join(args.backup_directory, test.name) logger.info("moving contents of '%s' to '%s'", test.output_directory, saved_output_directory) # Copy "empty" OUTPUT directories too. args.dry_run or os.makedirs(saved_output_directory, exist_ok=True) for name in os.listdir(test.output_directory): src = os.path.join(test.output_directory, name) dst = os.path.join(saved_output_directory, name) logger.debug("moving '%s' to '%s'", src, dst) args.dry_run or os.replace(src, dst) debugfile = None result = None # At least one iteration; above will have filtered out # skips and ignores for attempt in range(args.attempts): test_stats.add("attempts", test) # Create the OUTPUT directory. try: if not args.dry_run: os.mkdir(test.output_directory) elif os.exists(test.output_directory): raise FileExistsError() except FileExistsError: # On first attempt, the OUTPUT directory will # be empty (see above) so no need to save. if attempt > 0: saved_output_directory = os.path.join(test.output_directory, str(attempt)) logger.info("moving contents of '%s' to '%s'", test.output_directory, saved_output_directory) args.dry_run or os.makedirs(saved_output_directory, exist_ok=True) for name in os.listdir(test.output_directory): if os.path.isfile(src): src = os.path.join(test.output_directory, name) dst = os.path.join(saved_output_directory, name) logger.debug("moving '%s' to '%s'", src, dst) args.dry_run or os.replace(src, dst) # Start a debug log in the OUTPUT directory; include # timing for this specific test attempt. with logutil.TIMER, logutil.Debug(logger, os.path.join(test.output_directory, "debug.log")): logger.info("****** test %s attempt %d of %d started at %s ******", test.name, attempt+1, args.attempts, datetime.now()) if saved_output_directory: logger.info("contents of '%s' moved to '%s'", test.output_directory, saved_output_directory) saved_output_directory = None ending = "undefined" try: if not args.dry_run: runner.run_test(test, max_workers=args.workers) ending = "finished" result = post.mortem(test, args, update=(not args.dry_run)) if not args.dry_run: # Store enough to fool the script # pluto-testlist-scan.sh. logger.info("storing result in '%s'", test.result_file) with open(test.result_file, "w") as f: f.write('"result": "%s"\n' % result) except pexpect.TIMEOUT as e: ending = "timeout" logger.exception("**** test %s timed out ****", test.name) result = post.mortem(test, args, update=(not args.dry_run)) # Since the OUTPUT directory exists, all paths to # here should have a non-null RESULT. test_stats.add("attempts(%s:%s)" % (ending, result), test) if result.errors: logger.info("****** test %s %s %s ******", test.name, result, result.errors) else: logger.info("****** test %s %s ******", test.name, result) if result.passed: break # Above will have set RESULT. During a control-c or crash # the below will not be executed. test_stats.add("tests(%s)" % result, test) result_stats.add_result(result, old_result) test_stats.log_summary(logger.info, header="updated stats:", prefix=" ") result_stats.log_summary(logger.info, header="updated results:", prefix=" ") except KeyboardInterrupt: logger.exception("**** test %s interrupted ****", test.name) return 1 level = args.verbose and logger.info or logger.debug test_stats.log_details(level, header="stat details:", prefix=" ") result_stats.log_details(logger.info, header="result details:", prefix=" ") test_stats.log_summary(logger.info, header="stat summary:", prefix=" ") result_stats.log_summary(logger.info, header="result summary:", prefix=" ") end_time = datetime.now() logger.info("run finished at %s after %s", end_time, end_time - start_time) return 0
def main(): parser = argparse.ArgumentParser( description= "list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified." ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument( "--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--dump-args", action="store_true") parser.add_argument("--prefix", action="store", type=Prefix, choices=[p for p in Prefix], help="prefix to display with each test") # how to parse --print directory,saved-directory,...? parser.add_argument("--print", action="append", default=[], choices=[p for p in Print], type=Print, help="what information to display about each test") parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") parser.add_argument( "--baseline", metavar="DIRECTORY", help="a %(metavar)s containing baseline testsuite output") parser.add_argument( "directories", metavar="DIRECTORY", nargs="+", help= "%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), a TESTLIST file, test output, or testsuite output" ) # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consumes all remaining arguments. parser.add_argument( "baseline", metavar="BASELINE-DIRECTORY", nargs="?", help= "an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run" ) post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # default to printing results if not args.print: args.print = [Print.result] # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Prefix: %s", args.prefix) post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: # Perhaps the baseline just contains output, magic up the # corresponding testsuite directory. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load( logger, args, testsuite_directory=baseline_directory, saved_testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") return 0
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser( description="list test results", epilog= "By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). SIGUSR1 will dump all thread stacks" ) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( "--exit-ok", action="store_true", help= ("return a zero exit status; normally, when there are failures, a non-zero exit status is returned" )) parser.add_argument( "--quick", action="store_true", help= ("Use the previously generated '.console.txt' and '.console.diff' files" )) parser.add_argument( "--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--dump-args", action="store_true") # how to parse --print directory,saved-directory,...? parser.add_argument( "--print", action="store", default=printer.Print(printer.Print.PATH, printer.Print.RESULT, printer.Print.ISSUES), type=printer.Print, metavar=str(printer.Print), help= "comman separate list of attributes to print for each test; default: '%(default)s'" ) parser.add_argument( "--stats", action="store", default=Stats.summary, type=Stats, choices=[c for c in Stats], help="provide overview statistics; default: \"%(default)s\"") baseline_metavar = "BASELINE-DIRECTORY" baseline_help = "additional %(metavar)s containing results to compare against; any divergence between the test and baseline results are displayed" parser.add_argument("--baseline", "-b", metavar=baseline_metavar, help=baseline_help) parser.add_argument( "--json", action="store_true", help= "output each result as an individual json object (pipe the output through 'jq -s .' to convert it to a well formed json list" ) parser.add_argument( "directories", metavar="DIRECTORY-OR-FILE", nargs="+", help= "a directory containing: a test, testsuite, test output, or testsuite output; or a file containing a 'TESTLIST'" ) # Note: this argument serves as documentation only. The RESULT # argument should consumes all remaining parameters. parser.add_argument("baseline_ignored", nargs="?", metavar=baseline_metavar, help=baseline_help) testsuite.add_arguments(parser) logutil.add_arguments(parser) # XXX: while checking for an UNTESTED test should be very cheap # (does OUTPUT/ exist?) it isn't. Currently it triggers a full # post-mortem analysis. skip.add_arguments(parser, skip.Skip.UNTESTED) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stderr) logger = logutil.getLogger("kvmresults") if args.dump_args: logger.info("Arguments:") logger.info(" Stats: %s", args.stats) logger.info(" Print: %s", args.print) logger.info(" Baseline: %s", args.baseline) logger.info(" Json: %s", args.json) logger.info(" Quick: %s", args.quick) logger.info(" Update: %s", args.update) logger.info(" Exit OK: %s", args.exit_ok) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) return 0 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving in how # it is loaded. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.baseline, error_level=logutil.DEBUG) if not baseline and os.path.isdir(args.baseline): # Perhaps, AKA BACKUP/YYYY-MM-DDD-..., the baseline # directory only contains a copy of the output. Magic up # a baseline by combining the output with the tests in # ARGS.TESTING_DIRECTORY. baseline_directory = os.path.join(args.testing_directory, "pluto") baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, logutil.DEBUG, args, testsuite_directory=args.directories[-1]) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 result_stats = stats.Results() exit_code = 125 # assume a 'git bisect' barf try: exit_code = results(logger, tests, baseline, args, result_stats) finally: if args.stats is Stats.details: result_stats.log_details(stderr_log, header="Details:", prefix=" ") if args.stats in [Stats.details, Stats.summary]: result_stats.log_summary(stderr_log, header="Summary:", prefix=" ") publish.json_results(logger, args) publish.json_summary(logger, args) return exit_code
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser(description="Run tests", epilog="SIGUSR1 will dump all thread stacks") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="a testsuite directory, a TESTLIST file, or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stdout) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") stop_time = datetime.now() logger.info("run finished at %s after %s", stop_time, stop_time - timing.START_TIME) return exit_code
def main(): parser = argparse.ArgumentParser(description="list all tests in the form: <test> [ <directory> ] [ <result> <details...> ]", epilog="By default this tool uses 'sanitizer.sh' and 'diff' to generate up-to-the-minuite test results (the previously generated files 'OUTPUT/*.console.txt' and 'OUTPUT/*.console.diff' are ignored). While this makes things a little slower, it has the benefit of always providing the most up-to-date and correct results (for instance, changes to known-good files are reflected immediately). If a BASELINE directory is specified, anywhere a test result is different to the baseline is also identified.") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--quick", action="store_true", help=("Use the previously generated '.console.txt' and '.console.diff' files")) parser.add_argument("--quick-sanitize", action="store_true", help=("Use the previously generated '.console.txt' file")) parser.add_argument("--quick-diff", action="store_true", help=("Use the previously generated '.console.diff' file")) parser.add_argument("--update", action="store_true", help=("Update the '.console.txt' and '.console.diff' files")) parser.add_argument("--update-sanitize", action="store_true", help=("Update the '.console.txt' file")) parser.add_argument("--update-diff", action="store_true", help=("Update the '.console.diff' file")) parser.add_argument("--prefix-directory", action="store_true") parser.add_argument("--prefix-name", action="store_true") parser.add_argument("--prefix-output-directory", action="store_true") parser.add_argument("--print-result", action="store_true") parser.add_argument("--print-diff", action="store_true") parser.add_argument("--print-args", action="store_true") parser.add_argument("--print-scripts", action="store_true") parser.add_argument("--print-domains", action="store_true") parser.add_argument("--print-initiators", action="store_true") parser.add_argument("--list-ignored", action="store_true", help="include ignored tests in the list") parser.add_argument("--list-untested", action="store_true", help="include untested tests in the list") parser.add_argument("--baseline", metavar="DIRECTORY", help="a %(metavar)s containing baseline testsuite output") parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="%(metavar)s containing: a test, a testsuite (contains a TESTLIST file), test output, or testsuite output") # Note: this argument serves as documentation only. The # TEST-DIRECTORY argument always consumes all remaining arguments. parser.add_argument("baseline", metavar="BASELINE-DIRECTORY", nargs="?", help="an optional testsuite directory (contains a TESTLIST file) containing output from a previous test run") post.add_arguments(parser) testsuite.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmresults") # default to printing results if not args.print_scripts \ and not args.print_result \ and not args.print_diff \ and not args.print_initiators \ and not args.print_domains: args.print_result = True # The option -vvvvvvv is a short circuit for these; make # re-ordering easy by using V as a counter. v = 0 args.prefix_directory = args.prefix_directory or args.verbose > v args.prefix_name = args.prefix_name or args.verbose > v args.print_result = args.print_result or args.verbose > v v += 1 args.prefix_output_directory = args.prefix_output_directory or args.verbose > v v += 1 args.list_untested = args.list_untested or args.verbose > v args.list_ignored = args.list_ignored or args.verbose > v v += 1 args.print_scripts = args.print_scripts or args.verbose > v v += 1 args.print_args = args.print_args or args.verbose > v if args.print_args: post.log_arguments(logger, args) testsuite.log_arguments(logger, args) logutil.log_arguments(logger, args) return 1 # Try to find a baseline. If present, pre-load it. baseline = None if args.baseline: # An explict baseline testsuite, can be more forgiving. baseline = testsuite.load(logger, args, testsuite_directory=args.baseline, testsuite_output_directory=None, error_level=logutil.DEBUG) if not baseline: # Assume that it is baseline output only. if args.testing_directory: baseline_directory = os.path.join(args.testing_directory, "pluto") else: baseline_directory = utils.directory("..", "pluto") baseline = testsuite.load(logger, args, testsuite_directory=baseline_directory, testsuite_output_directory=args.baseline, error_level=logutil.DEBUG) if not baseline: logger.info("'%s' is not a baseline", args.baseline) return 1 elif len(args.directories) > 1: # If there is more than one directory then, perhaps, the last # one is a baseline. A baseline might be: a complete # testsuite snapshot; or just output saved as # testing/pluto/OUTPUT/TESTDIR. baseline = testsuite.load(logger, args, testsuite_directory=args.directories[-1], testsuite_output_directory=None, error_level=logutil.DEBUG) if baseline: # discard the last argument as consumed above. logger.debug("discarding baseline testsuite argument '%s'", args.directories[-1]) args.directories.pop() tests = testsuite.load_testsuite_or_tests(logger, args.directories, args) # And check if not tests: logger.error("Invalid testsuite or test directories") return 1 # When an explicit list of directories was specified always print # all of them (otherwise, tests seem to get lost). if isinstance(tests, list): args.list_untested = True result_stats = stats.Results() try: results(logger, tests, baseline, args, result_stats) finally: result_stats.log_summary(stderr_log, prefix=" ") return 0