def main(): parser = argparse.ArgumentParser(description="Run tests") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="a testsuite directory, a TESTLIST file, or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) post.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) post.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") end_time = datetime.now() logger.info("run finished at %s after %s", end_time, end_time - timing.START_TIME) return exit_code
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser( description="Run tests", epilog="SIGUSR1 will dump all thread stacks") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--pid-file", default="", help="file to store process id of KVMRUNNER") parser.add_argument( "directories", metavar="DIRECTORY", nargs="+", help= "a testsuite directory, a TESTLIST file, or a list of test directories" ) testsuite.add_arguments(parser) runner.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stdout) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) logger.info(" pid-file: %s", args.pid_file) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) if args.pid_file: pid = os.getpid() logger.info("writing pid %d to '%s'", pid, args.pid_file) with open(args.pid_file, "wt") as pidfile: pidfile.write("%d\n" % os.getpid()) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 if len(tests) == 1 and args.run_post_mortem is None: logger.warning( "skipping post-mortem.sh as only one test; use --run-post-mortem true to override this" ) args.run_post_mortem = False test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") stop_time = datetime.now() logger.info("run finished at %s after %s", stop_time, stop_time - timing.START_TIME) return exit_code
def main(): parser = argparse.ArgumentParser(description="Run tests") # This argument's behaviour is overloaded; the shorter word "try" # is a python word. parser.add_argument("--retry", type=int, metavar="COUNT", default=1, help="which previously run tests should be retried: 0 selects not-started tests; 1 selects not-started+failed tests; -1 selects not-started+failed+passed tests (default is %(default)s)") parser.add_argument("--attempts", type=int, default=1, help="number of times to attempt a test before giving up; default %(default)s") parser.add_argument("--dry-run", "-n", action="store_true") parser.add_argument("--verbose", "-v", action="count", default=0) # Default to BACKUP under the current directory. Name is # arbitrary, chosen for its hopefully unique first letter # (avoiding Makefile, OBJ, README, ... :-). parser.add_argument("--backup-directory", metavar="DIRECTORY", default=os.path.join("BACKUP", time.strftime("%Y%m%d%H%M%S", time.localtime())), help="backup existing <test>/OUTPUT to %(metavar)s/<test> (default: %(default)s)") parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="either a testsuite directory or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) post.add_arguments(parser) logutil.add_arguments(parser) args = parser.parse_args() logutil.config(args) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" retry: %s", args.retry) logger.info(" attempts: %s", args.attempts) logger.info(" dry-run: %s", args.dry_run) logger.info(" backup-directory: %s", args.backup_directory) logger.info(" directories: %s", args.directories) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) post.log_arguments(logger, args) logutil.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() start_time = datetime.now() try: logger.info("run started at %s", start_time) test_count = 0 for test in tests: test_stats.add("total", test) test_count += 1 # Would the number of tests to be [re]run be better? test_prefix = "****** %s (test %d of %d)" % (test.name, test_count, len(tests)) ignore = testsuite.ignore(test, args) if ignore: result_stats.add_ignore(test, ignore) test_stats.add("ignored", test) # No need to log all the ignored tests when an # explicit sub-set of tests is being run. For # instance, when running just one test. if not args.test_name: logger.info("%s: ignore (%s)", test_prefix, ignore) continue # Implement "--retry" as described above: if retry is -ve, # the test is always run; if there's no result, the test # is always run; skip passed tests; else things get a # little wierd. # Be lazy with gathering the results, don't run the # sanitizer or diff. old_result = post.mortem(test, args, skip_diff=True, skip_sanitize=True) if args.retry >= 0: if old_result: if old_result.passed: logger.info("%s: passed", test_prefix) test_stats.add("skipped", test) result_stats.add_skip(old_result) continue if args.retry == 0: logger.info("%s: %s (delete '%s' to re-test)", test_prefix, result, test.output_directory) test_stats.add("skipped", test) result_stats.add_skip(old_result) continue test_stats.add("retry", test) logger.info("%s: starting ...", test_prefix) test_stats.add("tests", test) # Move the contents of the existing OUTPUT directory to # BACKUP_DIRECTORY. Do it file-by-file so that, at no # point, the directory is empty. # # By moving each test just before it is started a trail of # what tests were attempted at each run is left. # # XXX: During boot, swan-transmogrify runs "chcon -R # testing/pluto". Of course this means that each time a # test is added and/or a test is run (adding files under # <test>/OUTPUT), the boot process (and consequently the # time taken to run a test) keeps increasing. # # Always moving the directory contents to the # BACKUP_DIRECTORY mitigates this some. saved_output_directory = None if os.path.exists(test.output_directory): saved_output_directory = os.path.join(args.backup_directory, test.name) logger.info("moving contents of '%s' to '%s'", test.output_directory, saved_output_directory) # Copy "empty" OUTPUT directories too. args.dry_run or os.makedirs(saved_output_directory, exist_ok=True) for name in os.listdir(test.output_directory): src = os.path.join(test.output_directory, name) dst = os.path.join(saved_output_directory, name) logger.debug("moving '%s' to '%s'", src, dst) args.dry_run or os.replace(src, dst) debugfile = None result = None # At least one iteration; above will have filtered out # skips and ignores for attempt in range(args.attempts): test_stats.add("attempts", test) # Create the OUTPUT directory. try: if not args.dry_run: os.mkdir(test.output_directory) elif os.exists(test.output_directory): raise FileExistsError() except FileExistsError: # On first attempt, the OUTPUT directory will # be empty (see above) so no need to save. if attempt > 0: saved_output_directory = os.path.join(test.output_directory, str(attempt)) logger.info("moving contents of '%s' to '%s'", test.output_directory, saved_output_directory) args.dry_run or os.makedirs(saved_output_directory, exist_ok=True) for name in os.listdir(test.output_directory): if os.path.isfile(src): src = os.path.join(test.output_directory, name) dst = os.path.join(saved_output_directory, name) logger.debug("moving '%s' to '%s'", src, dst) args.dry_run or os.replace(src, dst) # Start a debug log in the OUTPUT directory; include # timing for this specific test attempt. with logutil.TIMER, logutil.Debug(logger, os.path.join(test.output_directory, "debug.log")): logger.info("****** test %s attempt %d of %d started at %s ******", test.name, attempt+1, args.attempts, datetime.now()) if saved_output_directory: logger.info("contents of '%s' moved to '%s'", test.output_directory, saved_output_directory) saved_output_directory = None ending = "undefined" try: if not args.dry_run: runner.run_test(test, max_workers=args.workers) ending = "finished" result = post.mortem(test, args, update=(not args.dry_run)) if not args.dry_run: # Store enough to fool the script # pluto-testlist-scan.sh. logger.info("storing result in '%s'", test.result_file) with open(test.result_file, "w") as f: f.write('"result": "%s"\n' % result) except pexpect.TIMEOUT as e: ending = "timeout" logger.exception("**** test %s timed out ****", test.name) result = post.mortem(test, args, update=(not args.dry_run)) # Since the OUTPUT directory exists, all paths to # here should have a non-null RESULT. test_stats.add("attempts(%s:%s)" % (ending, result), test) if result.errors: logger.info("****** test %s %s %s ******", test.name, result, result.errors) else: logger.info("****** test %s %s ******", test.name, result) if result.passed: break # Above will have set RESULT. During a control-c or crash # the below will not be executed. test_stats.add("tests(%s)" % result, test) result_stats.add_result(result, old_result) test_stats.log_summary(logger.info, header="updated stats:", prefix=" ") result_stats.log_summary(logger.info, header="updated results:", prefix=" ") except KeyboardInterrupt: logger.exception("**** test %s interrupted ****", test.name) return 1 level = args.verbose and logger.info or logger.debug test_stats.log_details(level, header="stat details:", prefix=" ") result_stats.log_details(logger.info, header="result details:", prefix=" ") test_stats.log_summary(logger.info, header="stat summary:", prefix=" ") result_stats.log_summary(logger.info, header="result summary:", prefix=" ") end_time = datetime.now() logger.info("run finished at %s after %s", end_time, end_time - start_time) return 0
def main(): # If SIGUSR1, backtrace all threads; hopefully this is early # enough. faulthandler.register(signal.SIGUSR1) parser = argparse.ArgumentParser(description="Run tests", epilog="SIGUSR1 will dump all thread stacks") parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("directories", metavar="DIRECTORY", nargs="+", help="a testsuite directory, a TESTLIST file, or a list of test directories") testsuite.add_arguments(parser) runner.add_arguments(parser) logutil.add_arguments(parser) skip.add_arguments(parser) ignore.add_arguments(parser) publish.add_arguments(parser) # These three calls go together args = parser.parse_args() logutil.config(args, sys.stdout) logger = logutil.getLogger("kvmrunner") logger.info("Options:") logger.info(" directories: %s", args.directories) logger.info(" verbose: %s", args.verbose) testsuite.log_arguments(logger, args) runner.log_arguments(logger, args) logutil.log_arguments(logger, args) skip.log_arguments(logger, args) ignore.log_arguments(logger, args) publish.log_arguments(logger, args) tests = testsuite.load_testsuite_or_tests(logger, args.directories, args, log_level=logutil.INFO) if not tests: logger.error("test or testsuite directory invalid: %s", args.directories) return 1 test_stats = stats.Tests() result_stats = stats.Results() try: exit_code = 0 logger.info("run started at %s", timing.START_TIME) runner.run_tests(logger, args, tests, test_stats, result_stats) except KeyboardInterrupt: logger.exception("**** interrupted ****") exit_code = 1 test_stats.log_details(args.verbose and logger.info or logger.debug, header="final stat details:", prefix=" ") result_stats.log_details(logger.info, header="final test details:", prefix=" ") test_stats.log_summary(logger.info, header="final test stats:", prefix=" ") result_stats.log_summary(logger.info, header="final test results:", prefix=" ") stop_time = datetime.now() logger.info("run finished at %s after %s", stop_time, stop_time - timing.START_TIME) return exit_code