def main(): parser = argparse.ArgumentParser( description="Runs the CMS functional test suite.") parser.add_argument( "-s", "--submissions", action="store", type=int, default=50, help="set the number of submissions to submit (default 50)") parser.add_argument( "-w", "--workers", action="store", type=int, default=4, help="set the number of workers to use (default 4)") parser.add_argument( "-v", "--verbose", action="count", default=0, help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = False test_list = [Test('batch', task=batch_50, filenames=['correct-stdio.%l'], languages=(LANG_C, ), checks=[]) for _ in range(args.submissions)] runner = TestRunner(test_list, workers=args.workers) runner.submit_tests(concurrent_submit_and_eval=False) runner.log_elapsed_time() failures = runner.wait_for_evaluation() runner.log_elapsed_time() if failures == []: logger.info("All tests passed!") return 0 else: logger.error("Some test failed!") return 1
def main(): parser = ArgumentParser(description="Runs the CMS functional test suite.") parser.add_argument( "-s", "--submissions", action="store", type=int, default=50, help="set the number of submissions to submit (default 50)") parser.add_argument("-w", "--workers", action="store", type=int, default=4, help="set the number of workers to use (default 4)") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = False test_list = [ Test('batch', task=batch_50, filenames=['correct-stdio.%l'], languages=LANG_C, checks=[]) for _ in range(args.submissions) ] runner = TestRunner(test_list, workers=args.workers) runner.startup() runner.submit_tests() runner.log_elapsed_time() failures = runner.wait_for_evaluation() runner.log_elapsed_time() if failures == []: logger.info("All tests passed!") return 0 else: logger.error("Some test failed!") return 1
def main(): parser = ArgumentParser(description="Runs the CMS functional test suite.") parser.add_argument( "regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="a regex to match to run a subset of tests") parser.add_argument( "-l", "--languages", action="store", type=utf8_decoder, default="", help="a comma-separated list of languages to test") parser.add_argument( "-c", "--contest", action="store", type=utf8_decoder, help="use an existing contest (and the tasks in it)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_TEST_FILENAME) parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = True # Pre-process our command-line arguments to figure out which tests to run. regexes = [re.compile(s) for s in args.regex] if args.languages: languages = frozenset(args.languages.split(',')) else: languages = frozenset() if args.retry_failed: test_list = load_failed_tests() else: test_list = ALL_TESTS test_list = filter_tests(test_list, regexes, languages) if not test_list: logger.info( "There are no tests to run! (was your filter too restrictive?)") return 0 tests = 0 for test in test_list: for language in test.languages: if args.dry_run: logger.info("Test %s in %s.", test.name, language) tests += 1 if test.user_tests: for language in test.languages: if args.dry_run: logger.info("Test %s in %s (for usertest).", test.name, language) tests += 1 if args.dry_run: return 0 if args.retry_failed: logger.info( "Re-running %s failed tests from last run.", len(test_list)) # Clear out any old coverage data. logging.info("Clearing old coverage data.") sh(sys.executable + " -m coverage erase") # Startup the test runner. runner = TestRunner(test_list, contest_id=args.contest, workers=4) # Submit and wait for all tests to complete. runner.submit_tests() failures = runner.wait_for_evaluation() write_test_case_list( [(test, lang) for test, lang, _ in failures], FAILED_TEST_FILENAME) # And good night! runner.shutdown() runner.log_elapsed_time() combine_coverage() logger.info("Executed: %s", tests) logger.info("Failed: %s", len(failures)) if not failures: logger.info("All tests passed!") return 0 else: logger.error("Some test failed!") logger.info("Run again with --retry-failed (or -r) to retry.") logger.info("Failed tests:") for test, lang, msg in failures: logger.info("%s (%s): %s\n", test.name, lang, msg) return 1
def main(): parser = ArgumentParser(description="Runs the CMS functional test suite.") parser.add_argument("regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="a regex to match to run a subset of tests") parser.add_argument("-l", "--languages", action="store", type=utf8_decoder, default="", help="a comma-separated list of languages to test") parser.add_argument("-c", "--contest", action="store", type=utf8_decoder, help="use an existing contest (and the tasks in it)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_TEST_FILENAME) parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = True # Pre-process our command-line arguments to figure out which tests to run. regexes = [re.compile(s) for s in args.regex] if args.languages: languages = frozenset(args.languages.split(',')) else: languages = frozenset() if args.retry_failed: test_list = load_failed_tests() else: test_list = ALL_TESTS test_list = filter_tests(test_list, regexes, languages) if not test_list: logger.info( "There are no tests to run! (was your filter too restrictive?)") return 0 tests = 0 for test in test_list: for language in test.languages: if args.dry_run: logger.info("Test %s in %s.", test.name, language) tests += 1 if test.user_tests: for language in test.languages: if args.dry_run: logger.info("Test %s in %s (for usertest).", test.name, language) tests += 1 if args.dry_run: return 0 if args.retry_failed: logger.info("Re-running %s failed tests from last run.", len(test_list)) # Clear out any old coverage data. logging.info("Clearing old coverage data.") sh(sys.executable + " -m coverage erase") # Startup the test runner. runner = TestRunner(test_list, contest_id=args.contest, workers=4) # Submit and wait for all tests to complete. runner.submit_tests() failures = runner.wait_for_evaluation() write_test_case_list([(test, lang) for test, lang, _ in failures], FAILED_TEST_FILENAME) # And good night! runner.shutdown() runner.log_elapsed_time() combine_coverage() logger.info("Executed: %s", tests) logger.info("Failed: %s", len(failures)) if not failures: logger.info("All tests passed!") return 0 else: logger.error("Some test failed!") logger.info("Run again with --retry-failed (or -r) to retry.") logger.info("Failed tests:") for test, lang, msg in failures: logger.info("%s (%s): %s\n", test.name, lang, msg) return 1
def main(): parser = argparse.ArgumentParser( description="Runs the CMS functional test suite.") parser.add_argument( "-s", "--submissions", action="store", type=int, default=50, help="set the number of submissions to submit (default 50)") parser.add_argument("-w", "--workers", action="store", type=int, default=4, help="set the number of workers to use (default 4)") parser.add_argument( "-l", "--cpu_limits", action="append", default=[], help="set maximum CPU percentage for a set of services, for example: " "'-l .*Server:40' limits servers to use 40%% of a CPU or less; " "can be specified multiple times (requires cputool)") parser.add_argument( "-v", "--verbose", action="count", default=0, help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = False test_list = [ Test('batch', task=batch_50, filenames=['correct-stdio.%l'], languages=(LANG_C, ), checks=[]) for _ in range(args.submissions) ] cpu_limits = [] for l in args.cpu_limits: if ":" not in l: parser.error("CPU limit must be in the form <regex>:<limit>.") regex, _, limit = l.rpartition(":") try: limit = int(limit) except ValueError: parser.error("CPU limit must be an integer.") cpu_limits.append((regex, limit)) runner = TestRunner(test_list, workers=args.workers, cpu_limits=cpu_limits) runner.submit_tests(concurrent_submit_and_eval=False) runner.log_elapsed_time() failures = runner.wait_for_evaluation() runner.log_elapsed_time() if failures == []: logger.info("All tests passed!") return 0 else: logger.error("Some test failed!") return 1