def main(): parser = ArgumentParser(description="Runs the CMS functional test suite.") parser.add_argument( "regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="a regex to match to run a subset of tests") parser.add_argument( "-l", "--languages", action="store", type=utf8_decoder, default="", help="a comma-separated list of languages to test") parser.add_argument( "-c", "--contest", action="store", type=utf8_decoder, help="use an existing contest (and the tasks in it)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_TEST_FILENAME) parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = True # Pre-process our command-line arguments to figure out which tests to run. regexes = [re.compile(s) for s in args.regex] if args.languages: languages = frozenset(args.languages.split(',')) else: languages = frozenset() if args.retry_failed: test_list = load_failed_tests() else: test_list = ALL_TESTS test_list = filter_tests(test_list, regexes, languages) if not test_list: logger.info( "There are no tests to run! (was your filter too restrictive?)") return 0 tests = 0 for test in test_list: for language in test.languages: if args.dry_run: logger.info("Test %s in %s.", test.name, language) tests += 1 if test.user_tests: for language in test.languages: if args.dry_run: logger.info("Test %s in %s (for usertest).", test.name, language) tests += 1 if args.dry_run: return 0 if args.retry_failed: logger.info( "Re-running %s failed tests from last run.", len(test_list)) # Clear out any old coverage data. logging.info("Clearing old coverage data.") sh(sys.executable + " -m coverage erase") # Startup the test runner. runner = TestRunner(test_list, contest_id=args.contest, workers=4) # Submit and wait for all tests to complete. runner.submit_tests() failures = runner.wait_for_evaluation() write_test_case_list( [(test, lang) for test, lang, _ in failures], FAILED_TEST_FILENAME) # And good night! runner.shutdown() runner.log_elapsed_time() combine_coverage() logger.info("Executed: %s", tests) logger.info("Failed: %s", len(failures)) if not failures: logger.info("All tests passed!") return 0 else: logger.error("Some test failed!") logger.info("Run again with --retry-failed (or -r) to retry.") logger.info("Failed tests:") for test, lang, msg in failures: logger.info("%s (%s): %s\n", test.name, lang, msg) return 1
def main(): parser = ArgumentParser(description="Runs the CMS unittest suite.") parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_UNITTEST_FILENAME) # Unused parameters. parser.add_argument( "regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="unused") parser.add_argument( "-l", "--languages", action="store", type=utf8_decoder, default="", help="unused") parser.add_argument( "-c", "--contest", action="store", type=utf8_decoder, help="unused") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose start_time = datetime.datetime.now() try: git_root = subprocess.check_output( "git rev-parse --show-toplevel", shell=True, stderr=io.open(os.devnull, "wb")).strip() except subprocess.CalledProcessError: print("Please run the unit tests from the git repository.") return 1 if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() if args.dry_run: for t in test_list: print(t[0].name, t[1]) return 0 if args.retry_failed: info("Re-running %d failed tests from last run." % len(test_list)) # Load config from cms.conf. CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. info("Clearing old coverage data.") sh("python -m coverage erase") # Run all of our test cases. passed, test_results = run_unittests(test_list) combine_coverage() print(test_results) end_time = datetime.datetime.now() print(time_difference(start_time, end_time)) if passed: return 0 else: return 1
def main(): parser = ArgumentParser(description="Runs the CMS functional test suite.") parser.add_argument("regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="a regex to match to run a subset of tests") parser.add_argument("-l", "--languages", action="store", type=utf8_decoder, default="", help="a comma-separated list of languages to test") parser.add_argument("-c", "--contest", action="store", type=utf8_decoder, help="use an existing contest (and the tasks in it)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_TEST_FILENAME) parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose start_time = datetime.datetime.now() # Pre-process our command-line arugments to figure out which tests to run. regexes = [re.compile(s) for s in args.regex] if args.languages: languages = frozenset(args.languages.split(',')) else: languages = frozenset() if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() test_list = filter_testcases(test_list, regexes, languages) if not test_list: info("There are no tests to run! (was your filter too restrictive?)") return 0 if args.dry_run: for t in test_list: print(t[0].name, t[1]) return 0 if args.retry_failed: info("Re-running %d failed tests from last run." % len(test_list)) # Load config from cms.conf. try: git_root = subprocess.check_output("git rev-parse --show-toplevel", shell=True, stderr=io.open(os.devnull, "wb")).strip() except subprocess.CalledProcessError: git_root = None CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" cms_config = get_cms_config() if not config_is_usable(cms_config): return 1 if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. info("Clearing old coverage data.") sh("python-coverage erase") # Fire us up! start_generic_services() if args.contest is None: contest_id = create_contest() else: contest_id = int(args.contest) user_id = create_or_get_user(contest_id) start_contest(contest_id) # Run all of our test cases. passed, test_results = run_testcases(contest_id, user_id, test_list) # And good night! shutdown_services() combine_coverage() print(test_results) end_time = datetime.datetime.now() print(time_difference(start_time, end_time)) if passed: return 0 else: return 1
def main(): parser = ArgumentParser(description="Runs the CMS test suite.") parser.add_argument("regex", metavar="regex", type=str, nargs='*', help="a regex to match to run a subset of tests") parser.add_argument("-l", "--languages", type=str, action="store", default="", help="a comma-separated list of languages to test") parser.add_argument("-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_TEST_FILENAME) parser.add_argument("-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose start_time = datetime.datetime.now() # Pre-process our command-line arugments to figure out which tests to run. regexes = [re.compile(s) for s in args.regex] if args.languages: languages = frozenset(args.languages.split(',')) else: languages = frozenset() if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() test_list = filter_testcases(test_list, regexes, languages) if not test_list: info("There are no tests to run! (was your filter too restrictive?)") return 0 if args.retry_failed: info("Re-running %d failed tests from last run." % len(test_list)) # Load config from cms.conf. try: git_root = subprocess.check_output( "git rev-parse --show-toplevel", shell=True, stderr=open(os.devnull, "w")).strip() except subprocess.CalledProcessError: git_root = None CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/examples/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" cms_config = get_cms_config() if not config_is_usable(cms_config): return 1 if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. info("Clearing old coverage data.") sh("python-coverage erase") # Fire us up! start_generic_services() contest_id = create_contest() user_id = create_a_user(contest_id) # Run all of our test cases. test_results = run_testcases(contest_id, user_id, test_list) # And good night! shutdown_services() combine_coverage() print test_results end_time = datetime.datetime.now() print time_difference(start_time, end_time)
def main(): parser = ArgumentParser(description="Runs the CMS unittest suite.") parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_UNITTEST_FILENAME) # Unused parameters. parser.add_argument("regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="unused") parser.add_argument("-l", "--languages", action="store", type=utf8_decoder, default="", help="unused") parser.add_argument("-c", "--contest", action="store", type=utf8_decoder, help="unused") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = True start_time = datetime.datetime.now() try: git_root = subprocess.check_output("git rev-parse --show-toplevel", shell=True, stderr=io.open(os.devnull, "wb")).strip() except subprocess.CalledProcessError: print("Please run the unit tests from the git repository.") return 1 if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() if args.dry_run: for t in test_list: print(t[0].name, t[1]) return 0 if args.retry_failed: info("Re-running %d failed tests from last run." % len(test_list)) # Load config from cms.conf. CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. info("Clearing old coverage data.") sh("python -m coverage erase") # Run all of our test cases. passed, test_results = run_unittests(test_list) combine_coverage() print(test_results) end_time = datetime.datetime.now() print(time_difference(start_time, end_time)) if passed: return 0 else: return 1
def main(): parser = argparse.ArgumentParser( description="Runs the CMS unittest suite.") parser.add_argument("regex", action="store", type=utf8_decoder, nargs='*', help="a regex to match to run a subset of tests") parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", default=0, help="print debug information (use multiple times for more)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_UNITTEST_FILENAME) # Unused parameters. parser.add_argument("-l", "--languages", action="store", type=utf8_decoder, default="", help="unused") parser.add_argument("-c", "--contest", action="store", type=utf8_decoder, help="unused") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = True start_time = datetime.datetime.now() try: git_root = subprocess.check_output( "git rev-parse --show-toplevel", shell=True, stderr=io.open(os.devnull, "wb")).decode('utf8').strip() except subprocess.CalledProcessError: print("Please run the unit tests from the git repository.") return 1 if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() if args.regex: # Require at least one regex to match to include it in the list. filter_regexps = [re.compile(regex) for regex in args.regex] def test_match(t): return any(r.search(t) is not None for r in filter_regexps) test_list = [t for t in test_list if test_match(' '.join(t))] if args.dry_run: for t in test_list: print(t[0], t[1]) return 0 if args.retry_failed: logger.info("Re-running %d failed tests from last run.", len(test_list)) # Load config from cms.conf. CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. logger.info("Clearing old coverage data.") sh([sys.executable, "-m", "coverage", "erase"]) # Run all of our test cases. passed, test_results = run_unittests(test_list) combine_coverage() print(test_results) end_time = datetime.datetime.now() print("Time elapsed: %s" % (end_time - start_time)) if passed: return 0 else: return 1
def main(): parser = ArgumentParser(description="Runs the CMS functional test suite.") parser.add_argument("regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="a regex to match to run a subset of tests") parser.add_argument("-l", "--languages", action="store", type=utf8_decoder, default="", help="a comma-separated list of languages to test") parser.add_argument("-c", "--contest", action="store", type=utf8_decoder, help="use an existing contest (and the tasks in it)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_TEST_FILENAME) parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = True # Pre-process our command-line arguments to figure out which tests to run. regexes = [re.compile(s) for s in args.regex] if args.languages: languages = frozenset(args.languages.split(',')) else: languages = frozenset() if args.retry_failed: test_list = load_failed_tests() else: test_list = ALL_TESTS test_list = filter_tests(test_list, regexes, languages) if not test_list: logger.info( "There are no tests to run! (was your filter too restrictive?)") return 0 tests = 0 for test in test_list: for language in test.languages: if args.dry_run: logger.info("Test %s in %s.", test.name, language) tests += 1 if test.user_tests: for language in test.languages: if args.dry_run: logger.info("Test %s in %s (for usertest).", test.name, language) tests += 1 if args.dry_run: return 0 if args.retry_failed: logger.info("Re-running %s failed tests from last run.", len(test_list)) # Clear out any old coverage data. logging.info("Clearing old coverage data.") sh(sys.executable + " -m coverage erase") # Startup the test runner. runner = TestRunner(test_list, contest_id=args.contest, workers=4) # Submit and wait for all tests to complete. runner.submit_tests() failures = runner.wait_for_evaluation() write_test_case_list([(test, lang) for test, lang, _ in failures], FAILED_TEST_FILENAME) # And good night! runner.shutdown() runner.log_elapsed_time() combine_coverage() logger.info("Executed: %s", tests) logger.info("Failed: %s", len(failures)) if not failures: logger.info("All tests passed!") return 0 else: logger.error("Some test failed!") logger.info("Run again with --retry-failed (or -r) to retry.") logger.info("Failed tests:") for test, lang, msg in failures: logger.info("%s (%s): %s\n", test.name, lang, msg) return 1
def main(): parser = argparse.ArgumentParser( description="Runs the CMS unittest suite.") parser.add_argument( "regex", action="store", type=utf8_decoder, nargs='*', help="a regex to match to run a subset of tests") parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", default=0, help="print debug information (use multiple times for more)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_UNITTEST_FILENAME) parser.add_argument( "--codecov", action="store_true", help="send coverage results to Codecov") # Unused parameters. parser.add_argument( "-l", "--languages", action="store", type=utf8_decoder, default="", help="unused") parser.add_argument( "-c", "--contest", action="store", type=utf8_decoder, help="unused") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = True start_time = datetime.datetime.now() try: git_root = subprocess.check_output( "git rev-parse --show-toplevel", shell=True, stderr=io.open(os.devnull, "wb")).decode('utf8').strip() except subprocess.CalledProcessError: print("Please run the unit tests from the git repository.") return 1 if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() if args.regex: # Require at least one regex to match to include it in the list. filter_regexps = [re.compile(regex) for regex in args.regex] def test_match(t): return any(r.search(t) is not None for r in filter_regexps) test_list = [t for t in test_list if test_match(' '.join(t))] if args.dry_run: for t in test_list: print(t[0], t[1]) return 0 if args.retry_failed: logger.info("Re-running %d failed tests from last run.", len(test_list)) # Load config from cms.conf. CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG clear_coverage() # Run all of our test cases. passed, test_results = run_unittests(test_list) combine_coverage() print(test_results) end_time = datetime.datetime.now() print("Time elapsed: %s" % (end_time - start_time)) if args.codecov: send_coverage_to_codecov("unittests") if passed: return 0 else: return 1