def main():
parser = argparse.ArgumentParser(
description="Runs the CMS functional test suite.")
parser.add_argument("regex",
action="store",
type=utf8_decoder,
nargs='*',
help="a regex to match to run a subset of tests")
parser.add_argument("-l",
"--languages",
action="store",
type=utf8_decoder,
default="",
help="a comma-separated list of languages to test")
parser.add_argument("-c",
"--contest",
action="store",
type=utf8_decoder,
help="use an existing contest (and the tasks in it)")
parser.add_argument(
"-r",
"--retry-failed",
action="store_true",
help="only run failed tests from the previous run (stored in %s)" %
FAILED_TEST_FILENAME)
parser.add_argument(
"-n",
"--dry-run",
action="store_true",
help="show what tests would be run, but do not run them")
parser.add_argument(
"-v",
"--verbose",
action="count",
default=0,
help="print debug information (use multiple times for more)")
parser.add_argument(
"--codecov",
action="store_true",
help="send coverage results to Codecov (requires --coverage)")
g = parser.add_mutually_exclusive_group()
g.add_argument("--coverage",
action="store_true",
help="compute line coverage information")
g.add_argument("--profiler",
choices=[PROFILER_YAPPI, PROFILER_KERNPROF],
default=PROFILER_NONE,
help="set profiler")
args = parser.parse_args()
if args.codecov and not args.coverage:
parser.error("--codecov requires --coverage")
CONFIG["VERBOSITY"] = args.verbose
CONFIG["COVERAGE"] = args.coverage
CONFIG["PROFILER"] = args.profiler
# Pre-process our command-line arguments to figure out which tests to run.
regexes = [re.compile(s) for s in args.regex]
if args.languages:
languages = frozenset(args.languages.split(','))
else:
languages = frozenset()
if args.retry_failed:
test_list = load_failed_tests()
else:
test_list = ALL_TESTS
test_list = filter_tests(test_list, regexes, languages)
if not test_list:
logger.info(
"There are no tests to run! (was your filter too restrictive?)")
return 0
tests = 0
for test in test_list:
for language in test.languages:
if args.dry_run:
logger.info("Test %s in %s.", test.name, language)
tests += 1
if test.user_tests:
for language in test.languages:
if args.dry_run:
logger.info("Test %s in %s (for usertest).", test.name,
language)
tests += 1
if args.dry_run:
return 0
if args.retry_failed:
logger.info("Re-running %s failed tests from last run.",
len(test_list))
clear_coverage()
# Startup the test runner.
runner = TestRunner(test_list, contest_id=args.contest, workers=4)
# Submit and wait for all tests to complete.
runner.submit_tests()
failures = runner.wait_for_evaluation()
write_test_case_list([(test, lang) for test, lang, _ in failures],
FAILED_TEST_FILENAME)
# And good night!
runner.shutdown()
runner.log_elapsed_time()
combine_coverage()
if args.codecov:
send_coverage_to_codecov("functionaltests")
logger.info("Executed: %s", tests)
logger.info("Failed: %s", len(failures))
if not failures:
logger.info("All tests passed!")
return 0
else:
logger.error("Some test failed!")
logger.info("Run again with --retry-failed (or -r) to retry.")
logger.info("Failed tests:")
for test, lang, msg in failures:
logger.info("%s (%s): %s\n", test.name, lang, msg)
return 1
def main():
parser = argparse.ArgumentParser(
description="Runs the CMS functional test suite.")
parser.add_argument(
"regex", action="store", type=utf8_decoder, nargs='*',
help="a regex to match to run a subset of tests")
parser.add_argument(
"-l", "--languages", action="store", type=utf8_decoder, default="",
help="a comma-separated list of languages to test")
parser.add_argument(
"-c", "--contest", action="store", type=utf8_decoder,
help="use an existing contest (and the tasks in it)")
parser.add_argument(
"-r", "--retry-failed", action="store_true",
help="only run failed tests from the previous run (stored in %s)" %
FAILED_TEST_FILENAME)
parser.add_argument(
"-n", "--dry-run", action="store_true",
help="show what tests would be run, but do not run them")
parser.add_argument(
"-v", "--verbose", action="count", default=0,
help="print debug information (use multiple times for more)")
parser.add_argument(
"--codecov", action="store_true",
help="send coverage results to Codecov (requires --coverage)")
g = parser.add_mutually_exclusive_group()
g.add_argument(
"--coverage", action="store_true",
help="compute line coverage information")
g.add_argument(
"--profiler", choices=[PROFILER_YAPPI, PROFILER_KERNPROF],
default=PROFILER_NONE, help="set profiler")
args = parser.parse_args()
if args.codecov and not args.coverage:
parser.error("--codecov requires --coverage")
CONFIG["VERBOSITY"] = args.verbose
CONFIG["COVERAGE"] = args.coverage
CONFIG["PROFILER"] = args.profiler
# Pre-process our command-line arguments to figure out which tests to run.
regexes = [re.compile(s) for s in args.regex]
if args.languages:
languages = frozenset(args.languages.split(','))
else:
languages = frozenset()
if args.retry_failed:
test_list = load_failed_tests()
else:
test_list = ALL_TESTS
test_list = filter_tests(test_list, regexes, languages)
if not test_list:
logger.info(
"There are no tests to run! (was your filter too restrictive?)")
return 0
tests = 0
for test in test_list:
for language in test.languages:
if args.dry_run:
logger.info("Test %s in %s.", test.name, language)
tests += 1
if test.user_tests:
for language in test.languages:
if args.dry_run:
logger.info("Test %s in %s (for usertest).",
test.name, language)
tests += 1
if args.dry_run:
return 0
if args.retry_failed:
logger.info(
"Re-running %s failed tests from last run.", len(test_list))
clear_coverage()
# Startup the test runner.
runner = TestRunner(test_list, contest_id=args.contest, workers=4)
# Submit and wait for all tests to complete.
runner.submit_tests()
failures = runner.wait_for_evaluation()
write_test_case_list(
[(test, lang) for test, lang, _ in failures],
FAILED_TEST_FILENAME)
# And good night!
runner.shutdown()
runner.log_elapsed_time()
combine_coverage()
if args.codecov:
send_coverage_to_codecov("functionaltests")
logger.info("Executed: %s", tests)
logger.info("Failed: %s", len(failures))
if not failures:
logger.info("All tests passed!")
return 0
else:
logger.error("Some test failed!")
logger.info("Run again with --retry-failed (or -r) to retry.")
logger.info("Failed tests:")
for test, lang, msg in failures:
logger.info("%s (%s): %s\n", test.name, lang, msg)
return 1
def main():
parser = argparse.ArgumentParser(
description="Runs the CMS unittest suite.")
parser.add_argument(
"regex", action="store", type=utf8_decoder, nargs='*',
help="a regex to match to run a subset of tests")
parser.add_argument(
"-n", "--dry-run", action="store_true",
help="show what tests would be run, but do not run them")
parser.add_argument(
"-v", "--verbose", action="count", default=0,
help="print debug information (use multiple times for more)")
parser.add_argument(
"-r", "--retry-failed", action="store_true",
help="only run failed tests from the previous run (stored in %s)" %
FAILED_UNITTEST_FILENAME)
parser.add_argument(
"--codecov", action="store_true",
help="send coverage results to Codecov (requires --coverage)")
g = parser.add_mutually_exclusive_group()
g.add_argument(
"--coverage", action="store_true",
help="compute line coverage information")
g.add_argument(
"--profiler", choices=[PROFILER_YAPPI, PROFILER_KERNPROF],
default=PROFILER_NONE, help="set profiler")
# Unused parameters.
parser.add_argument(
"-l", "--languages", action="store", type=utf8_decoder, default="",
help="unused")
parser.add_argument(
"-c", "--contest", action="store", type=utf8_decoder,
help="unused")
args = parser.parse_args()
if args.codecov and not args.coverage:
parser.error("--codecov requires --coverage")
CONFIG["VERBOSITY"] = args.verbose
CONFIG["COVERAGE"] = args.coverage
CONFIG["PROFILER"] = args.profiler
start_time = datetime.datetime.now()
try:
git_root = subprocess.check_output(
"git rev-parse --show-toplevel", shell=True,
stderr=io.open(os.devnull, "wb")).decode('utf8').strip()
except subprocess.CalledProcessError:
print("Please run the unit tests from the git repository.")
return 1
if args.retry_failed:
test_list = load_failed_tests()
else:
test_list = get_all_tests()
if args.regex:
# Require at least one regex to match to include it in the list.
filter_regexps = [re.compile(regex) for regex in args.regex]
def test_match(t):
return any(r.search(t) is not None for r in filter_regexps)
test_list = [t for t in test_list if test_match(' '.join(t))]
if args.dry_run:
for t in test_list:
print(t[0], t[1])
return 0
if args.retry_failed:
logger.info("Re-running %d failed tests from last run.",
len(test_list))
# Load config from cms.conf.
CONFIG["TEST_DIR"] = git_root
CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"]
if CONFIG["TEST_DIR"] is None:
CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf"
if CONFIG["TEST_DIR"] is not None:
# Set up our expected environment.
os.chdir("%(TEST_DIR)s" % CONFIG)
os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG
clear_coverage()
# Run all of our test cases.
passed, test_results = run_unittests(test_list)
combine_coverage()
print(test_results)
end_time = datetime.datetime.now()
print("Time elapsed: %s" % (end_time - start_time))
if args.codecov:
send_coverage_to_codecov("unittests")
if passed:
return 0
else:
return 1
def main():
parser = argparse.ArgumentParser(
description="Runs the CMS unittest suite.")
parser.add_argument(
"regex", action="store", type=utf8_decoder, nargs='*',
help="a regex to match to run a subset of tests")
parser.add_argument(
"-n", "--dry-run", action="store_true",
help="show what tests would be run, but do not run them")
parser.add_argument(
"-v", "--verbose", action="count", default=0,
help="print debug information (use multiple times for more)")
parser.add_argument(
"-r", "--retry-failed", action="store_true",
help="only run failed tests from the previous run (stored in %s)" %
FAILED_UNITTEST_FILENAME)
parser.add_argument(
"--codecov", action="store_true",
help="send coverage results to Codecov (requires --coverage)")
g = parser.add_mutually_exclusive_group()
g.add_argument(
"--coverage", action="store_true",
help="compute line coverage information")
g.add_argument(
"--profiler", choices=[PROFILER_YAPPI, PROFILER_KERNPROF],
default=PROFILER_NONE, help="set profiler")
# Unused parameters.
parser.add_argument(
"-l", "--languages", action="store", type=utf8_decoder, default="",
help="unused")
parser.add_argument(
"-c", "--contest", action="store", type=utf8_decoder,
help="unused")
args = parser.parse_args()
if args.codecov and not args.coverage:
parser.error("--codecov requires --coverage")
CONFIG["VERBOSITY"] = args.verbose
CONFIG["COVERAGE"] = args.coverage
CONFIG["PROFILER"] = args.profiler
start_time = datetime.datetime.now()
try:
git_root = subprocess.check_output(
"git rev-parse --show-toplevel", shell=True,
stderr=subprocess.DEVNULL).decode('utf8').strip()
except subprocess.CalledProcessError:
print("Please run the unit tests from the git repository.")
return 1
if args.retry_failed:
test_list = load_failed_tests()
else:
test_list = get_all_tests()
if args.regex:
# Require at least one regex to match to include it in the list.
filter_regexps = [re.compile(regex) for regex in args.regex]
def test_match(t):
return any(r.search(t) is not None for r in filter_regexps)
test_list = [t for t in test_list if test_match(' '.join(t))]
if args.dry_run:
for t in test_list:
print(t[0], t[1])
return 0
if args.retry_failed:
logger.info("Re-running %d failed tests from last run.",
len(test_list))
# Load config from cms.conf.
CONFIG["TEST_DIR"] = git_root
CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"]
if CONFIG["TEST_DIR"] is None:
CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf"
if CONFIG["TEST_DIR"] is not None:
# Set up our expected environment.
os.chdir("%(TEST_DIR)s" % CONFIG)
os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG
clear_coverage()
# Run all of our test cases.
passed, test_results = run_unittests(test_list)
combine_coverage()
print(test_results)
end_time = datetime.datetime.now()
print("Time elapsed: %s" % (end_time - start_time))
if args.codecov:
send_coverage_to_codecov("unittests")
if passed:
return 0
else:
return 1