def _cleanup(): try: # Clean up tree. info("Cleaning up test directory %(TEST_DIR)s" % CONFIG) shutil.rmtree("%(TEST_DIR)s" % CONFIG) except: pass
def get_task_id(contest_id, user_id, task_module): # Create a task in the contest if we haven't already. if task_module not in task_id_map: # add the task itself. task_id = add_task( contest_id=contest_id, **task_module.task_info) # add the task's test data. data_path = os.path.join( os.path.dirname(task_module.__file__), "data") for input_file, output_file, public in task_module.test_cases: ipath = os.path.join(data_path, input_file) opath = os.path.join(data_path, output_file) add_testcase(task_id, ipath, opath, public) task_id_map[task_module] = task_id info("Creating task %s as id %d" % ( task_module.task_info['name'], task_id)) # We need to restart ScoringService to ensure it has picked up the # new task. restart_service("ScoringService", contest=contest_id) else: task_id = task_id_map[task_module] return task_id
def create_or_get_user(contest_id): global num_users num_users += 1 def enumerify(x): if 11 <= x <= 13: return 'th' return {1: 'st', 2: 'nd', 3: 'rd'}.get(x % 10, 'th') username = "******" % num_users # Find a user that may already exist (from a previous contest). users = get_users(contest_id) user_create_args = { "username": username, "password": "******", "first_name": "Ms. Test", "last_name": "Wabbit the %d%s" % (num_users, enumerify(num_users)), } if username in users: user_id = users[username]['id'] add_existing_user(contest_id, user_id, **user_create_args) info("Using existing user with id %d." % user_id) else: user_id = add_user(contest_id, **user_create_args) info("Created user with id %d." % user_id) return user_id
def setup_cms(): info("Creating database called %(DB_NAME)s accessible by %(DB_USER)s." % CONFIG) sh("sudo -u postgres createdb %(DB_NAME)s -O %(DB_USER)s" % CONFIG) info("Checking out code.") sh("git clone %(GIT_ORIGIN)s %(TEST_DIR)s" % CONFIG) os.chdir("%(TEST_DIR)s/cms" % CONFIG) sh("git checkout %(GIT_REVISION)s" % CONFIG) info("Configuring CMS.") configure_cms( { "database": '"postgresql+psycopg2://' "%(DB_USER)s:%(DB_PASSWORD)s@" '%(DB_HOST)s/%(DB_NAME)s"' % CONFIG, "keep_sandbox": "false", "contest_listen_address": '["127.0.0.1"]', "admin_listen_address": '"127.0.0.1"', "min_submission_interval": "0", } ) info("Setting environment.") os.environ["PYTHONPATH"] = "%(TEST_DIR)s/cms" % CONFIG info("Creating tables.") sh("python db/SQLAlchemyAll.py")
def setup_cms(): info("Creating database called %(DB_NAME)s accessible by %(DB_USER)s." % CONFIG) sh("sudo -u postgres createdb %(DB_NAME)s -O %(DB_USER)s" % CONFIG) info("Checking out code.") sh("git clone %(GIT_ORIGIN)s %(TEST_DIR)s" % CONFIG) os.chdir("%(TEST_DIR)s/cms" % CONFIG) sh("git checkout %(GIT_REVISION)s" % CONFIG) info("Configuring CMS.") configure_cms( { "database": '"postgresql+psycopg2://' \ '%(DB_USER)s:%(DB_PASSWORD)s@' \ '%(DB_HOST)s/%(DB_NAME)s"' % CONFIG, "keep_sandbox": "false", "contest_listen_address": '["127.0.0.1"]', "admin_listen_address": '"127.0.0.1"', "min_submission_interval": '0', }) info("Setting environment.") os.environ["PYTHONPATH"] = "%(TEST_DIR)s/cms" % CONFIG info("Creating tables.") sh("python db/SQLAlchemyAll.py")
def create_a_user(contest_id): global num_users num_users += 1 def enumerify(x): if 11 <= x <= 13: return 'th' return {1: 'st', 2: 'nd', 3: 'rd'}.get(x % 10, 'th') info("Creating user.") user_id = add_user(contest_id=contest_id, username="******" % num_users, password="******", first_name="Ms. Test", last_name="Wabbit the %d%s" % (num_users, enumerify(num_users))) return user_id
def create_a_user(contest_id): global num_users num_users += 1 def enumerify(x): if 11 <= x <= 13: return 'th' return {1: 'st', 2: 'nd', 3: 'rd'}.get(x % 10, 'th') info("Creating user.") user_id = add_user( contest_id=contest_id, username="******" % num_users, password="******", first_name="Ms. Test", last_name="Wabbit the %d%s" % (num_users, enumerify(num_users))) return user_id
def create_contest(): start_time = datetime.datetime.utcnow() stop_time = start_time + datetime.timedelta(1, 0, 0) contest_id = add_contest( name="testcontest1", description="A test contest #1.", start=start_time.strftime("%Y-%m-%d %H:%M:%S.%f"), stop=stop_time.strftime("%Y-%m-%d %H:%M:%S.%f"), timezone=get_system_timezone(), token_initial="100", token_max="100", token_total="100", token_min_interval="0", token_gen_time="0", token_gen_number="0", ) info("Created contest %d." % contest_id) return contest_id
def get_task_id(contest_id, user_id, task_module): # Create a task in the contest if we haven't already. if task_module not in task_id_map: # add the task itself. task_id = add_task( contest_id=contest_id, token_initial="100", token_max="100", token_total="100", token_min_interval="0", token_gen_time="0", token_gen_number="0", max_submission_number="100", max_user_test_number="100", min_submission_interval=None, min_user_test_interval=None, **task_module.task_info) # add the task's test data. data_path = os.path.join( os.path.dirname(task_module.__file__), "data") for num, (input_file, output_file, public) \ in enumerate(task_module.test_cases): ipath = os.path.join(data_path, input_file) opath = os.path.join(data_path, output_file) add_testcase(task_id, str(num), ipath, opath, public) task_id_map[task_module] = task_id info("Creating task %s as id %d" % ( task_module.task_info['name'], task_id)) # We need to restart ScoringService to ensure it has picked up the # new task. restart_service("ScoringService", contest=contest_id) else: task_id = task_id_map[task_module] return task_id
def create_contest(): start_time = datetime.datetime.utcnow() stop_time = start_time + datetime.timedelta(1, 0, 0) contest_id = add_contest( name="testcontest" + str(rand), description="A test contest #%s." % rand, languages=LANGUAGES, start=start_time.strftime("%Y-%m-%d %H:%M:%S.%f"), stop=stop_time.strftime("%Y-%m-%d %H:%M:%S.%f"), timezone=get_system_timezone(), token_mode="finite", token_max_number="100", token_min_interval="0", token_gen_initial="100", token_gen_number="0", token_gen_interval="1", token_gen_max="100", ) info("Created contest %d." % contest_id) return contest_id
def create_contest(): info("Creating contest.") start_time = datetime.datetime.now() end_time = start_time + datetime.timedelta(1, 0, 0) contest_id = add_contest( name="testcontest1", description="A test contest #1.", start=start_time.strftime("%d/%m/%Y %H:%M:%S"), end=end_time.strftime("%d/%m/%Y %H:%M:%S"), token_initial="100", #token_max="", #token_total="", #token_min_interval="", #token_gen_time="", #token_gen_number="", ) start_service("ScoringService", contest=contest_id) start_service("EvaluationService", contest=contest_id) start_server("ContestWebServer", contest=contest_id) return contest_id
def create_contest(): info("Creating contest.") start_time = datetime.datetime.utcnow() stop_time = start_time + datetime.timedelta(1, 0, 0) contest_id = add_contest( name="testcontest1", description="A test contest #1.", start=start_time.strftime("%Y-%m-%d %H:%M:%S.%f"), stop=stop_time.strftime("%Y-%m-%d %H:%M:%S.%f"), timezone=get_system_timezone(), token_initial="100", token_max="100", token_total="100", token_min_interval="0", token_gen_time="0", token_gen_number="0", ) start_service("ScoringService", contest=contest_id) start_service("EvaluationService", contest=contest_id) start_server("ContestWebServer", contest=contest_id) return contest_id
def get_task_id(contest_id, user_id, task_module): # Create a task in the contest if we haven't already. if task_module not in task_id_map: # add the task itself. task_id = add_task(contest_id=contest_id, token_initial="100", token_max="100", token_total="100", token_min_interval="0", token_gen_time="0", token_gen_number="0", max_submission_number="100", max_usertest_number="100", min_submission_interval="0", min_usertest_interval="0", **task_module.task_info) # add the task's test data. data_path = os.path.join(os.path.dirname(task_module.__file__), "data") for input_file, output_file, public in task_module.test_cases: ipath = os.path.join(data_path, input_file) opath = os.path.join(data_path, output_file) add_testcase(task_id, ipath, opath, public) task_id_map[task_module] = task_id info("Creating task %s as id %d" % (task_module.task_info['name'], task_id)) # We need to restart ScoringService to ensure it has picked up the # new task. restart_service("ScoringService", contest=contest_id) else: task_id = task_id_map[task_module] return task_id
def run_testcases(contest_id, user_id, test_list): """Run all test cases specified by the Tests module. contest_id and user_id must specify an already-created contest and user under which the tests are submitted. test_list should be a list of 2-tuples, each representing a test. The first element of each tuple is a Test object, and the second is the language for which it should be executed. """ info("Running test cases ...") failures = [] num_tests_to_execute = len(test_list) # For all tests... for i, (test, lang) in enumerate(test_list): # This installs the task into the contest if we haven't already. task_id = get_task_id(contest_id, user_id, test.task_module) info("Running test %d/%d: %s (%s)" % ( i + 1, num_tests_to_execute, test.name, lang)) try: test.run(contest_id, task_id, user_id, lang) except TestFailure as f: info(" (FAILED: %s)" % f.message) # Add this case to our list of failures, if we haven't already. failures.append((test, lang, f.message)) results = "\n\n" if not failures: results += "================== ALL TESTS PASSED! ==================\n" else: results += "------ TESTS FAILED: ------\n" results += " Executed: %d\n" % num_tests_to_execute results += " Failed: %d\n" % len(failures) results += "\n" for test, lang, msg in failures: results += " %s (%s): %s\n" % (test.name, lang, msg) if failures: write_test_case_list( [(test, lang) for test, lang, _ in failures], FAILED_TEST_FILENAME) results += "\n" results += "Failed tests stored in %s.\n" % FAILED_TEST_FILENAME results += "Run again with --retry-failed (or -r) to retry.\n" return results
def run_testcases(contest_id, user_id, test_list): """Run all test cases specified by the Tests module. contest_id and user_id must specify an already-created contest and user under which the tests are submitted. test_list should be a list of 2-tuples, each representing a test. The first element of each tuple is a Test object, and the second is the language for which it should be executed. """ info("Running test cases ...") failures = [] num_tests_to_execute = len(test_list) # For all tests... for i, (test, lang) in enumerate(test_list): # This installs the task into the contest if we haven't already. task_id = get_task_id(contest_id, user_id, test.task_module) info("Running test %d/%d: %s (%s)" % ( i + 1, num_tests_to_execute, test.name, lang)) try: test.run(contest_id, task_id, user_id, lang) except TestFailure as f: info(" (FAILED: %s)" % f.message) # Add this case to our list of failures, if we haven't already. failures.append((test, lang, f.message)) results = "\n\n" if not failures: results += "================== ALL TESTS PASSED! ==================\n" else: results += "------ TESTS FAILED: ------\n" results += " Executed: %d\n" % num_tests_to_execute results += " Failed: %d\n" % len(failures) results += "\n" for test, lang, msg in failures: results += " %s (%s): %s\n" % (test.name, lang, msg) if failures: write_test_case_list( [(test, lang) for test, lang, _ in failures], FAILED_TEST_FILENAME) results += "\n" results += "Failed tests stored in %s.\n" % FAILED_TEST_FILENAME results += "Run again with --retry-failed (or -r) to retry.\n" return len(failures) == 0, results
def run_unittests(test_list): """Run all needed unit tests. test_list ([(string, string)]): a list of test to run in the format (path, filename.py). return (int): """ info("Running unit tests...") failures = [] num_tests_to_execute = len(test_list) # For all tests... for i, (path, filename) in enumerate(test_list): info("Running test %d/%d: %s.%s" % ( i + 1, num_tests_to_execute, path, filename)) try: sh('python -m coverage run -p --source=cms %s' % os.path.join(path, filename)) except FrameworkException: info(" (FAILED: %s)" % filename) # Add this case to our list of failures, if we haven't already. failures.append((path, filename)) results = "\n\n" if not failures: results += "================== ALL TESTS PASSED! ==================\n" else: results += "------ TESTS FAILED: ------\n" results += " Executed: %d\n" % num_tests_to_execute results += " Failed: %d\n" % len(failures) results += "\n" for path, filename in failures: results += " %s.%s\n" % (path, filename) if failures: with io.open(FAILED_UNITTEST_FILENAME, "wt", encoding="utf-8") as failed_filename: for path, filename in failures: failed_filename.write("%s %s\n" % (path, filename)) results += "\n" results += "Failed tests stored in %s.\n" % FAILED_UNITTEST_FILENAME results += "Run again with --retry-failed (or -r) to retry.\n" return len(failures) == 0, results
def run_unittests(test_list): """Run all needed unit tests. test_list ([(string, string)]): a list of test to run in the format (path, filename.py). return (int): """ info("Running unit tests...") failures = [] num_tests_to_execute = len(test_list) # For all tests... for i, (path, filename) in enumerate(test_list): info("Running test %d/%d: %s.%s" % ( i + 1, num_tests_to_execute, path, filename)) try: sh('python-coverage run -p --source=cms %s' % os.path.join(path, filename)) except FrameworkException: info(" (FAILED: %s)" % filename) # Add this case to our list of failures, if we haven't already. failures.append((path, filename)) results = "\n\n" if not failures: results += "================== ALL TESTS PASSED! ==================\n" else: results += "------ TESTS FAILED: ------\n" results += " Executed: %d\n" % num_tests_to_execute results += " Failed: %d\n" % len(failures) results += "\n" for path, filename in failures: results += " %s.%s\n" % (path, filename) if failures: with io.open(FAILED_UNITTEST_FILENAME, "wt", encoding="utf-8") as failed_filename: for path, filename in failures: failed_filename.write("%s %s\n" % (path, filename)) results += "\n" results += "Failed tests stored in %s.\n" % FAILED_UNITTEST_FILENAME results += "Run again with --retry-failed (or -r) to retry.\n" return len(failures) == 0, results
def setup_cms(): info("Creating database called %(DB_NAME)s accessible by %(DB_USER)s." % CONFIG) sh("sudo -u postgres createdb %(DB_NAME)s -O %(DB_USER)s" % CONFIG) info("Checking out code.") sh(["git", "clone", "--recursive", CONFIG["GIT_ORIGIN"], CONFIG["TEST_DIR"]]) os.chdir("%(TEST_DIR)s" % CONFIG) sh(["git", "checkout", CONFIG["GIT_REVISION"]]) info("Configuring CMS.") configure_cms( {"database": '"postgresql+psycopg2://' '%(DB_USER)s:%(DB_PASSWORD)s@' '%(DB_HOST)s/%(DB_NAME)s"' % CONFIG, "keep_sandbox": "false", "contest_listen_address": '["127.0.0.1"]', "admin_listen_address": '"127.0.0.1"', "min_submission_interval": '0', }) info("Setting environment.") os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG info("Building cms.") sh("./setup.py build") # Add permission bits to isolate. sh("sudo chown root:root isolate/isolate") sh("sudo chmod 4755 isolate/isolate") # Ensure our logs get preserved. Point them into the checkout instead of # the tempdir that we blow away. sh(["mkdir", "-p", "%(GIT_ORIGIN)s/log" % CONFIG]) sh(["ln", "-s", "%(GIT_ORIGIN)s/log" % CONFIG, "log"]) info("Creating tables.") sh("python scripts/cmsInitDB")
def main(): parser = ArgumentParser(description="Runs the CMS functional test suite.") parser.add_argument("regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="a regex to match to run a subset of tests") parser.add_argument("-l", "--languages", action="store", type=utf8_decoder, default="", help="a comma-separated list of languages to test") parser.add_argument("-c", "--contest", action="store", type=utf8_decoder, help="use an existing contest (and the tasks in it)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_TEST_FILENAME) parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose start_time = datetime.datetime.now() # Pre-process our command-line arugments to figure out which tests to run. regexes = [re.compile(s) for s in args.regex] if args.languages: languages = frozenset(args.languages.split(',')) else: languages = frozenset() if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() test_list = filter_testcases(test_list, regexes, languages) if not test_list: info("There are no tests to run! (was your filter too restrictive?)") return 0 if args.dry_run: for t in test_list: print(t[0].name, t[1]) return 0 if args.retry_failed: info("Re-running %d failed tests from last run." % len(test_list)) # Load config from cms.conf. try: git_root = subprocess.check_output("git rev-parse --show-toplevel", shell=True, stderr=io.open(os.devnull, "wb")).strip() except subprocess.CalledProcessError: git_root = None CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" cms_config = get_cms_config() if not config_is_usable(cms_config): return 1 if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. info("Clearing old coverage data.") sh("python-coverage erase") # Fire us up! start_generic_services() if args.contest is None: contest_id = create_contest() else: contest_id = int(args.contest) user_id = create_or_get_user(contest_id) start_contest(contest_id) # Run all of our test cases. passed, test_results = run_testcases(contest_id, user_id, test_list) # And good night! shutdown_services() combine_coverage() print(test_results) end_time = datetime.datetime.now() print(time_difference(start_time, end_time)) if passed: return 0 else: return 1
def get_task_id(contest_id, user_id, task_module): name = task_module.task_info['name'] # Have we done this before? Pull it out of our cache if so. if task_module in task_id_map: # Ensure we don't have multiple modules with the same task name. assert task_id_map[task_module][1] == task_module return task_id_map[name][0] task_create_args = { "token_mode": "finite", "token_max_number": "100", "token_min_interval": "0", "token_gen_initial": "100", "token_gen_number": "0", "token_gen_interval": "1", "token_gen_max": "100", "max_submission_number": "100", "max_user_test_number": "100", "min_submission_interval": None, "min_user_test_interval": None, } task_create_args.update(task_module.task_info) # Find if the task already exists in the contest. tasks = get_tasks(contest_id) if name in tasks: # Then just use the existing one. task = tasks[name] task_id = task['id'] task_id_map[name] = (task_id, task_module) add_existing_task(contest_id, task_id, **task_create_args) return task_id # Otherwise, we need to add the task ourselves. task_id = add_task(contest_id, **task_create_args) # add any managers code_path = os.path.join(os.path.dirname(task_module.__file__), "code") if hasattr(task_module, 'managers'): for manager in task_module.managers: mpath = os.path.join(code_path, manager) add_manager(task_id, mpath) # add the task's test data. data_path = os.path.join(os.path.dirname(task_module.__file__), "data") for num, (input_file, output_file, public) \ in enumerate(task_module.test_cases): ipath = os.path.join(data_path, input_file) opath = os.path.join(data_path, output_file) add_testcase(task_id, num, ipath, opath, public) task_id_map[name] = (task_id, task_module) info("Created task %s as id %d" % (name, task_id)) # We need to restart ProxyService to ensure it reinitializes, # picking up the new task and sending it to RWS. restart_service("ProxyService", contest=contest_id) return task_id
def setup_cms(): info("Creating database called %(DB_NAME)s accessible by %(DB_USER)s." % CONFIG) sh("sudo -u postgres createdb %(DB_NAME)s -O %(DB_USER)s" % CONFIG) info("Checking out code.") sh([ "git", "clone", "--recursive", CONFIG["GIT_ORIGIN"], CONFIG["TEST_DIR"] ]) os.chdir("%(TEST_DIR)s" % CONFIG) sh(["git", "checkout", CONFIG["GIT_REVISION"]]) info("Configuring CMS.") configure_cms({ "database": '"postgresql+psycopg2://' '%(DB_USER)s:%(DB_PASSWORD)s@' '%(DB_HOST)s/%(DB_NAME)s"' % CONFIG, "keep_sandbox": "false", "contest_listen_address": '["127.0.0.1"]', "admin_listen_address": '"127.0.0.1"', "min_submission_interval": '0', }) info("Setting environment.") os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG info("Building cms.") sh("./setup.py build") # Add permission bits to isolate. sh("sudo chown root:root isolate/isolate") sh("sudo chmod 4755 isolate/isolate") # Ensure our logs get preserved. Point them into the checkout instead of # the tempdir that we blow away. sh(["mkdir", "-p", "%(GIT_ORIGIN)s/log" % CONFIG]) sh(["ln", "-s", "%(GIT_ORIGIN)s/log" % CONFIG, "log"]) info("Creating tables.") sh("python scripts/cmsInitDB")
def drop_old_data(): info("Dropping any old databases called %(DB_NAME)s." % CONFIG) sh("sudo -u postgres dropdb %(DB_NAME)s" % CONFIG, ignore_failure=True) info("Purging old checkout from %(TEST_DIR)s." % CONFIG) shutil.rmtree("%(TEST_DIR)s" % CONFIG)
else: CONFIG["GIT_REVISION"] = args.revision if not args.keep_working: def _cleanup(): try: # Clean up tree. info("Cleaning up test directory %(TEST_DIR)s" % CONFIG) shutil.rmtree("%(TEST_DIR)s" % CONFIG) except: pass atexit.register(_cleanup) info("Testing `%(GIT_REVISION)s' in %(TEST_DIR)s" % CONFIG) reinitialize_everything = True if reinitialize_everything: drop_old_data() setup_cms() else: os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG read_cms_config() # Now run the tests from the checkout. sh(["./cmstestsuite/RunTests.py"] + args.arguments) # We export the contest, import it again and re-run the tests on the
def main(): parser = ArgumentParser(description="Runs the CMS unittest suite.") parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_UNITTEST_FILENAME) # Unused parameters. parser.add_argument( "regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="unused") parser.add_argument( "-l", "--languages", action="store", type=utf8_decoder, default="", help="unused") parser.add_argument( "-c", "--contest", action="store", type=utf8_decoder, help="unused") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose start_time = datetime.datetime.now() try: git_root = subprocess.check_output( "git rev-parse --show-toplevel", shell=True, stderr=io.open(os.devnull, "wb")).strip() except subprocess.CalledProcessError: print("Please run the unit tests from the git repository.") return 1 if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() if args.dry_run: for t in test_list: print(t[0].name, t[1]) return 0 if args.retry_failed: info("Re-running %d failed tests from last run." % len(test_list)) # Load config from cms.conf. CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. info("Clearing old coverage data.") sh("python -m coverage erase") # Run all of our test cases. passed, test_results = run_unittests(test_list) combine_coverage() print(test_results) end_time = datetime.datetime.now() print(time_difference(start_time, end_time)) if passed: return 0 else: return 1
CONFIG["GIT_REVISION"] = \ subprocess.check_output("git rev-parse HEAD", shell=True).strip() else: CONFIG["GIT_REVISION"] = args.revision if not args.keep_working: def _cleanup(): try: # Clean up tree. info("Cleaning up test directory %(TEST_DIR)s" % CONFIG) shutil.rmtree("%(TEST_DIR)s" % CONFIG) except: pass atexit.register(_cleanup) info("Testing `%(GIT_REVISION)s' in %(TEST_DIR)s" % CONFIG) reinitialize_everything = True if reinitialize_everything: drop_old_data() setup_cms() else: os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG read_cms_config() # Now run the tests from the checkout. sh(["./cmstestsuite/RunTests.py"] + args.arguments) # We export the contest, import it again and re-run the tests on the
def main(): parser = ArgumentParser(description="Runs the CMS unittest suite.") parser.add_argument( "-n", "--dry-run", action="store_true", help="show what tests would be run, but do not run them") parser.add_argument( "-v", "--verbose", action="count", help="print debug information (use multiple times for more)") parser.add_argument( "-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_UNITTEST_FILENAME) # Unused parameters. parser.add_argument("regex", action="store", type=utf8_decoder, nargs='*', metavar="regex", help="unused") parser.add_argument("-l", "--languages", action="store", type=utf8_decoder, default="", help="unused") parser.add_argument("-c", "--contest", action="store", type=utf8_decoder, help="unused") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose CONFIG["COVERAGE"] = True start_time = datetime.datetime.now() try: git_root = subprocess.check_output("git rev-parse --show-toplevel", shell=True, stderr=io.open(os.devnull, "wb")).strip() except subprocess.CalledProcessError: print("Please run the unit tests from the git repository.") return 1 if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() if args.dry_run: for t in test_list: print(t[0].name, t[1]) return 0 if args.retry_failed: info("Re-running %d failed tests from last run." % len(test_list)) # Load config from cms.conf. CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/config/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. info("Clearing old coverage data.") sh("python -m coverage erase") # Run all of our test cases. passed, test_results = run_unittests(test_list) combine_coverage() print(test_results) end_time = datetime.datetime.now() print(time_difference(start_time, end_time)) if passed: return 0 else: return 1
def main(): parser = ArgumentParser(description="Runs the CMS test suite.") parser.add_argument("regex", metavar="regex", type=str, nargs='*', help="a regex to match to run a subset of tests") parser.add_argument("-l", "--languages", type=str, action="store", default="", help="a comma-separated list of languages to test") parser.add_argument("-r", "--retry-failed", action="store_true", help="only run failed tests from the previous run (stored in %s)" % FAILED_TEST_FILENAME) parser.add_argument("-v", "--verbose", action="count", help="print debug information (use multiple times for more)") args = parser.parse_args() CONFIG["VERBOSITY"] = args.verbose start_time = datetime.datetime.now() # Pre-process our command-line arugments to figure out which tests to run. regexes = [re.compile(s) for s in args.regex] if args.languages: languages = frozenset(args.languages.split(',')) else: languages = frozenset() if args.retry_failed: test_list = load_failed_tests() else: test_list = get_all_tests() test_list = filter_testcases(test_list, regexes, languages) if not test_list: info("There are no tests to run! (was your filter too restrictive?)") return 0 if args.retry_failed: info("Re-running %d failed tests from last run." % len(test_list)) # Load config from cms.conf. try: git_root = subprocess.check_output( "git rev-parse --show-toplevel", shell=True, stderr=open(os.devnull, "w")).strip() except subprocess.CalledProcessError: git_root = None CONFIG["TEST_DIR"] = git_root CONFIG["CONFIG_PATH"] = "%s/examples/cms.conf" % CONFIG["TEST_DIR"] if CONFIG["TEST_DIR"] is None: CONFIG["CONFIG_PATH"] = "/usr/local/etc/cms.conf" cms_config = get_cms_config() if not config_is_usable(cms_config): return 1 if CONFIG["TEST_DIR"] is not None: # Set up our expected environment. os.chdir("%(TEST_DIR)s" % CONFIG) os.environ["PYTHONPATH"] = "%(TEST_DIR)s" % CONFIG # Clear out any old coverage data. info("Clearing old coverage data.") sh("python-coverage erase") # Fire us up! start_generic_services() contest_id = create_contest() user_id = create_a_user(contest_id) # Run all of our test cases. test_results = run_testcases(contest_id, user_id, test_list) # And good night! shutdown_services() combine_coverage() print test_results end_time = datetime.datetime.now() print time_difference(start_time, end_time)
def get_task_id(contest_id, user_id, task_module): name = task_module.task_info['name'] # Have we done this before? Pull it out of our cache if so. if task_module in task_id_map: # Ensure we don't have multiple modules with the same task name. assert task_id_map[task_module][1] == task_module return task_id_map[name][0] task_create_args = { "token_mode": "finite", "token_max_number": "100", "token_min_interval": "0", "token_gen_initial": "100", "token_gen_number": "0", "token_gen_interval": "1", "token_gen_max": "100", "max_submission_number": "100", "max_user_test_number": "100", "min_submission_interval": None, "min_user_test_interval": None, } task_create_args.update(task_module.task_info) # Find if the task already exists in the contest. tasks = get_tasks(contest_id) if name in tasks: # Then just use the existing one. task = tasks[name] task_id = task['id'] task_id_map[name] = (task_id, task_module) add_existing_task(contest_id, task_id, **task_create_args) return task_id # Otherwise, we need to add the task ourselves. task_id = add_task(contest_id, **task_create_args) # add any managers code_path = os.path.join( os.path.dirname(task_module.__file__), "code") if hasattr(task_module, 'managers'): for manager in task_module.managers: mpath = os.path.join(code_path, manager) add_manager(task_id, mpath) # add the task's test data. data_path = os.path.join( os.path.dirname(task_module.__file__), "data") for num, (input_file, output_file, public) \ in enumerate(task_module.test_cases): ipath = os.path.join(data_path, input_file) opath = os.path.join(data_path, output_file) add_testcase(task_id, num, ipath, opath, public) task_id_map[name] = (task_id, task_module) info("Created task %s as id %d" % (name, task_id)) # We need to restart ProxyService to ensure it reinitializes, # picking up the new task and sending it to RWS. restart_service("ProxyService", contest=contest_id) return task_id