def main( verbose: bool, expansion_file: str, evg_api_config: str, selected_tests_config: str, ): """ Select tasks to be run based on changed files in a patch build. :param verbose: Log extra debug information. :param expansion_file: Configuration file. :param evg_api_config: Location of configuration file to connect to evergreen. :param selected_tests_config: Location of config file to connect to elected-tests service. """ _configure_logging(verbose) evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config) evg_conf = parse_evergreen_file(EVERGREEN_FILE) selected_tests_service = SelectedTestsService.from_file( selected_tests_config) repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] buildscripts.resmokelib.parser.set_run_options() task_expansions = read_config.read_config_file(expansion_file) origin_build_variants = task_expansions[ "selected_tests_buildvariants"].split(" ") config_dict_of_suites_and_tasks = run(evg_api, evg_conf, selected_tests_service, task_expansions, repos, origin_build_variants) write_file_dict(SELECTED_TESTS_CONFIG_DIR, config_dict_of_suites_and_tasks)
def main( verbose: bool, expansion_file: str, evg_api_config: str, build_variant: str, selected_tests_config: str, ): """ Select tasks to be run based on changed files in a patch build. :param verbose: Log extra debug information. :param expansion_file: Configuration file. :param evg_api_config: Location of configuration file to connect to evergreen. :param build_variant: Build variant to query tasks from. :param selected_tests_config: Location of config file to connect to elected-tests service. """ _configure_logging(verbose) evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config) evg_conf = parse_evergreen_file(EVERGREEN_FILE) selected_tests_service = SelectedTestsService.from_file( selected_tests_config) repo = Repo(".") changed_files = find_changed_files(repo) buildscripts.resmokelib.parser.set_options() LOGGER.debug("Found changed files", files=changed_files) config_dict_of_suites_and_tasks = run(evg_api, evg_conf, expansion_file, selected_tests_service, changed_files, build_variant) write_file_dict(SELECTED_TESTS_CONFIG_DIR, config_dict_of_suites_and_tasks)
def generate_evg_tasks(self, burn_in_test=None, burn_in_idx=0): # pylint: disable=too-many-locals """ Generate evergreen tasks for multiversion tests. The number of tasks generated equals (the number of version configs) * (the number of generated suites). :param burn_in_test: The test to be run as part of the burn in multiversion suite. """ is_sharded = is_suite_sharded(TEST_SUITE_DIR, self.options.suite) if is_sharded: version_configs = SHARDED_MIXED_VERSION_CONFIGS else: version_configs = REPL_MIXED_VERSION_CONFIGS if self.options.is_jstestfuzz: return self._generate_fuzzer_tasks(version_configs, is_sharded) # Divide tests into suites based on run-time statistics for the last # LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed # in the misc suite. gen_suites = generate_resmoke.GenerateSubSuites( self.evg_api, self.options) end_date = datetime.datetime.utcnow().replace(microsecond=0) start_date = end_date - datetime.timedelta( days=generate_resmoke.LOOKBACK_DURATION_DAYS) suites = gen_suites.calculate_suites(start_date, end_date) # Render the given suites into yml files that can be used by resmoke.py. config_file_dict = generate_resmoke.render_suite_files( suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, self.options.create_misc_suite) generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict) if burn_in_test is not None: # Generate the subtasks to run burn_in_test against the appropriate mixed version # configurations. The display task is defined later as part of generating the burn # in tests. self._generate_burn_in_execution_tasks(version_configs, suites, burn_in_test, burn_in_idx, is_sharded) return self.evg_config for version_config in version_configs: idx = 0 for suite in suites: # Generate the newly divided test suites source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml") self._generate_sub_task(version_config, self.task, idx, source_suite, len(suites), is_sharded) idx += 1 # Also generate the misc task. misc_suite_name = "{0}_misc".format(self.options.suite) misc_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml") self._generate_sub_task(version_config, self.task, idx, misc_suite, 1, is_sharded) idx += 1 self.create_display_task(self.task, self.task_specs, self.task_names) return self.evg_config
def _write_fuzzer_yaml(options): """Write the fuzzer yaml to CONFIG_DIRECTORY.""" suite_file = options.suite + ".yml" source_config = generate_resmoke.read_yaml(TEST_SUITE_DIR, suite_file) gen_yml = generate_resmoke.generate_resmoke_suite_config( source_config, suite_file) file_dict = {f"{options.suite}.yml": gen_yml} generate_resmoke.write_file_dict(CONFIG_DIRECTORY, file_dict)
def generate_exclude_yaml(suite: str, task_path_suffix: str, is_generated_suite: bool) -> None: # pylint: disable=too-many-locals """ Update the given multiversion suite configuration yaml to exclude tests. Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the last-stable branch to determine which tests should be blacklisted. """ enable_logging() suite_name = generate_resmoke.remove_gen_suffix(suite) files_to_exclude = get_exclude_files(suite_name, task_path_suffix) if not files_to_exclude: LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.") return suite_yaml_dict = {} if not is_generated_suite: # Populate the config values to get the resmoke config directory. buildscripts.resmokelib.parser.set_options() suites_dir = os.path.join(_config.CONFIG_DIR, "suites") # Update the static suite config with the excluded files and write to disk. file_name = f"{suite_name}.yml" suite_config = generate_resmoke.read_yaml(suites_dir, file_name) suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(files_to_exclude)) else: # We expect the generated suites to already have been generated in the generated config # directory. suites_dir = CONFIG_DIR for file_name in os.listdir(suites_dir): # Update the 'exclude_files' for each of the appropriate generated suites. if file_name.endswith('misc.yml'): # New tests will be run as part of misc.yml. We want to make sure to properly # exclude these tests if they have been blacklisted. suite_config = generate_resmoke.read_yaml(CONFIG_DIR, file_name) exclude_files = suite_config["selector"]["exclude_files"] add_to_excludes = [test for test in files_to_exclude if test not in exclude_files] exclude_files += add_to_excludes suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(exclude_files)) elif file_name.endswith('.yml'): suite_config = generate_resmoke.read_yaml(CONFIG_DIR, file_name) selected_files = suite_config["selector"]["roots"] # Only exclude the files that we want to exclude in the first place and have been # selected to run as part of the generated suite yml. intersection = [test for test in selected_files if test in files_to_exclude] if not intersection: continue suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(intersection)) generate_resmoke.write_file_dict(suites_dir, suite_yaml_dict)
def generate_resmoke_suites(self) -> List[Suite]: """Generate the resmoke configuration files for this generator.""" # Divide tests into suites based on run-time statistics for the last # LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed # in the misc suite. gen_suites = generate_resmoke.GenerateSubSuites(self.evg_api, self.options) end_date = datetime.datetime.utcnow().replace(microsecond=0) start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS) suites = gen_suites.calculate_suites(start_date, end_date) # Render the given suites into yml files that can be used by resmoke.py. config_file_dict = generate_resmoke.render_suite_files(suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, self.options.create_misc_suite) generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict) return suites
def _generate_fuzzer_tasks(self, config): suite_file = self.options.suite + ".yml" # Update the jstestfuzz yml suite with the proper multiversion configurations. source_config = generate_resmoke.read_yaml(TEST_SUITE_DIR, suite_file) config.update_yaml(source_config) updated_yml = generate_resmoke.generate_resmoke_suite_config(source_config, suite_file) file_dict = {f"{self.options.suite}.yml": updated_yml} dt = DisplayTaskDefinition(self.task) for version_config in config.version_configs: fuzzer_config = self._get_fuzzer_options(version_config, suite_file) gen_fuzzer.generate_evg_tasks(fuzzer_config, self.evg_config, task_name_suffix=version_config, display_task=dt) generate_resmoke.write_file_dict(CONFIG_DIR, file_dict) dt.execution_task(f"{fuzzer_config.name}_gen") self.evg_config.variant(self.options.variant).display_task(dt) return self.evg_config
def generate_evg_tasks(self): """ Generate evergreen tasks for multiversion tests. The number of tasks generated equals (the number of configs in MIXED_VERSION_CONFIGS) * (the number of generated suites). """ idx = 0 # Divide tests into suites based on run-time statistics for the last # LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed # in the misc suite. gen_suites = generate_resmoke.GenerateSubSuites(self.evg_api, self.options) end_date = datetime.datetime.utcnow().replace(microsecond=0) start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS) suites = gen_suites.calculate_suites(start_date, end_date) # Render the given suites into yml files that can be used by resmoke.py. if self.options.is_sharded: config = MultiversionConfig(update_suite_config_for_multiversion_sharded, SHARDED_MIXED_VERSION_CONFIGS) else: config = MultiversionConfig(update_suite_config_for_multiversion_replset, MIXED_VERSION_CONFIGS) config_file_dict = generate_resmoke.render_suite_files( suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, config.update_yaml) for version_config in config.version_configs: for suite in suites: # Generate the newly divided test suites source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml") self._generate_sub_task(version_config, idx, source_suite, len(suites)) idx += 1 # Also generate the misc task. misc_suite_name = "{0}_misc".format(self.options.suite) source_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml") self._generate_sub_task(version_config, idx, source_suite, 1) idx += 1 generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict) dt = DisplayTaskDefinition(self.task).execution_tasks(self.task_names)\ .execution_task("{0}_gen".format(self.task)) self.evg_config.variant(self.options.variant).tasks(self.task_specs).display_task(dt) return self.evg_config
def generate_exclude_yaml(suite, task_path_suffix, is_generated_suite): """ Update the given multiversion suite configuration yaml to exclude tests. Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the last-stable branch to determine which tests should be blacklisted. """ enable_logging() suite_name = generate_resmoke.remove_gen_suffix(suite) # Get the backports_required_for_multiversion_tests.yml on the current version branch. backports_required_latest = generate_resmoke.read_yaml( ETC_DIR, BACKPORTS_REQUIRED_FILE) if suite_name not in backports_required_latest: LOGGER.info( f"Generating exclude files not supported for '{suite_name}''.") return latest_suite_yaml = backports_required_latest[suite_name] if not latest_suite_yaml: LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.") return # Get the state of the backports_required_for_multiversion_tests.yml file for the last-stable # binary we are running tests against. We do this by using the commit hash from the last-stable # mongo shell executable. last_stable_commit_hash = get_backports_required_last_stable_hash( task_path_suffix) # Get the yaml contents under the 'suite_name' key from the last-stable commit. last_stable_suite_yaml = get_last_stable_yaml(last_stable_commit_hash, suite_name) if last_stable_suite_yaml is None: files_to_exclude = set(elem["test_file"] for elem in latest_suite_yaml) else: files_to_exclude = set(elem["test_file"] for elem in latest_suite_yaml if elem not in last_stable_suite_yaml) if not files_to_exclude: LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.") return suite_yaml_dict = {} if not is_generated_suite: # Populate the config values to get the resmoke config directory. buildscripts.resmokelib.parser.set_options() suites_dir = os.path.join(_config.CONFIG_DIR, "suites") # Update the static suite config with the excluded files and write to disk. file_name = f"{suite_name}.yml" suite_config = generate_resmoke.read_yaml(suites_dir, file_name) suite_yaml_dict[ file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(files_to_exclude)) else: # We expect the generated suites to already have been generated in the generated config # directory. for file_name in os.listdir(CONFIG_DIR): suites_dir = CONFIG_DIR # Update the 'exclude_files' for each of the appropriate generated suites. if suite_name in file_name and file_name.endswith('.yml'): suite_config = generate_resmoke.read_yaml( CONFIG_DIR, file_name) selected_files = suite_config["selector"]["roots"] # Only exclude the files that we want to exclude in the first place and have been # selected to run as part of the generated suite yml. intersection = [ test for test in selected_files if test in files_to_exclude ] if not intersection: continue suite_yaml_dict[ file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(intersection)) generate_resmoke.write_file_dict(suites_dir, suite_yaml_dict)