def test_adds_exclude_file(self, get_exclude_files): get_exclude_files.return_value = set() get_exclude_files.return_value.add( 'jstests/core/count_plan_summary.js') with open(self._tmpdir.name + "/sharding_jscore_passthrough_00.yml", mode='w') as fh: fh.write(CONF) runner = CliRunner() result = runner.invoke(under_test.generate_exclude_yaml, [ '--suite=test', '--task-path-suffix=/data/multiversion', '--is-generated-suite=true' ]) self.assertEqual(result.exit_code, 0, result) self.assertEqual(get_exclude_files.call_count, 1) new_conf = read_yaml(self._tmpdir.name, "sharding_jscore_passthrough_00.yml") self.assertEqual(new_conf["selector"]["exclude_files"], ["jstests/core/count_plan_summary.js"])
def get_last_stable_yaml(last_stable_commit_hash, suite_name): """Download BACKPORTS_REQUIRED_FILE from the last stable commit and return the yaml.""" LOGGER.info( f"Downloading file from commit hash of last-stable branch {last_stable_commit_hash}" ) response = requests.get( f'{BACKPORTS_REQUIRED_BASE_URL}/{last_stable_commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}' ) # If the response was successful, no exception will be raised. response.raise_for_status() last_stable_file = f"{last_stable_commit_hash}_{BACKPORTS_REQUIRED_FILE}" temp_dir = tempfile.mkdtemp() with open(os.path.join(temp_dir, last_stable_file), "w") as fileh: fileh.write(response.text) backports_required_last_stable = generate_resmoke.read_yaml( temp_dir, last_stable_file) return backports_required_last_stable[suite_name]
def _generate_fuzzer_tasks(self, config): suite_file = self.options.suite + ".yml" # Update the jstestfuzz yml suite with the proper multiversion configurations. source_config = generate_resmoke.read_yaml(TEST_SUITE_DIR, suite_file) config.update_yaml(source_config) updated_yml = generate_resmoke.generate_resmoke_suite_config( source_config, suite_file) file_dict = {f"{self.options.suite}.yml": updated_yml} dt = DisplayTaskDefinition(self.task) for version_config in config.version_configs: fuzzer_config = self._get_fuzzer_options(version_config, suite_file) gen_fuzzer.generate_evg_tasks(fuzzer_config, self.evg_config, task_name_suffix=version_config, display_task=dt) generate_resmoke.write_file_dict(CONFIG_DIR, file_dict) dt.execution_task(f"{fuzzer_config.name}_gen") self.evg_config.variant(self.options.variant).display_task(dt) return self.evg_config
def is_suite_sharded(suite_dir: str, suite_name: str) -> bool: """Return true if a suite uses ShardedClusterFixture.""" source_config = generate_resmoke.read_yaml(suite_dir, suite_name + ".yml") return source_config["executor"]["fixture"][ "class"] == "ShardedClusterFixture"
def generate_exclude_yaml(suite: str, task_path_suffix: str, is_generated_suite: bool) -> None: # pylint: disable=too-many-locals """ Update the given multiversion suite configuration yaml to exclude tests. Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the last-stable branch to determine which tests should be blacklisted. """ enable_logging() suite_name = generate_resmoke.remove_gen_suffix(suite) files_to_exclude = get_exclude_files(suite_name, task_path_suffix) if not files_to_exclude: LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.") return suite_yaml_dict = {} if not is_generated_suite: # Populate the config values to get the resmoke config directory. buildscripts.resmokelib.parser.set_run_options() suites_dir = os.path.join(_config.CONFIG_DIR, "suites") # Update the static suite config with the excluded files and write to disk. file_name = f"{suite_name}.yml" suite_config = generate_resmoke.read_yaml(suites_dir, file_name) suite_yaml_dict[ file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(files_to_exclude)) else: # We expect the generated suites to already have been generated in the generated config # directory. suites_dir = CONFIG_DIR for file_name in os.listdir(suites_dir): # Update the 'exclude_files' for each of the appropriate generated suites. if file_name.endswith('misc.yml'): # New tests will be run as part of misc.yml. We want to make sure to properly # exclude these tests if they have been blacklisted. suite_config = generate_resmoke.read_yaml( CONFIG_DIR, file_name) exclude_files = suite_config["selector"]["exclude_files"] add_to_excludes = [ test for test in files_to_exclude if test not in exclude_files ] exclude_files += add_to_excludes suite_yaml_dict[ file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(exclude_files)) elif file_name.endswith('.yml'): suite_config = generate_resmoke.read_yaml( CONFIG_DIR, file_name) selected_files = suite_config["selector"]["roots"] # Only exclude the files that we want to exclude in the first place and have been # selected to run as part of the generated suite yml. intersection = [ test for test in selected_files if test in files_to_exclude ] if not intersection: continue suite_yaml_dict[ file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(intersection)) generate_resmoke.write_file_dict(suites_dir, suite_yaml_dict)
def assert_contents(self, expected): actual = read_yaml(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE) self.assertEqual(actual, expected)
def generate_exclude_yaml(task_path_suffix: str, output: str) -> None: # pylint: disable=too-many-locals """ Create a tag file associating multiversion tests to tags for exclusion. Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the last-lts branch to determine which tests should be blacklisted. """ enable_logging() location, _ = os.path.split(os.path.abspath(output)) if not os.path.isdir(location): LOGGER.info(f"Cannot write to {output}. Not generating tag file.") return backports_required_latest = generate_resmoke.read_yaml( ETC_DIR, BACKPORTS_REQUIRED_FILE) # Get the state of the backports_required_for_multiversion_tests.yml file for the last-lts # binary we are running tests against. We do this by using the commit hash from the last-lts # mongo shell executable. last_lts_commit_hash = get_backports_required_last_lts_hash( task_path_suffix) # Get the yaml contents from the last-lts commit. backports_required_last_lts = get_last_lts_yaml(last_lts_commit_hash) def diff(list1, list2): return [elem for elem in (list1 or []) if elem not in (list2 or [])] suites_latest = backports_required_latest["suites"] or {} # Check if the changed syntax for etc/backports_required_multiversion.yml has been backported. # This variable and all branches where it's not set can be deleted after backporting the change. change_backported = "all" in backports_required_last_lts.keys() if change_backported: always_exclude = diff(backports_required_latest["all"], backports_required_last_lts["all"]) suites_last_lts: defaultdict = defaultdict( list, backports_required_last_lts["suites"]) else: always_exclude = backports_required_latest["all"] or [] suites_last_lts = defaultdict(list, backports_required_last_lts) for suite in suites_latest.keys(): for elem in suites_last_lts[suite] or []: if elem in always_exclude: always_exclude.remove(elem) tags = _tags.TagsConfig() # Tag tests that are excluded from every suite. for elem in always_exclude: tags.add_tag("js_test", elem["test_file"], BACKPORT_REQUIRED_TAG) # Tag tests that are excluded on a suite-by-suite basis. for suite in suites_latest.keys(): test_set = set() for elem in diff(suites_latest[suite], suites_last_lts[suite]): test_set.add(elem["test_file"]) for test in test_set: tags.add_tag("js_test", test, f"{suite}_{BACKPORT_REQUIRED_TAG}") LOGGER.info(f"Writing exclude tags to {output}.") tags.write_file( filename=output, preamble="Tag file that specifies exclusions from multiversion suites." )
def generate_exclude_yaml(suite, task_path_suffix, is_generated_suite): """ Update the given multiversion suite configuration yaml to exclude tests. Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the last-stable branch to determine which tests should be blacklisted. """ enable_logging() suite_name = generate_resmoke.remove_gen_suffix(suite) # Get the backports_required_for_multiversion_tests.yml on the current version branch. backports_required_latest = generate_resmoke.read_yaml( ETC_DIR, BACKPORTS_REQUIRED_FILE) if suite_name not in backports_required_latest: LOGGER.info( f"Generating exclude files not supported for '{suite_name}''.") return latest_suite_yaml = backports_required_latest[suite_name] if not latest_suite_yaml: LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.") return # Get the state of the backports_required_for_multiversion_tests.yml file for the last-stable # binary we are running tests against. We do this by using the commit hash from the last-stable # mongo shell executable. last_stable_commit_hash = get_backports_required_last_stable_hash( task_path_suffix) # Get the yaml contents under the 'suite_name' key from the last-stable commit. last_stable_suite_yaml = get_last_stable_yaml(last_stable_commit_hash, suite_name) if last_stable_suite_yaml is None: files_to_exclude = set(elem["test_file"] for elem in latest_suite_yaml) else: files_to_exclude = set(elem["test_file"] for elem in latest_suite_yaml if elem not in last_stable_suite_yaml) if not files_to_exclude: LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.") return suite_yaml_dict = {} if not is_generated_suite: # Populate the config values to get the resmoke config directory. buildscripts.resmokelib.parser.set_options() suites_dir = os.path.join(_config.CONFIG_DIR, "suites") # Update the static suite config with the excluded files and write to disk. file_name = f"{suite_name}.yml" suite_config = generate_resmoke.read_yaml(suites_dir, file_name) suite_yaml_dict[ file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(files_to_exclude)) else: # We expect the generated suites to already have been generated in the generated config # directory. for file_name in os.listdir(CONFIG_DIR): suites_dir = CONFIG_DIR # Update the 'exclude_files' for each of the appropriate generated suites. if suite_name in file_name and file_name.endswith('.yml'): suite_config = generate_resmoke.read_yaml( CONFIG_DIR, file_name) selected_files = suite_config["selector"]["roots"] # Only exclude the files that we want to exclude in the first place and have been # selected to run as part of the generated suite yml. intersection = [ test for test in selected_files if test in files_to_exclude ] if not intersection: continue suite_yaml_dict[ file_name] = generate_resmoke.generate_resmoke_suite_config( suite_config, file_name, excludes=list(intersection)) generate_resmoke.write_file_dict(suites_dir, suite_yaml_dict)