def test_adds_exclude_file(self, get_exclude_files):
get_exclude_files.return_value = set()
get_exclude_files.return_value.add(
'jstests/core/count_plan_summary.js')
with open(self._tmpdir.name + "/sharding_jscore_passthrough_00.yml",
mode='w') as fh:
fh.write(CONF)
runner = CliRunner()
result = runner.invoke(under_test.generate_exclude_yaml, [
'--suite=test', '--task-path-suffix=/data/multiversion',
'--is-generated-suite=true'
])
self.assertEqual(result.exit_code, 0, result)
self.assertEqual(get_exclude_files.call_count, 1)
new_conf = read_yaml(self._tmpdir.name,
"sharding_jscore_passthrough_00.yml")
self.assertEqual(new_conf["selector"]["exclude_files"],
["jstests/core/count_plan_summary.js"])
def get_last_stable_yaml(last_stable_commit_hash, suite_name):
"""Download BACKPORTS_REQUIRED_FILE from the last stable commit and return the yaml."""
LOGGER.info(
f"Downloading file from commit hash of last-stable branch {last_stable_commit_hash}"
)
response = requests.get(
f'{BACKPORTS_REQUIRED_BASE_URL}/{last_stable_commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}'
)
# If the response was successful, no exception will be raised.
response.raise_for_status()
last_stable_file = f"{last_stable_commit_hash}_{BACKPORTS_REQUIRED_FILE}"
temp_dir = tempfile.mkdtemp()
with open(os.path.join(temp_dir, last_stable_file), "w") as fileh:
fileh.write(response.text)
backports_required_last_stable = generate_resmoke.read_yaml(
temp_dir, last_stable_file)
return backports_required_last_stable[suite_name]
def _generate_fuzzer_tasks(self, config):
suite_file = self.options.suite + ".yml"
# Update the jstestfuzz yml suite with the proper multiversion configurations.
source_config = generate_resmoke.read_yaml(TEST_SUITE_DIR, suite_file)
config.update_yaml(source_config)
updated_yml = generate_resmoke.generate_resmoke_suite_config(
source_config, suite_file)
file_dict = {f"{self.options.suite}.yml": updated_yml}
dt = DisplayTaskDefinition(self.task)
for version_config in config.version_configs:
fuzzer_config = self._get_fuzzer_options(version_config,
suite_file)
gen_fuzzer.generate_evg_tasks(fuzzer_config,
self.evg_config,
task_name_suffix=version_config,
display_task=dt)
generate_resmoke.write_file_dict(CONFIG_DIR, file_dict)
dt.execution_task(f"{fuzzer_config.name}_gen")
self.evg_config.variant(self.options.variant).display_task(dt)
return self.evg_config
def is_suite_sharded(suite_dir: str, suite_name: str) -> bool:
"""Return true if a suite uses ShardedClusterFixture."""
source_config = generate_resmoke.read_yaml(suite_dir, suite_name + ".yml")
return source_config["executor"]["fixture"][
"class"] == "ShardedClusterFixture"
def generate_exclude_yaml(suite: str, task_path_suffix: str,
is_generated_suite: bool) -> None:
# pylint: disable=too-many-locals
"""
Update the given multiversion suite configuration yaml to exclude tests.
Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the
last-stable branch to determine which tests should be blacklisted.
"""
enable_logging()
suite_name = generate_resmoke.remove_gen_suffix(suite)
files_to_exclude = get_exclude_files(suite_name, task_path_suffix)
if not files_to_exclude:
LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
return
suite_yaml_dict = {}
if not is_generated_suite:
# Populate the config values to get the resmoke config directory.
buildscripts.resmokelib.parser.set_run_options()
suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
# Update the static suite config with the excluded files and write to disk.
file_name = f"{suite_name}.yml"
suite_config = generate_resmoke.read_yaml(suites_dir, file_name)
suite_yaml_dict[
file_name] = generate_resmoke.generate_resmoke_suite_config(
suite_config, file_name, excludes=list(files_to_exclude))
else:
# We expect the generated suites to already have been generated in the generated config
# directory.
suites_dir = CONFIG_DIR
for file_name in os.listdir(suites_dir):
# Update the 'exclude_files' for each of the appropriate generated suites.
if file_name.endswith('misc.yml'):
# New tests will be run as part of misc.yml. We want to make sure to properly
# exclude these tests if they have been blacklisted.
suite_config = generate_resmoke.read_yaml(
CONFIG_DIR, file_name)
exclude_files = suite_config["selector"]["exclude_files"]
add_to_excludes = [
test for test in files_to_exclude
if test not in exclude_files
]
exclude_files += add_to_excludes
suite_yaml_dict[
file_name] = generate_resmoke.generate_resmoke_suite_config(
suite_config, file_name, excludes=list(exclude_files))
elif file_name.endswith('.yml'):
suite_config = generate_resmoke.read_yaml(
CONFIG_DIR, file_name)
selected_files = suite_config["selector"]["roots"]
# Only exclude the files that we want to exclude in the first place and have been
# selected to run as part of the generated suite yml.
intersection = [
test for test in selected_files if test in files_to_exclude
]
if not intersection:
continue
suite_yaml_dict[
file_name] = generate_resmoke.generate_resmoke_suite_config(
suite_config, file_name, excludes=list(intersection))
generate_resmoke.write_file_dict(suites_dir, suite_yaml_dict)
def assert_contents(self, expected):
actual = read_yaml(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE)
self.assertEqual(actual, expected)
def generate_exclude_yaml(task_path_suffix: str, output: str) -> None:
# pylint: disable=too-many-locals
"""
Create a tag file associating multiversion tests to tags for exclusion.
Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the
last-lts branch to determine which tests should be blacklisted.
"""
enable_logging()
location, _ = os.path.split(os.path.abspath(output))
if not os.path.isdir(location):
LOGGER.info(f"Cannot write to {output}. Not generating tag file.")
return
backports_required_latest = generate_resmoke.read_yaml(
ETC_DIR, BACKPORTS_REQUIRED_FILE)
# Get the state of the backports_required_for_multiversion_tests.yml file for the last-lts
# binary we are running tests against. We do this by using the commit hash from the last-lts
# mongo shell executable.
last_lts_commit_hash = get_backports_required_last_lts_hash(
task_path_suffix)
# Get the yaml contents from the last-lts commit.
backports_required_last_lts = get_last_lts_yaml(last_lts_commit_hash)
def diff(list1, list2):
return [elem for elem in (list1 or []) if elem not in (list2 or [])]
suites_latest = backports_required_latest["suites"] or {}
# Check if the changed syntax for etc/backports_required_multiversion.yml has been backported.
# This variable and all branches where it's not set can be deleted after backporting the change.
change_backported = "all" in backports_required_last_lts.keys()
if change_backported:
always_exclude = diff(backports_required_latest["all"],
backports_required_last_lts["all"])
suites_last_lts: defaultdict = defaultdict(
list, backports_required_last_lts["suites"])
else:
always_exclude = backports_required_latest["all"] or []
suites_last_lts = defaultdict(list, backports_required_last_lts)
for suite in suites_latest.keys():
for elem in suites_last_lts[suite] or []:
if elem in always_exclude:
always_exclude.remove(elem)
tags = _tags.TagsConfig()
# Tag tests that are excluded from every suite.
for elem in always_exclude:
tags.add_tag("js_test", elem["test_file"], BACKPORT_REQUIRED_TAG)
# Tag tests that are excluded on a suite-by-suite basis.
for suite in suites_latest.keys():
test_set = set()
for elem in diff(suites_latest[suite], suites_last_lts[suite]):
test_set.add(elem["test_file"])
for test in test_set:
tags.add_tag("js_test", test, f"{suite}_{BACKPORT_REQUIRED_TAG}")
LOGGER.info(f"Writing exclude tags to {output}.")
tags.write_file(
filename=output,
preamble="Tag file that specifies exclusions from multiversion suites."
)
def generate_exclude_yaml(suite, task_path_suffix, is_generated_suite):
"""
Update the given multiversion suite configuration yaml to exclude tests.
Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the
last-stable branch to determine which tests should be blacklisted.
"""
enable_logging()
suite_name = generate_resmoke.remove_gen_suffix(suite)
# Get the backports_required_for_multiversion_tests.yml on the current version branch.
backports_required_latest = generate_resmoke.read_yaml(
ETC_DIR, BACKPORTS_REQUIRED_FILE)
if suite_name not in backports_required_latest:
LOGGER.info(
f"Generating exclude files not supported for '{suite_name}''.")
return
latest_suite_yaml = backports_required_latest[suite_name]
if not latest_suite_yaml:
LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
return
# Get the state of the backports_required_for_multiversion_tests.yml file for the last-stable
# binary we are running tests against. We do this by using the commit hash from the last-stable
# mongo shell executable.
last_stable_commit_hash = get_backports_required_last_stable_hash(
task_path_suffix)
# Get the yaml contents under the 'suite_name' key from the last-stable commit.
last_stable_suite_yaml = get_last_stable_yaml(last_stable_commit_hash,
suite_name)
if last_stable_suite_yaml is None:
files_to_exclude = set(elem["test_file"] for elem in latest_suite_yaml)
else:
files_to_exclude = set(elem["test_file"] for elem in latest_suite_yaml
if elem not in last_stable_suite_yaml)
if not files_to_exclude:
LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
return
suite_yaml_dict = {}
if not is_generated_suite:
# Populate the config values to get the resmoke config directory.
buildscripts.resmokelib.parser.set_options()
suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
# Update the static suite config with the excluded files and write to disk.
file_name = f"{suite_name}.yml"
suite_config = generate_resmoke.read_yaml(suites_dir, file_name)
suite_yaml_dict[
file_name] = generate_resmoke.generate_resmoke_suite_config(
suite_config, file_name, excludes=list(files_to_exclude))
else:
# We expect the generated suites to already have been generated in the generated config
# directory.
for file_name in os.listdir(CONFIG_DIR):
suites_dir = CONFIG_DIR
# Update the 'exclude_files' for each of the appropriate generated suites.
if suite_name in file_name and file_name.endswith('.yml'):
suite_config = generate_resmoke.read_yaml(
CONFIG_DIR, file_name)
selected_files = suite_config["selector"]["roots"]
# Only exclude the files that we want to exclude in the first place and have been
# selected to run as part of the generated suite yml.
intersection = [
test for test in selected_files if test in files_to_exclude
]
if not intersection:
continue
suite_yaml_dict[
file_name] = generate_resmoke.generate_resmoke_suite_config(
suite_config, file_name, excludes=list(intersection))
generate_resmoke.write_file_dict(suites_dir, suite_yaml_dict)
