def render_suite_files(suites: List, suite_name: str,
generated_suite_filename: str, test_list: List[str],
suite_dir, create_misc_suite: bool) -> Dict:
"""
Render the given list of suites.
This will create a dictionary of all the resmoke config files to create with the
filename of each file as the key and the contents as the value.
:param suites: List of suites to render.
:param suite_name: Base name of suites.
:param generated_suite_filename: The name to use as the file name for generated suite file.
:param test_list: List of tests used in suites.
:param suite_dir: Directory containing test suite configurations.
:param create_misc_suite: Whether or not a _misc suite file should be created.
:return: Dictionary of rendered resmoke config files.
"""
# pylint: disable=too-many-arguments
source_config = read_yaml_file(os.path.join(suite_dir,
suite_name + ".yml"))
suite_configs = {
f"{os.path.basename(suite.name)}.yml":
suite.generate_resmoke_config(source_config)
for suite in suites
}
if create_misc_suite:
suite_configs[
f"{generated_suite_filename}_misc.yml"] = generate_resmoke_suite_config(
source_config, generated_suite_filename, excludes=test_list)
return suite_configs
def from_yaml_file(cls, path: str) -> "EvgExpansions":
"""
Read the evergreen expansions from the given YAML file.
:param path: Path to expansions YAML file.
:return: Expansions read from file.
"""
return cls(**read_yaml_file(path))
def from_yaml_file(cls, path: str) -> "EvgExpansions":
"""
Read the generation configuration from the given file.
:param path: Path to file.
:return: Parse evergreen expansions.
"""
return cls(**read_yaml_file(path))
def read_suite_config(suite_dir, suite_name) -> Dict[str, Any]:
"""
Read the given resmoke suite configuration.
:param suite_dir: Directory containing suite configurations.
:param suite_name: Name of suite to read.
:return: Configuration of specified suite.
"""
return read_yaml_file(os.path.join(suite_dir, f"{suite_name}.yml"))
def get_old_yaml(commit_hash):
"""Download BACKPORTS_REQUIRED_FILE from the old commit and return the yaml."""
response = requests.get(
f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}'
)
# If the response was successful, no exception will be raised.
response.raise_for_status()
old_yaml_file = f"{commit_hash}_{BACKPORTS_REQUIRED_FILE}"
temp_dir = tempfile.mkdtemp()
temp_old_yaml_file = os.path.join(temp_dir, old_yaml_file)
with open(temp_old_yaml_file, "w") as fileh:
fileh.write(response.text)
backports_required_old = read_yaml_file(temp_old_yaml_file)
return backports_required_old
def get_last_lts_yaml(commit_hash):
"""Download BACKPORTS_REQUIRED_FILE from the last LTS commit and return the yaml."""
LOGGER.info(
f"Downloading file from commit hash of last-lts branch {commit_hash}")
response = requests.get(
f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}'
)
# If the response was successful, no exception will be raised.
response.raise_for_status()
last_lts_file = f"{commit_hash}_{BACKPORTS_REQUIRED_FILE}"
temp_dir = tempfile.mkdtemp()
with open(os.path.join(temp_dir, last_lts_file), "w") as fileh:
fileh.write(response.text)
backports_required_last_lts = read_yaml_file(
os.path.join(temp_dir, last_lts_file))
return backports_required_last_lts
def get_existing_credentials_from_file(
cls, file_path: str) -> Optional[OAuthCredentials]:
"""
Try to get OAuth credentials from a file location.
Will return None if credentials either don't exist or are expired.
:param file_path: Location to check for OAuth credentials.
:return: Valid OAuth credentials or None if valid credentials don't exist
"""
try:
creds = OAuthCredentials(**read_yaml_file(file_path))
if (creds.access_token and creds.created_time and creds.expires_in
and creds.user_name and not creds.are_expired()):
return creds
else:
return None
except ValidationError:
return None
except OSError:
return None
def from_yaml_file(cls, path: str) -> "EvgExpansions":
"""Read the generation configuration from the given file."""
return cls(**read_yaml_file(path))
def assert_contents(self, expected):
actual = read_yaml_file(
os.path.join(self._tmpdir.name, EXCLUDE_TAGS_FILE))
self.assertEqual(actual, expected)
def generate_exclude_yaml(old_bin_version: str, output: str,
logger: logging.Logger) -> None:
"""
Create a tag file associating multiversion tests to tags for exclusion.
Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the
last-lts and/or last-continuous branch to determine which tests should be denylisted.
"""
output = os.path.abspath(output)
location, _ = os.path.split(output)
if not os.path.isdir(location):
logger.info(f"Cannot write to {output}. Not generating tag file.")
return
backports_required_latest = read_yaml_file(
os.path.join(ETC_DIR, BACKPORTS_REQUIRED_FILE))
# Get the state of the backports_required_for_multiversion_tests.yml file for the old
# binary we are running tests against. We do this by using the commit hash from the old
# mongo shell executable.
from buildscripts.resmokelib import multiversionconstants
shell_version = {
MultiversionOptions.LAST_LTS:
multiversionconstants.LAST_LTS_MONGO_BINARY,
MultiversionOptions.LAST_CONTINUOUS:
multiversionconstants.LAST_CONTINUOUS_MONGO_BINARY,
}[old_bin_version]
old_version_commit_hash = get_backports_required_hash_for_shell_version(
mongo_shell_path=shell_version)
# Get the yaml contents from the old commit.
logger.info(
f"Downloading file from commit hash of old branch {old_version_commit_hash}"
)
backports_required_old = get_old_yaml(old_version_commit_hash)
def diff(list1, list2):
return [elem for elem in (list1 or []) if elem not in (list2 or [])]
def get_suite_exclusions(version_key):
_suites_latest = backports_required_latest[version_key]["suites"] or {}
# Check if the changed syntax for etc/backports_required_for_multiversion_tests.yml has been
# backported.
# This variable and all branches where it's not set can be deleted after backporting the change.
change_backported = version_key in backports_required_old.keys()
if change_backported:
_always_exclude = diff(
backports_required_latest[version_key]["all"],
backports_required_old[version_key]["all"])
_suites_old: defaultdict = defaultdict(
list, backports_required_old[version_key]["suites"] or {})
else:
_always_exclude = diff(
backports_required_latest[version_key]["all"],
backports_required_old["all"])
_suites_old: defaultdict = defaultdict(
list, backports_required_old["suites"] or {})
return _suites_latest, _suites_old, _always_exclude
suites_latest, suites_old, always_exclude = get_suite_exclusions(
old_bin_version.replace("_", "-"))
tags = _tags.TagsConfig()
# Tag tests that are excluded from every suite.
for elem in always_exclude:
tags.add_tag("js_test", elem["test_file"], BACKPORT_REQUIRED_TAG)
# Tag tests that are excluded on a suite-by-suite basis.
for suite in suites_latest.keys():
test_set = set()
for elem in diff(suites_latest[suite], suites_old[suite]):
test_set.add(elem["test_file"])
for test in test_set:
tags.add_tag("js_test", test, f"{suite}_{BACKPORT_REQUIRED_TAG}")
logger.info(f"Writing exclude tags to {output}.")
tags.write_file(
filename=output,
preamble="Tag file that specifies exclusions from multiversion suites."
)
def is_suite_sharded(suite_dir: str, suite_name: str) -> bool:
"""Return true if a suite uses ShardedClusterFixture."""
source_config = read_yaml_file(os.path.join(suite_dir,
suite_name + ".yml"))
return source_config["executor"]["fixture"][
"class"] == "ShardedClusterFixture"
def generate_exclude_yaml(task_path_suffix: str, output: str) -> None:
# pylint: disable=too-many-locals
"""
Create a tag file associating multiversion tests to tags for exclusion.
Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the
last-lts branch to determine which tests should be denylisted.
"""
enable_logging()
location, _ = os.path.split(os.path.abspath(output))
if not os.path.isdir(location):
LOGGER.info(f"Cannot write to {output}. Not generating tag file.")
return
backports_required_latest = read_yaml_file(
os.path.join(ETC_DIR, BACKPORTS_REQUIRED_FILE))
# Get the state of the backports_required_for_multiversion_tests.yml file for the last-lts
# binary we are running tests against. We do this by using the commit hash from the last-lts
# mongo shell executable.
last_lts_commit_hash = get_backports_required_last_lts_hash(
task_path_suffix)
# Get the yaml contents from the last-lts commit.
backports_required_last_lts = get_last_lts_yaml(last_lts_commit_hash)
def diff(list1, list2):
return [elem for elem in (list1 or []) if elem not in (list2 or [])]
suites_latest = backports_required_latest["suites"] or {}
# Check if the changed syntax for etc/backports_required_multiversion.yml has been backported.
# This variable and all branches where it's not set can be deleted after backporting the change.
change_backported = "all" in backports_required_last_lts.keys()
if change_backported:
always_exclude = diff(backports_required_latest["all"],
backports_required_last_lts["all"])
suites_last_lts: defaultdict = defaultdict(
list, backports_required_last_lts["suites"])
else:
always_exclude = backports_required_latest["all"] or []
suites_last_lts = defaultdict(list, backports_required_last_lts)
for suite in suites_latest.keys():
for elem in suites_last_lts[suite] or []:
if elem in always_exclude:
always_exclude.remove(elem)
tags = _tags.TagsConfig()
# Tag tests that are excluded from every suite.
for elem in always_exclude:
tags.add_tag("js_test", elem["test_file"], BACKPORT_REQUIRED_TAG)
# Tag tests that are excluded on a suite-by-suite basis.
for suite in suites_latest.keys():
test_set = set()
for elem in diff(suites_latest[suite], suites_last_lts[suite]):
test_set.add(elem["test_file"])
for test in test_set:
tags.add_tag("js_test", test, f"{suite}_{BACKPORT_REQUIRED_TAG}")
LOGGER.info(f"Writing exclude tags to {output}.")
tags.write_file(
filename=output,
preamble="Tag file that specifies exclusions from multiversion suites."
)
def from_yaml_file(cls, path: str) -> "EvgExpansions":
"""Read the evergreen expansions from the given file."""
return cls(**read_yaml_file(path))
def from_yaml_file(cls, path: str) -> "GoldenTestConfig":
"""Read the golden test configuration from the given file."""
return cls(**read_yaml_file(path))
def from_yaml_file(
cls,
path: str = GENERATE_CONFIG_FILE) -> "GenerationConfiguration":
"""Read the generation configuration from the given file."""
return cls(**read_yaml_file(path))