def run_generate_tasks(expansion_file, evergreen_config=None):
"""
Create a configuration for generate tasks to create sub suites for the specified resmoke suite.
Tests using ReplicaSetFixture will be generated to use 3 nodes and linear_chain=True.
Tests using ShardedClusterFixture will be generated to use 2 shards with 2 nodes each.
The different binary version configurations tested are stored in REPL_MIXED_VERSION_CONFIGS
and SHARDED_MIXED_VERSION_CONFIGS.
The `--expansion-file` should contain all the configuration needed to generate the tasks.
\f
:param expansion_file: Configuration file.
:param evergreen_config: Evergreen configuration file.
"""
evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config)
prepare_directory_for_suite(CONFIG_DIR)
evg_config = Configuration()
config_options = generate_resmoke.ConfigOptions.from_file(
expansion_file, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN)
config_generator = EvergreenConfigGenerator(evg_api, evg_config,
config_options)
config_generator.run()
def main(
verbose: bool,
expansion_file: str,
evg_api_config: str,
selected_tests_config: str,
):
"""
Select tasks to be run based on changed files in a patch build.
:param verbose: Log extra debug information.
:param expansion_file: Configuration file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param selected_tests_config: Location of config file to connect to elected-tests service.
"""
_configure_logging(verbose)
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
selected_tests_service = SelectedTestsService.from_file(
selected_tests_config)
repo = Repo(".")
changed_files = find_changed_files(repo)
buildscripts.resmokelib.parser.set_options()
LOGGER.debug("Found changed files", files=changed_files)
selected_tests_variant_expansions = read_config.read_config_file(
expansion_file)
origin_build_variants = selected_tests_variant_expansions[
"selected_tests_buildvariants"].split(" ")
config_dict_of_suites_and_tasks = run(evg_api, evg_conf,
selected_tests_service,
selected_tests_variant_expansions,
changed_files, origin_build_variants)
write_file_dict(SELECTED_TESTS_CONFIG_DIR, config_dict_of_suites_and_tasks)
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, evg_api_config, verbose, task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
generate_config.validate(evg_conf)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
change_detector = EvergreenFileChangeDetector(task_id, evg_api)
changed_tests = change_detector.find_changed_tests(repos)
tests_by_task = create_tests_by_task(generate_config.build_variant,
evg_conf, changed_tests)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
# We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include
# MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
build_variant = create_multiversion_generate_tasks_config(
tests_by_task, evg_api, generate_config)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
if not validate_task_generation_limit(shrub_project):
sys.exit(1)
write_file(generate_tasks_file, shrub_project.json())
"""
if not os.path.exists(CONFIG_DIRECTORY):
os.makedirs(CONFIG_DIRECTORY)
with open(os.path.join(CONFIG_DIRECTORY, CONFIG_FILE), "w") as file_handle:
file_handle.write(shrub_config.to_json())
def main(evergreen_api, repo):
"""Execute Main program."""
parser = argparse.ArgumentParser(description=main.__doc__)
parser.add_argument(
"--expansion-file",
dest="expansion_file",
type=str,
help="Location of expansions file generated by evergreen.")
cmd_line_options = parser.parse_args()
expansions_file_data = read_config.read_config_file(
cmd_line_options.expansion_file)
shrub_config = Configuration()
buildvariant_map = _create_evg_buildvariant_map(expansions_file_data)
_generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
buildvariant_map, repo)
_write_to_file(shrub_config)
if __name__ == '__main__':
main(RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE), Repo("."))
)
options, args = parse_command_line()
resmoke_cmd = _set_resmoke_cmd(options, args)
# Load the dict of tests to run.
if options.test_list_file:
tests_by_task = _load_tests_file(options.test_list_file)
# If there are no tests to run, carry on.
if tests_by_task is None:
test_results = {"failures": 0, "results": []}
_write_json_file(test_results, options.report_file)
sys.exit(0)
# Run the executor finder.
else:
tests_by_task = create_tests_by_task(options, evg_api)
if options.test_list_outfile:
_write_json_file(tests_by_task, options.test_list_outfile)
if options.generate_tasks_file:
create_generate_tasks_file(evg_api, options, tests_by_task)
else:
run_tests(options.no_exec, tests_by_task, resmoke_cmd, options.report_file)
if __name__ == "__main__":
main(RetryingEvergreenApi.get_api(config_file=CONFIG_FILE))
def main( # pylint: disable=too-many-arguments,too-many-locals,too-many-statements
project, build_variant, revision, patch_file, out_file, json_artifact):
"""
Create a file with expansions that can be used to bypass compile.
If for any reason bypass compile is false, we do not write out the expansion. Only if we
determine to bypass compile do we write out the expansions.
\f
:param project: The evergreen project.
:param build_variant: The build variant whose artifacts we want to use.
:param revision: Base revision of the build.
:param patch_file: A list of all files modified in patch build.
:param out_file: File to write expansions to.
:param json_artifact: The JSON file to write out the metadata of files to attach to task.
"""
logging.basicConfig(
format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s",
level=logging.DEBUG,
stream=sys.stdout,
)
# Determine if we should bypass compile based on modified patch files.
if should_bypass_compile(patch_file, build_variant):
evergreen_api = RetryingEvergreenApi.get_api(
config_file=EVG_CONFIG_FILE)
build_id = find_build_for_previous_compile_task(
evergreen_api, revision, project, build_variant)
if not build_id:
LOGGER.warning(
"Could not find build id. Default compile bypass to false.",
revision=revision,
project=project)
return
task = find_previous_compile_task(evergreen_api, build_id, revision)
if task is None or not task.is_success():
LOGGER.warning(
"Could not retrieve artifacts because the compile task for base commit"
" was not available. Default compile bypass to false.",
task_id=task.task_id)
return
LOGGER.info("Fetching pre-existing artifacts from compile task",
task_id=task.task_id)
artifacts = []
for artifact in task.artifacts:
filename = os.path.basename(artifact.url)
if filename.startswith(build_id):
LOGGER.info("Retrieving archive", filename=filename)
# This is the artifacts.tgz as referenced in evergreen.yml.
try:
urllib.request.urlretrieve(artifact.url, filename)
except urllib.error.ContentTooShortError:
LOGGER.warning(
"The artifact could not be completely downloaded. Default"
" compile bypass to false.",
filename=filename)
return
# Need to extract certain files from the pre-existing artifacts.tgz.
extract_files = [
executable_name("mongobridge"),
executable_name("mongoebench"),
executable_name("mongoed"),
executable_name("mongotmock"),
executable_name("wt"),
]
with tarfile.open(filename, "r:gz") as tar:
# The repo/ directory contains files needed by the package task. May
# need to add other files that would otherwise be generated by SCons
# if we did not bypass compile.
subdir = [
tarinfo for tarinfo in tar.getmembers()
if tarinfo.name.startswith("repo/")
or tarinfo.name in extract_files
]
LOGGER.info("Extracting the files...",
filename=filename,
files="\n".join(tarinfo.name
for tarinfo in subdir))
tar.extractall(members=subdir)
elif filename.startswith("mongo-src"):
LOGGER.info("Retrieving mongo source", filename=filename)
# This is the distsrc.[tgz|zip] as referenced in evergreen.yml.
try:
urllib.request.urlretrieve(artifact.url, filename)
except urllib.error.ContentTooShortError:
LOGGER.warn(
"The artifact could not be completely downloaded. Default"
" compile bypass to false.",
filename=filename)
return
extension = os.path.splitext(filename)[1]
distsrc_filename = "distsrc{}".format(extension)
LOGGER.info("Renaming",
filename=filename,
rename=distsrc_filename)
os.rename(filename, distsrc_filename)
else:
LOGGER.info("Linking base artifact to this patch build",
filename=filename)
# For other artifacts we just add their URLs to the JSON file to upload.
files = {
"name": artifact.name,
"link": artifact.url,
"visibility": "private",
}
# Check the link exists, else raise an exception. Compile bypass is disabled.
requests.head(artifact.url).raise_for_status()
artifacts.append(files)
# SERVER-21492 related issue where without running scons the jstests/libs/key1
# and key2 files are not chmod to 0600. Need to change permissions here since we
# bypass SCons.
os.chmod("jstests/libs/key1", 0o600)
os.chmod("jstests/libs/key2", 0o600)
os.chmod("jstests/libs/keyForRollover", 0o600)
# This is the artifacts.json file.
write_out_artifacts(json_artifact, artifacts)
# Need to apply these expansions for bypassing SCons.
expansions = generate_bypass_expansions(project, build_variant,
revision, build_id)
write_out_bypass_compile_expansions(out_file, **expansions)