def _distro_to_run_task_on(task: VariantTask,
evg_proj_config: EvergreenProjectConfig,
build_variant: str) -> str:
"""
Determine what distro an task should be run on.
For normal tasks, the distro will be the default for the build variant unless the task spec
specifies a particular distro to run on.
For generated tasks, the distro will be the default for the build variant unless (1) the
"use_large_distro" flag is set as a "var" in the "generate resmoke tasks" command of the
task definition and (2) the build variant defines the "large_distro_name" in its expansions.
:param task: Task being run.
:param evg_proj_config: Evergreen project configuration.
:param build_variant: Build Variant task is being run on.
:return: Distro task should be run on.
"""
task_def = evg_proj_config.get_task(task.name)
if task_def.is_generate_resmoke_task:
resmoke_vars = task_def.generate_resmoke_tasks_command["vars"]
if "use_large_distro" in resmoke_vars:
bv = evg_proj_config.get_variant(build_variant)
if "large_distro_name" in bv.raw["expansions"]:
return bv.raw["expansions"]["large_distro_name"]
return task.run_on[0]
def _check_variant(build_variant: str, evg_conf: EvergreenProjectConfig):
"""
Check if the build_variant is found in the evergreen file.
:param build_variant: Build variant to check.
:param evg_conf: Evergreen configuration to check against.
"""
if not evg_conf.get_variant(build_variant):
raise ValueError(f"Build variant '{build_variant}' not found in Evergreen file")
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str, str], repos: List[Repo],
origin_build_variants: List[str]) -> Dict[str, str]:
# pylint: disable=too-many-locals
"""
Run code to select tasks to run based on test and task mappings for each of the build variants.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param repos: List of repos containing changed files.
:param origin_build_variants: Build variants to collect task info from.
:return: Dict of files and file contents for generated tasks.
"""
config_dict_of_suites_and_tasks = {}
task_id = selected_tests_variant_expansions[TASK_ID_EXPANSION]
revision_map = generate_revision_map_from_manifest(repos, task_id, evg_api)
changed_files = find_changed_files_in_repos(repos, revision_map)
changed_files = {
_remove_repo_path_prefix(file_path)
for file_path in changed_files
}
LOGGER.debug("Found changed files", files=changed_files)
shrub_project = ShrubProject()
for build_variant in origin_build_variants:
shrub_build_variant = BuildVariant(build_variant)
build_variant_config = evg_conf.get_variant(build_variant)
origin_variant_expansions = build_variant_config.expansions
task_configs = _get_task_configs(evg_conf, selected_tests_service,
selected_tests_variant_expansions,
build_variant_config, changed_files)
for task_config in task_configs.values():
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
selected_tests_variant_expansions,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_build_variant,
config_options,
config_dict_of_suites_and_tasks)
shrub_project.add_build_variant(shrub_build_variant)
config_dict_of_suites_and_tasks[
"selected_tests_config.json"] = shrub_project.json()
return config_dict_of_suites_and_tasks
def create_task_list(evergreen_conf: EvergreenProjectConfig,
build_variant: str, suites: Dict, exclude_tasks: [str]):
"""
Find associated tasks for the specified build_variant and suites.
Returns a dict keyed by task_name, with executor, resmoke_args & tests, i.e.,
{'jsCore_small_oplog':
{'resmoke_args': '--suites=core_small_oplog --storageEngine=inMemory',
'tests': ['jstests/core/all2.js', 'jstests/core/all3.js'],
'use_multiversion': '/data/multiversion'}
}
:param evergreen_conf: Evergreen configuration for project.
:param build_variant: Build variant to select tasks from.
:param suites: Suites to be run.
:param exclude_tasks: Tasks to exclude.
:return: Dict of tasks to run with run configuration.
"""
log = LOGGER.bind(build_variant=build_variant)
log.debug("creating task list for suites",
suites=suites,
exclude_tasks=exclude_tasks)
evg_build_variant = evergreen_conf.get_variant(build_variant)
if not evg_build_variant:
log.warning("Buildvariant not found in evergreen config")
raise ValueError(
f"Buildvariant ({build_variant} not found in evergreen configuration"
)
# Find all the build variant tasks.
exclude_tasks_set = set(exclude_tasks)
all_variant_tasks = {
_get_task_name(task): task
for task in evg_build_variant.tasks
if task.name not in exclude_tasks_set and task.combined_resmoke_args
}
# Return the list of tasks to run for the specified suite.
task_list = {
task_name: {
"resmoke_args": _set_resmoke_args(task),
"tests": suites[task.resmoke_suite],
"use_multiversion": task.multiversion_path
}
for task_name, task in all_variant_tasks.items()
if task.resmoke_suite in suites
}
log.debug("Found task list", task_list=task_list)
return task_list
def _get_evg_build_variant_by_name(evergreen_conf: EvergreenProjectConfig, name: str) -> Variant:
"""
Get the evergreen build variant by name from the evergreen config file.
:param evergreen_conf: The evergreen config file.
:param name: The build variant name to find.
:return: The evergreen build variant.
"""
evg_build_variant = evergreen_conf.get_variant(name)
if not evg_build_variant:
LOGGER.warning("Build variant not found in evergreen config")
raise ValueError(f"Build variant ({name} not found in evergreen configuration")
return evg_build_variant
def _generate_evg_tasks(evergreen_api: EvergreenApi,
shrub_project: ShrubProject,
task_expansions: Dict[str, Any],
build_variant_map: Dict[str, str], repos: List[Repo],
evg_conf: EvergreenProjectConfig,
install_dir: str) -> None:
"""
Generate burn in tests tasks for a given shrub config and group of build variants.
:param evergreen_api: Evergreen.py object.
:param shrub_project: Shrub config object that the build variants will be built upon.
:param task_expansions: Dictionary of expansions for the running task.
:param build_variant_map: Map of base buildvariants to their generated buildvariant.
:param repos: Git repositories.
"""
for build_variant, run_build_variant in build_variant_map.items():
config_options = _get_config_options(task_expansions, build_variant,
run_build_variant)
task_id = task_expansions[TASK_ID_EXPANSION]
change_detector = EvergreenFileChangeDetector(task_id, evergreen_api,
os.environ)
changed_tests = change_detector.find_changed_tests(repos)
tests_by_task = create_tests_by_task(build_variant, evg_conf,
changed_tests, install_dir)
if tests_by_task:
shrub_build_variant = _generate_evg_build_variant(
evg_conf.get_variant(build_variant), run_build_variant,
task_expansions["build_variant"])
gen_config = GenerateConfig(
build_variant,
config_options.project,
run_build_variant,
config_options.distro,
include_gen_task=False).validate(evg_conf)
repeat_config = RepeatConfig(
repeat_tests_min=config_options.repeat_tests_min,
repeat_tests_max=config_options.repeat_tests_max,
repeat_tests_secs=config_options.repeat_tests_secs)
burn_in_generator = GenerateBurnInExecutor(gen_config,
repeat_config,
evergreen_api)
burn_in_generator.generate_tasks_for_variant(
tests_by_task, shrub_build_variant)
shrub_project.add_build_variant(shrub_build_variant)
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str,
str], changed_files: Set[str],
origin_build_variants: List[str]) -> Dict[str, dict]:
"""
Run code to select tasks to run based on test and task mappings for each of the build variants.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param changed_files: Set of changed_files.
:param origin_build_variants: Build variants to collect task info from.
:return: Dict of files and file contents for generated tasks.
"""
shrub_config = Configuration()
config_dict_of_suites_and_tasks = {}
for build_variant in origin_build_variants:
build_variant_config = evg_conf.get_variant(build_variant)
origin_variant_expansions = build_variant_config.expansions
task_configs = _get_task_configs(evg_conf, selected_tests_service,
selected_tests_variant_expansions,
build_variant_config, changed_files)
for task_config in task_configs.values():
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
selected_tests_variant_expansions,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_config, config_options,
config_dict_of_suites_and_tasks)
config_dict_of_suites_and_tasks[
"selected_tests_config.json"] = shrub_config.to_json()
return config_dict_of_suites_and_tasks
def _generate_evg_tasks(evergreen_api: EvergreenApi,
shrub_project: ShrubProject,
task_expansions: Dict[str, Any],
build_variant_map: Dict[str,
str], repos: Iterable[Repo],
evg_conf: EvergreenProjectConfig) -> None:
"""
Generate burn in tests tasks for a given shrub config and group of build variants.
:param evergreen_api: Evergreen.py object.
:param shrub_project: Shrub config object that the build variants will be built upon.
:param task_expansions: Dictionary of expansions for the running task.
:param build_variant_map: Map of base buildvariants to their generated buildvariant.
:param repos: Git repositories.
"""
for build_variant, run_build_variant in build_variant_map.items():
config_options = _get_config_options(task_expansions, build_variant,
run_build_variant)
changed_tests = find_changed_tests(repos)
tests_by_task = create_tests_by_task(build_variant, evg_conf,
changed_tests)
if tests_by_task:
shrub_build_variant = _generate_evg_build_variant(
evg_conf.get_variant(build_variant), run_build_variant,
task_expansions["build_variant"])
gen_config = GenerateConfig(
build_variant, config_options.project, run_build_variant,
config_options.distro).validate(evg_conf)
repeat_config = RepeatConfig(
repeat_tests_min=config_options.repeat_tests_min,
repeat_tests_max=config_options.repeat_tests_max,
repeat_tests_secs=config_options.repeat_tests_secs)
create_generate_tasks_config(shrub_build_variant,
tests_by_task,
gen_config,
repeat_config,
evergreen_api,
include_gen_task=False)
shrub_project.add_build_variant(shrub_build_variant)
def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig,
changed_tests: Set[str]) -> Dict[str, TaskInfo]:
"""
Create a list of tests by task.
:param build_variant: Build variant to collect tasks from.
:param evg_conf: Evergreen configuration.
:param changed_tests: Set of changed test files.
:return: Tests by task.
"""
exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE)
if not evg_conf.get_variant(build_variant).is_enterprise_build():
exclude_tests.append(f"{ENTERPRISE_MODULE_PATH}/**/*")
changed_tests = filter_tests(changed_tests, exclude_tests)
buildscripts.resmokelib.parser.set_run_options()
if changed_tests:
return create_task_list_for_tests(changed_tests, build_variant, evg_conf, exclude_suites,
exclude_tasks)
LOGGER.info("No new or modified tests found.")
return {}
def _get_evg_task_configuration(
evg_conf: EvergreenProjectConfig,
build_variant: str,
task_name: str,
test_list_info: dict,
):
"""
Look up task config of the task to be generated.
:param evg_conf: Evergreen configuration.
:param build_variant: Build variant to collect task info from.
:param task_name: Name of task to get info for.
:param test_list_info: The value for a given task_name in the tests_by_task dict.
return: Task configuration values.
"""
evg_build_variant = evg_conf.get_variant(build_variant)
task = evg_build_variant.get_task(task_name)
if task.is_generate_resmoke_task:
task_vars = task.generate_resmoke_tasks_command["vars"]
else:
task_vars = task.run_tests_command["vars"]
task_vars.update({"fallback_num_sub_suites": "1"})
suite_name = ResmokeArgs.get_arg(task_vars["resmoke_args"], "suites")
if suite_name:
task_vars.update({"suite": suite_name})
resmoke_args_without_suites = ResmokeArgs.remove_arg(
task_vars["resmoke_args"], "suites")
task_vars["resmoke_args"] = resmoke_args_without_suites
return {
"task_name": task_name,
"build_variant": build_variant,
"selected_tests_to_run": set(test_list_info["tests"]),
**task_vars
}
def create_task_list(evergreen_conf: EvergreenProjectConfig, build_variant: str,
tests_by_suite: Dict[str, List[str]],
exclude_tasks: [str]) -> Dict[str, TaskInfo]:
"""
Find associated tasks for the specified build_variant and suites.
:param evergreen_conf: Evergreen configuration for project.
:param build_variant: Build variant to select tasks from.
:param tests_by_suite: Suites to be run.
:param exclude_tasks: Tasks to exclude.
:return: Dict of tasks to run with run configuration.
"""
log = LOGGER.bind(build_variant=build_variant)
log.debug("creating task list for suites", suites=tests_by_suite, exclude_tasks=exclude_tasks)
evg_build_variant = evergreen_conf.get_variant(build_variant)
if not evg_build_variant:
log.warning("Buildvariant not found in evergreen config")
raise ValueError(f"Buildvariant ({build_variant} not found in evergreen configuration")
# Find all the build variant tasks.
exclude_tasks_set = set(exclude_tasks)
all_variant_tasks = {
task.name: task
for task in evg_build_variant.tasks
if task.name not in exclude_tasks_set and task.combined_resmoke_args
}
# Return the list of tasks to run for the specified suite.
task_list = {
task_name: TaskInfo.from_task(task, tests_by_suite, evergreen_conf, build_variant)
for task_name, task in all_variant_tasks.items() if task.resmoke_suite in tests_by_suite
}
log.debug("Found task list", task_list=task_list)
return task_list
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
expansion_file: str, selected_tests_service: SelectedTestsService,
changed_files: Set[str], build_variant: str) -> Dict[str, dict]:
"""
Run code to select tasks to run based on test mappings and task mappings.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param expansion_file: Configuration file.
:param selected_tests_config: Location of config file to connect to elected-tests service.
:param changed_files: Set of changed_files.
:param build_variant: Build variant to collect task info from.
:return: Dict of files and file contents for generated tasks.
"""
shrub_config = Configuration()
config_dict_of_suites_and_tasks = {}
task_configs = {}
build_variant_config = evg_conf.get_variant(build_variant)
related_test_files = _find_selected_test_files(selected_tests_service,
changed_files)
LOGGER.debug("related test files found",
related_test_files=related_test_files)
if related_test_files:
tests_by_task = create_task_list_for_tests(related_test_files,
build_variant, evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
test_mapping_task_configs = _get_task_configs_for_test_mappings(
expansion_file, tests_by_task, build_variant_config)
task_configs.update(test_mapping_task_configs)
related_tasks = _find_selected_tasks(selected_tests_service, changed_files,
build_variant_config)
LOGGER.debug("related tasks found", related_tasks=related_tasks)
if related_tasks:
task_mapping_task_configs = _get_task_configs_for_task_mappings(
expansion_file, related_tasks, build_variant_config)
# task_mapping_task_configs will overwrite test_mapping_task_configs
# because task_mapping_task_configs will run all tests rather than a subset of tests and we
# should err on the side of running all tests
task_configs.update(task_mapping_task_configs)
origin_variant_expansions = build_variant_config.expansions
for task_config in task_configs.values():
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
expansion_file,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_config, config_options,
config_dict_of_suites_and_tasks)
config_dict_of_suites_and_tasks[
"selected_tests_config.json"] = shrub_config.to_json()
return config_dict_of_suites_and_tasks
