def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str, str],
repos: List[Repo]) -> Dict[str, str]:
# pylint: disable=too-many-locals
"""
Run code to select tasks to run based on test and task mappings for each of the build variants.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param repos: List of repos containing changed files.
:return: Dict of files and file contents for generated tasks.
"""
config_dict_of_suites_and_tasks = {}
task_id = selected_tests_variant_expansions[TASK_ID_EXPANSION]
revision_map = generate_revision_map_from_manifest(repos, task_id, evg_api)
changed_files = find_changed_files_in_repos(repos, revision_map)
changed_files = {
_remove_repo_path_prefix(file_path)
for file_path in changed_files
}
LOGGER.info("Found changed files", files=changed_files)
shrub_project = ShrubProject()
for build_variant_config in evg_conf.get_required_variants():
shrub_build_variant = BuildVariant(build_variant_config.name)
origin_variant_expansions = build_variant_config.expansions
task_configs = _get_task_configs(evg_conf, selected_tests_service,
selected_tests_variant_expansions,
build_variant_config, changed_files)
remove_task_configs_already_in_build(
task_configs, evg_api, build_variant_config,
selected_tests_variant_expansions["version_id"])
for task_config in task_configs.values():
Suite.reset_current_index()
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
selected_tests_variant_expansions,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_build_variant,
config_options,
config_dict_of_suites_and_tasks)
shrub_project.add_build_variant(shrub_build_variant)
config_dict_of_suites_and_tasks[
"selected_tests_config.json"] = shrub_project.json()
return config_dict_of_suites_and_tasks
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str, str], repos: List[Repo],
origin_build_variants: List[str]) -> Dict[str, str]:
# pylint: disable=too-many-locals
"""
Run code to select tasks to run based on test and task mappings for each of the build variants.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param repos: List of repos containing changed files.
:param origin_build_variants: Build variants to collect task info from.
:return: Dict of files and file contents for generated tasks.
"""
config_dict_of_suites_and_tasks = {}
changed_files = find_changed_files_in_repos(repos)
changed_files = {_remove_repo_path_prefix(file_path) for file_path in changed_files}
LOGGER.debug("Found changed files", files=changed_files)
shrub_project = ShrubProject()
for build_variant in origin_build_variants:
shrub_build_variant = BuildVariant(build_variant)
build_variant_config = evg_conf.get_variant(build_variant)
origin_variant_expansions = build_variant_config.expansions
task_configs = _get_task_configs(evg_conf, selected_tests_service,
selected_tests_variant_expansions, build_variant_config,
changed_files)
for task_config in task_configs.values():
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
selected_tests_variant_expansions,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_build_variant, config_options,
config_dict_of_suites_and_tasks)
shrub_project.add_build_variant(shrub_build_variant)
config_dict_of_suites_and_tasks["selected_tests_config.json"] = shrub_project.json()
return config_dict_of_suites_and_tasks
def _generate_evg_tasks(evergreen_api: EvergreenApi,
shrub_project: ShrubProject,
task_expansions: Dict[str, Any],
build_variant_map: Dict[str, str], repos: List[Repo],
evg_conf: EvergreenProjectConfig,
install_dir: str) -> None:
"""
Generate burn in tests tasks for a given shrub config and group of build variants.
:param evergreen_api: Evergreen.py object.
:param shrub_project: Shrub config object that the build variants will be built upon.
:param task_expansions: Dictionary of expansions for the running task.
:param build_variant_map: Map of base buildvariants to their generated buildvariant.
:param repos: Git repositories.
"""
for build_variant, run_build_variant in build_variant_map.items():
config_options = _get_config_options(task_expansions, build_variant,
run_build_variant)
task_id = task_expansions[TASK_ID_EXPANSION]
change_detector = EvergreenFileChangeDetector(task_id, evergreen_api,
os.environ)
changed_tests = change_detector.find_changed_tests(repos)
tests_by_task = create_tests_by_task(build_variant, evg_conf,
changed_tests, install_dir)
if tests_by_task:
shrub_build_variant = _generate_evg_build_variant(
evg_conf.get_variant(build_variant), run_build_variant,
task_expansions["build_variant"])
gen_config = GenerateConfig(
build_variant,
config_options.project,
run_build_variant,
config_options.distro,
include_gen_task=False).validate(evg_conf)
repeat_config = RepeatConfig(
repeat_tests_min=config_options.repeat_tests_min,
repeat_tests_max=config_options.repeat_tests_max,
repeat_tests_secs=config_options.repeat_tests_secs)
burn_in_generator = GenerateBurnInExecutor(gen_config,
repeat_config,
evergreen_api)
burn_in_generator.generate_tasks_for_variant(
tests_by_task, shrub_build_variant)
shrub_project.add_build_variant(shrub_build_variant)
def _generate_evg_tasks(evergreen_api: EvergreenApi,
shrub_project: ShrubProject,
task_expansions: Dict[str, Any],
build_variant_map: Dict[str,
str], repos: Iterable[Repo],
evg_conf: EvergreenProjectConfig) -> None:
"""
Generate burn in tests tasks for a given shrub config and group of build variants.
:param evergreen_api: Evergreen.py object.
:param shrub_project: Shrub config object that the build variants will be built upon.
:param task_expansions: Dictionary of expansions for the running task.
:param build_variant_map: Map of base buildvariants to their generated buildvariant.
:param repos: Git repositories.
"""
for build_variant, run_build_variant in build_variant_map.items():
config_options = _get_config_options(task_expansions, build_variant,
run_build_variant)
changed_tests = find_changed_tests(repos)
tests_by_task = create_tests_by_task(build_variant, evg_conf,
changed_tests)
if tests_by_task:
shrub_build_variant = _generate_evg_build_variant(
evg_conf.get_variant(build_variant), run_build_variant,
task_expansions["build_variant"])
gen_config = GenerateConfig(
build_variant, config_options.project, run_build_variant,
config_options.distro).validate(evg_conf)
repeat_config = RepeatConfig(
repeat_tests_min=config_options.repeat_tests_min,
repeat_tests_max=config_options.repeat_tests_max,
repeat_tests_secs=config_options.repeat_tests_secs)
create_generate_tasks_config(shrub_build_variant,
tests_by_task,
gen_config,
repeat_config,
evergreen_api,
include_gen_task=False)
shrub_project.add_build_variant(shrub_build_variant)