def _distro_to_run_task_on(task: VariantTask, evg_proj_config: EvergreenProjectConfig, build_variant: str) -> str: """ Determine what distro an task should be run on. For normal tasks, the distro will be the default for the build variant unless the task spec specifies a particular distro to run on. For generated tasks, the distro will be the default for the build variant unless (1) the "use_large_distro" flag is set as a "var" in the "generate resmoke tasks" command of the task definition and (2) the build variant defines the "large_distro_name" in its expansions. :param task: Task being run. :param evg_proj_config: Evergreen project configuration. :param build_variant: Build Variant task is being run on. :return: Distro task should be run on. """ task_def = evg_proj_config.get_task(task.name) if task_def.is_generate_resmoke_task: resmoke_vars = task_def.generate_resmoke_tasks_command["vars"] if "use_large_distro" in resmoke_vars: bv = evg_proj_config.get_variant(build_variant) if "large_distro_name" in bv.raw["expansions"]: return bv.raw["expansions"]["large_distro_name"] return task.run_on[0]
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, selected_tests_service: SelectedTestsService, selected_tests_variant_expansions: Dict[str, str], repos: List[Repo]) -> Dict[str, str]: # pylint: disable=too-many-locals """ Run code to select tasks to run based on test and task mappings for each of the build variants. :param evg_api: Evergreen API object. :param evg_conf: Evergreen configuration. :param selected_tests_service: Selected-tests service. :param selected_tests_variant_expansions: Expansions of the selected-tests variant. :param repos: List of repos containing changed files. :return: Dict of files and file contents for generated tasks. """ config_dict_of_suites_and_tasks = {} task_id = selected_tests_variant_expansions[TASK_ID_EXPANSION] revision_map = generate_revision_map_from_manifest(repos, task_id, evg_api) changed_files = find_changed_files_in_repos(repos, revision_map) changed_files = { _remove_repo_path_prefix(file_path) for file_path in changed_files } LOGGER.info("Found changed files", files=changed_files) shrub_project = ShrubProject() for build_variant_config in evg_conf.get_required_variants(): shrub_build_variant = BuildVariant(build_variant_config.name) origin_variant_expansions = build_variant_config.expansions task_configs = _get_task_configs(evg_conf, selected_tests_service, selected_tests_variant_expansions, build_variant_config, changed_files) remove_task_configs_already_in_build( task_configs, evg_api, build_variant_config, selected_tests_variant_expansions["version_id"]) for task_config in task_configs.values(): Suite.reset_current_index() config_options = SelectedTestsConfigOptions.from_file( origin_variant_expansions, selected_tests_variant_expansions, task_config, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES, CONFIG_FORMAT_FN, ) _update_config_with_task(evg_api, shrub_build_variant, config_options, config_dict_of_suites_and_tasks) shrub_project.add_build_variant(shrub_build_variant) config_dict_of_suites_and_tasks[ "selected_tests_config.json"] = shrub_project.json() return config_dict_of_suites_and_tasks
def _check_variant(build_variant: str, evg_conf: EvergreenProjectConfig): """ Check if the build_variant is found in the evergreen file. :param build_variant: Build variant to check. :param evg_conf: Evergreen configuration to check against. """ if not evg_conf.get_variant(build_variant): raise ValueError(f"Build variant '{build_variant}' not found in Evergreen file")
def create_generate_tasks_config( evg_config: Configuration, tests_by_task: Dict, generate_config: GenerateConfig, repeat_config: RepeatConfig, evg_api: Optional[EvergreenApi], evg_project_config: EvergreenProjectConfig, include_gen_task: bool = True, task_prefix: str = "burn_in") -> Configuration: # pylint: disable=too-many-arguments,too-many-locals """ Create the config for the Evergreen generate.tasks file. :param evg_config: Shrub configuration to add to. :param tests_by_task: Dictionary of tests to generate tasks for. :param generate_config: Configuration of what to generate. :param repeat_config: Configuration of how to repeat tests. :param evg_api: Evergreen API. :param include_gen_task: Should generating task be include in display task. :param task_prefix: Prefix all task names with this. :return: Shrub configuration with added tasks. """ task_list = TaskList(evg_config) resmoke_options = repeat_config.generate_resmoke_options() for task in sorted(tests_by_task): test_list = tests_by_task[task]["tests"] for index, test in enumerate(test_list): if task in evg_project_config.get_task_names_by_tag(RANDOM_MULTIVERSION_REPLSETS_TAG): # Exclude files that should be blacklisted from multiversion testing. task_name = gen_resmoke.remove_gen_suffix(task) files_to_exclude = gen_multiversion.get_exclude_files(task_name, TASK_PATH_SUFFIX) if test in files_to_exclude: LOGGER.debug("Files to exclude", files_to_exclude=files_to_exclude, test=test, suite=task) continue multiversion_path = tests_by_task[task].get("use_multiversion") display_task_name = tests_by_task[task]["display_task_name"] task_runtime_stats = _get_task_runtime_history( evg_api, generate_config.project, display_task_name, generate_config.build_variant) resmoke_args = tests_by_task[task]["resmoke_args"] distro = tests_by_task[task].get("distro", generate_config.distro) # Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join # here, just use the forward slash; otherwise the path separator will be treated as # the escape character on Windows. sub_task_name = name_generated_task(f"{task_prefix}:{display_task_name}", index, len(test_list), generate_config.run_build_variant) LOGGER.debug("Generating sub-task", sub_task=sub_task_name) test_unix_style = test.replace('\\', '/') run_tests_vars = {"resmoke_args": f"{resmoke_args} {resmoke_options} {test_unix_style}"} if multiversion_path: run_tests_vars["task_path_suffix"] = multiversion_path timeout = _generate_timeouts(repeat_config, test, task_runtime_stats) commands = resmoke_commands("run tests", run_tests_vars, timeout, multiversion_path) task_list.add_task(sub_task_name, commands, ["compile"], distro) existing_tasks = [BURN_IN_TESTS_GEN_TASK] if include_gen_task else None task_list.add_to_variant(generate_config.run_build_variant, BURN_IN_TESTS_TASK, existing_tasks) return evg_config
def build_mock_evg_project_config(): tasks = [{"name": "task name"}] buildvariants = [{ "name": "build variant", "tasks": tasks, }] project_conf = { "buildvariants": buildvariants, "tasks": tasks, } return EvergreenProjectConfig(project_conf)
def create_task_list(evergreen_conf: EvergreenProjectConfig, build_variant: str, suites: Dict, exclude_tasks: [str]): """ Find associated tasks for the specified build_variant and suites. Returns a dict keyed by task_name, with executor, resmoke_args & tests, i.e., {'jsCore_small_oplog': {'resmoke_args': '--suites=core_small_oplog --storageEngine=inMemory', 'tests': ['jstests/core/all2.js', 'jstests/core/all3.js'], 'use_multiversion': '/data/multiversion'} } :param evergreen_conf: Evergreen configuration for project. :param build_variant: Build variant to select tasks from. :param suites: Suites to be run. :param exclude_tasks: Tasks to exclude. :return: Dict of tasks to run with run configuration. """ log = LOGGER.bind(build_variant=build_variant) log.debug("creating task list for suites", suites=suites, exclude_tasks=exclude_tasks) evg_build_variant = evergreen_conf.get_variant(build_variant) if not evg_build_variant: log.warning("Buildvariant not found in evergreen config") raise ValueError( f"Buildvariant ({build_variant} not found in evergreen configuration" ) # Find all the build variant tasks. exclude_tasks_set = set(exclude_tasks) all_variant_tasks = { _get_task_name(task): task for task in evg_build_variant.tasks if task.name not in exclude_tasks_set and task.combined_resmoke_args } # Return the list of tasks to run for the specified suite. task_list = { task_name: { "resmoke_args": _set_resmoke_args(task), "tests": suites[task.resmoke_suite], "use_multiversion": task.multiversion_path } for task_name, task in all_variant_tasks.items() if task.resmoke_suite in suites } log.debug("Found task list", task_list=task_list) return task_list
def _get_evg_build_variant_by_name(evergreen_conf: EvergreenProjectConfig, name: str) -> Variant: """ Get the evergreen build variant by name from the evergreen config file. :param evergreen_conf: The evergreen config file. :param name: The build variant name to find. :return: The evergreen build variant. """ evg_build_variant = evergreen_conf.get_variant(name) if not evg_build_variant: LOGGER.warning("Build variant not found in evergreen config") raise ValueError(f"Build variant ({name} not found in evergreen configuration") return evg_build_variant
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, selected_tests_service: SelectedTestsService, selected_tests_variant_expansions: Dict[str, str], repos: List[Repo], origin_build_variants: List[str]) -> Dict[str, str]: # pylint: disable=too-many-locals """ Run code to select tasks to run based on test and task mappings for each of the build variants. :param evg_api: Evergreen API object. :param evg_conf: Evergreen configuration. :param selected_tests_service: Selected-tests service. :param selected_tests_variant_expansions: Expansions of the selected-tests variant. :param repos: List of repos containing changed files. :param origin_build_variants: Build variants to collect task info from. :return: Dict of files and file contents for generated tasks. """ config_dict_of_suites_and_tasks = {} changed_files = find_changed_files_in_repos(repos) changed_files = {_remove_repo_path_prefix(file_path) for file_path in changed_files} LOGGER.debug("Found changed files", files=changed_files) shrub_project = ShrubProject() for build_variant in origin_build_variants: shrub_build_variant = BuildVariant(build_variant) build_variant_config = evg_conf.get_variant(build_variant) origin_variant_expansions = build_variant_config.expansions task_configs = _get_task_configs(evg_conf, selected_tests_service, selected_tests_variant_expansions, build_variant_config, changed_files) for task_config in task_configs.values(): config_options = SelectedTestsConfigOptions.from_file( origin_variant_expansions, selected_tests_variant_expansions, task_config, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES, CONFIG_FORMAT_FN, ) _update_config_with_task(evg_api, shrub_build_variant, config_options, config_dict_of_suites_and_tasks) shrub_project.add_build_variant(shrub_build_variant) config_dict_of_suites_and_tasks["selected_tests_config.json"] = shrub_project.json() return config_dict_of_suites_and_tasks
def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmoke_args: str, generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig, repos: Iterable[Repo], evg_api: EvergreenApi): """ Run burn_in_tests with the given configuration. :param repeat_config: Config on how much to repeat tests. :param generate_config: Config on how to generate tests. :param resmoke_args: Arguments to pass to resmoke. :param generate_tasks_file: File to write generated config to. :param no_exec: Do not execute tests, just discover tests to run. :param evg_conf: Evergreen configuration. :param repos: Git repositories to check. :param evg_api: Evergreen API client. """ # Populate the config values in order to use the helpers from resmokelib.suitesconfig. resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args)) tests_by_task = create_tests_by_task(generate_config.build_variant, repos, evg_conf) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) if generate_tasks_file: if generate_config.use_multiversion: multiversion_tasks = evg_conf.get_task_names_by_tag( MULTIVERSION_PASSTHROUGH_TAG) LOGGER.debug("Multiversion tasks by tag", tasks=multiversion_tasks, tag=MULTIVERSION_PASSTHROUGH_TAG) # We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in # multiversion_suites. Multiversion passthrough suites must include # MULTIVERSION_CONFIG_KEY as a root level key and must be set to true. multiversion_suites = get_named_suites_with_root_level_key( MULTIVERSION_CONFIG_KEY) assert len(multiversion_tasks) == len(multiversion_suites) json_config = create_generate_tasks_file(tests_by_task, generate_config, repeat_config, evg_api, evg_conf) _write_json_file(json_config, generate_tasks_file) elif not no_exec: run_tests(tests_by_task, resmoke_cmd) else: LOGGER.info("Not running tests due to 'no_exec' option.")
def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject, task_expansions: Dict[str, Any], build_variant_map: Dict[str, str], repos: List[Repo], evg_conf: EvergreenProjectConfig, install_dir: str) -> None: """ Generate burn in tests tasks for a given shrub config and group of build variants. :param evergreen_api: Evergreen.py object. :param shrub_project: Shrub config object that the build variants will be built upon. :param task_expansions: Dictionary of expansions for the running task. :param build_variant_map: Map of base buildvariants to their generated buildvariant. :param repos: Git repositories. """ for build_variant, run_build_variant in build_variant_map.items(): config_options = _get_config_options(task_expansions, build_variant, run_build_variant) task_id = task_expansions[TASK_ID_EXPANSION] change_detector = EvergreenFileChangeDetector(task_id, evergreen_api, os.environ) changed_tests = change_detector.find_changed_tests(repos) tests_by_task = create_tests_by_task(build_variant, evg_conf, changed_tests, install_dir) if tests_by_task: shrub_build_variant = _generate_evg_build_variant( evg_conf.get_variant(build_variant), run_build_variant, task_expansions["build_variant"]) gen_config = GenerateConfig( build_variant, config_options.project, run_build_variant, config_options.distro, include_gen_task=False).validate(evg_conf) repeat_config = RepeatConfig( repeat_tests_min=config_options.repeat_tests_min, repeat_tests_max=config_options.repeat_tests_max, repeat_tests_secs=config_options.repeat_tests_secs) burn_in_generator = GenerateBurnInExecutor(gen_config, repeat_config, evergreen_api) burn_in_generator.generate_tasks_for_variant( tests_by_task, shrub_build_variant) shrub_project.add_build_variant(shrub_build_variant)
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, selected_tests_service: SelectedTestsService, selected_tests_variant_expansions: Dict[str, str], changed_files: Set[str], origin_build_variants: List[str]) -> Dict[str, dict]: """ Run code to select tasks to run based on test and task mappings for each of the build variants. :param evg_api: Evergreen API object. :param evg_conf: Evergreen configuration. :param selected_tests_service: Selected-tests service. :param selected_tests_variant_expansions: Expansions of the selected-tests variant. :param changed_files: Set of changed_files. :param origin_build_variants: Build variants to collect task info from. :return: Dict of files and file contents for generated tasks. """ shrub_config = Configuration() config_dict_of_suites_and_tasks = {} for build_variant in origin_build_variants: build_variant_config = evg_conf.get_variant(build_variant) origin_variant_expansions = build_variant_config.expansions task_configs = _get_task_configs(evg_conf, selected_tests_service, selected_tests_variant_expansions, build_variant_config, changed_files) for task_config in task_configs.values(): config_options = SelectedTestsConfigOptions.from_file( origin_variant_expansions, selected_tests_variant_expansions, task_config, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES, CONFIG_FORMAT_FN, ) _update_config_with_task(evg_api, shrub_config, config_options, config_dict_of_suites_and_tasks) config_dict_of_suites_and_tasks[ "selected_tests_config.json"] = shrub_config.to_json() return config_dict_of_suites_and_tasks
def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject, task_expansions: Dict[str, Any], build_variant_map: Dict[str, str], repos: Iterable[Repo], evg_conf: EvergreenProjectConfig) -> None: """ Generate burn in tests tasks for a given shrub config and group of build variants. :param evergreen_api: Evergreen.py object. :param shrub_project: Shrub config object that the build variants will be built upon. :param task_expansions: Dictionary of expansions for the running task. :param build_variant_map: Map of base buildvariants to their generated buildvariant. :param repos: Git repositories. """ for build_variant, run_build_variant in build_variant_map.items(): config_options = _get_config_options(task_expansions, build_variant, run_build_variant) changed_tests = find_changed_tests(repos) tests_by_task = create_tests_by_task(build_variant, evg_conf, changed_tests) if tests_by_task: shrub_build_variant = _generate_evg_build_variant( evg_conf.get_variant(build_variant), run_build_variant, task_expansions["build_variant"]) gen_config = GenerateConfig( build_variant, config_options.project, run_build_variant, config_options.distro).validate(evg_conf) repeat_config = RepeatConfig( repeat_tests_min=config_options.repeat_tests_min, repeat_tests_max=config_options.repeat_tests_max, repeat_tests_secs=config_options.repeat_tests_secs) create_generate_tasks_config(shrub_build_variant, tests_by_task, gen_config, repeat_config, evergreen_api, include_gen_task=False) shrub_project.add_build_variant(shrub_build_variant)
def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, changed_tests: Set[str]) -> Dict[str, TaskInfo]: """ Create a list of tests by task. :param build_variant: Build variant to collect tasks from. :param evg_conf: Evergreen configuration. :param changed_tests: Set of changed test files. :return: Tests by task. """ exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE) if not evg_conf.get_variant(build_variant).is_enterprise_build(): exclude_tests.append(f"{ENTERPRISE_MODULE_PATH}/**/*") changed_tests = filter_tests(changed_tests, exclude_tests) buildscripts.resmokelib.parser.set_run_options() if changed_tests: return create_task_list_for_tests(changed_tests, build_variant, evg_conf, exclude_suites, exclude_tasks) LOGGER.info("No new or modified tests found.") return {}
def _get_evg_task_configuration( evg_conf: EvergreenProjectConfig, build_variant: str, task_name: str, test_list_info: dict, ): """ Look up task config of the task to be generated. :param evg_conf: Evergreen configuration. :param build_variant: Build variant to collect task info from. :param task_name: Name of task to get info for. :param test_list_info: The value for a given task_name in the tests_by_task dict. return: Task configuration values. """ evg_build_variant = evg_conf.get_variant(build_variant) task = evg_build_variant.get_task(task_name) if task.is_generate_resmoke_task: task_vars = task.generate_resmoke_tasks_command["vars"] else: task_vars = task.run_tests_command["vars"] task_vars.update({"fallback_num_sub_suites": "1"}) suite_name = ResmokeArgs.get_arg(task_vars["resmoke_args"], "suites") if suite_name: task_vars.update({"suite": suite_name}) resmoke_args_without_suites = ResmokeArgs.remove_arg( task_vars["resmoke_args"], "suites") task_vars["resmoke_args"] = resmoke_args_without_suites return { "task_name": task_name, "build_variant": build_variant, "selected_tests_to_run": set(test_list_info["tests"]), **task_vars }
def create_task_list(evergreen_conf: EvergreenProjectConfig, build_variant: str, tests_by_suite: Dict[str, List[str]], exclude_tasks: [str]) -> Dict[str, TaskInfo]: """ Find associated tasks for the specified build_variant and suites. :param evergreen_conf: Evergreen configuration for project. :param build_variant: Build variant to select tasks from. :param tests_by_suite: Suites to be run. :param exclude_tasks: Tasks to exclude. :return: Dict of tasks to run with run configuration. """ log = LOGGER.bind(build_variant=build_variant) log.debug("creating task list for suites", suites=tests_by_suite, exclude_tasks=exclude_tasks) evg_build_variant = evergreen_conf.get_variant(build_variant) if not evg_build_variant: log.warning("Buildvariant not found in evergreen config") raise ValueError(f"Buildvariant ({build_variant} not found in evergreen configuration") # Find all the build variant tasks. exclude_tasks_set = set(exclude_tasks) all_variant_tasks = { task.name: task for task in evg_build_variant.tasks if task.name not in exclude_tasks_set and task.combined_resmoke_args } # Return the list of tasks to run for the specified suite. task_list = { task_name: TaskInfo.from_task(task, tests_by_suite, evergreen_conf, build_variant) for task_name, task in all_variant_tasks.items() if task.resmoke_suite in tests_by_suite } log.debug("Found task list", task_list=task_list) return task_list
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, expansion_file: str, selected_tests_service: SelectedTestsService, changed_files: Set[str], build_variant: str) -> Dict[str, dict]: """ Run code to select tasks to run based on test mappings and task mappings. :param evg_api: Evergreen API object. :param evg_conf: Evergreen configuration. :param expansion_file: Configuration file. :param selected_tests_config: Location of config file to connect to elected-tests service. :param changed_files: Set of changed_files. :param build_variant: Build variant to collect task info from. :return: Dict of files and file contents for generated tasks. """ shrub_config = Configuration() config_dict_of_suites_and_tasks = {} task_configs = {} build_variant_config = evg_conf.get_variant(build_variant) related_test_files = _find_selected_test_files(selected_tests_service, changed_files) LOGGER.debug("related test files found", related_test_files=related_test_files) if related_test_files: tests_by_task = create_task_list_for_tests(related_test_files, build_variant, evg_conf) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) test_mapping_task_configs = _get_task_configs_for_test_mappings( expansion_file, tests_by_task, build_variant_config) task_configs.update(test_mapping_task_configs) related_tasks = _find_selected_tasks(selected_tests_service, changed_files, build_variant_config) LOGGER.debug("related tasks found", related_tasks=related_tasks) if related_tasks: task_mapping_task_configs = _get_task_configs_for_task_mappings( expansion_file, related_tasks, build_variant_config) # task_mapping_task_configs will overwrite test_mapping_task_configs # because task_mapping_task_configs will run all tests rather than a subset of tests and we # should err on the side of running all tests task_configs.update(task_mapping_task_configs) origin_variant_expansions = build_variant_config.expansions for task_config in task_configs.values(): config_options = SelectedTestsConfigOptions.from_file( origin_variant_expansions, expansion_file, task_config, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES, CONFIG_FORMAT_FN, ) _update_config_with_task(evg_api, shrub_config, config_options, config_dict_of_suites_and_tasks) config_dict_of_suites_and_tasks[ "selected_tests_config.json"] = shrub_config.to_json() return config_dict_of_suites_and_tasks