def get_task_config(self, build_variant_config: Variant, changed_files: Set[str]) -> Dict[str, Dict]: """ Get task configurations for the tasks to be generated. :param build_variant_config: Config of build variant to collect task info from. :param changed_files: Set of changed_files. :return: Task configurations. """ existing_tasks = self.get_existing_tasks( self.evg_expansions.version_id, build_variant_config.name) task_configs = {} related_test_files = self.selected_tests_service.find_selected_test_files( changed_files) LOGGER.info("related test files found", related_test_files=related_test_files, variant=build_variant_config.name) if related_test_files: tests_by_task = create_task_list_for_tests( related_test_files, build_variant_config.name, self.evg_conf) LOGGER.info("tests and tasks found", tests_by_task=tests_by_task) tests_by_task = { task: tests for task, tests in tests_by_task.items() if task not in existing_tasks } test_mapping_task_configs = self.task_config_service.get_task_configs_for_test_mappings( tests_by_task, build_variant_config) task_configs.update(test_mapping_task_configs) related_tasks = self.selected_tests_service.find_selected_tasks( changed_files) LOGGER.info("related tasks found", related_tasks=related_tasks, variant=build_variant_config.name) related_tasks = { task for task in related_tasks if task not in existing_tasks } if related_tasks: task_mapping_task_configs = self.task_config_service.get_task_configs_for_task_mappings( list(related_tasks), build_variant_config) # task_mapping_task_configs will overwrite test_mapping_task_configs # because task_mapping_task_configs will run all tests rather than a subset of tests # and we should err on the side of running all tests task_configs.update(task_mapping_task_configs) return task_configs
def _get_task_configs(evg_conf: EvergreenProjectConfig, selected_tests_service: SelectedTestsService, selected_tests_variant_expansions: Dict[str, str], build_variant_config: Variant, changed_files: Set[str]) -> Dict[str, Dict]: """ Get task configurations for the tasks to be generated. :param evg_conf: Evergreen configuration. :param selected_tests_service: Selected-tests service. :param selected_tests_variant_expansions: Expansions of the selected-tests variant. :param build_variant_config: Config of build variant to collect task info from. :param changed_files: Set of changed_files. :return: Task configurations. """ task_configs = {} related_test_files = _find_selected_test_files(selected_tests_service, changed_files) LOGGER.debug("related test files found", related_test_files=related_test_files, variant=build_variant_config.name) if related_test_files: tests_by_task = create_task_list_for_tests(related_test_files, build_variant_config.name, evg_conf) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) test_mapping_task_configs = _get_task_configs_for_test_mappings( selected_tests_variant_expansions, tests_by_task, build_variant_config) task_configs.update(test_mapping_task_configs) related_tasks = _find_selected_tasks(selected_tests_service, changed_files, build_variant_config) LOGGER.debug("related tasks found", related_tasks=related_tasks, variant=build_variant_config.name) if related_tasks: task_mapping_task_configs = _get_task_configs_for_task_mappings( selected_tests_variant_expansions, related_tasks, build_variant_config) # task_mapping_task_configs will overwrite test_mapping_task_configs # because task_mapping_task_configs will run all tests rather than a subset of tests and we # should err on the side of running all tests task_configs.update(task_mapping_task_configs) return task_configs
def main( verbose: bool, expansion_file: str, evg_api_config: str, build_variant: str, selected_tests_config: str, ): """ Select tasks to be run based on changed files in a patch build. :param verbose: Log extra debug information. :param expansion_file: Configuration file. :param evg_api_config: Location of configuration file to connect to evergreen. :param build_variant: Build variant to query tasks from. :param selected_tests_config: Location of config file to connect to elected-tests service. """ _configure_logging(verbose) evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config) evg_conf = parse_evergreen_file(EVERGREEN_FILE) selected_tests_service = SelectedTestsService.from_file( selected_tests_config) repo = Repo(".") changed_files = find_changed_files(repo) buildscripts.resmokelib.parser.set_options() LOGGER.debug("Found changed files", files=changed_files) related_test_files = _find_related_test_files(selected_tests_service, changed_files) LOGGER.debug("related test files found", related_test_files=related_test_files) if related_test_files: tests_by_task = create_task_list_for_tests(related_test_files, build_variant, evg_conf) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) config_dict_of_generated_tasks = _generate_shrub_config( evg_api, evg_conf, expansion_file, tests_by_task, build_variant) write_file_dict(SELECTED_TESTS_CONFIG_DIR, config_dict_of_generated_tasks)
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, expansion_file: str, selected_tests_service: SelectedTestsService, changed_files: Set[str], build_variant: str) -> Dict[str, dict]: """ Run code to select tasks to run based on test mappings and task mappings. :param evg_api: Evergreen API object. :param evg_conf: Evergreen configuration. :param expansion_file: Configuration file. :param selected_tests_config: Location of config file to connect to elected-tests service. :param changed_files: Set of changed_files. :param build_variant: Build variant to collect task info from. :return: Dict of files and file contents for generated tasks. """ shrub_config = Configuration() config_dict_of_suites_and_tasks = {} task_configs = {} build_variant_config = evg_conf.get_variant(build_variant) related_test_files = _find_selected_test_files(selected_tests_service, changed_files) LOGGER.debug("related test files found", related_test_files=related_test_files) if related_test_files: tests_by_task = create_task_list_for_tests(related_test_files, build_variant, evg_conf) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) test_mapping_task_configs = _get_task_configs_for_test_mappings( expansion_file, tests_by_task, build_variant_config) task_configs.update(test_mapping_task_configs) related_tasks = _find_selected_tasks(selected_tests_service, changed_files, build_variant_config) LOGGER.debug("related tasks found", related_tasks=related_tasks) if related_tasks: task_mapping_task_configs = _get_task_configs_for_task_mappings( expansion_file, related_tasks, build_variant_config) # task_mapping_task_configs will overwrite test_mapping_task_configs # because task_mapping_task_configs will run all tests rather than a subset of tests and we # should err on the side of running all tests task_configs.update(task_mapping_task_configs) origin_variant_expansions = build_variant_config.expansions for task_config in task_configs.values(): config_options = SelectedTestsConfigOptions.from_file( origin_variant_expansions, expansion_file, task_config, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES, CONFIG_FORMAT_FN, ) _update_config_with_task(evg_api, shrub_config, config_options, config_dict_of_suites_and_tasks) config_dict_of_suites_and_tasks[ "selected_tests_config.json"] = shrub_config.to_json() return config_dict_of_suites_and_tasks