def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, selected_tests_service: SelectedTestsService, selected_tests_variant_expansions: Dict[str, str], repos: List[Repo]) -> Dict[str, str]: # pylint: disable=too-many-locals """ Run code to select tasks to run based on test and task mappings for each of the build variants. :param evg_api: Evergreen API object. :param evg_conf: Evergreen configuration. :param selected_tests_service: Selected-tests service. :param selected_tests_variant_expansions: Expansions of the selected-tests variant. :param repos: List of repos containing changed files. :return: Dict of files and file contents for generated tasks. """ config_dict_of_suites_and_tasks = {} task_id = selected_tests_variant_expansions[TASK_ID_EXPANSION] revision_map = generate_revision_map_from_manifest(repos, task_id, evg_api) changed_files = find_changed_files_in_repos(repos, revision_map) changed_files = { _remove_repo_path_prefix(file_path) for file_path in changed_files } LOGGER.info("Found changed files", files=changed_files) shrub_project = ShrubProject() for build_variant_config in evg_conf.get_required_variants(): shrub_build_variant = BuildVariant(build_variant_config.name) origin_variant_expansions = build_variant_config.expansions task_configs = _get_task_configs(evg_conf, selected_tests_service, selected_tests_variant_expansions, build_variant_config, changed_files) remove_task_configs_already_in_build( task_configs, evg_api, build_variant_config, selected_tests_variant_expansions["version_id"]) for task_config in task_configs.values(): Suite.reset_current_index() config_options = SelectedTestsConfigOptions.from_file( origin_variant_expansions, selected_tests_variant_expansions, task_config, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES, CONFIG_FORMAT_FN, ) _update_config_with_task(evg_api, shrub_build_variant, config_options, config_dict_of_suites_and_tasks) shrub_project.add_build_variant(shrub_build_variant) config_dict_of_suites_and_tasks[ "selected_tests_config.json"] = shrub_project.json() return config_dict_of_suites_and_tasks
def find_changed_tests(repos: Iterable[Repo]) -> Set[str]: """ Find the changed tests. Use git to find which test files have changed in this patch. The returned file paths are in normalized form (see os.path.normpath(path)). :param repos: List of repos containing changed files. :returns: Set of changed tests. """ changed_files = find_changed_files_in_repos(repos) return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
def find_changed_tests(self, repos: List[Repo]) -> Set[str]: """ Find the changed tests. Use git to find which test files have changed in this patch. The returned file paths are in normalized form (see os.path.normpath(path)). :param repos: List of repos containing changed files. :return: Set of changed tests. """ revision_map = self.create_revision_map(repos) LOGGER.info("Calculated revision map", revision_map=revision_map) changed_files = find_changed_files_in_repos(repos, revision_map) return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
def find_changed_tests(repos: Iterable[Repo], origin_rev: Optional[str] = None) -> Set[str]: """ Find the changed tests. Use git to find which test files have changed in this patch. The returned file paths are in normalized form (see os.path.normpath(path)). :param repos: List of repos containing changed files. :param origin_rev: The revision that local changes will be compared against. :returns: Set of changed tests. """ revision_map = get_revision_map(repos, origin_rev) LOGGER.info("Calculated revision map", revision_map=revision_map) changed_files = find_changed_files_in_repos(repos, revision_map) return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
def test_changed_files_in_multiple_repos(self, changed_files_mock): repos_mock = [MagicMock(), MagicMock()] first_repo_file_changes = [ os.path.join("jstests", "test1.js"), os.path.join("jstests", "test1.cpp"), ] second_repo_file_changes = [ os.path.join("jstests", "test2.js"), ] changed_files_mock.side_effect = [ first_repo_file_changes, second_repo_file_changes ] self.assertEqual( 3, len(under_test.find_changed_files_in_repos(repos_mock)))
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, selected_tests_service: SelectedTestsService, selected_tests_variant_expansions: Dict[str, str], repos: List[Repo], origin_build_variants: List[str]) -> Dict[str, str]: # pylint: disable=too-many-locals """ Run code to select tasks to run based on test and task mappings for each of the build variants. :param evg_api: Evergreen API object. :param evg_conf: Evergreen configuration. :param selected_tests_service: Selected-tests service. :param selected_tests_variant_expansions: Expansions of the selected-tests variant. :param repos: List of repos containing changed files. :param origin_build_variants: Build variants to collect task info from. :return: Dict of files and file contents for generated tasks. """ config_dict_of_suites_and_tasks = {} changed_files = find_changed_files_in_repos(repos) changed_files = {_remove_repo_path_prefix(file_path) for file_path in changed_files} LOGGER.debug("Found changed files", files=changed_files) shrub_project = ShrubProject() for build_variant in origin_build_variants: shrub_build_variant = BuildVariant(build_variant) build_variant_config = evg_conf.get_variant(build_variant) origin_variant_expansions = build_variant_config.expansions task_configs = _get_task_configs(evg_conf, selected_tests_service, selected_tests_variant_expansions, build_variant_config, changed_files) for task_config in task_configs.values(): config_options = SelectedTestsConfigOptions.from_file( origin_variant_expansions, selected_tests_variant_expansions, task_config, REQUIRED_CONFIG_KEYS, DEFAULT_CONFIG_VALUES, CONFIG_FORMAT_FN, ) _update_config_with_task(evg_api, shrub_build_variant, config_options, config_dict_of_suites_and_tasks) shrub_project.add_build_variant(shrub_build_variant) config_dict_of_suites_and_tasks["selected_tests_config.json"] = shrub_project.json() return config_dict_of_suites_and_tasks
def find_changed_files(self, repos: List[Repo], task_id: str) -> Set[str]: """ Determine what files have changed in the given repos. :param repos: List of git repos to query. :param task_id: ID of task being run. :return: Set of files that contain changes. """ revision_map = generate_revision_map_from_manifest(repos, task_id, self.evg_api) changed_files = find_changed_files_in_repos(repos, revision_map) changed_files = {_remove_repo_path_prefix(file_path) for file_path in changed_files} changed_files = { file_path for file_path in changed_files if not file_path.startswith("src/third_party") } LOGGER.info("Found changed files", files=changed_files) return changed_files
def gather_changed_files_for_lint(is_interesting_file: Callable) -> List[str]: """ Get the files that have changes since the last git commit. :param is_interesting_file: Filter for whether a file should be returned. :return: List of files for linting. """ repos, revision_map = _get_repos_and_revisions() LOGGER.info("revisions", revision=revision_map) candidate_files = find_changed_files_in_repos(repos, revision_map) files = [ filename for filename in candidate_files if _filter_file(filename, is_interesting_file) ] LOGGER.info("Found files to lint", files=files) return files
def find_changed_tests(repos: List[Repo], origin_rev: Optional[str] = None, evg_api: Optional[EvergreenApi] = None, task_id: Optional[str] = None) -> Set[str]: """ Find the changed tests. Use git to find which test files have changed in this patch. The returned file paths are in normalized form (see os.path.normpath(path)). :param repos: List of repos containing changed files. :param origin_rev: The revision that local changes will be compared against. :param evg_api: Evergreen API client. :param task_id: Evergreen task ID. :return: Set of changed tests. """ revision_map = _create_revision_map(repos, origin_rev, evg_api, task_id) LOGGER.info("Calculated revision map", revision_map=revision_map) changed_files = find_changed_files_in_repos(repos, revision_map) return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
def lint_git_diff(eslint: Optional[str]) -> bool: """ Lint the files that have changes since the last git commit. :param eslint: Path to eslint command. :return: True if lint was successful. """ repos = [Repo(path) for path in git.get_module_paths()] candidate_files = find_changed_files_in_repos(repos) LOGGER.info("Found candidate_files", candidate_files=candidate_files) files = [ filename for filename in candidate_files if is_interesting_file(filename) ] LOGGER.info("Found files to lint", files=files) # Patch may have files that we do not want to check which is fine if files: return _lint_files(eslint, files) return True
def test_nothing_found(self, changed_files_mock): repos_mock = [MagicMock()] changed_files_mock.return_value = set() self.assertEqual( 0, len(under_test.find_changed_files_in_repos(repos_mock)))