def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str, str],
repos: List[Repo]) -> Dict[str, str]:
# pylint: disable=too-many-locals
"""
Run code to select tasks to run based on test and task mappings for each of the build variants.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param repos: List of repos containing changed files.
:return: Dict of files and file contents for generated tasks.
"""
config_dict_of_suites_and_tasks = {}
task_id = selected_tests_variant_expansions[TASK_ID_EXPANSION]
revision_map = generate_revision_map_from_manifest(repos, task_id, evg_api)
changed_files = find_changed_files_in_repos(repos, revision_map)
changed_files = {
_remove_repo_path_prefix(file_path)
for file_path in changed_files
}
LOGGER.info("Found changed files", files=changed_files)
shrub_project = ShrubProject()
for build_variant_config in evg_conf.get_required_variants():
shrub_build_variant = BuildVariant(build_variant_config.name)
origin_variant_expansions = build_variant_config.expansions
task_configs = _get_task_configs(evg_conf, selected_tests_service,
selected_tests_variant_expansions,
build_variant_config, changed_files)
remove_task_configs_already_in_build(
task_configs, evg_api, build_variant_config,
selected_tests_variant_expansions["version_id"])
for task_config in task_configs.values():
Suite.reset_current_index()
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
selected_tests_variant_expansions,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_build_variant,
config_options,
config_dict_of_suites_and_tasks)
shrub_project.add_build_variant(shrub_build_variant)
config_dict_of_suites_and_tasks[
"selected_tests_config.json"] = shrub_project.json()
return config_dict_of_suites_and_tasks
def find_changed_tests(repos: Iterable[Repo]) -> Set[str]:
"""
Find the changed tests.
Use git to find which test files have changed in this patch.
The returned file paths are in normalized form (see os.path.normpath(path)).
:param repos: List of repos containing changed files.
:returns: Set of changed tests.
"""
changed_files = find_changed_files_in_repos(repos)
return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
def find_changed_tests(self, repos: List[Repo]) -> Set[str]:
"""
Find the changed tests.
Use git to find which test files have changed in this patch.
The returned file paths are in normalized form (see os.path.normpath(path)).
:param repos: List of repos containing changed files.
:return: Set of changed tests.
"""
revision_map = self.create_revision_map(repos)
LOGGER.info("Calculated revision map", revision_map=revision_map)
changed_files = find_changed_files_in_repos(repos, revision_map)
return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
def find_changed_tests(repos: Iterable[Repo], origin_rev: Optional[str] = None) -> Set[str]:
"""
Find the changed tests.
Use git to find which test files have changed in this patch.
The returned file paths are in normalized form (see os.path.normpath(path)).
:param repos: List of repos containing changed files.
:param origin_rev: The revision that local changes will be compared against.
:returns: Set of changed tests.
"""
revision_map = get_revision_map(repos, origin_rev)
LOGGER.info("Calculated revision map", revision_map=revision_map)
changed_files = find_changed_files_in_repos(repos, revision_map)
return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
def test_changed_files_in_multiple_repos(self, changed_files_mock):
repos_mock = [MagicMock(), MagicMock()]
first_repo_file_changes = [
os.path.join("jstests", "test1.js"),
os.path.join("jstests", "test1.cpp"),
]
second_repo_file_changes = [
os.path.join("jstests", "test2.js"),
]
changed_files_mock.side_effect = [
first_repo_file_changes, second_repo_file_changes
]
self.assertEqual(
3, len(under_test.find_changed_files_in_repos(repos_mock)))
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str, str], repos: List[Repo],
origin_build_variants: List[str]) -> Dict[str, str]:
# pylint: disable=too-many-locals
"""
Run code to select tasks to run based on test and task mappings for each of the build variants.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param repos: List of repos containing changed files.
:param origin_build_variants: Build variants to collect task info from.
:return: Dict of files and file contents for generated tasks.
"""
config_dict_of_suites_and_tasks = {}
changed_files = find_changed_files_in_repos(repos)
changed_files = {_remove_repo_path_prefix(file_path) for file_path in changed_files}
LOGGER.debug("Found changed files", files=changed_files)
shrub_project = ShrubProject()
for build_variant in origin_build_variants:
shrub_build_variant = BuildVariant(build_variant)
build_variant_config = evg_conf.get_variant(build_variant)
origin_variant_expansions = build_variant_config.expansions
task_configs = _get_task_configs(evg_conf, selected_tests_service,
selected_tests_variant_expansions, build_variant_config,
changed_files)
for task_config in task_configs.values():
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
selected_tests_variant_expansions,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_build_variant, config_options,
config_dict_of_suites_and_tasks)
shrub_project.add_build_variant(shrub_build_variant)
config_dict_of_suites_and_tasks["selected_tests_config.json"] = shrub_project.json()
return config_dict_of_suites_and_tasks
def find_changed_files(self, repos: List[Repo], task_id: str) -> Set[str]:
"""
Determine what files have changed in the given repos.
:param repos: List of git repos to query.
:param task_id: ID of task being run.
:return: Set of files that contain changes.
"""
revision_map = generate_revision_map_from_manifest(repos, task_id, self.evg_api)
changed_files = find_changed_files_in_repos(repos, revision_map)
changed_files = {_remove_repo_path_prefix(file_path) for file_path in changed_files}
changed_files = {
file_path
for file_path in changed_files if not file_path.startswith("src/third_party")
}
LOGGER.info("Found changed files", files=changed_files)
return changed_files
def gather_changed_files_for_lint(is_interesting_file: Callable) -> List[str]:
"""
Get the files that have changes since the last git commit.
:param is_interesting_file: Filter for whether a file should be returned.
:return: List of files for linting.
"""
repos, revision_map = _get_repos_and_revisions()
LOGGER.info("revisions", revision=revision_map)
candidate_files = find_changed_files_in_repos(repos, revision_map)
files = [
filename for filename in candidate_files
if _filter_file(filename, is_interesting_file)
]
LOGGER.info("Found files to lint", files=files)
return files
def find_changed_tests(repos: List[Repo], origin_rev: Optional[str] = None,
evg_api: Optional[EvergreenApi] = None,
task_id: Optional[str] = None) -> Set[str]:
"""
Find the changed tests.
Use git to find which test files have changed in this patch.
The returned file paths are in normalized form (see os.path.normpath(path)).
:param repos: List of repos containing changed files.
:param origin_rev: The revision that local changes will be compared against.
:param evg_api: Evergreen API client.
:param task_id: Evergreen task ID.
:return: Set of changed tests.
"""
revision_map = _create_revision_map(repos, origin_rev, evg_api, task_id)
LOGGER.info("Calculated revision map", revision_map=revision_map)
changed_files = find_changed_files_in_repos(repos, revision_map)
return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
def lint_git_diff(eslint: Optional[str]) -> bool:
"""
Lint the files that have changes since the last git commit.
:param eslint: Path to eslint command.
:return: True if lint was successful.
"""
repos = [Repo(path) for path in git.get_module_paths()]
candidate_files = find_changed_files_in_repos(repos)
LOGGER.info("Found candidate_files", candidate_files=candidate_files)
files = [
filename for filename in candidate_files
if is_interesting_file(filename)
]
LOGGER.info("Found files to lint", files=files)
# Patch may have files that we do not want to check which is fine
if files:
return _lint_files(eslint, files)
return True
def test_nothing_found(self, changed_files_mock):
repos_mock = [MagicMock()]
changed_files_mock.return_value = set()
self.assertEqual(
0, len(under_test.find_changed_files_in_repos(repos_mock)))
