def get_task_config(self, build_variant_config: Variant,
changed_files: Set[str]) -> Dict[str, Dict]:
"""
Get task configurations for the tasks to be generated.
:param build_variant_config: Config of build variant to collect task info from.
:param changed_files: Set of changed_files.
:return: Task configurations.
"""
existing_tasks = self.get_existing_tasks(
self.evg_expansions.version_id, build_variant_config.name)
task_configs = {}
related_test_files = self.selected_tests_service.find_selected_test_files(
changed_files)
LOGGER.info("related test files found",
related_test_files=related_test_files,
variant=build_variant_config.name)
if related_test_files:
tests_by_task = create_task_list_for_tests(
related_test_files, build_variant_config.name, self.evg_conf)
LOGGER.info("tests and tasks found", tests_by_task=tests_by_task)
tests_by_task = {
task: tests
for task, tests in tests_by_task.items()
if task not in existing_tasks
}
test_mapping_task_configs = self.task_config_service.get_task_configs_for_test_mappings(
tests_by_task, build_variant_config)
task_configs.update(test_mapping_task_configs)
related_tasks = self.selected_tests_service.find_selected_tasks(
changed_files)
LOGGER.info("related tasks found",
related_tasks=related_tasks,
variant=build_variant_config.name)
related_tasks = {
task
for task in related_tasks if task not in existing_tasks
}
if related_tasks:
task_mapping_task_configs = self.task_config_service.get_task_configs_for_task_mappings(
list(related_tasks), build_variant_config)
# task_mapping_task_configs will overwrite test_mapping_task_configs
# because task_mapping_task_configs will run all tests rather than a subset of tests
# and we should err on the side of running all tests
task_configs.update(task_mapping_task_configs)
return task_configs
def _get_task_configs(evg_conf: EvergreenProjectConfig,
selected_tests_service: SelectedTestsService,
selected_tests_variant_expansions: Dict[str, str],
build_variant_config: Variant,
changed_files: Set[str]) -> Dict[str, Dict]:
"""
Get task configurations for the tasks to be generated.
:param evg_conf: Evergreen configuration.
:param selected_tests_service: Selected-tests service.
:param selected_tests_variant_expansions: Expansions of the selected-tests variant.
:param build_variant_config: Config of build variant to collect task info from.
:param changed_files: Set of changed_files.
:return: Task configurations.
"""
task_configs = {}
related_test_files = _find_selected_test_files(selected_tests_service,
changed_files)
LOGGER.debug("related test files found",
related_test_files=related_test_files,
variant=build_variant_config.name)
if related_test_files:
tests_by_task = create_task_list_for_tests(related_test_files,
build_variant_config.name,
evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
test_mapping_task_configs = _get_task_configs_for_test_mappings(
selected_tests_variant_expansions, tests_by_task,
build_variant_config)
task_configs.update(test_mapping_task_configs)
related_tasks = _find_selected_tasks(selected_tests_service, changed_files,
build_variant_config)
LOGGER.debug("related tasks found",
related_tasks=related_tasks,
variant=build_variant_config.name)
if related_tasks:
task_mapping_task_configs = _get_task_configs_for_task_mappings(
selected_tests_variant_expansions, related_tasks,
build_variant_config)
# task_mapping_task_configs will overwrite test_mapping_task_configs
# because task_mapping_task_configs will run all tests rather than a subset of tests and we
# should err on the side of running all tests
task_configs.update(task_mapping_task_configs)
return task_configs
def main(
verbose: bool,
expansion_file: str,
evg_api_config: str,
build_variant: str,
selected_tests_config: str,
):
"""
Select tasks to be run based on changed files in a patch build.
:param verbose: Log extra debug information.
:param expansion_file: Configuration file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param build_variant: Build variant to query tasks from.
:param selected_tests_config: Location of config file to connect to elected-tests service.
"""
_configure_logging(verbose)
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
selected_tests_service = SelectedTestsService.from_file(
selected_tests_config)
repo = Repo(".")
changed_files = find_changed_files(repo)
buildscripts.resmokelib.parser.set_options()
LOGGER.debug("Found changed files", files=changed_files)
related_test_files = _find_related_test_files(selected_tests_service,
changed_files)
LOGGER.debug("related test files found",
related_test_files=related_test_files)
if related_test_files:
tests_by_task = create_task_list_for_tests(related_test_files,
build_variant, evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
config_dict_of_generated_tasks = _generate_shrub_config(
evg_api, evg_conf, expansion_file, tests_by_task, build_variant)
write_file_dict(SELECTED_TESTS_CONFIG_DIR,
config_dict_of_generated_tasks)
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
expansion_file: str, selected_tests_service: SelectedTestsService,
changed_files: Set[str], build_variant: str) -> Dict[str, dict]:
"""
Run code to select tasks to run based on test mappings and task mappings.
:param evg_api: Evergreen API object.
:param evg_conf: Evergreen configuration.
:param expansion_file: Configuration file.
:param selected_tests_config: Location of config file to connect to elected-tests service.
:param changed_files: Set of changed_files.
:param build_variant: Build variant to collect task info from.
:return: Dict of files and file contents for generated tasks.
"""
shrub_config = Configuration()
config_dict_of_suites_and_tasks = {}
task_configs = {}
build_variant_config = evg_conf.get_variant(build_variant)
related_test_files = _find_selected_test_files(selected_tests_service,
changed_files)
LOGGER.debug("related test files found",
related_test_files=related_test_files)
if related_test_files:
tests_by_task = create_task_list_for_tests(related_test_files,
build_variant, evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
test_mapping_task_configs = _get_task_configs_for_test_mappings(
expansion_file, tests_by_task, build_variant_config)
task_configs.update(test_mapping_task_configs)
related_tasks = _find_selected_tasks(selected_tests_service, changed_files,
build_variant_config)
LOGGER.debug("related tasks found", related_tasks=related_tasks)
if related_tasks:
task_mapping_task_configs = _get_task_configs_for_task_mappings(
expansion_file, related_tasks, build_variant_config)
# task_mapping_task_configs will overwrite test_mapping_task_configs
# because task_mapping_task_configs will run all tests rather than a subset of tests and we
# should err on the side of running all tests
task_configs.update(task_mapping_task_configs)
origin_variant_expansions = build_variant_config.expansions
for task_config in task_configs.values():
config_options = SelectedTestsConfigOptions.from_file(
origin_variant_expansions,
expansion_file,
task_config,
REQUIRED_CONFIG_KEYS,
DEFAULT_CONFIG_VALUES,
CONFIG_FORMAT_FN,
)
_update_config_with_task(evg_api, shrub_config, config_options,
config_dict_of_suites_and_tasks)
config_dict_of_suites_and_tasks[
"selected_tests_config.json"] = shrub_config.to_json()
return config_dict_of_suites_and_tasks