Beispiel #1
0
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
        selected_tests_service: SelectedTestsService,
        selected_tests_variant_expansions: Dict[str, str],
        repos: List[Repo]) -> Dict[str, str]:
    # pylint: disable=too-many-locals
    """
    Run code to select tasks to run based on test and task mappings for each of the build variants.

    :param evg_api: Evergreen API object.
    :param evg_conf: Evergreen configuration.
    :param selected_tests_service: Selected-tests service.
    :param selected_tests_variant_expansions: Expansions of the selected-tests variant.
    :param repos: List of repos containing changed files.
    :return: Dict of files and file contents for generated tasks.
    """
    config_dict_of_suites_and_tasks = {}

    task_id = selected_tests_variant_expansions[TASK_ID_EXPANSION]
    revision_map = generate_revision_map_from_manifest(repos, task_id, evg_api)
    changed_files = find_changed_files_in_repos(repos, revision_map)
    changed_files = {
        _remove_repo_path_prefix(file_path)
        for file_path in changed_files
    }
    LOGGER.info("Found changed files", files=changed_files)

    shrub_project = ShrubProject()
    for build_variant_config in evg_conf.get_required_variants():
        shrub_build_variant = BuildVariant(build_variant_config.name)
        origin_variant_expansions = build_variant_config.expansions

        task_configs = _get_task_configs(evg_conf, selected_tests_service,
                                         selected_tests_variant_expansions,
                                         build_variant_config, changed_files)

        remove_task_configs_already_in_build(
            task_configs, evg_api, build_variant_config,
            selected_tests_variant_expansions["version_id"])

        for task_config in task_configs.values():
            Suite.reset_current_index()
            config_options = SelectedTestsConfigOptions.from_file(
                origin_variant_expansions,
                selected_tests_variant_expansions,
                task_config,
                REQUIRED_CONFIG_KEYS,
                DEFAULT_CONFIG_VALUES,
                CONFIG_FORMAT_FN,
            )
            _update_config_with_task(evg_api, shrub_build_variant,
                                     config_options,
                                     config_dict_of_suites_and_tasks)

        shrub_project.add_build_variant(shrub_build_variant)

    config_dict_of_suites_and_tasks[
        "selected_tests_config.json"] = shrub_project.json()
    return config_dict_of_suites_and_tasks
def run(evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig,
        selected_tests_service: SelectedTestsService,
        selected_tests_variant_expansions: Dict[str, str], repos: List[Repo],
        origin_build_variants: List[str]) -> Dict[str, str]:
    # pylint: disable=too-many-locals
    """
    Run code to select tasks to run based on test and task mappings for each of the build variants.

    :param evg_api: Evergreen API object.
    :param evg_conf: Evergreen configuration.
    :param selected_tests_service: Selected-tests service.
    :param selected_tests_variant_expansions: Expansions of the selected-tests variant.
    :param repos: List of repos containing changed files.
    :param origin_build_variants: Build variants to collect task info from.
    :return: Dict of files and file contents for generated tasks.
    """
    config_dict_of_suites_and_tasks = {}

    changed_files = find_changed_files_in_repos(repos)
    changed_files = {_remove_repo_path_prefix(file_path) for file_path in changed_files}
    LOGGER.debug("Found changed files", files=changed_files)

    shrub_project = ShrubProject()
    for build_variant in origin_build_variants:
        shrub_build_variant = BuildVariant(build_variant)
        build_variant_config = evg_conf.get_variant(build_variant)
        origin_variant_expansions = build_variant_config.expansions

        task_configs = _get_task_configs(evg_conf, selected_tests_service,
                                         selected_tests_variant_expansions, build_variant_config,
                                         changed_files)

        for task_config in task_configs.values():
            config_options = SelectedTestsConfigOptions.from_file(
                origin_variant_expansions,
                selected_tests_variant_expansions,
                task_config,
                REQUIRED_CONFIG_KEYS,
                DEFAULT_CONFIG_VALUES,
                CONFIG_FORMAT_FN,
            )
            _update_config_with_task(evg_api, shrub_build_variant, config_options,
                                     config_dict_of_suites_and_tasks)

        shrub_project.add_build_variant(shrub_build_variant)

    config_dict_of_suites_and_tasks["selected_tests_config.json"] = shrub_project.json()
    return config_dict_of_suites_and_tasks
Beispiel #3
0
def _generate_evg_tasks(evergreen_api: EvergreenApi,
                        shrub_project: ShrubProject,
                        task_expansions: Dict[str, Any],
                        build_variant_map: Dict[str, str], repos: List[Repo],
                        evg_conf: EvergreenProjectConfig,
                        install_dir: str) -> None:
    """
    Generate burn in tests tasks for a given shrub config and group of build variants.

    :param evergreen_api: Evergreen.py object.
    :param shrub_project: Shrub config object that the build variants will be built upon.
    :param task_expansions: Dictionary of expansions for the running task.
    :param build_variant_map: Map of base buildvariants to their generated buildvariant.
    :param repos: Git repositories.
    """
    for build_variant, run_build_variant in build_variant_map.items():
        config_options = _get_config_options(task_expansions, build_variant,
                                             run_build_variant)
        task_id = task_expansions[TASK_ID_EXPANSION]
        change_detector = EvergreenFileChangeDetector(task_id, evergreen_api,
                                                      os.environ)
        changed_tests = change_detector.find_changed_tests(repos)
        tests_by_task = create_tests_by_task(build_variant, evg_conf,
                                             changed_tests, install_dir)
        if tests_by_task:
            shrub_build_variant = _generate_evg_build_variant(
                evg_conf.get_variant(build_variant), run_build_variant,
                task_expansions["build_variant"])
            gen_config = GenerateConfig(
                build_variant,
                config_options.project,
                run_build_variant,
                config_options.distro,
                include_gen_task=False).validate(evg_conf)
            repeat_config = RepeatConfig(
                repeat_tests_min=config_options.repeat_tests_min,
                repeat_tests_max=config_options.repeat_tests_max,
                repeat_tests_secs=config_options.repeat_tests_secs)

            burn_in_generator = GenerateBurnInExecutor(gen_config,
                                                       repeat_config,
                                                       evergreen_api)
            burn_in_generator.generate_tasks_for_variant(
                tests_by_task, shrub_build_variant)
            shrub_project.add_build_variant(shrub_build_variant)
Beispiel #4
0
def _generate_evg_tasks(evergreen_api: EvergreenApi,
                        shrub_project: ShrubProject,
                        task_expansions: Dict[str, Any],
                        build_variant_map: Dict[str,
                                                str], repos: Iterable[Repo],
                        evg_conf: EvergreenProjectConfig) -> None:
    """
    Generate burn in tests tasks for a given shrub config and group of build variants.

    :param evergreen_api: Evergreen.py object.
    :param shrub_project: Shrub config object that the build variants will be built upon.
    :param task_expansions: Dictionary of expansions for the running task.
    :param build_variant_map: Map of base buildvariants to their generated buildvariant.
    :param repos: Git repositories.
    """
    for build_variant, run_build_variant in build_variant_map.items():
        config_options = _get_config_options(task_expansions, build_variant,
                                             run_build_variant)
        changed_tests = find_changed_tests(repos)
        tests_by_task = create_tests_by_task(build_variant, evg_conf,
                                             changed_tests)
        if tests_by_task:
            shrub_build_variant = _generate_evg_build_variant(
                evg_conf.get_variant(build_variant), run_build_variant,
                task_expansions["build_variant"])
            gen_config = GenerateConfig(
                build_variant, config_options.project, run_build_variant,
                config_options.distro).validate(evg_conf)
            repeat_config = RepeatConfig(
                repeat_tests_min=config_options.repeat_tests_min,
                repeat_tests_max=config_options.repeat_tests_max,
                repeat_tests_secs=config_options.repeat_tests_secs)

            create_generate_tasks_config(shrub_build_variant,
                                         tests_by_task,
                                         gen_config,
                                         repeat_config,
                                         evergreen_api,
                                         include_gen_task=False)
            shrub_project.add_build_variant(shrub_build_variant)