def test_build_variant_not_in_evg_project_config(self): variant = "novariant" evg_conf_mock = MagicMock() evg_conf_mock.get_variant.return_value = None with self.assertRaises(ValueError): under_test.create_tests_by_task(variant, evg_conf_mock, set(), "install-dir/bin")
def _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, build_variant_map, repos, evg_conf): """ Generate burn in tests tasks for a given shrub config and group of buildvariants. :param evergreen_api: Evergreen.py object. :param shrub_config: Shrub config object that the build variants will be built upon. :param expansions_file_data: Config data file to use. :param build_variant_map: Map of base buildvariants to their generated buildvariant. :param repos: Git repositories. """ for build_variant, run_build_variant in build_variant_map.items(): config_options = _get_config_options(expansions_file_data, build_variant, run_build_variant) tests_by_task = create_tests_by_task(build_variant, repos, evg_conf) if tests_by_task: _generate_evg_build_variant(shrub_config, build_variant, run_build_variant, expansions_file_data["build_variant"], evg_conf) gen_config = GenerateConfig(build_variant, config_options.project, run_build_variant, config_options.distro).validate(evg_conf) repeat_config = RepeatConfig(repeat_tests_min=config_options.repeat_tests_min, repeat_tests_max=config_options.repeat_tests_max, repeat_tests_secs=config_options.repeat_tests_secs) create_generate_tasks_config(shrub_config, tests_by_task, gen_config, repeat_config, evergreen_api, evg_conf, include_gen_task=False)
def find_changes(self, repos: List[Repo], generate_config: GenerateConfig) -> Dict[str, TaskInfo]: """ Find tests and tasks to run based on test changes. :param repos: List of repos to check for changed tests. :param generate_config: Configuration for how to generate tasks. :return: Dictionary of tasks with tests to run in them. """ changed_tests = self.change_detector.find_changed_tests(repos) tests_by_task = create_tests_by_task(generate_config.build_variant, self.evg_config, changed_tests) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) return tests_by_task
def _generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, buildvariant_map): """ Generate burn in tests tasks for a given shrub config and group of buildvariants. :param evergreen_api: Evergreen.py object. :param shrub_config: Shrub config object that the build variants will be built upon. :param expansions_file_data: Config data file to use. :param buildvariant_map: Map of base buildvariants to their generated buildvariant. """ for buildvariant, run_buildvariant in buildvariant_map.items(): config_options = _get_config_options(expansions_file_data, buildvariant, run_buildvariant) tests_by_task = create_tests_by_task(config_options) if tests_by_task: _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant) create_generate_tasks_config(evergreen_api, shrub_config, config_options, tests_by_task, False)
def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject, task_expansions: Dict[str, Any], build_variant_map: Dict[str, str], repos: List[Repo], evg_conf: EvergreenProjectConfig, install_dir: str) -> None: """ Generate burn in tests tasks for a given shrub config and group of build variants. :param evergreen_api: Evergreen.py object. :param shrub_project: Shrub config object that the build variants will be built upon. :param task_expansions: Dictionary of expansions for the running task. :param build_variant_map: Map of base buildvariants to their generated buildvariant. :param repos: Git repositories. """ for build_variant, run_build_variant in build_variant_map.items(): config_options = _get_config_options(task_expansions, build_variant, run_build_variant) task_id = task_expansions[TASK_ID_EXPANSION] change_detector = EvergreenFileChangeDetector(task_id, evergreen_api, os.environ) changed_tests = change_detector.find_changed_tests(repos) tests_by_task = create_tests_by_task(build_variant, evg_conf, changed_tests, install_dir) if tests_by_task: shrub_build_variant = _generate_evg_build_variant( evg_conf.get_variant(build_variant), run_build_variant, task_expansions["build_variant"]) gen_config = GenerateConfig( build_variant, config_options.project, run_build_variant, config_options.distro, include_gen_task=False).validate(evg_conf) repeat_config = RepeatConfig( repeat_tests_min=config_options.repeat_tests_min, repeat_tests_max=config_options.repeat_tests_max, repeat_tests_secs=config_options.repeat_tests_secs) burn_in_generator = GenerateBurnInExecutor(gen_config, repeat_config, evergreen_api) burn_in_generator.generate_tasks_for_variant( tests_by_task, shrub_build_variant) shrub_project.add_build_variant(shrub_build_variant)
def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject, task_expansions: Dict[str, Any], build_variant_map: Dict[str, str], repos: Iterable[Repo], evg_conf: EvergreenProjectConfig) -> None: """ Generate burn in tests tasks for a given shrub config and group of build variants. :param evergreen_api: Evergreen.py object. :param shrub_project: Shrub config object that the build variants will be built upon. :param task_expansions: Dictionary of expansions for the running task. :param build_variant_map: Map of base buildvariants to their generated buildvariant. :param repos: Git repositories. """ for build_variant, run_build_variant in build_variant_map.items(): config_options = _get_config_options(task_expansions, build_variant, run_build_variant) changed_tests = find_changed_tests(repos) tests_by_task = create_tests_by_task(build_variant, evg_conf, changed_tests) if tests_by_task: shrub_build_variant = _generate_evg_build_variant( evg_conf.get_variant(build_variant), run_build_variant, task_expansions["build_variant"]) gen_config = GenerateConfig( build_variant, config_options.project, run_build_variant, config_options.distro).validate(evg_conf) repeat_config = RepeatConfig( repeat_tests_min=config_options.repeat_tests_min, repeat_tests_max=config_options.repeat_tests_max, repeat_tests_secs=config_options.repeat_tests_secs) create_generate_tasks_config(shrub_build_variant, tests_by_task, gen_config, repeat_config, evergreen_api, include_gen_task=False) shrub_project.add_build_variant(shrub_build_variant)
def main(build_variant, run_build_variant, distro, project, generate_tasks_file, evg_api_config, verbose, task_id): """ Run new or changed tests in repeated mode to validate their stability. Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are excluded from the required build variants and are only run in certain daily build variants. As such, we only expect the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments should be None when executing this script. There are two modes that burn_in_tests_multiversion can run in: (1) Normal mode: by default burn_in_tests will attempt to run all detected tests the configured number of times. This is useful if you have a test or tests you would like to check before submitting a patch to evergreen. (2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks to do all the test executions. This is the mode used to run tests in patch builds. NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen can handle. \f :param build_variant: Build variant to query tasks from. :param run_build_variant:Build variant to actually run against. :param distro: Distro to run tests on. :param project: Project to run tests on. :param generate_tasks_file: Create a generate tasks configuration in this file. :param evg_api_config: Location of configuration file to connect to evergreen. :param verbose: Log extra debug information. """ _configure_logging(verbose) evg_conf = parse_evergreen_file(EVERGREEN_FILE) generate_config = GenerateConfig(build_variant=build_variant, run_build_variant=run_build_variant, distro=distro, project=project, task_id=task_id) # yapf: disable generate_config.validate(evg_conf) repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config) change_detector = EvergreenFileChangeDetector(task_id, evg_api) changed_tests = change_detector.find_changed_tests(repos) tests_by_task = create_tests_by_task(generate_config.build_variant, evg_conf, changed_tests) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) multiversion_tasks = evg_conf.get_task_names_by_tag( MULTIVERSION_PASSTHROUGH_TAG) LOGGER.debug("Multiversion tasks by tag", tasks=multiversion_tasks, tag=MULTIVERSION_PASSTHROUGH_TAG) # We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in # multiversion_suites. Multiversion passthrough suites must include # MULTIVERSION_CONFIG_KEY as a root level key and must be set to true. multiversion_suites = get_named_suites_with_root_level_key( MULTIVERSION_CONFIG_KEY) assert len(multiversion_tasks) == len(multiversion_suites) build_variant = create_multiversion_generate_tasks_config( tests_by_task, evg_api, generate_config) shrub_project = ShrubProject.empty() shrub_project.add_build_variant(build_variant) if not validate_task_generation_limit(shrub_project): sys.exit(1) write_file(generate_tasks_file, shrub_project.json())
def main(build_variant, run_build_variant, distro, project, generate_tasks_file, no_exec, resmoke_args, evg_api_config, verbose, task_id): """ Run new or changed tests in repeated mode to validate their stability. Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are excluded from the required build variants and are only run in certain daily build variants. As such, we only expect the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments should be None when executing this script. There are two modes that burn_in_tests_multiversion can run in: (1) Normal mode: by default burn_in_tests will attempt to run all detected tests the configured number of times. This is useful if you have a test or tests you would like to check before submitting a patch to evergreen. (2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks to do all the test executions. This is the mode used to run tests in patch builds. NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen can handle. \f :param build_variant: Build variant to query tasks from. :param run_build_variant:Build variant to actually run against. :param distro: Distro to run tests on. :param project: Project to run tests on. :param generate_tasks_file: Create a generate tasks configuration in this file. :param no_exec: Just perform test discover, do not execute the tests. :param resmoke_args: Arguments to pass through to resmoke. :param evg_api_config: Location of configuration file to connect to evergreen. :param verbose: Log extra debug information. """ _configure_logging(verbose) evg_conf = parse_evergreen_file(EVERGREEN_FILE) repeat_config = RepeatConfig() # yapf: disable generate_config = GenerateConfig(build_variant=build_variant, run_build_variant=run_build_variant, distro=distro, project=project, task_id=task_id) # yapf: disable if generate_tasks_file: generate_config.validate(evg_conf) evg_api = _get_evg_api(evg_api_config, False) repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args)) tests_by_task = create_tests_by_task(generate_config.build_variant, repos, evg_conf) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) if generate_tasks_file: multiversion_tasks = evg_conf.get_task_names_by_tag( MULTIVERSION_PASSTHROUGH_TAG) LOGGER.debug("Multiversion tasks by tag", tasks=multiversion_tasks, tag=MULTIVERSION_PASSTHROUGH_TAG) # We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in # multiversion_suites. Multiversion passthrough suites must include # MULTIVERSION_CONFIG_KEY as a root level key and must be set to true. multiversion_suites = get_named_suites_with_root_level_key( MULTIVERSION_CONFIG_KEY) assert len(multiversion_tasks) == len(multiversion_suites) evg_config = Configuration() evg_config = create_multiversion_generate_tasks_config( evg_config, tests_by_task, evg_api, generate_config) json_config = evg_config.to_map() tasks_to_create = len(json_config.get('tasks', [])) if tasks_to_create > MAX_TASKS_TO_CREATE: LOGGER.warning( "Attempting to create more tasks than max, aborting", tasks=tasks_to_create, max=MAX_TASKS_TO_CREATE) sys.exit(1) _write_json_file(json_config, generate_tasks_file) elif not no_exec: run_tests(tests_by_task, resmoke_cmd) else: LOGGER.info("Not running tests due to 'no_exec' option.")