def generate_version(self, changed_files: Set[str]) -> GeneratedConfiguration: """ Generate selected tests configuration for the given file changes. :param changed_files: Set of files that contain changes. :return: Configuration to generate selected-tests tasks. """ builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter for build_variant_config in self.evg_conf.get_required_variants(): self.generate_build_variant(build_variant_config, changed_files, builder) return builder.build("selected_tests_config.json")
def generate_build_variant(self, builder: EvgConfigBuilder, build_variant_name: str) -> EvgConfigBuilder: """ Generate task configuration for a build variant. :param builder: Evergreen configuration builder to use. :param build_variant_name: Name of build variant to generate. :return: Evergreen configuration builder with build variant configuration. """ LOGGER.info("Generating config", build_variant=build_variant_name) start_time = perf_counter() task_list = self.evg_project_config.get_variant( build_variant_name).task_names tasks_to_hide = set() with Executor(max_workers=MAX_WORKERS) as exe: jobs = [] for task_name in task_list: task_def = self.evg_project_config.get_task(task_name) if task_def.is_generate_resmoke_task: tasks_to_hide.add(task_name) run_vars = task_def.generate_resmoke_tasks_command.get( "vars", {}) requires_npm = run_vars.get("is_jstestfuzz", False) if requires_npm: fuzzer_params = self.task_def_to_fuzzer_params( task_def, build_variant_name) jobs.append( exe.submit(builder.generate_fuzzer, fuzzer_params)) else: split_params = self.task_def_to_split_params( task_def, build_variant_name) gen_params = self.task_def_to_gen_params( task_def, build_variant_name) jobs.append( exe.submit(builder.generate_suite, split_params, gen_params)) [j.result() for j in jobs] # pylint: disable=expression-not-assigned end_time = perf_counter() duration = end_time - start_time LOGGER.info("Finished BV", build_variant=build_variant_name, duration=duration, task_count=len(tasks_to_hide)) builder.add_display_task(GEN_PARENT_TASK, tasks_to_hide, build_variant_name) self.adjust_gen_tasks_priority(tasks_to_hide) return builder
def generate_build_variant(self, build_variant_config: Variant, changed_files: Set[str], builder: EvgConfigBuilder) -> None: """ Generate the selected tasks on the specified build variant. :param build_variant_config: Configuration of build variant to generate. :param changed_files: List of file changes to determine what to run. :param builder: Builder to create new configuration. """ build_variant_name = build_variant_config.name LOGGER.info("Generating build variant", build_variant=build_variant_name) task_configs = self.get_task_config(build_variant_config, changed_files) for task_config in task_configs.values(): task_def = Task(task_config) test_filter = None if "selected_tests_to_run" in task_config: test_filter = partial( filter_set, input_set=task_config["selected_tests_to_run"]) split_params = SuiteSplitParameters( build_variant=build_variant_name, task_name=task_config["task_name"], suite_name=task_config.get("suite", task_config["task_name"]), filename=task_config.get("suite", task_config["task_name"]), test_file_filter=test_filter, is_asan=build_variant_config.is_asan_build(), ) gen_params = ResmokeGenTaskParams( use_large_distro=task_config.get("use_large_distro", False), large_distro_name=task_config.get("large_distro_name"), require_multiversion_setup=task_def.require_multiversion_setup( ), require_multiversion_version_combo=False, repeat_suites=task_config.get("repeat_suites", 1), resmoke_args=task_config.get("resmoke_args", ""), resmoke_jobs_max=task_config.get("resmoke_jobs_max"), config_location=self.evg_expansions.get_config_location(), dependencies={ depends_on["name"] for depends_on in task_def.depends_on }, ) builder.generate_suite(split_params, gen_params)
def generate_task(self, task_id: str, split_params: SuiteSplitParameters, gen_params: ResmokeGenTaskParams) -> None: """ Generate the specified resmoke task. :param task_id: Task ID of generating task. :param split_params: Parameters describing how the task should be split. :param gen_params: Parameters describing how the task should be generated. """ LOGGER.debug("config options", split_params=split_params, gen_params=gen_params) if not self.gen_task_validation.should_task_be_generated(task_id): LOGGER.info( "Not generating configuration due to previous successful generation." ) return builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter builder.generate_suite(split_params, gen_params) builder.add_display_task(GEN_PARENT_TASK, {f"{split_params.task_name}{GEN_SUFFIX}"}, split_params.build_variant) generated_config = builder.build(split_params.task_name + ".json") generated_config.write_all_to_dir( self.gen_task_options.generated_config_dir)
def generate(self, task_id: str, build_variant_name: str, output_file: str) -> None: """ Write task configuration for a build variant to disk. :param task_id: ID of running task. :param build_variant_name: Name of build variant to generate. :param output_file: Filename to write generated configuration to. """ if not self.gen_task_validation.should_task_be_generated(task_id): LOGGER.info("Not generating configuration due to previous successful generation.") return builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter builder = self.generate_build_variant(builder, build_variant_name) generated_config = builder.build(output_file) generated_config.write_all_to_dir(self.gen_task_options.generated_config_dir) with open('gtcl_update_expansions.yml', "w+") as fh: fh.write(f"gtcl: {self.evg_expansions.config_location()}")
def generate_fuzzer( self, evg_expansions: EvgExpansions) -> GeneratedConfiguration: """ Generate configuration for the fuzzer task specified by the expansions. :param evg_expansions: Evergreen expansions describing what to generate. :return: Configuration to generate the specified task. """ suite = evg_expansions.suite is_sharded = self.multiversion_util.is_suite_sharded(suite) version_config_list = get_version_configs(is_sharded) builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter fuzzer_task_set = set() for version_config in version_config_list: fuzzer_params = evg_expansions.get_fuzzer_params( version_config, is_sharded) fuzzer_task = builder.generate_fuzzer(fuzzer_params) fuzzer_task_set = fuzzer_task_set.union(fuzzer_task.sub_tasks) existing_tasks = {ExistingTask(task) for task in fuzzer_task_set} existing_tasks.add({ExistingTask(f"{suite}_multiversion_gen")}) builder.add_display_task(evg_expansions.task, existing_tasks, evg_expansions.build_variant) return builder.build(f"{evg_expansions.task}.json")
def generate_config( fuzzer_params: FuzzerGenTaskParams) -> GeneratedConfiguration: """ Generate a fuzzer task based on the given parameters. :param fuzzer_params: Parameters describing how fuzzer should be generated. :return: Configuration to generate the specified fuzzer. """ builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter builder.generate_fuzzer(fuzzer_params) builder.add_display_task(GEN_PARENT_TASK, {f"{fuzzer_params.task_name}_gen"}, fuzzer_params.variant) return builder.build(fuzzer_params.task_name + ".json")
def generate_resmoke_suite( self, evg_expansions: EvgExpansions) -> GeneratedConfiguration: """ Generate configuration for the resmoke task specified by the expansions. :param evg_expansions: Evergreen expansions describing what to generate. :return: Configuration to generate the specified task. """ suite = evg_expansions.suite or evg_expansions.task is_sharded = self.multiversion_util.is_suite_sharded(suite) split_params = evg_expansions.get_split_params() gen_params = evg_expansions.get_generation_params(is_sharded) builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter builder.add_multiversion_suite(split_params, gen_params) builder.add_display_task(GEN_PARENT_TASK, {f"{split_params.task_name}"}, evg_expansions.build_variant) return builder.build(f"{evg_expansions.task}.json")
def generate_configuration(self, tests_by_task: Dict[str, TaskInfo], target_file: str, build_variant: str) -> GeneratedConfiguration: """ Generate configuration for the given tasks and tests. :param tests_by_task: Map of what to generate. :param target_file: Location to write generated configuration. :param build_variant: Name of build variant being generated on. :return: Generated configuration to create requested tasks and tests. """ builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter build_variant_config = self.evg_config.get_variant(build_variant) is_asan = build_variant_config.is_asan_build() tasks = set() if tests_by_task: # Get the multiversion suites that will run in as part of burn_in_multiversion. multiversion_suites = get_named_suites_with_root_level_key( MULTIVERSION_CONFIG_KEY) for suite in multiversion_suites: task_name = suite["origin"] if task_name not in tests_by_task.keys(): # Only generate burn in multiversion tasks for suites that would run the # detected changed tests. continue LOGGER.debug("Generating multiversion suite", suite=suite["multiversion_name"]) test_list = tests_by_task[task_name].tests split_params = SuiteSplitParameters( task_name=suite["multiversion_name"], suite_name=task_name, filename=task_name, test_file_filter=partial(filter_list, input_list=test_list), build_variant=build_variant, is_asan=is_asan) version_configs = self.multiversion_util.get_version_configs_for_suite( task_name) gen_params = MultiversionGenTaskParams( mixed_version_configs=version_configs, is_sharded=self.multiversion_util.is_suite_sharded( task_name), resmoke_args="", parent_task_name="burn_in_tests_multiversion", origin_suite=task_name, use_large_distro=False, large_distro_name=None, name_prefix="burn_in_multiversion", create_misc_suite=False, add_to_display_task=False, config_location=self.burn_in_config.build_config_location( ), ) tasks = tasks.union( builder.add_multiversion_burn_in_test( split_params, gen_params)) if len(tasks) == 0: builder.get_build_variant(build_variant) executions_tasks = {task.name for task in tasks} executions_tasks.add("burn_in_tests_multiversion_gen") builder.add_display_task(display_task_name="burn_in_multiversion", execution_task_names=executions_tasks, build_variant=build_variant) return builder.build(target_file)
def generate_build_variant(self, builder: EvgConfigBuilder, build_variant_name: str) -> EvgConfigBuilder: """ Generate task configuration for a build variant. :param builder: Evergreen configuration builder to use. :param build_variant_name: Name of build variant to generate. :return: Evergreen configuration builder with build variant configuration. """ LOGGER.info("Generating config", build_variant=build_variant_name) start_time = perf_counter() task_list = self.evg_project_config.get_variant( build_variant_name).task_names tasks_to_hide = set() with Executor(max_workers=MAX_WORKERS) as exe: jobs = [] for task_name in task_list: task_def = self.evg_project_config.get_task(task_name) if task_def.is_generate_resmoke_task: tasks_to_hide.add(task_name) is_sharded = None version_list = None run_vars = task_def.generate_resmoke_tasks_command["vars"] suite = run_vars.get("suite") is_jstestfuzz = run_vars.get("is_jstestfuzz", False) implicit_multiversion = run_vars.get( "implicit_multiversion", False) if implicit_multiversion: assert suite is not None is_sharded = self.multiversion_util.is_suite_sharded( suite) version_list = get_version_configs(is_sharded) if is_jstestfuzz: fuzzer_params = self.task_def_to_fuzzer_params( task_def, build_variant_name, is_sharded, version_list) jobs.append( exe.submit(builder.generate_fuzzer, fuzzer_params)) else: split_params = self.task_def_to_split_params( task_def, build_variant_name) if implicit_multiversion: gen_params = self.task_def_to_mv_gen_params( task_def, build_variant_name, is_sharded, version_list) jobs.append( exe.submit(builder.add_multiversion_suite, split_params, gen_params)) else: gen_params = self.task_def_to_gen_params( task_def, build_variant_name) jobs.append( exe.submit(builder.generate_suite, split_params, gen_params)) [j.result() for j in jobs] # pylint: disable=expression-not-assigned builder.generate_archive_dist_test_debug_activator_task( build_variant_name) # TODO: SERVER-59102 Check if this still causes a circular dependency on generator_tasks. # tasks_to_hide.add(ACTIVATE_ARCHIVE_DIST_TEST_DEBUG_TASK) end_time = perf_counter() duration = end_time - start_time LOGGER.info("Finished BV", build_variant=build_variant_name, duration=duration, task_count=len(tasks_to_hide)) builder.add_display_task(GEN_PARENT_TASK, tasks_to_hide, build_variant_name) self.adjust_gen_tasks_priority(tasks_to_hide) return builder