def generate_version(self, changed_files: Set[str]) -> GeneratedConfiguration:
"""
Generate selected tests configuration for the given file changes.
:param changed_files: Set of files that contain changes.
:return: Configuration to generate selected-tests tasks.
"""
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
for build_variant_config in self.evg_conf.get_required_variants():
self.generate_build_variant(build_variant_config, changed_files, builder)
return builder.build("selected_tests_config.json")
def generate_build_variant(self, builder: EvgConfigBuilder,
build_variant_name: str) -> EvgConfigBuilder:
"""
Generate task configuration for a build variant.
:param builder: Evergreen configuration builder to use.
:param build_variant_name: Name of build variant to generate.
:return: Evergreen configuration builder with build variant configuration.
"""
LOGGER.info("Generating config", build_variant=build_variant_name)
start_time = perf_counter()
task_list = self.evg_project_config.get_variant(
build_variant_name).task_names
tasks_to_hide = set()
with Executor(max_workers=MAX_WORKERS) as exe:
jobs = []
for task_name in task_list:
task_def = self.evg_project_config.get_task(task_name)
if task_def.is_generate_resmoke_task:
tasks_to_hide.add(task_name)
run_vars = task_def.generate_resmoke_tasks_command.get(
"vars", {})
requires_npm = run_vars.get("is_jstestfuzz", False)
if requires_npm:
fuzzer_params = self.task_def_to_fuzzer_params(
task_def, build_variant_name)
jobs.append(
exe.submit(builder.generate_fuzzer, fuzzer_params))
else:
split_params = self.task_def_to_split_params(
task_def, build_variant_name)
gen_params = self.task_def_to_gen_params(
task_def, build_variant_name)
jobs.append(
exe.submit(builder.generate_suite, split_params,
gen_params))
[j.result() for j in jobs] # pylint: disable=expression-not-assigned
end_time = perf_counter()
duration = end_time - start_time
LOGGER.info("Finished BV",
build_variant=build_variant_name,
duration=duration,
task_count=len(tasks_to_hide))
builder.add_display_task(GEN_PARENT_TASK, tasks_to_hide,
build_variant_name)
self.adjust_gen_tasks_priority(tasks_to_hide)
return builder
def generate_build_variant(self, build_variant_config: Variant,
changed_files: Set[str],
builder: EvgConfigBuilder) -> None:
"""
Generate the selected tasks on the specified build variant.
:param build_variant_config: Configuration of build variant to generate.
:param changed_files: List of file changes to determine what to run.
:param builder: Builder to create new configuration.
"""
build_variant_name = build_variant_config.name
LOGGER.info("Generating build variant",
build_variant=build_variant_name)
task_configs = self.get_task_config(build_variant_config,
changed_files)
for task_config in task_configs.values():
task_def = Task(task_config)
test_filter = None
if "selected_tests_to_run" in task_config:
test_filter = partial(
filter_set, input_set=task_config["selected_tests_to_run"])
split_params = SuiteSplitParameters(
build_variant=build_variant_name,
task_name=task_config["task_name"],
suite_name=task_config.get("suite", task_config["task_name"]),
filename=task_config.get("suite", task_config["task_name"]),
test_file_filter=test_filter,
is_asan=build_variant_config.is_asan_build(),
)
gen_params = ResmokeGenTaskParams(
use_large_distro=task_config.get("use_large_distro", False),
large_distro_name=task_config.get("large_distro_name"),
require_multiversion_setup=task_def.require_multiversion_setup(
),
require_multiversion_version_combo=False,
repeat_suites=task_config.get("repeat_suites", 1),
resmoke_args=task_config.get("resmoke_args", ""),
resmoke_jobs_max=task_config.get("resmoke_jobs_max"),
config_location=self.evg_expansions.get_config_location(),
dependencies={
depends_on["name"]
for depends_on in task_def.depends_on
},
)
builder.generate_suite(split_params, gen_params)
def generate_task(self, task_id: str, split_params: SuiteSplitParameters,
gen_params: ResmokeGenTaskParams) -> None:
"""
Generate the specified resmoke task.
:param task_id: Task ID of generating task.
:param split_params: Parameters describing how the task should be split.
:param gen_params: Parameters describing how the task should be generated.
"""
LOGGER.debug("config options",
split_params=split_params,
gen_params=gen_params)
if not self.gen_task_validation.should_task_be_generated(task_id):
LOGGER.info(
"Not generating configuration due to previous successful generation."
)
return
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
builder.generate_suite(split_params, gen_params)
builder.add_display_task(GEN_PARENT_TASK,
{f"{split_params.task_name}{GEN_SUFFIX}"},
split_params.build_variant)
generated_config = builder.build(split_params.task_name + ".json")
generated_config.write_all_to_dir(
self.gen_task_options.generated_config_dir)
def generate(self, task_id: str, build_variant_name: str, output_file: str) -> None:
"""
Write task configuration for a build variant to disk.
:param task_id: ID of running task.
:param build_variant_name: Name of build variant to generate.
:param output_file: Filename to write generated configuration to.
"""
if not self.gen_task_validation.should_task_be_generated(task_id):
LOGGER.info("Not generating configuration due to previous successful generation.")
return
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
builder = self.generate_build_variant(builder, build_variant_name)
generated_config = builder.build(output_file)
generated_config.write_all_to_dir(self.gen_task_options.generated_config_dir)
with open('gtcl_update_expansions.yml', "w+") as fh:
fh.write(f"gtcl: {self.evg_expansions.config_location()}")
def generate_fuzzer(
self, evg_expansions: EvgExpansions) -> GeneratedConfiguration:
"""
Generate configuration for the fuzzer task specified by the expansions.
:param evg_expansions: Evergreen expansions describing what to generate.
:return: Configuration to generate the specified task.
"""
suite = evg_expansions.suite
is_sharded = self.multiversion_util.is_suite_sharded(suite)
version_config_list = get_version_configs(is_sharded)
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
fuzzer_task_set = set()
for version_config in version_config_list:
fuzzer_params = evg_expansions.get_fuzzer_params(
version_config, is_sharded)
fuzzer_task = builder.generate_fuzzer(fuzzer_params)
fuzzer_task_set = fuzzer_task_set.union(fuzzer_task.sub_tasks)
existing_tasks = {ExistingTask(task) for task in fuzzer_task_set}
existing_tasks.add({ExistingTask(f"{suite}_multiversion_gen")})
builder.add_display_task(evg_expansions.task, existing_tasks,
evg_expansions.build_variant)
return builder.build(f"{evg_expansions.task}.json")
def generate_config(
fuzzer_params: FuzzerGenTaskParams) -> GeneratedConfiguration:
"""
Generate a fuzzer task based on the given parameters.
:param fuzzer_params: Parameters describing how fuzzer should be generated.
:return: Configuration to generate the specified fuzzer.
"""
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
builder.generate_fuzzer(fuzzer_params)
builder.add_display_task(GEN_PARENT_TASK,
{f"{fuzzer_params.task_name}_gen"},
fuzzer_params.variant)
return builder.build(fuzzer_params.task_name + ".json")
def generate_resmoke_suite(
self, evg_expansions: EvgExpansions) -> GeneratedConfiguration:
"""
Generate configuration for the resmoke task specified by the expansions.
:param evg_expansions: Evergreen expansions describing what to generate.
:return: Configuration to generate the specified task.
"""
suite = evg_expansions.suite or evg_expansions.task
is_sharded = self.multiversion_util.is_suite_sharded(suite)
split_params = evg_expansions.get_split_params()
gen_params = evg_expansions.get_generation_params(is_sharded)
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
builder.add_multiversion_suite(split_params, gen_params)
builder.add_display_task(GEN_PARENT_TASK,
{f"{split_params.task_name}"},
evg_expansions.build_variant)
return builder.build(f"{evg_expansions.task}.json")
def generate_configuration(self, tests_by_task: Dict[str, TaskInfo],
target_file: str,
build_variant: str) -> GeneratedConfiguration:
"""
Generate configuration for the given tasks and tests.
:param tests_by_task: Map of what to generate.
:param target_file: Location to write generated configuration.
:param build_variant: Name of build variant being generated on.
:return: Generated configuration to create requested tasks and tests.
"""
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
build_variant_config = self.evg_config.get_variant(build_variant)
is_asan = build_variant_config.is_asan_build()
tasks = set()
if tests_by_task:
# Get the multiversion suites that will run in as part of burn_in_multiversion.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
for suite in multiversion_suites:
task_name = suite["origin"]
if task_name not in tests_by_task.keys():
# Only generate burn in multiversion tasks for suites that would run the
# detected changed tests.
continue
LOGGER.debug("Generating multiversion suite",
suite=suite["multiversion_name"])
test_list = tests_by_task[task_name].tests
split_params = SuiteSplitParameters(
task_name=suite["multiversion_name"],
suite_name=task_name,
filename=task_name,
test_file_filter=partial(filter_list,
input_list=test_list),
build_variant=build_variant,
is_asan=is_asan)
version_configs = self.multiversion_util.get_version_configs_for_suite(
task_name)
gen_params = MultiversionGenTaskParams(
mixed_version_configs=version_configs,
is_sharded=self.multiversion_util.is_suite_sharded(
task_name),
resmoke_args="",
parent_task_name="burn_in_tests_multiversion",
origin_suite=task_name,
use_large_distro=False,
large_distro_name=None,
name_prefix="burn_in_multiversion",
create_misc_suite=False,
add_to_display_task=False,
config_location=self.burn_in_config.build_config_location(
),
)
tasks = tasks.union(
builder.add_multiversion_burn_in_test(
split_params, gen_params))
if len(tasks) == 0:
builder.get_build_variant(build_variant)
executions_tasks = {task.name for task in tasks}
executions_tasks.add("burn_in_tests_multiversion_gen")
builder.add_display_task(display_task_name="burn_in_multiversion",
execution_task_names=executions_tasks,
build_variant=build_variant)
return builder.build(target_file)
def generate_build_variant(self, builder: EvgConfigBuilder,
build_variant_name: str) -> EvgConfigBuilder:
"""
Generate task configuration for a build variant.
:param builder: Evergreen configuration builder to use.
:param build_variant_name: Name of build variant to generate.
:return: Evergreen configuration builder with build variant configuration.
"""
LOGGER.info("Generating config", build_variant=build_variant_name)
start_time = perf_counter()
task_list = self.evg_project_config.get_variant(
build_variant_name).task_names
tasks_to_hide = set()
with Executor(max_workers=MAX_WORKERS) as exe:
jobs = []
for task_name in task_list:
task_def = self.evg_project_config.get_task(task_name)
if task_def.is_generate_resmoke_task:
tasks_to_hide.add(task_name)
is_sharded = None
version_list = None
run_vars = task_def.generate_resmoke_tasks_command["vars"]
suite = run_vars.get("suite")
is_jstestfuzz = run_vars.get("is_jstestfuzz", False)
implicit_multiversion = run_vars.get(
"implicit_multiversion", False)
if implicit_multiversion:
assert suite is not None
is_sharded = self.multiversion_util.is_suite_sharded(
suite)
version_list = get_version_configs(is_sharded)
if is_jstestfuzz:
fuzzer_params = self.task_def_to_fuzzer_params(
task_def, build_variant_name, is_sharded,
version_list)
jobs.append(
exe.submit(builder.generate_fuzzer, fuzzer_params))
else:
split_params = self.task_def_to_split_params(
task_def, build_variant_name)
if implicit_multiversion:
gen_params = self.task_def_to_mv_gen_params(
task_def, build_variant_name, is_sharded,
version_list)
jobs.append(
exe.submit(builder.add_multiversion_suite,
split_params, gen_params))
else:
gen_params = self.task_def_to_gen_params(
task_def, build_variant_name)
jobs.append(
exe.submit(builder.generate_suite,
split_params, gen_params))
[j.result() for j in jobs] # pylint: disable=expression-not-assigned
builder.generate_archive_dist_test_debug_activator_task(
build_variant_name)
# TODO: SERVER-59102 Check if this still causes a circular dependency on generator_tasks.
# tasks_to_hide.add(ACTIVATE_ARCHIVE_DIST_TEST_DEBUG_TASK)
end_time = perf_counter()
duration = end_time - start_time
LOGGER.info("Finished BV",
build_variant=build_variant_name,
duration=duration,
task_count=len(tasks_to_hide))
builder.add_display_task(GEN_PARENT_TASK, tasks_to_hide,
build_variant_name)
self.adjust_gen_tasks_priority(tasks_to_hide)
return builder