def generate_fuzzer(
self, evg_expansions: EvgExpansions) -> GeneratedConfiguration:
"""
Generate configuration for the fuzzer task specified by the expansions.
:param evg_expansions: Evergreen expansions describing what to generate.
:return: Configuration to generate the specified task.
"""
suite = evg_expansions.suite
is_sharded = self.multiversion_util.is_suite_sharded(suite)
version_config_list = get_version_configs(is_sharded)
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
fuzzer_task_set = set()
for version_config in version_config_list:
fuzzer_params = evg_expansions.get_fuzzer_params(
version_config, is_sharded)
fuzzer_task = builder.generate_fuzzer(fuzzer_params)
fuzzer_task_set = fuzzer_task_set.union(fuzzer_task.sub_tasks)
existing_tasks = {ExistingTask(task) for task in fuzzer_task_set}
existing_tasks.add({ExistingTask(f"{suite}_multiversion_gen")})
builder.add_display_task(evg_expansions.task, existing_tasks,
evg_expansions.build_variant)
return builder.build(f"{evg_expansions.task}.json")
def _generate_evg_build_variant(
source_build_variant: Variant,
run_build_variant: str,
bypass_build_variant: str,
) -> BuildVariant:
"""
Generate a shrub build variant for the given run build variant.
:param source_build_variant: The build variant to base configuration on.
:param run_build_variant: The build variant to generate.
:param bypass_build_variant: The build variant to get compile artifacts from.
:return: Shrub build variant configuration.
"""
display_name = f"! {source_build_variant.display_name}"
run_on = source_build_variant.run_on
modules = source_build_variant.modules
expansions = source_build_variant.expansions
expansions["burn_in_bypass"] = bypass_build_variant
build_variant = BuildVariant(run_build_variant,
display_name,
expansions=expansions,
modules=modules,
run_on=run_on)
build_variant.add_existing_task(ExistingTask(COMPILE_TASK))
return build_variant
def test_display_task_with_different_task_types(self):
bv = under_test.BuildVariant("build variant")
n_tasks = 5
tasks = {Task(f"task {i}", []) for i in range(n_tasks)}
n_task_groups = 3
task_groups = {TaskGroup(f"task group {i}", []) for i in range(n_task_groups)}
n_existing_tasks = 4
existing_tasks = {ExistingTask(f"existing task {i}") for i in range(n_existing_tasks)}
bv.display_task(
"display",
execution_tasks=tasks,
execution_task_groups=task_groups,
execution_existing_tasks=existing_tasks,
distros=["the distro"],
)
d = bv.as_dict()
assert d["name"] == "build variant"
assert len(d["tasks"]) == n_tasks + n_task_groups # Existing tasks should already exist.
assert len(d["display_tasks"]) == 1
display_task = d["display_tasks"][0]
assert display_task["name"] == "display"
assert len(display_task["execution_tasks"]) == n_tasks + n_task_groups + n_existing_tasks
all_generated_execution_tasks = {t for t in display_task["execution_tasks"]}
for task in chain(tasks, task_groups, existing_tasks):
assert task.name in all_generated_execution_tasks
for task in d["tasks"]:
assert "the distro" in task["distros"]
def _generate_fuzzer_tasks(self, build_variant: BuildVariant,
version_configs: List[str],
is_sharded: bool) -> None:
"""
Generate fuzzer tasks and add them to the given build variant.
:param build_variant: Build variant to add tasks to.
:param version_configs: Version configurations to generate.
:param is_sharded: Should configuration be generated for sharding.
"""
tasks = set()
for version_config in version_configs:
fuzzer_config = self._get_fuzzer_options(version_config,
is_sharded)
task_name = f"{fuzzer_config.name}_{version_config}"
sub_tasks = gen_fuzzer.generate_fuzzer_sub_tasks(
task_name, fuzzer_config)
tasks = tasks.union(sub_tasks)
existing_tasks = {
ExistingTask(f"{self.options.suite}_multiversion_gen")
}
build_variant.display_task(self.task,
tasks,
execution_existing_tasks=existing_tasks)
def create_generate_tasks_config(build_variant: BuildVariant,
tests_by_task: Dict,
generate_config: GenerateConfig,
repeat_config: RepeatConfig,
evg_api: Optional[EvergreenApi],
include_gen_task: bool = True,
task_prefix: str = "burn_in") -> None:
# pylint: disable=too-many-arguments,too-many-locals
"""
Create the config for the Evergreen generate.tasks file.
:param build_variant: Shrub configuration to add to.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param generate_config: Configuration of what to generate.
:param repeat_config: Configuration of how to repeat tests.
:param evg_api: Evergreen API.
:param include_gen_task: Should generating task be include in display task.
:param task_prefix: Prefix all task names with this.
"""
tasks = create_generated_tasks(tests_by_task, task_prefix, generate_config,
repeat_config, evg_api)
existing_tasks = {ExistingTask(BURN_IN_TESTS_GEN_TASK)
} if include_gen_task else None
build_variant.display_task(BURN_IN_TESTS_TASK,
tasks,
execution_existing_tasks=existing_tasks)
def main(expansions_file: str = "expansions.yml",
output_file: str = "powercycle_tasks.json") -> None:
"""Generate multiple powercycle tasks to run in evergreen."""
config = make_config(expansions_file)
build_variant = BuildVariant(config.build_variant)
sub_tasks = set()
for task_name in config.task_names:
if "skip_compile" in task_name:
commands, task_dependency = get_skip_compile_setup_commands()
else:
commands, task_dependency = get_setup_commands()
commands.extend([
FunctionCall("set up remote credentials", config.remote_credentials_vars),
BuiltInCommand("timeout.update", config.timeout_params),
FunctionCall("set up EC2 instance", config.set_up_ec2_instance_vars),
FunctionCall("run powercycle test", config.run_powercycle_vars),
])
sub_tasks.update({
Task(
name_generated_task(task_name, index, config.num_tasks, config.build_variant),
commands, task_dependency)
for index in range(config.num_tasks)
})
build_variant.display_task(
config.current_task_name.replace("_gen", ""), sub_tasks, distros=[config.distro],
execution_existing_tasks={ExistingTask(config.current_task_name)})
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
write_file(output_file, shrub_project.json())
def generate_archive_dist_test_debug_activator_task(self, variant: str):
"""
Generate dummy task to activate the task that archives debug symbols.
We can't activate it directly as it's not generated.
"""
with self.lock:
build_variant = self.get_build_variant(variant)
build_variant.add_existing_task(
ExistingTask(ACTIVATE_ARCHIVE_DIST_TEST_DEBUG_TASK))
def create_multiversion_generate_tasks_config(
tests_by_task: Dict[str, TaskInfo], evg_api: EvergreenApi,
generate_config: GenerateConfig) -> BuildVariant:
"""
Create the multiversion config for the Evergreen generate.tasks file.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param evg_api: Evergreen API.
:param generate_config: Configuration of what to generate.
:return: Shrub configuration with added tasks.
"""
build_variant = BuildVariant(generate_config.build_variant)
tasks = set()
if tests_by_task:
# Get the multiversion suites that will run in as part of burn_in_multiversion.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
for suite in multiversion_suites:
idx = 0
if suite["origin"] not in tests_by_task.keys():
# Only generate burn in multiversion tasks for suites that would run the detected
# changed tests.
continue
LOGGER.debug("Generating multiversion suite",
suite=suite["multiversion_name"])
# We hardcode the number of fallback sub suites and the target resmoke time here
# since burn_in_tests cares about individual tests and not entire suites. The config
# options here are purely used to generate the proper multiversion suites to run
# tests against.
config_options = {
"suite": suite["origin"],
"fallback_num_sub_suites": 1,
"project": generate_config.project,
"build_variant": generate_config.build_variant,
"task_id": generate_config.task_id,
"task_name": suite["multiversion_name"],
"target_resmoke_time": 60,
}
config_options.update(gen_resmoke.DEFAULT_CONFIG_VALUES)
config_generator = gen_multiversion.EvergreenMultiversionConfigGenerator(
evg_api, gen_resmoke.ConfigOptions(config_options))
test_list = tests_by_task[suite["origin"]].tests
for test in test_list:
# Generate the multiversion tasks for each test.
sub_tasks = config_generator.get_burn_in_tasks(test, idx)
tasks = tasks.union(sub_tasks)
idx += 1
existing_tasks = {ExistingTask(f"{BURN_IN_MULTIVERSION_TASK}_gen")}
build_variant.display_task(BURN_IN_MULTIVERSION_TASK,
tasks,
execution_existing_tasks=existing_tasks)
return build_variant
def add_display_task(self, display_task_name: str, execution_task_names: Set[str],
build_variant: str) -> None:
"""
Add configuration to generate the specified display task.
:param display_task_name: Name of display task to create.
:param execution_task_names: Name of execution tasks to include in display task.
:param build_variant: Name of build variant to add to.
"""
execution_tasks = {ExistingTask(task_name) for task_name in execution_task_names}
with self.lock:
build_variant = self.get_build_variant(build_variant)
build_variant.display_task(display_task_name, execution_existing_tasks=execution_tasks)
def generate_config(self, build_variant: BuildVariant) -> None:
"""
Generate evergreen configuration.
:param build_variant: Build variant to add generated configuration to.
"""
self.build_tasks = self.evg_api.tasks_by_build(self.options.build_id)
tasks = self._generate_all_tasks()
generating_task = {ExistingTask(task_name) for task_name in self.options.gen_task_set}
distros = self._get_distro(build_variant.name)
build_variant.display_task(self.options.display_task_name, execution_tasks=tasks,
execution_existing_tasks=generating_task, distros=distros)
def create_fuzzer_task(options: ConfigOptions,
build_variant: BuildVariant) -> None:
"""
Generate an evergreen configuration for fuzzers and add it to the given build variant.
:param options: task options.
:param build_variant: Build variant to add tasks to.
"""
task_name = options.name
sub_tasks = generate_fuzzer_sub_tasks(task_name, options)
build_variant.display_task(
task_name,
sub_tasks,
execution_existing_tasks={ExistingTask(f"{options.name}_gen")})
def generate_evg_tasks(self, build_variant: BuildVariant) -> None:
# pylint: disable=too-many-locals
"""
Generate evergreen tasks for multiversion tests.
The number of tasks generated equals
(the number of version configs) * (the number of generated suites).
:param build_variant: Build variant to add generated configuration to.
"""
is_sharded = is_suite_sharded(TEST_SUITE_DIR, self.options.suite)
version_configs = get_version_configs(is_sharded)
if self.options.is_jstestfuzz:
self._generate_fuzzer_tasks(build_variant, version_configs,
is_sharded)
return
suites = self.generate_resmoke_suites()
sub_tasks = set()
for version_config in version_configs:
idx = 0
for suite in suites:
# Generate the newly divided test suites
source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
sub_tasks.add(
self._generate_sub_task(version_config,
self.task, idx, source_suite,
len(suites), is_sharded))
idx += 1
# Also generate the misc task.
misc_suite_name = "{0}_misc".format(self.options.suite)
misc_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml")
sub_tasks.add(
self._generate_sub_task(version_config, self.task, idx,
misc_suite, 1, is_sharded))
idx += 1
build_variant.display_task(
self.task,
sub_tasks,
execution_existing_tasks={ExistingTask(f"{self.task}_gen")})
def get_existing_tasks(self) -> Optional[Set[ExistingTask]]:
"""Get any existing tasks that should be included in the generated display task."""
if self.generate_config.include_gen_task:
return {ExistingTask(BURN_IN_TESTS_GEN_TASK)}
return None