def generate_task(self, generated_suite: GeneratedSuite,
build_variant: BuildVariant,
gen_params: ResmokeGenTaskParams) -> None:
"""
Generate evergreen configuration for the given suite and add it to the build_variant.
:param generated_suite: Suite to add.
:param build_variant: Build variant to add generated configuration to.
:param gen_params: Parameters to configuration how tasks are generated.
"""
execution_tasks = self.resmoke_gen_task_service.generate_tasks(
generated_suite, gen_params)
distros = self._get_distro(build_variant.name,
gen_params.use_large_distro,
gen_params.large_distro_name)
build_variant.display_task(generated_suite.display_task_name(),
execution_tasks=execution_tasks,
distros=distros,
activate=False)
def generate_evg_tasks(self, build_variant: BuildVariant) -> None:
# pylint: disable=too-many-locals
"""
Generate evergreen tasks for multiversion tests.
The number of tasks generated equals
(the number of version configs) * (the number of generated suites).
:param build_variant: Build variant to add generated configuration to.
"""
is_sharded = is_suite_sharded(TEST_SUITE_DIR, self.options.suite)
version_configs = get_version_configs(is_sharded)
if self.options.is_jstestfuzz:
self._generate_fuzzer_tasks(build_variant, version_configs,
is_sharded)
return
suites = self.generate_resmoke_suites()
sub_tasks = set()
for version_config in version_configs:
idx = 0
for suite in suites:
# Generate the newly divided test suites
source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
sub_tasks.add(
self._generate_sub_task(version_config,
self.task, idx, source_suite,
len(suites), is_sharded))
idx += 1
# Also generate the misc task.
misc_suite_name = "{0}_misc".format(self.options.suite)
misc_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml")
sub_tasks.add(
self._generate_sub_task(version_config, self.task, idx,
misc_suite, 1, is_sharded))
idx += 1
build_variant.display_task(
self.task,
sub_tasks,
execution_existing_tasks={ExistingTask(f"{self.task}_gen")})
def create_generate_tasks_config(build_variant: BuildVariant, tests_by_task: Dict,
generate_config: GenerateConfig, repeat_config: RepeatConfig,
evg_api: Optional[EvergreenApi], include_gen_task: bool = True,
task_prefix: str = "burn_in") -> None:
# pylint: disable=too-many-arguments,too-many-locals
"""
Create the config for the Evergreen generate.tasks file.
:param build_variant: Shrub configuration to add to.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param generate_config: Configuration of what to generate.
:param repeat_config: Configuration of how to repeat tests.
:param evg_api: Evergreen API.
:param include_gen_task: Should generating task be include in display task.
:param task_prefix: Prefix all task names with this.
"""
tasks = create_generated_tasks(tests_by_task, task_prefix, generate_config, repeat_config,
evg_api)
existing_tasks = {ExistingTask(BURN_IN_TESTS_GEN_TASK)} if include_gen_task else None
build_variant.display_task(BURN_IN_TESTS_TASK, tasks, execution_existing_tasks=existing_tasks)
def generate_tasks_for_variant(self, tests_by_task: Dict[str, TaskInfo], variant: BuildVariant):
"""Add tasks to the passed in variant."""
tasks = set()
for task in sorted(tests_by_task):
task_info = tests_by_task[task]
task_history = self.get_task_runtime_history(task_info.display_task_name)
gen_suite = _tests_dict_to_generated_suites(task_info, task_history)
task_generator = BurnInGenTaskService(self.generate_config, self.repeat_config,
task_history)
params = BurnInGenTaskParams(
resmoke_args=task_info.resmoke_args,
require_multiversion_setup=task_info.require_multiversion_setup,
distro=task_info.distro,
)
new_tasks = task_generator.generate_tasks(gen_suite, params)
tasks = tasks.union(new_tasks)
variant.display_task(BURN_IN_TESTS_TASK, tasks,
execution_existing_tasks=self._get_existing_tasks())
def generate_fuzzer_task(self, params: FuzzerGenTaskParams,
build_variant: BuildVariant) -> FuzzerTask:
"""
Generate evergreen configuration for the given fuzzer and add it to the build_variant.
:param params: Parameters for how fuzzer should be generated.
:param build_variant: Build variant to add generated configuration to.
"""
fuzzer_task = self.fuzzer_gen_task_service.generate_tasks(params)
distros = self._get_distro(build_variant.name, params.use_large_distro,
params.large_distro_name)
if params.add_to_display_task:
build_variant.display_task(fuzzer_task.task_name,
fuzzer_task.sub_tasks,
distros=distros,
activate=False)
else:
build_variant.add_tasks(fuzzer_task.sub_tasks,
distros=distros,
activate=False)
return fuzzer_task
def main(expansions_file: str = "expansions.yml",
output_file: str = "powercycle_tasks.json") -> None:
"""Generate multiple powercycle tasks to run in evergreen."""
config = make_config(expansions_file)
build_variant = BuildVariant(config.build_variant)
sub_tasks = set()
for task_name in config.task_names:
if "skip_compile" in task_name:
commands, task_dependency = get_skip_compile_setup_commands()
else:
commands, task_dependency = get_setup_commands()
commands.extend([
FunctionCall("set up remote credentials",
config.remote_credentials_vars),
BuiltInCommand("timeout.update", config.timeout_params),
FunctionCall("set up EC2 instance",
config.set_up_ec2_instance_vars),
FunctionCall("run powercycle test", config.run_powercycle_vars),
])
sub_tasks.update({
Task(
name_generated_task(task_name, index, config.num_tasks,
config.build_variant), commands,
task_dependency)
for index in range(config.num_tasks)
})
build_variant.display_task(
config.current_task_name.replace("_gen", ""),
sub_tasks,
distros=[config.distro],
execution_existing_tasks={ExistingTask(config.current_task_name)})
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
write_file(output_file, shrub_project.json())
def generate_multiversion_burnin_task(
self, generated_suite: GeneratedSuite,
gen_params: MultiversionGenTaskParams,
build_variant: BuildVariant) -> Set[Task]:
"""
Generate burn_in configuration for the given suite and add it to the build_variant.
:param generated_suite: Suite to add.
:param build_variant: Build variant to add generated configuration to.
:param gen_params: Parameters to configuration how tasks are generated.
:return: Set of tasks that were generated.
"""
tasks = self.multiversion_gen_task_service.generate_tasks(
generated_suite, gen_params)
distros = self._get_distro(build_variant.name,
gen_params.use_large_distro,
gen_params.large_distro_name)
if gen_params.add_to_display_task:
build_variant.display_task(generated_suite.task_name,
tasks,
distros=distros)
else:
build_variant.add_tasks(tasks, distros=distros)
return tasks
def create_multiversion_generate_tasks_config(
tests_by_task: Dict, evg_api: EvergreenApi,
generate_config: GenerateConfig) -> BuildVariant:
"""
Create the multiversion config for the Evergreen generate.tasks file.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param evg_api: Evergreen API.
:param generate_config: Configuration of what to generate.
:return: Shrub configuration with added tasks.
"""
build_variant = BuildVariant(generate_config.build_variant)
tasks = set()
if tests_by_task:
# Get the multiversion suites that will run in as part of burn_in_multiversion.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
for suite in multiversion_suites:
idx = 0
if suite["origin"] not in tests_by_task.keys():
# Only generate burn in multiversion tasks for suites that would run the detected
# changed tests.
continue
LOGGER.debug("Generating multiversion suite",
suite=suite["multiversion_name"])
# We hardcode the number of fallback sub suites and the target resmoke time here
# since burn_in_tests cares about individual tests and not entire suites. The config
# options here are purely used to generate the proper multiversion suites to run
# tests against.
config_options = {
"suite": suite["origin"],
"fallback_num_sub_suites": 1,
"project": generate_config.project,
"build_variant": generate_config.build_variant,
"task_id": generate_config.task_id,
"task_name": suite["multiversion_name"],
"target_resmoke_time": 60,
}
config_options.update(gen_resmoke.DEFAULT_CONFIG_VALUES)
config_generator = gen_multiversion.EvergreenMultiversionConfigGenerator(
evg_api, gen_resmoke.ConfigOptions(config_options))
test_list = tests_by_task[suite["origin"]]["tests"]
for test in test_list:
# Exclude files that should be blacklisted from multiversion testing.
files_to_exclude = gen_multiversion.get_exclude_files(
suite["multiversion_name"], TASK_PATH_SUFFIX)
LOGGER.debug("Files to exclude",
files_to_exclude=files_to_exclude,
test=test,
suite=suite["multiversion_name"])
if test not in files_to_exclude:
# Generate the multiversion tasks for each test.
sub_tasks = config_generator.get_burn_in_tasks(test, idx)
tasks = tasks.union(sub_tasks)
idx += 1
existing_tasks = {ExistingTask(f"{BURN_IN_MULTIVERSION_TASK}_gen")}
build_variant.display_task(BURN_IN_MULTIVERSION_TASK,
tasks,
execution_existing_tasks=existing_tasks)
return build_variant