def create_multiversion_generate_tasks_config(evg_config: Configuration, tests_by_task: Dict,
evg_api: EvergreenApi,
generate_config: GenerateConfig) -> Configuration:
"""
Create the multiversion config for the Evergreen generate.tasks file.
:param evg_config: Shrub configuration to add to.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param evg_api: Evergreen API.
:param generate_config: Configuration of what to generate.
:return: Shrub configuration with added tasks.
"""
dt = DisplayTaskDefinition(BURN_IN_MULTIVERSION_TASK)
if tests_by_task:
# Get the multiversion suites that will run in as part of burn_in_multiversion.
multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY)
for suite in multiversion_suites:
idx = 0
if suite["origin"] not in tests_by_task.keys():
# Only generate burn in multiversion tasks for suites that would run the detected
# changed tests.
continue
LOGGER.debug("Generating multiversion suite", suite=suite["multiversion_name"])
# We hardcode the number of fallback sub suites and the target resmoke time here
# since burn_in_tests cares about individual tests and not entire suites. The config
# options here are purely used to generate the proper multiversion suites to run
# tests against.
config_options = {
"suite": suite["origin"],
"fallback_num_sub_suites": 1,
"project": generate_config.project,
"build_variant": generate_config.build_variant,
"task_id": generate_config.task_id,
"task_name": suite["multiversion_name"],
"target_resmoke_time": 60,
}
config_options.update(gen_resmoke.DEFAULT_CONFIG_VALUES)
config_generator = gen_multiversion.EvergreenConfigGenerator(
evg_api, evg_config, gen_resmoke.ConfigOptions(config_options))
test_list = tests_by_task[suite["origin"]]["tests"]
for test in test_list:
# Exclude files that should be blacklisted from multiversion testing.
files_to_exclude = gen_multiversion.get_exclude_files(suite["multiversion_name"],
TASK_PATH_SUFFIX)
LOGGER.debug("Files to exclude", files_to_exclude=files_to_exclude, test=test,
suite=suite["multiversion_name"])
if test not in files_to_exclude:
# Generate the multiversion tasks for each test.
config_generator.generate_evg_tasks(test, idx)
idx += 1
dt.execution_tasks(config_generator.task_names)
evg_config.variant(generate_config.build_variant).tasks(config_generator.task_specs)
dt.execution_task(f"{BURN_IN_MULTIVERSION_TASK}_gen")
evg_config.variant(generate_config.build_variant).display_task(dt)
return evg_config
def create_multiversion_generate_tasks_config(
tests_by_task: Dict[str, TaskInfo], evg_api: EvergreenApi,
generate_config: GenerateConfig) -> BuildVariant:
"""
Create the multiversion config for the Evergreen generate.tasks file.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param evg_api: Evergreen API.
:param generate_config: Configuration of what to generate.
:return: Shrub configuration with added tasks.
"""
build_variant = BuildVariant(generate_config.build_variant)
tasks = set()
if tests_by_task:
# Get the multiversion suites that will run in as part of burn_in_multiversion.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
for suite in multiversion_suites:
idx = 0
if suite["origin"] not in tests_by_task.keys():
# Only generate burn in multiversion tasks for suites that would run the detected
# changed tests.
continue
LOGGER.debug("Generating multiversion suite",
suite=suite["multiversion_name"])
# We hardcode the number of fallback sub suites and the target resmoke time here
# since burn_in_tests cares about individual tests and not entire suites. The config
# options here are purely used to generate the proper multiversion suites to run
# tests against.
config_options = {
"suite": suite["origin"],
"fallback_num_sub_suites": 1,
"project": generate_config.project,
"build_variant": generate_config.build_variant,
"task_id": generate_config.task_id,
"task_name": suite["multiversion_name"],
"target_resmoke_time": 60,
}
config_options.update(gen_resmoke.DEFAULT_CONFIG_VALUES)
config_generator = gen_multiversion.EvergreenMultiversionConfigGenerator(
evg_api, gen_resmoke.ConfigOptions(config_options))
test_list = tests_by_task[suite["origin"]].tests
for test in test_list:
# Generate the multiversion tasks for each test.
sub_tasks = config_generator.get_burn_in_tasks(test, idx)
tasks = tasks.union(sub_tasks)
idx += 1
existing_tasks = {ExistingTask(f"{BURN_IN_MULTIVERSION_TASK}_gen")}
build_variant.display_task(BURN_IN_MULTIVERSION_TASK,
tasks,
execution_existing_tasks=existing_tasks)
return build_variant
def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig,
resmoke_args: str, generate_tasks_file: str, no_exec: bool,
evg_conf: EvergreenProjectConfig, repos: Iterable[Repo],
evg_api: EvergreenApi):
"""
Run burn_in_tests with the given configuration.
:param repeat_config: Config on how much to repeat tests.
:param generate_config: Config on how to generate tests.
:param resmoke_args: Arguments to pass to resmoke.
:param generate_tasks_file: File to write generated config to.
:param no_exec: Do not execute tests, just discover tests to run.
:param evg_conf: Evergreen configuration.
:param repos: Git repositories to check.
:param evg_api: Evergreen API client.
"""
# Populate the config values in order to use the helpers from resmokelib.suitesconfig.
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
tests_by_task = create_tests_by_task(generate_config.build_variant, repos,
evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
if generate_config.use_multiversion:
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
# We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include
# MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
json_config = create_generate_tasks_file(tests_by_task,
generate_config,
repeat_config, evg_api,
evg_conf)
_write_json_file(json_config, generate_tasks_file)
elif not no_exec:
run_tests(tests_by_task, resmoke_cmd)
else:
LOGGER.info("Not running tests due to 'no_exec' option.")
def validate_multiversion_tasks_and_suites(self) -> None:
"""
Validate that the multiversion suites and tasks match up.
We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
multiversion_suites. Multiversion passthrough suites must include
MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
Throws an exception if there are inconsistencies.
"""
multiversion_tasks = self.evg_config.get_task_names_by_tag(
MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, evg_api_config, verbose, task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
generate_config.validate(evg_conf)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config)
change_detector = EvergreenFileChangeDetector(task_id, evg_api)
changed_tests = change_detector.find_changed_tests(repos)
tests_by_task = create_tests_by_task(generate_config.build_variant,
evg_conf, changed_tests)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
# We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include
# MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
build_variant = create_multiversion_generate_tasks_config(
tests_by_task, evg_api, generate_config)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
if not validate_task_generation_limit(shrub_project):
sys.exit(1)
write_file(generate_tasks_file, shrub_project.json())
def generate_configuration(self, tests_by_task: Dict[str, TaskInfo],
target_file: str,
build_variant: str) -> GeneratedConfiguration:
"""
Generate configuration for the given tasks and tests.
:param tests_by_task: Map of what to generate.
:param target_file: Location to write generated configuration.
:param build_variant: Name of build variant being generated on.
:return: Generated configuration to create requested tasks and tests.
"""
builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter
build_variant_config = self.evg_config.get_variant(build_variant)
is_asan = build_variant_config.is_asan_build()
tasks = set()
if tests_by_task:
# Get the multiversion suites that will run in as part of burn_in_multiversion.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
for suite in multiversion_suites:
task_name = suite["origin"]
if task_name not in tests_by_task.keys():
# Only generate burn in multiversion tasks for suites that would run the
# detected changed tests.
continue
LOGGER.debug("Generating multiversion suite",
suite=suite["multiversion_name"])
test_list = tests_by_task[task_name].tests
split_params = SuiteSplitParameters(
task_name=suite["multiversion_name"],
suite_name=task_name,
filename=task_name,
test_file_filter=partial(filter_list,
input_list=test_list),
build_variant=build_variant,
is_asan=is_asan)
version_configs = self.multiversion_util.get_version_configs_for_suite(
task_name)
gen_params = MultiversionGenTaskParams(
mixed_version_configs=version_configs,
is_sharded=self.multiversion_util.is_suite_sharded(
task_name),
resmoke_args="",
parent_task_name="burn_in_tests_multiversion",
origin_suite=task_name,
use_large_distro=False,
large_distro_name=None,
name_prefix="burn_in_multiversion",
create_misc_suite=False,
add_to_display_task=False,
config_location=self.burn_in_config.build_config_location(
),
)
tasks = tasks.union(
builder.add_multiversion_burn_in_test(
split_params, gen_params))
if len(tasks) == 0:
builder.get_build_variant(build_variant)
executions_tasks = {task.name for task in tasks}
executions_tasks.add("burn_in_tests_multiversion_gen")
builder.add_display_task(display_task_name="burn_in_multiversion",
execution_task_names=executions_tasks,
build_variant=build_variant)
return builder.build(target_file)
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, no_exec, resmoke_args, evg_api_config, verbose,
task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param no_exec: Just perform test discover, do not execute the tests.
:param resmoke_args: Arguments to pass through to resmoke.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
repeat_config = RepeatConfig() # yapf: disable
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
if generate_tasks_file:
generate_config.validate(evg_conf)
evg_api = _get_evg_api(evg_api_config, False)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
tests_by_task = create_tests_by_task(generate_config.build_variant, repos,
evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
# We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include
# MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
evg_config = Configuration()
evg_config = create_multiversion_generate_tasks_config(
evg_config, tests_by_task, evg_api, generate_config)
json_config = evg_config.to_map()
tasks_to_create = len(json_config.get('tasks', []))
if tasks_to_create > MAX_TASKS_TO_CREATE:
LOGGER.warning(
"Attempting to create more tasks than max, aborting",
tasks=tasks_to_create,
max=MAX_TASKS_TO_CREATE)
sys.exit(1)
_write_json_file(json_config, generate_tasks_file)
elif not no_exec:
run_tests(tests_by_task, resmoke_cmd)
else:
LOGGER.info("Not running tests due to 'no_exec' option.")
