def create_multiversion_generate_tasks_config(
evg_config: Configuration, tests_by_task: Dict, evg_api: EvergreenApi,
generate_config: GenerateConfig) -> Configuration:
"""
Create the multiversion config for the Evergreen generate.tasks file.
:param evg_config: Shrub configuration to add to.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param evg_api: Evergreen API.
:param generate_config: Configuration of what to generate.
:return: Shrub configuration with added tasks.
"""
dt = DisplayTaskDefinition(BURN_IN_MULTIVERSION_TASK)
if tests_by_task:
multiversion_suites = get_named_suites_with_root_level_key_and_value(
MULTIVERSION_CONFIG_KEY, True)
for suite in multiversion_suites:
idx = 0
if suite not in tests_by_task.keys():
# Only generate burn in multiversion tasks for suites that would run the detected
# changed tests.
continue
LOGGER.debug("Generating multiversion suite", suite=suite)
# We hardcode the number of fallback sub suites and the target resmoke time here
# since burn_in_tests cares about individual tests and not entire suites. The config
# options here are purely used to generate the proper multiversion suites to run
# tests against.
config_options = {
"suite": suite,
"fallback_num_sub_suites": 1,
"project": generate_config.project,
"build_variant": generate_config.build_variant,
"task_id": generate_config.task_id,
"task_name": suite,
"target_resmoke_time": 60,
}
config_options.update(gen_resmoke.DEFAULT_CONFIG_VALUES)
config_generator = gen_multiversion.EvergreenConfigGenerator(
evg_api, evg_config, gen_resmoke.ConfigOptions(config_options))
test_list = tests_by_task[suite]["tests"]
for test in test_list:
# Generate the multiversion tasks for each test.
config_generator.generate_evg_tasks(test, idx)
idx += 1
dt.execution_tasks(config_generator.task_names)
evg_config.variant(generate_config.build_variant).tasks(
config_generator.task_specs)
dt.execution_task(f"{BURN_IN_MULTIVERSION_TASK}_gen")
evg_config.variant(generate_config.build_variant).display_task(dt)
return evg_config
def create_multiversion_tests_by_task_mock(n_tasks, n_tests):
multiversion_suites = get_named_suites_with_root_level_key_and_value(
under_test.MULTIVERSION_CONFIG_KEY, True)
mock_suites = [
"replica_sets_jscore_passthrough", "sharded_causally_consistent_jscore_passthrough"
]
assert n_tasks <= len(mock_suites)
for suite in mock_suites:
# We have to hardcode mock_suites because 'create_multiversion_generate_tasks_config'
# is only meant to generate the tests in multiversion_suites.
assert suite in multiversion_suites
return {
f"{mock_suites[i % len(mock_suites)]}": {
"resmoke_args": f"--suites=suite_{i}",
"tests": [f"jstests/tests_{j}" for j in range(n_tests)]
}
for i in range(n_tasks)
}
def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig,
resmoke_args: str, generate_tasks_file: str, no_exec: bool,
evg_conf: EvergreenProjectConfig, repo: Repo,
evg_api: EvergreenApi):
"""
Run burn_in_tests with the given configuration.
:param repeat_config: Config on how much to repeat tests.
:param generate_config: Config on how to generate tests.
:param resmoke_args: Arguments to pass to resmoke.
:param generate_tasks_file: File to write generated config to.
:param no_exec: Do not execute tests, just discover tests to run.
:param evg_conf: Evergreen configuration.
:param repo: Git repository.
:param evg_api: Evergreen API client.
"""
# Populate the config values in order to use the helpers from resmokelib.suitesconfig.
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
tests_by_task = create_tests_by_task(generate_config.build_variant, repo,
evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
if generate_config.use_multiversion:
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_TAG)
# We expect the number of suites with MULTIVERSION_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include BURN_IN_CONFIG_KEY
# as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key_and_value(
MULTIVERSION_CONFIG_KEY, True)
assert len(multiversion_tasks) == len(multiversion_suites)
json_config = create_generate_tasks_file(tests_by_task,
generate_config,
repeat_config, evg_api)
_write_json_file(json_config, generate_tasks_file)
elif not no_exec:
run_tests(tests_by_task, resmoke_cmd)
else:
LOGGER.info("Not running tests due to 'no_exec' option.")
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, no_exec, repeat_tests_num, repeat_tests_min,
repeat_tests_max, repeat_tests_secs, resmoke_args, local_mode,
evg_api_config, verbose, use_multiversion, task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
burn_in_tests detects jstests that are new or changed since the last git command and then
runs those tests in a loop to validate their reliability.
The `--repeat-*` arguments allow configuration of how burn_in_tests repeats tests. Tests can
either be repeated a specified number of times with the `--repeat-tests` option, or they can
be repeated for a certain time period with the `--repeat-tests-secs` option.
When the `--use-multiversion` flag is set to True, burn_in_tests will run new or changed tests
against the appropriate generated multiversion suites. The purpose of these tests are to signal
bugs in the generated multiversion suites as these tasks are excluded from the required build
variants and are only run in certain daily build variants. As such, we only expect the burn-in
multiversion tests to be run once for each binary version configuration, and `--repeat-*`
arguments should be None when `--use-multiversion` is True.
There are two modes that burn_in_tests can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param no_exec: Just perform test discover, do not execute the tests.
:param repeat_tests_num: Repeat each test this number of times.
:param repeat_tests_min: Repeat each test at least this number of times.
:param repeat_tests_max: Once this number of repetitions has been reached, stop repeating.
:param repeat_tests_secs: Continue repeating tests for this number of seconds.
:param resmoke_args: Arguments to pass through to resmoke.
:param local_mode: Don't call out to the evergreen API (used for testing).
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
# When --
repeat_config = RepeatConfig(repeat_tests_secs=repeat_tests_secs,
repeat_tests_min=repeat_tests_min,
repeat_tests_max=repeat_tests_max,
repeat_tests_num=repeat_tests_num).validate(use_multiversion) # yapf: disable
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id).validate(evg_conf, use_multiversion,
local_mode) # yapf: disable
evg_api = _get_evg_api(evg_api_config, local_mode)
repo = Repo(".")
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
tests_by_task = create_tests_by_task(build_variant, repo, evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
if use_multiversion:
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_TAG)
# We expect the number of suites with MULTIVERSION_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include BURN_IN_CONFIG_KEY
# as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key_and_value(
MULTIVERSION_CONFIG_KEY, True)
assert len(multiversion_tasks) == len(multiversion_suites)
json_config = create_generate_tasks_file(
tests_by_task,
generate_config,
repeat_config,
evg_api,
use_multiversion=use_multiversion)
_write_json_file(json_config, generate_tasks_file)
elif not no_exec:
run_tests(tests_by_task, resmoke_cmd)
else:
LOGGER.info("Not running tests due to 'no_exec' option.")
