def generate_evg_tasks(self, burn_in_test=None, burn_in_idx=0):
# pylint: disable=too-many-locals
"""
Generate evergreen tasks for multiversion tests.
The number of tasks generated equals
(the number of version configs) * (the number of generated suites).
:param burn_in_test: The test to be run as part of the burn in multiversion suite.
"""
is_sharded = is_suite_sharded(TEST_SUITE_DIR, self.options.suite)
if is_sharded:
version_configs = SHARDED_MIXED_VERSION_CONFIGS
else:
version_configs = REPL_MIXED_VERSION_CONFIGS
if self.options.is_jstestfuzz:
return self._generate_fuzzer_tasks(version_configs, is_sharded)
# Divide tests into suites based on run-time statistics for the last
# LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed
# in the misc suite.
gen_suites = generate_resmoke.GenerateSubSuites(
self.evg_api, self.options)
end_date = datetime.datetime.utcnow().replace(microsecond=0)
start_date = end_date - datetime.timedelta(
days=generate_resmoke.LOOKBACK_DURATION_DAYS)
suites = gen_suites.calculate_suites(start_date, end_date)
# Render the given suites into yml files that can be used by resmoke.py.
config_file_dict = generate_resmoke.render_suite_files(
suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR,
self.options.create_misc_suite)
generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
if burn_in_test is not None:
# Generate the subtasks to run burn_in_test against the appropriate mixed version
# configurations. The display task is defined later as part of generating the burn
# in tests.
self._generate_burn_in_execution_tasks(version_configs, suites,
burn_in_test, burn_in_idx,
is_sharded)
return self.evg_config
for version_config in version_configs:
idx = 0
for suite in suites:
# Generate the newly divided test suites
source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
self._generate_sub_task(version_config, self.task, idx,
source_suite, len(suites), is_sharded)
idx += 1
# Also generate the misc task.
misc_suite_name = "{0}_misc".format(self.options.suite)
misc_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml")
self._generate_sub_task(version_config, self.task, idx, misc_suite,
1, is_sharded)
idx += 1
self.create_display_task(self.task, self.task_specs, self.task_names)
return self.evg_config
def generate_resmoke_suites(self) -> List[Suite]:
"""Generate the resmoke configuration files for this generator."""
# Divide tests into suites based on run-time statistics for the last
# LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed
# in the misc suite.
gen_suites = generate_resmoke.GenerateSubSuites(self.evg_api, self.options)
end_date = datetime.datetime.utcnow().replace(microsecond=0)
start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS)
suites = gen_suites.calculate_suites(start_date, end_date)
# Render the given suites into yml files that can be used by resmoke.py.
config_file_dict = generate_resmoke.render_suite_files(suites, self.options.suite,
gen_suites.test_list, TEST_SUITE_DIR,
self.options.create_misc_suite)
generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
return suites
def generate_evg_tasks(self):
"""
Generate evergreen tasks for multiversion tests.
The number of tasks generated equals
(the number of configs in MIXED_VERSION_CONFIGS) * (the number of generated suites).
"""
idx = 0
# Divide tests into suites based on run-time statistics for the last
# LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed
# in the misc suite.
gen_suites = generate_resmoke.GenerateSubSuites(self.evg_api, self.options)
end_date = datetime.datetime.utcnow().replace(microsecond=0)
start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS)
suites = gen_suites.calculate_suites(start_date, end_date)
# Render the given suites into yml files that can be used by resmoke.py.
if self.options.is_sharded:
config = MultiversionConfig(update_suite_config_for_multiversion_sharded,
SHARDED_MIXED_VERSION_CONFIGS)
else:
config = MultiversionConfig(update_suite_config_for_multiversion_replset,
MIXED_VERSION_CONFIGS)
config_file_dict = generate_resmoke.render_suite_files(
suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, config.update_yaml)
for version_config in config.version_configs:
for suite in suites:
# Generate the newly divided test suites
source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
self._generate_sub_task(version_config, idx, source_suite, len(suites))
idx += 1
# Also generate the misc task.
misc_suite_name = "{0}_misc".format(self.options.suite)
source_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml")
self._generate_sub_task(version_config, idx, source_suite, 1)
idx += 1
generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
dt = DisplayTaskDefinition(self.task).execution_tasks(self.task_names)\
.execution_task("{0}_gen".format(self.task))
self.evg_config.variant(self.options.variant).tasks(self.task_specs).display_task(dt)
return self.evg_config
