def _generate_task(self,
sub_suite_name,
sub_task_name,
target_dir,
max_test_runtime=None,
expected_suite_runtime=None):
"""Generate evergreen config for a resmoke task."""
# pylint: disable=too-many-arguments
LOGGER.debug("Generating task", sub_suite=sub_suite_name)
spec = TaskSpec(sub_task_name)
self._set_task_distro(spec)
self.task_specs.append(spec)
self.task_names.append(sub_task_name)
task = self.evg_config.task(sub_task_name)
target_suite_file = os.path.join(target_dir,
os.path.basename(sub_suite_name))
run_tests_vars = self._get_run_tests_vars(target_suite_file)
use_multiversion = self.options.use_multiversion
timeout_info = self._get_timeout_command(
max_test_runtime, expected_suite_runtime,
self.options.use_default_timeouts)
commands = resmoke_commands("run generated tests", run_tests_vars,
timeout_info, use_multiversion)
self._add_dependencies(task).commands(commands)
def create_task(self, index: int, test_name: str) -> Task:
"""
Create the task configuration for the given test using the given index.
:param index: Index of sub-task being created.
:param test_name: Name of test that should be executed.
:return: Configuration for generating the specified task.
"""
resmoke_args = self.task_info.resmoke_args
sub_task_name = self.generate_name(index)
LOGGER.debug("Generating sub-task", sub_task=sub_task_name)
test_unix_style = test_name.replace('\\', '/')
run_tests_vars = {
"resmoke_args":
f"{resmoke_args} {self.repeat_config.generate_resmoke_options()} {test_unix_style}"
}
timeout = self.generate_timeouts(test_name)
commands = resmoke_commands("run tests", run_tests_vars, timeout,
self.task_info.require_multiversion)
dependencies = {TaskDependency(ARCHIVE_DIST_TEST_TASK)}
return Task(sub_task_name, commands, dependencies)
def generate_tasks(self, gen_suite: GeneratedSuite,
params: BurnInGenTaskParams) -> List[Task]:
"""Create the task configuration for the given test using the given index."""
tasks = set()
for index, suite in enumerate(gen_suite.sub_suites):
if len(suite.test_list) != 1:
raise ValueError(
f"Can only run one test per suite in burn-in; got {suite.test_list}"
)
test_name = suite.test_list[0]
test_unix_style = test_name.replace('\\', '/')
run_tests_vars = {
"suite":
gen_suite.suite_name,
"resmoke_args":
self._generate_resmoke_args(gen_suite.task_name, params,
test_unix_style)
}
timeout_cmd = self.generate_timeouts(test_name)
commands = resmoke_commands("run tests", run_tests_vars,
timeout_cmd,
params.require_multiversion_setup)
dependencies = {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)}
tasks.add(
Task(self._generate_task_name(gen_suite, index), commands,
dependencies))
return tasks
def _generate_task(self, sub_suite_name: str, sub_task_name: str,
timeout_est: TimeoutEstimate,
params: ResmokeGenTaskParams,
suite: GeneratedSuite) -> Task:
"""
Generate a shrub evergreen config for a resmoke task.
:param sub_suite_name: Name of suite being generated.
:param sub_task_name: Name of task to generate.
:param timeout_est: Estimated runtime to use for calculating timeouts.
:param params: Parameters describing how tasks should be generated.
:param suite: Parent suite being created.
:return: Shrub configuration for the described task.
"""
# pylint: disable=too-many-arguments
LOGGER.debug("Generating task", sub_suite=sub_suite_name)
target_suite_file = self.gen_task_options.suite_location(
sub_suite_name)
run_tests_vars = self._get_run_tests_vars(target_suite_file,
suite.suite_name, params,
suite.build_variant)
require_multiversion = params.require_multiversion
timeout_cmd = timeout_est.generate_timeout_cmd(
self.gen_task_options.is_patch, params.repeat_suites,
self.gen_task_options.use_default_timeouts)
commands = resmoke_commands("run generated tests", run_tests_vars,
timeout_cmd, require_multiversion)
return Task(sub_task_name, commands, self._get_dependencies())
def _generate_task(self, sub_suite_name: str, sub_task_name: str,
target_dir: str, timeout_est: TimeoutEstimate) -> Task:
"""
Generate a shrub evergreen config for a resmoke task.
:param sub_suite_name: Name of suite being generated.
:param sub_task_name: Name of task to generate.
:param target_dir: Directory containing generated suite files.
:param timeout_est: Estimated runtime to use for calculating timeouts.
:return: Shrub configuration for the described task.
"""
# pylint: disable=too-many-arguments
LOGGER.debug("Generating task", sub_suite=sub_suite_name)
# Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join
# here, just use the forward slash; otherwise the path separator will be treated as
# the escape character on Windows.
target_suite_file = '/'.join(
[target_dir, os.path.basename(sub_suite_name)])
run_tests_vars = self._get_run_tests_vars(target_suite_file)
use_multiversion = self.options.use_multiversion
timeout_cmd = timeout_est.generate_timeout_cmd(
self.options.is_patch, self.options.repeat_suites,
self.options.use_default_timeouts)
commands = resmoke_commands("run generated tests", run_tests_vars,
timeout_cmd, use_multiversion)
return Task(sub_task_name, commands, self._get_dependencies())
def create_generate_tasks_config(
evg_config: Configuration,
tests_by_task: Dict,
generate_config: GenerateConfig,
repeat_config: RepeatConfig,
evg_api: Optional[EvergreenApi],
include_gen_task: bool = True,
task_prefix: str = "burn_in") -> Configuration:
# pylint: disable=too-many-arguments,too-many-locals
"""
Create the config for the Evergreen generate.tasks file.
:param evg_config: Shrub configuration to add to.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param generate_config: Configuration of what to generate.
:param repeat_config: Configuration of how to repeat tests.
:param evg_api: Evergreen API.
:param include_gen_task: Should generating task be include in display task.
:param task_prefix: Prefix all task names with this.
:return: Shrub configuration with added tasks.
"""
task_list = TaskList(evg_config)
resmoke_options = repeat_config.generate_resmoke_options()
for task in sorted(tests_by_task):
test_list = tests_by_task[task]["tests"]
for index, test in enumerate(test_list):
# TODO: Extract multiversion related code into separate tooling - SERVER-47137
multiversion_path = tests_by_task[task].get("use_multiversion")
display_task_name = tests_by_task[task]["display_task_name"]
task_runtime_stats = _get_task_runtime_history(
evg_api, generate_config.project, display_task_name,
generate_config.build_variant)
resmoke_args = tests_by_task[task]["resmoke_args"]
distro = tests_by_task[task].get("distro", generate_config.distro)
# Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join
# here, just use the forward slash; otherwise the path separator will be treated as
# the escape character on Windows.
sub_task_name = name_generated_task(
f"{task_prefix}:{display_task_name}", index, len(test_list),
generate_config.run_build_variant)
LOGGER.debug("Generating sub-task", sub_task=sub_task_name)
test_unix_style = test.replace('\\', '/')
run_tests_vars = {
"resmoke_args":
f"{resmoke_args} {resmoke_options} {test_unix_style}"
}
if multiversion_path:
run_tests_vars["task_path_suffix"] = multiversion_path
timeout = _generate_timeouts(repeat_config, test,
task_runtime_stats)
commands = resmoke_commands("run tests", run_tests_vars, timeout,
multiversion_path)
task_list.add_task(sub_task_name, commands, ["compile"], distro)
existing_tasks = [BURN_IN_TESTS_GEN_TASK] if include_gen_task else None
task_list.add_to_variant(generate_config.run_build_variant,
BURN_IN_TESTS_TASK, existing_tasks)
return evg_config
def _generate_task(self,
sub_suite_name,
sub_task_name,
target_dir,
max_test_runtime=None,
expected_suite_runtime=None):
"""Generate evergreen config for a resmoke task."""
# pylint: disable=too-many-arguments
LOGGER.debug("Generating task", sub_suite=sub_suite_name)
spec = TaskSpec(sub_task_name)
self._set_task_distro(spec)
self.task_specs.append(spec)
self.task_names.append(sub_task_name)
task = self.evg_config.task(sub_task_name)
# Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join
# here, just use the forward slash; otherwise the path separator will be treated as
# the escape character on Windows.
target_suite_file = '/'.join(
[target_dir, os.path.basename(sub_suite_name)])
run_tests_vars = self._get_run_tests_vars(target_suite_file)
use_multiversion = self.options.use_multiversion
timeout_info = self._get_timeout_command(
max_test_runtime, expected_suite_runtime,
self.options.use_default_timeouts)
commands = resmoke_commands("run generated tests", run_tests_vars,
timeout_info, use_multiversion)
self._add_dependencies(task).commands(commands)
def test_basic_command(self):
run_tests = "run tests"
test_vars = {}
timeout_info = under_test.TimeoutInfo.default_timeout()
commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info)
# 4 expected command = 1 for setup + 1 for evergreen credentials + 1 for running tests +
# 1 for validate resmoke tests runtime.
self.assertEqual(4, len(commands))
def test_basic_command(self):
run_tests = "run tests"
test_vars = {}
timeout_info = under_test.TimeoutInfo.default_timeout()
commands = under_test.resmoke_commands(run_tests, test_vars,
timeout_info)
# 2 expected command = 1 for setup + 1 for running tests.
self.assertEqual(2, len(commands))
def test_with_timeout(self):
run_tests = "run tests"
test_vars = {}
timeout_info = under_test.TimeoutInfo.overridden(timeout=5)
commands = under_test.resmoke_commands(run_tests, test_vars,
timeout_info)
# 3 expected command = 1 for setup + 1 for running tests + 1 for timeout.
self.assertEqual(3, len(commands))
def test_with_timeout(self):
run_tests = "run tests"
test_vars = {}
timeout_info = under_test.TimeoutInfo.overridden(timeout=5)
commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info)
# 5 expected command = 1 for setup + 1 for evergreen credentials + 1 for running tests +
# 1 for timeout + 1 for validate resmoke tests runtime.
self.assertEqual(5, len(commands))
def test_with_everything(self):
run_tests = "run tests"
test_vars = {}
timeout_info = under_test.TimeoutInfo.overridden(timeout=5)
commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info,
require_multiversion_setup=True)
# 8 expected command = 1 for setup + 1 for evergreen credentials + 3 for multiversion setup +
# 1 for running tests + 1 for timeout + 1 for validate resmoke tests runtime.
self.assertEqual(8, len(commands))
def test_with_multiversion(self):
run_tests = "run tests"
test_vars = {}
timeout_info = under_test.TimeoutInfo.default_timeout()
commands = under_test.resmoke_commands(run_tests,
test_vars,
timeout_info,
require_multiversion_setup=True)
# 4 expected command = 1 for setup + 1 for running tests + 3 for multiversion setup.
self.assertEqual(6, len(commands))
def test_with_multiversion(self):
run_tests = "run tests"
test_vars = {}
timeout_info = under_test.TimeoutInfo.default_timeout()
commands = under_test.resmoke_commands(run_tests,
test_vars,
timeout_info,
use_multiversion="multiversion")
# 3 expected command = 1 for setup + 1 for running tests + 1 for multiversion setup.
self.assertEqual(3, len(commands))
def create_generate_tasks_config(
evg_config: Configuration,
tests_by_task: Dict,
generate_config: GenerateConfig,
repeat_config: RepeatConfig,
evg_api: Optional[EvergreenApi],
include_gen_task: bool = True,
task_prefix: str = "burn_in") -> Configuration:
# pylint: disable=too-many-arguments,too-many-locals
"""
Create the config for the Evergreen generate.tasks file.
:param evg_config: Shrub configuration to add to.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param generate_config: Configuration of what to generate.
:param repeat_config: Configuration of how to repeat tests.
:param evg_api: Evergreen API.
:param include_gen_task: Should generating task be include in display task.
:param task_prefix: Prefix all task names with this.
:return: Shrub configuration with added tasks.
"""
task_list = TaskList(evg_config)
resmoke_options = repeat_config.generate_resmoke_options()
for task in sorted(tests_by_task):
multiversion_path = tests_by_task[task].get("use_multiversion")
task_runtime_stats = _get_task_runtime_history(
evg_api, generate_config.project, task,
generate_config.build_variant)
resmoke_args = tests_by_task[task]["resmoke_args"]
test_list = tests_by_task[task]["tests"]
distro = tests_by_task[task].get("distro", generate_config.distro)
for index, test in enumerate(test_list):
sub_task_name = name_generated_task(
f"{task_prefix}:{task}", index, len(test_list),
generate_config.run_build_variant)
LOGGER.debug("Generating sub-task", sub_task=sub_task_name)
run_tests_vars = {
"resmoke_args": f"{resmoke_args} {resmoke_options} {test}"
}
if multiversion_path:
run_tests_vars["task_path_suffix"] = multiversion_path
timeout = _generate_timeouts(repeat_config.repeat_tests_secs, test,
task_runtime_stats)
commands = resmoke_commands("run tests", run_tests_vars, timeout,
multiversion_path)
task_list.add_task(sub_task_name, commands, ["compile"], distro)
existing_tasks = [BURN_IN_TESTS_GEN_TASK] if include_gen_task else None
task_list.add_to_variant(generate_config.run_build_variant,
BURN_IN_TESTS_TASK, existing_tasks)
return evg_config
def test_with_everything(self):
run_tests = "run tests"
test_vars = {}
timeout_info = under_test.TimeoutInfo.overridden(timeout=5)
commands = under_test.resmoke_commands(run_tests,
test_vars,
timeout_info,
use_multiversion="multiversion")
# 4 expected command = 1 for setup + 1 for running tests + 1 for multiversion setup +
# 1 for timeout.
self.assertEqual(4, len(commands))
def _generate_task(self, sub_suite_file, sub_task_name: str,
timeout_est: TimeoutEstimate,
params: ResmokeGenTaskParams,
suite: GeneratedSuite) -> Task:
"""
Generate a shrub evergreen config for a resmoke task.
:param sub_suite_file: Name of the suite file to run in the generated task.
:param sub_task_name: Name of task to generate.
:param timeout_est: Estimated runtime to use for calculating timeouts.
:param params: Parameters describing how tasks should be generated.
:param suite: Parent suite being created.
:return: Shrub configuration for the described task.
"""
# pylint: disable=too-many-arguments
LOGGER.debug("Generating task running suite",
sub_task_name=sub_task_name,
sub_suite_file=sub_suite_file)
# Some splits don't generate new physical config files.
if params.config_location is not None:
sub_suite_file_path = self.gen_task_options.suite_location(
sub_suite_file)
else:
sub_suite_file_path = None
run_tests_vars = self._get_run_tests_vars(
sub_suite_file_path=sub_suite_file_path,
suite_file=suite.suite_name,
task_name=suite.task_name,
params=params)
require_multiversion_setup = params.require_multiversion_setup
timeout_cmd = timeout_est.generate_timeout_cmd(
self.gen_task_options.is_patch, params.repeat_suites,
self.gen_task_options.use_default_timeouts)
commands = resmoke_commands("run generated tests", run_tests_vars,
timeout_cmd, require_multiversion_setup)
return Task(sub_task_name, commands, self._get_dependencies())
def _create_task(index: int, test_count: int, test: str, task_data: Dict,
task_runtime_stats: List[TestRuntime],
generate_config: GenerateConfig, repeat_config: RepeatConfig,
task_prefix: str) -> Task:
# pylint: disable=too-many-arguments,too-many-locals
"""
Create the described shrub sub task.
:param index: Index of task being created.
:param test_count: Total number of testing being created.
:param test: Test task is being generated for.
:param task_data: Data about task to create.
:param task_runtime_stats: Historical runtime of test.
:param generate_config: Configuration of how to generate the task.
:param repeat_config: Configuration of how the task should be repeated.
:param task_prefix: String to prefix generated task with.
:return: Shrub task for given configuration.
"""
# TODO: Extract multiversion related code into separate tooling - SERVER-47137
multiversion_path = task_data.get("use_multiversion")
display_task_name = task_data["display_task_name"]
resmoke_args = task_data["resmoke_args"]
sub_task_name = name_generated_task(f"{task_prefix}:{display_task_name}",
index, test_count,
generate_config.run_build_variant)
LOGGER.debug("Generating sub-task", sub_task=sub_task_name)
test_unix_style = test.replace('\\', '/')
run_tests_vars = {
"resmoke_args":
f"{resmoke_args} {repeat_config.generate_resmoke_options()} {test_unix_style}"
}
if multiversion_path:
run_tests_vars["task_path_suffix"] = multiversion_path
timeout = _generate_timeouts(repeat_config, test, task_runtime_stats)
commands = resmoke_commands("run tests", run_tests_vars, timeout,
multiversion_path)
dependencies = {TaskDependency("compile")}
return Task(sub_task_name, commands, dependencies)