def _generate_task(self, sub_suite_name, sub_task_name, target_dir, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" # pylint: disable=too-many-arguments LOGGER.debug("Generating task", sub_suite=sub_suite_name) spec = TaskSpec(sub_task_name) self._set_task_distro(spec) self.task_specs.append(spec) self.task_names.append(sub_task_name) task = self.evg_config.task(sub_task_name) target_suite_file = os.path.join(target_dir, os.path.basename(sub_suite_name)) run_tests_vars = self._get_run_tests_vars(target_suite_file) use_multiversion = self.options.use_multiversion timeout_info = self._get_timeout_command( max_test_runtime, expected_suite_runtime, self.options.use_default_timeouts) commands = resmoke_commands("run generated tests", run_tests_vars, timeout_info, use_multiversion) self._add_dependencies(task).commands(commands)
def create_task(self, index: int, test_name: str) -> Task: """ Create the task configuration for the given test using the given index. :param index: Index of sub-task being created. :param test_name: Name of test that should be executed. :return: Configuration for generating the specified task. """ resmoke_args = self.task_info.resmoke_args sub_task_name = self.generate_name(index) LOGGER.debug("Generating sub-task", sub_task=sub_task_name) test_unix_style = test_name.replace('\\', '/') run_tests_vars = { "resmoke_args": f"{resmoke_args} {self.repeat_config.generate_resmoke_options()} {test_unix_style}" } timeout = self.generate_timeouts(test_name) commands = resmoke_commands("run tests", run_tests_vars, timeout, self.task_info.require_multiversion) dependencies = {TaskDependency(ARCHIVE_DIST_TEST_TASK)} return Task(sub_task_name, commands, dependencies)
def generate_tasks(self, gen_suite: GeneratedSuite, params: BurnInGenTaskParams) -> List[Task]: """Create the task configuration for the given test using the given index.""" tasks = set() for index, suite in enumerate(gen_suite.sub_suites): if len(suite.test_list) != 1: raise ValueError( f"Can only run one test per suite in burn-in; got {suite.test_list}" ) test_name = suite.test_list[0] test_unix_style = test_name.replace('\\', '/') run_tests_vars = { "suite": gen_suite.suite_name, "resmoke_args": self._generate_resmoke_args(gen_suite.task_name, params, test_unix_style) } timeout_cmd = self.generate_timeouts(test_name) commands = resmoke_commands("run tests", run_tests_vars, timeout_cmd, params.require_multiversion_setup) dependencies = {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)} tasks.add( Task(self._generate_task_name(gen_suite, index), commands, dependencies)) return tasks
def _generate_task(self, sub_suite_name: str, sub_task_name: str, timeout_est: TimeoutEstimate, params: ResmokeGenTaskParams, suite: GeneratedSuite) -> Task: """ Generate a shrub evergreen config for a resmoke task. :param sub_suite_name: Name of suite being generated. :param sub_task_name: Name of task to generate. :param timeout_est: Estimated runtime to use for calculating timeouts. :param params: Parameters describing how tasks should be generated. :param suite: Parent suite being created. :return: Shrub configuration for the described task. """ # pylint: disable=too-many-arguments LOGGER.debug("Generating task", sub_suite=sub_suite_name) target_suite_file = self.gen_task_options.suite_location( sub_suite_name) run_tests_vars = self._get_run_tests_vars(target_suite_file, suite.suite_name, params, suite.build_variant) require_multiversion = params.require_multiversion timeout_cmd = timeout_est.generate_timeout_cmd( self.gen_task_options.is_patch, params.repeat_suites, self.gen_task_options.use_default_timeouts) commands = resmoke_commands("run generated tests", run_tests_vars, timeout_cmd, require_multiversion) return Task(sub_task_name, commands, self._get_dependencies())
def _generate_task(self, sub_suite_name: str, sub_task_name: str, target_dir: str, timeout_est: TimeoutEstimate) -> Task: """ Generate a shrub evergreen config for a resmoke task. :param sub_suite_name: Name of suite being generated. :param sub_task_name: Name of task to generate. :param target_dir: Directory containing generated suite files. :param timeout_est: Estimated runtime to use for calculating timeouts. :return: Shrub configuration for the described task. """ # pylint: disable=too-many-arguments LOGGER.debug("Generating task", sub_suite=sub_suite_name) # Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join # here, just use the forward slash; otherwise the path separator will be treated as # the escape character on Windows. target_suite_file = '/'.join( [target_dir, os.path.basename(sub_suite_name)]) run_tests_vars = self._get_run_tests_vars(target_suite_file) use_multiversion = self.options.use_multiversion timeout_cmd = timeout_est.generate_timeout_cmd( self.options.is_patch, self.options.repeat_suites, self.options.use_default_timeouts) commands = resmoke_commands("run generated tests", run_tests_vars, timeout_cmd, use_multiversion) return Task(sub_task_name, commands, self._get_dependencies())
def create_generate_tasks_config( evg_config: Configuration, tests_by_task: Dict, generate_config: GenerateConfig, repeat_config: RepeatConfig, evg_api: Optional[EvergreenApi], include_gen_task: bool = True, task_prefix: str = "burn_in") -> Configuration: # pylint: disable=too-many-arguments,too-many-locals """ Create the config for the Evergreen generate.tasks file. :param evg_config: Shrub configuration to add to. :param tests_by_task: Dictionary of tests to generate tasks for. :param generate_config: Configuration of what to generate. :param repeat_config: Configuration of how to repeat tests. :param evg_api: Evergreen API. :param include_gen_task: Should generating task be include in display task. :param task_prefix: Prefix all task names with this. :return: Shrub configuration with added tasks. """ task_list = TaskList(evg_config) resmoke_options = repeat_config.generate_resmoke_options() for task in sorted(tests_by_task): test_list = tests_by_task[task]["tests"] for index, test in enumerate(test_list): # TODO: Extract multiversion related code into separate tooling - SERVER-47137 multiversion_path = tests_by_task[task].get("use_multiversion") display_task_name = tests_by_task[task]["display_task_name"] task_runtime_stats = _get_task_runtime_history( evg_api, generate_config.project, display_task_name, generate_config.build_variant) resmoke_args = tests_by_task[task]["resmoke_args"] distro = tests_by_task[task].get("distro", generate_config.distro) # Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join # here, just use the forward slash; otherwise the path separator will be treated as # the escape character on Windows. sub_task_name = name_generated_task( f"{task_prefix}:{display_task_name}", index, len(test_list), generate_config.run_build_variant) LOGGER.debug("Generating sub-task", sub_task=sub_task_name) test_unix_style = test.replace('\\', '/') run_tests_vars = { "resmoke_args": f"{resmoke_args} {resmoke_options} {test_unix_style}" } if multiversion_path: run_tests_vars["task_path_suffix"] = multiversion_path timeout = _generate_timeouts(repeat_config, test, task_runtime_stats) commands = resmoke_commands("run tests", run_tests_vars, timeout, multiversion_path) task_list.add_task(sub_task_name, commands, ["compile"], distro) existing_tasks = [BURN_IN_TESTS_GEN_TASK] if include_gen_task else None task_list.add_to_variant(generate_config.run_build_variant, BURN_IN_TESTS_TASK, existing_tasks) return evg_config
def _generate_task(self, sub_suite_name, sub_task_name, target_dir, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" # pylint: disable=too-many-arguments LOGGER.debug("Generating task", sub_suite=sub_suite_name) spec = TaskSpec(sub_task_name) self._set_task_distro(spec) self.task_specs.append(spec) self.task_names.append(sub_task_name) task = self.evg_config.task(sub_task_name) # Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join # here, just use the forward slash; otherwise the path separator will be treated as # the escape character on Windows. target_suite_file = '/'.join( [target_dir, os.path.basename(sub_suite_name)]) run_tests_vars = self._get_run_tests_vars(target_suite_file) use_multiversion = self.options.use_multiversion timeout_info = self._get_timeout_command( max_test_runtime, expected_suite_runtime, self.options.use_default_timeouts) commands = resmoke_commands("run generated tests", run_tests_vars, timeout_info, use_multiversion) self._add_dependencies(task).commands(commands)
def test_basic_command(self): run_tests = "run tests" test_vars = {} timeout_info = under_test.TimeoutInfo.default_timeout() commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info) # 4 expected command = 1 for setup + 1 for evergreen credentials + 1 for running tests + # 1 for validate resmoke tests runtime. self.assertEqual(4, len(commands))
def test_basic_command(self): run_tests = "run tests" test_vars = {} timeout_info = under_test.TimeoutInfo.default_timeout() commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info) # 2 expected command = 1 for setup + 1 for running tests. self.assertEqual(2, len(commands))
def test_with_timeout(self): run_tests = "run tests" test_vars = {} timeout_info = under_test.TimeoutInfo.overridden(timeout=5) commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info) # 3 expected command = 1 for setup + 1 for running tests + 1 for timeout. self.assertEqual(3, len(commands))
def test_with_timeout(self): run_tests = "run tests" test_vars = {} timeout_info = under_test.TimeoutInfo.overridden(timeout=5) commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info) # 5 expected command = 1 for setup + 1 for evergreen credentials + 1 for running tests + # 1 for timeout + 1 for validate resmoke tests runtime. self.assertEqual(5, len(commands))
def test_with_everything(self): run_tests = "run tests" test_vars = {} timeout_info = under_test.TimeoutInfo.overridden(timeout=5) commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info, require_multiversion_setup=True) # 8 expected command = 1 for setup + 1 for evergreen credentials + 3 for multiversion setup + # 1 for running tests + 1 for timeout + 1 for validate resmoke tests runtime. self.assertEqual(8, len(commands))
def test_with_multiversion(self): run_tests = "run tests" test_vars = {} timeout_info = under_test.TimeoutInfo.default_timeout() commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info, require_multiversion_setup=True) # 4 expected command = 1 for setup + 1 for running tests + 3 for multiversion setup. self.assertEqual(6, len(commands))
def test_with_multiversion(self): run_tests = "run tests" test_vars = {} timeout_info = under_test.TimeoutInfo.default_timeout() commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info, use_multiversion="multiversion") # 3 expected command = 1 for setup + 1 for running tests + 1 for multiversion setup. self.assertEqual(3, len(commands))
def create_generate_tasks_config( evg_config: Configuration, tests_by_task: Dict, generate_config: GenerateConfig, repeat_config: RepeatConfig, evg_api: Optional[EvergreenApi], include_gen_task: bool = True, task_prefix: str = "burn_in") -> Configuration: # pylint: disable=too-many-arguments,too-many-locals """ Create the config for the Evergreen generate.tasks file. :param evg_config: Shrub configuration to add to. :param tests_by_task: Dictionary of tests to generate tasks for. :param generate_config: Configuration of what to generate. :param repeat_config: Configuration of how to repeat tests. :param evg_api: Evergreen API. :param include_gen_task: Should generating task be include in display task. :param task_prefix: Prefix all task names with this. :return: Shrub configuration with added tasks. """ task_list = TaskList(evg_config) resmoke_options = repeat_config.generate_resmoke_options() for task in sorted(tests_by_task): multiversion_path = tests_by_task[task].get("use_multiversion") task_runtime_stats = _get_task_runtime_history( evg_api, generate_config.project, task, generate_config.build_variant) resmoke_args = tests_by_task[task]["resmoke_args"] test_list = tests_by_task[task]["tests"] distro = tests_by_task[task].get("distro", generate_config.distro) for index, test in enumerate(test_list): sub_task_name = name_generated_task( f"{task_prefix}:{task}", index, len(test_list), generate_config.run_build_variant) LOGGER.debug("Generating sub-task", sub_task=sub_task_name) run_tests_vars = { "resmoke_args": f"{resmoke_args} {resmoke_options} {test}" } if multiversion_path: run_tests_vars["task_path_suffix"] = multiversion_path timeout = _generate_timeouts(repeat_config.repeat_tests_secs, test, task_runtime_stats) commands = resmoke_commands("run tests", run_tests_vars, timeout, multiversion_path) task_list.add_task(sub_task_name, commands, ["compile"], distro) existing_tasks = [BURN_IN_TESTS_GEN_TASK] if include_gen_task else None task_list.add_to_variant(generate_config.run_build_variant, BURN_IN_TESTS_TASK, existing_tasks) return evg_config
def test_with_everything(self): run_tests = "run tests" test_vars = {} timeout_info = under_test.TimeoutInfo.overridden(timeout=5) commands = under_test.resmoke_commands(run_tests, test_vars, timeout_info, use_multiversion="multiversion") # 4 expected command = 1 for setup + 1 for running tests + 1 for multiversion setup + # 1 for timeout. self.assertEqual(4, len(commands))
def _generate_task(self, sub_suite_file, sub_task_name: str, timeout_est: TimeoutEstimate, params: ResmokeGenTaskParams, suite: GeneratedSuite) -> Task: """ Generate a shrub evergreen config for a resmoke task. :param sub_suite_file: Name of the suite file to run in the generated task. :param sub_task_name: Name of task to generate. :param timeout_est: Estimated runtime to use for calculating timeouts. :param params: Parameters describing how tasks should be generated. :param suite: Parent suite being created. :return: Shrub configuration for the described task. """ # pylint: disable=too-many-arguments LOGGER.debug("Generating task running suite", sub_task_name=sub_task_name, sub_suite_file=sub_suite_file) # Some splits don't generate new physical config files. if params.config_location is not None: sub_suite_file_path = self.gen_task_options.suite_location( sub_suite_file) else: sub_suite_file_path = None run_tests_vars = self._get_run_tests_vars( sub_suite_file_path=sub_suite_file_path, suite_file=suite.suite_name, task_name=suite.task_name, params=params) require_multiversion_setup = params.require_multiversion_setup timeout_cmd = timeout_est.generate_timeout_cmd( self.gen_task_options.is_patch, params.repeat_suites, self.gen_task_options.use_default_timeouts) commands = resmoke_commands("run generated tests", run_tests_vars, timeout_cmd, require_multiversion_setup) return Task(sub_task_name, commands, self._get_dependencies())
def _create_task(index: int, test_count: int, test: str, task_data: Dict, task_runtime_stats: List[TestRuntime], generate_config: GenerateConfig, repeat_config: RepeatConfig, task_prefix: str) -> Task: # pylint: disable=too-many-arguments,too-many-locals """ Create the described shrub sub task. :param index: Index of task being created. :param test_count: Total number of testing being created. :param test: Test task is being generated for. :param task_data: Data about task to create. :param task_runtime_stats: Historical runtime of test. :param generate_config: Configuration of how to generate the task. :param repeat_config: Configuration of how the task should be repeated. :param task_prefix: String to prefix generated task with. :return: Shrub task for given configuration. """ # TODO: Extract multiversion related code into separate tooling - SERVER-47137 multiversion_path = task_data.get("use_multiversion") display_task_name = task_data["display_task_name"] resmoke_args = task_data["resmoke_args"] sub_task_name = name_generated_task(f"{task_prefix}:{display_task_name}", index, test_count, generate_config.run_build_variant) LOGGER.debug("Generating sub-task", sub_task=sub_task_name) test_unix_style = test.replace('\\', '/') run_tests_vars = { "resmoke_args": f"{resmoke_args} {repeat_config.generate_resmoke_options()} {test_unix_style}" } if multiversion_path: run_tests_vars["task_path_suffix"] = multiversion_path timeout = _generate_timeouts(repeat_config, test, task_runtime_stats) commands = resmoke_commands("run tests", run_tests_vars, timeout, multiversion_path) dependencies = {TaskDependency("compile")} return Task(sub_task_name, commands, dependencies)