def create_task(self, index: int, test_name: str) -> Task: """ Create the task configuration for the given test using the given index. :param index: Index of sub-task being created. :param test_name: Name of test that should be executed. :return: Configuration for generating the specified task. """ resmoke_args = self.task_info.resmoke_args sub_task_name = self.generate_name(index) LOGGER.debug("Generating sub-task", sub_task=sub_task_name) test_unix_style = test_name.replace('\\', '/') run_tests_vars = { "resmoke_args": f"{resmoke_args} {self.repeat_config.generate_resmoke_options()} {test_unix_style}" } timeout = self.generate_timeouts(test_name) commands = resmoke_commands("run tests", run_tests_vars, timeout, self.task_info.require_multiversion) dependencies = {TaskDependency(ARCHIVE_DIST_TEST_TASK)} return Task(sub_task_name, commands, dependencies)
def generate_tasks(self, gen_suite: GeneratedSuite, params: BurnInGenTaskParams) -> List[Task]: """Create the task configuration for the given test using the given index.""" tasks = set() for index, suite in enumerate(gen_suite.sub_suites): if len(suite.test_list) != 1: raise ValueError( f"Can only run one test per suite in burn-in; got {suite.test_list}" ) test_name = suite.test_list[0] test_unix_style = test_name.replace('\\', '/') run_tests_vars = { "suite": gen_suite.suite_name, "resmoke_args": self._generate_resmoke_args(gen_suite.task_name, params, test_unix_style) } timeout_cmd = self.generate_timeouts(test_name) commands = resmoke_commands("run tests", run_tests_vars, timeout_cmd, params.require_multiversion_setup) dependencies = {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)} tasks.add( Task(self._generate_task_name(gen_suite, index), commands, dependencies)) return tasks
def _get_dependencies(params: ResmokeGenTaskParams) -> Set[TaskDependency]: """Get the set of dependency tasks for these suites.""" dependencies = { TaskDependency(dependency) for dependency in params.dependencies } return dependencies
def _generate_task(self, sub_task_name: str, sub_suite_name: str, mixed_version_config: str, params: MultiversionGenTaskParams, build_variant: str) -> Task: """ Generate a sub task to be run with the provided suite and mixed version config. :param sub_task_name: Name of task being generated. :param sub_suite_name: Name of suite to run. :param mixed_version_config: Versions task is being generated for. :param params: Parameters for how tasks should be generated. :return: Shrub configuration for task specified. """ suite_file = self.gen_task_options.suite_location(f"{sub_suite_name}_{build_variant}.yml") run_tests_vars = { "resmoke_args": self._build_resmoke_args(suite_file, mixed_version_config, params), "task": params.parent_task_name, "gen_task_config_location": params.config_location, } commands = [ FunctionCall("do setup"), # Fetch and download the proper mongod binaries before running multiversion tests. FunctionCall("configure evergreen api credentials"), FunctionCall("do multiversion setup"), FunctionCall("run generated tests", run_tests_vars), ] return Task(sub_task_name, commands, {TaskDependency("archive_dist_test_debug")})
def build_fuzzer_sub_task(task_index: int, params: FuzzerGenTaskParams) -> Task: """ Build a shrub task to run the fuzzer. :param task_index: Index of sub task being generated. :param params: Parameters describing how tasks should be generated. :return: Shrub task to run the fuzzer. """ sub_task_name = taskname.name_generated_task(params.task_name, task_index, params.num_tasks, params.variant) run_tests_vars = { "continue_on_failure": params.continue_on_failure, "resmoke_args": params.get_resmoke_args(), "resmoke_jobs_max": params.resmoke_jobs_max, "should_shuffle": params.should_shuffle, "require_multiversion_setup": params.require_multiversion_setup, "timeout_secs": params.timeout_secs, "task": params.task_name, "gen_task_config_location": params.config_location, "suite": params.suite, } # yapf: disable timeout_info = TimeoutInfo.overridden(timeout=params.timeout_secs) commands = [ timeout_info.cmd, FunctionCall("do setup"), FunctionCall(CONFIGURE_EVG_CREDENTIALS), FunctionCall("setup jstestfuzz"), FunctionCall("run jstestfuzz", params.jstestfuzz_params()), FunctionCall(RUN_GENERATED_TESTS, run_tests_vars) ] return Task(sub_task_name, commands, {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)})
def build_fuzzer_sub_task(task_index: int, params: FuzzerGenTaskParams, version: str) -> Task: """ Build a shrub task to run the fuzzer. :param task_index: Index of sub task being generated. :param params: Parameters describing how tasks should be generated. :param version: Multiversion version to generate against. :return: Shrub task to run the fuzzer. """ sub_task_name = taskname.name_generated_task( params.get_task_name(version), task_index, params.num_tasks, params.variant) suite_arg = f"--suites={params.suite}" run_tests_vars = { "continue_on_failure": params.continue_on_failure, "resmoke_args": f"{suite_arg} {params.get_resmoke_args()}", "resmoke_jobs_max": params.resmoke_jobs_max, "should_shuffle": params.should_shuffle, "require_multiversion": params.require_multiversion, "timeout_secs": params.timeout_secs, "task": params.get_task_name(version), "gen_task_config_location": params.config_location, } # yapf: disable timeout_info = TimeoutInfo.overridden(timeout=params.timeout_secs) commands = [] if params.require_multiversion: commands += [FunctionCall("git get project no modules")] commands += [ timeout_info.cmd, FunctionCall("do setup"), FunctionCall("configure evergreen api credentials") if params.require_multiversion else None, FunctionCall("do multiversion setup") if params.require_multiversion else None, FunctionCall("setup jstestfuzz"), FunctionCall("run jstestfuzz", params.jstestfuzz_params()), FunctionCall("run generated tests", run_tests_vars) ] commands = [command for command in commands if command is not None] return Task(sub_task_name, commands, {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)})
def build_fuzzer_sub_task(task_index: int, params: FuzzerGenTaskParams) -> Task: """ Build a shrub task to run the fuzzer. :param task_index: Index of sub task being generated. :param params: Parameters describing how tasks should be generated. :return: Shrub task to run the fuzzer. """ sub_task_name = taskname.name_generated_task(params.task_name, task_index, params.num_tasks, params.variant) run_tests_vars = { "continue_on_failure": params.continue_on_failure, "resmoke_args": params.get_resmoke_args(), "resmoke_jobs_max": params.resmoke_jobs_max, "should_shuffle": params.should_shuffle, "require_multiversion_setup": params.require_multiversion_setup, "timeout_secs": params.timeout_secs, "task": params.task_name, # This expansion's name was shortened to reduce the overall size of # the generated configuration file # gtcl = gen_task_config_location "gtcl": params.config_location, "suite": params.suite, } # yapf: disable timeout_info = TimeoutInfo.overridden(timeout=params.timeout_secs) commands = [ timeout_info.cmd, FunctionCall("do setup"), FunctionCall(CONFIGURE_EVG_CREDENTIALS), FunctionCall("setup jstestfuzz"), FunctionCall("run jstestfuzz", params.jstestfuzz_params()), FunctionCall(RUN_GENERATED_TESTS, run_tests_vars), FunctionCall("minimize jstestfuzz") ] dependencies = { TaskDependency(dependency) for dependency in params.dependencies } return Task(sub_task_name, commands, dependencies)
def _generate_sub_task(self, mixed_version_config: str, task: str, task_index: int, suite: str, num_suites: int, is_sharded: bool, burn_in_test: Optional[str] = None) -> Task: # pylint: disable=too-many-arguments """ Generate a sub task to be run with the provided suite and mixed version config. :param mixed_version_config: mixed version configuration. :param task: Name of task. :param task_index: Index of task to generate. :param suite: Name of suite being generated. :param num_suites: Number os suites being generated. :param is_sharded: If this is being generated for a sharded configuration. :param burn_in_test: If generation is for burn_in, burn_in options to use. :return: Shrub configuration for task specified. """ # Create a sub task name appended with the task_index and build variant name. task_name = f"{task}_{mixed_version_config}" sub_task_name = taskname.name_generated_task(task_name, task_index, num_suites, self.options.variant) gen_task_name = BURN_IN_TASK if burn_in_test is not None else self.task run_tests_vars = { "resmoke_args": _generate_resmoke_args(suite, mixed_version_config, is_sharded, self.options, burn_in_test), "task": gen_task_name, } commands = [ FunctionCall("do setup"), # Fetch and download the proper mongod binaries before running multiversion tests. FunctionCall("configure evergreen api credentials"), FunctionCall("do multiversion setup"), FunctionCall("run generated tests", run_tests_vars), ] return Task(sub_task_name, commands, {TaskDependency("archive_dist_test_debug")})
def build_fuzzer_sub_task(task_name: str, task_index: int, options: ConfigOptions) -> Task: """ Build a shrub task to run the fuzzer. :param task_name: Parent name of task. :param task_index: Index of sub task being generated. :param options: Options to use for task. :return: Shrub task to run the fuzzer. """ sub_task_name = taskname.name_generated_task(task_name, task_index, options.num_tasks, options.variant) run_jstestfuzz_vars = { "jstestfuzz_vars": "--numGeneratedFiles {0} {1}".format(options.num_files, options.jstestfuzz_vars), "npm_command": options.npm_command, } suite_arg = f"--suites={options.suite}" run_tests_vars = { "continue_on_failure": options.continue_on_failure, "resmoke_args": f"{suite_arg} {options.resmoke_args}", "resmoke_jobs_max": options.resmoke_jobs_max, "should_shuffle": options.should_shuffle, "task_path_suffix": options.use_multiversion, "timeout_secs": options.timeout_secs, "task": options.name } # yapf: disable commands = [ FunctionCall("do setup"), FunctionCall("configure evergreen api credentials") if options.use_multiversion else None, FunctionCall("do multiversion setup") if options.use_multiversion else None, FunctionCall("setup jstestfuzz"), FunctionCall("run jstestfuzz", run_jstestfuzz_vars), FunctionCall("run generated tests", run_tests_vars) ] commands = [command for command in commands if command is not None] return Task(sub_task_name, commands, {TaskDependency("archive_dist_test_debug")})
def build_fuzzer_sub_task(task_index: int, params: FuzzerGenTaskParams) -> Task: """ Build a shrub task to run the fuzzer. :param task_index: Index of sub task being generated. :param params: Parameters describing how tasks should be generated. :return: Shrub task to run the fuzzer. """ sub_task_name = taskname.name_generated_task(params.task_name, task_index, params.num_tasks, params.variant) suite_arg = f"--suites={params.suite}" run_tests_vars = { "continue_on_failure": params.continue_on_failure, "resmoke_args": f"{suite_arg} {params.resmoke_args}", "resmoke_jobs_max": params.resmoke_jobs_max, "should_shuffle": params.should_shuffle, "require_multiversion": params.require_multiversion, "timeout_secs": params.timeout_secs, "task": params.task_name, "gen_task_config_location": params.config_location, } # yapf: disable commands = [ FunctionCall("do setup"), FunctionCall("configure evergreen api credentials") if params.require_multiversion else None, FunctionCall("do multiversion setup") if params.require_multiversion else None, FunctionCall("setup jstestfuzz"), FunctionCall("run jstestfuzz", params.jstestfuzz_params()), FunctionCall("run generated tests", run_tests_vars) ] commands = [command for command in commands if command is not None] return Task(sub_task_name, commands, {TaskDependency("archive_dist_test_debug")})
def _get_dependencies() -> Set[TaskDependency]: """Get the set of dependency tasks for these suites.""" dependencies = {TaskDependency("archive_dist_test_debug")} return dependencies
def get_setup_commands() -> Tuple[List[FunctionCall], Set[TaskDependency]]: """Return setup commands.""" return [ FunctionCall("do setup"), ], {TaskDependency("archive_dist_test_debug")}