def construct_variant_json(workloads, variants): """ :param list workloads: a list of filenames of workloads to schedule tasks for, each in the format subdirectory/Task.yml :param list variants: a list of buildvariants (strings) that the specified tasks should be run on. :return: json representation of variants running the given workloads, that can be provided to evergreen's generate.tasks command. Note: this function only generates variants, no tasks. It assumes that the tasks have already been generated (i.e. by calling generate.tasks with the result of construct_all_tasks_json()). """ task_specs = [] c = Configuration() for fname in workloads: basename = os.path.basename(fname) base_parts = os.path.splitext(basename) if base_parts[1] != '.yml': # Not a .yml workload file, ignore it. continue task_name = to_snake_case(base_parts[0]) prepare_environment_vars = get_prepare_environment_vars( task_name, fname) for prep_var in prepare_environment_vars: task_specs.append(TaskSpec(prep_var['test'])) for v in variants: c.variant(v).tasks(task_specs) return c.to_json()
def create_multiversion_generate_tasks_config(evg_config: Configuration, tests_by_task: Dict, evg_api: EvergreenApi, generate_config: GenerateConfig) -> Configuration: """ Create the multiversion config for the Evergreen generate.tasks file. :param evg_config: Shrub configuration to add to. :param tests_by_task: Dictionary of tests to generate tasks for. :param evg_api: Evergreen API. :param generate_config: Configuration of what to generate. :return: Shrub configuration with added tasks. """ dt = DisplayTaskDefinition(BURN_IN_MULTIVERSION_TASK) if tests_by_task: # Get the multiversion suites that will run in as part of burn_in_multiversion. multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY) for suite in multiversion_suites: idx = 0 if suite["origin"] not in tests_by_task.keys(): # Only generate burn in multiversion tasks for suites that would run the detected # changed tests. continue LOGGER.debug("Generating multiversion suite", suite=suite["multiversion_name"]) # We hardcode the number of fallback sub suites and the target resmoke time here # since burn_in_tests cares about individual tests and not entire suites. The config # options here are purely used to generate the proper multiversion suites to run # tests against. config_options = { "suite": suite["origin"], "fallback_num_sub_suites": 1, "project": generate_config.project, "build_variant": generate_config.build_variant, "task_id": generate_config.task_id, "task_name": suite["multiversion_name"], "target_resmoke_time": 60, } config_options.update(gen_resmoke.DEFAULT_CONFIG_VALUES) config_generator = gen_multiversion.EvergreenConfigGenerator( evg_api, evg_config, gen_resmoke.ConfigOptions(config_options)) test_list = tests_by_task[suite["origin"]]["tests"] for test in test_list: # Exclude files that should be blacklisted from multiversion testing. files_to_exclude = gen_multiversion.get_exclude_files(suite["multiversion_name"], TASK_PATH_SUFFIX) LOGGER.debug("Files to exclude", files_to_exclude=files_to_exclude, test=test, suite=suite["multiversion_name"]) if test not in files_to_exclude: # Generate the multiversion tasks for each test. config_generator.generate_evg_tasks(test, idx) idx += 1 dt.execution_tasks(config_generator.task_names) evg_config.variant(generate_config.build_variant).tasks(config_generator.task_specs) dt.execution_task(f"{BURN_IN_MULTIVERSION_TASK}_gen") evg_config.variant(generate_config.build_variant).display_task(dt) return evg_config
def create_generate_tasks_file(options, tests_by_task): """Create the Evergreen generate.tasks file.""" evg_config = Configuration() task_specs = [] task_names = [BURN_IN_TESTS_GEN_TASK] for task in sorted(tests_by_task): multiversion_path = tests_by_task[task].get("use_multiversion") for test_num, test in enumerate(tests_by_task[task]["tests"]): sub_task_name = _sub_task_name(options.buildvariant, task, test_num) task_names.append(sub_task_name) evg_sub_task = evg_config.task(sub_task_name) evg_sub_task.dependency(TaskDependency("compile")) task_spec = TaskSpec(sub_task_name) if options.distro: task_spec.distro(options.distro) task_specs.append(task_spec) run_tests_vars = { "resmoke_args": "{} {} {}".format(tests_by_task[task]["resmoke_args"], get_resmoke_repeat_options(options), test), } commands = [] commands.append(CommandDefinition().function("do setup")) if multiversion_path: run_tests_vars["task_path_suffix"] = multiversion_path commands.append(CommandDefinition().function("do multiversion setup")) commands.append(CommandDefinition().function("run tests").vars(run_tests_vars)) evg_sub_task.commands(commands) display_task = DisplayTaskDefinition(BURN_IN_TESTS_TASK).execution_tasks(task_names) evg_config.variant(_get_run_buildvariant(options)).tasks(task_specs).display_task(display_task) _write_json_file(evg_config.to_map(), options.generate_tasks_file)
def generate_evg_config(suites, options): """Generate evergreen configuration for the given suites.""" evg_config = Configuration() task_names = [] task_specs = [] def generate_task(sub_suite_name, sub_task_name, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" task_names.append(sub_task_name) spec = TaskSpec(sub_task_name) if options.use_large_distro: spec.distro(options.large_distro_name) task_specs.append(spec) task = evg_config.task(sub_task_name) target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) run_tests_vars = { "resmoke_args": "--suites={0}.yml {1}".format(target_suite_file, options.resmoke_args), "run_multiple_jobs": options.run_multiple_jobs, "task": options.task, } if options.resmoke_jobs_max: run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max commands = [] if max_test_runtime or expected_suite_runtime: cmd_timeout = CmdTimeoutUpdate() if max_test_runtime: cmd_timeout.timeout(int(math.ceil(max_test_runtime * 3))) if expected_suite_runtime: cmd_timeout.exec_timeout(int(math.ceil(expected_suite_runtime * 3))) commands.append(cmd_timeout.validate().resolve()) commands += [ CommandDefinition().function("do setup"), CommandDefinition().function("run generated tests").vars(run_tests_vars) ] task.dependency(TaskDependency("compile")).commands(commands) for idx, suite in enumerate(suites): sub_task_name = taskname.name_generated_task(options.task, idx, len(suites), options.variant) generate_task(suite.name, sub_task_name, suite.max_runtime, suite.get_runtime()) # Add the misc suite misc_suite_name = "{0}_misc".format(options.suite) generate_task(misc_suite_name, "{0}_misc_{1}".format(options.task, options.variant)) dt = DisplayTaskDefinition(options.task).execution_tasks(task_names) \ .execution_task("{0}_gen".format(options.task)) evg_config.variant(options.variant).tasks(task_specs).display_task(dt) return evg_config
def _generate_evg_tasks(options): """ Generate an evergreen configuration for fuzzers based on the options given. :param options: task options. :return: An evergreen configuration. """ evg_config = Configuration() task_names = [] task_specs = [] for task_index in range(options.num_tasks): name = taskname.name_generated_task(options.name, task_index, options.num_tasks, options.variant) task_names.append(name) task_specs.append(TaskSpec(name)) task = evg_config.task(name) commands = [CommandDefinition().function("do setup")] if options.use_multiversion: commands.append( CommandDefinition().function("do multiversion setup")) commands.append(CommandDefinition().function("setup jstestfuzz")) commands.append(CommandDefinition().function("run jstestfuzz").vars({ "jstestfuzz_vars": "--numGeneratedFiles {0} {1}".format(options.num_files, options.jstestfuzz_vars), "npm_command": options.npm_command })) run_tests_vars = { "continue_on_failure": options.continue_on_failure, "resmoke_args": options.resmoke_args, "resmoke_jobs_max": options.resmoke_jobs_max, "should_shuffle": options.should_shuffle, "task_path_suffix": options.use_multiversion, "timeout_secs": options.timeout_secs, } commands.append( CommandDefinition().function("run tests").vars(run_tests_vars)) task.dependency(TaskDependency("compile")).commands(commands) dt = DisplayTaskDefinition(options.name).execution_tasks(task_names)\ .execution_task("{0}_gen".format(options.name)) evg_config.variant(options.variant).tasks(task_specs).display_task(dt) return evg_config
def generate_evg_config(suites, options): """Generate evergreen configuration for the given suites.""" evg_config = Configuration() task_names = [] task_specs = [] def generate_task(sub_suite_name, sub_task_name): """Generate evergreen config for a resmoke task.""" task_names.append(sub_task_name) task_specs.append(TaskSpec(sub_task_name)) task = evg_config.task(sub_task_name) target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) run_tests_vars = { "resmoke_args": "--suites={0} {1}".format(target_suite_file, options.resmoke_args), "run_multiple_jobs": options.run_multiple_jobs, } if options.resmoke_jobs_max: run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max commands = [ CommandDefinition().function("do setup"), CommandDefinition().function("run tests").vars(run_tests_vars) ] task.dependency(TaskDependency("compile")).commands(commands) for idx, suite in enumerate(suites): sub_task_name = taskname.name_generated_task(options.task, idx, len(suites), options.variant) generate_task(suite.name, sub_task_name) # Add the misc suite misc_suite_name = "{0}_misc".format(options.suite) generate_task(misc_suite_name, "{0}_misc_{1}".format(options.task, options.variant)) dt = DisplayTaskDefinition(options.task).execution_tasks(task_names) \ .execution_task("{0}_gen".format(options.task)) evg_config.variant(options.variant).tasks(task_specs).display_task(dt) return evg_config
def _generate_evg_tasks(options): """ Generate an evergreen configuration for fuzzers based on the options given. :param options: task options. :return: An evergreen configuration. """ evg_config = Configuration() task_names = [] task_specs = [] for task_index in range(options.num_tasks): name = taskname.name_generated_task(options.name, task_index, options.num_tasks, options.variant) task_names.append(name) task_specs.append(TaskSpec(name)) task = evg_config.task(name) commands = [CommandDefinition().function("do setup")] if options.use_multiversion: commands.append(CommandDefinition().function("do multiversion setup")) commands.append(CommandDefinition().function("run jstestfuzz").vars({ "jstestfuzz_vars": "--numGeneratedFiles {0} {1}".format(options.num_files, options.jstestfuzz_vars), "npm_command": options.npm_command })) run_tests_vars = { "continue_on_failure": options.continue_on_failure, "resmoke_args": options.resmoke_args, "resmoke_jobs_max": options.resmoke_jobs_max, "should_shuffle": options.should_shuffle, "task_path_suffix": options.use_multiversion, "timeout_secs": options.timeout_secs, } commands.append(CommandDefinition().function("run tests").vars(run_tests_vars)) task.dependency(TaskDependency("compile")).commands(commands) dt = DisplayTaskDefinition(options.name).execution_tasks(task_names)\ .execution_task("{0}_gen".format(options.name)) evg_config.variant(options.variant).tasks(task_specs).display_task(dt) return evg_config
def configure(): n_tasks = 10 c = Configuration() task_names = [] task_specs = [] for i in range(n_tasks): name = "aggregation_multiversion_fuzzer_{0:03d}".format(i) task_names.append(name) task_specs.append(TaskSpec(name)) t = c.task(name) t.dependency(TaskDependency("compile")).commands([ CommandDefinition().function("do setup"), CommandDefinition().function("do multiversion setup"), CommandDefinition().function("run jstestfuzz").vars({ "jstestfuzz_var": "--numGeneratedFiles 5", "npm_command": "agg-fuzzer", }), CommandDefinition().function("run tests").vars({ "continue_on_failure": "false", "resmoke_args": "--suites=generational_fuzzer", "should_shuffle": "false", "task_path_suffix": "false", "timeout_secs": "1800", }) ]) dt = DisplayTaskDefinition("aggregation_multiversion_fuzzer")\ .execution_tasks(task_names) c.variant("linux-64").tasks(task_specs).display_task(dt) return c
def gen_lint_config(): targets = ["src", "test"] max_hosts = 5 variant_name = "lint variant" task_group_name = "lint group" config = Configuration() tasks = [] for t in targets: name = "make-lint-" + t config.task(name).function_with_vars("run-make", {"target": name}) tasks.append(name) group = config.task_group(task_group_name).max_hosts(max_hosts) group.setup_group().type("system").command("git.get_project").param( "directory", "src") group.setup_group().function("set-up-credentials") group.teardown_group().function("attach-test-results") group.teardown_group().function("remove-test-results") group.tasks(tasks) config.variant(variant_name).task(TaskSpec(task_group_name)) return config
def test_variants(self): c = Configuration() c.variant("variant 0") v1 = c.variant("variant 1") c.variant("variant 2") v1.batch_time(100) assert 100 == c.variant("variant 1").to_map()["batchtime"] assert 3 == len(c.to_map()["buildvariants"])
class EvergreenConfigGenerator(object): """Generate evergreen configurations.""" def __init__(self, suites, options, evg_api): """Create new EvergreenConfigGenerator object.""" self.suites = suites self.options = options self.evg_api = evg_api self.evg_config = Configuration() self.task_specs = [] self.task_names = [] self.build_tasks = None def _set_task_distro(self, task_spec): if self.options.use_large_distro and self.options.large_distro_name: task_spec.distro(self.options.large_distro_name) def _generate_resmoke_args(self, suite_file): resmoke_args = "--suite={0}.yml {1}".format(suite_file, self.options.resmoke_args) if self.options.repeat_suites and "repeatSuites" not in resmoke_args: resmoke_args += " --repeatSuites={0} ".format(self.options.repeat_suites) return resmoke_args def _get_run_tests_vars(self, suite_file): variables = { "resmoke_args": self._generate_resmoke_args(suite_file), "run_multiple_jobs": self.options.run_multiple_jobs, "task": self.options.task, } if self.options.resmoke_jobs_max: variables["resmoke_jobs_max"] = self.options.resmoke_jobs_max if self.options.use_multiversion: variables["task_path_suffix"] = self.options.use_multiversion return variables def _add_timeout_command(self, commands, max_test_runtime, expected_suite_runtime): repeat_factor = self.options.repeat_suites if max_test_runtime or expected_suite_runtime: cmd_timeout = CmdTimeoutUpdate() if max_test_runtime: timeout = calculate_timeout(max_test_runtime, 3) * repeat_factor LOGGER.debug("Setting timeout to: %d (max=%d, repeat=%d)", timeout, max_test_runtime, repeat_factor) cmd_timeout.timeout(timeout) if expected_suite_runtime: exec_timeout = calculate_timeout(expected_suite_runtime, 3) * repeat_factor LOGGER.debug("Setting exec_timeout to: %d (runtime=%d, repeat=%d)", exec_timeout, expected_suite_runtime, repeat_factor) cmd_timeout.exec_timeout(exec_timeout) commands.append(cmd_timeout.validate().resolve()) @staticmethod def _is_task_dependency(task, possible_dependency): return re.match("{0}_(\\d|misc)".format(task), possible_dependency) def _get_tasks_for_depends_on(self, dependent_task): return [ str(task["display_name"]) for task in self.build_tasks if self._is_task_dependency(dependent_task, str(task["display_name"])) ] def _add_dependencies(self, task): task.dependency(TaskDependency("compile")) if not self.options.is_patch: # Don"t worry about task dependencies in patch builds, only mainline. if self.options.depends_on: for dep in self.options.depends_on: depends_on_tasks = self._get_tasks_for_depends_on(dep) for dependency in depends_on_tasks: task.dependency(TaskDependency(dependency)) return task def _generate_task(self, sub_suite_name, sub_task_name, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" LOGGER.debug("Generating task for: %s", sub_suite_name) spec = TaskSpec(sub_task_name) self._set_task_distro(spec) self.task_specs.append(spec) self.task_names.append(sub_task_name) task = self.evg_config.task(sub_task_name) target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) run_tests_vars = self._get_run_tests_vars(target_suite_file) commands = [] self._add_timeout_command(commands, max_test_runtime, expected_suite_runtime) commands.append(CommandDefinition().function("do setup")) if self.options.use_multiversion: commands.append(CommandDefinition().function("do multiversion setup")) commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars)) self._add_dependencies(task).commands(commands) def _generate_all_tasks(self): for idx, suite in enumerate(self.suites): sub_task_name = taskname.name_generated_task(self.options.task, idx, len(self.suites), self.options.variant) max_runtime = None total_runtime = None if suite.should_overwrite_timeout(): max_runtime = suite.max_runtime total_runtime = suite.get_runtime() self._generate_task(suite.name, sub_task_name, max_runtime, total_runtime) # Add the misc suite misc_suite_name = "{0}_misc".format(self.options.suite) self._generate_task(misc_suite_name, "{0}_misc_{1}".format(self.options.task, self.options.variant)) def _generate_display_task(self): dt = DisplayTaskDefinition(self.options.task)\ .execution_tasks(self.task_names) \ .execution_task("{0}_gen".format(self.options.task)) return dt def _generate_variant(self): self._generate_all_tasks() self.evg_config.variant(self.options.variant)\ .tasks(self.task_specs)\ .display_task(self._generate_display_task()) def generate_config(self): """Generate evergreen configuration.""" self.build_tasks = self.evg_api.tasks_by_build_id(self.options.build_id) self._generate_variant() return self.evg_config
class EvergreenConfigGenerator(object): """Generate evergreen configurations.""" def __init__(self, suites, options, evg_api): """Create new EvergreenConfigGenerator object.""" self.suites = suites self.options = options self.evg_api = evg_api self.evg_config = Configuration() self.task_specs = [] self.task_names = [] self.build_tasks = None def _set_task_distro(self, task_spec): if self.options.use_large_distro and self.options.large_distro_name: task_spec.distro(self.options.large_distro_name) def _get_run_tests_vars(self, suite_file): variables = { "resmoke_args": "--suites={0}.yml {1}".format(suite_file, self.options.resmoke_args), "run_multiple_jobs": self.options.run_multiple_jobs, "task": self.options.task, } if self.options.resmoke_jobs_max: variables["resmoke_jobs_max"] = self.options.resmoke_jobs_max if self.options.use_multiversion: variables["task_path_suffix"] = self.options.use_multiversion return variables @staticmethod def _add_timeout_command(commands, max_test_runtime, expected_suite_runtime): if max_test_runtime or expected_suite_runtime: cmd_timeout = CmdTimeoutUpdate() if max_test_runtime: cmd_timeout.timeout(calculate_timeout(max_test_runtime, 3)) if expected_suite_runtime: cmd_timeout.exec_timeout(calculate_timeout(expected_suite_runtime, 3)) commands.append(cmd_timeout.validate().resolve()) @staticmethod def _is_task_dependency(task, possible_dependency): return re.match("{0}_(\\d|misc)".format(task), possible_dependency) def _get_tasks_for_depends_on(self, dependent_task): return [ str(task["display_name"]) for task in self.build_tasks if self._is_task_dependency(dependent_task, str(task["display_name"])) ] def _add_dependencies(self, task): task.dependency(TaskDependency("compile")) if not self.options.is_patch: # Don"t worry about task dependencies in patch builds, only mainline. if self.options.depends_on: for dep in self.options.depends_on: depends_on_tasks = self._get_tasks_for_depends_on(dep) for dependency in depends_on_tasks: task.dependency(TaskDependency(dependency)) return task def _generate_task(self, sub_suite_name, sub_task_name, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" spec = TaskSpec(sub_task_name) self._set_task_distro(spec) self.task_specs.append(spec) self.task_names.append(sub_task_name) task = self.evg_config.task(sub_task_name) target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) run_tests_vars = self._get_run_tests_vars(target_suite_file) commands = [] self._add_timeout_command(commands, max_test_runtime, expected_suite_runtime) commands.append(CommandDefinition().function("do setup")) if self.options.use_multiversion: commands.append(CommandDefinition().function("do multiversion setup")) commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars)) self._add_dependencies(task).commands(commands) def _generate_all_tasks(self): for idx, suite in enumerate(self.suites): sub_task_name = taskname.name_generated_task(self.options.task, idx, len(self.suites), self.options.variant) self._generate_task(suite.name, sub_task_name, suite.max_runtime, suite.get_runtime()) # Add the misc suite misc_suite_name = "{0}_misc".format(self.options.suite) self._generate_task(misc_suite_name, "{0}_misc_{1}".format(self.options.task, self.options.variant)) def _generate_display_task(self): dt = DisplayTaskDefinition(self.options.task)\ .execution_tasks(self.task_names) \ .execution_task("{0}_gen".format(self.options.task)) return dt def _generate_variant(self): self._generate_all_tasks() self.evg_config.variant(self.options.variant)\ .tasks(self.task_specs)\ .display_task(self._generate_display_task()) def generate_config(self): """Generate evergreen configuration.""" self.build_tasks = self.evg_api.tasks_by_build_id(self.options.build_id) self._generate_variant() return self.evg_config
class EvergreenConfigGenerator(object): """Generate evergreen configurations.""" def __init__(self, suites, options, evg_api): """Create new EvergreenConfigGenerator object.""" self.suites = suites self.options = options self.evg_api = evg_api self.evg_config = Configuration() self.task_specs = [] self.task_names = [] self.build_tasks = None def _set_task_distro(self, task_spec): if self.options.use_large_distro and self.options.large_distro_name: task_spec.distro(self.options.large_distro_name) def _generate_resmoke_args(self, suite_file): resmoke_args = "--suite={0}.yml --originSuite={1} {2}".format( suite_file, self.options.suite, self.options.resmoke_args) if self.options.repeat_suites and not string_contains_any_of_args( resmoke_args, ["repeatSuites", "repeat"]): resmoke_args += " --repeatSuites={0} ".format( self.options.repeat_suites) return resmoke_args def _get_run_tests_vars(self, suite_file): variables = { "resmoke_args": self._generate_resmoke_args(suite_file), "run_multiple_jobs": self.options.run_multiple_jobs, "task": self.options.task, } if self.options.resmoke_jobs_max: variables["resmoke_jobs_max"] = self.options.resmoke_jobs_max if self.options.use_multiversion: variables["task_path_suffix"] = self.options.use_multiversion return variables def _get_timeout_command(self, max_test_runtime, expected_suite_runtime, use_default): """ Add an evergreen command to override the default timeouts to the list of commands. :param max_test_runtime: Maximum runtime of any test in the sub-suite. :param expected_suite_runtime: Expected runtime of the entire sub-suite. :param use_default: Use default timeouts. :return: Timeout information. """ repeat_factor = self.options.repeat_suites if (max_test_runtime or expected_suite_runtime) and not use_default: timeout = None exec_timeout = None if max_test_runtime: timeout = calculate_timeout(max_test_runtime, 3) * repeat_factor LOGGER.debug("Setting timeout", timeout=timeout, max_runtime=max_test_runtime, factor=repeat_factor) if expected_suite_runtime: exec_timeout = calculate_timeout(expected_suite_runtime, 3) * repeat_factor LOGGER.debug("Setting exec_timeout", exec_timeout=exec_timeout, suite_runtime=expected_suite_runtime, factor=repeat_factor) return TimeoutInfo.overridden(timeout=timeout, exec_timeout=exec_timeout) return TimeoutInfo.default_timeout() @staticmethod def _is_task_dependency(task, possible_dependency): return re.match("{0}_(\\d|misc)".format(task), possible_dependency) def _get_tasks_for_depends_on(self, dependent_task): return [ str(task.display_name) for task in self.build_tasks if self._is_task_dependency(dependent_task, str(task.display_name)) ] def _add_dependencies(self, task): task.dependency(TaskDependency("compile")) if not self.options.is_patch: # Don"t worry about task dependencies in patch builds, only mainline. if self.options.depends_on: for dep in self.options.depends_on: depends_on_tasks = self._get_tasks_for_depends_on(dep) for dependency in depends_on_tasks: task.dependency(TaskDependency(dependency)) return task def _generate_task(self, sub_suite_name, sub_task_name, target_dir, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" # pylint: disable=too-many-arguments LOGGER.debug("Generating task", sub_suite=sub_suite_name) spec = TaskSpec(sub_task_name) self._set_task_distro(spec) self.task_specs.append(spec) self.task_names.append(sub_task_name) task = self.evg_config.task(sub_task_name) target_suite_file = os.path.join(target_dir, os.path.basename(sub_suite_name)) run_tests_vars = self._get_run_tests_vars(target_suite_file) use_multiversion = self.options.use_multiversion timeout_info = self._get_timeout_command( max_test_runtime, expected_suite_runtime, self.options.use_default_timeouts) commands = resmoke_commands("run generated tests", run_tests_vars, timeout_info, use_multiversion) self._add_dependencies(task).commands(commands) def _generate_all_tasks(self): for idx, suite in enumerate(self.suites): sub_task_name = taskname.name_generated_task( self.options.task, idx, len(self.suites), self.options.variant) max_runtime = None total_runtime = None if suite.should_overwrite_timeout(): max_runtime = suite.max_runtime total_runtime = suite.get_runtime() self._generate_task(suite.name, sub_task_name, self.options.generated_config_dir, max_runtime, total_runtime) # Add the misc suite misc_suite_name = f"{os.path.basename(self.options.suite)}_misc" misc_task_name = f"{self.options.task}_misc_{self.options.variant}" self._generate_task(misc_suite_name, misc_task_name, self.options.generated_config_dir) def _generate_display_task(self): dt = DisplayTaskDefinition(self.options.task)\ .execution_tasks(self.task_names) \ .execution_task("{0}_gen".format(self.options.task)) return dt def _generate_variant(self): self._generate_all_tasks() self.evg_config.variant(self.options.variant)\ .tasks(self.task_specs)\ .display_task(self._generate_display_task()) def generate_config(self): """Generate evergreen configuration.""" self.build_tasks = self.evg_api.tasks_by_build(self.options.build_id) self._generate_variant() return self.evg_config
class EvergreenConfigGenerator(object): """Generate evergreen configurations.""" def __init__(self, suites, options, evg_api): """Create new EvergreenConfigGenerator object.""" self.suites = suites self.options = options self.evg_api = evg_api self.evg_config = Configuration() self.task_specs = [] self.task_names = [] self.build_tasks = None def _set_task_distro(self, task_spec): if self.options.use_large_distro and self.options.large_distro_name: task_spec.distro(self.options.large_distro_name) def _generate_resmoke_args(self, suite_file): resmoke_args = "--suite={0}.yml --originSuite={1} {2}".format( suite_file, self.options.suite, self.options.resmoke_args) if self.options.repeat_suites and "repeatSuites" not in resmoke_args: resmoke_args += " --repeatSuites={0} ".format(self.options.repeat_suites) return resmoke_args def _get_run_tests_vars(self, suite_file): variables = { "resmoke_args": self._generate_resmoke_args(suite_file), "run_multiple_jobs": self.options.run_multiple_jobs, "task": self.options.task, } if self.options.resmoke_jobs_max: variables["resmoke_jobs_max"] = self.options.resmoke_jobs_max if self.options.use_multiversion: variables["task_path_suffix"] = self.options.use_multiversion return variables def _add_timeout_command(self, commands, max_test_runtime, expected_suite_runtime): """ Add an evergreen command to override the default timeouts to the list of commands. :param commands: List of commands to add timeout command to. :param max_test_runtime: Maximum runtime of any test in the sub-suite. :param expected_suite_runtime: Expected runtime of the entire sub-suite. """ repeat_factor = self.options.repeat_suites if max_test_runtime or expected_suite_runtime: cmd_timeout = CmdTimeoutUpdate() if max_test_runtime: timeout = calculate_timeout(max_test_runtime, 3) * repeat_factor LOGGER.debug("Setting timeout to: %d (max=%d, repeat=%d)", timeout, max_test_runtime, repeat_factor) cmd_timeout.timeout(timeout) if expected_suite_runtime: exec_timeout = calculate_timeout(expected_suite_runtime, 3) * repeat_factor LOGGER.debug("Setting exec_timeout to: %d (runtime=%d, repeat=%d)", exec_timeout, expected_suite_runtime, repeat_factor) cmd_timeout.exec_timeout(exec_timeout) commands.append(cmd_timeout.validate().resolve()) @staticmethod def _is_task_dependency(task, possible_dependency): return re.match("{0}_(\\d|misc)".format(task), possible_dependency) def _get_tasks_for_depends_on(self, dependent_task): return [ str(task["display_name"]) for task in self.build_tasks if self._is_task_dependency(dependent_task, str(task["display_name"])) ] def _add_dependencies(self, task): task.dependency(TaskDependency("compile")) if not self.options.is_patch: # Don"t worry about task dependencies in patch builds, only mainline. if self.options.depends_on: for dep in self.options.depends_on: depends_on_tasks = self._get_tasks_for_depends_on(dep) for dependency in depends_on_tasks: task.dependency(TaskDependency(dependency)) return task def _generate_task(self, sub_suite_name, sub_task_name, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" LOGGER.debug("Generating task for: %s", sub_suite_name) spec = TaskSpec(sub_task_name) self._set_task_distro(spec) self.task_specs.append(spec) self.task_names.append(sub_task_name) task = self.evg_config.task(sub_task_name) target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) run_tests_vars = self._get_run_tests_vars(target_suite_file) commands = [] if not self.options.use_default_timeouts: self._add_timeout_command(commands, max_test_runtime, expected_suite_runtime) commands.append(CommandDefinition().function("do setup")) if self.options.use_multiversion: commands.append(CommandDefinition().function("do multiversion setup")) commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars)) self._add_dependencies(task).commands(commands) def _generate_all_tasks(self): for idx, suite in enumerate(self.suites): sub_task_name = taskname.name_generated_task(self.options.task, idx, len(self.suites), self.options.variant) max_runtime = None total_runtime = None if suite.should_overwrite_timeout(): max_runtime = suite.max_runtime total_runtime = suite.get_runtime() self._generate_task(suite.name, sub_task_name, max_runtime, total_runtime) # Add the misc suite misc_suite_name = "{0}_misc".format(self.options.suite) self._generate_task(misc_suite_name, "{0}_misc_{1}".format(self.options.task, self.options.variant)) def _generate_display_task(self): dt = DisplayTaskDefinition(self.options.task)\ .execution_tasks(self.task_names) \ .execution_task("{0}_gen".format(self.options.task)) return dt def _generate_variant(self): self._generate_all_tasks() self.evg_config.variant(self.options.variant)\ .tasks(self.task_specs)\ .display_task(self._generate_display_task()) def generate_config(self): """Generate evergreen configuration.""" self.build_tasks = self.evg_api.tasks_by_build_id(self.options.build_id) self._generate_variant() return self.evg_config
def generate_evg_config(suites, options): """Generate evergreen configuration for the given suites.""" evg_config = Configuration() task_names = [] task_specs = [] def generate_task(sub_suite_name, sub_task_name, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" task_names.append(sub_task_name) spec = TaskSpec(sub_task_name) if options.use_large_distro: spec.distro(options.large_distro_name) task_specs.append(spec) task = evg_config.task(sub_task_name) target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) run_tests_vars = { "resmoke_args": "--suites={0}.yml {1}".format(target_suite_file, options.resmoke_args), "run_multiple_jobs": options.run_multiple_jobs, "task": options.task, } if options.resmoke_jobs_max: run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max commands = [] if max_test_runtime or expected_suite_runtime: cmd_timeout = CmdTimeoutUpdate() if max_test_runtime: cmd_timeout.timeout(int(math.ceil(max_test_runtime * 3))) if expected_suite_runtime: cmd_timeout.exec_timeout( int(math.ceil(expected_suite_runtime * 3))) commands.append(cmd_timeout.validate().resolve()) commands += [ CommandDefinition().function("do setup"), CommandDefinition().function("run generated tests").vars( run_tests_vars) ] task.dependency(TaskDependency("compile")).commands(commands) for idx, suite in enumerate(suites): sub_task_name = taskname.name_generated_task(options.task, idx, len(suites), options.variant) generate_task(suite.name, sub_task_name, suite.max_runtime, suite.get_runtime()) # Add the misc suite misc_suite_name = "{0}_misc".format(options.suite) generate_task(misc_suite_name, "{0}_misc_{1}".format(options.task, options.variant)) dt = DisplayTaskDefinition(options.task).execution_tasks(task_names) \ .execution_task("{0}_gen".format(options.task)) evg_config.variant(options.variant).tasks(task_specs).display_task(dt) return evg_config
def test_variant_throws_exception_for_invalid_value(self): c = Configuration() with pytest.raises(TypeError): c.variant(42)
class EvergreenConfigGenerator(object): """Generate evergreen configurations.""" def __init__(self, suites, options, evg_api): """Create new EvergreenConfigGenerator object.""" self.suites = suites self.options = options self.evg_api = evg_api self.evg_config = Configuration() self.task_specs = [] self.task_names = [] self.build_tasks = None def _set_task_distro(self, task_spec): if self.options.use_large_distro and self.options.large_distro_name: task_spec.distro(self.options.large_distro_name) def _generate_resmoke_args(self, suite_file): resmoke_args = "--suite={0}.yml --originSuite={1} {2}".format( suite_file, self.options.suite, self.options.resmoke_args) if self.options.repeat_suites and not string_contains_any_of_args( resmoke_args, ["repeatSuites", "repeat"]): resmoke_args += " --repeatSuites={0} ".format( self.options.repeat_suites) return resmoke_args def _get_run_tests_vars(self, suite_file): variables = { "resmoke_args": self._generate_resmoke_args(suite_file), "run_multiple_jobs": self.options.run_multiple_jobs, "task": self.options.task, } if self.options.resmoke_jobs_max: variables["resmoke_jobs_max"] = self.options.resmoke_jobs_max if self.options.use_multiversion: variables["task_path_suffix"] = self.options.use_multiversion return variables def _get_timeout_command(self, max_test_runtime, expected_suite_runtime, use_default): """ Add an evergreen command to override the default timeouts to the list of commands. :param max_test_runtime: Maximum runtime of any test in the sub-suite. :param expected_suite_runtime: Expected runtime of the entire sub-suite. :param use_default: Use default timeouts. :return: Timeout information. """ repeat_factor = self.options.repeat_suites if (max_test_runtime or expected_suite_runtime) and not use_default: timeout = None exec_timeout = None if max_test_runtime: timeout = calculate_timeout(max_test_runtime, 3) * repeat_factor LOGGER.debug("Setting timeout", timeout=timeout, max_runtime=max_test_runtime, factor=repeat_factor) if expected_suite_runtime: exec_timeout = calculate_timeout(expected_suite_runtime, 3) * repeat_factor LOGGER.debug("Setting exec_timeout", exec_timeout=exec_timeout, suite_runtime=expected_suite_runtime, factor=repeat_factor) if timeout > MAX_EXPECTED_TIMEOUT or exec_timeout > MAX_EXPECTED_TIMEOUT: frameinfo = getframeinfo(currentframe()) LOGGER.error( "This task looks like it is expected to run far longer than normal. This is " "likely due to setting the suite 'repeat' value very high. If you are sure " "this is something you want to do, comment this check out in your patch build " "and resubmit", repeat_value=repeat_factor, timeout=timeout, exec_timeout=exec_timeout, code_file=frameinfo.filename, code_line=frameinfo.lineno, max_timeout=MAX_EXPECTED_TIMEOUT) raise ValueError("Failing due to expected runtime.") return TimeoutInfo.overridden(timeout=timeout, exec_timeout=exec_timeout) return TimeoutInfo.default_timeout() @staticmethod def _is_task_dependency(task, possible_dependency): return re.match("{0}_(\\d|misc)".format(task), possible_dependency) def _get_tasks_for_depends_on(self, dependent_task): return [ str(task.display_name) for task in self.build_tasks if self._is_task_dependency(dependent_task, str(task.display_name)) ] def _add_dependencies(self, task): task.dependency(TaskDependency("compile")) if not self.options.is_patch: # Don"t worry about task dependencies in patch builds, only mainline. if self.options.depends_on: for dep in self.options.depends_on: depends_on_tasks = self._get_tasks_for_depends_on(dep) for dependency in depends_on_tasks: task.dependency(TaskDependency(dependency)) return task def _generate_task(self, sub_suite_name, sub_task_name, target_dir, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" # pylint: disable=too-many-arguments LOGGER.debug("Generating task", sub_suite=sub_suite_name) spec = TaskSpec(sub_task_name) self._set_task_distro(spec) self.task_specs.append(spec) self.task_names.append(sub_task_name) task = self.evg_config.task(sub_task_name) # Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join # here, just use the forward slash; otherwise the path separator will be treated as # the escape character on Windows. target_suite_file = '/'.join( [target_dir, os.path.basename(sub_suite_name)]) run_tests_vars = self._get_run_tests_vars(target_suite_file) use_multiversion = self.options.use_multiversion timeout_info = self._get_timeout_command( max_test_runtime, expected_suite_runtime, self.options.use_default_timeouts) commands = resmoke_commands("run generated tests", run_tests_vars, timeout_info, use_multiversion) self._add_dependencies(task).commands(commands) def _generate_all_tasks(self): for idx, suite in enumerate(self.suites): sub_task_name = taskname.name_generated_task( self.options.task, idx, len(self.suites), self.options.variant) max_runtime = None total_runtime = None if suite.should_overwrite_timeout(): max_runtime = suite.max_runtime total_runtime = suite.get_runtime() self._generate_task(suite.name, sub_task_name, self.options.generated_config_dir, max_runtime, total_runtime) # Add the misc suite misc_suite_name = f"{os.path.basename(self.options.suite)}_misc" misc_task_name = f"{self.options.task}_misc_{self.options.variant}" self._generate_task(misc_suite_name, misc_task_name, self.options.generated_config_dir) def _generate_display_task(self): dt = DisplayTaskDefinition(self.options.task)\ .execution_tasks(self.task_names) \ .execution_task("{0}_gen".format(self.options.task)) return dt def _generate_variant(self): self._generate_all_tasks() self.evg_config.variant(self.options.variant)\ .tasks(self.task_specs)\ .display_task(self._generate_display_task()) def generate_config(self): """Generate evergreen configuration.""" self.build_tasks = self.evg_api.tasks_by_build(self.options.build_id) self._generate_variant() return self.evg_config
def variant_tasks(tasks: List[GeneratedTask], variant: str) -> Configuration: c = Configuration() c.variant(variant).tasks([TaskSpec(task.name) for task in tasks]) return c
class EvergreenConfigGenerator(object): """Generate evergreen configurations.""" def __init__(self, suites, options): """Create new EvergreenConfigGenerator object.""" self.suites = suites self.options = options self.evg_config = Configuration() self.task_specs = [] self.task_names = [] def _set_task_distro(self, task_spec): if self.options.use_large_distro and self.options.large_distro_name: task_spec.distro(self.options.large_distro_name) def _get_run_tests_vars(self, suite_file): variables = { "resmoke_args": "--suites={0}.yml {1}".format(suite_file, self.options.resmoke_args), "run_multiple_jobs": self.options.run_multiple_jobs, "task": self.options.task, } if self.options.resmoke_jobs_max: variables["resmoke_jobs_max"] = self.options.resmoke_jobs_max if self.options.use_multiversion: variables["task_path_suffix"] = self.options.use_multiversion return variables @staticmethod def _add_timeout_command(commands, max_test_runtime, expected_suite_runtime): if max_test_runtime or expected_suite_runtime: cmd_timeout = CmdTimeoutUpdate() if max_test_runtime: cmd_timeout.timeout(int(math.ceil(max_test_runtime * 3))) if expected_suite_runtime: cmd_timeout.exec_timeout(int(math.ceil(expected_suite_runtime * 3))) commands.append(cmd_timeout.validate().resolve()) def _add_dependencies(self, task): task.dependency(TaskDependency("compile")) if not self.options.is_patch: # Don't worry about task dependencies in patch builds, only mainline. if self.options.depends_on: for dep in self.options.depends_on: task.dependency(TaskDependency(dep)) if self.options.requires: for dep in self.options.requires: task.requires(TaskDependency(dep)) return task def _generate_task(self, sub_suite_name, sub_task_name, max_test_runtime=None, expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" spec = TaskSpec(sub_task_name) self._set_task_distro(spec) self.task_specs.append(spec) self.task_names.append(sub_task_name) task = self.evg_config.task(sub_task_name) target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) run_tests_vars = self._get_run_tests_vars(target_suite_file) commands = [] self._add_timeout_command(commands, max_test_runtime, expected_suite_runtime) commands.append(CommandDefinition().function("do setup")) if self.options.use_multiversion: commands.append(CommandDefinition().function("do multiversion setup")) commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars)) self._add_dependencies(task).commands(commands) def _generate_all_tasks(self): for idx, suite in enumerate(self.suites): sub_task_name = taskname.name_generated_task(self.options.task, idx, len(self.suites), self.options.variant) self._generate_task(suite.name, sub_task_name, suite.max_runtime, suite.get_runtime()) # Add the misc suite misc_suite_name = "{0}_misc".format(self.options.suite) self._generate_task(misc_suite_name, "{0}_misc_{1}".format(self.options.task, self.options.variant)) def _generate_display_task(self): dt = DisplayTaskDefinition(self.options.task)\ .execution_tasks(self.task_names) \ .execution_task("{0}_gen".format(self.options.task)) return dt def _generate_variant(self): self._generate_all_tasks() self.evg_config.variant(self.options.variant)\ .tasks(self.task_specs)\ .display_task(self._generate_display_task()) def generate_config(self): """Generate evergreen configuration.""" self._generate_variant() return self.evg_config