def create_generate_tasks_file(options, tests_by_task):
"""Create the Evergreen generate.tasks file."""
evg_config = Configuration()
task_specs = []
task_names = [BURN_IN_TESTS_GEN_TASK]
for task in sorted(tests_by_task):
multiversion_path = tests_by_task[task].get("use_multiversion")
for test_num, test in enumerate(tests_by_task[task]["tests"]):
sub_task_name = _sub_task_name(options.buildvariant, task, test_num)
task_names.append(sub_task_name)
evg_sub_task = evg_config.task(sub_task_name)
evg_sub_task.dependency(TaskDependency("compile"))
task_spec = TaskSpec(sub_task_name)
if options.distro:
task_spec.distro(options.distro)
task_specs.append(task_spec)
run_tests_vars = {
"resmoke_args":
"{} {} {}".format(tests_by_task[task]["resmoke_args"],
get_resmoke_repeat_options(options), test),
}
commands = []
commands.append(CommandDefinition().function("do setup"))
if multiversion_path:
run_tests_vars["task_path_suffix"] = multiversion_path
commands.append(CommandDefinition().function("do multiversion setup"))
commands.append(CommandDefinition().function("run tests").vars(run_tests_vars))
evg_sub_task.commands(commands)
display_task = DisplayTaskDefinition(BURN_IN_TESTS_TASK).execution_tasks(task_names)
evg_config.variant(_get_run_buildvariant(options)).tasks(task_specs).display_task(display_task)
_write_json_file(evg_config.to_map(), options.generate_tasks_file)
def generate_task(sub_suite_name, sub_task_name, max_test_runtime=None,
expected_suite_runtime=None):
"""Generate evergreen config for a resmoke task."""
task_names.append(sub_task_name)
spec = TaskSpec(sub_task_name)
if options.use_large_distro:
spec.distro(options.large_distro_name)
task_specs.append(spec)
task = evg_config.task(sub_task_name)
target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name)
run_tests_vars = {
"resmoke_args": "--suites={0}.yml {1}".format(target_suite_file, options.resmoke_args),
"run_multiple_jobs": options.run_multiple_jobs,
"task": options.task,
}
if options.resmoke_jobs_max:
run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max
commands = []
if max_test_runtime or expected_suite_runtime:
cmd_timeout = CmdTimeoutUpdate()
if max_test_runtime:
cmd_timeout.timeout(int(math.ceil(max_test_runtime * 3)))
if expected_suite_runtime:
cmd_timeout.exec_timeout(int(math.ceil(expected_suite_runtime * 3)))
commands.append(cmd_timeout.validate().resolve())
commands += [
CommandDefinition().function("do setup"),
CommandDefinition().function("run generated tests").vars(run_tests_vars)
]
task.dependency(TaskDependency("compile")).commands(commands)
def _generate_evg_buildvariant(shrub_config, buildvariant, run_buildvariant):
"""
Generate buildvariants for a given shrub config.
:param shrub_config: Shrub config object that the generated buildvariant will be built upon.
:param buildvariant: The base variant that the generated run_buildvariant will be based on.
:param run_buildvariant: The generated buildvariant.
"""
evergreen_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
base_variant_config = evergreen_conf.get_variant(buildvariant)
new_variant_display_name = f"! {base_variant_config.display_name}"
new_variant_run_on = base_variant_config.run_on[0]
task_spec = TaskSpec("compile_TG")
task_spec.distro("rhel62-large")
new_variant = shrub_config.variant(run_buildvariant)
new_variant.display_name(new_variant_display_name)
new_variant.run_on(new_variant_run_on)
new_variant.task(task_spec)
base_variant_expansions = base_variant_config.expansions
new_variant.expansions(base_variant_expansions)
modules = base_variant_config.modules
new_variant.modules(modules)
def test_task_spec(self):
ts = TaskSpec("task name").stepback().distro("linux")
obj = ts.to_map()
assert "task name" == obj["name"]
assert obj["stepback"]
assert ["linux"] == obj["distros"]
def test_tasks_can_be_added(self):
v = Variant("variant name")
v.task(TaskSpec("task 0")).tasks(
[TaskSpec("task 1"), TaskSpec("task 2")])
obj = v.to_map()
assert "task 0" == obj["tasks"][0]["name"]
assert "task 1" == obj["tasks"][1]["name"]
assert "task 2" == obj["tasks"][2]["name"]
def construct_variant_json(workloads, variants):
"""
:param list workloads: a list of filenames of workloads to schedule tasks for, each in the format subdirectory/Task.yml
:param list variants: a list of buildvariants (strings) that the specified tasks should be run on.
:return: json representation of variants running the given workloads, that can be provided to evergreen's generate.tasks command.
Note: this function only generates variants, no tasks. It assumes that the tasks have already been generated (i.e. by calling generate.tasks with the result of construct_all_tasks_json()).
"""
task_specs = []
c = Configuration()
for fname in workloads:
basename = os.path.basename(fname)
base_parts = os.path.splitext(basename)
if base_parts[1] != '.yml':
# Not a .yml workload file, ignore it.
continue
task_name = to_snake_case(base_parts[0])
prepare_environment_vars = get_prepare_environment_vars(
task_name, fname)
for prep_var in prepare_environment_vars:
task_specs.append(TaskSpec(prep_var['test']))
for v in variants:
c.variant(v).tasks(task_specs)
return c.to_json()
def _generate_sub_task(self, mixed_version_config, task, task_index, suite, num_suites,
burn_in_test=None):
# pylint: disable=too-many-arguments
"""Generate a sub task to be run with the provided suite and mixed version config."""
# Create a sub task name appended with the task_index and build variant name.
task_name = "{0}_{1}".format(task, mixed_version_config)
sub_task_name = taskname.name_generated_task(task_name, task_index, num_suites,
self.options.variant)
self.task_names.append(sub_task_name)
self.task_specs.append(TaskSpec(sub_task_name))
task = self.evg_config.task(sub_task_name)
gen_task_name = BURN_IN_TASK if burn_in_test is not None else self.task
commands = [
CommandDefinition().function("do setup"),
# Fetch and download the proper mongod binaries before running multiversion tests.
CommandDefinition().function("do multiversion setup")
]
# TODO(SERVER-43306): Remove --dryRun command line option once we start turning on
# multiversion tests.
run_tests_vars = {
"resmoke_args":
"{0} --suite={1} --mixedBinVersions={2} --excludeWithAnyTags={3} ".format(
self.options.resmoke_args, suite, mixed_version_config, EXCLUDE_TAGS),
"task":
gen_task_name,
}
if burn_in_test is not None:
run_tests_vars["resmoke_args"] += burn_in_test
commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars))
task.dependency(TaskDependency("compile")).commands(commands)
def _generate_evg_build_variant(shrub_config, build_variant, run_build_variant,
burn_in_tags_gen_variant, evg_conf):
"""
Generate buildvariants for a given shrub config.
:param shrub_config: Shrub config object that the generated buildvariant will be built upon.
:param build_variant: The base variant that the generated run_buildvariant will be based on.
:param run_build_variant: The generated buildvariant.
:param burn_in_tags_gen_variant: The buildvariant on which the burn_in_tags_gen task runs.
"""
base_variant_config = evg_conf.get_variant(build_variant)
new_variant_display_name = f"! {base_variant_config.display_name}"
new_variant_run_on = base_variant_config.run_on[0]
task_spec = TaskSpec("compile_without_package_TG")
new_variant = shrub_config.variant(run_build_variant).expansion("burn_in_bypass",
burn_in_tags_gen_variant)
new_variant.display_name(new_variant_display_name)
new_variant.run_on(new_variant_run_on)
new_variant.task(task_spec)
base_variant_expansions = base_variant_config.expansions
new_variant.expansions(base_variant_expansions)
modules = base_variant_config.modules
new_variant.modules(modules)
def _generate_sub_task(self, mixed_version_config, task_index, suite, num_suites):
"""Generate a sub task to be run with the provided suite and mixed version config."""
# Create a sub task name appended with the task_index and build variant name.
task = "{0}_{1}".format(self.task, mixed_version_config)
sub_task_name = taskname.name_generated_task(task, task_index, num_suites,
self.options.variant)
self.task_names.append(sub_task_name)
self.task_specs.append(TaskSpec(sub_task_name))
task = self.evg_config.task(sub_task_name)
commands = [CommandDefinition().function("do setup")]
# Fetch and download the proper mongod binaries before running multiversion tests.
commands.append(CommandDefinition().function("do multiversion setup"))
exclude_tags = "requires_fcv_44"
# TODO(SERVER-43306): Remove --dryRun command line option once we start turning on multiversion tests.
run_tests_vars = {
"resmoke_args":
"{0} --suite={1} --mixedBinVersions={2} --excludeWithAnyTags={3} --dryRun=tests".
format(self.options.resmoke_args, suite, mixed_version_config, exclude_tags),
"task":
self.task,
}
commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars))
task.dependency(TaskDependency("compile")).commands(commands)
def test_invalid_modules_cannot_be_added(self):
v = Variant("variant name")
with pytest.raises(TypeError):
v.module(25)
with pytest.raises(TypeError):
v.modules(TaskSpec("not array"))
def _generate_task(self,
sub_suite_name,
sub_task_name,
target_dir,
max_test_runtime=None,
expected_suite_runtime=None):
"""Generate evergreen config for a resmoke task."""
# pylint: disable=too-many-arguments
LOGGER.debug("Generating task", sub_suite=sub_suite_name)
spec = TaskSpec(sub_task_name)
self._set_task_distro(spec)
self.task_specs.append(spec)
self.task_names.append(sub_task_name)
task = self.evg_config.task(sub_task_name)
target_suite_file = os.path.join(target_dir,
os.path.basename(sub_suite_name))
run_tests_vars = self._get_run_tests_vars(target_suite_file)
use_multiversion = self.options.use_multiversion
timeout_info = self._get_timeout_command(
max_test_runtime, expected_suite_runtime,
self.options.use_default_timeouts)
commands = resmoke_commands("run generated tests", run_tests_vars,
timeout_info, use_multiversion)
self._add_dependencies(task).commands(commands)
def _generate_task(self,
sub_suite_name,
sub_task_name,
max_test_runtime=None,
expected_suite_runtime=None):
"""Generate evergreen config for a resmoke task."""
spec = TaskSpec(sub_task_name)
self._set_task_distro(spec)
self.task_specs.append(spec)
self.task_names.append(sub_task_name)
task = self.evg_config.task(sub_task_name)
target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name)
run_tests_vars = self._get_run_tests_vars(target_suite_file)
commands = []
self._add_timeout_command(commands, max_test_runtime,
expected_suite_runtime)
commands.append(CommandDefinition().function("do setup"))
if self.options.use_multiversion:
commands.append(
CommandDefinition().function("do multiversion setup"))
commands.append(CommandDefinition().function(
"run generated tests").vars(run_tests_vars))
self._add_dependencies(task).commands(commands)
def _generate_task(self,
sub_suite_name,
sub_task_name,
target_dir,
max_test_runtime=None,
expected_suite_runtime=None):
"""Generate evergreen config for a resmoke task."""
# pylint: disable=too-many-arguments
LOGGER.debug("Generating task", sub_suite=sub_suite_name)
spec = TaskSpec(sub_task_name)
self._set_task_distro(spec)
self.task_specs.append(spec)
self.task_names.append(sub_task_name)
task = self.evg_config.task(sub_task_name)
# Evergreen always uses a unix shell, even on Windows, so instead of using os.path.join
# here, just use the forward slash; otherwise the path separator will be treated as
# the escape character on Windows.
target_suite_file = '/'.join(
[target_dir, os.path.basename(sub_suite_name)])
run_tests_vars = self._get_run_tests_vars(target_suite_file)
use_multiversion = self.options.use_multiversion
timeout_info = self._get_timeout_command(
max_test_runtime, expected_suite_runtime,
self.options.use_default_timeouts)
commands = resmoke_commands("run generated tests", run_tests_vars,
timeout_info, use_multiversion)
self._add_dependencies(task).commands(commands)
def test_invalid_tasks_cannot_be_added(self):
v = Variant("variant name")
with pytest.raises(TypeError):
v.task("I'm not really a task")
with pytest.raises(TypeError):
v.tasks(TaskSpec("not array"))
def generate_task(sub_suite_name,
sub_task_name,
max_test_runtime=None,
expected_suite_runtime=None):
"""Generate evergreen config for a resmoke task."""
task_names.append(sub_task_name)
spec = TaskSpec(sub_task_name)
if options.use_large_distro:
spec.distro(options.large_distro_name)
task_specs.append(spec)
task = evg_config.task(sub_task_name)
target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name)
run_tests_vars = {
"resmoke_args":
"--suites={0}.yml {1}".format(target_suite_file,
options.resmoke_args),
"run_multiple_jobs":
options.run_multiple_jobs,
"task":
options.task,
}
if options.resmoke_jobs_max:
run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max
commands = []
if max_test_runtime or expected_suite_runtime:
cmd_timeout = CmdTimeoutUpdate()
if max_test_runtime:
cmd_timeout.timeout(int(math.ceil(max_test_runtime * 3)))
if expected_suite_runtime:
cmd_timeout.exec_timeout(
int(math.ceil(expected_suite_runtime * 3)))
commands.append(cmd_timeout.validate().resolve())
commands += [
CommandDefinition().function("do setup"),
CommandDefinition().function("run generated tests").vars(
run_tests_vars)
]
task.dependency(TaskDependency("compile")).commands(commands)
def create_generate_tasks_config(evergreen_api, evg_config, options,
tests_by_task, include_gen_task):
"""Create the config for the Evergreen generate.tasks file."""
# pylint: disable=too-many-locals
task_specs = []
task_names = []
if include_gen_task:
task_names.append(BURN_IN_TESTS_GEN_TASK)
for task in sorted(tests_by_task):
multiversion_path = tests_by_task[task].get("use_multiversion")
task_avg_test_runtime_stats = _get_task_runtime_history(
evergreen_api, options.project, task, options.buildvariant)
for test_num, test in enumerate(tests_by_task[task]["tests"]):
sub_task_name = _sub_task_name(options, task, test_num)
task_names.append(sub_task_name)
evg_sub_task = evg_config.task(sub_task_name)
evg_sub_task.dependency(TaskDependency("compile"))
task_spec = TaskSpec(sub_task_name)
if options.distro:
task_spec.distro(options.distro)
task_specs.append(task_spec)
run_tests_vars = {
"resmoke_args":
"{} {} {}".format(tests_by_task[task]["resmoke_args"],
get_resmoke_repeat_options(options), test),
}
commands = []
_generate_timeouts(options, commands, test,
task_avg_test_runtime_stats)
commands.append(CommandDefinition().function("do setup"))
if multiversion_path:
run_tests_vars["task_path_suffix"] = multiversion_path
commands.append(
CommandDefinition().function("do multiversion setup"))
commands.append(
CommandDefinition().function("run tests").vars(run_tests_vars))
evg_sub_task.commands(commands)
display_task = DisplayTaskDefinition(BURN_IN_TESTS_TASK).execution_tasks(
task_names)
evg_config.variant(_get_run_buildvariant(options)).tasks(
task_specs).display_task(display_task)
return evg_config
def add_task(self, name: str, commands: [CommandDefinition],
depends_on: Optional[List[str]] = None, distro: Optional[str] = None):
"""
Add a new task to the task list.
:param name: Name of task to add.
:param commands: List of commands comprising task.
:param depends_on: Any dependencies for the task.
:param distro: Distro task should be run on.
"""
task = self.evg_config.task(name)
task.commands(commands)
if depends_on:
for dep in depends_on:
task.dependency(TaskDependency(dep))
task_spec = TaskSpec(name)
if distro:
task_spec.distro(distro)
self.task_specs.append(task_spec)
self.task_names.append(name)
def _generate_evg_tasks(options):
"""
Generate an evergreen configuration for fuzzers based on the options given.
:param options: task options.
:return: An evergreen configuration.
"""
evg_config = Configuration()
task_names = []
task_specs = []
for task_index in range(options.num_tasks):
name = taskname.name_generated_task(options.name, task_index,
options.num_tasks, options.variant)
task_names.append(name)
task_specs.append(TaskSpec(name))
task = evg_config.task(name)
commands = [CommandDefinition().function("do setup")]
if options.use_multiversion:
commands.append(
CommandDefinition().function("do multiversion setup"))
commands.append(CommandDefinition().function("setup jstestfuzz"))
commands.append(CommandDefinition().function("run jstestfuzz").vars({
"jstestfuzz_vars":
"--numGeneratedFiles {0} {1}".format(options.num_files,
options.jstestfuzz_vars),
"npm_command":
options.npm_command
}))
run_tests_vars = {
"continue_on_failure": options.continue_on_failure,
"resmoke_args": options.resmoke_args,
"resmoke_jobs_max": options.resmoke_jobs_max,
"should_shuffle": options.should_shuffle,
"task_path_suffix": options.use_multiversion,
"timeout_secs": options.timeout_secs,
}
commands.append(
CommandDefinition().function("run tests").vars(run_tests_vars))
task.dependency(TaskDependency("compile")).commands(commands)
dt = DisplayTaskDefinition(options.name).execution_tasks(task_names)\
.execution_task("{0}_gen".format(options.name))
evg_config.variant(options.variant).tasks(task_specs).display_task(dt)
return evg_config
def configure():
n_tasks = 10
c = Configuration()
task_names = []
task_specs = []
for i in range(n_tasks):
name = "aggregation_multiversion_fuzzer_{0:03d}".format(i)
task_names.append(name)
task_specs.append(TaskSpec(name))
t = c.task(name)
t.dependency(TaskDependency("compile")).commands([
CommandDefinition().function("do setup"),
CommandDefinition().function("do multiversion setup"),
CommandDefinition().function("run jstestfuzz").vars({
"jstestfuzz_var":
"--numGeneratedFiles 5",
"npm_command":
"agg-fuzzer",
}),
CommandDefinition().function("run tests").vars({
"continue_on_failure":
"false",
"resmoke_args":
"--suites=generational_fuzzer",
"should_shuffle":
"false",
"task_path_suffix":
"false",
"timeout_secs":
"1800",
})
])
dt = DisplayTaskDefinition("aggregation_multiversion_fuzzer")\
.execution_tasks(task_names)
c.variant("linux-64").tasks(task_specs).display_task(dt)
return c
def generate_task(sub_suite_name, sub_task_name):
"""Generate evergreen config for a resmoke task."""
task_names.append(sub_task_name)
task_specs.append(TaskSpec(sub_task_name))
task = evg_config.task(sub_task_name)
target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name)
run_tests_vars = {
"resmoke_args":
"--suites={0} {1}".format(target_suite_file, options.resmoke_args),
"run_multiple_jobs":
options.run_multiple_jobs,
}
if options.resmoke_jobs_max:
run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max
commands = [
CommandDefinition().function("do setup"),
CommandDefinition().function("run tests").vars(run_tests_vars)
]
task.dependency(TaskDependency("compile")).commands(commands)
def gen_lint_config():
targets = ["src", "test"]
max_hosts = 5
variant_name = "lint variant"
task_group_name = "lint group"
config = Configuration()
tasks = []
for t in targets:
name = "make-lint-" + t
config.task(name).function_with_vars("run-make", {"target": name})
tasks.append(name)
group = config.task_group(task_group_name).max_hosts(max_hosts)
group.setup_group().type("system").command("git.get_project").param(
"directory", "src")
group.setup_group().function("set-up-credentials")
group.teardown_group().function("attach-test-results")
group.teardown_group().function("remove-test-results")
group.tasks(tasks)
config.variant(variant_name).task(TaskSpec(task_group_name))
return config
def generate_evg_tasks(options,
evg_config,
task_name_suffix=None,
display_task=None):
"""
Generate an evergreen configuration for fuzzers based on the options given.
:param options: task options.
:param evg_config: evergreen configuration.
:param task_name_suffix: suffix to be appended to each task name.
:param display_task: an existing display task definition to append to.
:return: An evergreen configuration.
"""
task_names = []
task_specs = []
for task_index in range(options.num_tasks):
task_name = options.name if not task_name_suffix else f"{options.name}_{task_name_suffix}"
name = taskname.name_generated_task(task_name, task_index,
options.num_tasks, options.variant)
task_names.append(name)
task_specs.append(TaskSpec(name))
task = evg_config.task(name)
commands = [CommandDefinition().function("do setup")]
if options.use_multiversion:
commands.append(
CommandDefinition().function("do multiversion setup"))
commands.append(CommandDefinition().function("setup jstestfuzz"))
commands.append(CommandDefinition().function("run jstestfuzz").vars({
"jstestfuzz_vars":
"--numGeneratedFiles {0} {1}".format(options.num_files,
options.jstestfuzz_vars),
"npm_command":
options.npm_command
}))
# Unix path separators are used because Evergreen only runs this script in unix shells,
# even on Windows.
suite_arg = f"--suites={CONFIG_DIRECTORY}/{options.suite}.yml"
run_tests_vars = {
"continue_on_failure": options.continue_on_failure,
"resmoke_args": f"{suite_arg} {options.resmoke_args}",
"resmoke_jobs_max": options.resmoke_jobs_max,
"should_shuffle": options.should_shuffle,
"task_path_suffix": options.use_multiversion,
"timeout_secs": options.timeout_secs,
"task": options.name
} # yapf: disable
commands.append(CommandDefinition().function(
"run generated tests").vars(run_tests_vars))
task.dependency(TaskDependency("compile")).commands(commands)
# Create a new DisplayTaskDefinition or append to the one passed in.
dt = DisplayTaskDefinition(task_name) if not display_task else display_task
dt.execution_tasks(task_names)
evg_config.variant(options.variant).tasks(task_specs)
if not display_task:
dt.execution_task("{0}_gen".format(options.name))
evg_config.variant(options.variant).display_task(dt)
return evg_config
def variant_tasks(tasks: List[GeneratedTask],
variant: str) -> Configuration:
c = Configuration()
c.variant(variant).tasks([TaskSpec(task.name) for task in tasks])
return c
def test_invalid_distro(self):
ts = TaskSpec("task name")
with pytest.raises(TypeError):
ts.distro(42)
