def test_post(self):
c = Configuration()
cs = CommandSequence()
cs.command().function("func 0")
cs.command().function("func 1")
c.post(cs)
assert "func 0" == c.to_map()["post"][0]["func"]
assert 2 == len(c.to_map()["post"])
def create_generate_tasks_file(evg_api, options, tests_by_task):
"""Create the Evergreen generate.tasks file."""
evg_config = Configuration()
evg_config = create_generate_tasks_config(evg_api, evg_config, options, tests_by_task,
include_gen_task=True)
_write_json_file(evg_config.to_map(), options.generate_tasks_file)
def test_generate_evg_tasks_one_test_changed(self, create_tests_by_task_mock):
evg_conf_mock = get_evergreen_config()
create_tests_by_task_mock.return_value = {
"aggregation_mongos_passthrough": {
"resmoke_args":
"--suites=aggregation_mongos_passthrough --storageEngine=wiredTiger",
"tests": ["jstests/aggregation/bugs/ifnull.js"],
"use_multiversion": None
}
} # yapf: disable
expansions_file_data = get_expansions_data()
buildvariant_map = {
"enterprise-rhel-62-64-bit-inmem": "enterprise-rhel-62-64-bit-inmem-required",
"enterprise-rhel-62-64-bit-majority-read-concern-off":
"enterprise-rhel-62-64-bit-majority-read-concern-off-required",
} # yapf: disable
shrub_config = Configuration()
evergreen_api = MagicMock()
repo = MagicMock()
evergreen_api.test_stats_by_project.return_value = [
MagicMock(test_file="dir/test2.js", avg_duration_pass=10)
]
under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
buildvariant_map, repo, evg_conf_mock)
generated_config = shrub_config.to_map()
self.assertEqual(len(generated_config["buildvariants"]), 2)
first_generated_build_variant = generated_config["buildvariants"][0]
self.assertEqual(first_generated_build_variant["display_tasks"][0]["name"], "burn_in_tests")
self.assertEqual(
first_generated_build_variant["display_tasks"][0]["execution_tasks"][0],
"burn_in:aggregation_mongos_passthrough_0_enterprise-rhel-62-64-bit-inmem-required")
def test_generate_evg_tasks_one_test_changed(self,
create_tests_by_task_mock,
evergreen_mock):
evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config(
)
create_tests_by_task_mock.return_value = {
"aggregation_mongers_passthrough": {
"resmoke_args":
"--suites=aggregation_mongers_passthrough --storageEngine=wiredTiger",
"tests": ["jstests/aggregation/bugs/ifnull.js"],
"use_multiversion": None
}
} # yapf: disable
expansions_file_data = get_expansions_data()
buildvariant_map = {
"enterprise-rhel-62-64-bit-inmem": "enterprise-rhel-62-64-bit-inmem-required",
"enterprise-rhel-62-64-bit-majority-read-concern-off":
"enterprise-rhel-62-64-bit-majority-read-concern-off-required",
} # yapf: disable
shrub_config = Configuration()
burn_in_tags._generate_evg_tasks(shrub_config, expansions_file_data,
buildvariant_map)
generated_config = shrub_config.to_map()
self.assertEqual(len(generated_config["buildvariants"]), 2)
first_generated_build_variant = generated_config["buildvariants"][0]
self.assertEqual(
first_generated_build_variant["display_tasks"][0]["name"],
"burn_in_tests")
self.assertEqual(
first_generated_build_variant["display_tasks"][0]
["execution_tasks"][0],
"burn_in:enterprise-rhel-62-64-bit-inmem-required_aggregation_mongers_passthrough_0"
)
def create_generate_tasks_file(tests_by_task: Dict,
generate_config: GenerateConfig,
repeat_config: RepeatConfig,
evg_api: Optional[EvergreenApi],
task_prefix: str = 'burn_in',
include_gen_task: bool = True) -> Dict:
"""
Create an Evergreen generate.tasks file to run the given tasks and tests.
:param tests_by_task: Dictionary of tests and tasks to run.
:param generate_config: Information about how burn_in should generate tasks.
:param repeat_config: Information about how burn_in should repeat tests.
:param evg_api: Evergreen api.
:param task_prefix: Prefix to start generated task's name with.
:param include_gen_task: Should the generating task be included in the display task.
:returns: Configuration to pass to 'generate.tasks'.
"""
evg_config = Configuration()
evg_config = create_generate_tasks_config(
evg_config,
tests_by_task,
generate_config,
repeat_config,
evg_api,
include_gen_task=include_gen_task,
task_prefix=task_prefix)
json_config = evg_config.to_map()
tasks_to_create = len(json_config.get('tasks', []))
if tasks_to_create > MAX_TASKS_TO_CREATE:
LOGGER.warning("Attempting to create more tasks than max, aborting",
tasks=tasks_to_create,
max=MAX_TASKS_TO_CREATE)
sys.exit(1)
return json_config
def test_configuration_tasks(self):
c = Configuration()
c.task("new task")
obj = c.to_map()
assert "new task" == obj["tasks"][0]["name"]
def test_generate_evg_buildvariant_one_base_variant(self):
evg_conf_mock = get_evergreen_config()
base_variant = "enterprise-rhel-62-64-bit-inmem"
generated_variant = "enterprise-rhel-62-64-bit-inmem-required"
burn_in_tags_gen_variant = "enterprise-rhel-62-64-bit"
shrub_config = Configuration()
under_test._generate_evg_build_variant(shrub_config, base_variant,
generated_variant,
burn_in_tags_gen_variant,
evg_conf_mock)
expected_variant_data = get_evergreen_config().get_variant(
base_variant)
generated_buildvariants = shrub_config.to_map()["buildvariants"]
self.assertEqual(len(generated_buildvariants), 1)
generated_build_variant = generated_buildvariants[0]
self.assertEqual(generated_build_variant["name"], generated_variant)
self.assertEqual(generated_build_variant["modules"],
expected_variant_data.modules)
generated_expansions = generated_build_variant["expansions"]
burn_in_bypass_expansion_value = generated_expansions.pop(
"burn_in_bypass")
self.assertEqual(burn_in_bypass_expansion_value,
burn_in_tags_gen_variant)
self.assertEqual(generated_expansions,
expected_variant_data.expansions)
def create_generate_tasks_file(options, tests_by_task):
"""Create the Evergreen generate.tasks file."""
evg_config = Configuration()
task_specs = []
task_names = [BURN_IN_TESTS_GEN_TASK]
for task in sorted(tests_by_task):
multiversion_path = tests_by_task[task].get("use_multiversion")
for test_num, test in enumerate(tests_by_task[task]["tests"]):
sub_task_name = _sub_task_name(options.buildvariant, task, test_num)
task_names.append(sub_task_name)
evg_sub_task = evg_config.task(sub_task_name)
evg_sub_task.dependency(TaskDependency("compile"))
task_spec = TaskSpec(sub_task_name)
if options.distro:
task_spec.distro(options.distro)
task_specs.append(task_spec)
run_tests_vars = {
"resmoke_args":
"{} {} {}".format(tests_by_task[task]["resmoke_args"],
get_resmoke_repeat_options(options), test),
}
commands = []
commands.append(CommandDefinition().function("do setup"))
if multiversion_path:
run_tests_vars["task_path_suffix"] = multiversion_path
commands.append(CommandDefinition().function("do multiversion setup"))
commands.append(CommandDefinition().function("run tests").vars(run_tests_vars))
evg_sub_task.commands(commands)
display_task = DisplayTaskDefinition(BURN_IN_TESTS_TASK).execution_tasks(task_names)
evg_config.variant(_get_run_buildvariant(options)).tasks(task_specs).display_task(display_task)
_write_json_file(evg_config.to_map(), options.generate_tasks_file)
def test_using_display_task_with_existing_tasks(self):
config = Configuration()
task_list = under_test.TaskList(config)
func = "test"
variant = "variant 1"
tasks = ["task 1", "task 2"]
for task in tasks:
task_list.add_task(task, [under_test._cmd_by_name(func)])
display_task = "display_task"
existing_tasks = ["other task 1", "other task 2"]
task_list.add_to_variant(variant, display_task, existing_tasks)
cfg_dict = config.to_map()
self.assertEqual(len(tasks), len(cfg_dict["tasks"]))
variant_dict = cfg_dict["buildvariants"][0]
self.assertEqual(len(tasks), len(variant_dict["tasks"]))
dt_dict = variant_dict["display_tasks"][0]
self.assertEqual(display_task, dt_dict["name"])
for task in tasks:
self.assertIn(task, dt_dict["execution_tasks"])
for task in existing_tasks:
self.assertIn(task, dt_dict["execution_tasks"])
def test_get_existing_task(self):
c = Configuration()
c.task("task 0")
t1 = c.task("task 1")
c.task("task 2")
t1.priority(42)
assert 42 == c.task("task 1").to_map()["priority"]
assert 3 == len(c.to_map()["tasks"])
def test_no_tasks_given(self):
evg_config = Configuration()
gen_config = MagicMock(run_build_variant="variant")
repeat_config = MagicMock()
evg_config = under_test.create_generate_tasks_config(
evg_config, {}, gen_config, repeat_config, None)
evg_config_dict = evg_config.to_map()
self.assertNotIn("tasks", evg_config_dict)
def test_task_groups(self):
c = Configuration()
c.task_group("tg 0")
c.task_group("tg 1")
tg2 = c.task_group("tg 2")
tg2.max_hosts(5)
assert 5 == c.task_group("tg 2").to_map()["max_hosts"]
assert 3 == len(c.to_map()["task_groups"])
def test_functions(self):
c = Configuration()
f = c.function("func 0")
c.function("func 1")
c.function("func 2")
f.add(CommandDefinition().function("f"))
assert "f" == c.function("func 0").to_map()[0]["func"]
assert 3 == len(c.to_map()["functions"])
def tests_no_tasks_given(self):
evg_config = Configuration()
gen_config = MagicMock(run_build_variant="variant", fallback_num_sub_suites=1,
project="project", build_variant="build_variant", task_id="task_id",
target_resmoke_time=60)
evg_api = MagicMock()
evg_config = under_test.create_multiversion_generate_tasks_config(
evg_config, {}, evg_api, gen_config)
evg_config_dict = evg_config.to_map()
self.assertNotIn("tasks", evg_config_dict)
def test_variants(self):
c = Configuration()
c.variant("variant 0")
v1 = c.variant("variant 1")
c.variant("variant 2")
v1.batch_time(100)
assert 100 == c.variant("variant 1").to_map()["batchtime"]
assert 3 == len(c.to_map()["buildvariants"])
def test_n_task_m_test(self):
n_tasks = 3
n_tests = 5
evg_config = Configuration()
gen_config = MagicMock(run_build_variant="variant", distro=None)
repeat_config = MagicMock()
tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
evg_config = under_test.create_generate_tasks_config(
evg_config, tests_by_task, gen_config, repeat_config, None)
evg_config_dict = evg_config.to_map()
self.assertEqual(n_tasks * n_tests, len(evg_config_dict["tasks"]))
def test_generate_evg_buildvariant_one_base_variant(self, evergreen_mock):
evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config()
base_variant = "enterprise-rhel-62-64-bit-inmem"
expected_variant_data = get_evergreen_config().get_variant(base_variant)
shrub_config = Configuration()
burn_in_tags._generate_evg_buildvariant(shrub_config, base_variant,
"enterprise-rhel-62-64-bit-inmem-required")
generated_buildvariants = shrub_config.to_map()["buildvariants"]
self.assertEqual(len(generated_buildvariants), 1)
generated_build_variant = generated_buildvariants[0]
self.assertEqual(generated_build_variant["name"], f"{expected_variant_data}-required")
self.assertEqual(generated_build_variant["modules"], expected_variant_data.modules)
self.assertEqual(generated_build_variant["expansions"], expected_variant_data.expansions)
def test_one_task_n_test(self):
n_tasks = 1
n_tests = 2
evg_config = Configuration()
gen_config = MagicMock(run_build_variant="variant", fallback_num_sub_suites=1,
project="project", build_variant="build_variant", task_id="task_id",
target_resmoke_time=60)
evg_api = MagicMock()
tests_by_task = create_multiversion_tests_by_task_mock(n_tasks, n_tests)
evg_config = under_test.create_multiversion_generate_tasks_config(
evg_config, tests_by_task, evg_api, gen_config)
evg_config_dict = evg_config.to_map()
tasks = evg_config_dict["tasks"]
self.assertEqual(len(tasks), NUM_REPL_MIXED_VERSION_CONFIGS * n_tests)
def test_generate_evg_tasks_no_tests_changed(self, create_tests_by_task_mock, evergreen_mock):
evergreen_mock.parse_evergreen_file.return_value = get_evergreen_config()
create_tests_by_task_mock.return_value = {}
expansions_file_data = get_expansions_data()
buildvariant_map = {
"enterprise-rhel-62-64-bit-inmem": "enterprise-rhel-62-64-bit-inmem-required",
"enterprise-rhel-62-64-bit-majority-read-concern-off":
"enterprise-rhel-62-64-bit-majority-read-concern-off-required",
} # yapf: disable
shrub_config = Configuration()
evergreen_api = Mock()
burn_in_tags._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
buildvariant_map)
self.assertEqual(shrub_config.to_map(), {})
def test_config_with_non_recurse_values(self):
c = Configuration()
c.exec_timeout(20).batch_time(300).ignore_file("file1").ignore_files(["file2", "file3"])
c.stepback().command_type("setup")
obj = c.to_map()
assert obj["exec_timeout_secs"] == 20
assert obj["batchtime"] == 300
assert "file1" in obj["ignore"]
assert "file2" in obj["ignore"]
assert "file3" in obj["ignore"]
assert obj["stepback"]
assert obj["command_type"] == "setup"
assert "tasks" not in obj
def test_tasks_not_in_multiversion_suites(self):
n_tasks = 1
n_tests = 1
evg_config = Configuration()
gen_config = MagicMock(run_build_variant="variant", fallback_num_sub_suites=1,
project="project", build_variant="build_variant", task_id="task_id",
target_resmoke_time=60)
evg_api = MagicMock()
# Create a tests_by_tasks dict that doesn't contain any multiversion suites.
tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
evg_config = under_test.create_multiversion_generate_tasks_config(
evg_config, tests_by_task, evg_api, gen_config)
evg_config_dict = evg_config.to_map()
# We should not generate any tasks that are not part of the burn_in_multiversion suite.
self.assertNotIn("tasks", evg_config_dict)
def create_generate_tasks_file(evg_api, options, tests_by_task):
"""Create the Evergreen generate.tasks file."""
evg_config = Configuration()
evg_config = create_generate_tasks_config(evg_api,
evg_config,
options,
tests_by_task,
include_gen_task=True)
json_config = evg_config.to_map()
tasks_to_create = len(json_config.get('tasks', []))
if tasks_to_create > MAX_TASKS_TO_CREATE:
LOGGER.warning(
"Attempting to create more tasks than max(%d), aborting",
tasks_to_create)
sys.exit(1)
_write_json_file(json_config, options.generate_tasks_file)
def test_adding_multiple_tasks(self):
config = Configuration()
task_list = under_test.TaskList(config)
func = "test"
variant = "variant 1"
tasks = ["task 1", "task 2"]
for task in tasks:
task_list.add_task(task, [under_test._cmd_by_name(func)])
task_list.add_to_variant(variant)
cfg_dict = config.to_map()
self.assertEqual(len(tasks), len(cfg_dict["tasks"]))
self.assertEqual(len(tasks),
len(cfg_dict["buildvariants"][0]["tasks"]))
def test_multiversion_path_is_used(self):
n_tasks = 1
n_tests = 1
evg_config = Configuration()
gen_config = MagicMock(run_build_variant="variant", distro=None)
repeat_config = MagicMock()
tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
first_task = "task_0"
multiversion_path = "multiversion_path"
tests_by_task[first_task]["use_multiversion"] = multiversion_path
evg_config = under_test.create_generate_tasks_config(evg_config, tests_by_task, gen_config,
repeat_config, None)
evg_config_dict = evg_config.to_map()
tasks = evg_config_dict["tasks"]
self.assertEqual(n_tasks * n_tests, len(tasks))
self.assertEqual(multiversion_path, tasks[0]["commands"][2]["vars"]["task_path_suffix"])
def test_adding_a_task(self):
config = Configuration()
task_list = under_test.TaskList(config)
func = "test"
task = "task 1"
variant = "variant 1"
task_list.add_task(task, [under_test._cmd_by_name(func)])
task_list.add_to_variant(variant)
cfg_dict = config.to_map()
cmd_dict = cfg_dict["tasks"][0]
self.assertEqual(task, cmd_dict["name"])
self.assertEqual(func, cmd_dict["commands"][0]["func"])
self.assertEqual(task,
cfg_dict["buildvariants"][0]["tasks"][0]["name"])
def test_one_task_one_test(self):
n_tasks = 1
n_tests = 1
resmoke_options = "options for resmoke"
evg_config = Configuration()
gen_config = MagicMock(run_build_variant="variant", distro=None)
repeat_config = MagicMock()
repeat_config.generate_resmoke_options.return_value = resmoke_options
tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
evg_config = under_test.create_generate_tasks_config(
evg_config, tests_by_task, gen_config, repeat_config, None)
evg_config_dict = evg_config.to_map()
tasks = evg_config_dict["tasks"]
self.assertEqual(n_tasks * n_tests, len(tasks))
cmd = tasks[0]["commands"]
self.assertIn(resmoke_options, cmd[1]["vars"]["resmoke_args"])
self.assertIn("--suites=suite_0", cmd[1]["vars"]["resmoke_args"])
self.assertIn("tests_0", cmd[1]["vars"]["resmoke_args"])
def test_n_task_m_test(self, mock_hash):
mock_hash.return_value = MONGO_4_2_HASH
n_tasks = 2
n_tests = 3
evg_config = Configuration()
gen_config = MagicMock(run_build_variant="variant",
fallback_num_sub_suites=1,
project="project",
build_variant="build_variant",
task_id="task_id",
target_resmoke_time=60)
evg_api = MagicMock()
tests_by_task = create_multiversion_tests_by_task_mock(
n_tasks, n_tests)
evg_config = under_test.create_multiversion_generate_tasks_config(
evg_config, tests_by_task, evg_api, gen_config)
evg_config_dict = evg_config.to_map()
tasks = evg_config_dict["tasks"]
self.assertEqual(len(tasks),
(NUM_REPL_MIXED_VERSION_CONFIGS +
NUM_SHARDED_MIXED_VERSION_CONFIGS) * n_tests)
def test_adding_a_task_with_dependecies(self):
config = Configuration()
task_list = under_test.TaskList(config)
func = "test"
task = "task 1"
variant = "variant 1"
dependencies = ["dep task 1", "dep task 2"]
task_list.add_task(task, [under_test._cmd_by_name(func)],
depends_on=dependencies)
task_list.add_to_variant(variant)
cfg_dict = config.to_map()
cmd_dict = cfg_dict["tasks"][0]
self.assertEqual(task, cmd_dict["name"])
self.assertEqual(func, cmd_dict["commands"][0]["func"])
for dep in dependencies:
self.assertIn(dep, {d["name"] for d in cmd_dict["depends_on"]})
task_dict = cfg_dict["buildvariants"][0]["tasks"][0]
self.assertEqual(task, task_dict["name"])
def main(build_variant, run_build_variant, distro, project,
generate_tasks_file, no_exec, resmoke_args, evg_api_config, verbose,
task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param no_exec: Just perform test discover, do not execute the tests.
:param resmoke_args: Arguments to pass through to resmoke.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
repeat_config = RepeatConfig() # yapf: disable
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
if generate_tasks_file:
generate_config.validate(evg_conf)
evg_api = _get_evg_api(evg_api_config, False)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
tests_by_task = create_tests_by_task(generate_config.build_variant, repos,
evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
multiversion_tasks = evg_conf.get_task_names_by_tag(
MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag",
tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
# We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include
# MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key(
MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
evg_config = Configuration()
evg_config = create_multiversion_generate_tasks_config(
evg_config, tests_by_task, evg_api, generate_config)
json_config = evg_config.to_map()
tasks_to_create = len(json_config.get('tasks', []))
if tasks_to_create > MAX_TASKS_TO_CREATE:
LOGGER.warning(
"Attempting to create more tasks than max, aborting",
tasks=tasks_to_create,
max=MAX_TASKS_TO_CREATE)
sys.exit(1)
_write_json_file(json_config, generate_tasks_file)
elif not no_exec:
run_tests(tests_by_task, resmoke_cmd)
else:
LOGGER.info("Not running tests due to 'no_exec' option.")