def test_filter_blacklist_files_for_windows(self):
tests_runtimes = [
TestRuntime(test_name="dir1/file1.js", runtime=20.32),
TestRuntime(test_name="dir2/file2.js", runtime=24.32),
TestRuntime(test_name="dir1/dir3/file3.js", runtime=36.32),
]
blacklisted_test = tests_runtimes[1][0]
with patch("os.path.exists") as exists_mock, patch(ns("suitesconfig")) as suitesconfig_mock:
exists_mock.return_value = True
evg = MagicMock()
suitesconfig_mock.get_suite.return_value.tests = [
runtime[0].replace("/", "\\") for runtime in tests_runtimes
if runtime[0] != blacklisted_test
]
config_options = MagicMock(suite="suite")
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
filtered_list = gen_sub_suites.filter_existing_tests(tests_runtimes)
self.assertNotIn(blacklisted_test, filtered_list)
self.assertIn(tests_runtimes[2], filtered_list)
self.assertIn(tests_runtimes[0], filtered_list)
self.assertEqual(2, len(filtered_list))
def test_filter_missing_files(self):
tests_runtimes = [
teststats_utils.TestRuntime(test_name="dir1/file1.js",
runtime=20.32),
teststats_utils.TestRuntime(test_name="dir2/file2.js",
runtime=24.32),
teststats_utils.TestRuntime(test_name="dir1/file3.js",
runtime=36.32),
]
with patch("os.path.exists") as exists_mock, patch(
ns("suitesconfig")) as suitesconfig_mock:
exists_mock.side_effect = [False, True, True]
evg = Mock()
suitesconfig_mock.get_suite.return_value.tests = \
[runtime[0] for runtime in tests_runtimes]
config_options = MagicMock(suite="suite")
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
filtered_list = gen_sub_suites.filter_existing_tests(
tests_runtimes)
self.assertEqual(2, len(filtered_list))
self.assertNotIn(tests_runtimes[0], filtered_list)
self.assertIn(tests_runtimes[2], filtered_list)
self.assertIn(tests_runtimes[1], filtered_list)
def test_is_asan_build_with_no_san_options(self):
evg = MagicMock()
config_options = MagicMock(suite="suite", san_options=None)
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
self.assertFalse(gen_sub_suites._is_asan_build())
def generate_evg_tasks(self, burn_in_test=None, burn_in_idx=0):
# pylint: disable=too-many-locals
"""
Generate evergreen tasks for multiversion tests.
The number of tasks generated equals
(the number of version configs) * (the number of generated suites).
:param burn_in_test: The test to be run as part of the burn in multiversion suite.
"""
is_sharded = is_suite_sharded(TEST_SUITE_DIR, self.options.suite)
if is_sharded:
version_configs = SHARDED_MIXED_VERSION_CONFIGS
else:
version_configs = REPL_MIXED_VERSION_CONFIGS
if self.options.is_jstestfuzz:
return self._generate_fuzzer_tasks(version_configs, is_sharded)
# Divide tests into suites based on run-time statistics for the last
# LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed
# in the misc suite.
gen_suites = generate_resmoke.GenerateSubSuites(
self.evg_api, self.options)
end_date = datetime.datetime.utcnow().replace(microsecond=0)
start_date = end_date - datetime.timedelta(
days=generate_resmoke.LOOKBACK_DURATION_DAYS)
suites = gen_suites.calculate_suites(start_date, end_date)
# Render the given suites into yml files that can be used by resmoke.py.
config_file_dict = generate_resmoke.render_suite_files(
suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR,
self.options.create_misc_suite)
generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
if burn_in_test is not None:
# Generate the subtasks to run burn_in_test against the appropriate mixed version
# configurations. The display task is defined later as part of generating the burn
# in tests.
self._generate_burn_in_execution_tasks(version_configs, suites,
burn_in_test, burn_in_idx,
is_sharded)
return self.evg_config
for version_config in version_configs:
idx = 0
for suite in suites:
# Generate the newly divided test suites
source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
self._generate_sub_task(version_config, self.task, idx,
source_suite, len(suites), is_sharded)
idx += 1
# Also generate the misc task.
misc_suite_name = "{0}_misc".format(self.options.suite)
misc_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml")
self._generate_sub_task(version_config, self.task, idx, misc_suite,
1, is_sharded)
idx += 1
self.create_display_task(self.task, self.task_specs, self.task_names)
return self.evg_config
def test_is_asan_build_with_san_options_non_asan(self):
evg = MagicMock()
config_options = MagicMock(suite="suite",
san_options="SAN_OPTIONS=\"check_initialization_order=true\"")
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
self.assertFalse(gen_sub_suites._is_asan_build())
def test_is_asan_build_on_asan_builds(self):
evg = MagicMock()
config_options = MagicMock(
suite="suite",
san_options="ASAN_OPTIONS=\"detect_leaks=1:check_initialization_order=true\"")
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
self.assertTrue(gen_sub_suites._is_asan_build())
def test_clean_every_n_cadence_on_asan(self):
evg = MagicMock()
config_options = MagicMock(
suite="suite",
san_options="ASAN_OPTIONS=\"detect_leaks=1:check_initialization_order=true\"")
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
cadence = gen_sub_suites._get_clean_every_n_cadence()
self.assertEqual(1, cadence)
def test_calculate_suites_error(self):
response = MagicMock()
response.status_code = requests.codes.INTERNAL_SERVER_ERROR
evg = MagicMock()
evg.test_stats_by_project.side_effect = requests.HTTPError(response=response)
config_options = self.get_mock_options()
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
gen_sub_suites.list_tests = MagicMock(return_value=self.get_test_list(100))
with self.assertRaises(requests.HTTPError):
gen_sub_suites.calculate_suites(_DATE, _DATE)
def test_clean_every_n_cadence_no_hook_config(self, mock_read_suite_config):
evg = MagicMock()
config_options = MagicMock(
suite="suite",
san_options=None,
)
mock_read_suite_config.return_value = {"executor": {"hooks": [{
"class": "hook1",
}, ]}}
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
cadence = gen_sub_suites._get_clean_every_n_cadence()
self.assertEqual(1, cadence)
def test_calculate_suites_uses_fallback_for_no_results(self):
n_tests = 100
evg = MagicMock()
evg.test_stats_by_project.return_value = []
config_options = self.get_mock_options()
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
gen_sub_suites.list_tests = MagicMock(return_value=self.get_test_list(n_tests))
suites = gen_sub_suites.calculate_suites(_DATE, _DATE)
self.assertEqual(gen_sub_suites.config_options.fallback_num_sub_suites, len(suites))
for suite in suites:
self.assertEqual(50, len(suite.tests))
self.assertEqual(n_tests, len(gen_sub_suites.test_list))
def test_calculate_suites_fallback(self):
n_tests = 100
evg = mock_test_stats_unavailable(MagicMock())
config_options = self.get_mock_options()
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
gen_sub_suites.list_tests = MagicMock(return_value=self.get_test_list(n_tests))
suites = gen_sub_suites.calculate_suites(_DATE, _DATE)
self.assertEqual(gen_sub_suites.config_options.fallback_num_sub_suites, len(suites))
for suite in suites:
self.assertEqual(50, len(suite.tests))
self.assertEqual(n_tests, len(gen_sub_suites.test_list))
def test_when_task_has_already_run_successfully(self):
"""
Given evergreen_generate_resmoke_tasks has already been run successfully by this task,
When it attempts to run again,
It does not generate any files.
"""
evg_api_mock = self._mock_evg_api(successful_task=True)
mock_config = self._mock_config()
config = self._config_options(mock_config)
with TemporaryDirectory() as tmpdir:
mock_config["generated_config_dir"] = tmpdir
under_test.GenerateSubSuites(evg_api_mock, config).run()
self.assertEqual(0, len(os.listdir(tmpdir)))
def generate_resmoke_suites(self) -> List[Suite]:
"""Generate the resmoke configuration files for this generator."""
# Divide tests into suites based on run-time statistics for the last
# LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed
# in the misc suite.
gen_suites = generate_resmoke.GenerateSubSuites(self.evg_api, self.options)
end_date = datetime.datetime.utcnow().replace(microsecond=0)
start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS)
suites = gen_suites.calculate_suites(start_date, end_date)
# Render the given suites into yml files that can be used by resmoke.py.
config_file_dict = generate_resmoke.render_suite_files(suites, self.options.suite,
gen_suites.test_list, TEST_SUITE_DIR,
self.options.create_misc_suite)
generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
return suites
def test_with_each_test_in_own_task(self, suites_config_mock):
"""
Given a task with all tests having a historic runtime over the target,
When evergreen_generate_resmoke_tasks attempts to generate suites,
It generates a suite for each test.
"""
evg_api_mock = self._mock_evg_api()
mock_config = self._mock_config()
config = self._config_options(mock_config)
task = mock_config["task_name"][:-4]
mock_config['target_resmoke_time'] = 10 # 10 minute max test time.
n_tests = 4
with TemporaryDirectory() as tmpdir:
target_directory, source_directory = self._prep_dirs(
tmpdir, mock_config)
suite_path = os.path.join(source_directory, task)
mock_config["suite"] = suite_path
test_list = self._mock_test_files(source_directory, n_tests,
15 * 60, evg_api_mock,
suites_config_mock)
mock_resmoke_config_file(test_list, suite_path + ".yml")
under_test.enable_logging(True)
under_test.GenerateSubSuites(evg_api_mock, config).run()
# Were all the config files created? There should be one for each suite as well as
# the evergreen json config.
generated_files = os.listdir(target_directory)
# The expected suite count is the number of tests + the _misc suite.
expected_suite_count = n_tests + 1
# We expect files for all the suites + the evergreen json config.
self.assertEqual(expected_suite_count + 1, len(generated_files))
# Taking a closer look at the evergreen json config.
expected_shrub_file = f"{config.task}.json"
self.assertIn(expected_shrub_file, generated_files)
with open(os.path.join(target_directory,
expected_shrub_file)) as fileh:
shrub_config = json.load(fileh)
# Is there a task in the config for all the suites we created?
self.assertEqual(expected_suite_count,
len(shrub_config["tasks"]))
def test_calculate_suites_fallback(self):
n_tests = 100
response = Mock()
response.status_code = requests.codes.SERVICE_UNAVAILABLE
evg = Mock()
evg.test_stats_by_project.side_effect = requests.HTTPError(response=response)
config_options = self.get_mock_options()
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
gen_sub_suites.list_tests = Mock(return_value=self.get_test_list(n_tests))
suites = gen_sub_suites.calculate_suites(_DATE, _DATE)
self.assertEqual(gen_sub_suites.config_options.fallback_num_sub_suites, len(suites))
for suite in suites:
self.assertEqual(50, len(suite.tests))
self.assertEqual(n_tests, len(gen_sub_suites.test_list))
def test_when_evg_test_stats_is_down(self, suites_config_mock):
"""
Given Evergreen historic test stats endpoint is disabled,
When evergreen_generate_resmoke_tasks attempts to generate suites,
It generates suites based on "fallback_num_sub_suites".
"""
evg_api_mock = mock_test_stats_unavailable(self._mock_evg_api())
mock_config = self._mock_config()
config = self._config_options(mock_config)
task = mock_config["task_name"][:-4]
n_tests = 100
with TemporaryDirectory() as tmpdir:
target_directory, source_directory = self._prep_dirs(
tmpdir, mock_config)
suite_path = os.path.join(source_directory, task)
mock_config["suite"] = suite_path
test_list = self._mock_test_files(source_directory, n_tests, 5,
evg_api_mock, suites_config_mock)
mock_resmoke_config_file(test_list, suite_path + ".yml")
under_test.GenerateSubSuites(evg_api_mock, config).run()
# Were all the config files created? There should be one for each suite as well as
# the evergreen json config.
generated_files = os.listdir(target_directory)
# The expected suite count is the number of fallback suites + the _misc suite.
expected_suite_count = mock_config["fallback_num_sub_suites"] + 1
# We expect files for all the suites + the evergreen json config.
self.assertEqual(expected_suite_count + 1, len(generated_files))
# Taking a closer look at the evergreen json config.
expected_shrub_file = f"{config.task}.json"
self.assertIn(expected_shrub_file, generated_files)
with open(os.path.join(target_directory,
expected_shrub_file)) as fileh:
shrub_config = json.load(fileh)
# Is there a task in the config for all the suites we created?
self.assertEqual(expected_suite_count,
len(shrub_config["tasks"]))
def generate_evg_tasks(self):
"""
Generate evergreen tasks for multiversion tests.
The number of tasks generated equals
(the number of configs in MIXED_VERSION_CONFIGS) * (the number of generated suites).
"""
idx = 0
# Divide tests into suites based on run-time statistics for the last
# LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed
# in the misc suite.
gen_suites = generate_resmoke.GenerateSubSuites(self.evg_api, self.options)
end_date = datetime.datetime.utcnow().replace(microsecond=0)
start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS)
suites = gen_suites.calculate_suites(start_date, end_date)
# Render the given suites into yml files that can be used by resmoke.py.
if self.options.is_sharded:
config = MultiversionConfig(update_suite_config_for_multiversion_sharded,
SHARDED_MIXED_VERSION_CONFIGS)
else:
config = MultiversionConfig(update_suite_config_for_multiversion_replset,
MIXED_VERSION_CONFIGS)
config_file_dict = generate_resmoke.render_suite_files(
suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, config.update_yaml)
for version_config in config.version_configs:
for suite in suites:
# Generate the newly divided test suites
source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
self._generate_sub_task(version_config, idx, source_suite, len(suites))
idx += 1
# Also generate the misc task.
misc_suite_name = "{0}_misc".format(self.options.suite)
source_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml")
self._generate_sub_task(version_config, idx, source_suite, 1)
idx += 1
generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
dt = DisplayTaskDefinition(self.task).execution_tasks(self.task_names)\
.execution_task("{0}_gen".format(self.task))
self.evg_config.variant(self.options.variant).tasks(self.task_specs).display_task(dt)
return self.evg_config
def test_calculate_suites_uses_fallback_if_only_results_are_filtered(self):
n_tests = 100
evg = MagicMock()
evg.test_stats_by_project.return_value = [
tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)
]
config_options = self.get_mock_options()
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
gen_sub_suites.list_tests = MagicMock(return_value=self.get_test_list(n_tests))
with patch("os.path.exists") as exists_mock:
exists_mock.return_value = False
suites = gen_sub_suites.calculate_suites(_DATE, _DATE)
self.assertEqual(gen_sub_suites.config_options.fallback_num_sub_suites, len(suites))
for suite in suites:
self.assertEqual(50, len(suite.tests))
self.assertEqual(n_tests, len(gen_sub_suites.test_list))
def test_calculate_suites(self):
evg = MagicMock()
evg.test_stats_by_project.return_value = [
tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)
]
config_options = self.get_mock_options()
config_options.max_sub_suites = 1000
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
with patch("os.path.exists") as exists_mock, patch(ns("suitesconfig")) as suitesconfig_mock:
exists_mock.return_value = True
suitesconfig_mock.get_suite.return_value.tests = \
[stat.test_file for stat in evg.test_stats_by_project.return_value]
suites = gen_sub_suites.calculate_suites(_DATE, _DATE)
# There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites.
self.assertEqual(10, len(suites))
for suite in suites:
self.assertEqual(10, len(suite.tests))
def test_clean_every_n_cadence_from_hook_config(self, mock_read_suite_config):
evg = MagicMock()
config_options = MagicMock(
suite="suite",
san_options=None,
)
expected_n = 42
mock_read_suite_config.return_value = {
"executor": {
"hooks": [{
"class": "hook1",
}, {
"class": under_test.CLEAN_EVERY_N_HOOK,
"n": expected_n,
}]
}
}
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
cadence = gen_sub_suites._get_clean_every_n_cadence()
self.assertEqual(expected_n, cadence)
def test_calculate_suites_with_selected_tests_to_run(self):
evg = MagicMock()
evg.test_stats_by_project.return_value = [
tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)
]
config_options = self.get_mock_options()
config_options.max_sub_suites = 1000
config_options.selected_tests_to_run = ["test1.js", "test2.js"]
gen_sub_suites = under_test.GenerateSubSuites(evg, config_options)
with patch("os.path.exists") as exists_mock, patch(ns("suitesconfig")) as suitesconfig_mock:
exists_mock.return_value = True
suitesconfig_mock.get_suite.return_value.tests = \
[stat.test_file for stat in evg.test_stats_by_project.return_value]
suites = gen_sub_suites.calculate_suites(_DATE, _DATE)
# There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites.
# However, since we have selected only 2 tests to run, test1.js and
# test2.js, only 1 suite should be created.
self.assertEqual(1, len(suites))
for suite in suites:
self.assertEqual(2, len(suite.tests))
