def test_non_running_at_all_is_reliable(self):
"""
Tests that tests that are tagged as unreliable but no longer running (either during the
reliable or the unreliable period) have their tags removed.
"""
config = self.CONFIG
test_history = utl.TestHistory("jstests/core/all.js")
test_history2 = utl.TestHistory("jstests/core/all2.js")
initial_tags = collections.OrderedDict([
("jstests/core/all2.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
# all2.js did not run at all
test_history.add_reliable_period_stats([self._stats()])
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
utl.update_tags(summary_lifecycle, config, test_history2)
updated_tags = self.assert_has_only_js_tests(lifecycle)
# The tags for variant and distro have been removed.
self.assertEqual(updated_tags, collections.OrderedDict([]))
def test_obeys_unreliable_min_runs(self):
"""
Tests that update_tags() only considers a test unreliable if it has more than
'unreliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),
unreliable_min_runs=100)
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
stats = [
self._stats(num_pass=0, num_fail=1),
self._stats(num_pass=0, num_fail=1, task="jsCore"),
self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"),
self._stats(num_pass=1, num_fail=0),
self._stats(num_pass=0, num_fail=1, distro="rhel55"),
]
test_history.add_reliable_period_stats(stats)
test_history.add_unreliable_period_stats(stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_remain_reliable(self):
"""
Tests that update_tags() preserves the absence of tags for reliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
stats = [
self._stats(num_pass=1, num_fail=0),
self._stats(num_pass=1, num_fail=0, task="jsCore"),
self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"),
self._stats(num_pass=0, num_fail=1),
self._stats(num_pass=1, num_fail=0, distro="rhel55"),
]
test_history.add_reliable_period_stats(stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_get_rates_by_distro(self):
th = utl.TestHistory("test1.js")
th.add_reliable_period_stats([
{
"test_file": "test1.js", "task_name": "task1", "variant": "v", "distro": "d1",
"num_pass": 1, "num_fail": 0
},
{
"test_file": "test1.js", "task_name": "task1", "variant": "v", "distro": "d1",
"num_pass": 0, "num_fail": 1
},
{
"test_file": "test1.js", "task_name": "task1", "variant": "v", "distro": "d2",
"num_pass": 0, "num_fail": 1
},
{
"test_file": "test1.js", "task_name": "task2", "variant": "v", "distro": "d2",
"num_pass": 1, "num_fail": 0
},
])
expected = [
(utl.TestCombination("test1.js", "task1", "v", "d1"), 0.5, 2),
(utl.TestCombination("test1.js", "task1", "v", "d2"), 1.0, 1),
(utl.TestCombination("test1.js", "task2", "v", "d2"), 0.0, 1),
]
rates = th.get_reliable_period_rates(group_by=utl.TestCombination.GROUP_BY_DISTRO)
self.assertListEqual(expected, rates)
def transition_from_unreliable_to_reliable(self, config, initial_tags):
"""
Tests that update_tags() untags a formerly unreliable combination after it has become
reliable again.
"""
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
stats = [
self._stats(num_pass=1, num_fail=0),
self._stats(num_pass=1, num_fail=0, task="jsCore"),
self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"),
self._stats(num_pass=0, num_fail=1),
self._stats(num_pass=1, num_fail=0, distro="rhel55"),
]
test_history.add_reliable_period_stats(stats)
test_history.add_unreliable_period_stats(stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
def test_get_rates_by_test(self):
th = utl.TestHistory("test1.js")
th.add_reliable_period_stats([
{
"test_file": "test1.js", "task_name": "task1", "variant": "v1", "distro": "d1",
"num_pass": 1, "num_fail": 0
},
{
"test_file": "test1.js", "task_name": "task1", "variant": "v1", "distro": "d2",
"num_pass": 0, "num_fail": 1
},
{
"test_file": "test1.js", "task_name": "task2", "variant": "v2", "distro": "d3",
"num_pass": 2, "num_fail": 1
},
{
"test_file": "test1.js", "task_name": "task3", "variant": "v2", "distro": "d4",
"num_pass": 1, "num_fail": 0
},
])
expected = [
(utl.TestCombination("test1.js", None, None, None), float(1) / 3, 6),
]
rates = th.get_reliable_period_rates(group_by=utl.TestCombination.GROUP_BY_TEST)
self.assertListEqual(expected, rates)
def test_non_running_in_reliable_period_is_reliable(self):
"""
Tests that tests that have a failure rate above the unacceptable rate during the unreliable
period but haven't run during the reliable period are marked as reliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1),
unreliable_time_period=datetime.timedelta(days=2))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
# The test did not run on the reliable period on linux-64.
reliable_period_stats = [
self._stats(num_pass=3,
num_fail=0,
variant="linux-alt",
distro="debian7"),
]
unreliable_period_stats = [
# Failing.
self._stats(num_pass=0, num_fail=2),
self._stats(num_pass=3,
num_fail=0,
variant="linux-alt",
distro="debian7"),
]
test_history.add_reliable_period_stats(reliable_period_stats)
test_history.add_unreliable_period_stats(unreliable_period_stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
# The tags for variant and distro have been removed.
self.assertEqual(
updated_tags,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable", "unreliable|jsCore_WT"])
]))