class TestUpdateTags(unittest.TestCase):
"""
Tests for the update_tags() function.
"""
CONFIG = update_test_lifecycle.Config(
test_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
task_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
variant_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
distro_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
reliable_min_runs=2,
reliable_time_period=datetime.timedelta(days=1),
unreliable_min_runs=2,
unreliable_time_period=datetime.timedelta(days=1))
ENTRY = test_failures.ReportEntry(test="jstests/core/all.js",
task="jsCore_WT",
variant="linux-64",
distro="rhel62",
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=0,
num_fail=0)
def assert_has_only_js_tests(self, lifecycle):
"""
Raises an AssertionError exception if 'lifecycle' is not of the following form:
selector:
js_test:
...
"""
self.assertIn("selector", lifecycle.raw)
self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw))
self.assertIn("js_test", lifecycle.raw["selector"])
self.assertEqual(1,
len(lifecycle.raw["selector"]),
msg=str(lifecycle.raw))
return lifecycle.raw["selector"]["js_test"]
def transition_from_reliable_to_unreliable(self, config, expected_tags):
"""
Tests that update_tags() tags a formerly reliable combination as being unreliable.
"""
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(collections.OrderedDict(),
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, expected_tags)
def test_transition_test_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task) combination as being
unreliable.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being
unreliable.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination
as being unreliable.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags multiple formerly reliable combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def transition_from_unreliable_to_reliable(self, config, initial_tags):
"""
Tests that update_tags() untags a formerly unreliable combination after it has become
reliable again.
"""
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
def test_non_running_in_reliable_period_is_reliable(self):
"""
Tests that tests that have a failure rate above the unacceptable rate during the unreliable
period but haven't run during the reliable period are marked as reliable.
"""
# Unreliable period is 2 days: 2017-06-03 to 2017-06-04.
# Reliable period is 1 day: 2016-06-04.
reliable_period_date = datetime.date(2017, 6, 4)
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1),
unreliable_time_period=datetime.timedelta(days=2))
tests = ["jstests/core/all.js"]
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
# The test did not run on the reliable period on linux-64.
report = test_failures.Report([
# Failing.
self.ENTRY._replace(num_pass=0, num_fail=2),
# Passing on a different variant.
self.ENTRY._replace(start_date=reliable_period_date,
end_date=reliable_period_date,
num_pass=3,
num_fail=0,
variant="linux-alt",
distro="debian7"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
# The tags for variant and distro have been removed.
self.assertEqual(
updated_tags,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable", "unreliable|jsCore_WT"])
]))
def test_non_running_at_all_is_reliable(self):
"""
Tests that tests that are tagged as unreliable but no longer running (either during the
reliable or the unreliable period) have their tags removed.
"""
config = self.CONFIG
tests = ["jstests/core/all.js", "jstests/core/all2.js"]
initial_tags = collections.OrderedDict([
("jstests/core/all2.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
# all2.js did not run at all
report = test_failures.Report([self.ENTRY])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
# The tags for variant and distro have been removed.
self.assertEqual(updated_tags, collections.OrderedDict([]))
def test_transition_test_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test,) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant) combination
after it has become reliable again.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant, distro)
combination after it has become reliable again.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags multiple formerly unreliable combination after it has become
reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_remain_reliable(self):
"""
Tests that update_tags() preserves the absence of tags for reliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_remain_unreliable(self):
"""
Tests that update_tags() preserves the tags for unreliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_reliable_min_runs(self):
"""
Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9),
reliable_min_runs=100)
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_reliable_time_period(self):
"""
Tests that update_tags() ignores passes from before 'reliable_time_period'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=1)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=1)),
num_pass=1,
num_fail=0),
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=2)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=2)),
num_pass=1,
num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(
updated_tags,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_unreliable_min_runs(self):
"""
Tests that update_tags() only considers a test unreliable if it has more than
'unreliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1),
unreliable_min_runs=100)
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_unreliable_time_period(self):
"""
Tests that update_tags() ignores failures from before 'unreliable_time_period'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=1)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=1)),
num_pass=0,
num_fail=1),
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=2)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=2)),
num_pass=0,
num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
class TestReportEntry(unittest.TestCase):
"""
Tests for the test_failures.ReportEntry class.
"""
ENTRY = test_failures.ReportEntry(test="jstests/core/all.js",
task="jsCore_WT",
variant="linux-64",
distro="rhel62",
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=0,
num_fail=0)
def test_fail_rate(self):
"""
Tests for the test_failures.ReportEntry.fail_rate property.
"""
entry = self.ENTRY._replace(num_pass=0, num_fail=1)
self.assertEqual(1, entry.fail_rate)
entry = self.ENTRY._replace(num_pass=9, num_fail=1)
self.assertAlmostEqual(0.1, entry.fail_rate)
# Verify that we don't attempt to divide by zero.
entry = self.ENTRY._replace(num_pass=0, num_fail=0)
self.assertEqual(0, entry.fail_rate)
def test_week_start_date_with_sunday(self):
"""
Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week
specified as different forms of the string "Sunday".
"""
entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))
self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("sunday"))
self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("Sunday"))
self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("SUNDAY"))
entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 4))
self.assertEqual(datetime.date(2017, 6, 4), entry.week_start_date("sunday"))
entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 5))
self.assertEqual(datetime.date(2017, 6, 4), entry.week_start_date("sunday"))
def test_week_start_date_with_monday(self):
"""
Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week
specified as different forms of the string "Monday".
"""
entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))
self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("monday"))
self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("Monday"))
self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("MONDAY"))
entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 4))
self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("monday"))
entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 5))
self.assertEqual(datetime.date(2017, 6, 5), entry.week_start_date("monday"))
entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 6))
self.assertEqual(datetime.date(2017, 6, 5), entry.week_start_date("monday"))
def test_week_start_date_with_date(self):
"""
Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week
specified as a datetime.date() value.
"""
entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))
date = datetime.date(2017, 5, 21)
self.assertEqual(6, date.weekday(), "2017 May 21 is a Sunday")
self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date(date))
date = datetime.date(2017, 5, 22)
self.assertEqual(0, date.weekday(), "2017 May 22 is a Monday")
self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date(date))
date = datetime.date(2017, 6, 6)
self.assertEqual(1, date.weekday(), "2017 Jun 06 is a Tuesday")
self.assertEqual(datetime.date(2017, 5, 30), entry.week_start_date(date))
date = datetime.date(2017, 6, 9)
self.assertEqual(4, date.weekday(), "2017 Jun 09 is a Friday")
self.assertEqual(datetime.date(2017, 6, 2), entry.week_start_date(date))
date = datetime.date(2017, 6, 3)
self.assertEqual(5, date.weekday(), "2017 Jun 03 is a Saturday")
self.assertEqual(datetime.date(2017, 6, 3), entry.week_start_date(date))
def test_sum_combines_test_results(self):
"""
Tests for test_failures.ReportEntry.sum() that verify the start_date, end_date, num_pass,
and num_fail attributes are accumulated correctly.
"""
entry1 = self.ENTRY._replace(start_date=datetime.date(2017, 6, 1),
end_date=datetime.date(2017, 6, 1),
num_pass=1,
num_fail=0)
entry2 = self.ENTRY._replace(start_date=datetime.date(2017, 6, 2),
end_date=datetime.date(2017, 6, 2),
num_pass=0,
num_fail=3)
entry3 = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=0,
num_fail=0)
entry4 = self.ENTRY._replace(start_date=datetime.date(2017, 6, 4),
end_date=datetime.date(2017, 6, 4),
num_pass=2,
num_fail=2)
entry_1234 = test_failures.ReportEntry.sum([entry1, entry2, entry3, entry4])
entry_1432 = test_failures.ReportEntry.sum([entry1, entry4, entry3, entry2])
entry_124 = test_failures.ReportEntry.sum([entry1, entry2, entry4])
entry_13 = test_failures.ReportEntry.sum([entry1, entry3])
entry_42 = test_failures.ReportEntry.sum([entry4, entry2])
self.assertEqual(datetime.date(2017, 6, 1), entry_1234.start_date)
self.assertEqual(datetime.date(2017, 6, 4), entry_1234.end_date)
self.assertEqual(3, entry_1234.num_pass)
self.assertEqual(5, entry_1234.num_fail)
self.assertEqual(entry_1234, entry_1432, "order of arguments shouldn't matter")
self.assertEqual(entry_1234, entry_124, "entry3 didn't have any test executions")
self.assertEqual(datetime.date(2017, 6, 1), entry_13.start_date)
self.assertEqual(datetime.date(2017, 6, 3), entry_13.end_date)
self.assertEqual(1, entry_13.num_pass)
self.assertEqual(0, entry_13.num_fail)
self.assertEqual(datetime.date(2017, 6, 2), entry_42.start_date)
self.assertEqual(datetime.date(2017, 6, 4), entry_42.end_date)
self.assertEqual(2, entry_42.num_pass)
self.assertEqual(5, entry_42.num_fail)
def test_sum_combines_test_info(self):
"""
Tests for test_failures.ReportEntry.sum() that verify the test, task, variant, and distro
attributes are accumulated correctly.
"""
entry1 = self.ENTRY._replace(test="jstests/core/all.js",
task="jsCore_WT",
variant="linux-64",
distro="rhel62")
entry2 = self.ENTRY._replace(test="jstests/core/all.js",
task="jsCore_WT",
variant="linux-64",
distro="rhel55")
entry3 = self.ENTRY._replace(test="jstests/core/all2.js",
task="jsCore_WT",
variant="linux-64-debug",
distro="rhel62")
entry4 = self.ENTRY._replace(test="jstests/core/all.js",
task="jsCore",
variant="linux-64-debug",
distro="rhel62")
entry_12 = test_failures.ReportEntry.sum([entry1, entry2])
self.assertEqual("jstests/core/all.js", entry_12.test)
self.assertEqual("jsCore_WT", entry_12.task)
self.assertEqual("linux-64", entry_12.variant)
self.assertIsInstance(entry_12.distro, test_failures.Wildcard)
entry_123 = test_failures.ReportEntry.sum([entry1, entry2, entry3])
self.assertIsInstance(entry_123.test, test_failures.Wildcard)
self.assertEqual("jsCore_WT", entry_123.task)
self.assertIsInstance(entry_123.variant, test_failures.Wildcard)
self.assertIsInstance(entry_123.distro, test_failures.Wildcard)
entry_1234 = test_failures.ReportEntry.sum([entry1, entry2, entry3, entry4])
self.assertIsInstance(entry_1234.test, test_failures.Wildcard)
self.assertIsInstance(entry_1234.task, test_failures.Wildcard)
self.assertIsInstance(entry_1234.variant, test_failures.Wildcard)
self.assertIsInstance(entry_1234.distro, test_failures.Wildcard)
entry_34 = test_failures.ReportEntry.sum([entry3, entry4])
self.assertIsInstance(entry_34.test, test_failures.Wildcard)
self.assertIsInstance(entry_34.task, test_failures.Wildcard)
self.assertEqual("linux-64-debug", entry_34.variant)
self.assertEqual("rhel62", entry_34.distro)
class TestReportSummarization(unittest.TestCase):
"""
Tests for test_failures.Report.summarize_by().
"""
ENTRY = test_failures.ReportEntry(test="jstests/core/all.js",
task="jsCore_WT",
variant="linux-64",
distro="rhel62",
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=0,
num_fail=0)
ENTRIES = [
ENTRY._replace(start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=1,
num_fail=0),
ENTRY._replace(task="jsCore",
start_date=datetime.date(2017, 6, 5),
end_date=datetime.date(2017, 6, 5),
num_pass=0,
num_fail=1),
ENTRY._replace(start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0),
# The following entry is intentionally not in timestamp order to verify that the
# 'time_period' parameter becomes part of the sort in summarize_by().
ENTRY._replace(start_date=datetime.date(2017, 6, 9),
end_date=datetime.date(2017, 6, 9),
num_pass=1,
num_fail=0),
ENTRY._replace(distro="rhel55",
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=0,
num_fail=1),
ENTRY._replace(test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0),
ENTRY._replace(variant="linux-64-debug",
start_date=datetime.date(2017, 6, 17),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1),
]
def test_group_all_by_test_task_variant_distro(self):
"""
Tests that summarize_by() correctly accumulates all unique combinations of
(test, task, variant, distro).
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST_TASK_VARIANT_DISTRO)
self.assertEqual(5, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
task="jsCore",
start_date=datetime.date(2017, 6, 5),
end_date=datetime.date(2017, 6, 5),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
distro="rhel55",
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 10),
num_pass=3,
num_fail=0,
))
self.assertEqual(summed_entries[3], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 17),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[4], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0,
))
def test_group_all_by_test_task_variant(self):
"""
Tests that summarize_by() correctly accumulates all unique combinations of
(test, task, variant).
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST_TASK_VARIANT)
self.assertEqual(4, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
task="jsCore",
start_date=datetime.date(2017, 6, 5),
end_date=datetime.date(2017, 6, 5),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 10),
num_pass=3,
num_fail=1,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 17),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[3], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0,
))
def test_group_all_by_test_task(self):
"""
Tests that summarize_by() correctly accumulates all unique combinations of (test, task).
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST_TASK)
self.assertEqual(3, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
task="jsCore",
start_date=datetime.date(2017, 6, 5),
end_date=datetime.date(2017, 6, 5),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
variant=test_failures.Wildcard("variants"),
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 17),
num_pass=3,
num_fail=2,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0,
))
def test_group_all_by_test(self):
"""
Tests that summarize_by() correctly accumulates all unique combinations of (test,).
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST)
self.assertEqual(2, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
task=test_failures.Wildcard("tasks"),
variant=test_failures.Wildcard("variants"),
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 17),
num_pass=3,
num_fail=3,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0,
))
def test_group_all_by_variant_task(self):
"""
Tests that summarize_by() correctly accumulates all unique combinations of (variant, task).
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(["variant", "task"])
self.assertEqual(3, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
task="jsCore",
start_date=datetime.date(2017, 6, 5),
end_date=datetime.date(2017, 6, 5),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
test=test_failures.Wildcard("tests"),
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 10),
num_pass=4,
num_fail=1,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 17),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
def test_group_weekly_by_test_starting_on_sunday(self):
"""
Tests that summarize_by() correctly accumulates by week when the beginning of the week is
specified as the string "sunday".
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST,
time_period=test_failures.Report.WEEKLY,
start_day_of_week=test_failures.Report.SUNDAY)
self.assertEqual(4, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=1,
num_fail=0,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
task=test_failures.Wildcard("tasks"),
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 4),
end_date=datetime.date(2017, 6, 10),
num_pass=2,
num_fail=2,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 11),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[3], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 4),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0,
))
def test_group_weekly_by_test_starting_on_monday(self):
"""
Tests that summarize_by() correctly accumulates by week when the beginning of the week is
specified as the string "monday".
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST,
time_period=test_failures.Report.WEEKLY,
start_day_of_week=test_failures.Report.MONDAY)
self.assertEqual(4, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 4),
num_pass=1,
num_fail=0,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
task=test_failures.Wildcard("tasks"),
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 5),
end_date=datetime.date(2017, 6, 11),
num_pass=2,
num_fail=2,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 12),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[3], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 5),
end_date=datetime.date(2017, 6, 11),
num_pass=1,
num_fail=0,
))
def test_group_weekly_by_test_starting_on_date(self):
"""
Tests that summarize_by() correctly accumulates by week when the beginning of the week is
specified as a datetime.date() value.
"""
date = datetime.date(2017, 6, 7)
self.assertEqual(2, date.weekday(), "2017 Jun 07 is a Wednesday")
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST,
time_period=test_failures.Report.WEEKLY,
start_day_of_week=date)
self.assertEqual(4, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
task=test_failures.Wildcard("tasks"),
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 6),
num_pass=1,
num_fail=1,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 7),
end_date=datetime.date(2017, 6, 13),
num_pass=2,
num_fail=1,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 14),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[3], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 7),
end_date=datetime.date(2017, 6, 13),
num_pass=1,
num_fail=0,
))
def test_group_daily_by_test(self):
"""
Tests that summarize_by() correctly accumulates by day.
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST,
time_period=test_failures.Report.DAILY)
self.assertEqual(6, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=1,
num_fail=0,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
task="jsCore",
start_date=datetime.date(2017, 6, 5),
end_date=datetime.date(2017, 6, 5),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
start_date=datetime.date(2017, 6, 9),
end_date=datetime.date(2017, 6, 9),
num_pass=1,
num_fail=0,
))
self.assertEqual(summed_entries[3], self.ENTRY._replace(
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=1,
))
self.assertEqual(summed_entries[4], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 17),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[5], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 10),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0,
))
def test_group_4days_by_test(self):
"""
Tests that summarize_by() correctly accumulates by multiple days.
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST,
time_period=datetime.timedelta(days=4))
self.assertEqual(4, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
task=test_failures.Wildcard("tasks"),
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 6),
num_pass=1,
num_fail=1,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 7),
end_date=datetime.date(2017, 6, 10),
num_pass=2,
num_fail=1,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 15),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[3], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 7),
end_date=datetime.date(2017, 6, 10),
num_pass=1,
num_fail=0,
))
def test_group_9days_by_test(self):
"""
Tests that summarize_by() correctly accumulates by multiple days, including time periods
greater than 1 week.
"""
report = test_failures.Report(self.ENTRIES)
summed_entries = report.summarize_by(test_failures.Report.TEST,
time_period=datetime.timedelta(days=9))
self.assertEqual(3, len(summed_entries))
self.assertEqual(summed_entries[0], self.ENTRY._replace(
task=test_failures.Wildcard("tasks"),
distro=test_failures.Wildcard("distros"),
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 11),
num_pass=3,
num_fail=2,
))
self.assertEqual(summed_entries[1], self.ENTRY._replace(
variant="linux-64-debug",
start_date=datetime.date(2017, 6, 12),
end_date=datetime.date(2017, 6, 17),
num_pass=0,
num_fail=1,
))
self.assertEqual(summed_entries[2], self.ENTRY._replace(
test="jstests/core/all2.js",
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 11),
num_pass=1,
num_fail=0,
))
class TestUpdateTags(unittest.TestCase):
"""
Tests for the update_tags() function.
"""
CONFIG = update_test_lifecycle.Config(
test_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
task_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
variant_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
distro_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
reliable_min_runs=2,
reliable_time_period=datetime.timedelta(days=1),
unreliable_min_runs=2,
unreliable_time_period=datetime.timedelta(days=1))
ENTRY = test_failures.ReportEntry(test="jstests/core/all.js",
task="jsCore_WT",
variant="linux-64",
distro="rhel62",
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=0,
num_fail=0)
def assert_has_only_js_tests(self, lifecycle):
"""
Raises an AssertionError exception if 'lifecycle' is not of the following form:
selector:
js_test:
...
"""
self.assertIn("selector", lifecycle.raw)
self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw))
self.assertIn("js_test", lifecycle.raw["selector"])
self.assertEqual(1,
len(lifecycle.raw["selector"]),
msg=str(lifecycle.raw))
return lifecycle.raw["selector"]["js_test"]
def transition_from_reliable_to_unreliable(self, config, expected_tags):
"""
Tests that update_tags() tags a formerly reliable combination as being unreliable.
"""
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(collections.OrderedDict(),
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, expected_tags)
def test_transition_test_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task) combination as being
unreliable.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being
unreliable.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination
as being unreliable.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags multiple formerly reliable combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def transition_from_unreliable_to_reliable(self, config, initial_tags):
"""
Tests that update_tags() untags a formerly unreliable combination after it has become
reliable again.
"""
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
def test_transition_test_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test,) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant) combination
after it has become reliable again.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant, distro)
combination after it has become reliable again.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags multiple formerly unreliable combination after it has become
reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_remain_reliable(self):
"""
Tests that update_tags() preserves the absence of tags for reliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_remain_unreliable(self):
"""
Tests that update_tags() preserves the tags for unreliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_reliable_min_runs(self):
"""
Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9),
reliable_min_runs=100)
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_reliable_time_period(self):
"""
Tests that update_tags() ignores passes from before 'reliable_time_period'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=1)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=1)),
num_pass=1,
num_fail=0),
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=2)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=2)),
num_pass=1,
num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(
updated_tags,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_unreliable_min_runs(self):
"""
Tests that update_tags() only considers a test unreliable if it has more than
'unreliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1),
unreliable_min_runs=100)
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_unreliable_time_period(self):
"""
Tests that update_tags() ignores failures from before 'unreliable_time_period'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=1)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=1)),
num_pass=0,
num_fail=1),
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=2)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=2)),
num_pass=0,
num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
