class TestUpdateTags(unittest.TestCase):
"""
Tests for the update_tags() function.
"""
CONFIG = update_test_lifecycle.Config(
test_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
task_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
variant_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
distro_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
reliable_min_runs=2,
reliable_time_period=datetime.timedelta(days=1),
unreliable_min_runs=2,
unreliable_time_period=datetime.timedelta(days=1))
ENTRY = test_failures.ReportEntry(test="jstests/core/all.js",
task="jsCore_WT",
variant="linux-64",
distro="rhel62",
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=0,
num_fail=0)
def assert_has_only_js_tests(self, lifecycle):
"""
Raises an AssertionError exception if 'lifecycle' is not of the following form:
selector:
js_test:
...
"""
self.assertIn("selector", lifecycle.raw)
self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw))
self.assertIn("js_test", lifecycle.raw["selector"])
self.assertEqual(1,
len(lifecycle.raw["selector"]),
msg=str(lifecycle.raw))
return lifecycle.raw["selector"]["js_test"]
def transition_from_reliable_to_unreliable(self, config, expected_tags):
"""
Tests that update_tags() tags a formerly reliable combination as being unreliable.
"""
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(collections.OrderedDict(),
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, expected_tags)
def test_transition_test_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task) combination as being
unreliable.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being
unreliable.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination
as being unreliable.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags multiple formerly reliable combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def transition_from_unreliable_to_reliable(self, config, initial_tags):
"""
Tests that update_tags() untags a formerly unreliable combination after it has become
reliable again.
"""
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
def test_non_running_in_reliable_period_is_reliable(self):
"""
Tests that tests that have a failure rate above the unacceptable rate during the unreliable
period but haven't run during the reliable period are marked as reliable.
"""
# Unreliable period is 2 days: 2017-06-03 to 2017-06-04.
# Reliable period is 1 day: 2016-06-04.
reliable_period_date = datetime.date(2017, 6, 4)
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1),
unreliable_time_period=datetime.timedelta(days=2))
tests = ["jstests/core/all.js"]
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
# The test did not run on the reliable period on linux-64.
report = test_failures.Report([
# Failing.
self.ENTRY._replace(num_pass=0, num_fail=2),
# Passing on a different variant.
self.ENTRY._replace(start_date=reliable_period_date,
end_date=reliable_period_date,
num_pass=3,
num_fail=0,
variant="linux-alt",
distro="debian7"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
# The tags for variant and distro have been removed.
self.assertEqual(
updated_tags,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable", "unreliable|jsCore_WT"])
]))
def test_non_running_at_all_is_reliable(self):
"""
Tests that tests that are tagged as unreliable but no longer running (either during the
reliable or the unreliable period) have their tags removed.
"""
config = self.CONFIG
tests = ["jstests/core/all.js", "jstests/core/all2.js"]
initial_tags = collections.OrderedDict([
("jstests/core/all2.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
# all2.js did not run at all
report = test_failures.Report([self.ENTRY])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
# The tags for variant and distro have been removed.
self.assertEqual(updated_tags, collections.OrderedDict([]))
def test_transition_test_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test,) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant) combination
after it has become reliable again.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant, distro)
combination after it has become reliable again.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags multiple formerly unreliable combination after it has become
reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_remain_reliable(self):
"""
Tests that update_tags() preserves the absence of tags for reliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_remain_unreliable(self):
"""
Tests that update_tags() preserves the tags for unreliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_reliable_min_runs(self):
"""
Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9),
reliable_min_runs=100)
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_reliable_time_period(self):
"""
Tests that update_tags() ignores passes from before 'reliable_time_period'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=1)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=1)),
num_pass=1,
num_fail=0),
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=2)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=2)),
num_pass=1,
num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(
updated_tags,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_unreliable_min_runs(self):
"""
Tests that update_tags() only considers a test unreliable if it has more than
'unreliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1),
unreliable_min_runs=100)
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_unreliable_time_period(self):
"""
Tests that update_tags() ignores failures from before 'unreliable_time_period'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
tests = ["jstests/core/all.js"]
report = test_failures.Report([
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=1)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=1)),
num_pass=0,
num_fail=1),
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=2)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=2)),
num_pass=0,
num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report,
tests)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
class TestUpdateTags(unittest.TestCase):
"""
Tests for the update_tags() function.
"""
CONFIG = update_test_lifecycle.Config(
test_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
task_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
variant_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
distro_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
reliable_min_runs=2,
reliable_time_period=datetime.timedelta(days=1),
unreliable_min_runs=2,
unreliable_time_period=datetime.timedelta(days=1))
ENTRY = test_failures.ReportEntry(test="jstests/core/all.js",
task="jsCore_WT",
variant="linux-64",
distro="rhel62",
start_date=datetime.date(2017, 6, 3),
end_date=datetime.date(2017, 6, 3),
num_pass=0,
num_fail=0)
def assert_has_only_js_tests(self, lifecycle):
"""
Raises an AssertionError exception if 'lifecycle' is not of the following form:
selector:
js_test:
...
"""
self.assertIn("selector", lifecycle.raw)
self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw))
self.assertIn("js_test", lifecycle.raw["selector"])
self.assertEqual(1,
len(lifecycle.raw["selector"]),
msg=str(lifecycle.raw))
return lifecycle.raw["selector"]["js_test"]
def transition_from_reliable_to_unreliable(self, config, expected_tags):
"""
Tests that update_tags() tags a formerly reliable combination as being unreliable.
"""
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(collections.OrderedDict(),
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, expected_tags)
def test_transition_test_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task) combination as being
unreliable.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being
unreliable.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination
as being unreliable.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags multiple formerly reliable combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
self.transition_from_reliable_to_unreliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def transition_from_unreliable_to_reliable(self, config, initial_tags):
"""
Tests that update_tags() untags a formerly unreliable combination after it has become
reliable again.
"""
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
def test_transition_test_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test,) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant) combination
after it has become reliable again.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant, distro)
combination after it has become reliable again.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags multiple formerly unreliable combination after it has become
reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_remain_reliable(self):
"""
Tests that update_tags() preserves the absence of tags for reliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_remain_unreliable(self):
"""
Tests that update_tags() preserves the tags for unreliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_reliable_min_runs(self):
"""
Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9),
reliable_min_runs=100)
self.transition_from_unreliable_to_reliable(
config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_reliable_time_period(self):
"""
Tests that update_tags() ignores passes from before 'reliable_time_period'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=1)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=1)),
num_pass=1,
num_fail=0),
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=2)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=2)),
num_pass=1,
num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(
updated_tags,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_unreliable_min_runs(self):
"""
Tests that update_tags() only considers a test unreliable if it has more than
'unreliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1),
unreliable_min_runs=100)
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(num_pass=0, num_fail=1),
self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),
self.ENTRY._replace(num_pass=0,
num_fail=1,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_unreliable_time_period(self):
"""
Tests that update_tags() ignores failures from before 'unreliable_time_period'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(
lifecycle)
self.assertEqual(initial_tags,
self.assert_has_only_js_tests(lifecycle))
report = test_failures.Report([
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=1)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=1)),
num_pass=0,
num_fail=1),
self.ENTRY._replace(start_date=(self.ENTRY.start_date -
datetime.timedelta(days=2)),
end_date=(self.ENTRY.end_date -
datetime.timedelta(days=2)),
num_pass=0,
num_fail=1),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0),
self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),
self.ENTRY._replace(num_pass=1,
num_fail=0,
variant="linux-64-debug"),
self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),
])
update_test_lifecycle.validate_config(config)
update_test_lifecycle.update_tags(summary_lifecycle, config, report)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
class TestValidateConfig(unittest.TestCase):
"""
Tests for the validate_config() function.
"""
CONFIG = update_test_lifecycle.Config(
test_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
task_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
variant_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
distro_fail_rates=update_test_lifecycle.Rates(acceptable=0,
unacceptable=1),
reliable_min_runs=2,
reliable_time_period=datetime.timedelta(days=1),
unreliable_min_runs=2,
unreliable_time_period=datetime.timedelta(days=1))
def test_acceptable_test_fail_rate(self):
"""
Tests the validation of the 'test_fail_rates.acceptable' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable="not a number"))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=2))
update_test_lifecycle.validate_config(config)
def test_unacceptable_test_fail_rate(self):
"""
Tests the validation of the 'test_fail_rates.unacceptable' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable="not a number"))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
unacceptable=2))
update_test_lifecycle.validate_config(config)
def test_test_fail_rates(self):
"""
Tests the validation of the 'test_fail_rates' attribute.
"""
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(
acceptable=0.9, unacceptable=0.1))
update_test_lifecycle.validate_config(config)
def test_acceptable_task_fail_rate(self):
"""
Tests the validation of the 'test_fail_rates.acceptable' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable="not a number"))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=2))
update_test_lifecycle.validate_config(config)
def test_unacceptable_task_fail_rate(self):
"""
Tests the validation of the 'task_fail_rates.unacceptable' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable="not a number"))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
unacceptable=2))
update_test_lifecycle.validate_config(config)
def test_task_fail_rates(self):
"""
Tests the validation of the 'task_fail_rates' attribute.
"""
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(
acceptable=0.9, unacceptable=0.1))
update_test_lifecycle.validate_config(config)
def test_acceptable_variant_fail_rate(self):
"""
Tests the validation of the 'variant_fail_rates.acceptable' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable="not a number"))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=2))
update_test_lifecycle.validate_config(config)
def test_unacceptable_variant_fail_rate(self):
"""
Tests the validation of the 'variant_fail_rates.unacceptable' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable="not a number"))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
unacceptable=2))
update_test_lifecycle.validate_config(config)
def test_variant_fail_rates(self):
"""
Tests the validation of the 'variant_fail_rates' attribute.
"""
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
acceptable=0.9, unacceptable=0.1))
update_test_lifecycle.validate_config(config)
def test_acceptable_distro_fail_rate(self):
"""
Tests the validation of the 'distro_fail_rates.acceptable' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable="not a number"))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=2))
update_test_lifecycle.validate_config(config)
def test_unacceptable_distro_fail_rate(self):
"""
Tests the validation of the 'distro_fail_rates.unacceptable' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable="not a number"))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
unacceptable=2))
update_test_lifecycle.validate_config(config)
def test_distro_fail_rates(self):
"""
Tests the validation of the 'distro_fail_rates' attribute.
"""
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
acceptable=0.9, unacceptable=0.1))
update_test_lifecycle.validate_config(config)
def test_reliable_min_runs(self):
"""
Tests the validation of the 'reliable_min_runs' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(reliable_min_runs="not a number")
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(reliable_min_runs=-1)
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(reliable_min_runs=0)
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(reliable_min_runs=1.5)
update_test_lifecycle.validate_config(config)
def test_reliable_time_period(self):
"""
Tests the validation of the 'reliable_time_period' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
reliable_time_period="not a datetime.timedelta")
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
reliable_time_period=datetime.timedelta(days=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
reliable_time_period=datetime.timedelta(days=0))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
reliable_time_period=datetime.timedelta(days=1, hours=1))
update_test_lifecycle.validate_config(config)
def test_unreliable_min_runs(self):
"""
Tests the validation of the 'unreliable_min_runs' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(unreliable_min_runs="not a number")
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(unreliable_min_runs=-1)
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(unreliable_min_runs=0)
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(unreliable_min_runs=1.5)
update_test_lifecycle.validate_config(config)
def test_unreliable_time_period(self):
"""
Tests the validation of the 'unreliable_time_period' attribute.
"""
with self.assertRaises(TypeError):
config = self.CONFIG._replace(
unreliable_time_period="not a datetime.timedelta")
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
unreliable_time_period=datetime.timedelta(days=-1))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
unreliable_time_period=datetime.timedelta(days=0))
update_test_lifecycle.validate_config(config)
with self.assertRaises(ValueError):
config = self.CONFIG._replace(
unreliable_time_period=datetime.timedelta(days=1, hours=1))
update_test_lifecycle.validate_config(config)
class TestUpdateTags(unittest.TestCase): # pylint: disable=too-many-public-methods
"""
Tests for the update_tags() function.
"""
# yapf: disable
CONFIG = utl.Config(
test_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
task_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
variant_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
distro_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
reliable_min_runs=2,
reliable_time_period=datetime.timedelta(days=1),
unreliable_min_runs=2,
unreliable_time_period=datetime.timedelta(days=1))
# yapf: enable
STATS = {
"test": "jstests/core/all.js", "task": "jsCore_WT", "variant": "linux-64",
"distro": "rhel62", "num_pass": 0, "num_fail": 0
}
@staticmethod
def _stats( # pylint: disable=too-many-arguments
test="jstests/core/all.js", task="jsCore_WT", variant="linux-64", distro="rhel62",
num_pass=0, num_fail=0):
return {
"test_file": test,
"task_name": task,
"variant": variant,
"distro": distro,
"num_pass": num_pass,
"num_fail": num_fail,
}
def assert_has_only_js_tests(self, lifecycle):
"""
Raises an AssertionError exception if 'lifecycle' is not of the following form:
selector:
js_test:
...
"""
self.assertIn("selector", lifecycle.raw)
self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw))
self.assertIn("js_test", lifecycle.raw["selector"])
self.assertEqual(1, len(lifecycle.raw["selector"]), msg=str(lifecycle.raw))
return lifecycle.raw["selector"]["js_test"]
def transition_from_reliable_to_unreliable(self, config, expected_tags):
"""
Tests that update_tags() tags a formerly reliable combination as being unreliable.
"""
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(collections.OrderedDict(), self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
stats = [
self._stats(num_pass=0, num_fail=1),
self._stats(num_pass=0, num_fail=1, task="jsCore"),
self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"),
self._stats(num_pass=1, num_fail=0),
self._stats(num_pass=0, num_fail=1, distro="rhel55"),
]
test_history.add_reliable_period_stats(stats)
test_history.add_unreliable_period_stats(stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, expected_tags)
def test_transition_test_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1))
self.transition_from_reliable_to_unreliable(config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task) combination as being
unreliable.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1))
self.transition_from_reliable_to_unreliable(config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being
unreliable.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1))
self.transition_from_reliable_to_unreliable(config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination
as being unreliable.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))
self.transition_from_reliable_to_unreliable(config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_reliable_to_unreliable(self):
"""
Tests that update_tags() tags multiple formerly reliable combination as being unreliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))
self.transition_from_reliable_to_unreliable(config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def transition_from_unreliable_to_reliable(self, config, initial_tags):
"""
Tests that update_tags() untags a formerly unreliable combination after it has become
reliable again.
"""
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
stats = [
self._stats(num_pass=1, num_fail=0),
self._stats(num_pass=1, num_fail=0, task="jsCore"),
self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"),
self._stats(num_pass=0, num_fail=1),
self._stats(num_pass=1, num_fail=0, distro="rhel55"),
]
test_history.add_reliable_period_stats(stats)
test_history.add_unreliable_period_stats(stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, collections.OrderedDict())
def test_non_running_in_reliable_period_is_reliable(self):
"""
Tests that tests that have a failure rate above the unacceptable rate during the unreliable
period but haven't run during the reliable period are marked as reliable.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),
unreliable_time_period=datetime.timedelta(days=2))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
# The test did not run on the reliable period on linux-64.
reliable_period_stats = [
self._stats(num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"),
]
unreliable_period_stats = [
# Failing.
self._stats(num_pass=0, num_fail=2),
self._stats(num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"),
]
test_history.add_reliable_period_stats(reliable_period_stats)
test_history.add_unreliable_period_stats(unreliable_period_stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
# The tags for variant and distro have been removed.
self.assertEqual(updated_tags,
collections.OrderedDict([("jstests/core/all.js",
["unreliable", "unreliable|jsCore_WT"])]))
def test_non_running_at_all_is_reliable(self):
"""
Tests that tests that are tagged as unreliable but no longer running (either during the
reliable or the unreliable period) have their tags removed.
"""
config = self.CONFIG
test_history = utl.TestHistory("jstests/core/all.js")
test_history2 = utl.TestHistory("jstests/core/all2.js")
initial_tags = collections.OrderedDict([
("jstests/core/all2.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
# all2.js did not run at all
test_history.add_reliable_period_stats([self._stats()])
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
utl.update_tags(summary_lifecycle, config, test_history2)
updated_tags = self.assert_has_only_js_tests(lifecycle)
# The tags for variant and distro have been removed.
self.assertEqual(updated_tags, collections.OrderedDict([]))
def test_transition_test_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test,) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9))
self.transition_from_unreliable_to_reliable(config,
collections.OrderedDict([
("jstests/core/all.js", ["unreliable"]),
]))
def test_transition_task_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task) combination after it has
become reliable again.
"""
config = self.CONFIG._replace(
task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9))
self.transition_from_unreliable_to_reliable(config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT"]),
]))
def test_transition_variant_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant) combination
after it has become reliable again.
"""
config = self.CONFIG._replace(
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9))
self.transition_from_unreliable_to_reliable(config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64"]),
]))
def test_transition_distro_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags a formerly unreliable (test, task, variant, distro)
combination after it has become reliable again.
"""
config = self.CONFIG._replace(
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))
self.transition_from_unreliable_to_reliable(config,
collections.OrderedDict([
("jstests/core/all.js",
["unreliable|jsCore_WT|linux-64|rhel62"]),
]))
def test_transition_from_unreliable_to_reliable(self):
"""
Tests that update_tags() untags multiple formerly unreliable combination after it has become
reliable again.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))
self.transition_from_unreliable_to_reliable(config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_remain_reliable(self):
"""
Tests that update_tags() preserves the absence of tags for reliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
stats = [
self._stats(num_pass=1, num_fail=0),
self._stats(num_pass=1, num_fail=0, task="jsCore"),
self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"),
self._stats(num_pass=0, num_fail=1),
self._stats(num_pass=1, num_fail=0, distro="rhel55"),
]
test_history.add_reliable_period_stats(stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_remain_unreliable(self):
"""
Tests that update_tags() preserves the tags for unreliable combinations.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))
initial_tags = collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
])
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
stats = [
self._stats(num_pass=0, num_fail=1),
self._stats(num_pass=0, num_fail=1, task="jsCore"),
self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"),
self._stats(num_pass=1, num_fail=0),
self._stats(num_pass=0, num_fail=1, distro="rhel55"),
]
test_history.add_reliable_period_stats(stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)
def test_obeys_reliable_min_runs(self):
"""
Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),
task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9),
reliable_min_runs=100)
self.transition_from_unreliable_to_reliable(config,
collections.OrderedDict([
("jstests/core/all.js", [
"unreliable",
"unreliable|jsCore_WT",
"unreliable|jsCore_WT|linux-64",
"unreliable|jsCore_WT|linux-64|rhel62",
]),
]))
def test_obeys_unreliable_min_runs(self):
"""
Tests that update_tags() only considers a test unreliable if it has more than
'unreliable_min_runs'.
"""
config = self.CONFIG._replace(
test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),
unreliable_min_runs=100)
initial_tags = collections.OrderedDict()
lifecycle = ci_tags.TagsConfig.from_dict(
dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
test_history = utl.TestHistory("jstests/core/all.js")
stats = [
self._stats(num_pass=0, num_fail=1),
self._stats(num_pass=0, num_fail=1, task="jsCore"),
self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"),
self._stats(num_pass=1, num_fail=0),
self._stats(num_pass=0, num_fail=1, distro="rhel55"),
]
test_history.add_reliable_period_stats(stats)
test_history.add_unreliable_period_stats(stats)
utl.validate_config(config)
utl.update_tags(summary_lifecycle, config, test_history)
updated_tags = self.assert_has_only_js_tests(lifecycle)
self.assertEqual(updated_tags, initial_tags)