def transition_from_reliable_to_unreliable(self, config, expected_tags): """ Tests that update_tags() tags a formerly reliable combination as being unreliable. """ initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) self.assertEqual(collections.OrderedDict(), self.assert_has_only_js_tests(lifecycle)) report = test_failures.Report([ self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(lifecycle, config, report) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, expected_tags)
def transition_from_unreliable_to_reliable(self, config, initial_tags): """ Tests that update_tags() untags a formerly unreliable combination after it has become reliable again. """ lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog( lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) tests = ["jstests/core/all.js"] report = test_failures.Report([ self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"), self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"), self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, collections.OrderedDict())
def test_obeys_reliable_time_period(self): """ Tests that update_tags() ignores passes from before 'reliable_time_period'. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace( acceptable=0.9), task_fail_rates=self.CONFIG.task_fail_rates._replace( acceptable=0.9), variant_fail_rates=self.CONFIG.variant_fail_rates._replace( acceptable=0.9), distro_fail_rates=self.CONFIG.distro_fail_rates._replace( acceptable=0.9)) initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog( lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) tests = ["jstests/core/all.js"] report = test_failures.Report([ self.ENTRY._replace(start_date=(self.ENTRY.start_date - datetime.timedelta(days=1)), end_date=(self.ENTRY.end_date - datetime.timedelta(days=1)), num_pass=1, num_fail=0), self.ENTRY._replace(start_date=(self.ENTRY.start_date - datetime.timedelta(days=2)), end_date=(self.ENTRY.end_date - datetime.timedelta(days=2)), num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual( updated_tags, collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]))
def test_obeys_unreliable_min_runs(self): """ Tests that update_tags() only considers a test unreliable if it has more than 'unreliable_min_runs'. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1), unreliable_min_runs=100) initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) tests = ["jstests/core/all.js"] report = test_failures.Report([ self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def test_remain_reliable(self): """ Tests that update_tags() preserves the absence of tags for reliable combinations. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9), task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9)) initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) test_history = utl.TestHistory("jstests/core/all.js") stats = [ self._stats(num_pass=1, num_fail=0), self._stats(num_pass=1, num_fail=0, task="jsCore"), self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"), self._stats(num_pass=0, num_fail=1), self._stats(num_pass=1, num_fail=0, distro="rhel55"), ] test_history.add_reliable_period_stats(stats) utl.validate_config(config) utl.update_tags(summary_lifecycle, config, test_history) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def test_non_running_at_all_is_reliable(self): """ Tests that tests that are tagged as unreliable but no longer running (either during the reliable or the unreliable period) have their tags removed. """ config = self.CONFIG tests = ["jstests/core/all.js", "jstests/core/all2.js"] initial_tags = collections.OrderedDict([ ("jstests/core/all2.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog( lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) # all2.js did not run at all report = test_failures.Report([self.ENTRY]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) # The tags for variant and distro have been removed. self.assertEqual(updated_tags, collections.OrderedDict([]))
def test_remain_reliable(self): """ Tests that update_tags() preserves the absence of tags for reliable combinations. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9), task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9)) initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) tests = ["jstests/core/all.js"] report = test_failures.Report([ self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"), self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"), self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def test_non_running_at_all_is_reliable(self): """ Tests that tests that are tagged as unreliable but no longer running (either during the reliable or the unreliable period) have their tags removed. """ config = self.CONFIG test_history = utl.TestHistory("jstests/core/all.js") test_history2 = utl.TestHistory("jstests/core/all2.js") initial_tags = collections.OrderedDict([ ("jstests/core/all2.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) # all2.js did not run at all test_history.add_reliable_period_stats([self._stats()]) utl.validate_config(config) utl.update_tags(summary_lifecycle, config, test_history) utl.update_tags(summary_lifecycle, config, test_history2) updated_tags = self.assert_has_only_js_tests(lifecycle) # The tags for variant and distro have been removed. self.assertEqual(updated_tags, collections.OrderedDict([]))
def transition_from_unreliable_to_reliable(self, config, initial_tags): """ Tests that update_tags() untags a formerly unreliable combination after it has become reliable again. """ lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) test_history = utl.TestHistory("jstests/core/all.js") stats = [ self._stats(num_pass=1, num_fail=0), self._stats(num_pass=1, num_fail=0, task="jsCore"), self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"), self._stats(num_pass=0, num_fail=1), self._stats(num_pass=1, num_fail=0, distro="rhel55"), ] test_history.add_reliable_period_stats(stats) test_history.add_unreliable_period_stats(stats) utl.validate_config(config) utl.update_tags(summary_lifecycle, config, test_history) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, collections.OrderedDict())
def test_remain_reliable(self): """ Tests that update_tags() preserves the absence of tags for reliable combinations. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9), task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9)) initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) report = test_failures.Report([ self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"), self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"), self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(lifecycle, config, report) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def test_obeys_unreliable_min_runs(self): """ Tests that update_tags() only considers a test unreliable if it has more than 'unreliable_min_runs'. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1), unreliable_min_runs=100) initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) report = test_failures.Report([ self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(lifecycle, config, report) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def transition_from_unreliable_to_reliable(self, config, initial_tags): """ Tests that update_tags() untags a formerly unreliable combination after it has become reliable again. """ lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) test_history = utl.TestHistory("jstests/core/all.js") stats = [ self._stats(num_pass=1, num_fail=0), self._stats(num_pass=1, num_fail=0, task="jsCore"), self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"), self._stats(num_pass=0, num_fail=1), self._stats(num_pass=1, num_fail=0, distro="rhel55"), ] test_history.add_reliable_period_stats(stats) test_history.add_unreliable_period_stats(stats) utl.validate_config(config) utl.update_tags(summary_lifecycle, config, test_history) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, collections.OrderedDict())
def test_obeys_unreliable_min_runs(self): """ Tests that update_tags() only considers a test unreliable if it has more than 'unreliable_min_runs'. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1), unreliable_min_runs=100) initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) test_history = utl.TestHistory("jstests/core/all.js") stats = [ self._stats(num_pass=0, num_fail=1), self._stats(num_pass=0, num_fail=1, task="jsCore"), self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"), self._stats(num_pass=1, num_fail=0), self._stats(num_pass=0, num_fail=1, distro="rhel55"), ] test_history.add_reliable_period_stats(stats) test_history.add_unreliable_period_stats(stats) utl.validate_config(config) utl.update_tags(summary_lifecycle, config, test_history) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def test_non_running_in_reliable_period_is_reliable(self): """ Tests that tests that have a failure rate above the unacceptable rate during the unreliable period but haven't run during the reliable period are marked as reliable. """ # Unreliable period is 2 days: 2017-06-03 to 2017-06-04. # Reliable period is 1 day: 2016-06-04. reliable_period_date = datetime.date(2017, 6, 4) config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace( unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace( unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace( unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace( unacceptable=0.1), unreliable_time_period=datetime.timedelta(days=2)) tests = ["jstests/core/all.js"] initial_tags = collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog( lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) # The test did not run on the reliable period on linux-64. report = test_failures.Report([ # Failing. self.ENTRY._replace(num_pass=0, num_fail=2), # Passing on a different variant. self.ENTRY._replace(start_date=reliable_period_date, end_date=reliable_period_date, num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) # The tags for variant and distro have been removed. self.assertEqual( updated_tags, collections.OrderedDict([ ("jstests/core/all.js", ["unreliable", "unreliable|jsCore_WT"]) ]))
def test_non_running_in_reliable_period_is_reliable(self): """ Tests that tests that have a failure rate above the unacceptable rate during the unreliable period but haven't run during the reliable period are marked as reliable. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace( unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace( unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace( unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace( unacceptable=0.1), unreliable_time_period=datetime.timedelta(days=2)) initial_tags = collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) test_history = utl.TestHistory("jstests/core/all.js") # The test did not run on the reliable period on linux-64. reliable_period_stats = [ self._stats(num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"), ] unreliable_period_stats = [ # Failing. self._stats(num_pass=0, num_fail=2), self._stats(num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"), ] test_history.add_reliable_period_stats(reliable_period_stats) test_history.add_unreliable_period_stats(unreliable_period_stats) utl.validate_config(config) utl.update_tags(summary_lifecycle, config, test_history) updated_tags = self.assert_has_only_js_tests(lifecycle) # The tags for variant and distro have been removed. self.assertEqual( updated_tags, collections.OrderedDict([ ("jstests/core/all.js", ["unreliable", "unreliable|jsCore_WT"]) ]))
def test_test_fail_rates(self): """ Tests the validation of the 'test_fail_rates' attribute. """ with self.assertRaises(ValueError): config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9, unacceptable=0.1)) update_test_lifecycle.validate_config(config)
def test_distro_fail_rates(self): """ Tests the validation of the 'distro_fail_rates' attribute. """ with self.assertRaises(ValueError): config = self.CONFIG._replace( distro_fail_rates=self.CONFIG.distro_fail_rates._replace( acceptable=0.9, unacceptable=0.1)) utl.validate_config(config)
def test_test_fail_rates(self): """ Tests the validation of the 'test_fail_rates' attribute. """ with self.assertRaises(ValueError): config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace( acceptable=0.9, unacceptable=0.1)) update_test_lifecycle.validate_config(config)
def test_distro_fail_rates(self): """ Tests the validation of the 'distro_fail_rates' attribute. """ with self.assertRaises(ValueError): config = self.CONFIG._replace( distro_fail_rates=self.CONFIG.distro_fail_rates._replace( acceptable=0.9, unacceptable=0.1)) utl.validate_config(config)
def test_non_running_in_reliable_period_is_reliable(self): """ Tests that tests that have a failure rate above the unacceptable rate during the unreliable period but haven't run during the reliable period are marked as reliable. """ # Unreliable period is 2 days: 2017-06-03 to 2017-06-04. # Reliable period is 1 day: 2016-06-04. reliable_period_date = datetime.date(2017, 6, 4) config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1), unreliable_time_period=datetime.timedelta(days=2)) tests = ["jstests/core/all.js"] initial_tags = collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) # The test did not run on the reliable period on linux-64. report = test_failures.Report([ # Failing. self.ENTRY._replace(num_pass=0, num_fail=2), # Passing on a different variant. self.ENTRY._replace(start_date=reliable_period_date, end_date=reliable_period_date, num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) # The tags for variant and distro have been removed. self.assertEqual(updated_tags, collections.OrderedDict([ ("jstests/core/all.js", ["unreliable", "unreliable|jsCore_WT"])]))
def test_non_running_in_reliable_period_is_reliable(self): """ Tests that tests that have a failure rate above the unacceptable rate during the unreliable period but haven't run during the reliable period are marked as reliable. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1), unreliable_time_period=datetime.timedelta(days=2)) initial_tags = collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) test_history = utl.TestHistory("jstests/core/all.js") # The test did not run on the reliable period on linux-64. reliable_period_stats = [ self._stats(num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"), ] unreliable_period_stats = [ # Failing. self._stats(num_pass=0, num_fail=2), self._stats(num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"), ] test_history.add_reliable_period_stats(reliable_period_stats) test_history.add_unreliable_period_stats(unreliable_period_stats) utl.validate_config(config) utl.update_tags(summary_lifecycle, config, test_history) updated_tags = self.assert_has_only_js_tests(lifecycle) # The tags for variant and distro have been removed. self.assertEqual( updated_tags, collections.OrderedDict([("jstests/core/all.js", ["unreliable", "unreliable|jsCore_WT"])]))
def test_obeys_unreliable_time_period(self): """ Tests that update_tags() ignores failures from before 'unreliable_time_period'. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1)) initial_tags = collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) tests = ["jstests/core/all.js"] report = test_failures.Report([ self.ENTRY._replace( start_date=(self.ENTRY.start_date - datetime.timedelta(days=1)), end_date=(self.ENTRY.end_date - datetime.timedelta(days=1)), num_pass=0, num_fail=1), self.ENTRY._replace( start_date=(self.ENTRY.start_date - datetime.timedelta(days=2)), end_date=(self.ENTRY.end_date - datetime.timedelta(days=2)), num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"), self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"), self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, collections.OrderedDict())
def test_remain_unreliable(self): """ Tests that update_tags() preserves the tags for unreliable combinations. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace( unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace( unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace( unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace( unacceptable=0.1)) initial_tags = collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) test_history = utl.TestHistory("jstests/core/all.js") stats = [ self._stats(num_pass=0, num_fail=1), self._stats(num_pass=0, num_fail=1, task="jsCore"), self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"), self._stats(num_pass=1, num_fail=0), self._stats(num_pass=0, num_fail=1, distro="rhel55"), ] test_history.add_reliable_period_stats(stats) utl.validate_config(config) utl.update_tags(summary_lifecycle, config, test_history) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def test_remain_unreliable(self): """ Tests that update_tags() preserves the tags for unreliable combinations. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace( unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace( unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace( unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace( unacceptable=0.1)) initial_tags = collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) report = test_failures.Report([ self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(lifecycle, config, report) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def transition_from_unreliable_to_reliable(self, config, initial_tags): """ Tests that update_tags() untags a formerly unreliable combination after it has become reliable again. """ lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) report = test_failures.Report([ self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"), self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"), self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(lifecycle, config, report) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, collections.OrderedDict())
def test_remain_unreliable(self): """ Tests that update_tags() preserves the tags for unreliable combinations. """ config = self.CONFIG._replace( test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1)) initial_tags = collections.OrderedDict([ ("jstests/core/all.js", [ "unreliable", "unreliable|jsCore_WT", "unreliable|jsCore_WT|linux-64", "unreliable|jsCore_WT|linux-64|rhel62", ]), ]) lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) tests = ["jstests/core/all.js"] report = test_failures.Report([ self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, initial_tags)
def transition_from_reliable_to_unreliable(self, config, expected_tags): """ Tests that update_tags() tags a formerly reliable combination as being unreliable. """ initial_tags = collections.OrderedDict() lifecycle = ci_tags.TagsConfig.from_dict( dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) self.assertEqual(collections.OrderedDict(), self.assert_has_only_js_tests(lifecycle)) tests = ["jstests/core/all.js"] report = test_failures.Report([ self.ENTRY._replace(num_pass=0, num_fail=1), self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), self.ENTRY._replace(num_pass=1, num_fail=0), self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), ]) update_test_lifecycle.validate_config(config) update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) updated_tags = self.assert_has_only_js_tests(lifecycle) self.assertEqual(updated_tags, expected_tags)
def test_reliable_time_period(self): """ Tests the validation of the 'reliable_time_period' attribute. """ with self.assertRaises(TypeError): config = self.CONFIG._replace(reliable_time_period="not a datetime.timedelta") update_test_lifecycle.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=-1)) update_test_lifecycle.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=0)) update_test_lifecycle.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=1, hours=1)) update_test_lifecycle.validate_config(config)
def test_unreliable_min_runs(self): """ Tests the validation of the 'unreliable_min_runs' attribute. """ with self.assertRaises(TypeError): config = self.CONFIG._replace(unreliable_min_runs="not a number") update_test_lifecycle.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(unreliable_min_runs=-1) update_test_lifecycle.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(unreliable_min_runs=0) update_test_lifecycle.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(unreliable_min_runs=1.5) update_test_lifecycle.validate_config(config)
def test_reliable_time_period(self): """ Tests the validation of the 'reliable_time_period' attribute. """ with self.assertRaises(TypeError): config = self.CONFIG._replace(reliable_time_period="not a datetime.timedelta") utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=-1)) utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=0)) utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=1, hours=1)) utl.validate_config(config)
def test_unreliable_min_runs(self): """ Tests the validation of the 'unreliable_min_runs' attribute. """ with self.assertRaises(TypeError): config = self.CONFIG._replace(unreliable_min_runs="not a number") utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(unreliable_min_runs=-1) utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(unreliable_min_runs=0) utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace(unreliable_min_runs=1.5) utl.validate_config(config)
def test_acceptable_task_fail_rate(self): """ Tests the validation of the 'test_fail_rates.acceptable' attribute. """ with self.assertRaises(TypeError): config = self.CONFIG._replace( task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable="not a number")) update_test_lifecycle.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace( task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=-1)) update_test_lifecycle.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace( task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=2)) update_test_lifecycle.validate_config(config)
def test_acceptable_distro_fail_rate(self): """ Tests the validation of the 'distro_fail_rates.acceptable' attribute. """ with self.assertRaises(TypeError): config = self.CONFIG._replace( distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable="not a number")) utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace( distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=-1)) utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace( distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=2)) utl.validate_config(config)
def test_acceptable_distro_fail_rate(self): """ Tests the validation of the 'distro_fail_rates.acceptable' attribute. """ with self.assertRaises(TypeError): config = self.CONFIG._replace( distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable="not a number")) utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace( distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=-1)) utl.validate_config(config) with self.assertRaises(ValueError): config = self.CONFIG._replace( distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=2)) utl.validate_config(config)