def testProcessTest_CreatesAnAnomaly_RefMovesToo_BenchmarkDuration(self): testing_common.AddTests(['ChromiumGPU'], ['linux-release'], { 'foo': { 'benchmark_duration': { 'ref': {} } }, }) ref = utils.TestKey( 'ChromiumGPU/linux-release/foo/benchmark_duration/ref').get() non_ref = utils.TestKey( 'ChromiumGPU/linux-release/foo/benchmark_duration').get() test_container_key = utils.GetTestContainerKey(ref.key) test_container_key_non_ref = utils.GetTestContainerKey(non_ref.key) for row in _TEST_ROW_DATA: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() graph_data.Row(id=row[0], value=row[1], parent=test_container_key_non_ref).put() ref.UpdateSheriff() ref.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([ref.key]) self.assertEqual(m.call_args_list, [mock.call(ref.key.id())]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies))
def testProcessTest_CreatesAnAnomaly_RefMovesToo_BenchmarkDuration(self): testing_common.AddTests(['ChromiumGPU'], ['linux-release'], { 'foo': { 'benchmark_duration': { 'ref': {} } }, }) ref = utils.TestKey( 'ChromiumGPU/linux-release/foo/benchmark_duration/ref').get() non_ref = utils.TestKey( 'ChromiumGPU/linux-release/foo/benchmark_duration').get() test_container_key = utils.GetTestContainerKey(ref.key) test_container_key_non_ref = utils.GetTestContainerKey(non_ref.key) for row in _TEST_ROW_DATA: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() graph_data.Row(id=row[0], value=row[1], parent=test_container_key_non_ref).put() sheriff.Sheriff(email='*****@*****.**', id='sheriff', patterns=[ref.test_path]).put() ref.UpdateSheriff() ref.put() find_anomalies.ProcessTests([ref.key]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies))
def testProcessTest_SkipsLastAlert_NotAvg(self, mock_process_stat): self._AddDataForTests(stats=('count', )) test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() a = anomaly.Anomaly(test=test.key, start_revision=10061, end_revision=10062, statistic='count') a.put() test.UpdateSheriff() test.put() @ndb.tasklet def _AssertParams(config, test_entity, stat, rows, ref_rows): del config del test_entity del stat del ref_rows assert rows[0][0] < a.end_revision mock_process_stat.side_effect = _AssertParams with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))): find_anomalies.ProcessTests([test.key]) self.ExecuteDeferredTasks('default')
def testProcessTest_LastAlertedRevisionTooHigh_PropertyReset( self, mock_logging_error): # If the last_alerted_revision property of the TestMetadata is too high, # then the property should be reset and an error should be logged. self._AddDataForTests() test = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() sheriff.Sheriff(email='*****@*****.**', id='sheriff', patterns=[test.test_path]).put() test.last_alerted_revision = 1234567890 test.UpdateSheriff() test.put() find_anomalies.ProcessTests([test.key]) self.assertIsNone(test.key.get().last_alerted_revision) calls = [ mock.call( 'last_alerted_revision %d is higher than highest rev %d for test ' '%s; setting last_alerted_revision to None.', 1234567890, 10066, 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), mock.call('No rows fetched for %s', 'ChromiumGPU/linux-release/scrolling_benchmark/ref') ] mock_logging_error.assert_has_calls(calls, any_order=True)
def testProcessTest_CreatesAnAnomaly(self): testing_common.AddTests(['ChromiumGPU'], ['linux-release'], { 'scrolling_benchmark': { 'ref': {} }, }) ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() test_container_key = utils.GetTestContainerKey(ref.key) for row in _TEST_ROW_DATA: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() sheriff.Sheriff(email='*****@*****.**', id='sheriff', patterns=[ref.test_path]).put() ref.UpdateSheriff() ref.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([ref.key]) self.assertEqual(m.call_args_list, [mock.call(ref.key.id())]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(241536, new_anomalies[0].start_revision) self.assertEqual(241537, new_anomalies[0].end_revision)
def testProcessTest_UsesLastAlert_Avg(self, mock_process_stat): mock_process_stat.side_effect = _MockTasklet self._AddDataForTests() test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() a = anomaly.Anomaly(test=test.key, start_revision=10061, end_revision=10062, statistic='avg') a.put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))): find_anomalies.ProcessTests([test.key]) self.ExecuteDeferredTasks('default') query = graph_data.Row.query( projection=['revision', 'timestamp', 'value']) query = query.filter(graph_data.Row.revision > 10062) query = query.filter( graph_data.Row.parent_test == utils.OldStyleTestKey(test.key)) row_data = query.fetch() rows = [(r.revision, r, r.value) for r in row_data] mock_process_stat.assert_called_with(mock.ANY, mock.ANY, mock.ANY, rows, None) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 1)
def testProcessTest_AnomaliesMatchRefSeries_NoAlertCreated(self): # Tests that a Anomaly entity is not created if both the test and its # corresponding ref build series have the same data. testing_common.AddTests( ['ChromiumGPU'], ['linux-release'], { 'scrolling_benchmark': {'ref': {}}, }) ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() non_ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark').get() test_container_key = utils.GetTestContainerKey(ref.key) test_container_key_non_ref = utils.GetTestContainerKey(non_ref.key) for row in _TEST_ROW_DATA: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() graph_data.Row(id=row[0], value=row[1], parent=test_container_key_non_ref).put() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[non_ref.test_path]).put() ref.UpdateSheriff() ref.put() non_ref.UpdateSheriff() non_ref.put() find_anomalies.ProcessTests([non_ref.key]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(0, len(new_anomalies))
def testProcessTest_AnomalyDoesNotMatchRefSeries_AlertCreated(self): # Tests that an Anomaly entity is created when non-ref series goes up, but # the ref series stays flat. testing_common.AddTests(['ChromiumGPU'], ['linux-release'], { 'scrolling_benchmark': { 'ref': {} }, }) ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() non_ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark').get() test_container_key = utils.GetTestContainerKey(ref.key) test_container_key_non_ref = utils.GetTestContainerKey(non_ref.key) for row in _TEST_ROW_DATA: graph_data.Row(id=row[0], value=2125.375, parent=test_container_key).put() graph_data.Row(id=row[0], value=row[1], parent=test_container_key_non_ref).put() ref.UpdateSheriff() ref.put() non_ref.UpdateSheriff() non_ref.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([non_ref.key]) self.assertEqual(m.call_args_list, [mock.call(non_ref.key.id())]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(new_anomalies), 1)
def testProcessTest_SkipsLastAlert_NotAvg(self, mock_process_stat): self._AddDataForTests(stats=('count',)) test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() a = anomaly.Anomaly( test=test.key, start_revision=10061, end_revision=10062, statistic='count') a.put() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[test_path]).put() test.UpdateSheriff() test.put() @ndb.tasklet def _AssertParams( config, sheriff_entity, test_entity, stat, rows, ref_rows): del config del sheriff_entity del test_entity del stat del ref_rows assert rows[0][0] < a.end_revision mock_process_stat.side_effect = _AssertParams find_anomalies.ProcessTests([test.key]) self.ExecuteDeferredTasks('default')
def testProcessTest_InternalOnlyTest(self, mock_email_sheriff): self._AddDataForTests() test = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() test.internal_only = True sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[test.test_path]).put() test.put() find_anomalies.ProcessTests([test.key]) expected_calls = [ mock.call(ModelMatcher('sheriff'), ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10011))] self.assertEqual(expected_calls, mock_email_sheriff.call_args_list) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 1) self.assertEqual(test.key, anomalies[0].test) self.assertEqual(100, anomalies[0].percent_changed) self.assertEqual(anomaly.UP, anomalies[0].direction) self.assertEqual(10007, anomalies[0].start_revision) self.assertEqual(10011, anomalies[0].end_revision) self.assertTrue(anomalies[0].internal_only)
def testProcessTest_AnomalyDoesNotMatchRefSeries_AlertCreated(self): # Tests that an Anomaly entity is created when non-ref series goes up, but # the ref series stays flat. testing_common.AddTests( ['ChromiumGPU'], ['linux-release'], { 'scrolling_benchmark': {'ref': {}}, }) ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() non_ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark').get() test_container_key = utils.GetTestContainerKey(ref.key) test_container_key_non_ref = utils.GetTestContainerKey(non_ref.key) for row in _TEST_ROW_DATA: graph_data.Row(id=row[0], value=2125.375, parent=test_container_key).put() graph_data.Row(id=row[0], value=row[1], parent=test_container_key_non_ref).put() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[ref.test_path]).put() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[non_ref.test_path]).put() ref.put() non_ref.put() find_anomalies.ProcessTests([non_ref.key]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(new_anomalies), 1)
def testProcessTest_InternalOnlyTest(self, mock_email_sheriff): self._AddDataForTests() test = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() test.internal_only = True test.UpdateSheriff() test.put() s = Subscription(name='sheriff', visibility=VISIBILITY.PUBLIC) with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([s], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.key.id())]) self.ExecuteDeferredTasks('default') expected_calls = [ mock.call([ModelMatcher('sheriff')], ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10011)) ] self.assertEqual(expected_calls, mock_email_sheriff.call_args_list) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 1) self.assertEqual(test.key, anomalies[0].test) self.assertEqual(100, anomalies[0].percent_changed) self.assertEqual(anomaly.UP, anomalies[0].direction) self.assertEqual(10007, anomalies[0].start_revision) self.assertEqual(10011, anomalies[0].end_revision) self.assertTrue(anomalies[0].internal_only)
def testProcessTest_AnomaliesMatchRefSeries_NoAlertCreated(self): # Tests that a Anomaly entity is not created if both the test and its # corresponding ref build series have the same data. testing_common.AddTests(['ChromiumGPU'], ['linux-release'], { 'scrolling_benchmark': { 'ref': {} }, }) ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() non_ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark').get() test_container_key = utils.GetTestContainerKey(ref.key) test_container_key_non_ref = utils.GetTestContainerKey(non_ref.key) for row in _TEST_ROW_DATA: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() graph_data.Row(id=row[0], value=row[1], parent=test_container_key_non_ref).put() ref.UpdateSheriff() ref.put() non_ref.UpdateSheriff() non_ref.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))): find_anomalies.ProcessTests([non_ref.key]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(0, len(new_anomalies))
def testProcessTest_NoSheriff_ErrorLogged(self, mock_logging_error): self._AddDataForTests() ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))): find_anomalies.ProcessTests([ref.key]) mock_logging_error.assert_called_with('No sheriff for %s', ref.key)
def testProcessTest_ImprovementMarkedAsImprovement(self): self._AddDataForTests() test = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[test.test_path]).put() test.improvement_direction = anomaly.DOWN test.put() find_anomalies.ProcessTests([test.key]) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 1) self.assertTrue(anomalies[0].is_improvement)
def testProcessTest_RefineAnomalyPlacement_MinSize0Max2Elements(self): testing_common.AddTests(['ChromiumPerf'], ['linux-perf'], {'sizes': { 'method_count': {} }}) test = utils.TestKey( ('ChromiumPerf/linux-perf/sizes/method_count')).get() test_container_key = utils.GetTestContainerKey(test.key) custom_config = { 'max_window_size': 10, 'min_absolute_change': 50, 'min_relative_change': 0, 'min_segment_size': 0, } anomaly_config.AnomalyConfig(config=custom_config, patterns=[test.test_path]).put() test.UpdateSheriff() test.put() self.assertEqual(custom_config, anomaly_config.GetAnomalyConfigDict(test)) sample_data = [ (6990, 100), (6991, 100), (6992, 100), (6993, 100), (6994, 100), (6995, 100), (6996, 100), (6997, 100), (6998, 100), (6999, 100), (7000, 100), (7001, 155), (7002, 155), (7003, 155), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() sheriff.Sheriff(email='*****@*****.**', id='sheriff', patterns=[test.test_path]).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(7001, new_anomalies[0].start_revision) self.assertEqual(7001, new_anomalies[0].end_revision)
def testProcessTest_FiltersOutImprovements(self, mock_email_sheriff): self._AddDataForTests() test = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[test.test_path]).put() test.improvement_direction = anomaly.UP test.put() find_anomalies.ProcessTests([test.key]) mock_email_sheriff.assert_called_once_with( ModelMatcher('sheriff'), ModelMatcher('ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10041))
def testProcessTest_ImprovementMarkedAsImprovement(self): self._AddDataForTests() test = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() test.improvement_direction = anomaly.DOWN test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.key.id())]) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 1) self.assertTrue(anomalies[0].is_improvement)
def testProcessTest_SkipsClankInternal(self, mock_process_stat): mock_process_stat.side_effect = _MockTasklet self._AddDataForTests(masters=['ClankInternal']) test_path = 'ClankInternal/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() a = anomaly.Anomaly( test=test.key, start_revision=10061, end_revision=10062, statistic='avg') a.put() find_anomalies.ProcessTests([test.key]) self.ExecuteDeferredTasks('default') self.assertFalse(mock_process_stat.called)
def testProcessTest_FiltersOutImprovements(self, mock_email_sheriff): self._AddDataForTests() test = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() test.improvement_direction = anomaly.UP test.UpdateSheriff() test.put() s = Subscription(name='sheriff', visibility=VISIBILITY.PUBLIC) with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([s], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.key.id())]) self.ExecuteDeferredTasks('default') mock_email_sheriff.assert_called_once_with( [ModelMatcher('sheriff')], ModelMatcher('ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10041))
def testProcessTest_CreatesAnAnomaly(self): testing_common.AddTests( ['ChromiumGPU'], ['linux-release'], { 'scrolling_benchmark': {'ref': {}}, }) ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() test_container_key = utils.GetTestContainerKey(ref.key) for row in _TEST_ROW_DATA: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[ref.test_path]).put() ref.put() find_anomalies.ProcessTests([ref.key]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(241536, new_anomalies[0].start_revision) self.assertEqual(241537, new_anomalies[0].end_revision)
def testProcessTest_SkipsClankInternal(self, mock_process_stat): mock_process_stat.side_effect = _MockTasklet self._AddDataForTests(masters=['ClankInternal']) test_path = 'ClankInternal/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() a = anomaly.Anomaly( test=test.key, start_revision=10061, end_revision=10062, statistic='avg') a.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, []) self.ExecuteDeferredTasks('default') self.assertFalse(mock_process_stat.called)
def testProcessTest_NoSheriff_ErrorLogged(self, mock_logging_error): self._AddDataForTests() ref = utils.TestKey( 'ChromiumGPU/linux-release/scrolling_benchmark/ref').get() find_anomalies.ProcessTests([ref.key]) mock_logging_error.assert_called_with('No sheriff for %s', ref.key)
def testProcessTest(self, mock_email_sheriff): self._AddDataForTests() test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[test_path]).put() test.put() find_anomalies.ProcessTests([test.key]) expected_calls = [ mock.call(ModelMatcher('sheriff'), ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10011)), mock.call(ModelMatcher('sheriff'), ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10041)), mock.call(ModelMatcher('sheriff'), ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10061))] self.assertEqual(expected_calls, mock_email_sheriff.call_args_list) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 3) def AnomalyExists( anomalies, test, percent_changed, direction, start_revision, end_revision, sheriff_name, internal_only, units, absolute_delta): for a in anomalies: if (a.test == test and a.percent_changed == percent_changed and a.direction == direction and a.start_revision == start_revision and a.end_revision == end_revision and a.sheriff.string_id() == sheriff_name and a.internal_only == internal_only and a.units == units and a.absolute_delta == absolute_delta): return True return False self.assertTrue( AnomalyExists( anomalies, test.key, percent_changed=100, direction=anomaly.UP, start_revision=10007, end_revision=10011, sheriff_name='sheriff', internal_only=False, units='ms', absolute_delta=50)) self.assertTrue( AnomalyExists( anomalies, test.key, percent_changed=-50, direction=anomaly.DOWN, start_revision=10037, end_revision=10041, sheriff_name='sheriff', internal_only=False, units='ms', absolute_delta=-100)) self.assertTrue( AnomalyExists( anomalies, test.key, percent_changed=sys.float_info.max, direction=anomaly.UP, start_revision=10057, end_revision=10061, sheriff_name='sheriff', internal_only=False, units='ms', absolute_delta=100)) # This is here just to verify that AnomalyExists returns False sometimes. self.assertFalse( AnomalyExists( anomalies, test.key, percent_changed=100, direction=anomaly.DOWN, start_revision=10037, end_revision=10041, sheriff_name='sheriff', internal_only=False, units='ms', absolute_delta=500))
def testProcessTest__RefineAnomalyPlacement_BalancedEstimator2(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (793468, 136.5382), (793486, 137.7192), (793495, 137.4038), (793504, 137.4919), (793505, 137.4465), (793518, 136.9279), (793525, 137.3501), (793528, 136.9622), (793543, 137.1027), (793550, 137.7351), (793555, 137.1511), (793559, 137.2094), (793560, 136.5192), (793565, 138.1536), (793580, 137.4172), (793590, 136.8746), (793601, 137.5016), (793609, 137.0773), (793625, 137.4702), (793646, 135.9019), (793657, 137.2827), (793702, 136.5978), (793712, 136.0732), (793721, 132.1820), (793742, 122.1631), (793760, 136.3152), (793774, 136.9616), (793788, 136.8438), (794016, 136.3022), (794024, 136.3495), (794027, 136.3145), (794036, 136.5502), (794043, 136.3861), (794051, 136.2035), (794059, 136.2348), (794066, 136.2594), (794074, 135.9686), (794088, 136.7375), (794107, 136.5570), (794132, 129.9924), # This one is a potential change point - but weak (794143, 135.8275), (794154, 107.2502), # This is a better change point (794158, 108.3948), (794160, 107.3564), (794196, 107.9707), (794236, 111.3168), (794268, 108.7905), (794281, 111.1065), (794319, 109.7699), (794320, 109.8082), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.DOWN, new_anomalies[0].direction) self.assertEqual(794144, new_anomalies[0].start_revision) self.assertEqual(794154, new_anomalies[0].end_revision)
def testProcessTest__RefineAnomalyPlacement_BalancedEstimator1(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (818289, 2009771), (818290, 1966080), (818291, 1966080), (818293, 1966080), (818294, 2053461), (818296, 2009771), (818298, 1966080), (818301, 2009771), (818303, 2009771), (818305, 2009771), (818306, 2009771), (818307, 1966080), (818308, 2009771), (818309, 2009771), (818310, 1966080), (818311, 2009771), (818312, 1966080), (818317, 1966080), (818318, 1966080), (818320, 2053461), (818322, 2009771), (818326, 1966080), (818331, 1966080), (818335, 1966080), (818340, 2009771), (818347, 2009771), (818350, 1966080), (818353, 1966080), (818354, 2009771), (818361, 2009771), (818362, 1966080), (818374, 2009771), (818379, 2009771), (818382, 2053461), (818389, 2009771), (818402, 1966080), (818409, 2009771), (818416, 1966080), (818420, 1966080), (818430, 2009771), (818440, 2228224), (818450, 2228224), (818461, 2228224), (818469, 2228224), (818481, 2228224), (818498, 2271915), (818514, 2228224), (818531, 2271915), (818571, 2271915), (818583, 2271915), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(818431, new_anomalies[0].start_revision) self.assertEqual(818440, new_anomalies[0].end_revision)
def testProcessTest_MultipleChangePoints(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (804863, 13830765), (804867, 16667862), (804879, 13929296), (804891, 13823876), (804896, 13908794), (804900, 13899281), (804907, 14901462), (804921, 13890597), (804935, 13969113), (804946, 13996520), (804957, 13913104), (805143, 16770364), (805175, 14858529), (805179, 14013942), (805185, 14857516), (805195, 14895168), (805196, 14944037), (805205, 13919484), (805211, 15736581), (805231, 14730142), (805236, 13892102), (805247, 14808876), (805253, 14903648), (805262, 13896626), (805276, 15797878), (805281, 14542593), (805285, 15733168), (805290, 13882841), (805302, 15727394), (805314, 15758058), (805333, 16074960), (805345, 16142162), (805359, 16138912), (805384, 17914289), (805412, 18368834), (805428, 18055197), (805457, 19673614), (805482, 19705606), (805502, 19609089), (805509, 19576745), (805531, 19600059), (805550, 19702969), (805564, 19660953), (805584, 19830273), (805600, 19800662), (805606, 19493150), (805620, 19700545), (805624, 19623731), (805628, 19683921), (805634, 19660001), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(2, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(805429, new_anomalies[0].start_revision) self.assertEqual(805457, new_anomalies[0].end_revision) self.assertEqual(805315, new_anomalies[1].start_revision) self.assertEqual(805428, new_anomalies[1].end_revision)
def testProcessTest_RefineAnomalyPlacement_OffByOneStable(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], { 'memory.desktop': { ('memory:chrome:all_processes:' 'reported_by_chrome:v8:effective_size_avg'): {} } }) test = utils.TestKey(( 'ChromiumPerf/linux-perf/memory.desktop/' 'memory:chrome:all_processes:reported_by_chrome:v8:effective_size_avg' )).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (733480, 1381203.0), (733494, 1381220.0), (733504, 1381212.0), (733524, 1381220.0), (733538, 1381211.0), (733544, 1381212.0), (733549, 1381220.0), (733563, 1381220.0), (733581, 1381220.0), (733597, 1381212.0), (733611, 1381228.0), (733641, 1381212.0), (733675, 1381204.0), (733721, 1381212.0), (733766, 1381211.0), (733804, 1381204.0), (733835, 1381219.0), (733865, 1381211.0), (733885, 1381219.0), (733908, 1381204.0), (733920, 1381211.0), (733937, 1381220.0), (734091, 1381211.0), (734133, 1381219.0), (734181, 1381204.0), (734211, 1381720.0), (734248, 1381712.0), (734277, 1381696.0), (734311, 1381704.0), (734341, 1381703.0), (734372, 1381704.0), (734405, 1381703.0), (734431, 1381711.0), (734456, 1381720.0), (734487, 1381703.0), (734521, 1381704.0), (734554, 1381726.0), (734598, 1381704.0), (734630, 1381703.0), # In crbug/1041688 this is where it was placed. (734673, 1529888.0), # This is where it should be. (734705, 1529888.0), (734739, 1529860.0), (734770, 1529860.0), (734793, 1529888.0), (734829, 1529860.0), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(734631, new_anomalies[0].start_revision) self.assertEqual(734673, new_anomalies[0].end_revision)
def testProcessTest_RefineAnomalyPlacement_OffByOneBefore(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (728446, 480.2504), (728462, 487.685), (728469, 486.6389), (728480, 477.6597), (728492, 471.2238), (728512, 480.4379), (728539, 464.5573), (728594, 489.0594), (728644, 484.4796), (728714, 489.5986), (728751, 489.474), (728788, 481.9336), (728835, 484.089), (728869, 485.4287), (728883, 476.8234), (728907, 487.4736), (728938, 490.601), (728986, 483.5039), (729021, 485.176), (729066, 484.5855), (729105, 483.9114), (729119, 483.559), (729161, 477.6875), (729201, 484.9668), (729240, 480.7091), (729270, 484.5506), (729292, 495.1445), (729309, 479.9111), (729329, 479.8815), (729391, 487.5683), (729430, 476.7355), (729478, 487.7251), (729525, 493.1012), (729568, 497.7565), (729608, 499.6481), (729642, 496.1591), (729658, 493.4581), (729687, 486.1097), (729706, 478.036), (729730, 480.4222), # In crbug/1041688 this was the original placement. (729764, 421.0342), # We instead should be setting it here. (729795, 428.0284), (729846, 433.8261), (729883, 429.49), (729920, 436.3342), (729975, 434.3996), (730011, 428.3672), (730054, 436.309), (730094, 435.3792), (730128, 433.0537), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.DOWN, new_anomalies[0].direction) self.assertEqual(729731, new_anomalies[0].start_revision) self.assertEqual(729764, new_anomalies[0].end_revision)
def testProcessTest(self, mock_email_sheriff): self._AddDataForTests() test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() test.UpdateSheriff() test.put() alert_group_key1 = alert_group.AlertGroup( name='scrolling_benchmark', subscription_name='sheriff1', status=alert_group.AlertGroup.Status.untriaged, active=True, revision=alert_group.RevisionRange(repository='chromium', start=10000, end=10070), ).put() alert_group_key2 = alert_group.AlertGroup( name='scrolling_benchmark', subscription_name='sheriff2', status=alert_group.AlertGroup.Status.untriaged, active=True, revision=alert_group.RevisionRange(repository='chromium', start=10000, end=10070), ).put() s1 = Subscription(name='sheriff1', visibility=VISIBILITY.PUBLIC) s2 = Subscription(name='sheriff2', visibility=VISIBILITY.PUBLIC) with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([s1, s2], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.key.id())]) self.ExecuteDeferredTasks('default') expected_calls = [ mock.call([ModelMatcher('sheriff1'), ModelMatcher('sheriff2')], ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10011)), mock.call([ModelMatcher('sheriff1'), ModelMatcher('sheriff2')], ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10041)), mock.call([ModelMatcher('sheriff1'), ModelMatcher('sheriff2')], ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10061)) ] self.assertEqual(expected_calls, mock_email_sheriff.call_args_list) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 3) for a in anomalies: self.assertEqual(a.groups, [alert_group_key1, alert_group_key2]) def AnomalyExists(anomalies, test, percent_changed, direction, start_revision, end_revision, subscription_names, internal_only, units, absolute_delta, statistic): for a in anomalies: if (a.test == test and a.percent_changed == percent_changed and a.direction == direction and a.start_revision == start_revision and a.end_revision == end_revision and a.subscription_names == subscription_names and a.internal_only == internal_only and a.units == units and a.absolute_delta == absolute_delta and a.statistic == statistic): return True return False self.assertTrue( AnomalyExists(anomalies, test.key, percent_changed=100, direction=anomaly.UP, start_revision=10007, end_revision=10011, subscription_names=['sheriff1', 'sheriff2'], internal_only=False, units='ms', absolute_delta=50, statistic='avg')) self.assertTrue( AnomalyExists(anomalies, test.key, percent_changed=-50, direction=anomaly.DOWN, start_revision=10037, end_revision=10041, subscription_names=['sheriff1', 'sheriff2'], internal_only=False, units='ms', absolute_delta=-100, statistic='avg')) self.assertTrue( AnomalyExists(anomalies, test.key, percent_changed=sys.float_info.max, direction=anomaly.UP, start_revision=10057, end_revision=10061, internal_only=False, units='ms', subscription_names=['sheriff1', 'sheriff2'], absolute_delta=100, statistic='avg')) # This is here just to verify that AnomalyExists returns False sometimes. self.assertFalse( AnomalyExists(anomalies, test.key, percent_changed=100, direction=anomaly.DOWN, start_revision=10037, end_revision=10041, subscription_names=['sheriff1', 'sheriff2'], internal_only=False, units='ms', absolute_delta=500, statistic='avg'))