def testPost_ExternalUserRequestsInternalOnlySheriff_ErrorMessage(self): sheriff.Sheriff(id='Foo', internal_only=True).put() self.assertFalse(utils.IsInternalUser()) response = self.testapp.post('/alerts?sheriff=Foo') error = self.GetJsonValue(response, 'error') self.assertIsNotNone(error)
def testProcessTest(self, mock_email_sheriff): self._AddDataForTests() test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() sheriff.Sheriff( email='*****@*****.**', id='sheriff', patterns=[test_path]).put() test.UpdateSheriff() test.put() find_anomalies.ProcessTests([test.key]) self.ExecuteDeferredTasks('default') expected_calls = [ mock.call(ModelMatcher('sheriff'), ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10011)), mock.call(ModelMatcher('sheriff'), ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10041)), mock.call(ModelMatcher('sheriff'), ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10061))] self.assertEqual(expected_calls, mock_email_sheriff.call_args_list) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 3) def AnomalyExists( anomalies, test, percent_changed, direction, start_revision, end_revision, sheriff_name, internal_only, units, absolute_delta, statistic): for a in anomalies: if (a.test == test and a.percent_changed == percent_changed and a.direction == direction and a.start_revision == start_revision and a.end_revision == end_revision and a.sheriff.string_id() == sheriff_name and a.internal_only == internal_only and a.units == units and a.absolute_delta == absolute_delta and a.statistic == statistic): return True return False self.assertTrue( AnomalyExists( anomalies, test.key, percent_changed=100, direction=anomaly.UP, start_revision=10007, end_revision=10011, sheriff_name='sheriff', internal_only=False, units='ms', absolute_delta=50, statistic='avg')) self.assertTrue( AnomalyExists( anomalies, test.key, percent_changed=-50, direction=anomaly.DOWN, start_revision=10037, end_revision=10041, sheriff_name='sheriff', internal_only=False, units='ms', absolute_delta=-100, statistic='avg')) self.assertTrue( AnomalyExists( anomalies, test.key, percent_changed=sys.float_info.max, direction=anomaly.UP, start_revision=10057, end_revision=10061, sheriff_name='sheriff', internal_only=False, units='ms', absolute_delta=100, statistic='avg')) # This is here just to verify that AnomalyExists returns False sometimes. self.assertFalse( AnomalyExists( anomalies, test.key, percent_changed=100, direction=anomaly.DOWN, start_revision=10037, end_revision=10041, sheriff_name='sheriff', internal_only=False, units='ms', absolute_delta=500, statistic='avg'))
def _AddSheriffs(self): sheriff1 = sheriff.Sheriff(id='Chromium Perf Sheriff', email='*****@*****.**').put() sheriff2 = sheriff.Sheriff(id='QA Perf Sheriff', email='*****@*****.**').put() return [sheriff1, sheriff2]
def testGet_WithNoAlerts_HasImageAndNoAlertsTable(self): sheriff.Sheriff(id='Chromium Perf Sheriff').put() response = self.testapp.get('/alerts') self.assertEqual(1, len(response.html('img'))) self.assertEqual(0, len(response.html('alerts-table')))
def testSheriffEmail_InternalOnly_OtherDomain_RaisesError(self): # If it's not within the set of allowed domains, an exception will be raised # while putting. entity = sheriff.Sheriff(internal_only=True, email='*****@*****.**') with self.assertRaises(sheriff.ValidationError): entity.put()
def _AddAlertsToDataStore(self): """Adds sample data, including triaged and non-triaged alerts.""" key_map = {} subscription = Subscription( name='Chromium Perf Sheriff', notification_email='*****@*****.**', ) # We still need sheriff information from here before sheriff-config # provide a way to fetch subsciber list. sheriff.Sheriff( id='Chromium Perf Sheriff', email='*****@*****.**').put() testing_common.AddTests(['ChromiumGPU'], ['linux-release'], { 'scrolling-benchmark': { 'first_paint': {}, 'first_paint_ref': {}, 'mean_frame_time': {}, 'mean_frame_time_ref': {}, } }) first_paint = utils.TestKey( 'ChromiumGPU/linux-release/scrolling-benchmark/first_paint') mean_frame_time = utils.TestKey( 'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time') # By default, all TestMetadata entities have an improvement_direction of # UNKNOWN, meaning that neither direction is considered an improvement. # Here we set the improvement direction so that some anomalies are # considered improvements. for test_key in [first_paint, mean_frame_time]: test = test_key.get() test.improvement_direction = anomaly.DOWN test.put() # Add some (12) non-triaged alerts. for end_rev in range(10000, 10120, 10): test_key = first_paint if end_rev % 20 == 0 else mean_frame_time ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key)) anomaly_entity = anomaly.Anomaly( start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=100, median_after_anomaly=200, ref_test=ref_test_key, subscriptions=[subscription], subscription_names=[subscription.name], ) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() key_map[end_rev] = anomaly_key.urlsafe() # Add some (2) already-triaged alerts. for end_rev in range(10120, 10140, 10): test_key = first_paint if end_rev % 20 == 0 else mean_frame_time ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key)) bug_id = -1 if end_rev % 20 == 0 else 12345 anomaly_entity = anomaly.Anomaly( start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=100, median_after_anomaly=200, ref_test=ref_test_key, bug_id=bug_id, subscriptions=[subscription], subscription_names=[subscription.name], ) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() key_map[end_rev] = anomaly_key.urlsafe() if bug_id > 0: bug_data.Bug(id=bug_id).put() # Add some (6) non-triaged improvements. for end_rev in range(10140, 10200, 10): test_key = mean_frame_time ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key)) anomaly_entity = anomaly.Anomaly( start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=200, median_after_anomaly=100, ref_test=ref_test_key, subscriptions=[subscription], subscription_names=[subscription.name], ) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() self.assertTrue(anomaly_entity.is_improvement) key_map[end_rev] = anomaly_key.urlsafe() return key_map
def testGet_WithAllOwnershipComponents(self): ownership_samples = [ { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': 'Abc>Xyz' }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', 'component': 'Def>123' } ] test_paths = ['ChromiumPerf/linux/scrolling/first_paint', 'ChromiumPerf/linux/scrolling/mean_frame_time'] test_keys = [utils.TestKey(test_path) for test_path in test_paths] sheriff_key = sheriff.Sheriff( id='Sheriff', labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put() anomaly_1 = anomaly.Anomaly( start_revision=1476193324, end_revision=1476201840, test=test_keys[0], median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[0]).put() anomaly_2 = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_keys[1], median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[1]).put() response = self.testapp.post( '/file_bug', [ ('keys', '%s' % (anomaly_1.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="checkbox" checked name="component" value="Abc>Xyz">', response.body) response_with_both_anomalies = self.testapp.post( '/file_bug', [ ('keys', '%s,%s' % (anomaly_1.urlsafe(), anomaly_2.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="checkbox" checked name="component" value="Abc>Xyz">', response_with_both_anomalies.body) self.assertIn( '<input type="checkbox" checked name="component" value="Def>123">', response_with_both_anomalies.body)
def testProcessTest(self, mock_email_sheriff): self._AddDataForTests() test_path = 'ChromiumGPU/linux-release/scrolling_benchmark/ref' test = utils.TestKey(test_path).get() sheriff.Sheriff(email='*****@*****.**', id='sheriff', patterns=[test_path]).put() test.UpdateSheriff() test.put() s1 = Subscription(name='sheriff1', visibility=VISIBILITY.PUBLIC) s2 = Subscription(name='sheriff2', visibility=VISIBILITY.PUBLIC) with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([s1, s2], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.key.id())]) self.ExecuteDeferredTasks('default') expected_calls = [ mock.call([ModelMatcher('sheriff1'), ModelMatcher('sheriff2')], ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10011)), mock.call([ModelMatcher('sheriff1'), ModelMatcher('sheriff2')], ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10041)), mock.call([ModelMatcher('sheriff1'), ModelMatcher('sheriff2')], ModelMatcher( 'ChromiumGPU/linux-release/scrolling_benchmark/ref'), EndRevisionMatcher(10061)) ] self.assertEqual(expected_calls, mock_email_sheriff.call_args_list) anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(len(anomalies), 3) def AnomalyExists(anomalies, test, percent_changed, direction, start_revision, end_revision, subscription_names, internal_only, units, absolute_delta, statistic): for a in anomalies: if (a.test == test and a.percent_changed == percent_changed and a.direction == direction and a.start_revision == start_revision and a.end_revision == end_revision and a.subscription_names == subscription_names and a.internal_only == internal_only and a.units == units and a.absolute_delta == absolute_delta and a.statistic == statistic): return True return False self.assertTrue( AnomalyExists(anomalies, test.key, percent_changed=100, direction=anomaly.UP, start_revision=10007, end_revision=10011, subscription_names=['sheriff1', 'sheriff2'], internal_only=False, units='ms', absolute_delta=50, statistic='avg')) self.assertTrue( AnomalyExists(anomalies, test.key, percent_changed=-50, direction=anomaly.DOWN, start_revision=10037, end_revision=10041, subscription_names=['sheriff1', 'sheriff2'], internal_only=False, units='ms', absolute_delta=-100, statistic='avg')) self.assertTrue( AnomalyExists(anomalies, test.key, percent_changed=sys.float_info.max, direction=anomaly.UP, start_revision=10057, end_revision=10061, internal_only=False, units='ms', subscription_names=['sheriff1', 'sheriff2'], absolute_delta=100, statistic='avg')) # This is here just to verify that AnomalyExists returns False sometimes. self.assertFalse( AnomalyExists(anomalies, test.key, percent_changed=100, direction=anomaly.DOWN, start_revision=10037, end_revision=10041, subscription_names=['sheriff1', 'sheriff2'], internal_only=False, units='ms', absolute_delta=500, statistic='avg'))
def testProcessTest_RefineAnomalyPlacement_OffByOneBefore(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], {'blink_perf.layout': { 'nested-percent-height-tables': {} }}) test = utils.TestKey( 'ChromiumPerf/linux-perf/blink_perf.layout/nested-percent-height-tables' ).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (728446, 480.2504), (728462, 487.685), (728469, 486.6389), (728480, 477.6597), (728492, 471.2238), (728512, 480.4379), (728539, 464.5573), (728594, 489.0594), (728644, 484.4796), (728714, 489.5986), (728751, 489.474), (728788, 481.9336), (728835, 484.089), (728869, 485.4287), (728883, 476.8234), (728907, 487.4736), (728938, 490.601), (728986, 483.5039), (729021, 485.176), (729066, 484.5855), (729105, 483.9114), (729119, 483.559), (729161, 477.6875), (729201, 484.9668), (729240, 480.7091), (729270, 484.5506), (729292, 495.1445), (729309, 479.9111), (729329, 479.8815), (729391, 487.5683), (729430, 476.7355), (729478, 487.7251), (729525, 493.1012), (729568, 497.7565), (729608, 499.6481), (729642, 496.1591), (729658, 493.4581), (729687, 486.1097), (729706, 478.036), (729730, 480.4222), # In crbug/1041688 this was the original placement. (729764, 421.0342), # We instead should be setting it here. (729795, 428.0284), (729846, 433.8261), (729883, 429.49), (729920, 436.3342), (729975, 434.3996), (730011, 428.3672), (730054, 436.309), (730094, 435.3792), (730128, 433.0537), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() sheriff.Sheriff(email='*****@*****.**', id='sheriff', patterns=[test.test_path]).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.DOWN, new_anomalies[0].direction) self.assertEqual(729731, new_anomalies[0].start_revision) self.assertEqual(729764, new_anomalies[0].end_revision)
def testGet_UsesFirstDefinedComponent(self): ownership_samples = [{ 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': '' }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': 'Abc>Def' }] now_datetime = datetime.datetime.now() test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint') sheriff_key = sheriff.Sheriff( id='Sheriff', labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put() alert_without_ownership = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, timestamp=now_datetime).put() alert_without_component = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[0], timestamp=now_datetime + datetime.timedelta(10)).put() alert_with_empty_component = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[1], timestamp=now_datetime + datetime.timedelta(20)).put() alert_with_component = anomaly.Anomaly(start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[2], timestamp=now_datetime + datetime.timedelta(30)).put() response = self.testapp.post('/file_bug', [ ('keys', '%s,%s,%s,%s' % (alert_without_ownership.urlsafe(), alert_without_component.urlsafe(), alert_with_empty_component.urlsafe(), alert_with_component.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="checkbox" checked name="component" value="Abc>Def">', response.body)
def setUp(self): super(BenchFindChangePointsTest, self).setUp() self.sheriff = sheriff.Sheriff( email='*****@*****.**', id=bench_find_anomalies._TEST_DATA_SHERIFF).put()
def _AddSheriffToDatastore(self): sheriff_key = sheriff.Sheriff(id='Chromium Perf Sheriff', email='*****@*****.**') sheriff_key.patterns = ['*/*/my_test_suite/*'] sheriff_key.put()
def _AddDataToDatastore(self): """Puts a set of entities; some internal-only, some not.""" # Need to be privileged to add TestMetadata and Row objects to the datastore # because there is a get() for the parent_test in the pre_put_hook. This # should work correctly in production because Rows and TestMetadata should # only be added by /add_point, which is privileged. self.SetCurrentUser('*****@*****.**') testing_common.AddTests( ['ChromiumPerf'], ['Win7External', 'FooInternal'], { 'TestInternal': { 'SubTestInternal': {} }, 'TestExternal': { 'SubTestExternal': {} }, }) internal_key = ['Master', 'ChromiumPerf', 'Bot', 'FooInternal'] internal_bot = ndb.Key(*internal_key).get() internal_bot.internal_only = True internal_bot.put() internal_test = ndb.Key( 'TestMetadata', 'ChromiumPerf/Win7External/TestInternal').get() internal_test.internal_only = True internal_test.put() internal_test = ndb.Key('TestMetadata', 'ChromiumPerf/FooInternal/TestInternal').get() internal_test.internal_only = True internal_test.put() internal_sub_test = ndb.Key( 'TestMetadata', 'ChromiumPerf/Win7External/TestInternal/SubTestInternal').get() internal_sub_test.internal_only = True internal_sub_test.put() internal_sub_test = ndb.Key( 'TestMetadata', 'ChromiumPerf/FooInternal/TestInternal/SubTestInternal').get() internal_sub_test.internal_only = True internal_sub_test.put() internal_key = internal_sub_test.key external_key = ndb.Key( 'TestMetadata', 'ChromiumPerf/Win7External/TestExternal/SubTestExternal') internal_test_container_key = utils.GetTestContainerKey(internal_key) external_test_container_key = utils.GetTestContainerKey(external_key) for i in range(0, 100, 10): graph_data.Row(parent=internal_test_container_key, id=i, value=float(i * 2), internal_only=True).put() graph_data.Row(parent=external_test_container_key, id=i, value=float(i * 2)).put() self.UnsetCurrentUser() sheriff.Sheriff(id='external', email='*****@*****.**', internal_only=False).put() sheriff.Sheriff(id='internal', email='*****@*****.**', internal_only=True).put()
def _AddMockAlertSummaryData(self): """Adds data to be used in the alert-summary stats tests below.""" correct_sheriff = sheriff.Sheriff(id='Chromium Perf Sheriff', email='*****@*****.**', patterns=[]).put() wrong_sheriff = sheriff.Sheriff(id='Some other sheriff', email='*****@*****.**', patterns=[]).put() linux_sunspider = 'ChromiumPerf/linux-release/sunspider/Total' linux_octane = 'ChromiumPerf/linux-release/octane/Total' linux_media = 'ChromiumPerf/linux-release/media.tough_media_cases/Total' windows_sunspider = 'ChromiumPerf/windows/sunspider/Total' windows_octane = 'ChromiumPerf/windows/octane/Total' # Should not be included: too early. anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 1), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=100.5).put() # Should not be included: too late. anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 6, 17), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=100.5).put() # Should not be included: wrong sheriff. anomaly.Anomaly(sheriff=wrong_sheriff, timestamp=datetime.datetime(2013, 5, 11), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=100.5).put() # Everything below should be included. anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=100.5).put() anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(linux_octane), median_before_anomaly=100, median_after_anomaly=101.5, bug_id=-1).put() anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(linux_media), median_before_anomaly=100, median_after_anomaly=101.5, bug_id=-2).put() anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(windows_sunspider), median_before_anomaly=100, median_after_anomaly=104.5, bug_id=12345).put() anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(windows_octane), median_before_anomaly=100, median_after_anomaly=600, bug_id=12345).put() anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 15), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=200).put() anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 20), test=utils.TestKey(windows_sunspider), median_before_anomaly=100, median_after_anomaly=115, bug_id=12345).put() anomaly.Anomaly(sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 21), test=utils.TestKey(windows_octane), median_before_anomaly=100, median_after_anomaly=104).put()
def _AddAlertsToDataStore(self): """Adds sample data, including triaged and non-triaged alerts.""" key_map = {} sheriff_key = sheriff.Sheriff(id='Chromium Perf Sheriff', email='*****@*****.**').put() testing_common.AddTests(['ChromiumGPU'], ['linux-release'], { 'scrolling-benchmark': { 'first_paint': {}, 'mean_frame_time': {}, } }) first_paint = utils.TestKey( 'ChromiumGPU/linux-release/scrolling-benchmark/first_paint') mean_frame_time = utils.TestKey( 'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time') # By default, all TestMetadata entities have an improvement_direction of # UNKNOWN, meaning that neither direction is considered an improvement. # Here we set the improvement direction so that some anomalies are # considered improvements. for test_key in [first_paint, mean_frame_time]: test = test_key.get() test.improvement_direction = anomaly.DOWN test.put() # Add some (12) non-triaged alerts. for end_rev in range(10000, 10120, 10): test_key = first_paint if end_rev % 20 == 0 else mean_frame_time anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() key_map[end_rev] = anomaly_key.urlsafe() # Add some (2) already-triaged alerts. for end_rev in range(10120, 10140, 10): test_key = first_paint if end_rev % 20 == 0 else mean_frame_time bug_id = -1 if end_rev % 20 == 0 else 12345 anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=100, median_after_anomaly=200, bug_id=bug_id, sheriff=sheriff_key) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() key_map[end_rev] = anomaly_key.urlsafe() if bug_id > 0: bug_data.Bug(id=bug_id).put() # Add some (6) non-triaged improvements. for end_rev in range(10140, 10200, 10): test_key = mean_frame_time anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=200, median_after_anomaly=100, sheriff=sheriff_key) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() self.assertTrue(anomaly_entity.is_improvement) key_map[end_rev] = anomaly_key.urlsafe() return key_map
def testProcessTest_RefineAnomalyPlacement_OffByOneStable(self): testing_common.AddTests( ['ChromiumPerf'], ['linux-perf'], { 'memory.desktop': { ('memory:chrome:all_processes:' 'reported_by_chrome:v8:effective_size_avg'): {} } }) test = utils.TestKey(( 'ChromiumPerf/linux-perf/memory.desktop/' 'memory:chrome:all_processes:reported_by_chrome:v8:effective_size_avg' )).get() test_container_key = utils.GetTestContainerKey(test.key) sample_data = [ (733480, 1381203.0), (733494, 1381220.0), (733504, 1381212.0), (733524, 1381220.0), (733538, 1381211.0), (733544, 1381212.0), (733549, 1381220.0), (733563, 1381220.0), (733581, 1381220.0), (733597, 1381212.0), (733611, 1381228.0), (733641, 1381212.0), (733675, 1381204.0), (733721, 1381212.0), (733766, 1381211.0), (733804, 1381204.0), (733835, 1381219.0), (733865, 1381211.0), (733885, 1381219.0), (733908, 1381204.0), (733920, 1381211.0), (733937, 1381220.0), (734091, 1381211.0), (734133, 1381219.0), (734181, 1381204.0), (734211, 1381720.0), (734248, 1381712.0), (734277, 1381696.0), (734311, 1381704.0), (734341, 1381703.0), (734372, 1381704.0), (734405, 1381703.0), (734431, 1381711.0), (734456, 1381720.0), (734487, 1381703.0), (734521, 1381704.0), (734554, 1381726.0), (734598, 1381704.0), (734630, 1381703.0), # In crbug/1041688 this is where it was placed. (734673, 1529888.0), # This is where it should be. (734705, 1529888.0), (734739, 1529860.0), (734770, 1529860.0), (734793, 1529888.0), (734829, 1529860.0), ] for row in sample_data: graph_data.Row(id=row[0], value=row[1], parent=test_container_key).put() sheriff.Sheriff(email='*****@*****.**', id='sheriff', patterns=[test.test_path]).put() test.UpdateSheriff() test.put() with mock.patch.object(SheriffConfigClient, 'Match', mock.MagicMock(return_value=([], None))) as m: find_anomalies.ProcessTests([test.key]) self.assertEqual(m.call_args_list, [mock.call(test.test_path)]) new_anomalies = anomaly.Anomaly.query().fetch() self.assertEqual(1, len(new_anomalies)) self.assertEqual(anomaly.UP, new_anomalies[0].direction) self.assertEqual(734631, new_anomalies[0].start_revision) self.assertEqual(734673, new_anomalies[0].end_revision)
def _AddSheriff(self): """Adds a Sheriff entity and returns the key.""" return sheriff.Sheriff(id='Chromium Perf Sheriff', email='*****@*****.**').put()
def testSheriffUrlValidator_ValidUrl_DoesntRaiseError(self): # By contrast, this URL should be accepted. sheriff.Sheriff(url='http://x.com/')
def testGet_OwnersAreEmptyEvenWithOwnership(self): ownership_samples = [ { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'emails': ['*****@*****.**'] }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', 'emails': ['*****@*****.**'] } ] test_paths = ['ChromiumPerf/linux/scrolling/first_paint', 'ChromiumPerf/linux/scrolling/mean_frame_time'] test_keys = [utils.TestKey(test_path) for test_path in test_paths] sheriff_key = sheriff.Sheriff( id='Sheriff', labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put() anomaly_1 = anomaly.Anomaly( start_revision=1476193324, end_revision=1476201840, test=test_keys[0], median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[0]).put() anomaly_2 = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_keys[1], median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[1]).put() response = self.testapp.post( '/file_bug', [ ('keys', '%s,%s' % (anomaly_1.urlsafe(), anomaly_2.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="text" name="owner" value="">', response.body) response_changed_order = self.testapp.post( '/file_bug', [ ('keys', '%s,%s' % (anomaly_2.urlsafe(), anomaly_1.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="text" name="owner" value="">', response_changed_order.body)
def testSheriffEmail_InvalidEmail_RaisesError(self): # If the given email address is not valid, an assertion error # will be raised while creating the entity with self.assertRaises(sheriff.ValidationError): sheriff.Sheriff(email='oops')
def testGet_UsesOnlyMostRecentComponents(self): ownership_samples = [ { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', 'component': 'Abc>Def' }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': '123>456' }, ] sheriff_key = sheriff.Sheriff( id='Sheriff', labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put() test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint') now_datetime = datetime.datetime.now() older_alert = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[0], timestamp=now_datetime).put() newer_alert = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[1], timestamp=now_datetime + datetime.timedelta(10)).put() response = self.testapp.post( '/file_bug', [ ('keys', '%s,%s' % (older_alert.urlsafe(), newer_alert.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertNotIn( '<input type="checkbox" checked name="component" value="Abc>Def">', response.body) self.assertIn( '<input type="checkbox" checked name="component" value="123>456">', response.body) response_inverted_order = self.testapp.post( '/file_bug', [ ('keys', '%s,%s' % (newer_alert.urlsafe(), older_alert.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertNotIn( '<input type="checkbox" checked name="component" value="Abc>Def">', response_inverted_order.body) self.assertIn( '<input type="checkbox" checked name="component" value="123>456">', response_inverted_order.body)
def _AddSheriff(self, name, email=None, url=None, internal_only=False, summarize=False, patterns=None): """Adds a Sheriff entity to the datastore.""" sheriff.Sheriff( id=name, email=email, url=url, internal_only=internal_only, summarize=summarize, patterns=patterns or []).put()