def testGetTestMetadataKey_TestMetadata(self): a = anomaly.Anomaly(test=utils.TestKey('a/b/c/d')) k = a.GetTestMetadataKey() self.assertEqual('TestMetadata', k.kind()) self.assertEqual('a/b/c/d', k.id()) self.assertEqual('a/b/c/d', utils.TestPath(k))
def testGetTestMetadataKey_None(self): a = anomaly.Anomaly() k = a.GetTestMetadataKey() self.assertIsNone(k)
def testGet_UsesOnlyMostRecentComponents(self): ownership_samples = [ { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', 'component': 'Abc>Def' }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': '123>456' }, ] subscription = Subscription( name='Sheriff', bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript']) test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint') now_datetime = datetime.datetime.now() older_alert = anomaly.Anomaly(start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, subscriptions=[subscription], subscription_names=[subscription.name], ownership=ownership_samples[0], timestamp=now_datetime).put() newer_alert = anomaly.Anomaly(start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, subscriptions=[subscription], subscription_names=[subscription.name], ownership=ownership_samples[1], timestamp=now_datetime + datetime.timedelta(10)).put() response = self.testapp.post('/file_bug', [ ('keys', '%s,%s' % (older_alert.urlsafe(), newer_alert.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertNotIn( '<input type="checkbox" checked name="component" value="Abc>Def">', response.body) self.assertIn( '<input type="checkbox" checked name="component" value="123>456">', response.body) response_inverted_order = self.testapp.post('/file_bug', [ ('keys', '%s,%s' % (newer_alert.urlsafe(), older_alert.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertNotIn( '<input type="checkbox" checked name="component" value="Abc>Def">', response_inverted_order.body) self.assertIn( '<input type="checkbox" checked name="component" value="123>456">', response_inverted_order.body)
def testGet_UsesFirstDefinedComponent(self): ownership_samples = [{ 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': '' }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': 'Abc>Def' }] now_datetime = datetime.datetime.now() test_key = utils.TestKey('ChromiumPerf/linux/scrolling/first_paint') subscription = Subscription( name='Sheriff', bug_labels=['Performance-Sheriff', 'Cr-Blink-Javascript']) alert_without_ownership = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, subscriptions=[subscription], subscription_names=[subscription.name], timestamp=now_datetime).put() alert_without_component = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, subscriptions=[subscription], subscription_names=[subscription.name], ownership=ownership_samples[0], timestamp=now_datetime + datetime.timedelta(10)).put() alert_with_empty_component = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, subscriptions=[subscription], subscription_names=[subscription.name], ownership=ownership_samples[1], timestamp=now_datetime + datetime.timedelta(20)).put() alert_with_component = anomaly.Anomaly( start_revision=1476193320, end_revision=1476201870, test=test_key, median_before_anomaly=100, median_after_anomaly=200, subscriptions=[subscription], subscription_names=[subscription.name], ownership=ownership_samples[2], timestamp=now_datetime + datetime.timedelta(30)).put() response = self.testapp.post('/file_bug', [ ('keys', '%s,%s,%s,%s' % (alert_without_ownership.urlsafe(), alert_without_component.urlsafe(), alert_with_empty_component.urlsafe(), alert_with_component.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="checkbox" checked name="component" value="Abc>Def">', response.body)
def _MakeAnomalyEntity(change_point, test, rows): """Creates an Anomaly entity. Args: change_point: A find_change_points.ChangePoint object. test: The TestMetadata entity that the anomalies were found on. rows: List of Row entities that the anomalies were found on. Returns: An Anomaly entity, which is not yet put in the datastore. """ end_rev = change_point.x_value start_rev = _GetImmediatelyPreviousRevisionNumber(end_rev, rows) + 1 display_start = display_end = None if test.master_name == 'ClankInternal': display_start, display_end = _GetDisplayRange(change_point.x_value, rows) median_before = change_point.median_before median_after = change_point.median_after suite_key = test.key.id().split('/')[:3] suite_key = '/'.join(suite_key) suite_key = utils.TestKey(suite_key) queried_diagnostics = yield ( histogram.SparseDiagnostic.GetMostRecentDataByNamesAsync( suite_key, set([ reserved_infos.BUG_COMPONENTS.name, reserved_infos.OWNERS.name ]))) bug_components = queried_diagnostics.get( reserved_infos.BUG_COMPONENTS.name, {}).get('values') ownership_information = { 'emails': queried_diagnostics.get(reserved_infos.OWNERS.name, {}).get('values'), 'component': (bug_components[0] if bug_components else None) } new_anomaly = anomaly.Anomaly( start_revision=start_rev, end_revision=end_rev, median_before_anomaly=median_before, median_after_anomaly=median_after, segment_size_before=change_point.size_before, segment_size_after=change_point.size_after, window_end_revision=change_point.window_end, std_dev_before_anomaly=change_point.std_dev_before, t_statistic=change_point.t_statistic, degrees_of_freedom=change_point.degrees_of_freedom, p_value=change_point.p_value, is_improvement=_IsImprovement(test, median_before, median_after), ref_test=_GetRefBuildKeyForTest(test), test=test.key, sheriff=test.sheriff, internal_only=test.internal_only, units=test.units, display_start=display_start, display_end=display_end, ownership=ownership_information) raise ndb.Return(new_anomaly)
def testGetGraphJson_WithAnomalies_ReturnsCorrectAnomalyAnnotations(self): self._AddTestColumns() anomaly1 = anomaly.Anomaly( start_revision=14999, end_revision=15000, test=utils.TestKey('ChromiumGPU/win7/dromaeo/dom'), median_before_anomaly=100, median_after_anomaly=200) anomaly1.SetIsImprovement() key1 = anomaly1.put() anomaly2 = anomaly.Anomaly( start_revision=15004, end_revision=15006, test=utils.TestKey('ChromiumGPU/win7/dromaeo/dom'), median_before_anomaly=200, median_after_anomaly=100, bug_id=12345) anomaly2.SetIsImprovement() key2 = anomaly2.put() test = utils.TestKey('ChromiumGPU/win7/dromaeo/dom').get() test.description = 'About this test' test.units = 'ms' test.buildername = 'Windows 7 (1)' test.put() flot_json_str = graph_json.GetGraphJson( { 'ChromiumGPU/win7/dromaeo/dom': [], }, rev=15000, num_points=8) flot = json.loads(flot_json_str) annotations = flot['annotations'] self.assertEqual(5, len(annotations['0'])) # Verify key fields of the annotation dictionary for the first anomaly. anomaly_one_annotation = annotations['0']['0']['g_anomaly'] self.assertEqual(14999, anomaly_one_annotation['start_revision']) self.assertEqual(15000, anomaly_one_annotation['end_revision']) self.assertEqual('100.0%', anomaly_one_annotation['percent_changed']) self.assertIsNone(anomaly_one_annotation['bug_id']) self.assertEqual(key1.urlsafe(), anomaly_one_annotation['key']) self.assertTrue(anomaly_one_annotation['improvement']) # Verify key fields of the annotation dictionary for th second anomaly. anomaly_two_annotation = annotations['0']['2']['g_anomaly'] self.assertEqual(15004, anomaly_two_annotation['start_revision']) self.assertEqual(15006, anomaly_two_annotation['end_revision']) self.assertEqual('50.0%', anomaly_two_annotation['percent_changed']) self.assertEqual(12345, anomaly_two_annotation['bug_id']) self.assertEqual(key2.urlsafe(), anomaly_two_annotation['key']) self.assertFalse(anomaly_two_annotation['improvement']) # Verify the tracing link annotations self.assertEqual('http://trace/15000', annotations['0']['0']['a_tracing_uri']) self.assertEqual('http://trace/15012', annotations['0']['4']['a_tracing_uri']) # Verify the tracing rerun options self.assertEqual({'foo': '--foo'}, annotations['0']['0']['a_trace_rerun_options']) # Verify the series annotations. self.assertEqual({ '0': { 'name': 'dom', 'path': 'ChromiumGPU/win7/dromaeo/dom', 'units': 'ms', 'better': 'Higher', 'description': 'About this test' } }, annotations['series'])
def _AddMockAlertSummaryData(self): """Adds data to be used in the alert-summary stats tests below.""" correct_sheriff = sheriff.Sheriff( id='Chromium Perf Sheriff', email='*****@*****.**', patterns=[]).put() wrong_sheriff = sheriff.Sheriff( id='Some other sheriff', email='*****@*****.**', patterns=[]).put() linux_sunspider = 'ChromiumPerf/linux-release/sunspider/Total' linux_octane = 'ChromiumPerf/linux-release/octane/Total' linux_media = 'ChromiumPerf/linux-release/media.tough_media_cases/Total' windows_sunspider = 'ChromiumPerf/windows/sunspider/Total' windows_octane = 'ChromiumPerf/windows/octane/Total' # Should not be included: too early. anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 1), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=100.5).put() # Should not be included: too late. anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 6, 17), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=100.5).put() # Should not be included: wrong sheriff. anomaly.Anomaly( sheriff=wrong_sheriff, timestamp=datetime.datetime(2013, 5, 11), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=100.5).put() # Everything below should be included. anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=100.5).put() anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(linux_octane), median_before_anomaly=100, median_after_anomaly=101.5, bug_id=-1).put() anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(linux_media), median_before_anomaly=100, median_after_anomaly=101.5, bug_id=-2).put() anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(windows_sunspider), median_before_anomaly=100, median_after_anomaly=104.5, bug_id=12345).put() anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 12), test=utils.TestKey(windows_octane), median_before_anomaly=100, median_after_anomaly=600, bug_id=12345).put() anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 15), test=utils.TestKey(linux_sunspider), median_before_anomaly=100, median_after_anomaly=200).put() anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 20), test=utils.TestKey(windows_sunspider), median_before_anomaly=100, median_after_anomaly=115, bug_id=12345).put() anomaly.Anomaly( sheriff=correct_sheriff, timestamp=datetime.datetime(2013, 5, 21), test=utils.TestKey(windows_octane), median_before_anomaly=100, median_after_anomaly=104).put()
def testGet_ComponentsChosenPerTest(self): ownership_samples = [ { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', 'component': 'Abc>Def' }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': '123>456' }, ] sheriff_key = sheriff.Sheriff( id='Sheriff', labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put() test_paths = [ 'ChromiumPerf/linux/scrolling/first_paint', 'ChromiumPerf/linux/scrolling/mean_frame_time' ] test_keys = [utils.TestKey(test_path) for test_path in test_paths] now_datetime = datetime.datetime.now() alert_test_key_0 = anomaly.Anomaly(start_revision=1476193320, end_revision=1476201870, test=test_keys[0], median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[0], timestamp=now_datetime).put() alert_test_key_1 = anomaly.Anomaly(start_revision=1476193320, end_revision=1476201870, test=test_keys[1], median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[1], timestamp=now_datetime + datetime.timedelta(10)).put() response = self.testapp.post('/file_bug', [ ('keys', '%s,%s' % (alert_test_key_0.urlsafe(), alert_test_key_1.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="checkbox" checked name="component" value="Abc>Def">', response.body) self.assertIn( '<input type="checkbox" checked name="component" value="123>456">', response.body)
def testGet_WithAllOwnershipComponents(self): ownership_samples = [{ 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb826', 'component': 'Abc>Xyz' }, { 'type': 'Ownership', 'guid': 'eb212e80-db58-4cbd-b331-c2245ecbb827', 'component': 'Def>123' }] test_paths = [ 'ChromiumPerf/linux/scrolling/first_paint', 'ChromiumPerf/linux/scrolling/mean_frame_time' ] test_keys = [utils.TestKey(test_path) for test_path in test_paths] sheriff_key = sheriff.Sheriff( id='Sheriff', labels=['Performance-Sheriff', 'Cr-Blink-Javascript']).put() anomaly_1 = anomaly.Anomaly(start_revision=1476193324, end_revision=1476201840, test=test_keys[0], median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[0]).put() anomaly_2 = anomaly.Anomaly(start_revision=1476193320, end_revision=1476201870, test=test_keys[1], median_before_anomaly=100, median_after_anomaly=200, sheriff=sheriff_key, ownership=ownership_samples[1]).put() response = self.testapp.post('/file_bug', [ ('keys', '%s' % (anomaly_1.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="checkbox" checked name="component" value="Abc>Xyz">', response.body) response_with_both_anomalies = self.testapp.post( '/file_bug', [ ('keys', '%s,%s' % (anomaly_1.urlsafe(), anomaly_2.urlsafe())), ('summary', 's'), ('description', 'd\n'), ('label', 'one'), ('label', 'two'), ('component', 'Foo>Bar'), ]) self.assertIn( '<input type="checkbox" checked name="component" value="Abc>Xyz">', response_with_both_anomalies.body) self.assertIn( '<input type="checkbox" checked name="component" value="Def>123">', response_with_both_anomalies.body)
def _AddAlertsToDataStore(self): """Adds sample data, including triaged and non-triaged alerts.""" key_map = {} sheriff_key = sheriff.Sheriff(id='Chromium Perf Sheriff', email='*****@*****.**').put() testing_common.AddTests( ['ChromiumGPU'], ['linux-release'], { 'scrolling-benchmark': { 'first_paint': {}, 'first_paint_ref': {}, 'mean_frame_time': {}, 'mean_frame_time_ref': {}, } }) first_paint = utils.TestKey( 'ChromiumGPU/linux-release/scrolling-benchmark/first_paint') mean_frame_time = utils.TestKey( 'ChromiumGPU/linux-release/scrolling-benchmark/mean_frame_time') # By default, all TestMetadata entities have an improvement_direction of # UNKNOWN, meaning that neither direction is considered an improvement. # Here we set the improvement direction so that some anomalies are # considered improvements. for test_key in [first_paint, mean_frame_time]: test = test_key.get() test.improvement_direction = anomaly.DOWN test.put() # Add some (12) non-triaged alerts. for end_rev in range(10000, 10120, 10): test_key = first_paint if end_rev % 20 == 0 else mean_frame_time ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key)) anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=100, median_after_anomaly=200, ref_test=ref_test_key, sheriff=sheriff_key) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() key_map[end_rev] = anomaly_key.urlsafe() # Add some (2) already-triaged alerts. for end_rev in range(10120, 10140, 10): test_key = first_paint if end_rev % 20 == 0 else mean_frame_time ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key)) bug_id = -1 if end_rev % 20 == 0 else 12345 anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=100, median_after_anomaly=200, ref_test=ref_test_key, bug_id=bug_id, sheriff=sheriff_key) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() key_map[end_rev] = anomaly_key.urlsafe() if bug_id > 0: bug_data.Bug(id=bug_id).put() # Add some (6) non-triaged improvements. for end_rev in range(10140, 10200, 10): test_key = mean_frame_time ref_test_key = utils.TestKey('%s_ref' % utils.TestPath(test_key)) anomaly_entity = anomaly.Anomaly(start_revision=end_rev - 5, end_revision=end_rev, test=test_key, median_before_anomaly=200, median_after_anomaly=100, ref_test=ref_test_key, sheriff=sheriff_key) anomaly_entity.SetIsImprovement() anomaly_key = anomaly_entity.put() self.assertTrue(anomaly_entity.is_improvement) key_map[end_rev] = anomaly_key.urlsafe() return key_map
def testStartNewBisectForBut_BlacklistedMaster_SucceedsIfAlternative( self, mock_new): # Even if there are blacklisted masters, the bisect request should still # succeed if there's a non-blacklisted master. namespaced_stored_object.Set('file_bug_bisect_blacklist', {'ChromiumPerf': []}) namespaced_stored_object.Set('bot_configurations', { 'linux-pinpoint': { 'dimensions': [{ 'key': 'foo', 'value': 'bar' }] }, }) namespaced_stored_object.Set('repositories', { 'chromium': { 'some': 'params' }, }) mock_new.return_value = {'jobId': 123, 'jobUrl': 'http://pinpoint/123'} testing_common.AddTests(['ChromiumPerf'], ['linux-pinpoint'], {'sunspider': { 'score': {} }}) testing_common.AddTests(['ChromiumPerf2'], ['linux-pinpoint'], {'sunspider': { 'score': {} }}) test_key = utils.TestKey('ChromiumPerf/linux-pinpoint/sunspider/score') test_key2 = utils.TestKey( 'ChromiumPerf2/linux-pinpoint/sunspider/score') rows = { 11999: { 'a_default_rev': 'r_chromium', 'r_chromium': '9e29b5bcd08357155b2859f87227d50ed60cf857' }, 12500: { 'a_default_rev': 'r_chromium', 'r_chromium': 'fc34e5346446854637311ad7793a95d56e314042' } } testing_common.AddRows('ChromiumPerf/linux-pinpoint/sunspider/score', rows) testing_common.AddRows('ChromiumPerf2/linux-pinpoint/sunspider/score', rows) anomaly.Anomaly(bug_id=333, test=test_key, start_revision=12000, end_revision=12500, median_before_anomaly=100, median_after_anomaly=200).put() a = anomaly.Anomaly(bug_id=333, test=test_key2, start_revision=12000, end_revision=12500, median_before_anomaly=100, median_after_anomaly=200).put() result = auto_bisect.StartNewBisectForBug(333) self.assertNotIn('error', result) self.assertEqual({ 'alert': a.urlsafe(), 'test_path': test_key2.id() }, json.loads(mock_new.call_args[0][0]['tags']))