def testMatchFailed(self): clt = SheriffConfigClient() clt._session = self._Session( self._Response(False, 'some error message')) res, err_msg = clt.Match('Foo2/a/Bar2/b') self.assertIsNone(res) self.assertIn('some error message', err_msg)
def _GetSheriffList(): """Returns a list of sheriff names for all sheriffs in the datastore.""" clt = SheriffConfigClient() subscriptions, err_msg = clt.List() if err_msg: raise InternalServerError(err_msg) return [s.name for s in subscriptions]
def _ProcessTestStat(config, test, stat, rows, ref_rows): test_key = test.key # If there were no rows fetched, then there's nothing to analyze. if not rows: logging.error('No rows fetched for %s', test.test_path) raise ndb.Return(None) # Get anomalies and check if they happen in ref build also. change_points = FindChangePointsForTest(rows, config) if ref_rows: ref_change_points = FindChangePointsForTest(ref_rows, config) # Filter using any jumps in ref change_points = _FilterAnomaliesFoundInRef( change_points, ref_change_points, test_key) anomalies = yield [ _MakeAnomalyEntity(c, test, stat, rows) for c in change_points] # If no new anomalies were found, then we're done. if not anomalies: raise ndb.Return(None) logging.info('Created %d anomalies', len(anomalies)) logging.info(' Test: %s', test_key.id()) logging.info(' Stat: %s', stat) # Get all the sheriff from sheriff-config match the path client = SheriffConfigClient() subscriptions, err_msg = client.Match(test.test_path) subscription_names = [s.name for s in subscriptions or []] # Breaks the process when Match failed to ensure find_anomaly do the best # effort to find the subscriber. Leave retrying to upstream. if err_msg is not None: raise RuntimeError(err_msg) if not subscriptions: logging.warning('No subscription for %s', test.test_path) for a in anomalies: a.subscriptions = subscriptions a.subscription_names = subscription_names a.internal_only = (any([s.visibility != subscription.VISIBILITY.PUBLIC for s in subscriptions]) or test.internal_only) a.groups = alert_group.AlertGroup.GetGroupsForAnomaly(a) yield ndb.put_multi_async(anomalies) # TODO(simonhatch): email_sheriff.EmailSheriff() isn't a tasklet yet, so this # code will run serially. # Email sheriff about any new regressions. for anomaly_entity in anomalies: if (anomaly_entity.bug_id is None and not anomaly_entity.is_improvement): deferred.defer(_EmailSheriff, anomaly_entity.subscriptions, test.key, anomaly_entity.key)
def _GetSheriffForTest(test): """Gets the Sheriff for a test, or None if no sheriff.""" # Get all the sheriff from sheriff-config match the path # Currently just used for logging client = SheriffConfigClient() match_res = client.Match(test.test_path) # Old one. Get the sheriff from TestMetadata sheriff = None if test.sheriff: sheriff = yield test.sheriff.get_async() raise ndb.Return((sheriff, match_res))
def _ProcesssTestStat(config, sheriff, test, stat, rows, ref_rows): test_key = test.key # If there were no rows fetched, then there's nothing to analyze. if not rows: logging.error('No rows fetched for %s', test.test_path) raise ndb.Return(None) # Get anomalies and check if they happen in ref build also. change_points = FindChangePointsForTest(rows, config) if ref_rows: ref_change_points = FindChangePointsForTest(ref_rows, config) # Filter using any jumps in ref change_points = _FilterAnomaliesFoundInRef(change_points, ref_change_points, test_key) anomalies = yield [ _MakeAnomalyEntity(c, test, stat, rows) for c in change_points ] # If no new anomalies were found, then we're done. if not anomalies: raise ndb.Return(None) logging.info('Created %d anomalies', len(anomalies)) logging.info(' Test: %s', test_key.id()) logging.info(' Stat: %s', stat) logging.info(' Sheriff: %s', test.sheriff.id()) yield ndb.put_multi_async(anomalies) # Get all the sheriff from sheriff-config match the path # Currently just used for logging client = SheriffConfigClient() new_sheriffs, err_msg = client.Match(test.test_path) new_sheriffs_keys = [s.key.string_id() for s in new_sheriffs or []] logging.info('Sheriff for %s: old: %s, new: %s', test.test_path, 'None' if sheriff is None else sheriff.key.string_id(), err_msg if new_sheriffs is None else new_sheriffs_keys) if sheriff and sheriff.key.string_id() not in new_sheriffs_keys: logging.warn('Sheriff do not match: %s', test_key.string_id()) # TODO(simonhatch): email_sheriff.EmailSheriff() isn't a tasklet yet, so this # code will run serially. # Email sheriff about any new regressions. for anomaly_entity in anomalies: if (anomaly_entity.bug_id is None and not anomaly_entity.is_improvement and not sheriff.summarize): deferred.defer(_EmailSheriff, sheriff.key, test.key, anomaly_entity.key)
def testMatch(self): clt = SheriffConfigClient() response_text = """ { "subscriptions": [ { "config_set": "projects/catapult", "revision": "c9d4943dc832e448f9786e244f918fdabc1e5303", "subscription": { "name": "Public Team1", "rotation_url": "https://some/url", "notification_email": "*****@*****.**", "bug_labels": [ "Lable1", "Lable2" ], "bug_components": [ "foo>bar" ], "visibility": "PUBLIC", "patterns": [ { "glob": "Foo2/*/Bar2/*" }, { "regex": ".*" } ] } } ] } """ clt._session = self._Session(self._Response(True, response_text)) expected = [ Subscription( revision='c9d4943dc832e448f9786e244f918fdabc1e5303', name='Public Team1', rotation_url='https://some/url', notification_email='*****@*****.**', visibility=VISIBILITY.PUBLIC, bug_labels=['Lable1', 'Lable2'], bug_components=['foo>bar'] ), ] self.assertEqual(clt.Match('Foo2/a/Bar2/b'), (expected, None))
def get(self): client = SheriffConfigClient() ok, err_msg = client.Update() if not ok: return webapp2.Response('FAILED: %s' % err_msg) return webapp2.Response('OK')
def get(self): """Displays UI for debugging the anomaly detection function. Request parameters: test_path: Full test path (Master/bot/suite/chart) for test with alert. rev: A revision (Row id number) to center the graph on. num_before: Maximum number of points after the given revision to get. num_after: Maximum number of points before the given revision. config: Config parameters for in JSON form. Outputs: A HTML page with a chart (if test_path is given) and a form. """ try: test = self._GetTest() num_before, num_after = self._GetNumBeforeAfter() config_name = self._GetConfigName(test) config_dict = anomaly_config.CleanConfigDict( self._GetConfigDict(test)) except QueryParameterError as e: self.RenderHtml('debug_alert.html', {'error': e.message}) return revision = self.request.get('rev') if revision: rows = _FetchRowsAroundRev(test, int(revision), num_before, num_after) else: rows = _FetchLatestRows(test, num_before) chart_series = _ChartSeries(rows) lookup = _RevisionList(rows) # Get the anomaly data from the new anomaly detection module. This will # also be passed to the template so that it can be shown on the page. change_points = SimulateAlertProcessing(chart_series, **config_dict) anomaly_indexes = [c.x_value for c in change_points] anomaly_points = [(i, chart_series[i][1]) for i in anomaly_indexes] anomaly_segments = _AnomalySegmentSeries(change_points) plot_data = _GetPlotData(chart_series, anomaly_points, anomaly_segments) subscriptions, err_msg = SheriffConfigClient().Match(test.test_path) subscription_names = ','.join([s.name for s in subscriptions or []]) if err_msg is not None: self.RenderHtml('debug_alert.html', {'error': err_msg}) # Render the debug_alert page with all of the parameters filled in. self.RenderHtml( 'debug_alert.html', { 'test_path': test.test_path, 'rev': revision or '', 'num_before': num_before, 'num_after': num_after, 'sheriff_name': subscription_names, 'config_name': config_name, 'config_json': json.dumps( config_dict, indent=2, sort_keys=True), 'plot_data': json.dumps(plot_data), 'lookup': json.dumps(lookup), 'anomalies': json.dumps([c.AsDict() for c in change_points]), 'csv_url': _CsvUrl(test.test_path, rows), 'graph_url': _GraphUrl(test, revision), 'stored_anomalies': _FetchStoredAnomalies(test, lookup), })
def _ProcessTestStat(config, test, stat, rows, ref_rows): test_key = test.key # If there were no rows fetched, then there's nothing to analyze. if not rows: logging.error('No rows fetched for %s', test.test_path) raise ndb.Return(None) # Get anomalies and check if they happen in ref build also. change_points = FindChangePointsForTest(rows, config) if ref_rows: ref_change_points = FindChangePointsForTest(ref_rows, config) # Filter using any jumps in ref change_points = _FilterAnomaliesFoundInRef( change_points, ref_change_points, test_key) anomalies = yield [ _MakeAnomalyEntity(c, test, stat, rows) for c in change_points] # If no new anomalies were found, then we're done. if not anomalies: raise ndb.Return(None) logging.info('Created %d anomalies', len(anomalies)) logging.info(' Test: %s', test_key.id()) logging.info(' Stat: %s', stat) # Get all the sheriff from sheriff-config match the path legacy_sheriff = yield _GetSheriffForTest(test) client = SheriffConfigClient() subscriptions, err_msg = client.Match(test.test_path) subscription_names = [s.name for s in subscriptions or []] if legacy_sheriff is not None: logging.info('Sheriff for %s: old: %s, new: %s', test.test_path, legacy_sheriff.key.string_id(), err_msg if subscriptions is None else subscription_names) if legacy_sheriff.key.string_id() not in subscription_names: logging.warn('Sheriff do not match: %s', test_key.string_id()) # We still check legacy sheriff for backward compatibility. So during the # migration, we should update both legacy and new sheriff_config to ensure # config take effect. if not legacy_sheriff: logging.error('No sheriff for %s', test_key) raise ndb.Return(None) for a in anomalies: a.sheriff = legacy_sheriff.key a.subscriptions = subscriptions a.subscription_names = subscription_names a.internal_only = (any([s.visibility != subscription.VISIBILITY.PUBLIC for s in subscriptions]) or test.internal_only) yield ndb.put_multi_async(anomalies) # TODO(simonhatch): email_sheriff.EmailSheriff() isn't a tasklet yet, so this # code will run serially. # Email sheriff about any new regressions. for anomaly_entity in anomalies: if (anomaly_entity.bug_id is None and not anomaly_entity.is_improvement): deferred.defer(_EmailSheriff, legacy_sheriff, test.key, anomaly_entity.key)