def _GetAlerts(self, *args): alert_list = None list_type = args[0] try: if list_type.startswith('bug_id'): bug_id = list_type.replace('bug_id/', '') alert_list, _ = group_report.GetAlertsWithBugId(bug_id) elif list_type.startswith('keys'): keys = list_type.replace('keys/', '').split(',') alert_list, _ = group_report.GetAlertsForKeys(keys) elif list_type.startswith('rev'): rev = list_type.replace('rev/', '') alert_list, _ = group_report.GetAlertsAroundRevision(rev) else: raise BadRequestError('Invalid alert type %s' % list_type) except request_handler.InvalidInputError as e: raise BadRequestError(e.message) anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) stoppage_alert_dicts = alerts.StoppageAlertDicts( [a for a in alert_list if a.key.kind() == 'StoppageAlert']) response = {} if anomaly_dicts: response['anomalies'] = anomaly_dicts if stoppage_alert_dicts: response['stoppage_alerts'] = stoppage_alert_dicts return response
def post(self): """Returns dynamic data for /group_report with some set of alerts. The set of alerts is determined by the sid, keys, bug ID, or revision given. Request parameters: keys: A comma-separated list of urlsafe Anomaly keys (optional). bug_id: A bug number on the Chromium issue tracker (optional). rev: A revision number (optional). sid: A hash of a group of keys from /short_uri (optional). Outputs: JSON for the /group_report page XHR request. """ bug_id = self.request.get('bug_id') rev = self.request.get('rev') keys = self.request.get('keys') hash_code = self.request.get('sid') # sid takes precedence. if hash_code: state = ndb.Key(page_state.PageState, hash_code).get() if state: keys = json.loads(state.value) elif keys: keys = keys.split(',') try: alert_list = None if bug_id: try: alert_list, _, _ = anomaly.Anomaly.QueryAsync( bug_id=bug_id, limit=_QUERY_LIMIT).get_result() except ValueError: raise request_handler.InvalidInputError( 'Invalid bug ID "%s".' % bug_id) elif keys: alert_list = GetAlertsForKeys(keys) elif rev: alert_list = GetAlertsAroundRevision(rev) else: raise request_handler.InvalidInputError( 'No anomalies specified.') alert_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) values = { 'alert_list': alert_dicts, 'test_suites': update_test_suites.FetchCachedTestSuites(), } if bug_id: values['bug_id'] = bug_id if keys: values['selected_keys'] = keys self.GetDynamicVariables(values) self.response.out.write(json.dumps(values)) except request_handler.InvalidInputError as error: self.response.out.write(json.dumps({'error': str(error)}))
def _ShowAlerts(self, alert_list, bug_id=None): """Responds to an XHR from /group_report page with a JSON list of alerts. Args: alert_list: A list of Anomaly and/or StoppageAlert entities. bug_id: An integer bug ID. """ anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) stoppage_alert_dicts = alerts.StoppageAlertDicts( [a for a in alert_list if a.key.kind() == 'StoppageAlert']) alert_dicts = anomaly_dicts + stoppage_alert_dicts values = { 'alert_list': alert_dicts[:_DISPLAY_LIMIT], # This code for getting the subtests is supposed to be used to sort out # which metrics are "core" vs "non-core". But it's extremely slow, and # also doesn't seem to work very well. Turn it off for now: # https://github.com/catapult-project/catapult/issues/2877 #'subtests': _GetSubTestsForAlerts(alert_dicts), 'bug_id': bug_id, 'test_suites': update_test_suites.FetchCachedTestSuites(), } self.GetDynamicVariables(values) self.response.out.write(json.dumps(values))
def _ShowAlerts(self, alert_list, bug_id=None): """Responds to an XHR from /group_report page with a JSON list of alerts. Args: alert_list: A list of Anomaly and/or StoppageAlert entities. bug_id: An integer bug ID. """ anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) stoppage_alert_dicts = alerts.StoppageAlertDicts( [a for a in alert_list if a.key.kind() == 'StoppageAlert']) alert_dicts = anomaly_dicts + stoppage_alert_dicts owner_info = None if bug_id and ndb.Key('Bug', bug_id).get(): owner_info = _GetOwnerInfo(alert_dicts) values = { 'alert_list': alert_dicts[:_DISPLAY_LIMIT], 'subtests': _GetSubTestsForAlerts(alert_dicts), 'bug_id': bug_id, 'owner_info': owner_info, 'test_suites': update_test_suites.FetchCachedTestSuites(), } self.GetDynamicVariables(values) self.response.out.write(json.dumps(values))
def AuthorizedPost(self, *args): """Returns alert data in response to API requests. Possible list types: keys: A comma-separated list of urlsafe Anomaly keys. bug_id: A bug number on the Chromium issue tracker. rev: A revision number. Outputs: Alerts data; see README.md. """ alert_list = None list_type = args[0] try: if list_type.startswith('bug_id'): bug_id = list_type.replace('bug_id/', '') alert_list = group_report.GetAlertsWithBugId(bug_id) elif list_type.startswith('keys'): keys = list_type.replace('keys/', '').split(',') alert_list = group_report.GetAlertsForKeys(keys) elif list_type.startswith('rev'): rev = list_type.replace('rev/', '') alert_list = group_report.GetAlertsAroundRevision(rev) elif list_type.startswith('history'): try: days = int(list_type.replace('history/', '')) except ValueError: days = 7 cutoff = datetime.datetime.now() - datetime.timedelta( days=days) sheriff_name = self.request.get('sheriff', 'Chromium Perf Sheriff') sheriff_key = ndb.Key('Sheriff', sheriff_name) sheriff = sheriff_key.get() if not sheriff: raise api_request_handler.BadRequestError( 'Invalid sheriff %s' % sheriff_name) include_improvements = bool(self.request.get('improvements')) query = anomaly.Anomaly.query( anomaly.Anomaly.sheriff == sheriff_key) query = query.filter(anomaly.Anomaly.timestamp > cutoff) if not include_improvements: query = query.filter( anomaly.Anomaly.is_improvement == False) query = query.order(-anomaly.Anomaly.timestamp) alert_list = query.fetch() else: raise api_request_handler.BadRequestError( 'Invalid alert type %s' % list_type) except request_handler.InvalidInputError as e: raise api_request_handler.BadRequestError(e.message) anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) response = {'anomalies': anomaly_dicts} return response
def _FetchAlerts(self): anomalies, _, _ = yield anomaly.Anomaly.QueryAsync( test_keys=self._test_keys, max_start_revision=self._max_revision, min_end_revision=self._min_revision) for alert in anomalies: if alert.internal_only: self._private = True datum = self._Datum(alert.end_revision) # TODO(benjhayden) bisect_status datum['alert'] = alerts.AnomalyDicts([alert], v2=True)[0]
def _GetAlerts(self, *args): alert_list = None list_type = args[0] try: if list_type.startswith('bug_id'): bug_id = list_type.replace('bug_id/', '') alert_list, _ = group_report.GetAlertsWithBugId(bug_id) elif list_type.startswith('keys'): keys = list_type.replace('keys/', '').split(',') alert_list, _ = group_report.GetAlertsForKeys(keys) elif list_type.startswith('rev'): rev = list_type.replace('rev/', '') alert_list, _ = group_report.GetAlertsAroundRevision(rev) elif list_type.startswith('history'): try: days = int(list_type.replace('history/', '')) except ValueError: days = 7 cutoff = datetime.datetime.now() - datetime.timedelta(days=days) sheriff_name = self.request.get('sheriff', 'Chromium Perf Sheriff') sheriff_key = ndb.Key('Sheriff', sheriff_name) sheriff = sheriff_key.get() if not sheriff: raise BadRequestError('Invalid sheriff %s' % sheriff_name) include_improvements = bool(self.request.get('improvements')) query = anomaly.Anomaly.query(anomaly.Anomaly.sheriff == sheriff_key) query = query.filter(anomaly.Anomaly.timestamp > cutoff) if not include_improvements: query = query.filter( anomaly.Anomaly.is_improvement == False) query = query.order(-anomaly.Anomaly.timestamp) alert_list = query.fetch() else: raise BadRequestError('Invalid alert type %s' % list_type) except request_handler.InvalidInputError as e: raise BadRequestError(e.message) anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) stoppage_alert_dicts = alerts.StoppageAlertDicts( [a for a in alert_list if a.key.kind() == 'StoppageAlert']) response = {} if anomaly_dicts: response['anomalies'] = anomaly_dicts if stoppage_alert_dicts: response['stoppage_alerts'] = stoppage_alert_dicts return response
def _GetResponseValuesForBenchmark(self, benchmark, num_days, master): values = {} # The cached test suite info contains info about monitoring and bots. query = anomaly.Anomaly.query( anomaly.Anomaly.benchmark_name == benchmark, anomaly.Anomaly.master_name == master, anomaly.Anomaly.is_improvement == False, anomaly.Anomaly.timestamp > datetime.datetime.now() - datetime.timedelta(days=num_days)) query = query.order(-anomaly.Anomaly.timestamp) anomalies = query.fetch() values['alerts'] = alerts.AnomalyDicts(anomalies) benchmarks = update_test_suites.FetchCachedTestSuites() if benchmarks[benchmark].get('mon'): values['monitored'] = True else: values['monitored'] = False values['bots'] = benchmarks[benchmark]['mas'][master].keys() return values
def _OutputAnomaliesJSON(self, table_name): """Obtains the entire alert list specified. Args: table_name: The name of the requested report. """ table_entity = ndb.Key('TableConfig', table_name).get() if not table_entity: self.response.out.write( json.dumps({'error': 'Invalid table name.'})) return rev_a = self.request.get('revA') rev_b = self.request.get('revB') milestone_param = self.request.get('m') if milestone_param: milestone_param = int(milestone_param) if milestone_param not in CHROMIUM_MILESTONES: self.response.out.write( json.dumps({'error': 'No data for that milestone.'})) return master_bot_pairs = _GetMasterBotPairs(table_entity.bots) rev_a, rev_b, _ = _GetRevisionsFromParams(rev_a, rev_b, milestone_param, table_entity, master_bot_pairs) revisions = [rev_b, rev_a] anomalies = _FetchAnomalies(table_entity, rev_a, rev_b) anomaly_dicts = alerts.AnomalyDicts(anomalies) values = {} self.GetDynamicVariables(values) self.response.out.write( json.dumps({ 'xsrf_token': values['xsrf_token'], 'revisions': revisions, 'anomalies': anomaly_dicts }))
def _GetResponseValuesForBenchmark(self, benchmark, num_days, master): values = {} # The cached test suite info contains info about monitoring and bots. benchmarks = update_test_suites.FetchCachedTestSuites() sheriff = self._GetSheriffForBenchmark(benchmark, master, benchmarks) if sheriff: query = anomaly.Anomaly.query(anomaly.Anomaly.sheriff == sheriff) query = query.filter(anomaly.Anomaly.is_improvement == False) query = query.filter( anomaly.Anomaly.timestamp > datetime.datetime.now() - datetime.timedelta(days=num_days)) query = query.order(-anomaly.Anomaly.timestamp) anomalies = query.fetch() anomalies = [a for a in anomalies if self._BenchmarkName(a) == benchmark] values['monitored'] = True values['alerts'] = alerts.AnomalyDicts(anomalies) else: values['monitored'] = False values['bots'] = benchmarks[benchmark]['mas'][master].keys() return values
def _ShowAlerts(self, alert_list, selected_keys=None, bug_id=None): """Responds to an XHR from /group_report page with a JSON list of alerts. Args: alert_list: A list of Anomaly and/or StoppageAlert entities. bug_id: An integer bug ID. """ anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) stoppage_alert_dicts = alerts.StoppageAlertDicts( [a for a in alert_list if a.key.kind() == 'StoppageAlert']) alert_dicts = anomaly_dicts + stoppage_alert_dicts values = { 'alert_list': alert_dicts[:_DISPLAY_LIMIT], 'bug_id': bug_id, 'test_suites': update_test_suites.FetchCachedTestSuites(), 'selected_keys': selected_keys, } self.GetDynamicVariables(values) self.response.out.write(json.dumps(values))
def _ShowAlerts(self, alert_list, bug_id=None): """Renders the template group_report.html with a list of alerts. Args: alert_list: A list of Anomaly and/or StoppageAlert entities. bug_id: An integer bug ID. """ anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) stoppage_alert_dicts = alerts.StoppageAlertDicts( [a for a in alert_list if a.key.kind() == 'StoppageAlert']) alert_dicts = anomaly_dicts + stoppage_alert_dicts owner_info = None if bug_id and ndb.Key('Bug', bug_id).get(): owner_info = _GetOwnerInfo(alert_dicts) self.RenderHtml('group_report.html', { 'alert_list': json.dumps(alert_dicts[:_DISPLAY_LIMIT]), 'subtests': json.dumps(_GetSubTestsForAlerts(alert_dicts)), 'bug_id': bug_id, 'owner_info': json.dumps(owner_info), 'test_suites': json.dumps(update_test_suites.FetchCachedTestSuites()), })
def UnprivilegedPost(self, *args): """Returns alert data in response to API requests. Possible list types: keys: A comma-separated list of urlsafe Anomaly keys. bug_id: A bug number on the Chromium issue tracker. rev: A revision number. Outputs: Alerts data; see README.md. """ alert_list = None response = {} try: if len(args) == 0: is_improvement = utils.ParseBool( self.request.get('is_improvement', None)) recovered = utils.ParseBool(self.request.get( 'recovered', None)) start_cursor = self.request.get('cursor', None) if start_cursor: start_cursor = datastore_query.Cursor(urlsafe=start_cursor) min_timestamp = utils.ParseISO8601( self.request.get('min_timestamp', None)) max_timestamp = utils.ParseISO8601( self.request.get('max_timestamp', None)) test_keys = [] for template_id in self.request.get_all('report'): test_keys.extend( report_template.TestKeysForReportTemplate(template_id)) try: alert_list, next_cursor, _ = anomaly.Anomaly.QueryAsync( bot_name=self.request.get('bot', None), bug_id=self.request.get('bug_id', None), is_improvement=is_improvement, key=self.request.get('key', None), limit=int(self.request.get('limit', 100)), master_name=self.request.get('master', None), max_end_revision=self.request.get( 'max_end_revision', None), max_start_revision=self.request.get( 'max_start_revision', None), max_timestamp=max_timestamp, min_end_revision=self.request.get( 'min_end_revision', None), min_start_revision=self.request.get( 'min_start_revision', None), min_timestamp=min_timestamp, recovered=recovered, sheriff=self.request.get('sheriff', None), start_cursor=start_cursor, test=self.request.get('test', None), test_keys=test_keys, test_suite_name=self.request.get('test_suite', None)).get_result() except AssertionError: alert_list, next_cursor = [], None if next_cursor: response['next_cursor'] = next_cursor.urlsafe() else: list_type = args[0] if list_type.startswith('bug_id'): bug_id = list_type.replace('bug_id/', '') try: bug_id = int(bug_id) except ValueError as e: raise api_request_handler.BadRequestError( 'Invalid bug ID "%s".' % bug_id) response['DEPRECATION WARNING'] = ( 'Please use /api/alerts?bug_id=%s' % bug_id) alert_list, _, _ = anomaly.Anomaly.QueryAsync( bug_id=bug_id).get_result() elif list_type.startswith('keys'): keys = list_type.replace('keys/', '').split(',') response['DEPRECATION WARNING'] = ( 'Please use /api/alerts?key=%s' % keys[0]) alert_list = group_report.GetAlertsForKeys(keys) elif list_type.startswith('rev'): rev = list_type.replace('rev/', '') response['DEPRECATION WARNING'] = ( 'Please use /api/alerts?max_end_revision=%s&min_start_revision=%s' % (rev, rev)) alert_list = group_report.GetAlertsAroundRevision(rev) elif list_type.startswith('history'): try: days = int(list_type.replace('history/', '')) except ValueError: days = 7 cutoff = datetime.datetime.now() - datetime.timedelta( days=days) sheriff_name = self.request.get('sheriff', 'Chromium Perf Sheriff') sheriff_key = ndb.Key('Sheriff', sheriff_name) sheriff = sheriff_key.get() if not sheriff: raise api_request_handler.BadRequestError( 'Invalid sheriff %s' % sheriff_name) response['DEPRECATION WARNING'] = ( 'Please use /api/alerts?min_timestamp=%s&sheriff=%s' % (urllib.quote( cutoff.isoformat()), urllib.quote(sheriff_name))) include_improvements = bool( self.request.get('improvements')) filter_for_benchmark = self.request.get('benchmark') is_improvement = None if not include_improvements: is_improvement = False response[ 'DEPRECATION WARNING'] += '&is_improvement=false' if filter_for_benchmark: response['DEPRECATION WARNING'] += ( '&test_suite=' + filter_for_benchmark) alert_list, _, _ = anomaly.Anomaly.QueryAsync( sheriff=sheriff_key.id(), min_timestamp=cutoff, is_improvement=is_improvement, test_suite_name=filter_for_benchmark).get_result() else: raise api_request_handler.BadRequestError( 'Invalid alert type %s' % list_type) except AssertionError: # The only known assertion is in InternalOnlyModel._post_get_hook when a # non-internal user requests an internal-only entity. raise api_request_handler.BadRequestError('Not found') except request_handler.InvalidInputError as e: raise api_request_handler.BadRequestError(e.message) anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) response['anomalies'] = anomaly_dicts return response
def Post(self): """Returns alert data in response to API requests. Possible list types: keys: A comma-separated list of urlsafe Anomaly keys. bug_id: A bug number on the Chromium issue tracker. rev: A revision number. Outputs: Alerts data; see README.md. """ alert_list = None response = {} try: is_improvement = utils.ParseBool( self.request.get('is_improvement', None)) recovered = utils.ParseBool(self.request.get('recovered', None)) start_cursor = self.request.get('cursor', None) if start_cursor: start_cursor = datastore_query.Cursor(urlsafe=start_cursor) min_timestamp = utils.ParseISO8601( self.request.get('min_timestamp', None)) max_timestamp = utils.ParseISO8601( self.request.get('max_timestamp', None)) test_keys = [] for template_id in self.request.get_all('report'): test_keys.extend( report_template.TestKeysForReportTemplate(template_id)) try: alert_list, next_cursor, count = anomaly.Anomaly.QueryAsync( bot_name=self.request.get('bot', None), bug_id=self.request.get('bug_id', None), is_improvement=is_improvement, key=self.request.get('key', None), limit=int(self.request.get('limit', 100)), count_limit=int(self.request.get('count_limit', 0)), master_name=self.request.get('master', None), max_end_revision=self.request.get('max_end_revision', None), max_start_revision=self.request.get( 'max_start_revision', None), max_timestamp=max_timestamp, min_end_revision=self.request.get('min_end_revision', None), min_start_revision=self.request.get( 'min_start_revision', None), min_timestamp=min_timestamp, recovered=recovered, sheriff=self.request.get('sheriff', None), start_cursor=start_cursor, test=self.request.get('test', None), test_keys=test_keys, test_suite_name=self.request.get('test_suite', None)).get_result() response['count'] = count except AssertionError: alert_list, next_cursor = [], None if next_cursor: response['next_cursor'] = next_cursor.urlsafe() except AssertionError: # The only known assertion is in InternalOnlyModel._post_get_hook when a # non-internal user requests an internal-only entity. raise api_request_handler.BadRequestError('Not found') except request_handler.InvalidInputError as e: raise api_request_handler.BadRequestError(e.message) response['anomalies'] = alerts.AnomalyDicts( alert_list, utils.ParseBool(self.request.get('v2', None))) return response
def post(self): """Returns dynamic data for /group_report with some set of alerts. The set of alerts is determined by the sid, keys, bug ID, or revision given. Request parameters: keys: A comma-separated list of urlsafe Anomaly keys (optional). bug_id: A bug number on the Chromium issue tracker (optional). rev: A revision number (optional). sid: A hash of a group of keys from /short_uri (optional). Outputs: JSON for the /group_report page XHR request. """ bug_id = self.request.get('bug_id') rev = self.request.get('rev') keys = self.request.get('keys') hash_code = self.request.get('sid') # sid takes precedence. if hash_code: state = ndb.Key(page_state.PageState, hash_code).get() if state: keys = json.loads(state.value) elif keys: keys = keys.split(',') try: alert_list = None if bug_id: alert_list, extra_columns = GetAlertsWithBugId(bug_id) elif keys: alert_list, extra_columns = GetAlertsForKeys(keys) elif rev: alert_list, extra_columns = GetAlertsAroundRevision(rev) else: # TODO(qyearsley): Instead of just showing an error here, show a form # where the user can input a bug ID or revision. raise request_handler.InvalidInputError( 'No anomalies specified.') anomaly_dicts = alerts.AnomalyDicts( [a for a in alert_list if a.key.kind() == 'Anomaly']) stoppage_alert_dicts = alerts.StoppageAlertDicts( [a for a in alert_list if a.key.kind() == 'StoppageAlert']) alert_dicts = anomaly_dicts + stoppage_alert_dicts values = { 'alert_list': alert_dicts[:_DISPLAY_LIMIT], 'extra_columns': extra_columns, 'test_suites': update_test_suites.FetchCachedTestSuites(), } if bug_id: values['bug_id'] = bug_id if keys: values['selected_keys'] = keys self.GetDynamicVariables(values) self.response.out.write(json.dumps(values)) except request_handler.InvalidInputError as error: self.response.out.write(json.dumps({'error': str(error)}))