def testTriage_GroupUntriaged_MultiSubscriptions(self): anomalies = [self._AddAnomaly(), self._AddAnomaly()] group = self._AddAlertGroup( anomalies[0], status=alert_group.AlertGroup.Status.untriaged, ) self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff'), subscription.Subscription( name='sheriff_not_bind', auto_triage_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, config=alert_group_workflow.AlertGroupWorkflow.Config( active_window=datetime.timedelta(days=7), triage_delay=datetime.timedelta(hours=0), ), ) w.Process( update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=None, )) self.assertIsNone(self._issue_tracker.new_bug_args)
def testBisect_GroupTriaged_MultiSubscriptions(self): anomalies = [ self._AddAnomaly(median_before_anomaly=0.2), self._AddAnomaly(median_before_anomaly=0.1), ] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff'), subscription.Subscription(name='sheriff_not_bind', auto_triage_enable=True, auto_bisect_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, pinpoint=self._pinpoint, crrev=self._crrev, ) w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertIsNone(self._pinpoint.new_job_request)
def testMultipleAltertsGroupingMultipleSheriff(self, mock_get_sheriff_client): self._SetUpMocks(mock_get_sheriff_client) mock_get_sheriff_client().Match.return_value = ([ subscription.Subscription(name='sheriff1', auto_triage_enable=True, auto_bisect_enable=True), subscription.Subscription(name='sheriff2', auto_triage_enable=True, auto_bisect_enable=True), ], None) self.testapp.get('/alert_groups_update') self.ExecuteDeferredTasks('default') # Add anomaly a1 = self._AddAnomaly() # Create Group self._CallHandler() # Update Group to associate alerts self._CallHandler() # Add another anomaly with part of the subscription mock_get_sheriff_client().Match.return_value = ([ subscription.Subscription(name='sheriff1', auto_triage_enable=True, auto_bisect_enable=True), ], None) a2 = self._AddAnomaly() # Update Group to associate alerts self._CallHandler() # Add another anomaly with different subscription mock_get_sheriff_client().Match.return_value = ([ subscription.Subscription(name='sheriff2', auto_triage_enable=True, auto_bisect_enable=True), subscription.Subscription(name='sheriff3', auto_triage_enable=True, auto_bisect_enable=True), ], None) a3 = self._AddAnomaly() # Create Group self._CallHandler() # Update Group to associate alerts self._CallHandler() groups = { g.subscription_name: g for g in alert_group.AlertGroup.Get( 'test_suite', alert_group.AlertGroup.Type.test_suite, None) } self.assertItemsEqual(groups.keys(), ['sheriff1', 'sheriff2', 'sheriff3']) self.assertItemsEqual(groups['sheriff1'].anomalies, [a1, a2]) self.assertItemsEqual(groups['sheriff2'].anomalies, [a1, a3]) self.assertItemsEqual(groups['sheriff3'].anomalies, [a3])
def testAlertGroups_OnePerProject(self): self.mock_get_sheriff_client.Match.return_value = ([ subscription.Subscription( name='chromium sheriff', auto_triage_enable=True, monorail_project_id='chromium'), subscription.Subscription( name='v8 sheriff', auto_triage_enable=True, monorail_project_id='v8') ], None) self.PatchObject(alert_group.sheriff_config_client, 'GetSheriffConfigClient', lambda: self.mock_get_sheriff_client) self.PatchObject(alert_group, '_IssueTracker', lambda: self.fake_issue_tracker) # First create the 'Ungrouped' AlertGroup. self._CallHandler() # Then create an anomaly. self._AddAnomaly() self._CallHandler() # Ensure that we have two different groups on different projects. groups = alert_group.AlertGroup.Get('test_suite', None) self.assertEqual(2, len(groups)) self.assertItemsEqual(['chromium', 'v8'], [g.project_id for g in groups]) for group in groups: group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2) group.put() # And that we've filed two issues. self._CallHandler() self.assertItemsEqual([{ 'method': 'NewBug', 'args': (mock.ANY, mock.ANY), 'kwargs': { 'project': 'v8', 'cc': [], 'labels': mock.ANY, 'components': mock.ANY, }, }, { 'method': 'NewBug', 'args': (mock.ANY, mock.ANY), 'kwargs': { 'project': 'chromium', 'cc': [], 'labels': mock.ANY, 'components': mock.ANY, }, }], self.fake_issue_tracker.calls)
def testTriage_GroupUntriaged_MultipleRange(self): anomalies = [ self._AddAnomaly(median_before_anomaly=0.2, start_revision=10), self._AddAnomaly(median_before_anomaly=0.1) ] group = self._AddAlertGroup( anomalies[0], status=alert_group.AlertGroup.Status.untriaged, ) self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff', auto_triage_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, revision_info=self._revision_info, config=alert_group_workflow.AlertGroupWorkflow.Config( active_window=datetime.timedelta(days=7), triage_delay=datetime.timedelta(hours=0), ), ) w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=None, )) self.assertIn('2 regressions', self._issue_tracker.new_bug_args[0]) self.assertIn( 'Chromium Commit Position: http://test-results.appspot.com/revision_range?start=0&end=100', self._issue_tracker.new_bug_args[1])
def testBisect_GroupTriaged_PinpointFailed(self): anomalies = [self._AddAnomaly(), self._AddAnomaly()] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._pinpoint.SetFailure() self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, pinpoint=self._pinpoint, crrev=self._crrev, ) w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertEqual(alert_group.AlertGroup.Status.bisected, group.get().status) self.assertEqual([], group.get().bisection_ids) self.assertEqual(['Chromeperf-Auto-NeedsAttention'], self._issue_tracker.add_comment_kwargs['labels'])
def testTriage_GroupTriaged_InfAnomaly(self): anomalies = [ self._AddAnomaly(median_before_anomaly=0), self._AddAnomaly() ] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff', auto_triage_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, ) w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertIn('inf', self._issue_tracker.add_comment_args[1])
def testArchive_GroupTriaged(self): anomalies = [self._AddAnomaly(), self._AddAnomaly()] group = self._AddAlertGroup( anomalies[0], anomalies=anomalies, issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff', auto_triage_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, config=alert_group_workflow.AlertGroupWorkflow.Config( active_window=datetime.timedelta(days=0), triage_delay=datetime.timedelta(hours=0), ), ) w.Process( update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertEqual(True, group.get().active)
def testStartAutoBisection(self, mock_get_sheriff_client): self._SetUpMocks(mock_get_sheriff_client) mock_get_sheriff_client().Match.return_value = ([ subscription.Subscription( name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], None) self._CallHandler() # Add anomalies self._AddAnomaly() # Create Group self._CallHandler() # Update Group to associate alerts self._CallHandler() # Set Create timestamp to 2 hours ago group = alert_group.AlertGroup.Get('test_suite', None)[0] group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2) group.put() # Submit issue self._CallHandler() group = alert_group.AlertGroup.Get('test_suite', None)[0] # Start bisection self._CallHandler() group = alert_group.AlertGroup.Get('test_suite', None)[0] self.assertItemsEqual(group.bisection_ids, ['123456'])
def testManualClosedIssuesWithNewRegressions(self, mock_get_sheriff_client): # pylint: disable=no-value-for-parameter self.testClosesIssueOnAllRecovered() self._SetUpMocks(mock_get_sheriff_client) mock_get_sheriff_client().Match.return_value = ([ subscription.Subscription(name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], None) self.fake_issue_tracker.comments.append({ 'id': 2, 'author': "*****@*****.**", 'updates': { 'status': 'WontFix', }, }) # Then we add a new anomaly which should cause the issue to be reopened. self._AddAnomaly( start_revision=50, end_revision=75, test='master/bot/test_suite/measurement/other_test_case') self._CallHandler() logging.debug('Rendered:\n%s', self.fake_issue_tracker.add_comment_args[1]) self.assertEqual(self.fake_issue_tracker.issue["state"], 'closed') self.assertRegexpMatches(self.fake_issue_tracker.add_comment_args[1], r'test_suite/measurement/other_test_case')
def testAddAlertsAfterTriage(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff', auto_triage_enable=True) mock_get_sheriff_client().Match.return_value = ([sheriff], None) self.PatchObject(alert_group, '_IssueTracker', lambda: MockIssueTrackerService) self.testapp.get('/alert_groups_update') # Add anomalies a = self._AddAnomaly() # Create Group self.testapp.get('/alert_groups_update') # Update Group to associate alerts self.testapp.get('/alert_groups_update') # Set Create timestamp to 2 hours ago group = alert_group.AlertGroup.Get('test_suite', None)[0] group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2) group.put() # Submit issue self.testapp.get('/alert_groups_update') # Add anomalies anomalies = [ self._AddAnomaly(), self._AddAnomaly(), ] self.testapp.get('/alert_groups_update') for a in anomalies: self.assertEqual(a.get().bug_id, 12345) self.assertEqual(MockIssueTrackerService.add_comment_args[0], 12345)
def testUpdate_GroupTriaged_AlertsPartRecovered(self): anomalies = [self._AddAnomaly(recovered=True), self._AddAnomaly()] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff', auto_triage_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, ) w.Process( update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertEqual('open', self._issue_tracker.issue.get('state'))
def _SetUpMocks(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff', auto_triage_enable=True) mock_get_sheriff_client().Match.return_value = ([sheriff], None) self.PatchObject(alert_group_workflow, '_IssueTracker', lambda: self.fake_issue_tracker) self.PatchObject(crrev_service, 'GetNumbering', lambda *args, **kargs: {'git_sha': 'abcd'}) new_job = mock.MagicMock(return_value={'jobId': '123456'}) self.PatchObject(pinpoint_service, 'NewJob', new_job)
def testMultipleAltertsGrouping(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff') mock_get_sheriff_client().Match.return_value = ([sheriff], None) self.testapp.get('/alert_groups_update') # Add anomalies a1 = self._AddAnomaly() a2 = self._AddAnomaly(start_revision=50, end_revision=150) # Create Group self.testapp.get('/alert_groups_update') # Update Group to associate alerts self.testapp.get('/alert_groups_update') group = alert_group.AlertGroup.Get('test_suite', None)[0] self.assertItemsEqual(group.anomalies, [a1, a2])
def testCreatingGroup(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff') mock_get_sheriff_client().Match.return_value = ([sheriff], None) # Ungrouped is created in first run self._CallHandler() # Put an anomaly after Ungrouped is created a1 = self._AddAnomaly() # Anomaly is associated with Ungrouped and AlertGroup Created self._CallHandler() # Anomaly is associated with its AlertGroup self._CallHandler() self.assertEqual(len(a1.get().groups), 1) self.assertEqual(a1.get().groups[0].get().name, 'test_suite')
def testBisect_GroupTriaged_MultiBot_AllInf(self): anomalies = [ self._AddAnomaly( test='master/bot1/test_suite/measurement/test_case1', median_before_anomaly=0.0, median_after_anomaly=1.0, ), self._AddAnomaly( test='master/bot1/test_suite/measurement/test_case2', median_before_anomaly=0.0, median_after_anomaly=2.0, ), self._AddAnomaly( test='master/bot2/test_suite/measurement/test_case2', median_before_anomaly=0.1, ), ] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._sheriff_config.patterns = { '*': [ subscription.Subscription( name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, pinpoint=self._pinpoint, crrev=self._crrev, ) w.Process( update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertEqual( anomalies[1].urlsafe(), json.loads(self._pinpoint.new_job_request['tags'])['alert']) self.assertEqual(['123456'], group.get().bisection_ids)
def testBisect_GroupTriaged_NoIgnored(self): anomalies = [ # This anomaly is manually ignored. self._AddAnomaly( median_before_anomaly=0.1, median_after_anomaly=1.0, bug_id=-2), self._AddAnomaly( median_before_anomaly=0.2, median_after_anomaly=1.0, start_revision=20), ] group = self._AddAlertGroup( anomalies[1], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, anomalies=anomalies, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._sheriff_config.patterns = { '*': [ subscription.Subscription( name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, pinpoint=self._pinpoint, crrev=self._crrev, ) w.Process( update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertIsNotNone(self._pinpoint.new_job_request) self.assertEqual(self._pinpoint.new_job_request['bug_id'], 12345) self.assertEqual(group.get().status, alert_group.AlertGroup.Status.bisected) # Check that we bisected the anomaly that is not ignored. ignored_anomaly = anomalies[0].get() bisected_anomaly = anomalies[1].get() self.assertNotEqual(ignored_anomaly.pinpoint_bisects, ['123456']) self.assertEqual(bisected_anomaly.pinpoint_bisects, ['123456'])
def testFileIssue_InChromiumExplicitly(self): self.mock_get_sheriff_client.Match.return_value = ([ subscription.Subscription(name='sheriff', auto_triage_enable=True, monorail_project_id='chromium') ], None) self.PatchObject(alert_group.sheriff_config_client, 'GetSheriffConfigClient', lambda: self.mock_get_sheriff_client) self._SetUpMocks(self.mock_get_sheriff_client) self._CallHandler() a = self._AddAnomaly() self._CallHandler() grouped_anomaly = a.get() self.assertEqual(grouped_anomaly.project_id, 'chromium')
def testAlertGroups_NonChromium(self): self.mock_get_sheriff_client.Match.return_value = ([ subscription.Subscription(name='non-chromium sheriff', auto_triage_enable=True, monorail_project_id='non-chromium') ], None) self.PatchObject(alert_group.sheriff_config_client, 'GetSheriffConfigClient', lambda: self.mock_get_sheriff_client) self._SetUpMocks(self.mock_get_sheriff_client) self._CallHandler() a = self._AddAnomaly() self._CallHandler() groups = alert_group.AlertGroup.Get( 'test_suite', alert_group.AlertGroup.Type.test_suite, None) self.assertEqual(1, len(groups)) self.assertEqual(['non-chromium'], [g.project_id for g in groups]) for group in groups: group.created = datetime.datetime.utcnow() - datetime.timedelta( hours=2) group.put() self._CallHandler() self.assertItemsEqual([{ 'method': 'NewBug', 'args': (mock.ANY, mock.ANY), 'kwargs': { 'project': 'non-chromium', 'cc': [], 'labels': mock.ANY, 'components': mock.ANY, } }], self.fake_issue_tracker.calls) a = a.get() self.assertEqual(a.project_id, 'non-chromium') stored_issue = self.fake_issue_tracker.GetIssue( a.bug_id, 'non-chromium') logging.debug('bug_id = %s', a.bug_id) self.assertIsNotNone(stored_issue) # Now let's ensure that when new anomalies come in, that we're grouping # them into the same group for non-chromium alerts. self._AddAnomaly(start_revision=2) self._CallHandler() groups = alert_group.AlertGroup.Get( 'test_suite', alert_group.AlertGroup.Type.test_suite, None) self.assertEqual(1, len(groups)) self.assertEqual(groups[0].project_id, 'non-chromium')
def testBisect_GroupTriaged(self): anomalies = [ self._AddAnomaly(median_before_anomaly=0.2), self._AddAnomaly(median_before_anomaly=0.1), ] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._sheriff_config.patterns = { '*': [ subscription.Subscription( name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, pinpoint=self._pinpoint, crrev=self._crrev, ) w.Process( update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) tags = json.loads(self._pinpoint.new_job_request['tags']) self.assertEqual(anomalies[1].urlsafe(), tags['alert']) # Tags must be a dict of key/value string pairs. for k, v in tags.items(): self.assertIsInstance(k, basestring) self.assertIsInstance(v, basestring) self.assertEqual(['123456'], group.get().bisection_ids) self.assertEqual(['Chromeperf-Auto-Bisected'], self._issue_tracker.add_comment_kwargs['labels'])
def testArchiveAltertsGroup(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff') mock_get_sheriff_client().Match.return_value = ([sheriff], None) self.testapp.get('/alert_groups_update') # Add anomalies self._AddAnomaly() # Create Group self.testapp.get('/alert_groups_update') # Update Group to associate alerts self.testapp.get('/alert_groups_update') # Set Update timestamp to 10 days ago group = alert_group.AlertGroup.Get('test_suite', None)[0] group.updated = datetime.datetime.utcnow() - datetime.timedelta(days=10) group.put() # Archive Group self.testapp.get('/alert_groups_update') group = alert_group.AlertGroup.Get('test_suite', None, active=False)[0] self.assertEqual(group.name, 'test_suite')
def testMultipleAltertsGrouping(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff') mock_get_sheriff_client().Match.return_value = ([sheriff], None) self.testapp.get('/alert_groups_update') self.ExecuteDeferredTasks('default') # Add anomalies a1 = self._AddAnomaly() a2 = self._AddAnomaly(start_revision=50, end_revision=150) a3 = self._AddAnomaly(test='master/bot/other/measurement/test_case') a4 = self._AddAnomaly(median_before_anomaly=0) # Create Group self._CallHandler() # Update Group to associate alerts self._CallHandler() group = alert_group.AlertGroup.Get('test_suite', None)[0] self.assertItemsEqual(group.anomalies, [a1, a2, a4]) group = alert_group.AlertGroup.Get('other', None)[0] self.assertItemsEqual(group.anomalies, [a3])
def testAddAnomalies_GroupTriaged_IssueOpen(self): anomalies = [self._AddAnomaly(), self._AddAnomaly()] added = [self._AddAnomaly(), self._AddAnomaly()] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, anomalies=anomalies, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff', auto_triage_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, ) w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies + added), issue=self._issue_tracker.issue, )) self.assertEqual(len(group.get().anomalies), 4) self.assertEqual(group.get().status, alert_group.AlertGroup.Status.triaged) for a in added: self.assertIn(a, group.get().anomalies) self.assertEqual(group.get().bug.bug_id, self._issue_tracker.add_comment_args[0]) self.assertIn('Added 2 regressions to the group', self._issue_tracker.add_comment_args[1]) self.assertIn('4 regressions in test_suite', self._issue_tracker.add_comment_kwargs['summary']) self.assertIn('sheriff', self._issue_tracker.add_comment_kwargs['summary'])
def testBisect_GroupTriaged_AlertWithBug(self): anomalies = [ self._AddAnomaly(median_before_anomaly=0.2), self._AddAnomaly( median_before_anomaly=0.1, bug_id=12340, project_id='v8', ), ] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', }) self._sheriff_config.patterns = { '*': [ subscription.Subscription( name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, pinpoint=self._pinpoint, crrev=self._crrev, ) w.Process( update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertEqual(self._issue_tracker.bug_id, self._pinpoint.new_job_request['bug_id']) self.assertEqual('chromium', self._pinpoint.new_job_request['project']) self.assertEqual(['123456'], group.get().bisection_ids)
def testReopensClosedIssuesWithNewRegressions(self, mock_get_sheriff_client): # pylint: disable=no-value-for-parameter self.testClosesIssueOnAllRecovered() self._SetUpMocks(mock_get_sheriff_client) mock_get_sheriff_client().Match.return_value = ([ subscription.Subscription( name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], None) # Then we add a new anomaly which should cause the issue to be reopened. self._AddAnomaly( start_revision=50, end_revision=75, test='master/bot/test_suite/measurement/other_test_case') self._CallHandler() logging.debug('Rendered:\n%s', self.fake_issue_tracker.add_comment_args[1]) self.assertEqual(self.fake_issue_tracker.issue["state"], 'open') self.assertRegexpMatches( self.fake_issue_tracker.add_comment_args[1], r'Reopened due to new regressions detected for this alert group:') self.assertRegexpMatches(self.fake_issue_tracker.add_comment_args[1], r'test_suite/measurement/other_test_case')
def testAddAnomalies_GroupUntriaged(self): anomalies = [self._AddAnomaly(), self._AddAnomaly()] added = [self._AddAnomaly(), self._AddAnomaly()] group = self._AddAlertGroup(anomalies[0], anomalies=anomalies) self._sheriff_config.patterns = { '*': [subscription.Subscription(name='sheriff')], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, ) w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies + added), issue={}, )) self.assertEqual(len(group.get().anomalies), 4) for a in added: self.assertIn(a, group.get().anomalies)
def testTriage_GroupUntriaged_NonChromiumProject(self): anomalies = [self._AddAnomaly()] # TODO(dberris): Figure out a way to not have to hack the fake service to # seed it with the correct issue in the correct project. self._issue_tracker.issues[( 'v8', self._issue_tracker.bug_id)] = self._issue_tracker.issues[( 'chromium', self._issue_tracker.bug_id)] del self._issue_tracker.issues[('chromium', self._issue_tracker.bug_id)] self._issue_tracker.issues[('v8', self._issue_tracker.bug_id)].update({ 'projectId': 'v8', }) group = self._AddAlertGroup( anomalies[0], status=alert_group.AlertGroup.Status.untriaged, project_id='v8') self._sheriff_config.patterns = { '*': [ subscription.Subscription(name='sheriff', auto_triage_enable=True, monorail_project_id='v8') ], } self.assertEqual(group.get().project_id, 'v8') w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, revision_info=self._revision_info, config=alert_group_workflow.AlertGroupWorkflow.Config( active_window=datetime.timedelta(days=7), triage_delay=datetime.timedelta(hours=0), )) w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=None)) self.assertEqual(group.get().bug.project, 'v8') self.assertEqual(anomalies[0].get().project_id, 'v8')
def testBisect_ExplicitOptOut(self): anomalies = [self._AddAnomaly(), self._AddAnomaly()] group = self._AddAlertGroup( anomalies[0], issue=self._issue_tracker.issue, status=alert_group.AlertGroup.Status.triaged, ) self._issue_tracker.issue.update({ 'state': 'open', 'labels': self._issue_tracker.issue.get('labels') + ['Chromeperf-Auto-BisectOptOut'] }) self._sheriff_config.patterns = { '*': [ subscription.Subscription( name='sheriff', auto_triage_enable=True, auto_bisect_enable=True) ], } w = alert_group_workflow.AlertGroupWorkflow( group.get(), sheriff_config=self._sheriff_config, issue_tracker=self._issue_tracker, pinpoint=self._pinpoint, crrev=self._crrev, ) self.assertIn('Chromeperf-Auto-BisectOptOut', self._issue_tracker.issue.get('labels')) w.Process( update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate( now=datetime.datetime.utcnow(), anomalies=ndb.get_multi(anomalies), issue=self._issue_tracker.issue, )) self.assertIsNone(self._pinpoint.new_job_request)
def testAddAlertsAfterTriage(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff', auto_triage_enable=True) mock_get_sheriff_client().Match.return_value = ([sheriff], None) self.PatchObject(alert_group, '_IssueTracker', lambda: FakeIssueTrackerService) self._CallHandler() # Add anomalies a = self._AddAnomaly() # Create Group self._CallHandler() # Update Group to associate alerts self._CallHandler() # Set Create timestamp to 2 hours ago group = alert_group.AlertGroup.Get('test_suite', None)[0] group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2) group.put() # Submit issue self._CallHandler() # Add anomalies anomalies = [ self._AddAnomaly(), self._AddAnomaly(median_before_anomaly=0), ] self._CallHandler() for a in anomalies: self.assertEqual(a.get().bug_id, 12345) logging.debug('Rendered:\n%s', FakeIssueTrackerService.add_comment_args[1]) self.assertEqual(FakeIssueTrackerService.add_comment_args[0], 12345) self.assertItemsEqual( FakeIssueTrackerService.add_comment_kwargs['components'], ['Foo>Bar']) self.assertItemsEqual(FakeIssueTrackerService.add_comment_kwargs['labels'], [ 'Pri-2', 'Restrict-View-Google', 'Type-Bug-Regression', 'Chromeperf-Auto-Triaged' ]) self.assertRegexpMatches(FakeIssueTrackerService.add_comment_args[1], r'Top 2 affected measurements in bot:')
def testTriageAltertsGroup(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff', auto_triage_enable=True) mock_get_sheriff_client().Match.return_value = ([sheriff], None) self.PatchObject(alert_group, '_IssueTracker', lambda: MockIssueTrackerService) self.testapp.get('/alert_groups_update') # Add anomalies a = self._AddAnomaly() # Create Group self.testapp.get('/alert_groups_update') # Update Group to associate alerts self.testapp.get('/alert_groups_update') # Set Create timestamp to 2 hours ago group = alert_group.AlertGroup.Get('test_suite', None)[0] group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2) group.put() # Submit issue self.testapp.get('/alert_groups_update') group = alert_group.AlertGroup.Get('test_suite', None)[0] self.assertEqual(group.status, alert_group.AlertGroup.Status.triaged) self.assertItemsEqual(MockIssueTrackerService.new_bug_kwargs['components'], ['Foo>Bar']) self.assertEqual(a.get().bug_id, 12345)