def testTriage_GroupUntriaged_MultiSubscriptions(self):
anomalies = [self._AddAnomaly(), self._AddAnomaly()]
group = self._AddAlertGroup(
anomalies[0],
status=alert_group.AlertGroup.Status.untriaged,
)
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff'),
subscription.Subscription(
name='sheriff_not_bind', auto_triage_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
config=alert_group_workflow.AlertGroupWorkflow.Config(
active_window=datetime.timedelta(days=7),
triage_delay=datetime.timedelta(hours=0),
),
)
w.Process(
update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=None,
))
self.assertIsNone(self._issue_tracker.new_bug_args)
def testBisect_GroupTriaged_MultiSubscriptions(self):
anomalies = [
self._AddAnomaly(median_before_anomaly=0.2),
self._AddAnomaly(median_before_anomaly=0.1),
]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff'),
subscription.Subscription(name='sheriff_not_bind',
auto_triage_enable=True,
auto_bisect_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
pinpoint=self._pinpoint,
crrev=self._crrev,
)
w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertIsNone(self._pinpoint.new_job_request)
def testMultipleAltertsGroupingMultipleSheriff(self,
mock_get_sheriff_client):
self._SetUpMocks(mock_get_sheriff_client)
mock_get_sheriff_client().Match.return_value = ([
subscription.Subscription(name='sheriff1',
auto_triage_enable=True,
auto_bisect_enable=True),
subscription.Subscription(name='sheriff2',
auto_triage_enable=True,
auto_bisect_enable=True),
], None)
self.testapp.get('/alert_groups_update')
self.ExecuteDeferredTasks('default')
# Add anomaly
a1 = self._AddAnomaly()
# Create Group
self._CallHandler()
# Update Group to associate alerts
self._CallHandler()
# Add another anomaly with part of the subscription
mock_get_sheriff_client().Match.return_value = ([
subscription.Subscription(name='sheriff1',
auto_triage_enable=True,
auto_bisect_enable=True),
], None)
a2 = self._AddAnomaly()
# Update Group to associate alerts
self._CallHandler()
# Add another anomaly with different subscription
mock_get_sheriff_client().Match.return_value = ([
subscription.Subscription(name='sheriff2',
auto_triage_enable=True,
auto_bisect_enable=True),
subscription.Subscription(name='sheriff3',
auto_triage_enable=True,
auto_bisect_enable=True),
], None)
a3 = self._AddAnomaly()
# Create Group
self._CallHandler()
# Update Group to associate alerts
self._CallHandler()
groups = {
g.subscription_name: g
for g in alert_group.AlertGroup.Get(
'test_suite', alert_group.AlertGroup.Type.test_suite, None)
}
self.assertItemsEqual(groups.keys(),
['sheriff1', 'sheriff2', 'sheriff3'])
self.assertItemsEqual(groups['sheriff1'].anomalies, [a1, a2])
self.assertItemsEqual(groups['sheriff2'].anomalies, [a1, a3])
self.assertItemsEqual(groups['sheriff3'].anomalies, [a3])
def testAlertGroups_OnePerProject(self):
self.mock_get_sheriff_client.Match.return_value = ([
subscription.Subscription(
name='chromium sheriff',
auto_triage_enable=True,
monorail_project_id='chromium'),
subscription.Subscription(
name='v8 sheriff',
auto_triage_enable=True,
monorail_project_id='v8')
], None)
self.PatchObject(alert_group.sheriff_config_client,
'GetSheriffConfigClient',
lambda: self.mock_get_sheriff_client)
self.PatchObject(alert_group, '_IssueTracker',
lambda: self.fake_issue_tracker)
# First create the 'Ungrouped' AlertGroup.
self._CallHandler()
# Then create an anomaly.
self._AddAnomaly()
self._CallHandler()
# Ensure that we have two different groups on different projects.
groups = alert_group.AlertGroup.Get('test_suite', None)
self.assertEqual(2, len(groups))
self.assertItemsEqual(['chromium', 'v8'], [g.project_id for g in groups])
for group in groups:
group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
group.put()
# And that we've filed two issues.
self._CallHandler()
self.assertItemsEqual([{
'method': 'NewBug',
'args': (mock.ANY, mock.ANY),
'kwargs': {
'project': 'v8',
'cc': [],
'labels': mock.ANY,
'components': mock.ANY,
},
}, {
'method': 'NewBug',
'args': (mock.ANY, mock.ANY),
'kwargs': {
'project': 'chromium',
'cc': [],
'labels': mock.ANY,
'components': mock.ANY,
},
}], self.fake_issue_tracker.calls)
def testTriage_GroupUntriaged_MultipleRange(self):
anomalies = [
self._AddAnomaly(median_before_anomaly=0.2, start_revision=10),
self._AddAnomaly(median_before_anomaly=0.1)
]
group = self._AddAlertGroup(
anomalies[0],
status=alert_group.AlertGroup.Status.untriaged,
)
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff',
auto_triage_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
revision_info=self._revision_info,
config=alert_group_workflow.AlertGroupWorkflow.Config(
active_window=datetime.timedelta(days=7),
triage_delay=datetime.timedelta(hours=0),
),
)
w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=None,
))
self.assertIn('2 regressions', self._issue_tracker.new_bug_args[0])
self.assertIn(
'Chromium Commit Position: http://test-results.appspot.com/revision_range?start=0&end=100',
self._issue_tracker.new_bug_args[1])
def testBisect_GroupTriaged_PinpointFailed(self):
anomalies = [self._AddAnomaly(), self._AddAnomaly()]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._pinpoint.SetFailure()
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff',
auto_triage_enable=True,
auto_bisect_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
pinpoint=self._pinpoint,
crrev=self._crrev,
)
w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertEqual(alert_group.AlertGroup.Status.bisected,
group.get().status)
self.assertEqual([], group.get().bisection_ids)
self.assertEqual(['Chromeperf-Auto-NeedsAttention'],
self._issue_tracker.add_comment_kwargs['labels'])
def testTriage_GroupTriaged_InfAnomaly(self):
anomalies = [
self._AddAnomaly(median_before_anomaly=0),
self._AddAnomaly()
]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff',
auto_triage_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
)
w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertIn('inf', self._issue_tracker.add_comment_args[1])
def testArchive_GroupTriaged(self):
anomalies = [self._AddAnomaly(), self._AddAnomaly()]
group = self._AddAlertGroup(
anomalies[0],
anomalies=anomalies,
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff', auto_triage_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
config=alert_group_workflow.AlertGroupWorkflow.Config(
active_window=datetime.timedelta(days=0),
triage_delay=datetime.timedelta(hours=0),
),
)
w.Process(
update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertEqual(True, group.get().active)
def testStartAutoBisection(self, mock_get_sheriff_client):
self._SetUpMocks(mock_get_sheriff_client)
mock_get_sheriff_client().Match.return_value = ([
subscription.Subscription(
name='sheriff', auto_triage_enable=True, auto_bisect_enable=True)
], None)
self._CallHandler()
# Add anomalies
self._AddAnomaly()
# Create Group
self._CallHandler()
# Update Group to associate alerts
self._CallHandler()
# Set Create timestamp to 2 hours ago
group = alert_group.AlertGroup.Get('test_suite', None)[0]
group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
group.put()
# Submit issue
self._CallHandler()
group = alert_group.AlertGroup.Get('test_suite', None)[0]
# Start bisection
self._CallHandler()
group = alert_group.AlertGroup.Get('test_suite', None)[0]
self.assertItemsEqual(group.bisection_ids, ['123456'])
def testManualClosedIssuesWithNewRegressions(self,
mock_get_sheriff_client):
# pylint: disable=no-value-for-parameter
self.testClosesIssueOnAllRecovered()
self._SetUpMocks(mock_get_sheriff_client)
mock_get_sheriff_client().Match.return_value = ([
subscription.Subscription(name='sheriff',
auto_triage_enable=True,
auto_bisect_enable=True)
], None)
self.fake_issue_tracker.comments.append({
'id': 2,
'author': "*****@*****.**",
'updates': {
'status': 'WontFix',
},
})
# Then we add a new anomaly which should cause the issue to be reopened.
self._AddAnomaly(
start_revision=50,
end_revision=75,
test='master/bot/test_suite/measurement/other_test_case')
self._CallHandler()
logging.debug('Rendered:\n%s',
self.fake_issue_tracker.add_comment_args[1])
self.assertEqual(self.fake_issue_tracker.issue["state"], 'closed')
self.assertRegexpMatches(self.fake_issue_tracker.add_comment_args[1],
r'test_suite/measurement/other_test_case')
def testAddAlertsAfterTriage(self, mock_get_sheriff_client):
sheriff = subscription.Subscription(name='sheriff', auto_triage_enable=True)
mock_get_sheriff_client().Match.return_value = ([sheriff], None)
self.PatchObject(alert_group, '_IssueTracker',
lambda: MockIssueTrackerService)
self.testapp.get('/alert_groups_update')
# Add anomalies
a = self._AddAnomaly()
# Create Group
self.testapp.get('/alert_groups_update')
# Update Group to associate alerts
self.testapp.get('/alert_groups_update')
# Set Create timestamp to 2 hours ago
group = alert_group.AlertGroup.Get('test_suite', None)[0]
group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
group.put()
# Submit issue
self.testapp.get('/alert_groups_update')
# Add anomalies
anomalies = [
self._AddAnomaly(),
self._AddAnomaly(),
]
self.testapp.get('/alert_groups_update')
for a in anomalies:
self.assertEqual(a.get().bug_id, 12345)
self.assertEqual(MockIssueTrackerService.add_comment_args[0], 12345)
def testUpdate_GroupTriaged_AlertsPartRecovered(self):
anomalies = [self._AddAnomaly(recovered=True), self._AddAnomaly()]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff', auto_triage_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
)
w.Process(
update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertEqual('open', self._issue_tracker.issue.get('state'))
def _SetUpMocks(self, mock_get_sheriff_client):
sheriff = subscription.Subscription(name='sheriff', auto_triage_enable=True)
mock_get_sheriff_client().Match.return_value = ([sheriff], None)
self.PatchObject(alert_group_workflow, '_IssueTracker',
lambda: self.fake_issue_tracker)
self.PatchObject(crrev_service, 'GetNumbering',
lambda *args, **kargs: {'git_sha': 'abcd'})
new_job = mock.MagicMock(return_value={'jobId': '123456'})
self.PatchObject(pinpoint_service, 'NewJob', new_job)
def testMultipleAltertsGrouping(self, mock_get_sheriff_client):
sheriff = subscription.Subscription(name='sheriff')
mock_get_sheriff_client().Match.return_value = ([sheriff], None)
self.testapp.get('/alert_groups_update')
# Add anomalies
a1 = self._AddAnomaly()
a2 = self._AddAnomaly(start_revision=50, end_revision=150)
# Create Group
self.testapp.get('/alert_groups_update')
# Update Group to associate alerts
self.testapp.get('/alert_groups_update')
group = alert_group.AlertGroup.Get('test_suite', None)[0]
self.assertItemsEqual(group.anomalies, [a1, a2])
def testCreatingGroup(self, mock_get_sheriff_client): sheriff = subscription.Subscription(name='sheriff') mock_get_sheriff_client().Match.return_value = ([sheriff], None) # Ungrouped is created in first run self._CallHandler() # Put an anomaly after Ungrouped is created a1 = self._AddAnomaly() # Anomaly is associated with Ungrouped and AlertGroup Created self._CallHandler() # Anomaly is associated with its AlertGroup self._CallHandler() self.assertEqual(len(a1.get().groups), 1) self.assertEqual(a1.get().groups[0].get().name, 'test_suite')
def testBisect_GroupTriaged_MultiBot_AllInf(self):
anomalies = [
self._AddAnomaly(
test='master/bot1/test_suite/measurement/test_case1',
median_before_anomaly=0.0,
median_after_anomaly=1.0,
),
self._AddAnomaly(
test='master/bot1/test_suite/measurement/test_case2',
median_before_anomaly=0.0,
median_after_anomaly=2.0,
),
self._AddAnomaly(
test='master/bot2/test_suite/measurement/test_case2',
median_before_anomaly=0.1,
),
]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(
name='sheriff',
auto_triage_enable=True,
auto_bisect_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
pinpoint=self._pinpoint,
crrev=self._crrev,
)
w.Process(
update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertEqual(
anomalies[1].urlsafe(),
json.loads(self._pinpoint.new_job_request['tags'])['alert'])
self.assertEqual(['123456'], group.get().bisection_ids)
def testBisect_GroupTriaged_NoIgnored(self):
anomalies = [
# This anomaly is manually ignored.
self._AddAnomaly(
median_before_anomaly=0.1, median_after_anomaly=1.0, bug_id=-2),
self._AddAnomaly(
median_before_anomaly=0.2,
median_after_anomaly=1.0,
start_revision=20),
]
group = self._AddAlertGroup(
anomalies[1],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
anomalies=anomalies,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(
name='sheriff',
auto_triage_enable=True,
auto_bisect_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
pinpoint=self._pinpoint,
crrev=self._crrev,
)
w.Process(
update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertIsNotNone(self._pinpoint.new_job_request)
self.assertEqual(self._pinpoint.new_job_request['bug_id'], 12345)
self.assertEqual(group.get().status, alert_group.AlertGroup.Status.bisected)
# Check that we bisected the anomaly that is not ignored.
ignored_anomaly = anomalies[0].get()
bisected_anomaly = anomalies[1].get()
self.assertNotEqual(ignored_anomaly.pinpoint_bisects, ['123456'])
self.assertEqual(bisected_anomaly.pinpoint_bisects, ['123456'])
def testFileIssue_InChromiumExplicitly(self):
self.mock_get_sheriff_client.Match.return_value = ([
subscription.Subscription(name='sheriff',
auto_triage_enable=True,
monorail_project_id='chromium')
], None)
self.PatchObject(alert_group.sheriff_config_client,
'GetSheriffConfigClient',
lambda: self.mock_get_sheriff_client)
self._SetUpMocks(self.mock_get_sheriff_client)
self._CallHandler()
a = self._AddAnomaly()
self._CallHandler()
grouped_anomaly = a.get()
self.assertEqual(grouped_anomaly.project_id, 'chromium')
def testAlertGroups_NonChromium(self):
self.mock_get_sheriff_client.Match.return_value = ([
subscription.Subscription(name='non-chromium sheriff',
auto_triage_enable=True,
monorail_project_id='non-chromium')
], None)
self.PatchObject(alert_group.sheriff_config_client,
'GetSheriffConfigClient',
lambda: self.mock_get_sheriff_client)
self._SetUpMocks(self.mock_get_sheriff_client)
self._CallHandler()
a = self._AddAnomaly()
self._CallHandler()
groups = alert_group.AlertGroup.Get(
'test_suite', alert_group.AlertGroup.Type.test_suite, None)
self.assertEqual(1, len(groups))
self.assertEqual(['non-chromium'], [g.project_id for g in groups])
for group in groups:
group.created = datetime.datetime.utcnow() - datetime.timedelta(
hours=2)
group.put()
self._CallHandler()
self.assertItemsEqual([{
'method': 'NewBug',
'args': (mock.ANY, mock.ANY),
'kwargs': {
'project': 'non-chromium',
'cc': [],
'labels': mock.ANY,
'components': mock.ANY,
}
}], self.fake_issue_tracker.calls)
a = a.get()
self.assertEqual(a.project_id, 'non-chromium')
stored_issue = self.fake_issue_tracker.GetIssue(
a.bug_id, 'non-chromium')
logging.debug('bug_id = %s', a.bug_id)
self.assertIsNotNone(stored_issue)
# Now let's ensure that when new anomalies come in, that we're grouping
# them into the same group for non-chromium alerts.
self._AddAnomaly(start_revision=2)
self._CallHandler()
groups = alert_group.AlertGroup.Get(
'test_suite', alert_group.AlertGroup.Type.test_suite, None)
self.assertEqual(1, len(groups))
self.assertEqual(groups[0].project_id, 'non-chromium')
def testBisect_GroupTriaged(self):
anomalies = [
self._AddAnomaly(median_before_anomaly=0.2),
self._AddAnomaly(median_before_anomaly=0.1),
]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(
name='sheriff',
auto_triage_enable=True,
auto_bisect_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
pinpoint=self._pinpoint,
crrev=self._crrev,
)
w.Process(
update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
tags = json.loads(self._pinpoint.new_job_request['tags'])
self.assertEqual(anomalies[1].urlsafe(), tags['alert'])
# Tags must be a dict of key/value string pairs.
for k, v in tags.items():
self.assertIsInstance(k, basestring)
self.assertIsInstance(v, basestring)
self.assertEqual(['123456'], group.get().bisection_ids)
self.assertEqual(['Chromeperf-Auto-Bisected'],
self._issue_tracker.add_comment_kwargs['labels'])
def testArchiveAltertsGroup(self, mock_get_sheriff_client):
sheriff = subscription.Subscription(name='sheriff')
mock_get_sheriff_client().Match.return_value = ([sheriff], None)
self.testapp.get('/alert_groups_update')
# Add anomalies
self._AddAnomaly()
# Create Group
self.testapp.get('/alert_groups_update')
# Update Group to associate alerts
self.testapp.get('/alert_groups_update')
# Set Update timestamp to 10 days ago
group = alert_group.AlertGroup.Get('test_suite', None)[0]
group.updated = datetime.datetime.utcnow() - datetime.timedelta(days=10)
group.put()
# Archive Group
self.testapp.get('/alert_groups_update')
group = alert_group.AlertGroup.Get('test_suite', None, active=False)[0]
self.assertEqual(group.name, 'test_suite')
def testMultipleAltertsGrouping(self, mock_get_sheriff_client):
sheriff = subscription.Subscription(name='sheriff')
mock_get_sheriff_client().Match.return_value = ([sheriff], None)
self.testapp.get('/alert_groups_update')
self.ExecuteDeferredTasks('default')
# Add anomalies
a1 = self._AddAnomaly()
a2 = self._AddAnomaly(start_revision=50, end_revision=150)
a3 = self._AddAnomaly(test='master/bot/other/measurement/test_case')
a4 = self._AddAnomaly(median_before_anomaly=0)
# Create Group
self._CallHandler()
# Update Group to associate alerts
self._CallHandler()
group = alert_group.AlertGroup.Get('test_suite', None)[0]
self.assertItemsEqual(group.anomalies, [a1, a2, a4])
group = alert_group.AlertGroup.Get('other', None)[0]
self.assertItemsEqual(group.anomalies, [a3])
def testAddAnomalies_GroupTriaged_IssueOpen(self):
anomalies = [self._AddAnomaly(), self._AddAnomaly()]
added = [self._AddAnomaly(), self._AddAnomaly()]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
anomalies=anomalies,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff',
auto_triage_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
)
w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies + added),
issue=self._issue_tracker.issue,
))
self.assertEqual(len(group.get().anomalies), 4)
self.assertEqual(group.get().status,
alert_group.AlertGroup.Status.triaged)
for a in added:
self.assertIn(a, group.get().anomalies)
self.assertEqual(group.get().bug.bug_id,
self._issue_tracker.add_comment_args[0])
self.assertIn('Added 2 regressions to the group',
self._issue_tracker.add_comment_args[1])
self.assertIn('4 regressions in test_suite',
self._issue_tracker.add_comment_kwargs['summary'])
self.assertIn('sheriff',
self._issue_tracker.add_comment_kwargs['summary'])
def testBisect_GroupTriaged_AlertWithBug(self):
anomalies = [
self._AddAnomaly(median_before_anomaly=0.2),
self._AddAnomaly(
median_before_anomaly=0.1,
bug_id=12340,
project_id='v8',
),
]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state': 'open',
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(
name='sheriff',
auto_triage_enable=True,
auto_bisect_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
pinpoint=self._pinpoint,
crrev=self._crrev,
)
w.Process(
update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertEqual(self._issue_tracker.bug_id,
self._pinpoint.new_job_request['bug_id'])
self.assertEqual('chromium', self._pinpoint.new_job_request['project'])
self.assertEqual(['123456'], group.get().bisection_ids)
def testReopensClosedIssuesWithNewRegressions(self, mock_get_sheriff_client):
# pylint: disable=no-value-for-parameter
self.testClosesIssueOnAllRecovered()
self._SetUpMocks(mock_get_sheriff_client)
mock_get_sheriff_client().Match.return_value = ([
subscription.Subscription(
name='sheriff', auto_triage_enable=True, auto_bisect_enable=True)
], None)
# Then we add a new anomaly which should cause the issue to be reopened.
self._AddAnomaly(
start_revision=50,
end_revision=75,
test='master/bot/test_suite/measurement/other_test_case')
self._CallHandler()
logging.debug('Rendered:\n%s', self.fake_issue_tracker.add_comment_args[1])
self.assertEqual(self.fake_issue_tracker.issue["state"], 'open')
self.assertRegexpMatches(
self.fake_issue_tracker.add_comment_args[1],
r'Reopened due to new regressions detected for this alert group:')
self.assertRegexpMatches(self.fake_issue_tracker.add_comment_args[1],
r'test_suite/measurement/other_test_case')
def testAddAnomalies_GroupUntriaged(self):
anomalies = [self._AddAnomaly(), self._AddAnomaly()]
added = [self._AddAnomaly(), self._AddAnomaly()]
group = self._AddAlertGroup(anomalies[0], anomalies=anomalies)
self._sheriff_config.patterns = {
'*': [subscription.Subscription(name='sheriff')],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
)
w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies + added),
issue={},
))
self.assertEqual(len(group.get().anomalies), 4)
for a in added:
self.assertIn(a, group.get().anomalies)
def testTriage_GroupUntriaged_NonChromiumProject(self):
anomalies = [self._AddAnomaly()]
# TODO(dberris): Figure out a way to not have to hack the fake service to
# seed it with the correct issue in the correct project.
self._issue_tracker.issues[(
'v8', self._issue_tracker.bug_id)] = self._issue_tracker.issues[(
'chromium', self._issue_tracker.bug_id)]
del self._issue_tracker.issues[('chromium',
self._issue_tracker.bug_id)]
self._issue_tracker.issues[('v8', self._issue_tracker.bug_id)].update({
'projectId':
'v8',
})
group = self._AddAlertGroup(
anomalies[0],
status=alert_group.AlertGroup.Status.untriaged,
project_id='v8')
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(name='sheriff',
auto_triage_enable=True,
monorail_project_id='v8')
],
}
self.assertEqual(group.get().project_id, 'v8')
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
revision_info=self._revision_info,
config=alert_group_workflow.AlertGroupWorkflow.Config(
active_window=datetime.timedelta(days=7),
triage_delay=datetime.timedelta(hours=0),
))
w.Process(update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=None))
self.assertEqual(group.get().bug.project, 'v8')
self.assertEqual(anomalies[0].get().project_id, 'v8')
def testBisect_ExplicitOptOut(self):
anomalies = [self._AddAnomaly(), self._AddAnomaly()]
group = self._AddAlertGroup(
anomalies[0],
issue=self._issue_tracker.issue,
status=alert_group.AlertGroup.Status.triaged,
)
self._issue_tracker.issue.update({
'state':
'open',
'labels':
self._issue_tracker.issue.get('labels') +
['Chromeperf-Auto-BisectOptOut']
})
self._sheriff_config.patterns = {
'*': [
subscription.Subscription(
name='sheriff',
auto_triage_enable=True,
auto_bisect_enable=True)
],
}
w = alert_group_workflow.AlertGroupWorkflow(
group.get(),
sheriff_config=self._sheriff_config,
issue_tracker=self._issue_tracker,
pinpoint=self._pinpoint,
crrev=self._crrev,
)
self.assertIn('Chromeperf-Auto-BisectOptOut',
self._issue_tracker.issue.get('labels'))
w.Process(
update=alert_group_workflow.AlertGroupWorkflow.GroupUpdate(
now=datetime.datetime.utcnow(),
anomalies=ndb.get_multi(anomalies),
issue=self._issue_tracker.issue,
))
self.assertIsNone(self._pinpoint.new_job_request)
def testAddAlertsAfterTriage(self, mock_get_sheriff_client):
sheriff = subscription.Subscription(name='sheriff', auto_triage_enable=True)
mock_get_sheriff_client().Match.return_value = ([sheriff], None)
self.PatchObject(alert_group, '_IssueTracker',
lambda: FakeIssueTrackerService)
self._CallHandler()
# Add anomalies
a = self._AddAnomaly()
# Create Group
self._CallHandler()
# Update Group to associate alerts
self._CallHandler()
# Set Create timestamp to 2 hours ago
group = alert_group.AlertGroup.Get('test_suite', None)[0]
group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
group.put()
# Submit issue
self._CallHandler()
# Add anomalies
anomalies = [
self._AddAnomaly(),
self._AddAnomaly(median_before_anomaly=0),
]
self._CallHandler()
for a in anomalies:
self.assertEqual(a.get().bug_id, 12345)
logging.debug('Rendered:\n%s', FakeIssueTrackerService.add_comment_args[1])
self.assertEqual(FakeIssueTrackerService.add_comment_args[0], 12345)
self.assertItemsEqual(
FakeIssueTrackerService.add_comment_kwargs['components'], ['Foo>Bar'])
self.assertItemsEqual(FakeIssueTrackerService.add_comment_kwargs['labels'],
[
'Pri-2', 'Restrict-View-Google',
'Type-Bug-Regression', 'Chromeperf-Auto-Triaged'
])
self.assertRegexpMatches(FakeIssueTrackerService.add_comment_args[1],
r'Top 2 affected measurements in bot:')
def testTriageAltertsGroup(self, mock_get_sheriff_client):
sheriff = subscription.Subscription(name='sheriff', auto_triage_enable=True)
mock_get_sheriff_client().Match.return_value = ([sheriff], None)
self.PatchObject(alert_group, '_IssueTracker',
lambda: MockIssueTrackerService)
self.testapp.get('/alert_groups_update')
# Add anomalies
a = self._AddAnomaly()
# Create Group
self.testapp.get('/alert_groups_update')
# Update Group to associate alerts
self.testapp.get('/alert_groups_update')
# Set Create timestamp to 2 hours ago
group = alert_group.AlertGroup.Get('test_suite', None)[0]
group.created = datetime.datetime.utcnow() - datetime.timedelta(hours=2)
group.put()
# Submit issue
self.testapp.get('/alert_groups_update')
group = alert_group.AlertGroup.Get('test_suite', None)[0]
self.assertEqual(group.status, alert_group.AlertGroup.Status.triaged)
self.assertItemsEqual(MockIssueTrackerService.new_bug_kwargs['components'],
['Foo>Bar'])
self.assertEqual(a.get().bug_id, 12345)
