def MarkAlertAndBugIfRecovered(self, alert_key_urlsafe, bug_id):
"""Checks whether an alert has recovered, and marks it if so.
An alert will be considered "recovered" if there's a change point in
the series after it with roughly equal magnitude and opposite direction.
Args:
alert_key_urlsafe: The original regression Anomaly.
Returns:
True if the Anomaly should be marked as recovered, False otherwise.
"""
alert_entity = ndb.Key(urlsafe=alert_key_urlsafe).get()
logging.info('Checking alert %s', alert_entity)
if not self._IsAlertRecovered(alert_entity):
return
logging.info('Recovered')
alert_entity.recovered = True
alert_entity.put()
if bug_id:
unrecovered, _, _ = anomaly.Anomaly.QueryAsync(
bug_id=bug_id, recovered=False).get_result()
if not unrecovered:
# All alerts recovered! Update bug.
logging.info('All alerts for bug %s recovered!', bug_id)
comment = 'Automatic message: All alerts recovered.\nGraphs: %s' % (
'https://chromeperf.appspot.com/group_report?bug_id=%s' %
bug_id)
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(
bug_id, comment, labels='Performance-Regression-Recovered')
def testAddBugComment_Basic(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._MakeCommentRequest = mock.Mock()
self.assertTrue(service.AddBugComment(12345, 'The comment'))
self.assertEqual(1, service._MakeCommentRequest.call_count)
service._MakeCommentRequest.assert_called_with(
12345, {'updates': {}, 'content': 'The comment'}, send_email=True)
def UpdatePostAndMergeDeferred(bug_update_builder, bug_id, tags, url, project):
if not bug_id:
return
commit_cache_key = bug_update_builder.GenerateCommitCacheKey()
bug_update = bug_update_builder.BuildUpdate(tags, url)
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
merge_details, cc_list = _ComputePostMergeDetails(issue_tracker,
commit_cache_key,
bug_update.cc_list)
current_bug_status = _GetBugStatus(issue_tracker, bug_id, project=project)
if not current_bug_status:
return
status = None
bug_owner = None
if current_bug_status in ['Untriaged', 'Unconfirmed', 'Available']:
# Set the bug status and owner if this bug is opened and unowned.
status = 'Assigned'
bug_owner = bug_update.owner
issue_tracker.AddBugComment(
bug_id,
bug_update.comment_text,
status=status,
cc_list=sorted(cc_list),
owner=bug_owner,
labels=bug_update.labels,
merge_issue=merge_details.get('id'),
project=project)
update_bug_with_results.UpdateMergeIssue(
commit_cache_key, merge_details, bug_id, project=project)
def _UpdatePostAndMergeDeferred(
difference_details, commit_infos, authors_deltas, bug_id, tags, url):
if not bug_id:
return
commit_cache_key = _GenerateCommitCacheKey(commit_infos)
# Bring it all together.
owner, sheriff, cc_list = _ComputePostOwnerSheriffCCList(commit_infos,
authors_deltas)
comment = _FormatComment(difference_details, commit_infos, sheriff, tags, url)
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
merge_details, cc_list = _ComputePostMergeDetails(
issue_tracker, commit_cache_key, cc_list)
current_bug_status = _GetBugStatus(issue_tracker, bug_id)
if not current_bug_status:
return
status = None
bug_owner = None
if current_bug_status in ['Untriaged', 'Unconfirmed', 'Available']:
# Set the bug status and owner if this bug is opened and unowned.
status = 'Assigned'
bug_owner = owner
issue_tracker.AddBugComment(bug_id, comment, status=status,
cc_list=sorted(cc_list), owner=bug_owner,
merge_issue=merge_details.get('id'))
update_bug_with_results.UpdateMergeIssue(
commit_cache_key, merge_details, bug_id)
def testNewBug_Failure_HTTPException(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._ExecuteRequest = mock.Mock(
side_effect=httplib.HTTPException('reason'))
response = service.NewBug('Bug title', 'body', owner='*****@*****.**')
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertIn('error', response)
def testNewBug_Success_NewBugReturnsId(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._ExecuteRequest = mock.Mock(return_value={'id': 333})
response = service.NewBug('Bug title', 'body', owner='*****@*****.**')
bug_id = response['bug_id']
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertEqual(333, bug_id)
def Start(self):
self.Schedule()
comment = 'Pinpoint job started.\n' + self.url
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, comment, send_email=False)
def _PostBugComment(self,
status,
include_differences=False,
send_email=True):
if not self.bug_id:
return
title = '%s Pinpoint job %s.' % (_ROUND_PUSHPIN, status)
header = '\n'.join((title, self.url))
change_details = []
if include_differences:
# Include list of Changes.
differences = tuple(self.state.Differences())
if differences:
if len(differences) == 1:
change_details.append(
'<b>Found significant differences after 1 commit:</b>')
else:
change_details.append(
'<b>Found significant differences after each of %d commits:</b>'
% len(differences))
for _, change in differences:
change_details.append(_FormatChangeForBug(change))
else:
change_details.append(
"<b>Couldn't reproduce a difference.</b>")
comment = '\n\n'.join([header] + change_details)
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id,
comment,
send_email=send_email)
def testNewBug_UsesExpectedParamsSansOwner(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._MakeCreateRequest = mock.Mock()
service.NewBug('Bug title',
'body',
cc=['*****@*****.**', '*****@*****.**'])
service._MakeCreateRequest.assert_called_with(
{
'title': 'Bug title',
'summary': 'Bug title',
'description': 'body',
'labels': [],
'components': [],
'status': 'Unconfirmed',
'cc': mock.ANY,
'projectId': 'chromium',
}, 'chromium')
self.assertItemsEqual([
{
'name': '*****@*****.**'
},
{
'name': '*****@*****.**'
},
], service._MakeCreateRequest.call_args[0][0].get('cc'))
def _FetchBugs(self):
http = oauth2_decorator.DECORATOR.http()
issue_tracker = issue_tracker_service.IssueTrackerService(http)
response = issue_tracker.List(q='opened-after:today-5',
label='Type-Bug-Regression,Performance',
sort='-id')
return response.get('items', []) if response else []
def _IssueTracker():
"""Get a cached IssueTracker instance."""
# pylint: disable=protected-access
if not hasattr(_IssueTracker, '_client'):
_IssueTracker._client = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
return _IssueTracker._client
def Fail(self, exception):
self.exception = str(exception)
comment = 'Pinpoint job failed!\n' + self.url
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, comment, send_email=False)
def _PostBugComment(self, *args, **kwargs):
if not self.bug_id:
return
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, *args, **kwargs)
def get(self):
"""The get handler method is called from a cron job.
It expects no parameters and has no output. It checks all current bisect try
jobs and send comments to an issue on the issue tracker if a bisect job has
completed.
"""
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
# Set privilege so we can also fetch internal try_job entities.
datastore_hooks.SetPrivilegedRequest()
jobs_to_check = try_job.TryJob.query(
try_job.TryJob.status.IN(['started', 'pending'])).fetch()
all_successful = True
for job in jobs_to_check:
try:
_CheckJob(job, issue_tracker)
except Exception as e: # pylint: disable=broad-except
logging.error('Caught Exception %s: %s\n%s',
type(e).__name__, e, traceback.format_exc())
all_successful = False
if all_successful:
utils.TickMonitoringCustomMetric('UpdateBugWithResults')
def testNewBug_UsesExpectedParams(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._MakeCreateRequest = mock.Mock()
service.NewBug('Bug title',
'body',
owner='*****@*****.**',
cc='[email protected], [email protected]')
service._MakeCreateRequest.assert_called_with({
'title':
'Bug title',
'summary':
'Bug title',
'description':
'body',
'labels': [],
'components': [],
'status':
'Assigned',
'owner': {
'name': '*****@*****.**'
},
'cc': [{
'name': '*****@*****.**'
}, {
'name': '*****@*****.**'
}],
})
def _UpdatePostAndMergeDeferred(
comment, commit_cache_key, bug_id, cc_list, owner):
if not bug_id:
return
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
merge_details, cc_list = _ComputePostMergeDetails(
issue_tracker, commit_cache_key, cc_list)
current_bug_status = _GetBugStatus(issue_tracker, bug_id)
if not current_bug_status:
return
status = None
bug_owner = None
if current_bug_status in ['Untriaged', 'Unconfirmed', 'Available']:
# Set the bug status and owner if this bug is opened and unowned.
status = 'Assigned'
bug_owner = owner
issue_tracker.AddBugComment(bug_id, comment, status=status,
cc_list=sorted(cc_list), owner=bug_owner,
merge_issue=merge_details.get('id'))
update_bug_with_results.UpdateMergeIssue(
commit_cache_key, merge_details, bug_id)
def testNewBug_Failure_NewBugReturnsError(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._ExecuteRequest = mock.Mock(return_value={})
response = service.NewBug('Bug title',
'body',
owner='*****@*****.**')
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertTrue('error' in response)
def testNewBug_Failure_NewBugReturnsNone(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._ExecuteRequest = mock.Mock(return_value={})
bug_id = service.NewBug('Bug title',
'body',
owner='*****@*****.**')
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertIsNone(bug_id)
def _GetBugStatus(self):
if not self.bug_id:
return None
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_data = issue_tracker.GetIssue(self.bug_id)
return issue_data.get('status')
def _FetchOpenBugs(self):
"""Fetches a list of open bugs on all sheriffing labels."""
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
bugs = issue_tracker.List(
can='open',
q='Performance=Sheriff OR Performance=Sheriff-V8',
maxResults=1000)
return bugs['items']
def _CreateBug(self, summary, description, labels, components, urlsafe_keys):
"""Creates a bug, associates it with the alerts, sends a HTML response.
Args:
summary: The new bug summary string.
description: The new bug description string.
labels: List of label strings for the new bug.
components: List of component strings for the new bug.
urlsafe_keys: Comma-separated alert keys in urlsafe format.
"""
alert_keys = [ndb.Key(urlsafe=k) for k in urlsafe_keys.split(',')]
alerts = ndb.get_multi(alert_keys)
if not description:
description = 'See the link to graphs below.'
milestone_label = _MilestoneLabel(alerts)
if milestone_label:
labels.append(milestone_label)
# Only project members (@chromium.org accounts) can be owners of bugs.
owner = self.request.get('owner')
if owner and not owner.endswith('@chromium.org'):
self.RenderHtml('bug_result.html', {
'error': 'Owner email address must end with @chromium.org.'
})
return
cc = self.request.get('cc')
http = oauth2_decorator.DECORATOR.http()
service = issue_tracker_service.IssueTrackerService(http)
bug_id = service.NewBug(
summary, description, labels=labels, components=components, owner=owner,
cc=cc)
if not bug_id:
self.RenderHtml('bug_result.html', {'error': 'Error creating bug!'})
return
bug_data.Bug(id=bug_id).put()
for alert_entity in alerts:
alert_entity.bug_id = bug_id
ndb.put_multi(alerts)
comment_body = _AdditionalDetails(bug_id, alerts)
service.AddBugComment(bug_id, comment_body)
template_params = {'bug_id': bug_id}
if all(k.kind() == 'Anomaly' for k in alert_keys):
bisect_result = auto_bisect.StartNewBisectForBug(bug_id)
if 'error' in bisect_result:
template_params['bisect_error'] = bisect_result['error']
else:
template_params.update(bisect_result)
self.RenderHtml('bug_result.html', template_params)
def _ComputePostMergeDetails(self, commit_cache_key, cc_list):
merge_details = {}
if commit_cache_key:
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
merge_details = update_bug_with_results.GetMergeIssueDetails(
issue_tracker, commit_cache_key)
if merge_details['id']:
cc_list = []
return merge_details, cc_list
def testNewBug_HttpError_NewBugReturnsError(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
error_content = {
'error': {'message': 'The user does not exist: [email protected]',
'code': 404}
}
service._ExecuteRequest = mock.Mock(side_effect=errors.HttpError(
mock.Mock(return_value={'status': 404}), json.dumps(error_content)))
response = service.NewBug('Bug title', 'body', owner='*****@*****.**')
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertTrue('error' in response)
def testMakeCommentRequest_UserCantOwn_RetryMakeCommentRequest(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
error_content = {
'error': {'message': 'Issue owner must be a project member',
'code': 400}
}
service._ExecuteRequest = mock.Mock(side_effect=errors.HttpError(
mock.Mock(return_value={'status': 404}), json.dumps(error_content)))
service.AddBugComment(12345, 'The comment', owner=['*****@*****.**'])
self.assertEqual(2, service._ExecuteRequest.call_count)
def testMakeCommentRequest_UserDoesNotExist_RetryMakeCommentRequest(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
error_content = {
'error': {'message': 'The user does not exist: [email protected]',
'code': 404}
}
service._ExecuteRequest = mock.Mock(side_effect=errors.HttpError(
mock.Mock(return_value={'status': 404}), json.dumps(error_content)))
service.AddBugComment(12345, 'The comment', cc_list=['*****@*****.**'],
owner=['*****@*****.**'])
self.assertEqual(2, service._ExecuteRequest.call_count)
def testMakeCommentRequest_IssueDeleted_ReturnsTrue(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
error_content = {
'error': {'message': 'User is not allowed to view this issue 12345',
'code': 403}
}
service._ExecuteRequest = mock.Mock(side_effect=errors.HttpError(
mock.Mock(return_value={'status': 403}), json.dumps(error_content)))
comment_posted = service.AddBugComment(12345, 'The comment',
owner='*****@*****.**')
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertEqual(True, comment_posted)
def _CommentOnRecoveredBug(cls, bug_id):
"""Adds a comment and close the bug on Issue tracker."""
bug = ndb.Key('Bug', bug_id).get()
if bug.status != bug_data.BUG_STATUS_OPENED:
return
bug.status = bug_data.BUG_STATUS_RECOVERED
bug.put()
comment = cls._RecoveredBugComment(bug_id)
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(bug_id, comment)
def testAddBugComment_MergeBug(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._MakeCommentRequest = mock.Mock()
self.assertTrue(service.AddBugComment(12345, 'Dupe',
merge_issue=54321))
self.assertEqual(1, service._MakeCommentRequest.call_count)
service._MakeCommentRequest.assert_called_with(12345, {
'updates': {
'status': 'Duplicate',
'mergedInto': 54321,
},
'content': 'Dupe'
},
send_email=True)
def PerformBisect(bisect_job):
"""Starts the bisect job.
This creates a patch, uploads it, then tells Rietveld to try the patch.
TODO(qyearsley): If we want to use other tryservers sometimes in the future,
then we need to have some way to decide which one to use. This could
perhaps be passed as part of the bisect bot name, or guessed from the bisect
bot name.
Args:
bisect_job: A TryJob entity.
Returns:
A dictionary containing the result; if successful, this dictionary contains
the field "issue_id" and "issue_url", otherwise it contains "error".
Raises:
AssertionError: Bot or config not set as expected.
request_handler.InvalidInputError: Some property of the bisect job
is invalid.
"""
assert bisect_job.bot and bisect_job.config
if not bisect_job.key:
bisect_job.put()
result = _PerformBuildbucketBisect(bisect_job)
if 'error' in result:
bisect_job.run_count += 1
bisect_job.SetFailed()
comment = 'Bisect job failed to kick off'
elif result.get('issue_url'):
comment = 'Started bisect job %s' % result['issue_url']
else:
comment = 'Started bisect job: %s' % result
if not bisect_job.results_data:
bisect_job.results_data = {'issue_url': 'N/A', 'issue_id': 'N/A'}
bisect_job.results_data.update(result)
bisect_job.put()
if bisect_job.bug_id:
logging.info('Commenting on bug %s for bisect job', bisect_job.bug_id)
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(bisect_job.bug_id,
comment,
send_email=False)
return result
def testNewBug_Success_ProjectIsNone(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
service._ExecuteRequest = mock.Mock(return_value={
'id': 333,
'projectId': 'chromium'
})
response = service.NewBug('Bug title',
'body',
owner='*****@*****.**',
project=None)
bug_id = response['bug_id']
project_id = response['project_id']
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertEqual(333, bug_id)
self.assertEqual('chromium', project_id)
