def testGetActionSettings(self):
self.assertEqual(
{
'cr_notification_build_threshold': 2,
'cr_notification_latency_limit_minutes': 30,
'revert_compile_culprit': True,
}, waterfall_config.GetActionSettings())
def CanAutoCreateRevert(culprit, parameters):
"""Checks if Findit can auto create a revert.
Args:
culprit (Basestring): Urlsafe key for the suspected cl.
parameters (CulpritActionParameters): Parameters to run culprit action
pipelines.
Findit can auto create a revert if:
1. Auto create revert for test is turned on;
2. The number of reverts in past 24 hours is less than the daily limit;
3. The culprit is also being suspected by the heuristic analysis.
"""
heuristic_cls = parameters.heuristic_cls
if culprit not in heuristic_cls:
return False
action_settings = waterfall_config.GetActionSettings()
# Auto revert has been turned off.
if not bool(action_settings.get('auto_create_revert')):
return False
auto_create_revert_daily_threshold_test = action_settings.get(
'auto_create_revert_daily_threshold_test',
_DEFAULT_AUTO_CREATE_REVERT_DAILY_THRESHOLD_TEST)
# Auto revert has exceeded daily limit.
if _GetDailyNumberOfRevertedCulprits(
auto_create_revert_daily_threshold_test
) >= auto_create_revert_daily_threshold_test:
logging.info(
'Auto reverts for test culprits on %s has met daily limit.',
time_util.FormatDatetime(time_util.GetUTCNow()))
return False
return True
def run(self, master_name, builder_name, build_number, repo_name,
revision):
if waterfall_config.GetActionSettings().get('revert_compile_culprit',
False):
return _RevertCulprit(master_name, builder_name, build_number,
repo_name, revision)
return None
def _CanCommitRevert(parameters, analysis_id, codereview_info):
"""Checks if an auto-created revert of a culprit can be committed.
This function will call several different functions to check the culprit
and/or revert from many different aspects and make the final decision.
The criteria included so far are:
+ Revert is created by Findit;
+ Can the revert be committed in current analysis;
+ Was the change committed within time;
+ Was the change to be reverted authored by an auto-roller;
+ Are there other changes by the culprit's author depending on the culprit.
"""
if not parameters.revert_status == constants.CREATED_BY_FINDIT:
return False
culprit = entity_util.GetEntityFromUrlsafeKey(parameters.cl_key)
assert culprit
action_settings = waterfall_config.GetActionSettings()
culprit_commit_limit_hours = action_settings.get(
'culprit_commit_limit_hours',
constants.DEFAULT_CULPRIT_COMMIT_LIMIT_HOURS)
return (_CanCommitRevertInAnalysis(parameters.cl_key, analysis_id)
and git.ChangeCommittedWithinTime(culprit.revision,
hours=culprit_commit_limit_hours)
and not git.IsAuthoredByNoAutoRevertAccount(culprit.revision)
and not gerrit.ExistCQedDependingChanges(codereview_info))
def SendNotificationForCulprit(parameters):
culprit = entity_util.GetEntityFromUrlsafeKey(parameters.cl_key)
assert culprit
revision = culprit.revision
repo_name = culprit.repo_name
force_notify = parameters.force_notify
revert_status = parameters.revert_status
sent = False
should_send = False
action_settings = waterfall_config.GetActionSettings()
# Set some impossible default values to prevent notification by default.
build_num_threshold = action_settings.get(
'cr_notification_build_threshold', 100000)
if _ShouldSendNotification(repo_name, revision, force_notify,
revert_status, build_num_threshold):
should_send = True
codereview_info = GetCodeReviewDataForACulprit(parameters.cl_key)
sent = gerrit.SendNotificationForCulprit(parameters, codereview_info)
MonitoringCulpritNotification(parameters.failure_type, 'culprit', sent,
should_send)
return sent
def GetRemainingPostAnalysisDailyBugUpdatesCount():
"""Returns how many FlakeIssue updates can be made by Flake Analyzer."""
action_settings = waterfall_config.GetActionSettings()
limit = action_settings.get(
'max_flake_analysis_bug_updates_per_day',
flake_constants.DEFAULT_MAX_BUG_UPDATES_PER_DAY)
utc_one_day_ago = time_util.GetUTCNow() - datetime.timedelta(days=1)
num_updated_issues_24h = FlakeIssue.query(
FlakeIssue.last_updated_time_with_analysis_results > utc_one_day_ago
).count()
return limit - num_updated_issues_24h
def UnderLimitForAutorevert():
"""Returns True if currently under the limit for autoreverts."""
action_settings = waterfall_config.GetActionSettings()
# Do not auto commit revert if not configured to do so.
limit = action_settings.get('auto_commit_revert_daily_threshold_flake')
query = MasterFlakeAnalysis.query(
MasterFlakeAnalysis.autorevert_submission_time >=
time_util.GetMostRecentUTCMidnight(),
MasterFlakeAnalysis.has_submitted_autorevert == True)
return query.count(limit) < limit
def testNonAdminCouldTurnOffAutoCommit(self, _):
self.mock_current_user(user_email='*****@*****.**', is_admin=False)
params = {
'xsrf_token': 'token',
'auto_commit_revert': 'false',
'update_reason': 'reason',
}
response = self.test_app.post('/trooper?format=json', params=params)
redirect_url = '/trooper'
self.assertTrue(
response.headers.get('Location', '').endswith(redirect_url))
self.assertFalse(
waterfall_config.GetActionSettings().get('auto_commit_revert'))
def HandleGet(self):
action_settings = waterfall_config.GetActionSettings()
auto_commit_revert_is_on = action_settings.get('auto_commit_revert', False)
code_coverage_settings = waterfall_config.GetCodeCoverageSettings()
code_coverage_is_on = (
code_coverage_settings.get('serve_presubmit_coverage_data', False))
return {
'template': 'trooper.html',
'data': {
'is_admin': users.is_current_user_admin(),
'auto_commit_revert_on': auto_commit_revert_is_on,
'code_coverage_on': code_coverage_is_on,
}
}
def testAdminCouldTurnOnAutoCommit(self, _):
self.mock_current_user(user_email='*****@*****.**', is_admin=True)
self.UpdateUnitTestConfigSettings('action_settings',
{'auto_commit_revert': False})
params = {
'xsrf_token': 'token',
'auto_commit_revert': 'true',
'update_reason': 'reason',
'format': 'json',
}
response = self.test_app.post('/trooper', params=params, status=302)
self.assertTrue(
response.headers.get('Location', '').endswith('/trooper'))
self.assertTrue(
waterfall_config.GetActionSettings().get('auto_commit_revert'))
def UnderDailyLimit():
# TODO(crbug.com/942222): Distinguish between Flake Analyzer and Flake
# Detection bug operations.
action_settings = waterfall_config.GetActionSettings()
daily_bug_limit = action_settings.get(
'max_flake_detection_bug_updates_per_day',
flake_constants.DEFAULT_MAX_BUG_UPDATES_PER_DAY)
query = master_flake_analysis.MasterFlakeAnalysis.query(
master_flake_analysis.MasterFlakeAnalysis.request_time >=
time_util.GetMostRecentUTCMidnight())
bugs_filed_today = 0
more = True
cursor = None
while more:
results, cursor, more = query.fetch_page(100, start_cursor=cursor)
for result in results:
if result.has_attempted_filing and result.bug_id:
bugs_filed_today += 1
return bugs_filed_today < daily_bug_limit
def testGetActionSettings(self):
self.assertEqual(
{
'auto_commit_revert':
True,
'auto_create_revert':
True,
'cr_notification_build_threshold':
2,
'cr_notification_latency_limit_minutes':
30,
'cr_notification_should_notify_flake_culprit':
True,
'culprit_commit_limit_hours':
24,
'auto_commit_revert_daily_threshold_compile':
4,
'auto_create_revert_daily_threshold_compile':
10,
'auto_commit_revert_daily_threshold_test':
4,
'auto_create_revert_daily_threshold_test':
10,
'auto_create_revert_daily_threshold_flake':
10,
'auto_commit_revert_daily_threshold_flake':
4,
'rotations_url':
('https://rota-ng.appspot.com/legacy/all_rotations.js'),
'max_flake_detection_bug_updates_per_day':
30,
'max_flake_analysis_bug_updates_per_day':
30,
'minimum_confidence_to_update_endpoints':
0.7,
'minimum_confidence_to_revert_flake_culprit':
1.0,
}, waterfall_config.GetActionSettings())
def CanAutoCommitRevertByFindit():
"""Checks if the revert can be auto committed by Findit.
The revert can be committed if:
1. Auto revert and Auto commit is turned on;
2. The number of commits of reverts in past 24 hours is less than the
daily limit;
3. Culprit author has not landed another change yet.
"""
action_settings = waterfall_config.GetActionSettings()
if (not bool(action_settings.get('auto_commit_revert'))
or not bool(action_settings.get('auto_create_revert'))):
return False
auto_commit_revert_daily_threshold_test = action_settings.get(
'auto_commit_revert_daily_threshold_test',
_DEFAULT_AUTO_COMMIT_REVERT_DAILY_THRESHOLD_TEST)
if _GetDailyNumberOfCommits(auto_commit_revert_daily_threshold_test
) >= auto_commit_revert_daily_threshold_test:
logging.info('Auto commits on %s has met daily limit.',
time_util.FormatDatetime(time_util.GetUTCNow()))
return False
return True
def _UpdateActionSettings(self, auto_commit_revert_is_on, user, message):
"""Updates the action settings.
Args:
auto_commit_revert_is_on (bool): Whether the auto commit revert feature is
turned on.
user: User who initiated the update.
message: The update message.
Returns:
A bool indicates whether the update is successful.
"""
action_settings = waterfall_config.GetActionSettings()
if auto_commit_revert_is_on == action_settings.get('auto_commit_revert'):
return False
updated_action_settings = copy.deepcopy(action_settings)
updated_action_settings['auto_commit_revert'] = auto_commit_revert_is_on
return wf_config.FinditConfig.Get().Update(
user,
acl.IsPrivilegedUser(user.email(), users.is_current_user_admin()),
message=message,
action_settings=updated_action_settings)
def run(self,
master_name,
builder_name,
build_number,
repo_name,
revision,
force_notify,
revert_status=None):
# This information is not needed at the moment.
# TODO(robertocn): Remove these arguments if we won't need them for
# notifications once auto-revert is enabled.
_build_information = [master_name, builder_name, build_number]
if revert_status == create_revert_cl_pipeline.CREATED_BY_FINDIT:
# Already notified when revert, bail out.
return False
if revert_status == create_revert_cl_pipeline.CREATED_BY_SHERIFF:
force_notify = True
action_settings = waterfall_config.GetActionSettings()
# Set some impossible default values to prevent notification by default.
build_num_threshold = action_settings.get(
'cr_notification_build_threshold', 100000)
culprit_info = suspected_cl_util.GetCulpritInfo(repo_name, revision)
commit_position = culprit_info['commit_position']
review_server_host = culprit_info['review_server_host']
review_change_id = culprit_info['review_change_id']
if not _ShouldSendNotification(repo_name, revision,
build_num_threshold, force_notify):
return False
return _SendNotificationForCulprit(repo_name, revision,
commit_position, review_server_host,
review_change_id, revert_status)
def CanAutoCreateRevert():
"""Checks if Findit can auto create a revert.
Findit can auto create a revert if both of below are True:
1. Auto create revert for compile is turned on;
2. The number of reverts in past 24 hours is less than the daily limit.
"""
action_settings = waterfall_config.GetActionSettings()
# Auto revert has been turned off.
if not bool(action_settings.get('auto_create_revert')):
return False
auto_create_revert_daily_threshold_compile = action_settings.get(
'auto_create_revert_daily_threshold_compile',
_DEFAULT_AUTO_CREATE_REVERT_DAILY_THRESHOLD_COMPILE)
# Auto revert has exceeded daily limit.
if _GetDailyNumberOfRevertedCulprits(
auto_create_revert_daily_threshold_compile
) >= auto_create_revert_daily_threshold_compile:
logging.info('Auto reverts for compile culprits on %s has met daily limit.',
time_util.FormatDatetime(time_util.GetUTCNow()))
return False
return True
def RevertCulprit(urlsafe_key, build_id, build_failure_type, sample_step_name,
codereview_info):
"""Creates a revert of a culprit and adds reviewers.
Args:
urlsafe_key (str): Key to the ndb model.
build_id (str): Id of the sample failed build.
build_failure_type (int): Failure type: compile, test or flake.
sample_step_name (str): Sample failed step in the failed build.
codereview_info (dict): Code review information about the culprit.
Returns:
(int, string, string):
- Status of the reverting;
- change_id of the revert;
- reason why revert is skipped if it is skipped.
"""
culprit = entity_util.GetEntityFromUrlsafeKey(urlsafe_key)
repo_name = culprit.repo_name
revision = culprit.revision
# 0. Gets information about this culprit.
culprit_commit_position = codereview_info['commit_position']
culprit_change_id = codereview_info['review_change_id']
codereview = _GetCodeReview(codereview_info)
print codereview
if not codereview or not culprit_change_id: # pragma: no cover
logging.error('Failed to get change id for %s/%s', repo_name, revision)
return services_constants.ERROR, None, None
culprit_cl_info = codereview.GetClDetails(culprit_change_id)
if not culprit_cl_info: # pragma: no cover
logging.error('Failed to get cl_info for %s/%s', repo_name, revision)
return services_constants.ERROR, None, None
# Checks if the culprit is a revert. If yes, bail out.
if _IsCulpritARevert(culprit_cl_info):
return (services_constants.SKIPPED, None,
services_constants.CULPRIT_IS_A_REVERT)
if culprit_cl_info.auto_revert_off:
return services_constants.SKIPPED, None, services_constants.AUTO_REVERT_OFF
# 1. Checks if a revert CL by sheriff has been created.
reverts = culprit_cl_info.GetRevertCLsByRevision(revision)
if reverts is None: # pragma: no cover
# if no reverts, reverts should be [], only when some error happens it will
# be None.
logging.error('Failed to find patchset_id for %s/%s' %
(repo_name, revision))
return services_constants.ERROR, None, None
findit_revert = None
for revert in reverts:
if _IsOwnerFindit(revert.reverting_user_email):
findit_revert = revert
break
if reverts and not findit_revert:
# Sheriff(s) created the revert CL(s).
return (services_constants.CREATED_BY_SHERIFF, None,
services_constants.REVERTED_BY_SHERIFF)
revert_change_id = None
if findit_revert:
revert_change_id = findit_revert.reverting_cl.change_id
# 2. Crreate revert CL.
# TODO (chanli): Better handle cases where 2 analyses are trying to revert
# at the same time.
if not revert_change_id:
revert_reason = culprit.GenerateRevertReason(build_id,
culprit_commit_position,
revision,
sample_step_name)
if not revert_reason: # pragma: no cover.
logging.error('Failed to get the reason for revert culprit %s',
culprit.key.urlsafe() if culprit else '')
return services_constants.ERROR, None, None
bug_id = _GetBugIdForCulprit(culprit)
revert_change_id = codereview.CreateRevert(
revert_reason,
culprit_change_id,
culprit_cl_info.GetPatchsetIdByRevision(revision),
bug_id=bug_id)
if not revert_change_id: # pragma: no cover
return services_constants.ERROR, None, None
# Save revert CL info and notification info to culprit.
revert_cl = None
if not culprit.revert_cl:
revert_cl = RevertCL()
revert_cl.revert_cl_url = codereview.GetCodeReviewUrl(revert_change_id)
revert_cl.created_time = time_util.GetUTCNow()
# 3. Add reviewers.
# If Findit cannot commit the revert, add sheriffs as reviewers and ask them
# to 'LGTM' and commit the revert.
action_settings = waterfall_config.GetActionSettings()
can_commit_revert = (action_settings.get('auto_commit_revert_compile')
if build_failure_type == failure_type.COMPILE else
action_settings.get('auto_commit_revert_test'))
if not can_commit_revert:
success = _AddReviewers(revision, culprit, codereview,
revert_change_id, False)
if not success: # pragma: no cover
logging.error('Failed to add reviewers for revert of'
' culprit %s/%s' % (repo_name, revision))
return services_constants.ERROR, revert_cl, None
return services_constants.CREATED_BY_FINDIT, revert_cl, None
def IsAutorevertEnabled():
action_settings = waterfall_config.GetActionSettings()
return (action_settings.get('auto_commit_revert', False)
and action_settings.get('auto_create_revert', False))
def IsConfiguredToNotifyCulprits():
action_settings = waterfall_config.GetActionSettings()
return action_settings.get('cr_notification_should_notify_flake_culprit',
False)
def GetMinimumConfidenceToNotifyCulprits():
action_settings = waterfall_config.GetActionSettings()
return action_settings.get(
'minimum_confidence_to_update_endpoints',
flake_constants.DEFAULT_MINIMUM_CONFIDENCE_SCORE_TO_UPDATE_ENDPOINTS)
def GetMinimumConfidenceToUpdateEndpoints():
return waterfall_config.GetActionSettings().get(
'minimum_confidence_to_update_endpoints',
flake_constants.DEFAULT_MINIMUM_CONFIDENCE_SCORE_TO_UPDATE_ENDPOINTS)
