def _SetResponseForGetRequestSwarmingList(self,
master_name,
builder_name,
build_number,
step_name=None):
if builder_name == 'download_failed':
return
url = ('https://%s/_ah/api/swarming/v1/tasks/'
'list?tags=%s&tags=%s&tags=%s') % (
FinditConfig().Get().swarming_settings.get('server_host'),
urllib.quote('master:%s' % master_name),
urllib.quote('buildername:%s' % builder_name),
urllib.quote('buildnumber:%d' % build_number))
if step_name:
url += '&tags=%s' % urllib.quote('stepname:%s' % step_name)
response = self._GetData('step')
else:
response = self._GetData('build')
cursor_swarming_data = {
'cursor': None,
'items': [],
'state': 'all',
'limit': 100,
'sort': 'created_ts'
}
cursor_url = ('%s&cursor=thisisacursor') % url
self.get_responses[url] = response
self.get_responses[cursor_url] = json.dumps(cursor_swarming_data)
def _SendNotificationForCulprit(repo_name, revision, commit_position,
review_server_host, change_id, revert_status):
code_review_settings = FinditConfig().Get().code_review_settings
codereview = codereview_util.GetCodeReviewForReview(
review_server_host, code_review_settings)
sent = False
if codereview and change_id:
# Occasionally, a commit was not uploaded for code-review.
culprit = WfSuspectedCL.Get(repo_name, revision)
action = 'identified'
if revert_status == create_revert_cl_pipeline.CREATED_BY_SHERIFF:
action = 'confirmed'
message = textwrap.dedent("""
Findit (https://goo.gl/kROfz5) %s this CL at revision %s as the culprit for
failures in the build cycles as shown on:
https://findit-for-me.appspot.com/waterfall/culprit?key=%s""") % (
action, commit_position or revision, culprit.key.urlsafe())
sent = codereview.PostMessage(change_id, message)
else:
logging.error('No code-review url for %s/%s', repo_name, revision)
_UpdateNotificationStatus(repo_name, revision,
status.COMPLETED if sent else status.ERROR)
return sent
def ShouldSkipTestTryJobs(wf_mastername, wf_buildername):
"""Returns True if test try jobs should be triggered.
By default, test try jobs should be supported unless the master/builder
specifies to bail out.
Args:
wf_mastername: The mastername of a waterfall builder.
wf_buildername: The buildername of a waterfall builder.
Returns:
True if test try jobs are to be skipped, False otherwise.
"""
trybot_config = FinditConfig.Get().builders_to_trybots.get(
wf_mastername, {}).get(wf_buildername, {})
return trybot_config.get('not_run_tests', False)
def UpdateUnitTestConfigSettings(self,
config_property=None,
override_data=None):
"""Sets up Findit's config for unit tests.
Args:
config_property: The name of the config property to update.
override_data: A dict to override any default settings.
"""
config_data = DEFAULT_CONFIG_DATA
if config_property and override_data:
config_data = copy.deepcopy(DEFAULT_CONFIG_DATA)
config_data[config_property].update(override_data)
FinditConfig.Get().Update(users.User(email='*****@*****.**'), True,
**config_data)
def HandleGet(self):
http_client = HttpClientAppengine()
supported_masters = _GetSupportedMasters()
main_waterfall_builders = _GetBuildersOnMasters(
supported_masters, http_client)
trybot_config = FinditConfig.Get().builders_to_trybots
covered_builders = _GetCoveredBuilders(trybot_config)
missing_builders = _GetDiffBetweenDicts(main_waterfall_builders,
covered_builders)
deprecated_builders = _GetDiffBetweenDicts(covered_builders,
main_waterfall_builders)
unused_variable_builders = _GetUnusedVariableBuilders(
trybot_config, http_client)
return {
'template': 'check_trybot_mapping.html',
'data': {
'missing': missing_builders,
'deprecated': deprecated_builders,
'unused_variable_builders': unused_variable_builders
}
}
def GetCodeReviewSettings(): return FinditConfig().Get().code_review_settings
def GetCodeCoverageSettings(): return FinditConfig().Get().code_coverage_settings
def GetFlakeDetectionSettings(): return FinditConfig.Get().flake_detection_settings
def GetCheckFlakeSettings(): return FinditConfig().Get().check_flake_settings
def GetActionSettings(): return FinditConfig().Get().action_settings
def GetDownloadBuildDataSettings(): return FinditConfig().Get().download_build_data_settings
def get_rotation_url():
return FinditConfig().Get().action_settings.get('rotations_url',
_ROTATIONS_URL)
def GetTryJobSettings(): return FinditConfig().Get().try_job_settings
def _CheckRevertStatusOfSuspectedCL(suspected_cl):
"""Updates suspected_cl with findings about what happened to its revert CL.
Args:
suspected_cl (wf_suspected_cl): A WfSuspectedCL entity.
Returns:
processed (bool): True if the suspected cl's revert outcome was determined,
False if reverting was not relevant. None if the cl needed reverting but
an outcome could not be determined.
url (str): The code review url for manual investigation.
status (str): The eventual outcome of the revert cl.
"""
revert_cl = suspected_cl.revert_cl
if not revert_cl and not suspected_cl.should_be_reverted:
# Findit did not deem this suspected cl as needing revert. No action needed.
return False, None, None
revision = suspected_cl.revision
culprit_info = git.GetCodeReviewInfoForACommit(revision)
review_server_host = culprit_info.get('review_server_host')
change_id = culprit_info.get('review_change_id')
if not review_server_host or not change_id: # pragma: no cover
# TODO(lijeffrey): Handle cases a patch was committed without review.
return None, None, None
code_review_settings = FinditConfig().Get().code_review_settings
codereview = Gerrit(
review_server_host
) if review_server_host and codereview_util.IsCodeReviewGerrit(
review_server_host, code_review_settings) else None
if not codereview:
logging.error('Could not get codereview for %s/q/%s',
review_server_host, change_id)
return None, None, None
cl_info = codereview.GetClDetails(change_id)
code_review_url = codereview.GetCodeReviewUrl(change_id)
if not cl_info:
logging.error('Could not get CL details for %s/q/%s',
review_server_host, change_id)
return None, code_review_url, None
reverts_to_check = cl_info.GetRevertCLsByRevision(revision)
if not reverts_to_check:
logging.error('Could not get revert info for %s/q/%s',
review_server_host, change_id)
return None, code_review_url, None
reverts_to_check.sort(key=lambda x: x.timestamp)
if revert_cl: # Findit attempted to create a revert cl.
any_revert_committed = False
# Check whose revert CL was first commited.
for revert in reverts_to_check: # pragma: no branch
reverting_user = revert.reverting_user_email
revert_commits = revert.reverting_cl.commits
if revert_commits: # pragma: no branch
any_revert_committed = True
revert_commit = revert_commits[0]
if reverting_user == constants.DEFAULT_SERVICE_ACCOUNT:
# The sheriff used Findit's reverting CL.
cq_attempt = revert.reverting_cl.commit_attempts[
revert_commit.patchset_id]
_UpdateSuspectedCL(
suspected_cl,
sheriff_action_time=cq_attempt.last_cq_timestamp,
revert_commit_timestamp=revert_commit.timestamp,
revert_status=revert_cl_status.COMMITTED)
break
else:
# Sheriff used own revert CL.
_UpdateSuspectedCL(
suspected_cl,
sheriff_action_time=revert_commit.timestamp,
revert_status=revert_cl_status.DUPLICATE)
break
# No revert was ever committed. False positive.
if not any_revert_committed:
# TODO(crbug.com/702056): Close the revert CLs that were not used.
_UpdateSuspectedCL(suspected_cl,
revert_status=revert_cl_status.FALSE_POSITIVE)
elif suspected_cl.should_be_reverted: # pragma: no branch
# Findit could have created a revert CL, but the sheriff was too fast.
# Find the first revert that was successfully landed.
for revert in reverts_to_check: # pragma: no branch
revert_commits = revert.reverting_cl.commits
if revert_commits: # pragma: no branch
_UpdateSuspectedCL(suspected_cl,
sheriff_action_time=revert.timestamp)
break
return True, code_review_url, (revert_cl.status if revert_cl else
revert_cl_status.DUPLICATE)
def EnableStrictRegexForCompileLinkFailures(wf_mastername, wf_buildername):
"""Returns True if strict regex should be used for the given builder."""
trybot_config = FinditConfig.Get().builders_to_trybots.get(
wf_mastername, {}).get(wf_buildername, {})
return trybot_config.get('strict_regex', False)
def _GetTrybotConfig(master, builder):
trybot_config = _ConvertOldTrybotFormatToNew(
FinditConfig.Get().builders_to_trybots)
return trybot_config.get(master, {}).get(builder, {})
req = urllib2.Request(_MILO_MASTER_ENDPOINT, None, headers)
f = urllib2.urlopen(req, json.dumps(values), timeout=60)
except Exception as e:
print(
'WARNING: Unable to reach builbot to retrieve trybot '
'information')
raise e
data = _ProcessMiloData(f.read())
return [bot for bot in data.get('builders', {}).keys()]
if __name__ == '__main__':
remote_api.EnableRemoteApi(app_id='findit-for-me')
trybots = FinditConfig.Get().builders_to_trybots
steps_for_masters_rules = FinditConfig.Get().steps_for_masters_rules
main_waterfall_cache = {}
variable_builders_cache = defaultdict(list)
tryservers = set()
print 'Determining missing support...'
supported_masters = steps_for_masters_rules.get('supported_masters',
{}).keys()
for master in supported_masters:
print 'Master: %s' % master
if not trybots.get(master):
print 'Not found. Tryjobs for %s may not be supported.' % master
print
def _SetResponseForGetRequestSwarmingResult(self, task_id):
url = ('https://%s/_ah/api/swarming/v1/task/%s/result') % (
FinditConfig().Get().swarming_settings.get('server_host'), task_id)
response = self._GetData('task')
self.get_responses[url] = response
def GetStepsForMastersRules(settings=None, version=None):
if settings is None:
settings = FinditConfig.Get(version)
return (settings.steps_for_masters_rules or
_ConvertOldMastersFormatToNew(settings.masters_to_blacklisted_steps))
def GetSwarmingSettings(): return FinditConfig().Get().swarming_settings
return html
if __name__ == '__main__':
# Set up the Remote API to use services on the live App Engine.
remote_api.EnableRemoteApi(app_id='findit-for-me')
START_DATE = datetime.datetime(2016, 3, 1)
END_DATE = datetime.datetime(2016, 4, 14)
wf_analysis_query = WfTryJobData.query(
WfTryJobData.request_time >= START_DATE,
WfTryJobData.request_time < END_DATE)
data_list = wf_analysis_query.fetch()
masters_to_builders = FinditConfig.Get().builders_to_trybots
data = _CategorizeTryJobData(data_list)
full_report_list = _GetReportListForMastersAndBuilders(
masters_to_builders, data, START_DATE, END_DATE)
findit_tmp_dir = os.environ.get('TMP_DIR')
if not findit_tmp_dir:
findit_tmp_dir = os.getcwd()
report_path = os.path.join(findit_tmp_dir, 'try_job_data_report.html')
with open(report_path, 'w') as f:
f.write(CreateHtmlPage(full_report_list, START_DATE, END_DATE))
print 'Try job metadata report available at file://%s' % report_path
def _GetSupportedMasters():
"""Lists the supported main waterfall masters Findit runs analyses on."""
return FinditConfig.Get().steps_for_masters_rules.get(
'supported_masters', {}).keys()