예제 #1
0
    def _SetResponseForGetRequestSwarmingList(self,
                                              master_name,
                                              builder_name,
                                              build_number,
                                              step_name=None):
        if builder_name == 'download_failed':
            return

        url = ('https://%s/_ah/api/swarming/v1/tasks/'
               'list?tags=%s&tags=%s&tags=%s') % (
                   FinditConfig().Get().swarming_settings.get('server_host'),
                   urllib.quote('master:%s' % master_name),
                   urllib.quote('buildername:%s' % builder_name),
                   urllib.quote('buildnumber:%d' % build_number))

        if step_name:
            url += '&tags=%s' % urllib.quote('stepname:%s' % step_name)
            response = self._GetData('step')
        else:
            response = self._GetData('build')

        cursor_swarming_data = {
            'cursor': None,
            'items': [],
            'state': 'all',
            'limit': 100,
            'sort': 'created_ts'
        }
        cursor_url = ('%s&cursor=thisisacursor') % url

        self.get_responses[url] = response
        self.get_responses[cursor_url] = json.dumps(cursor_swarming_data)
예제 #2
0
def _SendNotificationForCulprit(repo_name, revision, commit_position,
                                review_server_host, change_id, revert_status):
    code_review_settings = FinditConfig().Get().code_review_settings
    codereview = codereview_util.GetCodeReviewForReview(
        review_server_host, code_review_settings)
    sent = False
    if codereview and change_id:
        # Occasionally, a commit was not uploaded for code-review.
        culprit = WfSuspectedCL.Get(repo_name, revision)

        action = 'identified'
        if revert_status == create_revert_cl_pipeline.CREATED_BY_SHERIFF:
            action = 'confirmed'

        message = textwrap.dedent("""
    Findit (https://goo.gl/kROfz5) %s this CL at revision %s as the culprit for
    failures in the build cycles as shown on:
    https://findit-for-me.appspot.com/waterfall/culprit?key=%s""") % (
            action, commit_position or revision, culprit.key.urlsafe())
        sent = codereview.PostMessage(change_id, message)
    else:
        logging.error('No code-review url for %s/%s', repo_name, revision)

    _UpdateNotificationStatus(repo_name, revision,
                              status.COMPLETED if sent else status.ERROR)
    return sent
예제 #3
0
def ShouldSkipTestTryJobs(wf_mastername, wf_buildername):
  """Returns True if test try jobs should be triggered.

    By default, test try jobs should be supported unless the master/builder
    specifies to bail out.

  Args:
    wf_mastername: The mastername of a waterfall builder.
    wf_buildername: The buildername of a waterfall builder.

  Returns:
    True if test try jobs are to be skipped, False otherwise.
  """
  trybot_config = FinditConfig.Get().builders_to_trybots.get(
      wf_mastername, {}).get(wf_buildername, {})
  return trybot_config.get('not_run_tests', False)
예제 #4
0
    def UpdateUnitTestConfigSettings(self,
                                     config_property=None,
                                     override_data=None):
        """Sets up Findit's config for unit tests.

    Args:
      config_property: The name of the config property to update.
      override_data: A dict to override any default settings.
    """
        config_data = DEFAULT_CONFIG_DATA

        if config_property and override_data:
            config_data = copy.deepcopy(DEFAULT_CONFIG_DATA)
            config_data[config_property].update(override_data)

        FinditConfig.Get().Update(users.User(email='*****@*****.**'), True,
                                  **config_data)
    def HandleGet(self):
        http_client = HttpClientAppengine()
        supported_masters = _GetSupportedMasters()
        main_waterfall_builders = _GetBuildersOnMasters(
            supported_masters, http_client)
        trybot_config = FinditConfig.Get().builders_to_trybots
        covered_builders = _GetCoveredBuilders(trybot_config)
        missing_builders = _GetDiffBetweenDicts(main_waterfall_builders,
                                                covered_builders)
        deprecated_builders = _GetDiffBetweenDicts(covered_builders,
                                                   main_waterfall_builders)
        unused_variable_builders = _GetUnusedVariableBuilders(
            trybot_config, http_client)

        return {
            'template': 'check_trybot_mapping.html',
            'data': {
                'missing': missing_builders,
                'deprecated': deprecated_builders,
                'unused_variable_builders': unused_variable_builders
            }
        }
예제 #6
0
def GetCodeReviewSettings():
  return FinditConfig().Get().code_review_settings
예제 #7
0
def GetCodeCoverageSettings():
  return FinditConfig().Get().code_coverage_settings
예제 #8
0
def GetFlakeDetectionSettings():
  return FinditConfig.Get().flake_detection_settings
예제 #9
0
def GetCheckFlakeSettings():
  return FinditConfig().Get().check_flake_settings
예제 #10
0
def GetActionSettings():
  return FinditConfig().Get().action_settings
예제 #11
0
def GetDownloadBuildDataSettings():
  return FinditConfig().Get().download_build_data_settings
예제 #12
0
def get_rotation_url():
    return FinditConfig().Get().action_settings.get('rotations_url',
                                                    _ROTATIONS_URL)
예제 #13
0
def GetTryJobSettings():
  return FinditConfig().Get().try_job_settings
예제 #14
0
def _CheckRevertStatusOfSuspectedCL(suspected_cl):
    """Updates suspected_cl with findings about what happened to its revert CL.

  Args:
    suspected_cl (wf_suspected_cl): A WfSuspectedCL entity.

  Returns:
    processed (bool): True if the suspected cl's revert outcome was determined,
      False if reverting was not relevant. None if the cl needed reverting but
      an outcome could not be determined.
    url (str): The code review url for manual investigation.
    status (str): The eventual outcome of the revert cl.
  """
    revert_cl = suspected_cl.revert_cl

    if not revert_cl and not suspected_cl.should_be_reverted:
        # Findit did not deem this suspected cl as needing revert. No action needed.
        return False, None, None

    revision = suspected_cl.revision
    culprit_info = git.GetCodeReviewInfoForACommit(revision)
    review_server_host = culprit_info.get('review_server_host')
    change_id = culprit_info.get('review_change_id')

    if not review_server_host or not change_id:  # pragma: no cover
        # TODO(lijeffrey): Handle cases a patch was committed without review.
        return None, None, None

    code_review_settings = FinditConfig().Get().code_review_settings
    codereview = Gerrit(
        review_server_host
    ) if review_server_host and codereview_util.IsCodeReviewGerrit(
        review_server_host, code_review_settings) else None
    if not codereview:
        logging.error('Could not get codereview for %s/q/%s',
                      review_server_host, change_id)
        return None, None, None

    cl_info = codereview.GetClDetails(change_id)
    code_review_url = codereview.GetCodeReviewUrl(change_id)

    if not cl_info:
        logging.error('Could not get CL details for %s/q/%s',
                      review_server_host, change_id)
        return None, code_review_url, None
    reverts_to_check = cl_info.GetRevertCLsByRevision(revision)

    if not reverts_to_check:
        logging.error('Could not get revert info for %s/q/%s',
                      review_server_host, change_id)
        return None, code_review_url, None

    reverts_to_check.sort(key=lambda x: x.timestamp)

    if revert_cl:  # Findit attempted to create a revert cl.
        any_revert_committed = False
        # Check whose revert CL was first commited.
        for revert in reverts_to_check:  # pragma: no branch
            reverting_user = revert.reverting_user_email
            revert_commits = revert.reverting_cl.commits

            if revert_commits:  # pragma: no branch
                any_revert_committed = True
                revert_commit = revert_commits[0]

                if reverting_user == constants.DEFAULT_SERVICE_ACCOUNT:
                    # The sheriff used Findit's reverting CL.
                    cq_attempt = revert.reverting_cl.commit_attempts[
                        revert_commit.patchset_id]
                    _UpdateSuspectedCL(
                        suspected_cl,
                        sheriff_action_time=cq_attempt.last_cq_timestamp,
                        revert_commit_timestamp=revert_commit.timestamp,
                        revert_status=revert_cl_status.COMMITTED)
                    break
                else:
                    # Sheriff used own revert CL.
                    _UpdateSuspectedCL(
                        suspected_cl,
                        sheriff_action_time=revert_commit.timestamp,
                        revert_status=revert_cl_status.DUPLICATE)
                    break

        # No revert was ever committed. False positive.
        if not any_revert_committed:
            # TODO(crbug.com/702056): Close the revert CLs that were not used.
            _UpdateSuspectedCL(suspected_cl,
                               revert_status=revert_cl_status.FALSE_POSITIVE)

    elif suspected_cl.should_be_reverted:  # pragma: no branch
        # Findit could have created a revert CL, but the sheriff was too fast.
        # Find the first revert that was successfully landed.
        for revert in reverts_to_check:  # pragma: no branch
            revert_commits = revert.reverting_cl.commits
            if revert_commits:  # pragma: no branch
                _UpdateSuspectedCL(suspected_cl,
                                   sheriff_action_time=revert.timestamp)
                break

    return True, code_review_url, (revert_cl.status if revert_cl else
                                   revert_cl_status.DUPLICATE)
예제 #15
0
def EnableStrictRegexForCompileLinkFailures(wf_mastername, wf_buildername):
  """Returns True if strict regex should be used for the given builder."""
  trybot_config = FinditConfig.Get().builders_to_trybots.get(
      wf_mastername, {}).get(wf_buildername, {})
  return trybot_config.get('strict_regex', False)
예제 #16
0
def _GetTrybotConfig(master, builder):
    trybot_config = _ConvertOldTrybotFormatToNew(
        FinditConfig.Get().builders_to_trybots)
    return trybot_config.get(master, {}).get(builder, {})
        req = urllib2.Request(_MILO_MASTER_ENDPOINT, None, headers)
        f = urllib2.urlopen(req, json.dumps(values), timeout=60)
    except Exception as e:
        print(
            'WARNING: Unable to reach builbot to retrieve trybot '
            'information')
        raise e

    data = _ProcessMiloData(f.read())
    return [bot for bot in data.get('builders', {}).keys()]


if __name__ == '__main__':
    remote_api.EnableRemoteApi(app_id='findit-for-me')

    trybots = FinditConfig.Get().builders_to_trybots
    steps_for_masters_rules = FinditConfig.Get().steps_for_masters_rules
    main_waterfall_cache = {}
    variable_builders_cache = defaultdict(list)
    tryservers = set()

    print 'Determining missing support...'

    supported_masters = steps_for_masters_rules.get('supported_masters',
                                                    {}).keys()
    for master in supported_masters:
        print 'Master: %s' % master

        if not trybots.get(master):
            print 'Not found. Tryjobs for %s may not be supported.' % master
            print
예제 #18
0
    def _SetResponseForGetRequestSwarmingResult(self, task_id):
        url = ('https://%s/_ah/api/swarming/v1/task/%s/result') % (
            FinditConfig().Get().swarming_settings.get('server_host'), task_id)

        response = self._GetData('task')
        self.get_responses[url] = response
예제 #19
0
def GetStepsForMastersRules(settings=None, version=None):
  if settings is None:
    settings = FinditConfig.Get(version)
  return (settings.steps_for_masters_rules or
          _ConvertOldMastersFormatToNew(settings.masters_to_blacklisted_steps))
예제 #20
0
def GetSwarmingSettings():
  return FinditConfig().Get().swarming_settings
예제 #21
0
    return html


if __name__ == '__main__':
    # Set up the Remote API to use services on the live App Engine.
    remote_api.EnableRemoteApi(app_id='findit-for-me')

    START_DATE = datetime.datetime(2016, 3, 1)
    END_DATE = datetime.datetime(2016, 4, 14)

    wf_analysis_query = WfTryJobData.query(
        WfTryJobData.request_time >= START_DATE,
        WfTryJobData.request_time < END_DATE)
    data_list = wf_analysis_query.fetch()

    masters_to_builders = FinditConfig.Get().builders_to_trybots
    data = _CategorizeTryJobData(data_list)
    full_report_list = _GetReportListForMastersAndBuilders(
        masters_to_builders, data, START_DATE, END_DATE)

    findit_tmp_dir = os.environ.get('TMP_DIR')
    if not findit_tmp_dir:
        findit_tmp_dir = os.getcwd()

    report_path = os.path.join(findit_tmp_dir, 'try_job_data_report.html')

    with open(report_path, 'w') as f:
        f.write(CreateHtmlPage(full_report_list, START_DATE, END_DATE))

    print 'Try job metadata report available at file://%s' % report_path
def _GetSupportedMasters():
    """Lists the supported main waterfall masters Findit runs analyses on."""
    return FinditConfig.Get().steps_for_masters_rules.get(
        'supported_masters', {}).keys()