Beispiel #1
0
 def __init__(self, config):
   self.config = config
   self.gerrit = Gerrit(config.gerrit_url, netrc=config.netrc)
   self.gerrit_admin = Gerrit(config.gerrit_url, netrc=config.netrc_admin)
   self.tag = 'autogenerated:review-o-matic'
   self.ignore_list = {}
   self.stats = TrollStats('{}'.format(self.config.stats_file))
Beispiel #2
0
    def main(self):
        """
        Send emails to requested users filed in the users folder (use '-a'
        for all users, or '-u user.address' for specific user) -- OR --
        create a dump for specified users for debugging purposes.
        """

        conf = Config()
        if conf.smtp.authentication and not self._gpg_recipient:
            logging.error("Cannot send email. SMTP authentication missing.")
            return

        cache = Cache(conf.cache_filename)
        cache.read()
        gerrit = Gerrit(cache, conf.gerrit_url)
        gerrit.get_cached_today()

        for user in self._users:
            try:
                user.add_content(gerrit.cache, conf.project, conf.gerrit_url)
                user.send_email(
                    conf.project, conf.smtp, self._gpg_recipient, self._dry_run
                )
            except IOError as err:
                logging.error("Could not send email for user %s: %s", user, str(err))
Beispiel #3
0
def main():
    """
    See module docstring.
    """

    conf = Config()
    gerrit = Gerrit(Cache(conf.cache_filename), conf.gerrit_url)
    gerrit.update()
def main():
    logging.info('Launching gerrit-stats')

    args = parse_commandline()

    settings = load_settings(args)

    gerrit = Gerrit(args, settings)
    cur = init_db(gerrit.my_cnf)
    gerrit.fetch_repos()

    start_date = settings.get('creation_date')
    yesterday = determine_yesterday()
    changesets = {}

    logging.info(
        'Queries will span timeframe: %s - %s.' % (start_date, yesterday))
    logging.info('Queries will always run up to \'yesterday\', so that we always have counts for full days.')

    changesets = load_commit_data(cur, changesets)
    changesets = load_patch_set_data(cur, changesets)
    changesets = load_review_data(cur, changesets)

    for changeset in changesets.itervalues():
        changeset.is_all_positive_reviews()
        changeset.calculate_wait_first_review()
        changeset.calculate_wait_plus2()
        changeset.is_self_reviewed()

        repo = gerrit.repos.get(changeset.dest_project_name)
        if repo:
            repo.increment(changeset)
        else:
            logging.info('Repo %s does not exist, ignored repos are: %s' % (changeset.dest_project_name, ','.join(gerrit.ignore_repos)))

    logging.info('Successfully parsed changesets data.')
    # create datasets that are collections of repositories
    create_aggregate_dataset(gerrit)

    for repo in gerrit.repos.itervalues():
#        if repo.name == 'mediawiki':
#            change_ids = list(repo.observations[yesterday.date()].changeset_ids)
#            change_ids.sort()
#            fh = open('/Users/diederik/Development/gerrit-stats/gerritstats/tests/backlog_gerrit-stats.txt','w')
#            for change_id in change_ids:
#                print change_id
#                fh.write('%s\n' % change_id)
#            fh.close()
        repo.fill_in_missing_days()
        repo.create_headings()
        repo.prune_observations()
        repo.write_dataset(gerrit)

    # save results for future use.
    successful_exit()
Beispiel #5
0
 def __init__(self, url, args):
     self.url = url
     self.args = args
     self.gerrit = Gerrit(url)
     self.tag = 'autogenerated:review-o-matic'
     self.blacklist = []
     self.stats = {
         ReviewType.SUCCESS: 0,
         ReviewType.BACKPORT: 0,
         ReviewType.ALTERED_UPSTREAM: 0,
         ReviewType.MISSING_FIELDS: 0,
         ReviewType.MISSING_HASH: 0,
         ReviewType.INCORRECT_PREFIX: 0,
         ReviewType.FIXES_REF: 0
     }
 def start(self):
     self.gerrit = Gerrit(['zuul'])
     self.omqtt = OpenstackMqtt()
     self.omqtt.on_connect = self.on_connect
     self.omqtt.on_message = self.on_message
     self.omqtt.connect()
     self.omqtt.client.loop_forever()
Beispiel #7
0
 def add_gerrit_remote(self,
                       name,
                       location,
                       project_name,
                       fetch=True,
                       fetch_changes=True):
     repo = Gerrit(name, location, project_name)
     self.addremote(repo, fetch=fetch)
     repo.local_track = TrackedRepo(name, self.directory, project_name)
     if fetch_changes:
         shell('git fetch %s +refs/changes/*:refs/remotes/%s/changes/*' %
               (name, name))
     try:
         os.stat(".git/hooks/commit-msg")
     except OSError:
         shell('scp -p %s:hooks/commit-msg .git/hooks/' % location)
    def __init__(self, scheduler: JobScheduler, config=Dict[str, str]):
        """
        Initialize a Gerrit based event source

        Args:
            @scheduler: scheduler to which job must be added
            @config: dictionary configuring the connection to gerrit. Allowed
                keys are 'hostname', 'username', 'port', 'keyfile' and
                'clone_url'
        """
        # Create connection to gerrit command line
        cfg = subdict(config, ['hostname', 'username', 'port', 'keyfile'])
        gerrit = Gerrit(**cfg)

        # determine settings for git clone when a project must be build
        clone_opts = {}
        clone_url = config.get('clone_url')
        if not clone_url:
            clone_url = 'ssh://{}@{}:{:d}'.format(gerrit.username,
                                                  gerrit.hostname,
                                                  int(gerrit.port))
            if gerrit.keyfile:
                clone_opts['git_ssh_cmd'] = 'ssh -i ' + gerrit.keyfile

        super().__init__(scheduler)
        self.gerrit_instance = gerrit
        self.clone_url = clone_url
        self.clone_opts = clone_opts
Beispiel #9
0
 def __init__(self, url, args):
     self.url = url
     self.args = args
     self.gerrit = Gerrit(url)
     self.tag = 'autogenerated:review-o-matic'
     self.blacklist = {}
     self.stats = {
         str(ReviewType.SUCCESS): 0,
         str(ReviewType.BACKPORT): 0,
         str(ReviewType.ALTERED_UPSTREAM): 0,
         str(ReviewType.MISSING_FIELDS): 0,
         str(ReviewType.MISSING_HASH): 0,
         str(ReviewType.INVALID_HASH): 0,
         str(ReviewType.MISSING_AM): 0,
         str(ReviewType.INCORRECT_PREFIX): 0,
         str(ReviewType.FIXES_REF): 0
     }
Beispiel #10
0
    def test_add_project(self):
        """
        Test that a project can be successfully managed
        """
        # Felix wants to add a project, he uses the gerrit module to do this
        gerrit = Gerrit(
            url=self._url,
            auth_type='http',
            auth_id=self._admin_username,
            auth_pw=self._admin_password,
        )
        created_project = gerrit.create_project(
            self._project,
            options={
                'description': 'my description',
                'branches': ['master'],
                'create_empty_commit': True,
            },
        )
        gotten_project = gerrit.get_project(self._project)
        # Using the same module he can get his project
        self.assertEqual(created_project, gotten_project)
        self.assertEqual(gotten_project.description, 'my description')

        # Felix uploads a new change
        change = gerrit.create_change(created_project, 'My change')

        # Wanting Mary to review the change, he adds her as a reviewer
        self.assertTrue(change.add_reviewer('mary'))

        # He can now see that Mary is a reviewer
        reviewers = change.list_reviewers()
        self.assertEqual(reviewers, [{
            'username': '******',
            'approvals': {
                'Code-Review': ' 0'
            },
            'name': 'Mary',
            '_account_id': 1000001
        }])

        # Felix made a mistake, Mary shouldn't be a reviewer.
        # He removes her.
        self.assertTrue(change.delete_reviewer('mary'))

        # He can now see that Mary is no longer a reviewer
        reviewers = change.list_reviewers()
        self.assertEqual(reviewers, [])

        # Happy with the change, Felix reviews and submits it
        change.set_review(labels={'Code-Review': '+2'})
        change.submit_change()
        self.assertEqual(change.status, 'MERGED')

        # Not needing the repo anymore Felix removes it
        self.assertTrue(created_project.delete({'force': True}))

        # Felix can no longer get the project
        with self.assertRaises(ValueError):
            gerrit.get_project(self._project)
    def __init__(self, last_cid, review, verify, ready):
        self.vote_review = 2 if review else None
        self.vote_verify = 1 if verify else None
        self.vote_trybot_ready = 1 if ready else None
        self.vote_cq_ready = 1 if ready else None

        self.tag = 'autogenerated:submit-o-matic'

        self.max_in_flight = 100  # 50 for the cq, 50 for the pre-cq
        self.in_flight = []

        self.changes = []
        self.gerrit = Gerrit('https://chromium-review.googlesource.com')
        last_change = self.gerrit.get_change(last_cid)
        self.changes = self.gerrit.get_related_changes(last_change)
        if not self.changes:
            self.changes = [last_change]
        else:
            self.changes.reverse()
    def test_add_project(self):
        # Felix wants to add a project, he uses the gerrit module to do this
        gerrit = Gerrit(
            url=self._url,
            auth_type='http',
            auth_id=self._admin_username,
            auth_pw=self._admin_password,
        )
        created_project = gerrit.create_project(
            self._project,
            options={
                'description': 'my description',
                'branches': ['master'],
                'create_empty_commit': True,
            },
        )
        gotten_project = gerrit.get_project(self._project)
        # Using the same module he can get his project
        self.assertEqual(created_project, gotten_project)
        self.assertEqual(gotten_project.description, 'my description')

        # Felix uploads a new change
        change = gerrit.create_change(created_project, 'My change')

        # Wanting Mary to review the change, he adds her as a reviewer
        self.assertTrue(change.add_reviewer('mary'))

        # He can now see that Mary is a reviewer
        reviewers = change.list_reviewers()
        self.assertEqual(
            reviewers,
            [{'username': '******',
              'approvals': {'Code-Review': ' 0'},
              'name': 'Mary',
              '_account_id': 1000001
             }]
        )

        # Felix made a mistake, Mary shouldn't be a reviewer.
        # He removes her.
        self.assertTrue(change.delete_reviewer('mary'))

        # He can now see that Mary is no longer a reviewer
        reviewers = change.list_reviewers()
        self.assertEqual(reviewers,[])

        # Happy with the change, Felix reviews and submits it
        change.set_review(labels={'Code-Review': '+2'})
        change.submit_change()
        self.assertEqual(change.status, 'MERGED')

        # Not needing the repo anymore Felix removes it
        self.assertTrue(created_project.delete({'force': True}))

        # Felix can no longer get the project
        with self.assertRaises(ValueError):
            gerrit.get_project(self._project)
Beispiel #13
0
    def __init__(self, last_cid, review, verify, ready, abandon, force_review,
                 dry_run):
        self.abandon = abandon
        self.vote_review = 2 if review else None
        self.vote_verify = 1 if verify else None
        self.vote_cq_ready = ready
        self.force_review = force_review

        self.dry_run = dry_run

        self.tag = 'autogenerated:submit-o-matic'

        self.max_in_flight = 100  # 50 for the cq, 50 for the pre-cq
        self.in_flight = []

        self.changes = []
        self.gerrit = Gerrit('https://chromium-review.googlesource.com',
                             use_internal=False)
        last_change = self.gerrit.get_change(last_cid)
        ancestor_changes = self.gerrit.get_ancestor_changes(last_change)
        for c in reversed(ancestor_changes):
            if c.status == 'NEW':
                self.changes.append(c)
        self.changes.append(last_change)
Beispiel #14
0
class ChangesFetcher(object):
    def __init__(self, host, port, username, pkey):
        self._gerrit = Gerrit(host, port, username, pkey)
        self._flow = _IDENTITY_FLOW

    def set_flow(self, flow):
        self._flow = flow

    def get_changes(self, query):
        changes = self._gerrit.query(query,
                                     options=[
                                         QueryOptions.Comments,
                                         QueryOptions.CurrentPatchSet,
                                         QueryOptions.CommitMessage
                                     ])
        return self._flow(changes)
Beispiel #15
0
def set_objects(config):
    """
    Sets bugzilla and gerrit objects

    :param config: dict of configuration keys and values
    :return: bugzilla and gerrit objects
    """

    # set bugzilla object
    bz_obj = Bugzilla(
        user=config['BZ_USER'],
        passwd=config['BZ_PASS'],
        url=config['BZ_URL'],
    )

    # set gerrit object
    gerrit_obj = Gerrit(config['GERRIT_SRV'])

    return bz_obj, gerrit_obj
Beispiel #16
0
            current_path_set != last_patch_set)
        if last_patch_set != current_path_set:
            change["diff_url"] = "http://%s/#/c/%s/%d..%d//COMMIT_MSG" % (
                host, change["number"], last_patch_set, current_path_set
            )
    else:
        change["change_since_last_comment"] = True
        change["last_checked_patch_set"] = -1

    return change


def fit_width(s, n):
    if len(s) > n:
        return s[:n-3] + "..."
    else:
        return s + " " * (n - len(s))

pkey = get_private_key()
g = Gerrit(host, port, username, pkey)
changes = g.query(query,
                  options=[QueryOptions.Comments,
                           QueryOptions.CurrentPatchSet])
changes = imap(remove_jenkins_comments, changes)
changes = imap(add_last_checked_information, changes)
changes = ifilter(not_mine, changes)
changes = ifilter(has_changed_since_comment, changes)
sys.stdout.write(str(Template(file=template_file,
                              searchList=[{"changes": changes,
                                           "fit_width": fit_width}])))
class Submitter(object):
    def __init__(self, last_cid, review, verify, ready):
        self.vote_review = 2 if review else None
        self.vote_verify = 1 if verify else None
        self.vote_trybot_ready = 1 if ready else None
        self.vote_cq_ready = 1 if ready else None

        self.tag = 'autogenerated:submit-o-matic'

        self.max_in_flight = 100  # 50 for the cq, 50 for the pre-cq
        self.in_flight = []

        self.changes = []
        self.gerrit = Gerrit('https://chromium-review.googlesource.com')
        last_change = self.gerrit.get_change(last_cid)
        self.changes = self.gerrit.get_related_changes(last_change)
        if not self.changes:
            self.changes = [last_change]
        else:
            self.changes.reverse()

    def change_needs_action(self, change):
        return change.is_merged() or \
               (self.vote_review and not change.is_reviewed()) or \
               (self.vote_verify and not change.is_verified()) or \
               (self.vote_cq_ready and not change.is_cq_ready())

    def num_changes(self):
        return len(self.changes)

    def num_in_flight(self):
        return len(self.in_flight)

    def review_changes(self):
        for i, c in enumerate(self.changes):
            sys.stdout.write('\rRunning reviewer (%d/%d)' %
                             (i, self.num_changes()))
            c = self.gerrit.get_change(c.change_id)
            if c.is_merged() or not self.change_needs_action(c):
                continue

            self.gerrit.review(c, self.tag, '', False, self.vote_review,
                               self.vote_verify, None, self.vote_trybot_ready)

    def submit_changes(self):
        self.in_flight = []
        merged = 0
        for i, c in enumerate(self.changes):
            if self.num_in_flight() >= self.max_in_flight:
                break

            sys.stdout.write('\rRunning submitter (%d/%d)' %
                             (i, self.num_changes()))
            c = self.gerrit.get_change(c.change_id)
            if c.is_merged():
                merged += 1
                continue

            if self.change_needs_action(c):
                self.gerrit.review(c, self.tag, '', False, self.vote_review,
                                   self.vote_verify, self.vote_cq_ready,
                                   self.vote_trybot_ready)

            self.in_flight.append(c)

        sys.stdout.write(
            '\r%d Changes:                                       \n' %
            self.num_changes())
        sys.stdout.write('-- %d merged\n' % merged)
        sys.stdout.write('-- %d in flight\n' % self.num_in_flight())

    def detect_change(self):
        if self.num_in_flight(
        ) == 0:  # everything is merged, so no detection needed
            return True

        c = self.in_flight[0]
        sys.stdout.write('\rDetecting: %s' % c.url())
        c = self.gerrit.get_change(c.change_id)
        if self.change_needs_action(c):
            return True

        return False
Beispiel #18
0
 def __init__(self, host, port, username, pkey):
     self._gerrit = Gerrit(host, port, username, pkey)
     self._flow = _IDENTITY_FLOW
Beispiel #19
0
class Troll(object):
  RETRY_REVIEW_KEY='retry-bot-review'

  def __init__(self, config):
    self.config = config
    self.gerrit = Gerrit(config.gerrit_url, netrc=config.netrc)
    self.gerrit_admin = Gerrit(config.gerrit_url, netrc=config.netrc_admin)
    self.tag = 'autogenerated:review-o-matic'
    self.ignore_list = {}
    self.stats = TrollStats('{}'.format(self.config.stats_file))

  def do_review(self, project, change, review):
    logger.info('Review for change: {}'.format(change.url()))
    logger.info('  Issues: {}, Feedback: {}, Vote:{}, Notify:{}'.format(
        review.issues.keys(), review.feedback.keys(), review.vote,
        review.notify))

    if review.dry_run:
      print(review.generate_review_message(self.RETRY_REVIEW_KEY))
      if review.inline_comments:
        print('')
        print('-- Inline comments:')
        for f,comments in review.inline_comments.items():
          for c in comments:
            print('{}:{}'.format(f, c['line']))
            print(c['message'])

      print('------')
      return

    self.stats.update_for_review(project, review)

    self.gerrit.review(change, self.tag,
                       review.generate_review_message(self.RETRY_REVIEW_KEY),
                       review.notify, vote_code_review=review.vote,
                       inline_comments=review.inline_comments)

    if self.config.results_file:
      with open(self.config.results_file, 'a+') as f:
        f.write('{}: Issues: {}, Feedback: {}, Vote:{}, Notify:{}\n'.format(
          change.url(), review.issues.keys(), review.feedback.keys(),
          review.vote, review.notify))

  def get_changes(self, project, prefix):
    message = '{}:'.format(prefix)
    after = datetime.date.today() - datetime.timedelta(days=5)
    changes = self.gerrit.query_changes(status='open', message=message,
                    after=after, project=project.gerrit_project,
                    branches=project.monitor_branches)
    return changes

  def add_change_to_ignore_list(self, change):
    self.ignore_list[change.number] = change.current_revision.number

  def is_change_in_ignore_list(self, change):
    return self.ignore_list.get(change.number) == change.current_revision.number

  def process_change(self, project, rev, c):
    if self.config.chatty:
      logger.debug('Processing change {}'.format(c.url()))

    force_review = self.config.force_cl or self.config.force_all

    # Look for a retry request in the topic
    retry_request = False
    topic_list = c.topic.split() if c.topic else None
    if topic_list and self.RETRY_REVIEW_KEY in topic_list:
      retry_request = True
      force_review = True
      logger.error('Received retry request on change {} (topic={})'.format(
                   c.url(), c.topic))

    # Look for prior reviews and retry requests
    last_review = None
    for m in c.get_messages():
      if not m.revision_num == c.current_revision.number:
        continue
      if m.tag == self.tag:
        last_review = m

    age_days = None
    if not force_review and last_review:
      age_days = (datetime.datetime.utcnow() - last_review.date).days

    if age_days != None and self.config.chatty:
      logger.debug('    Reviewed {} days ago'.format(age_days))

    # Find a reviewer or ignore if not found
    reviewer = None
    if not ChangeReviewer.can_review_change(project, c, age_days):
      # Some patches are blanket unreviewable, check these first
      reviewer = None
    elif FromlistChangeReviewer.can_review_change(project, c, age_days):
      reviewer = FromlistChangeReviewer(project, rev, c,
                                        self.config.gerrit_msg_limit,
                                        self.config.dry_run)
    elif FromgitChangeReviewer.can_review_change(project, c, age_days):
      reviewer = FromgitChangeReviewer(project, rev, c,
                                       self.config.gerrit_msg_limit,
                                       self.config.dry_run, age_days)
    elif UpstreamChangeReviewer.can_review_change(project, c, age_days):
      reviewer = UpstreamChangeReviewer(project, rev, c,
                                        self.config.gerrit_msg_limit,
                                        self.config.dry_run)
    elif ChromiumChangeReviewer.can_review_change(project, c, age_days):
      reviewer = ChromiumChangeReviewer(project, rev, c,
                                        self.config.gerrit_msg_limit,
                                        self.config.dry_run,
                                        self.config.verbose)

    # Clear the retry request from the topic
    if retry_request:
      topic_list.remove(self.RETRY_REVIEW_KEY)
      c.topic = ' '.join(topic_list)
      if not self.gerrit_admin.set_topic(c):
        logger.error('ERROR: Failed to clear retry request from change')
        return None

    if not reviewer:
      self.add_change_to_ignore_list(c)
      return None

    if not force_review and self.is_change_in_ignore_list(c):
      return None

    return reviewer.review_patch()

  def process_changes(self, project, changes):
    rev = Reviewer(git_dir=project.local_repo, verbose=self.config.verbose,
                   chatty=self.config.chatty)
    ret = 0
    for c in changes:
      ignore = False
      for b in project.ignore_branches:
        if re.match(b, c.branch):
          ignore = True
          break
      if ignore:
        if self.config.chatty:
          logger.debug('Ignoring change {}'.format(c))
        self.add_change_to_ignore_list(c)
        continue

      try:
        result = self.process_change(project, rev, c)
        if result:
          self.do_review(project, c, result)
          ret += 1
        self.add_change_to_ignore_list(c)
      except GerritFetchError as e:
        logger.error('Gerrit fetch failed, will retry, {}'.format(c.url()))
        logger.exception('Exception: {}'.format(e))
        # Don't add change to ignore list, we want to retry next time
      except Exception as e:
        logger.error('Exception processing change {}'.format(c.url()))
        logger.exception('Exception: {}'.format(e))
        self.add_change_to_ignore_list(c)

    return ret

  def run(self):
    if self.config.force_cl:
      c = self.gerrit.get_change(self.config.force_cl, self.config.force_rev)
      logger.info('Force reviewing change  {}'.format(c))
      project = self.config.get_project(c.project)
      if not project:
        raise ValueError('Could not find project!')
      self.process_changes(project, [c])
      return

    while True:
      try:
        did_review = 0
        for project in self.config.projects.values():
          if (self.config.force_project and
              project.name != self.config.force_project):
            continue
          if self.config.chatty:
            logger.debug('Running for project {}'.format(project.name))
          for p in project.prefixes:
            changes = self.get_changes(project, p)
            if self.config.chatty:
              logger.debug('{} changes for prefix {}'.format(len(changes), p))
            did_review += self.process_changes(project, changes)

        if did_review > 0:
          self.stats.summarize(logging.INFO)
          if not self.config.dry_run:
            self.stats.save()

        if not self.config.daemon:
          return
        if self.config.chatty:
          logger.debug('Finished! Going to sleep until next run')

      except (requests.exceptions.HTTPError, OSError) as e:
        logger.error('Error getting changes: ({})'.format(str(e)))
        logger.exception('Exception getting changes: {}'.format(e))
        time.sleep(60)

      time.sleep(120)
Beispiel #20
0
    else:
        change["change_since_last_comment"] = True
        change["last_checked_patch_set"] = -1

    return change


def fit_width(s, n):
    if len(s) > n:
        return s[: n - 3] + "..."
    else:
        return s + " " * (n - len(s))


pkey = get_private_key()
g = Gerrit(host, port, username, pkey)
changes = g.query(query, options=[QueryOptions.Comments, QueryOptions.CurrentPatchSet, QueryOptions.CommitMessage])

changes = imap(remove_jenkins_comments, changes)
changes = imap(add_last_checked_information, changes)
changes = imap(extract_headers, changes)
changes = imap(does_relate_to_bug, changes)
changes = imap(is_spec, changes)
# changes = ifilter(not_mine, changes)
changes = ifilter(has_changed_since_comment, changes)
sys.stdout.write(
    str(
        Template(
            file=template_path,
            searchList=[{"changes": changes, "fit_width": fit_width, "terminal_size": get_terminal_size()}],
        )
Beispiel #21
0
class Troll(object):
    def __init__(self, url, args):
        self.url = url
        self.args = args
        self.gerrit = Gerrit(url)
        self.tag = 'autogenerated:review-o-matic'
        self.blacklist = {}
        self.stats = {
            str(ReviewType.SUCCESS): 0,
            str(ReviewType.BACKPORT): 0,
            str(ReviewType.ALTERED_UPSTREAM): 0,
            str(ReviewType.MISSING_FIELDS): 0,
            str(ReviewType.MISSING_HASH): 0,
            str(ReviewType.INVALID_HASH): 0,
            str(ReviewType.MISSING_AM): 0,
            str(ReviewType.INCORRECT_PREFIX): 0,
            str(ReviewType.FIXES_REF): 0
        }

    def inc_stat(self, review_type):
        if self.args.dry_run:
            return
        key = str(review_type)
        if not self.stats.get(key):
            self.stats[key] = 1
        else:
            self.stats[key] += 1

    def do_review(self, change, review):
        print('Review for change: {}'.format(change.url()))
        print('  Issues: {}, Feedback: {}, Vote:{}, Notify:{}'.format(
            review.issues.keys(), review.feedback.keys(), review.vote,
            review.notify))

        if review.dry_run:
            print(review.generate_review_message())
            print('------')
            return

        for i in review.issues:
            self.inc_stat(i)
        for f in review.feedback:
            self.inc_stat(f)
        self.gerrit.review(change,
                           self.tag,
                           review.generate_review_message(),
                           review.notify,
                           vote_code_review=review.vote)

    def get_changes(self, prefix):
        message = '{}:'.format(prefix)
        after = datetime.date.today() - datetime.timedelta(days=5)
        changes = self.gerrit.query_changes(
            status='open',
            message=message,
            after=after,
            project='chromiumos/third_party/kernel')
        return changes

    def add_change_to_blacklist(self, change):
        self.blacklist[change.number] = change.current_revision.number

    def is_change_in_blacklist(self, change):
        return self.blacklist.get(
            change.number) == change.current_revision.number

    def process_changes(self, changes):
        rev = Reviewer(git_dir=self.args.git_dir,
                       verbose=self.args.verbose,
                       chatty=self.args.chatty)
        ret = 0
        for c in changes:
            if self.args.verbose:
                print('Processing change {}'.format(c.url()))

            # Blacklist if we've already reviewed this revision
            for m in c.messages:
                if m.tag == self.tag and m.revision_num == c.current_revision.number:
                    self.add_change_to_blacklist(c)

            # Find a reviewer and blacklist if not found
            reviewer = None
            if FromlistChangeReviewer.can_review_change(c):
                reviewer = FromlistChangeReviewer(rev, c, self.args.dry_run)
            elif FromgitChangeReviewer.can_review_change(c):
                reviewer = FromgitChangeReviewer(rev, c, self.args.dry_run)
            elif UpstreamChangeReviewer.can_review_change(c):
                reviewer = UpstreamChangeReviewer(rev, c, self.args.dry_run)
            if not reviewer:
                self.add_change_to_blacklist(c)
                continue

            force_review = self.args.force_cl or self.args.force_all
            if not force_review and self.is_change_in_blacklist(c):
                continue

            result = reviewer.review_patch()
            if result:
                self.do_review(c, result)
                ret += 1

            self.add_change_to_blacklist(c)

        return ret

    def update_stats(self):
        if not self.args.dry_run and self.args.stats_file:
            with open(self.args.stats_file, 'wt') as f:
                json.dump(self.stats, f)
        print('--')
        summary = '  Summary: '
        total = 0
        for k, v in self.stats.items():
            summary += '{}={} '.format(k, v)
            total += v
        summary += 'total={}'.format(total)
        print(summary)
        print('')

    def run(self):
        if self.args.force_cl:
            c = self.gerrit.get_change(self.args.force_cl)
            print('Force reviewing change  {}'.format(c))
            self.process_changes([c])
            return

        if self.args.stats_file:
            try:
                with open(self.args.stats_file, 'rt') as f:
                    self.stats = json.load(f)
            except FileNotFoundError:
                self.update_stats()

        prefixes = ['UPSTREAM', 'BACKPORT', 'FROMGIT', 'FROMLIST']
        while True:
            try:
                did_review = 0
                for p in prefixes:
                    changes = self.get_changes(p)
                    if self.args.verbose:
                        print('{} changes for prefix {}'.format(
                            len(changes), p))
                    did_review += self.process_changes(changes)
                if did_review > 0:
                    self.update_stats()
                if not self.args.daemon:
                    break
                if self.args.verbose:
                    print('Finished! Going to sleep until next run')

            except (requests.exceptions.HTTPError, OSError) as e:
                sys.stderr.write('Error getting changes: ({})\n'.format(
                    str(e)))
                time.sleep(60)

            time.sleep(120)
Beispiel #22
0
class Troll(object):
    STRING_HEADER = '''
-- Automated message --
'''
    STRING_SUCCESS = '''
This change does not differ from its upstream source. It is certified {}
by review-o-matic!
'''
    STRING_INCORRECT_PREFIX = '''
This change has a BACKPORT prefix, however it does not differ from its upstream
source. The BACKPORT prefix should be primarily used for patches which were
altered during the cherry-pick (due to conflicts or downstream inconsistencies).

Consider changing your subject prefix to UPSTREAM (or FROMGIT/FROMLIST as
appropriate) to better reflect the contents of this patch.
'''
    STRING_MISSING_FIELDS = '''
Your commit message is missing the following required field(s):
    {}
'''
    STRING_MISSING_FIELDS_SUCCESS = '''
Don't worry, there is good news! Your patch does not differ from its upstream
source. Once the missing fields are present, it will be certified {} (or some
other similarly official-sounding certification) by review-o-matic!
'''
    STRING_MISSING_FIELDS_DIFF = '''
In addition to the missing fields, this patch differs from its upstream source.
This may be expected, this message is posted to make reviewing backports easier.
'''
    STRING_MISSING_HASH_HEADER = '''
Your commit message is missing the upstream commit hash. It should be in the
form:
'''
    STRING_MISSING_HASH_FMT_FROMGIT = '''
    (cherry picked from commit <commit SHA>
     <remote git url> <remote git tree>)
'''
    STRING_MISSING_HASH_FMT_UPSTREAM = '''
    (cherry picked from commit <commit SHA>)
'''
    STRING_MISSING_HASH_FOOTER = '''
Hint: Use the '-x' argument of git cherry-pick to add this automagically
'''
    STRING_MISSING_AM = '''
Your commit message is missing the patchwork URL. It should be in the
form:
    (am from https://patchwork.kernel.org/.../)
'''
    STRING_UNSUCCESSFUL_HEADER = '''
This patch differs from the source commit.

'''
    STRING_UPSTREAM_DIFF = '''
Since this is an UPSTREAM labeled patch, it shouldn't. Either this reviewing
script is incorrect (totally possible, pls send patches!), or something changed
when this was backported. If the backport required changes, please consider
using the BACKPORT label with a description of you downstream changes in your
commit message.
'''
    STRING_BACKPORT_DIFF = '''
This is expected, and this message is posted to make reviewing backports easier.
'''
    STRING_FROMGIT_DIFF = '''
This may be expected, this message is posted to make reviewing backports easier.
'''
    STRING_UNSUCCESSFUL_FOOTER = '''
Below is a diff of the upstream patch referenced in this commit message, vs this
patch.

'''
    STRING_FOUND_FIXES_REF = '''
!! NOTE: This patch has been referenced in the Fixes: tag of another commit. If
!!       you haven't already, consider backporting the following patch:
!!  {}
'''
    STRING_FOOTER = '''
---
To learn more about backporting kernel patches to Chromium OS, check out:
  https://chromium.googlesource.com/chromiumos/docs/+/master/kernel_faq.md#UPSTREAM_BACKPORT_FROMLIST_and-you

If you're curious about how this message was generated, head over to:
  https://github.com/atseanpaul/review-o-matic

This link is not useful:
  https://thats.poorly.run/
'''

    SWAG = [
        'Frrrresh', 'Crisper Than Cabbage', 'Awesome', 'Ahhhmazing',
        'Cool As A Cucumber', 'Most Excellent', 'Eximious', 'Prestantious',
        'Supernacular', 'Bodacious', 'Blue Chip', 'Blue Ribbon', 'Cracking',
        'Dandy', 'Dynamite', 'Fab', 'Fabulous', 'Fantabulous',
        'Scrumtrulescent', 'First Class', 'First Rate', 'First String',
        'Five Star', 'Gangbusters', 'Grand', 'Groovy', 'HYPE', 'Jim-Dandy',
        'Snazzy', 'Marvelous', 'Nifty', 'Par Excellence', 'Peachy Keen',
        'PHAT', 'Prime', 'Prizewinning', 'Quality', 'Radical', 'Righteous',
        'Sensational', 'Slick', 'Splendid', 'Lovely', 'Stellar', 'Sterling',
        'Superb', 'Superior', 'Superlative', 'Supernal', 'Swell', 'Terrific',
        'Tip-Top', 'Top Notch', 'Top Shelf', 'Unsurpassed', 'Wonderful'
    ]

    def __init__(self, url, args):
        self.url = url
        self.args = args
        self.gerrit = Gerrit(url)
        self.tag = 'autogenerated:review-o-matic'
        self.blacklist = []
        self.stats = {
            ReviewType.SUCCESS: 0,
            ReviewType.BACKPORT: 0,
            ReviewType.ALTERED_UPSTREAM: 0,
            ReviewType.MISSING_FIELDS: 0,
            ReviewType.MISSING_HASH: 0,
            ReviewType.INCORRECT_PREFIX: 0,
            ReviewType.FIXES_REF: 0
        }

    def do_review(self, review_type, change, fixes_ref, msg, notify, vote):
        final_msg = self.STRING_HEADER
        if fixes_ref:
            print('Adding fixes ref for change {}'.format(change.url()))
            self.stats[ReviewType.FIXES_REF] += 1
            final_msg += self.STRING_FOUND_FIXES_REF.format(fixes_ref)
        final_msg += msg
        final_msg += self.STRING_FOOTER

        self.stats[review_type] += 1
        if not self.args.dry_run:
            self.gerrit.review(change,
                               self.tag,
                               final_msg,
                               notify,
                               vote_code_review=vote)
        else:
            print('Review for change: {}'.format(change.url()))
            print('  Type:{}, Vote:{}, Notify:{}'.format(
                review_type, vote, notify))
            print(final_msg)
            print('------')

    def handle_successful_review(self, change, prefix, fixes_ref):
        # TODO: We should tag FROMLIST: BACKPORT: patches as incorrect, if needed
        if prefix == 'BACKPORT':
            print('Adding incorrect prefix review for change {}'.format(
                change.url()))
            msg = self.STRING_INCORRECT_PREFIX
            self.do_review(ReviewType.INCORRECT_PREFIX, change, fixes_ref, msg,
                           True, 0)
        else:
            print('Adding successful review for change {}'.format(
                change.url()))
            msg = self.STRING_SUCCESS.format(random.choice(self.SWAG))
            self.do_review(ReviewType.SUCCESS, change, fixes_ref, msg, True, 1)

    def handle_missing_fields_review(self, change, fields, result, fixes_ref):
        print('Adding missing fields review for change {}'.format(
            change.url()))
        missing = []
        if not fields['bug']:
            missing.append('BUG=')
        if not fields['test']:
            missing.append('TEST=')
        if not fields['sob']:
            cur_rev = change.current_revision
            missing.append('Signed-off-by: {} <{}>'.format(
                cur_rev.uploader_name, cur_rev.uploader_email))

        msg = self.STRING_MISSING_FIELDS.format(', '.join(missing))
        if len(result) == 0:
            msg += self.STRING_MISSING_FIELDS_SUCCESS.format(
                random.choice(self.SWAG))
        else:
            msg += self.STRING_MISSING_FIELDS_DIFF
            msg += self.STRING_UNSUCCESSFUL_FOOTER
            for l in result:
                msg += '{}\n'.format(l)

        self.do_review(ReviewType.MISSING_FIELDS, change, fixes_ref, msg, True,
                       -1)

    def handle_missing_hash_review(self, change, prefix):
        print('Adding missing hash review for change {}'.format(change.url()))
        msg = self.STRING_MISSING_HASH_HEADER
        if prefix == 'FROMGIT':
            msg += self.STRING_MISSING_HASH_FMT_FROMGIT
        else:
            msg += self.STRING_MISSING_HASH_FMT_UPSTREAM
        msg += self.STRING_MISSING_HASH_FOOTER
        self.do_review(ReviewType.MISSING_HASH, change, None, msg, True, -1)

    def handle_missing_am_review(self, change, prefix):
        print('Adding missing am URL for change {}'.format(change.url()))
        self.do_review(ReviewType.MISSING_HASH, change, None,
                       self.STRING_MISSING_AM, True, -1)

    def handle_unsuccessful_review(self, change, prefix, result, fixes_ref):
        vote = 0
        notify = False
        review_type = ReviewType.BACKPORT

        msg = self.STRING_UNSUCCESSFUL_HEADER
        if prefix == 'UPSTREAM':
            review_type = ReviewType.ALTERED_UPSTREAM
            vote = -1
            notify = True
            msg += self.STRING_UPSTREAM_DIFF
        elif prefix == 'BACKPORT':
            msg += self.STRING_BACKPORT_DIFF
        elif prefix == 'FROMGIT' or prefix == 'FROMLIST':
            msg += self.STRING_FROMGIT_DIFF

        msg += self.STRING_UNSUCCESSFUL_FOOTER

        for l in result:
            msg += '{}\n'.format(l)

        print('Adding unsuccessful review (vote={}) for change {}'.format(
            vote, change.url()))

        self.do_review(review_type, change, fixes_ref, msg, notify, vote)

    def get_changes(self, prefix):
        message = '{}:'.format(prefix)
        after = datetime.date.today() - datetime.timedelta(days=5)
        changes = self.gerrit.query_changes(
            status='open',
            message=message,
            after=after,
            project='chromiumos/third_party/kernel')
        return changes

    def print_error(self, error):
        if self.args.verbose:
            sys.stderr.write('\n')
        sys.stderr.write(error)

    def process_changes(self, prefix, changes):
        rev = Reviewer(git_dir=self.args.git_dir,
                       verbose=self.args.verbose,
                       chatty=self.args.chatty)
        num_changes = len(changes)
        cur_change = 1
        line_feed = False
        ret = False
        for c in changes:
            cur_rev = c.current_revision

            if self.args.chatty:
                print('Processing change {}'.format(c.url()))
            elif self.args.verbose:
                sys.stdout.write('{}Processing change {}/{}'.format(
                    '\r' if line_feed else '', cur_change, num_changes))
                cur_change += 1

            line_feed = True

            if c in self.blacklist:
                continue

            if not c.subject.startswith(prefix):
                continue

            skip = False
            for m in c.messages:
                if m.tag == self.tag and m.revision_num == cur_rev.number:
                    skip = True
            if skip and not self.args.force_cl:
                continue

            ret = True
            line_feed = False
            if self.args.verbose:
                print('')

            gerrit_patch = rev.get_commit_from_remote('cros', cur_rev.ref)

            if prefix == 'FROMLIST':
                upstream_patchworks = rev.get_am_from_from_patch(gerrit_patch)
                if not upstream_patchworks:
                    self.handle_missing_am_review(c, prefix)
                    continue

                upstream_patch = None
                for u in reversed(upstream_patchworks):
                    try:
                        upstream_patch = rev.get_commit_from_patchwork(u)
                        break
                    except:
                        continue

                if not upstream_patch:
                    self.print_error(
                        'ERROR: patch missing from patchwork, or patchwork host '
                        'not whitelisted for {} ({})\n'.format(
                            c, upstream_patchworks))
                    self.blacklist.append(c)
                    continue
            else:
                upstream_shas = rev.get_cherry_pick_shas_from_patch(
                    gerrit_patch)
                if not upstream_shas:
                    self.handle_missing_hash_review(c, prefix)
                    continue

                upstream_patch = None
                upstream_sha = None
                for s in reversed(upstream_shas):
                    try:
                        upstream_patch = rev.get_commit_from_sha(s)
                        upstream_sha = s
                        break
                    except:
                        continue

                if not upstream_patch:
                    self.print_error(
                        'ERROR: SHA missing from git for {} ({})\n'.format(
                            c, upstream_shas))
                    self.blacklist.append(c)
                    continue

            if prefix != 'FROMLIST':
                fixes_ref = rev.find_fixes_reference(upstream_sha)
            else:
                fixes_ref = None

            result = rev.compare_diffs(upstream_patch, gerrit_patch)

            fields = {'sob': False, 'bug': False, 'test': False}
            sob_re = re.compile('Signed-off-by:\s+{}'.format(
                cur_rev.uploader_name))
            for l in cur_rev.commit_message.splitlines():
                if l.startswith('BUG='):
                    fields['bug'] = True
                    continue
                if l.startswith('TEST='):
                    fields['test'] = True
                    continue
                if sob_re.match(l):
                    fields['sob'] = True
                    continue
            if not fields['bug'] or not fields['test'] or not fields['sob']:
                self.handle_missing_fields_review(c, fields, result, fixes_ref)
                continue

            if len(result) == 0:
                self.handle_successful_review(c, prefix, fixes_ref)
                continue

            self.handle_unsuccessful_review(c, prefix, result, fixes_ref)

        if self.args.verbose:
            print('')

        return ret

    def run(self):
        if self.args.force_cl != None:
            c = self.gerrit.get_change(self.args.force_cl)
            prefix = c.subject.split(':')[0]
            print('Force reviewing change  {}'.format(c))
            self.process_changes(prefix, [c])
            return

        while True:
            try:
                prefixes = ['UPSTREAM', 'BACKPORT', 'FROMGIT']
                did_review = False
                for p in prefixes:
                    changes = self.get_changes(p)
                    if self.args.verbose:
                        print('{} changes for prefix {}'.format(
                            len(changes), p))
                    did_review |= self.process_changes(p, changes)
                if did_review:
                    print('--')
                    summary = '  Summary: '
                    for k, v in self.stats.items():
                        summary += '{}={} '.format(k, v)
                    print(summary)
                    print('')
                if not self.args.daemon:
                    break
                if self.args.verbose:
                    print('Finished! Going to sleep until next run')

            except requests.exceptions.HTTPError as e:
                self.print_error('HTTPError ({})\n'.format(
                    e.response.status_code))
                time.sleep(60)

            time.sleep(120)
Beispiel #23
0
class Submitter(object):
    def __init__(self, last_cid, review, verify, ready, abandon, force_review,
                 dry_run):
        self.abandon = abandon
        self.vote_review = 2 if review else None
        self.vote_verify = 1 if verify else None
        self.vote_cq_ready = ready
        self.force_review = force_review

        self.dry_run = dry_run

        self.tag = 'autogenerated:submit-o-matic'

        self.max_in_flight = 100  # 50 for the cq, 50 for the pre-cq
        self.in_flight = []

        self.changes = []
        self.gerrit = Gerrit('https://chromium-review.googlesource.com',
                             use_internal=False)
        last_change = self.gerrit.get_change(last_cid)
        ancestor_changes = self.gerrit.get_ancestor_changes(last_change)
        for c in reversed(ancestor_changes):
            if c.status == 'NEW':
                self.changes.append(c)
        self.changes.append(last_change)

    def change_needs_action(self, change):
        return change.is_merged() or \
               (self.vote_review and not change.is_reviewed()) or \
               (self.vote_verify and not change.is_verified()) or \
               (self.vote_cq_ready and not change.is_cq_ready())

    def num_changes(self):
        return len(self.changes)

    def num_in_flight(self):
        return len(self.in_flight)

    def review_changes(self):
        if not self.vote_review and not self.vote_verify and not self.abandon:
            return

        for i, c in enumerate(self.changes):
            sys.stdout.write('\rRunning reviewer (%d/%d)' %
                             (i, self.num_changes()))
            c = self.gerrit.get_change(c.number)

            if self.abandon:
                if not self.dry_run:
                    self.gerrit.abandon(c)
                else:
                    print('DRYRUN abandon {}'.format(c))
                continue

            if (c.is_merged() or
                    not self.change_needs_action(c)) and not self.force_review:
                continue

            if not self.dry_run:
                self.gerrit.review(c, self.tag, '', False, self.vote_review,
                                   self.vote_verify, None)
            else:
                print('DRYRUN review (r={}, v={}) {}'.format(
                    self.vote_review, self.vote_verify, c))

    def submit_changes(self):
        if self.abandon:
            return

        self.in_flight = []
        merged = 0
        for i, c in enumerate(self.changes):
            if self.num_in_flight() >= self.max_in_flight:
                break

            sys.stdout.write('\rRunning submitter (%d/%d)' %
                             (i, self.num_changes()))
            c = self.gerrit.get_change(c.id)
            if c.is_merged():
                merged += 1
                continue

            if self.change_needs_action(c):
                if not self.dry_run:
                    self.gerrit.review(c, self.tag, '', False,
                                       self.vote_review, self.vote_verify,
                                       self.vote_cq_ready)
                else:
                    print('DRYRUN review (r={}, v={} cq={}) {}'.format(
                        self.vote_review, self.vote_verify, self.vote_cq_ready,
                        c))

            self.in_flight.append(c)

        sys.stdout.write(
            '\r%d Changes:                                       \n' %
            self.num_changes())
        sys.stdout.write('-- %d merged\n' % merged)
        sys.stdout.write('-- %d in flight\n' % self.num_in_flight())

    def detect_change(self):
        if self.num_in_flight(
        ) == 0:  # everything is merged, so no detection needed
            return True

        c = self.in_flight[0]
        sys.stdout.write('\rDetecting: %s' % c.url())
        c = self.gerrit.get_change(c.id)
        if self.change_needs_action(c):
            return True

        return False