class GerritStatusPush(StatusReceiverMultiService): """Add a comment to a gerrit code review indicating the result of a build.""" def __init__(self, gerrit_url, buildbot_url): StatusReceiverMultiService.__init__(self) self.agent = GerritAgent(gerrit_url) self.buildbot_url = buildbot_url def startService(self): StatusReceiverMultiService.startService(self) self.status = self.parent.getStatus() # pylint: disable=W0201 self.status.subscribe(self) def builderAdded(self, name, builder): return self # subscribe to this builder def getMessage(self, builderName, build, result): message = "Buildbot finished compiling your patchset\n" message += "on configuration: %s\n" % builderName message += "The result is: %s\n" % Results[result].upper() message += '%sbuilders/%s/builds/%s\n' % ( self.buildbot_url, urllib.quote(build.getProperty('buildername')), build.getProperty('buildnumber')) return message def buildFinished(self, builderName, build, result): if 'event.change.number' not in build.getProperties(): return change_number = build.getProperty('event.change.number') revision = build.getProperty('revision') message = self.getMessage(builderName, build, result) verified = '+1' if (result == 0) else '-1' path = '/changes/%s/revisions/%s/review' % (change_number, revision) body = {'message': message, 'labels': {'Verified': verified}} self.agent.request('POST', path, body=body)
class GerritStatusPush(StatusReceiverMultiService): """Add a comment to a gerrit code review indicating the result of a build.""" def __init__(self, gerrit_url, buildbot_url): StatusReceiverMultiService.__init__(self) self.agent = GerritAgent(gerrit_url) self.buildbot_url = buildbot_url def startService(self): StatusReceiverMultiService.startService(self) self.status = self.parent.getStatus() # pylint: disable=W0201 self.status.subscribe(self) def builderAdded(self, name, builder): return self # subscribe to this builder def getMessage(self, builderName, build, result): message = "Buildbot finished compiling your patchset\n" message += "on configuration: %s\n" % builderName message += "The result is: %s\n" % Results[result].upper() message += '%sbuilders/%s/builds/%s\n' % ( self.buildbot_url, urllib.quote(build.getProperty('buildername')), build.getProperty('buildnumber')) return message def buildFinished(self, builderName, build, result): if 'event.change.number' not in build.getProperties(): return change_number = build.getProperty('event.change.number') revision = build.getProperty('revision') message = self.getMessage(builderName, build, result) verified = '+1' if (result == 0) else '-1' path = '/changes/%s/revisions/%s/review' % (change_number, revision) body = {'message': message, 'labels': {'Verified': verified}} self.agent.request('POST', path, None, body)
class GerritPoller(base.PollingChangeSource): """A poller which queries a gerrit server for new changes and patchsets.""" change_category = 'patchset-created' def __init__(self, gerrit_host, gerrit_projects=None, pollInterval=None, dry_run=None): if isinstance(gerrit_projects, basestring): gerrit_projects = [gerrit_projects] self.gerrit_projects = gerrit_projects if pollInterval: self.pollInterval = pollInterval self.initLock = defer.DeferredLock() self.last_timestamp = None self.agent = GerritAgent(gerrit_host) if dry_run is None: dry_run = 'POLLER_DRY_RUN' in os.environ self.dry_run = dry_run @staticmethod def _parse_timestamp(tm): tm = tm[:tm.index('.')+7] return datetime.datetime.strptime(tm, '%Y-%m-%d %H:%M:%S.%f') def startService(self): if not self.dry_run: self.initLastTimeStamp() base.PollingChangeSource.startService(self) def getChangeQuery(self): # pylint: disable=R0201 return 'status:open' @deferredLocked('initLock') def initLastTimeStamp(self): log.msg('GerritPoller: Getting latest timestamp from gerrit server.') path = '/changes/?q=%s&n=1' % self.getChangeQuery() d = self.agent.request('GET', path) def _get_timestamp(j): if len(j) == 0: self.last_timestamp = datetime.datetime.now() else: self.last_timestamp = self._parse_timestamp(j[0]['updated']) d.addCallback(_get_timestamp) return d def getChanges(self, sortkey=None): path = '/changes/?q=%s&n=10' % self.getChangeQuery() if sortkey: path += '&N=%s' % sortkey return self.agent.request('GET', path) def _is_interesting_message(self, message): # pylint: disable=R0201 return message['message'].startswith('Uploaded patch set ') def checkForNewPatchset(self, change, since): o_params = '&'.join('o=%s' % x for x in ( 'MESSAGES', 'ALL_REVISIONS', 'ALL_COMMITS', 'ALL_FILES')) path = '/changes/%s?%s' % (change['_number'], o_params) d = self.agent.request('GET', path) def _parse_messages(j): if not j or 'messages' not in j: return for m in reversed(j['messages']): if self._parse_timestamp(m['date']) <= since: break if self._is_interesting_message(m): return j, m d.addCallback(_parse_messages) return d def getChangeUrl(self, change): """Generates a URL for a Gerrit change.""" # GerritAgent stores its URL as protocol and host. return '%s://%s/#/c/%s' % (self.agent.gerrit_protocol, self.agent.gerrit_host, change['_number']) def addBuildbotChange(self, change, revision=None): """Adds a buildbot change into the database. Args: change: ChangeInfo Gerrit object. Documentation: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-info revision: the sha of the buildbot change revision to use. Defaults to the value of change['current_revision'] Returns the new buildbot change as Deferred. """ revision = revision or change['current_revision'] revision_details = change['revisions'][revision] commit = revision_details['commit'] properties = { 'event.change.number': change['_number'], 'event.change.id': change['id'], 'event.change.url': self.getChangeUrl(change), } if change['status'] == 'NEW': ref = revision_details.get('fetch', {}).get('http', {}).get('ref') if ref: properties['event.patchSet.ref'] = ref elif change['status'] in ('SUBMITTED', 'MERGED'): properties['event.refUpdate.newRev'] = revision chdict = { 'author': '%s <%s>' % ( commit['author']['name'], commit['author']['email']), 'project': change['project'], 'branch': change['branch'], 'revision': revision, 'comments': commit['subject'], 'files': revision_details.get('files', {'UNKNOWN': None}).keys(), 'category': self.change_category, 'when_timestamp': self._parse_timestamp(commit['committer']['date']), 'revlink': '%s://%s/#/c/%s' % ( self.agent.gerrit_protocol, self.agent.gerrit_host, change['_number']), 'repository': '%s://%s/%s' % ( self.agent.gerrit_protocol, self.agent.gerrit_host, change['project']), 'properties': properties, } d = self.master.addChange(**chdict) d.addErrback(log.err, 'GerritPoller: Could not add buildbot change for ' 'gerrit change %s.' % revision_details['_number']) return d @staticmethod def findRevisionShaForMessage(change, message): def warn(text): log.msg('GerritPoller warning: %s. Change: %s, message: %s' % (text, change['id'], message['message'])) revision_number = message.get('_revision_number') if revision_number is None: warn('A message doesn\'t have a _revision_number') return None for sha, revision in change['revisions'].iteritems(): if revision['_number'] == revision_number: return sha warn('a revision wasn\'t found for message') def addChange(self, change, message): revision = self.findRevisionShaForMessage(change, message) return self.addBuildbotChange(change, revision) def processChanges(self, j, since): need_more = bool(j) for change in j: tm = self._parse_timestamp(change['updated']) if tm <= since: need_more = False break if self.gerrit_projects and change['project'] not in self.gerrit_projects: continue d = self.checkForNewPatchset(change, since) d.addCallback(lambda x: self.addChange(*x) if x else None) if need_more and j[-1].get('_more_changes'): d = self.getChanges(sortkey=j[-1]['_sortkey']) d.addCallback(self.processChanges, since=since) else: d = defer.succeed(None) return d @deferredLocked('initLock') def poll(self): if self.dry_run: return log.msg('GerritPoller: getting latest changes...') since = self.last_timestamp d = self.getChanges() def _update_last_timestamp(j): if j: self.last_timestamp = self._parse_timestamp(j[0]['updated']) return j d.addCallback(_update_last_timestamp) d.addCallback(self.processChanges, since=since) return d
class TryJobGerritStatus(StatusReceiverMultiService): """Posts results of a try job back to a Gerrit change.""" def __init__(self, gerrit_host, review_factory=None): """Creates a TryJobGerritStatus. Args: gerrit_host: a URL of the Gerrit instance. review_factory: a function (self, builder_name, build, result) => review, where review is a dict described in Gerrit docs: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input """ StatusReceiverMultiService.__init__(self) self.review_factory = review_factory or TryJobGerritStatus.createReview self.agent = GerritAgent(gerrit_host) self.status = None def createReview(self, builder_name, build, result): review = {} if result is not None: message = ('A try job has finished on builder %s: %s' % (builder_name, Results[result].upper())) else: message = 'A try job has started on builder %s' % builder_name # Do not send email about this. review['notify'] = 'NONE' # Append build url. # A line break in a Gerrit message is \n\n. assert self.status build_url = self.status.getURLForThing(build) message = '%s\n\n%s' % (message, build_url) review['message'] = message return review def sendUpdate(self, builder_name, build, result): """Posts a message and labels, if any, on a Gerrit change.""" props = build.properties change_id = (props.getProperty('event.change.id') or props.getProperty('parent_event.change.id')) revision = props.getProperty('revision') if change_id and revision: review = self.review_factory(self, builder_name, build, result) if review: log.msg('Sending a review for change %s: %s' % (change_id, review)) path = '/changes/%s/revisions/%s/review' % (change_id, revision) return self.agent.request('POST', path, body=review) def startService(self): StatusReceiverMultiService.startService(self) self.status = self.parent.getStatus() self.status.subscribe(self) def builderAdded(self, name, builder): # Subscribe to this builder. return self def buildStarted(self, builder_name, build): self.sendUpdate(builder_name, build, None) def buildFinished(self, builder_name, build, result): self.sendUpdate(builder_name, build, result)
class GerritPoller(base.PollingChangeSource): """A poller which queries a gerrit server for new changes and patchsets.""" # TODO(szager): Due to the way query continuation works in gerrit (using # the 'S=%d' URL parameter), there are two distinct error scenarios that # are currently unhandled: # # - A new patch set is uploaded. # - When the poller runs, the change is #11, meaning it doesn't come in # the first batch of query results. # - In between the first and second queries, another patch set is # uploaded to the same change, bumping the change up to #1 in the list. # - The second query skips ahead by 10, and never sees the change. # # - A new patch set is uploaded. # - When the poller runs, the change is #10, and appears in the first set # of query results. # - In between the first and second queries, some other change gets a new # patch set and moves up to #1, bumping the current #10 to #11. # - The second query skips 10, getting changes 11-20. So, the change that # was already processes is processed again. # # Both of these problems need the same solution: keep some state in poller of # 'patch sets already processed'; and relax the 'since' parameter to # processChanges so that it goes further back in time than the last polling # event (maybe pollInterval*3). change_category = 'patchset-created' def __init__(self, gerrit_host, gerrit_projects=None, pollInterval=None, dry_run=None): """Constructs a new Gerrit poller. Args: gerrit_host: (str or GerritAgent) If supplied as a GerritAgent, the Gerrit Agent to use when polling; otherwise, the host parameter to use to construct the GerritAgent to poll through. gerrit_projects: (list) A list of project names (str) to poll. pollInterval: (int or datetime.timedelta) The amount of time to wait in between polls. dry_run: (bool) If 'True', then polls will not actually be executed. """ if isinstance(pollInterval, datetime.timedelta): pollInterval = pollInterval.total_seconds() if isinstance(gerrit_projects, basestring): gerrit_projects = [gerrit_projects] self.gerrit_projects = gerrit_projects if pollInterval: self.pollInterval = pollInterval self.initLock = defer.DeferredLock() self.last_timestamp = None if dry_run is None: dry_run = 'POLLER_DRY_RUN' in os.environ self.dry_run = dry_run self.agent = gerrit_host if not isinstance(self.agent, GerritAgent): self.agent = GerritAgent(self.agent) def startService(self): if not self.dry_run: self.initLastTimeStamp() base.PollingChangeSource.startService(self) @staticmethod def buildQuery(terms, operator=None): """Builds a Gerrit query from terms. This function will go away once the new GerritAgent lands. """ connective = ('+%s+' % operator) if operator else '+' terms_with_parens = [('(%s)' % t) if ('+' in t) else t for t in terms] return connective.join(terms_with_parens) def getChangeQuery(self): # pylint: disable=R0201 # Fetch only open issues. terms = ['status:open'] # Filter by projects. if self.gerrit_projects: project_terms = ['project:%s' % urllib.quote(p, safe='') for p in self.gerrit_projects] terms.append(self.buildQuery(project_terms, 'OR')) return self.buildQuery(terms) def request(self, path, method='GET'): log.msg('Gerrit request: %s' % path, logLevel=logging.DEBUG) return self.agent.request(method, path) @deferredLocked('initLock') def initLastTimeStamp(self): log.msg('GerritPoller: Getting latest timestamp from gerrit server.') query = self.getChangeQuery() path = '/changes/?q=%s&n=1' % query d = self.request(path) def _get_timestamp(j): if len(j) == 0: self.last_timestamp = datetime.datetime.now() else: self.last_timestamp = ParseGerritTime(j[0]['updated']) d.addCallback(_get_timestamp) return d def getChanges(self, skip=None): path = '/changes/?q=%s&n=10' % self.getChangeQuery() if skip: path += '&S=%d' % skip return self.request(path) def _is_interesting_message(self, message): # pylint: disable=R0201 return any((check_str in message['message']) for check_str in ( 'Uploaded patch set ', 'Published edit on patch set ',)) def checkForNewPatchset(self, change, since): o_params = '&'.join('o=%s' % x for x in ( 'MESSAGES', 'ALL_REVISIONS', 'ALL_COMMITS', 'ALL_FILES')) path = '/changes/%s?%s' % (change['_number'], o_params) d = self.request(path) def _parse_messages(j): if not j or 'messages' not in j: return for m in reversed(j['messages']): if ParseGerritTime(m['date']) <= since: break if self._is_interesting_message(m): return j, m d.addCallback(_parse_messages) return d def getChangeUrl(self, change): """Generates a URL for a Gerrit change.""" # GerritAgent stores its URL as protocol and host. return '%s/#/c/%s' % (self.agent.base_url, change['_number']) def getRepositoryUrl(self, change): """Generates a URL for a Gerrit repository containing a change""" return '%s/%s' % (self.agent.base_url, change['project']) def addBuildbotChange(self, change, revision=None, additional_chdict=None): """Adds a buildbot change into the database. Args: change: ChangeInfo Gerrit object. Documentation: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-info revision: the sha of the buildbot change revision to use. Defaults to the value of change['current_revision'] Returns the new buildbot change as Deferred. """ revision = revision or change['current_revision'] revision_details = change['revisions'][revision] commit = revision_details['commit'] properties = { 'event.change.number': change['_number'], 'event.change.id': change['id'], 'event.change.url': self.getChangeUrl(change), } if change['status'] == 'NEW': ref = revision_details.get('fetch', {}).get('http', {}).get('ref') if ref: properties['event.patchSet.ref'] = ref elif change['status'] in ('SUBMITTED', 'MERGED'): properties['event.refUpdate.newRev'] = revision chdict = { 'author': '%s <%s>' % ( commit['author']['name'], commit['author']['email']), 'project': change['project'], 'branch': change['branch'], 'revision': revision, 'comments': commit['subject'], 'files': revision_details.get('files', {'UNKNOWN': None}).keys(), 'category': self.change_category, 'when_timestamp': ParseGerritTime(commit['committer']['date']), 'revlink': self.getChangeUrl(change), 'repository': self.getRepositoryUrl(change), 'properties': properties, } # Factor in external 'chdict' overrides. if additional_chdict is not None: properties.update(additional_chdict.pop('properties', {})) chdict.update(additional_chdict) chdict['properties'] = properties d = self.master.addChange(**chdict) d.addErrback(log.err, 'GerritPoller: Could not add buildbot change for ' 'gerrit change %s.' % revision_details['_number']) return d @staticmethod def findRevisionShaForMessage(change, message): def warn(text): log.msg('GerritPoller warning: %s. Change: %s, message: %s' % (text, change['id'], message['message'])) revision_number = message.get('_revision_number') if revision_number is None: warn('A message doesn\'t have a _revision_number') return None for sha, revision in change['revisions'].iteritems(): if revision['_number'] == revision_number: return sha warn('a revision wasn\'t found for message') def addChange(self, change, message): revision = self.findRevisionShaForMessage(change, message) return self.addBuildbotChange(change, revision) def processChanges(self, j, since, skip=0): need_more = bool(j) for change in j: skip += 1 tm = ParseGerritTime(change['updated']) if tm <= since: need_more = False break if self.gerrit_projects and change['project'] not in self.gerrit_projects: continue d = self.checkForNewPatchset(change, since) d.addCallback(lambda x: self.addChange(*x) if x else None) if need_more and j[-1].get('_more_changes'): d = self.getChanges(skip=skip) d.addCallback(self.processChanges, since=since, skip=skip) else: d = defer.succeed(None) return d @deferredLocked('initLock') def poll(self): if self.dry_run: return log.msg('GerritPoller: getting latest changes...') since = self.last_timestamp d = self.getChanges() def _update_last_timestamp(j): if j: self.last_timestamp = ParseGerritTime(j[0]['updated']) return j d.addCallback(_update_last_timestamp) d.addCallback(self.processChanges, since=since) return d
class TryJobGerritStatus(StatusReceiverMultiService): """Posts results of a try job back to a Gerrit change.""" def __init__(self, gerrit_host, review_factory=None, cq_builders=None, **kwargs): """Creates a TryJobGerritStatus. Args: gerrit_host: a URL of the Gerrit instance. review_factory: a function (self, builder_name, build, result) => review, where review is a dict described in Gerrit docs: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input cq_builders: a list of buildernames, if specified, patchset will be submitted if all builders have completed successfully. kwargs: keyword arguments passed to GerritAgent. """ StatusReceiverMultiService.__init__(self) self.review_factory = review_factory or TryJobGerritStatus.createReview self.agent = GerritAgent(gerrit_host, **kwargs) self.status = None self.cq_builders = cq_builders def createReview(self, builder_name, build, result): review = {} if result is not None: message = ('A try job has finished on builder %s: %s' % (builder_name, Results[result].upper())) else: message = 'A try job has started on builder %s' % builder_name # Do not send email about this. review['notify'] = 'NONE' # Append build url. # A line break in a Gerrit message is \n\n. assert self.status build_url = self.status.getURLForThing(build) message = '%s\n\n%s' % (message, build_url) review['message'] = message return review def sendUpdate(self, builder_name, build, result): """Posts a message and labels, if any, on a Gerrit change.""" props = build.properties change_id = (props.getProperty('event.change.id') or props.getProperty('parent_event.change.id')) revision = props.getProperty('revision') if change_id and revision: review = self.review_factory(self, builder_name, build, result) if review: log.msg('Sending a review for change %s: %s' % (change_id, review)) path = '/changes/%s/revisions/%s/review' % (change_id, revision) return self.agent.request('POST', path, body=review) @defer.inlineCallbacks def _add_verified_label(self, change_id, revision, patchset_id, commit): message = 'All tryjobs have passed for patchset %s' % patchset_id path = '/changes/%s/revisions/%s/review' % (change_id, revision) body = {'message': message, 'labels': {'Verified': '+1'}} yield self.agent.request('POST', path, body=body) # commit the change if commit: path = 'changes/%s/submit' % change_id body = {'wait_for_merge': True} yield self.agent.request('POST', path, body=body) MESSAGE_REGEX_TRYJOB_RESULT = re.compile( 'A try job has finished on builder (.+): SUCCESS', re.I | re.M) def submitPatchSetIfNecessary(self, builder_name, build, result): """This is a temporary hack until Gerrit CQ is deployed.""" if not self.cq_builders: return if not (result == results.SUCCESS or results == results.WARNINGS): return props = build.properties change_id = (props.getProperty('event.change.id') or props.getProperty('parent_event.change.id')) revision = props.getProperty('revision') patchset_id = props.getProperty('event.patchSet.ref').rsplit('/', 1)[1] builders = [x for x in self.cq_builders if x != builder_name] o_params = '&'.join('o=%s' % x for x in ('MESSAGES', 'ALL_REVISIONS', 'ALL_COMMITS', 'ALL_FILES', 'LABELS')) path = '/changes/%s?%s' % (change_id, o_params) d = self.agent.request('GET', path) def _parse_messages(j): if not j: return commit = 'approved' in j.get('labels', {}).get('Commit-Queue', {}) if len(builders) == 0: self._add_verified_label(change_id, revision, patchset_id, commit) return if 'messages' not in j: return for m in reversed(j['messages']): if m['_revision_number'] == int(patchset_id): match = self.MESSAGE_REGEX_TRYJOB_RESULT.search( m['message']) if match: builder = match.groups()[0] if builder in builders: builders.remove(builder) if len(builders) == 0: self._add_verified_label( change_id, revision, patchset_id, commit) break d.addCallback(_parse_messages) return d def startService(self): StatusReceiverMultiService.startService(self) self.status = self.parent.getStatus() self.status.subscribe(self) def builderAdded(self, name, builder): # Subscribe to this builder. return self def buildStarted(self, builder_name, build): self.sendUpdate(builder_name, build, None) def buildFinished(self, builder_name, build, result): self.sendUpdate(builder_name, build, result) self.submitPatchSetIfNecessary(builder_name, build, result)
class GerritPoller(base.PollingChangeSource): """A poller which queries a gerrit server for new changes and patchsets.""" # TODO(szager): Due to the way query continuation works in gerrit (using # the 'S=%d' URL parameter), there are two distinct error scenarios that # are currently unhandled: # # - A new patch set is uploaded. # - When the poller runs, the change is #11, meaning it doesn't come in # the first batch of query results. # - In between the first and second queries, another patch set is # uploaded to the same change, bumping the change up to #1 in the list. # - The second query skips ahead by 10, and never sees the change. # # - A new patch set is uploaded. # - When the poller runs, the change is #10, and appears in the first set # of query results. # - In between the first and second queries, some other change gets a new # patch set and moves up to #1, bumping the current #10 to #11. # - The second query skips 10, getting changes 11-20. So, the change that # was already processes is processed again. # # Both of these problems need the same solution: keep some state in poller of # 'patch sets already processed'; and relax the 'since' parameter to # processChanges so that it goes further back in time than the last polling # event (maybe pollInterval*3). change_category = 'patchset-created' def __init__(self, gerrit_host, gerrit_projects=None, pollInterval=None, dry_run=None): """Constructs a new Gerrit poller. Args: gerrit_host: (str or GerritAgent) If supplied as a GerritAgent, the Gerrit Agent to use when polling; otherwise, the host parameter to use to construct the GerritAgent to poll through. gerrit_projects: (list) A list of project names (str) to poll. pollInterval: (int or datetime.timedelta) The amount of time to wait in between polls. dry_run: (bool) If 'True', then polls will not actually be executed. """ if isinstance(pollInterval, datetime.timedelta): pollInterval = pollInterval.total_seconds() if isinstance(gerrit_projects, basestring): gerrit_projects = [gerrit_projects] self.gerrit_projects = gerrit_projects if pollInterval: self.pollInterval = pollInterval self.initLock = defer.DeferredLock() self.last_timestamp = None if dry_run is None: dry_run = 'POLLER_DRY_RUN' in os.environ self.dry_run = dry_run self.agent = gerrit_host if not isinstance(self.agent, GerritAgent): self.agent = GerritAgent(self.agent) def startService(self): if not self.dry_run: self.initLastTimeStamp() base.PollingChangeSource.startService(self) @staticmethod def buildQuery(terms, operator=None): """Builds a Gerrit query from terms. This function will go away once the new GerritAgent lands. """ connective = ('+%s+' % operator) if operator else '+' terms_with_parens = [('(%s)' % t) if ('+' in t) else t for t in terms] return connective.join(terms_with_parens) def getChangeQuery(self): # pylint: disable=R0201 # Fetch only open issues. terms = ['status:open'] # Filter by projects. if self.gerrit_projects: project_terms = [ 'project:%s' % urllib.quote(p, safe='') for p in self.gerrit_projects ] terms.append(self.buildQuery(project_terms, 'OR')) return self.buildQuery(terms) def request(self, path, method='GET'): log.msg('Gerrit request: %s' % path, logLevel=logging.DEBUG) return self.agent.request(method, path) @deferredLocked('initLock') def initLastTimeStamp(self): log.msg('GerritPoller: Getting latest timestamp from gerrit server.') query = self.getChangeQuery() path = '/changes/?q=%s&n=1' % query d = self.request(path) def _get_timestamp(j): if len(j) == 0: self.last_timestamp = datetime.datetime.now() else: self.last_timestamp = ParseGerritTime(j[0]['updated']) d.addCallback(_get_timestamp) return d def getChanges(self, skip=None): path = '/changes/?q=%s&n=10' % self.getChangeQuery() if skip: path += '&S=%d' % skip return self.request(path) def _is_interesting_message(self, message): # pylint: disable=R0201 return message['message'].startswith('Uploaded patch set ') def checkForNewPatchset(self, change, since): o_params = '&'.join('o=%s' % x for x in ('MESSAGES', 'ALL_REVISIONS', 'ALL_COMMITS', 'ALL_FILES')) path = '/changes/%s?%s' % (change['_number'], o_params) d = self.request(path) def _parse_messages(j): if not j or 'messages' not in j: return for m in reversed(j['messages']): if ParseGerritTime(m['date']) <= since: break if self._is_interesting_message(m): return j, m d.addCallback(_parse_messages) return d def getChangeUrl(self, change): """Generates a URL for a Gerrit change.""" # GerritAgent stores its URL as protocol and host. return '%s/#/c/%s' % (self.agent.base_url, change['_number']) def getRepositoryUrl(self, change): """Generates a URL for a Gerrit repository containing a change""" return '%s/%s' % (self.agent.base_url, change['project']) def addBuildbotChange(self, change, revision=None, additional_chdict=None): """Adds a buildbot change into the database. Args: change: ChangeInfo Gerrit object. Documentation: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-info revision: the sha of the buildbot change revision to use. Defaults to the value of change['current_revision'] Returns the new buildbot change as Deferred. """ revision = revision or change['current_revision'] revision_details = change['revisions'][revision] commit = revision_details['commit'] properties = { 'event.change.number': change['_number'], 'event.change.id': change['id'], 'event.change.url': self.getChangeUrl(change), } if change['status'] == 'NEW': ref = revision_details.get('fetch', {}).get('http', {}).get('ref') if ref: properties['event.patchSet.ref'] = ref elif change['status'] in ('SUBMITTED', 'MERGED'): properties['event.refUpdate.newRev'] = revision chdict = { 'author': '%s <%s>' % (commit['author']['name'], commit['author']['email']), 'project': change['project'], 'branch': change['branch'], 'revision': revision, 'comments': commit['subject'], 'files': revision_details.get('files', { 'UNKNOWN': None }).keys(), 'category': self.change_category, 'when_timestamp': ParseGerritTime(commit['committer']['date']), 'revlink': self.getChangeUrl(change), 'repository': self.getRepositoryUrl(change), 'properties': properties, } # Factor in external 'chdict' overrides. if additional_chdict is not None: properties.update(additional_chdict.pop('properties', {})) chdict.update(additional_chdict) chdict['properties'] = properties d = self.master.addChange(**chdict) d.addErrback( log.err, 'GerritPoller: Could not add buildbot change for ' 'gerrit change %s.' % revision_details['_number']) return d @staticmethod def findRevisionShaForMessage(change, message): def warn(text): log.msg('GerritPoller warning: %s. Change: %s, message: %s' % (text, change['id'], message['message'])) revision_number = message.get('_revision_number') if revision_number is None: warn('A message doesn\'t have a _revision_number') return None for sha, revision in change['revisions'].iteritems(): if revision['_number'] == revision_number: return sha warn('a revision wasn\'t found for message') def addChange(self, change, message): revision = self.findRevisionShaForMessage(change, message) return self.addBuildbotChange(change, revision) def processChanges(self, j, since, skip=0): need_more = bool(j) for change in j: skip += 1 tm = ParseGerritTime(change['updated']) if tm <= since: need_more = False break if self.gerrit_projects and change[ 'project'] not in self.gerrit_projects: continue d = self.checkForNewPatchset(change, since) d.addCallback(lambda x: self.addChange(*x) if x else None) if need_more and j[-1].get('_more_changes'): d = self.getChanges(skip=skip) d.addCallback(self.processChanges, since=since, skip=skip) else: d = defer.succeed(None) return d @deferredLocked('initLock') def poll(self): if self.dry_run: return log.msg('GerritPoller: getting latest changes...') since = self.last_timestamp d = self.getChanges() def _update_last_timestamp(j): if j: self.last_timestamp = ParseGerritTime(j[0]['updated']) return j d.addCallback(_update_last_timestamp) d.addCallback(self.processChanges, since=since) return d
class TryJobGerritStatus(StatusReceiverMultiService): """Posts results of a try job back to a Gerrit change.""" def __init__(self, gerrit_host, review_factory=None, cq_builders=None, **kwargs): """Creates a TryJobGerritStatus. Args: gerrit_host: a URL of the Gerrit instance. review_factory: a function (self, builder_name, build, result) => review, where review is a dict described in Gerrit docs: https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input cq_builders: a list of buildernames, if specified, patchset will be submitted if all builders have completed successfully. kwargs: keyword arguments passed to GerritAgent. """ StatusReceiverMultiService.__init__(self) self.review_factory = review_factory or TryJobGerritStatus.createReview self.agent = GerritAgent(gerrit_host, **kwargs) self.status = None self.cq_builders = cq_builders def createReview(self, builder_name, build, result): review = {} if result is not None: message = ('A try job has finished on builder %s: %s' % (builder_name, Results[result].upper())) else: message = 'A try job has started on builder %s' % builder_name # Do not send email about this. review['notify'] = 'NONE' # Append build url. # A line break in a Gerrit message is \n\n. assert self.status build_url = self.status.getURLForThing(build) message = '%s\n\n%s' % (message, build_url) review['message'] = message return review def sendUpdate(self, builder_name, build, result): """Posts a message and labels, if any, on a Gerrit change.""" props = build.properties change_id = (props.getProperty('event.change.id') or props.getProperty('parent_event.change.id')) revision = props.getProperty('revision') if change_id and revision: review = self.review_factory(self, builder_name, build, result) if review: log.msg('Sending a review for change %s: %s' % (change_id, review)) path = '/changes/%s/revisions/%s/review' % (change_id, revision) return self.agent.request('POST', path, body=review) @defer.inlineCallbacks def _add_verified_label(self, change_id, revision, patchset_id, commit): message = 'All tryjobs have passed for patchset %s' % patchset_id path = '/changes/%s/revisions/%s/review' % (change_id, revision) body = {'message': message, 'labels': {'Verified': '+1'}} yield self.agent.request('POST', path, body=body) # commit the change if commit: path = 'changes/%s/submit' % change_id body = {'wait_for_merge': True} yield self.agent.request('POST', path, body=body) MESSAGE_REGEX_TRYJOB_RESULT = re.compile( 'A try job has finished on builder (.+): SUCCESS', re.I | re.M) def submitPatchSetIfNecessary(self, builder_name, build, result): """This is a temporary hack until Gerrit CQ is deployed.""" if not self.cq_builders: return if not (result == results.SUCCESS or results == results.WARNINGS): return props = build.properties change_id = (props.getProperty('event.change.id') or props.getProperty('parent_event.change.id')) revision = props.getProperty('revision') patchset_id = props.getProperty('event.patchSet.ref').rsplit('/', 1)[1] builders = [x for x in self.cq_builders if x != builder_name] o_params = '&'.join('o=%s' % x for x in ( 'MESSAGES', 'ALL_REVISIONS', 'ALL_COMMITS', 'ALL_FILES', 'LABELS')) path = '/changes/%s?%s' % (change_id, o_params) d = self.agent.request('GET', path) def _parse_messages(j): if not j: return commit = 'approved' in j.get('labels', {}).get('Commit-Queue', {}) if len(builders) == 0: self._add_verified_label(change_id, revision, patchset_id, commit) return if 'messages' not in j: return for m in reversed(j['messages']): if m['_revision_number'] == int(patchset_id): match = self.MESSAGE_REGEX_TRYJOB_RESULT.search(m['message']) if match: builder = match.groups()[0] if builder in builders: builders.remove(builder) if len(builders) == 0: self._add_verified_label(change_id, revision, patchset_id, commit) break d.addCallback(_parse_messages) return d def startService(self): StatusReceiverMultiService.startService(self) self.status = self.parent.getStatus() self.status.subscribe(self) def builderAdded(self, name, builder): # Subscribe to this builder. return self def buildStarted(self, builder_name, build): self.sendUpdate(builder_name, build, None) def buildFinished(self, builder_name, build, result): self.sendUpdate(builder_name, build, result) self.submitPatchSetIfNecessary(builder_name, build, result)