def __init__(self,
gerrit_host,
gerrit_projects=None,
pollInterval=None,
dry_run=None):
"""Constructs a new Gerrit poller.
Args:
gerrit_host: (str or GerritAgent) If supplied as a GerritAgent, the
Gerrit Agent to use when polling; otherwise, the host parameter to
use to construct the GerritAgent to poll through.
gerrit_projects: (list) A list of project names (str) to poll.
pollInterval: (int or datetime.timedelta) The amount of time to wait in
between polls.
dry_run: (bool) If 'True', then polls will not actually be executed.
"""
if isinstance(pollInterval, datetime.timedelta):
pollInterval = pollInterval.total_seconds()
if isinstance(gerrit_projects, basestring):
gerrit_projects = [gerrit_projects]
self.gerrit_projects = gerrit_projects
if pollInterval:
self.pollInterval = pollInterval
self.initLock = defer.DeferredLock()
self.last_timestamp = None
if dry_run is None:
dry_run = 'POLLER_DRY_RUN' in os.environ
self.dry_run = dry_run
self.agent = gerrit_host
if not isinstance(self.agent, GerritAgent):
self.agent = GerritAgent(self.agent)
class GerritStatusPush(StatusReceiverMultiService):
"""Add a comment to a gerrit code review indicating the result of a build."""
def __init__(self, gerrit_url, buildbot_url):
StatusReceiverMultiService.__init__(self)
self.agent = GerritAgent(gerrit_url)
self.buildbot_url = buildbot_url
def startService(self):
StatusReceiverMultiService.startService(self)
self.status = self.parent.getStatus() # pylint: disable=W0201
self.status.subscribe(self)
def builderAdded(self, name, builder):
return self # subscribe to this builder
def getMessage(self, builderName, build, result):
message = "Buildbot finished compiling your patchset\n"
message += "on configuration: %s\n" % builderName
message += "The result is: %s\n" % Results[result].upper()
message += '%sbuilders/%s/builds/%s\n' % (
self.buildbot_url, urllib.quote(build.getProperty('buildername')),
build.getProperty('buildnumber'))
return message
def buildFinished(self, builderName, build, result):
if 'event.change.number' not in build.getProperties():
return
change_number = build.getProperty('event.change.number')
revision = build.getProperty('revision')
message = self.getMessage(builderName, build, result)
verified = '+1' if (result == 0) else '-1'
path = '/changes/%s/revisions/%s/review' % (change_number, revision)
body = {'message': message, 'labels': {'Verified': verified}}
self.agent.request('POST', path, body=body)
def __init__(self, gerrit_host, review_factory=None):
"""Creates a TryJobGerritStatus.
Args:
gerrit_host: a URL of the Gerrit instance.
review_factory: a function (self, builder_name, build, result) => review,
where review is a dict described in Gerrit docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
"""
StatusReceiverMultiService.__init__(self)
self.review_factory = review_factory or TryJobGerritStatus.createReview
self.agent = GerritAgent(gerrit_host)
self.status = None
def __init__(self, gerrit_host, gerrit_projects=None, pollInterval=None,
dry_run=None):
if isinstance(gerrit_projects, basestring):
gerrit_projects = [gerrit_projects]
self.gerrit_projects = gerrit_projects
if pollInterval:
self.pollInterval = pollInterval
self.initLock = defer.DeferredLock()
self.last_timestamp = None
self.agent = GerritAgent(gerrit_host)
if dry_run is None:
dry_run = 'POLLER_DRY_RUN' in os.environ
self.dry_run = dry_run
def __init__(self, gerrit_host, gerrit_projects=None, pollInterval=None,
dry_run=None):
"""Constructs a new Gerrit poller.
Args:
gerrit_host: (str or GerritAgent) If supplied as a GerritAgent, the
Gerrit Agent to use when polling; otherwise, the host parameter to
use to construct the GerritAgent to poll through.
gerrit_projects: (list) A list of project names (str) to poll.
pollInterval: (int or datetime.timedelta) The amount of time to wait in
between polls.
dry_run: (bool) If 'True', then polls will not actually be executed.
"""
if isinstance(pollInterval, datetime.timedelta):
pollInterval = pollInterval.total_seconds()
if isinstance(gerrit_projects, basestring):
gerrit_projects = [gerrit_projects]
self.gerrit_projects = gerrit_projects
if pollInterval:
self.pollInterval = pollInterval
self.initLock = defer.DeferredLock()
self.last_timestamp = None
if dry_run is None:
dry_run = 'POLLER_DRY_RUN' in os.environ
self.dry_run = dry_run
self.agent = gerrit_host
if not isinstance(self.agent, GerritAgent):
self.agent = GerritAgent(self.agent)
def __init__(self,
gerrit_host,
review_factory=None,
cq_builders=None,
**kwargs):
"""Creates a TryJobGerritStatus.
Args:
gerrit_host: a URL of the Gerrit instance.
review_factory: a function (self, builder_name, build, result) => review,
where review is a dict described in Gerrit docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
cq_builders: a list of buildernames, if specified, patchset will be
submitted if all builders have completed successfully.
kwargs: keyword arguments passed to GerritAgent.
"""
StatusReceiverMultiService.__init__(self)
self.review_factory = review_factory or TryJobGerritStatus.createReview
self.agent = GerritAgent(gerrit_host, **kwargs)
self.status = None
self.cq_builders = cq_builders
def __init__(self, gerrit_host, review_factory=None):
"""Creates a TryJobGerritStatus.
Args:
gerrit_host: a URL of the Gerrit instance.
review_factory: a function (self, builder_name, build, result) => review,
where review is a dict described in Gerrit docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
"""
StatusReceiverMultiService.__init__(self)
self.review_factory = review_factory or TryJobGerritStatus.createReview
self.agent = GerritAgent(gerrit_host)
self.status = None
class GerritStatusPush(StatusReceiverMultiService):
"""Add a comment to a gerrit code review indicating the result of a build."""
def __init__(self, gerrit_url, buildbot_url):
StatusReceiverMultiService.__init__(self)
self.agent = GerritAgent(gerrit_url)
self.buildbot_url = buildbot_url
def startService(self):
StatusReceiverMultiService.startService(self)
self.status = self.parent.getStatus() # pylint: disable=W0201
self.status.subscribe(self)
def builderAdded(self, name, builder):
return self # subscribe to this builder
def getMessage(self, builderName, build, result):
message = "Buildbot finished compiling your patchset\n"
message += "on configuration: %s\n" % builderName
message += "The result is: %s\n" % Results[result].upper()
message += '%sbuilders/%s/builds/%s\n' % (
self.buildbot_url,
urllib.quote(build.getProperty('buildername')),
build.getProperty('buildnumber'))
return message
def buildFinished(self, builderName, build, result):
if 'event.change.number' not in build.getProperties():
return
change_number = build.getProperty('event.change.number')
revision = build.getProperty('revision')
message = self.getMessage(builderName, build, result)
verified = '+1' if (result == 0) else '-1'
path = '/changes/%s/revisions/%s/review' % (change_number, revision)
body = {'message': message, 'labels': {'Verified': verified}}
self.agent.request('POST', path, None, body)
def __init__(self, gerrit_host, review_factory=None, cq_builders=None,
**kwargs):
"""Creates a TryJobGerritStatus.
Args:
gerrit_host: a URL of the Gerrit instance.
review_factory: a function (self, builder_name, build, result) => review,
where review is a dict described in Gerrit docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
cq_builders: a list of buildernames, if specified, patchset will be
submitted if all builders have completed successfully.
kwargs: keyword arguments passed to GerritAgent.
"""
StatusReceiverMultiService.__init__(self)
self.review_factory = review_factory or TryJobGerritStatus.createReview
self.agent = GerritAgent(gerrit_host, **kwargs)
self.status = None
self.cq_builders = cq_builders
class TryJobGerritStatus(StatusReceiverMultiService):
"""Posts results of a try job back to a Gerrit change."""
def __init__(self, gerrit_host, review_factory=None):
"""Creates a TryJobGerritStatus.
Args:
gerrit_host: a URL of the Gerrit instance.
review_factory: a function (self, builder_name, build, result) => review,
where review is a dict described in Gerrit docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
"""
StatusReceiverMultiService.__init__(self)
self.review_factory = review_factory or TryJobGerritStatus.createReview
self.agent = GerritAgent(gerrit_host)
self.status = None
def createReview(self, builder_name, build, result):
review = {}
if result is not None:
message = ('A try job has finished on builder %s: %s' %
(builder_name, Results[result].upper()))
else:
message = 'A try job has started on builder %s' % builder_name
# Do not send email about this.
review['notify'] = 'NONE'
# Append build url.
# A line break in a Gerrit message is \n\n.
assert self.status
build_url = self.status.getURLForThing(build)
message = '%s\n\n%s' % (message, build_url)
review['message'] = message
return review
def sendUpdate(self, builder_name, build, result):
"""Posts a message and labels, if any, on a Gerrit change."""
props = build.properties
change_id = (props.getProperty('event.change.id') or
props.getProperty('parent_event.change.id'))
revision = props.getProperty('revision')
if change_id and revision:
review = self.review_factory(self, builder_name, build, result)
if review:
log.msg('Sending a review for change %s: %s' % (change_id, review))
path = '/changes/%s/revisions/%s/review' % (change_id, revision)
return self.agent.request('POST', path, body=review)
def startService(self):
StatusReceiverMultiService.startService(self)
self.status = self.parent.getStatus()
self.status.subscribe(self)
def builderAdded(self, name, builder):
# Subscribe to this builder.
return self
def buildStarted(self, builder_name, build):
self.sendUpdate(builder_name, build, None)
def buildFinished(self, builder_name, build, result):
self.sendUpdate(builder_name, build, result)
class GerritPoller(base.PollingChangeSource):
"""A poller which queries a gerrit server for new changes and patchsets."""
# TODO(szager): Due to the way query continuation works in gerrit (using
# the 'S=%d' URL parameter), there are two distinct error scenarios that
# are currently unhandled:
#
# - A new patch set is uploaded.
# - When the poller runs, the change is #11, meaning it doesn't come in
# the first batch of query results.
# - In between the first and second queries, another patch set is
# uploaded to the same change, bumping the change up to #1 in the list.
# - The second query skips ahead by 10, and never sees the change.
#
# - A new patch set is uploaded.
# - When the poller runs, the change is #10, and appears in the first set
# of query results.
# - In between the first and second queries, some other change gets a new
# patch set and moves up to #1, bumping the current #10 to #11.
# - The second query skips 10, getting changes 11-20. So, the change that
# was already processes is processed again.
#
# Both of these problems need the same solution: keep some state in poller of
# 'patch sets already processed'; and relax the 'since' parameter to
# processChanges so that it goes further back in time than the last polling
# event (maybe pollInterval*3).
change_category = 'patchset-created'
def __init__(self, gerrit_host, gerrit_projects=None, pollInterval=None,
dry_run=None):
"""Constructs a new Gerrit poller.
Args:
gerrit_host: (str or GerritAgent) If supplied as a GerritAgent, the
Gerrit Agent to use when polling; otherwise, the host parameter to
use to construct the GerritAgent to poll through.
gerrit_projects: (list) A list of project names (str) to poll.
pollInterval: (int or datetime.timedelta) The amount of time to wait in
between polls.
dry_run: (bool) If 'True', then polls will not actually be executed.
"""
if isinstance(pollInterval, datetime.timedelta):
pollInterval = pollInterval.total_seconds()
if isinstance(gerrit_projects, basestring):
gerrit_projects = [gerrit_projects]
self.gerrit_projects = gerrit_projects
if pollInterval:
self.pollInterval = pollInterval
self.initLock = defer.DeferredLock()
self.last_timestamp = None
if dry_run is None:
dry_run = 'POLLER_DRY_RUN' in os.environ
self.dry_run = dry_run
self.agent = gerrit_host
if not isinstance(self.agent, GerritAgent):
self.agent = GerritAgent(self.agent)
def startService(self):
if not self.dry_run:
self.initLastTimeStamp()
base.PollingChangeSource.startService(self)
@staticmethod
def buildQuery(terms, operator=None):
"""Builds a Gerrit query from terms.
This function will go away once the new GerritAgent lands.
"""
connective = ('+%s+' % operator) if operator else '+'
terms_with_parens = [('(%s)' % t) if ('+' in t) else t
for t in terms]
return connective.join(terms_with_parens)
def getChangeQuery(self): # pylint: disable=R0201
# Fetch only open issues.
terms = ['status:open']
# Filter by projects.
if self.gerrit_projects:
project_terms = ['project:%s' % urllib.quote(p, safe='')
for p in self.gerrit_projects]
terms.append(self.buildQuery(project_terms, 'OR'))
return self.buildQuery(terms)
def request(self, path, method='GET'):
log.msg('Gerrit request: %s' % path, logLevel=logging.DEBUG)
return self.agent.request(method, path)
@deferredLocked('initLock')
def initLastTimeStamp(self):
log.msg('GerritPoller: Getting latest timestamp from gerrit server.')
query = self.getChangeQuery()
path = '/changes/?q=%s&n=1' % query
d = self.request(path)
def _get_timestamp(j):
if len(j) == 0:
self.last_timestamp = datetime.datetime.now()
else:
self.last_timestamp = ParseGerritTime(j[0]['updated'])
d.addCallback(_get_timestamp)
return d
def getChanges(self, skip=None):
path = '/changes/?q=%s&n=10' % self.getChangeQuery()
if skip:
path += '&S=%d' % skip
return self.request(path)
def _is_interesting_message(self, message): # pylint: disable=R0201
return any((check_str in message['message'])
for check_str in (
'Uploaded patch set ',
'Published edit on patch set ',))
def checkForNewPatchset(self, change, since):
o_params = '&'.join('o=%s' % x for x in (
'MESSAGES', 'ALL_REVISIONS', 'ALL_COMMITS', 'ALL_FILES'))
path = '/changes/%s?%s' % (change['_number'], o_params)
d = self.request(path)
def _parse_messages(j):
if not j or 'messages' not in j:
return
for m in reversed(j['messages']):
if ParseGerritTime(m['date']) <= since:
break
if self._is_interesting_message(m):
return j, m
d.addCallback(_parse_messages)
return d
def getChangeUrl(self, change):
"""Generates a URL for a Gerrit change."""
# GerritAgent stores its URL as protocol and host.
return '%s/#/c/%s' % (self.agent.base_url,
change['_number'])
def getRepositoryUrl(self, change):
"""Generates a URL for a Gerrit repository containing a change"""
return '%s/%s' % (self.agent.base_url,
change['project'])
def addBuildbotChange(self, change, revision=None, additional_chdict=None):
"""Adds a buildbot change into the database.
Args:
change: ChangeInfo Gerrit object. Documentation:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-info
revision: the sha of the buildbot change revision to use. Defaults to the
value of change['current_revision']
Returns the new buildbot change as Deferred.
"""
revision = revision or change['current_revision']
revision_details = change['revisions'][revision]
commit = revision_details['commit']
properties = {
'event.change.number': change['_number'],
'event.change.id': change['id'],
'event.change.url': self.getChangeUrl(change),
}
if change['status'] == 'NEW':
ref = revision_details.get('fetch', {}).get('http', {}).get('ref')
if ref:
properties['event.patchSet.ref'] = ref
elif change['status'] in ('SUBMITTED', 'MERGED'):
properties['event.refUpdate.newRev'] = revision
chdict = {
'author': '%s <%s>' % (
commit['author']['name'], commit['author']['email']),
'project': change['project'],
'branch': change['branch'],
'revision': revision,
'comments': commit['subject'],
'files': revision_details.get('files', {'UNKNOWN': None}).keys(),
'category': self.change_category,
'when_timestamp': ParseGerritTime(commit['committer']['date']),
'revlink': self.getChangeUrl(change),
'repository': self.getRepositoryUrl(change),
'properties': properties,
}
# Factor in external 'chdict' overrides.
if additional_chdict is not None:
properties.update(additional_chdict.pop('properties', {}))
chdict.update(additional_chdict)
chdict['properties'] = properties
d = self.master.addChange(**chdict)
d.addErrback(log.err, 'GerritPoller: Could not add buildbot change for '
'gerrit change %s.' % revision_details['_number'])
return d
@staticmethod
def findRevisionShaForMessage(change, message):
def warn(text):
log.msg('GerritPoller warning: %s. Change: %s, message: %s' %
(text, change['id'], message['message']))
revision_number = message.get('_revision_number')
if revision_number is None:
warn('A message doesn\'t have a _revision_number')
return None
for sha, revision in change['revisions'].iteritems():
if revision['_number'] == revision_number:
return sha
warn('a revision wasn\'t found for message')
def addChange(self, change, message):
revision = self.findRevisionShaForMessage(change, message)
return self.addBuildbotChange(change, revision)
def processChanges(self, j, since, skip=0):
need_more = bool(j)
for change in j:
skip += 1
tm = ParseGerritTime(change['updated'])
if tm <= since:
need_more = False
break
if self.gerrit_projects and change['project'] not in self.gerrit_projects:
continue
d = self.checkForNewPatchset(change, since)
d.addCallback(lambda x: self.addChange(*x) if x else None)
if need_more and j[-1].get('_more_changes'):
d = self.getChanges(skip=skip)
d.addCallback(self.processChanges, since=since, skip=skip)
else:
d = defer.succeed(None)
return d
@deferredLocked('initLock')
def poll(self):
if self.dry_run:
return
log.msg('GerritPoller: getting latest changes...')
since = self.last_timestamp
d = self.getChanges()
def _update_last_timestamp(j):
if j:
self.last_timestamp = ParseGerritTime(j[0]['updated'])
return j
d.addCallback(_update_last_timestamp)
d.addCallback(self.processChanges, since=since)
return d
class TryJobGerritStatus(StatusReceiverMultiService):
"""Posts results of a try job back to a Gerrit change."""
def __init__(self, gerrit_host, review_factory=None):
"""Creates a TryJobGerritStatus.
Args:
gerrit_host: a URL of the Gerrit instance.
review_factory: a function (self, builder_name, build, result) => review,
where review is a dict described in Gerrit docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
"""
StatusReceiverMultiService.__init__(self)
self.review_factory = review_factory or TryJobGerritStatus.createReview
self.agent = GerritAgent(gerrit_host)
self.status = None
def createReview(self, builder_name, build, result):
review = {}
if result is not None:
message = ('A try job has finished on builder %s: %s' %
(builder_name, Results[result].upper()))
else:
message = 'A try job has started on builder %s' % builder_name
# Do not send email about this.
review['notify'] = 'NONE'
# Append build url.
# A line break in a Gerrit message is \n\n.
assert self.status
build_url = self.status.getURLForThing(build)
message = '%s\n\n%s' % (message, build_url)
review['message'] = message
return review
def sendUpdate(self, builder_name, build, result):
"""Posts a message and labels, if any, on a Gerrit change."""
props = build.properties
change_id = (props.getProperty('event.change.id')
or props.getProperty('parent_event.change.id'))
revision = props.getProperty('revision')
if change_id and revision:
review = self.review_factory(self, builder_name, build, result)
if review:
log.msg('Sending a review for change %s: %s' %
(change_id, review))
path = '/changes/%s/revisions/%s/review' % (change_id,
revision)
return self.agent.request('POST', path, body=review)
def startService(self):
StatusReceiverMultiService.startService(self)
self.status = self.parent.getStatus()
self.status.subscribe(self)
def builderAdded(self, name, builder):
# Subscribe to this builder.
return self
def buildStarted(self, builder_name, build):
self.sendUpdate(builder_name, build, None)
def buildFinished(self, builder_name, build, result):
self.sendUpdate(builder_name, build, result)
def __init__(self, gerrit_url, buildbot_url): StatusReceiverMultiService.__init__(self) self.agent = GerritAgent(gerrit_url) self.buildbot_url = buildbot_url
class GerritPoller(base.PollingChangeSource):
"""A poller which queries a gerrit server for new changes and patchsets."""
# TODO(szager): Due to the way query continuation works in gerrit (using
# the 'S=%d' URL parameter), there are two distinct error scenarios that
# are currently unhandled:
#
# - A new patch set is uploaded.
# - When the poller runs, the change is #11, meaning it doesn't come in
# the first batch of query results.
# - In between the first and second queries, another patch set is
# uploaded to the same change, bumping the change up to #1 in the list.
# - The second query skips ahead by 10, and never sees the change.
#
# - A new patch set is uploaded.
# - When the poller runs, the change is #10, and appears in the first set
# of query results.
# - In between the first and second queries, some other change gets a new
# patch set and moves up to #1, bumping the current #10 to #11.
# - The second query skips 10, getting changes 11-20. So, the change that
# was already processes is processed again.
#
# Both of these problems need the same solution: keep some state in poller of
# 'patch sets already processed'; and relax the 'since' parameter to
# processChanges so that it goes further back in time than the last polling
# event (maybe pollInterval*3).
change_category = 'patchset-created'
def __init__(self,
gerrit_host,
gerrit_projects=None,
pollInterval=None,
dry_run=None):
"""Constructs a new Gerrit poller.
Args:
gerrit_host: (str or GerritAgent) If supplied as a GerritAgent, the
Gerrit Agent to use when polling; otherwise, the host parameter to
use to construct the GerritAgent to poll through.
gerrit_projects: (list) A list of project names (str) to poll.
pollInterval: (int or datetime.timedelta) The amount of time to wait in
between polls.
dry_run: (bool) If 'True', then polls will not actually be executed.
"""
if isinstance(pollInterval, datetime.timedelta):
pollInterval = pollInterval.total_seconds()
if isinstance(gerrit_projects, basestring):
gerrit_projects = [gerrit_projects]
self.gerrit_projects = gerrit_projects
if pollInterval:
self.pollInterval = pollInterval
self.initLock = defer.DeferredLock()
self.last_timestamp = None
if dry_run is None:
dry_run = 'POLLER_DRY_RUN' in os.environ
self.dry_run = dry_run
self.agent = gerrit_host
if not isinstance(self.agent, GerritAgent):
self.agent = GerritAgent(self.agent)
def startService(self):
if not self.dry_run:
self.initLastTimeStamp()
base.PollingChangeSource.startService(self)
@staticmethod
def buildQuery(terms, operator=None):
"""Builds a Gerrit query from terms.
This function will go away once the new GerritAgent lands.
"""
connective = ('+%s+' % operator) if operator else '+'
terms_with_parens = [('(%s)' % t) if ('+' in t) else t for t in terms]
return connective.join(terms_with_parens)
def getChangeQuery(self): # pylint: disable=R0201
# Fetch only open issues.
terms = ['status:open']
# Filter by projects.
if self.gerrit_projects:
project_terms = [
'project:%s' % urllib.quote(p, safe='')
for p in self.gerrit_projects
]
terms.append(self.buildQuery(project_terms, 'OR'))
return self.buildQuery(terms)
def request(self, path, method='GET'):
log.msg('Gerrit request: %s' % path, logLevel=logging.DEBUG)
return self.agent.request(method, path)
@deferredLocked('initLock')
def initLastTimeStamp(self):
log.msg('GerritPoller: Getting latest timestamp from gerrit server.')
query = self.getChangeQuery()
path = '/changes/?q=%s&n=1' % query
d = self.request(path)
def _get_timestamp(j):
if len(j) == 0:
self.last_timestamp = datetime.datetime.now()
else:
self.last_timestamp = ParseGerritTime(j[0]['updated'])
d.addCallback(_get_timestamp)
return d
def getChanges(self, skip=None):
path = '/changes/?q=%s&n=10' % self.getChangeQuery()
if skip:
path += '&S=%d' % skip
return self.request(path)
def _is_interesting_message(self, message): # pylint: disable=R0201
return message['message'].startswith('Uploaded patch set ')
def checkForNewPatchset(self, change, since):
o_params = '&'.join('o=%s' % x for x in ('MESSAGES', 'ALL_REVISIONS',
'ALL_COMMITS', 'ALL_FILES'))
path = '/changes/%s?%s' % (change['_number'], o_params)
d = self.request(path)
def _parse_messages(j):
if not j or 'messages' not in j:
return
for m in reversed(j['messages']):
if ParseGerritTime(m['date']) <= since:
break
if self._is_interesting_message(m):
return j, m
d.addCallback(_parse_messages)
return d
def getChangeUrl(self, change):
"""Generates a URL for a Gerrit change."""
# GerritAgent stores its URL as protocol and host.
return '%s/#/c/%s' % (self.agent.base_url, change['_number'])
def getRepositoryUrl(self, change):
"""Generates a URL for a Gerrit repository containing a change"""
return '%s/%s' % (self.agent.base_url, change['project'])
def addBuildbotChange(self, change, revision=None, additional_chdict=None):
"""Adds a buildbot change into the database.
Args:
change: ChangeInfo Gerrit object. Documentation:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-info
revision: the sha of the buildbot change revision to use. Defaults to the
value of change['current_revision']
Returns the new buildbot change as Deferred.
"""
revision = revision or change['current_revision']
revision_details = change['revisions'][revision]
commit = revision_details['commit']
properties = {
'event.change.number': change['_number'],
'event.change.id': change['id'],
'event.change.url': self.getChangeUrl(change),
}
if change['status'] == 'NEW':
ref = revision_details.get('fetch', {}).get('http', {}).get('ref')
if ref:
properties['event.patchSet.ref'] = ref
elif change['status'] in ('SUBMITTED', 'MERGED'):
properties['event.refUpdate.newRev'] = revision
chdict = {
'author':
'%s <%s>' % (commit['author']['name'], commit['author']['email']),
'project':
change['project'],
'branch':
change['branch'],
'revision':
revision,
'comments':
commit['subject'],
'files':
revision_details.get('files', {
'UNKNOWN': None
}).keys(),
'category':
self.change_category,
'when_timestamp':
ParseGerritTime(commit['committer']['date']),
'revlink':
self.getChangeUrl(change),
'repository':
self.getRepositoryUrl(change),
'properties':
properties,
}
# Factor in external 'chdict' overrides.
if additional_chdict is not None:
properties.update(additional_chdict.pop('properties', {}))
chdict.update(additional_chdict)
chdict['properties'] = properties
d = self.master.addChange(**chdict)
d.addErrback(
log.err, 'GerritPoller: Could not add buildbot change for '
'gerrit change %s.' % revision_details['_number'])
return d
@staticmethod
def findRevisionShaForMessage(change, message):
def warn(text):
log.msg('GerritPoller warning: %s. Change: %s, message: %s' %
(text, change['id'], message['message']))
revision_number = message.get('_revision_number')
if revision_number is None:
warn('A message doesn\'t have a _revision_number')
return None
for sha, revision in change['revisions'].iteritems():
if revision['_number'] == revision_number:
return sha
warn('a revision wasn\'t found for message')
def addChange(self, change, message):
revision = self.findRevisionShaForMessage(change, message)
return self.addBuildbotChange(change, revision)
def processChanges(self, j, since, skip=0):
need_more = bool(j)
for change in j:
skip += 1
tm = ParseGerritTime(change['updated'])
if tm <= since:
need_more = False
break
if self.gerrit_projects and change[
'project'] not in self.gerrit_projects:
continue
d = self.checkForNewPatchset(change, since)
d.addCallback(lambda x: self.addChange(*x) if x else None)
if need_more and j[-1].get('_more_changes'):
d = self.getChanges(skip=skip)
d.addCallback(self.processChanges, since=since, skip=skip)
else:
d = defer.succeed(None)
return d
@deferredLocked('initLock')
def poll(self):
if self.dry_run:
return
log.msg('GerritPoller: getting latest changes...')
since = self.last_timestamp
d = self.getChanges()
def _update_last_timestamp(j):
if j:
self.last_timestamp = ParseGerritTime(j[0]['updated'])
return j
d.addCallback(_update_last_timestamp)
d.addCallback(self.processChanges, since=since)
return d
def __init__(self, gerrit_url, buildbot_url):
StatusReceiverMultiService.__init__(self)
self.agent = GerritAgent(gerrit_url)
self.buildbot_url = buildbot_url
class GerritPoller(base.PollingChangeSource):
"""A poller which queries a gerrit server for new changes and patchsets."""
change_category = 'patchset-created'
def __init__(self, gerrit_host, gerrit_projects=None, pollInterval=None,
dry_run=None):
if isinstance(gerrit_projects, basestring):
gerrit_projects = [gerrit_projects]
self.gerrit_projects = gerrit_projects
if pollInterval:
self.pollInterval = pollInterval
self.initLock = defer.DeferredLock()
self.last_timestamp = None
self.agent = GerritAgent(gerrit_host)
if dry_run is None:
dry_run = 'POLLER_DRY_RUN' in os.environ
self.dry_run = dry_run
@staticmethod
def _parse_timestamp(tm):
tm = tm[:tm.index('.')+7]
return datetime.datetime.strptime(tm, '%Y-%m-%d %H:%M:%S.%f')
def startService(self):
if not self.dry_run:
self.initLastTimeStamp()
base.PollingChangeSource.startService(self)
def getChangeQuery(self): # pylint: disable=R0201
return 'status:open'
@deferredLocked('initLock')
def initLastTimeStamp(self):
log.msg('GerritPoller: Getting latest timestamp from gerrit server.')
path = '/changes/?q=%s&n=1' % self.getChangeQuery()
d = self.agent.request('GET', path)
def _get_timestamp(j):
if len(j) == 0:
self.last_timestamp = datetime.datetime.now()
else:
self.last_timestamp = self._parse_timestamp(j[0]['updated'])
d.addCallback(_get_timestamp)
return d
def getChanges(self, sortkey=None):
path = '/changes/?q=%s&n=10' % self.getChangeQuery()
if sortkey:
path += '&N=%s' % sortkey
return self.agent.request('GET', path)
def _is_interesting_message(self, message): # pylint: disable=R0201
return message['message'].startswith('Uploaded patch set ')
def checkForNewPatchset(self, change, since):
o_params = '&'.join('o=%s' % x for x in (
'MESSAGES', 'ALL_REVISIONS', 'ALL_COMMITS', 'ALL_FILES'))
path = '/changes/%s?%s' % (change['_number'], o_params)
d = self.agent.request('GET', path)
def _parse_messages(j):
if not j or 'messages' not in j:
return
for m in reversed(j['messages']):
if self._parse_timestamp(m['date']) <= since:
break
if self._is_interesting_message(m):
return j, m
d.addCallback(_parse_messages)
return d
def getChangeUrl(self, change):
"""Generates a URL for a Gerrit change."""
# GerritAgent stores its URL as protocol and host.
return '%s://%s/#/c/%s' % (self.agent.gerrit_protocol,
self.agent.gerrit_host,
change['_number'])
def addBuildbotChange(self, change, revision=None):
"""Adds a buildbot change into the database.
Args:
change: ChangeInfo Gerrit object. Documentation:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#change-info
revision: the sha of the buildbot change revision to use. Defaults to the
value of change['current_revision']
Returns the new buildbot change as Deferred.
"""
revision = revision or change['current_revision']
revision_details = change['revisions'][revision]
commit = revision_details['commit']
properties = {
'event.change.number': change['_number'],
'event.change.id': change['id'],
'event.change.url': self.getChangeUrl(change),
}
if change['status'] == 'NEW':
ref = revision_details.get('fetch', {}).get('http', {}).get('ref')
if ref:
properties['event.patchSet.ref'] = ref
elif change['status'] in ('SUBMITTED', 'MERGED'):
properties['event.refUpdate.newRev'] = revision
chdict = {
'author': '%s <%s>' % (
commit['author']['name'], commit['author']['email']),
'project': change['project'],
'branch': change['branch'],
'revision': revision,
'comments': commit['subject'],
'files': revision_details.get('files', {'UNKNOWN': None}).keys(),
'category': self.change_category,
'when_timestamp': self._parse_timestamp(commit['committer']['date']),
'revlink': '%s://%s/#/c/%s' % (
self.agent.gerrit_protocol, self.agent.gerrit_host,
change['_number']),
'repository': '%s://%s/%s' % (
self.agent.gerrit_protocol, self.agent.gerrit_host,
change['project']),
'properties': properties,
}
d = self.master.addChange(**chdict)
d.addErrback(log.err, 'GerritPoller: Could not add buildbot change for '
'gerrit change %s.' % revision_details['_number'])
return d
@staticmethod
def findRevisionShaForMessage(change, message):
def warn(text):
log.msg('GerritPoller warning: %s. Change: %s, message: %s' %
(text, change['id'], message['message']))
revision_number = message.get('_revision_number')
if revision_number is None:
warn('A message doesn\'t have a _revision_number')
return None
for sha, revision in change['revisions'].iteritems():
if revision['_number'] == revision_number:
return sha
warn('a revision wasn\'t found for message')
def addChange(self, change, message):
revision = self.findRevisionShaForMessage(change, message)
return self.addBuildbotChange(change, revision)
def processChanges(self, j, since):
need_more = bool(j)
for change in j:
tm = self._parse_timestamp(change['updated'])
if tm <= since:
need_more = False
break
if self.gerrit_projects and change['project'] not in self.gerrit_projects:
continue
d = self.checkForNewPatchset(change, since)
d.addCallback(lambda x: self.addChange(*x) if x else None)
if need_more and j[-1].get('_more_changes'):
d = self.getChanges(sortkey=j[-1]['_sortkey'])
d.addCallback(self.processChanges, since=since)
else:
d = defer.succeed(None)
return d
@deferredLocked('initLock')
def poll(self):
if self.dry_run:
return
log.msg('GerritPoller: getting latest changes...')
since = self.last_timestamp
d = self.getChanges()
def _update_last_timestamp(j):
if j:
self.last_timestamp = self._parse_timestamp(j[0]['updated'])
return j
d.addCallback(_update_last_timestamp)
d.addCallback(self.processChanges, since=since)
return d
class TryJobGerritStatus(StatusReceiverMultiService):
"""Posts results of a try job back to a Gerrit change."""
def __init__(self,
gerrit_host,
review_factory=None,
cq_builders=None,
**kwargs):
"""Creates a TryJobGerritStatus.
Args:
gerrit_host: a URL of the Gerrit instance.
review_factory: a function (self, builder_name, build, result) => review,
where review is a dict described in Gerrit docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
cq_builders: a list of buildernames, if specified, patchset will be
submitted if all builders have completed successfully.
kwargs: keyword arguments passed to GerritAgent.
"""
StatusReceiverMultiService.__init__(self)
self.review_factory = review_factory or TryJobGerritStatus.createReview
self.agent = GerritAgent(gerrit_host, **kwargs)
self.status = None
self.cq_builders = cq_builders
def createReview(self, builder_name, build, result):
review = {}
if result is not None:
message = ('A try job has finished on builder %s: %s' %
(builder_name, Results[result].upper()))
else:
message = 'A try job has started on builder %s' % builder_name
# Do not send email about this.
review['notify'] = 'NONE'
# Append build url.
# A line break in a Gerrit message is \n\n.
assert self.status
build_url = self.status.getURLForThing(build)
message = '%s\n\n%s' % (message, build_url)
review['message'] = message
return review
def sendUpdate(self, builder_name, build, result):
"""Posts a message and labels, if any, on a Gerrit change."""
props = build.properties
change_id = (props.getProperty('event.change.id')
or props.getProperty('parent_event.change.id'))
revision = props.getProperty('revision')
if change_id and revision:
review = self.review_factory(self, builder_name, build, result)
if review:
log.msg('Sending a review for change %s: %s' %
(change_id, review))
path = '/changes/%s/revisions/%s/review' % (change_id,
revision)
return self.agent.request('POST', path, body=review)
@defer.inlineCallbacks
def _add_verified_label(self, change_id, revision, patchset_id, commit):
message = 'All tryjobs have passed for patchset %s' % patchset_id
path = '/changes/%s/revisions/%s/review' % (change_id, revision)
body = {'message': message, 'labels': {'Verified': '+1'}}
yield self.agent.request('POST', path, body=body)
# commit the change
if commit:
path = 'changes/%s/submit' % change_id
body = {'wait_for_merge': True}
yield self.agent.request('POST', path, body=body)
MESSAGE_REGEX_TRYJOB_RESULT = re.compile(
'A try job has finished on builder (.+): SUCCESS', re.I | re.M)
def submitPatchSetIfNecessary(self, builder_name, build, result):
"""This is a temporary hack until Gerrit CQ is deployed."""
if not self.cq_builders:
return
if not (result == results.SUCCESS or results == results.WARNINGS):
return
props = build.properties
change_id = (props.getProperty('event.change.id')
or props.getProperty('parent_event.change.id'))
revision = props.getProperty('revision')
patchset_id = props.getProperty('event.patchSet.ref').rsplit('/', 1)[1]
builders = [x for x in self.cq_builders if x != builder_name]
o_params = '&'.join('o=%s' % x
for x in ('MESSAGES', 'ALL_REVISIONS',
'ALL_COMMITS', 'ALL_FILES', 'LABELS'))
path = '/changes/%s?%s' % (change_id, o_params)
d = self.agent.request('GET', path)
def _parse_messages(j):
if not j:
return
commit = 'approved' in j.get('labels', {}).get('Commit-Queue', {})
if len(builders) == 0:
self._add_verified_label(change_id, revision, patchset_id,
commit)
return
if 'messages' not in j:
return
for m in reversed(j['messages']):
if m['_revision_number'] == int(patchset_id):
match = self.MESSAGE_REGEX_TRYJOB_RESULT.search(
m['message'])
if match:
builder = match.groups()[0]
if builder in builders:
builders.remove(builder)
if len(builders) == 0:
self._add_verified_label(
change_id, revision, patchset_id, commit)
break
d.addCallback(_parse_messages)
return d
def startService(self):
StatusReceiverMultiService.startService(self)
self.status = self.parent.getStatus()
self.status.subscribe(self)
def builderAdded(self, name, builder):
# Subscribe to this builder.
return self
def buildStarted(self, builder_name, build):
self.sendUpdate(builder_name, build, None)
def buildFinished(self, builder_name, build, result):
self.sendUpdate(builder_name, build, result)
self.submitPatchSetIfNecessary(builder_name, build, result)
def __init__(self,
repo_url,
branches=None,
pollInterval=30,
category=None,
project=None,
revlinktmpl=None,
agent=None,
svn_mode=False,
svn_branch=None,
change_filter=None,
comparator=None,
cursor_file=default_cursor_file):
"""Args:
repo_url: URL of the gitiles service to be polled.
branches: List of strings and/or compiled regular expressions, specifying
the branches to be polled.
pollInterval: Number of seconds between polling operations.
category: Category to be applied to generated Change objects.
project: Project to be applied to generated Change objects.
revlinktmpl: String template, taking a single 'revision' parameter, used to
generate a web link to a revision.
agent: A GerritAgent object used to make requests to the gitiles service.
svn_mode: When polling a mirror of an svn repository, create changes using
the svn revision number.
svn_branch: When svn_mode=True, this is used to determine the svn branch
name for each change. It can be either a static string, or a function
that takes (gitiles_commit_json, git_branch) as arguments and returns
a static string.
comparator: A GitilesRevisionComparator object, or None. This is used to
share a single comparator between multiple pollers.
cursor_file: Load/save the latest polled revisions for each
repository/branch to this file. If None, no state will be preserved,
and polling will begin at branch HEAD at the time of master start.
"""
u = urlparse(repo_url)
self.repo_url = repo_url
self.repo_host = u.netloc
self.repo_path = urllib.quote(u.path)
if branches is None:
branches = ['master']
elif isinstance(branches, basestring):
branches = [branches]
self.branches = []
for b in branches:
if not isinstance(b, self.re_pattern_type):
b = b.lstrip('/')
if not b.startswith('refs/'):
b = 'refs/heads/' + b
self.branches.append(b)
self.branch_heads = {}
self.pollInterval = pollInterval
self.category = category
if project is None:
project = os.path.basename(repo_url)
if project.endswith('.git'):
project = project[:-4]
self.project = project
self.revlinktmpl = revlinktmpl or '%s/+/%%s' % repo_url
self.svn_mode = svn_mode
self.svn_branch = svn_branch
if svn_mode and not svn_branch:
self.svn_branch = project
if agent is None:
agent = GerritAgent('%s://%s' % (u.scheme, u.netloc),
read_only=True)
self.agent = agent
self.dry_run = os.environ.get('POLLER_DRY_RUN')
self.change_filter = change_filter
self.comparator = comparator or GitilesRevisionComparator()
self.cursor_file = cursor_file
class TryJobGerritStatus(StatusReceiverMultiService):
"""Posts results of a try job back to a Gerrit change."""
def __init__(self, gerrit_host, review_factory=None, cq_builders=None,
**kwargs):
"""Creates a TryJobGerritStatus.
Args:
gerrit_host: a URL of the Gerrit instance.
review_factory: a function (self, builder_name, build, result) => review,
where review is a dict described in Gerrit docs:
https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input
cq_builders: a list of buildernames, if specified, patchset will be
submitted if all builders have completed successfully.
kwargs: keyword arguments passed to GerritAgent.
"""
StatusReceiverMultiService.__init__(self)
self.review_factory = review_factory or TryJobGerritStatus.createReview
self.agent = GerritAgent(gerrit_host, **kwargs)
self.status = None
self.cq_builders = cq_builders
def createReview(self, builder_name, build, result):
review = {}
if result is not None:
message = ('A try job has finished on builder %s: %s' %
(builder_name, Results[result].upper()))
else:
message = 'A try job has started on builder %s' % builder_name
# Do not send email about this.
review['notify'] = 'NONE'
# Append build url.
# A line break in a Gerrit message is \n\n.
assert self.status
build_url = self.status.getURLForThing(build)
message = '%s\n\n%s' % (message, build_url)
review['message'] = message
return review
def sendUpdate(self, builder_name, build, result):
"""Posts a message and labels, if any, on a Gerrit change."""
props = build.properties
change_id = (props.getProperty('event.change.id') or
props.getProperty('parent_event.change.id'))
revision = props.getProperty('revision')
if change_id and revision:
review = self.review_factory(self, builder_name, build, result)
if review:
log.msg('Sending a review for change %s: %s' % (change_id, review))
path = '/changes/%s/revisions/%s/review' % (change_id, revision)
return self.agent.request('POST', path, body=review)
@defer.inlineCallbacks
def _add_verified_label(self, change_id, revision, patchset_id, commit):
message = 'All tryjobs have passed for patchset %s' % patchset_id
path = '/changes/%s/revisions/%s/review' % (change_id, revision)
body = {'message': message, 'labels': {'Verified': '+1'}}
yield self.agent.request('POST', path, body=body)
# commit the change
if commit:
path = 'changes/%s/submit' % change_id
body = {'wait_for_merge': True}
yield self.agent.request('POST', path, body=body)
MESSAGE_REGEX_TRYJOB_RESULT = re.compile(
'A try job has finished on builder (.+): SUCCESS', re.I | re.M)
def submitPatchSetIfNecessary(self, builder_name, build, result):
"""This is a temporary hack until Gerrit CQ is deployed."""
if not self.cq_builders:
return
if not (result == results.SUCCESS or results == results.WARNINGS):
return
props = build.properties
change_id = (props.getProperty('event.change.id') or
props.getProperty('parent_event.change.id'))
revision = props.getProperty('revision')
patchset_id = props.getProperty('event.patchSet.ref').rsplit('/', 1)[1]
builders = [x for x in self.cq_builders if x != builder_name]
o_params = '&'.join('o=%s' % x for x in (
'MESSAGES', 'ALL_REVISIONS', 'ALL_COMMITS', 'ALL_FILES', 'LABELS'))
path = '/changes/%s?%s' % (change_id, o_params)
d = self.agent.request('GET', path)
def _parse_messages(j):
if not j:
return
commit = 'approved' in j.get('labels', {}).get('Commit-Queue', {})
if len(builders) == 0:
self._add_verified_label(change_id, revision, patchset_id, commit)
return
if 'messages' not in j:
return
for m in reversed(j['messages']):
if m['_revision_number'] == int(patchset_id):
match = self.MESSAGE_REGEX_TRYJOB_RESULT.search(m['message'])
if match:
builder = match.groups()[0]
if builder in builders:
builders.remove(builder)
if len(builders) == 0:
self._add_verified_label(change_id, revision, patchset_id,
commit)
break
d.addCallback(_parse_messages)
return d
def startService(self):
StatusReceiverMultiService.startService(self)
self.status = self.parent.getStatus()
self.status.subscribe(self)
def builderAdded(self, name, builder):
# Subscribe to this builder.
return self
def buildStarted(self, builder_name, build):
self.sendUpdate(builder_name, build, None)
def buildFinished(self, builder_name, build, result):
self.sendUpdate(builder_name, build, result)
self.submitPatchSetIfNecessary(builder_name, build, result)
