def __init__(
self,
goanna_host,
goanna_port,
project,
branch=None,
goanna_hosted_path=None,
paths=[],
category=None,
pollInterval=60 * 10,
pollinterval=-2,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password=None,
):
# for backward compatibility; official buildbot ChangSource pollers
# used to provide "pollinterval" in older versions but newer versions
# now prefer "pollInterval". We allow for both.
if pollinterval != -2:
pollInterval = pollinterval
# Goanna Timestamp of the last revision acted upon. This is used
# during each poll to retrieve revisions newer than the timestamp.
# - On startup, this value is set based on the last known state
# (see self.startService()
# - If not previous states are found, we query Goanna for the
# latest revision and use that as the initial state
# - This value is updated (and the state stored) each time changes
# are found.
self.lastRevTimestamp = None
self.project = project
self.branch = branch
self.category = category
self.web_client = ProxyWebClient(proxy_host, proxy_port, proxy_username, proxy_password, timeout=pollInterval)
if not goanna_hosted_path:
goanna_hosted_path = ""
self.query_url = "http://%s:%s%s/query/%s" % (goanna_host, goanna_port, goanna_hosted_path, project)
if branch:
self.query_url += "/%s" % branch
if isinstance(paths, basestring):
paths = [paths]
self.paths = sorted(set(normalise_path(p) for p in paths))
if any("|" in p for p in paths):
config.error("GoannaPoller: invalid value given as path")
# older versions of PollingChangeSource does not have __init__()
if hasattr(base.PollingChangeSource, "__init__"):
base.PollingChangeSource.__init__(self, name=self.query_url, pollInterval=pollInterval)
class GoannaPoller(base.PollingChangeSource, StateMixin):
"""
Polls a Goanna server for changes in a specific project/branch/path
"""
compare_attrs = ["project", "branch", "path", "pollInterval", "goanna_url"]
last_change = None
def __init__(
self,
goanna_host,
goanna_port,
project,
branch=None,
goanna_hosted_path=None,
paths=[],
category=None,
pollInterval=60 * 10,
pollinterval=-2,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password=None,
):
# for backward compatibility; official buildbot ChangSource pollers
# used to provide "pollinterval" in older versions but newer versions
# now prefer "pollInterval". We allow for both.
if pollinterval != -2:
pollInterval = pollinterval
# Goanna Timestamp of the last revision acted upon. This is used
# during each poll to retrieve revisions newer than the timestamp.
# - On startup, this value is set based on the last known state
# (see self.startService()
# - If not previous states are found, we query Goanna for the
# latest revision and use that as the initial state
# - This value is updated (and the state stored) each time changes
# are found.
self.lastRevTimestamp = None
self.project = project
self.branch = branch
self.category = category
self.web_client = ProxyWebClient(proxy_host, proxy_port, proxy_username, proxy_password, timeout=pollInterval)
if not goanna_hosted_path:
goanna_hosted_path = ""
self.query_url = "http://%s:%s%s/query/%s" % (goanna_host, goanna_port, goanna_hosted_path, project)
if branch:
self.query_url += "/%s" % branch
if isinstance(paths, basestring):
paths = [paths]
self.paths = sorted(set(normalise_path(p) for p in paths))
if any("|" in p for p in paths):
config.error("GoannaPoller: invalid value given as path")
# older versions of PollingChangeSource does not have __init__()
if hasattr(base.PollingChangeSource, "__init__"):
base.PollingChangeSource.__init__(self, name=self.query_url, pollInterval=pollInterval)
def startService(self):
d.addCallback(lambda _: base.PollingChangeSource.StartService(self))
return d
def describe(self):
if not self.master:
status = "[STOPPED - check log]"
return "GoannaPoller: watching %s" % self.query_url
@defer.inlineCallbacks
def poll(self):
yield self.verify_state()
d = self.get_changes()
d.addCallback(self.parse_changes)
d.addCallback(self.submit_changes)
d.addCallback(self.update_state)
d.addErrback(self._processFailure)
yield d
def _processFailure(self, f):
log.msg("GoannaPoller: failure while polling " + self.query_url)
log.err(f)
return None
@defer.inlineCallbacks
def verify_state(self):
if self.lastRevTimestamp:
yield defer.succeed(None)
else:
yield self.restore_previous_state()
@defer.inlineCallbacks
def restore_previous_state(self):
d = self.getState("lastRevTimestamp", None)
def initialise_timestamp(st):
self.lastRevTimestamp = st
d.addCallback(initialise_timestamp)
yield d
if not self.lastRevTimestamp:
yield self.initialise_with_latest_revision()
else:
log.msg("GoannaPoller: restoring state. Last revision %s" % r["revision"])
def initialise_with_latest_revision(self):
# No previous state, query Goanna for latest revision
url = self.query_url + self._get_querystring(brief=True)
d = self.web_client.getPage(url)
def load_query_results(data):
r = unpack(data)
self.lastRevTimestamp = r["ts"]
log.msg("GoannaPoller: initialising at revision %s" % r["revision"])
d.addCallback(load_query_results)
d.addErrback(log.err, "could not load latest revision from " + url)
return d
def get_changes(self):
url = self.query_url + self._get_querystring(self.lastRevTimestamp)
d = self.web_client.getPage(url)
return d
def parse_changes(self, change_data):
def _get_chdict(c):
chdict = dict(
project=c["project"],
author=c["author"],
files=c["files"],
comments=c["comments"],
revision=c["revision"],
ts=c["ts"],
) # to be stripped out before submission
# insert optional data
for key in ("branch", "revlink", "repository", "codebase"):
if key in c:
chdict[key] = c[key]
if "change_ts" in c:
chdict["when_timestamp"] = epoch2datetime(c["change_ts"])
return chdict
return [_get_chdict(c) for c in unpack(change_data)]
@defer.inlineCallbacks
def submit_changes(self, changes):
if changes:
ts_max = 0
for chdict in changes:
ts_max = max(ts_max, float(chdict.pop("ts")))
r = chdict["revision"]
if self._is_relevant_change(chdict):
log.msg("GoannaPoller: adding change (rev: %s)" % r)
yield self.master.addChange(category=self.category, src="goanna", **chdict)
else:
log.msg("GoannaPoller: skipping change (rev: %s)" % r)
yield defer.returnValue(ts_max)
else:
yield defer.succeed(None)
def update_state(self, ts_max):
if ts_max:
self.lastRevTimestamp = ts_max
return self.setState("lastRevTimestamp", self.lastRevTimestamp)
else:
return defer.succeed(None)
def _is_relevant_change(self, change):
if not self.paths: # if unfiltered, all changes are relevant
return True
changed_files = change["files"]
assert isinstance(changed_files, list)
files = (normalise_path(f) for f in sorted(changed_files))
paths = (p for p in self.paths) # already sorted and normalised
try:
f = next(files)
p = next(paths)
while True:
if is_subdir(f, p):
return True
if p > f:
f = next(files)
else:
p = next(paths)
except StopIteration:
return False
def _get_querystring(self, ts_from=None, ts_to=None, brief=False):
params = []
if ts_from:
params.append("from=%s" % str(ts_from))
if ts_to:
params.append("to=%s" % str(ts_to))
if brief:
params.append("brief")
if not params:
return ""
else:
return "?%s" % ";".join(params)