Esempio n. 1
0
    def __init__(
        self,
        goanna_host,
        goanna_port,
        project,
        branch=None,
        goanna_hosted_path=None,
        paths=[],
        category=None,
        pollInterval=60 * 10,
        pollinterval=-2,
        proxy_host=None,
        proxy_port=None,
        proxy_username=None,
        proxy_password=None,
    ):

        # for backward compatibility; official buildbot ChangSource pollers
        # used to provide "pollinterval" in older versions but newer versions
        # now prefer "pollInterval". We allow for both.
        if pollinterval != -2:
            pollInterval = pollinterval

        # Goanna Timestamp of the last revision acted upon. This is used
        # during each poll to retrieve revisions newer than the timestamp.
        #  - On startup, this value is set based on the last known state
        #    (see self.startService()
        #  - If not previous states are found, we query Goanna for the
        #    latest revision and use that as the initial state
        #  - This value is updated (and the state stored) each time changes
        #    are found.
        self.lastRevTimestamp = None

        self.project = project
        self.branch = branch
        self.category = category
        self.web_client = ProxyWebClient(proxy_host, proxy_port, proxy_username, proxy_password, timeout=pollInterval)

        if not goanna_hosted_path:
            goanna_hosted_path = ""
        self.query_url = "http://%s:%s%s/query/%s" % (goanna_host, goanna_port, goanna_hosted_path, project)
        if branch:
            self.query_url += "/%s" % branch

        if isinstance(paths, basestring):
            paths = [paths]
        self.paths = sorted(set(normalise_path(p) for p in paths))
        if any("|" in p for p in paths):
            config.error("GoannaPoller: invalid value given as path")

        # older versions of PollingChangeSource does not have __init__()
        if hasattr(base.PollingChangeSource, "__init__"):
            base.PollingChangeSource.__init__(self, name=self.query_url, pollInterval=pollInterval)
Esempio n. 2
0
class GoannaPoller(base.PollingChangeSource, StateMixin):
    """
    Polls a Goanna server for changes in a specific project/branch/path
    """

    compare_attrs = ["project", "branch", "path", "pollInterval", "goanna_url"]
    last_change = None

    def __init__(
        self,
        goanna_host,
        goanna_port,
        project,
        branch=None,
        goanna_hosted_path=None,
        paths=[],
        category=None,
        pollInterval=60 * 10,
        pollinterval=-2,
        proxy_host=None,
        proxy_port=None,
        proxy_username=None,
        proxy_password=None,
    ):

        # for backward compatibility; official buildbot ChangSource pollers
        # used to provide "pollinterval" in older versions but newer versions
        # now prefer "pollInterval". We allow for both.
        if pollinterval != -2:
            pollInterval = pollinterval

        # Goanna Timestamp of the last revision acted upon. This is used
        # during each poll to retrieve revisions newer than the timestamp.
        #  - On startup, this value is set based on the last known state
        #    (see self.startService()
        #  - If not previous states are found, we query Goanna for the
        #    latest revision and use that as the initial state
        #  - This value is updated (and the state stored) each time changes
        #    are found.
        self.lastRevTimestamp = None

        self.project = project
        self.branch = branch
        self.category = category
        self.web_client = ProxyWebClient(proxy_host, proxy_port, proxy_username, proxy_password, timeout=pollInterval)

        if not goanna_hosted_path:
            goanna_hosted_path = ""
        self.query_url = "http://%s:%s%s/query/%s" % (goanna_host, goanna_port, goanna_hosted_path, project)
        if branch:
            self.query_url += "/%s" % branch

        if isinstance(paths, basestring):
            paths = [paths]
        self.paths = sorted(set(normalise_path(p) for p in paths))
        if any("|" in p for p in paths):
            config.error("GoannaPoller: invalid value given as path")

        # older versions of PollingChangeSource does not have __init__()
        if hasattr(base.PollingChangeSource, "__init__"):
            base.PollingChangeSource.__init__(self, name=self.query_url, pollInterval=pollInterval)

    def startService(self):

        d.addCallback(lambda _: base.PollingChangeSource.StartService(self))
        return d

    def describe(self):
        if not self.master:
            status = "[STOPPED - check log]"
        return "GoannaPoller: watching %s" % self.query_url

    @defer.inlineCallbacks
    def poll(self):
        yield self.verify_state()
        d = self.get_changes()
        d.addCallback(self.parse_changes)
        d.addCallback(self.submit_changes)
        d.addCallback(self.update_state)
        d.addErrback(self._processFailure)
        yield d

    def _processFailure(self, f):
        log.msg("GoannaPoller: failure while polling " + self.query_url)
        log.err(f)
        return None

    @defer.inlineCallbacks
    def verify_state(self):
        if self.lastRevTimestamp:
            yield defer.succeed(None)
        else:
            yield self.restore_previous_state()

    @defer.inlineCallbacks
    def restore_previous_state(self):
        d = self.getState("lastRevTimestamp", None)

        def initialise_timestamp(st):
            self.lastRevTimestamp = st

        d.addCallback(initialise_timestamp)
        yield d

        if not self.lastRevTimestamp:
            yield self.initialise_with_latest_revision()
        else:
            log.msg("GoannaPoller: restoring state. Last revision %s" % r["revision"])

    def initialise_with_latest_revision(self):
        # No previous state, query Goanna for latest revision
        url = self.query_url + self._get_querystring(brief=True)
        d = self.web_client.getPage(url)

        def load_query_results(data):
            r = unpack(data)
            self.lastRevTimestamp = r["ts"]
            log.msg("GoannaPoller: initialising at revision %s" % r["revision"])

        d.addCallback(load_query_results)
        d.addErrback(log.err, "could not load latest revision from " + url)
        return d

    def get_changes(self):
        url = self.query_url + self._get_querystring(self.lastRevTimestamp)
        d = self.web_client.getPage(url)
        return d

    def parse_changes(self, change_data):
        def _get_chdict(c):
            chdict = dict(
                project=c["project"],
                author=c["author"],
                files=c["files"],
                comments=c["comments"],
                revision=c["revision"],
                ts=c["ts"],
            )  # to be stripped out before submission

            # insert optional data
            for key in ("branch", "revlink", "repository", "codebase"):
                if key in c:
                    chdict[key] = c[key]

            if "change_ts" in c:
                chdict["when_timestamp"] = epoch2datetime(c["change_ts"])

            return chdict

        return [_get_chdict(c) for c in unpack(change_data)]

    @defer.inlineCallbacks
    def submit_changes(self, changes):
        if changes:
            ts_max = 0
            for chdict in changes:
                ts_max = max(ts_max, float(chdict.pop("ts")))
                r = chdict["revision"]
                if self._is_relevant_change(chdict):
                    log.msg("GoannaPoller: adding change (rev: %s)" % r)
                    yield self.master.addChange(category=self.category, src="goanna", **chdict)
                else:
                    log.msg("GoannaPoller: skipping change (rev: %s)" % r)
            yield defer.returnValue(ts_max)
        else:
            yield defer.succeed(None)

    def update_state(self, ts_max):
        if ts_max:
            self.lastRevTimestamp = ts_max
            return self.setState("lastRevTimestamp", self.lastRevTimestamp)
        else:
            return defer.succeed(None)

    def _is_relevant_change(self, change):
        if not self.paths:  # if unfiltered, all changes are relevant
            return True

        changed_files = change["files"]
        assert isinstance(changed_files, list)

        files = (normalise_path(f) for f in sorted(changed_files))
        paths = (p for p in self.paths)  # already sorted and normalised
        try:
            f = next(files)
            p = next(paths)
            while True:
                if is_subdir(f, p):
                    return True
                if p > f:
                    f = next(files)
                else:
                    p = next(paths)
        except StopIteration:
            return False

    def _get_querystring(self, ts_from=None, ts_to=None, brief=False):
        params = []
        if ts_from:
            params.append("from=%s" % str(ts_from))
        if ts_to:
            params.append("to=%s" % str(ts_to))
        if brief:
            params.append("brief")

        if not params:
            return ""
        else:
            return "?%s" % ";".join(params)