Пример #1
0
class PulseDispatcher(object):
    instance = None

    def __init__(self, msg, config):
        self.msg = msg
        self.config = config
        self.dispatch = defaultdict(set)
        self.max_checkins = 10
        self.applabel = None
        self.shutting_down = False

        if not config.parser.has_option('pulse', 'user'):
            raise Exception('Missing configuration: pulse.user')

        if not config.parser.has_option('pulse', 'password'):
            raise Exception('Missing configuration: pulse.password')

        if (config.parser.has_option('bugzilla', 'server')
                and config.parser.has_option('bugzilla', 'password')
                and config.parser.has_option('bugzilla', 'user')):
            server = config.bugzilla.server
            if not server.lower().startswith('https://'):
                raise Exception('bugzilla.server must be a HTTPS url')

            self.bugzilla = Bugzilla(server,
                                     config.bugzilla.user,
                                     config.bugzilla.password)
        else:
            self.bugzilla = None

        if config.parser.has_option('bugzilla', 'pulse'):
            self.bugzilla_branches = config.bugzilla.get_list('pulse')

        self.pulse = pulse.PulseListener(
            config.pulse.user,
            config.pulse.password,
            config.pulse.applabel
            if config.parser.has_option('pulse', 'applabel') else None
        )

        if config.parser.has_option('pulse', 'channels'):
            for chan in config.pulse.get_list('channels'):
                confchan = chan[1:] if chan[0] == '#' else chan
                if config.parser.has_option('pulse', confchan):
                    for branch in config.pulse.get_list(confchan):
                        self.dispatch[branch].add(chan)

        if config.parser.has_option('pulse', 'max_checkins'):
            self.max_checkins = config.pulse.max_checkins

        if self.dispatch:
            self.bugzilla_queue = Queue(42)
            self.reporter_thread = threading.Thread(target=self.change_reporter)
            self.bugzilla_thread = threading.Thread(target=self.bugzilla_reporter)
            self.reporter_thread.start()
            self.bugzilla_thread.start()

    def change_reporter(self):
        for rev, branch, revlink, data in self.pulse:
            repo = REVLINK_RE.sub('', revlink)
            pushes_url = '%s/json-pushes?full=1&changeset=%s' \
                % (repo, rev)
            messages = []
            urls_for_bugs = defaultdict(list)
            try:
                r = requests.get(pushes_url)
                if r.status_code == 500:
                    # If we got an error 500, try again once.
                    r = requests.get(pushes_url)
                if r.status_code != 200:
                    r.raise_for_status()

                data = r.json()

                for d in data.values():
                    changesets = d['changesets']
                    group_changesets = len(changesets) > self.max_checkins
                    if group_changesets:
                        short_rev = rev[:12]
                        messages.append('%s/pushloghtml?changeset=%s'
                            ' - %d changesets'
                            % (repo, short_rev, len(changesets)))

                    for cs in changesets:
                        short_node = cs['node'][:12]
                        revlink = '%s/rev/%s' \
                            % (repo, short_node)
                        desc = cs['desc'].splitlines()[0].strip()

                        if self.bugzilla and branch in self.bugzilla_branches:
                            bugs = parse_bugs(desc)
                            if bugs:
                                urls_for_bugs[bugs[0]].append((
                                    revlink,
                                    extract_summary(desc),
                                    bool(BACKOUT_RE.match(desc)),
                                ))

                        if not group_changesets:
                            author = cs['author']
                            author = author.split(' <')[0].strip()
                            messages.append("%s - %s - %s"
                                % (revlink, author, desc))
            except:
                self.msg(self.config.core.owner, self.config.core.owner,
                    "Failure on %s:" % pushes_url)
                for line in traceback.format_exc().splitlines():
                    self.msg(self.config.core.owner, self.config.core.owner,
                        line)
                self.msg(self.config.core.owner, self.config.core.owner,
                    "Message data was: %s" % data, 10)
                continue

            for msg in messages:
                for chan in self.dispatch.get(branch, set()) | \
                        self.dispatch.get('*', set()):
                    self.msg(chan, chan, "Check-in: %s" % msg)

            for bug, urls in urls_for_bugs.iteritems():
                self.bugzilla_queue.put((bug, urls))

    def bugzilla_reporter(self):
        delayed_comments = []
        def get_one():
            if delayed_comments:
                when, bug, urls = delayed_comments[0]
                if when <= time.time():
                    delayed_comments.pop(0)
                    return bug, urls, True
            try:
                bug, urls = self.bugzilla_queue.get(timeout=1)
                return bug, urls, False
            except Empty:
                return None, None, None

        while not self.shutting_down:
            bug, urls, delayed = get_one()
            if bug is None:
                continue

            try:
                comments = '\n'.join(self.bugzilla.get_comments(bug))
            except BugzillaError:
                # Don't do anything on errors, such as "You are not authorized
                # to access bug #xxxxx".
                continue

            urls_to_write = []
            summary = {}
            backouts = set()
            for url, desc, is_backout in urls:
                # Only write about a changeset if it's never been mentioned
                # at all. This makes us not emit changesets that e.g. land
                # on mozilla-inbound when they were mentioned when landing
                # on mozilla-central.
                if url[-12:] not in comments:
                    urls_to_write.append(url)
                    summary[url] = desc
                if is_backout:
                    backouts.add(url)

            if urls_to_write:
                def comment():
                    if all(url in backouts for url in urls_to_write):
                        yield 'Backout:'
                        for url in urls_to_write:
                            yield url + summary[url]
                    else:
                        for url in urls_to_write:
                            if url in backouts:
                                yield '%s (backout)%s' % (url, summary[url])
                            else:
                                yield url + summary[url]

                try:
                    fields = ('whiteboard', 'keywords')
                    values = self.bugzilla.get_fields(bug, fields)
                    # Delay comments for backouts and checkin-needed in
                    # whiteboard
                    delay_comment = (
                        not delayed
                        and (all(url in backouts for url in urls_to_write)
                             or 'checkin-needed' in values.get('whiteboard', ''))
                    )
                    if delay_comment:
                        delayed_comments.append((time.time() + 600, bug, urls))
                    else:
                        message = '\n'.join(comment())
                        kwargs = {}
                        if 'checkin-needed' in values.get('keywords', {}):
                            kwargs['keywords'] = {
                                'remove': ['checkin-needed']
                            }
                        if kwargs:
                            kwargs['comment'] = {'body': message}
                            self.bugzilla.update_bug(bug, **kwargs)
                        else:
                            self.bugzilla.post_comment(bug, message)
                except:
                    self.msg(self.config.core.owner, self.config.core.owner,
                        "Failed to send comment to bug %d" % bug)

    def shutdown(self):
        self.shutting_down = True
        self.pulse.shutdown()
        self.reporter_thread.join()
        self.bugzilla_thread.join()
Пример #2
0
class PulseDispatcher(object):
    instance = None

    def __init__(self, msg, config):
        self.msg = msg
        self.config = config
        self.hgpushes = PulseHgPushes(config)
        self.max_checkins = 10
        self.shutting_down = False
        self.backout_delay = 600

        if (config.parser.has_option('bugzilla', 'server')
                and config.parser.has_option('bugzilla', 'api_key')):
            self.bugzilla = Bugzilla(config.bugzilla.server,
                                     config.bugzilla.api_key)

        if config.parser.has_option('pulse', 'max_checkins'):
            self.max_checkins = config.pulse.max_checkins

        if self.config.dispatch or self.config.bugzilla_branches:
            self.reporter_thread = threading.Thread(
                target=self.change_reporter)
            self.reporter_thread.start()

        if self.config.bugzilla_branches:
            self.bugzilla_queue = Queue(42)
            self.bugzilla_thread = threading.Thread(
                target=self.bugzilla_reporter)
            self.bugzilla_thread.start()

    def change_reporter(self):
        for push in self.hgpushes:
            self.report_one_push(push)

    def report_one_push(self, push):
        url = urlparse.urlparse(push['pushlog'])
        branch = os.path.dirname(url.path).strip('/')

        channels = self.config.dispatch.get(branch)

        if channels:
            for msg in self.create_messages(push, self.max_checkins):
                for chan in channels:
                    self.msg(chan, chan, "Check-in: %s" % msg)

        if branch in self.config.bugzilla_branches:
            for info in self.munge_for_bugzilla(push):
                if branch in self.config.bugzilla_leave_open:
                    info.leave_open = True
                self.bugzilla_queue.put(info)

    @staticmethod
    def create_messages(push, max_checkins=sys.maxsize, max_bugs=5):
        max_bugs = max(1, max_bugs)
        changesets = push['changesets']
        group = ''
        group_bugs = []

        # Kind of gross
        last_desc = changesets[-1]['desc'] if changesets else ''
        merge = changesets[-1].get('is_merge') if changesets else False

        group_changesets = merge or len(changesets) > max_checkins

        if not merge:
            for cs in changesets:
                revlink = cs['revlink']
                desc = cs['desc']

                if group_changesets:
                    bugs = parse_bugs(desc)
                    if bugs and bugs[0] not in group_bugs:
                        group_bugs.append(bugs[0])
                else:
                    author = cs['author']
                    yield "%s - %s - %s" % (revlink, author, desc)

        if group_changesets:
            group = '%s - %d changesets' % (push['pushlog'], len(changesets))

        if merge:
            group += ' - %s' % last_desc

        if group:
            if group_bugs and not merge:
                group += ' (bug%s %s%s)' % (
                    's' if len(group_bugs) > 1 else '', ', '.join(
                        str(b) for b in group_bugs[:max_bugs]),
                    ' and %d other bug%s' %
                    (len(group_bugs) - max_bugs,
                     's' if len(group_bugs) > max_bugs + 1 else '')
                    if len(group_bugs) > max_bugs else '')
            yield group

    @staticmethod
    def munge_for_bugzilla(push):
        info_for_bugs = {}

        for cs in push['changesets']:
            if cs.get('source-repo', '').startswith('https://github.com/'):
                continue
            bugs = parse_bugs(cs['desc'])
            if bugs:
                if bugs[0] not in info_for_bugs:
                    info_for_bugs[bugs[0]] = BugInfo(bugs[0], push['user'])
                info_for_bugs[bugs[0]].add_changeset(cs)

        for info in info_for_bugs.itervalues():
            yield info

    @staticmethod
    def bugzilla_summary(cs):
        yield cs['revlink']

        desc = cs['desc']
        matches = [
            m for m in BUG_RE.finditer(desc) if int(m.group(2)) < 100000000
        ]

        match = matches[0]
        if match.start() == 0:
            desc = desc[match.end():].lstrip(' \t-,.:')
        else:
            backout = BACKOUT_RE.match(desc)
            if backout and not desc[backout.end():match.start()].strip():
                desc = desc[:backout.end()] + desc[match.end():]
            elif (desc[match.start() - 1] == '('
                  and desc[match.end():match.end() + 1] == ')'):
                desc = (desc[:match.start() - 1].rstrip() + ' ' +
                        desc[match.end() + 1:].lstrip())

        yield desc

    def bugzilla_reporter(self):
        delayed_comments = []

        def get_one():
            if delayed_comments:
                when, info = delayed_comments[0]
                if when <= time.time():
                    delayed_comments.pop(0)
                    return info, True
            try:
                info = self.bugzilla_queue.get(timeout=1)
                return info, False
            except Empty:
                return None, None

        while True:
            info, delayed = get_one()
            if info is None:
                if self.shutting_down:
                    break
                continue

            try:
                comments = '\n'.join(self.bugzilla.get_comments(info.bug))
            except BugzillaError:
                # Don't do anything on errors, such as "You are not authorized
                # to access bug #xxxxx".
                continue

            cs_to_write = []
            for cs_info in info:
                url = cs_info['revlink']
                # Only write about a changeset if it's never been mentioned
                # at all. This makes us not emit changesets that e.g. land
                # on mozilla-inbound when they were mentioned when landing
                # on mozilla-central.
                if url[-12:] not in comments:
                    cs_to_write.append(cs_info)

            if cs_to_write:
                is_backout = all(cs['is_backout'] for cs in cs_to_write)

                def comment():
                    if is_backout:
                        if info.pusher:
                            yield 'Backout by %s:' % info.pusher
                        else:
                            yield 'Backout:'
                    elif info.pusher:
                        yield 'Pushed by %s:' % info.pusher
                    for cs in cs_to_write:
                        for line in self.bugzilla_summary(cs):
                            yield line

                try:
                    fields = ('whiteboard', 'keywords')
                    values = self.bugzilla.get_fields(info.bug, fields)
                    # Delay comments for backouts and checkin-needed in
                    # whiteboard
                    delay_comment = (not delayed
                                     and (is_backout or 'checkin-needed'
                                          in values.get('whiteboard', '')))
                    if delay_comment:
                        delayed_comments.append(
                            (time.time() + self.backout_delay, info))
                    else:
                        message = '\n'.join(comment())
                        kwargs = {}
                        if 'checkin-needed' in values.get('keywords', {}):
                            kwargs['keywords'] = {'remove': ['checkin-needed']}
                        # TODO: reopen closed bugs on backout
                        if ('leave-open' not in values.get('keywords', {})
                                and not is_backout and not info.leave_open):
                            kwargs['status'] = 'RESOLVED'
                            kwargs['resolution'] = 'FIXED'
                        if kwargs:
                            kwargs['comment'] = {'body': message}
                            self.bugzilla.update_bug(info.bug, **kwargs)
                        else:
                            self.bugzilla.post_comment(info.bug, message)
                except:
                    logging.getLogger('pulsebot.buzilla').error(
                        "Failed to send comment to bug %d", info.bug)

    def shutdown(self):
        self.hgpushes.shutdown()
        if self.config.dispatch or self.config.bugzilla_branches:
            self.reporter_thread.join()
        self.shutting_down = True
        if self.config.bugzilla_branches:
            self.bugzilla_thread.join()
Пример #3
0
class PulseDispatcher(object):
    instance = None

    def __init__(self, config):
        self.config = config
        self.hgpushes = PulseHgPushes(config)
        self.max_checkins = 10
        self.shutting_down = False
        self.backout_delay = 600

        if config.bugzilla_server and config.bugzilla_api_key:
            self.bugzilla = Bugzilla(config.bugzilla_server,
                                     config.bugzilla_api_key)

        if config.pulse_max_checkins:
            self.max_checkins = config.pulse_max_checkins

        if self.config.bugzilla_branches:
            self.bugzilla_queue = Queue(42)
            self.bugzilla_thread = threading.Thread(
                target=self.bugzilla_reporter)
            self.bugzilla_thread.start()

    def change_reporter(self):
        for push in self.hgpushes:
            self.report_one_push(push)

    def report_one_push(self, push):
        url = urlparse(push["pushlog"])
        branch = os.path.dirname(url.path).strip("/")
        logger.info(f'report_one_push: {url.netloc}{url.path} {branch}')
        if branch in self.config.bugzilla_branches:
            for info in self.munge_for_bugzilla(push):
                if branch in self.config.bugzilla_leave_open:
                    info.leave_open = True
                self.bugzilla_queue.put(info)

    @staticmethod
    def create_messages(push, max_checkins=sys.maxsize, max_bugs=5):
        max_bugs = max(1, max_bugs)
        changesets = push["changesets"]
        group = ""
        group_bugs = []

        # Kind of gross
        last_desc = changesets[-1]["desc"] if changesets else ""
        merge = changesets[-1].get("is_merge") if changesets else False

        group_changesets = merge or len(changesets) > max_checkins

        if not merge:
            for cs in changesets:
                revlink = cs["revlink"]
                desc = cs["desc"]

                if group_changesets:
                    bugs = parse_bugs(desc)
                    if bugs and bugs[0] not in group_bugs:
                        logger.info(f'bug found: {bugs[0]}')
                        group_bugs.append(bugs[0])
                else:
                    author = cs["author"]
                    yield "%s - %s - %s" % (revlink, author, desc)

        if group_changesets:
            group = "%s - %d changesets" % (push["pushlog"], len(changesets))

        if merge:
            group += " - %s" % last_desc

        if group:
            if group_bugs and not merge:
                group += " (bug%s %s%s)" % (
                    "s" if len(group_bugs) > 1 else "",
                    ", ".join(str(b) for b in group_bugs[:max_bugs]),
                    " and %d other bug%s" % (
                        len(group_bugs) - max_bugs,
                        "s" if len(group_bugs) > max_bugs + 1 else "",
                    ) if len(group_bugs) > max_bugs else "",
                )
            yield group

    @staticmethod
    def munge_for_bugzilla(push):
        info_for_bugs = {}

        for cs in push["changesets"]:
            if cs.get("source-repo", "").startswith("https://github.com/"):
                continue
            bugs = parse_bugs(cs["desc"])
            if bugs:
                logger.info(f'bug found: {bugs[0]}')
                if bugs[0] not in info_for_bugs:
                    info_for_bugs[bugs[0]] = BugInfo(bugs[0], push["user"])
                info_for_bugs[bugs[0]].add_changeset(cs)

        for info in info_for_bugs.values():
            yield info

    @staticmethod
    def bugzilla_summary(cs):
        yield cs["revlink"]

        desc = cs["desc"]
        matches = [
            m for m in BUG_RE.finditer(desc) if int(m.group(2)) < 100000000
        ]

        match = matches[0]
        if match.start() == 0:
            desc = desc[match.end():].lstrip(" \t-,.:")
        else:
            backout = BACKOUT_RE.match(desc)
            if backout and not desc[backout.end():match.start()].strip():
                desc = desc[:backout.end()] + desc[match.end():]
            elif (desc[match.start() - 1] == "("
                  and desc[match.end():match.end() + 1] == ")"):
                desc = (desc[:match.start() - 1].rstrip() + " " +
                        desc[match.end() + 1:].lstrip())

        yield desc

    def bugzilla_reporter(self):
        delayed_comments = []

        def get_one():
            if delayed_comments:
                when, info = delayed_comments[0]
                if when <= time.time():
                    delayed_comments.pop(0)
                    return info, True
            try:
                info = self.bugzilla_queue.get(timeout=1)
                return info, False
            except Empty:
                return None, None

        while True:
            info, delayed = get_one()
            if info is None:
                if self.shutting_down:
                    break
                continue

            try:
                comments = "\n".join(self.bugzilla.get_comments(info.bug))
            except BugzillaError:
                # Don't do anything on errors, such as "You are not authorized
                # to access bug #xxxxx".
                continue

            cs_to_write = []
            for cs_info in info:
                url = cs_info["revlink"]
                # Only write about a changeset if it's never been mentioned
                # at all. This makes us not emit changesets that e.g. land
                # on mozilla-inbound when they were mentioned when landing
                # on mozilla-central.
                if url[-12:] not in comments:
                    cs_to_write.append(cs_info)

            if cs_to_write:
                is_backout = all(cs["is_backout"] for cs in cs_to_write)

                def comment():
                    if is_backout:
                        if info.pusher:
                            yield "Backout by %s:" % info.pusher
                        else:
                            yield "Backout:"
                    elif info.pusher:
                        yield "Pushed by %s:" % info.pusher
                    for cs in cs_to_write:
                        for line in self.bugzilla_summary(cs):
                            yield line

                try:
                    fields = ("whiteboard", "keywords", "status")
                    values = self.bugzilla.get_fields(info.bug, fields)
                    # Delay comments for backouts and checkin-needed in
                    # whiteboard
                    delay_comment = not delayed and (
                        is_backout
                        or "checkin-needed" in values.get("whiteboard", ""))
                    if delay_comment:
                        delayed_comments.append(
                            (time.time() + self.backout_delay, info))
                    else:
                        message = "\n".join(comment())
                        kwargs = {}
                        remove_keywords = [
                            kw
                            for kw in ["checkin-needed", "checkin-needed-tb"]
                            if kw in values.get("keywords", {})
                        ]
                        if remove_keywords:
                            kwargs["keywords"] = {"remove": remove_keywords}
                        # TODO: reopen closed bugs on backout
                        if ("leave-open" not in values.get("keywords", {})
                                and not is_backout and not info.leave_open
                                and values.get("status", "")
                                not in ("VERIFIED", "CLOSED", "RESOLVED")):
                            kwargs["status"] = "RESOLVED"
                            kwargs["resolution"] = "FIXED"
                        if kwargs:
                            kwargs["comment"] = {"body": message}
                            self.bugzilla.update_bug(info.bug, **kwargs)
                        else:
                            self.bugzilla.post_comment(info.bug, message)
                except Exception:
                    logger.exception(
                        f"Failed to send comment to bug {info.bug}")

    def shutdown(self):
        self.hgpushes.shutdown()
        self.shutting_down = True
        if self.config.bugzilla_branches:
            self.bugzilla_thread.join()