class PulseDispatcher(object):
instance = None
def __init__(self, msg, config):
self.msg = msg
self.config = config
self.dispatch = defaultdict(set)
self.max_checkins = 10
self.applabel = None
self.shutting_down = False
if not config.parser.has_option('pulse', 'user'):
raise Exception('Missing configuration: pulse.user')
if not config.parser.has_option('pulse', 'password'):
raise Exception('Missing configuration: pulse.password')
if (config.parser.has_option('bugzilla', 'server')
and config.parser.has_option('bugzilla', 'password')
and config.parser.has_option('bugzilla', 'user')):
server = config.bugzilla.server
if not server.lower().startswith('https://'):
raise Exception('bugzilla.server must be a HTTPS url')
self.bugzilla = Bugzilla(server,
config.bugzilla.user,
config.bugzilla.password)
else:
self.bugzilla = None
if config.parser.has_option('bugzilla', 'pulse'):
self.bugzilla_branches = config.bugzilla.get_list('pulse')
self.pulse = pulse.PulseListener(
config.pulse.user,
config.pulse.password,
config.pulse.applabel
if config.parser.has_option('pulse', 'applabel') else None
)
if config.parser.has_option('pulse', 'channels'):
for chan in config.pulse.get_list('channels'):
confchan = chan[1:] if chan[0] == '#' else chan
if config.parser.has_option('pulse', confchan):
for branch in config.pulse.get_list(confchan):
self.dispatch[branch].add(chan)
if config.parser.has_option('pulse', 'max_checkins'):
self.max_checkins = config.pulse.max_checkins
if self.dispatch:
self.bugzilla_queue = Queue(42)
self.reporter_thread = threading.Thread(target=self.change_reporter)
self.bugzilla_thread = threading.Thread(target=self.bugzilla_reporter)
self.reporter_thread.start()
self.bugzilla_thread.start()
def change_reporter(self):
for rev, branch, revlink, data in self.pulse:
repo = REVLINK_RE.sub('', revlink)
pushes_url = '%s/json-pushes?full=1&changeset=%s' \
% (repo, rev)
messages = []
urls_for_bugs = defaultdict(list)
try:
r = requests.get(pushes_url)
if r.status_code == 500:
# If we got an error 500, try again once.
r = requests.get(pushes_url)
if r.status_code != 200:
r.raise_for_status()
data = r.json()
for d in data.values():
changesets = d['changesets']
group_changesets = len(changesets) > self.max_checkins
if group_changesets:
short_rev = rev[:12]
messages.append('%s/pushloghtml?changeset=%s'
' - %d changesets'
% (repo, short_rev, len(changesets)))
for cs in changesets:
short_node = cs['node'][:12]
revlink = '%s/rev/%s' \
% (repo, short_node)
desc = cs['desc'].splitlines()[0].strip()
if self.bugzilla and branch in self.bugzilla_branches:
bugs = parse_bugs(desc)
if bugs:
urls_for_bugs[bugs[0]].append((
revlink,
extract_summary(desc),
bool(BACKOUT_RE.match(desc)),
))
if not group_changesets:
author = cs['author']
author = author.split(' <')[0].strip()
messages.append("%s - %s - %s"
% (revlink, author, desc))
except:
self.msg(self.config.core.owner, self.config.core.owner,
"Failure on %s:" % pushes_url)
for line in traceback.format_exc().splitlines():
self.msg(self.config.core.owner, self.config.core.owner,
line)
self.msg(self.config.core.owner, self.config.core.owner,
"Message data was: %s" % data, 10)
continue
for msg in messages:
for chan in self.dispatch.get(branch, set()) | \
self.dispatch.get('*', set()):
self.msg(chan, chan, "Check-in: %s" % msg)
for bug, urls in urls_for_bugs.iteritems():
self.bugzilla_queue.put((bug, urls))
def bugzilla_reporter(self):
delayed_comments = []
def get_one():
if delayed_comments:
when, bug, urls = delayed_comments[0]
if when <= time.time():
delayed_comments.pop(0)
return bug, urls, True
try:
bug, urls = self.bugzilla_queue.get(timeout=1)
return bug, urls, False
except Empty:
return None, None, None
while not self.shutting_down:
bug, urls, delayed = get_one()
if bug is None:
continue
try:
comments = '\n'.join(self.bugzilla.get_comments(bug))
except BugzillaError:
# Don't do anything on errors, such as "You are not authorized
# to access bug #xxxxx".
continue
urls_to_write = []
summary = {}
backouts = set()
for url, desc, is_backout in urls:
# Only write about a changeset if it's never been mentioned
# at all. This makes us not emit changesets that e.g. land
# on mozilla-inbound when they were mentioned when landing
# on mozilla-central.
if url[-12:] not in comments:
urls_to_write.append(url)
summary[url] = desc
if is_backout:
backouts.add(url)
if urls_to_write:
def comment():
if all(url in backouts for url in urls_to_write):
yield 'Backout:'
for url in urls_to_write:
yield url + summary[url]
else:
for url in urls_to_write:
if url in backouts:
yield '%s (backout)%s' % (url, summary[url])
else:
yield url + summary[url]
try:
fields = ('whiteboard', 'keywords')
values = self.bugzilla.get_fields(bug, fields)
# Delay comments for backouts and checkin-needed in
# whiteboard
delay_comment = (
not delayed
and (all(url in backouts for url in urls_to_write)
or 'checkin-needed' in values.get('whiteboard', ''))
)
if delay_comment:
delayed_comments.append((time.time() + 600, bug, urls))
else:
message = '\n'.join(comment())
kwargs = {}
if 'checkin-needed' in values.get('keywords', {}):
kwargs['keywords'] = {
'remove': ['checkin-needed']
}
if kwargs:
kwargs['comment'] = {'body': message}
self.bugzilla.update_bug(bug, **kwargs)
else:
self.bugzilla.post_comment(bug, message)
except:
self.msg(self.config.core.owner, self.config.core.owner,
"Failed to send comment to bug %d" % bug)
def shutdown(self):
self.shutting_down = True
self.pulse.shutdown()
self.reporter_thread.join()
self.bugzilla_thread.join()
class PulseDispatcher(object):
instance = None
def __init__(self, msg, config):
self.msg = msg
self.config = config
self.hgpushes = PulseHgPushes(config)
self.max_checkins = 10
self.shutting_down = False
self.backout_delay = 600
if (config.parser.has_option('bugzilla', 'server')
and config.parser.has_option('bugzilla', 'api_key')):
self.bugzilla = Bugzilla(config.bugzilla.server,
config.bugzilla.api_key)
if config.parser.has_option('pulse', 'max_checkins'):
self.max_checkins = config.pulse.max_checkins
if self.config.dispatch or self.config.bugzilla_branches:
self.reporter_thread = threading.Thread(
target=self.change_reporter)
self.reporter_thread.start()
if self.config.bugzilla_branches:
self.bugzilla_queue = Queue(42)
self.bugzilla_thread = threading.Thread(
target=self.bugzilla_reporter)
self.bugzilla_thread.start()
def change_reporter(self):
for push in self.hgpushes:
self.report_one_push(push)
def report_one_push(self, push):
url = urlparse.urlparse(push['pushlog'])
branch = os.path.dirname(url.path).strip('/')
channels = self.config.dispatch.get(branch)
if channels:
for msg in self.create_messages(push, self.max_checkins):
for chan in channels:
self.msg(chan, chan, "Check-in: %s" % msg)
if branch in self.config.bugzilla_branches:
for info in self.munge_for_bugzilla(push):
if branch in self.config.bugzilla_leave_open:
info.leave_open = True
self.bugzilla_queue.put(info)
@staticmethod
def create_messages(push, max_checkins=sys.maxsize, max_bugs=5):
max_bugs = max(1, max_bugs)
changesets = push['changesets']
group = ''
group_bugs = []
# Kind of gross
last_desc = changesets[-1]['desc'] if changesets else ''
merge = changesets[-1].get('is_merge') if changesets else False
group_changesets = merge or len(changesets) > max_checkins
if not merge:
for cs in changesets:
revlink = cs['revlink']
desc = cs['desc']
if group_changesets:
bugs = parse_bugs(desc)
if bugs and bugs[0] not in group_bugs:
group_bugs.append(bugs[0])
else:
author = cs['author']
yield "%s - %s - %s" % (revlink, author, desc)
if group_changesets:
group = '%s - %d changesets' % (push['pushlog'], len(changesets))
if merge:
group += ' - %s' % last_desc
if group:
if group_bugs and not merge:
group += ' (bug%s %s%s)' % (
's' if len(group_bugs) > 1 else '', ', '.join(
str(b) for b in group_bugs[:max_bugs]),
' and %d other bug%s' %
(len(group_bugs) - max_bugs,
's' if len(group_bugs) > max_bugs + 1 else '')
if len(group_bugs) > max_bugs else '')
yield group
@staticmethod
def munge_for_bugzilla(push):
info_for_bugs = {}
for cs in push['changesets']:
if cs.get('source-repo', '').startswith('https://github.com/'):
continue
bugs = parse_bugs(cs['desc'])
if bugs:
if bugs[0] not in info_for_bugs:
info_for_bugs[bugs[0]] = BugInfo(bugs[0], push['user'])
info_for_bugs[bugs[0]].add_changeset(cs)
for info in info_for_bugs.itervalues():
yield info
@staticmethod
def bugzilla_summary(cs):
yield cs['revlink']
desc = cs['desc']
matches = [
m for m in BUG_RE.finditer(desc) if int(m.group(2)) < 100000000
]
match = matches[0]
if match.start() == 0:
desc = desc[match.end():].lstrip(' \t-,.:')
else:
backout = BACKOUT_RE.match(desc)
if backout and not desc[backout.end():match.start()].strip():
desc = desc[:backout.end()] + desc[match.end():]
elif (desc[match.start() - 1] == '('
and desc[match.end():match.end() + 1] == ')'):
desc = (desc[:match.start() - 1].rstrip() + ' ' +
desc[match.end() + 1:].lstrip())
yield desc
def bugzilla_reporter(self):
delayed_comments = []
def get_one():
if delayed_comments:
when, info = delayed_comments[0]
if when <= time.time():
delayed_comments.pop(0)
return info, True
try:
info = self.bugzilla_queue.get(timeout=1)
return info, False
except Empty:
return None, None
while True:
info, delayed = get_one()
if info is None:
if self.shutting_down:
break
continue
try:
comments = '\n'.join(self.bugzilla.get_comments(info.bug))
except BugzillaError:
# Don't do anything on errors, such as "You are not authorized
# to access bug #xxxxx".
continue
cs_to_write = []
for cs_info in info:
url = cs_info['revlink']
# Only write about a changeset if it's never been mentioned
# at all. This makes us not emit changesets that e.g. land
# on mozilla-inbound when they were mentioned when landing
# on mozilla-central.
if url[-12:] not in comments:
cs_to_write.append(cs_info)
if cs_to_write:
is_backout = all(cs['is_backout'] for cs in cs_to_write)
def comment():
if is_backout:
if info.pusher:
yield 'Backout by %s:' % info.pusher
else:
yield 'Backout:'
elif info.pusher:
yield 'Pushed by %s:' % info.pusher
for cs in cs_to_write:
for line in self.bugzilla_summary(cs):
yield line
try:
fields = ('whiteboard', 'keywords')
values = self.bugzilla.get_fields(info.bug, fields)
# Delay comments for backouts and checkin-needed in
# whiteboard
delay_comment = (not delayed
and (is_backout or 'checkin-needed'
in values.get('whiteboard', '')))
if delay_comment:
delayed_comments.append(
(time.time() + self.backout_delay, info))
else:
message = '\n'.join(comment())
kwargs = {}
if 'checkin-needed' in values.get('keywords', {}):
kwargs['keywords'] = {'remove': ['checkin-needed']}
# TODO: reopen closed bugs on backout
if ('leave-open' not in values.get('keywords', {})
and not is_backout and not info.leave_open):
kwargs['status'] = 'RESOLVED'
kwargs['resolution'] = 'FIXED'
if kwargs:
kwargs['comment'] = {'body': message}
self.bugzilla.update_bug(info.bug, **kwargs)
else:
self.bugzilla.post_comment(info.bug, message)
except:
logging.getLogger('pulsebot.buzilla').error(
"Failed to send comment to bug %d", info.bug)
def shutdown(self):
self.hgpushes.shutdown()
if self.config.dispatch or self.config.bugzilla_branches:
self.reporter_thread.join()
self.shutting_down = True
if self.config.bugzilla_branches:
self.bugzilla_thread.join()
class PulseDispatcher(object):
instance = None
def __init__(self, config):
self.config = config
self.hgpushes = PulseHgPushes(config)
self.max_checkins = 10
self.shutting_down = False
self.backout_delay = 600
if config.bugzilla_server and config.bugzilla_api_key:
self.bugzilla = Bugzilla(config.bugzilla_server,
config.bugzilla_api_key)
if config.pulse_max_checkins:
self.max_checkins = config.pulse_max_checkins
if self.config.bugzilla_branches:
self.bugzilla_queue = Queue(42)
self.bugzilla_thread = threading.Thread(
target=self.bugzilla_reporter)
self.bugzilla_thread.start()
def change_reporter(self):
for push in self.hgpushes:
self.report_one_push(push)
def report_one_push(self, push):
url = urlparse(push["pushlog"])
branch = os.path.dirname(url.path).strip("/")
logger.info(f'report_one_push: {url.netloc}{url.path} {branch}')
if branch in self.config.bugzilla_branches:
for info in self.munge_for_bugzilla(push):
if branch in self.config.bugzilla_leave_open:
info.leave_open = True
self.bugzilla_queue.put(info)
@staticmethod
def create_messages(push, max_checkins=sys.maxsize, max_bugs=5):
max_bugs = max(1, max_bugs)
changesets = push["changesets"]
group = ""
group_bugs = []
# Kind of gross
last_desc = changesets[-1]["desc"] if changesets else ""
merge = changesets[-1].get("is_merge") if changesets else False
group_changesets = merge or len(changesets) > max_checkins
if not merge:
for cs in changesets:
revlink = cs["revlink"]
desc = cs["desc"]
if group_changesets:
bugs = parse_bugs(desc)
if bugs and bugs[0] not in group_bugs:
logger.info(f'bug found: {bugs[0]}')
group_bugs.append(bugs[0])
else:
author = cs["author"]
yield "%s - %s - %s" % (revlink, author, desc)
if group_changesets:
group = "%s - %d changesets" % (push["pushlog"], len(changesets))
if merge:
group += " - %s" % last_desc
if group:
if group_bugs and not merge:
group += " (bug%s %s%s)" % (
"s" if len(group_bugs) > 1 else "",
", ".join(str(b) for b in group_bugs[:max_bugs]),
" and %d other bug%s" % (
len(group_bugs) - max_bugs,
"s" if len(group_bugs) > max_bugs + 1 else "",
) if len(group_bugs) > max_bugs else "",
)
yield group
@staticmethod
def munge_for_bugzilla(push):
info_for_bugs = {}
for cs in push["changesets"]:
if cs.get("source-repo", "").startswith("https://github.com/"):
continue
bugs = parse_bugs(cs["desc"])
if bugs:
logger.info(f'bug found: {bugs[0]}')
if bugs[0] not in info_for_bugs:
info_for_bugs[bugs[0]] = BugInfo(bugs[0], push["user"])
info_for_bugs[bugs[0]].add_changeset(cs)
for info in info_for_bugs.values():
yield info
@staticmethod
def bugzilla_summary(cs):
yield cs["revlink"]
desc = cs["desc"]
matches = [
m for m in BUG_RE.finditer(desc) if int(m.group(2)) < 100000000
]
match = matches[0]
if match.start() == 0:
desc = desc[match.end():].lstrip(" \t-,.:")
else:
backout = BACKOUT_RE.match(desc)
if backout and not desc[backout.end():match.start()].strip():
desc = desc[:backout.end()] + desc[match.end():]
elif (desc[match.start() - 1] == "("
and desc[match.end():match.end() + 1] == ")"):
desc = (desc[:match.start() - 1].rstrip() + " " +
desc[match.end() + 1:].lstrip())
yield desc
def bugzilla_reporter(self):
delayed_comments = []
def get_one():
if delayed_comments:
when, info = delayed_comments[0]
if when <= time.time():
delayed_comments.pop(0)
return info, True
try:
info = self.bugzilla_queue.get(timeout=1)
return info, False
except Empty:
return None, None
while True:
info, delayed = get_one()
if info is None:
if self.shutting_down:
break
continue
try:
comments = "\n".join(self.bugzilla.get_comments(info.bug))
except BugzillaError:
# Don't do anything on errors, such as "You are not authorized
# to access bug #xxxxx".
continue
cs_to_write = []
for cs_info in info:
url = cs_info["revlink"]
# Only write about a changeset if it's never been mentioned
# at all. This makes us not emit changesets that e.g. land
# on mozilla-inbound when they were mentioned when landing
# on mozilla-central.
if url[-12:] not in comments:
cs_to_write.append(cs_info)
if cs_to_write:
is_backout = all(cs["is_backout"] for cs in cs_to_write)
def comment():
if is_backout:
if info.pusher:
yield "Backout by %s:" % info.pusher
else:
yield "Backout:"
elif info.pusher:
yield "Pushed by %s:" % info.pusher
for cs in cs_to_write:
for line in self.bugzilla_summary(cs):
yield line
try:
fields = ("whiteboard", "keywords", "status")
values = self.bugzilla.get_fields(info.bug, fields)
# Delay comments for backouts and checkin-needed in
# whiteboard
delay_comment = not delayed and (
is_backout
or "checkin-needed" in values.get("whiteboard", ""))
if delay_comment:
delayed_comments.append(
(time.time() + self.backout_delay, info))
else:
message = "\n".join(comment())
kwargs = {}
remove_keywords = [
kw
for kw in ["checkin-needed", "checkin-needed-tb"]
if kw in values.get("keywords", {})
]
if remove_keywords:
kwargs["keywords"] = {"remove": remove_keywords}
# TODO: reopen closed bugs on backout
if ("leave-open" not in values.get("keywords", {})
and not is_backout and not info.leave_open
and values.get("status", "")
not in ("VERIFIED", "CLOSED", "RESOLVED")):
kwargs["status"] = "RESOLVED"
kwargs["resolution"] = "FIXED"
if kwargs:
kwargs["comment"] = {"body": message}
self.bugzilla.update_bug(info.bug, **kwargs)
else:
self.bugzilla.post_comment(info.bug, message)
except Exception:
logger.exception(
f"Failed to send comment to bug {info.bug}")
def shutdown(self):
self.hgpushes.shutdown()
self.shutting_down = True
if self.config.bugzilla_branches:
self.bugzilla_thread.join()