def generate_mbox(messages, full_tags):
mbox_dir = config.get_mbox_path()
mid = escape_message_id(messages[0][0].get_message_id())
tmp_mbox_path = "%s/tmp-%s" % (mbox_dir, mid)
mbox_path = "%s/mbox-%s" % (mbox_dir, mid)
mbox = mailbox.mbox(tmp_mbox_path, create=True)
for message, tags in messages:
new_payload = add_tags(message, merge_tags(full_tags, tags))
msg = message.get_message_parts()[0]
# Drop content transfer encoding so msg.set_payload() will re-encode
if "content-transfer-encoding" in msg:
del msg["content-transfer-encoding"]
# Change charset to UTF-8 to guarantee that we can represent all
# characters. Think about the case where the patch email was ASCII and
# a reviewer with a non-ASCII name replied with a Reviewed-by tag, now
# the patch can no longer be represented by ASCII.
msg.set_payload(new_payload.encode("utf-8"), "utf-8")
mbox.add(msg)
mbox.flush()
mbox.close()
os.rename(tmp_mbox_path, mbox_path)
return config.get_mbox_prefix() + ("mbox-%s" % mid)
def generate_mbox(messages, full_tags):
mbox_dir = config.get_mbox_path()
mid = escape_message_id(messages[0][0].get_message_id())
tmp_mbox_path = '%s/tmp-%s' % (mbox_dir, mid)
mbox_path = '%s/mbox-%s' % (mbox_dir, mid)
mbox = mailbox.mbox(tmp_mbox_path, create=True)
for message, tags in messages:
new_payload = add_tags(message, merge_tags(full_tags, tags))
msg = message.get_message_parts()[0]
# Drop content transfer encoding so msg.set_payload() will re-encode
if 'content-transfer-encoding' in msg:
del msg['content-transfer-encoding']
# Change charset to UTF-8 to guarantee that we can represent all
# characters. Think about the case where the patch email was ASCII and
# a reviewer with a non-ASCII name replied with a Reviewed-by tag, now
# the patch can no longer be represented by ASCII.
msg.set_payload(new_payload.encode('utf-8'), 'utf-8')
mbox.add(msg)
mbox.flush()
mbox.close()
os.rename(tmp_mbox_path, mbox_path)
return config.get_mbox_prefix() + ('mbox-%s' % mid)
def generate_mbox(messages, full_tags):
mbox_dir = config.get_mbox_path()
mid = messages[0][0].get_message_id()
tmp_mbox_path = '%s/tmp-%s' % (mbox_dir, mid)
mbox_path = '%s/mbox-%s' % (mbox_dir, mid)
mbox = mailbox.mbox(tmp_mbox_path, create=True)
for message, tags in messages:
msg = message.get_message_parts()[0]
msg.set_payload(add_tags(msg, merge_tags(full_tags, tags)))
mbox.add(msg)
mbox.flush()
mbox.close()
os.rename(tmp_mbox_path, mbox_path)
return config.get_mbox_prefix() + ('mbox-%s' % mid)
def build_patches(notmuch_dir, search_days, mail_query, trees):
db = notmuch.Database(notmuch_dir)
now = long(time())
then = now - days_to_seconds(search_days)
query = '%s (subject:PATCH or subject:PULL) %s..%s' % (mail_query, then, now)
q = notmuch.Query(db, query)
oldest = build_thread_leaders(q, then)
# A pull request may contain patches older than the posted commits. That's
# because a commit doesn't happen *after* the post like what normally
# happens with a patch but rather the post happens after the commit.
# There's no obvious way to handle this other than the hack below.
# Give some extra time for pull request commits
oldest -= (30 * 24 * 60 * 60)
commits = gitcmd.get_commits(oldest, trees)
merged_heads = gitcmd.get_merges(oldest)
mbox.setup_mboxes()
patches = []
for thread in q.search_threads():
try:
top = list(thread.get_toplevel_messages())[0]
except notmuch.errors.NullPointerError:
continue
if not message.is_patch(top):
continue
# The parser chokes on emails too often, simply report the error and
# skip the thread so that scan can complete.
try:
patch = build_patch(commits, merged_heads,
top, trees, leader=True)
except:
import traceback
import sys
sys.stderr.write('Message-Id: %s\n' % top.get_message_id())
traceback.print_exc()
continue
patch_list = [ patch ]
message_list = []
for reply in top.get_replies():
# notmuch won't let us call get_replies twice so we have to do
# everything in a single loop.
# any first level replies are replies to the top level post.
if not message.is_patch(reply):
new_tags, to, cc = message.find_extra_tags(reply, False)
patch_list[0]['tags'] = message.merge_tags(patch_list[0]['tags'], new_tags)
patch_list[0]['to'] = message.dedup(patch_list[0]['to'] + to)
patch_list[0]['cc'] = message.dedup(patch_list[0]['cc'] + cc)
if message.is_thanks_applied(reply):
patch_list[0]['applied-by'] = message.parse_email_address(message.get_header(reply, 'From'))
else:
patch = build_patch(commits, merged_heads, reply, trees)
patch_list.append(patch)
message_list.append((reply, patch['tags']))
# now we're done with replies so tags for the top patch are known
if not message.is_cover(patch_list[0]):
message_list.insert(0, (top, patch_list[0]['tags']))
series = { 'messages': patch_list,
'total_messages': thread.get_total_messages() }
if series_.is_pull_request(series):
series = fixup_pull_request(series, merged_heads)
message_list.sort(message.cmp_patch)
m = message.parse_subject(top)[1]
if len(message_list) != m:
series['broken'] = True
if (not series_.is_broken(series) and not series_.is_obsolete(series) and
not series_.any_committed(series) and not series_.is_pull_request(series) and
not series_.is_applied(series)):
if message.is_cover(series['messages'][0]):
tags = series['messages'][0]['tags']
else:
tags = {}
series['mbox_path'] = mbox.generate_mbox(message_list, tags)
series['mbox_hash'] = mbox.get_hash(series['mbox_path'])
patches.append(series)
return patches
def build_patches(notmuch_dir, search_days, mail_query, trees):
db = notmuch.Database(notmuch_dir)
now = long(time())
then = now - days_to_seconds(search_days)
query = '%s (subject:PATCH or subject:PULL) %s..%s' % (mail_query, then,
now)
q = notmuch.Query(db, query)
oldest = build_thread_leaders(q, then)
# A pull request may contain patches older than the posted commits. That's
# because a commit doesn't happen *after* the post like what normally
# happens with a patch but rather the post happens after the commit.
# There's no obvious way to handle this other than the hack below.
# Give some extra time for pull request commits
oldest -= (30 * 24 * 60 * 60)
commits = gitcmd.get_commits(oldest, trees)
merged_heads = gitcmd.get_merges(oldest)
mbox.setup_mboxes()
patches = []
for thread in q.search_threads():
try:
top = list(thread.get_toplevel_messages())[0]
except notmuch.errors.NullPointerError:
continue
if not message.is_patch(top):
continue
# The parser chokes on emails too often, simply report the error and
# skip the thread so that scan can complete.
try:
patch = build_patch(commits, merged_heads, top, trees, leader=True)
except:
import traceback
import sys
sys.stderr.write('Message-Id: %s\n' % top.get_message_id())
traceback.print_exc()
continue
patch_list = [patch]
message_list = []
for reply in top.get_replies():
# notmuch won't let us call get_replies twice so we have to do
# everything in a single loop.
# any first level replies are replies to the top level post.
if not message.is_patch(reply):
new_tags, to, cc = message.find_extra_tags(reply, False)
patch_list[0]['tags'] = message.merge_tags(
patch_list[0]['tags'], new_tags)
patch_list[0]['to'] = message.dedup(patch_list[0]['to'] + to)
patch_list[0]['cc'] = message.dedup(patch_list[0]['cc'] + cc)
if message.is_thanks_applied(reply):
patch_list[0]['applied-by'] = message.parse_email_address(
message.get_header(reply, 'From'))
else:
patch = build_patch(commits, merged_heads, reply, trees)
patch_list.append(patch)
message_list.append((reply, patch['tags']))
# now we're done with replies so tags for the top patch are known
if not message.is_cover(patch_list[0]):
message_list.insert(0, (top, patch_list[0]['tags']))
series = {
'messages': patch_list,
'total_messages': thread.get_total_messages()
}
if series_.is_pull_request(series):
series = fixup_pull_request(series, merged_heads)
message_list.sort(message.cmp_patch)
m = message.parse_subject(top)[1]
if len(message_list) != m:
series['broken'] = True
if (not series_.is_broken(series) and not series_.is_obsolete(series)
and not series_.any_committed(series)
and not series_.is_pull_request(series)
and not series_.is_applied(series)):
if message.is_cover(series['messages'][0]):
tags = series['messages'][0]['tags']
else:
tags = {}
series['mbox_path'] = mbox.generate_mbox(message_list, tags)
series['mbox_hash'] = mbox.get_hash(series['mbox_path'])
patches.append(series)
return patches
