def recursive_first_plaintext(self, container, html_instead=False):
pl = container.get_payload()
if isinstance(pl, str):
# This was not a multipart, but it leaked... Give up!
return None
for p in pl:
if p.get_params() is None:
# MIME multipart/mixed, but no MIME type on the part
log.status("Found multipart/mixed in message '%s', but no MIME type on part. Trying text/plain." % self.msgid)
return self.get_payload_as_unicode(p)
if p.get_params()[0][0].lower() == 'text/plain':
# Don't include it if it looks like an attachment
if 'Content-Disposition' in p and p['Content-Disposition'].startswith('attachment'):
continue
return self.get_payload_as_unicode(p)
if html_instead and p.get_params()[0][0].lower() == 'text/html':
# Don't include it if it looks like an attachment
if 'Content-Disposition' in p and p['Content-Disposition'].startswith('attachment'):
continue
return self.get_payload_as_unicode(p)
if p.is_multipart():
b = self.recursive_first_plaintext(p, html_instead)
if b or b == '':
return b
# Yikes, nothing here! Hopefully we'll find something when
# we continue looping at a higher level.
return None
def clean_messageid(self, messageid, ignorebroken=False):
m = self.re_msgid.match(messageid)
if not m:
if ignorebroken:
log.status("Could not parse messageid '%s', ignoring it" % messageid)
return None
raise IgnorableException("Could not parse message id '%s'" % messageid)
return m.groups(1)[0].replace(' ', '')
def clean_messageid(self, messageid, ignorebroken=False):
m = self.re_msgid.match(messageid)
if not m:
if ignorebroken:
log.status("Could not parse messageid '%s', ignoring it" %
messageid)
return None
raise IgnorableException("Could not parse message id '%s'" %
messageid)
return m.groups(1)[0].replace(' ', '')
def analyze(self, date_override=None):
self.msgid = self.clean_messageid(self.decode_mime_header(self.get_mandatory('Message-ID')))
self._from = self.decode_mime_header(self.get_mandatory('From'), True)
self.to = self.decode_mime_header(self.get_optional('To'), True)
self.cc = self.decode_mime_header(self.get_optional('CC'), True)
self.subject = self.decode_mime_header(self.get_optional('Subject'))
if date_override:
self.date = self.forgiving_date_decode(date_override)
else:
self.date = self.forgiving_date_decode(self.decode_mime_header(self.get_mandatory('Date')))
# Accept times up to 4 hours in the future, for badly synced clocks
maxdate = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=4)
if self.date > maxdate:
# Date is in the future, we don't trust that. Instead, let's see if we can find
# it in the raw text of the message.
def _extract_date(d):
m = _re_received.match(d)
if m:
try:
return self.forgiving_date_decode(m.group(1).strip())
except IgnorableException:
pass
lowdate = min((x for x in map(_extract_date, self.msg.get_all('Received')) if x and x < maxdate))
if lowdate:
self.date = lowdate
# Else we're going to go with what we found
self.bodytxt = self.get_body()
self.attachments = []
self.get_attachments()
if len(self.attachments) > 0:
log.status("Found %s attachments" % len(self.attachments))
# Build an list of the message id's we are interested in
self.parents = []
# The first one is in-reply-to, if it exists
if self.get_optional('in-reply-to'):
m = self.clean_messageid(self.decode_mime_header(self.get_optional('in-reply-to')), True)
if m:
self.parents.append(m)
# Then we add all References values, in backwards order
if self.get_optional('references'):
cleaned_msgids = [self.clean_messageid(x, True) for x in reversed(self.decode_mime_header(self.get_optional('references')).split())]
# Can't do this with a simple self.parents.extend() due to broken
# mailers that add the same reference more than once. And we can't
# use a set() to make it unique, because order is very important
for m in cleaned_msgids:
if m and m not in self.parents:
self.parents.append(m)
def _get_body(self):
# This is where the magic happens - try to figure out what the body
# of this message should render as.
hasempty = False
# First see if this is a single-part message that we can just
# decode and go.
b = self.get_payload_as_unicode(self.msg)
if b:
return b
if b == '':
# We found something, but it was empty. We'll keep looking as
# there might be something better available, but make a note
# that empty exists.
hasempty = True
# Ok, it's multipart. Find the first part that is text/plain,
# and use that one. Do this recursively, since we may have something
# like:
# multipart/mixed:
# multipart/alternative:
# text/plain
# text/html
# application/octet-stream (attachment)
b = self.recursive_first_plaintext(self.msg)
if b:
return b
if b == '':
hasempty = True
# Couldn't find a plaintext. Look for the first HTML in that case.
# Fallback, but what can we do at this point...
b = self.recursive_first_plaintext(self.msg, True)
if b:
b = self.html_clean(b)
if b:
return b
if b == '' or b is None:
hasempty = True
if hasempty:
log.status('Found empty body in %s' % self.msgid)
return ''
raise IgnorableException("Don't know how to read the body from %s" %
self.msgid)
def _get_body(self):
# This is where the magic happens - try to figure out what the body
# of this message should render as.
hasempty = False
# First see if this is a single-part message that we can just
# decode and go.
b = self.get_payload_as_unicode(self.msg)
if b:
return b
if b == '':
# We found something, but it was empty. We'll keep looking as
# there might be something better available, but make a note
# that empty exists.
hasempty = True
# Ok, it's multipart. Find the first part that is text/plain,
# and use that one. Do this recursively, since we may have something
# like:
# multipart/mixed:
# multipart/alternative:
# text/plain
# text/html
# application/octet-stream (attachment)
b = self.recursive_first_plaintext(self.msg)
if b:
return b
if b == '':
hasempty = True
# Couldn't find a plaintext. Look for the first HTML in that case.
# Fallback, but what can we do at this point...
b = self.recursive_first_plaintext(self.msg, True)
if b:
b = self.html_clean(b)
if b:
return b
if b == '' or b is None:
hasempty = True
if hasempty:
log.status('Found empty body in %s' % self.msgid)
return ''
raise IgnorableException("Don't know how to read the body from %s" % self.msgid)
def recursive_first_plaintext(self, container, html_instead=False):
pl = container.get_payload()
if isinstance(pl, str):
# This was not a multipart, but it leaked... Give up!
return None
for p in pl:
if p.get_params() is None:
# MIME multipart/mixed, but no MIME type on the part
log.status(
"Found multipart/mixed in message '%s', but no MIME type on part. Trying text/plain."
% self.msgid)
return self.get_payload_as_unicode(p)
if p.get_params()[0][0].lower() == 'text/plain':
# Don't include it if it looks like an attachment
if 'Content-Disposition' in p and p[
'Content-Disposition'].startswith('attachment'):
continue
t = self.get_payload_as_unicode(p)
if t:
return t
if html_instead and p.get_params()[0][0].lower() == 'text/html':
# Don't include it if it looks like an attachment
if 'Content-Disposition' in p and p[
'Content-Disposition'].startswith('attachment'):
continue
t = self.get_payload_as_unicode(p)
if t:
return t
if p.is_multipart():
b = self.recursive_first_plaintext(p, html_instead)
if b or b == '':
return b
# Yikes, nothing here! Hopefully we'll find something when
# we continue looping at a higher level.
return None
curs.execute("SELECT listid FROM lists WHERE listname=%(list)s", {
'list': opt.list
})
r = curs.fetchall()
if len(r) != 1:
log.error("List %s not found" % opt.list)
conn.close()
sys.exit(1)
listid = r[0][0]
purges = set()
if opt.directory:
# Parse all files in directory
for x in os.listdir(opt.directory):
log.status("Parsing file %s" % x)
with open(os.path.join(opt.directory, x)) as f:
ap = ArchivesParserStorage()
ap.parse(f)
if opt.filter_msgid and not ap.is_msgid(opt.filter_msgid):
continue
try:
ap.analyze(date_override=opt.force_date)
except IgnorableException as e:
log_failed_message(listid, "directory", os.path.join(opt.directory, x), ap, e)
opstatus.failed += 1
continue
ap.store(conn, listid, opt.overwrite, opt.overwrite)
purges.update(ap.purges)
if opt.interactive:
print("Interactive mode, committing transaction")
def analyze(self, date_override=None):
self.msgid = self.clean_messageid(
self.decode_mime_header(self.get_mandatory('Message-ID')))
self._from = self.decode_mime_header(self.get_mandatory('From'), True)
self.to = self.decode_mime_header(self.get_optional('To'), True)
self.cc = self.decode_mime_header(self.get_optional('CC'), True)
self.subject = self.decode_mime_header(self.get_optional('Subject'))
if date_override:
self.date = self.forgiving_date_decode(date_override)
else:
self.date = self.forgiving_date_decode(
self.decode_mime_header(self.get_mandatory('Date')))
# Accept times up to 4 hours in the future, for badly synced clocks
maxdate = datetime.datetime.now(
datetime.timezone.utc) + datetime.timedelta(hours=4)
if self.date > maxdate:
# Date is in the future, we don't trust that. Instead, let's see if we can find
# it in the raw text of the message.
def _extract_date(d):
m = _re_received.match(d)
if m:
try:
return self.forgiving_date_decode(
m.group(1).strip())
except IgnorableException:
pass
lowdate = min(
(x
for x in map(_extract_date, self.msg.get_all('Received'))
if x and x < maxdate))
if lowdate:
self.date = lowdate
# Else we're going to go with what we found
self.bodytxt = self.get_body()
self.attachments = []
self.get_attachments()
if len(self.attachments) > 0:
log.status("Found %s attachments" % len(self.attachments))
# Build an list of the message id's we are interested in
self.parents = []
# The first one is in-reply-to, if it exists
if self.get_optional('in-reply-to'):
m = self.clean_messageid(
self.decode_mime_header(self.get_optional('in-reply-to')),
True)
if m:
self.parents.append(m)
# Then we add all References values, in backwards order
if self.get_optional('references'):
cleaned_msgids = [
self.clean_messageid(x, True) for x in reversed(
self.decode_mime_header(self.get_optional(
'references')).split())
]
# Can't do this with a simple self.parents.extend() due to broken
# mailers that add the same reference more than once. And we can't
# use a set() to make it unique, because order is very important
for m in cleaned_msgids:
if m and m not in self.parents:
self.parents.append(m)
def diff(self, conn, f, fromonlyf, oldid):
curs = conn.cursor()
# Fetch the old one so we have something to diff against
curs.execute(
"SELECT id, _from, _to, cc, subject, date, has_attachment, bodytxt FROM messages WHERE messageid=%(msgid)s",
{
'msgid': self.msgid,
})
try:
id, _from, to, cc, subject, date, has_attachment, bodytxt = curs.fetchone(
)
except TypeError as e:
f.write("---- %s ----\n" % self.msgid)
f.write("Could not re-find in archives (old id was %s): %s\n" %
(oldid, e))
f.write("\n-------------------------------\n\n")
return
if (_from.rstrip(), to.rstrip(), cc.rstrip(), subject.rstrip(),
date) != (self._from, self.to, self.cc, self.subject,
self.date):
log.status("Message %s has header changes " % self.msgid)
f.write("==== %s ====\n" % self.msgid)
for fn in ['_from', 'to', 'cc', 'subject', 'date']:
if getattr(self, fn) != eval(fn):
s = "- {0}: {1}\n".format(fn, eval(fn))
d = "+ {0}: {1}\n".format(fn, getattr(self, fn))
f.write(s)
f.write(d)
f.write("\n\n")
if bodytxt != self.bodytxt:
log.status("Message %s has body changes " % self.msgid)
tempdiff = list(
difflib.unified_diff(bodytxt.splitlines(),
self.bodytxt.splitlines(),
fromfile='old',
tofile='new',
n=0,
lineterm=''))
if (len(tempdiff) - 2) % 3 == 0:
# 3 rows to a diff, two header rows.
# Then verify that each slice of 3 contains one @@ row (header), one -From and one +>From,
# which indicates the only change is in the From.
ok = True
tempdiff = tempdiff[2:]
while tempdiff:
a, b, c = (tempdiff.pop(0), tempdiff.pop(0),
tempdiff.pop(0))
if not (a.startswith('@@ ') and b.startswith('-From ')
and c.startswith('+>From ')):
ok = False
break
if ok:
fromonlyf.write("%s\n" % self.msgid)
return
# Generate a nicer diff
d = list(
difflib.unified_diff(bodytxt.splitlines(),
self.bodytxt.splitlines(),
fromfile='old',
tofile='new',
n=0,
lineterm=''))
if len(d) > 0:
f.write("---- %s ----\n" % self.msgid)
f.write("\n".join(d))
f.write("\n\n")
else:
log.status("Message %s unchanged." % self.msgid)
def store(self, conn, listid, overwrite=False, overwrite_raw=False):
curs = conn.cursor()
# Potentially add the information that there exists a mail for
# this month. We do that this early since we're always going to
# make the check anyway, and this keeps the code in one place..
if not overwrite:
curs.execute(
"INSERT INTO list_months (listid, year, month) SELECT %(listid)s, %(year)s, %(month)s WHERE NOT EXISTS (SELECT listid FROM list_months WHERE listid=%(listid)s AND year=%(year)s AND month=%(month)s)",
{
'listid': listid,
'year': self.date.year,
'month': self.date.month,
})
curs.execute(
"SELECT threadid, EXISTS(SELECT threadid FROM list_threads lt WHERE lt.listid=%(listid)s AND lt.threadid=m.threadid), id FROM messages m WHERE m.messageid=%(messageid)s",
{
'messageid': self.msgid,
'listid': listid,
})
r = curs.fetchall()
if len(r) > 0:
# Has to be 1 row, since we have a unique index on id
if not r[0][1] and not overwrite:
log.status("Tagging message %s with list %s" %
(self.msgid, listid))
curs.execute(
"INSERT INTO list_threads (threadid, listid) VALUES (%(threadid)s, %(listid)s)",
{
'threadid': r[0][0],
'listid': listid,
})
opstatus.tagged += 1
self.purge_list(listid, self.date.year, self.date.month)
self.purge_thread(r[0][0])
else:
opstatus.dupes += 1
if overwrite:
pk = r[0][2]
self.purge_thread(r[0][0])
if overwrite_raw:
# For full overwrite, we also update the raw text of the message. This is an
# uncommon enough operation that we'll just do it as a separate command.
log.status("Full raw overwrite of %s" % self.msgid)
curs.execute(
"UPDATE messages SET rawtxt=%(raw)s WHERE id=%(id)s", {
'raw': bytearray(self.rawtxt),
'id': pk,
})
# Overwrite an existing message. We do not attempt to
# "re-thread" a message, we just update the contents. We
# do remove all attachments and rewrite them. Of course, we
# don't change the messageid (since it's our primary
# identifyer), and we don't update the raw text of the message.
# (since we are expected to have used that raw text to do
# the re-parsing initially)
# We update bodytext as a separate step so as not to rewrite
# the TOAST table unnecessarily...
curs.execute(
"UPDATE messages SET bodytxt=%(bodytxt)s WHERE id=%(id)s AND NOT (bodytxt=%(bodytxt)s) RETURNING id",
{
'id': pk,
'bodytxt': self.bodytxt,
})
rc = curs.rowcount
curs.execute(
"UPDATE messages SET _from=%(from)s, _to=%(to)s, cc=%(cc)s, subject=%(subject)s, date=%(date)s, has_attachment=%(has_attachment)s WHERE id=%(id)s AND NOT (_from=%(from)s AND _to=%(to)s AND cc=%(cc)s AND subject=%(subject)s AND date=%(date)s AND has_attachment=%(has_attachment)s) RETURNING id",
{
'id': pk,
'from': self._from,
'to': self.to or '',
'cc': self.cc or '',
'subject': self.subject or '',
'date': self.date,
'has_attachment': len(self.attachments) > 0,
})
rc += curs.rowcount
if rc == 0:
log.status("Message %s unchanged" % self.msgid)
return False
curs.execute(
"DELETE FROM attachments WHERE message=%(message)s", {
'message': pk,
})
if len(self.attachments):
curs.executemany(
"INSERT INTO attachments (message, filename, contenttype, attachment) VALUES (%(message)s, %(filename)s, %(contenttype)s, %(attachment)s)",
[{
'message': pk,
'filename': a[0] or 'unknown_filename',
'contenttype': a[1],
'attachment': bytearray(a[2]),
} for a in self.attachments])
opstatus.overwritten += 1
log.status("Message %s overwritten" % self.msgid)
else:
log.status("Message %s already stored" % self.msgid)
return True
if overwrite:
raise Exception(
"Attempt to overwrite message (%s) that doesn't exist on list %s!"
% (self.msgid, listid))
# Always purge the primary list for this thread
self.purge_list(listid, self.date.year, self.date.month)
# Resolve own thread
curs.execute(
"SELECT id, messageid, threadid FROM messages WHERE messageid=ANY(%(parents)s)",
{
'parents': self.parents,
})
all_parents = curs.fetchall()
if len(all_parents):
# At least one of the parents exist. Now try to figure out which one
best_parent = len(self.parents) + 1
best_threadid = -1
best_parentid = None
for i in range(0, len(all_parents)):
for j in range(0, len(self.parents)):
if self.parents[j] == all_parents[i][1]:
# This messageid found. Better than the last one?
if j < best_parent:
best_parent = j
best_parentid = all_parents[i][0]
best_threadid = all_parents[i][2]
if best_threadid == -1:
raise Exception(
"Message %s, resolve failed in a way it shouldn't :P" %
selg.msgid)
self.parentid = best_parentid
self.threadid = best_threadid
# Slice away all matches that are worse than the one we wanted
self.parents = self.parents[:best_parent]
log.status(
"Message %s resolved to existing thread %s, waiting for %s better messages"
% (self.msgid, self.threadid, len(self.parents)))
else:
# No parent exist. But don't create the threadid just yet, since
# it's possible that we're somebody elses parent!
self.parentid = None
self.threadid = None
# Now see if we are somebody elses *parent*...
curs.execute(
"SELECT message, priority, threadid FROM unresolved_messages INNER JOIN messages ON messages.id=unresolved_messages.message WHERE unresolved_messages.msgid=%(msgid)s ORDER BY threadid",
{
'msgid': self.msgid,
})
childrows = curs.fetchall()
if len(childrows):
# We are some already existing message's parent (meaning the
# messages arrived out of order)
# In the best case, the threadid is the same for all threads.
# But it might be different if this it the "glue message" that's
# holding other threads together.
if self.threadid:
# Already have a threadid, means that we have a glue message
print(
"Message %s resolved to existing thread %s, while being somebodys parent"
% (self.msgid, self.threadid))
else:
print(
"Message %s did not resolve to existing thread, but is somebodys parent"
% self.msgid)
# In this case, just pick the first thread from the list and merge into that
# one.
self.threadid = childrows[0][2]
# Get a unique list (set) of all threads *except* the primary one,
# because we'll be merging into that one.
mergethreads = set([r[2] for r in childrows
]).difference(set((self.threadid, )))
if len(mergethreads):
# We have one or more merge threads
log.status("Merging threads %s into thread %s" %
(",".join(str(s)
for s in mergethreads), self.threadid))
curs.execute(
"UPDATE messages SET threadid=%(threadid)s WHERE threadid=ANY(%(oldthreadids)s)",
{
'threadid': self.threadid,
'oldthreadids': list(mergethreads),
})
# Insert any lists that were tagged on the merged threads
curs.execute(
"INSERT INTO list_threads (threadid, listid) SELECT DISTINCT %(threadid)s,listid FROM list_threads lt2 WHERE lt2.threadid=ANY(%(oldthreadids)s) AND listid NOT IN (SELECT listid FROM list_threads lt3 WHERE lt3.threadid=%(threadid)s)",
{
'threadid': self.threadid,
'oldthreadids': list(mergethreads),
})
# Remove all old leftovers
curs.execute(
"DELETE FROM list_threads WHERE threadid=ANY(%(oldthreadids)s)",
{
'oldthreadids': list(mergethreads),
})
# Purge varnish records for all the threads we just removed
for t in mergethreads:
self.purge_thread(t)
# Batch all the children for repointing. We can't do the actual
# repointing until later, since we don't know our own id yet.
self.children = [r[0] for r in childrows]
log.status(
"Children set to %s with mergethreads being %s (from childrows %s and threadid %s)"
% (self.children, mergethreads, childrows, self.threadid))
# Finally, remove all the pending messages that had a higher
# priority value (meaning less important) than us
curs.executemany(
"DELETE FROM unresolved_messages WHERE message=%(msg)s AND priority >= %(prio)s",
[{
'msg': msg,
'prio': prio,
} for msg, prio, tid in childrows])
else:
self.children = []
if not self.threadid:
# No parent and no child exists - create a new threadid, just for us!
curs.execute("SELECT nextval('threadid_seq')")
self.threadid = curs.fetchall()[0][0]
log.status(
"Message %s resolved to no parent (out of %s) and no child, new thread %s"
% (self.msgid, len(self.parents), self.threadid))
else:
# We have a threadid already, so we're not a new thread. Thus,
# we need to purge the old thread
self.purge_thread(self.threadid)
# Insert a thread tag if we're on a new list
curs.execute(
"INSERT INTO list_threads (threadid, listid) SELECT %(threadid)s, %(listid)s WHERE NOT EXISTS (SELECT * FROM list_threads t2 WHERE t2.threadid=%(threadid)s AND t2.listid=%(listid)s) RETURNING threadid",
{
'threadid': self.threadid,
'listid': listid,
})
if len(curs.fetchall()):
log.status("Tagged thread %s with listid %s" %
(self.threadid, listid))
curs.execute(
"INSERT INTO messages (parentid, threadid, _from, _to, cc, subject, date, has_attachment, messageid, bodytxt, rawtxt) VALUES (%(parentid)s, %(threadid)s, %(from)s, %(to)s, %(cc)s, %(subject)s, %(date)s, %(has_attachment)s, %(messageid)s, %(bodytxt)s, %(rawtxt)s) RETURNING id",
{
'parentid': self.parentid,
'threadid': self.threadid,
'from': self._from,
'to': self.to or '',
'cc': self.cc or '',
'subject': self.subject or '',
'date': self.date,
'has_attachment': len(self.attachments) > 0,
'messageid': self.msgid,
'bodytxt': self.bodytxt,
'rawtxt': bytearray(self.rawtxt),
})
id = curs.fetchall()[0][0]
log.status("Message %s, got id %s, set thread %s, parent %s" %
(self.msgid, id, self.threadid, self.parentid))
if len(self.attachments):
# Insert attachments
curs.executemany(
"INSERT INTO attachments (message, filename, contenttype, attachment) VALUES (%(message)s, %(filename)s, %(contenttype)s, %(attachment)s)",
[{
'message': id,
'filename': a[0] or 'unknown_filename',
'contenttype': a[1],
'attachment': bytearray(a[2]),
} for a in self.attachments])
if len(self.children):
log.status("Setting %s other messages to children of %s" %
(len(self.children), self.msgid))
curs.executemany(
"UPDATE messages SET parentid=%(parent)s WHERE id=%(id)s",
[{
'parent': id,
'id': c
} for c in self.children])
if len(self.parents):
# There are remaining parents we'd rather have to get ourselves
# properly threaded - so store them in the db.
curs.executemany(
"INSERT INTO unresolved_messages (message, priority, msgid) VALUES (%(id)s, %(priority)s, %(msgid)s)",
[{
'id': id,
'priority': i,
'msgid': self.parents[i]
} for i in range(0, len(self.parents))])
opstatus.stored += 1
return True
curs.execute("SELECT listid FROM lists WHERE listname=%(list)s", {
'list': opt.list
})
r = curs.fetchall()
if len(r) != 1:
log.error("List %s not found" % opt.list)
conn.close()
sys.exit(1)
listid = r[0][0]
purges = set()
if opt.directory:
# Parse all files in directory
for x in os.listdir(opt.directory):
log.status("Parsing file %s" % x)
with open(os.path.join(opt.directory, x)) as f:
ap = ArchivesParserStorage()
ap.parse(f)
if opt.filter_msgid and not ap.is_msgid(opt.filter_msgid):
continue
try:
ap.analyze(date_override=opt.force_date)
except IgnorableException as e:
log_failed_message(listid, "directory", os.path.join(opt.directory, x), ap, e)
opstatus.failed += 1
continue
ap.store(conn, listid)
purges.update(ap.purges)
if opt.interactive:
print("Interactive mode, committing transaction")