def import_email(self, email_id):
mailbox = self.mailbox
# log.debug( "running fetch for message: "+email_id)
try:
messages = self.mailbox.fetch([email_id], [b"RFC822"])
# log.debug( repr(messages))
message_string = messages[email_id][b"RFC822"]
assert message_string
message_string = AbstractMailbox.guess_encoding(message_string)
try:
if self.source.message_ok_to_import(message_string):
(email_object, dummy, error) = self.source.parse_email(message_string)
if error:
raise ReaderError(error)
self.source.db.add(email_object)
else:
log.info("Skipped message with imap id %s (bounce or vacation message)" % (email_id))
# log.debug( "Setting self.source.last_imported_email_uid to "+email_id)
self.source.last_imported_email_uid = email_id
self.source.db.commit()
finally:
self.source = ContentSource.get(self.source.id)
except (IMAP4.abort, IMAP4.error) as e:
raise ClientError(e)
def process_email_ids(self, email_ids):
self.set_status(ReaderStatus.READING)
self.refresh_source()
log.info("Processing messages from IMAP: %d "% (len(email_ids)))
for email_id in email_ids:
self.import_email(email_id)
if self.status != ReaderStatus.READING:
break
# We imported mails, we need to re-thread
self.source.db.flush()
# Rethread emails globally (sigh)
emails = self.source.db.query(Post).filter_by(
discussion_id=self.source.discussion_id
).options(undefer(ImportedPost.imported_blob)).all()
AbstractMailbox.thread_mails(emails)
self.source.db.commit()
def check_striping_html(original, expected, fail_msg):
result = AbstractMailbox.strip_full_message_quoting_html(original)
expected = lxml.html.tostring(lxml.html.fromstring(expected), pretty_print=True)
original = lxml.html.tostring(lxml.html.fromstring(original), pretty_print=True)
result = lxml.html.tostring(lxml.html.fromstring(result), pretty_print=True)
assert result == expected, "Failed striping quotations for case %s, message was: \n------\n%s\n------\nExpected: \n------\n%s\n------\nInstead received: \n------\n%s\n------\n" % (
fail_msg,
original,
expected,
result)
def check_striping_html(original, expected, fail_msg):
result = AbstractMailbox.strip_full_message_quoting_html(original)
expected = lxml.html.tostring(lxml.html.fromstring(expected),
pretty_print=True)
original = lxml.html.tostring(lxml.html.fromstring(original),
pretty_print=True)
result = lxml.html.tostring(lxml.html.fromstring(result),
pretty_print=True)
assert result == expected, "Failed striping quotations for case %s, message was: \n------\n%s\n------\nExpected: \n------\n%s\n------\nInstead received: \n------\n%s\n------\n" % (
fail_msg, original, expected, result)
def mailbox(request, discussion, test_session):
from assembl.models import AbstractMailbox
m = AbstractMailbox(
discussion=discussion, name='mailbox')
test_session.add(m)
test_session.flush()
def fin():
print "finalizer mailbox"
test_session.delete(m)
test_session.flush()
request.addfinalizer(fin)
return m
def abstract_mailbox(request, discussion, test_session):
"""An AbstractMailbox fixture with type of abstract_mailbox"""
from assembl.models import AbstractMailbox
ps = AbstractMailbox(
discussion=discussion, name='a source', type='abstract_mailbox')
test_session.add(ps)
test_session.flush()
def fin():
print("finalizer abstract_mailbox")
test_session.delete(ps)
test_session.flush()
request.addfinalizer(fin)
return ps
def check_striping_plaintext(original, expected, fail_msg):
expected = expected.strip()
result = AbstractMailbox.strip_full_message_quoting_plaintext(original).strip()
assert result == expected, "Failed striping quotations for case %s, message was: \n------\n%s\n------\nExpected: \n------\n%s\n------\nInstead received: \n------\n%s\n------\n" % (fail_msg, original,expected,result)
def check_striping_plaintext(original, expected, fail_msg):
expected = expected.strip()
result = AbstractMailbox.strip_full_message_quoting_plaintext(
original).strip()
assert result == expected, "Failed striping quotations for case %s, message was: \n------\n%s\n------\nExpected: \n------\n%s\n------\nInstead received: \n------\n%s\n------\n" % (
fail_msg, original, expected, result)
def do_read(self):
only_new = not self.reimporting
try:
self.set_status(ReaderStatus.READING)
mailbox = self.mailbox
command = "ALL"
search_status = None
email_ids = None
if only_new and self.source.last_imported_email_uid:
command = "(UID %s:*)" % self.source.last_imported_email_uid
search_status, search_result = mailbox.uid(
'search', None, command)
if not is_ok((search_status, )):
raise ReaderError(search_result)
#log.debug( "UID searched with: "+ command + ", got result "+repr(search_status)+" and found "+repr(search_result))
email_ids = search_result[0].split()
#log.debug( email_ids)
if (only_new and search_status == 'OK' and email_ids
and email_ids[0] == self.source.last_imported_email_uid):
# Note: the email_ids[0]==self.source.last_imported_email_uid test is
# necessary beacuse according to https://tools.ietf.org/html/rfc3501
# seq-range like "3291:* includes the UID of the last message in
# the mailbox, even if that value is less than 3291."
# discard the first message, it should be the last imported email.
del email_ids[0]
else:
# Either:
# a) we don't import only new messages or
# b) the message with self.source.last_imported_email_uid hasn't been found
# (may have been deleted)
# In this case we request all messages and rely on duplicate
# detection
command = "ALL"
search_status, search_result = mailbox.uid(
'search', None, command)
if not is_ok((search_status, )):
raise ReaderError(search_result)
#log.debug( "UID searched with: "+ command + ", got result "+repr(search_status)+" and found "+repr(search_result))
email_ids = search_result[0].split()
if len(email_ids):
log.info("Processing messages from IMAP: %d " %
(len(email_ids)))
for email_id in email_ids:
self.import_email(email_id)
if self.status != ReaderStatus.READING:
break
from assembl.models import Post, AbstractMailbox
# We imported mails, we need to re-thread
self.source.db.flush()
# Rethread emails globally (sigh)
emails = self.source.db.query(Post).filter_by(
discussion_id=self.source.discussion_id).options(
undefer(ImportedPost.imported_blob)).all()
AbstractMailbox.thread_mails(emails)
self.source.db.flush()
else:
log.debug("No IMAP messages to process")
self.successful_read()
self.set_status(ReaderStatus.PAUSED)
except IMAP4.abort as e:
raise IrrecoverableError(e)
except IMAP4.error as e:
raise ClientError(e)
