def main(self):
self.txn.begin()
# NB: This somewhat duplicates handleMail, but there it's mixed in
# with handling a mailbox, which we're avoiding here.
if len(self.args) >= 1:
from_file = file(self.args[0], 'rb')
else:
from_file = sys.stdin
self.logger.debug("reading message from %r" % (from_file,))
raw_mail = from_file.read()
self.logger.debug("got %d bytes" % len(raw_mail))
file_alias = save_mail_to_librarian(raw_mail)
self.logger.debug("saved to librarian as %r" % (file_alias,))
parsed_mail = signed_message_from_string(raw_mail)
# Kinda kludgey way to cause sendmail to just print it.
config.sendmail_to_stdout = True
handle_one_mail(
self.logger, parsed_mail,
file_alias, file_alias.http_url,
signature_timestamp_checker=None)
self.logger.debug("mail handling complete")
self.txn.commit()
def main(self):
self.txn.begin()
# NB: This somewhat duplicates handleMail, but there it's mixed in
# with handling a mailbox, which we're avoiding here.
if len(self.args) >= 1:
from_file = file(self.args[0], 'rb')
else:
from_file = sys.stdin
self.logger.debug("reading message from %r" % (from_file, ))
raw_mail = from_file.read()
self.logger.debug("got %d bytes" % len(raw_mail))
file_alias = save_mail_to_librarian(raw_mail)
self.logger.debug("saved to librarian as %r" % (file_alias, ))
parsed_mail = signed_message_from_string(raw_mail)
# Kinda kludgey way to cause sendmail to just print it.
config.sendmail_to_stdout = True
handle_one_mail(self.logger,
parsed_mail,
file_alias,
file_alias.http_url,
signature_timestamp_checker=None)
self.logger.debug("mail handling complete")
self.txn.commit()
def handleMail(trans=transaction, signature_timestamp_checker=None):
log = logging.getLogger('process-mail')
mailbox = getUtility(IMailBox)
log.info("Opening the mail box.")
mailbox.open()
try:
for mail_id, raw_mail in mailbox.items():
log.info("Processing mail %s" % mail_id)
trans.begin()
try:
file_alias = save_mail_to_librarian(raw_mail)
# Let's save the url of the file alias, otherwise we might not
# be able to access it later if we get a DB exception.
file_alias_url = file_alias.http_url
log.debug('Uploaded mail to librarian %s' % (file_alias_url, ))
# If something goes wrong when handling the mail, the
# transaction will be aborted. Therefore we need to commit the
# transaction now, to ensure that the mail gets stored in the
# Librarian.
trans.commit()
except UploadFailed:
# Something went wrong in the Librarian. It could be that it's
# not running, but not necessarily. Log the error and skip the
# message, but don't delete it: retrying might help.
log.exception('Upload to Librarian failed')
continue
try:
mail = signed_message_from_string(raw_mail)
except email.errors.MessageError:
# If we can't parse the message, we can't send a reply back to
# the user, but logging an exception will let us investigate.
log.exception("Couldn't convert email to email.message: %s" %
(file_alias_url, ))
mailbox.delete(mail_id)
continue
try:
trans.begin()
handle_one_mail(log, mail, file_alias, file_alias_url,
signature_timestamp_checker)
trans.commit()
mailbox.delete(mail_id)
except (KeyboardInterrupt, SystemExit):
raise
except:
# This bare except is needed in order to prevent any bug
# in the email handling from causing the email interface
# to lock up. We simply log the error, then send an oops, and
# continue through the mailbox, so that it doesn't stop the
# rest of the emails from being processed.
log.exception(
"An exception was raised inside the handler:\n%s" %
(file_alias_url, ))
# Delete the troublesome email before attempting to send the
# OOPS in case something goes wrong. Retrying probably
# wouldn't work and we'd get stuck on the bad message.
mailbox.delete(mail_id)
_send_email_oops(trans, log, mail, "Unhandled exception",
file_alias_url)
finally:
log.info("Closing the mail box.")
mailbox.close()
def fromEmail(self, email_message, owner=None, filealias=None,
parsed_message=None, create_missing_persons=False,
fallback_parent=None, date_created=None, restricted=False):
"""See IMessageSet.fromEmail."""
# It does not make sense to handle Unicode strings, as email
# messages may contain chunks encoded in differing character sets.
# Passing Unicode in here indicates a bug.
if not zisinstance(email_message, str):
raise TypeError(
'email_message must be a normal string. Got: %r'
% email_message)
# Parse the raw message into an email.Message.Message instance,
# if we haven't been given one already.
if parsed_message is None:
parsed_message = email.message_from_string(email_message)
# We could easily generate a default, but a missing message-id
# almost certainly means a developer is using this method when
# they shouldn't (by creating emails by hand and passing them here),
# which is broken because they will almost certainly have Unicode
# errors.
rfc822msgid = parsed_message.get('message-id')
if not rfc822msgid:
raise InvalidEmailMessage('Missing Message-Id')
# Over-long messages are checked for at the handle_on_message level.
# If it's a restricted mail (IE: for a private bug), or it hasn't been
# uploaded, do so now.
from lp.services.mail.helpers import save_mail_to_librarian
if restricted or filealias is None:
raw_email_message = save_mail_to_librarian(
email_message, restricted=restricted)
else:
raw_email_message = filealias
# Find the message subject
subject = self._decode_header(parsed_message.get('subject', ''))
subject = subject.strip()
if owner is None:
# Try and determine the owner. We raise a NotFoundError
# if the sender does not exist, unless we were asked to
# create_missing_persons.
person_set = getUtility(IPersonSet)
from_hdr = self._decode_header(
parsed_message.get('from', '')).strip()
replyto_hdr = self._decode_header(
parsed_message.get('reply-to', '')).strip()
from_addrs = [from_hdr, replyto_hdr]
from_addrs = [parseaddr(addr) for addr in from_addrs if addr]
if len(from_addrs) == 0:
raise InvalidEmailMessage('No From: or Reply-To: header')
for from_addr in from_addrs:
owner = person_set.getByEmail(from_addr[1].lower().strip())
if owner is not None:
break
if owner is None:
if not create_missing_persons:
raise UnknownSender(from_addrs[0][1])
# autocreate a person
sendername = ensure_unicode(from_addrs[0][0].strip())
senderemail = from_addrs[0][1].lower().strip()
# XXX: Guilherme Salgado 2006-08-31 bug=62344:
# It's hard to define what rationale to use here, and to
# make things worst, it's almost impossible to provide a
# meaningful comment having only the email message.
owner = person_set.ensurePerson(
senderemail, sendername,
PersonCreationRationale.FROMEMAILMESSAGE)
if owner is None:
raise UnknownSender(senderemail)
# Get the parent of the message, if available in the db. We'll
# go through all the message's parents until we find one that's
# in the db.
parent = None
for parent_msgid in reversed(get_parent_msgids(parsed_message)):
try:
# we assume it's the first matching message
parent = self.get(parent_msgid)[0]
break
except NotFoundError:
pass
else:
parent = fallback_parent
# Figure out the date of the message.
if date_created is not None:
datecreated = date_created
else:
datecreated = utcdatetime_from_field(parsed_message['date'])
# Make sure we don't create an email with a datecreated in the
# distant past or future.
now = datetime.now(pytz.timezone('UTC'))
thedistantpast = datetime(1990, 1, 1, tzinfo=pytz.timezone('UTC'))
if datecreated < thedistantpast or datecreated > now:
datecreated = UTC_NOW
message = Message(subject=subject, owner=owner,
rfc822msgid=rfc822msgid, parent=parent,
raw=raw_email_message, datecreated=datecreated)
sequence = 1
# Don't store the preamble or epilogue -- they are only there
# to give hints to non-MIME aware clients
#
# Determine the encoding to use for non-multipart messages, and the
# preamble and epilogue of multipart messages. We default to
# iso-8859-1 as it seems fairly harmless to cope with old, broken
# mail clients (The RFCs state US-ASCII as the default character
# set).
# default_charset = (parsed_message.get_content_charset() or
# 'iso-8859-1')
#
# XXX: kiko 2005-09-23: Is default_charset only useful here?
#
# if getattr(parsed_message, 'preamble', None):
# # We strip a leading and trailing newline - the email parser
# # seems to arbitrarily add them :-/
# preamble = parsed_message.preamble.decode(
# default_charset, 'replace')
# if preamble.strip():
# if preamble[0] == '\n':
# preamble = preamble[1:]
# if preamble[-1] == '\n':
# preamble = preamble[:-1]
# MessageChunk(
# message=message, sequence=sequence, content=preamble
# )
# sequence += 1
for part in parsed_message.walk():
mime_type = part.get_content_type()
# Skip the multipart section that walk gives us. This part
# is the entire message.
if part.is_multipart():
continue
# Decode the content of this part.
content = part.get_payload(decode=True)
# Store the part as a MessageChunk
#
# We want only the content type text/plain as "main content".
# Exceptions to this rule:
# - if the content disposition header explicitly says that
# this part is an attachment, text/plain content is stored
# as a blob,
# - if the content-disposition header provides a filename,
# text/plain content is stored as a blob.
content_disposition = part.get('Content-disposition', '').lower()
no_attachment = not content_disposition.startswith('attachment')
if (mime_type == 'text/plain' and no_attachment
and part.get_filename() is None):
# Get the charset for the message part. If one isn't
# specified, default to latin-1 to prevent
# UnicodeDecodeErrors.
charset = part.get_content_charset()
if charset is None or str(charset).lower() == 'x-unknown':
charset = 'latin-1'
content = self.decode(content, charset)
if content.strip():
MessageChunk(
message=message, sequence=sequence, content=content)
sequence += 1
else:
filename = part.get_filename() or 'unnamed'
# Strip off any path information.
filename = os.path.basename(filename)
# Note we use the Content-Type header instead of
# part.get_content_type() here to ensure we keep
# parameters as sent. If Content-Type is None we default
# to application/octet-stream.
if part['content-type'] is None:
content_type = 'application/octet-stream'
else:
content_type = part['content-type']
if len(content) > 0:
blob = getUtility(ILibraryFileAliasSet).create(
name=filename, size=len(content),
file=cStringIO(content), contentType=content_type,
restricted=restricted)
MessageChunk(message=message, sequence=sequence, blob=blob)
sequence += 1
# Don't store the epilogue
# if getattr(parsed_message, 'epilogue', None):
# epilogue = parsed_message.epilogue.decode(
# default_charset, 'replace')
# if epilogue.strip():
# if epilogue[0] == '\n':
# epilogue = epilogue[1:]
# if epilogue[-1] == '\n':
# epilogue = epilogue[:-1]
# MessageChunk(
# message=message, sequence=sequence, content=epilogue
# )
# XXX 2008-05-27 jamesh:
# Ensure that BugMessages get flushed in same order as they
# are created.
Store.of(message).flush()
return message
def handleMail(trans=transaction, signature_timestamp_checker=None):
log = logging.getLogger('process-mail')
mailbox = getUtility(IMailBox)
log.info("Opening the mail box.")
mailbox.open()
try:
for mail_id, raw_mail in mailbox.items():
log.info("Processing mail %s" % mail_id)
trans.begin()
try:
file_alias = save_mail_to_librarian(raw_mail)
# Let's save the url of the file alias, otherwise we might not
# be able to access it later if we get a DB exception.
file_alias_url = file_alias.http_url
log.debug('Uploaded mail to librarian %s' % (file_alias_url,))
# If something goes wrong when handling the mail, the
# transaction will be aborted. Therefore we need to commit the
# transaction now, to ensure that the mail gets stored in the
# Librarian.
trans.commit()
except UploadFailed:
# Something went wrong in the Librarian. It could be that it's
# not running, but not necessarily. Log the error and skip the
# message, but don't delete it: retrying might help.
log.exception('Upload to Librarian failed')
continue
try:
mail = signed_message_from_string(raw_mail)
except email.Errors.MessageError:
# If we can't parse the message, we can't send a reply back to
# the user, but logging an exception will let us investigate.
log.exception(
"Couldn't convert email to email.Message: %s" % (
file_alias_url, ))
mailbox.delete(mail_id)
continue
try:
trans.begin()
handle_one_mail(log, mail, file_alias, file_alias_url,
signature_timestamp_checker)
trans.commit()
mailbox.delete(mail_id)
except (KeyboardInterrupt, SystemExit):
raise
except:
# This bare except is needed in order to prevent any bug
# in the email handling from causing the email interface
# to lock up. We simply log the error, then send an oops, and
# continue through the mailbox, so that it doesn't stop the
# rest of the emails from being processed.
log.exception(
"An exception was raised inside the handler:\n%s"
% (file_alias_url,))
# Delete the troublesome email before attempting to send the
# OOPS in case something goes wrong. Retrying probably
# wouldn't work and we'd get stuck on the bad message.
mailbox.delete(mail_id)
_send_email_oops(trans, log, mail,
"Unhandled exception", file_alias_url)
finally:
log.info("Closing the mail box.")
mailbox.close()
def fromEmail(self,
email_message,
owner=None,
filealias=None,
parsed_message=None,
create_missing_persons=False,
fallback_parent=None,
date_created=None,
restricted=False):
"""See IMessageSet.fromEmail."""
# It does not make sense to handle Unicode strings, as email
# messages may contain chunks encoded in differing character sets.
# Passing Unicode in here indicates a bug.
if not zisinstance(email_message, str):
raise TypeError('email_message must be a normal string. Got: %r' %
email_message)
# Parse the raw message into an email.message.Message instance,
# if we haven't been given one already.
if parsed_message is None:
parsed_message = email.message_from_string(email_message)
# We could easily generate a default, but a missing message-id
# almost certainly means a developer is using this method when
# they shouldn't (by creating emails by hand and passing them here),
# which is broken because they will almost certainly have Unicode
# errors.
rfc822msgid = parsed_message.get('message-id')
if not rfc822msgid:
raise InvalidEmailMessage('Missing Message-Id')
# Over-long messages are checked for at the handle_on_message level.
# If it's a restricted mail (IE: for a private bug), or it hasn't been
# uploaded, do so now.
from lp.services.mail.helpers import save_mail_to_librarian
if restricted or filealias is None:
raw_email_message = save_mail_to_librarian(email_message,
restricted=restricted)
else:
raw_email_message = filealias
# Find the message subject
subject = self._decode_header(parsed_message.get('subject', ''))
subject = subject.strip()
if owner is None:
# Try and determine the owner. We raise a NotFoundError
# if the sender does not exist, unless we were asked to
# create_missing_persons.
person_set = getUtility(IPersonSet)
from_hdr = self._decode_header(parsed_message.get('from',
'')).strip()
replyto_hdr = self._decode_header(
parsed_message.get('reply-to', '')).strip()
from_addrs = [from_hdr, replyto_hdr]
from_addrs = [parseaddr(addr) for addr in from_addrs if addr]
if len(from_addrs) == 0:
raise InvalidEmailMessage('No From: or Reply-To: header')
for from_addr in from_addrs:
owner = person_set.getByEmail(from_addr[1].lower().strip())
if owner is not None:
break
if owner is None:
if not create_missing_persons:
raise UnknownSender(from_addrs[0][1])
# autocreate a person
sendername = ensure_unicode(from_addrs[0][0].strip())
senderemail = from_addrs[0][1].lower().strip()
# XXX: Guilherme Salgado 2006-08-31 bug=62344:
# It's hard to define what rationale to use here, and to
# make things worst, it's almost impossible to provide a
# meaningful comment having only the email message.
owner = person_set.ensurePerson(
senderemail, sendername,
PersonCreationRationale.FROMEMAILMESSAGE)
if owner is None:
raise UnknownSender(senderemail)
# Get the parent of the message, if available in the db. We'll
# go through all the message's parents until we find one that's
# in the db.
parent = None
for parent_msgid in reversed(get_parent_msgids(parsed_message)):
try:
# we assume it's the first matching message
parent = self.get(parent_msgid)[0]
break
except NotFoundError:
pass
else:
parent = fallback_parent
# Figure out the date of the message.
if date_created is not None:
datecreated = date_created
else:
datecreated = utcdatetime_from_field(parsed_message['date'])
# Make sure we don't create an email with a datecreated in the
# distant past or future.
now = datetime.now(pytz.timezone('UTC'))
thedistantpast = datetime(1990, 1, 1, tzinfo=pytz.timezone('UTC'))
if datecreated < thedistantpast or datecreated > now:
datecreated = UTC_NOW
message = Message(subject=subject,
owner=owner,
rfc822msgid=rfc822msgid,
parent=parent,
raw=raw_email_message,
datecreated=datecreated)
sequence = 1
# Don't store the preamble or epilogue -- they are only there
# to give hints to non-MIME aware clients
#
# Determine the encoding to use for non-multipart messages, and the
# preamble and epilogue of multipart messages. We default to
# iso-8859-1 as it seems fairly harmless to cope with old, broken
# mail clients (The RFCs state US-ASCII as the default character
# set).
# default_charset = (parsed_message.get_content_charset() or
# 'iso-8859-1')
#
# XXX: kiko 2005-09-23: Is default_charset only useful here?
#
# if getattr(parsed_message, 'preamble', None):
# # We strip a leading and trailing newline - the email parser
# # seems to arbitrarily add them :-/
# preamble = parsed_message.preamble.decode(
# default_charset, 'replace')
# if preamble.strip():
# if preamble[0] == '\n':
# preamble = preamble[1:]
# if preamble[-1] == '\n':
# preamble = preamble[:-1]
# MessageChunk(
# message=message, sequence=sequence, content=preamble
# )
# sequence += 1
for part in parsed_message.walk():
mime_type = part.get_content_type()
# Skip the multipart section that walk gives us. This part
# is the entire message.
if part.is_multipart():
continue
# Decode the content of this part.
content = part.get_payload(decode=True)
# Store the part as a MessageChunk
#
# We want only the content type text/plain as "main content".
# Exceptions to this rule:
# - if the content disposition header explicitly says that
# this part is an attachment, text/plain content is stored
# as a blob,
# - if the content-disposition header provides a filename,
# text/plain content is stored as a blob.
content_disposition = part.get('Content-disposition', '').lower()
no_attachment = not content_disposition.startswith('attachment')
if (mime_type == 'text/plain' and no_attachment
and part.get_filename() is None):
# Get the charset for the message part. If one isn't
# specified, default to latin-1 to prevent
# UnicodeDecodeErrors.
charset = part.get_content_charset()
if charset is None or str(charset).lower() == 'x-unknown':
charset = 'latin-1'
content = self.decode(content, charset)
if content.strip():
MessageChunk(message=message,
sequence=sequence,
content=content)
sequence += 1
else:
filename = part.get_filename() or 'unnamed'
# Strip off any path information.
filename = os.path.basename(filename)
# Note we use the Content-Type header instead of
# part.get_content_type() here to ensure we keep
# parameters as sent. If Content-Type is None we default
# to application/octet-stream.
if part['content-type'] is None:
content_type = 'application/octet-stream'
else:
content_type = part['content-type']
if len(content) > 0:
blob = getUtility(ILibraryFileAliasSet).create(
name=filename,
size=len(content),
file=cStringIO(content),
contentType=content_type,
restricted=restricted)
MessageChunk(message=message, sequence=sequence, blob=blob)
sequence += 1
# Don't store the epilogue
# if getattr(parsed_message, 'epilogue', None):
# epilogue = parsed_message.epilogue.decode(
# default_charset, 'replace')
# if epilogue.strip():
# if epilogue[0] == '\n':
# epilogue = epilogue[1:]
# if epilogue[-1] == '\n':
# epilogue = epilogue[:-1]
# MessageChunk(
# message=message, sequence=sequence, content=epilogue
# )
# XXX 2008-05-27 jamesh:
# Ensure that BugMessages get flushed in same order as they
# are created.
Store.of(message).flush()
return message
