def main(self): self.txn.begin() # NB: This somewhat duplicates handleMail, but there it's mixed in # with handling a mailbox, which we're avoiding here. if len(self.args) >= 1: from_file = file(self.args[0], 'rb') else: from_file = sys.stdin self.logger.debug("reading message from %r" % (from_file,)) raw_mail = from_file.read() self.logger.debug("got %d bytes" % len(raw_mail)) file_alias = save_mail_to_librarian(raw_mail) self.logger.debug("saved to librarian as %r" % (file_alias,)) parsed_mail = signed_message_from_string(raw_mail) # Kinda kludgey way to cause sendmail to just print it. config.sendmail_to_stdout = True handle_one_mail( self.logger, parsed_mail, file_alias, file_alias.http_url, signature_timestamp_checker=None) self.logger.debug("mail handling complete") self.txn.commit()
def main(self): self.txn.begin() # NB: This somewhat duplicates handleMail, but there it's mixed in # with handling a mailbox, which we're avoiding here. if len(self.args) >= 1: from_file = file(self.args[0], 'rb') else: from_file = sys.stdin self.logger.debug("reading message from %r" % (from_file, )) raw_mail = from_file.read() self.logger.debug("got %d bytes" % len(raw_mail)) file_alias = save_mail_to_librarian(raw_mail) self.logger.debug("saved to librarian as %r" % (file_alias, )) parsed_mail = signed_message_from_string(raw_mail) # Kinda kludgey way to cause sendmail to just print it. config.sendmail_to_stdout = True handle_one_mail(self.logger, parsed_mail, file_alias, file_alias.http_url, signature_timestamp_checker=None) self.logger.debug("mail handling complete") self.txn.commit()
def handleMail(trans=transaction, signature_timestamp_checker=None): log = logging.getLogger('process-mail') mailbox = getUtility(IMailBox) log.info("Opening the mail box.") mailbox.open() try: for mail_id, raw_mail in mailbox.items(): log.info("Processing mail %s" % mail_id) trans.begin() try: file_alias = save_mail_to_librarian(raw_mail) # Let's save the url of the file alias, otherwise we might not # be able to access it later if we get a DB exception. file_alias_url = file_alias.http_url log.debug('Uploaded mail to librarian %s' % (file_alias_url, )) # If something goes wrong when handling the mail, the # transaction will be aborted. Therefore we need to commit the # transaction now, to ensure that the mail gets stored in the # Librarian. trans.commit() except UploadFailed: # Something went wrong in the Librarian. It could be that it's # not running, but not necessarily. Log the error and skip the # message, but don't delete it: retrying might help. log.exception('Upload to Librarian failed') continue try: mail = signed_message_from_string(raw_mail) except email.errors.MessageError: # If we can't parse the message, we can't send a reply back to # the user, but logging an exception will let us investigate. log.exception("Couldn't convert email to email.message: %s" % (file_alias_url, )) mailbox.delete(mail_id) continue try: trans.begin() handle_one_mail(log, mail, file_alias, file_alias_url, signature_timestamp_checker) trans.commit() mailbox.delete(mail_id) except (KeyboardInterrupt, SystemExit): raise except: # This bare except is needed in order to prevent any bug # in the email handling from causing the email interface # to lock up. We simply log the error, then send an oops, and # continue through the mailbox, so that it doesn't stop the # rest of the emails from being processed. log.exception( "An exception was raised inside the handler:\n%s" % (file_alias_url, )) # Delete the troublesome email before attempting to send the # OOPS in case something goes wrong. Retrying probably # wouldn't work and we'd get stuck on the bad message. mailbox.delete(mail_id) _send_email_oops(trans, log, mail, "Unhandled exception", file_alias_url) finally: log.info("Closing the mail box.") mailbox.close()
def fromEmail(self, email_message, owner=None, filealias=None, parsed_message=None, create_missing_persons=False, fallback_parent=None, date_created=None, restricted=False): """See IMessageSet.fromEmail.""" # It does not make sense to handle Unicode strings, as email # messages may contain chunks encoded in differing character sets. # Passing Unicode in here indicates a bug. if not zisinstance(email_message, str): raise TypeError( 'email_message must be a normal string. Got: %r' % email_message) # Parse the raw message into an email.Message.Message instance, # if we haven't been given one already. if parsed_message is None: parsed_message = email.message_from_string(email_message) # We could easily generate a default, but a missing message-id # almost certainly means a developer is using this method when # they shouldn't (by creating emails by hand and passing them here), # which is broken because they will almost certainly have Unicode # errors. rfc822msgid = parsed_message.get('message-id') if not rfc822msgid: raise InvalidEmailMessage('Missing Message-Id') # Over-long messages are checked for at the handle_on_message level. # If it's a restricted mail (IE: for a private bug), or it hasn't been # uploaded, do so now. from lp.services.mail.helpers import save_mail_to_librarian if restricted or filealias is None: raw_email_message = save_mail_to_librarian( email_message, restricted=restricted) else: raw_email_message = filealias # Find the message subject subject = self._decode_header(parsed_message.get('subject', '')) subject = subject.strip() if owner is None: # Try and determine the owner. We raise a NotFoundError # if the sender does not exist, unless we were asked to # create_missing_persons. person_set = getUtility(IPersonSet) from_hdr = self._decode_header( parsed_message.get('from', '')).strip() replyto_hdr = self._decode_header( parsed_message.get('reply-to', '')).strip() from_addrs = [from_hdr, replyto_hdr] from_addrs = [parseaddr(addr) for addr in from_addrs if addr] if len(from_addrs) == 0: raise InvalidEmailMessage('No From: or Reply-To: header') for from_addr in from_addrs: owner = person_set.getByEmail(from_addr[1].lower().strip()) if owner is not None: break if owner is None: if not create_missing_persons: raise UnknownSender(from_addrs[0][1]) # autocreate a person sendername = ensure_unicode(from_addrs[0][0].strip()) senderemail = from_addrs[0][1].lower().strip() # XXX: Guilherme Salgado 2006-08-31 bug=62344: # It's hard to define what rationale to use here, and to # make things worst, it's almost impossible to provide a # meaningful comment having only the email message. owner = person_set.ensurePerson( senderemail, sendername, PersonCreationRationale.FROMEMAILMESSAGE) if owner is None: raise UnknownSender(senderemail) # Get the parent of the message, if available in the db. We'll # go through all the message's parents until we find one that's # in the db. parent = None for parent_msgid in reversed(get_parent_msgids(parsed_message)): try: # we assume it's the first matching message parent = self.get(parent_msgid)[0] break except NotFoundError: pass else: parent = fallback_parent # Figure out the date of the message. if date_created is not None: datecreated = date_created else: datecreated = utcdatetime_from_field(parsed_message['date']) # Make sure we don't create an email with a datecreated in the # distant past or future. now = datetime.now(pytz.timezone('UTC')) thedistantpast = datetime(1990, 1, 1, tzinfo=pytz.timezone('UTC')) if datecreated < thedistantpast or datecreated > now: datecreated = UTC_NOW message = Message(subject=subject, owner=owner, rfc822msgid=rfc822msgid, parent=parent, raw=raw_email_message, datecreated=datecreated) sequence = 1 # Don't store the preamble or epilogue -- they are only there # to give hints to non-MIME aware clients # # Determine the encoding to use for non-multipart messages, and the # preamble and epilogue of multipart messages. We default to # iso-8859-1 as it seems fairly harmless to cope with old, broken # mail clients (The RFCs state US-ASCII as the default character # set). # default_charset = (parsed_message.get_content_charset() or # 'iso-8859-1') # # XXX: kiko 2005-09-23: Is default_charset only useful here? # # if getattr(parsed_message, 'preamble', None): # # We strip a leading and trailing newline - the email parser # # seems to arbitrarily add them :-/ # preamble = parsed_message.preamble.decode( # default_charset, 'replace') # if preamble.strip(): # if preamble[0] == '\n': # preamble = preamble[1:] # if preamble[-1] == '\n': # preamble = preamble[:-1] # MessageChunk( # message=message, sequence=sequence, content=preamble # ) # sequence += 1 for part in parsed_message.walk(): mime_type = part.get_content_type() # Skip the multipart section that walk gives us. This part # is the entire message. if part.is_multipart(): continue # Decode the content of this part. content = part.get_payload(decode=True) # Store the part as a MessageChunk # # We want only the content type text/plain as "main content". # Exceptions to this rule: # - if the content disposition header explicitly says that # this part is an attachment, text/plain content is stored # as a blob, # - if the content-disposition header provides a filename, # text/plain content is stored as a blob. content_disposition = part.get('Content-disposition', '').lower() no_attachment = not content_disposition.startswith('attachment') if (mime_type == 'text/plain' and no_attachment and part.get_filename() is None): # Get the charset for the message part. If one isn't # specified, default to latin-1 to prevent # UnicodeDecodeErrors. charset = part.get_content_charset() if charset is None or str(charset).lower() == 'x-unknown': charset = 'latin-1' content = self.decode(content, charset) if content.strip(): MessageChunk( message=message, sequence=sequence, content=content) sequence += 1 else: filename = part.get_filename() or 'unnamed' # Strip off any path information. filename = os.path.basename(filename) # Note we use the Content-Type header instead of # part.get_content_type() here to ensure we keep # parameters as sent. If Content-Type is None we default # to application/octet-stream. if part['content-type'] is None: content_type = 'application/octet-stream' else: content_type = part['content-type'] if len(content) > 0: blob = getUtility(ILibraryFileAliasSet).create( name=filename, size=len(content), file=cStringIO(content), contentType=content_type, restricted=restricted) MessageChunk(message=message, sequence=sequence, blob=blob) sequence += 1 # Don't store the epilogue # if getattr(parsed_message, 'epilogue', None): # epilogue = parsed_message.epilogue.decode( # default_charset, 'replace') # if epilogue.strip(): # if epilogue[0] == '\n': # epilogue = epilogue[1:] # if epilogue[-1] == '\n': # epilogue = epilogue[:-1] # MessageChunk( # message=message, sequence=sequence, content=epilogue # ) # XXX 2008-05-27 jamesh: # Ensure that BugMessages get flushed in same order as they # are created. Store.of(message).flush() return message
def handleMail(trans=transaction, signature_timestamp_checker=None): log = logging.getLogger('process-mail') mailbox = getUtility(IMailBox) log.info("Opening the mail box.") mailbox.open() try: for mail_id, raw_mail in mailbox.items(): log.info("Processing mail %s" % mail_id) trans.begin() try: file_alias = save_mail_to_librarian(raw_mail) # Let's save the url of the file alias, otherwise we might not # be able to access it later if we get a DB exception. file_alias_url = file_alias.http_url log.debug('Uploaded mail to librarian %s' % (file_alias_url,)) # If something goes wrong when handling the mail, the # transaction will be aborted. Therefore we need to commit the # transaction now, to ensure that the mail gets stored in the # Librarian. trans.commit() except UploadFailed: # Something went wrong in the Librarian. It could be that it's # not running, but not necessarily. Log the error and skip the # message, but don't delete it: retrying might help. log.exception('Upload to Librarian failed') continue try: mail = signed_message_from_string(raw_mail) except email.Errors.MessageError: # If we can't parse the message, we can't send a reply back to # the user, but logging an exception will let us investigate. log.exception( "Couldn't convert email to email.Message: %s" % ( file_alias_url, )) mailbox.delete(mail_id) continue try: trans.begin() handle_one_mail(log, mail, file_alias, file_alias_url, signature_timestamp_checker) trans.commit() mailbox.delete(mail_id) except (KeyboardInterrupt, SystemExit): raise except: # This bare except is needed in order to prevent any bug # in the email handling from causing the email interface # to lock up. We simply log the error, then send an oops, and # continue through the mailbox, so that it doesn't stop the # rest of the emails from being processed. log.exception( "An exception was raised inside the handler:\n%s" % (file_alias_url,)) # Delete the troublesome email before attempting to send the # OOPS in case something goes wrong. Retrying probably # wouldn't work and we'd get stuck on the bad message. mailbox.delete(mail_id) _send_email_oops(trans, log, mail, "Unhandled exception", file_alias_url) finally: log.info("Closing the mail box.") mailbox.close()
def fromEmail(self, email_message, owner=None, filealias=None, parsed_message=None, create_missing_persons=False, fallback_parent=None, date_created=None, restricted=False): """See IMessageSet.fromEmail.""" # It does not make sense to handle Unicode strings, as email # messages may contain chunks encoded in differing character sets. # Passing Unicode in here indicates a bug. if not zisinstance(email_message, str): raise TypeError('email_message must be a normal string. Got: %r' % email_message) # Parse the raw message into an email.message.Message instance, # if we haven't been given one already. if parsed_message is None: parsed_message = email.message_from_string(email_message) # We could easily generate a default, but a missing message-id # almost certainly means a developer is using this method when # they shouldn't (by creating emails by hand and passing them here), # which is broken because they will almost certainly have Unicode # errors. rfc822msgid = parsed_message.get('message-id') if not rfc822msgid: raise InvalidEmailMessage('Missing Message-Id') # Over-long messages are checked for at the handle_on_message level. # If it's a restricted mail (IE: for a private bug), or it hasn't been # uploaded, do so now. from lp.services.mail.helpers import save_mail_to_librarian if restricted or filealias is None: raw_email_message = save_mail_to_librarian(email_message, restricted=restricted) else: raw_email_message = filealias # Find the message subject subject = self._decode_header(parsed_message.get('subject', '')) subject = subject.strip() if owner is None: # Try and determine the owner. We raise a NotFoundError # if the sender does not exist, unless we were asked to # create_missing_persons. person_set = getUtility(IPersonSet) from_hdr = self._decode_header(parsed_message.get('from', '')).strip() replyto_hdr = self._decode_header( parsed_message.get('reply-to', '')).strip() from_addrs = [from_hdr, replyto_hdr] from_addrs = [parseaddr(addr) for addr in from_addrs if addr] if len(from_addrs) == 0: raise InvalidEmailMessage('No From: or Reply-To: header') for from_addr in from_addrs: owner = person_set.getByEmail(from_addr[1].lower().strip()) if owner is not None: break if owner is None: if not create_missing_persons: raise UnknownSender(from_addrs[0][1]) # autocreate a person sendername = ensure_unicode(from_addrs[0][0].strip()) senderemail = from_addrs[0][1].lower().strip() # XXX: Guilherme Salgado 2006-08-31 bug=62344: # It's hard to define what rationale to use here, and to # make things worst, it's almost impossible to provide a # meaningful comment having only the email message. owner = person_set.ensurePerson( senderemail, sendername, PersonCreationRationale.FROMEMAILMESSAGE) if owner is None: raise UnknownSender(senderemail) # Get the parent of the message, if available in the db. We'll # go through all the message's parents until we find one that's # in the db. parent = None for parent_msgid in reversed(get_parent_msgids(parsed_message)): try: # we assume it's the first matching message parent = self.get(parent_msgid)[0] break except NotFoundError: pass else: parent = fallback_parent # Figure out the date of the message. if date_created is not None: datecreated = date_created else: datecreated = utcdatetime_from_field(parsed_message['date']) # Make sure we don't create an email with a datecreated in the # distant past or future. now = datetime.now(pytz.timezone('UTC')) thedistantpast = datetime(1990, 1, 1, tzinfo=pytz.timezone('UTC')) if datecreated < thedistantpast or datecreated > now: datecreated = UTC_NOW message = Message(subject=subject, owner=owner, rfc822msgid=rfc822msgid, parent=parent, raw=raw_email_message, datecreated=datecreated) sequence = 1 # Don't store the preamble or epilogue -- they are only there # to give hints to non-MIME aware clients # # Determine the encoding to use for non-multipart messages, and the # preamble and epilogue of multipart messages. We default to # iso-8859-1 as it seems fairly harmless to cope with old, broken # mail clients (The RFCs state US-ASCII as the default character # set). # default_charset = (parsed_message.get_content_charset() or # 'iso-8859-1') # # XXX: kiko 2005-09-23: Is default_charset only useful here? # # if getattr(parsed_message, 'preamble', None): # # We strip a leading and trailing newline - the email parser # # seems to arbitrarily add them :-/ # preamble = parsed_message.preamble.decode( # default_charset, 'replace') # if preamble.strip(): # if preamble[0] == '\n': # preamble = preamble[1:] # if preamble[-1] == '\n': # preamble = preamble[:-1] # MessageChunk( # message=message, sequence=sequence, content=preamble # ) # sequence += 1 for part in parsed_message.walk(): mime_type = part.get_content_type() # Skip the multipart section that walk gives us. This part # is the entire message. if part.is_multipart(): continue # Decode the content of this part. content = part.get_payload(decode=True) # Store the part as a MessageChunk # # We want only the content type text/plain as "main content". # Exceptions to this rule: # - if the content disposition header explicitly says that # this part is an attachment, text/plain content is stored # as a blob, # - if the content-disposition header provides a filename, # text/plain content is stored as a blob. content_disposition = part.get('Content-disposition', '').lower() no_attachment = not content_disposition.startswith('attachment') if (mime_type == 'text/plain' and no_attachment and part.get_filename() is None): # Get the charset for the message part. If one isn't # specified, default to latin-1 to prevent # UnicodeDecodeErrors. charset = part.get_content_charset() if charset is None or str(charset).lower() == 'x-unknown': charset = 'latin-1' content = self.decode(content, charset) if content.strip(): MessageChunk(message=message, sequence=sequence, content=content) sequence += 1 else: filename = part.get_filename() or 'unnamed' # Strip off any path information. filename = os.path.basename(filename) # Note we use the Content-Type header instead of # part.get_content_type() here to ensure we keep # parameters as sent. If Content-Type is None we default # to application/octet-stream. if part['content-type'] is None: content_type = 'application/octet-stream' else: content_type = part['content-type'] if len(content) > 0: blob = getUtility(ILibraryFileAliasSet).create( name=filename, size=len(content), file=cStringIO(content), contentType=content_type, restricted=restricted) MessageChunk(message=message, sequence=sequence, blob=blob) sequence += 1 # Don't store the epilogue # if getattr(parsed_message, 'epilogue', None): # epilogue = parsed_message.epilogue.decode( # default_charset, 'replace') # if epilogue.strip(): # if epilogue[0] == '\n': # epilogue = epilogue[1:] # if epilogue[-1] == '\n': # epilogue = epilogue[:-1] # MessageChunk( # message=message, sequence=sequence, content=epilogue # ) # XXX 2008-05-27 jamesh: # Ensure that BugMessages get flushed in same order as they # are created. Store.of(message).flush() return message