def test_message_issue_82_bis(self): """ Ensure that the email object is good before and after calling _get_dehydrated_message() """ message = self._get_email_object('email_issue_82.eml') success = True # this is the code of _process_message() msg = Message() # if STORE_ORIGINAL_MESSAGE: # msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message), # save=False) msg.mailbox = self.mailbox if 'subject' in message: msg.subject = convert_header_to_unicode(message['subject'])[0:255] if 'message-id' in message: msg.message_id = message['message-id'][0:255] if 'from' in message: msg.from_header = convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = convert_header_to_unicode(message['Delivered-To']) msg.save() # here the message is ok str_msg = message.as_string() message = self.mailbox._get_dehydrated_message(message, msg) try: # here as_string raises UnicodeEncodeError str_msg = message.as_string() except: success = False msg.set_body(str_msg) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'] )[0] except IndexError: pass msg.save() self.assertEqual(True, success)
def get_filename(self): """Returns the original filename of this attachment.""" file_name = self._get_rehydrated_headers().get_filename() if isinstance(file_name, six.string_types): result = utils.convert_header_to_unicode(file_name) if result is None: return file_name return result else: return None
def _process_message(self, message): msg = Message() msg._email_object = message settings = utils.get_settings() if settings['store_original_message']: self._process_save_original_message(message, msg) msg.mailbox = self if 'subject' in message: msg.subject = (utils.convert_header_to_unicode( message['subject'])[0:255]) if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To']) msg.save() message = self._get_dehydrated_message(message, msg) try: body = message.as_string() except KeyError as exc: # email.message.replace_header may raise 'KeyError' if the header # 'content-transfer-encoding' is missing logger.warning( "Failed to parse message: %s", exc, ) return None msg.set_body(body) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip())[0] except IndexError: pass msg.save() return msg
def _process_message(self, message): msg = Message() msg._email_object = message settings = utils.get_settings() if settings['store_original_message']: self._process_save_original_message(message, msg) msg.mailbox = self if 'subject' in message: msg.subject = ( utils.convert_header_to_unicode(message['subject'])[0:255] ) if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To'] ) msg.save() message = self._get_dehydrated_message(message, msg) try: body = message.as_string() except KeyError as exc: # email.message.replace_header may raise 'KeyError' if the header # 'content-transfer-encoding' is missing logger.warning("Failed to parse message: %s", exc,) return None msg.set_body(body) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip() )[0] except IndexError: pass msg.save() return msg
def process_message(self, message): msg = Message() settings = utils.get_settings() if settings['store_original_message']: msg.eml.save( '%s.eml' % uuid.uuid4(), ContentFile(message.as_string()), save=False ) msg.mailbox = self if 'subject' in message: msg.subject = ( utils.convert_header_to_unicode(message['subject'])[0:255] ) if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To'] ) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip() )[0] except IndexError: pass with open('/home/ubuntu/msg', 'wb') as output: output.write(msg.get_email_object().as_string()) msg.save() return msg
def _process_message(self, message): msg = Message() settings = utils.get_settings() if settings['store_original_message']: msg.eml.save( '%s.eml' % uuid.uuid4(), ContentFile(message.as_string()), save=False ) msg.mailbox = self if 'subject' in message: msg.subject = ( utils.convert_header_to_unicode(message['subject'])[0:255] ) if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To'] ) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip() )[0] except IndexError: pass msg.save() return msg
def convert_header_to_unicode(header: str) -> str: from django_mailbox import utils if six.PY2 and isinstance(header, six.text_type): return header try: return utils.convert_header_to_unicode(header) except LookupError: pass import codecs import webencodings import email default_charset = utils.get_settings()['default_charset'] def factory(decoder): def _decode(value, encoding): if isinstance(value, six.text_type): return value if not encoding or encoding == 'unknown-8bit': encoding = default_charset return decoder(value, encoding) return _decode for _decode in [ factory(lambda value, encoding: codecs.decode( value, encoding, 'replace')), factory(lambda value, encoding: webencodings.decode( value, encoding, 'replace')[0]), ]: try: return ''.join([ (_decode(bytestr, encoding)) for bytestr, encoding in email.header.decode_header(header) ]) except LookupError as e: last_error = e raise last_error
def get_new_mail(self, condition=None): """Connect to this transport and fetch new messages.""" new_mail = [] connection = self.get_connection() if not connection: return for message in connection.get_message(condition): # if username find in message headers - it's outgoing message if self.username in utils.convert_header_to_unicode(message['from']): msg = self.record_outgoing_message(message) else: msg = self.process_incoming_message(message) if not msg is None: yield msg self.last_polling = now() if django.VERSION >= (1, 5): # Django 1.5 introduces update_fields self.save(update_fields=['last_polling']) else: self.save()
def _get_dehydrated_message(self, msg, record): settings = utils.get_settings() new = EmailMessage() if msg.is_multipart(): for header, value in msg.items(): new[header] = value for part in msg.get_payload(): new.attach( self._get_dehydrated_message(part, record) ) elif ( settings['strip_unallowed_mimetypes'] and not msg.get_content_type() in settings['allowed_mimetypes'] ): for header, value in msg.items(): new[header] = value # Delete header, otherwise when attempting to deserialize the # payload, it will be expecting a body for this. del new['Content-Transfer-Encoding'] new[settings['altered_message_header']] = ( 'Stripped; Content type %s not allowed' % ( msg.get_content_type() ) ) new.set_payload('') elif ( ( msg.get_content_type() not in settings['text_stored_mimetypes'] ) or ('attachment' in msg.get('Content-Disposition', '')) ): filename = None raw_filename = msg.get_filename() if raw_filename: filename = utils.convert_header_to_unicode(raw_filename) if not filename: extension = mimetypes.guess_extension(msg.get_content_type()) else: _, extension = os.path.splitext(filename) if not extension: extension = '.bin' attachment = MessageAttachment() attachment.document.save( uuid.uuid4().hex + extension, ContentFile( six.BytesIO( msg.get_payload(decode=True) ).getvalue() ) ) attachment.message = record for key, value in msg.items(): attachment[key] = value attachment.save() placeholder = EmailMessage() placeholder[ settings['attachment_interpolation_header'] ] = str(attachment.pk) new = placeholder else: content_charset = msg.get_content_charset() if not content_charset: content_charset = 'ascii' try: # Make sure that the payload can be properly decoded in the # defined charset, if it can't, let's mash some things # inside the payload :-\ msg.get_payload(decode=True).decode(content_charset) except LookupError: logger.warning( "Unknown encoding %s; interpreting as ASCII!", content_charset ) msg.set_payload( msg.get_payload(decode=True).decode( 'ascii', 'ignore' ) ) except ValueError: logger.warning( "Decoding error encountered; interpreting %s as ASCII!", content_charset ) msg.set_payload( msg.get_payload(decode=True).decode( 'ascii', 'ignore' ) ) new = msg return new
def subject(self, msg): return convert_header_to_unicode(msg.subject)
def subject(self, msg): return convert_header_to_unicode(msg.subject)