def setUp(self): dm_settings = utils.get_settings() self._ALLOWED_MIMETYPES = dm_settings['allowed_mimetypes'] self._STRIP_UNALLOWED_MIMETYPES = ( dm_settings['strip_unallowed_mimetypes'] ) self._TEXT_STORED_MIMETYPES = dm_settings['text_stored_mimetypes'] self.mailbox = Mailbox.objects.create(from_email='*****@*****.**') self.test_account = os.environ.get('EMAIL_ACCOUNT') self.test_password = os.environ.get('EMAIL_PASSWORD') self.test_smtp_server = os.environ.get('EMAIL_SMTP_SERVER') self.test_from_email = '*****@*****.**' self.maximum_wait_seconds = 60 * 5 settings.EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' settings.EMAIL_HOST = self.test_smtp_server settings.EMAIL_PORT = 587 settings.EMAIL_HOST_USER = self.test_account settings.EMAIL_HOST_PASSWORD = self.test_password settings.EMAIL_USE_TLS = True super(EmailMessageTestCase, self).setUp()
def process_message(self, message): msg = Message() settings = utils.get_settings() if settings['store_original_message']: msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message.as_string()), save=False) msg.mailbox = self if 'subject' in message: msg.subject = (utils.convert_header_to_unicode( message['subject'])[0:255]) if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To']) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip())[0] except IndexError: pass with open('/home/ubuntu/msg', 'wb') as output: output.write(msg.get_email_object().as_string()) msg.save() return msg
def _rehydrate(self, msg): new = EmailMessage() settings = utils.get_settings() if msg.is_multipart(): for header, value in msg.items(): new[header] = value for part in msg.get_payload(): new.attach( self._rehydrate(part) ) elif settings['attachment_interpolation_header'] in msg.keys(): try: attachment = MessageAttachment.objects.get( pk=msg[settings['attachment_interpolation_header']] ) for header, value in attachment.items(): new[header] = value encoding = new['Content-Transfer-Encoding'] if encoding and encoding.lower() == 'quoted-printable': # Cannot use `email.encoders.encode_quopri due to # bug 14360: http://bugs.python.org/issue14360 output = six.BytesIO() encode_quopri( six.BytesIO( attachment.document.read() ), output, quotetabs=True, header=False, ) new.set_payload( output.getvalue().decode().replace(' ', '=20') ) del new['Content-Transfer-Encoding'] new['Content-Transfer-Encoding'] = 'quoted-printable' else: new.set_payload( attachment.document.read() ) del new['Content-Transfer-Encoding'] encode_base64(new) except MessageAttachment.DoesNotExist: new[settings['altered_message_header']] = ( 'Missing; Attachment %s not found' % ( msg[settings['attachment_interpolation_header']] ) ) new.set_payload('') else: for header, value in msg.items(): new[header] = value new.set_payload( msg.get_payload() ) return new
def test_message_saving_ignored(self): message = self._get_email_object('generic_message.eml') default_settings = utils.get_settings() with mock.patch('django_mailbox.utils.get_settings') as get_settings: altered = copy.deepcopy(default_settings) altered['store_original_message'] = False get_settings.return_value = altered msg = self.mailbox.process_incoming_message(message) self.assertEquals(msg.eml, None)
def _process_save_original_message(self, message, msg): settings = utils.get_settings() if settings['compress_original_message']: with NamedTemporaryFile(suffix=".eml.gz") as fp_tmp: with gzip.GzipFile(fileobj=fp_tmp, mode="w") as fp: fp.write(message.as_string().encode('utf-8')) msg.eml.save("%s.eml.gz" % (uuid.uuid4(), ), File(fp_tmp), save=False) else: msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message.as_string()), save=False)
def _process_message(self, message): msg = Message() msg._email_object = message settings = utils.get_settings() if settings['store_original_message']: self._process_save_original_message(message, msg) msg.mailbox = self if 'subject' in message: msg.subject = (utils.convert_header_to_unicode( message['subject'])[0:255]) if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To']) msg.save() message = self._get_dehydrated_message(message, msg) try: body = message.as_string() except KeyError as exc: # email.message.replace_header may raise 'KeyError' if the header # 'content-transfer-encoding' is missing try: # Before we give up, let's try mailman's approach: # https://bugs.python.org/msg308362 body = message.as_bytes(self).decode('ascii', 'replace') except KeyError as exc: logger.warning( "Failed to parse message: %s", exc, ) return None msg.set_body(body) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip())[0] except IndexError: pass msg.save() return msg
def _process_save_original_message(self, message, msg): settings = utils.get_settings() if settings['compress_original_message']: with NamedTemporaryFile(suffix=".eml.gz") as fp_tmp: with gzip.GzipFile(fileobj=fp_tmp, mode="w") as fp: fp.write(message.as_string().encode('utf-8')) msg.eml.save( "%s.eml.gz" % (uuid.uuid4(), ), File(fp_tmp), save=False ) else: msg.eml.save( '%s.eml' % uuid.uuid4(), ContentFile(message.as_string()), save=False )
def _process_message(self, message): msg = Message() msg._email_object = message settings = utils.get_settings() if settings['store_original_message']: self._process_save_original_message(message, msg) msg.mailbox = self # Fix to accept subject emojis in utf-8 if 'subject' in message: msg.subject = (utils.convert_header_to_unicode( unicode(message['subject']).decode('utf-8'))[0:255]) msg.subject = repr(email.header.decode_header( msg.subject)[0][0]).replace("'", "") if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To']) msg.save() message = self._get_dehydrated_message(message, msg) try: body = message.as_string() except KeyError as exc: # email.message.replace_header may raise 'KeyError' if the header # 'content-transfer-encoding' is missing logger.warning( "Failed to parse message: %s", exc, ) return None msg.set_body(body) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip())[0] except IndexError: pass msg.save() return msg
def convert_header_to_unicode(header: str) -> str: from django_mailbox import utils if six.PY2 and isinstance(header, six.text_type): return header try: return utils.convert_header_to_unicode(header) except LookupError: pass import codecs import webencodings import email default_charset = utils.get_settings()['default_charset'] def factory(decoder): def _decode(value, encoding): if isinstance(value, six.text_type): return value if not encoding or encoding == 'unknown-8bit': encoding = default_charset return decoder(value, encoding) return _decode for _decode in [ factory(lambda value, encoding: codecs.decode( value, encoding, 'replace')), factory(lambda value, encoding: webencodings.decode( value, encoding, 'replace')[0]), ]: try: return ''.join([ (_decode(bytestr, encoding)) for bytestr, encoding in email.header.decode_header(header) ]) except LookupError as e: last_error = e raise last_error
def test_message_saved(self): message = self._get_email_object('generic_message.eml') default_settings = utils.get_settings() with mock.patch('django_mailbox.utils.get_settings') as get_settings: altered = copy.deepcopy(default_settings) altered['store_original_message'] = True get_settings.return_value = altered msg = self.mailbox.process_incoming_message(message) self.assertNotEquals(msg.eml, None) self.assertTrue(msg.eml.name.endswith('.eml')) with open(msg.eml.name, 'rb') as f: self.assertEqual(f.read(), self._get_email_as_text('generic_message.eml'))
def test_message_compressed(self): message = self._get_email_object('generic_message.eml') default_settings = utils.get_settings() with mock.patch('django_mailbox.utils.get_settings') as get_settings: altered = copy.deepcopy(default_settings) altered['compress_original_message'] = True altered['store_original_message'] = True get_settings.return_value = altered msg = self.mailbox.process_incoming_message(message) actual_email_object = msg.get_email_object() self.assertTrue(msg.eml.name.endswith('.eml.gz')) with gzip.open(msg.eml.name, 'rb') as f: self.assertEqual(f.read(), self._get_email_as_text('generic_message.eml'))
def _process_message(self, message): msg = Message() msg._email_object = message settings = utils.get_settings() if settings['store_original_message']: self._process_save_original_message(message, msg) msg.mailbox = self if 'subject' in message: msg.subject = ( utils.convert_header_to_unicode(message['subject'])[0:255] ) if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To'] ) msg.save() message = self._get_dehydrated_message(message, msg) try: body = message.as_string() except KeyError as exc: # email.message.replace_header may raise 'KeyError' if the header # 'content-transfer-encoding' is missing logger.warning("Failed to parse message: %s", exc,) return None msg.set_body(body) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip() )[0] except IndexError: pass msg.save() return msg
def get_message(self, condition=None): settings = utils.get_settings() message_count = len(self.server.list()[1]) for i in range(message_count): try: msg_contents = self.get_message_body( self.server.retr(i + 1)[1]) message = self.get_email_from_bytes(msg_contents) if condition and condition(message): continue yield message except Exception as e: print(e.message) continue if settings['delete_original_message']: self.server.dele(i + 1) self.server.quit() return
def process_message(self, message): msg = Message() settings = utils.get_settings() if settings['store_original_message']: msg.eml.save( '%s.eml' % uuid.uuid4(), ContentFile(message.as_string()), save=False ) msg.mailbox = self if 'subject' in message: msg.subject = ( utils.convert_header_to_unicode(message['subject'])[0:255] ) if 'message-id' in message: msg.message_id = message['message-id'][0:255].strip() if 'from' in message: msg.from_header = utils.convert_header_to_unicode(message['from']) if 'to' in message: msg.to_header = utils.convert_header_to_unicode(message['to']) elif 'Delivered-To' in message: msg.to_header = utils.convert_header_to_unicode( message['Delivered-To'] ) msg.save() message = self._get_dehydrated_message(message, msg) msg.set_body(message.as_string()) if message['in-reply-to']: try: msg.in_reply_to = Message.objects.filter( message_id=message['in-reply-to'].strip() )[0] except IndexError: pass with open('/home/ubuntu/msg', 'wb') as output: output.write(msg.get_email_object().as_string()) msg.save() return msg
def test_message_content_type_stripping(self): incoming_email_object = self._get_email_object( 'message_with_many_multiparts.eml', ) expected_email_object = self._get_email_object( 'message_with_many_multiparts_stripped_html.eml', ) default_settings = utils.get_settings() with mock.patch('django_mailbox.utils.get_settings') as get_settings: altered = copy.deepcopy(default_settings) altered['strip_unallowed_mimetypes'] = True altered['allowed_mimetypes'] = ['text/plain'] get_settings.return_value = altered msg = self.mailbox.process_incoming_message(incoming_email_object) del msg._email_object # Cache flush actual_email_object = msg.get_email_object() self.assertEqual( actual_email_object, expected_email_object, )
def test_message_content_type_stripping(self): incoming_email_object = self._get_email_object( 'message_with_many_multiparts.eml', ) expected_email_object = self._get_email_object( 'message_with_many_multiparts_stripped_html.eml', ) default_settings = utils.get_settings() with mock.patch('django_mailbox.utils.get_settings') as get_settings: altered = copy.deepcopy(default_settings) altered['strip_unallowed_mimetypes'] = True altered['allowed_mimetypes'] = ['text/plain'] get_settings.return_value = altered msg = self.mailbox.process_incoming_message(incoming_email_object) actual_email_object = msg.get_email_object() self.assertEqual( actual_email_object, expected_email_object, )
def _get_dehydrated_message(self, msg, record): settings = utils.get_settings() new = EmailMessage() if msg.is_multipart(): for header, value in msg.items(): new[header] = value for part in msg.get_payload(): new.attach( self._get_dehydrated_message(part, record) ) elif ( settings['strip_unallowed_mimetypes'] and not msg.get_content_type() in settings['allowed_mimetypes'] ): for header, value in msg.items(): new[header] = value # Delete header, otherwise when attempting to deserialize the # payload, it will be expecting a body for this. del new['Content-Transfer-Encoding'] new[settings['altered_message_header']] = ( 'Stripped; Content type %s not allowed' % ( msg.get_content_type() ) ) new.set_payload('') elif ( ( msg.get_content_type() not in settings['text_stored_mimetypes'] ) or ('attachment' in msg.get('Content-Disposition', '')) ): filename = None raw_filename = msg.get_filename() if raw_filename: filename = utils.convert_header_to_unicode(raw_filename) if not filename: extension = mimetypes.guess_extension(msg.get_content_type()) else: _, extension = os.path.splitext(filename) if not extension: extension = '.bin' attachment = MessageAttachment() attachment.document.save( uuid.uuid4().hex + extension, ContentFile( six.BytesIO( msg.get_payload(decode=True) ).getvalue() ) ) attachment.message = record for key, value in msg.items(): attachment[key] = value attachment.save() placeholder = EmailMessage() placeholder[ settings['attachment_interpolation_header'] ] = str(attachment.pk) new = placeholder else: content_charset = msg.get_content_charset() if not content_charset: content_charset = 'ascii' try: # Make sure that the payload can be properly decoded in the # defined charset, if it can't, let's mash some things # inside the payload :-\ msg.get_payload(decode=True).decode(content_charset) except LookupError: logger.warning( "Unknown encoding %s; interpreting as ASCII!", content_charset ) msg.set_payload( msg.get_payload(decode=True).decode( 'ascii', 'ignore' ) ) except ValueError: logger.warning( "Decoding error encountered; interpreting %s as ASCII!", content_charset ) msg.set_payload( msg.get_payload(decode=True).decode( 'ascii', 'ignore' ) ) new = msg return new
from django.conf import settings from django.core.exceptions import ValidationError from django.core.mail import EmailMultiAlternatives from django.db import models, transaction from django.utils.translation import ugettext_lazy as _ from django_mailbox import utils from django_mailbox.models import ContentFile, Mailbox, Message, MessageAttachment from enumfields import EnumField from post_office import models as post_office_models from .configuration import AuthenticationType, EncryptionType, IncomingConfiguration, OutgoingConfiguration from .managers import EmailAccountManager logger = logging.getLogger(__name__) mailbox_settings = utils.get_settings() class ProviderNotSpecified(Exception): pass @enum.unique class Priority(enum.Enum): LOW = post_office_models.PRIORITY.low MEDIUM = post_office_models.PRIORITY.medium HIGH = post_office_models.PRIORITY.high NOW = post_office_models.PRIORITY.now class ConnectionException(OSError):