def setUp(self):
dm_settings = utils.get_settings()
self._ALLOWED_MIMETYPES = dm_settings['allowed_mimetypes']
self._STRIP_UNALLOWED_MIMETYPES = (
dm_settings['strip_unallowed_mimetypes']
)
self._TEXT_STORED_MIMETYPES = dm_settings['text_stored_mimetypes']
self.mailbox = Mailbox.objects.create(from_email='*****@*****.**')
self.test_account = os.environ.get('EMAIL_ACCOUNT')
self.test_password = os.environ.get('EMAIL_PASSWORD')
self.test_smtp_server = os.environ.get('EMAIL_SMTP_SERVER')
self.test_from_email = '*****@*****.**'
self.maximum_wait_seconds = 60 * 5
settings.EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
settings.EMAIL_HOST = self.test_smtp_server
settings.EMAIL_PORT = 587
settings.EMAIL_HOST_USER = self.test_account
settings.EMAIL_HOST_PASSWORD = self.test_password
settings.EMAIL_USE_TLS = True
super(EmailMessageTestCase, self).setUp()
def setUp(self):
dm_settings = utils.get_settings()
self._ALLOWED_MIMETYPES = dm_settings['allowed_mimetypes']
self._STRIP_UNALLOWED_MIMETYPES = (
dm_settings['strip_unallowed_mimetypes']
)
self._TEXT_STORED_MIMETYPES = dm_settings['text_stored_mimetypes']
self.mailbox = Mailbox.objects.create(from_email='*****@*****.**')
self.test_account = os.environ.get('EMAIL_ACCOUNT')
self.test_password = os.environ.get('EMAIL_PASSWORD')
self.test_smtp_server = os.environ.get('EMAIL_SMTP_SERVER')
self.test_from_email = '*****@*****.**'
self.maximum_wait_seconds = 60 * 5
settings.EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
settings.EMAIL_HOST = self.test_smtp_server
settings.EMAIL_PORT = 587
settings.EMAIL_HOST_USER = self.test_account
settings.EMAIL_HOST_PASSWORD = self.test_password
settings.EMAIL_USE_TLS = True
super(EmailMessageTestCase, self).setUp()
def process_message(self, message):
msg = Message()
settings = utils.get_settings()
if settings['store_original_message']:
msg.eml.save('%s.eml' % uuid.uuid4(),
ContentFile(message.as_string()),
save=False)
msg.mailbox = self
if 'subject' in message:
msg.subject = (utils.convert_header_to_unicode(
message['subject'])[0:255])
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To'])
msg.save()
message = self._get_dehydrated_message(message, msg)
msg.set_body(message.as_string())
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip())[0]
except IndexError:
pass
with open('/home/ubuntu/msg', 'wb') as output:
output.write(msg.get_email_object().as_string())
msg.save()
return msg
def _rehydrate(self, msg):
new = EmailMessage()
settings = utils.get_settings()
if msg.is_multipart():
for header, value in msg.items():
new[header] = value
for part in msg.get_payload():
new.attach(
self._rehydrate(part)
)
elif settings['attachment_interpolation_header'] in msg.keys():
try:
attachment = MessageAttachment.objects.get(
pk=msg[settings['attachment_interpolation_header']]
)
for header, value in attachment.items():
new[header] = value
encoding = new['Content-Transfer-Encoding']
if encoding and encoding.lower() == 'quoted-printable':
# Cannot use `email.encoders.encode_quopri due to
# bug 14360: http://bugs.python.org/issue14360
output = six.BytesIO()
encode_quopri(
six.BytesIO(
attachment.document.read()
),
output,
quotetabs=True,
header=False,
)
new.set_payload(
output.getvalue().decode().replace(' ', '=20')
)
del new['Content-Transfer-Encoding']
new['Content-Transfer-Encoding'] = 'quoted-printable'
else:
new.set_payload(
attachment.document.read()
)
del new['Content-Transfer-Encoding']
encode_base64(new)
except MessageAttachment.DoesNotExist:
new[settings['altered_message_header']] = (
'Missing; Attachment %s not found' % (
msg[settings['attachment_interpolation_header']]
)
)
new.set_payload('')
else:
for header, value in msg.items():
new[header] = value
new.set_payload(
msg.get_payload()
)
return new
def test_message_saving_ignored(self):
message = self._get_email_object('generic_message.eml')
default_settings = utils.get_settings()
with mock.patch('django_mailbox.utils.get_settings') as get_settings:
altered = copy.deepcopy(default_settings)
altered['store_original_message'] = False
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(message)
self.assertEquals(msg.eml, None)
def test_message_saving_ignored(self):
message = self._get_email_object('generic_message.eml')
default_settings = utils.get_settings()
with mock.patch('django_mailbox.utils.get_settings') as get_settings:
altered = copy.deepcopy(default_settings)
altered['store_original_message'] = False
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(message)
self.assertEquals(msg.eml, None)
def _process_save_original_message(self, message, msg):
settings = utils.get_settings()
if settings['compress_original_message']:
with NamedTemporaryFile(suffix=".eml.gz") as fp_tmp:
with gzip.GzipFile(fileobj=fp_tmp, mode="w") as fp:
fp.write(message.as_string().encode('utf-8'))
msg.eml.save("%s.eml.gz" % (uuid.uuid4(), ),
File(fp_tmp),
save=False)
else:
msg.eml.save('%s.eml' % uuid.uuid4(),
ContentFile(message.as_string()),
save=False)
def _process_message(self, message):
msg = Message()
msg._email_object = message
settings = utils.get_settings()
if settings['store_original_message']:
self._process_save_original_message(message, msg)
msg.mailbox = self
if 'subject' in message:
msg.subject = (utils.convert_header_to_unicode(
message['subject'])[0:255])
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To'])
msg.save()
message = self._get_dehydrated_message(message, msg)
try:
body = message.as_string()
except KeyError as exc:
# email.message.replace_header may raise 'KeyError' if the header
# 'content-transfer-encoding' is missing
try:
# Before we give up, let's try mailman's approach:
# https://bugs.python.org/msg308362
body = message.as_bytes(self).decode('ascii', 'replace')
except KeyError as exc:
logger.warning(
"Failed to parse message: %s",
exc,
)
return None
msg.set_body(body)
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip())[0]
except IndexError:
pass
msg.save()
return msg
def _process_save_original_message(self, message, msg):
settings = utils.get_settings()
if settings['compress_original_message']:
with NamedTemporaryFile(suffix=".eml.gz") as fp_tmp:
with gzip.GzipFile(fileobj=fp_tmp, mode="w") as fp:
fp.write(message.as_string().encode('utf-8'))
msg.eml.save(
"%s.eml.gz" % (uuid.uuid4(), ),
File(fp_tmp),
save=False
)
else:
msg.eml.save(
'%s.eml' % uuid.uuid4(),
ContentFile(message.as_string()),
save=False
)
def _process_message(self, message):
msg = Message()
msg._email_object = message
settings = utils.get_settings()
if settings['store_original_message']:
self._process_save_original_message(message, msg)
msg.mailbox = self
# Fix to accept subject emojis in utf-8
if 'subject' in message:
msg.subject = (utils.convert_header_to_unicode(
unicode(message['subject']).decode('utf-8'))[0:255])
msg.subject = repr(email.header.decode_header(
msg.subject)[0][0]).replace("'", "")
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To'])
msg.save()
message = self._get_dehydrated_message(message, msg)
try:
body = message.as_string()
except KeyError as exc:
# email.message.replace_header may raise 'KeyError' if the header
# 'content-transfer-encoding' is missing
logger.warning(
"Failed to parse message: %s",
exc,
)
return None
msg.set_body(body)
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip())[0]
except IndexError:
pass
msg.save()
return msg
def convert_header_to_unicode(header: str) -> str:
from django_mailbox import utils
if six.PY2 and isinstance(header, six.text_type):
return header
try:
return utils.convert_header_to_unicode(header)
except LookupError:
pass
import codecs
import webencodings
import email
default_charset = utils.get_settings()['default_charset']
def factory(decoder):
def _decode(value, encoding):
if isinstance(value, six.text_type):
return value
if not encoding or encoding == 'unknown-8bit':
encoding = default_charset
return decoder(value, encoding)
return _decode
for _decode in [
factory(lambda value, encoding: codecs.decode(
value, encoding, 'replace')),
factory(lambda value, encoding: webencodings.decode(
value, encoding, 'replace')[0]),
]:
try:
return ''.join([
(_decode(bytestr, encoding))
for bytestr, encoding in email.header.decode_header(header)
])
except LookupError as e:
last_error = e
raise last_error
def test_message_saved(self):
message = self._get_email_object('generic_message.eml')
default_settings = utils.get_settings()
with mock.patch('django_mailbox.utils.get_settings') as get_settings:
altered = copy.deepcopy(default_settings)
altered['store_original_message'] = True
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(message)
self.assertNotEquals(msg.eml, None)
self.assertTrue(msg.eml.name.endswith('.eml'))
with open(msg.eml.name, 'rb') as f:
self.assertEqual(f.read(),
self._get_email_as_text('generic_message.eml'))
def test_message_compressed(self):
message = self._get_email_object('generic_message.eml')
default_settings = utils.get_settings()
with mock.patch('django_mailbox.utils.get_settings') as get_settings:
altered = copy.deepcopy(default_settings)
altered['compress_original_message'] = True
altered['store_original_message'] = True
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(message)
actual_email_object = msg.get_email_object()
self.assertTrue(msg.eml.name.endswith('.eml.gz'))
with gzip.open(msg.eml.name, 'rb') as f:
self.assertEqual(f.read(),
self._get_email_as_text('generic_message.eml'))
def _process_message(self, message):
msg = Message()
msg._email_object = message
settings = utils.get_settings()
if settings['store_original_message']:
self._process_save_original_message(message, msg)
msg.mailbox = self
if 'subject' in message:
msg.subject = (
utils.convert_header_to_unicode(message['subject'])[0:255]
)
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To']
)
msg.save()
message = self._get_dehydrated_message(message, msg)
try:
body = message.as_string()
except KeyError as exc:
# email.message.replace_header may raise 'KeyError' if the header
# 'content-transfer-encoding' is missing
logger.warning("Failed to parse message: %s", exc,)
return None
msg.set_body(body)
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip()
)[0]
except IndexError:
pass
msg.save()
return msg
def get_message(self, condition=None):
settings = utils.get_settings()
message_count = len(self.server.list()[1])
for i in range(message_count):
try:
msg_contents = self.get_message_body(
self.server.retr(i + 1)[1])
message = self.get_email_from_bytes(msg_contents)
if condition and condition(message):
continue
yield message
except Exception as e:
print(e.message)
continue
if settings['delete_original_message']:
self.server.dele(i + 1)
self.server.quit()
return
def process_message(self, message):
msg = Message()
settings = utils.get_settings()
if settings['store_original_message']:
msg.eml.save(
'%s.eml' % uuid.uuid4(),
ContentFile(message.as_string()),
save=False
)
msg.mailbox = self
if 'subject' in message:
msg.subject = (
utils.convert_header_to_unicode(message['subject'])[0:255]
)
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To']
)
msg.save()
message = self._get_dehydrated_message(message, msg)
msg.set_body(message.as_string())
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip()
)[0]
except IndexError:
pass
with open('/home/ubuntu/msg', 'wb') as output:
output.write(msg.get_email_object().as_string())
msg.save()
return msg
def test_message_content_type_stripping(self):
incoming_email_object = self._get_email_object(
'message_with_many_multiparts.eml', )
expected_email_object = self._get_email_object(
'message_with_many_multiparts_stripped_html.eml', )
default_settings = utils.get_settings()
with mock.patch('django_mailbox.utils.get_settings') as get_settings:
altered = copy.deepcopy(default_settings)
altered['strip_unallowed_mimetypes'] = True
altered['allowed_mimetypes'] = ['text/plain']
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(incoming_email_object)
del msg._email_object # Cache flush
actual_email_object = msg.get_email_object()
self.assertEqual(
actual_email_object,
expected_email_object,
)
def test_message_content_type_stripping(self):
incoming_email_object = self._get_email_object(
'message_with_many_multiparts.eml',
)
expected_email_object = self._get_email_object(
'message_with_many_multiparts_stripped_html.eml',
)
default_settings = utils.get_settings()
with mock.patch('django_mailbox.utils.get_settings') as get_settings:
altered = copy.deepcopy(default_settings)
altered['strip_unallowed_mimetypes'] = True
altered['allowed_mimetypes'] = ['text/plain']
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(incoming_email_object)
actual_email_object = msg.get_email_object()
self.assertEqual(
actual_email_object,
expected_email_object,
)
def _get_dehydrated_message(self, msg, record):
settings = utils.get_settings()
new = EmailMessage()
if msg.is_multipart():
for header, value in msg.items():
new[header] = value
for part in msg.get_payload():
new.attach(
self._get_dehydrated_message(part, record)
)
elif (
settings['strip_unallowed_mimetypes']
and not msg.get_content_type() in settings['allowed_mimetypes']
):
for header, value in msg.items():
new[header] = value
# Delete header, otherwise when attempting to deserialize the
# payload, it will be expecting a body for this.
del new['Content-Transfer-Encoding']
new[settings['altered_message_header']] = (
'Stripped; Content type %s not allowed' % (
msg.get_content_type()
)
)
new.set_payload('')
elif (
(
msg.get_content_type() not in settings['text_stored_mimetypes']
) or
('attachment' in msg.get('Content-Disposition', ''))
):
filename = None
raw_filename = msg.get_filename()
if raw_filename:
filename = utils.convert_header_to_unicode(raw_filename)
if not filename:
extension = mimetypes.guess_extension(msg.get_content_type())
else:
_, extension = os.path.splitext(filename)
if not extension:
extension = '.bin'
attachment = MessageAttachment()
attachment.document.save(
uuid.uuid4().hex + extension,
ContentFile(
six.BytesIO(
msg.get_payload(decode=True)
).getvalue()
)
)
attachment.message = record
for key, value in msg.items():
attachment[key] = value
attachment.save()
placeholder = EmailMessage()
placeholder[
settings['attachment_interpolation_header']
] = str(attachment.pk)
new = placeholder
else:
content_charset = msg.get_content_charset()
if not content_charset:
content_charset = 'ascii'
try:
# Make sure that the payload can be properly decoded in the
# defined charset, if it can't, let's mash some things
# inside the payload :-\
msg.get_payload(decode=True).decode(content_charset)
except LookupError:
logger.warning(
"Unknown encoding %s; interpreting as ASCII!",
content_charset
)
msg.set_payload(
msg.get_payload(decode=True).decode(
'ascii',
'ignore'
)
)
except ValueError:
logger.warning(
"Decoding error encountered; interpreting %s as ASCII!",
content_charset
)
msg.set_payload(
msg.get_payload(decode=True).decode(
'ascii',
'ignore'
)
)
new = msg
return new
from django.conf import settings
from django.core.exceptions import ValidationError
from django.core.mail import EmailMultiAlternatives
from django.db import models, transaction
from django.utils.translation import ugettext_lazy as _
from django_mailbox import utils
from django_mailbox.models import ContentFile, Mailbox, Message, MessageAttachment
from enumfields import EnumField
from post_office import models as post_office_models
from .configuration import AuthenticationType, EncryptionType, IncomingConfiguration, OutgoingConfiguration
from .managers import EmailAccountManager
logger = logging.getLogger(__name__)
mailbox_settings = utils.get_settings()
class ProviderNotSpecified(Exception):
pass
@enum.unique
class Priority(enum.Enum):
LOW = post_office_models.PRIORITY.low
MEDIUM = post_office_models.PRIORITY.medium
HIGH = post_office_models.PRIORITY.high
NOW = post_office_models.PRIORITY.now
class ConnectionException(OSError):
