def test_message_content_type_stripping(self):
incoming_email_object = self._get_email_object(
"message_with_many_multiparts.eml",
)
expected_email_object = self._get_email_object(
"message_with_many_multiparts_stripped_html.eml",
)
default_settings = utils.get_settings()
with mock.patch("django_mailbox2.utils.get_settings") as get_settings:
altered = copy.deepcopy(default_settings)
altered["strip_unallowed_mimetypes"] = True
altered["allowed_mimetypes"] = ["text/plain"]
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(incoming_email_object)
del msg._email_object # Cache flush
actual_email_object = msg.get_email_object()
self.assertEqual(
actual_email_object,
expected_email_object,
)
def _process_message(self, message):
try:
msg = Message(message_id=message["message-id"][0:255].strip())
except Exception:
msg = Message()
msg._email_object = message
settings = utils.get_settings()
if settings["store_original_message"]:
self._process_save_original_message(message, msg)
msg.mailbox = self
if "subject" in message:
msg.subject = utils.convert_header_to_unicode(
message["subject"])[0:255]
if "message-id" in message:
msg.message_id = message["message-id"][0:255].strip()
if "from" in message:
msg.from_header = utils.convert_header_to_unicode(message["from"])
if "to" in message:
msg.to_header = utils.convert_header_to_unicode(message["to"])
elif "Delivered-To" in message:
msg.to_header = utils.convert_header_to_unicode(
message["Delivered-To"])
try:
msg.save()
message = self._get_dehydrated_message(message, msg)
except django.db.utils.IntegrityError:
logger.debug("Already loaded")
already_loaded_limiter = getattr(
django_settings, "DJANGO_MAILBOX2_ALREADY_LOADED_LIMITER",
False)
if already_loaded_limiter:
check_already_loaded_count()
return None
try:
body = message.as_string()
except KeyError as exc:
# email.message.replace_header may raise 'KeyError' if the header
# 'content-transfer-encoding' is missing
logger.warning(
"Failed to parse message: %s",
exc,
)
return None
msg.set_body(body)
if message["in-reply-to"]:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message["in-reply-to"].strip())[0]
except IndexError:
pass
msg.save()
return msg
def test_message_saving_ignored(self):
message = self._get_email_object("generic_message.eml")
default_settings = utils.get_settings()
with mock.patch("django_mailbox2.utils.get_settings") as get_settings:
altered = copy.deepcopy(default_settings)
altered["store_original_message"] = False
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(message)
self.assertEqual(msg.eml, None)
def _process_save_original_message(self, message, msg):
settings = utils.get_settings()
if settings["compress_original_message"]:
with NamedTemporaryFile(suffix=".eml.gz") as fp_tmp:
with gzip.GzipFile(fileobj=fp_tmp, mode="w") as fp:
fp.write(message.as_string().encode("utf-8"))
msg.eml.save("{}.eml.gz".format(uuid.uuid4()),
File(fp_tmp),
save=False)
else:
msg.eml.save("%s.eml" % uuid.uuid4(),
ContentFile(message.as_string()),
save=False)
def test_message_saved(self):
message = self._get_email_object("generic_message.eml")
default_settings = utils.get_settings()
with mock.patch("django_mailbox2.utils.get_settings") as get_settings:
altered = copy.deepcopy(default_settings)
altered["store_original_message"] = True
get_settings.return_value = altered
msg = self.mailbox.process_incoming_message(message)
self.assertNotEqual(msg.eml, None)
self.assertTrue(msg.eml.name.endswith(".eml"))
with open(msg.eml.name, "rb") as f:
self.assertEqual(f.read(), self._get_email_as_text("generic_message.eml"))
def _rehydrate(self, msg):
new = EmailMessage()
settings = utils.get_settings()
if msg.is_multipart():
for header, value in msg.items():
new[header] = value
for part in msg.get_payload():
new.attach(self._rehydrate(part))
elif settings["attachment_interpolation_header"] in msg.keys():
try:
attachment = MessageAttachment.objects.get(
pk=msg[settings["attachment_interpolation_header"]])
for header, value in attachment.items():
new[header] = value
encoding = new["Content-Transfer-Encoding"]
if encoding and encoding.lower() == "quoted-printable":
# Cannot use `email.encoders.encode_quopri due to
# bug 14360: http://bugs.python.org/issue14360
output = BytesIO()
encode_quopri(
BytesIO(attachment.document.read()),
output,
quotetabs=True,
header=False,
)
new.set_payload(output.getvalue().decode().replace(
" ", "=20"))
del new["Content-Transfer-Encoding"]
new["Content-Transfer-Encoding"] = "quoted-printable"
else:
new.set_payload(attachment.document.read())
del new["Content-Transfer-Encoding"]
encode_base64(new)
except MessageAttachment.DoesNotExist:
new[settings[
"altered_message_header"]] = "Missing; Attachment %s not found" % (
msg[settings["attachment_interpolation_header"]])
new.set_payload("")
else:
for header, value in msg.items():
new[header] = value
new.set_payload(msg.get_payload())
return new
def setUp(self):
dm_settings = utils.get_settings()
self._ALLOWED_MIMETYPES = dm_settings["allowed_mimetypes"]
self._STRIP_UNALLOWED_MIMETYPES = dm_settings[
"strip_unallowed_mimetypes"]
self._TEXT_STORED_MIMETYPES = dm_settings["text_stored_mimetypes"]
self.mailbox = Mailbox.objects.create(from_email="*****@*****.**")
self.test_account = os.environ.get("EMAIL_ACCOUNT")
self.test_password = os.environ.get("EMAIL_PASSWORD")
self.test_smtp_server = os.environ.get("EMAIL_SMTP_SERVER")
self.test_from_email = "*****@*****.**"
self.maximum_wait_seconds = 60 * 5
settings.EMAIL_BACKEND = "django.core.mail.backends.smtp.EmailBackend"
settings.EMAIL_HOST = self.test_smtp_server
settings.EMAIL_PORT = 587
settings.EMAIL_HOST_USER = self.test_account
settings.EMAIL_HOST_PASSWORD = self.test_password
settings.EMAIL_USE_TLS = True
super().setUp()
def _get_dehydrated_message(self, msg, record):
settings = utils.get_settings()
new = EmailMessage()
if msg.is_multipart():
for header, value in msg.items():
new[header] = value
for part in msg.get_payload():
new.attach(self._get_dehydrated_message(part, record))
elif (settings["strip_unallowed_mimetypes"]
and not msg.get_content_type() in settings["allowed_mimetypes"]):
for header, value in msg.items():
new[header] = value
# Delete header, otherwise when attempting to deserialize the
# payload, it will be expecting a body for this.
del new["Content-Transfer-Encoding"]
new[settings[
"altered_message_header"]] = "Stripped; Content type %s not allowed" % (
msg.get_content_type())
new.set_payload("")
elif (msg.get_content_type() not in settings["text_stored_mimetypes"]
) or ("attachment" in msg.get("Content-Disposition", "")):
filename = None
raw_filename = msg.get_filename()
if raw_filename:
filename = utils.convert_header_to_unicode(raw_filename)
if not filename:
extension = mimetypes.guess_extension(msg.get_content_type())
else:
_, extension = os.path.splitext(filename)
if not extension:
extension = ".bin"
attachment = MessageAttachment()
attachment.document.save(
uuid.uuid4().hex + extension,
ContentFile(BytesIO(msg.get_payload(decode=True)).getvalue()),
)
attachment.message = record
for key, value in msg.items():
attachment[key] = value
attachment.save()
placeholder = EmailMessage()
placeholder[settings["attachment_interpolation_header"]] = str(
attachment.pk)
new = placeholder
else:
content_charset = msg.get_content_charset()
if not content_charset:
content_charset = "ascii"
try:
# Make sure that the payload can be properly decoded in the
# defined charset, if it can't, let's mash some things
# inside the payload :-\
msg.get_payload(decode=True).decode(content_charset)
except LookupError:
logger.warning("Unknown encoding %s; interpreting as ASCII!",
content_charset)
msg.set_payload(
msg.get_payload(decode=True).decode("ascii", "ignore"))
except ValueError:
logger.warning(
"Decoding error encountered; interpreting %s as ASCII!",
content_charset,
)
msg.set_payload(
msg.get_payload(decode=True).decode("ascii", "ignore"))
new = msg
return new