def test_message_issue_82_bis(self):
""" Ensure that the email object is good before and after
calling _get_dehydrated_message()
"""
message = self._get_email_object('email_issue_82.eml')
success = True
# this is the code of _process_message()
msg = Message()
# if STORE_ORIGINAL_MESSAGE:
# msg.eml.save('%s.eml' % uuid.uuid4(), ContentFile(message),
# save=False)
msg.mailbox = self.mailbox
if 'subject' in message:
msg.subject = convert_header_to_unicode(message['subject'])[0:255]
if 'message-id' in message:
msg.message_id = message['message-id'][0:255]
if 'from' in message:
msg.from_header = convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = convert_header_to_unicode(message['Delivered-To'])
msg.save()
# here the message is ok
str_msg = message.as_string()
message = self.mailbox._get_dehydrated_message(message, msg)
try:
# here as_string raises UnicodeEncodeError
str_msg = message.as_string()
except:
success = False
msg.set_body(str_msg)
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to']
)[0]
except IndexError:
pass
msg.save()
self.assertEqual(True, success)
def get_filename(self):
"""Returns the original filename of this attachment."""
file_name = self._get_rehydrated_headers().get_filename()
if isinstance(file_name, six.string_types):
result = utils.convert_header_to_unicode(file_name)
if result is None:
return file_name
return result
else:
return None
def _process_message(self, message):
msg = Message()
msg._email_object = message
settings = utils.get_settings()
if settings['store_original_message']:
self._process_save_original_message(message, msg)
msg.mailbox = self
if 'subject' in message:
msg.subject = (utils.convert_header_to_unicode(
message['subject'])[0:255])
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To'])
msg.save()
message = self._get_dehydrated_message(message, msg)
try:
body = message.as_string()
except KeyError as exc:
# email.message.replace_header may raise 'KeyError' if the header
# 'content-transfer-encoding' is missing
logger.warning(
"Failed to parse message: %s",
exc,
)
return None
msg.set_body(body)
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip())[0]
except IndexError:
pass
msg.save()
return msg
def _process_message(self, message):
msg = Message()
msg._email_object = message
settings = utils.get_settings()
if settings['store_original_message']:
self._process_save_original_message(message, msg)
msg.mailbox = self
if 'subject' in message:
msg.subject = (
utils.convert_header_to_unicode(message['subject'])[0:255]
)
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To']
)
msg.save()
message = self._get_dehydrated_message(message, msg)
try:
body = message.as_string()
except KeyError as exc:
# email.message.replace_header may raise 'KeyError' if the header
# 'content-transfer-encoding' is missing
logger.warning("Failed to parse message: %s", exc,)
return None
msg.set_body(body)
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip()
)[0]
except IndexError:
pass
msg.save()
return msg
def process_message(self, message):
msg = Message()
settings = utils.get_settings()
if settings['store_original_message']:
msg.eml.save(
'%s.eml' % uuid.uuid4(),
ContentFile(message.as_string()),
save=False
)
msg.mailbox = self
if 'subject' in message:
msg.subject = (
utils.convert_header_to_unicode(message['subject'])[0:255]
)
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To']
)
msg.save()
message = self._get_dehydrated_message(message, msg)
msg.set_body(message.as_string())
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip()
)[0]
except IndexError:
pass
with open('/home/ubuntu/msg', 'wb') as output:
output.write(msg.get_email_object().as_string())
msg.save()
return msg
def _process_message(self, message):
msg = Message()
settings = utils.get_settings()
if settings['store_original_message']:
msg.eml.save(
'%s.eml' % uuid.uuid4(),
ContentFile(message.as_string()),
save=False
)
msg.mailbox = self
if 'subject' in message:
msg.subject = (
utils.convert_header_to_unicode(message['subject'])[0:255]
)
if 'message-id' in message:
msg.message_id = message['message-id'][0:255].strip()
if 'from' in message:
msg.from_header = utils.convert_header_to_unicode(message['from'])
if 'to' in message:
msg.to_header = utils.convert_header_to_unicode(message['to'])
elif 'Delivered-To' in message:
msg.to_header = utils.convert_header_to_unicode(
message['Delivered-To']
)
msg.save()
message = self._get_dehydrated_message(message, msg)
msg.set_body(message.as_string())
if message['in-reply-to']:
try:
msg.in_reply_to = Message.objects.filter(
message_id=message['in-reply-to'].strip()
)[0]
except IndexError:
pass
msg.save()
return msg
def convert_header_to_unicode(header: str) -> str:
from django_mailbox import utils
if six.PY2 and isinstance(header, six.text_type):
return header
try:
return utils.convert_header_to_unicode(header)
except LookupError:
pass
import codecs
import webencodings
import email
default_charset = utils.get_settings()['default_charset']
def factory(decoder):
def _decode(value, encoding):
if isinstance(value, six.text_type):
return value
if not encoding or encoding == 'unknown-8bit':
encoding = default_charset
return decoder(value, encoding)
return _decode
for _decode in [
factory(lambda value, encoding: codecs.decode(
value, encoding, 'replace')),
factory(lambda value, encoding: webencodings.decode(
value, encoding, 'replace')[0]),
]:
try:
return ''.join([
(_decode(bytestr, encoding))
for bytestr, encoding in email.header.decode_header(header)
])
except LookupError as e:
last_error = e
raise last_error
def get_new_mail(self, condition=None):
"""Connect to this transport and fetch new messages."""
new_mail = []
connection = self.get_connection()
if not connection:
return
for message in connection.get_message(condition):
# if username find in message headers - it's outgoing message
if self.username in utils.convert_header_to_unicode(message['from']):
msg = self.record_outgoing_message(message)
else:
msg = self.process_incoming_message(message)
if not msg is None:
yield msg
self.last_polling = now()
if django.VERSION >= (1, 5): # Django 1.5 introduces update_fields
self.save(update_fields=['last_polling'])
else:
self.save()
def _get_dehydrated_message(self, msg, record):
settings = utils.get_settings()
new = EmailMessage()
if msg.is_multipart():
for header, value in msg.items():
new[header] = value
for part in msg.get_payload():
new.attach(
self._get_dehydrated_message(part, record)
)
elif (
settings['strip_unallowed_mimetypes']
and not msg.get_content_type() in settings['allowed_mimetypes']
):
for header, value in msg.items():
new[header] = value
# Delete header, otherwise when attempting to deserialize the
# payload, it will be expecting a body for this.
del new['Content-Transfer-Encoding']
new[settings['altered_message_header']] = (
'Stripped; Content type %s not allowed' % (
msg.get_content_type()
)
)
new.set_payload('')
elif (
(
msg.get_content_type() not in settings['text_stored_mimetypes']
) or
('attachment' in msg.get('Content-Disposition', ''))
):
filename = None
raw_filename = msg.get_filename()
if raw_filename:
filename = utils.convert_header_to_unicode(raw_filename)
if not filename:
extension = mimetypes.guess_extension(msg.get_content_type())
else:
_, extension = os.path.splitext(filename)
if not extension:
extension = '.bin'
attachment = MessageAttachment()
attachment.document.save(
uuid.uuid4().hex + extension,
ContentFile(
six.BytesIO(
msg.get_payload(decode=True)
).getvalue()
)
)
attachment.message = record
for key, value in msg.items():
attachment[key] = value
attachment.save()
placeholder = EmailMessage()
placeholder[
settings['attachment_interpolation_header']
] = str(attachment.pk)
new = placeholder
else:
content_charset = msg.get_content_charset()
if not content_charset:
content_charset = 'ascii'
try:
# Make sure that the payload can be properly decoded in the
# defined charset, if it can't, let's mash some things
# inside the payload :-\
msg.get_payload(decode=True).decode(content_charset)
except LookupError:
logger.warning(
"Unknown encoding %s; interpreting as ASCII!",
content_charset
)
msg.set_payload(
msg.get_payload(decode=True).decode(
'ascii',
'ignore'
)
)
except ValueError:
logger.warning(
"Decoding error encountered; interpreting %s as ASCII!",
content_charset
)
msg.set_payload(
msg.get_payload(decode=True).decode(
'ascii',
'ignore'
)
)
new = msg
return new
def subject(self, msg):
return convert_header_to_unicode(msg.subject)
def subject(self, msg):
return convert_header_to_unicode(msg.subject)
