def _parse_mimepart(self, mimepart, mid, index, namespace_id):
"""Parse a single MIME part into a Block and Part object linked to this
message."""
from inbox.models.block import Block, Part
disposition, disposition_params = mimepart.content_disposition
if (disposition is not None and
disposition not in ['inline', 'attachment']):
cd = mimepart.content_disposition
log.error('Unknown Content-Disposition',
mid=mid, bad_content_disposition=cd,
parsed_content_disposition=disposition)
self._mark_error()
return
block = Block()
block.namespace_id = namespace_id
block.content_type = mimepart.content_type.value
block.filename = _trim_filename(
mimepart.content_type.params.get('name'), mid)
new_part = Part(block=block)
new_part.walk_index = index
# TODO maybe also trim other headers?
if disposition is not None:
new_part.content_disposition = disposition
if disposition == 'attachment':
new_part.block.filename = _trim_filename(
disposition_params.get('filename'), mid)
if mimepart.body is None:
data_to_write = ''
elif new_part.block.content_type.startswith('text'):
data_to_write = mimepart.body.encode('utf-8', 'strict')
# normalize mac/win/unix newlines
data_to_write = data_to_write.replace('\r\n', '\n'). \
replace('\r', '\n')
else:
data_to_write = mimepart.body
if data_to_write is None:
data_to_write = ''
new_part.content_id = mimepart.headers.get('Content-Id')
block.data = data_to_write
# Wait until end so we don't create incomplete blocks/parts for MIME
# parts which fail to parse.
new_part.message = self
def _parse_mimepart(self, mimepart, mid, index, namespace_id):
"""Parse a single MIME part into a Block and Part object linked to this
message."""
from inbox.models.block import Block, Part
disposition, disposition_params = mimepart.content_disposition
if (disposition is not None
and disposition not in ['inline', 'attachment']):
cd = mimepart.content_disposition
log.error('Unknown Content-Disposition',
mid=mid,
bad_content_disposition=cd,
parsed_content_disposition=disposition)
self._mark_error()
return
block = Block()
block.namespace_id = namespace_id
block.content_type = mimepart.content_type.value
block.filename = _trim_filename(
mimepart.content_type.params.get('name'), mid)
new_part = Part(block=block, message=self)
new_part.walk_index = index
# TODO maybe also trim other headers?
if disposition is not None:
new_part.content_disposition = disposition
if disposition == 'attachment':
new_part.block.filename = _trim_filename(
disposition_params.get('filename'), mid)
if mimepart.body is None:
data_to_write = ''
elif new_part.block.content_type.startswith('text'):
data_to_write = mimepart.body.encode('utf-8', 'strict')
# normalize mac/win/unix newlines
data_to_write = data_to_write.replace('\r\n', '\n'). \
replace('\r', '\n')
else:
data_to_write = mimepart.body
if data_to_write is None:
data_to_write = ''
new_part.content_id = mimepart.headers.get('Content-Id')
block.data = data_to_write
def _parse_mimepart(self, mimepart, mid, index, namespace_id):
"""Parse a single MIME part into a Block and Part object linked to this
message."""
from inbox.models.block import Block, Part
block = Block()
block.namespace_id = namespace_id
block.content_type = mimepart.content_type.value
block.filename = _trim_filename(
mimepart.content_type.params.get('name'), mid)
new_part = Part(block=block, message=self)
new_part.walk_index = index
# TODO maybe also trim other headers?
if mimepart.content_disposition[0] is not None:
value, params = mimepart.content_disposition
if value not in ['inline', 'attachment']:
cd = mimepart.content_disposition
log.error('Unknown Content-Disposition',
mid=mid, bad_content_disposition=cd,
parsed_content_disposition=value)
return
else:
new_part.content_disposition = value
if value == 'attachment':
new_part.block.filename = _trim_filename(
params.get('filename'), mid)
if mimepart.body is None:
data_to_write = ''
elif new_part.block.content_type.startswith('text'):
data_to_write = mimepart.body.encode('utf-8', 'strict')
# normalize mac/win/unix newlines
data_to_write = data_to_write.replace('\r\n', '\n'). \
replace('\r', '\n')
else:
data_to_write = mimepart.body
if data_to_write is None:
data_to_write = ''
new_part.content_id = mimepart.headers.get('Content-Id')
block.data = data_to_write
self.parts.append(new_part)
def add_fake_msg_with_calendar_part(db_session, account, ics_str):
thread = add_fake_thread(db_session, account.namespace.id)
msg = add_fake_message(db_session, account.namespace.id, thread=thread)
if not msg.parts:
walk_index = 1
else:
walk_index = msg.parts[-1].walk_index + 1
ics_part = Part(
message=msg,
block=Block(
data=ics_str,
namespace_id=msg.namespace_id),
walk_index=walk_index)
ics_part.block.content_type = 'text/calendar'
assert msg.has_attached_events
db_session.add(msg)
db_session.commit()
return msg
def create_from_synced(cls, account, mid, folder_name, received_date,
body_string):
""" Parses message data and writes out db metadata and MIME blocks.
Returns the new Message, which links to the new Part and Block objects
through relationships. All new objects are uncommitted.
Threads are not computed here; you gotta do that separately.
Parameters
----------
mid : int
The account backend-specific message identifier; it's only used for
logging errors.
raw_message : str
The full message including headers (encoded).
"""
_rqd = [account, mid, folder_name, body_string]
if not all([v is not None for v in _rqd]):
raise ValueError(
"Required keyword arguments: account, mid, folder_name, "
"body_string")
# stop trickle-down bugs
assert account.namespace is not None
assert not isinstance(body_string, unicode)
msg = Message()
try:
msg.namespace_id = account.namespace.id
parsed = mime.from_string(body_string)
mime_version = parsed.headers.get('Mime-Version')
# sometimes MIME-Version is "1.0 (1.0)", hence the .startswith()
if mime_version is not None and not mime_version.startswith('1.0'):
log.warning('Unexpected MIME-Version',
account_id=account.id,
folder_name=folder_name,
mid=mid,
mime_version=mime_version)
msg.data_sha256 = sha256(body_string).hexdigest()
# clean_subject strips re:, fwd: etc.
msg.subject = parsed.clean_subject
msg.from_addr = parse_mimepart_address_header(parsed, 'From')
msg.sender_addr = parse_mimepart_address_header(parsed, 'Sender')
msg.reply_to = parse_mimepart_address_header(parsed, 'Reply-To')
msg.to_addr = parse_mimepart_address_header(parsed, 'To')
msg.cc_addr = parse_mimepart_address_header(parsed, 'Cc')
msg.bcc_addr = parse_mimepart_address_header(parsed, 'Bcc')
msg.in_reply_to = parsed.headers.get('In-Reply-To')
msg.message_id_header = parsed.headers.get('Message-Id')
msg.received_date = received_date if received_date else \
get_internaldate(parsed.headers.get('Date'),
parsed.headers.get('Received'))
# Custom Inbox header
msg.inbox_uid = parsed.headers.get('X-INBOX-ID')
# In accordance with JWZ (http://www.jwz.org/doc/threading.html)
msg.references = parse_references(
parsed.headers.get('References', ''),
parsed.headers.get('In-Reply-To', ''))
msg.size = len(body_string) # includes headers text
i = 0 # for walk_index
from inbox.models.block import Block, Part
# Store all message headers as object with index 0
block = Block()
block.namespace_id = account.namespace.id
block.data = json.dumps(parsed.headers.items())
headers_part = Part(block=block, message=msg)
headers_part.walk_index = i
msg.parts.append(headers_part)
for mimepart in parsed.walk(
with_self=parsed.content_type.is_singlepart()):
i += 1
if mimepart.content_type.is_multipart():
log.warning('multipart sub-part found',
account_id=account.id,
folder_name=folder_name,
mid=mid)
continue # TODO should we store relations?
msg._parse_mimepart(mimepart, mid, i, account.namespace.id)
msg.calculate_sanitized_body()
except (mime.DecodingError, AttributeError, RuntimeError) as e:
# Message parsing can fail for several reasons. Occasionally iconv
# will fail via maximum recursion depth. EAS messages may be
# missing Date and Received headers. In such cases, we still keep
# the metadata and mark it as b0rked.
_log_decode_error(account.id, folder_name, mid, body_string)
err_filename = _get_errfilename(account.id, folder_name, mid)
log.error('Message parsing error',
folder_name=folder_name,
account_id=account.id,
err_filename=err_filename,
error=e)
msg._mark_error()
# Occasionally people try to send messages to way too many
# recipients. In such cases, empty the field and treat as a parsing
# error so that we don't break the entire sync.
for field in ('to_addr', 'cc_addr', 'bcc_addr', 'references'):
value = getattr(msg, field)
if json_field_too_long(value):
_log_decode_error(account.id, folder_name, mid, body_string)
err_filename = _get_errfilename(account.id, folder_name, mid)
log.error('Recipient field too long',
field=field,
account_id=account.id,
folder_name=folder_name,
mid=mid)
setattr(msg, field, [])
msg._mark_error()
return msg
def __init__(self, account=None, mid=None, folder_name=None,
received_date=None, flags=None, body_string=None,
*args, **kwargs):
""" Parses message data and writes out db metadata and MIME blocks.
Returns the new Message, which links to the new Block objects through
relationships. All new objects are uncommitted.
Threads are not computed here; you gotta do that separately.
Parameters
----------
mid : int
The account backend-specific message identifier; it's only used for
logging errors.
raw_message : str
The full message including headers (encoded).
"""
_rqd = [account, mid, folder_name, flags, body_string]
MailSyncBase.__init__(self, *args, **kwargs)
# for drafts
if not any(_rqd):
return
if any(_rqd) and not all([v is not None for v in _rqd]):
raise ValueError(
"Required keyword arguments: account, mid, folder_name, "
"flags, body_string")
# stop trickle-down bugs
assert account.namespace is not None
assert not isinstance(body_string, unicode)
try:
parsed = mime.from_string(body_string)
mime_version = parsed.headers.get('Mime-Version')
# sometimes MIME-Version is "1.0 (1.0)", hence the .startswith()
if mime_version is not None and not mime_version.startswith('1.0'):
log.warning('Unexpected MIME-Version',
account_id=account.id, folder_name=folder_name,
mid=mid, mime_version=mime_version)
self.data_sha256 = sha256(body_string).hexdigest()
# clean_subject strips re:, fwd: etc.
self.subject = parsed.clean_subject
self.from_addr = parse_email_address_list(
parsed.headers.get('From'))
self.sender_addr = parse_email_address_list(
parsed.headers.get('Sender'))
self.reply_to = parse_email_address_list(
parsed.headers.get('Reply-To'))
self.to_addr = parse_email_address_list(
parsed.headers.getall('To'))
self.cc_addr = parse_email_address_list(
parsed.headers.getall('Cc'))
self.bcc_addr = parse_email_address_list(
parsed.headers.getall('Bcc'))
self.in_reply_to = parsed.headers.get('In-Reply-To')
self.message_id_header = parsed.headers.get('Message-Id')
self.received_date = received_date if received_date else \
get_internaldate(parsed.headers.get('Date'),
parsed.headers.get('Received'))
# Custom Inbox header
self.inbox_uid = parsed.headers.get('X-INBOX-ID')
# In accordance with JWZ (http://www.jwz.org/doc/threading.html)
self.references = parse_references(
parsed.headers.get('References', ''),
parsed.headers.get('In-Reply-To', ''))
self.size = len(body_string) # includes headers text
i = 0 # for walk_index
from inbox.models.block import Part
# Store all message headers as object with index 0
headers_part = Part()
headers_part.namespace_id = account.namespace.id
headers_part.message = self
headers_part.walk_index = i
headers_part.data = json.dumps(parsed.headers.items())
self.parts.append(headers_part)
for mimepart in parsed.walk(
with_self=parsed.content_type.is_singlepart()):
i += 1
if mimepart.content_type.is_multipart():
log.warning('multipart sub-part found',
account_id=account.id, folder_name=folder_name,
mid=mid)
continue # TODO should we store relations?
new_part = Part()
new_part.namespace_id = account.namespace.id
new_part.message = self
new_part.walk_index = i
new_part.content_type = mimepart.content_type.value
new_part.filename = _trim_filename(
mimepart.content_type.params.get('name'),
account.id, mid)
# TODO maybe also trim other headers?
if mimepart.content_disposition[0] is not None:
value, params = mimepart.content_disposition
if value not in ['inline', 'attachment']:
log.error('Unknown Content-Disposition',
account_id=account.id, mid=mid,
folder_name=folder_name,
bad_content_disposition=
mimepart.content_disposition,
parsed_content_disposition=value)
continue
else:
new_part.content_disposition = value
if value == 'attachment':
new_part.filename = _trim_filename(
params.get('filename'), account.id, mid)
if mimepart.body is None:
data_to_write = ''
elif new_part.content_type.startswith('text'):
data_to_write = mimepart.body.encode('utf-8', 'strict')
# normalize mac/win/unix newlines
data_to_write = data_to_write \
.replace('\r\n', '\n').replace('\r', '\n')
else:
data_to_write = mimepart.body
if data_to_write is None:
data_to_write = ''
new_part.content_id = mimepart.headers.get('Content-Id')
new_part.data = data_to_write
self.parts.append(new_part)
self.calculate_sanitized_body()
except mime.DecodingError:
# Occasionally iconv will fail via maximum recursion depth. We
# still keep the metadata and mark it as b0rked.
_log_decode_error(account.id, folder_name, mid, body_string)
log.error('Message parsing DecodeError', account_id=account.id,
folder_name=folder_name, err_filename=_get_errfilename(
account.id, folder_name, mid))
self.mark_error()
return
except AttributeError:
# For EAS messages that are missing Date + Received headers, due
# to the processing we do in inbox.util.misc.get_internaldate()
_log_decode_error(account.id, folder_name, mid, body_string)
log.error('Message parsing AttributeError', account_id=account.id,
folder_name=folder_name, err_filename=_get_errfilename(
account.id, folder_name, mid))
self.mark_error()
return
except RuntimeError:
_log_decode_error(account.id, folder_name, mid, body_string)
log.error('Message parsing RuntimeError<iconv>'.format(
err_filename=_get_errfilename(account.id, folder_name, mid)))
self.mark_error()
return
def create_from_synced(cls, account, mid, folder_name,
received_date, body_string):
""" Parses message data and writes out db metadata and MIME blocks.
Returns the new Message, which links to the new Part and Block objects
through relationships. All new objects are uncommitted.
Threads are not computed here; you gotta do that separately.
Parameters
----------
mid : int
The account backend-specific message identifier; it's only used for
logging errors.
raw_message : str
The full message including headers (encoded).
"""
_rqd = [account, mid, folder_name, body_string]
if not all([v is not None for v in _rqd]):
raise ValueError(
"Required keyword arguments: account, mid, folder_name, "
"body_string")
# stop trickle-down bugs
assert account.namespace is not None
assert not isinstance(body_string, unicode)
msg = Message()
try:
msg.namespace_id = account.namespace.id
parsed = mime.from_string(body_string)
mime_version = parsed.headers.get('Mime-Version')
# sometimes MIME-Version is "1.0 (1.0)", hence the .startswith()
if mime_version is not None and not mime_version.startswith('1.0'):
log.warning('Unexpected MIME-Version',
account_id=account.id, folder_name=folder_name,
mid=mid, mime_version=mime_version)
msg.data_sha256 = sha256(body_string).hexdigest()
# clean_subject strips re:, fwd: etc.
msg.subject = parsed.clean_subject
msg.from_addr = parse_mimepart_address_header(parsed, 'From')
msg.sender_addr = parse_mimepart_address_header(parsed, 'Sender')
msg.reply_to = parse_mimepart_address_header(parsed, 'Reply-To')
msg.to_addr = parse_mimepart_address_header(parsed, 'To')
msg.cc_addr = parse_mimepart_address_header(parsed, 'Cc')
msg.bcc_addr = parse_mimepart_address_header(parsed, 'Bcc')
msg.in_reply_to = parsed.headers.get('In-Reply-To')
msg.message_id_header = parsed.headers.get('Message-Id')
msg.received_date = received_date if received_date else \
get_internaldate(parsed.headers.get('Date'),
parsed.headers.get('Received'))
# Custom Inbox header
msg.inbox_uid = parsed.headers.get('X-INBOX-ID')
# In accordance with JWZ (http://www.jwz.org/doc/threading.html)
msg.references = parse_references(
parsed.headers.get('References', ''),
parsed.headers.get('In-Reply-To', ''))
msg.size = len(body_string) # includes headers text
i = 0 # for walk_index
from inbox.models.block import Block, Part
# Store all message headers as object with index 0
block = Block()
block.namespace_id = account.namespace.id
block.data = json.dumps(parsed.headers.items())
headers_part = Part(block=block, message=msg)
headers_part.walk_index = i
msg.parts.append(headers_part)
for mimepart in parsed.walk(
with_self=parsed.content_type.is_singlepart()):
i += 1
if mimepart.content_type.is_multipart():
log.warning('multipart sub-part found',
account_id=account.id, folder_name=folder_name,
mid=mid)
continue # TODO should we store relations?
msg._parse_mimepart(mimepart, mid, i, account.namespace.id)
msg.calculate_sanitized_body()
except (mime.DecodingError, AttributeError, RuntimeError) as e:
# Message parsing can fail for several reasons. Occasionally iconv
# will fail via maximum recursion depth. EAS messages may be
# missing Date and Received headers. In such cases, we still keep
# the metadata and mark it as b0rked.
_log_decode_error(account.id, folder_name, mid, body_string)
err_filename = _get_errfilename(account.id, folder_name, mid)
log.error('Message parsing error',
folder_name=folder_name, account_id=account.id,
err_filename=err_filename, error=e)
msg._mark_error()
return msg
def create_from_synced(cls, account, mid, folder_name, received_date,
body_string):
"""
Parses message data and writes out db metadata and MIME blocks.
Returns the new Message, which links to the new Part and Block objects
through relationships. All new objects are uncommitted.
Threads are not computed here; you gotta do that separately.
Parameters
----------
mid : int
The account backend-specific message identifier; it's only used for
logging errors.
raw_message : str
The full message including headers (encoded).
"""
_rqd = [account, mid, folder_name, body_string]
if not all([v is not None for v in _rqd]):
raise ValueError(
'Required keyword arguments: account, mid, folder_name, '
'body_string')
# stop trickle-down bugs
assert account.namespace is not None
assert not isinstance(body_string, unicode)
msg = Message()
from inbox.models.block import Block, Part
body_block = Block()
body_block.namespace_id = account.namespace.id
body_block.data = body_string
body_block.content_type = "text/plain"
msg.full_body = body_block
msg.namespace_id = account.namespace.id
try:
parsed = mime.from_string(body_string)
i = 0 # for walk_index
# Store all message headers as object with index 0
block = Block()
block.namespace_id = account.namespace.id
block.data = json.dumps(parsed.headers.items())
headers_part = Part(block=block, message=msg)
headers_part.walk_index = i
msg._parse_metadata(parsed, body_string, received_date, account.id,
folder_name, mid)
except (mime.DecodingError, AttributeError, RuntimeError, TypeError,
ValueError) as e:
parsed = None
log.error('Error parsing message metadata',
folder_name=folder_name, account_id=account.id, error=e)
msg._mark_error()
if parsed is not None:
for mimepart in parsed.walk(
with_self=parsed.content_type.is_singlepart()):
i += 1
try:
if mimepart.content_type.is_multipart():
log.warning('multipart sub-part found',
account_id=account.id,
folder_name=folder_name,
mid=mid)
continue # TODO should we store relations?
msg._parse_mimepart(mimepart, mid, i, account.namespace.id)
except (mime.DecodingError, AttributeError, RuntimeError,
TypeError, ValueError) as e:
log.error('Error parsing message MIME parts',
folder_name=folder_name, account_id=account.id,
error=e)
msg._mark_error()
msg.calculate_body()
# Occasionally people try to send messages to way too many
# recipients. In such cases, empty the field and treat as a parsing
# error so that we don't break the entire sync.
for field in ('to_addr', 'cc_addr', 'bcc_addr', 'references'):
value = getattr(msg, field)
if json_field_too_long(value):
log.error('Recipient field too long', field=field,
account_id=account.id, folder_name=folder_name,
mid=mid)
setattr(msg, field, [])
msg._mark_error()
return msg
def __init__(self, account=None, mid=None, folder_name=None,
received_date=None, flags=None, body_string=None,
*args, **kwargs):
""" Use .create() instead to handle common errors!
(Can't abort object creation in a constructor.)
"""
_rqd = [account, mid, folder_name, received_date, flags, body_string]
# for drafts - skip parsing
if not any(_rqd):
MailSyncBase.__init__(self, *args, **kwargs)
return
parsed = mime.from_string(body_string)
mime_version = parsed.headers.get('Mime-Version')
# NOTE: sometimes MIME-Version is set to "1.0 (1.0)", hence the
# .startswith
if mime_version is not None and not mime_version.startswith('1.0'):
log.error('Unexpected MIME-Version: {0}'.format(mime_version))
self.data_sha256 = sha256(body_string).hexdigest()
# clean_subject strips re:, fwd: etc.
self.subject = parsed.clean_subject
self.from_addr = parse_email_address_list(
parsed.headers.get('From'))
self.sender_addr = parse_email_address_list(
parsed.headers.get('Sender'))
self.reply_to = parse_email_address_list(
parsed.headers.get('Reply-To'))
self.to_addr = parse_email_address_list(
parsed.headers.getall('To'))
self.cc_addr = parse_email_address_list(
parsed.headers.getall('Cc'))
self.bcc_addr = parse_email_address_list(
parsed.headers.getall('Bcc'))
self.in_reply_to = parsed.headers.get('In-Reply-To')
self.message_id_header = parsed.headers.get('Message-Id')
self.received_date = received_date
# Optional mailing list headers
self.mailing_list_headers = parse_ml_headers(parsed.headers)
# Custom Inbox header
self.inbox_uid = parsed.headers.get('X-INBOX-ID')
# In accordance with JWZ (http://www.jwz.org/doc/threading.html)
self.references = parse_references(
parsed.headers.get('References', ''),
parsed.headers.get('In-Reply-To', ''))
self.size = len(body_string) # includes headers text
i = 0 # for walk_index
from inbox.models.block import Part
# Store all message headers as object with index 0
headers_part = Part()
headers_part.namespace_id = account.namespace.id
headers_part.message = self
headers_part.walk_index = i
headers_part.data = json.dumps(parsed.headers.items())
self.parts.append(headers_part)
for mimepart in parsed.walk(
with_self=parsed.content_type.is_singlepart()):
i += 1
if mimepart.content_type.is_multipart():
log.warning("multipart sub-part found! on {}"
.format(self.g_msgid))
continue # TODO should we store relations?
new_part = Part()
new_part.namespace_id = account.namespace.id
new_part.message = self
new_part.walk_index = i
new_part.misc_keyval = mimepart.headers.items() # everything
new_part.content_type = mimepart.content_type.value
new_part.filename = _trim_filename(
mimepart.content_type.params.get('name'),
log=log)
# TODO maybe also trim other headers?
if mimepart.content_disposition[0] is not None:
value, params = mimepart.content_disposition
if value not in ['inline', 'attachment']:
errmsg = """
Unknown Content-Disposition on message {0} found in {1}.
Bad Content-Disposition was: '{2}'
Parsed Content-Disposition was: '{3}'""".format(
mid, folder_name, mimepart.content_disposition)
log.error(errmsg)
continue
else:
new_part.content_disposition = value
if value == 'attachment':
new_part.filename = _trim_filename(
params.get('filename'),
log=log)
if mimepart.body is None:
data_to_write = ''
elif new_part.content_type.startswith('text'):
data_to_write = mimepart.body.encode('utf-8', 'strict')
# normalize mac/win/unix newlines
data_to_write = data_to_write \
.replace('\r\n', '\n').replace('\r', '\n')
else:
data_to_write = mimepart.body
if data_to_write is None:
data_to_write = ''
new_part.content_id = mimepart.headers.get('Content-Id')
new_part.data = data_to_write
self.parts.append(new_part)
self.calculate_sanitized_body()
MailSyncBase.__init__(self, *args, **kwargs)
