def parsemail(mail, logger='none'):
log = logging.getLogger(logger)
if len(mail) == 0:
raise MailParserException('Empty mail')
data = { 'type': 'mail' }
# parse mail
try:
message = email.parser.Parser().parsestr(mail)
except UnicodeEncodeError:
message = email.parser.Parser().parsestr(mail.encode('latin_1'))
# test defects and try to save defect mails
if len(message.defects) != 0:
raise MailParserException("Parser signaled defect:\n %s" % (str(message.defects)))
# encoded word is not decoded here, because it only should appear in the
# display name that is discarded by the last map function
# parse from and sender addresses
addresses = itertools.chain(*(message.get_all(field) for field in ('from', 'sender') if message.has_key(field)))
data['from'] = map(lambda adrs: adrs[1], set(email.utils.getaddresses(addresses)))
log.info("From: %s"%(' '.join(data['from'])))
# parse recipient addresses
addresses = itertools.chain(*(message.get_all(field) for field in ('to', 'cc') if message.has_key(field)))
data['to'] = map(lambda adrs: adrs[1], set(email.utils.getaddresses(addresses)))
log.info("To: %s"%(' '.join(data['to'])))
# parse date and convert it to standard format in UTC
if message.get('Date', None):
try:
# guesses format and parses 10-tuple
parsedtime = email.utils.parsedate_tz(message.get('Date'))
# seconds since epoch
utc_timestamp = calendar.timegm(parsedtime[0:9])-parsedtime[9]
# formatted
data['date'] = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(utc_timestamp))
log.info("Date: %s", data['date'])
except Exception as e:
raise MailParserException("Could not convert %s to YYYY-MM-DD hh:mm:ss\n %s" % (message.get('Date'), str(e)))
# format current UTC time
data['upload_date'] = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()))
log.info("upload-date: %s", data['upload_date'])
# add labels
data['labels'] = []
if message.get('Status', None):
if not 'R' in message.get('Status'):
data['labels'].append('unread')
else: data['labels'].append('unread')
if config.autolabels:
labeller = labels.Labeller(path=config.autolabels)
labeller.check(data)
log.info("Labels: %s", ' '.join(data['labels']))
return data
def extract_and_upload_attachments(message: message.Message, realm: Realm) -> str:
user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT)
attachment_links = []
payload = message.get_payload()
if not isinstance(payload, list):
# This is not a multipart message, so it can't contain attachments.
return ""
for part in payload:
content_type = part.get_content_type()
filename = part.get_filename()
if filename:
attachment = part.get_payload(decode=True)
if isinstance(attachment, bytes):
s3_url = upload_message_file(filename, len(attachment), content_type,
attachment,
user_profile,
target_realm=realm)
formatted_link = "[%s](%s)" % (filename, s3_url)
attachment_links.append(formatted_link)
else:
logger.warning("Payload is not bytes (invalid attachment %s in message from %s)." %
(filename, message.get("From")))
return "\n".join(attachment_links)
def process_message(message: message.Message,
rcpt_to: Optional[str] = None,
pre_checked: bool = False) -> None:
subject_header = str(message.get("Subject", "")).strip()
if subject_header == "":
subject_header = "(no topic)"
encoded_subject, encoding = decode_header(subject_header)[0]
if encoding is None:
subject = cast(str, encoded_subject
) # encoded_subject has type str when encoding is None
else:
try:
subject = encoded_subject.decode(encoding)
except (UnicodeDecodeError, LookupError):
subject = "(unreadable subject)"
debug_info = {}
try:
if rcpt_to is not None:
to = rcpt_to
else:
to = find_emailgateway_recipient(message)
debug_info["to"] = to
if is_missed_message_address(to):
process_missed_message(to, message, pre_checked)
else:
process_stream_message(to, subject, message, debug_info)
except ZulipEmailForwardError as e:
# TODO: notify sender of error, retry if appropriate.
log_and_report(message, str(e), debug_info)
def extract_and_upload_attachments(message: message.Message, realm: Realm) -> str:
user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT)
attachment_links = []
for part in message.walk():
content_type = part.get_content_type()
encoded_filename = part.get_filename()
if not encoded_filename:
continue
filename = handle_header_content(encoded_filename)
if filename:
attachment = part.get_payload(decode=True)
if isinstance(attachment, bytes):
s3_url = upload_message_file(filename, len(attachment), content_type,
attachment,
user_profile,
target_realm=realm)
formatted_link = "[%s](%s)" % (filename, s3_url)
attachment_links.append(formatted_link)
else:
logger.warning("Payload is not bytes (invalid attachment %s in message from %s)." %
(filename, message.get("From")))
return '\n'.join(attachment_links)
def process_message(message, rcpt_to=None, pre_checked=False):
# type: (message.Message, Optional[text_type], bool) -> None
subject_header = message.get("Subject", "(no subject)")
encoded_subject, encoding = decode_header(subject_header)[0] # type: ignore # https://github.com/python/typeshed/pull/333
if encoding is None:
subject = force_text(encoded_subject) # encoded_subject has type str when encoding is None
else:
try:
subject = encoded_subject.decode(encoding)
except (UnicodeDecodeError, LookupError):
subject = u"(unreadable subject)"
debug_info = {}
try:
if rcpt_to is not None:
to = rcpt_to
else:
to = find_emailgateway_recipient(message)
debug_info["to"] = to
if is_missed_message_address(to):
process_missed_message(to, message, pre_checked)
else:
process_stream_message(to, subject, message, debug_info)
except ZulipEmailForwardError as e:
# TODO: notify sender of error, retry if appropriate.
log_and_report(message, str(e), debug_info)
def extract_and_upload_attachments(message, realm):
# type: (message.Message, Realm) -> Text
user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT)
attachment_links = []
payload = message.get_payload()
if not isinstance(payload, list):
# This is not a multipart message, so it can't contain attachments.
return ""
for part in payload:
content_type = part.get_content_type()
filename = part.get_filename()
if filename:
attachment = part.get_payload(decode=True)
if isinstance(attachment, binary_type):
s3_url = upload_message_image(filename,
len(attachment),
content_type,
attachment,
user_profile,
target_realm=realm)
formatted_link = u"[%s](%s)" % (filename, s3_url)
attachment_links.append(formatted_link)
else:
logger.warning(
"Payload is not bytes (invalid attachment %s in message from %s)."
% (filename, message.get("From")))
return u"\n".join(attachment_links)
def collect_frequency_data(self, message, headers=None):
"""
Store data about frequency of message submission from sender of this
message. 'headers', if specified, is a list of header names to store
along with times, for use as discriminators.
"""
user = message['From']
date = message.get('Date')
if date is not None:
date = parsedate(date)
if date is not None:
date = datetime.datetime(*date[:6])
else:
date = datetime.datetime.now()
if headers is None:
headers = {}
else:
headers = dict([(name, message[name]) for name in headers])
times = self._freq_data.get(user)
if times is None:
times = _FreqData()
self._freq_data[user] = times
times.append((date,headers))
def pop_next(self):
"""
Retrieve the next message in the queue, removing it from the queue.
"""
key = iter(self._messages.keys()).next()
message = self._messages.pop(key)
return message.get()
def process_message(message: message.Message, rcpt_to: Optional[str]=None, pre_checked: bool=False) -> None:
subject_header = str(message.get("Subject", "")).strip()
if subject_header == "":
subject_header = "(no topic)"
encoded_subject, encoding = decode_header(subject_header)[0]
if encoding is None:
subject = force_text(encoded_subject) # encoded_subject has type str when encoding is None
else:
try:
subject = encoded_subject.decode(encoding)
except (UnicodeDecodeError, LookupError):
subject = "(unreadable subject)"
debug_info = {}
try:
if rcpt_to is not None:
to = rcpt_to
else:
to = find_emailgateway_recipient(message)
debug_info["to"] = to
if is_missed_message_address(to):
process_missed_message(to, message, pre_checked)
else:
process_stream_message(to, subject, message, debug_info)
except ZulipEmailForwardError as e:
# TODO: notify sender of error, retry if appropriate.
log_and_report(message, str(e), debug_info)
def process_message(message, rcpt_to=None, pre_checked=False):
# type: (message.Message, Optional[Text], bool) -> None
subject_header = message.get("Subject", "(no subject)")
encoded_subject, encoding = decode_header(subject_header)[0]
if encoding is None:
subject = force_text(encoded_subject) # encoded_subject has type str when encoding is None
else:
try:
subject = encoded_subject.decode(encoding)
except (UnicodeDecodeError, LookupError):
subject = u"(unreadable subject)"
debug_info = {}
try:
if rcpt_to is not None:
to = rcpt_to
else:
to = find_emailgateway_recipient(message)
debug_info["to"] = to
if is_missed_message_address(to):
process_missed_message(to, message, pre_checked)
else:
process_stream_message(to, subject, message, debug_info)
except ZulipEmailForwardError as e:
# TODO: notify sender of error, retry if appropriate.
log_and_report(message, str(e), debug_info)
def test_003_send_reply(self):
itip_events = itip.events_from_message(
message_from_string(itip_non_multipart))
itip.send_reply(
"*****@*****.**", itip_events,
"SUMMARY=%(summary)s; STATUS=%(status)s; NAME=%(name)s;")
self.assertEqual(len(self.smtplog), 1)
self.assertEqual(self.smtplog[0][0],
'*****@*****.**',
"From attendee")
self.assertEqual(self.smtplog[0][1], '*****@*****.**',
"To organizer")
_accepted = participant_status_label('ACCEPTED')
message = message_from_string(self.smtplog[0][2])
self.assertEqual(
message.get('Subject'),
_("Invitation for %(summary)s was %(status)s") % {
'summary': 'test',
'status': _accepted
})
text = str(message.get_payload(0))
self.assertIn('SUMMARY=3Dtest', text)
self.assertIn('STATUS=3D' + _accepted, text)
def process_stream_message(to: str, message: message.Message) -> None:
subject_header = str(make_header(decode_header(message.get("Subject", ""))))
subject = strip_from_subject(subject_header) or "(no topic)"
stream, show_sender = extract_and_validate(to)
# Don't remove quotations if message is forwarded:
remove_quotations = not is_forwarded(subject_header)
body = construct_zulip_body(message, stream.realm, show_sender, remove_quotations)
send_zulip(settings.EMAIL_GATEWAY_BOT, stream, subject, body)
logger.info("Successfully processed email to %s (%s)" % (
stream.name, stream.realm.string_id))
def remove_from_quarantine(self, message):
"""
Removes the given message from the quarantine.
"""
id = message.get('X-Postoffice-Id')
if id is None:
raise ValueError("Message is not in the quarantine.")
id = int(id)
if id not in self._quarantine:
raise ValueError("Message is not in the quarantine.")
del self._quarantine[id]
del message['X-Postoffice-Id']
def process_stream_message(to: str, message: message.Message,
debug_info: Dict[str, Any]) -> None:
subject_header = str(make_header(decode_header(message.get("Subject", ""))))
subject = strip_from_subject(subject_header) or "(no topic)"
stream, show_sender = extract_and_validate(to)
# Don't remove quotations if message is forwarded:
remove_quotations = not is_forwarded(subject_header)
body = construct_zulip_body(message, stream.realm, show_sender, remove_quotations)
debug_info["stream"] = stream
send_zulip(settings.EMAIL_GATEWAY_BOT, stream, subject, body)
logger.info("Successfully processed email to %s (%s)" % (
stream.name, stream.realm.string_id))
def process_stream_message(to: str, message: message.Message) -> None:
subject_header = str(make_header(decode_header(message.get("Subject", ""))))
subject = strip_from_subject(subject_header) or "(no topic)"
stream, options = decode_stream_email_address(to)
# Don't remove quotations if message is forwarded, unless otherwise specified:
if 'include_quotes' not in options:
options['include_quotes'] = is_forwarded(subject_header)
body = construct_zulip_body(message, stream.realm, **options)
send_zulip(settings.EMAIL_GATEWAY_BOT, stream, subject, body)
logger.info("Successfully processed email to %s (%s)" % (
stream.name, stream.realm.string_id))
def test_send_a_single_email_to_multiple_recipients(self):
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent)
self.maxDiff = None
to_emails = [
To('*****@*****.**', 'Example To Name 0'),
To('*****@*****.**', 'Example To Name 1')
]
message = Mail(
from_email=From('*****@*****.**', 'Example From Name'),
to_emails=to_emails,
subject=Subject('Sending with SendGrid is Fun'),
plain_text_content=PlainTextContent(
'and easy to do anywhere, even with Python'),
html_content=HtmlContent(
'<strong>and easy to do anywhere, even with Python</strong>'))
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"to": [
{
"email": "*****@*****.**",
"name": "Example To Name 0"
},
{
"email": "*****@*****.**",
"name": "Example To Name 1"
}
]
}
],
"subject": "Sending with SendGrid is Fun"
}''')
)
def test_send_a_single_email_to_multiple_recipients(self):
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent)
self.maxDiff = None
to_emails = [
To('*****@*****.**', 'Example To Name 0'),
To('*****@*****.**', 'Example To Name 1')
]
message = Mail(
from_email=From('*****@*****.**', 'Example From Name'),
to_emails=to_emails,
subject=Subject('Sending with SendGrid is Fun'),
plain_text_content=PlainTextContent(
'and easy to do anywhere, even with Python'),
html_content=HtmlContent(
'<strong>and easy to do anywhere, even with Python</strong>'))
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"to": [
{
"email": "*****@*****.**",
"name": "Example To Name 0"
},
{
"email": "*****@*****.**",
"name": "Example To Name 1"
}
]
}
],
"subject": "Sending with SendGrid is Fun"
}'''))
def test_003_send_reply(self):
itip_events = itip.events_from_message(message_from_string(itip_non_multipart))
itip.send_reply("*****@*****.**", itip_events, "SUMMARY=%(summary)s; STATUS=%(status)s; NAME=%(name)s;")
self.assertEqual(len(self.smtplog), 1)
self.assertEqual(self.smtplog[0][0], '*****@*****.**', "From attendee")
self.assertEqual(self.smtplog[0][1], '*****@*****.**', "To organizer")
_accepted = participant_status_label('ACCEPTED')
message = message_from_string(self.smtplog[0][2])
self.assertEqual(message.get('Subject'), _("Invitation for %(summary)s was %(status)s") % { 'summary':'test', 'status':_accepted })
text = str(message.get_payload(0));
self.assertIn('SUMMARY=3Dtest', text)
self.assertIn('STATUS=3D' + _accepted, text)
def test_unicode_values_in_substitutions_helper(self):
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent)
self.maxDiff = None
message = Mail(
from_email=From('*****@*****.**', 'Example From Name'),
to_emails=To('*****@*****.**', 'Example To Name'),
subject=Subject('Sending with SendGrid is Fun'),
plain_text_content=PlainTextContent(
'and easy to do anywhere, even with Python'),
html_content=HtmlContent(
'<strong>and easy to do anywhere, even with Python</strong>'))
message.substitution = Substitution('%city%', u'Αθήνα', p=1)
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"to": [
{
"email": "*****@*****.**",
"name": "Example To Name"
}
]
},
{
"substitutions": {
"%city%": "Αθήνα"
}
}
],
"subject": "Sending with SendGrid is Fun"
}''')
)
def construct_zulip_body(message: message.Message, realm: Realm, show_sender: bool=False,
remove_quotations: bool=True) -> str:
body = extract_body(message, remove_quotations)
# Remove null characters, since Zulip will reject
body = body.replace("\x00", "")
body = filter_footer(body)
body += extract_and_upload_attachments(message, realm)
body = body.strip()
if not body:
body = '(No email body)'
if show_sender:
sender = message.get("From")
body = "From: %s\n%s" % (sender, body)
return body
def _is_cancel(self, message):
"""
Check if the passed message is a CANCEL message.
This will try to match :attr:`FUCore.CANCEL_EXP` against all
**Control:** headers contained in *message*.
:param message: A :class:`email.message` object.
:returns: :const:`True` or :const:`False`
"""
if FUCore.CANCEL_EXP.findall(message.get('Control', '')):
self._log('--- Message contains *this* header, you know?')
self._log('--- Therefore I\'m going to delete it.')
return True
return False
def construct_zulip_body(message: message.Message, realm: Realm, show_sender: bool=False,
remove_quotations: bool=True) -> str:
body = extract_body(message, remove_quotations)
# Remove null characters, since Zulip will reject
body = body.replace("\x00", "")
body = filter_footer(body)
body += extract_and_upload_attachments(message, realm)
body = body.strip()
if not body:
body = '(No email body)'
if show_sender:
sender = message.get("From")
body = "From: %s\n%s" % (sender, body)
return body
def _is_cancel(self, message):
"""
Check if the passed message is a CANCEL message.
This will try to match :attr:`FUCore.CANCEL_EXP` against all
**Control:** headers contained in *message*.
:param message: A :class:`email.message` object.
:returns: :const:`True` or :const:`False`
"""
if FUCore.CANCEL_EXP.findall(message.get('Control', '')):
self._log('--- Message contains *this* header, you know?')
self._log('--- Therefore I\'m going to delete it.')
return True
return False
def test_unicode_values_in_substitutions_helper(self):
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent)
self.maxDiff = None
message = Mail(
from_email=From('*****@*****.**', 'Example From Name'),
to_emails=To('*****@*****.**', 'Example To Name'),
subject=Subject('Sending with SendGrid is Fun'),
plain_text_content=PlainTextContent(
'and easy to do anywhere, even with Python'),
html_content=HtmlContent(
'<strong>and easy to do anywhere, even with Python</strong>'))
message.substitution = Substitution('%city%', u'Αθήνα', p=1)
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"to": [
{
"email": "*****@*****.**",
"name": "Example To Name"
}
]
},
{
"substitutions": {
"%city%": "Αθήνα"
}
}
],
"subject": "Sending with SendGrid is Fun"
}'''))
def test_single_email_to_a_single_recipient_content_reversed(self):
"""Tests bug found in Issue-451 with Content ordering causing a crash
"""
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent)
self.maxDiff = None
message = Mail()
message.from_email = From('*****@*****.**', 'Example From Name')
message.to = To('*****@*****.**', 'Example To Name')
message.subject = Subject('Sending with SendGrid is Fun')
message.content = HtmlContent(
'<strong>and easy to do anywhere, even with Python</strong>')
message.content = PlainTextContent(
'and easy to do anywhere, even with Python')
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"to": [
{
"email": "*****@*****.**",
"name": "Example To Name"
}
]
}
],
"subject": "Sending with SendGrid is Fun"
}''')
)
def test_single_email_to_a_single_recipient_content_reversed(self):
"""Tests bug found in Issue-451 with Content ordering causing a crash
"""
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent)
self.maxDiff = None
message = Mail()
message.from_email = From('*****@*****.**', 'Example From Name')
message.to = To('*****@*****.**', 'Example To Name')
message.subject = Subject('Sending with SendGrid is Fun')
message.content = HtmlContent(
'<strong>and easy to do anywhere, even with Python</strong>')
message.content = PlainTextContent(
'and easy to do anywhere, even with Python')
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"to": [
{
"email": "*****@*****.**",
"name": "Example To Name"
}
]
}
],
"subject": "Sending with SendGrid is Fun"
}'''))
def process_message(message, rcpt_to=None, pre_checked=False):
# type: (message.Message, Optional[text_type], bool) -> None
subject = decode_header(message.get("Subject", "(no subject)"))[0][0]
debug_info = {}
try:
if rcpt_to is not None:
to = rcpt_to
else:
to = find_emailgateway_recipient(message)
debug_info["to"] = to
if is_missed_message_address(to):
process_missed_message(to, message, pre_checked)
else:
process_stream_message(to, subject, message, debug_info)
except ZulipEmailForwardError as e:
# TODO: notify sender of error, retry if appropriate.
log_and_report(message, e.message, debug_info)
def construct_zulip_body(message: message.Message, realm: Realm, show_sender: bool=False,
include_quotes: bool=False, include_footer: bool=False,
prefer_text: bool=True) -> str:
body = extract_body(message, include_quotes, prefer_text)
# Remove null characters, since Zulip will reject
body = body.replace("\x00", "")
if not include_footer:
body = filter_footer(body)
if not body.endswith('\n'):
body += '\n'
body += extract_and_upload_attachments(message, realm)
body = body.strip()
if not body:
body = '(No email body)'
if show_sender:
sender = handle_header_content(message.get("From", ""))
body = "From: %s\n%s" % (sender, body)
return body
def process_message(message: message.Message,
rcpt_to: Optional[str] = None,
pre_checked: bool = False) -> None:
subject_header = make_header(decode_header(message.get("Subject", "")))
subject = strip_from_subject(str(subject_header)) or "(no topic)"
debug_info = {}
try:
if rcpt_to is not None:
to = rcpt_to
else:
to = find_emailgateway_recipient(message)
debug_info["to"] = to
if is_missed_message_address(to):
process_missed_message(to, message, pre_checked)
else:
process_stream_message(to, subject, message, debug_info)
except ZulipEmailForwardError as e:
# TODO: notify sender of error, retry if appropriate.
log_and_report(message, str(e), debug_info)
def test_create_mailing_from_message_with_encoded_headers(self):
parser = email.parser.Parser()
msg = parser.parsestr("""Content-Transfer-Encoding: 7bit
Content-Type: multipart/alternative; boundary="===============2840728917476054151=="
Subject: Great news!
From: =?UTF-8?B?Q2VkcmljIFJJQ0FSRA==?= <*****@*****.**>
To: <*****@*****.**>
Date: Wed, 05 Jun 2013 06:05:56 -0000
This is a multi-part message in MIME format.
--===============2840728917476054151==
Content-Type: text/plain; charset="windows-1252"
Content-Transfer-Encoding: quoted-printable
This is a very simple mailing. I=92m happy.
--===============2840728917476054151==
Content-Type: text/html; charset="windows-1252"
Content-Transfer-Encoding: quoted-printable
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<html><head>
<META http-equiv=3DContent-Type content=3D"text/html; charset=3Diso-8859-1">
</head>
<body>
This is <strong> a very simple</strong> <u>mailing</u>. =
I=92m happy! Nothing else to say...
</body></html>
--===============2840728917476054151==--
""")
mailing = Mailing.create_from_message(msg, scheduled_start=None, scheduled_duration=None)
message = parser.parsestr(mailing.header + mailing.body)
assert(isinstance(message, email.message.Message))
mail_from = header_to_unicode(message.get("From"))
self.assertEquals(u"Cedric RICARD <*****@*****.**>", mail_from)
def process_message(message: message.Message, rcpt_to: Optional[str]=None, pre_checked: bool=False) -> None:
subject_header = make_header(decode_header(message.get("Subject", "")))
subject = strip_from_subject(str(subject_header)) or "(no topic)"
debug_info = {}
try:
if rcpt_to is not None:
to = rcpt_to
else:
to = find_emailgateway_recipient(message)
debug_info["to"] = to
if is_missed_message_address(to):
process_missed_message(to, message, pre_checked)
else:
process_stream_message(to, subject, message, debug_info)
except ZulipEmailForwardError as e:
if isinstance(e, ZulipEmailForwardUserError):
# TODO: notify sender of error, retry if appropriate.
logging.warning(str(e))
else:
log_and_report(message, str(e), debug_info)
def construct_gmail_message(payload):
message = email.message.Message()
for header in payload['headers']:
message[header['name']] = header['value']
if message.get_content_maintype() == 'multipart':
message.set_payload(
[construct_gmail_message(part) for part in payload['parts']])
else:
cte = message.get('Content-Transfer-Encoding')
if cte is not None:
del message['Content-Transfer-Encoding']
message['X-Original-Content-Transfer-Encoding'] = cte
try:
external_id = payload['body']['attachmentId']
ct = message.get_content_type()
message.replace_header('Content-Type', 'text/plain')
message.set_payload(
'Attachment with type %s, ID %r omitted; retrieve separately' %
(ct, external_id))
except KeyError:
body = payload['body']['data']
body += '=' * (4 - len(body) % 4)
message.set_payload(base64.urlsafe_b64decode(body))
return message
def test_kitchen_sink(self):
from sendgrid.helpers.mail import (
Mail, From, To, Cc, Bcc, Subject, Substitution, Header, CustomArg,
SendAt, Content, MimeType, Attachment, FileName, FileContent,
FileType, Disposition, ContentId, TemplateId, Section, ReplyTo,
Category, BatchId, Asm, GroupId, GroupsToDisplay, IpPoolName,
MailSettings, BccSettings, BccSettingsEmail, BypassListManagement,
FooterSettings, FooterText, FooterHtml, SandBoxMode, SpamCheck,
SpamThreshold, SpamUrl, TrackingSettings, ClickTracking,
SubscriptionTracking, SubscriptionText, SubscriptionHtml,
SubscriptionSubstitutionTag, OpenTracking,
OpenTrackingSubstitutionTag, Ganalytics, UtmSource, UtmMedium,
UtmTerm, UtmContent, UtmCampaign)
self.maxDiff = None
message = Mail()
# Define Personalizations
message.to = To('*****@*****.**', 'Example User1', p=0)
message.to = [
To('*****@*****.**', 'Example User2', p=0),
To('*****@*****.**', 'Example User3', p=0)
]
message.cc = Cc('*****@*****.**', 'Example User4', p=0)
message.cc = [
Cc('*****@*****.**', 'Example User5', p=0),
Cc('*****@*****.**', 'Example User6', p=0)
]
message.bcc = Bcc('*****@*****.**', 'Example User7', p=0)
message.bcc = [
Bcc('*****@*****.**', 'Example User8', p=0),
Bcc('*****@*****.**', 'Example User9', p=0)
]
message.subject = Subject('Sending with SendGrid is Fun 0', p=0)
message.header = Header('X-Test1', 'Test1', p=0)
message.header = Header('X-Test2', 'Test2', p=0)
message.header = [
Header('X-Test3', 'Test3', p=0),
Header('X-Test4', 'Test4', p=0)
]
message.substitution = Substitution('%name1%', 'Example Name 1', p=0)
message.substitution = Substitution('%city1%', 'Example City 1', p=0)
message.substitution = [
Substitution('%name2%', 'Example Name 2', p=0),
Substitution('%city2%', 'Example City 2', p=0)
]
message.custom_arg = CustomArg('marketing1', 'true', p=0)
message.custom_arg = CustomArg('transactional1', 'false', p=0)
message.custom_arg = [
CustomArg('marketing2', 'false', p=0),
CustomArg('transactional2', 'true', p=0)
]
message.send_at = SendAt(1461775051, p=0)
message.to = To('*****@*****.**', 'Example User10', p=1)
message.to = [
To('*****@*****.**', 'Example User11', p=1),
To('*****@*****.**', 'Example User12', p=1)
]
message.cc = Cc('*****@*****.**', 'Example User13', p=1)
message.cc = [
Cc('*****@*****.**', 'Example User14', p=1),
Cc('*****@*****.**', 'Example User15', p=1)
]
message.bcc = Bcc('*****@*****.**', 'Example User16', p=1)
message.bcc = [
Bcc('*****@*****.**', 'Example User17', p=1),
Bcc('*****@*****.**', 'Example User18', p=1)
]
message.header = Header('X-Test5', 'Test5', p=1)
message.header = Header('X-Test6', 'Test6', p=1)
message.header = [
Header('X-Test7', 'Test7', p=1),
Header('X-Test8', 'Test8', p=1)
]
message.substitution = Substitution('%name3%', 'Example Name 3', p=1)
message.substitution = Substitution('%city3%', 'Example City 3', p=1)
message.substitution = [
Substitution('%name4%', 'Example Name 4', p=1),
Substitution('%city4%', 'Example City 4', p=1)
]
message.custom_arg = CustomArg('marketing3', 'true', p=1)
message.custom_arg = CustomArg('transactional3', 'false', p=1)
message.custom_arg = [
CustomArg('marketing4', 'false', p=1),
CustomArg('transactional4', 'true', p=1)
]
message.send_at = SendAt(1461775052, p=1)
message.subject = Subject('Sending with SendGrid is Fun 1', p=1)
# The values below this comment are global to entire message
message.from_email = From('*****@*****.**', 'Twilio SendGrid')
message.reply_to = ReplyTo('*****@*****.**',
'Twilio SendGrid Reply')
message.subject = Subject('Sending with SendGrid is Fun 2')
message.content = Content(MimeType.text,
'and easy to do anywhere, even with Python')
message.content = Content(
MimeType.html,
'<strong>and easy to do anywhere, even with Python</strong>')
message.content = [
Content('text/calendar', 'Party Time!!'),
Content('text/custom', 'Party Time 2!!')
]
message.attachment = Attachment(
FileContent('base64 encoded content 1'),
FileName('balance_001.pdf'), FileType('application/pdf'),
Disposition('attachment'), ContentId('Content ID 1'))
message.attachment = [
Attachment(FileContent('base64 encoded content 2'),
FileName('banner.png'), FileType('image/png'),
Disposition('inline'), ContentId('Content ID 2')),
Attachment(FileContent('base64 encoded content 3'),
FileName('banner2.png'), FileType('image/png'),
Disposition('inline'), ContentId('Content ID 3'))
]
message.template_id = TemplateId(
'13b8f94f-bcae-4ec6-b752-70d6cb59f932')
message.section = Section('%section1%',
'Substitution for Section 1 Tag')
message.section = [
Section('%section2%', 'Substitution for Section 2 Tag'),
Section('%section3%', 'Substitution for Section 3 Tag')
]
message.header = Header('X-Test9', 'Test9')
message.header = Header('X-Test10', 'Test10')
message.header = [
Header('X-Test11', 'Test11'),
Header('X-Test12', 'Test12')
]
message.category = Category('Category 1')
message.category = Category('Category 2')
message.category = [Category('Category 1'), Category('Category 2')]
message.custom_arg = CustomArg('marketing5', 'false')
message.custom_arg = CustomArg('transactional5', 'true')
message.custom_arg = [
CustomArg('marketing6', 'true'),
CustomArg('transactional6', 'false')
]
message.send_at = SendAt(1461775053)
message.batch_id = BatchId("HkJ5yLYULb7Rj8GKSx7u025ouWVlMgAi")
message.asm = Asm(GroupId(1), GroupsToDisplay([1, 2, 3, 4]))
message.ip_pool_name = IpPoolName("IP Pool Name")
mail_settings = MailSettings()
mail_settings.bcc_settings = BccSettings(
False, BccSettingsEmail("*****@*****.**"))
mail_settings.bypass_list_management = BypassListManagement(False)
mail_settings.footer_settings = FooterSettings(
True, FooterText("w00t"), FooterHtml("<string>w00t!<strong>"))
mail_settings.sandbox_mode = SandBoxMode(True)
mail_settings.spam_check = SpamCheck(True, SpamThreshold(5),
SpamUrl("https://example.com"))
message.mail_settings = mail_settings
tracking_settings = TrackingSettings()
tracking_settings.click_tracking = ClickTracking(True, False)
tracking_settings.open_tracking = OpenTracking(
True, OpenTrackingSubstitutionTag("open_tracking"))
tracking_settings.subscription_tracking = SubscriptionTracking(
True, SubscriptionText("Goodbye"),
SubscriptionHtml("<strong>Goodbye!</strong>"),
SubscriptionSubstitutionTag("unsubscribe"))
tracking_settings.ganalytics = Ganalytics(True,
UtmSource("utm_source"),
UtmMedium("utm_medium"),
UtmTerm("utm_term"),
UtmContent("utm_content"),
UtmCampaign("utm_campaign"))
message.tracking_settings = tracking_settings
self.assertEqual(
message.get(),
json.loads(r'''{
"asm": {
"group_id": 1,
"groups_to_display": [
1,
2,
3,
4
]
},
"attachments": [
{
"content": "base64 encoded content 3",
"content_id": "Content ID 3",
"disposition": "inline",
"filename": "banner2.png",
"type": "image/png"
},
{
"content": "base64 encoded content 2",
"content_id": "Content ID 2",
"disposition": "inline",
"filename": "banner.png",
"type": "image/png"
},
{
"content": "base64 encoded content 1",
"content_id": "Content ID 1",
"disposition": "attachment",
"filename": "balance_001.pdf",
"type": "application/pdf"
}
],
"batch_id": "HkJ5yLYULb7Rj8GKSx7u025ouWVlMgAi",
"categories": [
"Category 2",
"Category 1",
"Category 2",
"Category 1"
],
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
},
{
"type": "text/calendar",
"value": "Party Time!!"
},
{
"type": "text/custom",
"value": "Party Time 2!!"
}
],
"custom_args": {
"marketing5": "false",
"marketing6": "true",
"transactional5": "true",
"transactional6": "false"
},
"from": {
"email": "*****@*****.**",
"name": "Twilio SendGrid"
},
"headers": {
"X-Test10": "Test10",
"X-Test11": "Test11",
"X-Test12": "Test12",
"X-Test9": "Test9"
},
"ip_pool_name": "IP Pool Name",
"mail_settings": {
"bcc": {
"email": "*****@*****.**",
"enable": false
},
"bypass_list_management": {
"enable": false
},
"footer": {
"enable": true,
"html": "<string>w00t!<strong>",
"text": "w00t"
},
"sandbox_mode": {
"enable": true
},
"spam_check": {
"enable": true,
"post_to_url": "https://example.com",
"threshold": 5
}
},
"personalizations": [
{
"bcc": [
{
"email": "*****@*****.**",
"name": "Example User7"
},
{
"email": "*****@*****.**",
"name": "Example User8"
},
{
"email": "*****@*****.**",
"name": "Example User9"
}
],
"cc": [
{
"email": "*****@*****.**",
"name": "Example User4"
},
{
"email": "*****@*****.**",
"name": "Example User5"
},
{
"email": "*****@*****.**",
"name": "Example User6"
}
],
"custom_args": {
"marketing1": "true",
"marketing2": "false",
"transactional1": "false",
"transactional2": "true"
},
"headers": {
"X-Test1": "Test1",
"X-Test2": "Test2",
"X-Test3": "Test3",
"X-Test4": "Test4"
},
"send_at": 1461775051,
"subject": "Sending with SendGrid is Fun 0",
"substitutions": {
"%city1%": "Example City 1",
"%city2%": "Example City 2",
"%name1%": "Example Name 1",
"%name2%": "Example Name 2"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example User1"
},
{
"email": "*****@*****.**",
"name": "Example User2"
},
{
"email": "*****@*****.**",
"name": "Example User3"
}
]
},
{
"bcc": [
{
"email": "*****@*****.**",
"name": "Example User16"
},
{
"email": "*****@*****.**",
"name": "Example User17"
},
{
"email": "*****@*****.**",
"name": "Example User18"
}
],
"cc": [
{
"email": "*****@*****.**",
"name": "Example User13"
},
{
"email": "*****@*****.**",
"name": "Example User14"
},
{
"email": "*****@*****.**",
"name": "Example User15"
}
],
"custom_args": {
"marketing3": "true",
"marketing4": "false",
"transactional3": "false",
"transactional4": "true"
},
"headers": {
"X-Test5": "Test5",
"X-Test6": "Test6",
"X-Test7": "Test7",
"X-Test8": "Test8"
},
"send_at": 1461775052,
"subject": "Sending with SendGrid is Fun 1",
"substitutions": {
"%city3%": "Example City 3",
"%city4%": "Example City 4",
"%name3%": "Example Name 3",
"%name4%": "Example Name 4"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example User10"
},
{
"email": "*****@*****.**",
"name": "Example User11"
},
{
"email": "*****@*****.**",
"name": "Example User12"
}
]
}
],
"reply_to": {
"email": "*****@*****.**",
"name": "Twilio SendGrid Reply"
},
"sections": {
"%section1%": "Substitution for Section 1 Tag",
"%section2%": "Substitution for Section 2 Tag",
"%section3%": "Substitution for Section 3 Tag"
},
"send_at": 1461775053,
"subject": "Sending with SendGrid is Fun 2",
"template_id": "13b8f94f-bcae-4ec6-b752-70d6cb59f932",
"tracking_settings": {
"click_tracking": {
"enable": true,
"enable_text": false
},
"ganalytics": {
"enable": true,
"utm_campaign": "utm_campaign",
"utm_content": "utm_content",
"utm_medium": "utm_medium",
"utm_source": "utm_source",
"utm_term": "utm_term"
},
"open_tracking": {
"enable": true,
"substitution_tag": "open_tracking"
},
"subscription_tracking": {
"enable": true,
"html": "<strong>Goodbye!</strong>",
"substitution_tag": "unsubscribe",
"text": "Goodbye"
}
}
}'''))
def test_dynamic_template_data(self):
self.maxDiff = None
to_emails = [
To(email='*****@*****.**',
name='Example To 0 Name',
dynamic_template_data=DynamicTemplateData(
{'name': 'Example 0 Name'})),
To(email='*****@*****.**',
name='Example To 1 Name',
dynamic_template_data={'name': 'Example 1 Name'})
]
message = Mail(from_email=From('*****@*****.**',
'Example From Name'),
to_emails=to_emails,
subject=Subject('Hi!'),
plain_text_content='Hello!',
html_content='<strong>Hello!</strong>',
is_multiple=True)
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "Hello!"
},
{
"type": "text/html",
"value": "<strong>Hello!</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"dynamic_template_data": {
"name": "Example 1 Name"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example To 1 Name"
}
]
},
{
"dynamic_template_data": {
"name": "Example 0 Name"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example To 0 Name"
}
]
}
],
"subject": "Hi!"
}'''))
def test_multiple_emails_to_multiple_recipients(self):
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent,
Substitution)
self.maxDiff = None
to_emails = [
To(email='*****@*****.**',
name='Example Name 0',
substitutions=[
Substitution('-name-', 'Example Name Substitution 0'),
Substitution('-github-', 'https://example.com/test0'),
],
subject=Subject('Override Global Subject')),
To(email='*****@*****.**',
name='Example Name 1',
substitutions=[
Substitution('-name-', 'Example Name Substitution 1'),
Substitution('-github-', 'https://example.com/test1'),
])
]
global_substitutions = Substitution('-time-', '2019-01-01 00:00:00')
message = Mail(
from_email=From('*****@*****.**', 'Example From Name'),
to_emails=to_emails,
subject=Subject('Hi -name-'),
plain_text_content=PlainTextContent(
'Hello -name-, your URL is -github-, email sent at -time-'),
html_content=HtmlContent(
'<strong>Hello -name-, your URL is <a href=\"-github-\">here</a></strong> email sent at -time-'
),
global_substitutions=global_substitutions,
is_multiple=True)
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "Hello -name-, your URL is -github-, email sent at -time-"
},
{
"type": "text/html",
"value": "<strong>Hello -name-, your URL is <a href=\"-github-\">here</a></strong> email sent at -time-"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"substitutions": {
"-github-": "https://example.com/test1",
"-name-": "Example Name Substitution 1",
"-time-": "2019-01-01 00:00:00"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example Name 1"
}
]
},
{
"subject": "Override Global Subject",
"substitutions": {
"-github-": "https://example.com/test0",
"-name-": "Example Name Substitution 0",
"-time-": "2019-01-01 00:00:00"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example Name 0"
}
]
}
],
"subject": "Hi -name-"
}'''))
def test_multiple_emails_to_multiple_recipients(self):
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent,
Substitution)
self.maxDiff = None
to_emails = [
To(email='*****@*****.**',
name='Example Name 0',
substitutions=[
Substitution('-name-', 'Example Name Substitution 0'),
Substitution('-github-', 'https://example.com/test0'),
],
subject=Subject('Override Global Subject')),
To(email='*****@*****.**',
name='Example Name 1',
substitutions=[
Substitution('-name-', 'Example Name Substitution 1'),
Substitution('-github-', 'https://example.com/test1'),
])
]
global_substitutions = Substitution('-time-', '2019-01-01 00:00:00')
message = Mail(
from_email=From('*****@*****.**', 'Example From Name'),
to_emails=to_emails,
subject=Subject('Hi -name-'),
plain_text_content=PlainTextContent(
'Hello -name-, your URL is -github-, email sent at -time-'),
html_content=HtmlContent(
'<strong>Hello -name-, your URL is <a href=\"-github-\">here</a></strong> email sent at -time-'),
global_substitutions=global_substitutions,
is_multiple=True)
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "Hello -name-, your URL is -github-, email sent at -time-"
},
{
"type": "text/html",
"value": "<strong>Hello -name-, your URL is <a href=\"-github-\">here</a></strong> email sent at -time-"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"substitutions": {
"-github-": "https://example.com/test1",
"-name-": "Example Name Substitution 1",
"-time-": "2019-01-01 00:00:00"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example Name 1"
}
]
},
{
"subject": "Override Global Subject",
"substitutions": {
"-github-": "https://example.com/test0",
"-name-": "Example Name Substitution 0",
"-time-": "2019-01-01 00:00:00"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example Name 0"
}
]
}
],
"subject": "Hi -name-"
}''')
)
def _apply_blacklist(self, message, mode, rec=0):
"""
Apply the global blacklist to this message.
This method will modify the headers of the message or decide to
discard the whole message based on the global blacklist.
:param messag: A :class:`email.message` object.
:param mode: One of :attr:`FUCore.BLACKLIST_MODES`
:returns: The modified message or None if the message should be dropped.
"""
if not mode.lower() in FUCore.BLACKLIST_MODES:
self._log(
'!!! Invalid blacklist mode "{0}", not modifying message.',
mode,
rec=rec)
return message
mfrom = message.get('From', '').split('<')[-1].split('>')[0]
sender = message.get('Sender', '').split('<')[-1].split('>')[0]
if not mfrom and not msender:
self._log('!!! Message has neiter "From:" nor "Sender:" headers!',
rec=rec)
return message
list_entry = self._blacklist.get(
mfrom.lower(), self._blacklist.get(sender.lower(), None))
if not list_entry:
return message
self._log('--- applying blacklist rule {0} to message',
str(list_entry),
rec=rec)
if not list_entry['action'].lower() in ['d', 'n', 'e', 'ne', 'en']:
# invalid
self._log('!!! unsupported blacklist rule "{0}"',
list_entry['action'],
rec=rec)
return message
add_expires = False
add_xnay = False
if list_entry['action'].lower(
) == 'd' and not mode.lower() == 'reactor':
self._log('--- blacklist rule "drop"', rec=rec, verbosity=2)
return None
elif mode.lower() == 'news2mail':
#D=N=E=NE/EN=drop
self._log('--- blacklist rule D=N=E=NE/EN "drop"',
rec=rec,
verbosity=2)
return None
elif mode.lower() == 'mail2news':
if list_entry['action'].lower() in ['ne', 'en', 'n']:
add_xnay = True
if list_entry['action'].lower() in ['ne', 'en', 'e']:
add_expires = True
elif mode.lower() == 'reactor':
if list_entry['action'].lower() in ['ne', 'en', 'n', 'd']:
add_xnay = True
if list_entry['action'].lower() in ['ne', 'en', 'e']:
add_expires = True
if add_xnay:
self._log('--- blacklist rule "xnay"', rec=rec, verbosity=2)
try:
message.replace_header('X-No-Archive', 'yes')
except KeyError:
message._headers.append(('X-No-Archive', 'yes'))
if add_expires:
if not list_entry.get('param', None):
self._log('!!! blacklist rule "expires" missing a parameter',
rec=rec)
else:
try:
delay = long(list_entry['param'])
expires = time.strftime(
r"%d %b %Y %H:%M:%S %z",
time.localtime(time.time() + (86400 * delay)))
self._log('--- blacklist rule "expires" => {0}',
expires,
rec=rec,
verbosity=2)
try:
message.replace_header('Expires', expires)
except KeyError:
message._headers.append(('Expires', expires))
except ValueError:
self.log(
'!!! blacklist rule "expires" needs a *numeric* parameter.'
)
return message
def _process(self, message, rec=0):
"""
Recursively scan and filter a MIME message.
_process will scan the passed message part for invalid headers
as well as mailman signatures and modify them according to
the global settings.
Generic modifications include:
* fixing of broken **References:** and **In-Reply-To** headers
* generic header filtering (see :meth:`FuCore._filter_headers`)
* removal of Mailman or Mailman-like headers (see :meth:`_mutate_part`)
Args:
message: a :class:`email.message` object containing a set of MIME parts
rec: Recursion level used to prettify log messages.
Returns:
A (probably) filtered / modified :class:`email.message` object.
"""
mm_parts = 0
text_parts = 0
mailman_sig = Reactor.MAILMAN_SIG
self._log('>>> processing {0}', message.get_content_type(), rec=rec)
if self._conf.complex_footer:
self._log('--- using complex mailman filter', rec=rec)
mailman_sig = Reactor.MAILMAN_COMPLEX
if message.is_multipart():
parts = message._payload
else:
parts = [
message,
]
list_tag = self._find_list_tag(message, rec)
reference = message.get('References', None)
in_reply = message.get('In-Reply-To', None)
x_mailman = message.get('X-Mailman-Version', None)
message._headers = self._filter_headers(list_tag, message._headers,
self._conf.outlook_hacks,
self._conf.fix_dateline, rec)
if in_reply and not reference and rec == 0:
# set References: to In-Reply-To: if where in toplevel
# and References was not set properly
self._log('--- set References: {0}', in_reply, rec=rec)
try:
# uncertain this will ever happen..
message.replace_header('References', in_reply)
except KeyError:
message._headers.append(('References', in_reply))
for i in xrange(len(parts) - 1, -1, -1):
# again, crude since we mutate the list while iterating it..
# the whole reason is the deeply nested structure of email.message
p = parts[i]
ct = p.get_content_maintype()
cs = p.get_content_subtype()
ce = p.get('Content-Transfer-Encoding', None)
cb = p.get_boundary()
self._log('-- [ct = {0}, cs = {1}, ce = <{2}>]',
ct,
cs,
ce,
rec=rec)
if ct == 'text':
text_parts += 1
payload = p.get_payload(decode=True)
self._log('--- scan: """{0}"""', payload, rec=rec, verbosity=3)
if mailman_sig[0].search(payload) and \
mailman_sig[1].match(payload.split('\n')[0]):
self._log('*** removing this part', rec=rec)
self._log('--- """{0}"""', payload, rec=rec, verbosity=2)
message._payload.remove(p)
text_parts -= 1
mm_parts += 1
elif mailman_sig[0].search(payload):
self._log('--- trying to mutate..', rec=rec)
(use, mutation) = self._mutate_part(payload, rec)
if use:
self._log('*** mutated this part', rec=rec)
self._log('--- """{0}"""',
payload,
rec=rec,
verbosity=2)
payload = mutation
mm_parts += 1
# if it was encoded we need to re-encode it
# to keep SMIME happy
if ce == 'base64':
payload = payload.encode('base64')
elif ce == 'quoted-printable':
payload = quopri.encodestring(payload)
p.set_payload(payload)
elif ct == 'message' or \
(ct == 'multipart' and cs in ['alternative', 'mixed']):
p = self._process(p, rec + 1)
else:
self._log('--- what about {0}?',
p.get_content_type(),
rec=rec)
if rec == 0:
self._log('--- [mm_parts: {0}, text_parts: {1}, x_mailman: {0}]',
mm_parts,
text_parts,
x_mailman,
rec=rec)
if x_mailman and mm_parts and not text_parts:
# if we have
# - modified the content
# - no text parts left in outer message
# - a valid X-Mailmann-Version:
# --> remove outer message
self._log('!!! beheading this one..', rec=rec)
mm = message._payload[0]
for h in message._headers:
if h[0] == 'Content-Type':
continue
try:
mm.replace_header(h[0], h[1])
except KeyError:
mm._headers.append(h)
return mm
return message
def test_single_email_to_a_single_recipient_with_dynamic_templates(self):
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent)
self.maxDiff = None
message = Mail(
from_email=From('*****@*****.**', 'Example From Name'),
to_emails=To('*****@*****.**', 'Example To Name'),
subject=Subject('Sending with SendGrid is Fun'),
plain_text_content=PlainTextContent(
'and easy to do anywhere, even with Python'),
html_content=HtmlContent(
'<strong>and easy to do anywhere, even with Python</strong>'))
message.dynamic_template_data = DynamicTemplateData({
"total": "$ 239.85",
"items": [
{
"text": "New Line Sneakers",
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/8dda1131320a6d978b515cc04ed479df259a458d5d45d58b6b381cae0bf9588113e80ef912f69e8c4cc1ef1a0297e8eefdb7b270064cc046b79a44e21b811802.png",
"price": "$ 79.95"
},
{
"text": "Old Line Sneakers",
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/3629f54390ead663d4eb7c53702e492de63299d7c5f7239efdc693b09b9b28c82c924225dcd8dcb65732d5ca7b7b753c5f17e056405bbd4596e4e63a96ae5018.png",
"price": "$ 79.95"
},
{
"text": "Blue Line Sneakers",
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/00731ed18eff0ad5da890d876c456c3124a4e44cb48196533e9b95fb2b959b7194c2dc7637b788341d1ff4f88d1dc88e23f7e3704726d313c57f350911dd2bd0.png",
"price": "$ 79.95"
}
],
"receipt": True,
"name": "Sample Name",
"address01": "1234 Fake St.",
"address02": "Apt. 123",
"city": "Place",
"state": "CO",
"zip": "80202"
})
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"dynamic_template_data": {
"address01": "1234 Fake St.",
"address02": "Apt. 123",
"city": "Place",
"items": [
{
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/8dda1131320a6d978b515cc04ed479df259a458d5d45d58b6b381cae0bf9588113e80ef912f69e8c4cc1ef1a0297e8eefdb7b270064cc046b79a44e21b811802.png",
"price": "$ 79.95",
"text": "New Line Sneakers"
},
{
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/3629f54390ead663d4eb7c53702e492de63299d7c5f7239efdc693b09b9b28c82c924225dcd8dcb65732d5ca7b7b753c5f17e056405bbd4596e4e63a96ae5018.png",
"price": "$ 79.95",
"text": "Old Line Sneakers"
},
{
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/00731ed18eff0ad5da890d876c456c3124a4e44cb48196533e9b95fb2b959b7194c2dc7637b788341d1ff4f88d1dc88e23f7e3704726d313c57f350911dd2bd0.png",
"price": "$ 79.95",
"text": "Blue Line Sneakers"
}
],
"name": "Sample Name",
"receipt": true,
"state": "CO",
"total": "$ 239.85",
"zip": "80202"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example To Name"
}
]
}
],
"subject": "Sending with SendGrid is Fun"
}''')
)
def store_and_build_forward_message(self, form, boundary=None,
max_bytes_per_blob=None,
max_bytes_total=None,
bucket_name=None):
"""Reads form data, stores blobs data and builds the forward request.
This finds all of the file uploads in a set of form fields, converting them
into blobs and storing them in the blobstore. It also generates the HTTP
request to forward to the user's application.
Args:
form: cgi.FieldStorage instance representing the whole form derived from
original POST data.
boundary: The optional boundary to use for the resulting form. If omitted,
one is randomly generated.
max_bytes_per_blob: The maximum size in bytes that any single blob
in the form is allowed to be.
max_bytes_total: The maximum size in bytes that the total of all blobs
in the form is allowed to be.
bucket_name: The name of the Google Storage bucket to store the uploaded
files.
Returns:
A tuple (content_type, content_text), where content_type is the value of
the Content-Type header, and content_text is a string containing the body
of the HTTP request to forward to the application.
Raises:
webob.exc.HTTPException: The upload failed.
"""
message = multipart.MIMEMultipart('form-data', boundary)
creation = self._now_func()
total_bytes_uploaded = 0
created_blobs = []
mime_type_error = None
too_many_conflicts = False
upload_too_large = False
filename_too_large = False
content_type_too_large = False
# Extract all of the individual form items out of the FieldStorage.
form_items = []
# Sorting of forms is done merely to make testing a little easier since
# it means blob-keys are generated in a predictable order.
for key in sorted(form):
form_item = form[key]
if isinstance(form_item, list):
form_items.extend(form_item)
else:
form_items.append(form_item)
for form_item in form_items:
disposition_parameters = {'name': form_item.name}
variable = email.message.Message()
if form_item.filename is None:
# Copy as is
variable.add_header('Content-Type', 'text/plain')
variable.set_payload(form_item.value)
else:
# If there is no filename associated with this field it means that the
# file form field was not filled in. This blob should not be created
# and forwarded to success handler.
if not form_item.filename:
continue
disposition_parameters['filename'] = form_item.filename
try:
main_type, sub_type = _split_mime_type(form_item.type)
except _InvalidMIMETypeFormatError as ex:
mime_type_error = str(ex)
break
# Seek to the end of file and use the pos as the length.
form_item.file.seek(0, os.SEEK_END)
content_length = form_item.file.tell()
form_item.file.seek(0)
total_bytes_uploaded += content_length
if max_bytes_per_blob is not None:
if content_length > max_bytes_per_blob:
upload_too_large = True
break
if max_bytes_total is not None:
if total_bytes_uploaded > max_bytes_total:
upload_too_large = True
break
if form_item.filename is not None:
if len(form_item.filename) > _MAX_STRING_NAME_LENGTH:
filename_too_large = True
break
if form_item.type is not None:
if len(form_item.type) > _MAX_STRING_NAME_LENGTH:
content_type_too_large = True
break
# Compute the MD5 hash of the upload.
digester = hashlib.md5()
while True:
block = form_item.file.read(1 << 20)
if not block:
break
digester.update(block)
form_item.file.seek(0)
# Create the external body message containing meta-data about the blob.
external = email.message.Message()
external.add_header('Content-Type', '%s/%s' % (main_type, sub_type),
**form_item.type_options)
# NOTE: This is in violation of RFC 2616 (Content-MD5 should be the
# base-64 encoding of the binary hash, not the hex digest), but it is
# consistent with production.
content_md5 = base64.urlsafe_b64encode(digester.hexdigest())
# Create header MIME message
headers = dict(form_item.headers)
for name in _STRIPPED_FILE_HEADERS:
if name in headers:
del headers[name]
headers['Content-Length'] = str(content_length)
headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
blobstore._format_creation(creation))
headers['Content-MD5'] = content_md5
gs_filename = None
if bucket_name:
random_key = str(self._generate_blob_key())
gs_filename = '%s/fake-%s' % (bucket_name, random_key)
headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = (
blobstore.GS_PREFIX + gs_filename)
for key, value in headers.items():
external.add_header(key, value)
# Add disposition parameters (a clone of the outer message's field).
if not external.get('Content-Disposition'):
external.add_header('Content-Disposition', 'form-data',
**disposition_parameters)
base64_encoding = (form_item.headers.get('Content-Transfer-Encoding') ==
'base64')
content_type, blob_file, filename = self._preprocess_data(
external['content-type'],
form_item.file,
form_item.filename,
base64_encoding)
# Store the actual contents to storage.
if gs_filename:
info_entity = self.store_gs_file(
content_type, gs_filename, blob_file, filename)
else:
try:
info_entity = self.store_blob(content_type, filename,
digester, blob_file, creation)
except _TooManyConflictsError:
too_many_conflicts = True
break
# Track created blobs in case we need to roll them back.
created_blobs.append(info_entity)
variable.add_header('Content-Type', 'message/external-body',
access_type=blobstore.BLOB_KEY_HEADER,
blob_key=info_entity.key().name())
variable.set_payload([external])
# Set common information.
variable.add_header('Content-Disposition', 'form-data',
**disposition_parameters)
message.attach(variable)
if (mime_type_error or too_many_conflicts or upload_too_large or
filename_too_large or content_type_too_large):
for blob in created_blobs:
datastore.Delete(blob)
if mime_type_error:
self.abort(400, detail=mime_type_error)
elif too_many_conflicts:
self.abort(500, detail='Could not generate a blob key.')
elif upload_too_large:
self.abort(413)
else:
if filename_too_large:
invalid_field = 'filename'
elif content_type_too_large:
invalid_field = 'Content-Type'
detail = 'The %s exceeds the maximum allowed length of %s.' % (
invalid_field, _MAX_STRING_NAME_LENGTH)
self.abort(400, detail=detail)
message_out = io.StringIO()
gen = email.generator.Generator(message_out, maxheaderlen=0)
gen.flatten(message, unixfrom=False)
# Get the content text out of the message.
message_text = message_out.getvalue()
content_start = message_text.find('\n\n') + 2
content_text = message_text[content_start:]
content_text = content_text.replace('\n', '\r\n')
return message.get('Content-Type'), content_text
def _process(self, message, rec=0):
"""
Recursively scan and filter a MIME message.
_process will scan the passed message part for invalid headers
as well as mailman signatures and modify them according to
the global settings.
Generic modifications include:
* fixing of broken **References:** and **In-Reply-To** headers
* generic header filtering (see :meth:`FuCore._filter_headers`)
* removal of Mailman or Mailman-like headers (see :meth:`_mutate_part`)
Args:
message: a :class:`email.message` object containing a set of MIME parts
rec: Recursion level used to prettify log messages.
Returns:
A (probably) filtered / modified :class:`email.message` object.
"""
mm_parts = 0
text_parts = 0
mailman_sig = Reactor.MAILMAN_SIG
self._log('>>> processing {0}', message.get_content_type(), rec=rec)
if self._conf.complex_footer:
self._log('--- using complex mailman filter', rec=rec)
mailman_sig = Reactor.MAILMAN_COMPLEX
if message.is_multipart():
parts = message._payload
else:
parts = [message,]
list_tag = self._find_list_tag(message, rec)
reference = message.get('References', None)
in_reply = message.get('In-Reply-To', None)
x_mailman = message.get('X-Mailman-Version', None)
message._headers = self._filter_headers(list_tag, message._headers,
self._conf.outlook_hacks,
self._conf.fix_dateline, rec)
if in_reply and not reference and rec == 0:
# set References: to In-Reply-To: if where in toplevel
# and References was not set properly
self._log('--- set References: {0}', in_reply, rec=rec)
try:
# uncertain this will ever happen..
message.replace_header('References', in_reply)
except KeyError:
message._headers.append(('References', in_reply))
for i in xrange(len(parts) - 1, -1, -1):
# again, crude since we mutate the list while iterating it..
# the whole reason is the deeply nested structure of email.message
p = parts[i]
ct = p.get_content_maintype()
cs = p.get_content_subtype()
ce = p.get('Content-Transfer-Encoding', None)
cb = p.get_boundary()
self._log('-- [ct = {0}, cs = {1}, ce = <{2}>]', ct, cs, ce,
rec=rec)
if ct == 'text':
text_parts += 1
payload = p.get_payload(decode=True)
self._log('--- scan: """{0}"""', payload, rec=rec, verbosity=3)
if mailman_sig[0].search(payload) and \
mailman_sig[1].match(payload.split('\n')[0]):
self._log('*** removing this part', rec=rec)
self._log('--- """{0}"""', payload, rec=rec, verbosity=2)
message._payload.remove(p)
text_parts -= 1
mm_parts += 1
elif mailman_sig[0].search(payload):
self._log('--- trying to mutate..', rec=rec)
(use, mutation) = self._mutate_part(payload, rec)
if use:
self._log('*** mutated this part', rec=rec)
self._log('--- """{0}"""', payload, rec=rec, verbosity=2)
payload = mutation
mm_parts += 1
# if it was encoded we need to re-encode it
# to keep SMIME happy
if ce == 'base64':
payload = payload.encode('base64')
elif ce == 'quoted-printable':
payload = quopri.encodestring(payload)
p.set_payload(payload)
elif ct == 'message' or \
(ct == 'multipart' and cs in ['alternative', 'mixed']):
p = self._process(p, rec + 1)
else:
self._log('--- what about {0}?', p.get_content_type(), rec=rec)
if rec == 0:
self._log('--- [mm_parts: {0}, text_parts: {1}, x_mailman: {0}]',
mm_parts, text_parts, x_mailman, rec=rec)
if x_mailman and mm_parts and not text_parts:
# if we have
# - modified the content
# - no text parts left in outer message
# - a valid X-Mailmann-Version:
# --> remove outer message
self._log('!!! beheading this one..', rec=rec)
mm = message._payload[0]
for h in message._headers:
if h[0] == 'Content-Type':
continue
try:
mm.replace_header(h[0], h[1])
except KeyError:
mm._headers.append(h)
return mm
return message
def get_quarantined_messages(self):
"""
Returns an iterator over the messages currently in the quarantine.
"""
for message, error in self._quarantine.values():
yield message.get(), error
def _find_list_tag(self, message, rec=0, plain=False):
"""
Filter *message* for a valid list tag.
This method will scan the passed in messages headers looking for a hint
to the used list-tag.
The following headers will be checked (in order):
* X-SynFU-Tags (explicit List-Tags supplied by synfu-news2mail)
* [X-]List-Post
* [X-]List-Id
* [X-]AF-Envelope-to
If any of these is found it will be converted into a regular expression
which can be used to remove the List-Tag from arbitrary headers.
:param message: A :class:`email.message` object.
:param rec: Optional recursion level used to indent messages.
:param plain: If :const:`True` return the plain List-Tag (no regexp)
:returns: Either a :class:`re.SRE_PATTERN` or a string
"""
tag = message.get('X-SynFU-Tags', None)
lp = message.get('List-Post', message.get('X-List-Post', None))
lid = message.get('List-Id' , message.get('X-List-Id', None))
evl = message.get('AF-Envelope-to', message.get('X-AF-Envelope-to', None))
tag_base = None
if tag:
tag = email.header.decode_header(tag)[0][0]
self._log('--- using supplied SynFU tag hints', rec=rec)
tag_base = '({0})'.format('|'.join(x.strip() for x in tag.split(',') if x.strip()))
# preffer List-Id if we have it
elif lid:
lid = email.header.decode_header(lid)[-1][0]
tag_base = lid.split('<')[-1].split('.')[0].strip()
elif lp:
lp = email.header.decode_header(lp)[0][0]
try:
tag_base = lp.split('mailto:')[1].split('@')[0]
except IndexError:
tag_base = None
elif evl:
evl = email.header.decode_header(evl)[0][0]
try:
tag_base = evl.split('@')[0]
except IndexError:
tag_base = None
if plain:
return tag_base
if tag_base:
self._log('--- list tag: "[*{0}*]"', format(tag_base), rec=rec)
return re.compile('(?i)\[[^[]*{0}[^]]*\]'.format(tag_base))
self._log('--- no list tag found', rec=rec)
# return 'moab'
return re.compile('(?i)\s*\[[^]]*(?# This is here to confuse people)\]\s*')
def _apply_blacklist(self, message, mode, rec=0):
"""
Apply the global blacklist to this message.
This method will modify the headers of the message or decide to
discard the whole message based on the global blacklist.
:param messag: A :class:`email.message` object.
:param mode: One of :attr:`FUCore.BLACKLIST_MODES`
:returns: The modified message or None if the message should be dropped.
"""
if not mode.lower() in FUCore.BLACKLIST_MODES:
self._log('!!! Invalid blacklist mode "{0}", not modifying message.', mode, rec=rec)
return message
mfrom = message.get('From', '').split('<')[-1].split('>')[0]
sender = message.get('Sender', '').split('<')[-1].split('>')[0]
if not mfrom and not msender:
self._log('!!! Message has neiter "From:" nor "Sender:" headers!', rec=rec)
return message
list_entry = self._blacklist.get(mfrom.lower(), self._blacklist.get(sender.lower(), None))
if not list_entry:
return message
self._log('--- applying blacklist rule {0} to message', str(list_entry), rec=rec)
if not list_entry['action'].lower() in ['d', 'n', 'e', 'ne', 'en']:
# invalid
self._log('!!! unsupported blacklist rule "{0}"', list_entry['action'], rec=rec)
return message
add_expires=False
add_xnay=False
if list_entry['action'].lower() == 'd' and not mode.lower() == 'reactor':
self._log('--- blacklist rule "drop"', rec=rec, verbosity=2)
return None
elif mode.lower() == 'news2mail':
#D=N=E=NE/EN=drop
self._log('--- blacklist rule D=N=E=NE/EN "drop"', rec=rec, verbosity=2)
return None
elif mode.lower() == 'mail2news':
if list_entry['action'].lower() in ['ne', 'en','n']:
add_xnay = True
if list_entry['action'].lower() in ['ne', 'en', 'e']:
add_expires = True
elif mode.lower() == 'reactor':
if list_entry['action'].lower() in ['ne','en','n','d']:
add_xnay = True
if list_entry['action'].lower() in ['ne','en','e']:
add_expires = True
if add_xnay:
self._log('--- blacklist rule "xnay"', rec=rec, verbosity=2)
try:
message.replace_header('X-No-Archive', 'yes')
except KeyError:
message._headers.append(('X-No-Archive', 'yes'))
if add_expires:
if not list_entry.get('param', None):
self._log('!!! blacklist rule "expires" missing a parameter', rec=rec)
else:
try:
delay=long(list_entry['param'])
expires=time.strftime(r"%d %b %Y %H:%M:%S %z", time.localtime(time.time() + (86400*delay)))
self._log('--- blacklist rule "expires" => {0}', expires, rec=rec, verbosity=2)
try:
message.replace_header('Expires', expires)
except KeyError:
message._headers.append(('Expires', expires))
except ValueError:
self.log('!!! blacklist rule "expires" needs a *numeric* parameter.')
return message
def test_single_email_to_a_single_recipient_with_dynamic_templates(self):
from sendgrid.helpers.mail import (Mail, From, To, Subject,
PlainTextContent, HtmlContent)
self.maxDiff = None
message = Mail(
from_email=From('*****@*****.**', 'Example From Name'),
to_emails=To('*****@*****.**', 'Example To Name'),
subject=Subject('Sending with SendGrid is Fun'),
plain_text_content=PlainTextContent(
'and easy to do anywhere, even with Python'),
html_content=HtmlContent(
'<strong>and easy to do anywhere, even with Python</strong>'))
message.dynamic_template_data = DynamicTemplateData({
"total":
"$ 239.85",
"items": [{
"text": "New Line Sneakers",
"image":
"https://marketing-image-production.s3.amazonaws.com/uploads/8dda1131320a6d978b515cc04ed479df259a458d5d45d58b6b381cae0bf9588113e80ef912f69e8c4cc1ef1a0297e8eefdb7b270064cc046b79a44e21b811802.png",
"price": "$ 79.95"
}, {
"text": "Old Line Sneakers",
"image":
"https://marketing-image-production.s3.amazonaws.com/uploads/3629f54390ead663d4eb7c53702e492de63299d7c5f7239efdc693b09b9b28c82c924225dcd8dcb65732d5ca7b7b753c5f17e056405bbd4596e4e63a96ae5018.png",
"price": "$ 79.95"
}, {
"text": "Blue Line Sneakers",
"image":
"https://marketing-image-production.s3.amazonaws.com/uploads/00731ed18eff0ad5da890d876c456c3124a4e44cb48196533e9b95fb2b959b7194c2dc7637b788341d1ff4f88d1dc88e23f7e3704726d313c57f350911dd2bd0.png",
"price": "$ 79.95"
}],
"receipt":
True,
"name":
"Sample Name",
"address01":
"1234 Fake St.",
"address02":
"Apt. 123",
"city":
"Place",
"state":
"CO",
"zip":
"80202"
})
self.assertEqual(
message.get(),
json.loads(r'''{
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
}
],
"from": {
"email": "*****@*****.**",
"name": "Example From Name"
},
"personalizations": [
{
"dynamic_template_data": {
"address01": "1234 Fake St.",
"address02": "Apt. 123",
"city": "Place",
"items": [
{
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/8dda1131320a6d978b515cc04ed479df259a458d5d45d58b6b381cae0bf9588113e80ef912f69e8c4cc1ef1a0297e8eefdb7b270064cc046b79a44e21b811802.png",
"price": "$ 79.95",
"text": "New Line Sneakers"
},
{
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/3629f54390ead663d4eb7c53702e492de63299d7c5f7239efdc693b09b9b28c82c924225dcd8dcb65732d5ca7b7b753c5f17e056405bbd4596e4e63a96ae5018.png",
"price": "$ 79.95",
"text": "Old Line Sneakers"
},
{
"image": "https://marketing-image-production.s3.amazonaws.com/uploads/00731ed18eff0ad5da890d876c456c3124a4e44cb48196533e9b95fb2b959b7194c2dc7637b788341d1ff4f88d1dc88e23f7e3704726d313c57f350911dd2bd0.png",
"price": "$ 79.95",
"text": "Blue Line Sneakers"
}
],
"name": "Sample Name",
"receipt": true,
"state": "CO",
"total": "$ 239.85",
"zip": "80202"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example To Name"
}
]
}
],
"subject": "Sending with SendGrid is Fun"
}'''))
def parse(data, codec=Codec): # noqa: C901
message = email.message_from_bytes(data)
# ################################################################## HEADERS
_, issuer = email.utils.parseaddr(message.get("from"))
subject = utils_parse.normalize(message.get("subject"))
recipient, identifier, domain = get_address_parts(message.get("to"))
new_tag, subject = utils_parse.consume_re(codec.re_new(), subject)
new = bool(new_tag)
external_id, subject = utils_parse.consume_re(codec.re_external_id(),
subject)
if not identifier and "in-reply-to" in message:
irt = get_address_parts(message.get("in-reply-to"))
if irt[0] == recipient and irt[2] == domain:
if irt[1] == "new":
new_tag = True
else:
identifier = irt[1]
identifier = irt[1]
if not identifier:
identifier, subject = utils_parse.consume_re(codec.re_id(), subject)
# ##################################################################### BODY
json_part, text_part, html_part, attachments = "", "", "", []
for part in message.walk():
# sub-parts are iterated over in this walk
if part.is_multipart():
continue
payload = part.get_payload(decode=True)
if geojson and part.get_content_type() == "application/geo+json":
try:
attachments.append((part.get_content_type(),
geojson.loads(payload.decode("utf-8"))))
except ValueError:
pass
elif part.get_content_type() == "application/json":
try:
json_part = json.loads(payload.decode("utf-8"))
except ValueError:
json_part = None
elif part.get_content_maintype() == "text":
payload = payload.decode(part.get_content_charset() or "utf-8")
# if multiple text/plain parts are given, concatenate
if part.get_content_subtype() == "plain":
text_part += payload
# if no text/plain version is given,
# get plain text version from HTML using BeautifulSoup
if part.get_content_subtype() == "html" and not text_part:
soup = bs4.BeautifulSoup(payload, "html.parser")
for item in soup(["script", "style"]):
item.extract()
html_part += "\n" + "\n".join(
l.strip() for l in soup.get_text().split("\n") if l)
# other subtypes of txt are considered attachments
if part.get_content_subtype() not in ("plain", "html"):
attachments.append((part.get_content_type(), payload))
# attachments
elif part.get_content_maintype() in ("image", "video", "application"):
attachments.append((part.get_content_type(), payload))
# ################################################################# FIX TEXT
text_part = (text_part or html_part).strip()
# remove signature
match = SIGNATURE_RE.search(text_part)
if match:
text_part = text_part[:match.start()].strip()
# ################################################################# FIX JSON
# if no `application/json` part was given, parse `text/plain` as YAML
# if we manage to parse YAML, remove this part from `text/plain`
# otherwise, leave the text part untouched
if not json_part:
try:
# only parse the first YAML document provided.
# this enables the user to provide YAML,
# use the '---' or '...' YAML document separators
# and provided plain text afterwards.
json_part = next(yaml.safe_load_all(text_part))
document_starts = [
e for e in yaml.parse(text_part)
if isinstance(e, yaml.events.DocumentStartEvent)
][1:]
if document_starts:
text_part = text_part[document_starts[1].end_mark.index + 1:]
else:
text_part = ""
except yaml.YAMLError:
json_part = None
return codec.update_item(
domain=domain,
issuer=issuer,
recipient=recipient,
identifier=identifier,
external_id=external_id,
new=new,
subject=subject,
json_part=json_part,
text_part=text_part,
attachments=attachments,
)
def test_kitchen_sink(self):
from sendgrid.helpers.mail import (
Mail, From, To, Cc, Bcc, Subject, Substitution, Header,
CustomArg, SendAt, Content, MimeType, Attachment, FileName,
FileContent, FileType, Disposition, ContentId, TemplateId,
Section, ReplyTo, Category, BatchId, Asm, GroupId, GroupsToDisplay,
IpPoolName, MailSettings, BccSettings, BccSettingsEmail,
BypassListManagement, FooterSettings, FooterText,
FooterHtml, SandBoxMode, SpamCheck, SpamThreshold, SpamUrl,
TrackingSettings, ClickTracking, SubscriptionTracking,
SubscriptionText, SubscriptionHtml, SubscriptionSubstitutionTag,
OpenTracking, OpenTrackingSubstitutionTag, Ganalytics,
UtmSource, UtmMedium, UtmTerm, UtmContent, UtmCampaign)
self.maxDiff = None
message = Mail()
# Define Personalizations
message.to = To('*****@*****.**', 'Example User1', p=0)
message.to = [
To('*****@*****.**', 'Example User2', p=0),
To('*****@*****.**', 'Example User3', p=0)
]
message.cc = Cc('*****@*****.**', 'Example User4', p=0)
message.cc = [
Cc('*****@*****.**', 'Example User5', p=0),
Cc('*****@*****.**', 'Example User6', p=0)
]
message.bcc = Bcc('*****@*****.**', 'Example User7', p=0)
message.bcc = [
Bcc('*****@*****.**', 'Example User8', p=0),
Bcc('*****@*****.**', 'Example User9', p=0)
]
message.subject = Subject('Sending with SendGrid is Fun 0', p=0)
message.header = Header('X-Test1', 'Test1', p=0)
message.header = Header('X-Test2', 'Test2', p=0)
message.header = [
Header('X-Test3', 'Test3', p=0),
Header('X-Test4', 'Test4', p=0)
]
message.substitution = Substitution('%name1%', 'Example Name 1', p=0)
message.substitution = Substitution('%city1%', 'Example City 1', p=0)
message.substitution = [
Substitution('%name2%', 'Example Name 2', p=0),
Substitution('%city2%', 'Example City 2', p=0)
]
message.custom_arg = CustomArg('marketing1', 'true', p=0)
message.custom_arg = CustomArg('transactional1', 'false', p=0)
message.custom_arg = [
CustomArg('marketing2', 'false', p=0),
CustomArg('transactional2', 'true', p=0)
]
message.send_at = SendAt(1461775051, p=0)
message.to = To('*****@*****.**', 'Example User10', p=1)
message.to = [
To('*****@*****.**', 'Example User11', p=1),
To('*****@*****.**', 'Example User12', p=1)
]
message.cc = Cc('*****@*****.**', 'Example User13', p=1)
message.cc = [
Cc('*****@*****.**', 'Example User14', p=1),
Cc('*****@*****.**', 'Example User15', p=1)
]
message.bcc = Bcc('*****@*****.**', 'Example User16', p=1)
message.bcc = [
Bcc('*****@*****.**', 'Example User17', p=1),
Bcc('*****@*****.**', 'Example User18', p=1)
]
message.header = Header('X-Test5', 'Test5', p=1)
message.header = Header('X-Test6', 'Test6', p=1)
message.header = [
Header('X-Test7', 'Test7', p=1),
Header('X-Test8', 'Test8', p=1)
]
message.substitution = Substitution('%name3%', 'Example Name 3', p=1)
message.substitution = Substitution('%city3%', 'Example City 3', p=1)
message.substitution = [
Substitution('%name4%', 'Example Name 4', p=1),
Substitution('%city4%', 'Example City 4', p=1)
]
message.custom_arg = CustomArg('marketing3', 'true', p=1)
message.custom_arg = CustomArg('transactional3', 'false', p=1)
message.custom_arg = [
CustomArg('marketing4', 'false', p=1),
CustomArg('transactional4', 'true', p=1)
]
message.send_at = SendAt(1461775052, p=1)
message.subject = Subject('Sending with SendGrid is Fun 1', p=1)
# The values below this comment are global to entire message
message.from_email = From('*****@*****.**', 'DX')
message.reply_to = ReplyTo('*****@*****.**', 'DX Reply')
message.subject = Subject('Sending with SendGrid is Fun 2')
message.content = Content(
MimeType.text,
'and easy to do anywhere, even with Python')
message.content = Content(
MimeType.html,
'<strong>and easy to do anywhere, even with Python</strong>')
message.content = [
Content('text/calendar', 'Party Time!!'),
Content('text/custom', 'Party Time 2!!')
]
message.attachment = Attachment(
FileContent('base64 encoded content 1'),
FileName('balance_001.pdf'),
FileType('application/pdf'),
Disposition('attachment'),
ContentId('Content ID 1'))
message.attachment = [
Attachment(
FileContent('base64 encoded content 2'),
FileName('banner.png'),
FileType('image/png'),
Disposition('inline'),
ContentId('Content ID 2')),
Attachment(
FileContent('base64 encoded content 3'),
FileName('banner2.png'),
FileType('image/png'),
Disposition('inline'),
ContentId('Content ID 3'))
]
message.template_id = TemplateId(
'13b8f94f-bcae-4ec6-b752-70d6cb59f932')
message.section = Section(
'%section1%', 'Substitution for Section 1 Tag')
message.section = [
Section('%section2%', 'Substitution for Section 2 Tag'),
Section('%section3%', 'Substitution for Section 3 Tag')
]
message.header = Header('X-Test9', 'Test9')
message.header = Header('X-Test10', 'Test10')
message.header = [
Header('X-Test11', 'Test11'),
Header('X-Test12', 'Test12')
]
message.category = Category('Category 1')
message.category = Category('Category 2')
message.category = [
Category('Category 1'),
Category('Category 2')
]
message.custom_arg = CustomArg('marketing5', 'false')
message.custom_arg = CustomArg('transactional5', 'true')
message.custom_arg = [
CustomArg('marketing6', 'true'),
CustomArg('transactional6', 'false')
]
message.send_at = SendAt(1461775053)
message.batch_id = BatchId("HkJ5yLYULb7Rj8GKSx7u025ouWVlMgAi")
message.asm = Asm(GroupId(1), GroupsToDisplay([1, 2, 3, 4]))
message.ip_pool_name = IpPoolName("IP Pool Name")
mail_settings = MailSettings()
mail_settings.bcc_settings = BccSettings(
False, BccSettingsEmail("*****@*****.**"))
mail_settings.bypass_list_management = BypassListManagement(False)
mail_settings.footer_settings = FooterSettings(
True, FooterText("w00t"), FooterHtml("<string>w00t!<strong>"))
mail_settings.sandbox_mode = SandBoxMode(True)
mail_settings.spam_check = SpamCheck(
True, SpamThreshold(5), SpamUrl("https://example.com"))
message.mail_settings = mail_settings
tracking_settings = TrackingSettings()
tracking_settings.click_tracking = ClickTracking(True, False)
tracking_settings.open_tracking = OpenTracking(
True, OpenTrackingSubstitutionTag("open_tracking"))
tracking_settings.subscription_tracking = SubscriptionTracking(
True,
SubscriptionText("Goodbye"),
SubscriptionHtml("<strong>Goodbye!</strong>"),
SubscriptionSubstitutionTag("unsubscribe"))
tracking_settings.ganalytics = Ganalytics(
True,
UtmSource("utm_source"),
UtmMedium("utm_medium"),
UtmTerm("utm_term"),
UtmContent("utm_content"),
UtmCampaign("utm_campaign"))
message.tracking_settings = tracking_settings
self.assertEqual(
message.get(),
json.loads(r'''{
"asm": {
"group_id": 1,
"groups_to_display": [
1,
2,
3,
4
]
},
"attachments": [
{
"content": "base64 encoded content 3",
"content_id": "Content ID 3",
"disposition": "inline",
"filename": "banner2.png",
"type": "image/png"
},
{
"content": "base64 encoded content 2",
"content_id": "Content ID 2",
"disposition": "inline",
"filename": "banner.png",
"type": "image/png"
},
{
"content": "base64 encoded content 1",
"content_id": "Content ID 1",
"disposition": "attachment",
"filename": "balance_001.pdf",
"type": "application/pdf"
}
],
"batch_id": "HkJ5yLYULb7Rj8GKSx7u025ouWVlMgAi",
"categories": [
"Category 2",
"Category 1",
"Category 2",
"Category 1"
],
"content": [
{
"type": "text/plain",
"value": "and easy to do anywhere, even with Python"
},
{
"type": "text/html",
"value": "<strong>and easy to do anywhere, even with Python</strong>"
},
{
"type": "text/calendar",
"value": "Party Time!!"
},
{
"type": "text/custom",
"value": "Party Time 2!!"
}
],
"custom_args": {
"marketing5": "false",
"marketing6": "true",
"transactional5": "true",
"transactional6": "false"
},
"from": {
"email": "*****@*****.**",
"name": "DX"
},
"headers": {
"X-Test10": "Test10",
"X-Test11": "Test11",
"X-Test12": "Test12",
"X-Test9": "Test9"
},
"ip_pool_name": "IP Pool Name",
"mail_settings": {
"bcc": {
"email": "*****@*****.**",
"enable": false
},
"bypass_list_management": {
"enable": false
},
"footer": {
"enable": true,
"html": "<string>w00t!<strong>",
"text": "w00t"
},
"sandbox_mode": {
"enable": true
},
"spam_check": {
"enable": true,
"post_to_url": "https://example.com",
"threshold": 5
}
},
"personalizations": [
{
"bcc": [
{
"email": "*****@*****.**",
"name": "Example User7"
},
{
"email": "*****@*****.**",
"name": "Example User8"
},
{
"email": "*****@*****.**",
"name": "Example User9"
}
],
"cc": [
{
"email": "*****@*****.**",
"name": "Example User4"
},
{
"email": "*****@*****.**",
"name": "Example User5"
},
{
"email": "*****@*****.**",
"name": "Example User6"
}
],
"custom_args": {
"marketing1": "true",
"marketing2": "false",
"transactional1": "false",
"transactional2": "true"
},
"headers": {
"X-Test1": "Test1",
"X-Test2": "Test2",
"X-Test3": "Test3",
"X-Test4": "Test4"
},
"send_at": 1461775051,
"subject": "Sending with SendGrid is Fun 0",
"substitutions": {
"%city1%": "Example City 1",
"%city2%": "Example City 2",
"%name1%": "Example Name 1",
"%name2%": "Example Name 2"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example User1"
},
{
"email": "*****@*****.**",
"name": "Example User2"
},
{
"email": "*****@*****.**",
"name": "Example User3"
}
]
},
{
"bcc": [
{
"email": "*****@*****.**",
"name": "Example User16"
},
{
"email": "*****@*****.**",
"name": "Example User17"
},
{
"email": "*****@*****.**",
"name": "Example User18"
}
],
"cc": [
{
"email": "*****@*****.**",
"name": "Example User13"
},
{
"email": "*****@*****.**",
"name": "Example User14"
},
{
"email": "*****@*****.**",
"name": "Example User15"
}
],
"custom_args": {
"marketing3": "true",
"marketing4": "false",
"transactional3": "false",
"transactional4": "true"
},
"headers": {
"X-Test5": "Test5",
"X-Test6": "Test6",
"X-Test7": "Test7",
"X-Test8": "Test8"
},
"send_at": 1461775052,
"subject": "Sending with SendGrid is Fun 1",
"substitutions": {
"%city3%": "Example City 3",
"%city4%": "Example City 4",
"%name3%": "Example Name 3",
"%name4%": "Example Name 4"
},
"to": [
{
"email": "*****@*****.**",
"name": "Example User10"
},
{
"email": "*****@*****.**",
"name": "Example User11"
},
{
"email": "*****@*****.**",
"name": "Example User12"
}
]
}
],
"reply_to": {
"email": "*****@*****.**",
"name": "DX Reply"
},
"sections": {
"%section1%": "Substitution for Section 1 Tag",
"%section2%": "Substitution for Section 2 Tag",
"%section3%": "Substitution for Section 3 Tag"
},
"send_at": 1461775053,
"subject": "Sending with SendGrid is Fun 2",
"template_id": "13b8f94f-bcae-4ec6-b752-70d6cb59f932",
"tracking_settings": {
"click_tracking": {
"enable": true,
"enable_text": false
},
"ganalytics": {
"enable": true,
"utm_campaign": "utm_campaign",
"utm_content": "utm_content",
"utm_medium": "utm_medium",
"utm_source": "utm_source",
"utm_term": "utm_term"
},
"open_tracking": {
"enable": true,
"substitution_tag": "open_tracking"
},
"subscription_tracking": {
"enable": true,
"html": "<strong>Goodbye!</strong>",
"substitution_tag": "unsubscribe",
"text": "Goodbye"
}
}
}''')
)
def get_msgid_from_stdin():
if not sys.stdin.isatty():
message = email.message_from_string(sys.stdin.read())
return message.get('Message-ID', None)
logger.error('Error: pipe a message or pass msgid as parameter')
sys.exit(1)
def store_and_build_forward_message(self,
form,
boundary=None,
max_bytes_per_blob=None,
max_bytes_total=None,
bucket_name=None):
"""Reads form data, stores blobs data and builds the forward request.
This finds all of the file uploads in a set of form fields, converting them
into blobs and storing them in the blobstore. It also generates the HTTP
request to forward to the user's application.
Args:
form: cgi.FieldStorage instance representing the whole form derived from
original POST data.
boundary: The optional boundary to use for the resulting form. If omitted,
one is randomly generated.
max_bytes_per_blob: The maximum size in bytes that any single blob
in the form is allowed to be.
max_bytes_total: The maximum size in bytes that the total of all blobs
in the form is allowed to be.
bucket_name: The name of the Google Storage bucket to store the uploaded
files.
Returns:
A tuple (content_type, content_text), where content_type is the value of
the Content-Type header, and content_text is a string containing the body
of the HTTP request to forward to the application.
Raises:
webob.exc.HTTPException: The upload failed.
"""
message = multipart.MIMEMultipart('form-data', boundary)
creation = self._now_func()
total_bytes_uploaded = 0
created_blobs = []
mime_type_error = None
too_many_conflicts = False
upload_too_large = False
filename_too_large = False
content_type_too_large = False
# Extract all of the individual form items out of the FieldStorage.
form_items = []
# Sorting of forms is done merely to make testing a little easier since
# it means blob-keys are generated in a predictable order.
for key in sorted(form):
form_item = form[key]
if isinstance(form_item, list):
form_items.extend(form_item)
else:
form_items.append(form_item)
for form_item in form_items:
disposition_parameters = {'name': form_item.name}
variable = email.message.Message()
if form_item.filename is None:
# Copy as is
variable.add_header('Content-Type', 'text/plain')
variable.set_payload(form_item.value)
else:
# If there is no filename associated with this field it means that the
# file form field was not filled in. This blob should not be created
# and forwarded to success handler.
if not form_item.filename:
continue
disposition_parameters['filename'] = form_item.filename
try:
main_type, sub_type = _split_mime_type(form_item.type)
except _InvalidMIMETypeFormatError as ex:
mime_type_error = str(ex)
break
# Seek to the end of file and use the pos as the length.
form_item.file.seek(0, os.SEEK_END)
content_length = form_item.file.tell()
form_item.file.seek(0)
total_bytes_uploaded += content_length
if max_bytes_per_blob is not None:
if content_length > max_bytes_per_blob:
upload_too_large = True
break
if max_bytes_total is not None:
if total_bytes_uploaded > max_bytes_total:
upload_too_large = True
break
if form_item.filename is not None:
if len(form_item.filename) > _MAX_STRING_NAME_LENGTH:
filename_too_large = True
break
if form_item.type is not None:
if len(form_item.type) > _MAX_STRING_NAME_LENGTH:
content_type_too_large = True
break
# Compute the MD5 hash of the upload.
digester = hashlib.md5()
while True:
block = form_item.file.read(1 << 20)
if not block:
break
digester.update(block)
form_item.file.seek(0)
# Create the external body message containing meta-data about the blob.
external = email.message.Message()
external.add_header('Content-Type',
'%s/%s' % (main_type, sub_type),
**form_item.type_options)
# NOTE: This is in violation of RFC 2616 (Content-MD5 should be the
# base-64 encoding of the binary hash, not the hex digest), but it is
# consistent with production.
content_md5 = base64.urlsafe_b64encode(digester.hexdigest())
# Create header MIME message
headers = dict(form_item.headers)
for name in _STRIPPED_FILE_HEADERS:
if name in headers:
del headers[name]
headers['Content-Length'] = str(content_length)
headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
blobstore._format_creation(creation))
headers['Content-MD5'] = content_md5
gs_filename = None
if bucket_name:
random_key = str(self._generate_blob_key())
gs_filename = '%s/fake-%s' % (bucket_name, random_key)
headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = (
blobstore.GS_PREFIX + gs_filename)
for key, value in six.iteritems(headers):
external.add_header(key, value)
# Add disposition parameters (a clone of the outer message's field).
if not external.get('Content-Disposition'):
external.add_header('Content-Disposition', 'form-data',
**disposition_parameters)
base64_encoding = (form_item.headers.get(
'Content-Transfer-Encoding') == 'base64')
content_type, blob_file, filename = self._preprocess_data(
external['content-type'], form_item.file,
form_item.filename, base64_encoding)
# Store the actual contents to storage.
if gs_filename:
info_entity = self.store_gs_file(content_type, gs_filename,
blob_file, filename)
else:
try:
info_entity = self.store_blob(content_type, filename,
digester, blob_file,
creation)
except _TooManyConflictsError:
too_many_conflicts = True
break
# Track created blobs in case we need to roll them back.
created_blobs.append(info_entity)
variable.add_header('Content-Type',
'message/external-body',
access_type=blobstore.BLOB_KEY_HEADER,
blob_key=info_entity.key().name())
variable.set_payload([external])
# Set common information.
variable.add_header('Content-Disposition', 'form-data',
**disposition_parameters)
message.attach(variable)
if (mime_type_error or too_many_conflicts or upload_too_large
or filename_too_large or content_type_too_large):
for blob in created_blobs:
datastore.Delete(blob)
if mime_type_error:
self.abort(400, detail=mime_type_error)
elif too_many_conflicts:
self.abort(500, detail='Could not generate a blob key.')
elif upload_too_large:
self.abort(413)
else:
if filename_too_large:
invalid_field = 'filename'
elif content_type_too_large:
invalid_field = 'Content-Type'
detail = 'The %s exceeds the maximum allowed length of %s.' % (
invalid_field, _MAX_STRING_NAME_LENGTH)
self.abort(400, detail=detail)
message_out = io.StringIO()
gen = email.generator.Generator(message_out, maxheaderlen=0)
gen.flatten(message, unixfrom=False)
# Get the content text out of the message.
message_text = message_out.getvalue()
content_start = message_text.find('\n\n') + 2
content_text = message_text[content_start:]
content_text = content_text.replace('\n', '\r\n')
return message.get('Content-Type'), content_text
def _run_customizer(self):
"""Executes the entire process of customize a mailing for a recipient
and returns its full path.
This may take some time and shouldn't be run from the reactor thread.
"""
try:
fullpath = os.path.join(
self.temp_path,
MailCustomizer.make_file_name(self.recipient.mailing.id,
self.recipient.id))
if os.path.exists(fullpath):
self.log.debug("Customized email found here: %s", fullpath)
parser = email.parser.Parser()
with file(fullpath, 'rt') as fd:
header = parser.parse(fd, headersonly=True)
return header['Message-ID'], fullpath
contact_data = self.make_contact_data_dict(self.recipient)
message = self._parse_message()
assert (isinstance(contact_data, dict))
assert (isinstance(message, Message))
#email.iterators._structure(message)
mixed_attachments = []
related_attachments = []
for attachment in contact_data.get('attachments', []):
if 'content-id' in attachment:
related_attachments.append(attachment)
else:
mixed_attachments.append(attachment)
#bodies = MailingBody.objects.filter(relay = self.recipient.mailing_queue).order_by('header_pos')
def convert_to_mixed(part, mixed_attachments, subtype):
import email.mime.multipart
part2 = email.mime.multipart.MIMEMultipart(_subtype=subtype)
part2.set_payload(part.get_payload())
del part['Content-Type']
part['Content-Type'] = 'multipart/mixed'
part.set_payload(None)
part.attach(part2)
for attachment in mixed_attachments:
part.attach(self._make_mime_part(attachment))
def personalise_bodies(part,
mixed_attachments=[],
related_attachments=[]):
import email.message
assert (isinstance(part, email.message.Message))
if part.is_multipart():
subtype = part.get_content_subtype()
if subtype == 'mixed':
personalise_bodies(
part.get_payload(0),
related_attachments=related_attachments)
for attachment in mixed_attachments:
part.attach(self._make_mime_part(attachment))
elif subtype == 'alternative':
for p in part.get_payload():
personalise_bodies(
p, related_attachments=related_attachments)
if mixed_attachments:
convert_to_mixed(part,
mixed_attachments,
subtype="alternative")
elif subtype == 'digest':
raise email.errors.MessageParseError, "multipart/digest not supported"
elif subtype == 'parallel':
raise email.errors.MessageParseError, "multipart/parallel not supported"
elif subtype == 'related':
personalise_bodies(part.get_payload(0))
for attachment in related_attachments:
part.attach(self._make_mime_part(attachment))
if mixed_attachments:
convert_to_mixed(part,
mixed_attachments,
subtype="related")
else:
self.log.warn("Unknown multipart subtype '%s'" %
subtype)
else:
maintype = part.get_content_maintype()
if maintype == 'text':
self._customize_message(part, contact_data)
if mixed_attachments:
import email.mime.text
part2 = email.mime.text.MIMEText(
part.get_payload(decode=True))
del part['Content-Type']
part['Content-Type'] = 'multipart/mixed'
part.set_payload(None)
part.attach(part2)
for attachment in mixed_attachments:
part.attach(self._make_mime_part(attachment))
else:
self.log.warn(
"personalise_bodies(): can't handle '%s' parts" %
part.get_content_type())
personalise_bodies(message, mixed_attachments, related_attachments)
# Customize the subject
subject = self._do_customization(
header_to_unicode(message.get("Subject", "")), contact_data)
# Remove some headers
for header in ('Subject', 'Received', 'To', 'From', 'User-Agent',
'Date', 'Message-ID', 'List-Unsubscribe',
'DKIM-Signature', 'Authentication-Results',
'Received-SPF', 'Received-SPF', 'X-Received',
'Delivered-To', 'Feedback-ID', 'Precedence',
'Return-Path'):
if header in message:
del message[header]
message['Subject'] = Header(subject)
# Adding missing headers
# message['Precedence'] = "bulk"
h = Header(self.recipient.sender_name or '')
h.append("<%s>" % self.recipient.mail_from)
message['From'] = h
h = Header()
h.append(contact_data.get('firstname') or '')
h.append(contact_data.get('lastname') or '')
h.append("<%s>" % contact_data['email'])
message['To'] = h
message['Date'] = email.utils.formatdate()
# message['Message-ID'] = email.utils.make_msgid() # very very slow on certain circumstance
message['Message-ID'] = "<%s.%d@cm.%s>" % (
self.recipient.id, self.recipient.mailing.id,
self.recipient.domain_name)
if self.unsubscribe_url:
message['List-Unsubscribe'] = self.unsubscribe_url
fp = cStringIO.StringIO()
generator = email.generator.Generator(fp, mangle_from_=False)
generator.flatten(message)
flattened_message = fp.getvalue()
flattened_message = self.add_dkim_signature(flattened_message)
flattened_message = self.add_fbl(flattened_message)
with open(fullpath + '.tmp', 'wt') as fp:
fp.write(flattened_message)
fp.close()
if os.path.exists(fullpath):
os.remove(fullpath)
os.rename(fullpath + '.tmp', fullpath)
return message['Message-ID'], fullpath
except Exception:
self.log.exception(
"Failed to customize mailing '%s' for recipient '%s'" %
(self.recipient.mail_from, self.recipient.email))
raise
def _find_list_tag(self, message, rec=0, plain=False):
"""
Filter *message* for a valid list tag.
This method will scan the passed in messages headers looking for a hint
to the used list-tag.
The following headers will be checked (in order):
* X-SynFU-Tags (explicit List-Tags supplied by synfu-news2mail)
* [X-]List-Post
* [X-]List-Id
* [X-]AF-Envelope-to
If any of these is found it will be converted into a regular expression
which can be used to remove the List-Tag from arbitrary headers.
:param message: A :class:`email.message` object.
:param rec: Optional recursion level used to indent messages.
:param plain: If :const:`True` return the plain List-Tag (no regexp)
:returns: Either a :class:`re.SRE_PATTERN` or a string
"""
tag = message.get('X-SynFU-Tags', None)
lp = message.get('List-Post', message.get('X-List-Post', None))
lid = message.get('List-Id', message.get('X-List-Id', None))
evl = message.get('AF-Envelope-to',
message.get('X-AF-Envelope-to', None))
tag_base = None
if tag:
tag = email.header.decode_header(tag)[0][0]
self._log('--- using supplied SynFU tag hints', rec=rec)
tag_base = '({0})'.format('|'.join(x.strip()
for x in tag.split(',')
if x.strip()))
# preffer List-Id if we have it
elif lid:
lid = email.header.decode_header(lid)[-1][0]
tag_base = lid.split('<')[-1].split('.')[0].strip()
elif lp:
lp = email.header.decode_header(lp)[0][0]
try:
tag_base = lp.split('mailto:')[1].split('@')[0]
except IndexError:
tag_base = None
elif evl:
evl = email.header.decode_header(evl)[0][0]
try:
tag_base = evl.split('@')[0]
except IndexError:
tag_base = None
if plain:
return tag_base
if tag_base:
self._log('--- list tag: "[*{0}*]"', format(tag_base), rec=rec)
return re.compile('(?i)\[[^[]*{0}[^]]*\]'.format(tag_base))
self._log('--- no list tag found', rec=rec)
# return 'moab'
return re.compile(
'(?i)\s*\[[^]]*(?# This is here to confuse people)\]\s*')
def test_parsing_message_with_custom_message_class(self):
message = email.message_from_string(self.attachment_eml, Message)
self.assertEqual(message.get('to'), '*****@*****.**')
class Application(object):
"""A WSGI middleware application for handling blobstore upload requests.
This application will handle all uploaded files in a POST request, store the
results in the blob-storage, close the upload session and forward the request
on to another WSGI application, with the environment transformed so that the
uploaded file contents are replaced with their blob keys.
"""
def __init__(self,
forward_app,
get_blob_storage=_get_blob_storage,
generate_blob_key=_generate_blob_key,
now_func=datetime.datetime.now):
"""Constructs a new Application.
Args:
forward_app: A WSGI application to forward successful upload requests to.
get_blob_storage: Callable that returns a BlobStorage instance. The
default is fine, but may be overridden for testing purposes.
generate_blob_key: Function used for generating unique blob keys.
now_func: Function that returns the current timestamp.
"""
self._forward_app = forward_app
self._blob_storage = get_blob_storage()
self._generate_blob_key = generate_blob_key
self._now_func = now_func
def abort(self, code, detail=None):
"""Aborts the application by raising a webob.exc.HTTPException.
Args:
code: HTTP status code int.
detail: Optional detail message str.
Raises:
webob.exc.HTTPException: Always.
"""
exception = webob.exc.status_map[code]()
if detail:
exception.detail = detail
raise exception
def store_blob(self, content_type, filename, md5_hash, blob_file,
creation):
"""Store a supplied form-data item to the blobstore.
The appropriate metadata is stored into the datastore.
Args:
content_type: The MIME content type of the uploaded file.
filename: The filename of the uploaded file.
md5_hash: MD5 hash of the file contents, as a hashlib hash object.
blob_file: A file-like object containing the contents of the file.
creation: datetime.datetime instance to associate with new blobs creation
time. This parameter is provided so that all blobs in the same upload
form can have the same creation date.
Returns:
datastore.Entity('__BlobInfo__') associated with the upload.
Raises:
_TooManyConflictsError if there were too many name conflicts generating a
blob key.
"""
blob_key = self._generate_blob_key()
# Store the blob contents in the blobstore.
self._blob_storage.StoreBlob(blob_key, blob_file)
# Store the blob metadata in the datastore as a __BlobInfo__ entity.
blob_entity = datastore.Entity('__BlobInfo__',
name=str(blob_key),
namespace='')
blob_entity['content_type'] = content_type
blob_entity['creation'] = creation
blob_entity['filename'] = filename
blob_entity['md5_hash'] = md5_hash.hexdigest()
blob_entity['size'] = blob_file.tell()
datastore.Put(blob_entity)
return blob_entity
def store_gs_file(self, content_type, gs_filename, blob_file, filename):
"""Store a supplied form-data item to GS.
Delegate all the work of gs file creation to CloudStorageStub.
Args:
content_type: The MIME content type of the uploaded file.
gs_filename: The gs filename to create of format bucket/filename.
blob_file: A file-like object containing the contents of the file.
filename: user provided filename.
Returns:
datastore.Entity('__GsFileInfo__') associated with the upload.
"""
gs_stub = cloudstorage_stub.CloudStorageStub(self._blob_storage)
blobkey = gs_stub.post_start_creation('/' + gs_filename,
{'content-type': content_type})
content = blob_file.read()
return gs_stub.put_continue_creation(blobkey,
content, (0, len(content) - 1),
len(content), filename)
def _preprocess_data(self, content_type, blob_file, filename,
base64_encoding):
"""Preprocess data and metadata before storing them.
Args:
content_type: The MIME content type of the uploaded file.
blob_file: A file-like object containing the contents of the file.
filename: The filename of the uploaded file.
base64_encoding: True, if the file contents are base-64 encoded.
Returns:
(content_type, blob_file, filename) after proper preprocessing.
Raises:
_InvalidMetadataError: when metadata are not utf-8 encoded.
"""
if base64_encoding:
blob_file = cStringIO.StringIO(
base64.urlsafe_b64decode(blob_file.read()))
# If content_type or filename are bytes, assume UTF-8 encoding.
try:
if not isinstance(content_type, unicode):
content_type = content_type.decode('utf-8')
if filename and not isinstance(filename, unicode):
filename = filename.decode('utf-8')
except UnicodeDecodeError:
raise _InvalidMetadataError(
'The uploaded entity contained invalid UTF-8 metadata. This may be '
'because the page containing the upload form was served with a '
'charset other than "utf-8".')
return content_type, blob_file, filename
def store_and_build_forward_message(self,
form,
boundary=None,
max_bytes_per_blob=None,
max_bytes_total=None,
bucket_name=None):
"""Reads form data, stores blobs data and builds the forward request.
This finds all of the file uploads in a set of form fields, converting them
into blobs and storing them in the blobstore. It also generates the HTTP
request to forward to the user's application.
Args:
form: cgi.FieldStorage instance representing the whole form derived from
original POST data.
boundary: The optional boundary to use for the resulting form. If omitted,
one is randomly generated.
max_bytes_per_blob: The maximum size in bytes that any single blob
in the form is allowed to be.
max_bytes_total: The maximum size in bytes that the total of all blobs
in the form is allowed to be.
bucket_name: The name of the Google Storage bucket to store the uploaded
files.
Returns:
A tuple (content_type, content_text), where content_type is the value of
the Content-Type header, and content_text is a string containing the body
of the HTTP request to forward to the application.
Raises:
webob.exc.HTTPException: The upload failed.
"""
message = multipart.MIMEMultipart('form-data', boundary)
creation = self._now_func()
total_bytes_uploaded = 0
created_blobs = []
mime_type_error = None
too_many_conflicts = False
upload_too_large = False
filename_too_large = False
content_type_too_large = False
# Extract all of the individual form items out of the FieldStorage.
form_items = []
# Sorting of forms is done merely to make testing a little easier since
# it means blob-keys are generated in a predictable order.
for key in sorted(form):
form_item = form[key]
if isinstance(form_item, list):
form_items.extend(form_item)
else:
form_items.append(form_item)
for form_item in form_items:
disposition_parameters = {'name': form_item.name}
variable = email.message.Message()
if form_item.filename is None:
# Copy as is
variable.add_header('Content-Type', 'text/plain')
variable.set_payload(form_item.value)
else:
# If there is no filename associated with this field it means that the
# file form field was not filled in. This blob should not be created
# and forwarded to success handler.
if not form_item.filename:
continue
disposition_parameters['filename'] = form_item.filename
try:
main_type, sub_type = _split_mime_type(form_item.type)
except _InvalidMIMETypeFormatError, ex:
mime_type_error = str(ex)
break
# Seek to the end of file and use the pos as the length.
form_item.file.seek(0, os.SEEK_END)
content_length = form_item.file.tell()
form_item.file.seek(0)
total_bytes_uploaded += content_length
if max_bytes_per_blob is not None:
if content_length > max_bytes_per_blob:
upload_too_large = True
break
if max_bytes_total is not None:
if total_bytes_uploaded > max_bytes_total:
upload_too_large = True
break
if form_item.filename is not None:
if len(form_item.filename) > _MAX_STRING_NAME_LENGTH:
filename_too_large = True
break
if form_item.type is not None:
if len(form_item.type) > _MAX_STRING_NAME_LENGTH:
content_type_too_large = True
break
# Compute the MD5 hash of the upload.
digester = hashlib.md5()
while True:
block = form_item.file.read(1 << 20)
if not block:
break
digester.update(block)
form_item.file.seek(0)
# Create the external body message containing meta-data about the blob.
external = email.message.Message()
external.add_header('Content-Type',
'%s/%s' % (main_type, sub_type),
**form_item.type_options)
# NOTE: This is in violation of RFC 2616 (Content-MD5 should be the
# base-64 encoding of the binary hash, not the hex digest), but it is
# consistent with production.
content_md5 = base64.urlsafe_b64encode(digester.hexdigest())
# Create header MIME message
headers = dict(form_item.headers)
for name in _STRIPPED_FILE_HEADERS:
if name in headers:
del headers[name]
headers['Content-Length'] = str(content_length)
headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = (
blobstore._format_creation(creation))
headers['Content-MD5'] = content_md5
gs_filename = None
if bucket_name:
random_key = str(self._generate_blob_key())
gs_filename = '%s/fake-%s' % (bucket_name, random_key)
headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = (
blobstore.GS_PREFIX + gs_filename)
for key, value in headers.iteritems():
external.add_header(key, value)
# Add disposition parameters (a clone of the outer message's field).
if not external.get('Content-Disposition'):
external.add_header('Content-Disposition', 'form-data',
**disposition_parameters)
base64_encoding = (form_item.headers.get(
'Content-Transfer-Encoding') == 'base64')
content_type, blob_file, filename = self._preprocess_data(
external['content-type'], form_item.file,
form_item.filename, base64_encoding)
# Store the actual contents to storage.
if gs_filename:
info_entity = self.store_gs_file(content_type, gs_filename,
blob_file, filename)
else:
try:
info_entity = self.store_blob(content_type, filename,
digester, blob_file,
creation)
except _TooManyConflictsError:
too_many_conflicts = True
break
# Track created blobs in case we need to roll them back.
created_blobs.append(info_entity)
variable.add_header('Content-Type',
'message/external-body',
access_type=blobstore.BLOB_KEY_HEADER,
blob_key=info_entity.key().name())
variable.set_payload([external])
# Set common information.
variable.add_header('Content-Disposition', 'form-data',
**disposition_parameters)
message.attach(variable)
if (mime_type_error or too_many_conflicts or upload_too_large
or filename_too_large or content_type_too_large):
for blob in created_blobs:
datastore.Delete(blob)
if mime_type_error:
self.abort(400, detail=mime_type_error)
elif too_many_conflicts:
self.abort(500, detail='Could not generate a blob key.')
elif upload_too_large:
self.abort(413)
else:
if filename_too_large:
invalid_field = 'filename'
elif content_type_too_large:
invalid_field = 'Content-Type'
detail = 'The %s exceeds the maximum allowed length of %s.' % (
invalid_field, _MAX_STRING_NAME_LENGTH)
self.abort(400, detail=detail)
message_out = cStringIO.StringIO()
gen = email.generator.Generator(message_out, maxheaderlen=0)
gen.flatten(message, unixfrom=False)
# Get the content text out of the message.
message_text = message_out.getvalue()
content_start = message_text.find('\n\n') + 2
content_text = message_text[content_start:]
content_text = content_text.replace('\n', '\r\n')
return message.get('Content-Type'), content_text
