def parsemail(mail, logger='none'): log = logging.getLogger(logger) if len(mail) == 0: raise MailParserException('Empty mail') data = { 'type': 'mail' } # parse mail try: message = email.parser.Parser().parsestr(mail) except UnicodeEncodeError: message = email.parser.Parser().parsestr(mail.encode('latin_1')) # test defects and try to save defect mails if len(message.defects) != 0: raise MailParserException("Parser signaled defect:\n %s" % (str(message.defects))) # encoded word is not decoded here, because it only should appear in the # display name that is discarded by the last map function # parse from and sender addresses addresses = itertools.chain(*(message.get_all(field) for field in ('from', 'sender') if message.has_key(field))) data['from'] = map(lambda adrs: adrs[1], set(email.utils.getaddresses(addresses))) log.info("From: %s"%(' '.join(data['from']))) # parse recipient addresses addresses = itertools.chain(*(message.get_all(field) for field in ('to', 'cc') if message.has_key(field))) data['to'] = map(lambda adrs: adrs[1], set(email.utils.getaddresses(addresses))) log.info("To: %s"%(' '.join(data['to']))) # parse date and convert it to standard format in UTC if message.get('Date', None): try: # guesses format and parses 10-tuple parsedtime = email.utils.parsedate_tz(message.get('Date')) # seconds since epoch utc_timestamp = calendar.timegm(parsedtime[0:9])-parsedtime[9] # formatted data['date'] = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(utc_timestamp)) log.info("Date: %s", data['date']) except Exception as e: raise MailParserException("Could not convert %s to YYYY-MM-DD hh:mm:ss\n %s" % (message.get('Date'), str(e))) # format current UTC time data['upload_date'] = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())) log.info("upload-date: %s", data['upload_date']) # add labels data['labels'] = [] if message.get('Status', None): if not 'R' in message.get('Status'): data['labels'].append('unread') else: data['labels'].append('unread') if config.autolabels: labeller = labels.Labeller(path=config.autolabels) labeller.check(data) log.info("Labels: %s", ' '.join(data['labels'])) return data
def extract_and_upload_attachments(message: message.Message, realm: Realm) -> str: user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT) attachment_links = [] payload = message.get_payload() if not isinstance(payload, list): # This is not a multipart message, so it can't contain attachments. return "" for part in payload: content_type = part.get_content_type() filename = part.get_filename() if filename: attachment = part.get_payload(decode=True) if isinstance(attachment, bytes): s3_url = upload_message_file(filename, len(attachment), content_type, attachment, user_profile, target_realm=realm) formatted_link = "[%s](%s)" % (filename, s3_url) attachment_links.append(formatted_link) else: logger.warning("Payload is not bytes (invalid attachment %s in message from %s)." % (filename, message.get("From"))) return "\n".join(attachment_links)
def process_message(message: message.Message, rcpt_to: Optional[str] = None, pre_checked: bool = False) -> None: subject_header = str(message.get("Subject", "")).strip() if subject_header == "": subject_header = "(no topic)" encoded_subject, encoding = decode_header(subject_header)[0] if encoding is None: subject = cast(str, encoded_subject ) # encoded_subject has type str when encoding is None else: try: subject = encoded_subject.decode(encoding) except (UnicodeDecodeError, LookupError): subject = "(unreadable subject)" debug_info = {} try: if rcpt_to is not None: to = rcpt_to else: to = find_emailgateway_recipient(message) debug_info["to"] = to if is_missed_message_address(to): process_missed_message(to, message, pre_checked) else: process_stream_message(to, subject, message, debug_info) except ZulipEmailForwardError as e: # TODO: notify sender of error, retry if appropriate. log_and_report(message, str(e), debug_info)
def extract_and_upload_attachments(message: message.Message, realm: Realm) -> str: user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT) attachment_links = [] for part in message.walk(): content_type = part.get_content_type() encoded_filename = part.get_filename() if not encoded_filename: continue filename = handle_header_content(encoded_filename) if filename: attachment = part.get_payload(decode=True) if isinstance(attachment, bytes): s3_url = upload_message_file(filename, len(attachment), content_type, attachment, user_profile, target_realm=realm) formatted_link = "[%s](%s)" % (filename, s3_url) attachment_links.append(formatted_link) else: logger.warning("Payload is not bytes (invalid attachment %s in message from %s)." % (filename, message.get("From"))) return '\n'.join(attachment_links)
def process_message(message, rcpt_to=None, pre_checked=False): # type: (message.Message, Optional[text_type], bool) -> None subject_header = message.get("Subject", "(no subject)") encoded_subject, encoding = decode_header(subject_header)[0] # type: ignore # https://github.com/python/typeshed/pull/333 if encoding is None: subject = force_text(encoded_subject) # encoded_subject has type str when encoding is None else: try: subject = encoded_subject.decode(encoding) except (UnicodeDecodeError, LookupError): subject = u"(unreadable subject)" debug_info = {} try: if rcpt_to is not None: to = rcpt_to else: to = find_emailgateway_recipient(message) debug_info["to"] = to if is_missed_message_address(to): process_missed_message(to, message, pre_checked) else: process_stream_message(to, subject, message, debug_info) except ZulipEmailForwardError as e: # TODO: notify sender of error, retry if appropriate. log_and_report(message, str(e), debug_info)
def extract_and_upload_attachments(message, realm): # type: (message.Message, Realm) -> Text user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT) attachment_links = [] payload = message.get_payload() if not isinstance(payload, list): # This is not a multipart message, so it can't contain attachments. return "" for part in payload: content_type = part.get_content_type() filename = part.get_filename() if filename: attachment = part.get_payload(decode=True) if isinstance(attachment, binary_type): s3_url = upload_message_image(filename, len(attachment), content_type, attachment, user_profile, target_realm=realm) formatted_link = u"[%s](%s)" % (filename, s3_url) attachment_links.append(formatted_link) else: logger.warning( "Payload is not bytes (invalid attachment %s in message from %s)." % (filename, message.get("From"))) return u"\n".join(attachment_links)
def collect_frequency_data(self, message, headers=None): """ Store data about frequency of message submission from sender of this message. 'headers', if specified, is a list of header names to store along with times, for use as discriminators. """ user = message['From'] date = message.get('Date') if date is not None: date = parsedate(date) if date is not None: date = datetime.datetime(*date[:6]) else: date = datetime.datetime.now() if headers is None: headers = {} else: headers = dict([(name, message[name]) for name in headers]) times = self._freq_data.get(user) if times is None: times = _FreqData() self._freq_data[user] = times times.append((date,headers))
def pop_next(self): """ Retrieve the next message in the queue, removing it from the queue. """ key = iter(self._messages.keys()).next() message = self._messages.pop(key) return message.get()
def process_message(message: message.Message, rcpt_to: Optional[str]=None, pre_checked: bool=False) -> None: subject_header = str(message.get("Subject", "")).strip() if subject_header == "": subject_header = "(no topic)" encoded_subject, encoding = decode_header(subject_header)[0] if encoding is None: subject = force_text(encoded_subject) # encoded_subject has type str when encoding is None else: try: subject = encoded_subject.decode(encoding) except (UnicodeDecodeError, LookupError): subject = "(unreadable subject)" debug_info = {} try: if rcpt_to is not None: to = rcpt_to else: to = find_emailgateway_recipient(message) debug_info["to"] = to if is_missed_message_address(to): process_missed_message(to, message, pre_checked) else: process_stream_message(to, subject, message, debug_info) except ZulipEmailForwardError as e: # TODO: notify sender of error, retry if appropriate. log_and_report(message, str(e), debug_info)
def process_message(message, rcpt_to=None, pre_checked=False): # type: (message.Message, Optional[Text], bool) -> None subject_header = message.get("Subject", "(no subject)") encoded_subject, encoding = decode_header(subject_header)[0] if encoding is None: subject = force_text(encoded_subject) # encoded_subject has type str when encoding is None else: try: subject = encoded_subject.decode(encoding) except (UnicodeDecodeError, LookupError): subject = u"(unreadable subject)" debug_info = {} try: if rcpt_to is not None: to = rcpt_to else: to = find_emailgateway_recipient(message) debug_info["to"] = to if is_missed_message_address(to): process_missed_message(to, message, pre_checked) else: process_stream_message(to, subject, message, debug_info) except ZulipEmailForwardError as e: # TODO: notify sender of error, retry if appropriate. log_and_report(message, str(e), debug_info)
def test_003_send_reply(self): itip_events = itip.events_from_message( message_from_string(itip_non_multipart)) itip.send_reply( "*****@*****.**", itip_events, "SUMMARY=%(summary)s; STATUS=%(status)s; NAME=%(name)s;") self.assertEqual(len(self.smtplog), 1) self.assertEqual(self.smtplog[0][0], '*****@*****.**', "From attendee") self.assertEqual(self.smtplog[0][1], '*****@*****.**', "To organizer") _accepted = participant_status_label('ACCEPTED') message = message_from_string(self.smtplog[0][2]) self.assertEqual( message.get('Subject'), _("Invitation for %(summary)s was %(status)s") % { 'summary': 'test', 'status': _accepted }) text = str(message.get_payload(0)) self.assertIn('SUMMARY=3Dtest', text) self.assertIn('STATUS=3D' + _accepted, text)
def process_stream_message(to: str, message: message.Message) -> None: subject_header = str(make_header(decode_header(message.get("Subject", "")))) subject = strip_from_subject(subject_header) or "(no topic)" stream, show_sender = extract_and_validate(to) # Don't remove quotations if message is forwarded: remove_quotations = not is_forwarded(subject_header) body = construct_zulip_body(message, stream.realm, show_sender, remove_quotations) send_zulip(settings.EMAIL_GATEWAY_BOT, stream, subject, body) logger.info("Successfully processed email to %s (%s)" % ( stream.name, stream.realm.string_id))
def remove_from_quarantine(self, message): """ Removes the given message from the quarantine. """ id = message.get('X-Postoffice-Id') if id is None: raise ValueError("Message is not in the quarantine.") id = int(id) if id not in self._quarantine: raise ValueError("Message is not in the quarantine.") del self._quarantine[id] del message['X-Postoffice-Id']
def process_stream_message(to: str, message: message.Message, debug_info: Dict[str, Any]) -> None: subject_header = str(make_header(decode_header(message.get("Subject", "")))) subject = strip_from_subject(subject_header) or "(no topic)" stream, show_sender = extract_and_validate(to) # Don't remove quotations if message is forwarded: remove_quotations = not is_forwarded(subject_header) body = construct_zulip_body(message, stream.realm, show_sender, remove_quotations) debug_info["stream"] = stream send_zulip(settings.EMAIL_GATEWAY_BOT, stream, subject, body) logger.info("Successfully processed email to %s (%s)" % ( stream.name, stream.realm.string_id))
def process_stream_message(to: str, message: message.Message) -> None: subject_header = str(make_header(decode_header(message.get("Subject", "")))) subject = strip_from_subject(subject_header) or "(no topic)" stream, options = decode_stream_email_address(to) # Don't remove quotations if message is forwarded, unless otherwise specified: if 'include_quotes' not in options: options['include_quotes'] = is_forwarded(subject_header) body = construct_zulip_body(message, stream.realm, **options) send_zulip(settings.EMAIL_GATEWAY_BOT, stream, subject, body) logger.info("Successfully processed email to %s (%s)" % ( stream.name, stream.realm.string_id))
def test_send_a_single_email_to_multiple_recipients(self): from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent) self.maxDiff = None to_emails = [ To('*****@*****.**', 'Example To Name 0'), To('*****@*****.**', 'Example To Name 1') ] message = Mail( from_email=From('*****@*****.**', 'Example From Name'), to_emails=to_emails, subject=Subject('Sending with SendGrid is Fun'), plain_text_content=PlainTextContent( 'and easy to do anywhere, even with Python'), html_content=HtmlContent( '<strong>and easy to do anywhere, even with Python</strong>')) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "to": [ { "email": "*****@*****.**", "name": "Example To Name 0" }, { "email": "*****@*****.**", "name": "Example To Name 1" } ] } ], "subject": "Sending with SendGrid is Fun" }''') )
def test_send_a_single_email_to_multiple_recipients(self): from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent) self.maxDiff = None to_emails = [ To('*****@*****.**', 'Example To Name 0'), To('*****@*****.**', 'Example To Name 1') ] message = Mail( from_email=From('*****@*****.**', 'Example From Name'), to_emails=to_emails, subject=Subject('Sending with SendGrid is Fun'), plain_text_content=PlainTextContent( 'and easy to do anywhere, even with Python'), html_content=HtmlContent( '<strong>and easy to do anywhere, even with Python</strong>')) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "to": [ { "email": "*****@*****.**", "name": "Example To Name 0" }, { "email": "*****@*****.**", "name": "Example To Name 1" } ] } ], "subject": "Sending with SendGrid is Fun" }'''))
def test_003_send_reply(self): itip_events = itip.events_from_message(message_from_string(itip_non_multipart)) itip.send_reply("*****@*****.**", itip_events, "SUMMARY=%(summary)s; STATUS=%(status)s; NAME=%(name)s;") self.assertEqual(len(self.smtplog), 1) self.assertEqual(self.smtplog[0][0], '*****@*****.**', "From attendee") self.assertEqual(self.smtplog[0][1], '*****@*****.**', "To organizer") _accepted = participant_status_label('ACCEPTED') message = message_from_string(self.smtplog[0][2]) self.assertEqual(message.get('Subject'), _("Invitation for %(summary)s was %(status)s") % { 'summary':'test', 'status':_accepted }) text = str(message.get_payload(0)); self.assertIn('SUMMARY=3Dtest', text) self.assertIn('STATUS=3D' + _accepted, text)
def test_unicode_values_in_substitutions_helper(self): from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent) self.maxDiff = None message = Mail( from_email=From('*****@*****.**', 'Example From Name'), to_emails=To('*****@*****.**', 'Example To Name'), subject=Subject('Sending with SendGrid is Fun'), plain_text_content=PlainTextContent( 'and easy to do anywhere, even with Python'), html_content=HtmlContent( '<strong>and easy to do anywhere, even with Python</strong>')) message.substitution = Substitution('%city%', u'Αθήνα', p=1) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "to": [ { "email": "*****@*****.**", "name": "Example To Name" } ] }, { "substitutions": { "%city%": "Αθήνα" } } ], "subject": "Sending with SendGrid is Fun" }''') )
def construct_zulip_body(message: message.Message, realm: Realm, show_sender: bool=False, remove_quotations: bool=True) -> str: body = extract_body(message, remove_quotations) # Remove null characters, since Zulip will reject body = body.replace("\x00", "") body = filter_footer(body) body += extract_and_upload_attachments(message, realm) body = body.strip() if not body: body = '(No email body)' if show_sender: sender = message.get("From") body = "From: %s\n%s" % (sender, body) return body
def _is_cancel(self, message): """ Check if the passed message is a CANCEL message. This will try to match :attr:`FUCore.CANCEL_EXP` against all **Control:** headers contained in *message*. :param message: A :class:`email.message` object. :returns: :const:`True` or :const:`False` """ if FUCore.CANCEL_EXP.findall(message.get('Control', '')): self._log('--- Message contains *this* header, you know?') self._log('--- Therefore I\'m going to delete it.') return True return False
def test_unicode_values_in_substitutions_helper(self): from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent) self.maxDiff = None message = Mail( from_email=From('*****@*****.**', 'Example From Name'), to_emails=To('*****@*****.**', 'Example To Name'), subject=Subject('Sending with SendGrid is Fun'), plain_text_content=PlainTextContent( 'and easy to do anywhere, even with Python'), html_content=HtmlContent( '<strong>and easy to do anywhere, even with Python</strong>')) message.substitution = Substitution('%city%', u'Αθήνα', p=1) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "to": [ { "email": "*****@*****.**", "name": "Example To Name" } ] }, { "substitutions": { "%city%": "Αθήνα" } } ], "subject": "Sending with SendGrid is Fun" }'''))
def test_single_email_to_a_single_recipient_content_reversed(self): """Tests bug found in Issue-451 with Content ordering causing a crash """ from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent) self.maxDiff = None message = Mail() message.from_email = From('*****@*****.**', 'Example From Name') message.to = To('*****@*****.**', 'Example To Name') message.subject = Subject('Sending with SendGrid is Fun') message.content = HtmlContent( '<strong>and easy to do anywhere, even with Python</strong>') message.content = PlainTextContent( 'and easy to do anywhere, even with Python') self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "to": [ { "email": "*****@*****.**", "name": "Example To Name" } ] } ], "subject": "Sending with SendGrid is Fun" }''') )
def test_single_email_to_a_single_recipient_content_reversed(self): """Tests bug found in Issue-451 with Content ordering causing a crash """ from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent) self.maxDiff = None message = Mail() message.from_email = From('*****@*****.**', 'Example From Name') message.to = To('*****@*****.**', 'Example To Name') message.subject = Subject('Sending with SendGrid is Fun') message.content = HtmlContent( '<strong>and easy to do anywhere, even with Python</strong>') message.content = PlainTextContent( 'and easy to do anywhere, even with Python') self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "to": [ { "email": "*****@*****.**", "name": "Example To Name" } ] } ], "subject": "Sending with SendGrid is Fun" }'''))
def process_message(message, rcpt_to=None, pre_checked=False): # type: (message.Message, Optional[text_type], bool) -> None subject = decode_header(message.get("Subject", "(no subject)"))[0][0] debug_info = {} try: if rcpt_to is not None: to = rcpt_to else: to = find_emailgateway_recipient(message) debug_info["to"] = to if is_missed_message_address(to): process_missed_message(to, message, pre_checked) else: process_stream_message(to, subject, message, debug_info) except ZulipEmailForwardError as e: # TODO: notify sender of error, retry if appropriate. log_and_report(message, e.message, debug_info)
def construct_zulip_body(message: message.Message, realm: Realm, show_sender: bool=False, include_quotes: bool=False, include_footer: bool=False, prefer_text: bool=True) -> str: body = extract_body(message, include_quotes, prefer_text) # Remove null characters, since Zulip will reject body = body.replace("\x00", "") if not include_footer: body = filter_footer(body) if not body.endswith('\n'): body += '\n' body += extract_and_upload_attachments(message, realm) body = body.strip() if not body: body = '(No email body)' if show_sender: sender = handle_header_content(message.get("From", "")) body = "From: %s\n%s" % (sender, body) return body
def process_message(message: message.Message, rcpt_to: Optional[str] = None, pre_checked: bool = False) -> None: subject_header = make_header(decode_header(message.get("Subject", ""))) subject = strip_from_subject(str(subject_header)) or "(no topic)" debug_info = {} try: if rcpt_to is not None: to = rcpt_to else: to = find_emailgateway_recipient(message) debug_info["to"] = to if is_missed_message_address(to): process_missed_message(to, message, pre_checked) else: process_stream_message(to, subject, message, debug_info) except ZulipEmailForwardError as e: # TODO: notify sender of error, retry if appropriate. log_and_report(message, str(e), debug_info)
def test_create_mailing_from_message_with_encoded_headers(self): parser = email.parser.Parser() msg = parser.parsestr("""Content-Transfer-Encoding: 7bit Content-Type: multipart/alternative; boundary="===============2840728917476054151==" Subject: Great news! From: =?UTF-8?B?Q2VkcmljIFJJQ0FSRA==?= <*****@*****.**> To: <*****@*****.**> Date: Wed, 05 Jun 2013 06:05:56 -0000 This is a multi-part message in MIME format. --===============2840728917476054151== Content-Type: text/plain; charset="windows-1252" Content-Transfer-Encoding: quoted-printable This is a very simple mailing. I=92m happy. --===============2840728917476054151== Content-Type: text/html; charset="windows-1252" Content-Transfer-Encoding: quoted-printable <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> <html><head> <META http-equiv=3DContent-Type content=3D"text/html; charset=3Diso-8859-1"> </head> <body> This is <strong> a very simple</strong> <u>mailing</u>. = I=92m happy! Nothing else to say... </body></html> --===============2840728917476054151==-- """) mailing = Mailing.create_from_message(msg, scheduled_start=None, scheduled_duration=None) message = parser.parsestr(mailing.header + mailing.body) assert(isinstance(message, email.message.Message)) mail_from = header_to_unicode(message.get("From")) self.assertEquals(u"Cedric RICARD <*****@*****.**>", mail_from)
def process_message(message: message.Message, rcpt_to: Optional[str]=None, pre_checked: bool=False) -> None: subject_header = make_header(decode_header(message.get("Subject", ""))) subject = strip_from_subject(str(subject_header)) or "(no topic)" debug_info = {} try: if rcpt_to is not None: to = rcpt_to else: to = find_emailgateway_recipient(message) debug_info["to"] = to if is_missed_message_address(to): process_missed_message(to, message, pre_checked) else: process_stream_message(to, subject, message, debug_info) except ZulipEmailForwardError as e: if isinstance(e, ZulipEmailForwardUserError): # TODO: notify sender of error, retry if appropriate. logging.warning(str(e)) else: log_and_report(message, str(e), debug_info)
def construct_gmail_message(payload): message = email.message.Message() for header in payload['headers']: message[header['name']] = header['value'] if message.get_content_maintype() == 'multipart': message.set_payload( [construct_gmail_message(part) for part in payload['parts']]) else: cte = message.get('Content-Transfer-Encoding') if cte is not None: del message['Content-Transfer-Encoding'] message['X-Original-Content-Transfer-Encoding'] = cte try: external_id = payload['body']['attachmentId'] ct = message.get_content_type() message.replace_header('Content-Type', 'text/plain') message.set_payload( 'Attachment with type %s, ID %r omitted; retrieve separately' % (ct, external_id)) except KeyError: body = payload['body']['data'] body += '=' * (4 - len(body) % 4) message.set_payload(base64.urlsafe_b64decode(body)) return message
def test_kitchen_sink(self): from sendgrid.helpers.mail import ( Mail, From, To, Cc, Bcc, Subject, Substitution, Header, CustomArg, SendAt, Content, MimeType, Attachment, FileName, FileContent, FileType, Disposition, ContentId, TemplateId, Section, ReplyTo, Category, BatchId, Asm, GroupId, GroupsToDisplay, IpPoolName, MailSettings, BccSettings, BccSettingsEmail, BypassListManagement, FooterSettings, FooterText, FooterHtml, SandBoxMode, SpamCheck, SpamThreshold, SpamUrl, TrackingSettings, ClickTracking, SubscriptionTracking, SubscriptionText, SubscriptionHtml, SubscriptionSubstitutionTag, OpenTracking, OpenTrackingSubstitutionTag, Ganalytics, UtmSource, UtmMedium, UtmTerm, UtmContent, UtmCampaign) self.maxDiff = None message = Mail() # Define Personalizations message.to = To('*****@*****.**', 'Example User1', p=0) message.to = [ To('*****@*****.**', 'Example User2', p=0), To('*****@*****.**', 'Example User3', p=0) ] message.cc = Cc('*****@*****.**', 'Example User4', p=0) message.cc = [ Cc('*****@*****.**', 'Example User5', p=0), Cc('*****@*****.**', 'Example User6', p=0) ] message.bcc = Bcc('*****@*****.**', 'Example User7', p=0) message.bcc = [ Bcc('*****@*****.**', 'Example User8', p=0), Bcc('*****@*****.**', 'Example User9', p=0) ] message.subject = Subject('Sending with SendGrid is Fun 0', p=0) message.header = Header('X-Test1', 'Test1', p=0) message.header = Header('X-Test2', 'Test2', p=0) message.header = [ Header('X-Test3', 'Test3', p=0), Header('X-Test4', 'Test4', p=0) ] message.substitution = Substitution('%name1%', 'Example Name 1', p=0) message.substitution = Substitution('%city1%', 'Example City 1', p=0) message.substitution = [ Substitution('%name2%', 'Example Name 2', p=0), Substitution('%city2%', 'Example City 2', p=0) ] message.custom_arg = CustomArg('marketing1', 'true', p=0) message.custom_arg = CustomArg('transactional1', 'false', p=0) message.custom_arg = [ CustomArg('marketing2', 'false', p=0), CustomArg('transactional2', 'true', p=0) ] message.send_at = SendAt(1461775051, p=0) message.to = To('*****@*****.**', 'Example User10', p=1) message.to = [ To('*****@*****.**', 'Example User11', p=1), To('*****@*****.**', 'Example User12', p=1) ] message.cc = Cc('*****@*****.**', 'Example User13', p=1) message.cc = [ Cc('*****@*****.**', 'Example User14', p=1), Cc('*****@*****.**', 'Example User15', p=1) ] message.bcc = Bcc('*****@*****.**', 'Example User16', p=1) message.bcc = [ Bcc('*****@*****.**', 'Example User17', p=1), Bcc('*****@*****.**', 'Example User18', p=1) ] message.header = Header('X-Test5', 'Test5', p=1) message.header = Header('X-Test6', 'Test6', p=1) message.header = [ Header('X-Test7', 'Test7', p=1), Header('X-Test8', 'Test8', p=1) ] message.substitution = Substitution('%name3%', 'Example Name 3', p=1) message.substitution = Substitution('%city3%', 'Example City 3', p=1) message.substitution = [ Substitution('%name4%', 'Example Name 4', p=1), Substitution('%city4%', 'Example City 4', p=1) ] message.custom_arg = CustomArg('marketing3', 'true', p=1) message.custom_arg = CustomArg('transactional3', 'false', p=1) message.custom_arg = [ CustomArg('marketing4', 'false', p=1), CustomArg('transactional4', 'true', p=1) ] message.send_at = SendAt(1461775052, p=1) message.subject = Subject('Sending with SendGrid is Fun 1', p=1) # The values below this comment are global to entire message message.from_email = From('*****@*****.**', 'Twilio SendGrid') message.reply_to = ReplyTo('*****@*****.**', 'Twilio SendGrid Reply') message.subject = Subject('Sending with SendGrid is Fun 2') message.content = Content(MimeType.text, 'and easy to do anywhere, even with Python') message.content = Content( MimeType.html, '<strong>and easy to do anywhere, even with Python</strong>') message.content = [ Content('text/calendar', 'Party Time!!'), Content('text/custom', 'Party Time 2!!') ] message.attachment = Attachment( FileContent('base64 encoded content 1'), FileName('balance_001.pdf'), FileType('application/pdf'), Disposition('attachment'), ContentId('Content ID 1')) message.attachment = [ Attachment(FileContent('base64 encoded content 2'), FileName('banner.png'), FileType('image/png'), Disposition('inline'), ContentId('Content ID 2')), Attachment(FileContent('base64 encoded content 3'), FileName('banner2.png'), FileType('image/png'), Disposition('inline'), ContentId('Content ID 3')) ] message.template_id = TemplateId( '13b8f94f-bcae-4ec6-b752-70d6cb59f932') message.section = Section('%section1%', 'Substitution for Section 1 Tag') message.section = [ Section('%section2%', 'Substitution for Section 2 Tag'), Section('%section3%', 'Substitution for Section 3 Tag') ] message.header = Header('X-Test9', 'Test9') message.header = Header('X-Test10', 'Test10') message.header = [ Header('X-Test11', 'Test11'), Header('X-Test12', 'Test12') ] message.category = Category('Category 1') message.category = Category('Category 2') message.category = [Category('Category 1'), Category('Category 2')] message.custom_arg = CustomArg('marketing5', 'false') message.custom_arg = CustomArg('transactional5', 'true') message.custom_arg = [ CustomArg('marketing6', 'true'), CustomArg('transactional6', 'false') ] message.send_at = SendAt(1461775053) message.batch_id = BatchId("HkJ5yLYULb7Rj8GKSx7u025ouWVlMgAi") message.asm = Asm(GroupId(1), GroupsToDisplay([1, 2, 3, 4])) message.ip_pool_name = IpPoolName("IP Pool Name") mail_settings = MailSettings() mail_settings.bcc_settings = BccSettings( False, BccSettingsEmail("*****@*****.**")) mail_settings.bypass_list_management = BypassListManagement(False) mail_settings.footer_settings = FooterSettings( True, FooterText("w00t"), FooterHtml("<string>w00t!<strong>")) mail_settings.sandbox_mode = SandBoxMode(True) mail_settings.spam_check = SpamCheck(True, SpamThreshold(5), SpamUrl("https://example.com")) message.mail_settings = mail_settings tracking_settings = TrackingSettings() tracking_settings.click_tracking = ClickTracking(True, False) tracking_settings.open_tracking = OpenTracking( True, OpenTrackingSubstitutionTag("open_tracking")) tracking_settings.subscription_tracking = SubscriptionTracking( True, SubscriptionText("Goodbye"), SubscriptionHtml("<strong>Goodbye!</strong>"), SubscriptionSubstitutionTag("unsubscribe")) tracking_settings.ganalytics = Ganalytics(True, UtmSource("utm_source"), UtmMedium("utm_medium"), UtmTerm("utm_term"), UtmContent("utm_content"), UtmCampaign("utm_campaign")) message.tracking_settings = tracking_settings self.assertEqual( message.get(), json.loads(r'''{ "asm": { "group_id": 1, "groups_to_display": [ 1, 2, 3, 4 ] }, "attachments": [ { "content": "base64 encoded content 3", "content_id": "Content ID 3", "disposition": "inline", "filename": "banner2.png", "type": "image/png" }, { "content": "base64 encoded content 2", "content_id": "Content ID 2", "disposition": "inline", "filename": "banner.png", "type": "image/png" }, { "content": "base64 encoded content 1", "content_id": "Content ID 1", "disposition": "attachment", "filename": "balance_001.pdf", "type": "application/pdf" } ], "batch_id": "HkJ5yLYULb7Rj8GKSx7u025ouWVlMgAi", "categories": [ "Category 2", "Category 1", "Category 2", "Category 1" ], "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" }, { "type": "text/calendar", "value": "Party Time!!" }, { "type": "text/custom", "value": "Party Time 2!!" } ], "custom_args": { "marketing5": "false", "marketing6": "true", "transactional5": "true", "transactional6": "false" }, "from": { "email": "*****@*****.**", "name": "Twilio SendGrid" }, "headers": { "X-Test10": "Test10", "X-Test11": "Test11", "X-Test12": "Test12", "X-Test9": "Test9" }, "ip_pool_name": "IP Pool Name", "mail_settings": { "bcc": { "email": "*****@*****.**", "enable": false }, "bypass_list_management": { "enable": false }, "footer": { "enable": true, "html": "<string>w00t!<strong>", "text": "w00t" }, "sandbox_mode": { "enable": true }, "spam_check": { "enable": true, "post_to_url": "https://example.com", "threshold": 5 } }, "personalizations": [ { "bcc": [ { "email": "*****@*****.**", "name": "Example User7" }, { "email": "*****@*****.**", "name": "Example User8" }, { "email": "*****@*****.**", "name": "Example User9" } ], "cc": [ { "email": "*****@*****.**", "name": "Example User4" }, { "email": "*****@*****.**", "name": "Example User5" }, { "email": "*****@*****.**", "name": "Example User6" } ], "custom_args": { "marketing1": "true", "marketing2": "false", "transactional1": "false", "transactional2": "true" }, "headers": { "X-Test1": "Test1", "X-Test2": "Test2", "X-Test3": "Test3", "X-Test4": "Test4" }, "send_at": 1461775051, "subject": "Sending with SendGrid is Fun 0", "substitutions": { "%city1%": "Example City 1", "%city2%": "Example City 2", "%name1%": "Example Name 1", "%name2%": "Example Name 2" }, "to": [ { "email": "*****@*****.**", "name": "Example User1" }, { "email": "*****@*****.**", "name": "Example User2" }, { "email": "*****@*****.**", "name": "Example User3" } ] }, { "bcc": [ { "email": "*****@*****.**", "name": "Example User16" }, { "email": "*****@*****.**", "name": "Example User17" }, { "email": "*****@*****.**", "name": "Example User18" } ], "cc": [ { "email": "*****@*****.**", "name": "Example User13" }, { "email": "*****@*****.**", "name": "Example User14" }, { "email": "*****@*****.**", "name": "Example User15" } ], "custom_args": { "marketing3": "true", "marketing4": "false", "transactional3": "false", "transactional4": "true" }, "headers": { "X-Test5": "Test5", "X-Test6": "Test6", "X-Test7": "Test7", "X-Test8": "Test8" }, "send_at": 1461775052, "subject": "Sending with SendGrid is Fun 1", "substitutions": { "%city3%": "Example City 3", "%city4%": "Example City 4", "%name3%": "Example Name 3", "%name4%": "Example Name 4" }, "to": [ { "email": "*****@*****.**", "name": "Example User10" }, { "email": "*****@*****.**", "name": "Example User11" }, { "email": "*****@*****.**", "name": "Example User12" } ] } ], "reply_to": { "email": "*****@*****.**", "name": "Twilio SendGrid Reply" }, "sections": { "%section1%": "Substitution for Section 1 Tag", "%section2%": "Substitution for Section 2 Tag", "%section3%": "Substitution for Section 3 Tag" }, "send_at": 1461775053, "subject": "Sending with SendGrid is Fun 2", "template_id": "13b8f94f-bcae-4ec6-b752-70d6cb59f932", "tracking_settings": { "click_tracking": { "enable": true, "enable_text": false }, "ganalytics": { "enable": true, "utm_campaign": "utm_campaign", "utm_content": "utm_content", "utm_medium": "utm_medium", "utm_source": "utm_source", "utm_term": "utm_term" }, "open_tracking": { "enable": true, "substitution_tag": "open_tracking" }, "subscription_tracking": { "enable": true, "html": "<strong>Goodbye!</strong>", "substitution_tag": "unsubscribe", "text": "Goodbye" } } }'''))
def test_dynamic_template_data(self): self.maxDiff = None to_emails = [ To(email='*****@*****.**', name='Example To 0 Name', dynamic_template_data=DynamicTemplateData( {'name': 'Example 0 Name'})), To(email='*****@*****.**', name='Example To 1 Name', dynamic_template_data={'name': 'Example 1 Name'}) ] message = Mail(from_email=From('*****@*****.**', 'Example From Name'), to_emails=to_emails, subject=Subject('Hi!'), plain_text_content='Hello!', html_content='<strong>Hello!</strong>', is_multiple=True) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "Hello!" }, { "type": "text/html", "value": "<strong>Hello!</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "dynamic_template_data": { "name": "Example 1 Name" }, "to": [ { "email": "*****@*****.**", "name": "Example To 1 Name" } ] }, { "dynamic_template_data": { "name": "Example 0 Name" }, "to": [ { "email": "*****@*****.**", "name": "Example To 0 Name" } ] } ], "subject": "Hi!" }'''))
def test_multiple_emails_to_multiple_recipients(self): from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent, Substitution) self.maxDiff = None to_emails = [ To(email='*****@*****.**', name='Example Name 0', substitutions=[ Substitution('-name-', 'Example Name Substitution 0'), Substitution('-github-', 'https://example.com/test0'), ], subject=Subject('Override Global Subject')), To(email='*****@*****.**', name='Example Name 1', substitutions=[ Substitution('-name-', 'Example Name Substitution 1'), Substitution('-github-', 'https://example.com/test1'), ]) ] global_substitutions = Substitution('-time-', '2019-01-01 00:00:00') message = Mail( from_email=From('*****@*****.**', 'Example From Name'), to_emails=to_emails, subject=Subject('Hi -name-'), plain_text_content=PlainTextContent( 'Hello -name-, your URL is -github-, email sent at -time-'), html_content=HtmlContent( '<strong>Hello -name-, your URL is <a href=\"-github-\">here</a></strong> email sent at -time-' ), global_substitutions=global_substitutions, is_multiple=True) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "Hello -name-, your URL is -github-, email sent at -time-" }, { "type": "text/html", "value": "<strong>Hello -name-, your URL is <a href=\"-github-\">here</a></strong> email sent at -time-" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "substitutions": { "-github-": "https://example.com/test1", "-name-": "Example Name Substitution 1", "-time-": "2019-01-01 00:00:00" }, "to": [ { "email": "*****@*****.**", "name": "Example Name 1" } ] }, { "subject": "Override Global Subject", "substitutions": { "-github-": "https://example.com/test0", "-name-": "Example Name Substitution 0", "-time-": "2019-01-01 00:00:00" }, "to": [ { "email": "*****@*****.**", "name": "Example Name 0" } ] } ], "subject": "Hi -name-" }'''))
def test_multiple_emails_to_multiple_recipients(self): from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent, Substitution) self.maxDiff = None to_emails = [ To(email='*****@*****.**', name='Example Name 0', substitutions=[ Substitution('-name-', 'Example Name Substitution 0'), Substitution('-github-', 'https://example.com/test0'), ], subject=Subject('Override Global Subject')), To(email='*****@*****.**', name='Example Name 1', substitutions=[ Substitution('-name-', 'Example Name Substitution 1'), Substitution('-github-', 'https://example.com/test1'), ]) ] global_substitutions = Substitution('-time-', '2019-01-01 00:00:00') message = Mail( from_email=From('*****@*****.**', 'Example From Name'), to_emails=to_emails, subject=Subject('Hi -name-'), plain_text_content=PlainTextContent( 'Hello -name-, your URL is -github-, email sent at -time-'), html_content=HtmlContent( '<strong>Hello -name-, your URL is <a href=\"-github-\">here</a></strong> email sent at -time-'), global_substitutions=global_substitutions, is_multiple=True) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "Hello -name-, your URL is -github-, email sent at -time-" }, { "type": "text/html", "value": "<strong>Hello -name-, your URL is <a href=\"-github-\">here</a></strong> email sent at -time-" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "substitutions": { "-github-": "https://example.com/test1", "-name-": "Example Name Substitution 1", "-time-": "2019-01-01 00:00:00" }, "to": [ { "email": "*****@*****.**", "name": "Example Name 1" } ] }, { "subject": "Override Global Subject", "substitutions": { "-github-": "https://example.com/test0", "-name-": "Example Name Substitution 0", "-time-": "2019-01-01 00:00:00" }, "to": [ { "email": "*****@*****.**", "name": "Example Name 0" } ] } ], "subject": "Hi -name-" }''') )
def _apply_blacklist(self, message, mode, rec=0): """ Apply the global blacklist to this message. This method will modify the headers of the message or decide to discard the whole message based on the global blacklist. :param messag: A :class:`email.message` object. :param mode: One of :attr:`FUCore.BLACKLIST_MODES` :returns: The modified message or None if the message should be dropped. """ if not mode.lower() in FUCore.BLACKLIST_MODES: self._log( '!!! Invalid blacklist mode "{0}", not modifying message.', mode, rec=rec) return message mfrom = message.get('From', '').split('<')[-1].split('>')[0] sender = message.get('Sender', '').split('<')[-1].split('>')[0] if not mfrom and not msender: self._log('!!! Message has neiter "From:" nor "Sender:" headers!', rec=rec) return message list_entry = self._blacklist.get( mfrom.lower(), self._blacklist.get(sender.lower(), None)) if not list_entry: return message self._log('--- applying blacklist rule {0} to message', str(list_entry), rec=rec) if not list_entry['action'].lower() in ['d', 'n', 'e', 'ne', 'en']: # invalid self._log('!!! unsupported blacklist rule "{0}"', list_entry['action'], rec=rec) return message add_expires = False add_xnay = False if list_entry['action'].lower( ) == 'd' and not mode.lower() == 'reactor': self._log('--- blacklist rule "drop"', rec=rec, verbosity=2) return None elif mode.lower() == 'news2mail': #D=N=E=NE/EN=drop self._log('--- blacklist rule D=N=E=NE/EN "drop"', rec=rec, verbosity=2) return None elif mode.lower() == 'mail2news': if list_entry['action'].lower() in ['ne', 'en', 'n']: add_xnay = True if list_entry['action'].lower() in ['ne', 'en', 'e']: add_expires = True elif mode.lower() == 'reactor': if list_entry['action'].lower() in ['ne', 'en', 'n', 'd']: add_xnay = True if list_entry['action'].lower() in ['ne', 'en', 'e']: add_expires = True if add_xnay: self._log('--- blacklist rule "xnay"', rec=rec, verbosity=2) try: message.replace_header('X-No-Archive', 'yes') except KeyError: message._headers.append(('X-No-Archive', 'yes')) if add_expires: if not list_entry.get('param', None): self._log('!!! blacklist rule "expires" missing a parameter', rec=rec) else: try: delay = long(list_entry['param']) expires = time.strftime( r"%d %b %Y %H:%M:%S %z", time.localtime(time.time() + (86400 * delay))) self._log('--- blacklist rule "expires" => {0}', expires, rec=rec, verbosity=2) try: message.replace_header('Expires', expires) except KeyError: message._headers.append(('Expires', expires)) except ValueError: self.log( '!!! blacklist rule "expires" needs a *numeric* parameter.' ) return message
def _process(self, message, rec=0): """ Recursively scan and filter a MIME message. _process will scan the passed message part for invalid headers as well as mailman signatures and modify them according to the global settings. Generic modifications include: * fixing of broken **References:** and **In-Reply-To** headers * generic header filtering (see :meth:`FuCore._filter_headers`) * removal of Mailman or Mailman-like headers (see :meth:`_mutate_part`) Args: message: a :class:`email.message` object containing a set of MIME parts rec: Recursion level used to prettify log messages. Returns: A (probably) filtered / modified :class:`email.message` object. """ mm_parts = 0 text_parts = 0 mailman_sig = Reactor.MAILMAN_SIG self._log('>>> processing {0}', message.get_content_type(), rec=rec) if self._conf.complex_footer: self._log('--- using complex mailman filter', rec=rec) mailman_sig = Reactor.MAILMAN_COMPLEX if message.is_multipart(): parts = message._payload else: parts = [ message, ] list_tag = self._find_list_tag(message, rec) reference = message.get('References', None) in_reply = message.get('In-Reply-To', None) x_mailman = message.get('X-Mailman-Version', None) message._headers = self._filter_headers(list_tag, message._headers, self._conf.outlook_hacks, self._conf.fix_dateline, rec) if in_reply and not reference and rec == 0: # set References: to In-Reply-To: if where in toplevel # and References was not set properly self._log('--- set References: {0}', in_reply, rec=rec) try: # uncertain this will ever happen.. message.replace_header('References', in_reply) except KeyError: message._headers.append(('References', in_reply)) for i in xrange(len(parts) - 1, -1, -1): # again, crude since we mutate the list while iterating it.. # the whole reason is the deeply nested structure of email.message p = parts[i] ct = p.get_content_maintype() cs = p.get_content_subtype() ce = p.get('Content-Transfer-Encoding', None) cb = p.get_boundary() self._log('-- [ct = {0}, cs = {1}, ce = <{2}>]', ct, cs, ce, rec=rec) if ct == 'text': text_parts += 1 payload = p.get_payload(decode=True) self._log('--- scan: """{0}"""', payload, rec=rec, verbosity=3) if mailman_sig[0].search(payload) and \ mailman_sig[1].match(payload.split('\n')[0]): self._log('*** removing this part', rec=rec) self._log('--- """{0}"""', payload, rec=rec, verbosity=2) message._payload.remove(p) text_parts -= 1 mm_parts += 1 elif mailman_sig[0].search(payload): self._log('--- trying to mutate..', rec=rec) (use, mutation) = self._mutate_part(payload, rec) if use: self._log('*** mutated this part', rec=rec) self._log('--- """{0}"""', payload, rec=rec, verbosity=2) payload = mutation mm_parts += 1 # if it was encoded we need to re-encode it # to keep SMIME happy if ce == 'base64': payload = payload.encode('base64') elif ce == 'quoted-printable': payload = quopri.encodestring(payload) p.set_payload(payload) elif ct == 'message' or \ (ct == 'multipart' and cs in ['alternative', 'mixed']): p = self._process(p, rec + 1) else: self._log('--- what about {0}?', p.get_content_type(), rec=rec) if rec == 0: self._log('--- [mm_parts: {0}, text_parts: {1}, x_mailman: {0}]', mm_parts, text_parts, x_mailman, rec=rec) if x_mailman and mm_parts and not text_parts: # if we have # - modified the content # - no text parts left in outer message # - a valid X-Mailmann-Version: # --> remove outer message self._log('!!! beheading this one..', rec=rec) mm = message._payload[0] for h in message._headers: if h[0] == 'Content-Type': continue try: mm.replace_header(h[0], h[1]) except KeyError: mm._headers.append(h) return mm return message
def test_single_email_to_a_single_recipient_with_dynamic_templates(self): from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent) self.maxDiff = None message = Mail( from_email=From('*****@*****.**', 'Example From Name'), to_emails=To('*****@*****.**', 'Example To Name'), subject=Subject('Sending with SendGrid is Fun'), plain_text_content=PlainTextContent( 'and easy to do anywhere, even with Python'), html_content=HtmlContent( '<strong>and easy to do anywhere, even with Python</strong>')) message.dynamic_template_data = DynamicTemplateData({ "total": "$ 239.85", "items": [ { "text": "New Line Sneakers", "image": "https://marketing-image-production.s3.amazonaws.com/uploads/8dda1131320a6d978b515cc04ed479df259a458d5d45d58b6b381cae0bf9588113e80ef912f69e8c4cc1ef1a0297e8eefdb7b270064cc046b79a44e21b811802.png", "price": "$ 79.95" }, { "text": "Old Line Sneakers", "image": "https://marketing-image-production.s3.amazonaws.com/uploads/3629f54390ead663d4eb7c53702e492de63299d7c5f7239efdc693b09b9b28c82c924225dcd8dcb65732d5ca7b7b753c5f17e056405bbd4596e4e63a96ae5018.png", "price": "$ 79.95" }, { "text": "Blue Line Sneakers", "image": "https://marketing-image-production.s3.amazonaws.com/uploads/00731ed18eff0ad5da890d876c456c3124a4e44cb48196533e9b95fb2b959b7194c2dc7637b788341d1ff4f88d1dc88e23f7e3704726d313c57f350911dd2bd0.png", "price": "$ 79.95" } ], "receipt": True, "name": "Sample Name", "address01": "1234 Fake St.", "address02": "Apt. 123", "city": "Place", "state": "CO", "zip": "80202" }) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "dynamic_template_data": { "address01": "1234 Fake St.", "address02": "Apt. 123", "city": "Place", "items": [ { "image": "https://marketing-image-production.s3.amazonaws.com/uploads/8dda1131320a6d978b515cc04ed479df259a458d5d45d58b6b381cae0bf9588113e80ef912f69e8c4cc1ef1a0297e8eefdb7b270064cc046b79a44e21b811802.png", "price": "$ 79.95", "text": "New Line Sneakers" }, { "image": "https://marketing-image-production.s3.amazonaws.com/uploads/3629f54390ead663d4eb7c53702e492de63299d7c5f7239efdc693b09b9b28c82c924225dcd8dcb65732d5ca7b7b753c5f17e056405bbd4596e4e63a96ae5018.png", "price": "$ 79.95", "text": "Old Line Sneakers" }, { "image": "https://marketing-image-production.s3.amazonaws.com/uploads/00731ed18eff0ad5da890d876c456c3124a4e44cb48196533e9b95fb2b959b7194c2dc7637b788341d1ff4f88d1dc88e23f7e3704726d313c57f350911dd2bd0.png", "price": "$ 79.95", "text": "Blue Line Sneakers" } ], "name": "Sample Name", "receipt": true, "state": "CO", "total": "$ 239.85", "zip": "80202" }, "to": [ { "email": "*****@*****.**", "name": "Example To Name" } ] } ], "subject": "Sending with SendGrid is Fun" }''') )
def store_and_build_forward_message(self, form, boundary=None, max_bytes_per_blob=None, max_bytes_total=None, bucket_name=None): """Reads form data, stores blobs data and builds the forward request. This finds all of the file uploads in a set of form fields, converting them into blobs and storing them in the blobstore. It also generates the HTTP request to forward to the user's application. Args: form: cgi.FieldStorage instance representing the whole form derived from original POST data. boundary: The optional boundary to use for the resulting form. If omitted, one is randomly generated. max_bytes_per_blob: The maximum size in bytes that any single blob in the form is allowed to be. max_bytes_total: The maximum size in bytes that the total of all blobs in the form is allowed to be. bucket_name: The name of the Google Storage bucket to store the uploaded files. Returns: A tuple (content_type, content_text), where content_type is the value of the Content-Type header, and content_text is a string containing the body of the HTTP request to forward to the application. Raises: webob.exc.HTTPException: The upload failed. """ message = multipart.MIMEMultipart('form-data', boundary) creation = self._now_func() total_bytes_uploaded = 0 created_blobs = [] mime_type_error = None too_many_conflicts = False upload_too_large = False filename_too_large = False content_type_too_large = False # Extract all of the individual form items out of the FieldStorage. form_items = [] # Sorting of forms is done merely to make testing a little easier since # it means blob-keys are generated in a predictable order. for key in sorted(form): form_item = form[key] if isinstance(form_item, list): form_items.extend(form_item) else: form_items.append(form_item) for form_item in form_items: disposition_parameters = {'name': form_item.name} variable = email.message.Message() if form_item.filename is None: # Copy as is variable.add_header('Content-Type', 'text/plain') variable.set_payload(form_item.value) else: # If there is no filename associated with this field it means that the # file form field was not filled in. This blob should not be created # and forwarded to success handler. if not form_item.filename: continue disposition_parameters['filename'] = form_item.filename try: main_type, sub_type = _split_mime_type(form_item.type) except _InvalidMIMETypeFormatError as ex: mime_type_error = str(ex) break # Seek to the end of file and use the pos as the length. form_item.file.seek(0, os.SEEK_END) content_length = form_item.file.tell() form_item.file.seek(0) total_bytes_uploaded += content_length if max_bytes_per_blob is not None: if content_length > max_bytes_per_blob: upload_too_large = True break if max_bytes_total is not None: if total_bytes_uploaded > max_bytes_total: upload_too_large = True break if form_item.filename is not None: if len(form_item.filename) > _MAX_STRING_NAME_LENGTH: filename_too_large = True break if form_item.type is not None: if len(form_item.type) > _MAX_STRING_NAME_LENGTH: content_type_too_large = True break # Compute the MD5 hash of the upload. digester = hashlib.md5() while True: block = form_item.file.read(1 << 20) if not block: break digester.update(block) form_item.file.seek(0) # Create the external body message containing meta-data about the blob. external = email.message.Message() external.add_header('Content-Type', '%s/%s' % (main_type, sub_type), **form_item.type_options) # NOTE: This is in violation of RFC 2616 (Content-MD5 should be the # base-64 encoding of the binary hash, not the hex digest), but it is # consistent with production. content_md5 = base64.urlsafe_b64encode(digester.hexdigest()) # Create header MIME message headers = dict(form_item.headers) for name in _STRIPPED_FILE_HEADERS: if name in headers: del headers[name] headers['Content-Length'] = str(content_length) headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = ( blobstore._format_creation(creation)) headers['Content-MD5'] = content_md5 gs_filename = None if bucket_name: random_key = str(self._generate_blob_key()) gs_filename = '%s/fake-%s' % (bucket_name, random_key) headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = ( blobstore.GS_PREFIX + gs_filename) for key, value in headers.items(): external.add_header(key, value) # Add disposition parameters (a clone of the outer message's field). if not external.get('Content-Disposition'): external.add_header('Content-Disposition', 'form-data', **disposition_parameters) base64_encoding = (form_item.headers.get('Content-Transfer-Encoding') == 'base64') content_type, blob_file, filename = self._preprocess_data( external['content-type'], form_item.file, form_item.filename, base64_encoding) # Store the actual contents to storage. if gs_filename: info_entity = self.store_gs_file( content_type, gs_filename, blob_file, filename) else: try: info_entity = self.store_blob(content_type, filename, digester, blob_file, creation) except _TooManyConflictsError: too_many_conflicts = True break # Track created blobs in case we need to roll them back. created_blobs.append(info_entity) variable.add_header('Content-Type', 'message/external-body', access_type=blobstore.BLOB_KEY_HEADER, blob_key=info_entity.key().name()) variable.set_payload([external]) # Set common information. variable.add_header('Content-Disposition', 'form-data', **disposition_parameters) message.attach(variable) if (mime_type_error or too_many_conflicts or upload_too_large or filename_too_large or content_type_too_large): for blob in created_blobs: datastore.Delete(blob) if mime_type_error: self.abort(400, detail=mime_type_error) elif too_many_conflicts: self.abort(500, detail='Could not generate a blob key.') elif upload_too_large: self.abort(413) else: if filename_too_large: invalid_field = 'filename' elif content_type_too_large: invalid_field = 'Content-Type' detail = 'The %s exceeds the maximum allowed length of %s.' % ( invalid_field, _MAX_STRING_NAME_LENGTH) self.abort(400, detail=detail) message_out = io.StringIO() gen = email.generator.Generator(message_out, maxheaderlen=0) gen.flatten(message, unixfrom=False) # Get the content text out of the message. message_text = message_out.getvalue() content_start = message_text.find('\n\n') + 2 content_text = message_text[content_start:] content_text = content_text.replace('\n', '\r\n') return message.get('Content-Type'), content_text
def _process(self, message, rec=0): """ Recursively scan and filter a MIME message. _process will scan the passed message part for invalid headers as well as mailman signatures and modify them according to the global settings. Generic modifications include: * fixing of broken **References:** and **In-Reply-To** headers * generic header filtering (see :meth:`FuCore._filter_headers`) * removal of Mailman or Mailman-like headers (see :meth:`_mutate_part`) Args: message: a :class:`email.message` object containing a set of MIME parts rec: Recursion level used to prettify log messages. Returns: A (probably) filtered / modified :class:`email.message` object. """ mm_parts = 0 text_parts = 0 mailman_sig = Reactor.MAILMAN_SIG self._log('>>> processing {0}', message.get_content_type(), rec=rec) if self._conf.complex_footer: self._log('--- using complex mailman filter', rec=rec) mailman_sig = Reactor.MAILMAN_COMPLEX if message.is_multipart(): parts = message._payload else: parts = [message,] list_tag = self._find_list_tag(message, rec) reference = message.get('References', None) in_reply = message.get('In-Reply-To', None) x_mailman = message.get('X-Mailman-Version', None) message._headers = self._filter_headers(list_tag, message._headers, self._conf.outlook_hacks, self._conf.fix_dateline, rec) if in_reply and not reference and rec == 0: # set References: to In-Reply-To: if where in toplevel # and References was not set properly self._log('--- set References: {0}', in_reply, rec=rec) try: # uncertain this will ever happen.. message.replace_header('References', in_reply) except KeyError: message._headers.append(('References', in_reply)) for i in xrange(len(parts) - 1, -1, -1): # again, crude since we mutate the list while iterating it.. # the whole reason is the deeply nested structure of email.message p = parts[i] ct = p.get_content_maintype() cs = p.get_content_subtype() ce = p.get('Content-Transfer-Encoding', None) cb = p.get_boundary() self._log('-- [ct = {0}, cs = {1}, ce = <{2}>]', ct, cs, ce, rec=rec) if ct == 'text': text_parts += 1 payload = p.get_payload(decode=True) self._log('--- scan: """{0}"""', payload, rec=rec, verbosity=3) if mailman_sig[0].search(payload) and \ mailman_sig[1].match(payload.split('\n')[0]): self._log('*** removing this part', rec=rec) self._log('--- """{0}"""', payload, rec=rec, verbosity=2) message._payload.remove(p) text_parts -= 1 mm_parts += 1 elif mailman_sig[0].search(payload): self._log('--- trying to mutate..', rec=rec) (use, mutation) = self._mutate_part(payload, rec) if use: self._log('*** mutated this part', rec=rec) self._log('--- """{0}"""', payload, rec=rec, verbosity=2) payload = mutation mm_parts += 1 # if it was encoded we need to re-encode it # to keep SMIME happy if ce == 'base64': payload = payload.encode('base64') elif ce == 'quoted-printable': payload = quopri.encodestring(payload) p.set_payload(payload) elif ct == 'message' or \ (ct == 'multipart' and cs in ['alternative', 'mixed']): p = self._process(p, rec + 1) else: self._log('--- what about {0}?', p.get_content_type(), rec=rec) if rec == 0: self._log('--- [mm_parts: {0}, text_parts: {1}, x_mailman: {0}]', mm_parts, text_parts, x_mailman, rec=rec) if x_mailman and mm_parts and not text_parts: # if we have # - modified the content # - no text parts left in outer message # - a valid X-Mailmann-Version: # --> remove outer message self._log('!!! beheading this one..', rec=rec) mm = message._payload[0] for h in message._headers: if h[0] == 'Content-Type': continue try: mm.replace_header(h[0], h[1]) except KeyError: mm._headers.append(h) return mm return message
def get_quarantined_messages(self): """ Returns an iterator over the messages currently in the quarantine. """ for message, error in self._quarantine.values(): yield message.get(), error
def _find_list_tag(self, message, rec=0, plain=False): """ Filter *message* for a valid list tag. This method will scan the passed in messages headers looking for a hint to the used list-tag. The following headers will be checked (in order): * X-SynFU-Tags (explicit List-Tags supplied by synfu-news2mail) * [X-]List-Post * [X-]List-Id * [X-]AF-Envelope-to If any of these is found it will be converted into a regular expression which can be used to remove the List-Tag from arbitrary headers. :param message: A :class:`email.message` object. :param rec: Optional recursion level used to indent messages. :param plain: If :const:`True` return the plain List-Tag (no regexp) :returns: Either a :class:`re.SRE_PATTERN` or a string """ tag = message.get('X-SynFU-Tags', None) lp = message.get('List-Post', message.get('X-List-Post', None)) lid = message.get('List-Id' , message.get('X-List-Id', None)) evl = message.get('AF-Envelope-to', message.get('X-AF-Envelope-to', None)) tag_base = None if tag: tag = email.header.decode_header(tag)[0][0] self._log('--- using supplied SynFU tag hints', rec=rec) tag_base = '({0})'.format('|'.join(x.strip() for x in tag.split(',') if x.strip())) # preffer List-Id if we have it elif lid: lid = email.header.decode_header(lid)[-1][0] tag_base = lid.split('<')[-1].split('.')[0].strip() elif lp: lp = email.header.decode_header(lp)[0][0] try: tag_base = lp.split('mailto:')[1].split('@')[0] except IndexError: tag_base = None elif evl: evl = email.header.decode_header(evl)[0][0] try: tag_base = evl.split('@')[0] except IndexError: tag_base = None if plain: return tag_base if tag_base: self._log('--- list tag: "[*{0}*]"', format(tag_base), rec=rec) return re.compile('(?i)\[[^[]*{0}[^]]*\]'.format(tag_base)) self._log('--- no list tag found', rec=rec) # return 'moab' return re.compile('(?i)\s*\[[^]]*(?# This is here to confuse people)\]\s*')
def _apply_blacklist(self, message, mode, rec=0): """ Apply the global blacklist to this message. This method will modify the headers of the message or decide to discard the whole message based on the global blacklist. :param messag: A :class:`email.message` object. :param mode: One of :attr:`FUCore.BLACKLIST_MODES` :returns: The modified message or None if the message should be dropped. """ if not mode.lower() in FUCore.BLACKLIST_MODES: self._log('!!! Invalid blacklist mode "{0}", not modifying message.', mode, rec=rec) return message mfrom = message.get('From', '').split('<')[-1].split('>')[0] sender = message.get('Sender', '').split('<')[-1].split('>')[0] if not mfrom and not msender: self._log('!!! Message has neiter "From:" nor "Sender:" headers!', rec=rec) return message list_entry = self._blacklist.get(mfrom.lower(), self._blacklist.get(sender.lower(), None)) if not list_entry: return message self._log('--- applying blacklist rule {0} to message', str(list_entry), rec=rec) if not list_entry['action'].lower() in ['d', 'n', 'e', 'ne', 'en']: # invalid self._log('!!! unsupported blacklist rule "{0}"', list_entry['action'], rec=rec) return message add_expires=False add_xnay=False if list_entry['action'].lower() == 'd' and not mode.lower() == 'reactor': self._log('--- blacklist rule "drop"', rec=rec, verbosity=2) return None elif mode.lower() == 'news2mail': #D=N=E=NE/EN=drop self._log('--- blacklist rule D=N=E=NE/EN "drop"', rec=rec, verbosity=2) return None elif mode.lower() == 'mail2news': if list_entry['action'].lower() in ['ne', 'en','n']: add_xnay = True if list_entry['action'].lower() in ['ne', 'en', 'e']: add_expires = True elif mode.lower() == 'reactor': if list_entry['action'].lower() in ['ne','en','n','d']: add_xnay = True if list_entry['action'].lower() in ['ne','en','e']: add_expires = True if add_xnay: self._log('--- blacklist rule "xnay"', rec=rec, verbosity=2) try: message.replace_header('X-No-Archive', 'yes') except KeyError: message._headers.append(('X-No-Archive', 'yes')) if add_expires: if not list_entry.get('param', None): self._log('!!! blacklist rule "expires" missing a parameter', rec=rec) else: try: delay=long(list_entry['param']) expires=time.strftime(r"%d %b %Y %H:%M:%S %z", time.localtime(time.time() + (86400*delay))) self._log('--- blacklist rule "expires" => {0}', expires, rec=rec, verbosity=2) try: message.replace_header('Expires', expires) except KeyError: message._headers.append(('Expires', expires)) except ValueError: self.log('!!! blacklist rule "expires" needs a *numeric* parameter.') return message
def test_single_email_to_a_single_recipient_with_dynamic_templates(self): from sendgrid.helpers.mail import (Mail, From, To, Subject, PlainTextContent, HtmlContent) self.maxDiff = None message = Mail( from_email=From('*****@*****.**', 'Example From Name'), to_emails=To('*****@*****.**', 'Example To Name'), subject=Subject('Sending with SendGrid is Fun'), plain_text_content=PlainTextContent( 'and easy to do anywhere, even with Python'), html_content=HtmlContent( '<strong>and easy to do anywhere, even with Python</strong>')) message.dynamic_template_data = DynamicTemplateData({ "total": "$ 239.85", "items": [{ "text": "New Line Sneakers", "image": "https://marketing-image-production.s3.amazonaws.com/uploads/8dda1131320a6d978b515cc04ed479df259a458d5d45d58b6b381cae0bf9588113e80ef912f69e8c4cc1ef1a0297e8eefdb7b270064cc046b79a44e21b811802.png", "price": "$ 79.95" }, { "text": "Old Line Sneakers", "image": "https://marketing-image-production.s3.amazonaws.com/uploads/3629f54390ead663d4eb7c53702e492de63299d7c5f7239efdc693b09b9b28c82c924225dcd8dcb65732d5ca7b7b753c5f17e056405bbd4596e4e63a96ae5018.png", "price": "$ 79.95" }, { "text": "Blue Line Sneakers", "image": "https://marketing-image-production.s3.amazonaws.com/uploads/00731ed18eff0ad5da890d876c456c3124a4e44cb48196533e9b95fb2b959b7194c2dc7637b788341d1ff4f88d1dc88e23f7e3704726d313c57f350911dd2bd0.png", "price": "$ 79.95" }], "receipt": True, "name": "Sample Name", "address01": "1234 Fake St.", "address02": "Apt. 123", "city": "Place", "state": "CO", "zip": "80202" }) self.assertEqual( message.get(), json.loads(r'''{ "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" } ], "from": { "email": "*****@*****.**", "name": "Example From Name" }, "personalizations": [ { "dynamic_template_data": { "address01": "1234 Fake St.", "address02": "Apt. 123", "city": "Place", "items": [ { "image": "https://marketing-image-production.s3.amazonaws.com/uploads/8dda1131320a6d978b515cc04ed479df259a458d5d45d58b6b381cae0bf9588113e80ef912f69e8c4cc1ef1a0297e8eefdb7b270064cc046b79a44e21b811802.png", "price": "$ 79.95", "text": "New Line Sneakers" }, { "image": "https://marketing-image-production.s3.amazonaws.com/uploads/3629f54390ead663d4eb7c53702e492de63299d7c5f7239efdc693b09b9b28c82c924225dcd8dcb65732d5ca7b7b753c5f17e056405bbd4596e4e63a96ae5018.png", "price": "$ 79.95", "text": "Old Line Sneakers" }, { "image": "https://marketing-image-production.s3.amazonaws.com/uploads/00731ed18eff0ad5da890d876c456c3124a4e44cb48196533e9b95fb2b959b7194c2dc7637b788341d1ff4f88d1dc88e23f7e3704726d313c57f350911dd2bd0.png", "price": "$ 79.95", "text": "Blue Line Sneakers" } ], "name": "Sample Name", "receipt": true, "state": "CO", "total": "$ 239.85", "zip": "80202" }, "to": [ { "email": "*****@*****.**", "name": "Example To Name" } ] } ], "subject": "Sending with SendGrid is Fun" }'''))
def parse(data, codec=Codec): # noqa: C901 message = email.message_from_bytes(data) # ################################################################## HEADERS _, issuer = email.utils.parseaddr(message.get("from")) subject = utils_parse.normalize(message.get("subject")) recipient, identifier, domain = get_address_parts(message.get("to")) new_tag, subject = utils_parse.consume_re(codec.re_new(), subject) new = bool(new_tag) external_id, subject = utils_parse.consume_re(codec.re_external_id(), subject) if not identifier and "in-reply-to" in message: irt = get_address_parts(message.get("in-reply-to")) if irt[0] == recipient and irt[2] == domain: if irt[1] == "new": new_tag = True else: identifier = irt[1] identifier = irt[1] if not identifier: identifier, subject = utils_parse.consume_re(codec.re_id(), subject) # ##################################################################### BODY json_part, text_part, html_part, attachments = "", "", "", [] for part in message.walk(): # sub-parts are iterated over in this walk if part.is_multipart(): continue payload = part.get_payload(decode=True) if geojson and part.get_content_type() == "application/geo+json": try: attachments.append((part.get_content_type(), geojson.loads(payload.decode("utf-8")))) except ValueError: pass elif part.get_content_type() == "application/json": try: json_part = json.loads(payload.decode("utf-8")) except ValueError: json_part = None elif part.get_content_maintype() == "text": payload = payload.decode(part.get_content_charset() or "utf-8") # if multiple text/plain parts are given, concatenate if part.get_content_subtype() == "plain": text_part += payload # if no text/plain version is given, # get plain text version from HTML using BeautifulSoup if part.get_content_subtype() == "html" and not text_part: soup = bs4.BeautifulSoup(payload, "html.parser") for item in soup(["script", "style"]): item.extract() html_part += "\n" + "\n".join( l.strip() for l in soup.get_text().split("\n") if l) # other subtypes of txt are considered attachments if part.get_content_subtype() not in ("plain", "html"): attachments.append((part.get_content_type(), payload)) # attachments elif part.get_content_maintype() in ("image", "video", "application"): attachments.append((part.get_content_type(), payload)) # ################################################################# FIX TEXT text_part = (text_part or html_part).strip() # remove signature match = SIGNATURE_RE.search(text_part) if match: text_part = text_part[:match.start()].strip() # ################################################################# FIX JSON # if no `application/json` part was given, parse `text/plain` as YAML # if we manage to parse YAML, remove this part from `text/plain` # otherwise, leave the text part untouched if not json_part: try: # only parse the first YAML document provided. # this enables the user to provide YAML, # use the '---' or '...' YAML document separators # and provided plain text afterwards. json_part = next(yaml.safe_load_all(text_part)) document_starts = [ e for e in yaml.parse(text_part) if isinstance(e, yaml.events.DocumentStartEvent) ][1:] if document_starts: text_part = text_part[document_starts[1].end_mark.index + 1:] else: text_part = "" except yaml.YAMLError: json_part = None return codec.update_item( domain=domain, issuer=issuer, recipient=recipient, identifier=identifier, external_id=external_id, new=new, subject=subject, json_part=json_part, text_part=text_part, attachments=attachments, )
def test_kitchen_sink(self): from sendgrid.helpers.mail import ( Mail, From, To, Cc, Bcc, Subject, Substitution, Header, CustomArg, SendAt, Content, MimeType, Attachment, FileName, FileContent, FileType, Disposition, ContentId, TemplateId, Section, ReplyTo, Category, BatchId, Asm, GroupId, GroupsToDisplay, IpPoolName, MailSettings, BccSettings, BccSettingsEmail, BypassListManagement, FooterSettings, FooterText, FooterHtml, SandBoxMode, SpamCheck, SpamThreshold, SpamUrl, TrackingSettings, ClickTracking, SubscriptionTracking, SubscriptionText, SubscriptionHtml, SubscriptionSubstitutionTag, OpenTracking, OpenTrackingSubstitutionTag, Ganalytics, UtmSource, UtmMedium, UtmTerm, UtmContent, UtmCampaign) self.maxDiff = None message = Mail() # Define Personalizations message.to = To('*****@*****.**', 'Example User1', p=0) message.to = [ To('*****@*****.**', 'Example User2', p=0), To('*****@*****.**', 'Example User3', p=0) ] message.cc = Cc('*****@*****.**', 'Example User4', p=0) message.cc = [ Cc('*****@*****.**', 'Example User5', p=0), Cc('*****@*****.**', 'Example User6', p=0) ] message.bcc = Bcc('*****@*****.**', 'Example User7', p=0) message.bcc = [ Bcc('*****@*****.**', 'Example User8', p=0), Bcc('*****@*****.**', 'Example User9', p=0) ] message.subject = Subject('Sending with SendGrid is Fun 0', p=0) message.header = Header('X-Test1', 'Test1', p=0) message.header = Header('X-Test2', 'Test2', p=0) message.header = [ Header('X-Test3', 'Test3', p=0), Header('X-Test4', 'Test4', p=0) ] message.substitution = Substitution('%name1%', 'Example Name 1', p=0) message.substitution = Substitution('%city1%', 'Example City 1', p=0) message.substitution = [ Substitution('%name2%', 'Example Name 2', p=0), Substitution('%city2%', 'Example City 2', p=0) ] message.custom_arg = CustomArg('marketing1', 'true', p=0) message.custom_arg = CustomArg('transactional1', 'false', p=0) message.custom_arg = [ CustomArg('marketing2', 'false', p=0), CustomArg('transactional2', 'true', p=0) ] message.send_at = SendAt(1461775051, p=0) message.to = To('*****@*****.**', 'Example User10', p=1) message.to = [ To('*****@*****.**', 'Example User11', p=1), To('*****@*****.**', 'Example User12', p=1) ] message.cc = Cc('*****@*****.**', 'Example User13', p=1) message.cc = [ Cc('*****@*****.**', 'Example User14', p=1), Cc('*****@*****.**', 'Example User15', p=1) ] message.bcc = Bcc('*****@*****.**', 'Example User16', p=1) message.bcc = [ Bcc('*****@*****.**', 'Example User17', p=1), Bcc('*****@*****.**', 'Example User18', p=1) ] message.header = Header('X-Test5', 'Test5', p=1) message.header = Header('X-Test6', 'Test6', p=1) message.header = [ Header('X-Test7', 'Test7', p=1), Header('X-Test8', 'Test8', p=1) ] message.substitution = Substitution('%name3%', 'Example Name 3', p=1) message.substitution = Substitution('%city3%', 'Example City 3', p=1) message.substitution = [ Substitution('%name4%', 'Example Name 4', p=1), Substitution('%city4%', 'Example City 4', p=1) ] message.custom_arg = CustomArg('marketing3', 'true', p=1) message.custom_arg = CustomArg('transactional3', 'false', p=1) message.custom_arg = [ CustomArg('marketing4', 'false', p=1), CustomArg('transactional4', 'true', p=1) ] message.send_at = SendAt(1461775052, p=1) message.subject = Subject('Sending with SendGrid is Fun 1', p=1) # The values below this comment are global to entire message message.from_email = From('*****@*****.**', 'DX') message.reply_to = ReplyTo('*****@*****.**', 'DX Reply') message.subject = Subject('Sending with SendGrid is Fun 2') message.content = Content( MimeType.text, 'and easy to do anywhere, even with Python') message.content = Content( MimeType.html, '<strong>and easy to do anywhere, even with Python</strong>') message.content = [ Content('text/calendar', 'Party Time!!'), Content('text/custom', 'Party Time 2!!') ] message.attachment = Attachment( FileContent('base64 encoded content 1'), FileName('balance_001.pdf'), FileType('application/pdf'), Disposition('attachment'), ContentId('Content ID 1')) message.attachment = [ Attachment( FileContent('base64 encoded content 2'), FileName('banner.png'), FileType('image/png'), Disposition('inline'), ContentId('Content ID 2')), Attachment( FileContent('base64 encoded content 3'), FileName('banner2.png'), FileType('image/png'), Disposition('inline'), ContentId('Content ID 3')) ] message.template_id = TemplateId( '13b8f94f-bcae-4ec6-b752-70d6cb59f932') message.section = Section( '%section1%', 'Substitution for Section 1 Tag') message.section = [ Section('%section2%', 'Substitution for Section 2 Tag'), Section('%section3%', 'Substitution for Section 3 Tag') ] message.header = Header('X-Test9', 'Test9') message.header = Header('X-Test10', 'Test10') message.header = [ Header('X-Test11', 'Test11'), Header('X-Test12', 'Test12') ] message.category = Category('Category 1') message.category = Category('Category 2') message.category = [ Category('Category 1'), Category('Category 2') ] message.custom_arg = CustomArg('marketing5', 'false') message.custom_arg = CustomArg('transactional5', 'true') message.custom_arg = [ CustomArg('marketing6', 'true'), CustomArg('transactional6', 'false') ] message.send_at = SendAt(1461775053) message.batch_id = BatchId("HkJ5yLYULb7Rj8GKSx7u025ouWVlMgAi") message.asm = Asm(GroupId(1), GroupsToDisplay([1, 2, 3, 4])) message.ip_pool_name = IpPoolName("IP Pool Name") mail_settings = MailSettings() mail_settings.bcc_settings = BccSettings( False, BccSettingsEmail("*****@*****.**")) mail_settings.bypass_list_management = BypassListManagement(False) mail_settings.footer_settings = FooterSettings( True, FooterText("w00t"), FooterHtml("<string>w00t!<strong>")) mail_settings.sandbox_mode = SandBoxMode(True) mail_settings.spam_check = SpamCheck( True, SpamThreshold(5), SpamUrl("https://example.com")) message.mail_settings = mail_settings tracking_settings = TrackingSettings() tracking_settings.click_tracking = ClickTracking(True, False) tracking_settings.open_tracking = OpenTracking( True, OpenTrackingSubstitutionTag("open_tracking")) tracking_settings.subscription_tracking = SubscriptionTracking( True, SubscriptionText("Goodbye"), SubscriptionHtml("<strong>Goodbye!</strong>"), SubscriptionSubstitutionTag("unsubscribe")) tracking_settings.ganalytics = Ganalytics( True, UtmSource("utm_source"), UtmMedium("utm_medium"), UtmTerm("utm_term"), UtmContent("utm_content"), UtmCampaign("utm_campaign")) message.tracking_settings = tracking_settings self.assertEqual( message.get(), json.loads(r'''{ "asm": { "group_id": 1, "groups_to_display": [ 1, 2, 3, 4 ] }, "attachments": [ { "content": "base64 encoded content 3", "content_id": "Content ID 3", "disposition": "inline", "filename": "banner2.png", "type": "image/png" }, { "content": "base64 encoded content 2", "content_id": "Content ID 2", "disposition": "inline", "filename": "banner.png", "type": "image/png" }, { "content": "base64 encoded content 1", "content_id": "Content ID 1", "disposition": "attachment", "filename": "balance_001.pdf", "type": "application/pdf" } ], "batch_id": "HkJ5yLYULb7Rj8GKSx7u025ouWVlMgAi", "categories": [ "Category 2", "Category 1", "Category 2", "Category 1" ], "content": [ { "type": "text/plain", "value": "and easy to do anywhere, even with Python" }, { "type": "text/html", "value": "<strong>and easy to do anywhere, even with Python</strong>" }, { "type": "text/calendar", "value": "Party Time!!" }, { "type": "text/custom", "value": "Party Time 2!!" } ], "custom_args": { "marketing5": "false", "marketing6": "true", "transactional5": "true", "transactional6": "false" }, "from": { "email": "*****@*****.**", "name": "DX" }, "headers": { "X-Test10": "Test10", "X-Test11": "Test11", "X-Test12": "Test12", "X-Test9": "Test9" }, "ip_pool_name": "IP Pool Name", "mail_settings": { "bcc": { "email": "*****@*****.**", "enable": false }, "bypass_list_management": { "enable": false }, "footer": { "enable": true, "html": "<string>w00t!<strong>", "text": "w00t" }, "sandbox_mode": { "enable": true }, "spam_check": { "enable": true, "post_to_url": "https://example.com", "threshold": 5 } }, "personalizations": [ { "bcc": [ { "email": "*****@*****.**", "name": "Example User7" }, { "email": "*****@*****.**", "name": "Example User8" }, { "email": "*****@*****.**", "name": "Example User9" } ], "cc": [ { "email": "*****@*****.**", "name": "Example User4" }, { "email": "*****@*****.**", "name": "Example User5" }, { "email": "*****@*****.**", "name": "Example User6" } ], "custom_args": { "marketing1": "true", "marketing2": "false", "transactional1": "false", "transactional2": "true" }, "headers": { "X-Test1": "Test1", "X-Test2": "Test2", "X-Test3": "Test3", "X-Test4": "Test4" }, "send_at": 1461775051, "subject": "Sending with SendGrid is Fun 0", "substitutions": { "%city1%": "Example City 1", "%city2%": "Example City 2", "%name1%": "Example Name 1", "%name2%": "Example Name 2" }, "to": [ { "email": "*****@*****.**", "name": "Example User1" }, { "email": "*****@*****.**", "name": "Example User2" }, { "email": "*****@*****.**", "name": "Example User3" } ] }, { "bcc": [ { "email": "*****@*****.**", "name": "Example User16" }, { "email": "*****@*****.**", "name": "Example User17" }, { "email": "*****@*****.**", "name": "Example User18" } ], "cc": [ { "email": "*****@*****.**", "name": "Example User13" }, { "email": "*****@*****.**", "name": "Example User14" }, { "email": "*****@*****.**", "name": "Example User15" } ], "custom_args": { "marketing3": "true", "marketing4": "false", "transactional3": "false", "transactional4": "true" }, "headers": { "X-Test5": "Test5", "X-Test6": "Test6", "X-Test7": "Test7", "X-Test8": "Test8" }, "send_at": 1461775052, "subject": "Sending with SendGrid is Fun 1", "substitutions": { "%city3%": "Example City 3", "%city4%": "Example City 4", "%name3%": "Example Name 3", "%name4%": "Example Name 4" }, "to": [ { "email": "*****@*****.**", "name": "Example User10" }, { "email": "*****@*****.**", "name": "Example User11" }, { "email": "*****@*****.**", "name": "Example User12" } ] } ], "reply_to": { "email": "*****@*****.**", "name": "DX Reply" }, "sections": { "%section1%": "Substitution for Section 1 Tag", "%section2%": "Substitution for Section 2 Tag", "%section3%": "Substitution for Section 3 Tag" }, "send_at": 1461775053, "subject": "Sending with SendGrid is Fun 2", "template_id": "13b8f94f-bcae-4ec6-b752-70d6cb59f932", "tracking_settings": { "click_tracking": { "enable": true, "enable_text": false }, "ganalytics": { "enable": true, "utm_campaign": "utm_campaign", "utm_content": "utm_content", "utm_medium": "utm_medium", "utm_source": "utm_source", "utm_term": "utm_term" }, "open_tracking": { "enable": true, "substitution_tag": "open_tracking" }, "subscription_tracking": { "enable": true, "html": "<strong>Goodbye!</strong>", "substitution_tag": "unsubscribe", "text": "Goodbye" } } }''') )
def get_msgid_from_stdin(): if not sys.stdin.isatty(): message = email.message_from_string(sys.stdin.read()) return message.get('Message-ID', None) logger.error('Error: pipe a message or pass msgid as parameter') sys.exit(1)
def store_and_build_forward_message(self, form, boundary=None, max_bytes_per_blob=None, max_bytes_total=None, bucket_name=None): """Reads form data, stores blobs data and builds the forward request. This finds all of the file uploads in a set of form fields, converting them into blobs and storing them in the blobstore. It also generates the HTTP request to forward to the user's application. Args: form: cgi.FieldStorage instance representing the whole form derived from original POST data. boundary: The optional boundary to use for the resulting form. If omitted, one is randomly generated. max_bytes_per_blob: The maximum size in bytes that any single blob in the form is allowed to be. max_bytes_total: The maximum size in bytes that the total of all blobs in the form is allowed to be. bucket_name: The name of the Google Storage bucket to store the uploaded files. Returns: A tuple (content_type, content_text), where content_type is the value of the Content-Type header, and content_text is a string containing the body of the HTTP request to forward to the application. Raises: webob.exc.HTTPException: The upload failed. """ message = multipart.MIMEMultipart('form-data', boundary) creation = self._now_func() total_bytes_uploaded = 0 created_blobs = [] mime_type_error = None too_many_conflicts = False upload_too_large = False filename_too_large = False content_type_too_large = False # Extract all of the individual form items out of the FieldStorage. form_items = [] # Sorting of forms is done merely to make testing a little easier since # it means blob-keys are generated in a predictable order. for key in sorted(form): form_item = form[key] if isinstance(form_item, list): form_items.extend(form_item) else: form_items.append(form_item) for form_item in form_items: disposition_parameters = {'name': form_item.name} variable = email.message.Message() if form_item.filename is None: # Copy as is variable.add_header('Content-Type', 'text/plain') variable.set_payload(form_item.value) else: # If there is no filename associated with this field it means that the # file form field was not filled in. This blob should not be created # and forwarded to success handler. if not form_item.filename: continue disposition_parameters['filename'] = form_item.filename try: main_type, sub_type = _split_mime_type(form_item.type) except _InvalidMIMETypeFormatError as ex: mime_type_error = str(ex) break # Seek to the end of file and use the pos as the length. form_item.file.seek(0, os.SEEK_END) content_length = form_item.file.tell() form_item.file.seek(0) total_bytes_uploaded += content_length if max_bytes_per_blob is not None: if content_length > max_bytes_per_blob: upload_too_large = True break if max_bytes_total is not None: if total_bytes_uploaded > max_bytes_total: upload_too_large = True break if form_item.filename is not None: if len(form_item.filename) > _MAX_STRING_NAME_LENGTH: filename_too_large = True break if form_item.type is not None: if len(form_item.type) > _MAX_STRING_NAME_LENGTH: content_type_too_large = True break # Compute the MD5 hash of the upload. digester = hashlib.md5() while True: block = form_item.file.read(1 << 20) if not block: break digester.update(block) form_item.file.seek(0) # Create the external body message containing meta-data about the blob. external = email.message.Message() external.add_header('Content-Type', '%s/%s' % (main_type, sub_type), **form_item.type_options) # NOTE: This is in violation of RFC 2616 (Content-MD5 should be the # base-64 encoding of the binary hash, not the hex digest), but it is # consistent with production. content_md5 = base64.urlsafe_b64encode(digester.hexdigest()) # Create header MIME message headers = dict(form_item.headers) for name in _STRIPPED_FILE_HEADERS: if name in headers: del headers[name] headers['Content-Length'] = str(content_length) headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = ( blobstore._format_creation(creation)) headers['Content-MD5'] = content_md5 gs_filename = None if bucket_name: random_key = str(self._generate_blob_key()) gs_filename = '%s/fake-%s' % (bucket_name, random_key) headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = ( blobstore.GS_PREFIX + gs_filename) for key, value in six.iteritems(headers): external.add_header(key, value) # Add disposition parameters (a clone of the outer message's field). if not external.get('Content-Disposition'): external.add_header('Content-Disposition', 'form-data', **disposition_parameters) base64_encoding = (form_item.headers.get( 'Content-Transfer-Encoding') == 'base64') content_type, blob_file, filename = self._preprocess_data( external['content-type'], form_item.file, form_item.filename, base64_encoding) # Store the actual contents to storage. if gs_filename: info_entity = self.store_gs_file(content_type, gs_filename, blob_file, filename) else: try: info_entity = self.store_blob(content_type, filename, digester, blob_file, creation) except _TooManyConflictsError: too_many_conflicts = True break # Track created blobs in case we need to roll them back. created_blobs.append(info_entity) variable.add_header('Content-Type', 'message/external-body', access_type=blobstore.BLOB_KEY_HEADER, blob_key=info_entity.key().name()) variable.set_payload([external]) # Set common information. variable.add_header('Content-Disposition', 'form-data', **disposition_parameters) message.attach(variable) if (mime_type_error or too_many_conflicts or upload_too_large or filename_too_large or content_type_too_large): for blob in created_blobs: datastore.Delete(blob) if mime_type_error: self.abort(400, detail=mime_type_error) elif too_many_conflicts: self.abort(500, detail='Could not generate a blob key.') elif upload_too_large: self.abort(413) else: if filename_too_large: invalid_field = 'filename' elif content_type_too_large: invalid_field = 'Content-Type' detail = 'The %s exceeds the maximum allowed length of %s.' % ( invalid_field, _MAX_STRING_NAME_LENGTH) self.abort(400, detail=detail) message_out = io.StringIO() gen = email.generator.Generator(message_out, maxheaderlen=0) gen.flatten(message, unixfrom=False) # Get the content text out of the message. message_text = message_out.getvalue() content_start = message_text.find('\n\n') + 2 content_text = message_text[content_start:] content_text = content_text.replace('\n', '\r\n') return message.get('Content-Type'), content_text
def _run_customizer(self): """Executes the entire process of customize a mailing for a recipient and returns its full path. This may take some time and shouldn't be run from the reactor thread. """ try: fullpath = os.path.join( self.temp_path, MailCustomizer.make_file_name(self.recipient.mailing.id, self.recipient.id)) if os.path.exists(fullpath): self.log.debug("Customized email found here: %s", fullpath) parser = email.parser.Parser() with file(fullpath, 'rt') as fd: header = parser.parse(fd, headersonly=True) return header['Message-ID'], fullpath contact_data = self.make_contact_data_dict(self.recipient) message = self._parse_message() assert (isinstance(contact_data, dict)) assert (isinstance(message, Message)) #email.iterators._structure(message) mixed_attachments = [] related_attachments = [] for attachment in contact_data.get('attachments', []): if 'content-id' in attachment: related_attachments.append(attachment) else: mixed_attachments.append(attachment) #bodies = MailingBody.objects.filter(relay = self.recipient.mailing_queue).order_by('header_pos') def convert_to_mixed(part, mixed_attachments, subtype): import email.mime.multipart part2 = email.mime.multipart.MIMEMultipart(_subtype=subtype) part2.set_payload(part.get_payload()) del part['Content-Type'] part['Content-Type'] = 'multipart/mixed' part.set_payload(None) part.attach(part2) for attachment in mixed_attachments: part.attach(self._make_mime_part(attachment)) def personalise_bodies(part, mixed_attachments=[], related_attachments=[]): import email.message assert (isinstance(part, email.message.Message)) if part.is_multipart(): subtype = part.get_content_subtype() if subtype == 'mixed': personalise_bodies( part.get_payload(0), related_attachments=related_attachments) for attachment in mixed_attachments: part.attach(self._make_mime_part(attachment)) elif subtype == 'alternative': for p in part.get_payload(): personalise_bodies( p, related_attachments=related_attachments) if mixed_attachments: convert_to_mixed(part, mixed_attachments, subtype="alternative") elif subtype == 'digest': raise email.errors.MessageParseError, "multipart/digest not supported" elif subtype == 'parallel': raise email.errors.MessageParseError, "multipart/parallel not supported" elif subtype == 'related': personalise_bodies(part.get_payload(0)) for attachment in related_attachments: part.attach(self._make_mime_part(attachment)) if mixed_attachments: convert_to_mixed(part, mixed_attachments, subtype="related") else: self.log.warn("Unknown multipart subtype '%s'" % subtype) else: maintype = part.get_content_maintype() if maintype == 'text': self._customize_message(part, contact_data) if mixed_attachments: import email.mime.text part2 = email.mime.text.MIMEText( part.get_payload(decode=True)) del part['Content-Type'] part['Content-Type'] = 'multipart/mixed' part.set_payload(None) part.attach(part2) for attachment in mixed_attachments: part.attach(self._make_mime_part(attachment)) else: self.log.warn( "personalise_bodies(): can't handle '%s' parts" % part.get_content_type()) personalise_bodies(message, mixed_attachments, related_attachments) # Customize the subject subject = self._do_customization( header_to_unicode(message.get("Subject", "")), contact_data) # Remove some headers for header in ('Subject', 'Received', 'To', 'From', 'User-Agent', 'Date', 'Message-ID', 'List-Unsubscribe', 'DKIM-Signature', 'Authentication-Results', 'Received-SPF', 'Received-SPF', 'X-Received', 'Delivered-To', 'Feedback-ID', 'Precedence', 'Return-Path'): if header in message: del message[header] message['Subject'] = Header(subject) # Adding missing headers # message['Precedence'] = "bulk" h = Header(self.recipient.sender_name or '') h.append("<%s>" % self.recipient.mail_from) message['From'] = h h = Header() h.append(contact_data.get('firstname') or '') h.append(contact_data.get('lastname') or '') h.append("<%s>" % contact_data['email']) message['To'] = h message['Date'] = email.utils.formatdate() # message['Message-ID'] = email.utils.make_msgid() # very very slow on certain circumstance message['Message-ID'] = "<%s.%d@cm.%s>" % ( self.recipient.id, self.recipient.mailing.id, self.recipient.domain_name) if self.unsubscribe_url: message['List-Unsubscribe'] = self.unsubscribe_url fp = cStringIO.StringIO() generator = email.generator.Generator(fp, mangle_from_=False) generator.flatten(message) flattened_message = fp.getvalue() flattened_message = self.add_dkim_signature(flattened_message) flattened_message = self.add_fbl(flattened_message) with open(fullpath + '.tmp', 'wt') as fp: fp.write(flattened_message) fp.close() if os.path.exists(fullpath): os.remove(fullpath) os.rename(fullpath + '.tmp', fullpath) return message['Message-ID'], fullpath except Exception: self.log.exception( "Failed to customize mailing '%s' for recipient '%s'" % (self.recipient.mail_from, self.recipient.email)) raise
def _find_list_tag(self, message, rec=0, plain=False): """ Filter *message* for a valid list tag. This method will scan the passed in messages headers looking for a hint to the used list-tag. The following headers will be checked (in order): * X-SynFU-Tags (explicit List-Tags supplied by synfu-news2mail) * [X-]List-Post * [X-]List-Id * [X-]AF-Envelope-to If any of these is found it will be converted into a regular expression which can be used to remove the List-Tag from arbitrary headers. :param message: A :class:`email.message` object. :param rec: Optional recursion level used to indent messages. :param plain: If :const:`True` return the plain List-Tag (no regexp) :returns: Either a :class:`re.SRE_PATTERN` or a string """ tag = message.get('X-SynFU-Tags', None) lp = message.get('List-Post', message.get('X-List-Post', None)) lid = message.get('List-Id', message.get('X-List-Id', None)) evl = message.get('AF-Envelope-to', message.get('X-AF-Envelope-to', None)) tag_base = None if tag: tag = email.header.decode_header(tag)[0][0] self._log('--- using supplied SynFU tag hints', rec=rec) tag_base = '({0})'.format('|'.join(x.strip() for x in tag.split(',') if x.strip())) # preffer List-Id if we have it elif lid: lid = email.header.decode_header(lid)[-1][0] tag_base = lid.split('<')[-1].split('.')[0].strip() elif lp: lp = email.header.decode_header(lp)[0][0] try: tag_base = lp.split('mailto:')[1].split('@')[0] except IndexError: tag_base = None elif evl: evl = email.header.decode_header(evl)[0][0] try: tag_base = evl.split('@')[0] except IndexError: tag_base = None if plain: return tag_base if tag_base: self._log('--- list tag: "[*{0}*]"', format(tag_base), rec=rec) return re.compile('(?i)\[[^[]*{0}[^]]*\]'.format(tag_base)) self._log('--- no list tag found', rec=rec) # return 'moab' return re.compile( '(?i)\s*\[[^]]*(?# This is here to confuse people)\]\s*')
def test_parsing_message_with_custom_message_class(self): message = email.message_from_string(self.attachment_eml, Message) self.assertEqual(message.get('to'), '*****@*****.**')
class Application(object): """A WSGI middleware application for handling blobstore upload requests. This application will handle all uploaded files in a POST request, store the results in the blob-storage, close the upload session and forward the request on to another WSGI application, with the environment transformed so that the uploaded file contents are replaced with their blob keys. """ def __init__(self, forward_app, get_blob_storage=_get_blob_storage, generate_blob_key=_generate_blob_key, now_func=datetime.datetime.now): """Constructs a new Application. Args: forward_app: A WSGI application to forward successful upload requests to. get_blob_storage: Callable that returns a BlobStorage instance. The default is fine, but may be overridden for testing purposes. generate_blob_key: Function used for generating unique blob keys. now_func: Function that returns the current timestamp. """ self._forward_app = forward_app self._blob_storage = get_blob_storage() self._generate_blob_key = generate_blob_key self._now_func = now_func def abort(self, code, detail=None): """Aborts the application by raising a webob.exc.HTTPException. Args: code: HTTP status code int. detail: Optional detail message str. Raises: webob.exc.HTTPException: Always. """ exception = webob.exc.status_map[code]() if detail: exception.detail = detail raise exception def store_blob(self, content_type, filename, md5_hash, blob_file, creation): """Store a supplied form-data item to the blobstore. The appropriate metadata is stored into the datastore. Args: content_type: The MIME content type of the uploaded file. filename: The filename of the uploaded file. md5_hash: MD5 hash of the file contents, as a hashlib hash object. blob_file: A file-like object containing the contents of the file. creation: datetime.datetime instance to associate with new blobs creation time. This parameter is provided so that all blobs in the same upload form can have the same creation date. Returns: datastore.Entity('__BlobInfo__') associated with the upload. Raises: _TooManyConflictsError if there were too many name conflicts generating a blob key. """ blob_key = self._generate_blob_key() # Store the blob contents in the blobstore. self._blob_storage.StoreBlob(blob_key, blob_file) # Store the blob metadata in the datastore as a __BlobInfo__ entity. blob_entity = datastore.Entity('__BlobInfo__', name=str(blob_key), namespace='') blob_entity['content_type'] = content_type blob_entity['creation'] = creation blob_entity['filename'] = filename blob_entity['md5_hash'] = md5_hash.hexdigest() blob_entity['size'] = blob_file.tell() datastore.Put(blob_entity) return blob_entity def store_gs_file(self, content_type, gs_filename, blob_file, filename): """Store a supplied form-data item to GS. Delegate all the work of gs file creation to CloudStorageStub. Args: content_type: The MIME content type of the uploaded file. gs_filename: The gs filename to create of format bucket/filename. blob_file: A file-like object containing the contents of the file. filename: user provided filename. Returns: datastore.Entity('__GsFileInfo__') associated with the upload. """ gs_stub = cloudstorage_stub.CloudStorageStub(self._blob_storage) blobkey = gs_stub.post_start_creation('/' + gs_filename, {'content-type': content_type}) content = blob_file.read() return gs_stub.put_continue_creation(blobkey, content, (0, len(content) - 1), len(content), filename) def _preprocess_data(self, content_type, blob_file, filename, base64_encoding): """Preprocess data and metadata before storing them. Args: content_type: The MIME content type of the uploaded file. blob_file: A file-like object containing the contents of the file. filename: The filename of the uploaded file. base64_encoding: True, if the file contents are base-64 encoded. Returns: (content_type, blob_file, filename) after proper preprocessing. Raises: _InvalidMetadataError: when metadata are not utf-8 encoded. """ if base64_encoding: blob_file = cStringIO.StringIO( base64.urlsafe_b64decode(blob_file.read())) # If content_type or filename are bytes, assume UTF-8 encoding. try: if not isinstance(content_type, unicode): content_type = content_type.decode('utf-8') if filename and not isinstance(filename, unicode): filename = filename.decode('utf-8') except UnicodeDecodeError: raise _InvalidMetadataError( 'The uploaded entity contained invalid UTF-8 metadata. This may be ' 'because the page containing the upload form was served with a ' 'charset other than "utf-8".') return content_type, blob_file, filename def store_and_build_forward_message(self, form, boundary=None, max_bytes_per_blob=None, max_bytes_total=None, bucket_name=None): """Reads form data, stores blobs data and builds the forward request. This finds all of the file uploads in a set of form fields, converting them into blobs and storing them in the blobstore. It also generates the HTTP request to forward to the user's application. Args: form: cgi.FieldStorage instance representing the whole form derived from original POST data. boundary: The optional boundary to use for the resulting form. If omitted, one is randomly generated. max_bytes_per_blob: The maximum size in bytes that any single blob in the form is allowed to be. max_bytes_total: The maximum size in bytes that the total of all blobs in the form is allowed to be. bucket_name: The name of the Google Storage bucket to store the uploaded files. Returns: A tuple (content_type, content_text), where content_type is the value of the Content-Type header, and content_text is a string containing the body of the HTTP request to forward to the application. Raises: webob.exc.HTTPException: The upload failed. """ message = multipart.MIMEMultipart('form-data', boundary) creation = self._now_func() total_bytes_uploaded = 0 created_blobs = [] mime_type_error = None too_many_conflicts = False upload_too_large = False filename_too_large = False content_type_too_large = False # Extract all of the individual form items out of the FieldStorage. form_items = [] # Sorting of forms is done merely to make testing a little easier since # it means blob-keys are generated in a predictable order. for key in sorted(form): form_item = form[key] if isinstance(form_item, list): form_items.extend(form_item) else: form_items.append(form_item) for form_item in form_items: disposition_parameters = {'name': form_item.name} variable = email.message.Message() if form_item.filename is None: # Copy as is variable.add_header('Content-Type', 'text/plain') variable.set_payload(form_item.value) else: # If there is no filename associated with this field it means that the # file form field was not filled in. This blob should not be created # and forwarded to success handler. if not form_item.filename: continue disposition_parameters['filename'] = form_item.filename try: main_type, sub_type = _split_mime_type(form_item.type) except _InvalidMIMETypeFormatError, ex: mime_type_error = str(ex) break # Seek to the end of file and use the pos as the length. form_item.file.seek(0, os.SEEK_END) content_length = form_item.file.tell() form_item.file.seek(0) total_bytes_uploaded += content_length if max_bytes_per_blob is not None: if content_length > max_bytes_per_blob: upload_too_large = True break if max_bytes_total is not None: if total_bytes_uploaded > max_bytes_total: upload_too_large = True break if form_item.filename is not None: if len(form_item.filename) > _MAX_STRING_NAME_LENGTH: filename_too_large = True break if form_item.type is not None: if len(form_item.type) > _MAX_STRING_NAME_LENGTH: content_type_too_large = True break # Compute the MD5 hash of the upload. digester = hashlib.md5() while True: block = form_item.file.read(1 << 20) if not block: break digester.update(block) form_item.file.seek(0) # Create the external body message containing meta-data about the blob. external = email.message.Message() external.add_header('Content-Type', '%s/%s' % (main_type, sub_type), **form_item.type_options) # NOTE: This is in violation of RFC 2616 (Content-MD5 should be the # base-64 encoding of the binary hash, not the hex digest), but it is # consistent with production. content_md5 = base64.urlsafe_b64encode(digester.hexdigest()) # Create header MIME message headers = dict(form_item.headers) for name in _STRIPPED_FILE_HEADERS: if name in headers: del headers[name] headers['Content-Length'] = str(content_length) headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = ( blobstore._format_creation(creation)) headers['Content-MD5'] = content_md5 gs_filename = None if bucket_name: random_key = str(self._generate_blob_key()) gs_filename = '%s/fake-%s' % (bucket_name, random_key) headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = ( blobstore.GS_PREFIX + gs_filename) for key, value in headers.iteritems(): external.add_header(key, value) # Add disposition parameters (a clone of the outer message's field). if not external.get('Content-Disposition'): external.add_header('Content-Disposition', 'form-data', **disposition_parameters) base64_encoding = (form_item.headers.get( 'Content-Transfer-Encoding') == 'base64') content_type, blob_file, filename = self._preprocess_data( external['content-type'], form_item.file, form_item.filename, base64_encoding) # Store the actual contents to storage. if gs_filename: info_entity = self.store_gs_file(content_type, gs_filename, blob_file, filename) else: try: info_entity = self.store_blob(content_type, filename, digester, blob_file, creation) except _TooManyConflictsError: too_many_conflicts = True break # Track created blobs in case we need to roll them back. created_blobs.append(info_entity) variable.add_header('Content-Type', 'message/external-body', access_type=blobstore.BLOB_KEY_HEADER, blob_key=info_entity.key().name()) variable.set_payload([external]) # Set common information. variable.add_header('Content-Disposition', 'form-data', **disposition_parameters) message.attach(variable) if (mime_type_error or too_many_conflicts or upload_too_large or filename_too_large or content_type_too_large): for blob in created_blobs: datastore.Delete(blob) if mime_type_error: self.abort(400, detail=mime_type_error) elif too_many_conflicts: self.abort(500, detail='Could not generate a blob key.') elif upload_too_large: self.abort(413) else: if filename_too_large: invalid_field = 'filename' elif content_type_too_large: invalid_field = 'Content-Type' detail = 'The %s exceeds the maximum allowed length of %s.' % ( invalid_field, _MAX_STRING_NAME_LENGTH) self.abort(400, detail=detail) message_out = cStringIO.StringIO() gen = email.generator.Generator(message_out, maxheaderlen=0) gen.flatten(message, unixfrom=False) # Get the content text out of the message. message_text = message_out.getvalue() content_start = message_text.find('\n\n') + 2 content_text = message_text[content_start:] content_text = content_text.replace('\n', '\r\n') return message.get('Content-Type'), content_text