def test_metrics_parse_list():
# parse_list
assert_equal(len(address.parse_list('[email protected], [email protected]', metrics=True)), 2)
p, m = address.parse_list('[email protected], [email protected]', metrics=True)
assert_equal('parsing' in m, True)
assert_equal(isinstance(address.parse_list('[email protected], [email protected]', metrics=False), address.AddressList), True)
assert_equal(isinstance(address.parse_list('[email protected], [email protected]'), address.AddressList), True)
def test_metrics_parse_list():
# parse_list
assert_equal(len(address.parse_list('[email protected], [email protected]', metrics=True)), 2)
p, m = address.parse_list('[email protected], [email protected]', metrics=True)
assert_equal('parsing' in m, True)
assert_equal(isinstance(address.parse_list('[email protected], [email protected]', metrics=False), address.AddressList), True)
assert_equal(isinstance(address.parse_list('[email protected], [email protected]'), address.AddressList), True)
def process(self, filename):
self.filename = filename
with open(filename, "rb") as f:
msg = mime.from_string(f.read())
self.logging.info('Start working on {0}'.format(filename))
# count how many time connect to neo4j
self.count = 0
# initialize message basic information
self.message_id = str(msg.message_id)
sender = address.parse(msg.headers.get('From'))
self.sender = {'address': sender.address, 'name': sender.display_name}
self.receivers = {'to': [], 'cc': [], 'bcc': []}
for value in address.parse_list(msg.headers.get('To')):
tmp = {'address': value.address, 'name': value.display_name}
self.receivers['to'].append(tmp)
for value in address.parse_list(msg.headers.get('Cc')):
tmp = {'address': value.address, 'name': value.display_name}
self.receivers['cc'].append(tmp)
self.tx = self.graph.begin()
try:
for func in self.processors:
self.logging.info('\t Start layer: {0}'.format(func))
getattr(self, func)(msg)
self.logging.info('\t End layer: {0}'.format(func))
except:
pass
finally:
# commit remainder
self.tx.commit()
def test_simple_invalid():
s = '''httd://foo.com:8080\r\n; "Ev K." <ev@ host.com>\n "Alex K" alex@ , "Tom, S" "tom+[" a]"@s.com'''
assert_equal(AddressList(), parse_list(s))
s = ""
assert_equal(AddressList(), parse_list(s))
s = "crap"
assert_equal(AddressList(), parse_list(s))
def test_simple_invalid():
s = '''httd://foo.com:8080\r\n; "Ev K." <ev@ host.com>\n "Alex K" alex@ , "Tom, S" "tom+[" a]"@s.com'''
assert_equal(address.AddressList(), address.parse_list(s))
s = ""
assert_equal(address.AddressList(), address.parse_list(s))
s = "crap"
assert_equal(address.AddressList(), address.parse_list(s))
def test_addresslist_with_apostrophe():
s = '''"Allan G\'o" <*****@*****.**>, "Os Wi" <*****@*****.**>'''
lst = parse_list(s)
eq_(2, len(lst))
eq_('Allan G\'o <*****@*****.**>', lst[0].full_spec())
eq_('Os Wi <*****@*****.**>', lst[1].full_spec())
lst = parse_list("=?UTF-8?Q?Eugueny_=CF=8E_Kontsevoy?= <*****@*****.**>")
eq_('=?utf-8?q?Eugueny_=CF=8E_Kontsevoy?= <*****@*****.**>', lst.full_spec())
eq_(u'Eugueny ώ Kontsevoy', lst[0].display_name)
def test_endpoints():
# expected result: [[email protected], [email protected]]
presult = parse_list('[email protected], bar, [email protected]', as_tuple=False)
assert isinstance(presult, AddressList)
assert_equal(0, len(presult))
# expected result: ([[email protected], [email protected]], ['bar'])
presult = parse_list(['*****@*****.**', 'bar', '*****@*****.**'], as_tuple=True)
assert type(presult) is tuple
assert_equal(2, len(presult[0]))
assert_equal(1, len(presult[1]))
def test_endpoints():
# expected result: [[email protected], [email protected]]
presult = parse_list('[email protected], bar, [email protected]',
as_tuple=False)
assert isinstance(presult, AddressList)
assert_equal(0, len(presult))
# expected result: ([[email protected], [email protected]], ['bar'])
presult = parse_list(['*****@*****.**', 'bar', '*****@*****.**'],
as_tuple=True)
assert type(presult) is tuple
assert_equal(2, len(presult[0]))
assert_equal(1, len(presult[1]))
def parse_headers(self, header, meta):
meta.title = header.get("Subject")
if header.get("Message-Id"):
meta.foreign_id = unicode(header.get("Message-Id"))
if header.get("From"):
addr = address.parse(header.get("From"))
if addr is not None:
meta.author = addr.to_unicode()
meta.add_email(addr.address)
for hdr in ["To", "CC", "BCC"]:
if header.get(hdr):
for addr in address.parse_list(header.get(hdr)):
meta.add_email(addr.address)
date = header.get("Date")
date = rfc822.parsedate(date)
if date is not None:
dt = datetime.fromtimestamp(mktime(date))
meta.add_date(dt)
meta.headers = dict([(k, unicode(v)) for k, v in header.items()])
return meta
def _smtp_attrs(msg):
""" Generate the SMTP recipients, RFC compliant SMTP message. """
# SMTP recipients include addresses in To-, Cc- and Bcc-
all_recipients = u', '.\
join([m for m in msg.headers.get('To'),
msg.headers.get('Cc'),
msg.headers.get('Bcc') if m])
recipients = [a.full_spec()
for a in address.parse_list(', '.join(all_recipients))]
# Keep Cc-, but strip Bcc-
# Gmail actually keeps Bcc- on the bcc- recipients (only!) but
# for now, we strip for all. Exchange does this too so it's not a big deal.
msg.remove_headers('Bcc')
# Create an RFC-2821 compliant SMTP message
rfcmsg = rfc_transform(msg)
in_reply_to = msg.headers['In-Reply-To']
references = msg.headers['References']
subject = msg.headers['Subject']
smtpmsg = SMTPMessage(inbox_uid=msg.headers.get('X-INBOX-ID'),
msg=rfcmsg,
recipients=recipients,
in_reply_to=in_reply_to,
references=references,
subject=subject)
return smtpmsg
def create_email(sender_name, sender_email, inbox_uid, recipients,
subject, body, attachments=None):
""" Create an email. """
mimemsg = base_create_email(sender_name, sender_email, inbox_uid,
recipients, subject, body, attachments)
return _smtp_attrs(mimemsg)
def create_reply(sender_name, sender_email, in_reply_to, references,
inbox_uid, recipients, subject, body, attachments=None):
""" Create an email reply. """
mimemsg = base_create_email(sender_name, sender_email, inbox_uid,
recipients, subject, body, attachments)
# Add general reply headers:
if in_reply_to:
mimemsg.headers['In-Reply-To'] = in_reply_to
if references:
mimemsg.headers['References'] = '\t'.join(references)
# Set the 'Subject' header of the reply, required for Gmail.
# Gmail requires the same subject as the original (adding Re:/Fwd: is fine
# though) to group messages in the same conversation,
# See: https://support.google.com/mail/answer/5900?hl=en
mimemsg.headers['Subject'] = REPLYSTR + subject
return _smtp_attrs(mimemsg)
def parse_headers(self, msg, meta):
meta.title = msg.subject
if msg.headers.get('Message-Id'):
meta.foreign_id = unicode(msg.headers.get('Message-Id'))
if msg.headers.get('From'):
addr = address.parse(msg.headers.get('From'))
if addr is not None:
meta.author = addr.to_unicode()
for hdr in ['To', 'CC', 'BCC']:
if msg.headers.get(hdr):
for addr in address.parse_list(msg.headers.get(hdr)):
meta.add_recipient(addr.to_unicode())
date = msg.headers.get('Date')
date = rfc822.parsedate(date)
if date is not None:
dt = datetime.fromtimestamp(mktime(date))
meta.add_date(dt)
meta.headers = dict([(k, unicode(v)) for k, v in
msg.headers.items()])
return meta
def parse_emails(self, text):
"""Parse an email list with the side effect of adding them to the
relevant result lists."""
parsed = address.parse_list(safe_string(text))
# If the snippet didn't parse, assume it is just a name.
if not len(parsed):
return [(text, None)]
values = []
for addr in parsed:
name = stringify(addr.display_name)
email = stringify(addr.address)
if not self.check_email(email):
email = None
if self.check_email(name):
email = email or name
name = None
self.result.emit_email(email)
self.result.emit_name(name)
values.append((name, email))
return values
def _smtp_attrs(msg):
""" Generate the SMTP recipients, RFC compliant SMTP message. """
# SMTP recipients include addresses in To-, Cc- and Bcc-
all_recipients = u', '.\
join([m for m in msg.headers.get('To'),
msg.headers.get('Cc'),
msg.headers.get('Bcc') if m])
recipients = [a.full_spec() for a in address.parse_list(all_recipients)]
# Keep Cc-, but strip Bcc-
# Gmail actually keeps Bcc- on the bcc- recipients (only!) but
# for now, we strip for all. Exchange does this too so it's not a big deal.
msg.remove_headers('Bcc')
# Create an RFC-2821 compliant SMTP message
rfcmsg = rfc_transform(msg)
in_reply_to = msg.headers['In-Reply-To']
references = msg.headers['References']
subject = msg.headers['Subject']
smtpmsg = SMTPMessage(inbox_uid=msg.headers.get('X-INBOX-ID'),
msg=rfcmsg,
recipients=recipients,
in_reply_to=in_reply_to,
references=references,
subject=subject)
return smtpmsg
def create_gmail_email(sender_info, recipients, subject, body, attachments):
msg = create_email(sender_info, recipients, subject, body, attachments)
# Gmail specific add-ons:
all_recipients = u', '.\
join([m for m in msg.headers.get('To'),
msg.headers.get('Cc'),
msg.headers.get('Bcc') if m])
# Return recipients, MimeMessage
recipients = [a.full_spec() for a in address.parse_list(all_recipients)]
mimemsg = MimeMessage(uid=msg.headers.get('X-INBOX-ID'),
msg=msg.to_string())
return recipients, mimemsg
def save_gmail_email(account_id, db_session, log, uid, email):
# TODO[k]: Check these -
uid = uuid.uuid4().int & (1 << 16) - 1
date = datetime.now()
folder_name = 'sent'
msg = RawMessage(uid=uid, internaldate=date, flags=set(), body=email,
g_thrid=0, g_msgid=0, g_labels=set(), created=True)
new_uids = create_db_objects(account_id, db_session, log, folder_name,
[msg], create_gmail_message)
assert len(new_uids) == 1
new_uids[0].created_date = date
commit_uids(db_session, log, new_uids)
return new_uids[0]
def smtp_attrs(msg):
""" Generate the SMTP recipients, RFC compliant SMTP message. """
# SMTP recipients include addresses in To-, Cc- and Bcc-
all_recipients = u', '.\
join([m for m in msg.headers.get('To'),
msg.headers.get('Cc'),
msg.headers.get('Bcc') if m])
recipients = [a.full_spec() for a in address.parse_list(all_recipients)]
# Keep Cc-, but strip Bcc-
msg.remove_headers('Bcc')
# Create an RFC-2821 compliant SMTP message
rfcmsg = rfc_transform(msg)
in_reply_to = msg.headers['In-Reply-To']
references = msg.headers['References']
subject = msg.headers['Subject']
smtpmsg = SMTPMessage(inbox_uid=msg.headers.get('X-INBOX-ID'),
msg=rfcmsg,
recipients=recipients,
in_reply_to=in_reply_to,
references=references,
subject=subject)
return smtpmsg
def test_addresslist_non_ascii_list_input():
al = [u'Aurélien Berger <*****@*****.**>', 'Os Wi <*****@*****.**>']
lst = parse_list(al)
eq_(2, len(lst))
eq_('=?utf-8?q?Aur=C3=A9lien_Berger?= <*****@*****.**>',
lst[0].full_spec())
eq_('Os Wi <*****@*****.**>', lst[1].full_spec())
def parse_headers(self, header, meta):
meta.title = header.get('Subject')
if header.get('Message-Id'):
meta.foreign_id = string_value(header.get('Message-Id'))
if header.get('From'):
addr = address.parse(header.get('From'))
if addr is not None:
meta.author = addr.to_unicode()
meta.add_email(addr.address)
for hdr in ['To', 'CC', 'BCC']:
if header.get(hdr):
for addr in address.parse_list(header.get(hdr)):
meta.add_email(addr.address)
date = header.get('Date')
date = rfc822.parsedate(date)
if date is not None:
dt = datetime.fromtimestamp(mktime(date))
meta.add_date(dt)
meta.headers = dict([(k, string_value(v)) for k, v in header.items()])
return meta
def test_addresslist_address_obj_list_input():
al = [EmailAddress(u'Aurélien Berger <*****@*****.**>'),
UrlAddress('https://www.example.com')]
lst = parse_list(al)
eq_(2, len(lst))
eq_('=?utf-8?q?Aur=C3=A9lien_Berger?= <*****@*****.**>',
lst[0].full_spec())
eq_('https://www.example.com', lst[1].full_spec())
def test_parse_list_relaxed():
addr_list = [
'foo <*****@*****.**>', 'foo [email protected]', 'not@valid <*****@*****.**>'
]
expected = [
'foo <*****@*****.**>', 'foo <*****@*****.**>', '"not@valid" <*****@*****.**>'
]
eq_(expected, [addr.to_unicode() for addr in parse_list(addr_list)])
def smtp_attrs(msg):
""" Generate the SMTP recipients, RFC compliant SMTP message. """
# SMTP recipients include addresses in To-, Cc- and Bcc-
all_recipients = u', '.\
join([m for m in msg.headers.get('To'),
msg.headers.get('Cc'),
msg.headers.get('Bcc') if m])
recipients = [a.full_spec() for a in address.parse_list(all_recipients)]
# Keep Cc-, but strip Bcc-
msg.remove_headers('Bcc')
# Create an RFC-2821 compliant SMTP message
rfcmsg = rfc_transform(msg)
in_reply_to = msg.headers['In-Reply-To']
references = msg.headers['References']
subject = msg.headers['Subject']
smtpmsg = SMTPMessage(inbox_uid=msg.headers.get('X-INBOX-ID'),
msg=rfcmsg,
recipients=recipients,
in_reply_to=in_reply_to,
references=references,
subject=subject)
return smtpmsg
def create_email(sender_name, sender_email, inbox_uid, recipients,
subject, body, attachments=None):
""" Create a Generic email. """
mimemsg = create_base_email(sender_name, sender_email, inbox_uid,
recipients, subject, body, attachments)
return smtp_attrs(mimemsg)
def create_reply(sender_name, sender_email, in_reply_to, references,
inbox_uid, recipients, subject, body, attachments=None):
""" Create a Generic email reply. """
mimemsg = create_base_email(sender_name, sender_email, inbox_uid,
recipients, subject, body, attachments)
# Add general reply headers:
if in_reply_to:
mimemsg.headers['In-Reply-To'] = in_reply_to
if references:
mimemsg.headers['References'] = references
# Set the 'Subject' header of the reply
# Some providers require the same subject as the original (adding Re:/Fwd:
# is fine though) to group messages in the same conversation, See:
# https://support.google.com/mail/answer/5900?hl=en
mimemsg.headers['Subject'] = REPLYSTR + subject
return smtp_attrs(mimemsg)
def clean_emails(self):
value = self.cleaned_data['emails']
value = value.replace('\n', ',').replace(';', ',')
#print "INPUT"
#print repr(value)
emails = address.parse_list(value)
#print "OUTPUT"
#print emails
return emails
def test_addresslist_address_obj_list_input():
skip_if_asked() # Bad direct EmailAddress creation, spec is not valid
al = [EmailAddress(u'Aurélien Berger <*****@*****.**>'),
UrlAddress('https://www.example.com')]
lst = parse_list(al)
eq_(2, len(lst))
eq_('=?utf-8?q?Aur=C3=A9lien_Berger?= <*****@*****.**>',
lst[0].full_spec())
eq_('https://www.example.com', lst[1].full_spec())
def find_emails(self):
for line in self.pdf_as_string.split('\n'):
for string in line.split(' '):
possible_email = address.parse_list(string.strip())
if possible_email:
if possible_email != []:
if possible_email != "":
if u"@" in unicode(possible_email):
self.emails.append('%s\t%s' % (pdf, possible_email))
return self.emails
def get_email_from_to(headers):
sender = address.parse(fix_addr_if_necessary(headers['from']))
if not sender:
print "could not parse sender: {}".format(headers['from'])
return None, None
if sender.hostname != 'enron.com':
# skipping b/c we only care about enron communications
return None, None
receivers = address.parse_list(headers['to'])
enron_receivers = [r for r in receivers if r.hostname == 'enron.com']
return sender, enron_receivers
def test_addresslist_basics():
lst = parse_list("http://foo.com:1000; [email protected] ")
eq_(2, len(lst))
eq_("http", lst[0].scheme)
eq_("kontsevoy.com", lst[1].hostname)
eq_("Biz", lst[1].mailbox)
ok_("*****@*****.**" in lst)
# test case-sensitivity: hostname must be lowercased, but the local-part needs
# to remain case-sensitive
ok_("*****@*****.**" in str(lst))
# check parsing:
spec = '''http://foo.com:8080, "Ev K." <*****@*****.**>, "Alex K" [email protected]; "Tom, S" "tom+[a]"@s.com'''
lst = parse_list(spec, True)
eq_(len(lst), 4)
eq_("http://foo.com:8080", lst[0].address)
eq_("*****@*****.**", lst[1].address)
eq_("*****@*****.**", lst[2].address)
eq_('"tom+[a]"@s.com', lst[3].address)
# string-based persistence:
s = str(lst)
clone = parse_list(s)
eq_(lst, clone)
# now clone using full spec:
s = lst.full_spec()
clone = parse_list(s)
eq_(lst, clone)
# hostnames:
eq_(set(['host.com', 'foo.com', 'yahoo.net', 's.com']), lst.hostnames)
eq_(set(['url', 'email']), lst.addr_types)
# add:
result = lst + parse_list("*****@*****.**") + ["*****@*****.**"]
ok_(isinstance(result, AddressList))
eq_(len(result), len(lst)+2)
ok_("*****@*****.**" in result)
def test_addresslist_basics():
lst = parse_list("http://foo.com:1000; [email protected] ")
eq_(2, len(lst))
eq_("http", lst[0].scheme)
eq_("kontsevoy.com", lst[1].hostname)
eq_("Biz", lst[1].mailbox)
ok_("*****@*****.**" in lst)
# test case-sensitivity: hostname must be lowercased, but the local-part needs
# to remain case-sensitive
ok_("*****@*****.**" in str(lst))
# check parsing:
spec = '''http://foo.com:8080, "Ev K." <*****@*****.**>, "Alex K" [email protected]; "Tom, S" "tom+[a]"@s.com'''
lst = parse_list(spec, True)
eq_(len(lst), 4)
eq_("http://foo.com:8080", lst[0].address)
eq_("*****@*****.**", lst[1].address)
eq_("*****@*****.**", lst[2].address)
eq_('"tom+[a]"@s.com', lst[3].address)
# string-based persistence:
s = str(lst)
clone = parse_list(s)
eq_(lst, clone)
# now clone using full spec:
s = lst.full_spec()
clone = parse_list(s)
eq_(lst, clone)
# hostnames:
eq_(set(['host.com', 'foo.com', 'yahoo.net', 's.com']), lst.hostnames)
eq_(set(['url', 'email']), lst.addr_types)
# add:
result = lst + parse_list("*****@*****.**") + ["*****@*****.**"]
ok_(isinstance(result, AddressList))
eq_(len(result), len(lst) + 2)
ok_("*****@*****.**" in result)
async def handle_DATA(self, server, session, envelope):
message = mime.from_string(
message_from_bytes(envelope.content).as_string()
)
timestamp = datetime.now(tz=timezone.utc)
payload = await get_message_payload(message)
parsed_from = address.parse(message.headers['From'])
parsed_to_list = address.parse_list(message.headers['To'])
for mail_to in parsed_to_list.addresses:
if mail_to not in envelope.rcpt_tos:
continue
document = {
'uuid': str(uuid4()),
'from_name': parsed_from.display_name,
'from_address': parsed_from.address,
'to': mail_to,
'subject': message.headers['Subject'],
'payload': payload,
'timestamp': timestamp.isoformat(timespec='seconds')
}
await self.db.emails.insert_one(document)
await self.db.mailboxes.update_one(
{'address': mail_to}, {
'$push': {
'emails': {
'uuid': document['uuid'],
'from_address': document['from_address'],
'from_name': document['from_name'],
'subject': document['subject'],
'timestamp': document['timestamp'],
'is_read': False,
}
}
}
)
if self.config['collect_statistic']:
self.db.income_counter.update_one(
{'from_address': parsed_from.address},
{'$inc': {'count': 1}},
upsert=True,
)
self.db.email_counter.update_one(
{},
{'$inc': {'count': 1},
'$setOnInsert': {'since': timestamp.date().isoformat()}},
upsert=True
)
return '250 OK'
def handle_mailing_list_email_route(request):
'''
Handles the Mailgun route action when email is sent to a Mailgun mailing list.
:param request:
:return JsonResponse:
'''
logger.debug('Full mailgun post: %s', request.POST)
from_ = addresslib_address.parse(request.POST.get('from'))
message_id = request.POST.get('Message-Id')
recipients = set(
addresslib_address.parse_list(request.POST.get('recipient')))
sender = addresslib_address.parse(
_remove_batv_prefix(request.POST.get('sender')))
subject = request.POST.get('subject')
user_alt_email_cache = CommChannelCache()
logger.info(
'Handling Mailgun mailing list email from %s (sender %s) to '
'%s, subject %s, message id %s', from_, sender, recipients, subject,
message_id)
# short circuit if we detect a bounce loop
sender_address = sender.address.lower() if sender else ''
from_address = from_.address.lower() if from_ else ''
if settings.NO_REPLY_ADDRESS.lower() in (sender_address, from_address):
logger.error('Caught a bounce loop, dropping it. POST data:\n%s\n',
json.dumps(request.POST))
return JsonResponse({'success': True})
for recipient in recipients:
# shortcut if we've already handled this message for this recipient
if message_id:
cache_key = (
settings.CACHE_KEY_MESSAGE_HANDLED_BY_MESSAGE_ID_AND_RECIPIENT
% (message_id, recipient))
if cache.get(cache_key):
logger.warning(
'Message-Id %s was posted to the route handler '
"for %s, but we've already handled that. "
'Skipping.', recipient, message_id)
continue
_handle_recipient(request, recipient, user_alt_email_cache)
return JsonResponse({'success': True})
def validate_form(form):
messages = []
for field in required_fields:
if field not in form or len(form.get(field, "")) <= 0:
messages.append({
'field':
field,
'message':
"Missing required field '{}'".format(field)
})
for field in email_fields:
if field in form:
email_address = form[field]
failed_emails = None
if "," in email_address:
parsed_email, failed_emails = address.parse_list(email_address,
as_tuple=True)
else:
parsed_email = address.parse(email_address)
if parsed_email is None:
messages.append({
'field':
field,
'message':
"Email field '{}' is not a valid email address {}".format(
field, form[field])
})
if failed_emails is not None and len(failed_emails) > 0:
messages.append({
'field':
field,
'message':
"Email field '{}' contains invalid email address(es)".
format(field),
'parsed':
",".join([str(e) for e in parsed_email] or ""),
'unparsed':
",".join([str(e) for e in failed_emails])
})
return messages
def __init__(self, email_string):
"""
Takes a raw email string and processes it into something useful
"""
self.str = email_string
self.raw = mime.from_string(self.str)
to = self.raw.headers['To']
if to is None:
self.recipients = []
else:
to = to.lower()
self.recipients = address.parse_list(to) if ',' in to else [address.parse(to)]
# It's possible a recipient is None if it is something like
# 'Undisclosed recipients:;'
self.recipients = [r for r in self.recipients if r is not None]
self.sender = address.parse(self.raw.headers['From'].lower())
self.subject = self.raw.subject
self.id = self.raw.message_id
self.date = parse(self.raw.headers['Date'])
self.content_encoding = self.raw.content_encoding[0]
# Extract plaintext body
if self.raw.content_type.is_singlepart():
self.full_body = self.raw.body
elif self.raw.content_type.is_multipart():
for p in self.raw.parts:
if p.content_type == 'text/plain':
self.full_body = p.body
break
# Try to get signature
self.body, self.signature = extract_signature(self.full_body)
# Try ML approach if necessary
if self.signature is None:
self.body, self.signature = signature.extract(self.full_body, sender=self.sender)
# Get replies only, not the quotes
self.body = quotations.extract_from(self.body, 'text/plain')
def test_address_properties_req_utf8():
for i, tc in enumerate([{
'desc': 'utf8',
'addr_list': u'"Федот" <стрелец@письмо.рф>, Марья <искусница@mail.gun>',
'repr': '[Федот <стрелец@письмо.рф>, Марья <искусница@mail.gun>]',
'str': 'стрелец@письмо.рф, искусница@mail.gun',
'unicode': u'Федот <стрелец@письмо.рф>, Марья <искусница@mail.gun>',
'full_spec': ValueError(),
}]):
print('Test case #%d' % i)
addr_list = parse_list(tc['addr_list'])
eq_(tc['repr'], repr(addr_list))
eq_(tc['str'], str(addr_list))
eq_(tc['unicode'], unicode(addr_list))
eq_(tc['unicode'], addr_list.to_unicode())
if isinstance(tc['full_spec'], Exception):
assert_raises(type(tc['full_spec']), addr_list.full_spec)
else:
eq_(tc['full_spec'], addr_list.full_spec())
def handle_mailing_list_email_route(request):
'''
Handles the Mailgun route action when email is sent to a Mailgun mailing list.
:param request:
:return JsonResponse:
'''
logger.debug(u'Full mailgun post: %s', request.POST)
from_ = address.parse(request.POST.get('from'))
message_id = request.POST.get('Message-Id')
recipients = set(address.parse_list(request.POST.get('recipient')))
sender = address.parse(_remove_batv_prefix(request.POST.get('sender')))
subject = request.POST.get('subject')
user_alt_email_cache = CommChannelCache()
logger.info(u'Handling Mailgun mailing list email from %s (sender %s) to '
u'%s, subject %s, message id %s',
from_, sender, recipients, subject, message_id)
# short circuit if we detect a bounce loop
sender_address = sender.address.lower() if sender else ''
from_address = from_.address.lower() if from_ else ''
if settings.NO_REPLY_ADDRESS.lower() in (sender_address, from_address):
logger.error(u'Caught a bounce loop, dropping it. POST data:\n%s\n',
json.dumps(request.POST))
return JsonResponse({'success': True})
for recipient in recipients:
# shortcut if we've already handled this message for this recipient
if message_id:
cache_key = (
settings.CACHE_KEY_MESSAGE_HANDLED_BY_MESSAGE_ID_AND_RECIPIENT
% (message_id, recipient))
if cache.get(cache_key):
logger.warning(u'Message-Id %s was posted to the route handler '
u"for %s, but we've already handled that. "
u'Skipping.', recipient, message_id)
continue
_handle_recipient(request, recipient, user_alt_email_cache)
return JsonResponse({'success': True})
def clean_addresses(emails):
"""Takes a string of emails and returns a list of tuples of name/address
pairs that are symanticly valid"""
# Parse our string of emails, discarding invalid/illegal addresses
valid_emails_list = address.parse_list(emails)
# If no valid email addresses are found, return an empty list
if not valid_emails_list:
return []
# If we have valid emails, use flanker's unicode address list creator to
# give us something to pass to Python's email library's getaddresses
valid_emails = valid_emails_list.to_unicode()
# Return a list, in ('Name', '*****@*****.**')] form, the resulting emails
email_list = getaddresses([valid_emails])
# Lowercase all the email addresses in the list
lowered_list = [(name, email.lower()) for name, email in email_list]
return lowered_list
def test_address_properties_req_utf8():
for i, tc in enumerate([{
'desc': 'utf8',
'addr_list': u'"Федот" <стрелец@письмо.рф>, Марья <искусница@mail.gun>',
'repr': '[Федот <стрелец@письмо.рф>, Марья <искусница@mail.gun>]',
'str': 'стрелец@письмо.рф, искусница@mail.gun',
'unicode': u'Федот <стрелец@письмо.рф>, Марья <искусница@mail.gun>',
'full_spec': ValueError(),
}]):
print('Test case #%d' % i)
addr_list = parse_list(tc['addr_list'])
eq_(tc['repr'], repr(addr_list))
eq_(tc['str'], str(addr_list))
if six.PY2:
eq_(tc['unicode'], unicode(addr_list))
eq_(tc['unicode'], addr_list.to_unicode())
if isinstance(tc['full_spec'], Exception):
assert_raises(type(tc['full_spec']), addr_list.full_spec)
else:
eq_(tc['full_spec'], addr_list.full_spec())
def clean_addresses(emails):
"""Takes a string of emails and returns a list of tuples of name/address
pairs that are symanticly valid"""
# Parse our string of emails, discarding invalid/illegal addresses
valid_emails_list = address.parse_list(emails)
# If no valid email addresses are found, return an empty list
if not valid_emails_list:
return []
# If we have valid emails, use flanker's unicode address list creator to
# give us something to pass to Python's email library's getaddresses
valid_emails = valid_emails_list.to_unicode()
# Return a list, in ('Name', '*****@*****.**')] form, the resulting emails
email_list = getaddresses([valid_emails])
# Lowercase all the email addresses in the list
lowered_list = [(name, email.lower()) for name, email in email_list]
return lowered_list
def extract_headers_metadata(self, headers):
headers = [(stringify(k), stringify(v)) for k, v in headers]
self.result.headers = dict(headers)
for field, value in headers:
field = field.lower()
if field is None or value is None:
continue
if field == 'subject':
self.update('title', value)
if field == 'message-id':
self.update('id', value)
if field == 'date':
try:
date = rfc822.parsedate(value)
date = datetime.fromtimestamp(mktime(date))
self.update('created_at', date)
except Exception as ex:
log.warning("Failed to parse [%s]: %s", date, ex)
if field == 'from':
addr = address.parse(value)
if addr is not None:
author = stringify(addr.display_name)
email = stringify(addr.address)
self.result.emails.append(email)
if author is not None and author != email:
self.update('author', author)
self.result.entities.append(author)
if field in ['to', 'cc', 'bcc']:
for addr in address.parse_list(value):
name = stringify(addr.display_name)
email = stringify(addr.address)
self.result.emails.append(email)
if name is not None and name != email:
self.result.entities.append(name)
async def handle_message(self,
message: EmailMessage,
internal: bool = False):
if "message-id" not in message:
return
mail_to: BaseHeader = message["To"]
mail_cc: BaseHeader = message["Cc"]
mail_bcc: BaseHeader = message["BCC"]
target_address_lists: List[address.AddressList] = []
for delivering_list in filter(lambda x: x,
(mail_to, mail_cc, mail_bcc)):
target_address_lists.append(
address.parse_list(delivering_list, strict=True))
should_be_delivered_to: List[str] = []
for list in target_address_lists:
for addr in list:
if addr.addr_type == "email":
if addr.hostname in self.mydomains:
should_be_delivered_to.append(
addr.address
) # TODO (rubicon): verify spf and dkim before local delivery
elif (isinstance(message["X-Peer"], str) and
(message["X-Peer"].startswith("127.0.0.1")
or message["X-Peer"].startswith("::1")
or message["X-Peer"].startswith("localhost"))
or internal):
should_be_delivered_to.append(addr.address)
queue_futures: List[Future] = []
for addr in should_be_delivered_to:
msg_copy = deepcopy(message)
if "delivered-to" in msg_copy:
del msg_copy["delivered-to"]
msg_copy["delivered-to"] = addr
queue_futures.append(
asyncio.ensure_future(self.queue.put(msg_copy)))
await asyncio.gather(*queue_futures)
self.__logger.info(
"handled message: {msg_id}".format(msg_id=message["message-id"]))
def run_test(string, expected_mlist):
mlist = parse_list(string)
assert_equal(mlist, expected_mlist)
def parse_email_address_list(email_addresses):
parsed = address.parse_list(email_addresses)
return [or_none(addr, lambda p:
(strip_quotes(p.display_name), p.address)) for addr in parsed]
from flanker.addresslib import address
ap = address.parse('Example [email protected]')
print(ap)
not_email = address.parse('Example @example.com')
print(not_email)
multi_address = address.parse_list(
'[email protected], [email protected]')
print(multi_address)
multi_address2 = address.parse_list(
'[email protected], [email protected]', as_tuple=True)
print(multi_address2)
multi_address3 = address.parse_list(
'[email protected], [email protected]', strict=True)
print(multi_address3)
from flanker.addresslib import validate
sa = validate.suggest_alternate('*****@*****.**')
print(sa)
msg = '''MIME-Version: 1.0
Content-Type: text/plain
From: Example1 <*****@*****.**>
To: Example2 <*****@*****.**>
Subject: hello, message
Date: Mon, 10 Sep 2019 12:43:03 -0700
from flanker.addresslib import address
print(address.parse('*****@*****.**'))
print(address.parse('@hotmail.com'))
#parse address list
print(address.parse_list(['*****@*****.**', '*****@*****.**']))
#validate email address
print(address.validate_address('*****@*****.**'))
def create_email(sender_info, recipients, subject, html, attachments):
"""
Creates a MIME email message and stores it in the local datastore.
Parameters
----------
sender_info:
recipients: a list of utf-8 encoded strings
body: a utf-8 encoded string
subject:
html: a utf-8 encoded string
attachments:
"""
full_name = sender_info.name if sender_info.name else ''
email_address = sender_info.email
recipients = address.parse_list(recipients)
plaintext = html2text(html)
# Create a multipart/alternative message
msg = mime.create.multipart('alternative')
msg.append(
mime.create.text('text', plaintext),
mime.create.text('html', html))
# Create an outer multipart/mixed message
if attachments:
text_msg = msg
msg = mime.create.multipart('mixed')
# The first part is the multipart/alternative text part
msg.append(text_msg)
# The subsequent parts are the attachment parts
for a in attachments:
# Disposition should be inline if we add Content-ID
msg.append(mime.create.attachment(
a['content_type'],
a['data'],
filename=a['filename'],
disposition='attachment'))
# Gmail sets the From: header to the default sending account. We can
# however set our own custom phrase i.e. the name that appears next to the
# email address (useful if the user has multiple aliases and wants to
# specify which to send as).
# See: http://lee-phillips.org/gmailRewriting/
from_addr = u'"{0}" <{1}>'.format(full_name, email_address)
from_addr = address.parse(from_addr)
msg.headers['From'] = from_addr.full_spec()
# The 'recipients' below are the ones who will actually receive mail,
# but we need to set this header so recipients know we sent it to them
# Note also that the To: header has different semantics than the envelope
# recipient. For example, you can use '"Tony Meyer" <*****@*****.**>'
# as an address in the To: header, but the envelope recipient must be only
# '*****@*****.**'.
msg.headers['To'] = u', '.join([addr.full_spec() for addr in recipients])
msg.headers['Subject'] = subject
add_custom_headers(msg)
return msg
def test_addresslist_non_ascii_list_input():
al = [u'Aurélien Berger <*****@*****.**>', 'Os Wi <*****@*****.**>']
lst = parse_list(al)
eq_(2, len(lst))
eq_('=?utf-8?q?Aur=C3=A9lien_Berger?= <*****@*****.**>', lst[0].full_spec())
eq_('Os Wi <*****@*****.**>', lst[1].full_spec())
def _handle_recipient(request, recipient, user_alt_email_cache):
'''
The logic behind whether an email will be forwarded to list members or
trigger a bounce email can be complicated. A (hopefully simpler to follow)
matrix can be found on the wiki:
https://confluence.huit.harvard.edu/display/TLT/LTI+Emailer
'''
attachments, inlines = _get_attachments_inlines(request)
body_html = request.POST.get('body-html', '')
body_plain = request.POST.get('body-plain', '')
cc_list = address.parse_list(request.POST.get('Cc'))
parsed_from = address.parse(request.POST.get('from'))
message_id = request.POST.get('Message-Id')
sender = request.POST.get('sender')
subject = request.POST.get('subject')
to_list = address.parse_list(request.POST.get('To'))
logger.debug(u'Handling recipient %s, from %s, subject %s, message id %s',
recipient, sender, subject, message_id)
sender = _remove_batv_prefix(sender)
# short circuit if the mailing list doesn't exist
try:
ml = MailingList.objects.get_or_create_or_delete_mailing_list_by_address(
recipient.address)
except MailingList.DoesNotExist:
logger.info(
u'Sending mailing list bounce back email to %s for mailing list %s '
u'because the mailing list does not exist', sender, recipient)
_send_bounce('mailgun/email/bounce_back_does_not_exist.html',
sender, recipient.full_spec(), subject,
body_plain or body_html, message_id)
return
logger.debug(u'Got the MailingList object: {}'.format(ml))
# try to determine the course instance, and from there the school
school_id = None
ci = CourseInstance.objects.get_primary_course_by_canvas_course_id(ml.canvas_course_id)
if ci:
school_id = ci.course.school_id
else:
logger.warning(
u'Could not determine the primary course instance for Canvas '
u'course id %s, so we cannot prepend a short title to the '
u'email subject, or check the super senders.', ml.canvas_course_id)
member_addresses = set([m['address'].lower() for m in ml.members])
logger.debug(u'Got member_addresses: %d', len(member_addresses))
# conditionally include staff addresses in the members list. If
# always_mail_staff is true all staff will receive the email
teaching_staff_addresses = ml.teaching_staff_addresses
logger.debug(u'Got teaching_staff_addresses: %d', len(teaching_staff_addresses))
# if the course settings object does not exist create it to initialize the
# defaults
if ml.course_settings is None:
# we need to call get or create here as there might already be a setting
# for the course in question that has not been applied to this list yet
course_settings, created = CourseSettings.objects.get_or_create(
canvas_course_id=ml.canvas_course_id)
ml.course_settings = course_settings
ml.save()
# for non-full-course mailing lists, only include teachers from other
# sections if the course settings say to do so
if ml.section_id is not None and ml.course_settings.always_mail_staff:
member_addresses = member_addresses.union(teaching_staff_addresses)
# create a list of staff plus members to use to check permissions against
# so all staff can email all lists
staff_plus_members = teaching_staff_addresses.union(member_addresses)
# if we can, grab the list of super senders
super_senders = set()
if school_id:
# use iexact here to be able to match on COLGSAS or colgsas
query = SuperSender.objects.filter(school_id__iexact=school_id)
# lowercase all addresses in the supersenders list
super_senders = {addr.lower() for addr
in query.values_list('email', flat=True)}
# if we want to check email addresses against the sender, we need to parse
# out the address from the display name.
parsed_sender = address.parse(sender)
sender_address = parsed_sender.address.lower()
from_address = parsed_from.address.lower() if parsed_from else None
# email to use as reply-to / sender; defaults to sender address, but if the
# from address is the actual list member and the sender address is an active
# communication channel in Canvas for said member we will use the from
# address instead (see logic below)
parsed_reply_to = None
# any validation that fails will set the bounce template
bounce_back_email_template = None
# is sender or from_address a supersender?
if from_address != sender_address and from_address in super_senders:
alt_emails = user_alt_email_cache.get_for(from_address)
if sender_address in alt_emails:
parsed_reply_to = parsed_from
elif sender_address in super_senders:
parsed_reply_to = parsed_sender
is_super_sender = bool(parsed_reply_to)
# is the mailing list open to everyone?
if not parsed_reply_to \
and ml.access_level == MailingList.ACCESS_LEVEL_EVERYONE:
parsed_reply_to = parsed_sender
if from_address != sender_address \
and from_address in staff_plus_members:
alt_emails = user_alt_email_cache.get_for(from_address)
if sender_address in alt_emails:
parsed_reply_to = parsed_from
if not parsed_reply_to:
if ml.access_level == MailingList.ACCESS_LEVEL_STAFF:
if from_address != sender_address \
and from_address in teaching_staff_addresses:
# check if email is being sent on behalf of a list member by an
# alternate email account
alt_emails = user_alt_email_cache.get_for(from_address)
if sender_address in alt_emails:
parsed_reply_to = parsed_from
else:
# the from address matches a teaching staff list member, but
# the sender address is not a valid communication channel
# for that member in Canvas
# * note that alternate emails are not cached by default, so
# changes to a user's alternate emails in Canvas should be
# recognized immediately by this handler logic
logger.info(
u'Sending mailing list bounce back email to sender for '
u'mailing list %s because the sender address %s is not '
u'one of the active email communication channels for '
u'the list member matching the from address %s',
recipient, sender_address, from_address)
bounce_back_email_template = 'mailgun/email/bounce_back_no_comm_channel_match.html'
elif sender_address in teaching_staff_addresses:
parsed_reply_to = parsed_sender
else:
logger.info(
u'Sending mailing list bounce back email to sender for '
u'mailing list %s because neither the sender address %s '
u'nor the from address %s was a staff member',
recipient, sender_address, from_address)
bounce_back_email_template = 'mailgun/email/bounce_back_access_denied.html'
if not parsed_reply_to and not bounce_back_email_template:
# is sender or from_ a member of the list?
if from_address != sender_address \
and from_address in staff_plus_members:
# check if email is being sent on behalf of a list member by an
# alternate email account
alt_emails = user_alt_email_cache.get_for(from_address)
if sender_address in alt_emails:
parsed_reply_to = parsed_from
else:
# the from address matches a list member, but the sender address
# is not a valid communication channel for that member in Canvas
# * note that alternate emails are not cached by default, so
# changes to a user's alternate emails in Canvas should be
# recognized immediately by this handler logic
logger.info(
u'Sending mailing list bounce back email to sender for '
u'mailing list %s because the sender address %s is not one '
u'of the active email communication channels for the list '
u'member matching the from address %s',
recipient, sender_address, from_address)
bounce_back_email_template = 'mailgun/email/bounce_back_no_comm_channel_match.html'
elif sender_address in staff_plus_members:
parsed_reply_to = parsed_sender
else:
# neither of the possible sender addresses matches a list member
logger.info(
u'Sending mailing list bounce back email to sender for '
u'mailing list %s because neither the sender address %s '
u'nor the from address %s was a member',
recipient, sender_address, from_address)
bounce_back_email_template = 'mailgun/email/bounce_back_not_subscribed.html'
if not is_super_sender and not bounce_back_email_template:
if ml.access_level == MailingList.ACCESS_LEVEL_READONLY:
logger.info(
u'Sending mailing list bounce back email to sender '
u'address %s (from address %s) for mailing list %s because '
u'the list is readonly', sender_address, from_address,
recipient)
bounce_back_email_template = 'mailgun/email/bounce_back_readonly_list.html'
# bounce and return if they don't have the correct permissions
if bounce_back_email_template:
_send_bounce(bounce_back_email_template, sender, recipient.full_spec(),
subject, body_plain or body_html, message_id)
return
# always send the email to the sender. Add 'parsed_reply_to to the
# member_addresses set(tlt-2960)
logger.debug(u'Adding parsed_reply_to (sender) address to the final '
u'recipient list:%s.', parsed_reply_to.address.lower())
member_addresses.add(parsed_reply_to.address.lower())
# finally, we can send the email to the list
member_addresses = list(member_addresses)
logger.debug(u'Full list of recipients: %s', member_addresses)
# if we found the course instance, insert [SHORT TITLE] into the subject
if ci and ci.short_title:
title_prefix = '[{}]'.format(ci.short_title)
if title_prefix not in subject:
subject = title_prefix + ' ' + subject
# we want to add 'via Canvas' to the sender's name. do our best to figure
# it out, then add 'via Canvas' as long as we could.
logger.debug(
u'Original sender: %s, original from: %s, using %s as reply-to and '
u'sender address', parsed_sender, parsed_from, parsed_reply_to)
reply_to_display_name = _get_sender_display_name(
parsed_reply_to, parsed_from, ml)
if reply_to_display_name:
reply_to_display_name += ' via Canvas'
logger.debug(u'Final sender name: %s, sender address: %s',
reply_to_display_name, parsed_reply_to.address.lower())
# make sure inline images actually show up inline, since fscking
# mailgun won't let us specify the cid on post. see their docs at
# https://documentation.mailgun.com/user_manual.html#sending-via-api
# where they explain that they use the inlined file's name attribute
# as the content-id.
if inlines:
for f in inlines:
logger.debug(u'Replacing "%s" with "%s" in body', f.cid, f.name)
body_plain = re.sub(f.cid, f.name, body_plain)
body_html = re.sub(f.cid, f.name, body_html)
# convert the original to/cc fields back to strings so we can send
# them along through the listserv
original_to_list = [a.full_spec() for a in to_list]
original_cc_list = [a.full_spec() for a in cc_list]
# and send it off
logger.debug(
u'Mailgun router handler sending email to %s from %s, subject %s',
member_addresses, parsed_reply_to.full_spec(), subject)
try:
ml.send_mail(
reply_to_display_name, parsed_reply_to.address.lower(),
member_addresses, subject, text=body_plain, html=body_html,
original_to_address=original_to_list, original_cc_address=original_cc_list,
attachments=attachments, inlines=inlines, message_id=message_id
)
except RuntimeError:
logger.exception(
u'Error attempting to send message from %s to %s, originally '
u'sent to list %s, with subject %s', parsed_reply_to.full_spec(),
member_addresses, ml.address, subject)
raise
return JsonResponse({'success': True})
def test_views():
for i, tc in enumerate([
{
# Pure ASCII
'addr': parse('*****@*****.**'),
'repr': '*****@*****.**',
'str': '*****@*****.**',
'unicode': u'*****@*****.**',
'full_spec': '*****@*****.**',
},
{
# Pure ASCII
'addr': parse('<*****@*****.**>'),
'repr': '*****@*****.**',
'str': '*****@*****.**',
'unicode': u'*****@*****.**',
'full_spec': '*****@*****.**',
},
{
# Pure ASCII
'addr': parse('foo <*****@*****.**>'),
'repr': 'foo <*****@*****.**>',
'str': '*****@*****.**',
'unicode': u'foo <*****@*****.**>',
'full_spec': 'foo <*****@*****.**>',
},
{
# UTF-8
'addr': parse(u'Федот <стрелец@письмо.рф>'),
'repr': 'Федот <стрелец@письмо.рф>',
'str': 'стрелец@письмо.рф',
'unicode': u'Федот <стрелец@письмо.рф>',
'full_spec': ValueError(),
},
{
# UTF-8
'addr': parse(u'"Федот" <стрелец@письмо.рф>'),
'repr': 'Федот <стрелец@письмо.рф>',
'str': 'стрелец@письмо.рф',
'unicode': u'Федот <стрелец@письмо.рф>',
'full_spec': ValueError(),
},
{
# ASCII with utf-8 encoded display name
'addr': parse('=?utf-8?b?0LDQtNC20LDQuQ==?= <*****@*****.**>'),
'repr': '=?utf-8?b?0LDQtNC20LDQuQ==?= <*****@*****.**>',
'str': '*****@*****.**',
'unicode': '=?utf-8?b?0LDQtNC20LDQuQ==?= <*****@*****.**>',
'full_spec': '=?utf-8?b?0LDQtNC20LDQuQ==?= <*****@*****.**>',
},
{
# IDNA domain
'addr': parse('foo <foo@экзампл.рус>'),
'repr': 'foo <foo@экзампл.рус>',
'str': 'foo@экзампл.рус',
'unicode': u'foo <foo@экзампл.рус>',
'full_spec': 'foo <[email protected]>',
},
{
# UTF-8 local part
'addr': parse(u'foo <аджай@bar.com>'),
'repr': 'foo <аджай@bar.com>',
'str': 'аджай@bar.com',
'unicode': u'foo <аджай@bar.com>',
'full_spec': ValueError(),
},
{
# UTF-8 address list
'addr':
parse_list(
u'"Федот" <стрелец@письмо.рф>, Марья <искусница@mail.gun>'),
'repr':
'[Федот <стрелец@письмо.рф>, Марья <искусница@mail.gun>]',
'str':
'стрелец@письмо.рф, искусница@mail.gun',
'unicode':
u'Федот <стрелец@письмо.рф>, Марья <искусница@mail.gun>',
'full_spec':
ValueError(),
}
]):
print('Test case #%d' % i)
eq_(tc['repr'], repr(tc['addr']))
eq_(tc['str'], str(tc['addr']))
eq_(tc['unicode'], unicode(tc['addr']))
eq_(tc['unicode'], tc['addr'].to_unicode())
if isinstance(tc['full_spec'], Exception):
assert_raises(type(tc['full_spec']), tc['addr'].full_spec)
else:
eq_(tc['full_spec'], tc['addr'].full_spec())
def run_relaxed_test(string, expected_mlist, expected_unpar):
mlist, unpar = address.parse_list(string, strict=False, as_tuple=True)
assert_equal(mlist, expected_mlist)
assert_equal(unpar, expected_unpar)
def run_strict_test(string, expected_mlist):
mlist = address.parse_list(string, strict=True)
assert_equal(mlist, expected_mlist)
def test_simple_valid():
s = '''http://foo.com:8080; "Ev K." <*****@*****.**>, "Alex K" [email protected], "Tom, S" "tom+[a]"@s.com'''
addrs = address.parse_list(s)
assert_equal(4, len(addrs))
assert_equal(addrs[0].addr_type, 'url')
assert_equal(addrs[0].address, 'http://foo.com:8080')
assert_equal(addrs[0].full_spec(), 'http://foo.com:8080')
assert_equal(addrs[1].addr_type, 'email')
assert_equal(addrs[1].display_name, '"Ev K."')
assert_equal(addrs[1].address, '*****@*****.**')
assert_equal(addrs[1].full_spec(), '"Ev K." <*****@*****.**>')
assert_equal(addrs[2].addr_type, 'email')
assert_equal(addrs[2].display_name, '"Alex K"')
assert_equal(addrs[2].address, '*****@*****.**')
assert_equal(addrs[2].full_spec(), '"Alex K" <*****@*****.**>')
assert_equal(addrs[3].addr_type, 'email')
assert_equal(addrs[3].display_name, '"Tom, S"')
assert_equal(addrs[3].address, '"tom+[a]"@s.com')
assert_equal(addrs[3].full_spec(), '"Tom, S" <"tom+[a]"@s.com>')
s = '''"Allan G\'o" <*****@*****.**>, "Os Wi" <*****@*****.**>'''
addrs = address.parse_list(s)
assert_equal(2, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, '"Allan G\'o"')
assert_equal(addrs[0].address, '*****@*****.**')
assert_equal(addrs[0].full_spec(), '"Allan G\'o" <*****@*****.**>')
assert_equal(addrs[1].addr_type, 'email')
assert_equal(addrs[1].display_name, '"Os Wi"')
assert_equal(addrs[1].address, '*****@*****.**')
assert_equal(addrs[1].full_spec(), '"Os Wi" <*****@*****.**>')
s = u'''I am also A <*****@*****.**>, Zeka <*****@*****.**> ;Gonzalo Bañuelos<*****@*****.**>'''
addrs = address.parse_list(s)
assert_equal(3, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, 'I am also A')
assert_equal(addrs[0].address, '*****@*****.**')
assert_equal(addrs[0].full_spec(), 'I am also A <*****@*****.**>')
assert_equal(addrs[1].addr_type, 'email')
assert_equal(addrs[1].display_name, 'Zeka')
assert_equal(addrs[1].address, '*****@*****.**')
assert_equal(addrs[1].full_spec(), 'Zeka <*****@*****.**>')
assert_equal(addrs[2].addr_type, 'email')
assert_equal(addrs[2].display_name, u'Gonzalo Bañuelos')
assert_equal(addrs[2].address, '*****@*****.**')
assert_equal(addrs[2].full_spec(), '=?utf-8?q?Gonzalo_Ba=C3=B1uelos?= <*****@*****.**>')
s = r'''"Escaped" "\e\s\c\a\p\e\d"@sld.com; http://userid:[email protected]:8080, "Dmitry" <my|'`!#_~%$&{}?^+-*@host.com>'''
addrs = address.parse_list(s)
assert_equal(3, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, '"Escaped"')
assert_equal(addrs[0].address, '"\e\s\c\\a\p\e\d"@sld.com')
assert_equal(addrs[0].full_spec(), '"Escaped" <"\e\s\c\\a\p\e\d"@sld.com>')
assert_equal(addrs[1].addr_type, 'url')
assert_equal(addrs[1].address, 'http://*****:*****@example.com:8080')
assert_equal(addrs[1].full_spec(), 'http://*****:*****@example.com:8080')
assert_equal(addrs[2].addr_type, 'email')
assert_equal(addrs[2].display_name, '"Dmitry"')
assert_equal(addrs[2].address, 'my|\'`!#_~%$&{}?^+-*@host.com')
assert_equal(addrs[2].full_spec(), '"Dmitry" <my|\'`!#_~%$&{}?^+-*@host.com>')
s = "http://foo.com/blah_blah_(wikipedia)"
addrs = address.parse_list(s)
assert_equal(1, len(addrs))
assert_equal(addrs[0].addr_type, 'url')
assert_equal(addrs[0].address, 'http://foo.com/blah_blah_(wikipedia)')
assert_equal(addrs[0].full_spec(), 'http://foo.com/blah_blah_(wikipedia)')
s = "Sasha Klizhentas <*****@*****.**>"
addrs = address.parse_list(s)
assert_equal(1, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, 'Sasha Klizhentas')
assert_equal(addrs[0].address, '*****@*****.**')
assert_equal(addrs[0].full_spec(), 'Sasha Klizhentas <*****@*****.**>')
s = "[email protected],[email protected]"
addrs = address.parse_list(s)
assert_equal(2, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, '')
assert_equal(addrs[0].address, '*****@*****.**')
assert_equal(addrs[0].full_spec(), '*****@*****.**')
assert_equal(addrs[1].addr_type, 'email')
assert_equal(addrs[1].display_name, '')
assert_equal(addrs[1].address, '*****@*****.**')
assert_equal(addrs[1].full_spec(), '*****@*****.**')
def run_test(string, expected_mlist):
mlist = parse_list(string, strict=True)
assert_equal(mlist, expected_mlist)
def test_simple_valid():
s = '''http://foo.com:8080; "Ev K." <*****@*****.**>, "Alex K" <*****@*****.**>, "Tom, S" <"tom+[a]"@s.com>'''
addrs = parse_list(s)
assert_equal(4, len(addrs))
assert_equal(addrs[0].addr_type, 'url')
assert_equal(addrs[0].address, 'http://foo.com:8080')
assert_equal(addrs[0].full_spec(), 'http://foo.com:8080')
assert_equal(addrs[1].addr_type, 'email')
assert_equal(addrs[1].display_name, 'Ev K.')
assert_equal(addrs[1].address, '*****@*****.**')
assert_equal(addrs[1].full_spec(), '"Ev K." <*****@*****.**>')
assert_equal(addrs[2].addr_type, 'email')
assert_equal(addrs[2].display_name, 'Alex K')
assert_equal(addrs[2].address, '*****@*****.**')
assert_equal(addrs[2].full_spec(), 'Alex K <*****@*****.**>')
assert_equal(addrs[3].addr_type, 'email')
assert_equal(addrs[3].display_name, 'Tom, S')
assert_equal(addrs[3].address, '"tom+[a]"@s.com')
assert_equal(addrs[3].full_spec(), '"Tom, S" <"tom+[a]"@s.com>')
s = '''"Allan G\'o" <*****@*****.**>, "Os Wi" <*****@*****.**>'''
addrs = parse_list(s)
assert_equal(2, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, 'Allan G\'o')
assert_equal(addrs[0].address, '*****@*****.**')
assert_equal(addrs[0].full_spec(), 'Allan G\'o <*****@*****.**>')
assert_equal(addrs[1].addr_type, 'email')
assert_equal(addrs[1].display_name, 'Os Wi')
assert_equal(addrs[1].address, '*****@*****.**')
assert_equal(addrs[1].full_spec(), 'Os Wi <*****@*****.**>')
s = u'''I am also A <*****@*****.**>, Zeka <*****@*****.**> ;Gonzalo Bañuelos<*****@*****.**>'''
addrs = parse_list(s)
assert_equal(3, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, 'I am also A')
assert_equal(addrs[0].address, '*****@*****.**')
assert_equal(addrs[0].full_spec(), 'I am also A <*****@*****.**>')
assert_equal(addrs[1].addr_type, 'email')
assert_equal(addrs[1].display_name, 'Zeka')
assert_equal(addrs[1].address, '*****@*****.**')
assert_equal(addrs[1].full_spec(), 'Zeka <*****@*****.**>')
assert_equal(addrs[2].addr_type, 'email')
assert_equal(addrs[2].display_name, u'Gonzalo Bañuelos')
assert_equal(addrs[2].address, '*****@*****.**')
assert_equal(addrs[2].full_spec(),
'=?utf-8?q?Gonzalo_Ba=C3=B1uelos?= <*****@*****.**>')
s = r'''"Escaped" <"\e\s\c\a\p\e\d"@sld.com>; http://userid:[email protected]:8080, "Dmitry" <my|'`!#_~%$&{}?^+-*@host.com>'''
addrs = parse_list(s)
assert_equal(3, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, 'Escaped')
assert_equal(addrs[0].address, '"\e\s\c\\a\p\e\d"@sld.com')
assert_equal(addrs[0].full_spec(), 'Escaped <"\e\s\c\\a\p\e\d"@sld.com>')
assert_equal(addrs[1].addr_type, 'url')
assert_equal(addrs[1].address, 'http://*****:*****@example.com:8080')
assert_equal(addrs[1].full_spec(),
'http://*****:*****@example.com:8080')
assert_equal(addrs[2].addr_type, 'email')
assert_equal(addrs[2].display_name, 'Dmitry')
assert_equal(addrs[2].address, 'my|\'`!#_~%$&{}?^+-*@host.com')
assert_equal(addrs[2].full_spec(),
'Dmitry <my|\'`!#_~%$&{}?^+-*@host.com>')
s = "http://foo.com/blah_blah_(wikipedia)"
addrs = parse_list(s)
assert_equal(1, len(addrs))
assert_equal(addrs[0].addr_type, 'url')
assert_equal(addrs[0].address, 'http://foo.com/blah_blah_(wikipedia)')
assert_equal(addrs[0].full_spec(), 'http://foo.com/blah_blah_(wikipedia)')
s = "Sasha Klizhentas <*****@*****.**>"
addrs = parse_list(s)
assert_equal(1, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, 'Sasha Klizhentas')
assert_equal(addrs[0].address, '*****@*****.**')
assert_equal(addrs[0].full_spec(),
'Sasha Klizhentas <*****@*****.**>')
s = "[email protected],[email protected]"
addrs = parse_list(s)
assert_equal(2, len(addrs))
assert_equal(addrs[0].addr_type, 'email')
assert_equal(addrs[0].display_name, '')
assert_equal(addrs[0].address, '*****@*****.**')
assert_equal(addrs[0].full_spec(), '*****@*****.**')
assert_equal(addrs[1].addr_type, 'email')
assert_equal(addrs[1].display_name, '')
assert_equal(addrs[1].address, '*****@*****.**')
assert_equal(addrs[1].full_spec(), '*****@*****.**')
msg = mime.from_string(mime_msg)
# print msg.headers
# print msg.subject
# # print msg.content_type.is_singlepart()
# if msg.content_type.is_multipart():
# for part in msg.parts:
# print part, part.size
# else:
# print msg.body, msg.size
# print dir(msg)
from_str = msg.headers.get('From')
from_list = address.parse_list(from_str)
from_email = ''
if from_list:
for from_addr in from_list.container:
from_email = from_addr.address
to_str = msg.headers.get('To')
to_list = address.parse_list(to_str)
cc_str = msg.headers.get('Cc')
cc_list = address.parse_list(cc_str)
if not cc_list:
cc_email = ''
def handle_mailing_list_email_route(request):
'''
Handles the Mailgun route action when email is sent to a Mailgun mailing list.
1. Verify that recipient address is a course mailing list
2. If access level of mailing list is 'members' and sender is not a member, send email response notifying the sender
that their email was not received because they are not a member of the list
3. If access level of mailing list is 'readonly', send email response notifying the sender that their email was not
received because the list is currently not accepting email from anyone
4. Verify if user can post to the list: If access level of mailing list is 'staff' and sender is not enrolled as a
teacher, the mail should not be sent
:param request:
:return:
'''
sender = request.POST.get('sender')
recipient = request.POST.get('recipient')
subject = request.POST.get('subject')
body_plain = request.POST.get('body-plain', '')
body_html = request.POST.get('body-html', '')
to_list = address.parse_list(request.POST.get('To'))
cc_list = address.parse_list(request.POST.get('Cc'))
attachments, inlines = _get_attachments_inlines(request)
logger.info(u'Handling Mailgun mailing list email from %s to %s',
sender, recipient)
logger.debug(u'Full mailgun post: %s', request.POST)
# if we want to check email addresses against the sender, we need to parse
# out just the address.
parsed_sender = address.parse(sender)
sender_address = parsed_sender.address.lower()
sender_display_name = parsed_sender.display_name
# make sure the mailing list exists
bounce_back_email_template = None
try:
ml = MailingList.objects.get_or_create_or_delete_mailing_list_by_address(recipient)
except MailingList.DoesNotExist:
logger.info(
u'Sending mailing list bounce back email to %s for mailing list %s '
u'because the mailing list does not exist', sender, recipient)
bounce_back_email_template = get_template('mailgun/email/bounce_back_does_not_exist.html')
content = bounce_back_email_template.render(Context({
'sender': sender,
'recipient': recipient,
'subject': subject,
'message_body': body_plain or body_html,
}))
listserv_client.send_mail(recipient, recipient, sender_address,
subject='Undeliverable mail', html=content)
return JsonResponse({'success': True})
# Always include teaching staff addresses with members addresses, so that they can email any list in the course
teaching_staff_addresses = ml.teaching_staff_addresses
member_addresses = teaching_staff_addresses.union([m['address'] for m in ml.members])
if ml.access_level == MailingList.ACCESS_LEVEL_MEMBERS and sender_address not in member_addresses:
logger.info(
u'Sending mailing list bounce back email to %s for mailing list %s '
u'because the sender was not a member', sender, recipient)
bounce_back_email_template = get_template('mailgun/email/bounce_back_access_denied.html')
elif ml.access_level == MailingList.ACCESS_LEVEL_STAFF and sender_address not in teaching_staff_addresses:
logger.info(
u'Sending mailing list bounce back email to %s for mailing list %s '
u'because the sender was not a staff member', sender, recipient)
bounce_back_email_template = get_template('mailgun/email/bounce_back_access_denied.html')
elif ml.access_level == MailingList.ACCESS_LEVEL_READONLY:
logger.info(
u'Sending mailing list bounce back email to %s for mailing list %s '
u'because the list is readonly', sender, recipient)
bounce_back_email_template = get_template('mailgun/email/bounce_back_readonly_list.html')
if bounce_back_email_template:
content = bounce_back_email_template.render(Context({
'sender': sender,
'recipient': recipient,
'subject': subject,
'message_body': body_plain or body_html,
}))
subject = 'Undeliverable mail'
ml.send_mail('', ml.address, sender_address, subject=subject,
html=content)
else:
# try to prepend [SHORT TITLE] to subject, keep going if lookup fails
try:
ci = CourseInstance.objects.get(canvas_course_id=ml.canvas_course_id)
except CourseInstance.DoesNotExist:
logger.warning(
u'Unable to find the course instance for Canvas course id %s, '
u'so we cannot prepend a short title to the email subject.',
ml.canvas_course_id)
except CourseInstance.MultipleObjectsReturned:
logger.warning(
u'Found multiple course instances for Canvas course id %s, '
u'so we cannot prepend a short title to the email subject.',
ml.canvas_course_id)
except RuntimeError:
logger.exception(
u'Received unexpected error trying to look up course instance '
u'for Canvas course id %s', ml.canvas_course_id)
else:
if ci.short_title:
title_prefix = '[{}]'.format(ci.short_title)
if title_prefix not in subject:
subject = title_prefix + ' ' + subject
# anyone in the to/cc field will already have gotten a copy of this
# email directly from the sender. let's not send them a duplicate.
# let's also not send a copy to the sender.
logger.debug(u'Full list of recipients: %s', member_addresses)
try:
logger.debug(u'Removing sender %s from the list of recipients',
sender_address)
member_addresses.remove(sender_address)
except KeyError:
logger.info(
u'Email sent to mailing list %s from non-member address %s',
ml.address, sender)
to_cc_list = {a.address for a in (to_list + cc_list)}
logger.debug(
u'Removing anyone in the to/cc list %s from the list of recipients',
list(to_cc_list))
member_addresses.difference_update(to_cc_list)
member_addresses = list(member_addresses)
logger.info(u'Final list of recipients: %s', member_addresses)
# double check to make sure the list is in the to/cc field somewhere,
# add it to cc if not. do this to ensure that, even if someone decided
# to bcc the list, it will be possible to reply-all to the list.
if ml.address not in to_cc_list:
cc_list.append(address.parse(ml.address))
# we want to add 'via Canvas' to the sender's name. so first make
# sure we know their name.
logger.debug(u'Original sender name: %s, address: %s',
sender_display_name, sender_address)
if not sender_display_name:
name = get_name_for_email(ml.canvas_course_id, sender_address)
if name:
sender_display_name = name
logger.debug(u'Looked up sender name: %s, address: %s',
sender_display_name, sender_address)
# now add in 'via Canvas'
if sender_display_name:
sender_display_name += ' via Canvas'
logger.debug(u'Final sender name: %s, address: %s',
sender_display_name, sender_address)
# make sure inline images actually show up inline, since fscking
# mailgun won't let us specify the cid on post. see their docs at
# https://documentation.mailgun.com/user_manual.html#sending-via-api
# where they explain that they use the inlined file's name attribute
# as the content-id.
if inlines:
for f in inlines:
logger.debug(u'Replacing "%s" with "%s" in body', f.cid, f.name)
body_plain = re.sub(f.cid, f.name, body_plain)
body_html = re.sub(f.cid, f.name, body_html)
# convert the original to/cc fields back to strings so we can send
# them along through the listserv
to_list = [a.full_spec() for a in to_list]
cc_list = [a.full_spec() for a in cc_list]
# and send it off
logger.debug(
u'Mailgun router handler sending email to %s from %s, subject %s',
member_addresses, parsed_sender.full_spec(), subject)
try:
ml.send_mail(
sender_display_name, sender_address,
member_addresses, subject, text=body_plain, html=body_html,
original_to_address=to_list, original_cc_address=cc_list,
attachments=attachments, inlines=inlines
)
except RuntimeError:
logger.exception(
u'Error attempting to send message from %s to %s, originally '
u'sent to list %s, with subject %s', parsed_sender.full_spec(),
member_addresses, ml.address, subject)
return JsonResponse({'success': False}, status=500)
return JsonResponse({'success': True})
def test_parse_list_relaxed():
addr_list = ['foo <*****@*****.**>', 'foo [email protected]', 'not@valid <*****@*****.**>']
expected = ['foo <*****@*****.**>', 'foo <*****@*****.**>', '"not@valid" <*****@*****.**>']
eq_(expected, [addr.to_unicode() for addr in parse_list(addr_list)])
def create_email(sender_info, recipients, subject, html, attachments):
"""
Creates a MIME email message (both body and sets the needed headers).
Parameters
----------
sender_info : SenderInfo(name, email)
recipients: Recipients(to, cc, bcc) namedtuple
to, cc, bcc are a lists of utf-8 encoded strings or None.
subject : string
a utf-8 encoded string
html : string
a utf-8 encoded string
attachments: list of dicts, optional
a list of dicts(filename, data, content_type)
"""
full_name = sender_info.name if sender_info.name else ''
email_address = sender_info.email
to = address.parse_list(recipients.to)
cc = address.parse_list(recipients.cc)
bcc = address.parse_list(recipients.bcc)
html = html if html else ''
plaintext = html2text(html)
# Create a multipart/alternative message
msg = mime.create.multipart('alternative')
msg.append(
mime.create.text('text', plaintext),
mime.create.text('html', html))
# Create an outer multipart/mixed message
if attachments:
text_msg = msg
msg = mime.create.multipart('mixed')
# The first part is the multipart/alternative text part
msg.append(text_msg)
# The subsequent parts are the attachment parts
for a in attachments:
# Disposition should be inline if we add Content-ID
msg.append(mime.create.attachment(
a['content_type'],
a['data'],
filename=a['filename'],
disposition='attachment'))
msg.headers['Subject'] = subject if subject else ''
# Gmail sets the From: header to the default sending account. We can
# however set our own custom phrase i.e. the name that appears next to the
# email address (useful if the user has multiple aliases and wants to
# specify which to send as), see: http://lee-phillips.org/gmailRewriting/
# For other providers, we simply use full_name = ''
from_addr = u'"{0}" <{1}>'.format(full_name, email_address)
from_addr = address.parse(from_addr)
msg.headers['From'] = from_addr.full_spec()
# Need to set these headers so recipients know we sent the email to them:
# Note also that the To: header has different semantics than the envelope
# recipient. For example, you can use '"Tony Meyer" <*****@*****.**>'
# as an address in the To: header, but the envelope recipient must be only
# '*****@*****.**'.
msg.headers['To'] = u', '.join([addr.full_spec() for addr in to])
msg.headers['Cc'] = u', '.join([addr.full_spec() for addr in cc])
msg.headers['Bcc'] = u', '.join([addr.full_spec() for addr in bcc])
if 'X-INBOX-ID' not in msg.headers:
add_inbox_headers(msg)
return msg
def test_parse_list_from_list():
for i, tc in enumerate([{
'desc': 'Empty',
'in': [],
'good': AddressList(),
'bad': []
}, {
'desc': 'All good',
'in': [u'Bill Gates <*****@*****.**>',
u'Федот <стрелец@почта.рф>',
u'*****@*****.**'],
'good': AddressList([
parse('Bill Gates <*****@*****.**>'),
parse(u'Федот <стрелец@почта.рф>'),
parse('*****@*****.**')]),
'bad': []
}, {
'desc': 'All bad',
'in': ['httd://foo.com:8080\r\n',
'"Ev K." <ev@ host.com>\n "Alex K" alex@',
'"Tom, S" "tom+[" a]"@s.com'],
'good': AddressList(),
'bad': ['httd://foo.com:8080\r\n',
'"Ev K." <ev@ host.com>\n "Alex K" alex@',
'"Tom, S" "tom+[" a]"@s.com'],
'bad_s': ['httd://foo.com:8080\r\n, "Ev K." <ev@ host.com>\n "Alex K" alex@, "Tom, S" "tom+[" a]"@s.com']
}, {
'desc': 'Some bad',
'in': [u'Bill Gates <*****@*****.**>',
u'crap',
'foo.bar.com',
u'Федот <стрелец@почта.рф>',
'torvalds@@kernel.org'],
'good': AddressList([
parse('Bill Gates <*****@*****.**>'),
parse(u'Федот <стрелец@почта.рф>')]),
'good_s': AddressList(),
'bad': ['crap',
'foo.bar.com',
'torvalds@@kernel.org'],
'bad_s': [u'Bill Gates <*****@*****.**>, crap, foo.bar.com, Федот <стрелец@почта.рф>, torvalds@@kernel.org'],
}, {
'desc': 'Bad IDN among addresses (UTF-8)',
'in': [u'Bill Gates <*****@*****.**>',
u'foo@ドメイン.채ᅳ',
u'Федот <стрелец@почта.рф>'],
'good': AddressList([
parse('Bill Gates <*****@*****.**>'),
parse(u'Федот <стрелец@почта.рф>')]),
'bad': [u'foo@ドメイン.채ᅳ']
}, {
'desc': 'Bad IDN among addresses (punycode)',
'in': [u'Bill Gates <*****@*****.**>',
u'[email protected]',
u'Федот <стрелец@почта.рф>'],
'good': AddressList([
parse('Bill Gates <*****@*****.**>'),
parse(u'Федот <стрелец@почта.рф>')]),
'bad': ['[email protected]']
}]):
print('Test case #%d: %s' % (i, tc['desc']))
# When
al = parse_list(tc['in'])
al_from_l, bad_from_l = parse_list(tc['in'], as_tuple=True)
al_from_s, bad_from_s = parse_list(', '.join(tc['in']), as_tuple=True)
# Then
eq_(tc['good'], al)
eq_(tc['good'], al_from_l)
for j in xrange(len(al_from_l)):
_strict_eq(tc['good'][j], al[j])
_strict_eq(tc['good'][j], al_from_l[j])
eq_(tc['bad'], bad_from_l)
eq_(tc.get('good_s', tc['good']), al_from_s)
for j in xrange(len(al_from_s)):
_strict_eq(tc.get('good_s', tc['good'])[j], al_from_s[j])
eq_(tc.get('bad_s', tc['bad']), bad_from_s)