def computereplacement(matchobj):
text, coding = Header.decode_header(matchobj.group(0))[0]
try:
return text.strip().decode(coding).encode('utf-8')
except Exception as e:
logging.error(e)
return text
def _extract_recipient(self, header):
"""Extract recipient name and email address from header"""
v_list = email_header.decode_header(header)
if len(v_list) == 2:
# User name and Email already split.
name = str(v_list[0][0].strip())
address = str(v_list[1][0].strip())
address = address.replace("<", "").replace(">", "").strip()
if self.encoding:
enc = chardet.detect(name)['encoding']
if not enc:
enc = 'ascii'
name = name.decode(enc).encode(self.encoding)
return (name, address)
else:
entry = v_list[0][0].strip()
parsed_addr = email.utils.parseaddr(entry)
name = parsed_addr[0]
address = parsed_addr[1]
if address:
address = address.strip()
if self.encoding:
enc = chardet.detect(name)['encoding']
if not enc:
enc = 'ascii'
name = name.decode(enc).encode(self.encoding)
return (name, address)
def computereplacement(matchobj):
text, coding = Header.decode_header(matchobj.group(0))[0]
try:
return text.strip().decode(coding).encode('utf-8')
except Exception as e:
logging.error(e)
return text
def decodeHeader(header, charset="utf-8"):
try:
h = Header.decode_header(header)
buf = [b[0].decode(b[1] or 'ascii') for b in h]
return u''.join(buf)
except(UnicodeError, UnicodeDecodeError, LookupError, \
email.errors.HeaderParseError):
return unicode("".join(header.splitlines()), charset, 'ignore')
def decodeHeader(header, charset="utf-8"):
try:
h = Header.decode_header(header)
buf = [b[0].decode(b[1] or 'ascii') for b in h]
return u''.join(buf)
except(UnicodeError, UnicodeDecodeError, LookupError, \
email.errors.HeaderParseError):
return unicode("".join(header.splitlines()), charset, 'ignore')
def decodeHeader(header, charset=constants.DEFAULT_CHARSET):
try:
decoded = Header.make_header(Header.decode_header(header))
unicodeStr = decoded.__unicode__()
return constants.EMPTY.join(unicodeStr.splitlines())
except(UnicodeError, UnicodeDecodeError, LookupError):
return unicode("".join(header.splitlines()), charset, 'ignore')
def _log_subject(mi, log):
encoded_subject = mi.msg.get('subject')
try:
subject, encoding = Header.decode_header(encoded_subject)[0]
except Header.HeaderParseError:
log.info("%s Subject cannot be parsed" % (mi.i,))
return
if encoding is None or encoding == 'iso-8859-1':
s = subject
else:
s = encoded_subject
log.info("%s Subject: %r" % (mi.i, s))
def _log_subject(mi, log):
encoded_subject = mi.msg.get('subject')
try:
subject, encoding = Header.decode_header(encoded_subject)[0]
except Header.HeaderParseError:
log.info("%s Subject cannot be parsed" % (mi.i, ))
return
if encoding is None or encoding == 'iso-8859-1':
s = subject
else:
s = encoded_subject
log.info("%s Subject: %r" % (mi.i, s))
def decode_header(s):
from email import Header
def d(s,c):
try:
return s.decode(c or 'latin1', 'ignore')
except LookupError:
return s.decode('latin1', 'ignore')
try:
s = u' '.join( d(s1,t1) for (s1,t1) in Header.decode_header(s) )
except Header.HeaderParseError:
s = s.decode('latin1', 'ignore')
return s
def headerToUnicode(header, fallbackEncoding='utf-8'):
"""Decode a MIME encoded header and return a unicode object.
@param header: a MIME encoded header to decode
@param fallbackEncoding: the encoding to use if an unknown encoding is encountered
"""
segments = []
for segment, charset in Header.decode_header(header):
try:
segments.append(segment.decode(charset or 'ascii', 'replace'))
except LookupError:
segments.append(segment.decode(fallbackEncoding, 'replace'))
return u' '.join(segments)
def headerToUnicode(header, fallbackEncoding='utf-8'):
"""Decode a MIME encoded header and return a unicode object.
@param header: a MIME encoded header to decode
@param fallbackEncoding: the encoding to use if an unknown encoding is encountered
"""
segments = []
for segment, charset in Header.decode_header(header):
try:
segments.append(segment.decode(charset or 'ascii', 'replace'))
except LookupError:
segments.append(segment.decode(fallbackEncoding, 'replace'))
return u' '.join(segments)
def decode_header(value):
"""Converts an encoded header into a unicode string::
>>> decode_header(
... 'Je les =?utf-8?b?ZMODwql0w4PCqHN0ZQ==?= oui?')
u'Je les d\\xc3\\xa9t\\xc3\\xa8ste oui?'
"""
encoded_strings = Header.decode_header(value)
header_val = Header.Header()
for string, encoding in encoded_strings:
header_val.append(string, encoding, errors='replace')
return unicode(header_val)
def decode_header(value):
"""Converts an encoded header into a unicode string::
>>> decode_header(
... 'Je les =?utf-8?b?ZMODwql0w4PCqHN0ZQ==?= oui?')
u'Je les d\\xc3\\xa9t\\xc3\\xa8ste oui?'
"""
encoded_strings = Header.decode_header(value)
header_val = Header.Header()
for string, encoding in encoded_strings:
header_val.append(string, encoding, errors='replace')
return unicode(header_val)
def DecodeSingleHeader(s):
"""
decode headers to UTF-8
"""
parts = Header.decode_header(s)
header = []
for part in parts:
s, enc = part
if enc:
s = unicode(s, enc).encode('utf8', 'replace')
header.append(s)
h = " ".join(header)
return h
def __decode_item(item):
if isinstance(item, str) and item.startswith('=?'):
_item = "?=\r\n=?".join(item.split("?==?"))
decodefrag = Header.decode_header(_item)
subj_fragments = []
for s, enc in decodefrag:
if enc is None:
subj_fragments.append(s)
continue
if enc.lower() in ('gb2312', 'gbk', 'gb_1988-80'):
enc = 'gb18030'
s = unicode(s, enc).encode('utf8', 'replace')
subj_fragments.append(s)
return ''.join(subj_fragments)
return item
def decode_item(item):
if isinstance(item, str) and item.startswith('=?') :
_item = "?=\r\n=?".join(item.split("?==?"))
decodefrag = Header.decode_header(_item)
subj_fragments = []
for s, enc in decodefrag:
if enc is None:
subj_fragments.append(s)
continue
if enc.lower() in ('gb2312', 'gbk', 'gb_1988-80'):
enc = 'gb18030'
s = unicode(s, enc).encode('utf8', 'replace')
subj_fragments.append(s)
return ''.join(subj_fragments)
return item
def __decode_item(item):
if isinstance(item, str) and item.startswith("=?"):
_item = "?=\r\n=?".join(item.split("?==?"))
decodefrag = Header.decode_header(_item)
subj_fragments = []
for s, enc in decodefrag:
if enc is None:
subj_fragments.append(s)
continue
if enc.lower() in ("gb2312", "gbk", "gb_1988-80"):
enc = "gb18030"
s = unicode(s, enc).encode("utf8", "replace")
subj_fragments.append(s)
return "".join(subj_fragments)
return item
def get_addresses(self, message, header_name):
"""Get addresses from the header_name.
This is usually 'From' or 'To', but other headers may contain
addresses too, so we allow all, unlike we used to do.
We expect just one From address and one To address, but
multiple addresses can also be checked.
May easily be something ugly like this:
=?utf-8?q?Portal_Administrator_?=<*****@*****.**>
From the Python docs:
decode_header(header)
Decode a message header value without converting charset.
Returns a list of (decoded_string, charset) pairs containing
each of the decoded parts of the header. Charset is None
for non-encoded parts of the header, otherwise a lower-case
string containing the name of the character set specified in
the encoded string.
An email.Errors.HeaderParseError may be raised when certain
decoding error occurs (e.g. a base64 decoding exception).
"""
if not header_name:
raise ValueError
address = message.get(header_name, '')
try:
decoded = Header.decode_header(address)
except HeaderParseError:
logger.warn("Could not parse header %r", address)
return []
logger.debug('Decoded header: %r', decoded)
for decoded_string, charset in decoded:
if charset is not None:
# Surely this is no email address but a name.
continue
if '@' not in decoded_string:
continue
return email_utils.getaddresses((decoded_string, ))
return []
def _decode_headers(headers, encoding=None):
"""Decode headers"""
if type(headers) is not list:
headers = [headers]
ret = []
for header in headers:
header = email_header.decode_header(header)
h_ret = []
for (value, h_encoding) in header:
decoded_hv = decode_value(value, h_encoding)
if encoding:
enc = chardet.detect(value)['encoding']
if not enc:
enc = 'ascii'
decoded_hv = decoded_hv.decode(enc).encode(encoding)
h_ret.append(decoded_hv)
if len(h_ret) == 1:
h_ret = h_ret[0]
ret.append(h_ret)
return ret
def decode_header(header_msg):
result = Header.decode_header(header_msg)
return decode_mail_string(''.join(t[0] for t in result))
#!/usr/bin/env python
# MIME Header Parsing - Chapter 9
# mime_parse_headers.py
# This program requires Python 2.2.2 or above
import sys, email, codecs
from email import Header
msg = email.message_from_file(sys.stdin)
for header, value in msg.items():
headerparts = Header.decode_header(value)
headerval = []
for part in headerparts:
data, charset = part
if charset is None:
charset = 'ascii'
dec = codecs.getdecoder(charset)
enc = codecs.getencoder('iso-8859-1')
data = enc(dec(data)[0])[0]
headerval.append(data)
print "%s: %s" % (header, " ".join(headerval))
def __call__(self):
if not self.enabled:
logger.debug('received mail-in request, but mail-in not enabled')
raise NotFound
mail = self.request.get('Mail', '')
mail = mail.strip()
if not mail:
msg = u'No mail found in request'
logger.warn(msg)
return msg
message = message_from_string(mail)
logger.debug('--------')
logger.debug(mail)
logger.debug('--------')
logger.debug(message)
from_addresses = self.get_addresses(message, 'From')
to_addresses = self.get_addresses(message, 'To')
if not from_addresses or not to_addresses:
msg = u'No From or To address found in request'
logger.warn(msg)
return msg
for from_name, from_address in from_addresses:
if from_address:
break
portal = getToolByName(self.context, 'portal_url').getPortalObject()
email_from_address = portal.getProperty('email_from_address')
if from_address.lower() == email_from_address.lower():
# This too easily means that a message sent by Poi ends up
# being added as a reply on an issue that we have just
# created.
msg = u'Ignoring mail from portal email_from_address'
logger.info(msg)
return msg
subject_line = message.get('Subject', '')
subjects = []
decoded = Header.decode_header(subject_line)
for decoded_string, charset in decoded:
if charset:
decoded_string = decoded_string.decode(charset)
subjects.append(decoded_string)
subject = u' '.join(subjects)
logger.debug("Forum at %s received mail from %r to %r with "
"subject %r", self.context.absolute_url(),
from_address, to_addresses, subject)
text, mimetype = self.get_text_and_mimetype(message)
if not text:
text = "Warning: no text found in email"
mimetype = 'text/plain'
logger.warn(text)
logger.debug('Got payload with mimetype %s from email.', mimetype)
# Create an attachment from the complete email.
attachment = File('email.eml', 'E-mail', mail)
tags = self.get_tags(message)
if tags:
logger.debug("Determined tags: %r", tags)
else:
logger.debug("Could not determine tags.")
# Store original security manager.
sm = getSecurityManager()
# Possibly switch to a different user.
self.switch_user(from_address)
if self.add_attachments:
attachments = self.get_attachments(message)
attachments = [File(filename, filename, data) for filename, data in attachments]
else:
attachments = [attachment]
target = self.find_conversation_or_thread(subject, text, tags, message)
if target is None:
# We don't allow creating conversations from mail, only replies
logger.info('Could not find something to reply to')
else:
try:
self.add_response(target, from_address, subject, text, mimetype, attachments)
except Unauthorized, exc:
logger.error(u'Unauthorized to add response: %s', exc)
return u'Unauthorized'
logger.info('Added mail as response to target %s',
target.absolute_url())
#!/usr/bin/env python
import sys, email, codecs
from email import Header
msg = email.message_from_file(sys.stdin)
for header, value in msg.items():
headerparts = Header.decode_header(value)
headerval = []
for part in headerparts:
data, charset = part
if charset is None:
charset = 'ascii'
dec = codecs.getdecoder(charset)
enc = codecs.getencoder('iso-8859-1')
data = enc(dec(data)[0])[0]
headerval.append(data)
print "%s: %s" % (header, " ".join(headerval))
#!/usr/bin/python
'''
File: mail_parser.py
Author: Sigurd Fosseng
Licence: MIT Licence
Description: Converts the subject of the email given through stdin to Unicode, and sends
it through xmlrpc
'''
from email.Parser import Parser
from email import Header
import xmlrpclib
import sys
#mime handling
data = sys.stdin.read()
message = Parser().parsestr(data)
subject = unicode(Header.make_header(Header.decode_header(message['subject'])))
#send to bot
s = xmlrpclib.Server("http://localhost:5656/")
s.say(subject)
def __call__(self):
mail = self.request.get('Mail')
mail = mail.strip()
if not mail:
msg = u'No mail found in request'
logger.warn(msg)
return msg
message = message_from_string(mail)
self.encoding = self._get_encoding(message)
logger.debug('--------')
logger.debug(mail)
logger.debug('--------')
logger.debug(message)
from_addresses = self.get_addresses(message, 'From')
to_addresses = self.get_addresses(message, 'To')
if not from_addresses or not to_addresses:
msg = u'No From or To address found in request'
logger.warn(msg)
return msg
# Pick the first one; strange anyway if there would be more.
from_name, from_address = from_addresses[0]
portal = getToolByName(self.context, 'portal_url').getPortalObject()
email_from_address = portal.getProperty('email_from_address')
if from_address.lower() == email_from_address.lower():
# This too easily means that a message sent by Poi ends up
# being added as a reply on an issue that we have just
# created.
msg = u'Ignoring mail from portal email_from_address'
logger.info(msg)
return msg
subject_line = message.get('Subject', '')
subjects = []
decoded = Header.decode_header(subject_line)
for decoded_string, charset in decoded:
if charset:
decoded_string = decoded_string.decode(charset)
subjects.append(decoded_string)
subject = u' '.join(subjects)
logger.info(
"Tracker at %s received mail from %r to %r with "
"subject %r", self.context.absolute_url(),
from_address, to_addresses, subject)
details, mimetype = self.get_details_and_mimetype(message)
logger.debug('Got payload with mimetype %s from email.', mimetype)
# Transform to an allowed mime type, if needed.
details, mimetype = self.to_allowed_mimetype(details, mimetype)
if not details:
# Details is a required field.
details = '.'
mimetype = 'text/plain'
logger.info('No details found in email.')
# Create an attachment from the complete email.
attachment = mail
tags = self.get_tags(message)
if tags:
logger.debug("Determined tags: %r", tags)
else:
logger.debug("Could not determine tags.")
# Get all info that we need from the mail.
mail_info = {
'subject': subject,
'tags': tags,
'message': message,
'details': details,
'from_address': from_address,
'attachment': attachment,
'mimetype': mimetype,
}
# Possibly switch to a different user.
user = self.find_user_for_switching(from_address)
if user is None:
result = self.create_content(**mail_info)
else:
with api.env.adopt_user(user=user):
current_user = api.user.get_current()
user_id = current_user.getId()
user_name = current_user.getUserName()
logger.info("Switched email=%s to user name=%s (id=%s)",
from_address, user_name, user_id)
role = self.find_role_to_fake()
if not role:
result = self.create_content(**mail_info)
else:
logger.info("Faking %s role for user %s", role, user_id)
# Fake a few extra roles as well. For example,
# TrackerManager may not have the 'Copy or Move'
# permission, but Anonymous may have. Go figure.
roles = [role, 'Member', 'Anonymous']
with api.env.adopt_roles(roles):
result = self.create_content(**mail_info)
# We need to return something.
if result:
# error
return result
return mail
def __call__(self):
mail = self.request.get('Mail')
mail = mail.strip()
if not mail:
msg = u'No mail found in request'
logger.warn(msg)
return msg
message = message_from_string(mail)
self.encoding = self._get_encoding(message)
logger.debug('--------')
logger.debug(mail)
logger.debug('--------')
logger.debug(message)
from_addresses = self.get_addresses(message, 'From')
to_addresses = self.get_addresses(message, 'To')
if not from_addresses or not to_addresses:
msg = u'No From or To address found in request'
logger.warn(msg)
return msg
# Pick the first one; strange anyway if there would be more.
from_name, from_address = from_addresses[0]
portal = getToolByName(self.context, 'portal_url').getPortalObject()
email_from_address = portal.getProperty('email_from_address')
if from_address.lower() == email_from_address.lower():
# This too easily means that a message sent by Poi ends up
# being added as a reply on an issue that we have just
# created.
msg = u'Ignoring mail from portal email_from_address'
logger.info(msg)
return msg
subject_line = message.get('Subject', '')
subjects = []
decoded = Header.decode_header(subject_line)
for decoded_string, charset in decoded:
if charset:
decoded_string = decoded_string.decode(charset)
subjects.append(decoded_string)
subject = u' '.join(subjects)
logger.debug("Tracker at %s received mail from %r to %r with "
"subject %r", self.context.absolute_url(),
from_address, to_addresses, subject)
details, mimetype = self.get_details_and_mimetype(message)
if not details:
details = "Warning: no details found in email"
mimetype = 'text/plain'
logger.warn(details)
logger.debug('Got payload with mimetype %s from email.', mimetype)
# Create an attachment from the complete email. Somehow the
# result is nicer when it is put in a response than in an
# issue. Not much we can do about that probably.
attachment = File('email.eml', 'E-mail', mail)
tags = self.get_tags(message)
if tags:
logger.debug("Determined tags: %r", tags)
else:
logger.debug("Could not determine tags.")
# Store original security manager.
sm = getSecurityManager()
# Possibly switch to a different user.
self.switch_user(from_address)
issue = self.find_issue(subject, tags, message)
if issue is None:
manager = self.get_manager(message, tags)
logger.debug("Determined manager: %s", manager)
if not subject:
# When there is no subject, we always want to create a
# new issue, as we do not want to try to match a empty
# or fake subject.
# We also want to flay the user alive...
subject = '[no subject]'
try:
issue = self.create_issue(
title=subject, details=details, contactEmail=from_address,
attachment=attachment, responsibleManager=manager,
subject=tags)
except Unauthorized, exc:
logger.error(u'Unauthorized to create issue: %s', exc)
return u'Unauthorized'
logger.info('Created issue from email at %s', issue.absolute_url())
