def decode_header(header, normalize=False):
"""
decode a header value to a unicode string
values are usually a mixture of different substrings
encoded in quoted printable using diffetrent encodings.
This turns it into a single unicode string
:param header: the header value
:type header: str
:param normalize: replace trailing spaces after newlines
:type normalize: bool
:rtype: unicode
"""
# If the value isn't ascii as RFC2822 prescribes,
# we just return the unicode bytestring as is
value = string_decode(header) # convert to unicode
try:
value = value.encode('ascii')
except UnicodeEncodeError:
return value
# otherwise we interpret RFC2822 encoding escape sequences
valuelist = email.header.decode_header(header)
decoded_list = []
for v, enc in valuelist:
v = string_decode(v, enc)
decoded_list.append(string_sanitize(v))
value = u' '.join(decoded_list)
if normalize:
value = re.sub(r'\n\s+', r' ', value)
return value
def _get_raw_html(msg):
mail = msg.get_email()
for part in mail.walk():
ctype = part.get_content_type()
if ctype != "text/html":
continue
cd = part.get('Content-Disposition', '')
if cd.startswith('attachment'):
continue
enc = part.get_content_charset() or 'utf-8'
raw = string_decode(part.get_payload(decode=True), enc)
return string_sanitize(raw)
return None
def _get_raw_html(msg):
mail = msg.get_email()
for part in mail.walk():
ctype = part.get_content_type()
if ctype != "text/html":
continue
cd = part.get('Content-Disposition', '')
if cd.startswith('attachment'):
continue
enc = part.get_content_charset() or 'utf-8'
raw = string_decode(part.get_payload(decode=True), enc)
return string_sanitize(raw)
return None
def decode_header(header, normalize=False):
"""
decode a header value to a unicode string
values are usually a mixture of different substrings
encoded in quoted printable using different encodings.
This turns it into a single unicode string
:param header: the header value
:type header: str
:param normalize: replace trailing spaces after newlines
:type normalize: bool
:rtype: unicode
"""
# If the value isn't ascii as RFC2822 prescribes,
# we just return the unicode bytestring as is
value = string_decode(header) # convert to unicode
try:
value = value.encode('ascii')
except UnicodeEncodeError:
return value
# some mailers send out incorrectly escaped headers
# and double quote the escaped realname part again. remove those
# RFC: 2047
regex = r'"(=\?.+?\?.+?\?[^ ?]+\?=)"'
value = re.sub(regex, r'\1', value)
logging.debug("unquoted header: |%s|", value)
# otherwise we interpret RFC2822 encoding escape sequences
valuelist = email.header.decode_header(value)
decoded_list = []
for v, enc in valuelist:
v = string_decode(v, enc)
decoded_list.append(string_sanitize(v))
value = u' '.join(decoded_list)
if normalize:
value = re.sub(r'\n\s+', r' ', value)
return value
def decode_header(header, normalize=False):
"""
decode a header value to a unicode string
values are usually a mixture of different substrings
encoded in quoted printable using different encodings.
This turns it into a single unicode string
:param header: the header value
:type header: str
:param normalize: replace trailing spaces after newlines
:type normalize: bool
:rtype: unicode
"""
# If the value isn't ascii as RFC2822 prescribes,
# we just return the unicode bytestring as is
value = string_decode(header) # convert to unicode
try:
value = value.encode('ascii')
except UnicodeEncodeError:
return value
# some mailers send out incorrectly escaped headers
# and double quote the escaped realname part again. remove those
# RFC: 2047
regex = r'"(=\?.+?\?.+?\?[^ ?]+\?=)"'
value = re.sub(regex, r'\1', value)
logging.debug("unquoted header: |%s|", value)
# otherwise we interpret RFC2822 encoding escape sequences
valuelist = email.header.decode_header(value)
decoded_list = []
for v, enc in valuelist:
v = string_decode(v, enc)
decoded_list.append(string_sanitize(v))
value = ' '.join(decoded_list)
if normalize:
value = re.sub(r'\n\s+', r' ', value)
return value
def extract_body(mail, types=None):
"""
returns a body text string for given mail.
If types is `None`, `text/*` is used:
The exact preferred type is specified by the prefer_plaintext config option
which defaults to text/html.
:param mail: the mail to use
:type mail: :class:`email.Message`
:param types: mime content types to use for body string
:type types: list of str
"""
preferred = 'text/plain' if settings.get('prefer_plaintext') else 'text/html'
has_preferred = False
# see if the mail has our preferred type
if types == None:
has_preferred = list(typed_subpart_iterator(mail, *preferred.split('/')))
body_parts = []
for part in mail.walk():
ctype = part.get_content_type()
if types is not None:
if ctype not in types:
continue
cd = part.get('Content-Disposition', '')
if cd.startswith('attachment'):
continue
# if the mail has our preferred type, we only keep this type
# note that if types != None, has_preferred always stays False
if has_preferred and ctype != preferred:
continue
enc = part.get_content_charset() or 'ascii'
raw_payload = part.get_payload(decode=True)
if ctype == 'text/plain':
raw_payload = string_decode(raw_payload, enc)
body_parts.append(string_sanitize(raw_payload))
else:
#get mime handler
key = 'copiousoutput'
handler, entry = settings.mailcap_find_match(ctype, key=key)
tempfile_name = None
stdin = None
if entry:
handler_raw_commandstring = entry['view']
# in case the mailcap defined command contains no '%s',
# we pipe the files content to the handling command via stdin
if '%s' in handler_raw_commandstring:
# open tempfile, respect mailcaps nametemplate
nametemplate = entry.get('nametemplate', '%s')
prefix, suffix = parse_mailcap_nametemplate(nametemplate)
tmpfile = tempfile.NamedTemporaryFile(delete=False,
prefix=prefix,
suffix=suffix)
# write payload to tmpfile
tmpfile.write(raw_payload)
tmpfile.close()
tempfile_name = tmpfile.name
else:
stdin = raw_payload
# read parameter, create handler command
parms = tuple(map('='.join, part.get_params()))
# create and call external command
cmd = mailcap.subst(entry['view'], ctype,
filename=tempfile_name, plist=parms)
logging.debug('command: %s' % cmd)
logging.debug('parms: %s' % str(parms))
cmdlist = split_commandstring(cmd)
# call handler
rendered_payload, errmsg, retval = helper.call_cmd(cmdlist, stdin=stdin)
# remove tempfile
if tempfile_name:
os.unlink(tempfile_name)
if rendered_payload: # handler had output
body_parts.append(string_sanitize(rendered_payload))
return u'\n\n'.join(body_parts)
def extract_body(mail, types=None):
"""
returns a body text string for given mail.
If types is `None`, `text/*` is used:
In case mail has a `text/html` part, it is prefered over
`text/plain` parts.
:param mail: the mail to use
:type mail: :class:`email.Message`
:param types: mime content types to use for body string
:type types: list of str
"""
html = list(typed_subpart_iterator(mail, 'text', 'html'))
# if no specific types are given, we favor text/html over text/plain
drop_plaintext = False
if html and not types:
drop_plaintext = True
body_parts = []
for part in mail.walk():
ctype = part.get_content_type()
if types is not None:
if ctype not in types:
continue
cd = part.get('Content-Disposition', '')
if cd.startswith('attachment'):
continue
enc = part.get_content_charset() or 'ascii'
raw_payload = part.get_payload(decode=True)
if ctype == 'text/plain' and not drop_plaintext:
raw_payload = string_decode(raw_payload, enc)
body_parts.append(string_sanitize(raw_payload))
else:
#get mime handler
key = 'copiousoutput'
handler, entry = settings.mailcap_find_match(ctype, key=key)
if entry:
# open tempfile, respect mailcaps nametemplate
nametemplate = entry.get('nametemplate', '%s')
prefix, suffix = parse_mailcap_nametemplate(nametemplate)
tmpfile = tempfile.NamedTemporaryFile(delete=False,
prefix=prefix,
suffix=suffix)
# write payload to tmpfile
tmpfile.write(raw_payload)
tmpfile.close()
# read parameter, create handler command
parms = tuple(map('='.join, part.get_params()))
# create and call external command
cmd = mailcap.subst(entry['view'], ctype,
filename=tmpfile.name, plist=parms)
logging.debug('command: %s' % cmd)
logging.debug('parms: %s' % str(parms))
cmdlist = split_commandstring(cmd)
# call handler
rendered_payload, errmsg, retval = helper.call_cmd(cmdlist)
# remove tempfile
os.unlink(tmpfile.name)
if rendered_payload: # handler had output
body_parts.append(string_sanitize(rendered_payload))
return u'\n\n'.join(body_parts)
def extract_body(mail, types=None):
"""
returns a body text string for given mail.
If types is `None`, `text/*` is used:
In case mail has a `text/html` part, it is prefered over
`text/plain` parts.
:param mail: the mail to use
:type mail: :class:`email.Message`
:param types: mime content types to use for body string
:type types: list of str
"""
html = list(typed_subpart_iterator(mail, 'text', 'html'))
# if no specific types are given, we favor text/html over text/plain
drop_plaintext = False
if html and not types:
drop_plaintext = True
body_parts = []
for part in mail.walk():
ctype = part.get_content_type()
logging.debug(ctype)
if types is not None:
if ctype not in types:
continue
cd = part.get('Content-Disposition', '')
if cd.startswith('attachment'):
continue
enc = part.get_content_charset() or 'ascii'
raw_payload = part.get_payload(decode=True)
if ctype == 'text/plain' and not drop_plaintext:
raw_payload = string_decode(raw_payload, enc)
body_parts.append(string_sanitize(raw_payload))
else:
#get mime handler
key = 'copiousoutput'
handler, entry = settings.mailcap_find_match(ctype, key=key)
if entry:
# open tempfile, respect mailcaps nametemplate
nametemplate = entry.get('nametemplate', '%s')
prefix, suffix = parse_mailcap_nametemplate(nametemplate)
tmpfile = tempfile.NamedTemporaryFile(delete=False,
prefix=prefix,
suffix=suffix)
# write payload to tmpfile
tmpfile.write(raw_payload)
tmpfile.close()
# read parameter, create handler command
parms = tuple(map('='.join, part.get_params()))
# create and call external command
cmd = mailcap.subst(entry['view'],
ctype,
filename=tmpfile.name,
plist=parms)
logging.debug('command: %s' % cmd)
logging.debug('parms: %s' % str(parms))
cmdlist = split_commandstring(cmd)
# call handler
rendered_payload, errmsg, retval = helper.call_cmd(cmdlist)
# remove tempfile
os.unlink(tmpfile.name)
if rendered_payload: # handler had output
body_parts.append(string_sanitize(rendered_payload))
return u'\n\n'.join(body_parts)
def extract_body(mail, types=None):
"""
returns a body text string for given mail.
If types is `None`, `text/*` is used:
The exact preferred type is specified by the prefer_plaintext config option
which defaults to text/html.
:param mail: the mail to use
:type mail: :class:`email.Message`
:param types: mime content types to use for body string
:type types: list of str
"""
preferred = 'text/plain' if settings.get(
'prefer_plaintext') else 'text/html'
has_preferred = False
# see if the mail has our preferred type
if types is None:
has_preferred = list(
typed_subpart_iterator(mail, *preferred.split('/')))
body_parts = []
for part in mail.walk():
ctype = part.get_content_type()
if types is not None:
if ctype not in types:
continue
cd = part.get('Content-Disposition', '')
if cd.startswith('attachment'):
continue
# if the mail has our preferred type, we only keep this type
# note that if types != None, has_preferred always stays False
if has_preferred and ctype != preferred:
continue
enc = part.get_content_charset() or 'ascii'
raw_payload = part.get_payload(decode=True)
if ctype == 'text/plain':
raw_payload = string_decode(raw_payload, enc)
body_parts.append(string_sanitize(raw_payload))
else:
# get mime handler
key = 'copiousoutput'
handler, entry = settings.mailcap_find_match(ctype, key=key)
tempfile_name = None
stdin = None
if entry:
handler_raw_commandstring = entry['view']
# in case the mailcap defined command contains no '%s',
# we pipe the files content to the handling command via stdin
if '%s' in handler_raw_commandstring:
# open tempfile, respect mailcaps nametemplate
nametemplate = entry.get('nametemplate', '%s')
prefix, suffix = parse_mailcap_nametemplate(nametemplate)
tmpfile = tempfile.NamedTemporaryFile(delete=False,
prefix=prefix,
suffix=suffix)
# write payload to tmpfile
tmpfile.write(raw_payload)
tmpfile.close()
tempfile_name = tmpfile.name
else:
stdin = raw_payload
# read parameter, create handler command
parms = tuple(map('='.join, part.get_params()))
# create and call external command
cmd = mailcap.subst(entry['view'],
ctype,
filename=tempfile_name,
plist=parms)
logging.debug('command: %s' % cmd)
logging.debug('parms: %s' % str(parms))
cmdlist = split_commandstring(cmd)
# call handler
rendered_payload, errmsg, retval = helper.call_cmd(cmdlist,
stdin=stdin)
# remove tempfile
if tempfile_name:
os.unlink(tempfile_name)
if rendered_payload: # handler had output
body_parts.append(string_sanitize(rendered_payload))
return u'\n\n'.join(body_parts)
def test_tabs(self):
base = 'foo\tbar\noink\n'
expected = 'foo' + ' ' * 5 + 'bar\noink\n'
actual = helper.string_sanitize(base)
self.assertEqual(actual, expected)
def test_tabs(self):
base = 'foo\tbar\noink\n'
expected = 'foo' + ' ' * 5 + 'bar\noink\n'
actual = helper.string_sanitize(base)
self.assertEqual(actual, expected)
