def extract_from_html(msg_body):
"""
Extract not quoted message from provided html message body
using tags and plain text algorithm.
Cut out the 'blockquote', 'gmail_quote' tags.
Cut Microsoft quotations.
Then use plain text algorithm to cut out splitter or
leftover quotation.
This works by adding checkpoint text to all html tags,
then converting html to text,
then extracting quotations from text,
then checking deleted checkpoints,
then deleting necessary tags.
"""
if msg_body.strip() == '':
return msg_body
html_tree = html.document_fromstring(
msg_body,
parser=html.HTMLParser(encoding="utf-8")
)
cut_quotations = (html_quotations.cut_gmail_quote(html_tree) or
html_quotations.cut_blockquote(html_tree) or
html_quotations.cut_microsoft_quote(html_tree) or
html_quotations.cut_by_id(html_tree) or
html_quotations.cut_from_block(html_tree)
)
html_tree_copy = deepcopy(html_tree)
number_of_checkpoints = html_quotations.add_checkpoint(html_tree, 0)
quotation_checkpoints = [False for i in range(number_of_checkpoints)]
msg_with_checkpoints = html.tostring(html_tree)
h = html2text.HTML2Text()
h.body_width = 0 # generate plain text without wrap
# html2text adds unnecessary star symbols. Remove them.
# Mask star symbols
msg_with_checkpoints = msg_with_checkpoints.decode('utf-8').replace('*', '3423oorkg432')
plain_text = h.handle(msg_with_checkpoints)
# Remove created star symbols
plain_text = plain_text.replace('*', '')
# Unmask saved star symbols
plain_text = plain_text.replace('3423oorkg432', '*')
delimiter = get_delimiter(plain_text)
plain_text = preprocess(plain_text, delimiter, content_type='text/html')
lines = plain_text.splitlines()
# Don't process too long messages
if len(lines) > MAX_LINES_COUNT:
return msg_body
# Collect checkpoints on each line
line_checkpoints = [
[int(i[4:-4]) # Only checkpoint number
for i in re.findall(html_quotations.CHECKPOINT_PATTERN, line)]
for line in lines]
# Remove checkpoints
lines = [re.sub(html_quotations.CHECKPOINT_PATTERN, '', line)
for line in lines]
# Use plain text quotation extracting algorithm
markers = mark_message_lines(lines)
return_flags = []
process_marked_lines(lines, markers, return_flags)
lines_were_deleted, first_deleted, last_deleted = return_flags
if lines_were_deleted:
#collect checkpoints from deleted lines
for i in range(first_deleted, last_deleted):
for checkpoint in line_checkpoints[i]:
quotation_checkpoints[checkpoint] = True
else:
if cut_quotations:
return html.tostring(html_tree_copy)
else:
return msg_body
# Remove tags with quotation checkpoints
html_quotations.delete_quotation_tags(
html_tree_copy, 0, quotation_checkpoints
)
return html.tostring(html_tree_copy).decode('utf-8')
def _extract_from_html(msg_body):
"""
Extract not quoted message from provided html message body
using tags and plain text algorithm.
Cut out the 'blockquote', 'gmail_quote' tags.
Cut Microsoft quotations.
Then use plain text algorithm to cut out splitter or
leftover quotation.
This works by adding checkpoint text to all html tags,
then converting html to text,
then extracting quotations from text,
then checking deleted checkpoints,
then deleting necessary tags.
"""
if len(msg_body) > MAX_HTML_LEN:
return msg_body
if msg_body.strip() == b'':
return msg_body
msg_body = msg_body.replace(b'\r\n', b'\n')
html_tree = html_document_fromstring(msg_body)
if html_tree is None:
return msg_body
cut_quotations = (html_quotations.cut_gmail_quote(html_tree)
or html_quotations.cut_zimbra_quote(html_tree)
or html_quotations.cut_blockquote(html_tree)
or html_quotations.cut_microsoft_quote(html_tree)
or html_quotations.cut_by_id(html_tree)
or html_quotations.cut_from_block(html_tree))
html_tree_copy = deepcopy(html_tree)
number_of_checkpoints = html_quotations.add_checkpoint(html_tree, 0)
quotation_checkpoints = [False] * number_of_checkpoints
plain_text = html_tree_to_text(html_tree)
plain_text = preprocess(plain_text, '\n', content_type='text/html')
lines = plain_text.splitlines()
# Don't process too long messages
if len(lines) > MAX_LINES_COUNT:
return msg_body
# Collect checkpoints on each line
line_checkpoints = [
[
int(i[4:-4]) # Only checkpoint number
for i in re.findall(html_quotations.CHECKPOINT_PATTERN, line)
] for line in lines
]
# Remove checkpoints
lines = [
re.sub(html_quotations.CHECKPOINT_PATTERN, '', line) for line in lines
]
# Use plain text quotation extracting algorithm
markers = mark_message_lines(lines)
return_flags = []
process_marked_lines(lines, markers, return_flags)
lines_were_deleted, first_deleted, last_deleted = return_flags
if not lines_were_deleted and not cut_quotations:
return msg_body
if lines_were_deleted:
#collect checkpoints from deleted lines
for i in range(first_deleted, last_deleted):
for checkpoint in line_checkpoints[i]:
quotation_checkpoints[checkpoint] = True
# Remove tags with quotation checkpoints
html_quotations.delete_quotation_tags(html_tree_copy, 0,
quotation_checkpoints)
if _readable_text_empty(html_tree_copy):
return msg_body
return html.tostring(html_tree_copy)
def extract_from_html(msg_body):
"""
Extract not quoted message from provided html message body
using tags and plain text algorithm.
Cut out the 'blockquote', 'gmail_quote' tags.
Cut Microsoft quotations.
Then use plain text algorithm to cut out splitter or
leftover quotation.
This works by adding checkpoint text to all html tags,
then converting html to text,
then extracting quotations from text,
then checking deleted checkpoints,
then deleting necessary tags.
"""
if msg_body.strip() == '':
return msg_body
html_tree = html.document_fromstring(
msg_body, parser=html.HTMLParser(encoding="utf-8"))
cut_quotations = (html_quotations.cut_gmail_quote(html_tree)
or html_quotations.cut_blockquote(html_tree)
or html_quotations.cut_microsoft_quote(html_tree)
or html_quotations.cut_by_id(html_tree)
or html_quotations.cut_from_block(html_tree))
html_tree_copy = deepcopy(html_tree)
number_of_checkpoints = html_quotations.add_checkpoint(html_tree, 0)
quotation_checkpoints = [False for i in xrange(number_of_checkpoints)]
msg_with_checkpoints = html.tostring(html_tree)
h = html2text.HTML2Text()
h.body_width = 0 # generate plain text without wrap
# html2text adds unnecessary star symbols. Remove them.
# Mask star symbols
msg_with_checkpoints = msg_with_checkpoints.replace('*', '3423oorkg432')
plain_text = h.handle(msg_with_checkpoints)
# Remove created star symbols
plain_text = plain_text.replace('*', '')
# Unmask saved star symbols
plain_text = plain_text.replace('3423oorkg432', '*')
delimiter = get_delimiter(plain_text)
plain_text = preprocess(plain_text, delimiter, content_type='text/html')
lines = plain_text.splitlines()
# Don't process too long messages
if len(lines) > MAX_LINES_COUNT:
return msg_body
# Collect checkpoints on each line
line_checkpoints = [
[
int(i[4:-4]) # Only checkpoint number
for i in re.findall(html_quotations.CHECKPOINT_PATTERN, line)
] for line in lines
]
# Remove checkpoints
lines = [
re.sub(html_quotations.CHECKPOINT_PATTERN, '', line) for line in lines
]
# Use plain text quotation extracting algorithm
markers = mark_message_lines(lines)
return_flags = []
process_marked_lines(lines, markers, return_flags)
lines_were_deleted, first_deleted, last_deleted = return_flags
if lines_were_deleted:
#collect checkpoints from deleted lines
for i in xrange(first_deleted, last_deleted):
for checkpoint in line_checkpoints[i]:
quotation_checkpoints[checkpoint] = True
else:
if cut_quotations:
return html.tostring(html_tree_copy)
else:
return msg_body
# Remove tags with quotation checkpoints
html_quotations.delete_quotation_tags(html_tree_copy, 0,
quotation_checkpoints)
return html.tostring(html_tree_copy)
def _extract_from_html(msg_body):
"""
Extract not quoted message from provided html message body
using tags and plain text algorithm.
Cut out first some encoding html tags such as xml and doctype
for avoiding conflict with unicode decoding
Cut out the 'blockquote', 'gmail_quote' tags.
Cut Microsoft quotations.
Then use plain text algorithm to cut out splitter or
leftover quotation.
This works by adding checkpoint text to all html tags,
then converting html to text,
then extracting quotations from text,
then checking deleted checkpoints,
then deleting necessary tags.
"""
if msg_body.strip() == b"":
return msg_body
msg_body = msg_body.replace(b"\r\n", b"\n")
msg_body = msg_body.decode("utf-8")
msg_body = re.sub(r"\<\?xml.+\?\>|\<\!DOCTYPE.+]\>", "", msg_body)
html_tree = html_document_fromstring(msg_body)
if html_tree is None:
return msg_body
cut_quotations = (html_quotations.cut_gmail_quote(html_tree)
or html_quotations.cut_zimbra_quote(html_tree)
or html_quotations.cut_blockquote(html_tree)
or html_quotations.cut_microsoft_quote(html_tree)
or html_quotations.cut_by_id(html_tree)
or html_quotations.cut_from_block(html_tree))
html_tree_copy = deepcopy(html_tree)
number_of_checkpoints = html_quotations.add_checkpoint(html_tree, 0)
quotation_checkpoints = [False] * number_of_checkpoints
plain_text = html_tree_to_text(html_tree)
plain_text = preprocess(plain_text, "\n", content_type="text/html")
lines = plain_text.splitlines()
# Don't process too long messages
if len(lines) > MAX_LINES_COUNT:
return msg_body
# Collect checkpoints on each line
line_checkpoints = [
[
int(i[4:-4])
for i in re.findall(html_quotations.CHECKPOINT_PATTERN, line)
] # Only checkpoint number
for line in lines
]
# Remove checkpoints
lines = [
re.sub(html_quotations.CHECKPOINT_PATTERN, "", line) for line in lines
]
# Use plain text quotation extracting algorithm
markers = mark_message_lines(lines)
return_flags = []
process_marked_lines(lines, markers, return_flags)
lines_were_deleted, first_deleted, last_deleted = return_flags
if not lines_were_deleted and not cut_quotations:
return msg_body
if lines_were_deleted:
# collect checkpoints from deleted lines
for i in range(first_deleted, last_deleted):
for checkpoint in line_checkpoints[i]:
quotation_checkpoints[checkpoint] = True
# Remove tags with quotation checkpoints
html_quotations.delete_quotation_tags(html_tree_copy, 0,
quotation_checkpoints)
if _readable_text_empty(html_tree_copy):
return msg_body
# NOTE: We remove_namespaces() because we are using an HTML5 Parser, HTML
# parsers do not recognize namespaces in HTML tags. As such the rendered
# HTML tags are no longer recognizable HTML tags. Example: <o:p> becomes
# <oU0003Ap>. When we port this to golang we should look into using an
# XML Parser NOT and HTML5 Parser since we do not know what input a
# customer will send us. Switching to a common XML parser in python
# opens us up to a host of vulnerabilities.
# See https://docs.python.org/3/library/xml.html#xml-vulnerabilities
#
# The down sides to removing the namespaces is that customers might
# judge the XML namespaces important. If that is the case then support
# should encourage customers to preform XML parsing of the un-stripped
# body to get the full unmodified XML payload.
#
# Alternatives to this approach are
# 1. Ignore the U0003A in tag names and let the customer deal with it.
# This is not ideal, as most customers use stripped-html for viewing
# emails sent from a recipient, as such they cannot control the HTML
# provided by a recipient.
# 2. Preform a string replace of 'U0003A' to ':' on the rendered HTML
# string. While this would solve the issue simply, it runs the risk
# of replacing data outside the <tag> which might be essential to
# the customer.
remove_namespaces(html_tree_copy)
return html.tostring(html_tree_copy)
def _extract_from_html(msg_body):
"""
Extract not quoted message from provided html message body
using tags and plain text algorithm.
Cut out the 'blockquote', 'gmail_quote' tags.
Cut Microsoft quotations.
Then use plain text algorithm to cut out splitter or
leftover quotation.
This works by adding checkpoint text to all html tags,
then converting html to text,
then extracting quotations from text,
then checking deleted checkpoints,
then deleting necessary tags.
"""
if msg_body.strip() == b'':
return msg_body
msg_body = msg_body.replace('\r\n', '').replace('\n', '')
html_tree = html.document_fromstring(
msg_body,
parser=html.HTMLParser(encoding="utf-8")
)
if html_tree is None:
return msg_body
cut_quotations = False
while True:
cut_quotations_tmp = html_quotations.cut_blockquote(html_tree)
cut_quotations_tmp = html_quotations.cut_gmail_quote(html_tree) or cut_quotations_tmp
cut_quotations_tmp = html_quotations.cut_microsoft_quote(html_tree) or cut_quotations_tmp
cut_quotations_tmp = html_quotations.cut_by_id(html_tree) or cut_quotations_tmp
cut_quotations_tmp = html_quotations.cut_from_block(html_tree) or cut_quotations_tmp
if cut_quotations_tmp == True:
cut_quotations = True
else:
break
html_tree_copy = deepcopy(html_tree)
number_of_checkpoints = html_quotations.add_checkpoint(html_tree, 0)
quotation_checkpoints = [False] * number_of_checkpoints
plain_text = html_tree_to_text(html_tree)
plain_text = preprocess(plain_text, '\n', content_type='text/html')
lines = plain_text.splitlines()
# Don't process too long messages
if len(lines) > MAX_LINES_COUNT:
return msg_body
# Collect checkpoints on each line
line_checkpoints = [
[int(i[4:-4]) # Only checkpoint number
for i in re.findall(html_quotations.CHECKPOINT_PATTERN, line)]
for line in lines]
# Remove checkpoints
lines = [re.sub(html_quotations.CHECKPOINT_PATTERN, '', line)
for line in lines]
# Use plain text quotation extracting algorithm
markers = mark_message_lines(lines)
return_flags = []
process_marked_lines(lines, markers, return_flags)
lines_were_deleted, first_deleted, last_deleted = return_flags
if not lines_were_deleted and not cut_quotations:
return msg_body
if lines_were_deleted:
#collect checkpoints from deleted lines
for i in range(first_deleted, last_deleted):
for checkpoint in line_checkpoints[i]:
quotation_checkpoints[checkpoint] = True
# Remove tags with quotation checkpoints
html_quotations.delete_quotation_tags(
html_tree_copy, 0, quotation_checkpoints
)
if _readable_text_empty(html_tree_copy):
return msg_body
return html.tostring(html_tree_copy)