def cleanpage(html):
# cleaner setup
cleaner = Cleaner()
cleaner.html = True
cleaner.page_structure = False
cleaner.meta = False
cleaner.safe_attrs_only = False
cleaner.links = False
cleaner.javascript = True # activate the javascript filter
cleaner.style = True # activate the styles & stylesheet filter
cleaner.links = False
cleaner.frames = True
cleaner.embedded = True
cleaner.comments = True
cleaner.annoying_tags = True
cleaner.inline_style = True
cleaner.page_structure = False
# cleaner.remove_tags = ['b','img','h']
cleaner.kill_tags = ['img','script']
#invoke cleaner
try:
content=cleaner.clean_html(html)
except:
#error: ValueError: Unicode strings with encoding declaration are not supported. Please use bytes input or XML fr
content = u""
return content
def lxml_extractor(html, url):
'''LXML PARSER'''
cleaner = Cleaner()
cleaner.javascript = True # This is True because we want to activate the javascript filter
cleaner.style = True # This is True because we want to activate the styles & stylesheet filter
cleaner.comments = True
cleaner.embedded = True
cleaner.forms= True
cleaner.frames = True
cleaner.annoying_tags = True
cleaner.kill_tags = NEGATIVE_K
cleaner.allow_tag = POSITIVE_K
cleaner.safe_attrs_only = True
#~ oc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
#~ File "/usr/local/lib/python2.7/dist-packages/lxml/html/__init__.py", line 752, in document_fromstring
#~ value = etree.fromstring(html, parser, **kw)
try:
html = lxml.html.fromstring(html, base_url="url")
tree = cleaner.clean_html(html)
#tree.make_links_absolute(url)
doc = lxml.html.tostring(tree)
doc = soup_extractor(doc, url)
except ValueError:
doc = soup_extractor(html, url)
#~ (title, doc, article, text) = read_extractor(html, url)
#~ print title
#~ doc = (self.doc).replace(unichr(160), " ")
#~ doc = re.sub(spaces,"",self.doc)
return doc
def clean_html(html_text,
javascript=True,
scripts=True,
style=True,
embedded=True,
links=True,
forms=True,
frames=True,
comments=True,
annoying_tags=True,
meta=True,
safe_attrs_only=True,
remove_unknown_tags=True,
processing_instructions=True):
"""Clean all the javascript and styles from the HTML returning the string with only the html content"""
# True = Remove | False = Keep
cleaner = Cleaner()
cleaner.javascript = javascript # This is True because we want to activate the javascript filter
cleaner.scripts = scripts # This is True because we want to activate the scripts filter
cleaner.style = style
cleaner.embedded = embedded
cleaner.links = links
cleaner.forms = forms
cleaner.frames = frames
cleaner.comments = comments
cleaner.page_structure = False # Keep page structure
cleaner.annoying_tags = annoying_tags
cleaner.meta = meta
cleaner.safe_attrs_only = safe_attrs_only
cleaner.remove_unknown_tags = remove_unknown_tags
cleaner.processing_instructions = processing_instructions
clean_content = cleaner.clean_html(lxml.html.fromstring(html_text))
return lxml.html.tostring(clean_content)
def clearTag_old(self, text: str) -> str:
import lxml
from lxml.html.clean import Cleaner
cleaner = Cleaner()
cleaner.javascript = True
cleaner.style = True
cleaner.links = True
cleaner.meta = True
cleaner.forms = True
cleaner.embedded = True
cleaner.frames = True
cleaner.remove_unknown_tags = True
cleaner.kill_tags = ["img"]
cleaner.remove_tags = [
"strong",
"div",
"body",
"br",
"a",
"p",
"blockquote",
"h3",
"ol",
"li",
"font",
]
return cleaner.clean_html(
lxml.html.document_fromstring(text)).decode("utf-8")
def cleaner_parameters():
reject_list = [
'script', 'noscript', 'style', 'meta', 'semantics', 'img', 'label',
'table', 'li', 'ul', 'ol', 'nav', 'dl', 'dd', 'sub', 'sup', 'math'
]
accept_list = [
'div', 'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'
'span', 'b', 'a', 'u', 'i', 'body'
]
html_cleaner = Cleaner()
html_cleaner.remove_unknown_tags = True
html_cleaner.processing_instructions = True
html_cleaner.style = True
html_cleaner.comments = True
html_cleaner.scripts = True
html_cleaner.javascript = True
html_cleaner.meta = True
html_cleaner.links = True
html_cleaner.embedded = True
html_cleaner.annoying_tags = True
html_cleaner.frames = True
html_cleaner.forms = True
html_cleaner.remove_tags = accept_list
html_cleaner.kill_tags = reject_list
return html_cleaner
def convertHtmlToDicts(url, content):
""" given a url and content, create file and article dictionaries
content has to include normal newlines, no \a or #N# replacers
returns None, None on error
"""
# lxml does not like unicode if the document has an explicit encoding
if " encoding=" not in content:
content = pubGeneric.forceToUnicode(content)
logging.debug("Converting to text: %s " % (repr(url)))
artDict = pubStore.createEmptyArticleDict(source="bing", fulltextUrl=url)
if not "<html" in content:
return None, None
try:
logging.debug("Parsing html with lxml, html size %d" % len(content))
tree = lxml.html.document_fromstring(content)
logging.debug("end parse html")
except lxml.etree.XMLSyntaxError:
return None, None
titleEl = tree.find("head/title")
if titleEl!=None:
title = titleEl.text
else:
logging.debug("No title found?")
title = ""
metaTags = tree.findall("head/meta")
artDict = parseMetaData(metaTags, artDict)
logging.debug("Cleaning html tree")
cleaner = Cleaner()
cleaner.javascript = True
cleaner.style = True
cleaner.meta = True
cleaner.embedded = True
cleaner.page_structure=True
#cleaner.remove_tags = ["a", "li", "td"]
cleanTree = cleaner.clean_html(tree)
logging.debug("Cleaning done, now converting to ASCII")
#text = cleanTree.text_content()
newlineTags = ["p", "br"]
asciiText = pubXml.treeToAsciiText(cleanTree, newlineTags)
logging.debug("ASCII conversion done")
logging.debug("title: %s" % title)
if "title" not in artDict or artDict["title"]=="":
artDict["title"] = title
if artDict["abstract"]=="":
abstract = unidecode.unidecode(asciiText[0:1500]).strip()
artDict["abstract"] = abstract
logging.debug("abstract: %s" % artDict["abstract"])
fileDict = pubStore.createEmptyFileDict(url=url, content=asciiText, mimeType="text/html")
logging.debug("meta data extract success: %s" % artDict)
return artDict, fileDict
def convertHtmlToDicts(url, content):
""" given a url and content, create file and article dictionaries
content has to include normal newlines, no \a or #N# replacers
returns None, None on error
"""
# lxml does not like unicode if the document has an explicit encoding
if " encoding=" not in content:
content = pubGeneric.forceToUnicode(content)
logging.debug("Converting to text: %s " % (repr(url)))
artDict = pubStore.createEmptyArticleDict(source="bing", fulltextUrl=url)
if not "<html" in content:
return None, None
try:
logging.debug("Parsing html with lxml, html size %d" % len(content))
tree = lxml.html.document_fromstring(content)
logging.debug("end parse html")
except lxml.etree.XMLSyntaxError:
return None, None
titleEl = tree.find("head/title")
if titleEl!=None:
title = titleEl.text
else:
logging.debug("No title found?")
title = ""
metaTags = tree.findall("head/meta")
artDict = parseMetaData(metaTags, artDict)
logging.debug("Cleaning html tree")
cleaner = Cleaner()
cleaner.javascript = True
cleaner.style = True
cleaner.meta = True
cleaner.embedded = True
cleaner.page_structure=True
#cleaner.remove_tags = ["a", "li", "td"]
cleanTree = cleaner.clean_html(tree)
logging.debug("Cleaning done, now converting to ASCII")
#text = cleanTree.text_content()
newlineTags = ["p", "br"]
asciiText = pubXml.treeToAsciiText(cleanTree, newlineTags)
logging.debug("ASCII conversion done")
logging.debug("title: %s" % title)
if "title" not in artDict or artDict["title"]=="":
artDict["title"] = title
if artDict["abstract"]=="":
abstract = unidecode.unidecode(asciiText[0:1500]).strip()
artDict["abstract"] = abstract
logging.debug("abstract: %s" % artDict["abstract"])
fileDict = pubStore.createEmptyFileDict(url=url, content=asciiText, mimeType="text/html")
logging.debug("meta data extract success: %s" % artDict)
return artDict, fileDict
def get_cleaner():
cleaner = Cleaner()
cleaner.embedded = True
cleaner.frames = True
cleaner.style = True
cleaner.remove_unknown_tags = True
cleaner.processing_instructions = True
cleaner.annoying_tags = True
cleaner.remove_tags = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'b', 'a', 'u', 'i', 'body', 'div', 'span', 'p']
cleaner.kill_tags = ['table', 'img', 'semantics', 'script', 'noscript', 'style', 'meta', 'label', 'li', 'ul',
'ol', 'sup', 'math', 'nav', 'dl', 'dd', 'sub']
return cleaner
import lxml
from lxml import etree
from lxml.html.clean import autolink_html
from lxml.html.clean import Cleaner
#LXML PARSER
cleaner = Cleaner()
cleaner.javascript = True # This is True because we want to activate the javascript filter
cleaner.style = True # This is True because we want to activate the styles & stylesheet filter
cleaner.comments = True
cleaner.embedded = True
cleaner.forms= True
cleaner.frames = True
#cleaner.safe_attrs_only = True
import re
notalpha = re.compile('[^a-zA-Z]')
#BS PARSER
from bs4 import BeautifulSoup as bs
from bs4 import Comment
def make_links_absolute(soup, url):
return [urlparse.urljoin(url, tag['href']) for tag in soup.findAll('a', href=True)]
def clean_html(soup):
soup = bs(" ".join([s.extract() for s in soup('script')]))
soup = bs(" ".join([s.extract() for s in soup('iframe')]))
soup = bs(" ".join([s.extract() for s in soup('form')]))
soup = bs(" ".join([s.extract() for s in soup('embed')]))
soup = bs(" ".join([s.extract() for s in soup('style')]))
from lxml import etree from lxml.html.clean import Cleaner from .filters import duplicate_test, textfilter from .settings import CUT_EMPTY_ELEMS, DEFAULT_CONFIG, MANUALLY_CLEANED, MANUALLY_STRIPPED from .utils import trim LOGGER = logging.getLogger(__name__) # HTML_CLEANER config # http://lxml.de/api/lxml.html.clean.Cleaner-class.html # https://lxml.de/apidoc/lxml.html.clean.html HTML_CLEANER = Cleaner() HTML_CLEANER.annoying_tags = False # True HTML_CLEANER.comments = True HTML_CLEANER.embedded = False # True HTML_CLEANER.forms = False # True HTML_CLEANER.frames = False # True HTML_CLEANER.javascript = False HTML_CLEANER.links = False HTML_CLEANER.meta = False HTML_CLEANER.page_structure = False HTML_CLEANER.processing_instructions = True HTML_CLEANER.remove_unknown_tags = False HTML_CLEANER.safe_attrs_only = False HTML_CLEANER.scripts = False HTML_CLEANER.style = False #HTML_CLEANER.remove_tags = MANUALLY_STRIPPED #HTML_CLEANER.kill_tags = MANUALLY_CLEANED
DATE_EXPRESSIONS = [
"//*[contains(@id, 'date') or contains(@id, 'Date') or contains(@id, 'datum') or contains(@id, 'Datum') or contains(@id, 'time') or contains(@class, 'post-meta-time')]",
"//*[contains(@class, 'date') or contains(@class, 'Date') or contains(@class, 'datum') or contains(@class, 'Datum')]",
"//*[contains(@class, 'postmeta') or contains(@class, 'post-meta') or contains(@class, 'entry-meta') or contains(@class, 'postMeta') or contains(@class, 'post_meta') or contains(@class, 'post__meta')]",
"//*[@class='meta' or @class='meta-before' or @class='asset-meta' or contains(@id, 'article-metadata') or contains(@class, 'article-metadata') or contains(@class, 'byline') or contains(@class, 'subline')]",
"//*[contains(@class, 'published') or contains(@class, 'posted') or contains(@class, 'submitted') or contains(@class, 'created-post')]",
"//*[contains(@id, 'lastmod') or contains(@itemprop, 'date') or contains(@class, 'time')]",
"//footer",
"//*[@class='post-footer' or @class='footer' or @id='footer']",
"//small",
"//*[contains(@class, 'author') or contains(@class, 'autor') or contains(@class, 'field-content') or @class='meta' or contains(@class, 'info') or contains(@class, 'fa-clock-o')]",
]
CLEANER = Cleaner()
CLEANER.comments = False
CLEANER.embedded = True
CLEANER.forms = False
CLEANER.frames = True
CLEANER.javascript = True
CLEANER.links = False
CLEANER.meta = False
CLEANER.page_structure = True
CLEANER.processing_instructions = True
CLEANER.remove_unknown_tags = False
CLEANER.safe_attrs_only = False
CLEANER.scripts = False
CLEANER.style = True
CLEANER.kill_tags = [
'audio', 'canvas', 'label', 'map', 'math', 'object', 'picture', 'rdf',
'svg', 'video'
] # 'embed', 'figure', 'img', 'table'
# './/aside', \ # conflicts with text extraction
]
COMMENTS_DISCARD_XPATH = ['.//*[(self::div or self::section)][starts-with(@id, "respond")]', \
'.//cite', \
'.//quote', \
'.//*[starts-with(@id, "reply-") or starts-with(@class, "reply-title")]', \
'.//*[contains(@id, "akismet") or contains(@class, "akismet")]', \
]
# HTML_CLEANER config # http://lxml.de/api/lxml.html.clean.Cleaner-class.html
HTML_CLEANER = Cleaner()
HTML_CLEANER.annoying_tags = True
HTML_CLEANER.comments = True
HTML_CLEANER.embedded = True
HTML_CLEANER.forms = True
HTML_CLEANER.frames = True
HTML_CLEANER.javascript = True
HTML_CLEANER.links = False
HTML_CLEANER.meta = False
HTML_CLEANER.page_structure = False
HTML_CLEANER.processing_instructions = True
HTML_CLEANER.remove_unknown_tags = False
HTML_CLEANER.safe_attrs_only = False
HTML_CLEANER.scripts = True
HTML_CLEANER.style = False
HTML_CLEANER.remove_tags = [
'a', 'abbr', 'acronym', 'address', 'big', 'cite', 'font', 'ins', 'meta',
'small', 'sub', 'sup', 'wbr'
] # 'center', 'table', 'tbody', 'td', 'th', 'tr', 'span',
# latest possible year
MAX_YEAR = datetime.date.today().year
## DateDataParser object
PARSERCONFIG = {
'PREFER_DAY_OF_MONTH': 'first',
'PREFER_DATES_FROM': 'past',
'DATE_ORDER': 'DMY'
}
logger.debug('settings: %s %s %s', MIN_YEAR, TODAY, MAX_YEAR)
logger.debug('dateparser configuration: %s', PARSERCONFIG)
cleaner = Cleaner()
cleaner.comments = True
cleaner.embedded = True
cleaner.forms = False
cleaner.frames = True
cleaner.javascript = False
cleaner.links = False
cleaner.meta = False
cleaner.page_structure = True
cleaner.processing_instructions = True
cleaner.remove_unknown_tags = False
cleaner.safe_attrs_only = False
cleaner.scripts = False
cleaner.style = False
cleaner.kill_tags = [
'audio', 'canvas', 'label', 'map', 'math', 'object', 'picture', 'table',
'svg', 'video'
]
def html_sanitizer(content):
""" sanitize malicious scripts """
cleaner = Cleaner()
cleaner.embedded = False
cleaner.safe_attrs_only = False
return cleaner.clean_html(content)
