def test_class_hits(): """If the class is in the list then it gets a weight""" test_div = '<div class="something post">Content</div>' node = fragment_fromstring(test_div) assert get_class_weight(node) == 25 test_div = '<div class="something comments">Content</div>' node = fragment_fromstring(test_div) assert get_class_weight(node) == -25
def test_class_hits(self): """If the class is in the list then it gets a weight""" test_div = '<div class="something post">Content</div>' node = fragment_fromstring(test_div) self.assertEqual(get_class_weight(node), 25) test_div = '<div class="something comments">Content</div>' node = fragment_fromstring(test_div) self.assertEqual(get_class_weight(node), -25)
def test_id_hits(): """If the id is in the list then it gets a weight""" test_div = '<div id="post">Content</div>' node = fragment_fromstring(test_div) assert get_class_weight(node) == 25 test_div = '<div id="comments">Content</div>' node = fragment_fromstring(test_div) assert get_class_weight(node) == -25
def test_scores_collide(): """We might hit both positive and negative scores. Positive and negative scoring is done independently so it's possible to hit both positive and negative scores and cancel each other out. """ test_div = '<div id="post" class="something comment">Content</div>' node = fragment_fromstring(test_div) assert get_class_weight(node) == 0 test_div = '<div id="post" class="post comment">Content</div>' node = fragment_fromstring(test_div) assert get_class_weight(node) == 25
def test_scores_collide(self): """We might hit both positive and negative scores. Positive and negative scoring is done independently so it's possible to hit both positive and negative scores and cancel each other out. """ test_div = '<div id="post" class="something comment">Content</div>' node = fragment_fromstring(test_div) self.assertEqual(get_class_weight(node), 0) test_div = '<div id="post" class="post comment">Content</div>' node = fragment_fromstring(test_div) self.assertEqual(get_class_weight(node), 25)
def clean_document(node): """Clean up the final document we return as the readable article""" if node is None or len(node) == 0: return LNODE.log(node, 2, "Processing doc") clean_list = ['object', 'h1'] to_drop = [] # If there is only one h2, they are probably using it as a header and # not a subheader, so remove it since we already have a header. if len(node.findall('.//h2')) == 1: LOG.debug('Adding H2 to list of nodes to clean.') clean_list.append('h2') for n in node.iter(): LNODE.log(n, 2, "Cleaning iter node") # clean out any in-line style properties if 'style' in n.attrib: n.set('style', '') # remove all of the following tags # Clean a node of all elements of type "tag". # (Unless it's a youtube/vimeo video. People love movies.) is_embed = True if n.tag in ['object', 'embed'] else False if n.tag in clean_list: allow = False # Allow youtube and vimeo videos through as people usually # want to see those. if is_embed: if ok_embedded_video(n): allow = True if not allow: LNODE.log(n, 2, "Dropping Node") to_drop.append(n) if n.tag in ['h1', 'h2', 'h3', 'h4']: # clean headings # if the heading has no css weight or a high link density, # remove it if get_class_weight(n) < 0 or get_link_density(n) > .33: LNODE.log(n, 2, "Dropping <hX>, it's insignificant") to_drop.append(n) # clean out extra <p> if n.tag == 'p': # if the p has no children and has no content...well then down # with it. if not n.getchildren() and len(n.text_content()) < 5: LNODE.log(n, 2, 'Dropping extra <p>') to_drop.append(n) # finally try out the conditional cleaning of the target node if clean_conditionally(n): to_drop.append(n) [n.drop_tree() for n in to_drop if n.getparent() is not None] return node
def test_scores_only_once(): """Scoring is not cumulative within a class hit.""" test_div = '<div class="post main">Content</div>' node = fragment_fromstring(test_div) assert get_class_weight(node) == 25
def test_no_matches_zero(): """If you don't have the attribute then you get a weight of 0""" node = fragment_fromstring("<div></div>") assert get_class_weight(node) == 0
def test_no_matches_zero(self): """If you don't have the attribute then you get a weight of 0""" node = fragment_fromstring("<div></div>") self.assertEqual(get_class_weight(node), 0)
def clean_conditionally(node): """Remove the clean_el if it looks like bad content based on rules.""" target_tags = ["form", "table", "ul", "div", "p"] if node.tag not in target_tags: # this is not the tag you're looking for return weight = get_class_weight(node) # content_score = LOOK up the content score for this node we found # before else default to 0 content_score = 0 if weight + content_score < 0: LNODE.log(node, 2, "Dropping conditional node") return True if node.text_content().count(",") < 10: LOG.debug("There aren't 10 ,s so we're processing more") # If there are not very many commas, and the number of # non-paragraph elements is more than paragraphs or other ominous # signs, remove the element. p = len(node.findall(".//p")) img = len(node.findall(".//img")) li = len(node.findall(".//li")) - 100 inputs = len(node.findall(".//input")) embed = 0 embeds = node.findall(".//embed") for e in embeds: if ok_embedded_video(e): embed += 1 link_density = get_link_density(node) content_length = len(node.text_content()) remove_node = False if img > p: # this one has shown to do some extra image removals. # we could get around this by checking for caption info in the # images to try to do some scoring of good v. bad images. # failing example: # arstechnica.com/science/news/2012/05/1859s-great-auroral-stormthe-week-the-sun-touched-the-earth.ars LNODE.log(node, 2, "Conditional drop: img > p") remove_node = True elif li > p and node.tag != "ul" and node.tag != "ol": LNODE.log(node, 2, "Conditional drop: li > p and not ul/ol") remove_node = True elif inputs > p / 3.0: LNODE.log(node, 2, "Conditional drop: inputs > p/3.0") remove_node = True elif content_length < 25 and (img == 0 or img > 2): LNODE.log(node, 2, "Conditional drop: len < 25 and 0/>2 images") remove_node = True elif weight < 25 and link_density > 0.2: LNODE.log(node, 2, "Conditional drop: weight small and link is dense") remove_node = True elif weight >= 25 and link_density > 0.5: LNODE.log(node, 2, "Conditional drop: weight big but link heavy") remove_node = True elif (embed == 1 and content_length < 75) or embed > 1: LNODE.log(node, 2, "Conditional drop: embed without much content or many embed") remove_node = True return remove_node # nope, don't remove anything return False
def clean_document(node): """Clean up the final document we return as the readable article""" LOG.debug("Cleaning document") clean_list = ["object", "h1"] # If there is only one h2, they are probably using it as a header and # not a subheader, so remove it since we already have a header. if len(node.findall(".//h2")) == 1: LOG.debug("Adding H2 to list of nodes to clean.") clean_list.append("h2") for n in node.iter(): # clean out any incline style properties if "style" in n.attrib: n.set("style", "") # remove all of the following tags # Clean a node of all elements of type "tag". # (Unless it's a youtube/vimeo video. People love movies.) is_embed = True if n.tag in ["object", "embed"] else False if n.tag in clean_list: allow = False # Allow youtube and vimeo videos through as people usually # want to see those. if is_embed: if ok_embedded_video(n): allow = True if not allow: LNODE.log(n, 2, "Dropping Node") n.drop_tree() # go on with next loop, this guy is gone continue if n.tag in ["h1", "h2", "h3", "h4"]: # clean headings # if the heading has no css weight or a high link density, # remove it if get_class_weight(n) < 0 or get_link_density(n) > 0.33: # for some reason we get nodes here without a parent if n.getparent() is not None: LNODE.log(n, 2, "Dropping <hX>, it's insignificant") n.drop_tree() # go on with next loop, this guy is gone continue # clean out extra <p> if n.tag == "p": # if the p has no children and has no content...well then down # with it. if not n.getchildren() and len(n.text_content()) < 5: LNODE.log(n, 2, "Dropping extra <p>") n.drop_tree() # go on with next loop, this guy is gone continue # finally try out the conditional cleaning of the target node if clean_conditionally(n): # For some reason the parent is none so we can't drop, we're # not in a tree that can take dropping this node. if n.getparent() is not None: n.drop_tree() return node
def clean_conditionally(node): """Remove the clean_el if it looks like bad content based on rules.""" target_tags = ['form', 'table', 'ul', 'div', 'p'] LNODE.log(node, 2, 'Cleaning conditionally node.') if node.tag not in target_tags: # this is not the tag you're looking for LNODE.log(node, 2, 'Node cleared.') return weight = get_class_weight(node) # content_score = LOOK up the content score for this node we found # before else default to 0 content_score = 0 if (weight + content_score < 0): LNODE.log(node, 2, 'Dropping conditional node') LNODE.log(node, 2, 'Weight + score < 0') return True if node.text_content().count(',') < 10: LOG.debug("There aren't 10 ,s so we're processing more") # If there are not very many commas, and the number of # non-paragraph elements is more than paragraphs or other ominous # signs, remove the element. p = len(node.findall('.//p')) img = len(node.findall('.//img')) li = len(node.findall('.//li')) - 100 inputs = len(node.findall('.//input')) embed = 0 embeds = node.findall('.//embed') for e in embeds: if ok_embedded_video(e): embed += 1 link_density = get_link_density(node) content_length = len(node.text_content()) remove_node = False if li > p and node.tag != 'ul' and node.tag != 'ol': LNODE.log(node, 2, 'Conditional drop: li > p and not ul/ol') remove_node = True elif inputs > p / 3.0: LNODE.log(node, 2, 'Conditional drop: inputs > p/3.0') remove_node = True elif content_length < 25 and (img == 0 or img > 2): LNODE.log(node, 2, 'Conditional drop: len < 25 and 0/>2 images') remove_node = True elif weight < 25 and link_density > 0.2: LNODE.log(node, 2, 'Conditional drop: weight small and link is dense') remove_node = True elif weight >= 25 and link_density > 0.5: LNODE.log(node, 2, 'Conditional drop: weight big but link heavy') remove_node = True elif (embed == 1 and content_length < 75) or embed > 1: LNODE.log( node, 2, 'Conditional drop: embed w/o much content or many embed') remove_node = True if remove_node: LNODE.log(node, 2, 'Node will be removed') else: LNODE.log(node, 2, 'Node cleared') return remove_node # nope, don't remove anything LNODE.log(node, 2, 'Node Cleared final.') return False
def test_scores_only_once(self): """Scoring is not cumulative within a class hit.""" test_div = '<div class="post main">Content</div>' node = fragment_fromstring(test_div) self.assertEqual(get_class_weight(node), 25)
def clean_conditionally(node): """Remove the clean_el if it looks like bad content based on rules.""" target_tags = ['form', 'table', 'ul', 'div', 'p'] LNODE.log(node, 2, 'Cleaning conditionally node.') if node.tag not in target_tags: # this is not the tag you're looking for LNODE.log(node, 2, 'Node cleared.') return weight = get_class_weight(node) # content_score = LOOK up the content score for this node we found # before else default to 0 content_score = 0 if (weight + content_score < 0): LNODE.log(node, 2, 'Dropping conditional node') LNODE.log(node, 2, 'Weight + score < 0') return True if node.text_content().count(',') < 10: LOG.debug("There aren't 10 ,s so we're processing more") # If there are not very many commas, and the number of # non-paragraph elements is more than paragraphs or other ominous # signs, remove the element. p = len(node.findall('.//p')) img = len(node.findall('.//img')) li = len(node.findall('.//li')) - 100 inputs = len(node.findall('.//input')) embed = 0 embeds = node.findall('.//embed') for e in embeds: if ok_embedded_video(e): embed += 1 link_density = get_link_density(node) content_length = len(node.text_content()) remove_node = False if li > p and node.tag != 'ul' and node.tag != 'ol': LNODE.log(node, 2, 'Conditional drop: li > p and not ul/ol') remove_node = True elif inputs > p / 3.0: LNODE.log(node, 2, 'Conditional drop: inputs > p/3.0') remove_node = True elif content_length < 25 and (img == 0 or img > 2): LNODE.log(node, 2, 'Conditional drop: len < 25 and 0/>2 images') remove_node = True elif weight < 25 and link_density > 0.2: LNODE.log(node, 2, 'Conditional drop: weight small and link is dense') remove_node = True elif weight >= 25 and link_density > 0.5: LNODE.log(node, 2, 'Conditional drop: weight big but link heavy') remove_node = True elif (embed == 1 and content_length < 75) or embed > 1: LNODE.log(node, 2, 'Conditional drop: embed w/o much content or many embed') remove_node = True if remove_node: LNODE.log(node, 2, 'Node will be removed') else: LNODE.log(node, 2, 'Node cleared') return remove_node # nope, don't remove anything LNODE.log(node, 2, 'Node Cleared final.') return False