def test_split_head_body(): ''' Whitebox test of the hueristics in this function ''' head, body = parse.split_head_body('x' * 100000) assert head == '' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<HeAd>' + 'x' * 100000) assert head == '' assert len(body) == 100007 head, body = parse.split_head_body('x' + '</HeAd>' + 'x' * 100000) assert head == 'x' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<BoDy>' + 'x' * 100000) assert head == 'x' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<heAd><boDy>' + 'x' * 100000) assert head == 'x<heAd>' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<hEad></heAd>' + 'x' * 100000) assert head == 'x<hEad>' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<heaD></Head><bOdy>' + 'x' * 100000) assert head == 'x<heaD>' assert len(body) == 100006
def test_split_head_body(): ''' Whitebox test of the heuristics in this function ''' head, body = parse.split_head_body('x'*100000) assert head == '' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<HeAd>' + 'x'*100000) assert head == '' assert len(body) == 100007 head, body = parse.split_head_body('x' + '</HeAd>' + 'x'*100000) assert head == 'x' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<BoDy>' + 'x'*100000) assert head == 'x' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<heAd><boDy>' + 'x'*100000) assert head == 'x<heAd>' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<hEad></heAd>' + 'x'*100000) assert head == 'x<hEad>' assert len(body) == 100000 head, body = parse.split_head_body('x' + '<heaD></Head><bOdy>' + 'x'*100000) assert head == 'x<heaD>' assert len(body) == 100006
def test_individual_parsers(): links, embeds = parse.find_html_links_re(test_html) assert len(links) == 6 assert len(embeds) == 0 assert 'foo2.htm' in links assert 'foo3.html ' in links assert 'foo.gif' in links assert 'torture"\n<url>' in links head, body = parse.split_head_body(test_html) links, embeds = parse.find_body_links_re(body) assert len(links) == 4 assert len(embeds) == 1 assert 'foo2.htm' in links assert 'foo3.html ' in links assert 'torture"\n<url>' in links assert 'foo.gif' in embeds head_soup = BeautifulSoup(head, 'lxml') links, embeds = parse.find_head_links_soup(head_soup) assert len(links) == 0 assert len(embeds) == 1 assert 'link.html' in embeds head_soup = BeautifulSoup(head, 'lxml') body_soup = BeautifulSoup(body, 'lxml') links, embeds = parse.find_head_links_soup(head_soup) lbody, ebody = parse.find_body_links_soup(body_soup) links.update(lbody) embeds.update(ebody) assert len(links) == 4 assert len(embeds) == 2 assert 'foo2.htm' in links assert 'foo3.html ' in links assert 'torture"\n<url>' in links assert 'link.html' in embeds assert 'foo.gif' in embeds head, body = parse.split_head_body(test_html_harder) body_soup = BeautifulSoup(body, 'lxml') lbody, ebody = parse.find_body_links_soup(body_soup) assert len(lbody) == 1 assert len(ebody) == 1 assert 'iframe.html' in lbody assert 'stylesheet.blah' in ebody
def test_individual_parsers(): links, embeds = parse.find_html_links_re(test_html) assert len(links) == 6 assert len(embeds) == 0 assert 'foo2.htm' in links assert 'foo3.html ' in links assert 'foo.gif' in links assert 'torture"\n<url>' in links head, body = parse.split_head_body(test_html) links, embeds = parse.find_body_links_re(body) assert len(links) == 4 assert len(embeds) == 1 assert 'foo2.htm' in links assert 'foo3.html ' in links assert 'torture"\n<url>' in links assert 'foo.gif' in embeds head_soup = BeautifulSoup(head, 'lxml') links, embeds = parse.find_head_links_soup(head_soup) assert len(links) == 0 assert len(embeds) == 1 assert 'link.html' in embeds head_soup = BeautifulSoup(head, 'lxml') body_soup = BeautifulSoup(body, 'lxml') links, embeds = parse.find_head_links_soup(head_soup) lbody, ebody = parse.find_body_links_soup(body_soup) links.update(lbody) embeds.update(ebody) assert len(links) == 4 assert len(embeds) == 2 assert 'foo2.htm' in links assert 'foo3.html ' in links assert 'torture"\n<url>' in links assert 'link.html' in embeds assert 'foo.gif' in embeds
def parse_all(name, string): all_links = [] # warmup head, body = parse.split_head_body(string) links, embeds = parse.find_html_links_re(string) # embeds is empty here by design links, embeds = parse.find_body_links_re(body) head_soup = BeautifulSoup(head, 'lxml') body_soup = BeautifulSoup(body, 'lxml') links, embeds = parse.find_head_links_soup(head_soup) links, embeds = parse.find_body_links_soup(body_soup) # measurement with stats.record_burn('split_head_body', url=name): head, body = parse.split_head_body(string) with stats.record_burn('find_html_links_re', url=name): links, embeds = parse.find_html_links_re(string) # embeds is empty here by design all_links.append(links.union(embeds)) with stats.record_burn('head_soup', url=name): head_soup = BeautifulSoup(head, 'lxml') with stats.record_burn('find_head_links_soup', url=name): head_links, head_embeds = parse.find_head_links_soup(head_soup) body = '<html>' + body # because a closing tag at the start of body screws up lxml with stats.record_burn('find_body_links_re', url=name): links, embeds = parse.find_body_links_re(body) all_links.append(links.union(embeds).union(head_links).union(head_embeds)) with stats.record_burn('body_soup', url=name): body_soup = BeautifulSoup(body, 'lxml') with stats.record_burn('find_body_links_soup', url=name): links, embeds = parse.find_body_links_soup(body_soup) all_links.append(links.union(embeds).union(head_links).union(head_embeds)) # evaluation biggest = functools.reduce(max, [len(x) for x in all_links]) for i, v in enumerate(all_links): if len(v) == biggest: biggest_index = i biggest_links = v names = 'find_html_links_re', 'find_body_links_re', 'find_body_links_soup' for i, v in enumerate(all_links): if len(v) != biggest: print('{} had different link counts of {} and {}'.format(name, biggest, len(v))) extra1 = v.difference(biggest_links) extra2 = biggest_links.difference(v) if extra1: print(' extra in {}: {!r}'.format(names[i], extra1)) else: print(' count was {} for {}'.format(len(v), names[i])) if extra2: print(' extra in {}: {!r}'.format(names[biggest_index], extra2)) else: print(' count was {} for {}'.format(len(biggest_links), names[biggest_index]))
def test_individual_parsers(): links, embeds = parse.find_html_links_re(test_html) assert len(links) == 6 assert len(embeds) == 0 linkset = set(parse.collapse_links(links)) assert 'foo2.htm' in linkset assert 'foo3.html ' in linkset assert 'foo.gif' in linkset assert 'torture"\n<url>' in linkset head, body = parse.split_head_body(test_html) links, embeds = parse.find_body_links_re(body) assert len(links) == 4 assert len(embeds) == 1 linkset = set(parse.collapse_links(links)) embedset = set(parse.collapse_links(embeds)) assert 'foo2.htm' in linkset assert 'foo3.html ' in linkset assert 'torture"\n<url>' in linkset assert 'foo.gif' in embedset links, embeds = parse.find_body_links_anchors_re(body) assert len(links) == 4 assert len(embeds) == 1 linkdict = dict([(l['href'], l['anchor']) for l in links]) # {('foo1.html', 'Anchor 1'), ('foo3.html ', 'Anchor 3'), ('foo2.htm', 'Anchor 2'), ('torture"\n<url>', 'torture\nanchor')} assert linkdict['foo2.htm'] == 'Anchor 2' assert linkdict['foo3.html '] == 'Anchor 3' assert linkdict['torture"\n<url>'] == 'torture\nanchor' assert 'foo.gif' in embeds[0]['src'] head_soup = BeautifulSoup(head, 'lxml') links, embeds = parse.find_head_links_soup(head_soup) embedset = set(parse.collapse_links(embeds)) assert len(links) == 0 assert len(embeds) == 1 assert 'link.html' in embedset head_soup = BeautifulSoup(head, 'lxml') body_soup = BeautifulSoup(body, 'lxml') links, embeds = parse.find_head_links_soup(head_soup) lbody, ebody = parse.find_body_links_soup(body_soup) links += lbody embeds += ebody linkset = set(parse.collapse_links(links)) embedset = set(parse.collapse_links(embeds)) assert len(links) == 4 assert len(embeds) == 2 assert 'foo2.htm' in linkset assert 'foo3.html ' in linkset assert 'torture"\n<url>' in linkset assert 'link.html' in embedset assert 'foo.gif' in embedset head, body = parse.split_head_body(test_html_harder) body_soup = BeautifulSoup(body, 'lxml') lbody, ebody = parse.find_body_links_soup(body_soup) assert len(lbody) == 1 assert len(ebody) == 1 assert 'iframe.html' == lbody[0]['src'] assert 'stylesheet.blah' == ebody[0]['href']
</head></head> <body> <a href="a-url">an-anchor</a> </body> </html> ''' all_soup = BeautifulSoup(html, 'lxml') print('all soup:', repr(all_soup)) # try to get the adblock key out print('got data-adblockkey of', all_soup.get('data-adblockkey')) # fails html_soup = all_soup.find('html') print('got data-adblockkey of', html_soup.get('data-adblockkey')) # works head, body = parse.split_head_body(html) print('head', head) head_soup = BeautifulSoup(head, 'lxml') print('head soup:', repr(head_soup)) print() print('body:', body) body_soup = BeautifulSoup(body, 'lxml') print('body soup:', repr(body_soup)) body = ''' <body> <a href="a-url">an-anchor</a> </body> </html>