def test_speed(parser_type, test_data_dir):
stream_items = []
for i in xrange(10):
stream_item = StreamItem()
stream_item.body = ContentItem()
path = os.path.join(test_data_dir, 'test' )
stream_item.body.clean_html = open(
os.path.join(path, 'nytimes-index-clean.html')).read()
stream_items.append( stream_item )
context = {}
start = time.time()
hl = hyperlink_labels(config={
'require_abs_url': True,
'all_domains': False,
'domain_substrings': ['nytimes.com'],
'offset_types': [parser_type],
})
for si in stream_items:
si = hl(si, context)
elapsed = time.time() - start
rate = len(stream_items) / elapsed
logger.debug('OffsetType: {}'.format(OffsetType))
logger.info('{:.1f} per second for {}'.format(rate, parser_type))
def test_target_parsing(test_data_dir):
path = os.path.join(test_data_dir, 'test')
test_html = open(os.path.join(path, 'target-test.html')).read()
html = make_clean_html( test_html )
assert 'logo' in html
assert 'target' in html
visible = make_clean_visible( html )
assert 'logo' not in visible
assert 'target' not in visible
stage = hyperlink_labels(config={
'offset_types': ['LINES'],
'require_abs_url': True,
'all_domains': True,
})
si = StreamItem(body=ContentItem(clean_html=html))
context = {}
stage( si, context )
html2 = si.body.clean_html
visible2 = make_clean_visible( html2 )
#print visible2
assert 'target' not in visible2
assert 'logo' not in visible2
def test_stage(test_data_dir):
stage = clean_html({}) # NB: not even defaults
path = os.path.join(test_data_dir, 'test')
with open(os.path.join(path, 'nytimes-index.html'), 'r') as f:
raw = f.read().decode('utf8')
si = StreamItem(body=ContentItem(raw=raw, media_type='text/html'))
si = stage(si, {})
with open(os.path.join(path, 'nytimes-index-clean-stable.html'), 'r') as f:
stable = f.read()
assert si.body.clean_html == stable
def test_long_doc(parser_type, test_data_dir):
stream_item = StreamItem()
stream_item.body = ContentItem()
path = os.path.join(test_data_dir, 'test' )
stream_item.body.clean_html = open(
os.path.join(path, 'company-test.html')).read()
context = {}
hl = hyperlink_labels(config={
'require_abs_url': True,
'all_domains': True,
'offset_types': [parser_type],
})
hl(stream_item, context)
