def test_seen(): c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}} config.set_config(c) dl = datalayer.Datalayer() assert not dl.seen(URL('http://example.com')) dl.add_seen(URL('http://example.com')) assert dl.seen(URL('http://example.com'))
def test_robots(): ''' There's already end-to-end testing for the normal functionality. Exercise only the weird stuff here. ''' config.set_config({ 'Robots': { 'MaxTries': 4, 'MaxRobotsPageSize': 500000 }, 'Logging': {} }) # XXX really I should use the defaults in config.py so that I don't have # to edit the above as I add mandatory args r = robots.Robots('foo', None, None) robots_txt = b'<' plausible, message = r.is_plausible_robots('example.com', robots_txt, 1.0) assert not plausible assert len(message) robots_txt = b'' # application/x-empty plausible, message = r.is_plausible_robots('example.com', robots_txt, 1.0) assert plausible assert not len(message) robots_txt = b'x' * 1000001 plausible, message = r.is_plausible_robots('example.com', robots_txt, 1.0) assert not plausible assert len(message) robots_txt = b'foo' plausible, message = r.is_plausible_robots('example.com', robots_txt, 1.0) assert plausible assert not len(message)
def test_seen(): c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}} config.set_config(c) dl = datalayer.Datalayer() assert not dl.seen_url(URL('http://example.com')) dl.add_seen_url(URL('http://example.com')) assert dl.seen_url(URL('http://example.com'))
def test_robotscache(): c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}} config.set_config(c) dl = datalayer.Datalayer() with pytest.raises(KeyError): dl.read_robots_cache('http://example.com') dl.cache_robots('http://example.com', b'THIS IS A TEST') assert dl.read_robots_cache('http://example.com') == b'THIS IS A TEST'
def test_robotscache(): c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}} config.set_config(c) dl = datalayer.Datalayer() with pytest.raises(KeyError): dl.read_robots_cache('http://example.com') dl.cache_robots('http://example.com', b'THIS IS A TEST') assert dl.read_robots_cache('http://example.com') == b'THIS IS A TEST'
def test_summarize(capsys): c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}} config.set_config(c) dl = datalayer.Datalayer() dl.add_seen(URL('http://example.com')) dl.add_seen(URL('http://example2.com')) dl.summarize() out, err = capsys.readouterr() assert len(err) == 0 assert out.startswith('2 seen')
def test_summarize(capsys): c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}} config.set_config(c) dl = datalayer.Datalayer() dl.add_seen_url(URL('http://example.com')) dl.add_seen_url(URL('http://example2.com')) dl.summarize() out, err = capsys.readouterr() assert len(err) == 0 assert out.startswith('2 seen_urls')
def main(): ARGS = argparse.ArgumentParser(description='bench_burn benchmark for burner thread overhead') ARGS.add_argument('--threads', type=int, default=2) ARGS.add_argument('--workers', type=int, default=100) ARGS.add_argument('--datasize', type=int, default=10000) ARGS.add_argument('--affinity', action='store_true') ARGS.add_argument('--duration', type=float, default=0.010) ARGS.add_argument('--count', type=int, default=10000) args = ARGS.parse_args() c = {'Multiprocess': {'BurnerThreads': args.threads, 'Affinity': args.affinity}} config.set_config(c) global b b = burner.Burner('parser') for _ in range(args.count): queue.put_nowait((args.duration, 'x' * args.datasize)) print('args are', args) print('Processing {} items of size {} kbytes and {:.3f} seconds of burn using {} burner threads'.format( args.count, int(args.datasize/1000), args.duration, args.threads)) t0 = time.time() c0 = time.process_time() try: loop.run_until_complete(crawl()) except KeyboardInterrupt: sys.stderr.flush() print('\nInterrupt. Exiting.\n') finally: loop.stop() loop.run_forever() loop.close() elapsed = time.time() - t0 print('Elapsed time is {:.1f} seconds.'.format(elapsed)) expected = args.count * args.duration / args.threads print('Expected is {:.1f} seconds.'.format(expected)) print('Burner-side overhead is {}% or {:.4f} seconds per call'.format( int((elapsed - expected)/expected*100), (elapsed - expected)/args.count)) celapsed = time.process_time() - c0 print('Main-thread overhead is {}%, {:.4f} seconds per call, {} calls per cpu-second'.format( int(celapsed/elapsed*100), celapsed/args.count, int(args.count/celapsed)))
def test_saveload(): tf = tempfile.NamedTemporaryFile(delete=False) name = tf.name c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}} config.set_config(c) dl = datalayer.Datalayer() dl.add_seen(URL('http://example.com')) assert dl.seen(URL('http://example.com')) with open(name, 'wb') as f: dl.save(f) dl.add_seen(URL('http://example2.com')) with open(name, 'rb') as f: dl.load(f) assert dl.seen(URL('http://example.com')) assert not dl.seen(URL('http://example2.com')) os.unlink(name) assert not os.path.exists(name)
def test_saveload(): tf = tempfile.NamedTemporaryFile(delete=False) name = tf.name c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}} config.set_config(c) dl = datalayer.Datalayer() dl.add_seen_url(URL('http://example.com')) assert dl.seen_url(URL('http://example.com')) with open(name, 'wb') as f: dl.save(f) dl.add_seen_url(URL('http://example2.com')) with open(name, 'rb') as f: dl.load(f) assert dl.seen_url(URL('http://example.com')) assert not dl.seen_url(URL('http://example2.com')) os.unlink(name) assert not os.path.exists(name)
import os import sys import logging import functools import asyncio import cocrawler.burner as burner import cocrawler.parse as parse import cocrawler.stats as stats import cocrawler.config as config c = {'Multiprocess': {'BurnerThreads': 2}} config.set_config(c) loop = asyncio.get_event_loop() b = burner.Burner('parser') queue = asyncio.Queue() def parse_all(name, string): links1, _ = parse.find_html_links(string) links2, embeds2 = parse.find_html_links_and_embeds(string) all2 = links2.union(embeds2) if len(links1) != len(all2): print('{} had different link counts of {} and {}'.format( name, len(links1), len(all2))) extra1 = links1.difference(all2) extra2 = all2.difference(links1) print(' extra in links: {!r}'.format(extra1))
import os import sys import logging import functools import asyncio import cocrawler.burner as burner import cocrawler.parse as parse import cocrawler.stats as stats import cocrawler.config as config c = {'Multiprocess': {'BurnerThreads': 2}} config.set_config(c) loop = asyncio.get_event_loop() b = burner.Burner('parser') queue = asyncio.Queue() def parse_all(name, string): links1, _ = parse.find_html_links(string) links2, embeds2 = parse.find_html_links_and_embeds(string) all2 = links2.union(embeds2) if len(links1) != len(all2): print('{} had different link counts of {} and {}'.format(name, len(links1), len(all2))) extra1 = links1.difference(all2) extra2 = all2.difference(links1) print(' extra in links: {!r}'.format(extra1)) print(' extra in links and embeds: {!r}'.format(extra2))
def test_useragent(): c = { 'UserAgent': { 'Style': 'crawler', 'MyPrefix': 'something', 'URL': 'http://example.com/cocrawler.html' } } config.set_config(c) version = '1.0' robotname, ua = useragent.useragent(version) assert version in ua assert 'http://example.com/cocrawler.html' in ua assert robotname == 'something-cocrawler' config.write('laptopplus', 'UserAgent', 'Style') robotname, ua = useragent.useragent(version) assert 'Mozilla/5.0' in ua config.write('tabletplus', 'UserAgent', 'Style') robotname, ua = useragent.useragent(version) assert 'Mozilla/5.0' in ua config.write('phoneplus', 'UserAgent', 'Style') robotname, ua = useragent.useragent(version) assert 'Mozilla/5.0' in ua config.set_config(c) config.write('error', 'UserAgent', 'Style') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version) config.set_config(c) config.write('ha ha I left this off', 'UserAgent', 'URL') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version) config.set_config(c) config.write('http://cocrawler.com/cocrawler.html', 'UserAgent', 'URL') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version) config.set_config(c) config.write('test', 'UserAgent', 'MyPrefix') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version) config.set_config(c) config.write('', 'UserAgent', 'MyPrefix') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version)
def test_useragent(): c = {'UserAgent': {'Style': 'crawler', 'MyPrefix': 'something', 'URL': 'http://example.com/cocrawler.html'}} config.set_config(c) version = '1.0' robotname, ua = useragent.useragent(version) assert version in ua assert 'http://example.com/cocrawler.html' in ua assert robotname == 'something-cocrawler' config.write('laptopplus', 'UserAgent', 'Style') robotname, ua = useragent.useragent(version) assert 'Mozilla/5.0' in ua config.write('tabletplus', 'UserAgent', 'Style') robotname, ua = useragent.useragent(version) assert 'Mozilla/5.0' in ua config.write('phoneplus', 'UserAgent', 'Style') robotname, ua = useragent.useragent(version) assert 'Mozilla/5.0' in ua config.set_config(c) config.write('error', 'UserAgent', 'Style') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version) config.set_config(c) config.write('ha ha I left this off', 'UserAgent', 'URL') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version) config.set_config(c) config.write('http://cocrawler.com/cocrawler.html', 'UserAgent', 'URL') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version) config.set_config(c) config.write('test', 'UserAgent', 'MyPrefix') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version) config.set_config(c) config.write('', 'UserAgent', 'MyPrefix') with pytest.raises(ValueError): robotname, ua = useragent.useragent(version)