def test_javascript_heavy_inline_monstrosity(self): scraper = JavaScriptScraper() request = Request('http://example.com/test.js') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'twitchplayspokemonfirered.html') with open(html_file_path, 'rb') as in_file: in_file.seek(0x147) shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertIn( 'http://cdn.bulbagarden.net/upload/archive/a/a4/' '20090718115357%21195Quagsire.png', inline_urls ) self.assertIn( 'http://www.google.com/url?q=http%3A%2F%2Fwww.reddit.com%2F' 'user%2FGoldenSandslash15&sa=D&sntz=1&' 'usg=AFQjCNElFBxZYdNm5mWoRSncf5tbdIJQ-A', linked_urls ) print('\n'.join(inline_urls)) print('\n'.join(linked_urls))
def test_xhtml_invalid(self): element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, '') response.body = Body() with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'xhtml_invalid.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual( { 'http://example.com/image.png', 'http://example.com/script.js', }, inline_urls ) self.assertEqual( { 'http://example.com/link' }, linked_urls )
def test_rss_as_html(self): element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, '') response.body = Body() response.fields['content-type'] = 'application/rss+xml' with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'rss.xml') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response) self.assertTrue(scrape_result) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertFalse( inline_urls ) self.assertEqual( { 'http://www.someexamplerssdomain.com/main.html', 'http://www.wikipedia.org/' }, linked_urls )
def test_html_not_quite_charset(self): element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, '') response.body = Body() with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'videogame_top.htm') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertIn( 'http://example.com/copyright_2001_2006_rtype.gif', inline_urls ) self.assertIn( 'http://www.geocities.jp/gamehouse_grindcrusher/', linked_urls )
def test_redirect_loop(self): http_client = MockHTTPClient() pool = RobotsTxtPool() client = RichClient(http_client, pool) session = MockRobotsTxtRichClientSession( client, Request.new('http://example.com') ) self.assertEqual(RobotsState.unknown, session._robots_state) for dummy in range(21): request = session.next_request self.assertTrue(request.url_info.url.endswith('robots.txt')) response = Response('HTTP/1.0', 302, 'See else') response.url_info = request.url_info response.fields['location'] = '/robots.txt' http_client.response = response yield session.fetch() request = session.next_request self.assertTrue(request) response = Response('HTTP/1.0', 200, 'OK') http_client.response = response yield session.fetch() self.assertEqual(RobotsState.ok, session._robots_state) print(session.next_request) self.assertTrue(session.done)
def test_html_krokozyabry(self): scraper = HTMLScraper() request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, '') response.fields['content-type'] = 'text/html; charset=KOI8-R' with wpull.util.reset_file_offset(response.body.content_file): html_file_path = os.path.join(os.path.dirname(__file__), 'testing', 'samples', 'krokozyabry.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body.content_file) scrape_info = scraper.scrape(request, response) inline_urls = scrape_info['inline_urls'] linked_urls = scrape_info['linked_urls'] self.assertEqual('koi8-r', scrape_info['encoding']) self.assertEqual( set(), inline_urls ) self.assertEqual( {'http://example.com/Кракозябры'}, linked_urls )
def test_javascript_heavy_inline_monstrosity(self): scraper = JavaScriptScraper() request = Request('http://example.com/test.js') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'twitchplayspokemonfirered.html') with open(html_file_path, 'rb') as in_file: in_file.seek(0x147) shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertIn( 'http://cdn.bulbagarden.net/upload/archive/a/a4/' '20090718115357%21195Quagsire.png', inline_urls) self.assertIn( 'http://www.google.com/url?q=http%3A%2F%2Fwww.reddit.com%2F' 'user%2FGoldenSandslash15&sa=D&sntz=1&' 'usg=AFQjCNElFBxZYdNm5mWoRSncf5tbdIJQ-A', linked_urls) print('\n'.join(inline_urls)) print('\n'.join(linked_urls))
def test_sitemap_scraper_xml_index(self): scraper = SitemapScraper(self.get_html_parser()) request = Request('http://example.com/sitemap.xml') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write(b'''<?xml version="1.0" encoding="UTF-8"?> <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <sitemap> <loc>http://www.example.com/sitemap1.xml.gz</loc> <lastmod>2004-10-01T18:23:17+00:00</lastmod> </sitemap> </sitemapindex> ''') scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual({ 'http://www.example.com/sitemap1.xml.gz', }, linked_urls) self.assertFalse(inline_urls)
def test_rss_as_html(self): scraper = HTMLScraper() request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, '') response.fields['content-type'] = 'application/rss+xml' with wpull.util.reset_file_offset(response.body.content_file): html_file_path = os.path.join(os.path.dirname(__file__), 'testing', 'samples', 'rss.xml') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body.content_file) scrape_info = scraper.scrape(request, response) self.assertTrue(scrape_info) inline_urls = scrape_info['inline_urls'] linked_urls = scrape_info['linked_urls'] self.assertFalse( inline_urls ) self.assertEqual( { 'http://www.someexamplerssdomain.com/main.html', 'http://www.wikipedia.org/' }, linked_urls )
def test_sitemap_scraper_xml_index(self): scraper = SitemapScraper(self.get_html_parser()) request = Request('http://example.com/sitemap.xml') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write( b'''<?xml version="1.0" encoding="UTF-8"?> <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <sitemap> <loc>http://www.example.com/sitemap1.xml.gz</loc> <lastmod>2004-10-01T18:23:17+00:00</lastmod> </sitemap> </sitemapindex> ''' ) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual({ 'http://www.example.com/sitemap1.xml.gz', }, linked_urls ) self.assertFalse(inline_urls)
def test_response(self): response = Response(200, 'OK') response.fields['Cake'] = 'dolphin' self.assertEqual((b'HTTP/1.1 200 OK\r\n' b'Cake: dolphin\r\n' b'\r\n'), response.to_bytes())
def test_sitemap_scraper_xml(self): scraper = SitemapScraper(self.get_html_parser()) request = Request('http://example.com/sitemap.xml') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write( b'''<?xml version="1.0" encoding="UTF-8"?> <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <url> <loc>http://www.example.com/</loc> <lastmod>2005-01-01</lastmod> <changefreq>monthly</changefreq> <priority>0.8</priority> </url> </urlset> ''' ) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual({ 'http://www.example.com/', }, linked_urls ) self.assertFalse(inline_urls)
def test_html_krokozyabry(self): element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, '') response.body = Body() response.fields['content-type'] = 'text/html; charset=KOI8-R' with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'krokozyabry.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual('koi8-r', scrape_result.encoding) self.assertEqual( set(), inline_urls ) self.assertEqual( {'http://example.com/Кракозябры'}, linked_urls )
def test_html_soup(self): scraper = HTMLScraper() request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, '') response.fields['Refresh'] = 'yes' with wpull.util.reset_file_offset(response.body.content_file): html_file_path = os.path.join(os.path.dirname(__file__), 'testing', 'samples', 'soup.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body.content_file) scrape_info = scraper.scrape(request, response) inline_urls = scrape_info['inline_urls'] linked_urls = scrape_info['linked_urls'] self.assertEqual( {'http://example.com/ABOUTM~1.JPG'}, inline_urls ) self.assertEqual( { 'http://example.com/BLOG', 'http://example.com/web ring/Join.htm', }, linked_urls )
def test_html_soup(self): element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, '') response.body = Body() response.fields['Refresh'] = 'yes' with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'soup.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual( {'http://example.com/ABOUTM~1.JPG'}, inline_urls ) self.assertEqual( { 'http://example.com/BLOG', 'http://example.com/web ring/Join.htm', }, linked_urls )
def test_fetch_allow(self): http_client = MockHTTPClient() pool = RobotsTxtPool() client = RichClient(http_client, pool) session = MockRobotsTxtRichClientSession( client, Request.new('http://example.com')) self.assertEqual(RobotsState.unknown, session._robots_state) request = session.next_request self.assertTrue(request.url_info.url.endswith('robots.txt')) response = Response('HTTP/1.0', 200, 'OK') response.body.content_file = io.StringIO('User-agent:*\nAllow: /\n') http_client.response = response yield session.fetch() self.assertEqual(RobotsState.ok, session._robots_state) request = session.next_request self.assertTrue(request.url_info.url.endswith('/')) response = Response('HTTP/1.0', 200, 'OK') http_client.response = response yield session.fetch() self.assertTrue(session.done)
def test_sitemap_scraper_xml(self): scraper = SitemapScraper(self.get_html_parser()) request = Request('http://example.com/sitemap.xml') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write(b'''<?xml version="1.0" encoding="UTF-8"?> <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <url> <loc>http://www.example.com/</loc> <lastmod>2005-01-01</lastmod> <changefreq>monthly</changefreq> <priority>0.8</priority> </url> </urlset> ''') scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual({ 'http://www.example.com/', }, linked_urls) self.assertFalse(inline_urls)
def get_http_header(self): '''Return the HTTP header. It only attempts to read the first 4 KiB of the payload. Returns: Response, None: Returns an instance of :class:`.http.request.Response` or None. ''' with wpull.util.reset_file_offset(self.block_file): data = self.block_file.read(4096) match = re.match(br'(.*?\r?\n\r?\n)', data) if not match: return status_line, dummy, field_str = match.group(1).partition(b'\n') try: version, code, reason = Response.parse_status_line(status_line) except ValueError: return response = Response(version, code, reason) try: response.fields.parse(field_str, strict=False) except ValueError: return return response
def test_http_response(self): response = Response(200, 'OK', version='HTTP/1.0') response.fields['hello'] = 'world' new_response = HTTPResponseInfoWrapper(response) info = new_response.info() self.assertEqual('world', info.get('hello'))
def test_response(self): response = Response(200, 'OK') response.fields['Cake'] = 'dolphin' self.assertEqual( (b'HTTP/1.1 200 OK\r\n' b'Cake: dolphin\r\n' b'\r\n'), response.to_bytes() )
def test_xml_detect(self): self.assertTrue( XMLDetector.is_file(io.BytesIO( '<?xml version='.encode('utf-16le')))) self.assertFalse( XMLDetector.is_file( io.BytesIO('<!DOCTYPE html><html><body>'.encode('utf-16le')))) self.assertFalse(XMLDetector.is_file(io.BytesIO(b'<html><body>hello'))) self.assertTrue(XMLDetector.is_file(io.BytesIO(b'<?xml version'))) self.assertTrue( XMLDetector.is_url(URLInfo.parse('example.com/index.xml'))) self.assertFalse( XMLDetector.is_url(URLInfo.parse('example.com/image.jpg'))) self.assertTrue( XMLDetector.is_request(Request('example.com/index.xml'))) self.assertFalse( XMLDetector.is_request(Request('example.com/image.jpg'))) response = Response(200, 'OK') response.fields['Content-Type'] = 'text/xml' self.assertTrue(XMLDetector.is_response(response)) response = Response(200, 'OK') response.fields['Content-Type'] = 'application/xml' self.assertTrue(XMLDetector.is_response(response)) response = Response(200, 'OK') response.fields['Content-Type'] = 'image/png' self.assertFalse(XMLDetector.is_response(response))
def test_to_dict_body(self): request = Request() request.body = Body() request_dict = request.to_dict() self.assertTrue(request_dict['body']) request.body.close() request = Request() request.body = NotImplemented request_dict = request.to_dict() self.assertFalse(request_dict['body']) response = Response() response.body = Body() response_dict = response.to_dict() self.assertTrue(response_dict['body']) response.body.close() response = Response() response.body = NotImplemented response_dict = response.to_dict() self.assertFalse(response_dict['body'])
def test_html_detect(self): self.assertTrue(HTMLReader.is_file( io.BytesIO('<html><body>hi</body></html>'.encode('utf-16le')) )) self.assertFalse(HTMLReader.is_file( io.BytesIO('hello world!'.encode('utf-16le')) )) self.assertTrue(HTMLReader.is_file( io.BytesIO(b'<title>hello</title>hi') )) self.assertTrue(HTMLReader.is_file( io.BytesIO(b'<html><body>hello') )) self.assertTrue(HTMLReader.is_file( io.BytesIO( b'The document has moved <a href="somewhere.html">here</a>' ) )) self.assertTrue( HTMLReader.is_url(URLInfo.parse('example.com/index.htm')) ) self.assertTrue( HTMLReader.is_url(URLInfo.parse('example.com/index.html')) ) self.assertTrue( HTMLReader.is_url(URLInfo.parse('example.com/index.dhtm')) ) self.assertTrue( HTMLReader.is_url(URLInfo.parse('example.com/index.xhtml')) ) self.assertTrue( HTMLReader.is_url(URLInfo.parse('example.com/index.xht')) ) self.assertFalse( HTMLReader.is_url(URLInfo.parse('example.com/image.jpg')) ) self.assertTrue( HTMLReader.is_request(Request.new('example.com/index.html')) ) self.assertFalse( HTMLReader.is_request(Request.new('example.com/image.jpg')) ) response = Response('HTTP/1.0', '200', 'OK') response.fields['Content-Type'] = 'text/html' self.assertTrue(HTMLReader.is_response(response)) response = Response('HTTP/1.0', '200', 'OK') response.fields['Content-Type'] = 'image/png' self.assertFalse(HTMLReader.is_response(response))
def test_warc_recorder_rollback(self): warc_filename = 'asdf.warc' warc_prefix = 'asdf' with open(warc_filename, 'wb') as warc_file: warc_file.write(b'a' * 10) warc_recorder = WARCRecorder( warc_prefix, params=WARCRecorderParams( compress=False, ) ) request = HTTPRequest('http://example.com/') request.address = ('0.0.0.0', 80) response = HTTPResponse(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write(b'KITTEH DOGE') with warc_recorder.session() as session: session.pre_request(request) session.request_data(request.to_bytes()) class BadRecord(WARCRecord): def __init__(self, original_record): super().__init__() self.block_file = original_record.block_file self.fields = original_record.fields def __iter__(self): for dummy in range(1000): yield b"where's my elephant?" raise OSError('Oops') session._child_session._request_record = \ BadRecord(session._child_session._request_record) original_offset = os.path.getsize(warc_filename) with self.assertRaises((OSError, IOError)): session.request(request) new_offset = os.path.getsize(warc_filename) self.assertEqual(new_offset, original_offset) self.assertFalse(os.path.exists(warc_filename + '-wpullinc')) _logger.debug('original offset {0}'.format(original_offset))
def test_html_serious_bad_encoding(self): scraper = HTMLScraper(encoding_override='utf8') request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, '') response.fields['content-type'] = 'text/html; charset=utf8' with wpull.util.reset_file_offset(response.body.content_file): html_file_path = os.path.join(os.path.dirname(__file__), 'testing', 'samples', 'xkcd_1_evil.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body.content_file) scrape_info = scraper.scrape(request, response) self.assertTrue(scrape_info)
def test_css_scraper_reject_type(self): scraper = CSSScraper() request = Request('http://example.com/styles.css') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'styles.css') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response, link_type=LinkType.html) self.assertFalse(scrape_result)
def test_warc_max_size_and_append(self): file_prefix = 'asdf' with open('asdf-00000.warc', 'w'): pass with open('asdf-00001.warc', 'w'): pass warc_recorder = WARCRecorder( file_prefix, params=WARCRecorderParams( compress=False, max_size=1, appending=True ), ) request = HTTPRequest('http://example.com/1') request.address = ('0.0.0.0', 80) response = HTTPResponse(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write(b'BLAH') with warc_recorder.session() as session: session.pre_request(request) session.request_data(request.to_bytes()) session.request(request) session.pre_response(response) session.response_data(response.to_bytes()) session.response_data(response.body.content()) session.response(response) warc_recorder.close() self.assertTrue(os.path.exists('asdf-00000.warc')) self.assertTrue(os.path.exists('asdf-00001.warc')) self.assertTrue(os.path.exists('asdf-00002.warc')) self.assertTrue(os.path.exists('asdf-00003.warc')) self.assertTrue(os.path.exists('asdf-meta.warc')) self.assertEqual(0, os.path.getsize('asdf-00000.warc')) self.assertEqual(0, os.path.getsize('asdf-00001.warc')) self.assertNotEqual(0, os.path.getsize('asdf-00002.warc')) self.assertNotEqual(0, os.path.getsize('asdf-00003.warc')) self.assertNotEqual(0, os.path.getsize('asdf-meta.warc'))
def test_sitemap_scraper_invalid_robots(self): scraper = SitemapScraper(self.get_html_parser()) request = Request('http://example.com/robots.txt') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write( b'dsfju3wrji kjasSItemapsdmjfkl wekie;er :Ads fkj3m /Dk') scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertFalse(linked_urls) self.assertFalse(inline_urls)
def test_javascript_heavy_inline_monstrosity(self): scraper = HTMLScraper() request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, 'OK') with wpull.util.reset_file_offset(response.body.content_file): html_file_path = os.path.join(os.path.dirname(__file__), 'testing', 'samples', 'twitchplayspokemonfirered.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body.content_file) scrape_info = scraper.scrape(request, response) inline_urls = scrape_info['inline_urls'] linked_urls = scrape_info['linked_urls'] self.assertIn( 'http://cdn.bulbagarden.net/upload/archive/a/a4/' '20090718115357%21195Quagsire.png', inline_urls ) self.assertIn( 'http://www.google.com/url?q=http%3A%2F%2Fwww.reddit.com%2F' 'user%2FGoldenSandslash15&sa=D&sntz=1&' 'usg=AFQjCNElFBxZYdNm5mWoRSncf5tbdIJQ-A', linked_urls )
def test_sitemap_scraper_xml(self): scraper = SitemapScraper() request = Request.new('http://example.com/sitemap.xml') response = Response('HTTP/1.0', 200, 'OK') with wpull.util.reset_file_offset(response.body.content_file): response.body.content_file.write( b'''<?xml version="1.0" encoding="UTF-8"?> <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <url> <loc>http://www.example.com/</loc> <lastmod>2005-01-01</lastmod> <changefreq>monthly</changefreq> <priority>0.8</priority> </url> </urlset> ''' ) scrape_info = scraper.scrape(request, response) inline_urls = scrape_info['inline_urls'] linked_urls = scrape_info['linked_urls'] self.assertEqual({ 'http://www.example.com/', }, linked_urls ) self.assertFalse(inline_urls)
def test_sitemap_scraper_xml_index(self): scraper = SitemapScraper() request = Request.new('http://example.com/sitemap.xml') response = Response('HTTP/1.0', 200, 'OK') with wpull.util.reset_file_offset(response.body.content_file): response.body.content_file.write( b'''<?xml version="1.0" encoding="UTF-8"?> <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"> <sitemap> <loc>http://www.example.com/sitemap1.xml.gz</loc> <lastmod>2004-10-01T18:23:17+00:00</lastmod> </sitemap> </sitemapindex> ''' ) scrape_info = scraper.scrape(request, response) inline_urls = scrape_info['inline_urls'] linked_urls = scrape_info['linked_urls'] self.assertEqual({ 'http://www.example.com/sitemap1.xml.gz', }, linked_urls ) self.assertFalse(inline_urls)
def test_html_encoding_lxml_name_mismatch(self): '''It should accept encoding names with underscore.''' scraper = HTMLScraper() request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, '') response.fields['content-type'] = 'text/html; charset=EUC_KR' with wpull.util.reset_file_offset(response.body.content_file): response.body.content_file.write( '힖'.encode('euc_kr') ) scrape_info = scraper.scrape(request, response) self.assertTrue(scrape_info) self.assertEqual('euc_kr', scrape_info['encoding'])
def test_html_garbage(self): scraper = HTMLScraper() request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, '') response.fields['content-type'] = 'text/html' with wpull.util.reset_file_offset(response.body.content_file): response.body.content_file.write( b'\x01\x00\x01\x00l~Z\xff\x0f`y\x80\x00p<\x7f' b'\xffndo\xff\xff-\x83{d\xec</\xfe\x80\x00\xb4Bo' b'\x7f\xff\xff\xffV\xc1\xff\x7f\xff7' ) scrape_info = scraper.scrape(request, response) self.assertTrue(scrape_info)
def test_xhtml_invalid(self): scraper = HTMLScraper() request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, '') with wpull.util.reset_file_offset(response.body.content_file): html_file_path = os.path.join(os.path.dirname(__file__), 'testing', 'samples', 'xhtml_invalid.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body.content_file) scrape_info = scraper.scrape(request, response) inline_urls = scrape_info['inline_urls'] linked_urls = scrape_info['linked_urls'] self.assertEqual( { 'http://example.com/image.png', 'http://example.com/script.js', }, inline_urls ) self.assertEqual( { 'http://example.com/link' }, linked_urls )
def test_server_error(self): http_client = MockHTTPClient() pool = RobotsTxtPool() client = RichClient(http_client, pool) session = MockRobotsTxtRichClientSession( client, Request.new('http://example.com') ) self.assertEqual(RobotsState.unknown, session._robots_state) for dummy in range(21): request = session.next_request self.assertTrue(request.url_info.url.endswith('robots.txt')) response = Response('HTTP/1.0', 500, 'Opps') http_client.response = response yield session.fetch() request = session.next_request self.assertIsNone(request) try: yield session.fetch() except RobotsDenied: pass else: self.fail() self.assertTrue(session.done)
def test_html_scraper_links_base_href(self): scraper = HTMLScraper() request = Request.new('http://example.com/') response = Response('HTTP/1.0', 200, 'OK') with wpull.util.reset_file_offset(response.body.content_file): html_file_path = os.path.join(os.path.dirname(__file__), 'testing', 'samples', 'basehref.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body.content_file) scrape_info = scraper.scrape(request, response) inline_urls = scrape_info['inline_urls'] linked_urls = scrape_info['linked_urls'] self.assertEqual('utf-8', scrape_info['encoding']) self.assertEqual({ 'http://cdn.example.com/stylesheet1.css', 'http://www.example.com/stylesheet2.css', 'http://example.com/a/stylesheet3.css', 'http://example.com/a/dir/image1.png', 'http://example.com/dir/image2.png', 'http://example.net/image3.png', 'http://example.com/dir/image4.png', }, inline_urls ) self.assertEqual({ 'http://example.com/a/' }, linked_urls )
def test_javascript_reject_type(self): scraper = JavaScriptScraper() request = Request('http://example.com/script.js') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'script.js') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response, link_type=LinkType.css) self.assertFalse(scrape_result)
def test_response_empty_reason_line(self): response = Response() response.parse(b'HTTP/1.0 200\r\n') response.parse(b'Cake: dolphin\r\n') response.parse(b'\r\n') self.assertEqual(200, response.status_code) self.assertEqual('', response.reason) self.assertEqual('dolphin', response.fields['Cake'])
def test_sitemap_scraper_invalid_robots(self): scraper = SitemapScraper(self.get_html_parser()) request = Request('http://example.com/robots.txt') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write( b'dsfju3wrji kjasSItemapsdmjfkl wekie;er :Ads fkj3m /Dk' ) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertFalse(linked_urls) self.assertFalse(inline_urls)
def test_sitemap_scraper_robots(self): scraper = SitemapScraper(self.get_html_parser()) request = Request('http://example.com/robots.txt') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write(b'Sitemap: http://example.com/sitemap00.xml') scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual({ 'http://example.com/sitemap00.xml', }, linked_urls) self.assertFalse(inline_urls)
def test_warc_recorder_journal(self): warc_filename = 'asdf.warc' warc_prefix = 'asdf' warc_recorder = WARCRecorder( warc_prefix, params=WARCRecorderParams( compress=False, ) ) request = HTTPRequest('http://example.com/') request.address = ('0.0.0.0', 80) response = HTTPResponse(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): response.body.write(b'KITTEH DOGE') with warc_recorder.session() as session: session.pre_request(request) session.request_data(request.to_bytes()) test_instance = self class MockRecord(WARCRecord): def __init__(self, original_record): super().__init__() self.block_file = original_record.block_file self.fields = original_record.fields def __iter__(self): print(list(os.walk('.'))) test_instance.assertTrue( os.path.exists(warc_filename + '-wpullinc') ) for dummy in range(1000): yield b"where's my elephant?" session._child_session._request_record = \ MockRecord(session._child_session._request_record) session.request(request) self.assertFalse(os.path.exists(warc_filename + '-wpullinc'))
def test_html_scraper_reject_type(self): element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, 'OK') response.body = Body() with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'many_urls.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response, link_type=LinkType.css) self.assertFalse(scrape_result)
def test_redirect_tracker(self): tracker = RedirectTracker(5) self.assertFalse(tracker.is_redirect()) self.assertFalse(tracker.is_repeat()) self.assertFalse(tracker.exceeded()) self.assertFalse(tracker.next_location(raw=True)) self.assertEqual(0, tracker.count()) response = Response(200, 'OK') tracker.load(response) self.assertFalse(tracker.is_redirect()) self.assertFalse(tracker.is_repeat()) self.assertFalse(tracker.exceeded()) self.assertFalse(tracker.next_location()) self.assertEqual(0, tracker.count()) response = Response(303, 'See other') response.fields['location'] = '/test' tracker.load(response) self.assertTrue(tracker.is_redirect()) self.assertFalse(tracker.is_repeat()) self.assertFalse(tracker.exceeded()) self.assertEqual('/test', tracker.next_location(raw=True)) self.assertEqual(1, tracker.count()) response = Response(307, 'Temporary redirect') response.fields['location'] = '/test' tracker.load(response) tracker.load(response) tracker.load(response) tracker.load(response) tracker.load(response) self.assertTrue(tracker.is_redirect()) self.assertTrue(tracker.is_repeat()) self.assertTrue(tracker.exceeded()) self.assertEqual('/test', tracker.next_location(raw=True)) self.assertEqual(6, tracker.count())
def test_redirect_tracker(self): tracker = RedirectTracker(5) self.assertFalse(tracker.is_redirect()) self.assertFalse(tracker.is_repeat()) self.assertFalse(tracker.exceeded()) self.assertFalse(tracker.next_location(raw=True)) self.assertEqual(0, tracker.count()) response = Response('HTTP/1.1', 200, '') tracker.load(response) self.assertFalse(tracker.is_redirect()) self.assertFalse(tracker.is_repeat()) self.assertFalse(tracker.exceeded()) self.assertFalse(tracker.next_location()) self.assertEqual(0, tracker.count()) response = Response('HTTP/1.1', 303, '') response.fields['location'] = '/test' tracker.load(response) self.assertTrue(tracker.is_redirect()) self.assertFalse(tracker.is_repeat()) self.assertFalse(tracker.exceeded()) self.assertEqual('/test', tracker.next_location(raw=True)) self.assertEqual(1, tracker.count()) response = Response('HTTP/1.1', 307, '') response.fields['location'] = '/test' tracker.load(response) tracker.load(response) tracker.load(response) tracker.load(response) tracker.load(response) self.assertTrue(tracker.is_redirect()) self.assertTrue(tracker.is_repeat()) self.assertTrue(tracker.exceeded()) self.assertEqual('/test', tracker.next_location(raw=True)) self.assertEqual(6, tracker.count())
def test_html_encoding_lxml_name_mismatch(self): '''It should accept encoding names with underscore.''' element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, '') response.body = Body() response.fields['content-type'] = 'text/html; charset=EUC_KR' with wpull.util.reset_file_offset(response.body): response.body.write( '힖'.encode('euc_kr') ) scrape_info = scraper.scrape(request, response) self.assertTrue(scrape_info) self.assertEqual('euc_kr', scrape_info['encoding'])
def test_html_garbage(self): element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, '') response.body = Body() response.fields['content-type'] = 'text/html' with wpull.util.reset_file_offset(response.body): response.body.write( b'\x01\x00\x01\x00l~Z\xff\x0f`y\x80\x00p<\x7f' b'\xffndo\xff\xff-\x83{d\xec</\xfe\x80\x00\xb4Bo' b'\x7f\xff\xff\xffV\xc1\xff\x7f\xff7' ) scrape_info = scraper.scrape(request, response) self.assertTrue(scrape_info)
def test_html_wrong_charset(self): element_walker = ElementWalker( css_scraper=CSSScraper(), javascript_scraper=JavaScriptScraper()) scraper = HTMLScraper(self.get_html_parser(), element_walker) request = Request('http://example.com/') response = Response(200, '') response.body = Body() with wpull.util.reset_file_offset(response.body): html_file_path = os.path.join(ROOT_PATH, 'testing', 'samples', 'kcna.html') with open(html_file_path, 'rb') as in_file: shutil.copyfileobj(in_file, response.body) scrape_result = scraper.scrape(request, response) inline_urls = scrape_result.inline_links linked_urls = scrape_result.linked_links self.assertEqual('utf-16-le', scrape_result.encoding) self.assertEqual( { 'http://example.com/utm/__utm.js', 'http://example.com/Knewskage.gif', 'http://example.com/Lline.gif', 'http://example.com/Sline.gif', 'http://example.com/korean01.gif', 'http://example.com/korean02.gif', 'http://example.com/english01.gif', 'http://example.com/english02.gif', 'http://example.com/Tongsinkage.gif', 'http://example.com/Knewskage.gif', }, inline_urls ) self.assertEqual( { 'http://example.com/index-k.htm', 'http://example.com/index-e.htm', }, linked_urls )