def test_extraction_encoding(self):
body = get_testdata('link_extractor', 'linkextractor_noenc.html')
response_utf8 = HtmlResponse(
url='http://example.com/utf8',
body=body,
headers={'Content-Type': ['text/html; charset=utf-8']})
response_noenc = HtmlResponse(url='http://example.com/noenc',
body=body)
body = get_testdata('link_extractor', 'linkextractor_latin1.html')
response_latin1 = HtmlResponse(url='http://example.com/latin1',
body=body)
lx = LxmlLinkExtractor(unique=False)
self.assertEqual(lx.extract_links(response_utf8), [
Link(url='http://example.com/sample_%C3%B1.html', text=''),
Link(url='http://example.com/sample_%E2%82%AC.html',
text='sample \xe2\x82\xac text'.decode('utf-8')),
])
self.assertEqual(lx.extract_links(response_noenc), [
Link(url='http://example.com/sample_%C3%B1.html', text=''),
Link(url='http://example.com/sample_%E2%82%AC.html',
text='sample \xe2\x82\xac text'.decode('utf-8')),
])
self.assertEqual(lx.extract_links(response_latin1), [
Link(url='http://example.com/sample_%F1.html', text=''),
Link(url='http://example.com/sample_%E1.html',
text='sample \xe1 text'.decode('latin1')),
])
def test_base_url(self):
html = '''<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
<body><p><a href="item/12.html">Item 12</a></p>
</body></html>'''
response = HtmlResponse('http://example.org/somepage/index.html',
body=html)
lx = LxmlLinkExtractor(unique=False)
self.assertEqual(lx.extract_links(response), [
Link(url='http://otherdomain.com/base/item/12.html',
text='Item 12')
])
# base url is an absolute path and relative to host
html = '''<html><head><title>Page title<title><base href="/" />
<body><p><a href="item/12.html">Item 12</a></p></body></html>'''
response = HtmlResponse('https://example.org/somepage/index.html',
body=html)
self.assertEqual(
lx.extract_links(response),
[Link(url='https://example.org/item/12.html', text='Item 12')])
# base url has no scheme
html = '''<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
<body><p><a href="item/12.html">Item 12</a></p></body></html>'''
response = HtmlResponse('https://example.org/somepage/index.html',
body=html)
self.assertEqual(lx.extract_links(response), [
Link(url='https://noschemedomain.com/path/to/item/12.html',
text='Item 12')
])
def test_max_redirect_times(self):
self.mw.max_redirect_times = 1
req = Request('http://test.org/max')
rsp = HtmlResponse(req.url, body=self._body(), request=req)
req = self.mw.process_response(rsp)
self.assertIsInstance(req, Request)
self.assertEqual(len(req.history), 1)
rsp.request = req
self.assertIsNone(self.mw.process_response(rsp))
def test_max_redirect_times(self):
self.mw.max_redirect_times = 1
req = Request('http://test.org/max')
rsp = HtmlResponse(req.url, body=self._body(), request=req)
req = self.mw.process_response(rsp)
self.assertIsInstance(req, Request)
self.assertEqual(len(req.history), 1)
rsp.request = req
self.assertIsNone(self.mw.process_response(rsp))
def test_nothing(self):
body = '''<html><head></head><body></body></html>'''
req = Request('http://a.com')
rsp = HtmlResponse(req.url, body=body, request=req)
rsp2 = self.mw.process_response(rsp)
self.assertIs(rsp, rsp2)
self.assertNotIn('canonical_url', rsp.meta)
def test_selector_nested(self):
'''Nested selector tests.'''
body = '''<body>
<div class='one'>
<ul>
<li>one</li><li>two</li>
</ul>
</div>
<div class='two'>
<ul>
<li>four</li><li>five</li><li>six</li>
</ul>
</div>
</body>'''
response = HtmlResponse(url='http://example.com"', body=body)
x = self.hxs_cls(response)
divtwo = x.select('//div[@class="two"]')
self.assertEqual(map(unicode.strip,
divtwo.select('//li').extract()),
[
'<li>one</li>', '<li>two</li>', '<li>four</li>',
'<li>five</li>', '<li>six</li>'
])
self.assertEqual(
map(unicode.strip,
divtwo.select('./ul/li').extract()),
['<li>four</li>', '<li>five</li>', '<li>six</li>'])
self.assertEqual(map(unicode.strip,
divtwo.select('.//li').extract()),
['<li>four</li>', '<li>five</li>', '<li>six</li>'])
self.assertEqual(divtwo.select('./li').extract(), [])
def test_deny_extensions(self):
html = '''<a href="page.html">asd</a> and <a href="photo.jpg">'''
response = HtmlResponse('http://example.org/', body=html)
lx = LxmlLinkExtractor()
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.org/page.html', text=u'asd'),
])
def test_meta_refresh_with_high_interval(self):
# meta-refresh with high intervals don't trigger redirects
req = Request(url='http://example.org')
rsp = HtmlResponse(url='http://example.org',
body=self._body(interval=1000),
request=req)
rsp2 = self.mw.process_response(rsp)
self.assertIs(rsp, rsp2)
def test_tag(self):
body = '''<html><head><link rel="canonical" href="%s" /></head></html>'''
# absolute url
req = Request('http://a.com/pom')
rsp = HtmlResponse(req.url,
body=body % 'https://b.sk/hello',
request=req)
rsp2 = self.mw.process_response(rsp)
self.assertIs(rsp, rsp2)
self.assertEqual(rsp.meta['canonical_url'], 'https://b.sk/hello')
# relative url
req = Request('http://a.com/pom')
rsp = HtmlResponse(req.url, body=body % '/hello/world', request=req)
rsp2 = self.mw.process_response(rsp)
self.assertIs(rsp, rsp2)
self.assertEqual(rsp.meta['canonical_url'], 'http://a.com/hello/world')
def test_encoded_url(self):
body = '''<html><body><div><a href="?page=2">BinB</a></body></html>'''
response = HtmlResponse("http://known.fm/AC%2FDC/",
body=body,
encoding='utf8')
lx = LxmlLinkExtractor()
self.assertEqual(lx.extract_links(response), [
Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB'),
])
def test_redirect_urls(self):
req1 = Request('http://test.org/first')
rsp1 = HtmlResponse(req1.url,
body=self._body(url='/redirected'),
request=req1)
req2 = self.mw.process_response(rsp1)
self.assertIsInstance(req2, Request)
rsp2 = HtmlResponse(req2.url,
body=self._body(url='/redirected2'),
request=req2)
req3 = self.mw.process_response(rsp2)
self.assertIsInstance(req3, Request)
self.assertEqual(req2.url, 'http://test.org/redirected')
self.assertListEqual(req2.history, ['http://test.org/first'])
self.assertEqual(req3.url, 'http://test.org/redirected2')
self.assertListEqual(
req3.history,
['http://test.org/first', 'http://test.org/redirected'])
def test_link_text_wrong_encoding(self):
html = '''<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>'''
response = HtmlResponse('http://www.example.com',
body=html,
encoding='utf-8')
lx = LxmlLinkExtractor(unique=False)
self.assertEqual(lx.extract_links(response), [
Link(url='http://www.example.com/item/12.html',
text=u'Wrong: \ufffd'),
])
def test_invalid(self):
# parser shouldn't fail or anything
html = '''<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" lang="cs">
<body>
</body>
</html>'''
response = HtmlResponse('http://www.example.com',
body=html,
encoding='utf-8')
lx = LxmlLinkExtractor(unique=False)
self.assertEqual(lx.extract_links(response), [])
def test_link_nofollow(self):
html = '''
<a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
<a href="about.html">About us</a>
'''
response = HtmlResponse('http://example.org/page.html', body=html)
lx = LxmlLinkExtractor()
self.assertEqual(lx.extract_links(response), [
Link(url='http://example.org/page.html?action=print',
text=u'Printer-friendly page',
nofollow=True),
Link(url='http://example.org/about.html',
text=u'About us',
nofollow=False)
])
def test_open_in_browser(self):
def browser_open(burl):
path = urlparse.urlparse(burl).path
if not os.path.exists(path):
path = burl.replace('file://', '')
bbody = open(path).read()
self.assertIn('<base href="%s">' % url, bbody,
'<base> tag not added')
return True
url = 'http:///www.example.com/some/page.html'
body = '<html> <head> <title>test page</title> </head> <body>test body</body> </html>'
response = HtmlResponse(url, body=body)
self.assertTrue(open_in_browser(response, _openfunc=browser_open),
'Browser not called')
self.assertRaises(TypeError, open_in_browser, Response(url, body=body))
def test_process_response_force_recalculate_encoding(self):
headers = {
'Content-Type': 'text/html',
'Content-Encoding': 'gzip',
}
f = StringIO()
plainbody = '''<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">'''
zf = GzipFile(fileobj=f, mode='wb')
zf.write(plainbody)
zf.close()
response = HtmlResponse('http;//www.example.com/page.html',
headers=headers,
body=f.getvalue())
new_response = self.mw.process_response(response)
self.assertIsInstance(new_response, HtmlResponse)
self.assertEqual(new_response.body, plainbody)
self.assertEqual(new_response.encoding, normalize_encoding('gb2312'))
def test_process_links(self):
def _process(link):
if link.url.endswith('othercat.html'):
return None
else:
link.url = 'http://gogo.com/'
return link
html = '''<html><head><title>Page title<title>
<p><a href="../othercat.html">Other category</a></p>
<p><a href="/">>></a></p>
<p><a href="/">mimi</a></p>
<p><a href="/hello">mimino</a></p>
</body></html>'''
response = HtmlResponse('http://example.org/', body=html)
lx = LxmlLinkExtractor()
self.assertEqual(lx.extract_links(response, process_links=_process),
[Link(url='http://gogo.com/', text=u'>>')])
def test_css(self):
body = get_testdata('pages', 'ip_page.html')
response = HtmlResponse(url='http://myip.com/list', body=body)
hxs = response.selector
valid_ts = S('_',
css='div#main',
quant='1',
children=[
S('all_ip', css='span.ip', quant='7'),
S('_',
css='ul#ip_list',
quant='1',
children=[S('list_ip', css='span.ip', quant='6')]),
])
parsed = valid_ts.parse(hxs)
self.assertRaises(TypeError, S, '_')
self.assertRaises(TypeError, S, '_', 'div[@id="main]', css='div#main')
def test_http_header_encoding_precedence(self):
# u'\xa3' = pound symbol in unicode
# u'\xc2\xa3' = pound symbol in utf-8
# u'\xa3' = pound symbol in latin-1 (iso-8859-1)
meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
head = u'<head>' + meta + u'</head>'
body_content = u'<span id="blank">\xa3</span>'
body = u'<body>' + body_content + u'</body>'
html = u'<html>' + head + body + u'</html>'
encoding = 'utf-8'
html_utf8 = html.encode(encoding)
headers = {'Content-Type': ['text/html; charset=utf-8']}
response = HtmlResponse(url='http://example.com',
headers=headers,
body=html_utf8)
x = self.hxs_cls(response)
self.assertEquals(
x.select('//span[@id="blank"]/text()').extract(), [u'\xa3'])
def test_relative_paths(self):
html = '''<html><head></head><body>
<a href="hello/world/"></a>
<a href="/hello/world/"></a>
<a href="/hello/././world"></a>
<a href="/hello/../world"></a>
<a href="../hello/world"></a>
<a href="./hello/world"></a>
<a href="../../hello/././world"></a>
</body></html>'''
response = HtmlResponse('http://exmaple.org/yay/', body=html)
lx = LxmlLinkExtractor(unique=False)
self.assertEqual([link.url for link in lx.extract_links(response)], [
'http://exmaple.org/yay/hello/world/',
'http://exmaple.org/hello/world/',
'http://exmaple.org/hello/world',
'http://exmaple.org/world',
'http://exmaple.org/hello/world',
'http://exmaple.org/yay/hello/world',
'http://exmaple.org/hello/world',
])
def test_meta_refresh_trough_posted_request(self):
req = Request(url='http://example.org',
method='POST',
body='test',
headers={
'Content-Type': 'text/plain',
'Content-length': '4'
})
rsp = HtmlResponse(req.url, body=self._body(), request=req)
req2 = self.mw.process_response(rsp)
self.assertIsInstance(req2, Request)
self.assertEqual(req2.url, 'http://example.org/newpage')
self.assertEqual(req2.method, 'GET')
self.assertNotIn(
'Content-Type', req2.headers,
'Content-Type header must not be present in redirected request')
self.assertNotIn(
'Content-Length', req2.headers,
'Content-Length header must not be present in redirected request')
self.assertNot(req2.body,
'Redirected body must be empty, not `%s`' % req2.body)
def test_basic(self):
html = '''<html><head><title>Page title<title>
<body><p><a href="item/12.html">Item 12</a></p>
<p><a href="/about.html">About us</a></p>
<img src="/logo.png" alt="Company logo (not a link)" />
<p><a href="../othercat.html">Other category</a></p>
<p><a href="/">>></a></p>
<p><a href="/" /></p>
</body></html>'''
response = HtmlResponse('http://example.org/somepage/index.html',
body=html)
lx = LxmlLinkExtractor(unique=False)
self.assertListEqual(lx.extract_links(response), [
Link(url='http://example.org/somepage/item/12.html',
text='Item 12'),
Link(url='http://example.org/about.html', text='About us'),
Link(url='http://example.org/othercat.html',
text='Other category'),
Link(url='http://example.org/', text='>>'),
Link(url='http://example.org/', text='')
])
def setUp(self):
self.resp = HtmlResponse('http://github.com/',
body='''<head>
<base href="http://www.w3schools.com/" target="_blank"></head>
<body></body>''')
def test_get_meta_refresh(self):
url = 'http://example.org'
body = '''
<html>
<head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
<body>blahablsdfsal&</body>
</html>'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(5, 'http://example.org/newpage'))
# refresh without url should return (None, None)
body = '''<meta http-equiv="refresh" content="5" />'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(None, None))
body = '''<meta http-equiv="refresh" content="5;
url=http://example.org/newpage" /></head>'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(5, 'http://example.org/newpage'))
# meta refresh in multiple lines
body = '''<html><head>
<META
HTTP-EQUIV="Refresh"
CONTENT="1; URL=http://example.org/newpage">'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(1, 'http://example.org/newpage'))
# entities in the redirect url
body = '''<meta http-equiv="refresh" content="3; url='http://www.example.com/other'">'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(3, 'http://www.example.com/other'))
url = 'http://example.com/page/this.html'
# relative redirects
body = '''<meta http-equiv="refresh" content="3; url=other.html">'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(3, 'http://example.com/page/other.html'))
# non-standard encodings (utf-16)
url = 'http://example.com'
body = '''<meta http-equiv="refresh" content="3; url=http://example.com/redirect">'''
body = body.decode('ascii').encode('utf-16')
self.assertEqual(
get_meta_refresh(HtmlResponse(url, body=body, encoding='utf-16')),
(3,
'http://example.com/%FF%FEh%00t%00t%00p%00:%00/%00/%00e%00x%00a%00m%00p%00l%00e%00.%00c%00o%00m%00/%00r%00e%00d%00i%00r%00e%00c%00t%00'
))
# non-ascii chars in the url (utf8 - default)
body = '''<meta http-equiv="refresh" content="3; url=http://example.com/to\xc2\xa3">'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(3, 'http://example.com/to%C2%A3'))
# non-ascii chars in the url (latin1)
body = '''<meta http-equiv="refresh" content="3; url=http://example.com/to\xa3">'''
self.assertEqual(
get_meta_refresh(HtmlResponse(url, body=body, encoding='latin1')),
(3, 'http://example.com/to%A3'))
# html commented meta refresh header must not directed
body = '''<!--<meta http-equiv="refresh" content="3; url=http://example.com/">-->'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(None, None))
# html comments must not interfere with uncommented meta refresh header
body = '''<!-- commented --><meta http-equiv="refresh" content="3; url=http://example.com/">-->'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(3, 'http://example.com/'))
# float refresh intervals
body = '''<meta http-equiv="refresh" content=".1;URL=index.html" />'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(0.1, 'http://example.com/index.html'))
body = '''<meta http-equiv="refresh" content="3.1;URL=index.html" />'''
self.assertEqual(get_meta_refresh(HtmlResponse(url, body=body)),
(3.1, 'http://example.com/index.html'))
def test_xpath(self):
body = get_testdata('pages', 'ip_page.html')
response = HtmlResponse(url='http://myip.com/list', body=body)
hxs = response.selector
# test valid parsing
valid_ts = S(
'_',
'//div[@id="main"]',
quant='1',
children=[
S('title', 'h1', quant='1', value='text()'),
S('full_title_script',
'h1|div[@id="subtitle"]/h2',
quant='2',
value='descendant-or-self::text()'),
# although the following statement gives the same elements, it gives them in different order
# S('full_title_script', '(h1|div[@id="subtitle"]/h2)/descendant-or-self::*', quant='+', value='text()'),
S('full_title_no_script',
'h1|div[@id="subtitle"]/h2',
quant='2',
value='descendant-or-self::*[name()!="script"]/text()'),
S('full_title_script_bad',
'(h1|div[@id="subtitle"]/h2)//*',
quant='+',
value='text()'),
S('_list',
'ul[@id="ip_list"]',
quant='1',
children=[
S('_ips',
'li',
quant='6',
group='ips',
children=[
S('ip',
'span[@class="ip"]',
quant='1',
value='text()'),
S('port',
'span[@class="port"]',
quant='1',
value='text()'),
S('ip_port',
'self::*',
value='descendant-or-self::text()'),
])
]),
S('url',
'descendant-or-self::a',
quant='1',
value='@href',
callback=S.absolute_url),
S('empty', 'div[@id="empty"]', quant='1', value='text()'),
S('footer',
'following-sibling::div[@id="footer"]',
quant='1',
children=[
S('footer_links',
'a',
quant='+',
value='@href',
callback=S.absolute_url)
]),
S('nonexistent', 'div/div/div', quant='?', value='text()')
])
# validation without context, when context is expected
self.assertRaises(SValidationError, valid_ts.parse, hxs)
parsed = valid_ts.parse(response)
self.assertItemsEqual(parsed, [
'title', 'full_title_script', 'full_title_no_script',
'full_title_script_bad', 'ips', 'url', 'empty', 'footer',
'footer_links'
]) # nonexistent is missing!
# title
self.assertIsInstance(parsed['title'], list)
self.assertListEqual(parsed['title'],
[u'Here is the list of some ', u' addresses '
]) # text inside strong is not pased
# full_title
self.assertListEqual(parsed['full_title_script'], [
u'Here is the list of some ', u'ip', u' addresses ', u'!!!',
u'Just ', u'some', u' ', u'this is bad', u' other text.'
]) # order of text nodes is perserved
self.assertListEqual(parsed['full_title_no_script'], [
u'Here is the list of some ', u'ip', u' addresses ', u'!!!',
u'Just ', u'some', u' ', u' other text.'
]) # same result as before, excluding script content
self.assertListEqual(parsed['full_title_script_bad'],
[u'ip', u'!!!', u'some', u'this is bad'
]) # this only took the inner nodes
# ips
self.assertIsInstance(parsed['ips'], list)
self.assertEqual(len(parsed['ips']), 6)
first = parsed['ips'][0]
self.assertIsInstance(first, defaultdict)
self.assertItemsEqual(first, ['ip', 'port', 'ip_port'])
self.assertListEqual(first['ip'], [u'123.44.1.9'])
self.assertIsInstance(first['ip'][0],
unicode) # parsed objects are always unicode
self.assertListEqual(first['port'], [u'80'])
self.assertListEqual(first['ip_port'], [u'123.44.1.9', u':', u'80'])
# url
self.assertListEqual(parsed['url'], [u'http://myip.com/url1'])
self.assertIsInstance(
parsed['url'][0],
unicode) # even urls are unicode after being processed
# empty
self.assertListEqual(
parsed['empty'], []
) # even though we matched 1 tag empty, the was no text and the returned list is empty
# footer
self.assertIsInstance(parsed['footer'][0], HtmlXPathSelector)
self.assertListEqual(parsed['footer_links'],
[u'http://myip.com/url2', u'http://google.com/'])
def test_priority_adjust(self):
req = Request('http://a.com')
rsp = HtmlResponse(req.url, body=self._body(), request=req)
req2 = self.mw.process_response(rsp)
self.assertTrue(req2.priority > req.priority)
def setUp(self):
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
self.response = HtmlResponse(url='http://example.com/index', body=body)
def test_meta_refresh(self):
req = Request(url='http://example.org')
rsp = HtmlResponse(req.url, body=self._body(), request=req)
req2 = self.mw.process_response(rsp)
self.assertIsInstance(req2, Request)
self.assertEqual(req2.url, 'http://example.org/newpage')
def setUp(self):
body = get_testdata('pages', 'ip_page.html')
response = HtmlResponse(url='http://myip.com/list', body=body)
hxs = response.selector
self.parsed = basic_ts.parse(hxs)
def test_empty_body(self):
lx = LxmlLinkExtractor()
response = HtmlResponse('http://www.example.com')
self.assertEqual(lx.extract_links(response), [])
response = HtmlResponse('http://www.example.com', body='\n\r\n\n')
self.assertEqual(lx.extract_links(response), [])