def test_find_urls_remove_subdomain(self):
self.url = 'http://www.example.com'
node = UrlNode(self.url)
body = '<a href="http://example.com/login/">login</a>'
self._mock_response(body)
node.process()
self.assertFalse(node.linked_urls)
def test_find_urls_different_subdomain(self):
self.url = 'http://www.example.com'
node = UrlNode(self.url)
body = '<a href="http://mail.example.com">e-mail</a>'
self._mock_response(body)
node.process()
self.assertFalse(node.linked_urls)
def test_find_static_script_relative(self):
self.url = 'http://example.com/folder/index.html'
node = UrlNode(self.url)
body = '<script src="../s.js"></script>'
self._mock_response(body)
node.process()
self.assertEqual(node.static_urls, {'http://example.com/s.js'})
def test_find_static_img_relative(self):
self.url = 'http://example.com/folder/index.html'
node = UrlNode(self.url)
body = '<link rel="stylesheet" href="/img.jpg">'
self._mock_response(body)
node.process()
self.assertEqual(node.static_urls, {'http://example.com/img.jpg'})
def test_find_urls_root_fragment(self):
self.url = 'http://www.example.com/#top'
node = UrlNode(self.url)
body = '<a name="bottom"></a><a href="#bottom">jump to top</a>'
# Manually add otherwise responses won't think we visited the page
# since the fragment is removed
self.responses.add(responses.GET,
'http://www.example.com/',
body=body,
status=200)
node.process()
self.assertFalse(node.linked_urls)
def test_strip_fragments(self):
self.node = UrlNode('https://example.com#top')
self.assertEqual(self.node.url, 'https://example.com')
def test_strip_trailing_slash(self):
self.node = UrlNode('https://example.com/')
self.assertEqual(self.node.url, 'https://example.com')
def setUp(self):
self.url = 'http://example.com'
self.node = UrlNode(self.url)