Python process_response Examples

Example #1

 def test_process_response_imgs(self):
     host_url = 'http://www.davidcmoss.co.uk/'
     content = "<img src='test.png' />"
     response = Response()
     response.url = host_url
     response._content = content
     page = process_response(host_url, response)
     assert host_url + 'test.png' in page.images

Example #2

 def test_process_response_ex_links(self):
     host_url = 'http://www.davidcmoss.co.uk/'
     content = "<a href='http://google.com' />"
     response = Response()
     response.url = host_url
     response._content = content
     page = process_response(host_url, response)
     assert 'http://google.com' in page.ex_links

Example #3

 def test_process_response_links(self):
     host_url = 'http://www.davidcmoss.co.uk/'
     content = "<a href='/test' />"
     response = Response()
     response.url = host_url
     response._content = content
     page = process_response(host_url, response)
     assert host_url + 'test' in page.links

Example #4

File: tests.py Project: davidmoss/crawler

 def test_process_response_imgs(self):
     host_url = 'http://www.davidcmoss.co.uk/'
     content = "<img src='test.png' />"
     response = Response()
     response.url = host_url
     response._content = content
     page = process_response(host_url, response)
     assert host_url + 'test.png' in page.images

Example #5

File: tests.py Project: davidmoss/crawler

 def test_process_response_ex_links(self):
     host_url = 'http://www.davidcmoss.co.uk/'
     content = "<a href='http://google.com' />"
     response = Response()
     response.url = host_url
     response._content = content
     page = process_response(host_url, response)
     assert 'http://google.com' in page.ex_links

Example #6

File: tests.py Project: davidmoss/crawler

 def test_process_response_links(self):
     host_url = 'http://www.davidcmoss.co.uk/'
     content = "<a href='/test' />"
     response = Response()
     response.url = host_url
     response._content = content
     page = process_response(host_url, response)
     assert host_url + 'test' in page.links