def test_host_header_seted_in_request_headers(self):
def _test(response):
self.assertEquals(response.body, 'example.com')
self.assertEquals(request.headers.get('Host'), 'example.com')
request = Request(self.getURL('host'), headers={'Host': 'example.com'})
return download_http(request, BaseSpider()).addCallback(_test)
d = download_http(request, BaseSpider())
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, 'example.com')
return d
def test_payload(self):
body = '1'*100 # PayloadResource requires body length to be 100
request = Request(self.getURL('payload'), method='POST', body=body)
d = download_http(request, BaseSpider())
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, body)
return d
def test_host_header_not_in_request_headers(self):
def _test(response):
self.assertEquals(response.body, '127.0.0.1:%d' % self.portno)
self.assertEquals(request.headers, {})
request = Request(self.getURL('host'))
return download_http(request, BaseSpider()).addCallback(_test)
def test_download_without_proxy(self):
def _test(response):
self.assertEquals(response.status, 200)
self.assertEquals(response.url, request.url)
self.assertEquals(response.body, '/path/to/resource')
request = Request(self.getURL('path/to/resource'))
return download_http(request, BaseSpider()).addCallback(_test)
def test_download_with_proxy(self):
def _test(response):
self.assertEquals(response.status, 200)
self.assertEquals(response.url, request.url)
self.assertEquals(response.body, 'https://example.com')
http_proxy = self.getURL('')
request = Request('https://example.com', meta={'proxy': http_proxy})
return download_http(request, BaseSpider()).addCallback(_test)
def test_timeout_download_from_spider(self):
spider = BaseSpider()
spider.download_timeout = 0.000001
request = Request(self.getURL('wait'))
d = download_http(request, spider)
return self.assertFailure(d, defer.TimeoutError)
def test_redirect_status(self):
request = Request(self.getURL('redirect'))
d = download_http(request, BaseSpider())
d.addCallback(lambda r: r.status)
d.addCallback(self.assertEquals, 302)
return d
def test_download(self):
request = Request(self.getURL('file'))
d = download_http(request, BaseSpider())
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, "0123456789")
return d
def test_broken_download(self):
request = Request(self.getURL('broken'))
d = download_http(request, BaseSpider())
return self.assertFailure(d, PartialDownloadError)
def test_download_head(self):
request = Request(self.getURL('file'), method='HEAD')
d = download_http(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, '')
return d
