def test_from_args(self):
mappings = [
({
'url': 'http://www.example.com/data.csv'
}, TextResponse),
# headers takes precedence over url
({
'headers':
Headers({'Content-Type': ['text/html; charset=utf-8']}),
'url': 'http://www.example.com/item/'
}, HtmlResponse),
({
'headers':
Headers({
'Content-Disposition':
['attachment; filename="data.xml.gz"']
}),
'url':
'http://www.example.com/page/'
}, Response),
]
for source, cls in mappings:
retcls = from_args(**source)
self.assertIs(
retcls, cls,
'Expected: %s Received: %s' % (cls.__name__, retcls.__name__))
def _cb_body_done(self, result, request, url):
txresponse, body, flags = result
status = int(txresponse.code)
headers = Headers(txresponse.headers.getAllRawHeaders())
response_cls = resp_factory.from_args(headers=headers, url=url)
return response_cls(url=url, status=status, headers=headers, body=body,
request=request)
def _cb_body_done(self, result, request, url):
txresponse, body, flags = result
status = int(txresponse.code)
headers = Headers(txresponse.headers.getAllRawHeaders())
response_cls = resp_factory.from_args(headers=headers, url=url)
return response_cls(url=url, status=status, headers=headers, body=body,
request=request)
def retrieve_response(self, request):
data = self._read_data(request)
if data is None:
return # not cached
url = data['url']
status = data['status']
headers = Headers(data['headers'])
body = data['body']
respcls = from_args(headers=headers, url=url)
response = respcls(url=url, headers=headers, status=status, body=body)
return response
def _build_response(self, body, request):
if self.invalid_headers:
raise BadHttpHeaderError('Invalid headers received: %s' %
self.invalid_headers)
response_cls = resp_factory.from_args(headers=self.response_headers,
url=self.url)
response = response_cls(
url=self.url, status=self.status, headers=self.response_headers,
body=body, request=request)
response.download_latency = self.headers_time - self.start_time
return response
def test_from_args(self):
mappings = [
({'url': 'http://www.example.com/data.csv'}, TextResponse),
# headers takes precedence over url
({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), 'url': 'http://www.example.com/item/'}, HtmlResponse),
({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response),
]
for source, cls in mappings:
retcls = from_args(**source)
self.assertIs(retcls, cls, 'Expected: %s Received: %s' % (cls.__name__, retcls.__name__))
def process_response(self, response):
content_encoding = response.headers.getlist('Content-Encoding')
if content_encoding and not is_gzipped(response):
max_length = self.settings.get_int('DOWNLOAD_SIZE_LIMIT', 0,
response.request)
encoding = content_encoding.pop()
if not content_encoding:
del response.headers['Content-Encoding']
decoded_body = self._decode(response.body, encoding.lower(),
max_length)
resp_cls = factory.from_args(headers=response.headers,
url=response.url)
response = response.replace(cls=resp_cls, body=decoded_body)
return response
def _build_response(self, body, request):
if self.invalid_headers:
raise BadHttpHeaderError('Invalid headers received: %s' %
self.invalid_headers)
response_cls = resp_factory.from_args(headers=self.response_headers,
url=self.url)
response = response_cls(url=self.url,
status=self.status,
headers=self.response_headers,
body=body,
request=request)
response.download_latency = self.headers_time - self.start_time
return response
def process_response(self, response):
content_encoding = response.headers.getlist('Content-Encoding')
if content_encoding and not is_gzipped(response):
max_length = self.settings.get_int('DOWNLOAD_SIZE_LIMIT', 0,
response.request)
encoding = content_encoding.pop()
if not content_encoding:
del response.headers['Content-Encoding']
decoded_body = self._decode(response.body, encoding.lower(),
max_length)
resp_cls = factory.from_args(headers=response.headers,
url=response.url)
response = response.replace(cls=resp_cls, body=decoded_body)
return response
def download():
filepath = file_uri_to_path(request.url)
body = open(filepath, 'rb').read()
response_cls = resp_factory.from_args(filename=filepath, body=body)
return response_cls(url=request.url, body=body, request=request)