def get_sitemap_body(response):
'''Return the sitemap body contained in the given response, or None if the
response is not a sitemap.
'''
if isinstance(response, XmlResponse):
return response.body
elif is_gzipped(response):
return gunzip(response.body)
elif response.url.endswith('.xml'):
return response.body
elif response.url.endswith('.xml.gz'):
return gunzip(response.body)
def process_response(self, response):
content_encoding = response.headers.getlist('Content-Encoding')
if content_encoding and not is_gzipped(response):
max_length = self.settings.get_int('DOWNLOAD_SIZE_LIMIT', 0,
response.request)
encoding = content_encoding.pop()
if not content_encoding:
del response.headers['Content-Encoding']
decoded_body = self._decode(response.body, encoding.lower(),
max_length)
resp_cls = factory.from_args(headers=response.headers,
url=response.url)
response = response.replace(cls=resp_cls, body=decoded_body)
return response
def process_response(self, response):
content_encoding = response.headers.getlist('Content-Encoding')
if content_encoding and not is_gzipped(response):
max_length = self.settings.get_int('DOWNLOAD_SIZE_LIMIT', 0,
response.request)
encoding = content_encoding.pop()
if not content_encoding:
del response.headers['Content-Encoding']
decoded_body = self._decode(response.body, encoding.lower(),
max_length)
resp_cls = factory.from_args(headers=response.headers,
url=response.url)
response = response.replace(cls=resp_cls, body=decoded_body)
return response
