Пример #1
0
 def _build_response(self, result, request, protocol):
     self.result = result
     respcls = responsetypes.from_args(url=request.url)
     protocol.close()
     body = protocol.filename or protocol.body.read()
     headers = {"local filename": protocol.filename or '', "size": protocol.size}
     return respcls(url=request.url, status=200, body=body, headers=headers)
Пример #2
0
 def retrieve_response(self, spider, request):
     data = self._read_data(spider, request)
     if data is None:
         return  # not cached
     url = data['url']
     status = data['status']
     headers = Headers(data['headers'])
     body = data['body']
     respcls = responsetypes.from_args(headers=headers, url=url)
     response = respcls(url=url, headers=headers, status=status, body=body)
     return response
Пример #3
0
 def retrieve_response(self, spider, request):
     """Return response if present in cache, or None otherwise."""
     metadata = self._read_meta(spider, request)
     if metadata is None:
         return  # not cached
     rpath = self._get_request_path(spider, request)
     with open(os.path.join(rpath, 'response_body'), 'rb') as f:
         body = f.read()
     with open(os.path.join(rpath, 'response_headers'), 'rb') as f:
         rawheaders = f.read()
     url = metadata.get('response_url')
     status = metadata['status']
     headers = Headers(headers_raw_to_dict(rawheaders))
     respcls = responsetypes.from_args(headers=headers, url=url)
     response = respcls(url=url, headers=headers, status=status, body=body)
     return response
Пример #4
0
    def process_response(self, request, response, spider):
        if isinstance(response, Response):
            content_encoding = response.headers.getlist("Content-Encoding")
            if content_encoding and not is_gzipped(response):
                encoding = content_encoding.pop()
                decoded_body = self._decode(response.body, encoding.lower())
                respcls = responsetypes.from_args(headers=response.headers, url=response.url)
                kwargs = dict(cls=respcls, body=decoded_body)
                if issubclass(respcls, TextResponse):
                    # force recalculating the encoding until we make sure the
                    # responsetypes guessing is reliable
                    kwargs["encoding"] = None
                response = response.replace(**kwargs)
                if not content_encoding:
                    del response.headers["Content-Encoding"]

        return response
Пример #5
0
 def _cb_bodydone(self, result, request, url):
     txresponse, body, flags = result
     status = int(txresponse.code)
     headers = Headers(txresponse.headers.getAllRawHeaders())
     respcls = responsetypes.from_args(headers=headers, url=url)
     return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
Пример #6
0
 def download_request(self, request, spider):
     filepath = file_uri_to_path(request.url)
     body = open(filepath, 'rb').read()
     respcls = responsetypes.from_args(filename=filepath, body=body)
     return respcls(url=request.url, body=body)
Пример #7
0
 def _build_response(self, body, request):
     request.meta['download_latency'] = self.headers_time-self.start_time
     status = int(self.status)
     headers = Headers(self.response_headers)
     respcls = responsetypes.from_args(headers=headers, url=self.url)
     return respcls(url=self.url, status=status, headers=headers, body=body)