コード例 #1
0
 def test_from_args(self):
     mappings = [
         ({
             'url': 'http://www.example.com/data.csv'
         }, TextResponse),
         # headers takes precedence over url
         ({
             'headers':
             Headers({'Content-Type': ['text/html; charset=utf-8']}),
             'url': 'http://www.example.com/item/'
         }, HtmlResponse),
         ({
             'headers':
             Headers({
                 'Content-Disposition':
                 ['attachment; filename="data.xml.gz"']
             }),
             'url':
             'http://www.example.com/page/'
         }, Response),
     ]
     for source, cls in mappings:
         retcls = from_args(**source)
         self.assertIs(
             retcls, cls,
             'Expected: %s  Received: %s' % (cls.__name__, retcls.__name__))
コード例 #2
0
ファイル: http_1_1_handler.py プロジェクト: Mimino666/crawlmi
 def _cb_body_done(self, result, request, url):
     txresponse, body, flags = result
     status = int(txresponse.code)
     headers = Headers(txresponse.headers.getAllRawHeaders())
     response_cls = resp_factory.from_args(headers=headers, url=url)
     return response_cls(url=url, status=status, headers=headers, body=body,
         request=request)
コード例 #3
0
 def _cb_body_done(self, result, request, url):
     txresponse, body, flags = result
     status = int(txresponse.code)
     headers = Headers(txresponse.headers.getAllRawHeaders())
     response_cls = resp_factory.from_args(headers=headers, url=url)
     return response_cls(url=url, status=status, headers=headers, body=body,
         request=request)
コード例 #4
0
ファイル: storage.py プロジェクト: dahuangfeng123/crawlmi
 def retrieve_response(self, request):
     data = self._read_data(request)
     if data is None:
         return  # not cached
     url = data['url']
     status = data['status']
     headers = Headers(data['headers'])
     body = data['body']
     respcls = from_args(headers=headers, url=url)
     response = respcls(url=url, headers=headers, status=status, body=body)
     return response
コード例 #5
0
ファイル: webclient.py プロジェクト: Mimino666/crawlmi
    def _build_response(self, body, request):
        if self.invalid_headers:
            raise BadHttpHeaderError('Invalid headers received: %s' %
                                     self.invalid_headers)

        response_cls = resp_factory.from_args(headers=self.response_headers,
                                              url=self.url)
        response = response_cls(
            url=self.url, status=self.status, headers=self.response_headers,
            body=body, request=request)
        response.download_latency = self.headers_time - self.start_time
        return response
コード例 #6
0
    def test_from_args(self):
        mappings = [
            ({'url': 'http://www.example.com/data.csv'}, TextResponse),
            # headers takes precedence over url
            ({'headers': Headers({'Content-Type': ['text/html; charset=utf-8']}), 'url': 'http://www.example.com/item/'}, HtmlResponse),
            ({'headers': Headers({'Content-Disposition': ['attachment; filename="data.xml.gz"']}), 'url': 'http://www.example.com/page/'}, Response),


        ]
        for source, cls in mappings:
            retcls = from_args(**source)
            self.assertIs(retcls, cls, 'Expected: %s  Received: %s' % (cls.__name__, retcls.__name__))
コード例 #7
0
ファイル: http_compression.py プロジェクト: Mimino666/crawlmi
 def process_response(self, response):
     content_encoding = response.headers.getlist('Content-Encoding')
     if content_encoding and not is_gzipped(response):
         max_length = self.settings.get_int('DOWNLOAD_SIZE_LIMIT', 0,
                                            response.request)
         encoding = content_encoding.pop()
         if not content_encoding:
             del response.headers['Content-Encoding']
         decoded_body = self._decode(response.body, encoding.lower(),
                                     max_length)
         resp_cls = factory.from_args(headers=response.headers,
                                      url=response.url)
         response = response.replace(cls=resp_cls, body=decoded_body)
     return response
コード例 #8
0
ファイル: webclient.py プロジェクト: dahuangfeng123/crawlmi
    def _build_response(self, body, request):
        if self.invalid_headers:
            raise BadHttpHeaderError('Invalid headers received: %s' %
                                     self.invalid_headers)

        response_cls = resp_factory.from_args(headers=self.response_headers,
                                              url=self.url)
        response = response_cls(url=self.url,
                                status=self.status,
                                headers=self.response_headers,
                                body=body,
                                request=request)
        response.download_latency = self.headers_time - self.start_time
        return response
コード例 #9
0
 def process_response(self, response):
     content_encoding = response.headers.getlist('Content-Encoding')
     if content_encoding and not is_gzipped(response):
         max_length = self.settings.get_int('DOWNLOAD_SIZE_LIMIT', 0,
                                            response.request)
         encoding = content_encoding.pop()
         if not content_encoding:
             del response.headers['Content-Encoding']
         decoded_body = self._decode(response.body, encoding.lower(),
                                     max_length)
         resp_cls = factory.from_args(headers=response.headers,
                                      url=response.url)
         response = response.replace(cls=resp_cls, body=decoded_body)
     return response
コード例 #10
0
 def download():
     filepath = file_uri_to_path(request.url)
     body = open(filepath, 'rb').read()
     response_cls = resp_factory.from_args(filename=filepath, body=body)
     return response_cls(url=request.url, body=body, request=request)