Beispiel #1
0
    def get_request(self, url):
        cache_key = "urls-%s" % url

        contents = self.redis.get(cache_key)

        if not contents:
            return url, None

        body = self.redis.get('%s-body' % cache_key)

        item = loads(contents)
        response = Response(
            url=url,
            status_code=item['status_code'],
            headers=item['headers'],
            cookies=item['cookies'],
            text=body,
            effective_url=item['effective_url'],
            error=item['error'],
            request_time=float(item['request_time'])
        )

        response.from_cache = True

        return url, response
Beispiel #2
0
    def get_request(self, url):
        cache_key = "urls-%s" % url

        contents = self.redis.get(cache_key)

        if not contents:
            return url, None

        item = msgpack.unpackb(contents)

        text = GzipFile(mode='r', fileobj=StringIO(item['body'])).read()

        response = Response(
            url=url,
            status_code=item['status_code'],
            headers=item['headers'],
            cookies=item['cookies'],
            text=text,
            effective_url=item['effective_url'],
            error=item['error'],
            request_time=float(item['request_time'])
        )

        response.from_cache = True

        return url, response
Beispiel #3
0
 def from_requests_response(self, url, response):
     return Response(
         url=url,
         status_code=response.status_code,
         headers=dict([(key, value)
                       for key, value in response.headers.items()]),
         cookies=dict([(key, value)
                       for key, value in response.cookies.items()]),
         text=response.text,
         effective_url=response.url,
         error=response.status_code > 399 and response.text or None,
         request_time=response.elapsed and response.elapsed.total_seconds
         or 0)
Beispiel #4
0
    def get_request(self, url):
        cache_key = "urls-%s" % url

        contents = self.redis.get(cache_key)

        if not contents:
            return url, None

        item = msgpack.unpackb(contents)

        text = GzipFile(mode='r', fileobj=StringIO(item['body'])).read()

        response = Response(url=url,
                            status_code=item['status_code'],
                            headers=item['headers'],
                            cookies=item['cookies'],
                            text=text,
                            effective_url=item['effective_url'],
                            error=item['error'],
                            request_time=float(item['request_time']))

        response.from_cache = True

        return url, response
    def async_get(self, url, handler, method='GET', **kw):
        response = requests.get(url, allow_redirects=False)
        resp = Response(
            url=url,
            status_code=response.status_code,
            headers=dict([(key, value)
                          for key, value in response.headers.items()]),
            cookies=dict([(key, value)
                          for key, value in response.cookies.items()]),
            text=response.text,
            effective_url=response.url,
            error=response.status_code > 399 and response.text or None,
            request_time=response.elapsed and response.elapsed.total_seconds()
            or 0)

        handler(url, resp)
Beispiel #6
0
    def from_tornado_response(cls, url, response):
        cookies = response.request.headers.get('Cookie', '')
        if cookies:
            cookies = dict(
                [cookie.split('=') for cookie in cookies.split(';')])

        return Response(url=url,
                        status_code=response.code,
                        headers=dict([
                            (key, value)
                            for key, value in response.headers.items()
                        ]),
                        cookies=cookies,
                        text=response.body,
                        effective_url=response.effective_url,
                        error=response.error and str(response.error) or None,
                        request_time=response.request_time)
Beispiel #7
0
    def test_can_create_response(self):
        response = Response(url="http://www.google.com",
                            status_code=200,
                            headers={'Accept': 'image/webp; */*'},
                            cookies={'whatever': 'some-value'},
                            text='some request body',
                            effective_url='http://www.google.com/',
                            error="some error message",
                            request_time=10.24)

        expect(response.url).to_equal('http://www.google.com')
        expect(response.status_code).to_equal(200)
        expect(response.headers).to_be_like({'Accept': 'image/webp; */*'})
        expect(response.cookies).to_be_like({'whatever': 'some-value'})
        expect(response.text).to_equal('some request body')
        expect(response.effective_url).to_equal('http://www.google.com/')
        expect(response.error).to_equal('some error message')
        expect(response.request_time).to_equal(10.24)