def get_request(self, url):
cache_key = "urls-%s" % url
contents = self.redis.get(cache_key)
if not contents:
return url, None
body = self.redis.get('%s-body' % cache_key)
item = loads(contents)
response = Response(
url=url,
status_code=item['status_code'],
headers=item['headers'],
cookies=item['cookies'],
text=body,
effective_url=item['effective_url'],
error=item['error'],
request_time=float(item['request_time'])
)
response.from_cache = True
return url, response
def get_request(self, url):
cache_key = "urls-%s" % url
contents = self.redis.get(cache_key)
if not contents:
return url, None
item = msgpack.unpackb(contents)
text = GzipFile(mode='r', fileobj=StringIO(item['body'])).read()
response = Response(
url=url,
status_code=item['status_code'],
headers=item['headers'],
cookies=item['cookies'],
text=text,
effective_url=item['effective_url'],
error=item['error'],
request_time=float(item['request_time'])
)
response.from_cache = True
return url, response
def from_requests_response(self, url, response):
return Response(
url=url,
status_code=response.status_code,
headers=dict([(key, value)
for key, value in response.headers.items()]),
cookies=dict([(key, value)
for key, value in response.cookies.items()]),
text=response.text,
effective_url=response.url,
error=response.status_code > 399 and response.text or None,
request_time=response.elapsed and response.elapsed.total_seconds
or 0)
def get_request(self, url):
cache_key = "urls-%s" % url
contents = self.redis.get(cache_key)
if not contents:
return url, None
item = msgpack.unpackb(contents)
text = GzipFile(mode='r', fileobj=StringIO(item['body'])).read()
response = Response(url=url,
status_code=item['status_code'],
headers=item['headers'],
cookies=item['cookies'],
text=text,
effective_url=item['effective_url'],
error=item['error'],
request_time=float(item['request_time']))
response.from_cache = True
return url, response
def async_get(self, url, handler, method='GET', **kw):
response = requests.get(url, allow_redirects=False)
resp = Response(
url=url,
status_code=response.status_code,
headers=dict([(key, value)
for key, value in response.headers.items()]),
cookies=dict([(key, value)
for key, value in response.cookies.items()]),
text=response.text,
effective_url=response.url,
error=response.status_code > 399 and response.text or None,
request_time=response.elapsed and response.elapsed.total_seconds()
or 0)
handler(url, resp)
def from_tornado_response(cls, url, response):
cookies = response.request.headers.get('Cookie', '')
if cookies:
cookies = dict(
[cookie.split('=') for cookie in cookies.split(';')])
return Response(url=url,
status_code=response.code,
headers=dict([
(key, value)
for key, value in response.headers.items()
]),
cookies=cookies,
text=response.body,
effective_url=response.effective_url,
error=response.error and str(response.error) or None,
request_time=response.request_time)
def test_can_create_response(self):
response = Response(url="http://www.google.com",
status_code=200,
headers={'Accept': 'image/webp; */*'},
cookies={'whatever': 'some-value'},
text='some request body',
effective_url='http://www.google.com/',
error="some error message",
request_time=10.24)
expect(response.url).to_equal('http://www.google.com')
expect(response.status_code).to_equal(200)
expect(response.headers).to_be_like({'Accept': 'image/webp; */*'})
expect(response.cookies).to_be_like({'whatever': 'some-value'})
expect(response.text).to_equal('some request body')
expect(response.effective_url).to_equal('http://www.google.com/')
expect(response.error).to_equal('some error message')
expect(response.request_time).to_equal(10.24)