def get_request(self, url):
cache_key = "urls-%s" % url
contents = self.redis.get(cache_key)
if not contents:
return url, None
body = self.redis.get('%s-body' % cache_key)
item = loads(contents)
response = Response(
url=url,
status_code=item['status_code'],
headers=item['headers'],
cookies=item['cookies'],
text=body,
effective_url=item['effective_url'],
error=item['error'],
request_time=float(item['request_time'])
)
response.from_cache = True
return url, response
def get_request(self, url):
cache_key = "urls-%s" % url
contents = self.redis.get(cache_key)
if not contents:
return url, None
item = msgpack.unpackb(contents)
text = GzipFile(mode='r', fileobj=StringIO(item['body'])).read()
response = Response(
url=url,
status_code=item['status_code'],
headers=item['headers'],
cookies=item['cookies'],
text=text,
effective_url=item['effective_url'],
error=item['error'],
request_time=float(item['request_time'])
)
response.from_cache = True
return url, response
def get_request(self, url):
cache_key = "urls-%s" % url
contents = self.redis.get(cache_key)
if not contents:
return url, None
item = msgpack.unpackb(contents)
text = GzipFile(mode='r', fileobj=StringIO(item['body'])).read()
response = Response(url=url,
status_code=item['status_code'],
headers=item['headers'],
cookies=item['cookies'],
text=text,
effective_url=item['effective_url'],
error=item['error'],
request_time=float(item['request_time']))
response.from_cache = True
return url, response
