def process_response(self, request, response, spider):
if (get_mw_setting('httpcache_enabled', self, spider, request) and
self.is_cacheable(request) and
self.is_cacheable_response(response) and
'cached' not in response.flags):
self.storage.store_response(spider, request, response)
self.stats.inc_value('httpcache/store', spider=spider)
return response
def _read_data(self, spider, request):
key = self._request_key(request)
db = self.db
tkey = '%s_time' % key
if not db.has_key(tkey):
return # not found
ts = db[tkey]
if 0 < get_mw_setting('httpcache_expiration_secs', self, spider, request) < time() - float(ts):
return # expired
return pickle.loads(db['%s_data' % key])
def _read_meta(self, spider, request):
rpath = self._get_request_path(spider, request)
metapath = join(rpath, 'pickled_meta')
if not exists(metapath):
return # not found
mtime = os.stat(rpath).st_mtime
if 0 < get_mw_setting('httpcache_expiration_secs', self, spider, request) < time() - mtime:
return # expired
with open(metapath, 'rb') as f:
return pickle.load(f)
def process_request(self, request, spider):
if (not get_mw_setting('httpcache_enabled', self, spider, request) or
not self.is_cacheable(request)):
return
response = self.storage.retrieve_response(spider, request)
if response and self.is_cacheable_response(response):
response.flags.append('cached')
self.stats.inc_value('httpcache/hits', spider=spider)
return response
self.stats.inc_value('httpcache/misses', spider=spider)
if self.ignore_missing:
raise IgnoreRequest("Ignored request not in cache: %s" % request)