def get(self):
"""
A full, synchronous get-with-expiration check.
The whole get-check-extract process is in three easily
accessible stages so that async cache APIs can call them
separately.
"""
if not self._policy.should_read_cache():
# this is now pretty normal for client. We should only log
# this when the client isn't requesting it.
# LOG.warn("memcache.skip.read", "Skipping cache", **self._log_kwds())
self._policy.add_cost('s')
return ('miss', None)
full_result = self.raw_get()
key = self.get_key()
if not self.check_result(key, full_result):
# would rather log this in the policy object, because this
# is the only use of tag?
LOG.notice("%s.cache.result" % self._policy.tag,
"",
key=key,
keyobj=self.key_obj,
**self._log_kwds(code="miss"))
if self._policy.allow_stale() and full_result:
return ('miss', self._policy.extract_result(full_result))
else:
return ('miss', None)
return ('hit', self._policy.extract_result(full_result))
def fix_xml_encoding(text_body):
"""
If the given xml document has an encoding declaration, then
reencode that document with that declaration. This sucks, but it's
the only sane way to deal with lxml.html
"""
encoding = encoding_declaration(text_body)
if encoding:
# reencode so lxml can be happy - this totally
# sucks because we just spent all this time
# encoding it.
LOG.notice("content.reencode",
"Dumb reencoding of blob body as %s" % encoding)
text_body = text_body.encode(encoding)
return text_body
def get(self):
"""
Gets all the cache entries - will return a triple of::
('hit', 'miss' or 'skip', value, CacheEntry)
for each cache entry passed in to the CacheEntryList constructor
"""
# no memcache hooked up?
if not self._policy.should_read_cache() or not self.cache:
LOG.warn("memcache.skip.read", "Skipping cache",
**self._log_kwds())
self._policy.add_cost('s')
return [('skip', None, ce) for ce in self.cache_entries]
self._policy.add_cost('r')
with MemcacheChecker(self.cache):
try:
memcache_result = self.cache.get_multi(
[ce.get_key() for ce in self.cache_entries])
except pylibmc.Error as e:
memcache_result = {}
LOG.error("memcache.error.get_multi",
"memcache get_multi failure",
error=e,
**self._log_kwds())
assert isinstance(memcache_result, dict)
result = []
# create an entry in the result for each cache entry
for ce in self.cache_entries:
key = ce.get_key()
mr = memcache_result.get(key)
if not ce.check_result(key, mr):
result.append(('miss', None, ce))
else:
result.append(('hit', ce._policy.extract_result(mr), ce))
misses = [miss for miss in result if miss[0] == 'miss']
hits = [hit for hit in result if hit[0] == 'hit']
miss_keys = [ce.key_obj for (status, value, ce) in misses]
miss_hashes = [ce.get_key() for (status, value, ce) in misses]
hit_keys = [ce.key_obj for (status, value, ce) in hits]
hit_hashes = [ce.get_key() for (status, value, ce) in hits]
if miss_keys:
code = "hits+misses" if hit_keys else "all miss"
else:
code = "all hit" if hit_keys else "empty"
LOG.notice("%s.cache.multiresult" % self._policy.tag,
"",
miss_hashes=miss_hashes,
hit_hashes=hit_hashes,
miss_count=len(misses),
hit_count=len(hits),
**self._log_kwds(code=code))
return result