def get_cached_post(url, postdata, host='127.0.0.1', port=11211):
"""Returns url data from url with post request"""
servers = ["%s:%d" % (host, port)]
m = hashlib.sha256()
m.update(url.encode('utf8'))
m.update(str(postdata).encode('utf8'))
key = m.hexdigest()
client = Client(servers)
c_data = client.get(key)
if c_data:
data = decompress(c_data)
else:
r = requests.post(url, postdata)
data = r.text
client.set(key, compress(data))
hp = lxml.etree.HTMLParser(encoding='utf-8')
root = lxml.html.fromstring(data, parser=hp)
return root
class CacheLayer(object):
"""Memcache Wrapper."""
def __init__(
self,
host,
port: int = 11211,
user: Optional[str] = None,
password: Optional[str] = None,
):
"""Init Cache Layer."""
self.client = Client((f"{host}:{port}",), user, password)
def get_image_from_cache(self, img_hash: str) -> Tuple[bytes, ImageType]:
"""
Get image body from cache layer.
Attributes
----------
img_hash : str
file url.
Returns
-------
img : bytes
image body.
ext : str
image ext
"""
content, ext = self.client.get(img_hash)
return content, ext
def set_image_cache(
self, img_hash: str, body: Tuple[bytes, ImageType], timeout: int = 432000
) -> bool:
"""
Set base64 encoded image body in cache layer.
Attributes
----------
img_hash : str
file url.
body : tuple
image body + ext
Returns
-------
bool
"""
try:
return self.client.set(img_hash, body, time=timeout)
except Exception:
return False
def get_cached_url(url,
timeout=DEFAULT_CACHE_TIMEOUT,
host='127.0.0.1',
port=11211):
"""Returns url data from url or from local memcached"""
servers = ["%s:%d" % (host, port)]
m = hashlib.sha256()
m.update(url.encode('utf8'))
key = m.hexdigest()
client = Client(servers)
c_data = client.get(key)
if c_data:
data = decompress(c_data)
else:
o = urlopen(url)
rurl = o.geturl()
data = o.read()
client.set(key, compress(data))
hp = lxml.etree.HTMLParser(encoding='utf-8')
root = lxml.html.fromstring(data, parser=hp)
return root
class MemcacheStorage(Storage):
'''
Storage class for a memcached cluster.
'''
def __init__(self, servers):
from bmemcached import Client
self.mc = Client(servers)
self.last_stats = {}
def get(self, key):
return self.mc.get(key)
def put(self, key, data):
return self.mc.set(key, data) != 0
