def get_cached_post(url, postdata, host='127.0.0.1', port=11211): """Returns url data from url with post request""" servers = ["%s:%d" % (host, port)] m = hashlib.sha256() m.update(url.encode('utf8')) m.update(str(postdata).encode('utf8')) key = m.hexdigest() client = Client(servers) c_data = client.get(key) if c_data: data = decompress(c_data) else: r = requests.post(url, postdata) data = r.text client.set(key, compress(data)) hp = lxml.etree.HTMLParser(encoding='utf-8') root = lxml.html.fromstring(data, parser=hp) return root
class CacheLayer(object): """Memcache Wrapper.""" def __init__( self, host, port: int = 11211, user: Optional[str] = None, password: Optional[str] = None, ): """Init Cache Layer.""" self.client = Client((f"{host}:{port}",), user, password) def get_image_from_cache(self, img_hash: str) -> Tuple[bytes, ImageType]: """ Get image body from cache layer. Attributes ---------- img_hash : str file url. Returns ------- img : bytes image body. ext : str image ext """ content, ext = self.client.get(img_hash) return content, ext def set_image_cache( self, img_hash: str, body: Tuple[bytes, ImageType], timeout: int = 432000 ) -> bool: """ Set base64 encoded image body in cache layer. Attributes ---------- img_hash : str file url. body : tuple image body + ext Returns ------- bool """ try: return self.client.set(img_hash, body, time=timeout) except Exception: return False
def get_cached_url(url, timeout=DEFAULT_CACHE_TIMEOUT, host='127.0.0.1', port=11211): """Returns url data from url or from local memcached""" servers = ["%s:%d" % (host, port)] m = hashlib.sha256() m.update(url.encode('utf8')) key = m.hexdigest() client = Client(servers) c_data = client.get(key) if c_data: data = decompress(c_data) else: o = urlopen(url) rurl = o.geturl() data = o.read() client.set(key, compress(data)) hp = lxml.etree.HTMLParser(encoding='utf-8') root = lxml.html.fromstring(data, parser=hp) return root
class MemcacheStorage(Storage): ''' Storage class for a memcached cluster. ''' def __init__(self, servers): from bmemcached import Client self.mc = Client(servers) self.last_stats = {} def get(self, key): return self.mc.get(key) def put(self, key, data): return self.mc.set(key, data) != 0