class MemCacheWrapper(object): """ Memcache client wrapper. No exception raise and add some useful function. """ def __init__(self, servers, logerr=None): self.cache = MCClient(servers=servers, debug=False) self.logerr = logerr def add(self, key, val=1, time=0): try: return self.cache.add(key, val, time) except Exception as e: _logger.warning("Exception during `add`: %s", e) return None def count(self, key, expires=0, delta=1): try: result = self.cache.incr(key, delta) if result is None: if not self.cache.add(key, delta, expires): result = self.cache.incr(key, delta) else: return delta return result except Exception as e: _logger.warning("Exception during `count`: %s", e) return None def get(self, key): result = None try: result = self.cache.get(str(key)) except Exception as e: _logger.warning("Exception during `get`: %s", e) return result def set(self, key, value, expires): result = False try: result = self.cache.set(str(key), value, expires) except Exception as e: _logger.warning("Exception during `set`: %s", e) return result def delete(self, key): result = False try: result = self.cache.delete(key) except Exception as e: _logger.warning("Exception during `del`: %s", e) return result
class WrappedClient(object): def __init__(self, *args): self.args = args self.mc = Client(*args, cache_cas=True, socket_timeout=10) self.del_que = [] import threading def gets(self, key): while True: result = self.mc.gets(key) if isinstance(result, tuple): return result[0] return result def cas(self, key, value): retry_count = 0 try: while True: result = self.mc.cas(key, value) if not isinstance(result, bool): if retry_count <= 10: retry_count += 1 wait_time = 0.001 * randint(0, 1 << retry_count) print "add fail, retry for sleep" sleep(wait_time) self.mc = Client(*self.args, cache_cas=True, socket_timeout=10) continue # raise ConnectionError return result except TypeError: return False def add(self, key, value): retry_count = 0 while True: result = self.mc.add(key, value) if not isinstance(result, bool): if retry_count <= 10: retry_count += 1 wait_time = 0.001 * randint(0, 1 << retry_count) print "add fail, retry for sleep" sleep(wait_time) print self.args self.mc = Client(*self.args, cache_cas=True, socket_timeout=10) continue # raise ConnectionError return result # delegation def __getattr__(self, attrname): return getattr(self.mc, attrname)
class Store(object): def __init__(self, host='127.0.0.1', port=11211): self.client = Client(['%s:%d' % (host, port),], pickler=JSONPickler, unpickler=JSONUnpickler) def get(self, key): result = self.client.get(key) if result: result = result.split('\t') return result def set(self, key, val): result = self.client.add(key, val) if not result: result = self.client.append(key, '\t%s' % val) return result
def lock(self, layer, coord, format): """ Acquire a cache lock for this tile. Returns nothing, but blocks until the lock has been acquired. """ mem = Client(self.servers) key = tile_key(layer, coord, format, self.revision, self.key_prefix) due = _time() + layer.stale_lock_timeout try: while _time() < due: if mem.add(key + '-lock', 'locked.', layer.stale_lock_timeout): return _sleep(.2) mem.set(key + '-lock', 'locked.', layer.stale_lock_timeout) return finally: mem.disconnect_all()
def lock(self, layer, coord, format): """ Acquire a cache lock for this tile. Returns nothing, but blocks until the lock has been acquired. """ mem = Client(self.servers) key = tile_key(layer, coord, format, self.revision, self.key_prefix) due = _time() + layer.stale_lock_timeout try: while _time() < due: if mem.add(key + "-lock", "locked.", layer.stale_lock_timeout): return _sleep(0.2) mem.set(key + "-lock", "locked.", layer.stale_lock_timeout) return finally: mem.disconnect_all()
class MemcachedCacheStore(AbstractCacheStore): servers = ("127.0.0.1:11211") def __init__(self, servers=None, debug=False): if servers is None: servers = self.servers from memcache import Client as MemcachedClient self._client = MemcachedClient(servers, debug) def set(self, key, val, time=0): self._client.set(key, val, time) def add(self, key, val, time=0): res = self._client.add(key, val, time) if not res: raise Error("a value for key %r is already in the cache" % key) self._data[key] = (val, time) def replace(self, key, val, time=0): res = self._client.replace(key, val, time) if not res: raise Error("a value for key %r is already in the cache" % key) self._data[key] = (val, time) def delete(self, key): res = self._client.delete(key, time=0) if not res: raise KeyError(key) def get(self, key): val = self._client.get(key) if val is None: raise KeyError(key) return val def clear(self): self._client.flush_all()
class MemcachedCacheStore(AbstractCacheStore): servers = ('127.0.0.1:11211') def __init__(self, servers=None, debug=False): if servers is None: servers = self.servers from memcache import Client as MemcachedClient self._client = MemcachedClient(servers, debug) def set(self, key, val, time=0): self._client.set(key, val, time) def add(self, key, val, time=0): res = self._client.add(key, val, time) if not res: raise Error('a value for key %r is already in the cache'%key) self._data[key] = (val, time) def replace(self, key, val, time=0): res = self._client.replace(key, val, time) if not res: raise Error('a value for key %r is already in the cache'%key) self._data[key] = (val, time) def delete(self, key): res = self._client.delete(key, time=0) if not res: raise KeyError(key) def get(self, key): val = self._client.get(key) if val is None: raise KeyError(key) else: return val def clear(self): self._client.flush_all()
class MemcachedCacheClient(CacheClient): """Memcached cache client implementation.""" def __init__(self, config): super(MemcachedCacheClient, self).__init__(config["host"], config["port"], config["cache"]) self.config = config if self.cache_name and self.cache_name != "" and self.cache_name != DEFAULT_MEMCACHED_CACHE_NAME: print "WARNING: memcached client doesn't support named caches. cache_name config value will be ignored and the cache name configured on the server will be used instead." self.memcached_client = Client([self.host + ':' + self.port], debug=0) return def put(self, key, value, version=None, lifespan=None, max_idle=None, put_if_absent=False): time = 0 if lifespan != None: if lifespan > MEMCACHED_LIFESPAN_MAX_SECONDS: self._error("Memcached cache client supports lifespan values only up to %s seconds (30 days)." % MEMCACHED_LIFESPAN_MAX_SECONDS) time = lifespan if max_idle != None: self._error("Memcached cache client doesn't support max idle time setting.") try: if (version == None): if (put_if_absent): if not self.memcached_client.add(key, value, time, 0): # current python-memcached doesn't recoginze these states # if self.memcached_client.last_set_status == "NOT_STORED": # raise ConflictError # else: # self._error("Operation unsuccessful. " + self.memcached_client.last_set_status) self._error("Operation unsuccessful. Possibly CONFLICT.") else: if not self.memcached_client.set(key, value, time, 0): # self._error("Operation unsuccessful. " + self.memcached_client.last_set_status) self._error("Operation unsuccessful.") else: try: self.memcached_client.cas_ids[key] = int(version) except ValueError: self._error("Please provide an integer version.") if not self.memcached_client.cas(key, value, time, 0): # if self.memcached_client.last_set_status == "EXISTS": # raise ConflictError # if self.memcached_client.last_set_status == "NOT_FOUND": # raise NotFoundError # else: # self._error("Operation unsuccessful. " + self.memcached_client.last_set_status) self._error("Operation unsuccessful. Possibly CONFLICT, NOT_FOUND.") except CacheClientError as e: raise e #rethrow except Exception as e: self._error(e) def get(self, key, get_version=False): try: if get_version: val = self.memcached_client.gets(key) if val == None: raise NotFoundError version = self.memcached_client.cas_ids[key] if version == None: self._error("Couldn't obtain version info from memcached server.") return version, val else: val = self.memcached_client.get(key) if val == None: raise NotFoundError return val except CacheClientError as e: raise e #rethrow except Exception as e: self._error(e.args) def delete(self, key, version=None): try: if version: self._error("versioned delete operation not available for memcached client") if self.memcached_client.delete(key, 0): if self.memcached_client.last_set_status == "NOT_FOUND": raise NotFoundError else: self._error("Operation unsuccessful. " + self.memcached_client.last_set_status) except CacheClientError as e: raise e #rethrow except Exception as e: self._error(e.args) def clear(self): try: self.memcached_client.flush_all() except CacheClientError as e: raise e #rethrow except Exception as e: self._error(e.args)
class TestMemcache(unittest.TestCase): def setUp(self): # TODO(): unix socket server stuff servers = ["127.0.0.1:11211"] self.mc = Client(servers, debug=1) def tearDown(self): self.mc.flush_all() self.mc.disconnect_all() def check_setget(self, key, val, noreply=False): self.mc.set(key, val, noreply=noreply) newval = self.mc.get(key) self.assertEqual(newval, val) def test_setget(self): self.check_setget("a_string", "some random string") self.check_setget("a_string_2", "some random string", noreply=True) self.check_setget("an_integer", 42) self.check_setget("an_integer_2", 42, noreply=True) def test_delete(self): self.check_setget("long", int(1 << 30)) result = self.mc.delete("long") self.assertEqual(result, True) self.assertEqual(self.mc.get("long"), None) @mock.patch.object(_Host, 'send_cmd') @mock.patch.object(_Host, 'readline') def test_touch(self, mock_readline, mock_send_cmd): with captured_stderr(): self.mc.touch('key') mock_send_cmd.assert_called_with(b'touch key 0') def test_get_multi(self): self.check_setget("gm_a_string", "some random string") self.check_setget("gm_an_integer", 42) self.assertEqual(self.mc.get_multi(["gm_a_string", "gm_an_integer"]), { "gm_an_integer": 42, "gm_a_string": "some random string" }) def test_get_unknown_value(self): self.mc.delete("unknown_value") self.assertEqual(self.mc.get("unknown_value"), None) def test_setget_foostruct(self): f = FooStruct() self.check_setget("foostruct", f) self.check_setget("foostruct_2", f, noreply=True) def test_incr(self): self.check_setget("i_an_integer", 42) self.assertEqual(self.mc.incr("i_an_integer", 1), 43) def test_incr_noreply(self): self.check_setget("i_an_integer_2", 42) self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None) self.assertEqual(self.mc.get("i_an_integer_2"), 43) def test_decr(self): self.check_setget("i_an_integer", 42) self.assertEqual(self.mc.decr("i_an_integer", 1), 41) def test_decr_noreply(self): self.check_setget("i_an_integer_2", 42) self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None) self.assertEqual(self.mc.get("i_an_integer_2"), 41) def test_sending_spaces(self): try: self.mc.set("this has spaces", 1) except Client.MemcachedKeyCharacterError as err: self.assertTrue("characters not allowed" in err.args[0]) else: self.fail( "Expected Client.MemcachedKeyCharacterError, nothing raised") def test_sending_control_characters(self): try: self.mc.set("this\x10has\x11control characters\x02", 1) except Client.MemcachedKeyCharacterError as err: self.assertTrue("characters not allowed" in err.args[0]) else: self.fail( "Expected Client.MemcachedKeyCharacterError, nothing raised") def test_sending_key_too_long(self): try: self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1) except Client.MemcachedKeyLengthError as err: self.assertTrue("length is >" in err.args[0]) else: self.fail( "Expected Client.MemcachedKeyLengthError, nothing raised") # These should work. self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1) self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True) def test_setget_boolean(self): """GitHub issue #75. Set/get with boolean values.""" self.check_setget("bool", True) def test_unicode_key(self): s = u'\u4f1a' maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8')) key = s * maxlen self.mc.set(key, 5) value = self.mc.get(key) self.assertEqual(value, 5) def test_unicode_value(self): key = 'key' value = u'Iñtërnâtiônàlizætiøn2' self.mc.set(key, value) cached_value = self.mc.get(key) self.assertEqual(value, cached_value) def test_binary_string(self): value = 'value_to_be_compressed' compressed_value = zlib.compress(value.encode()) self.mc.set('binary1', compressed_value) compressed_result = self.mc.get('binary1') self.assertEqual(compressed_value, compressed_result) self.assertEqual(value, zlib.decompress(compressed_result).decode()) self.mc.add('binary1-add', compressed_value) compressed_result = self.mc.get('binary1-add') self.assertEqual(compressed_value, compressed_result) self.assertEqual(value, zlib.decompress(compressed_result).decode()) self.mc.set_multi({'binary1-set_many': compressed_value}) compressed_result = self.mc.get('binary1-set_many') self.assertEqual(compressed_value, compressed_result) self.assertEqual(value, zlib.decompress(compressed_result).decode()) def test_ignore_too_large_value(self): # NOTE: "MemCached: while expecting[...]" is normal... key = 'keyhere' value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2) self.assertTrue(self.mc.set(key, value)) self.assertEqual(self.mc.get(key), value) value = 'a' * SERVER_MAX_VALUE_LENGTH with captured_stderr() as log: self.assertIs(self.mc.set(key, value), False) self.assertEqual( log.getvalue(), "MemCached: while expecting 'STORED', got unexpected response " "'SERVER_ERROR object too large for cache'\n") # This test fails if the -I option is used on the memcached server self.assertTrue(self.mc.get(key) is None) def test_get_set_multi_key_prefix(self): """Testing set_multi() with no memcacheds running.""" prefix = 'pfx_' values = {'key1': 'a', 'key2': 'b'} errors = self.mc.set_multi(values, key_prefix=prefix) self.assertEqual(errors, []) keys = list(values) self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values) def test_set_multi_dead_servers(self): """Testing set_multi() with no memcacheds running.""" self.mc.disconnect_all() with captured_stderr() as log: for server in self.mc.servers: server.mark_dead('test') self.assertIn('Marking dead.', log.getvalue()) errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'}) self.assertEqual(sorted(errors), ['key1', 'key2']) def test_disconnect_all_delete_multi(self): """Testing delete_multi() with no memcacheds running.""" self.mc.disconnect_all() with captured_stderr() as output: ret = self.mc.delete_multi(('keyhere', 'keythere')) self.assertEqual(ret, 1) self.assertEqual( output.getvalue(), "MemCached: while expecting 'DELETED', got unexpected response " "'NOT_FOUND'\n" "MemCached: while expecting 'DELETED', got unexpected response " "'NOT_FOUND'\n") @mock.patch.object(_Host, 'send_cmd') # Don't send any commands. @mock.patch.object(_Host, 'readline') def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd): """touch() logs an error upon receiving an unexpected reply.""" mock_readline.return_value = 'SET' # the unexpected reply with captured_stderr() as output: self.mc.touch('key') self.assertEqual( output.getvalue(), "MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED')
class PrintFavicon(BaseHandler): def __init__(self): super(PrintFavicon, self).__init__() default_icon_data = self.open(DEFAULT_FAVICON_LOC, time()).read() self.default_icon = Icon(data=default_icon_data, location=DEFAULT_FAVICON_LOC, type=DEFAULT_FAVICON_TYPE) self.env = Environment(loader=FileSystemLoader( os.path.join(cherrypy.config['favicon.root'], 'templates'))) self.mc = Client( ['%(memcache.host)s:%(memcache.port)d' % cherrypy.config], debug=2) # Initialize counters for counter in ['requests', 'hits', 'defaults']: self.mc.add('counter-%s' % counter, '0') def open(self, url, start, headers=None): time_spent = int(time() - start) if time_spent >= TIMEOUT: raise TimeoutError(time_spent) if not headers: headers = dict() headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; ' + 'rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13' }) opener = build_opener(HTTPRedirectHandler(), HTTPCookieProcessor()) return opener.open(Request(url, headers=headers), timeout=min(CONNECTION_TIMEOUT, TIMEOUT - time_spent)) def validateIconResponse(self, iconResponse): if iconResponse.getcode() != 200: cherrypy.log('Non-success response:%d fetching url:%s' % \ (iconResponse.getcode(), iconResponse.geturl()), severity=INFO) return None iconContentType = iconResponse.info().gettype() if iconContentType in ICON_MIMETYPE_BLACKLIST: cherrypy.log('Url:%s favicon content-Type:%s blacklisted' % \ (iconResponse.geturl(), iconContentType), severity=INFO) return None icon = iconResponse.read() iconLength = len(icon) if iconLength == 0: cherrypy.log('Url:%s null content length' % iconResponse.geturl(), severity=INFO) return None if iconLength < MIN_ICON_LENGTH or iconLength > MAX_ICON_LENGTH: # Issue warning, but accept nonetheless! cherrypy.log('Warning: url:%s favicon size:%d out of bounds' % \ (iconResponse.geturl(), iconLength), severity=INFO) return Icon(data=icon, type=iconContentType) # Icon at [domain]/favicon.ico? def iconAtRoot(self, targetDomain, start): cherrypy.log('Attempting to locate favicon for domain:%s at root' % \ targetDomain, severity=INFO) rootIconPath = targetDomain + '/favicon.ico' try: rootDomainFaviconResult = self.open(rootIconPath, start) rootIcon = self.validateIconResponse(rootDomainFaviconResult) if rootIcon: cherrypy.log('Found favicon for domain:%s at root' % targetDomain, severity=INFO) self.cacheIcon(targetDomain, rootIcon.data, rootIconPath) rootIcon.location = rootIconPath return rootIcon except: cherrypy.log('Error fetching favicon at domain root:%s, err:%s, msg:%s' % \ (targetDomain, sys.exc_info()[0], sys.exc_info()[1]), severity=INFO) # Icon specified in page? def iconInPage(self, targetDomain, targetPath, start, refresh=True): cherrypy.log('Attempting to locate embedded favicon link in page:%s' % \ targetPath, severity=INFO) try: rootDomainPageResult = self.open(targetPath, start) if rootDomainPageResult.getcode() == 200: pageSoup = BeautifulSoup(rootDomainPageResult.read()) pageSoupIcon = pageSoup.find( 'link', rel=compile('^(shortcut|icon|shortcut icon)$', IGNORECASE)) if pageSoupIcon: pageIconHref = pageSoupIcon.get('href') if pageIconHref: pageIconPath = urljoin(targetPath, pageIconHref) cherrypy.log('Found embedded favicon link:%s for domain:%s' % \ (pageIconPath, targetDomain), severity=INFO) cookies = rootDomainPageResult.headers.getheaders( "Set-Cookie") headers = None if cookies: headers = {'Cookie': ';'.join(cookies)} pagePathFaviconResult = self.open(pageIconPath, start, headers=headers) pageIcon = self.validateIconResponse( pagePathFaviconResult) if pageIcon: cherrypy.log('Found favicon at:%s for domain:%s' % \ (pageIconPath, targetDomain), severity=INFO) self.cacheIcon(targetDomain, pageIcon.data, pageIconPath) pageIcon.location = pageIconPath return pageIcon else: if refresh: for meta in pageSoup.findAll('meta'): if meta.get('http-equiv', '').lower() == 'refresh': match = search('url=([^;]+)', meta.get('content', ''), flags=IGNORECASE) if match: refreshPath = urljoin( rootDomainPageResult.geturl(), match.group(1)) cherrypy.log('Processing refresh directive:%s for domain:%s' % \ (refreshPath, targetDomain), severity=INFO) return self.iconInPage(targetDomain, refreshPath, start, refresh=False) cherrypy.log('No link tag found:%s' % targetPath, severity=INFO) else: cherrypy.log('Non-success response:%d for url:%s' % \ (rootDomainPageResult.getcode(), targetPath), severity=INFO) except: cherrypy.log('Error extracting favicon from page:%s, err:%s, msg:%s' % \ (targetPath, sys.exc_info()[0], sys.exc_info()[1]), severity=WARNING) def cacheIcon(self, domain, icon, loc): cherrypy.log('Caching icon at location:%s for domain:%s' % (loc, domain), severity=INFO) if not self.mc.set('icon-%s' % domain, icon, time=MC_CACHE_TIME): cherrypy.log('Could not cache icon for domain:%s' % domain, severity=ERROR) def iconInCache(self, targetDomain, start): icon = self.mc.get('icon-%s' % targetDomain) if icon: self.mc.incr('counter-hits') cherrypy.log('Cache hit:%s' % targetDomain, severity=INFO) cherrypy.response.headers['X-Cache'] = 'Hit' if icon == 'DEFAULT': self.mc.incr('counter-defaults') cherrypy.response.headers['X-Cache'] = 'Hit' return self.default_icon else: return Icon(data=icon) def writeIcon(self, icon): self.writeHeaders(icon) return icon.data def writeHeaders(self, icon, fmt='%a, %d %b %Y %H:%M:%S %z'): # MIME Type cherrypy.response.headers['Content-Type'] = icon.type or 'image/x-icon' # Set caching headers cherrypy.response.headers['Cache-Control'] = 'public, max-age=2592000' cherrypy.response.headers['Expires'] = \ (datetime.now() + timedelta(days=30)).strftime(fmt) def parse(self, url): # Get page path targetPath = self.urldecode(url) if not targetPath.startswith('http'): targetPath = 'http://%s' % targetPath cherrypy.log('Decoded URL:%s' % targetPath, severity=INFO) # Split path to get domain targetURL = urlparse(targetPath) if not targetURL or not targetURL.scheme or not targetURL.netloc: raise cherrypy.HTTPError(400, 'Malformed URL:%s' % url) targetDomain = '%s://%s' % (targetURL.scheme, targetURL.netloc) cherrypy.log('URL:%s, domain:%s' % (targetPath, targetDomain), severity=INFO) return (targetPath, targetDomain) @cherrypy.expose def index(self): status = {'status': 'ok', 'counters': dict()} for counter in ['requests', 'hits', 'defaults']: status['counters'][counter] = self.mc.get('counter-%s' % counter) return json.dumps(status) @cherrypy.expose def test(self): topSites = open( os.path.join(cherrypy.config['favicon.root'], 'topsites.txt'), 'r').read().split() template = self.env.get_template('test.html') return template.render(topSites=topSites) @cherrypy.expose def clear(self, url): cherrypy.log('Incoming cache invalidation request:%s' % url, severity=INFO) targetPath, targetDomain = self.parse(str(url)) self.mc.delete('icon_loc-%s' % targetDomain) cherrypy.log('Evicted cache entry for %s' % targetDomain, severity=INFO) @cherrypy.expose def s(self, url, skipCache='false'): start = time() if skipCache.lower() == 'true': skipCache = True else: skipCache = False cherrypy.log('Incoming request:%s (skipCache=%s)' % (url, skipCache), severity=INFO) self.mc.incr('counter-requests') targetPath, targetDomain = self.parse(str(url)) icon = (not skipCache and self.iconInCache(targetDomain, start)) or \ self.iconInPage(targetDomain, targetPath, start) or \ self.iconAtRoot(targetDomain, start) if not icon: cherrypy.log('Falling back to default icon for:%s' % targetDomain, severity=INFO) self.cacheIcon(targetDomain, 'DEFAULT', 'DEFAULT_LOC') self.mc.incr('counter-defaults') icon = self.default_icon cherrypy.log('Time taken to process domain:%s %f' % \ (targetDomain, time() - start), severity=INFO) return self.writeIcon(icon)
class PrintFavicon(BaseHandler): def __init__(self): super(PrintFavicon, self).__init__() default_icon_data = self.open(DEFAULT_FAVICON_LOC, time()).read() self.default_icon = Icon(data=default_icon_data, location=DEFAULT_FAVICON_LOC, type=DEFAULT_FAVICON_TYPE) self.env = Environment(loader=FileSystemLoader(os.path.join(cherrypy.config["favicon.root"], "templates"))) self.mc = Client(["%(memcache.host)s:%(memcache.port)d" % cherrypy.config], debug=2) # Initialize counters for counter in ["requests", "hits", "defaults"]: self.mc.add("counter-%s" % counter, "0") def open(self, url, start, headers=None): time_spent = int(time() - start) if time_spent >= TIMEOUT: raise TimeoutError(time_spent) if not headers: headers = dict() headers.update( { "User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; " + "rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13" } ) opener = build_opener(HTTPRedirectHandler(), HTTPCookieProcessor()) return opener.open(Request(url, headers=headers), timeout=min(CONNECTION_TIMEOUT, TIMEOUT - time_spent)) def validateIconResponse(self, iconResponse): if iconResponse.getcode() != 200: cherrypy.log( "Non-success response:%d fetching url:%s" % (iconResponse.getcode(), iconResponse.geturl()), severity=INFO, ) return None iconContentType = iconResponse.info().gettype() if iconContentType in ICON_MIMETYPE_BLACKLIST: cherrypy.log( "Url:%s favicon content-Type:%s blacklisted" % (iconResponse.geturl(), iconContentType), severity=INFO ) return None icon = iconResponse.read() iconLength = len(icon) if iconLength == 0: cherrypy.log("Url:%s null content length" % iconResponse.geturl(), severity=INFO) return None if iconLength < MIN_ICON_LENGTH or iconLength > MAX_ICON_LENGTH: # Issue warning, but accept nonetheless! cherrypy.log( "Warning: url:%s favicon size:%d out of bounds" % (iconResponse.geturl(), iconLength), severity=INFO ) return Icon(data=icon, type=iconContentType) # Icon at [domain]/favicon.ico? def iconAtRoot(self, targetDomain, start): cherrypy.log("Attempting to locate favicon for domain:%s at root" % targetDomain, severity=INFO) rootIconPath = targetDomain + "/favicon.ico" try: rootDomainFaviconResult = self.open(rootIconPath, start) rootIcon = self.validateIconResponse(rootDomainFaviconResult) if rootIcon: cherrypy.log("Found favicon for domain:%s at root" % targetDomain, severity=INFO) self.cacheIcon(targetDomain, rootIcon.data, rootIconPath) rootIcon.location = rootIconPath return rootIcon except: cherrypy.log( "Error fetching favicon at domain root:%s, err:%s, msg:%s" % (targetDomain, sys.exc_info()[0], sys.exc_info()[1]), severity=INFO, ) # Icon specified in page? def iconInPage(self, targetDomain, targetPath, start, refresh=True): cherrypy.log("Attempting to locate embedded favicon link in page:%s" % targetPath, severity=INFO) try: rootDomainPageResult = self.open(targetPath, start) if rootDomainPageResult.getcode() == 200: pageSoup = BeautifulSoup(rootDomainPageResult.read()) pageSoupIcon = pageSoup.find("link", rel=compile("^(shortcut|icon|shortcut icon)$", IGNORECASE)) if pageSoupIcon: pageIconHref = pageSoupIcon.get("href") if pageIconHref: pageIconPath = urljoin(targetPath, pageIconHref) cherrypy.log( "Found embedded favicon link:%s for domain:%s" % (pageIconPath, targetDomain), severity=INFO ) cookies = rootDomainPageResult.headers.getheaders("Set-Cookie") headers = None if cookies: headers = {"Cookie": ";".join(cookies)} pagePathFaviconResult = self.open(pageIconPath, start, headers=headers) pageIcon = self.validateIconResponse(pagePathFaviconResult) if pageIcon: cherrypy.log( "Found favicon at:%s for domain:%s" % (pageIconPath, targetDomain), severity=INFO ) self.cacheIcon(targetDomain, pageIcon.data, pageIconPath) pageIcon.location = pageIconPath return pageIcon else: if refresh: for meta in pageSoup.findAll("meta"): if meta.get("http-equiv", "").lower() == "refresh": match = search("url=([^;]+)", meta.get("content", ""), flags=IGNORECASE) if match: refreshPath = urljoin(rootDomainPageResult.geturl(), match.group(1)) cherrypy.log( "Processing refresh directive:%s for domain:%s" % (refreshPath, targetDomain), severity=INFO, ) return self.iconInPage(targetDomain, refreshPath, start, refresh=False) cherrypy.log("No link tag found:%s" % targetPath, severity=INFO) else: cherrypy.log( "Non-success response:%d for url:%s" % (rootDomainPageResult.getcode(), targetPath), severity=INFO ) except: cherrypy.log( "Error extracting favicon from page:%s, err:%s, msg:%s" % (targetPath, sys.exc_info()[0], sys.exc_info()[1]), severity=WARNING, ) def cacheIcon(self, domain, icon, loc): cherrypy.log("Caching icon at location:%s for domain:%s" % (loc, domain), severity=INFO) if not self.mc.set("icon-%s" % domain, icon, time=MC_CACHE_TIME): cherrypy.log("Could not cache icon for domain:%s" % domain, severity=ERROR) def iconInCache(self, targetDomain, start): icon = self.mc.get("icon-%s" % targetDomain) if icon: self.mc.incr("counter-hits") cherrypy.log("Cache hit:%s" % targetDomain, severity=INFO) cherrypy.response.headers["X-Cache"] = "Hit" if icon == "DEFAULT": self.mc.incr("counter-defaults") cherrypy.response.headers["X-Cache"] = "Hit" return self.default_icon else: return Icon(data=icon) def writeIcon(self, icon): self.writeHeaders(icon) return icon.data def writeHeaders(self, icon, fmt="%a, %d %b %Y %H:%M:%S %z"): # MIME Type cherrypy.response.headers["Content-Type"] = icon.type or "image/x-icon" # Set caching headers cherrypy.response.headers["Cache-Control"] = "public, max-age=2592000" cherrypy.response.headers["Expires"] = (datetime.now() + timedelta(days=30)).strftime(fmt) def parse(self, url): # Get page path targetPath = self.urldecode(url) if not targetPath.startswith("http"): targetPath = "http://%s" % targetPath cherrypy.log("Decoded URL:%s" % targetPath, severity=INFO) # Split path to get domain targetURL = urlparse(targetPath) if not targetURL or not targetURL.scheme or not targetURL.netloc: raise cherrypy.HTTPError(400, "Malformed URL:%s" % url) targetDomain = "%s://%s" % (targetURL.scheme, targetURL.netloc) cherrypy.log("URL:%s, domain:%s" % (targetPath, targetDomain), severity=INFO) return (targetPath, targetDomain) @cherrypy.expose def index(self): status = {"status": "ok", "counters": dict()} for counter in ["requests", "hits", "defaults"]: status["counters"][counter] = self.mc.get("counter-%s" % counter) return json.dumps(status) @cherrypy.expose def test(self): topSites = open(os.path.join(cherrypy.config["favicon.root"], "topsites.txt"), "r").read().split() template = self.env.get_template("test.html") return template.render(topSites=topSites) @cherrypy.expose def clear(self, url): cherrypy.log("Incoming cache invalidation request:%s" % url, severity=INFO) targetPath, targetDomain = self.parse(str(url)) self.mc.delete("icon_loc-%s" % targetDomain) cherrypy.log("Evicted cache entry for %s" % targetDomain, severity=INFO) @cherrypy.expose def s(self, url, skipCache="false"): start = time() if skipCache.lower() == "true": skipCache = True else: skipCache = False cherrypy.log("Incoming request:%s (skipCache=%s)" % (url, skipCache), severity=INFO) self.mc.incr("counter-requests") targetPath, targetDomain = self.parse(str(url)) icon = ( (not skipCache and self.iconInCache(targetDomain, start)) or self.iconInPage(targetDomain, targetPath, start) or self.iconAtRoot(targetDomain, start) ) if not icon: cherrypy.log("Falling back to default icon for:%s" % targetDomain, severity=INFO) self.cacheIcon(targetDomain, "DEFAULT", "DEFAULT_LOC") self.mc.incr("counter-defaults") icon = self.default_icon cherrypy.log("Time taken to process domain:%s %f" % (targetDomain, time() - start), severity=INFO) return self.writeIcon(icon)
class MemcachedCacheClient(CacheClient): """Memcached cache client implementation.""" def __init__(self, config): super(MemcachedCacheClient, self).__init__(config["host"], config["port"], config["cache"]) self.config = config if self.cache_name != DEFAULT_CACHE_NAME: print "WARNING: memcached client doesn't support named caches. cache_name config value will be ignored and default cache will be used instead." self.memcached_client = Client([self.host + ':' + self.port], debug=0) return def put(self, key, value, version=None, lifespan=None, max_idle=None, put_if_absent=False): time = 0 if lifespan != None: if lifespan > MEMCACHED_LIFESPAN_MAX_SECONDS: self._error("Memcached cache client supports lifespan values only up to %s seconds (30 days)." % MEMCACHED_LIFESPAN_MAX_SECONDS) time = lifespan if max_idle != None: self._error("Memcached cache client doesn't support max idle time setting.") try: if (version == None): if (put_if_absent): if not self.memcached_client.add(key, value, time, 0): # current python-memcached doesn't recoginze these states # if self.memcached_client.last_set_status == "NOT_STORED": # raise ConflictError # else: # self._error("Operation unsuccessful. " + self.memcached_client.last_set_status) self._error("Operation unsuccessful. Possibly CONFLICT.") else: if not self.memcached_client.set(key, value, time, 0): # self._error("Operation unsuccessful. " + self.memcached_client.last_set_status) self._error("Operation unsuccessful.") else: try: self.memcached_client.cas_ids[key] = int(version) except ValueError: self._error("Please provide an integer version.") if not self.memcached_client.cas(key, value, time, 0): # if self.memcached_client.last_set_status == "EXISTS": # raise ConflictError # if self.memcached_client.last_set_status == "NOT_FOUND": # raise NotFoundError # else: # self._error("Operation unsuccessful. " + self.memcached_client.last_set_status) self._error("Operation unsuccessful. Possibly CONFLICT, NOT_FOUND.") except CacheClientError as e: raise e #rethrow except Exception as e: self._error(e) def get(self, key, get_version=False): try: if get_version: val = self.memcached_client.gets(key) if val == None: raise NotFoundError version = self.memcached_client.cas_ids[key] if version == None: self._error("Couldn't obtain version info from memcached server.") return version, val else: val = self.memcached_client.get(key) if val == None: raise NotFoundError return val except CacheClientError as e: raise e #rethrow except Exception as e: self._error(e.args) def delete(self, key, version=None): try: if version: self._error("versioned delete operation not available for memcached client") if self.memcached_client.delete(key, 0): if self.memcached_client.last_set_status == "NOT_FOUND": raise NotFoundError else: self._error("Operation unsuccessful. " + self.memcached_client.last_set_status) except CacheClientError as e: raise e #rethrow except Exception as e: self._error(e.args) def clear(self): try: self.memcached_client.flush_all() except CacheClientError as e: raise e #rethrow except Exception as e: self._error(e.args)
class MemcacheFeatureStorage(FeatureStorage): PREFIX = 'georest_buckets' support_version = False def __init__(self, hosts): """ Feature storage implemented in Memcache :param list hosts: list of hosts 1. Strings of the form C{"host:port"} 2. Tuples of the form C{("host:port", weight)} :rtype :class:`MemcacheFeatureStorage` """ self._client = Client(servers=hosts) def create_bucket(self, name, overwrite=False, **kwargs): bucket_name = self._make_bucket_name(name) timestamp = time.time() try: add_ok = self._client.add(key=bucket_name, val=timestamp) except Exception as e: raise StorageInternalError(message='add error', e=e) if not add_ok: if overwrite: try: rep_ok = self._client.replace( key=bucket_name, val=timestamp) except Exception as e: raise StorageInternalError('replace error', e=e) if not rep_ok: raise StorageInternalError(message='failed to replace') else: raise DuplicatedBucket(name) return MemcacheFeatureBucket(name, self._client, str(timestamp)) def get_bucket(self, name): bucket_name = self._make_bucket_name(name) try: timestamp = self._client.get(bucket_name) except Exception as e: raise StorageInternalError(message='get error', e=e) if not timestamp: raise BucketNotFound(name) return MemcacheFeatureBucket(name, self._client, str(timestamp)) def delete_bucket(self, name): bucket_name = self._make_bucket_name(name) try: delete_ok = self._client.delete(bucket_name) except Exception as e: raise StorageInternalError(message='delete error', e=e) if not delete_ok: raise BucketNotFound(name) return True def has_bucket(self, name): bucket_name = self._make_bucket_name(name) try: get_ok = self._client.get(bucket_name) except Exception as e: raise StorageInternalError(message='get error', e=e) return get_ok is not None def close(self): pass def _make_bucket_name(self, name): if isinstance(name, unicode): name = name.encode('utf-8') return '.'.join((self.PREFIX, name))
from memcache import Client """ 安装memcache时,遇到的问题,提示libevent 解决 yum install libevent yum install libevent-devel """ # 这里是个list,可以吧memcache集群这么搞 MC_SERVERS = ['192.168.52.3:11211', '192.168.52.3:11212'] CONN = Client(MC_SERVERS) status = CONN.set('key1', 'val2', 0) print(status) status = CONN.delete('key') print(status) status = CONN.add('key', 'val', 20) print(status) status = CONN.replace('key', 'val1', 0) print(status) status = CONN.append('key', ',val2') print(status) data = CONN.get('key') print(data)
class TestMemcache(unittest.TestCase): def setUp(self): # TODO(): unix socket server stuff servers = ["127.0.0.1:11211"] self.mc = Client(servers, debug=1) def tearDown(self): self.mc.flush_all() self.mc.disconnect_all() def check_setget(self, key, val, noreply=False): self.mc.set(key, val, noreply=noreply) newval = self.mc.get(key) self.assertEqual(newval, val) def test_setget(self): self.check_setget("a_string", "some random string") self.check_setget("a_string_2", "some random string", noreply=True) self.check_setget("an_integer", 42) self.check_setget("an_integer_2", 42, noreply=True) def test_delete(self): self.check_setget("long", int(1 << 30)) result = self.mc.delete("long") self.assertEqual(result, True) self.assertEqual(self.mc.get("long"), None) @mock.patch.object(_Host, 'send_cmd') @mock.patch.object(_Host, 'readline') def test_touch(self, mock_readline, mock_send_cmd): with captured_stderr(): self.mc.touch('key') mock_send_cmd.assert_called_with(b'touch key 0') def test_get_multi(self): self.check_setget("gm_a_string", "some random string") self.check_setget("gm_an_integer", 42) self.assertEqual( self.mc.get_multi(["gm_a_string", "gm_an_integer"]), {"gm_an_integer": 42, "gm_a_string": "some random string"}) def test_get_unknown_value(self): self.mc.delete("unknown_value") self.assertEqual(self.mc.get("unknown_value"), None) def test_setget_foostruct(self): f = FooStruct() self.check_setget("foostruct", f) self.check_setget("foostruct_2", f, noreply=True) def test_incr(self): self.check_setget("i_an_integer", 42) self.assertEqual(self.mc.incr("i_an_integer", 1), 43) def test_incr_noreply(self): self.check_setget("i_an_integer_2", 42) self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None) self.assertEqual(self.mc.get("i_an_integer_2"), 43) def test_decr(self): self.check_setget("i_an_integer", 42) self.assertEqual(self.mc.decr("i_an_integer", 1), 41) def test_decr_noreply(self): self.check_setget("i_an_integer_2", 42) self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None) self.assertEqual(self.mc.get("i_an_integer_2"), 41) def test_sending_spaces(self): try: self.mc.set("this has spaces", 1) except Client.MemcachedKeyCharacterError as err: self.assertTrue("characters not allowed" in err.args[0]) else: self.fail( "Expected Client.MemcachedKeyCharacterError, nothing raised") def test_sending_control_characters(self): try: self.mc.set("this\x10has\x11control characters\x02", 1) except Client.MemcachedKeyCharacterError as err: self.assertTrue("characters not allowed" in err.args[0]) else: self.fail( "Expected Client.MemcachedKeyCharacterError, nothing raised") def test_sending_key_too_long(self): try: self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1) except Client.MemcachedKeyLengthError as err: self.assertTrue("length is >" in err.args[0]) else: self.fail( "Expected Client.MemcachedKeyLengthError, nothing raised") # These should work. self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1) self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True) def test_setget_boolean(self): """GitHub issue #75. Set/get with boolean values.""" self.check_setget("bool", True) def test_unicode_key(self): s = u'\u4f1a' maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8')) key = s * maxlen self.mc.set(key, 5) value = self.mc.get(key) self.assertEqual(value, 5) def test_unicode_value(self): key = 'key' value = u'Iñtërnâtiônàlizætiøn2' self.mc.set(key, value) cached_value = self.mc.get(key) self.assertEqual(value, cached_value) def test_binary_string(self): value = 'value_to_be_compressed' compressed_value = zlib.compress(value.encode()) self.mc.set('binary1', compressed_value) compressed_result = self.mc.get('binary1') self.assertEqual(compressed_value, compressed_result) self.assertEqual(value, zlib.decompress(compressed_result).decode()) self.mc.add('binary1-add', compressed_value) compressed_result = self.mc.get('binary1-add') self.assertEqual(compressed_value, compressed_result) self.assertEqual(value, zlib.decompress(compressed_result).decode()) self.mc.set_multi({'binary1-set_many': compressed_value}) compressed_result = self.mc.get('binary1-set_many') self.assertEqual(compressed_value, compressed_result) self.assertEqual(value, zlib.decompress(compressed_result).decode()) def test_ignore_too_large_value(self): # NOTE: "MemCached: while expecting[...]" is normal... key = 'keyhere' value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2) self.assertTrue(self.mc.set(key, value)) self.assertEqual(self.mc.get(key), value) value = 'a' * SERVER_MAX_VALUE_LENGTH with captured_stderr() as log: self.assertIs(self.mc.set(key, value), False) self.assertEqual( log.getvalue(), "MemCached: while expecting 'STORED', got unexpected response " "'SERVER_ERROR object too large for cache'\n" ) # This test fails if the -I option is used on the memcached server self.assertTrue(self.mc.get(key) is None) def test_get_set_multi_key_prefix(self): """Testing set_multi() with no memcacheds running.""" prefix = 'pfx_' values = {'key1': 'a', 'key2': 'b'} errors = self.mc.set_multi(values, key_prefix=prefix) self.assertEqual(errors, []) keys = list(values) self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values) def test_set_multi_dead_servers(self): """Testing set_multi() with no memcacheds running.""" self.mc.disconnect_all() with captured_stderr() as log: for server in self.mc.servers: server.mark_dead('test') self.assertIn('Marking dead.', log.getvalue()) errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'}) self.assertEqual(sorted(errors), ['key1', 'key2']) def test_disconnect_all_delete_multi(self): """Testing delete_multi() with no memcacheds running.""" self.mc.disconnect_all() with captured_stderr() as output: ret = self.mc.delete_multi(('keyhere', 'keythere')) self.assertEqual(ret, 1) self.assertEqual( output.getvalue(), "MemCached: while expecting 'DELETED', got unexpected response " "'NOT_FOUND'\n" "MemCached: while expecting 'DELETED', got unexpected response " "'NOT_FOUND'\n" ) @mock.patch.object(_Host, 'send_cmd') # Don't send any commands. @mock.patch.object(_Host, 'readline') def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd): """touch() logs an error upon receiving an unexpected reply.""" mock_readline.return_value = 'SET' # the unexpected reply with captured_stderr() as output: self.mc.touch('key') self.assertEqual( output.getvalue(), "MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED' )