def remove(self, layer, coord, format):
""" Remove a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision)
mem.delete(key)
class MemcacheStore(DictProxy):
def __init__(self, host, port):
from memcache import Client
self.mc= Client(['%s:%s' % (host, port)])
def update(self, updates):
for update in updates:
(processId, data)= update
self.mc.set(processId, data)
def get(self, processId, default= None):
data= self.mc.get(processId)
if data == None:
return default
return data
def pop(self, processId):
data= self.mc.get(processId)
self.mc.delete(processId)
return data
if data == None:
return default
def __len__(self):
return int(self.mc.get_stats()[0][1].get('curr_items'))
class OODictMongoMemcache(OODictMongo):
"""Add memcache caching capabilities to a OODictMongo."""
def __init__(self,
memcache_host,
mongo_uri,
mongo_db,
mongo_coll,
mongo_id="_id",
mongo_client=None,
memcache_client=None,
_dict=None):
super(OODictMongoMemcache,
self).__init__(mongo_uri, mongo_db, mongo_coll, mongo_id,
mongo_client, _dict)
self._memcache_host = memcache_host
## self._memcache_lock = memache_lock
if memcache_client is None:
self._memcache = MemcacheClient(memcache_host)
else:
self._memcache = memcache_client
def _memcache_key(self, mongo_id=None):
return str("%s:%s:%s" % (self._mongo_db, self._mongo_coll, mongo_id
or self._dict.get(self._mongo_id, '')))
def get_from_field(self, key, value, flush=False):
"""Get user by a key:value pair from mongo or memcache."""
# if searching by id key, then we can find it in memcache
if not flush and key == self._mongo_id:
item = self._memcache.get(self._memcache_key(value))
if item:
log.info("Cache hit.")
return self._reinit(item)
log.info("Cache miss.")
# didn't find it in memcache, search in mongo and update cache
super(OODictMongoMemcache, self).get_from_field(key, value)
item = self._dict
if item:
self._memcache.set(self._memcache_key(), item)
return self._reinit(item)
def refresh(self, flush=False):
"""Refresh self data from memcache. If flush is True, then
flush memcache entry and force a refresh from mongo.
"""
self.get_from_field(self._mongo_id, self._dict[self._mongo_id], flush)
def save(self):
"""Save user data to storage."""
self._memcache.set(self._memcache_key(), self._dict)
super(OODictMongoMemcache, self).save()
def delete(self):
"""Delete user from storage."""
self._memcache.delete(self._memcache_key())
super(OODictMongoMemcache, self).delete()
def unlock(self, layer, coord, format):
""" Release a cache lock for this tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
mem.delete(key + "-lock")
mem.disconnect_all()
def remove(self, layer, coord, format):
""" Remove a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
mem.delete(key)
mem.disconnect_all()
def unlock(self, layer, coord, format):
""" Release a cache lock for this tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
mem.delete(key + '-lock')
mem.disconnect_all()
def remove(self, layer, coord, format):
""" Remove a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
mem.delete(key)
mem.disconnect_all()
class OODictMongoMemcache(OODictMongo):
"""Add memcache caching capabilities to a OODictMongo."""
def __init__(self, memcache_host, mongo_uri, mongo_db, mongo_coll,
mongo_id="_id", mongo_client=None, memcache_client=None,
_dict=None):
super(OODictMongoMemcache, self).__init__(
mongo_uri, mongo_db, mongo_coll, mongo_id,
mongo_client, _dict
)
self._memcache_host = memcache_host
## self._memcache_lock = memache_lock
if memcache_client is None:
self._memcache = MemcacheClient(memcache_host)
else:
self._memcache = memcache_client
def _memcache_key(self, mongo_id=None):
return str("%s:%s:%s" % (self._mongo_db, self._mongo_coll,
mongo_id or self._dict.get(self._mongo_id, '')))
def get_from_field(self, key, value, flush=False):
"""Get user by a key:value pair from mongo or memcache."""
# if searching by id key, then we can find it in memcache
if not flush and key == self._mongo_id:
item = self._memcache.get(self._memcache_key(value))
if item:
log.info("Cache hit.")
return self._reinit(item)
log.info("Cache miss.")
# didn't find it in memcache, search in mongo and update cache
super(OODictMongoMemcache, self).get_from_field(key, value)
item = self._dict
if item:
self._memcache.set(self._memcache_key(), item)
return self._reinit(item)
def refresh(self, flush=False):
"""Refresh self data from memcache. If flush is True, then
flush memcache entry and force a refresh from mongo.
"""
self.get_from_field(self._mongo_id, self._dict[self._mongo_id], flush)
def save(self):
"""Save user data to storage."""
self._memcache.set(self._memcache_key(), self._dict)
super(OODictMongoMemcache, self).save()
def delete(self):
"""Delete user from storage."""
self._memcache.delete(self._memcache_key())
super(OODictMongoMemcache, self).delete()
class MemcachedManager():
class Prefix():
Session = 'SESSION'
SysCache = 'ORDER_SYS_CACHE'
def __init__(self):
# if settings.ENVIRONMENT['dev']:
# pass
# else:
self.conn = Client(settings.CACHE['clients'])
def get(self, *arg):
if len(arg) ==0:
return None
arg = [str(i) for i in arg]
key = PRODUCT_PREFIX+ '|'.join(arg)
try:
# if settings.ENVIRONMENT['dev']:
# return _localCache.get(key)
# else:
return self.conn.get(key)
# self.conn.
except:# Client.MemcachedKeyNoneError:
return None
def set(self, value, timeout, *arg):
if len(arg) ==0:
return None
arg = [str(i) for i in arg]
key = PRODUCT_PREFIX+'|'.join(arg)
# if settings.ENVIRONMENT['dev']:
# _localCache[key] = value
# else:
self.conn.set(key, value, timeout)
def delete(self, *arg):
if len(arg) ==0:
return None
arg = [str(i) for i in arg]
key = PRODUCT_PREFIX+'|'.join(arg)
try:
# if settings.ENVIRONMENT['dev']:
# if _localCache.has_key(key):
# _localCache.pop(key)
# else:
self.conn.delete(key)
except: # Client.MemcachedKeyNoneError:
return None
class SessionHandler:
inst = None
def __init__(self):
SessionHandler.inst = self
self.conn = Client(["127.0.0.1:11211"])
def setuid(self, uid):
key = uuid4().get_hex()
self.conn.set(key, uid, 3600)
return key
def getuid(self, key):
return self.conn.get(key)
def deluid(self, key):
self.conn.delete(key)
class ResetCodeMemcache(ResetCode):
""" Implements the reset code methods for auth backends.
"""
def __init__(self,
product='auth',
nodes=None,
debug=0,
expiration=_6HOURS,
**kw):
if nodes is None:
nodes = ['127.0.0.1:11211']
self._engine = Client(nodes, debug)
self.product = product
self.expiration = expiration
#
# Private methods
#
def _get_reset_code(self, user_id):
return self._engine.get(self._generate_key(user_id))
def _generate_key(self, user_id):
return "reset:%s:%s" % (user_id, self.product)
def _set_reset_code(self, user_id):
code = self._generate_reset_code()
key = self._generate_key(user_id)
if not self._engine.set(key, code, self.expiration):
raise BackendError()
return code
#
# Public methods
#
def generate_reset_code(self, user, overwrite=False):
user_id = self._get_user_id(user)
if not overwrite:
stored_code = self._get_reset_code(user_id)
if stored_code is not None:
return stored_code
return self._set_reset_code(user_id)
def verify_reset_code(self, user, code):
user_id = self._get_user_id(user)
if not self._check_reset_code(code):
return False
stored_code = self._get_reset_code(user_id)
if stored_code is None:
return False
return stored_code == code
def clear_reset_code(self, user):
user_id = self._get_user_id(user)
return self._engine.delete(self._generate_key(user_id))
class MemCacheWrapper(object):
"""
Memcache client wrapper.
No exception raise and add some useful function.
"""
def __init__(self, servers, logerr=None):
self.cache = MCClient(servers=servers, debug=False)
self.logerr = logerr
def add(self, key, val=1, time=0):
try:
return self.cache.add(key, val, time)
except Exception as e:
_logger.warning("Exception during `add`: %s", e)
return None
def count(self, key, expires=0, delta=1):
try:
result = self.cache.incr(key, delta)
if result is None:
if not self.cache.add(key, delta, expires):
result = self.cache.incr(key, delta)
else:
return delta
return result
except Exception as e:
_logger.warning("Exception during `count`: %s", e)
return None
def get(self, key):
result = None
try:
result = self.cache.get(str(key))
except Exception as e:
_logger.warning("Exception during `get`: %s", e)
return result
def set(self, key, value, expires):
result = False
try:
result = self.cache.set(str(key), value, expires)
except Exception as e:
_logger.warning("Exception during `set`: %s", e)
return result
def delete(self, key):
result = False
try:
result = self.cache.delete(key)
except Exception as e:
_logger.warning("Exception during `del`: %s", e)
return result
class Memcache(object):
"""
address=127.0.0.1:11211
# TODO 链接断开后没有自动重连机制
"""
def __init__(self, address):
from memcache import Client
self.conn = Client([address])
def get(self, key):
return self.conn.get(key)
def set(self, key, value, timeout):
self.conn.set(key, value, timeout)
def delete(self, key):
self.conn.delete(key)
class TransactionalMemcacheClient(object):
def __init__(self, *args):
from memcache import Client
self.mc = Client(*args)
self.del_que = []
self.random = Random()
self.random.seed()
from threading import Thread
self.del_thread = Thread(target = lambda:self._async_delete())
self.del_thread.setDaemon(True)
self.del_thread.start()
def __getattr__(self, attrname):
return getattr(self.mc, attrname)
def _async_delete(self):
while True:
try:
sleep(5)
while 0 < len(self.del_que):
target = self.del_que.pop(0)
if target != None:
self.mc.delete(target)
except Exception, e:
print e
exit()
class TestCacheDecoratorWithMemcache(unittest.TestCase):
def setUp(self):
server = ["127.0.0.1:11211"]
self.cache = Client(server)
self.user_id = 5
def test_cache_is_empty(self):
value = self.cache.get(str(self.user_id))
self.assertTrue(value is None)
def test_get_long_response_set_value_to_cache(self):
cached, result = get_long_response(self.user_id)
self.assertTrue(cached is False)
self.assertTrue(result == self.cache.get(str(self.user_id)))
def test_get_long_response_gets_value_from_cache(self):
cached_first_call, result_first_call = get_long_response(self.user_id)
cached_second_call, result_second_call = get_long_response(
self.user_id)
self.assertTrue(cached_second_call is True)
self.assertTrue(result_second_call == result_first_call)
def test_get_long_response_uses_user_id_as_unique_key(self):
another_user_id = 7
cached_user1_1, result_user1_1 = get_long_response(self.user_id)
cached_user1_2, result_user1_2 = get_long_response(self.user_id)
cached_user2_1, result_user2_1 = get_long_response(another_user_id)
cached_user2_2, result_user2_2 = get_long_response(another_user_id)
self.assertTrue(cached_user1_2 is True)
self.assertTrue(cached_user2_2 is True)
self.assertTrue(result_user1_1 == result_user1_2)
self.assertTrue(result_user2_1 == result_user2_2)
self.assertFalse(result_user2_2 == result_user1_2)
# remove entry in memcached.
self.cache.delete(str(7))
def test_delete_entry_in_cache_from_outside(self):
cached, result = get_long_response(self.user_id)
self.assertTrue(cached is False)
value_in_cache = self.cache.get(str(self.user_id))
self.assertTrue(result == value_in_cache)
self.cache.delete(str(self.user_id))
self.assertTrue(self.cache.get(str(self.user_id)) is None)
def tearDown(self):
self.cache.delete(str(self.user_id))
self.cache.disconnect_all()
class MemcacheMemory(Memory):
"""
Memory gateway to a Memcache server
"""
def __init__(self, servers=["127.0.0.1:11211"], expire=0, debug=False):
"""
:param servers: List of servers to use. Please, read
memcache.Client help.
"""
self._client = MemcacheClient(servers)
self._expire = expire
logging.basicConfig(level=logging.WARNING)
self.log = logging.getLogger("Memcache-Gateway")
if debug:
self.log.setLevel(logging.DEBUG)
def __getitem__(self, key):
self.log.debug("Accessing key %s", key)
value = self._client.get(key)
if isinstance(value, NotSet):
value = None
elif value is None:
raise KeyError
return value
def __setitem__(self, key, value):
self.log.debug("Setting key")
if value is None:
value = NotSet()
self._client.set(key, value, self._expire)
def __delitem__(self, key):
self.log.debug("Deleting key %s", key)
if self._client.delete(key) == 0:
raise KeyError
def open(self, key):
return KeyFile(self, key)
class MemcachedCacheStore(AbstractCacheStore):
servers = ("127.0.0.1:11211")
def __init__(self, servers=None, debug=False):
if servers is None:
servers = self.servers
from memcache import Client as MemcachedClient
self._client = MemcachedClient(servers, debug)
def set(self, key, val, time=0):
self._client.set(key, val, time)
def add(self, key, val, time=0):
res = self._client.add(key, val, time)
if not res:
raise Error("a value for key %r is already in the cache" % key)
self._data[key] = (val, time)
def replace(self, key, val, time=0):
res = self._client.replace(key, val, time)
if not res:
raise Error("a value for key %r is already in the cache" % key)
self._data[key] = (val, time)
def delete(self, key):
res = self._client.delete(key, time=0)
if not res:
raise KeyError(key)
def get(self, key):
val = self._client.get(key)
if val is None:
raise KeyError(key)
return val
def clear(self):
self._client.flush_all()
class MemcachedCacheStore(AbstractCacheStore):
servers = ('127.0.0.1:11211')
def __init__(self, servers=None, debug=False):
if servers is None:
servers = self.servers
from memcache import Client as MemcachedClient
self._client = MemcachedClient(servers, debug)
def set(self, key, val, time=0):
self._client.set(key, val, time)
def add(self, key, val, time=0):
res = self._client.add(key, val, time)
if not res:
raise Error('a value for key %r is already in the cache'%key)
self._data[key] = (val, time)
def replace(self, key, val, time=0):
res = self._client.replace(key, val, time)
if not res:
raise Error('a value for key %r is already in the cache'%key)
self._data[key] = (val, time)
def delete(self, key):
res = self._client.delete(key, time=0)
if not res:
raise KeyError(key)
def get(self, key):
val = self._client.get(key)
if val is None:
raise KeyError(key)
else:
return val
def clear(self):
self._client.flush_all()
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.flush_all()
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
@mock.patch.object(_Host, 'send_cmd')
@mock.patch.object(_Host, 'readline')
def test_touch(self, mock_readline, mock_send_cmd):
with captured_stderr():
self.mc.touch('key')
mock_send_cmd.assert_called_with(b'touch key 0')
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = u'\u4f1a'
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_unicode_value(self):
key = 'key'
value = u'Iñtërnâtiônàlizætiøn2'
self.mc.set(key, value)
cached_value = self.mc.get(key)
self.assertEqual(value, cached_value)
def test_binary_string(self):
value = 'value_to_be_compressed'
compressed_value = zlib.compress(value.encode())
self.mc.set('binary1', compressed_value)
compressed_result = self.mc.get('binary1')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.add('binary1-add', compressed_value)
compressed_result = self.mc.get('binary1-add')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.set_multi({'binary1-set_many': compressed_value})
compressed_result = self.mc.get('binary1-set_many')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
with captured_stderr() as log:
self.assertIs(self.mc.set(key, value), False)
self.assertEqual(
log.getvalue(),
"MemCached: while expecting 'STORED', got unexpected response "
"'SERVER_ERROR object too large for cache'\n"
)
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix),
values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as log:
for server in self.mc.servers:
server.mark_dead('test')
self.assertIn('Marking dead.', log.getvalue())
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as output:
ret = self.mc.delete_multi(('keyhere', 'keythere'))
self.assertEqual(ret, 1)
self.assertEqual(
output.getvalue(),
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
)
@mock.patch.object(_Host, 'send_cmd') # Don't send any commands.
@mock.patch.object(_Host, 'readline')
def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd):
"""touch() logs an error upon receiving an unexpected reply."""
mock_readline.return_value = 'SET' # the unexpected reply
with captured_stderr() as output:
self.mc.touch('key')
self.assertEqual(
output.getvalue(),
"MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED'
)
#host = 'localhost'
memcached_port = '9150'
redis_port = '9250'
password = ''
hostname = 'ec2-54-183-15-168.us-west-1.compute.amazonaws.com'
mc = Client([hostname + ':' + memcached_port], debug=0)
#mc = Redis(host=hostname, port=redis_port, password=password)
start_time = datetime.now()
mc.set("first_key", "first value")
value = mc.get("first_key")
print(value)
mc.set("second_key", 3)
mc.delete("second_key")
mc.set("key", "1") # note that the key used for incr/decr must be
# a string.
value = mc.get('key')
print(value)
mc.incr("key")
value = mc.get('key')
print(value)
mc.decr("key")
value = mc.get('key')
print(value)
end_time = datetime.now()
elapsed_time = end_time - start_time
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.flush_all()
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
@mock.patch.object(_Host, 'send_cmd')
@mock.patch.object(_Host, 'readline')
def test_touch(self, mock_readline, mock_send_cmd):
with captured_stderr():
self.mc.touch('key')
mock_send_cmd.assert_called_with(b'touch key 0')
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(self.mc.get_multi(["gm_a_string", "gm_an_integer"]), {
"gm_an_integer": 42,
"gm_a_string": "some random string"
})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = u'\u4f1a'
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_unicode_value(self):
key = 'key'
value = u'Iñtërnâtiônàlizætiøn2'
self.mc.set(key, value)
cached_value = self.mc.get(key)
self.assertEqual(value, cached_value)
def test_binary_string(self):
value = 'value_to_be_compressed'
compressed_value = zlib.compress(value.encode())
self.mc.set('binary1', compressed_value)
compressed_result = self.mc.get('binary1')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.add('binary1-add', compressed_value)
compressed_result = self.mc.get('binary1-add')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.set_multi({'binary1-set_many': compressed_value})
compressed_result = self.mc.get('binary1-set_many')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
with captured_stderr() as log:
self.assertIs(self.mc.set(key, value), False)
self.assertEqual(
log.getvalue(),
"MemCached: while expecting 'STORED', got unexpected response "
"'SERVER_ERROR object too large for cache'\n")
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as log:
for server in self.mc.servers:
server.mark_dead('test')
self.assertIn('Marking dead.', log.getvalue())
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as output:
ret = self.mc.delete_multi(('keyhere', 'keythere'))
self.assertEqual(ret, 1)
self.assertEqual(
output.getvalue(),
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n")
@mock.patch.object(_Host, 'send_cmd') # Don't send any commands.
@mock.patch.object(_Host, 'readline')
def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd):
"""touch() logs an error upon receiving an unexpected reply."""
mock_readline.return_value = 'SET' # the unexpected reply
with captured_stderr() as output:
self.mc.touch('key')
self.assertEqual(
output.getvalue(),
"MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED')
class TestMemcache(TestCase):
def setUp(self):
# TODO: unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
pass
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def delete(self, key, **kwargs):
return Client.delete(self, self.uniqeKey + str(key), **kwargs)
class PrintFavicon(BaseHandler):
def __init__(self):
super(PrintFavicon, self).__init__()
default_icon_data = self.open(DEFAULT_FAVICON_LOC, time()).read()
self.default_icon = Icon(data=default_icon_data,
location=DEFAULT_FAVICON_LOC,
type=DEFAULT_FAVICON_TYPE)
self.env = Environment(loader=FileSystemLoader(
os.path.join(cherrypy.config['favicon.root'], 'templates')))
self.mc = Client(
['%(memcache.host)s:%(memcache.port)d' % cherrypy.config], debug=2)
# Initialize counters
for counter in ['requests', 'hits', 'defaults']:
self.mc.add('counter-%s' % counter, '0')
def open(self, url, start, headers=None):
time_spent = int(time() - start)
if time_spent >= TIMEOUT:
raise TimeoutError(time_spent)
if not headers:
headers = dict()
headers.update({
'User-Agent':
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; ' +
'rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13'
})
opener = build_opener(HTTPRedirectHandler(), HTTPCookieProcessor())
return opener.open(Request(url, headers=headers),
timeout=min(CONNECTION_TIMEOUT,
TIMEOUT - time_spent))
def validateIconResponse(self, iconResponse):
if iconResponse.getcode() != 200:
cherrypy.log('Non-success response:%d fetching url:%s' % \
(iconResponse.getcode(), iconResponse.geturl()),
severity=INFO)
return None
iconContentType = iconResponse.info().gettype()
if iconContentType in ICON_MIMETYPE_BLACKLIST:
cherrypy.log('Url:%s favicon content-Type:%s blacklisted' % \
(iconResponse.geturl(), iconContentType),
severity=INFO)
return None
icon = iconResponse.read()
iconLength = len(icon)
if iconLength == 0:
cherrypy.log('Url:%s null content length' % iconResponse.geturl(),
severity=INFO)
return None
if iconLength < MIN_ICON_LENGTH or iconLength > MAX_ICON_LENGTH:
# Issue warning, but accept nonetheless!
cherrypy.log('Warning: url:%s favicon size:%d out of bounds' % \
(iconResponse.geturl(), iconLength),
severity=INFO)
return Icon(data=icon, type=iconContentType)
# Icon at [domain]/favicon.ico?
def iconAtRoot(self, targetDomain, start):
cherrypy.log('Attempting to locate favicon for domain:%s at root' % \
targetDomain,
severity=INFO)
rootIconPath = targetDomain + '/favicon.ico'
try:
rootDomainFaviconResult = self.open(rootIconPath, start)
rootIcon = self.validateIconResponse(rootDomainFaviconResult)
if rootIcon:
cherrypy.log('Found favicon for domain:%s at root' %
targetDomain,
severity=INFO)
self.cacheIcon(targetDomain, rootIcon.data, rootIconPath)
rootIcon.location = rootIconPath
return rootIcon
except:
cherrypy.log('Error fetching favicon at domain root:%s, err:%s, msg:%s' % \
(targetDomain, sys.exc_info()[0], sys.exc_info()[1]),
severity=INFO)
# Icon specified in page?
def iconInPage(self, targetDomain, targetPath, start, refresh=True):
cherrypy.log('Attempting to locate embedded favicon link in page:%s' % \
targetPath,
severity=INFO)
try:
rootDomainPageResult = self.open(targetPath, start)
if rootDomainPageResult.getcode() == 200:
pageSoup = BeautifulSoup(rootDomainPageResult.read())
pageSoupIcon = pageSoup.find(
'link',
rel=compile('^(shortcut|icon|shortcut icon)$', IGNORECASE))
if pageSoupIcon:
pageIconHref = pageSoupIcon.get('href')
if pageIconHref:
pageIconPath = urljoin(targetPath, pageIconHref)
cherrypy.log('Found embedded favicon link:%s for domain:%s' % \
(pageIconPath, targetDomain),
severity=INFO)
cookies = rootDomainPageResult.headers.getheaders(
"Set-Cookie")
headers = None
if cookies:
headers = {'Cookie': ';'.join(cookies)}
pagePathFaviconResult = self.open(pageIconPath,
start,
headers=headers)
pageIcon = self.validateIconResponse(
pagePathFaviconResult)
if pageIcon:
cherrypy.log('Found favicon at:%s for domain:%s' % \
(pageIconPath, targetDomain),
severity=INFO)
self.cacheIcon(targetDomain, pageIcon.data,
pageIconPath)
pageIcon.location = pageIconPath
return pageIcon
else:
if refresh:
for meta in pageSoup.findAll('meta'):
if meta.get('http-equiv', '').lower() == 'refresh':
match = search('url=([^;]+)',
meta.get('content', ''),
flags=IGNORECASE)
if match:
refreshPath = urljoin(
rootDomainPageResult.geturl(),
match.group(1))
cherrypy.log('Processing refresh directive:%s for domain:%s' % \
(refreshPath, targetDomain),
severity=INFO)
return self.iconInPage(targetDomain,
refreshPath,
start,
refresh=False)
cherrypy.log('No link tag found:%s' % targetPath,
severity=INFO)
else:
cherrypy.log('Non-success response:%d for url:%s' % \
(rootDomainPageResult.getcode(), targetPath),
severity=INFO)
except:
cherrypy.log('Error extracting favicon from page:%s, err:%s, msg:%s' % \
(targetPath, sys.exc_info()[0], sys.exc_info()[1]),
severity=WARNING)
def cacheIcon(self, domain, icon, loc):
cherrypy.log('Caching icon at location:%s for domain:%s' %
(loc, domain),
severity=INFO)
if not self.mc.set('icon-%s' % domain, icon, time=MC_CACHE_TIME):
cherrypy.log('Could not cache icon for domain:%s' % domain,
severity=ERROR)
def iconInCache(self, targetDomain, start):
icon = self.mc.get('icon-%s' % targetDomain)
if icon:
self.mc.incr('counter-hits')
cherrypy.log('Cache hit:%s' % targetDomain, severity=INFO)
cherrypy.response.headers['X-Cache'] = 'Hit'
if icon == 'DEFAULT':
self.mc.incr('counter-defaults')
cherrypy.response.headers['X-Cache'] = 'Hit'
return self.default_icon
else:
return Icon(data=icon)
def writeIcon(self, icon):
self.writeHeaders(icon)
return icon.data
def writeHeaders(self, icon, fmt='%a, %d %b %Y %H:%M:%S %z'):
# MIME Type
cherrypy.response.headers['Content-Type'] = icon.type or 'image/x-icon'
# Set caching headers
cherrypy.response.headers['Cache-Control'] = 'public, max-age=2592000'
cherrypy.response.headers['Expires'] = \
(datetime.now() + timedelta(days=30)).strftime(fmt)
def parse(self, url):
# Get page path
targetPath = self.urldecode(url)
if not targetPath.startswith('http'):
targetPath = 'http://%s' % targetPath
cherrypy.log('Decoded URL:%s' % targetPath, severity=INFO)
# Split path to get domain
targetURL = urlparse(targetPath)
if not targetURL or not targetURL.scheme or not targetURL.netloc:
raise cherrypy.HTTPError(400, 'Malformed URL:%s' % url)
targetDomain = '%s://%s' % (targetURL.scheme, targetURL.netloc)
cherrypy.log('URL:%s, domain:%s' % (targetPath, targetDomain),
severity=INFO)
return (targetPath, targetDomain)
@cherrypy.expose
def index(self):
status = {'status': 'ok', 'counters': dict()}
for counter in ['requests', 'hits', 'defaults']:
status['counters'][counter] = self.mc.get('counter-%s' % counter)
return json.dumps(status)
@cherrypy.expose
def test(self):
topSites = open(
os.path.join(cherrypy.config['favicon.root'], 'topsites.txt'),
'r').read().split()
template = self.env.get_template('test.html')
return template.render(topSites=topSites)
@cherrypy.expose
def clear(self, url):
cherrypy.log('Incoming cache invalidation request:%s' % url,
severity=INFO)
targetPath, targetDomain = self.parse(str(url))
self.mc.delete('icon_loc-%s' % targetDomain)
cherrypy.log('Evicted cache entry for %s' % targetDomain,
severity=INFO)
@cherrypy.expose
def s(self, url, skipCache='false'):
start = time()
if skipCache.lower() == 'true':
skipCache = True
else:
skipCache = False
cherrypy.log('Incoming request:%s (skipCache=%s)' % (url, skipCache),
severity=INFO)
self.mc.incr('counter-requests')
targetPath, targetDomain = self.parse(str(url))
icon = (not skipCache and self.iconInCache(targetDomain, start)) or \
self.iconInPage(targetDomain, targetPath, start) or \
self.iconAtRoot(targetDomain, start)
if not icon:
cherrypy.log('Falling back to default icon for:%s' % targetDomain,
severity=INFO)
self.cacheIcon(targetDomain, 'DEFAULT', 'DEFAULT_LOC')
self.mc.incr('counter-defaults')
icon = self.default_icon
cherrypy.log('Time taken to process domain:%s %f' % \
(targetDomain, time() - start),
severity=INFO)
return self.writeIcon(icon)
class PrintFavicon(BaseHandler):
def __init__(self):
super(PrintFavicon, self).__init__()
default_icon_data = self.open(DEFAULT_FAVICON_LOC, time()).read()
self.default_icon = Icon(data=default_icon_data, location=DEFAULT_FAVICON_LOC, type=DEFAULT_FAVICON_TYPE)
self.env = Environment(loader=FileSystemLoader(os.path.join(cherrypy.config["favicon.root"], "templates")))
self.mc = Client(["%(memcache.host)s:%(memcache.port)d" % cherrypy.config], debug=2)
# Initialize counters
for counter in ["requests", "hits", "defaults"]:
self.mc.add("counter-%s" % counter, "0")
def open(self, url, start, headers=None):
time_spent = int(time() - start)
if time_spent >= TIMEOUT:
raise TimeoutError(time_spent)
if not headers:
headers = dict()
headers.update(
{
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; "
+ "rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13"
}
)
opener = build_opener(HTTPRedirectHandler(), HTTPCookieProcessor())
return opener.open(Request(url, headers=headers), timeout=min(CONNECTION_TIMEOUT, TIMEOUT - time_spent))
def validateIconResponse(self, iconResponse):
if iconResponse.getcode() != 200:
cherrypy.log(
"Non-success response:%d fetching url:%s" % (iconResponse.getcode(), iconResponse.geturl()),
severity=INFO,
)
return None
iconContentType = iconResponse.info().gettype()
if iconContentType in ICON_MIMETYPE_BLACKLIST:
cherrypy.log(
"Url:%s favicon content-Type:%s blacklisted" % (iconResponse.geturl(), iconContentType), severity=INFO
)
return None
icon = iconResponse.read()
iconLength = len(icon)
if iconLength == 0:
cherrypy.log("Url:%s null content length" % iconResponse.geturl(), severity=INFO)
return None
if iconLength < MIN_ICON_LENGTH or iconLength > MAX_ICON_LENGTH:
# Issue warning, but accept nonetheless!
cherrypy.log(
"Warning: url:%s favicon size:%d out of bounds" % (iconResponse.geturl(), iconLength), severity=INFO
)
return Icon(data=icon, type=iconContentType)
# Icon at [domain]/favicon.ico?
def iconAtRoot(self, targetDomain, start):
cherrypy.log("Attempting to locate favicon for domain:%s at root" % targetDomain, severity=INFO)
rootIconPath = targetDomain + "/favicon.ico"
try:
rootDomainFaviconResult = self.open(rootIconPath, start)
rootIcon = self.validateIconResponse(rootDomainFaviconResult)
if rootIcon:
cherrypy.log("Found favicon for domain:%s at root" % targetDomain, severity=INFO)
self.cacheIcon(targetDomain, rootIcon.data, rootIconPath)
rootIcon.location = rootIconPath
return rootIcon
except:
cherrypy.log(
"Error fetching favicon at domain root:%s, err:%s, msg:%s"
% (targetDomain, sys.exc_info()[0], sys.exc_info()[1]),
severity=INFO,
)
# Icon specified in page?
def iconInPage(self, targetDomain, targetPath, start, refresh=True):
cherrypy.log("Attempting to locate embedded favicon link in page:%s" % targetPath, severity=INFO)
try:
rootDomainPageResult = self.open(targetPath, start)
if rootDomainPageResult.getcode() == 200:
pageSoup = BeautifulSoup(rootDomainPageResult.read())
pageSoupIcon = pageSoup.find("link", rel=compile("^(shortcut|icon|shortcut icon)$", IGNORECASE))
if pageSoupIcon:
pageIconHref = pageSoupIcon.get("href")
if pageIconHref:
pageIconPath = urljoin(targetPath, pageIconHref)
cherrypy.log(
"Found embedded favicon link:%s for domain:%s" % (pageIconPath, targetDomain), severity=INFO
)
cookies = rootDomainPageResult.headers.getheaders("Set-Cookie")
headers = None
if cookies:
headers = {"Cookie": ";".join(cookies)}
pagePathFaviconResult = self.open(pageIconPath, start, headers=headers)
pageIcon = self.validateIconResponse(pagePathFaviconResult)
if pageIcon:
cherrypy.log(
"Found favicon at:%s for domain:%s" % (pageIconPath, targetDomain), severity=INFO
)
self.cacheIcon(targetDomain, pageIcon.data, pageIconPath)
pageIcon.location = pageIconPath
return pageIcon
else:
if refresh:
for meta in pageSoup.findAll("meta"):
if meta.get("http-equiv", "").lower() == "refresh":
match = search("url=([^;]+)", meta.get("content", ""), flags=IGNORECASE)
if match:
refreshPath = urljoin(rootDomainPageResult.geturl(), match.group(1))
cherrypy.log(
"Processing refresh directive:%s for domain:%s" % (refreshPath, targetDomain),
severity=INFO,
)
return self.iconInPage(targetDomain, refreshPath, start, refresh=False)
cherrypy.log("No link tag found:%s" % targetPath, severity=INFO)
else:
cherrypy.log(
"Non-success response:%d for url:%s" % (rootDomainPageResult.getcode(), targetPath), severity=INFO
)
except:
cherrypy.log(
"Error extracting favicon from page:%s, err:%s, msg:%s"
% (targetPath, sys.exc_info()[0], sys.exc_info()[1]),
severity=WARNING,
)
def cacheIcon(self, domain, icon, loc):
cherrypy.log("Caching icon at location:%s for domain:%s" % (loc, domain), severity=INFO)
if not self.mc.set("icon-%s" % domain, icon, time=MC_CACHE_TIME):
cherrypy.log("Could not cache icon for domain:%s" % domain, severity=ERROR)
def iconInCache(self, targetDomain, start):
icon = self.mc.get("icon-%s" % targetDomain)
if icon:
self.mc.incr("counter-hits")
cherrypy.log("Cache hit:%s" % targetDomain, severity=INFO)
cherrypy.response.headers["X-Cache"] = "Hit"
if icon == "DEFAULT":
self.mc.incr("counter-defaults")
cherrypy.response.headers["X-Cache"] = "Hit"
return self.default_icon
else:
return Icon(data=icon)
def writeIcon(self, icon):
self.writeHeaders(icon)
return icon.data
def writeHeaders(self, icon, fmt="%a, %d %b %Y %H:%M:%S %z"):
# MIME Type
cherrypy.response.headers["Content-Type"] = icon.type or "image/x-icon"
# Set caching headers
cherrypy.response.headers["Cache-Control"] = "public, max-age=2592000"
cherrypy.response.headers["Expires"] = (datetime.now() + timedelta(days=30)).strftime(fmt)
def parse(self, url):
# Get page path
targetPath = self.urldecode(url)
if not targetPath.startswith("http"):
targetPath = "http://%s" % targetPath
cherrypy.log("Decoded URL:%s" % targetPath, severity=INFO)
# Split path to get domain
targetURL = urlparse(targetPath)
if not targetURL or not targetURL.scheme or not targetURL.netloc:
raise cherrypy.HTTPError(400, "Malformed URL:%s" % url)
targetDomain = "%s://%s" % (targetURL.scheme, targetURL.netloc)
cherrypy.log("URL:%s, domain:%s" % (targetPath, targetDomain), severity=INFO)
return (targetPath, targetDomain)
@cherrypy.expose
def index(self):
status = {"status": "ok", "counters": dict()}
for counter in ["requests", "hits", "defaults"]:
status["counters"][counter] = self.mc.get("counter-%s" % counter)
return json.dumps(status)
@cherrypy.expose
def test(self):
topSites = open(os.path.join(cherrypy.config["favicon.root"], "topsites.txt"), "r").read().split()
template = self.env.get_template("test.html")
return template.render(topSites=topSites)
@cherrypy.expose
def clear(self, url):
cherrypy.log("Incoming cache invalidation request:%s" % url, severity=INFO)
targetPath, targetDomain = self.parse(str(url))
self.mc.delete("icon_loc-%s" % targetDomain)
cherrypy.log("Evicted cache entry for %s" % targetDomain, severity=INFO)
@cherrypy.expose
def s(self, url, skipCache="false"):
start = time()
if skipCache.lower() == "true":
skipCache = True
else:
skipCache = False
cherrypy.log("Incoming request:%s (skipCache=%s)" % (url, skipCache), severity=INFO)
self.mc.incr("counter-requests")
targetPath, targetDomain = self.parse(str(url))
icon = (
(not skipCache and self.iconInCache(targetDomain, start))
or self.iconInPage(targetDomain, targetPath, start)
or self.iconAtRoot(targetDomain, start)
)
if not icon:
cherrypy.log("Falling back to default icon for:%s" % targetDomain, severity=INFO)
self.cacheIcon(targetDomain, "DEFAULT", "DEFAULT_LOC")
self.mc.incr("counter-defaults")
icon = self.default_icon
cherrypy.log("Time taken to process domain:%s %f" % (targetDomain, time() - start), severity=INFO)
return self.writeIcon(icon)
class MemcachedCacheClient(CacheClient):
"""Memcached cache client implementation."""
def __init__(self, config):
super(MemcachedCacheClient, self).__init__(config["host"], config["port"], config["cache"])
self.config = config
if self.cache_name != DEFAULT_CACHE_NAME:
print "WARNING: memcached client doesn't support named caches. cache_name config value will be ignored and default cache will be used instead."
self.memcached_client = Client([self.host + ':' + self.port], debug=0)
return
def put(self, key, value, version=None, lifespan=None, max_idle=None, put_if_absent=False):
time = 0
if lifespan != None:
if lifespan > MEMCACHED_LIFESPAN_MAX_SECONDS:
self._error("Memcached cache client supports lifespan values only up to %s seconds (30 days)." % MEMCACHED_LIFESPAN_MAX_SECONDS)
time = lifespan
if max_idle != None:
self._error("Memcached cache client doesn't support max idle time setting.")
try:
if (version == None):
if (put_if_absent):
if not self.memcached_client.add(key, value, time, 0):
# current python-memcached doesn't recoginze these states
# if self.memcached_client.last_set_status == "NOT_STORED":
# raise ConflictError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT.")
else:
if not self.memcached_client.set(key, value, time, 0):
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful.")
else:
try:
self.memcached_client.cas_ids[key] = int(version)
except ValueError:
self._error("Please provide an integer version.")
if not self.memcached_client.cas(key, value, time, 0):
# if self.memcached_client.last_set_status == "EXISTS":
# raise ConflictError
# if self.memcached_client.last_set_status == "NOT_FOUND":
# raise NotFoundError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT, NOT_FOUND.")
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e)
def get(self, key, get_version=False):
try:
if get_version:
val = self.memcached_client.gets(key)
if val == None:
raise NotFoundError
version = self.memcached_client.cas_ids[key]
if version == None:
self._error("Couldn't obtain version info from memcached server.")
return version, val
else:
val = self.memcached_client.get(key)
if val == None:
raise NotFoundError
return val
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def delete(self, key, version=None):
try:
if version:
self._error("versioned delete operation not available for memcached client")
if self.memcached_client.delete(key, 0):
if self.memcached_client.last_set_status == "NOT_FOUND":
raise NotFoundError
else:
self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def clear(self):
try:
self.memcached_client.flush_all()
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(self.mc.get_multi(["gm_a_string", "gm_an_integer"]), {
"gm_an_integer": 42,
"gm_a_string": "some random string"
})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = six.u('\u4f1a')
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
self.assertFalse(self.mc.set(key, value))
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
for server in self.mc.servers:
server.mark_dead('test')
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
ret = self.mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
self.assertEqual(ret, 1)
from memcache import Client
"""
安装memcache时，遇到的问题，提示libevent
解决 yum install libevent
yum install libevent-devel
"""
# 这里是个list，可以吧memcache集群这么搞
MC_SERVERS = ['192.168.52.3:11211', '192.168.52.3:11212']
CONN = Client(MC_SERVERS)
status = CONN.set('key1', 'val2', 0)
print(status)
status = CONN.delete('key')
print(status)
status = CONN.add('key', 'val', 20)
print(status)
status = CONN.replace('key', 'val1', 0)
print(status)
status = CONN.append('key', ',val2')
print(status)
data = CONN.get('key')
print(data)
class MemcacheFeatureStorage(FeatureStorage):
PREFIX = 'georest_buckets'
support_version = False
def __init__(self, hosts):
""" Feature storage implemented in Memcache
:param list hosts: list of hosts
1. Strings of the form C{"host:port"}
2. Tuples of the form C{("host:port", weight)}
:rtype :class:`MemcacheFeatureStorage`
"""
self._client = Client(servers=hosts)
def create_bucket(self, name, overwrite=False, **kwargs):
bucket_name = self._make_bucket_name(name)
timestamp = time.time()
try:
add_ok = self._client.add(key=bucket_name, val=timestamp)
except Exception as e:
raise StorageInternalError(message='add error', e=e)
if not add_ok:
if overwrite:
try:
rep_ok = self._client.replace(
key=bucket_name, val=timestamp)
except Exception as e:
raise StorageInternalError('replace error', e=e)
if not rep_ok:
raise StorageInternalError(message='failed to replace')
else:
raise DuplicatedBucket(name)
return MemcacheFeatureBucket(name, self._client, str(timestamp))
def get_bucket(self, name):
bucket_name = self._make_bucket_name(name)
try:
timestamp = self._client.get(bucket_name)
except Exception as e:
raise StorageInternalError(message='get error', e=e)
if not timestamp:
raise BucketNotFound(name)
return MemcacheFeatureBucket(name, self._client, str(timestamp))
def delete_bucket(self, name):
bucket_name = self._make_bucket_name(name)
try:
delete_ok = self._client.delete(bucket_name)
except Exception as e:
raise StorageInternalError(message='delete error', e=e)
if not delete_ok:
raise BucketNotFound(name)
return True
def has_bucket(self, name):
bucket_name = self._make_bucket_name(name)
try:
get_ok = self._client.get(bucket_name)
except Exception as e:
raise StorageInternalError(message='get error', e=e)
return get_ok is not None
def close(self):
pass
def _make_bucket_name(self, name):
if isinstance(name, unicode):
name = name.encode('utf-8')
return '.'.join((self.PREFIX, name))
class MemcachedCacheClient(CacheClient):
"""Memcached cache client implementation."""
def __init__(self, config):
super(MemcachedCacheClient, self).__init__(config["host"], config["port"], config["cache"])
self.config = config
if self.cache_name and self.cache_name != "" and self.cache_name != DEFAULT_MEMCACHED_CACHE_NAME:
print "WARNING: memcached client doesn't support named caches. cache_name config value will be ignored and the cache name configured on the server will be used instead."
self.memcached_client = Client([self.host + ':' + self.port], debug=0)
return
def put(self, key, value, version=None, lifespan=None, max_idle=None, put_if_absent=False):
time = 0
if lifespan != None:
if lifespan > MEMCACHED_LIFESPAN_MAX_SECONDS:
self._error("Memcached cache client supports lifespan values only up to %s seconds (30 days)." % MEMCACHED_LIFESPAN_MAX_SECONDS)
time = lifespan
if max_idle != None:
self._error("Memcached cache client doesn't support max idle time setting.")
try:
if (version == None):
if (put_if_absent):
if not self.memcached_client.add(key, value, time, 0):
# current python-memcached doesn't recoginze these states
# if self.memcached_client.last_set_status == "NOT_STORED":
# raise ConflictError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT.")
else:
if not self.memcached_client.set(key, value, time, 0):
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful.")
else:
try:
self.memcached_client.cas_ids[key] = int(version)
except ValueError:
self._error("Please provide an integer version.")
if not self.memcached_client.cas(key, value, time, 0):
# if self.memcached_client.last_set_status == "EXISTS":
# raise ConflictError
# if self.memcached_client.last_set_status == "NOT_FOUND":
# raise NotFoundError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT, NOT_FOUND.")
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e)
def get(self, key, get_version=False):
try:
if get_version:
val = self.memcached_client.gets(key)
if val == None:
raise NotFoundError
version = self.memcached_client.cas_ids[key]
if version == None:
self._error("Couldn't obtain version info from memcached server.")
return version, val
else:
val = self.memcached_client.get(key)
if val == None:
raise NotFoundError
return val
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def delete(self, key, version=None):
try:
if version:
self._error("versioned delete operation not available for memcached client")
if self.memcached_client.delete(key, 0):
if self.memcached_client.last_set_status == "NOT_FOUND":
raise NotFoundError
else:
self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def clear(self):
try:
self.memcached_client.flush_all()
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO: unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = six.u('\u4f1a')
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
self.assertFalse(self.mc.set(key, value))
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix),
values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
for server in self.mc.servers:
server.mark_dead('test')
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
ret = self.mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
self.assertEqual(ret, 1)
def test_tags_set(self):
self.mc.disconnect_all()
self.mc.set('key', 'val', tags=['t1'])
self.mc.set('key1', 'val2', tags=['t2'])
self.mc.delete_by_tag('t1')
self.assertIsNone(self.mc.get('key'))
self.assertEqual(self.mc.get('key1'), 'val2')
def test_tags_set_multi(self):
self.mc.disconnect_all()
mapping = {
'k': 1,
'k2': 2,
}
self.mc.set_multi(mapping, tags=['t1'])
self.mc.delete_by_tag('t1')
self.assertIsNone(self.mc.get('k'))
self.assertIsNone(self.mc.get('k1'))
