class MemcacheStoreTest(unittest.TestCase, SimpleKVTest):
def setUp(self):
self.mc = Client([testconf.get('memcache', 'server')])
self.mc.flush_all()
self.store = MemcacheStore(self.mc)
def tearDown(self):
self.mc.flush_all()
def test_memcache_connection(self):
pass
# disabled tests (not fully API support for memcache)
test_has_key = None
test_has_key_with_delete = None
test_key_iterator = None
test_keys = None
def test_keys_throws_io_error(self):
with self.assertRaises(IOError):
self.store.keys()
with self.assertRaises(IOError):
self.store.iter_keys()
with self.assertRaises(IOError):
iter(self.store)
def test_contains_throws_io_error_or_succeeds(self):
try:
'a' in self.store
except IOError:
pass
class CacheMgr(object):
def __init__(self, servers=["127.0.0.1:11211"]):
self.__mc = Client(servers)
pass
def tearDown(self):
self.__mc.flush_all()
self.__mc.disconnect_all()
def get(self, key):
val = self.__mc.get(key)
return val
def set(self, key, val):
# TODO: noreply setting
self.__mc.set(key, val)
class MemcachedCacheStore(AbstractCacheStore):
servers = ("127.0.0.1:11211")
def __init__(self, servers=None, debug=False):
if servers is None:
servers = self.servers
from memcache import Client as MemcachedClient
self._client = MemcachedClient(servers, debug)
def set(self, key, val, time=0):
self._client.set(key, val, time)
def add(self, key, val, time=0):
res = self._client.add(key, val, time)
if not res:
raise Error("a value for key %r is already in the cache" % key)
self._data[key] = (val, time)
def replace(self, key, val, time=0):
res = self._client.replace(key, val, time)
if not res:
raise Error("a value for key %r is already in the cache" % key)
self._data[key] = (val, time)
def delete(self, key):
res = self._client.delete(key, time=0)
if not res:
raise KeyError(key)
def get(self, key):
val = self._client.get(key)
if val is None:
raise KeyError(key)
return val
def clear(self):
self._client.flush_all()
class MemcachedCacheStore(AbstractCacheStore):
servers = ('127.0.0.1:11211')
def __init__(self, servers=None, debug=False):
if servers is None:
servers = self.servers
from memcache import Client as MemcachedClient
self._client = MemcachedClient(servers, debug)
def set(self, key, val, time=0):
self._client.set(key, val, time)
def add(self, key, val, time=0):
res = self._client.add(key, val, time)
if not res:
raise Error('a value for key %r is already in the cache'%key)
self._data[key] = (val, time)
def replace(self, key, val, time=0):
res = self._client.replace(key, val, time)
if not res:
raise Error('a value for key %r is already in the cache'%key)
self._data[key] = (val, time)
def delete(self, key):
res = self._client.delete(key, time=0)
if not res:
raise KeyError(key)
def get(self, key):
val = self._client.get(key)
if val is None:
raise KeyError(key)
else:
return val
def clear(self):
self._client.flush_all()
class MemcachedCacheClient(CacheClient):
"""Memcached cache client implementation."""
def __init__(self, config):
super(MemcachedCacheClient, self).__init__(config["host"], config["port"], config["cache"])
self.config = config
if self.cache_name and self.cache_name != "" and self.cache_name != DEFAULT_MEMCACHED_CACHE_NAME:
print "WARNING: memcached client doesn't support named caches. cache_name config value will be ignored and the cache name configured on the server will be used instead."
self.memcached_client = Client([self.host + ':' + self.port], debug=0)
return
def put(self, key, value, version=None, lifespan=None, max_idle=None, put_if_absent=False):
time = 0
if lifespan != None:
if lifespan > MEMCACHED_LIFESPAN_MAX_SECONDS:
self._error("Memcached cache client supports lifespan values only up to %s seconds (30 days)." % MEMCACHED_LIFESPAN_MAX_SECONDS)
time = lifespan
if max_idle != None:
self._error("Memcached cache client doesn't support max idle time setting.")
try:
if (version == None):
if (put_if_absent):
if not self.memcached_client.add(key, value, time, 0):
# current python-memcached doesn't recoginze these states
# if self.memcached_client.last_set_status == "NOT_STORED":
# raise ConflictError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT.")
else:
if not self.memcached_client.set(key, value, time, 0):
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful.")
else:
try:
self.memcached_client.cas_ids[key] = int(version)
except ValueError:
self._error("Please provide an integer version.")
if not self.memcached_client.cas(key, value, time, 0):
# if self.memcached_client.last_set_status == "EXISTS":
# raise ConflictError
# if self.memcached_client.last_set_status == "NOT_FOUND":
# raise NotFoundError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT, NOT_FOUND.")
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e)
def get(self, key, get_version=False):
try:
if get_version:
val = self.memcached_client.gets(key)
if val == None:
raise NotFoundError
version = self.memcached_client.cas_ids[key]
if version == None:
self._error("Couldn't obtain version info from memcached server.")
return version, val
else:
val = self.memcached_client.get(key)
if val == None:
raise NotFoundError
return val
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def delete(self, key, version=None):
try:
if version:
self._error("versioned delete operation not available for memcached client")
if self.memcached_client.delete(key, 0):
if self.memcached_client.last_set_status == "NOT_FOUND":
raise NotFoundError
else:
self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def clear(self):
try:
self.memcached_client.flush_all()
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.flush_all()
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
@mock.patch.object(_Host, 'send_cmd')
@mock.patch.object(_Host, 'readline')
def test_touch(self, mock_readline, mock_send_cmd):
with captured_stderr():
self.mc.touch('key')
mock_send_cmd.assert_called_with(b'touch key 0')
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(self.mc.get_multi(["gm_a_string", "gm_an_integer"]), {
"gm_an_integer": 42,
"gm_a_string": "some random string"
})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = u'\u4f1a'
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_unicode_value(self):
key = 'key'
value = u'Iñtërnâtiônàlizætiøn2'
self.mc.set(key, value)
cached_value = self.mc.get(key)
self.assertEqual(value, cached_value)
def test_binary_string(self):
value = 'value_to_be_compressed'
compressed_value = zlib.compress(value.encode())
self.mc.set('binary1', compressed_value)
compressed_result = self.mc.get('binary1')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.add('binary1-add', compressed_value)
compressed_result = self.mc.get('binary1-add')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.set_multi({'binary1-set_many': compressed_value})
compressed_result = self.mc.get('binary1-set_many')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
with captured_stderr() as log:
self.assertIs(self.mc.set(key, value), False)
self.assertEqual(
log.getvalue(),
"MemCached: while expecting 'STORED', got unexpected response "
"'SERVER_ERROR object too large for cache'\n")
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as log:
for server in self.mc.servers:
server.mark_dead('test')
self.assertIn('Marking dead.', log.getvalue())
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as output:
ret = self.mc.delete_multi(('keyhere', 'keythere'))
self.assertEqual(ret, 1)
self.assertEqual(
output.getvalue(),
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n")
@mock.patch.object(_Host, 'send_cmd') # Don't send any commands.
@mock.patch.object(_Host, 'readline')
def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd):
"""touch() logs an error upon receiving an unexpected reply."""
mock_readline.return_value = 'SET' # the unexpected reply
with captured_stderr() as output:
self.mc.touch('key')
self.assertEqual(
output.getvalue(),
"MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED')
class MemcachedCacheClient(CacheClient):
"""Memcached cache client implementation."""
def __init__(self, config):
super(MemcachedCacheClient, self).__init__(config["host"], config["port"], config["cache"])
self.config = config
if self.cache_name != DEFAULT_CACHE_NAME:
print "WARNING: memcached client doesn't support named caches. cache_name config value will be ignored and default cache will be used instead."
self.memcached_client = Client([self.host + ':' + self.port], debug=0)
return
def put(self, key, value, version=None, lifespan=None, max_idle=None, put_if_absent=False):
time = 0
if lifespan != None:
if lifespan > MEMCACHED_LIFESPAN_MAX_SECONDS:
self._error("Memcached cache client supports lifespan values only up to %s seconds (30 days)." % MEMCACHED_LIFESPAN_MAX_SECONDS)
time = lifespan
if max_idle != None:
self._error("Memcached cache client doesn't support max idle time setting.")
try:
if (version == None):
if (put_if_absent):
if not self.memcached_client.add(key, value, time, 0):
# current python-memcached doesn't recoginze these states
# if self.memcached_client.last_set_status == "NOT_STORED":
# raise ConflictError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT.")
else:
if not self.memcached_client.set(key, value, time, 0):
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful.")
else:
try:
self.memcached_client.cas_ids[key] = int(version)
except ValueError:
self._error("Please provide an integer version.")
if not self.memcached_client.cas(key, value, time, 0):
# if self.memcached_client.last_set_status == "EXISTS":
# raise ConflictError
# if self.memcached_client.last_set_status == "NOT_FOUND":
# raise NotFoundError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT, NOT_FOUND.")
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e)
def get(self, key, get_version=False):
try:
if get_version:
val = self.memcached_client.gets(key)
if val == None:
raise NotFoundError
version = self.memcached_client.cas_ids[key]
if version == None:
self._error("Couldn't obtain version info from memcached server.")
return version, val
else:
val = self.memcached_client.get(key)
if val == None:
raise NotFoundError
return val
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def delete(self, key, version=None):
try:
if version:
self._error("versioned delete operation not available for memcached client")
if self.memcached_client.delete(key, 0):
if self.memcached_client.last_set_status == "NOT_FOUND":
raise NotFoundError
else:
self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def clear(self):
try:
self.memcached_client.flush_all()
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
from memtransaction import rr_transaction
from memcache import Client
def init(setter, getter):
setter('counter',0)
if __name__ == '__main__':
mc = Client(["127.0.0.1:11211"])
mc.flush_all()
def _(setter, getter):
getter('counter')
print 'counter:',_['counter']
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.flush_all()
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
@mock.patch.object(_Host, 'send_cmd')
@mock.patch.object(_Host, 'readline')
def test_touch(self, mock_readline, mock_send_cmd):
with captured_stderr():
self.mc.touch('key')
mock_send_cmd.assert_called_with(b'touch key 0')
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = u'\u4f1a'
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_unicode_value(self):
key = 'key'
value = u'Iñtërnâtiônàlizætiøn2'
self.mc.set(key, value)
cached_value = self.mc.get(key)
self.assertEqual(value, cached_value)
def test_binary_string(self):
value = 'value_to_be_compressed'
compressed_value = zlib.compress(value.encode())
self.mc.set('binary1', compressed_value)
compressed_result = self.mc.get('binary1')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.add('binary1-add', compressed_value)
compressed_result = self.mc.get('binary1-add')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.set_multi({'binary1-set_many': compressed_value})
compressed_result = self.mc.get('binary1-set_many')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
with captured_stderr() as log:
self.assertIs(self.mc.set(key, value), False)
self.assertEqual(
log.getvalue(),
"MemCached: while expecting 'STORED', got unexpected response "
"'SERVER_ERROR object too large for cache'\n"
)
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix),
values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as log:
for server in self.mc.servers:
server.mark_dead('test')
self.assertIn('Marking dead.', log.getvalue())
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as output:
ret = self.mc.delete_multi(('keyhere', 'keythere'))
self.assertEqual(ret, 1)
self.assertEqual(
output.getvalue(),
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
)
@mock.patch.object(_Host, 'send_cmd') # Don't send any commands.
@mock.patch.object(_Host, 'readline')
def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd):
"""touch() logs an error upon receiving an unexpected reply."""
mock_readline.return_value = 'SET' # the unexpected reply
with captured_stderr() as output:
self.mc.touch('key')
self.assertEqual(
output.getvalue(),
"MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED'
)
