class MCStore(object):
def __init__(self, server):
self.server = server
self.mc = Client([server], do_split=0)
def __del__(self):
self.close()
def __repr__(self):
return '<MCStore(server=%s)>' % repr(self.server)
def __str__(self):
return self.server
def set(self, key, data, rev=0):
return bool(self.mc.set(key, data, rev))
def set_raw(self, key, data, rev=0, flag=0): # interface for sync data
if rev < 0:
raise str(rev)
return self.mc.set_raw(key, data, rev, flag)
def get(self, key):
try:
r = self.mc.get(key)
if r is None and self.mc.get_last_error() != 0:
raise IOError(self.mc.get_last_error())
return r
except ValueError:
self.mc.delete(key)
def get_raw(self, key):
r, flag = self.mc.get_raw(key)
if r is None and self.mc.get_last_error() != 0:
raise IOError(
self.mc.get_last_error(), self.mc.get_last_strerror())
return r, flag
def get_multi(self, keys):
return self.mc.get_multi(keys)
def delete(self, key):
return bool(self.mc.delete(key))
def close(self):
if self.mc is not None:
self.mc = None
class MCStore(object):
def __init__(self, server):
self.server = server
self.mc = Client([server], do_split=0)
def __del__(self):
self.close()
def __repr__(self):
return '<MCStore(server=%s)>' % repr(self.server)
def __str__(self):
return self.server
def set(self, key, data, rev=0):
return bool(self.mc.set(key, data, rev))
def set_raw(self, key, data, rev=0, flag=0): # interface for sync data
if rev < 0:
raise str(rev)
return self.mc.set_raw(key, data, rev, flag)
def get(self, key):
try:
r = self.mc.get(key)
if r is None and self.mc.get_last_error() != 0:
raise IOError(self.mc.get_last_error())
return r
except ValueError:
self.mc.delete(key)
def get_raw(self, key):
r, flag = self.mc.get_raw(key)
if r is None and self.mc.get_last_error() != 0:
raise IOError(
self.mc.get_last_error(), self.mc.get_last_strerror())
return r, flag
def get_multi(self, keys):
return self.mc.get_multi(keys)
def delete(self, key):
return bool(self.mc.delete(key))
def close(self):
if self.mc is not None:
self.mc = None
class SingleServerCase(unittest.TestCase):
def setUp(self):
self.mc = Client(["127.0.0.1:21211"])
self.compressed_mc = Client(["127.0.0.1:21211"], comp_threshold=1024)
self.noreply_mc = Client(["127.0.0.1:21211"], noreply=True)
def test_attribute(self):
"Test attributes are accessible from Python code"
mc = self.mc
assert hasattr(mc, 'comp_threshold')
assert hasattr(mc, 'servers')
def test_misc(self):
mc = self.mc
mc.get_multi(['foo', 'tuiche'])
mc.delete('foo')
mc.delete('tuiche')
assert mc.get('foo') is None
assert mc.get('tuiche') is None
mc.set('foo', 'biu')
mc.set('tuiche', 'bb')
assert mc.get('foo') == 'biu'
assert mc.get('tuiche') == 'bb'
assert (mc.get_multi(['foo', 'tuiche']) == {
'foo': 'biu',
'tuiche': 'bb'
})
mc.set_multi({'foo': 1024, 'tuiche': '8964'})
assert (mc.get_multi(['foo', 'tuiche']) == {
'foo': 1024,
'tuiche': '8964'
})
def test_delete(self):
mc = self.mc
assert mc.set('smmf', 0xCA909) is True
assert mc.get('smmf') == 0xCA909
assert mc.delete('smmf') is True
assert mc.delete_multi(['smmf']) is True
assert mc.set('smmf', 0xCA909) is True
assert mc.delete_multi(['smmf']) is True
assert mc.delete('smmf') is True
assert mc.delete('smmf') is True
def test_incr_decr(self):
mc = self.mc
mc.set('wazi', 99)
assert mc.incr('wazi', 1) == 100
assert mc.decr('wazi', 1) == 99
mc.delete('wazi')
assert mc.incr('wazi', 1) is None
assert mc.decr('wazi', 1) is None
def test_cas(self):
mc = self.mc
mc.delete('bilinda')
k1 = 'bilinda'
v1 = 'butchers'
assert mc.gets(k1) is None
mc.set(k1, v1)
v1_, ck = mc.gets('bilinda')
v2 = 'ding'
assert v1 == v1_
assert mc.cas(k1, v2, 0, ck) is True
assert mc.cas(k1, v1, 0, ck) is not True
v2_, ck = mc.gets('bilinda')
assert v2 == v2_
def test_large(self):
mc = self.mc
BUF_500KB = 'i' * (500 * 1000)
BUF_1MB = 'i' * (1000 * 1000)
key_500kb = 'test_500kb'
key_1mb = 'test_1mb'
mc.delete(key_500kb)
mc.delete(key_1mb)
mc.set(key_500kb, BUF_500KB)
assert mc.get(key_500kb) == BUF_500KB
mc.set(key_1mb, BUF_1MB)
assert mc.get(key_1mb) == BUF_1MB
mc.delete(key_500kb)
mc.delete(key_1mb)
dct = {key_500kb: BUF_500KB, key_1mb: BUF_1MB}
mc.set_multi(dct)
assert mc.get_multi(dct.keys()) == dct
def test_extra_large(self):
threshold = 1000000
mc = self.mc
key_xl = 'test_very_large'
val = 'i' * threshold
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val += 'b'
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val = 'i' * (threshold + threshold)
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val += 'b'
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
def test_noreply(self):
mc = self.noreply_mc
assert mc.set('foo', 'bar')
assert mc.touch('foo', 30)
v, ck = mc.gets('foo')
assert mc.cas('foo', 'bar2', 0, ck)
mc.delete('foo')
assert mc.set('foo', 1024)
assert mc.incr('foo', 1) is None
assert mc.get('foo') == 1025
assert mc.decr('foo', 3) is None
assert mc.get('foo') == 1022
def test_injection(self):
# credit to The New Page of Injections Book:
# Memcached Injections @ blackhat2014 [pdf](http://t.cn/RP0J10Z)
mc = self.mc
assert mc.delete('injected')
assert mc.set('a' * 250, 'biu')
assert not mc.set('a' * 251, 'biu')
mc.set('a' * 251, 'set injected 0 3600 10\r\n1234567890')
assert mc.get('injected') is None
mc.delete('injected')
mc.set('key1', '1234567890')
mc.set(
'key1 0', '123456789012345678901234567890\r\n'
'set injected 0 3600 3\r\nINJ\r\n')
assert mc.get('injected') is None
def test_maxiov(self):
key_tmpl = 'not_existed.%s'
assert self.mc.get_multi([key_tmpl % i for i in range(10000)]) == {}
def test_get_set_raw(self):
self.mc.set('foo', 233)
assert self.mc.get_raw('foo') == ('233', 2)
self.mc.set_raw('foo', '2335', 0, 2)
assert self.mc.get('foo') == 2335
def test_stats(self):
stats = self.mc.stats()
for addr, dct in stats.iteritems():
assert isinstance(dct['version'], str)
assert (isinstance(dct['rusage_system'], float)
or isinstance(dct['rusage_user'], float))
assert isinstance(dct['curr_connections'], int)
def test_get_set_large_raw(self):
key = 'large_raw_key'
key_dup = '%s_dup' % key
val = 'i' * int(_DOUBAN_CHUNK_SIZE * 1.5)
for mc in (self.mc, self.compressed_mc):
mc.set(key, val)
assert mc.get(key) == val
raw_val1, flags1 = mc.get_raw(key)
assert mc.set_raw(key_dup, raw_val1, 0, flags1)
assert mc.get_raw(key) == mc.get_raw(key_dup)
def test_patch_no_compress(self):
key = 'no_compress'
val = '1024'
self.mc.set(key, val, compress=False)
large_patch = 'hahahaha' * 512
self.mc.prepend(key, large_patch)
assert self.mc.get(key) == large_patch + val
self.mc.delete(key)
self.mc.set(key, val, compress=False)
large_patch = 'hahahaha' * 512
self.mc.append(key, large_patch)
assert self.mc.get(key) == val + large_patch
self.mc.delete(key)
class SingleServerCase(unittest.TestCase):
def setUp(self):
self.mc = Client(["127.0.0.1:21211"])
self.compressed_mc = Client(["127.0.0.1:21211"], comp_threshold=1024)
self.noreply_mc = Client(["127.0.0.1:21211"], noreply=True)
def test_attribute(self):
"Test attributes are accessible from Python code"
mc = self.mc
assert hasattr(mc, 'comp_threshold')
assert hasattr(mc, 'servers')
def test_misc(self):
mc = self.mc
mc.get_multi(['foo', 'tuiche'])
mc.delete('foo')
mc.delete('tuiche')
assert mc.get('foo') is None
assert mc.get('tuiche') is None
mc.set('foo', 'biu')
mc.set('tuiche', 'bb')
assert mc.get('foo') == 'biu'
assert mc.get('tuiche') == 'bb'
assert (mc.get_multi(['foo', 'tuiche']) ==
{'foo': 'biu', 'tuiche': 'bb'})
mc.set_multi({'foo': 1024, 'tuiche': '8964'})
assert (mc.get_multi(['foo', 'tuiche']) ==
{'foo': 1024, 'tuiche': '8964'})
def test_delete(self):
mc = self.mc
assert mc.set('smmf', 0xCA909) is True
assert mc.get('smmf') == 0xCA909
assert mc.delete('smmf') is True
assert mc.delete_multi(['smmf']) is True
assert mc.set('smmf', 0xCA909) is True
assert mc.delete_multi(['smmf']) is True
assert mc.delete('smmf') is True
assert mc.delete('smmf') is True
def test_incr_decr(self):
mc = self.mc
mc.set('wazi', 99)
assert mc.incr('wazi', 1) == 100
assert mc.decr('wazi', 1) == 99
mc.delete('wazi')
assert mc.incr('wazi', 1) is None
assert mc.decr('wazi', 1) is None
def test_cas(self):
mc = self.mc
mc.delete('bilinda')
k1 = 'bilinda'
v1 = 'butchers'
assert mc.gets(k1) is None
mc.set(k1, v1)
v1_, ck = mc.gets('bilinda')
v2 = 'ding'
assert v1 == v1_
assert mc.cas(k1, v2, 0, ck) is True
assert mc.cas(k1, v1, 0, ck) is not True
v2_, ck = mc.gets('bilinda')
assert v2 == v2_
def test_large(self):
mc = self.mc
BUF_500KB = 'i' * (500 * 1000)
BUF_1MB = 'i' * (1000 * 1000)
key_500kb = 'test_500kb'
key_1mb = 'test_1mb'
mc.delete(key_500kb)
mc.delete(key_1mb)
mc.set(key_500kb, BUF_500KB)
assert mc.get(key_500kb) == BUF_500KB
mc.set(key_1mb, BUF_1MB)
assert mc.get(key_1mb) == BUF_1MB
mc.delete(key_500kb)
mc.delete(key_1mb)
dct = {key_500kb: BUF_500KB, key_1mb: BUF_1MB}
mc.set_multi(dct)
assert mc.get_multi(dct.keys()) == dct
def test_extra_large(self):
threshold = 1000000
mc = self.mc
key_xl = 'test_very_large'
val = 'i' * threshold
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val += 'b'
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val = 'i' * (threshold + threshold)
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val += 'b'
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
def test_noreply(self):
mc = self.noreply_mc
assert mc.set('foo', 'bar')
assert mc.touch('foo', 30)
v, ck = mc.gets('foo')
assert mc.cas('foo', 'bar2', 0, ck)
mc.delete('foo')
assert mc.set('foo', 1024)
assert mc.incr('foo', 1) is None
assert mc.get('foo') == 1025
assert mc.decr('foo', 3) is None
assert mc.get('foo') == 1022
def test_injection(self):
# credit to The New Page of Injections Book:
# Memcached Injections @ blackhat2014 [pdf](http://t.cn/RP0J10Z)
mc = self.mc
assert mc.delete('injected')
assert mc.set('a' * 250, 'biu')
assert not mc.set('a' * 251, 'biu')
mc.set('a' * 251, 'set injected 0 3600 10\r\n1234567890')
assert mc.get('injected') is None
mc.delete('injected')
mc.set('key1', '1234567890')
mc.set('key1 0', '123456789012345678901234567890\r\n'
'set injected 0 3600 3\r\nINJ\r\n')
assert mc.get('injected') is None
def test_maxiov(self):
key_tmpl = 'not_existed.%s'
assert self.mc.get_multi([key_tmpl % i for i in range(10000)]) == {}
def test_get_set_raw(self):
self.mc.set('foo', 233)
assert self.mc.get_raw('foo') == ('233', 2)
self.mc.set_raw('foo', '2335', 0, 2)
assert self.mc.get('foo') == 2335
def test_stats(self):
stats = self.mc.stats()
for addr, dct in stats.iteritems():
assert isinstance(dct['version'], str)
assert (isinstance(dct['rusage_system'], float) or
isinstance(dct['rusage_user'], float))
assert isinstance(dct['curr_connections'], int)
def test_get_set_large_raw(self):
key = 'large_raw_key'
key_dup = '%s_dup' % key
val = 'i' * int(_DOUBAN_CHUNK_SIZE * 1.5)
for mc in (self.mc, self.compressed_mc):
mc.set(key, val)
assert mc.get(key) == val
raw_val1, flags1 = mc.get_raw(key)
assert mc.set_raw(key_dup, raw_val1, 0, flags1)
assert mc.get_raw(key) == mc.get_raw(key_dup)
def test_patch_no_compress(self):
key = 'no_compress'
val = '1024'
self.mc.set(key, val, compress=False)
large_patch = 'hahahaha' * 512
self.mc.prepend(key, large_patch)
assert self.mc.get(key) == large_patch + val
self.mc.delete(key)
self.mc.set(key, val, compress=False)
large_patch = 'hahahaha' * 512
self.mc.append(key, large_patch)
assert self.mc.get(key) == val + large_patch
self.mc.delete(key)
class CmemcachedRegressionCase(unittest.TestCase):
def setUp(self):
host = "127.0.0.1"
port = 21211
self.server_addr = '%s:%d' % (host, port)
self.mc = Client([self.server_addr], comp_threshold=1024)
if cmemcached is not None:
self.old_mc = cmemcached.Client([self.server_addr],
comp_threshold=1024)
def test_set_get(self):
self.mc.set("key", "value")
self.assertEqual(self.mc.get("key"), "value")
self.mc.set("key_int", 1)
self.assertEqual(self.mc.get("key_int"), True)
self.mc.set("key_long", int(1234567890))
self.assertEqual(self.mc.get("key_long"), int(1234567890))
self.mc.set("key_object", BigObject())
self.assertEqual(self.mc.get("key_object"), BigObject())
big_object = BigObject('x', 1000001)
self.mc.set("key_big_object", big_object)
self.assertEqual(self.mc.get("key_big_object"), big_object)
def test_set_get_none(self):
self.assertEqual(self.mc.set('key', None), True)
self.assertEqual(self.mc.get('key'), None)
def test_chinese_set_get(self):
key = '豆瓣'
value = '在炎热的夏天我们无法停止上豆瓣'
self.assertEqual(self.mc.set(key, value), True)
self.assertEqual(self.mc.get(key), value)
def test_unicode_set_get(self):
key = "test_unicode_set_get"
value = u"中文"
self.assertEqual(self.mc.set(key, value), True)
self.assertEqual(self.mc.get(key), value)
def test_special_key(self):
key = 'keke a kid'
value = 1024
self.assertEqual(self.mc.set(key, value), False)
self.assertEqual(self.mc.get(key), None)
key = 'u:keke a kid'
self.assertEqual(self.mc.set(key, value), False)
self.assertEqual(self.mc.get(key), None)
def test_unicode_key(self):
key1 = u"answer"
key2 = u"答案"
bytes_key1 = "answer"
bytes_key2 = "答案"
value = 42
self.assertEqual(self.mc.set(key1, value), True)
self.assertEqual(self.mc.get(key1), value)
self.assertEqual(self.mc.set(key2, value), True)
self.assertEqual(self.mc.get(key2), value)
self.assertEqual(self.mc.incr(key2), value + 1)
self.assertEqual(self.mc.get(key2), value + 1)
self.assertEqual(self.mc.delete(key1), True)
self.assertEqual(self.mc.get(key1), None)
self.assertEqual(self.mc.add(key1, value), True)
self.assertEqual(self.mc.get(key1), value)
self.assertEqual(self.mc.add(key1, value), False)
self.assertEqual(self.mc.set(key1, value), True)
self.assertEqual(self.mc.get(bytes_key1), self.mc.get(key1))
self.assertEqual(self.mc.get(bytes_key2), self.mc.get(key2))
def test_add(self):
key = 'test_add'
self.mc.delete(key)
self.assertEqual(self.mc.add(key, 'tt'), True)
self.assertEqual(self.mc.get(key), 'tt')
self.assertEqual(self.mc.add(key, 'tt'), 0)
self.mc.delete(key + '2')
self.assertEqual(self.mc.add(key + '2', range(10)), True)
def test_replace(self):
key = 'test_replace'
self.mc.delete(key)
self.assertEqual(self.mc.replace(key, ''), 0)
self.assertEqual(self.mc.set(key, 'b'), True)
self.assertEqual(self.mc.replace(key, 'a'), True)
self.assertEqual(self.mc.get(key), 'a')
def test_append(self):
key = "test_append"
value = b"append\n"
self.mc.delete(key)
self.assertEqual(self.mc.append(key, value), 0)
self.mc.set(key, b"")
self.assertEqual(self.mc.append(key, value), True)
self.assertEqual(self.mc.append(key, value), True)
self.assertEqual(self.mc.prepend(key, b'before\n'), True)
self.assertEqual(self.mc.get(key), b'before\n' + value * 2)
def test_set_multi(self):
values = dict(('key%s' % k, ('value%s' % k) * 100)
for k in range(1000))
values.update({' ': ''})
self.assertEqual(self.mc.set_multi(values), False)
del values[' ']
self.assertEqual(self.mc.get_multi(list(values.keys())), values)
def test_append_large(self):
k = 'test_append_large'
self.mc.set(k, b'a' * 2048)
self.mc.append(k, b'bbbb')
assert b'bbbb' not in self.mc.get(k)
self.mc.set(k, b'a' * 2048, compress=False)
self.mc.append(k, b'bbbb')
assert b'bbbb' in self.mc.get(k)
def test_incr(self):
key = "Not_Exist"
self.assertEqual(self.mc.incr(key), None)
# key="incr:key1"
# self.mc.set(key, "not_numerical")
# self.assertEqual(self.mc.incr(key), 0)
key = "incr:key2"
self.mc.set(key, 2007)
self.assertEqual(self.mc.incr(key), 2008)
def test_decr(self):
key = "Not_Exist"
self.assertEqual(self.mc.decr(key), None)
# key="decr:key1"
# self.mc.set(key, "not_numerical")
# self.assertEqual(self.mc.decr(key),0)
key = "decr:key2"
self.mc.set(key, 2009)
self.assertEqual(self.mc.decr(key), 2008)
def test_get_multi(self):
keys = ["hello1", "hello2", "hello3"]
values = ["vhello1", "vhello2", "vhello3"]
for x in range(3):
self.mc.set(keys[x], values[x])
self.assertEqual(self.mc.get(keys[x]), values[x])
result = self.mc.get_multi(keys)
for x in range(3):
self.assertEqual(result[keys[x]], values[x])
def test_get_multi_invalid(self):
keys = ["hello1", "hello2", "hello3"]
values = ["vhello1", "vhello2", "vhello3"]
for x in range(3):
self.mc.set(keys[x], values[x])
self.assertEqual(self.mc.get(keys[x]), values[x])
invalid_keys = keys + ['hoho\r\n']
result = self.mc.get_multi(invalid_keys)
for x in range(3):
self.assertEqual(result[keys[x]], values[x])
result_new = self.mc.get_multi(keys)
for x in range(3):
self.assertEqual(result_new[keys[x]], values[x])
def test_get_multi_big(self):
keys = ["hello1", "hello2", "hello3"]
values = [BigObject(str(i), 1000001) for i in range(3)]
for x in range(3):
self.mc.set(keys[x], values[x])
self.assertEqual(self.mc.get(keys[x]), values[x])
result = self.mc.get_multi(keys)
for x in range(3):
self.assertEqual(result[keys[x]], values[x])
def test_get_multi_with_empty_string(self):
keys = ["hello1", "hello2", "hello3"]
for k in keys:
self.mc.set(k, '')
self.assertEqual(self.mc.get_multi(keys), dict(zip(keys, [""] * 3)))
def test_bool(self):
self.mc.set("bool", True)
value = self.mc.get("bool")
self.assertEqual(value, True)
self.mc.set("bool_", False)
value = self.mc.get("bool_")
self.assertEqual(value, False)
def testEmptyString(self):
self.assertTrue(self.mc.set("str", ''))
value = self.mc.get("str")
self.assertEqual(value, '')
def test_pickle(self):
v = [{"v": BigObject('a', 10)}]
self.mc.set("a", v)
self.assertEqual(self.mc.get("a"), v)
# TODO
'''
raw, flags = self.mc.get_raw("a")
self.assertEqual(raw, pickle.dumps(v, -1))
'''
def test_no_pickle(self):
v = NoPickle()
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.assertEqual(self.mc.set("nopickle", v), None)
self.assertEqual(self.mc.get("nopickle"), None)
def test_marshal(self):
v = [{2: {"a": 337}}]
self.mc.set("a", v)
self.assertEqual(self.mc.get("a"), v)
# TODO
'''
raw, flags = self.mc.get_raw("a")
self.assertEqual(raw, marshal.dumps(v, 2))
'''
def test_big_list(self):
v = range(1024 * 1024)
r = self.mc.set('big_list', v)
self.assertEqual(r, True)
self.assertEqual(self.mc.get('big_list'), v)
def test_touch(self):
self.mc.set('test', True)
self.assertEqual(self.mc.get('test'), True)
self.assertEqual(self.mc.touch('test', -1), True)
self.assertEqual(self.mc.get('test'), None)
self.mc.set('test', True)
self.assertEqual(self.mc.get('test'), True)
self.assertEqual(self.mc.touch('test', 1), True)
time.sleep(1)
self.assertEqual(self.mc.get('test'), None)
def test_client_pickable(self):
import pickle
d = pickle.dumps(self.mc)
self.mc = pickle.loads(d)
self.test_stats()
def test_stats(self):
s = self.mc.stats()
self.assertEqual(self.server_addr in s, True)
st = s[self.server_addr]
st_keys = {
"pid",
"uptime",
"time",
"version",
"pointer_size",
"rusage_user",
"rusage_system",
"curr_items",
"total_items",
"bytes",
"curr_connections",
"total_connections",
"connection_structures",
"cmd_get",
"cmd_set",
"get_hits",
"get_misses",
"evictions",
"bytes_read",
"bytes_written",
"limit_maxbytes",
"threads",
}
self.assertTrue(set(st.keys()) >= st_keys)
''' TODO
mc = cmemcached.Client([INVALID_SERVER_ADDR, self.server_addr])
s = mc.stats()
self.assertEqual(len(s), 2)
'''
def test_append_multi(self):
N = 10
K = "test_append_multi_%d"
data = b"after\n"
for i in range(N):
self.assertEqual(self.mc.set(K % i, b"before\n"), True)
keys = [K % i for i in range(N)]
# append
self.assertEqual(self.mc.append_multi(keys, data), True)
self.assertEqual(self.mc.get_multi(keys),
dict(zip(keys, [b"before\n" + data] * N)))
# prepend
self.assertEqual(self.mc.prepend_multi(keys, data), True)
self.assertEqual(self.mc.get_multi(keys),
dict(zip(keys, [data + b"before\n" + data] * N)))
# delete
self.assertEqual(self.mc.delete_multi(keys), True)
self.assertEqual(self.mc.get_multi(keys), {})
def test_append_multi_performance(self):
N = 70000
K = "test_append_multi_%d"
data = b"after\n"
keys = [K % i for i in range(N)]
t = time.time()
self.mc.append_multi(keys, data)
t = time.time() - t
threshold = 5
assert t < threshold, 'should append 7w key in %s secs, ' \
'actual val: %f' % (threshold, t)
def test_get_host(self):
host = self.mc.get_host_by_key("str")
self.assertEqual(host, self.server_addr)
def test_get_list(self):
self.mc.set("a", 'a')
self.mc.delete('b')
v = self.mc.get_list(['a', 'b'])
self.assertEqual(v, ['a', None])
@pytest.mark.skipif(np is None or cmemcached is None,
reason='cmemcached and numpy are not installed')
def test_general_get_set_regression(self):
key = 'anykey'
key_dup = '%s_dup' % key
for val in ('x', np.uint32(1), np.int32(2), 0, int(0), False, True):
self.old_mc.set(key, val)
val2 = self.mc.get(key)
assert val2 == val
self.mc.set(key_dup, val)
val3 = self.old_mc.get(key_dup)
assert val3 == val
@pytest.mark.skipif(cmemcached is None,
reason='cmemcached is not installed')
def test_large_mc_split_regression(self):
key = 'anykey'
key_dup = '%s_dup' % key
for val in ('i' * int(_DOUBAN_CHUNK_SIZE * 1.5), 'small_value'):
self.old_mc.set(key, val)
assert self.mc.get(key) == self.old_mc.get(key) == val
self.mc.set(key, val)
assert self.mc.get(key) == self.old_mc.get(key) == val
raw_val1, flags1 = self.old_mc.get_raw(key)
assert len(raw_val1) <= len(val) # compressed
assert self.old_mc.set_raw(key_dup, raw_val1, 0, flags1)
assert self.old_mc.get_raw(key) == self.old_mc.get_raw(key_dup)
raw_val2, flags2 = self.mc.get_raw(key)
assert len(raw_val2) <= len(val) # compressed
assert self.mc.set_raw(key_dup, raw_val2, 0, flags2)
assert self.mc.get_raw(key) == self.mc.get_raw(key_dup)
class CmemcachedRegressionCase(unittest.TestCase):
def setUp(self):
host = "127.0.0.1"
port = 21211
self.server_addr = '%s:%d' % (host, port)
self.mc = Client([self.server_addr], comp_threshold=1024)
if cmemcached is not None:
self.old_mc = cmemcached.Client([self.server_addr],
comp_threshold=1024)
def test_set_get(self):
self.mc.set("key", "value")
self.assertEqual(self.mc.get("key"), "value")
self.mc.set("key_int", 1)
self.assertEqual(self.mc.get("key_int"), True)
self.mc.set("key_long", int(1234567890))
self.assertEqual(self.mc.get("key_long"), int(1234567890))
self.mc.set("key_object", BigObject())
self.assertEqual(self.mc.get("key_object"), BigObject())
big_object = BigObject('x', 1000001)
self.mc.set("key_big_object", big_object)
self.assertEqual(self.mc.get("key_big_object"), big_object)
def test_set_get_none(self):
self.assertEqual(self.mc.set('key', None), True)
self.assertEqual(self.mc.get('key'), None)
def test_chinese_set_get(self):
key = '豆瓣'
value = '在炎热的夏天我们无法停止上豆瓣'
self.assertEqual(self.mc.set(key, value), True)
self.assertEqual(self.mc.get(key), value)
def test_unicode_set_get(self):
key = "test_unicode_set_get"
value = u"中文"
self.assertEqual(self.mc.set(key, value), True)
self.assertEqual(self.mc.get(key), value)
def test_special_key(self):
key = 'keke a kid'
value = 1024
self.assertEqual(self.mc.set(key, value), False)
self.assertEqual(self.mc.get(key), None)
key = 'u:keke a kid'
self.assertEqual(self.mc.set(key, value), False)
self.assertEqual(self.mc.get(key), None)
def test_unicode_key(self):
key1 = u"answer"
key2 = u"答案"
bytes_key1 = "answer"
bytes_key2 = "答案"
value = 42
self.assertEqual(self.mc.set(key1, value), True)
self.assertEqual(self.mc.get(key1), value)
self.assertEqual(self.mc.set(key2, value), True)
self.assertEqual(self.mc.get(key2), value)
self.assertEqual(self.mc.incr(key2), value + 1)
self.assertEqual(self.mc.get(key2), value + 1)
self.assertEqual(self.mc.delete(key1), True)
self.assertEqual(self.mc.get(key1), None)
self.assertEqual(self.mc.add(key1, value), True)
self.assertEqual(self.mc.get(key1), value)
self.assertEqual(self.mc.add(key1, value), False)
self.assertEqual(self.mc.set(key1, value), True)
self.assertEqual(self.mc.get(bytes_key1), self.mc.get(key1))
self.assertEqual(self.mc.get(bytes_key2), self.mc.get(key2))
def test_add(self):
key = 'test_add'
self.mc.delete(key)
self.assertEqual(self.mc.add(key, 'tt'), True)
self.assertEqual(self.mc.get(key), 'tt')
self.assertEqual(self.mc.add(key, 'tt'), 0)
self.mc.delete(key + '2')
self.assertEqual(self.mc.add(key + '2', range(10)), True)
def test_replace(self):
key = 'test_replace'
self.mc.delete(key)
self.assertEqual(self.mc.replace(key, ''), 0)
self.assertEqual(self.mc.set(key, 'b'), True)
self.assertEqual(self.mc.replace(key, 'a'), True)
self.assertEqual(self.mc.get(key), 'a')
def test_append(self):
key = "test_append"
value = b"append\n"
self.mc.delete(key)
self.assertEqual(self.mc.append(key, value), 0)
self.mc.set(key, b"")
self.assertEqual(self.mc.append(key, value), True)
self.assertEqual(self.mc.append(key, value), True)
self.assertEqual(self.mc.prepend(key, b'before\n'), True)
self.assertEqual(self.mc.get(key), b'before\n' + value * 2)
def test_set_multi(self):
values = dict(
('key%s' % k, ('value%s' % k) * 100) for k in range(1000))
values.update({' ': ''})
self.assertEqual(self.mc.set_multi(values), False)
del values[' ']
self.assertEqual(self.mc.get_multi(list(values.keys())), values)
def test_append_large(self):
k = 'test_append_large'
self.mc.set(k, b'a' * 2048)
self.mc.append(k, b'bbbb')
assert b'bbbb' not in self.mc.get(k)
self.mc.set(k, b'a' * 2048, compress=False)
self.mc.append(k, b'bbbb')
assert b'bbbb' in self.mc.get(k)
def test_incr(self):
key = "Not_Exist"
self.assertEqual(self.mc.incr(key), None)
# key="incr:key1"
# self.mc.set(key, "not_numerical")
# self.assertEqual(self.mc.incr(key), 0)
key = "incr:key2"
self.mc.set(key, 2007)
self.assertEqual(self.mc.incr(key), 2008)
def test_decr(self):
key = "Not_Exist"
self.assertEqual(self.mc.decr(key), None)
# key="decr:key1"
# self.mc.set(key, "not_numerical")
# self.assertEqual(self.mc.decr(key),0)
key = "decr:key2"
self.mc.set(key, 2009)
self.assertEqual(self.mc.decr(key), 2008)
def test_get_multi(self):
keys = ["hello1", "hello2", "hello3"]
values = ["vhello1", "vhello2", "vhello3"]
for x in range(3):
self.mc.set(keys[x], values[x])
self.assertEqual(self.mc.get(keys[x]), values[x])
result = self.mc.get_multi(keys)
for x in range(3):
self.assertEqual(result[keys[x]], values[x])
def test_get_multi_invalid(self):
keys = ["hello1", "hello2", "hello3"]
values = ["vhello1", "vhello2", "vhello3"]
for x in range(3):
self.mc.set(keys[x], values[x])
self.assertEqual(self.mc.get(keys[x]), values[x])
invalid_keys = keys + ['hoho\r\n']
result = self.mc.get_multi(invalid_keys)
for x in range(3):
self.assertEqual(result[keys[x]], values[x])
result_new = self.mc.get_multi(keys)
for x in range(3):
self.assertEqual(result_new[keys[x]], values[x])
def test_get_multi_big(self):
keys = ["hello1", "hello2", "hello3"]
values = [BigObject(str(i), 1000001) for i in range(3)]
for x in range(3):
self.mc.set(keys[x], values[x])
self.assertEqual(self.mc.get(keys[x]), values[x])
result = self.mc.get_multi(keys)
for x in range(3):
self.assertEqual(result[keys[x]], values[x])
def test_get_multi_with_empty_string(self):
keys = ["hello1", "hello2", "hello3"]
for k in keys:
self.mc.set(k, '')
self.assertEqual(self.mc.get_multi(keys), dict(zip(keys, [""] * 3)))
def test_bool(self):
self.mc.set("bool", True)
value = self.mc.get("bool")
self.assertEqual(value, True)
self.mc.set("bool_", False)
value = self.mc.get("bool_")
self.assertEqual(value, False)
def testEmptyString(self):
self.assertTrue(self.mc.set("str", ''))
value = self.mc.get("str")
self.assertEqual(value, '')
def test_pickle(self):
v = [{"v": BigObject('a', 10)}]
self.mc.set("a", v)
self.assertEqual(self.mc.get("a"), v)
# TODO
'''
raw, flags = self.mc.get_raw("a")
self.assertEqual(raw, pickle.dumps(v, -1))
'''
def test_no_pickle(self):
v = NoPickle()
self.assertEqual(self.mc.set("nopickle", v), None)
self.assertEqual(self.mc.get("nopickle"), None)
def test_marshal(self):
v = [{2: {"a": 337}}]
self.mc.set("a", v)
self.assertEqual(self.mc.get("a"), v)
# TODO
'''
raw, flags = self.mc.get_raw("a")
self.assertEqual(raw, marshal.dumps(v, 2))
'''
def test_big_list(self):
v = range(1024 * 1024)
r = self.mc.set('big_list', v)
self.assertEqual(r, True)
self.assertEqual(self.mc.get('big_list'), v)
def test_touch(self):
self.mc.set('test', True)
self.assertEqual(self.mc.get('test'), True)
self.assertEqual(self.mc.touch('test', -1), True)
self.assertEqual(self.mc.get('test'), None)
self.mc.set('test', True)
self.assertEqual(self.mc.get('test'), True)
self.assertEqual(self.mc.touch('test', 1), True)
time.sleep(1)
self.assertEqual(self.mc.get('test'), None)
def test_client_pickable(self):
import pickle
d = pickle.dumps(self.mc)
self.mc = pickle.loads(d)
self.test_stats()
def test_stats(self):
s = self.mc.stats()
self.assertEqual(self.server_addr in s, True)
st = s[self.server_addr]
st_keys = {
"pid",
"uptime",
"time",
"version",
"pointer_size",
"rusage_user",
"rusage_system",
"curr_items",
"total_items",
"bytes",
"curr_connections",
"total_connections",
"connection_structures",
"cmd_get",
"cmd_set",
"get_hits",
"get_misses",
"evictions",
"bytes_read",
"bytes_written",
"limit_maxbytes",
"threads",
}
self.assertTrue(set(st.keys()) >= st_keys)
''' TODO
mc = cmemcached.Client([INVALID_SERVER_ADDR, self.server_addr])
s = mc.stats()
self.assertEqual(len(s), 2)
'''
def test_append_multi(self):
N = 10
K = "test_append_multi_%d"
data = b"after\n"
for i in range(N):
self.assertEqual(self.mc.set(K % i, b"before\n"), True)
keys = [K % i for i in range(N)]
# append
self.assertEqual(self.mc.append_multi(keys, data), True)
self.assertEqual(self.mc.get_multi(keys),
dict(zip(keys, [b"before\n" + data] * N)))
# prepend
self.assertEqual(self.mc.prepend_multi(keys, data), True)
self.assertEqual(self.mc.get_multi(keys),
dict(zip(keys, [data + b"before\n" + data] * N)))
# delete
self.assertEqual(self.mc.delete_multi(keys), True)
self.assertEqual(self.mc.get_multi(keys), {})
def test_append_multi_performance(self):
N = 70000
K = "test_append_multi_%d"
data = b"after\n"
keys = [K % i for i in range(N)]
t = time.time()
self.mc.append_multi(keys, data)
t = time.time() - t
threshold = 5
assert t < threshold, 'should append 7w key in %s secs, ' \
'actual val: %f' % (threshold, t)
def test_get_host(self):
host = self.mc.get_host_by_key("str")
self.assertEqual(host, self.server_addr)
def test_get_list(self):
self.mc.set("a", 'a')
self.mc.delete('b')
v = self.mc.get_list(['a', 'b'])
self.assertEqual(v, ['a', None])
@pytest.mark.skipif(np is None or cmemcached is None,
reason='cmemcached and numpy are not installed')
def test_general_get_set_regression(self):
key = 'anykey'
key_dup = '%s_dup' % key
for val in ('x', np.uint32(1), np.int32(2), 0, int(0), False, True):
self.old_mc.set(key, val)
val2 = self.mc.get(key)
assert val2 == val
self.mc.set(key_dup, val)
val3 = self.old_mc.get(key_dup)
assert val3 == val
@pytest.mark.skipif(cmemcached is None,
reason='cmemcached is not installed')
def test_large_mc_split_regression(self):
key = 'anykey'
key_dup = '%s_dup' % key
for val in ('i' * int(_DOUBAN_CHUNK_SIZE * 1.5), 'small_value'):
self.old_mc.set(key, val)
assert self.mc.get(key) == self.old_mc.get(key) == val
self.mc.set(key, val)
assert self.mc.get(key) == self.old_mc.get(key) == val
raw_val1, flags1 = self.old_mc.get_raw(key)
assert len(raw_val1) <= len(val) # compressed
assert self.old_mc.set_raw(key_dup, raw_val1, 0, flags1)
assert self.old_mc.get_raw(key) == self.old_mc.get_raw(key_dup)
raw_val2, flags2 = self.mc.get_raw(key)
assert len(raw_val2) <= len(val) # compressed
assert self.mc.set_raw(key_dup, raw_val2, 0, flags2)
assert self.mc.get_raw(key) == self.mc.get_raw(key_dup)
class SingleServerCase(unittest.TestCase):
def setUp(self):
self.mc = Client(["127.0.0.1:21211"])
self.mc_alt = Client(["127.0.0.1:21211"])
self.compressed_mc = Client(["127.0.0.1:21211"], comp_threshold=1024)
self.noreply_mc = Client(["127.0.0.1:21211"], noreply=True)
def test_attribute(self):
"Test attributes are accessible from Python code"
mc = self.mc
assert hasattr(mc, 'comp_threshold')
assert hasattr(mc, 'servers')
def test_misc(self):
mc = self.mc
mc.get_multi(['foo', 'tuiche'])
mc.delete('foo')
mc.delete('tuiche')
assert mc.get('foo') is None
assert mc.get('tuiche') is None
mc.set('foo', 'biu')
mc.set('tuiche', 'bb')
assert mc.get('foo') == 'biu'
assert mc.get('tuiche') == 'bb'
assert (mc.get_multi(['foo', 'tuiche']) == {
'foo': 'biu',
'tuiche': 'bb'
})
mc.set_multi({'foo': 1024, 'tuiche': '8964'})
assert (mc.get_multi(['foo', 'tuiche']) == {
'foo': 1024,
'tuiche': '8964'
})
def test_delete(self):
mc = self.mc
assert mc.set('smmf', 0xCA909) is True
assert mc.get('smmf') == 0xCA909
assert mc.delete('smmf') is True
assert mc.delete_multi(['smmf']) is True
assert mc.set('smmf', 0xCA909) is True
assert mc.delete_multi(['smmf']) is True
assert mc.delete('smmf') is True
assert mc.delete('smmf') is True
def test_incr_decr(self):
mc = self.mc
mc.set('wazi', 99)
assert mc.incr('wazi', 1) == 100
assert mc.decr('wazi', 1) == 99
mc.delete('wazi')
assert mc.incr('wazi', 1) is None
assert mc.decr('wazi', 1) is None
def test_cas(self):
mc = self.mc
mc.delete('bilinda')
k1 = 'bilinda'
v1 = 'butchers'
assert mc.gets(k1) is None
mc.set(k1, v1)
v1_, ck = mc.gets('bilinda')
v2 = 'ding'
assert v1 == v1_
assert mc.cas(k1, v2, 0, ck) is True
assert mc.cas(k1, v1, 0, ck) is not True
v2_, ck = mc.gets('bilinda')
assert v2 == v2_
def test_large(self):
mc = self.mc
BUF_500KB = 'i' * (500 * 1000)
BUF_1MB = 'i' * (1000 * 1000)
key_500kb = 'test_500kb'
key_1mb = 'test_1mb'
mc.delete(key_500kb)
mc.delete(key_1mb)
mc.set(key_500kb, BUF_500KB)
assert mc.get(key_500kb) == BUF_500KB
mc.set(key_1mb, BUF_1MB)
assert mc.get(key_1mb) == BUF_1MB
mc.delete(key_500kb)
mc.delete(key_1mb)
dct = {key_500kb: BUF_500KB, key_1mb: BUF_1MB}
mc.set_multi(dct)
assert mc.get_multi(list(dct.keys())) == dct
def test_extra_large(self):
threshold = 1000000
mc = self.mc
key_xl = 'test_very_large'
val = 'i' * threshold
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val += 'b'
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val = 'i' * (threshold + threshold)
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
val += 'b'
assert mc.set(key_xl, val)
assert mc.get(key_xl) == val
def test_noreply(self):
mc = self.noreply_mc
assert mc.set('foo', 'bar')
assert mc.touch('foo', 30)
v, ck = mc.gets('foo')
assert mc.cas('foo', 'bar2', 0, ck)
mc.delete('foo')
assert mc.set('foo', 1024)
assert mc.incr('foo', 1) is None
assert mc.get('foo') == 1025
assert mc.decr('foo', 3) is None
assert mc.get('foo') == 1022
def test_injection(self):
# credit to The New Page of Injections Book:
# Memcached Injections @ blackhat2014 [pdf](http://t.cn/RP0J10Z)
mc = self.mc
assert mc.delete('injected')
assert mc.set('a' * 250, 'biu')
assert not mc.set('a' * 251, 'biu')
mc.set('a' * 251, 'set injected 0 3600 10\r\n1234567890')
assert mc.get('injected') is None
mc.delete('injected')
mc.set('key1', '1234567890')
mc.set(
'key1 0', '123456789012345678901234567890\r\n'
'set injected 0 3600 3\r\nINJ\r\n')
assert mc.get('injected') is None
def test_maxiov(self):
key_tmpl = 'not_existed.%s'
assert self.mc.get_multi([key_tmpl % i for i in range(10000)]) == {}
def test_get_set_raw(self):
self.mc.set('foo', 233)
assert self.mc.get_raw('foo') == (b'233', 2)
self.mc.set_raw('foo', b'2335', 0, 2)
assert self.mc.get('foo') == 2335
def test_stats(self):
stats = self.mc.stats()
for addr, dct in stats.items():
assert isinstance(dct['version'], basestring)
assert (isinstance(dct['rusage_system'], float)
or isinstance(dct['rusage_user'], float))
assert isinstance(dct['curr_connections'], int)
def test_get_set_large_raw(self):
key = 'large_raw_key'
key_dup = '%s_dup' % key
val = 'i' * int(_DOUBAN_CHUNK_SIZE * 1.5)
for mc in (self.mc, self.compressed_mc):
mc.set(key, val)
assert mc.get(key) == val
raw_val1, flags1 = mc.get_raw(key)
assert mc.set_raw(key_dup, raw_val1, 0, flags1)
assert mc.get_raw(key) == mc.get_raw(key_dup)
def test_patch_no_compress(self):
key = 'no_compress'
val = b'1024'
self.mc.set(key, val, compress=False)
large_patch = b'hahahaha' * 512
self.mc.prepend(key, large_patch)
assert self.mc.get_raw(key)[0] == large_patch + val
self.mc.delete(key)
self.mc.set(key, val, compress=False)
large_patch = b'hahahaha' * 512
self.mc.append(key, large_patch)
assert self.mc.get_raw(key)[0] == val + large_patch
self.mc.delete(key)
def test_quit(self):
assert self.mc.delete('all_is_well')
assert self.mc.set('all_is_well', 'bingo')
assert self.mc_alt.delete('all_is_well')
assert self.mc_alt.set('all_is_well', 'bingo')
assert self.mc.version() # establish all conns
assert self.mc_alt.version() # establish all conns
nc1 = self.mc.stats()[self.mc.servers[0]]['curr_connections']
nc2 = self.mc_alt.stats()[self.mc.servers[0]]['curr_connections']
assert nc1 == nc2
assert self.mc.quit()
max_wait = 3
while nc1 - 1 != nc2 and max_wait > 0:
nc2 = self.mc_alt.stats()[self.mc.servers[0]]['curr_connections']
max_wait -= 1
time.sleep(1)
assert nc1 - 1 == nc2
# back to life immediately
assert self.mc.get('all_is_well') == 'bingo'
def test_flush_all(self):
keys = ["flush_all_check_%s" % i for i in range(1000)]
value = "testing_flush_all"
dict_to_set = {key: value for key in keys}
self.mc.set_multi(dict_to_set)
retrieved_dct = self.mc.get_multi(keys)
assert retrieved_dct == dict_to_set
with pytest.raises(RuntimeError, match=r".*toggle.*"):
rtn = self.mc.flush_all()
self.mc.toggle_flush_all_feature(True)
rtn = self.mc.flush_all()
assert isinstance(rtn, list)
assert rtn == self.mc.servers
assert {} == self.mc.get_multi(keys)
self.mc.toggle_flush_all_feature(False)
with pytest.raises(RuntimeError, match=r".*toggle.*"):
rtn = self.mc.flush_all()
