def test_normal(self):
proxy = MCStore(self.proxy.addr)
key = '/test/normal/key'
val = 'val1'
self.assertTrue(proxy.set(key, val))
self.assertEqual(proxy.get(key), val)
for db in self.dbs:
self._assert_val(db.addr, key, val)
def start(self):
assert self.popen is None
self.popen = start_cmd(self.cmd)
try_times = 0
while True:
try:
store = MCStore(self.addr)
store.get("@")
return
except IOError:
try_times += 1
if try_times > 20:
raise Exception('connect error for addr: %s', self.addr)
time.sleep(0.5)
def test_one_server_down(self):
proxy = MCStore(self.proxy.addr)
key = '/test/one/server/down'
val = 'val'
bad_server_idx = 0
self.dbs[bad_server_idx].stop()
self.assertTrue(proxy.set(key, val))
self.assertEqual(proxy.get(key), val)
for i, db in enumerate(self.dbs):
if i != bad_server_idx:
self._assert_val(db.addr, key, val)
# 没有写备节点
for db in self.backup_dbs:
self._assert_val(db.addr, key, None)
def test_two_server_down(self):
proxy = MCStore(self.proxy.addr)
key = '/test/two/server/down'
val = 'val'
bad_server_idxs = [0, 1]
for i in bad_server_idxs:
self.dbs[i].stop()
self.assertTrue(proxy.set(key, val))
self.assertEqual(proxy.get(key), val)
for i, db in enumerate(self.dbs):
if i not in bad_server_idxs:
self._assert_val(db.addr, key, val)
# 写备节点了. 这里假设测试配置中只有一个备节点
for db in self.backup_dbs:
self._assert_val(db.addr, key, val)
def test_big_value(self):
store = MCStore(self.proxy.addr)
key = 'largekey'
size = 10 * 1024 * 1024
rsize = (((size + len(key) + 24) >> 8) + 1) << 8
string_large = random_string(size / 10) * 10
self.assertTrue(store.set(key, string_large))
self.assertEqual(store.get(key), string_large)
self.update_pos(rsize)
self.assertEqual(self.get_meta(store, key), (1, 0, self.last_pos))
self.assertTrue(store.set(key, 'aaa'))
self.update_pos(256)
self.assertEqual(self.get_meta(store, key), (2, 0, self.last_pos))
self.checkCounterZero()
def test_three_server_down(self):
proxy = MCStore(self.proxy.addr)
key = '/test/three/server/down'
val = 'val'
bad_server_idxs = [0, 1, 2]
for i in bad_server_idxs:
self.dbs[i].stop()
self.assertFalse(proxy.set(key, val))
with self.assertRaises(IOError):
proxy.get(key)
for i, db in enumerate(self.dbs):
if i not in bad_server_idxs:
self._assert_val(db.addr, key, val)
# 写备节点了
for db in self.backup_dbs:
self._assert_val(db.addr, key, val)
def test_delete_version(self):
store = MCStore(self.proxy.addr)
key = 'key1'
store.set(key, 'aaa')
self.update_pos(256)
self.assertEqual(self.get_meta(store, key), (1, 0, self.last_pos))
store.delete(key)
self.update_pos(256)
self.assertEqual(store.get(key), None)
self.assertEqual(self.get_meta(store, key), (-2, 0, self.last_pos))
self.checkCounterZero()
store.set(key, 'bbb')
self.update_pos(256)
self.assertEqual(store.get(key), 'bbb')
self.assertEqual(self.get_meta(store, key), (3, 0, self.last_pos))
self.checkCounterZero()
def test_special_key(self):
store = MCStore(self.proxy.addr)
kvs = [('a' * 200, 1), ('a', range(1000))]
for k, v in kvs:
self.assertTrue(store.set(k, v))
self.assertEqual(store.get(k), v)
# restart
self.proxy.stop()
self.proxy.start()
store = MCStore(self.proxy.addr)
for (k, v) in kvs:
v2 = store.get(k)
self.assertEqual(v2, v, "key %s, value %s, not %s" % (k, v, v2))
self.checkCounterZero()
def _test_compress(self, overflow):
store = MCStore(self.db.addr)
value = string.letters
compressed_value = zlib.compress(value, 0)
key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0))
value_easy_compress = 'v' * len(compressed_value)
self.assertTrue(store.set(key, value_easy_compress))
self.assertEqual(store.get(key), value_easy_compress)
self.update_pos(256)
self.assertEqual(self.get_meta(store, key), (1, 0, self.last_pos))
self.assertTrue(store.set_raw(key, compressed_value, flag=0x00000010))
self.assertEqual(store.get(key), value)
self.update_pos(512 if overflow else 256)
self.assertEqual(self.get_meta(store, key), (2, 0, self.last_pos))
self.assertTrue(store.set(key, 'aaa'))
self.update_pos(256)
self.assertEqual(self.get_meta(store, key), (3, 0, self.last_pos))
self.checkCounterZero()
def setUp(self):
BaseTest.setUp(self)
self.store = MCStore(self.db.addr)
self.invalid_key = '/this/is/a/bad/key/%s' % chr(15)
class AbnormalCmdTest(BaseTest):
def setUp(self):
BaseTest.setUp(self)
self.store = MCStore(self.db.addr)
self.invalid_key = '/this/is/a/bad/key/%s' % chr(15)
def run_cmd_by_telnet(self, cmd, expected, timeout=2):
addr, port = self.db.addr.split(':')
t = telnetlib.Telnet(addr, port)
t.write('%s\r\n' % cmd)
out = t.read_until('\n', timeout=timeout)
t.write('quit\n')
t.close()
r = out.strip('\r\n')
self.assertEqual(r, expected)
def test_get(self):
# get not exist key
cmd = 'get /test/get'
self.run_cmd_by_telnet(cmd, 'END')
# invalid key
cmd = 'get %s' % self.invalid_key
self.run_cmd_by_telnet(cmd, 'END')
self.checkCounterZero()
def test_set(self):
# invalid key
cmd = 'set %s 0 0 3\r\naaa' % self.invalid_key
self.run_cmd_by_telnet(cmd, 'NOT_STORED')
cmd = 'set /test/set 0 0 3\r\naaaa'
self.run_cmd_by_telnet(cmd, 'CLIENT_ERROR bad data chunk')
self.checkCounterZero()
def test_incr(self):
key = '/test/incr'
self.assertEqual(self.store.delete(key), True)
cmd = 'incr %s 10' % key
self.run_cmd_by_telnet(cmd, '10')
self.assertEqual(self.store.get(key), 10)
# incr 一个 value 为字符串的 key
key = '/test/incr2'
self.assertEqual(self.store.set(key, 'aaa'), True)
cmd = 'incr %s 10' % key
self.run_cmd_by_telnet(cmd, '0')
self.assertEqual(self.store.get(key), 'aaa')
self.checkCounterZero()
def test_delete(self):
key = '/delete/not/exist/key'
cmd = 'delete %s' % key
self.run_cmd_by_telnet(cmd, 'NOT_FOUND')
cmd = 'delete %s' % self.invalid_key
self.run_cmd_by_telnet(cmd, 'NOT_FOUND')
self.checkCounterZero()
def test_get_meta_by_key(self):
key = '/get_meta_by_key/not/exist/key'
cmd = 'get ?%s' % key
self.run_cmd_by_telnet(cmd, 'END')
cmd = 'get ?%s' % self.invalid_key
self.run_cmd_by_telnet(cmd, 'END')
self.checkCounterZero()
def _assert_val(self, addr, key, val, msg=None):
store = MCStore(addr)
self.assertEqual(store.get(key), val, msg)
def test_set_version(self):
store = MCStore(self.proxy.addr)
key = 'key1'
store.set(key, 'aaa')
self.update_pos(256)
self.assertEqual(store.get(key), 'aaa')
self.assertEqual(self.get_meta(store, key), (1, 0, self.last_pos))
store.set_raw(key, 'bbb', rev=3)
self.update_pos(256)
self.assertEqual(self.get_meta(store, key), (3, 0, self.last_pos))
store.set_raw(key, 'bbb', rev=4)
self.assertEqual(self.get_meta(store, key), (4, 0, self.last_pos))
store.set_raw(key, 'ccc', rev=2)
self.assertEqual(store.get(key), 'bbb')
self.assertEqual(self.get_meta(store, key), (4, 0, self.last_pos))
self.checkCounterZero()
def test_incr(self):
store = MCStore(self.proxy.addr)
key = 'key1'
store.incr(key, 10)
self.assertEqual(store.get(key), 10)
self.checkCounterZero()
def test_collision(self):
# keyhash = "c80f795945b78f6b"
store = MCStore(self.db.addr)
# key1 and key2 have the same keyhash "c80f795945b78f6b"
# key_other and key_other2 is in bucket c too
# key_other is used to test with key1, key2 for get/set/gc...
# key_other2 is used to make it possible to gc [0, 1]
key1 = "processed_log_backup_text_20140912102821_1020_13301733"
key2 = "/subject/10460967/props"
key_other = "/ark/p/385242854/"
key_other2 = "/doumail/849134123/source"
keys = [key1, key2, key_other]
for k in keys:
self.assertTrue(store.set(k, k))
self.assertEqual(store.get(k), k)
for i, k in enumerate(keys):
self.assertEqual(store.get(k), k)
ver = 2 if i == 1 else 1
self.assertEqual(self.get_meta(store, k), (ver, 0, i * 256))
self.db.stop()
self.db.start()
store = MCStore(self.db.addr)
for k in keys:
self.assertEqual(store.get(k), k)
for k in keys:
self.assertTrue(store.set(k, k + k))
for i, k in enumerate(keys):
self.assertEqual(store.get(k), k + k)
ver = 3 if i == 1 else 2
self.assertEqual(self.get_meta(store, k), (ver, 1, i * 256))
self.db.stop()
self.db.start()
store = MCStore(self.db.addr)
self.assertTrue(store.set(key_other2, 1))
self.db.stop()
self.db.start()
store = MCStore(self.db.addr)
self.gc(12)
time.sleep(1)
for i, k in enumerate(keys):
self.assertEqual(store.get(k), k + k)
ver = 3 if i == 1 else 2
self.assertEqual(self.get_meta(store, k), (ver, 0, i * 256))