def _check_data(self, data, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
try:
self.assertEqual(store.get(key), data)
except Exception, e:
return self.fail("fail to check key %s: %s" % (key, str(e)))
def test_get_broken(self):
print
print "test get broken data"
self.backend1.start()
self._gen_data("some value", prefix='test1', loop_num=1024, sector=0)
tempered_key = None
for key in self.backend1.generate_key(prefix="test2", count=1, sector=0):
tempered_key = key
store = MCStore(self.backend1_addr)
store.set(tempered_key, string_large)
store.close()
self._gen_data("other value", prefix='test3', loop_num=1024, sector=0)
self.assertEqual(self.backend1.item_count(), 2049)
# flush hint
self.backend1.stop()
self.backend1.start()
#make sure we produce a crc error
temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=False)
self.assertEqual(self.backend1.item_count(), 2049)
print "when beansdb encounter broken data when reading, it should delete it from htree"
store = MCStore(self.backend1_addr)
store.get(tempered_key) is None
store.close()
self.assertEqual(self.backend1.item_count(), 2048)
self._check_data("some value", prefix="test1", loop_num=1024, sector=0)
self._check_data("other value", prefix="test3", loop_num=1024, sector=0)
self.backend1.stop()
def _delete_data(self, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix,
count=loop_num,
sector=0):
if not store.delete(key):
return self.fail("fail to delete %s" % (key))
def test_gc_broken(self):
print
print "test gc broken data"
self.backend1.start()
self._gen_data("some value", prefix='test1', loop_num=1024, sector=0)
tempered_key = None
for key in self.backend1.generate_key(prefix="test2", count=1, sector=0):
tempered_key = key
store = MCStore(self.backend1_addr)
store.set(tempered_key, string_large)
store.close()
self._gen_data("other value", prefix='test1', loop_num=1024, sector=0)
self.assertEqual(self.backend1.item_count(), 1025)
# flush hint
self.backend1.stop()
self.backend1.start()
#make sure we produce a crc error
temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=False)
self._start_gc()
while True:
status = self._gc_status()
if status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
self.assertEqual(self.backend1.item_count(), 1024)
self._check_data("other value", prefix="test1", loop_num=1024, sector=0)
def _check_data(self, data, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
try:
self.assertEqual(store.get(key), data)
except Exception, e:
return self.fail("fail to check key %s: %s" % (key, str(e)))
def _gen_data(self, data, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix,
count=loop_num,
sector=0):
if not store.set(key, data):
return self.fail("fail to set %s" % (key))
def test_special_key(self):
self.backend1.start()
kvs = [('a'*250, 1), ("a", range(1000))]
store = MCStore(self.backend1_addr)
for k,v in kvs:
assert(store.set(k, v))
assert(v == store.get(k))
self.backend1.stop()
self.backend1.start()
store = MCStore(self.backend1_addr)
for (k,v) in kvs:
assert(v == store.get(k))
def test_delete_version(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key2"
store.set(key, "aaa")
self.assertEqual(self._get_version(store, key), 1)
store.delete(key)
self.assertEqual(self._get_version(store, key), None)
store.set(key, "bbb")
self.assertEqual(store.get(key), 'bbb')
self.assertEqual(self._get_version(store, key), 3)
def test_big_value(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "largekey"
size = 10 * 1024 * 1024
rsize = (((size + len(key) + 24) >> 8) + 1) << 8
string_large = random_string(size)
assert(store.set(key, string_large))
assert(store.get(key) == string_large)
self.append(rsize)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert(store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
def test_big_value(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "largekey"
size = 10 * 1024 * 1024
rsize = (((size + len(key) + 24) >> 8) + 1) << 8
string_large = random_string(size)
assert (store.set(key, string_large))
assert (store.get(key) == string_large)
self.append(rsize)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert (store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
def test_gc(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
ver_key = 'test_version_key'
store.set(ver_key, 1)
store.set(ver_key, 1, rev=3) # will only raise version in htree
self.assertEqual(self._get_version(store, ver_key), 3)
self._gen_data(1)
print "done set data to 1"
time.sleep(10)
self._gen_data(2)
self._gen_data(1, prefix='delete_group')
time.sleep(2)
self.assertEqual(self.backend1.item_count(), 20481)
self._delete_data(prefix='delete_group')
self.assertEqual(self.backend1.item_count(), 10241)
self.assert_(not store.delete('key not exists'))
self.assertEqual(self.backend1.item_count(), 10241)
print "stop beansdb to rotate data file and produce hint"
self.backend1.stop()
self.backend1.start()
print "deleted key should exists in data"
assert locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=1)
assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=-2)
print "done set data to 2"
self._start_gc(0)
print "gc started"
while True:
status = self._gc_status()
if status.find('running') >= 0:
self._check_data(2)
continue
elif status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
else:
self.fail(status)
self._check_data(2)
store = MCStore(self.backend1_addr)
self.assertEqual(self._get_version(store, ver_key), 3) # version 3 should be in data
print "check test key version, old version should not exist"
assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=3)
assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=1)
print "check data & hint"
check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)
print "deleted key got deleted from data file during gc"
assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0")
self.assertEqual(self.backend1.item_count(), 10241)
self.backend1.stop()
def test_delete_version(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.append(256)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
store.delete(key)
self.append(256)
self.assertEqual(self._get_meta(store, key), (-2, 0, self.last_pos))
store.set(key, "bbb")
self.append(256)
self.assertEqual(store.get(key), 'bbb')
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def test_bc_scan(self):
print ""
print "test bc scan broken data"
self.backend1.start()
self._gen_data("some value", prefix='test1', loop_num=1024, sector=0)
tempered_key = None
for key in self.backend1.generate_key(prefix="test2",
count=1,
sector=0):
tempered_key = key
store = MCStore(self.backend1_addr)
store.set(tempered_key, string_large)
store.close()
self._gen_data("other value", prefix='test3', loop_num=1024, sector=0)
self.assertEqual(self.backend1.item_count(), 2049)
self.backend1.stop()
#make sure we produce a crc error
assert temper_with_key_value(self.backend1.db_home,
self.backend1.db_depth,
tempered_key,
delete_hint=True)
delete_hint_and_htree(self.backend1.db_home, self.backend1.db_depth)
self.backend1.start()
self.assertEqual(self.backend1.item_count(), 2048)
self._check_data("some value", prefix="test1", loop_num=1024, sector=0)
self._check_data("other value",
prefix="test3",
loop_num=1024,
sector=0)
self.backend1.stop()
def test_special_key(self):
self.backend1.start()
kvs = [('a' * 250, 1), ("a", range(1000))]
store = MCStore(self.backend1_addr)
for k, v in kvs:
assert (store.set(k, v))
assert (v == store.get(k))
self.backend1.stop()
self.backend1.start()
store = MCStore(self.backend1_addr)
for (k, v) in kvs:
assert (v == store.get(k))
def test_bc_scan(self):
print ""
print "test bc scan broken data"
self.backend1.start()
self._gen_data("some value", prefix='test1', loop_num=1024, sector=0)
tempered_key = None
for key in self.backend1.generate_key(prefix="test2", count=1, sector=0):
tempered_key = key
store = MCStore(self.backend1_addr)
store.set(tempered_key, string_large)
store.close()
self._gen_data("other value", prefix='test3', loop_num=1024, sector=0)
self.assertEqual(self.backend1.item_count(), 2049)
self.backend1.stop()
#make sure we produce a crc error
assert temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=True)
delete_hint_and_htree(self.backend1.db_home, self.backend1.db_depth)
self.backend1.start()
self.assertEqual(self.backend1.item_count(), 2048)
self._check_data("some value", prefix="test1", loop_num=1024, sector=0)
self._check_data("other value", prefix="test3", loop_num=1024, sector=0)
self.backend1.stop()
def test_delete_version(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.append(256)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
store.delete(key)
self.append(256)
self.assertEqual(self._get_meta(store, key), (-2, 0, self.last_pos))
store.set(key, "bbb")
self.append(256)
self.assertEqual(store.get(key), 'bbb')
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def _test_compress(self, overflow):
self.backend1.start()
value = string.letters
store = MCStore(self.backend1_addr)
compressed_value = zlib.compress(value, 0)
key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0))
value_easy_compress = 'v' * len(compressed_value)
assert (store.set(key, value_easy_compress))
assert (store.get(key) == value_easy_compress)
self.append(256)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert (store.set_raw(key, compressed_value, flag=0x00000010))
assert (store.get(key) == value)
self.append(512 if overflow else 256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
assert (store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def test_gc_broken(self):
print
print "test gc broken data"
self.backend1.start()
self._gen_data("some value", prefix='test1', loop_num=1024, sector=0)
tempered_key = None
for key in self.backend1.generate_key(prefix="test2",
count=1,
sector=0):
tempered_key = key
store = MCStore(self.backend1_addr)
store.set(tempered_key, string_large)
store.close()
self._gen_data("other value", prefix='test1', loop_num=1024, sector=0)
self.assertEqual(self.backend1.item_count(), 1025)
# flush hint
self.backend1.stop()
self.backend1.start()
#make sure we produce a crc error
temper_with_key_value(self.backend1.db_home,
self.backend1.db_depth,
tempered_key,
delete_hint=False)
self._start_gc()
while True:
status = self._gc_status()
if status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
self.assertEqual(self.backend1.item_count(), 1024)
self._check_data("other value",
prefix="test1",
loop_num=1024,
sector=0)
def _test_compress(self, overflow):
self.backend1.start()
value = string.letters
store = MCStore(self.backend1_addr)
compressed_value = zlib.compress(value, 0)
key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0))
value_easy_compress = 'v'* len(compressed_value)
assert(store.set(key, value_easy_compress))
assert(store.get(key) == value_easy_compress)
self.append(256)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert(store.set_raw(key, compressed_value, flag = 0x00000010))
assert(store.get(key) == value)
self.append(512 if overflow else 256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
assert(store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def test_gen_data(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
self.assert_(store.set("largekey", string_large))
self.assert_(store.get("largekey") == string_large)
loop_num = 16 * 1024
for i in xrange(loop_num):
key = "test%s" % (i)
if not store.set(key, 1):
print "failed to set %s" % (key)
return self.fail("fail")
if not store.set(key, 2):
print "failed to set %s" % (key)
return self.fail("fail")
print "done set"
for i in xrange(loop_num):
key = "test%s" % (i)
try:
self.assertEqual(store.get(key), 2)
except Exception, e:
print key, "error", e
return self.fail("fail")
def test_set_verion(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.append(256)
self.assertEqual(store.get(key), "aaa")
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
store.set_raw(key, "bbb", rev=3)
self.append(256)
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
store.set_raw(key, "bbb", rev=4)
self.assertEqual(
self._get_meta(store, key),
(4, 0, self.last_pos)) # current behavior will raise version
store.set_raw(key, "ccc", rev=2)
self.assertEqual(store.get(key), "bbb")
self.assertEqual(
self._get_meta(store, key),
(4, 0, self.last_pos)) # current behavior will raise version
def test_set_verion(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.append(256)
self.assertEqual(store.get(key), "aaa")
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
store.set_raw(key, "bbb", rev=3)
self.append(256)
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
store.set_raw(key, "bbb", rev=4)
self.assertEqual(self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version
store.set_raw(key, "ccc", rev=2)
self.assertEqual(store.get(key), "bbb")
self.assertEqual(self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version
def _delete_data(self, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
if not store.delete(key):
return self.fail("fail to delete %s" % (key))
def test_set_verion(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.assertEqual(store.get(key), "aaa")
self.assertEqual(self._get_version(store, key), 1)
store.set_raw(key, "bbb", rev=3)
self.assertEqual(self._get_version(store, key), 3)
store.set_raw(key, "bbb", rev=4)
self.assertEqual(self._get_version(store, key), 4) # current behavior will raise version
store.set_raw(key, "ccc", rev=2)
self.assertEqual(store.get(key), "bbb")
self.assertEqual(self._get_version(store, key), 4) # current behavior will raise version
def _gen_data(self, data, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
if not store.set(key, data):
return self.fail("fail to set %s" % (key))
def test_gc(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
ver_key = 'test_version_key'
store.set(ver_key, 1)
store.set(ver_key, 1, rev=3) # will only raise version in htree
self.assertEqual(self._get_version(store, ver_key), 3)
self._gen_data(1)
print "done set data to 1"
time.sleep(10)
self._gen_data(2)
self._gen_data(1, prefix='delete_group')
time.sleep(2)
self.assertEqual(self.backend1.item_count(), 20481)
self._delete_data(prefix='delete_group')
self.assertEqual(self.backend1.item_count(), 10241)
self.assert_(not store.delete('key not exists'))
self.assertEqual(self.backend1.item_count(), 10241)
print "stop beansdb to rotate data file and produce hint"
self.backend1.stop()
self.backend1.start()
print "deleted key should exists in data"
assert locate_key_iterate(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key="delete_group" + "test0",
ver_=1)
assert locate_key_with_hint(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key="delete_group" + "test0",
ver_=-2)
print "done set data to 2"
self._start_gc(0)
print "gc started"
while True:
status = self._gc_status()
if status.find('running') >= 0:
self._check_data(2)
continue
elif status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
else:
self.fail(status)
self._check_data(2)
store = MCStore(self.backend1_addr)
self.assertEqual(self._get_version(store, ver_key),
3) # version 3 should be in data
print "check test key version, old version should not exist"
assert locate_key_with_hint(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key=ver_key,
ver_=3)
assert not locate_key_iterate(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key=ver_key,
ver_=1)
print "check data & hint"
check_data_hint_integrity(self.backend1.db_home,
db_depth=self.backend1.db_depth)
print "deleted key got deleted from data file during gc"
assert not locate_key_iterate(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key="delete_group" + "test0")
self.assertEqual(self.backend1.item_count(), 10241)
self.backend1.stop()
def test_get_broken(self):
print
print "test get broken data"
self.backend1.start()
self._gen_data("some value", prefix='test1', loop_num=1024, sector=0)
tempered_key = None
for key in self.backend1.generate_key(prefix="test2",
count=1,
sector=0):
tempered_key = key
store = MCStore(self.backend1_addr)
store.set(tempered_key, string_large)
store.close()
self._gen_data("other value", prefix='test3', loop_num=1024, sector=0)
self.assertEqual(self.backend1.item_count(), 2049)
# flush hint
self.backend1.stop()
self.backend1.start()
#make sure we produce a crc error
temper_with_key_value(self.backend1.db_home,
self.backend1.db_depth,
tempered_key,
delete_hint=False)
self.assertEqual(self.backend1.item_count(), 2049)
print "when beansdb encounter broken data when reading, it should delete it from htree"
store = MCStore(self.backend1_addr)
store.get(tempered_key) is None
store.close()
self.assertEqual(self.backend1.item_count(), 2048)
self._check_data("some value", prefix="test1", loop_num=1024, sector=0)
self._check_data("other value",
prefix="test3",
loop_num=1024,
sector=0)
self.backend1.stop()
