def _check_data(self, data, prefix='', loop_num=10 * 1024): store = MCStore(self.backend1_addr) for key in self.backend1.generate_key(prefix=prefix, count=loop_num): try: self.assertEqual(store.get(key), data) except Exception, e: return self.fail("fail to check key %s: %s" % (key, str(e)))
def test_get_broken(self): print print "test get broken data" self.backend1.start() self._gen_data("some value", prefix='test1', loop_num=1024, sector=0) tempered_key = None for key in self.backend1.generate_key(prefix="test2", count=1, sector=0): tempered_key = key store = MCStore(self.backend1_addr) store.set(tempered_key, string_large) store.close() self._gen_data("other value", prefix='test3', loop_num=1024, sector=0) self.assertEqual(self.backend1.item_count(), 2049) # flush hint self.backend1.stop() self.backend1.start() #make sure we produce a crc error temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=False) self.assertEqual(self.backend1.item_count(), 2049) print "when beansdb encounter broken data when reading, it should delete it from htree" store = MCStore(self.backend1_addr) store.get(tempered_key) is None store.close() self.assertEqual(self.backend1.item_count(), 2048) self._check_data("some value", prefix="test1", loop_num=1024, sector=0) self._check_data("other value", prefix="test3", loop_num=1024, sector=0) self.backend1.stop()
def _delete_data(self, prefix='', loop_num=10 * 1024): store = MCStore(self.backend1_addr) for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=0): if not store.delete(key): return self.fail("fail to delete %s" % (key))
def test_gc_broken(self): print print "test gc broken data" self.backend1.start() self._gen_data("some value", prefix='test1', loop_num=1024, sector=0) tempered_key = None for key in self.backend1.generate_key(prefix="test2", count=1, sector=0): tempered_key = key store = MCStore(self.backend1_addr) store.set(tempered_key, string_large) store.close() self._gen_data("other value", prefix='test1', loop_num=1024, sector=0) self.assertEqual(self.backend1.item_count(), 1025) # flush hint self.backend1.stop() self.backend1.start() #make sure we produce a crc error temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=False) self._start_gc() while True: status = self._gc_status() if status == 'success': print "done gc" break elif status == 'fail': return self.fail("optimize_stat = fail") self.assertEqual(self.backend1.item_count(), 1024) self._check_data("other value", prefix="test1", loop_num=1024, sector=0)
def _gen_data(self, data, prefix='', loop_num=10 * 1024): store = MCStore(self.backend1_addr) for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=0): if not store.set(key, data): return self.fail("fail to set %s" % (key))
def test_special_key(self): self.backend1.start() kvs = [('a'*250, 1), ("a", range(1000))] store = MCStore(self.backend1_addr) for k,v in kvs: assert(store.set(k, v)) assert(v == store.get(k)) self.backend1.stop() self.backend1.start() store = MCStore(self.backend1_addr) for (k,v) in kvs: assert(v == store.get(k))
def test_delete_version(self): self.backend1.start() store = MCStore(self.backend1_addr) key = "key2" store.set(key, "aaa") self.assertEqual(self._get_version(store, key), 1) store.delete(key) self.assertEqual(self._get_version(store, key), None) store.set(key, "bbb") self.assertEqual(store.get(key), 'bbb') self.assertEqual(self._get_version(store, key), 3)
def test_big_value(self): self.backend1.start() store = MCStore(self.backend1_addr) key = "largekey" size = 10 * 1024 * 1024 rsize = (((size + len(key) + 24) >> 8) + 1) << 8 string_large = random_string(size) assert(store.set(key, string_large)) assert(store.get(key) == string_large) self.append(rsize) self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) assert(store.set(key, 'aaa')) self.append(256) self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
def test_big_value(self): self.backend1.start() store = MCStore(self.backend1_addr) key = "largekey" size = 10 * 1024 * 1024 rsize = (((size + len(key) + 24) >> 8) + 1) << 8 string_large = random_string(size) assert (store.set(key, string_large)) assert (store.get(key) == string_large) self.append(rsize) self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) assert (store.set(key, 'aaa')) self.append(256) self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
def test_gc(self): self.backend1.start() store = MCStore(self.backend1_addr) ver_key = 'test_version_key' store.set(ver_key, 1) store.set(ver_key, 1, rev=3) # will only raise version in htree self.assertEqual(self._get_version(store, ver_key), 3) self._gen_data(1) print "done set data to 1" time.sleep(10) self._gen_data(2) self._gen_data(1, prefix='delete_group') time.sleep(2) self.assertEqual(self.backend1.item_count(), 20481) self._delete_data(prefix='delete_group') self.assertEqual(self.backend1.item_count(), 10241) self.assert_(not store.delete('key not exists')) self.assertEqual(self.backend1.item_count(), 10241) print "stop beansdb to rotate data file and produce hint" self.backend1.stop() self.backend1.start() print "deleted key should exists in data" assert locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=1) assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=-2) print "done set data to 2" self._start_gc(0) print "gc started" while True: status = self._gc_status() if status.find('running') >= 0: self._check_data(2) continue elif status == 'success': print "done gc" break elif status == 'fail': return self.fail("optimize_stat = fail") else: self.fail(status) self._check_data(2) store = MCStore(self.backend1_addr) self.assertEqual(self._get_version(store, ver_key), 3) # version 3 should be in data print "check test key version, old version should not exist" assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=3) assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=1) print "check data & hint" check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth) print "deleted key got deleted from data file during gc" assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0") self.assertEqual(self.backend1.item_count(), 10241) self.backend1.stop()
def test_delete_version(self): self.backend1.start() store = MCStore(self.backend1_addr) key = "key1" store.set(key, "aaa") self.append(256) self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) store.delete(key) self.append(256) self.assertEqual(self._get_meta(store, key), (-2, 0, self.last_pos)) store.set(key, "bbb") self.append(256) self.assertEqual(store.get(key), 'bbb') self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def test_bc_scan(self): print "" print "test bc scan broken data" self.backend1.start() self._gen_data("some value", prefix='test1', loop_num=1024, sector=0) tempered_key = None for key in self.backend1.generate_key(prefix="test2", count=1, sector=0): tempered_key = key store = MCStore(self.backend1_addr) store.set(tempered_key, string_large) store.close() self._gen_data("other value", prefix='test3', loop_num=1024, sector=0) self.assertEqual(self.backend1.item_count(), 2049) self.backend1.stop() #make sure we produce a crc error assert temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=True) delete_hint_and_htree(self.backend1.db_home, self.backend1.db_depth) self.backend1.start() self.assertEqual(self.backend1.item_count(), 2048) self._check_data("some value", prefix="test1", loop_num=1024, sector=0) self._check_data("other value", prefix="test3", loop_num=1024, sector=0) self.backend1.stop()
def test_special_key(self): self.backend1.start() kvs = [('a' * 250, 1), ("a", range(1000))] store = MCStore(self.backend1_addr) for k, v in kvs: assert (store.set(k, v)) assert (v == store.get(k)) self.backend1.stop() self.backend1.start() store = MCStore(self.backend1_addr) for (k, v) in kvs: assert (v == store.get(k))
def _test_compress(self, overflow): self.backend1.start() value = string.letters store = MCStore(self.backend1_addr) compressed_value = zlib.compress(value, 0) key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0)) value_easy_compress = 'v' * len(compressed_value) assert (store.set(key, value_easy_compress)) assert (store.get(key) == value_easy_compress) self.append(256) self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) assert (store.set_raw(key, compressed_value, flag=0x00000010)) assert (store.get(key) == value) self.append(512 if overflow else 256) self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos)) assert (store.set(key, 'aaa')) self.append(256) self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def _test_compress(self, overflow): self.backend1.start() value = string.letters store = MCStore(self.backend1_addr) compressed_value = zlib.compress(value, 0) key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0)) value_easy_compress = 'v'* len(compressed_value) assert(store.set(key, value_easy_compress)) assert(store.get(key) == value_easy_compress) self.append(256) self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) assert(store.set_raw(key, compressed_value, flag = 0x00000010)) assert(store.get(key) == value) self.append(512 if overflow else 256) self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos)) assert(store.set(key, 'aaa')) self.append(256) self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def test_gen_data(self): self.backend1.start() store = MCStore(self.backend1_addr) self.assert_(store.set("largekey", string_large)) self.assert_(store.get("largekey") == string_large) loop_num = 16 * 1024 for i in xrange(loop_num): key = "test%s" % (i) if not store.set(key, 1): print "failed to set %s" % (key) return self.fail("fail") if not store.set(key, 2): print "failed to set %s" % (key) return self.fail("fail") print "done set" for i in xrange(loop_num): key = "test%s" % (i) try: self.assertEqual(store.get(key), 2) except Exception, e: print key, "error", e return self.fail("fail")
def test_set_verion(self): self.backend1.start() store = MCStore(self.backend1_addr) key = "key1" store.set(key, "aaa") self.append(256) self.assertEqual(store.get(key), "aaa") self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) store.set_raw(key, "bbb", rev=3) self.append(256) self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos)) store.set_raw(key, "bbb", rev=4) self.assertEqual( self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version store.set_raw(key, "ccc", rev=2) self.assertEqual(store.get(key), "bbb") self.assertEqual( self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version
def test_set_verion(self): self.backend1.start() store = MCStore(self.backend1_addr) key = "key1" store.set(key, "aaa") self.append(256) self.assertEqual(store.get(key), "aaa") self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) store.set_raw(key, "bbb", rev=3) self.append(256) self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos)) store.set_raw(key, "bbb", rev=4) self.assertEqual(self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version store.set_raw(key, "ccc", rev=2) self.assertEqual(store.get(key), "bbb") self.assertEqual(self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version
def _delete_data(self, prefix='', loop_num=10 * 1024): store = MCStore(self.backend1_addr) for key in self.backend1.generate_key(prefix=prefix, count=loop_num): if not store.delete(key): return self.fail("fail to delete %s" % (key))
def test_set_verion(self): self.backend1.start() store = MCStore(self.backend1_addr) key = "key1" store.set(key, "aaa") self.assertEqual(store.get(key), "aaa") self.assertEqual(self._get_version(store, key), 1) store.set_raw(key, "bbb", rev=3) self.assertEqual(self._get_version(store, key), 3) store.set_raw(key, "bbb", rev=4) self.assertEqual(self._get_version(store, key), 4) # current behavior will raise version store.set_raw(key, "ccc", rev=2) self.assertEqual(store.get(key), "bbb") self.assertEqual(self._get_version(store, key), 4) # current behavior will raise version
def _gen_data(self, data, prefix='', loop_num=10 * 1024): store = MCStore(self.backend1_addr) for key in self.backend1.generate_key(prefix=prefix, count=loop_num): if not store.set(key, data): return self.fail("fail to set %s" % (key))