def test_gc(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
ver_key = 'test_version_key'
store.set(ver_key, 1)
store.set(ver_key, 1, rev=3) # will only raise version in htree
self.assertEqual(self._get_version(store, ver_key), 3)
self._gen_data(1)
print "done set data to 1"
time.sleep(10)
self._gen_data(2)
self._gen_data(1, prefix='delete_group')
time.sleep(2)
self.assertEqual(self.backend1.item_count(), 20481)
self._delete_data(prefix='delete_group')
self.assertEqual(self.backend1.item_count(), 10241)
self.assert_(not store.delete('key not exists'))
self.assertEqual(self.backend1.item_count(), 10241)
print "stop beansdb to rotate data file and produce hint"
self.backend1.stop()
self.backend1.start()
print "deleted key should exists in data"
assert locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=1)
assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=-2)
print "done set data to 2"
self._start_gc(0)
print "gc started"
while True:
status = self._gc_status()
if status.find('running') >= 0:
self._check_data(2)
continue
elif status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
else:
self.fail(status)
self._check_data(2)
store = MCStore(self.backend1_addr)
self.assertEqual(self._get_version(store, ver_key), 3) # version 3 should be in data
print "check test key version, old version should not exist"
assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=3)
assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=1)
print "check data & hint"
check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)
print "deleted key got deleted from data file during gc"
assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0")
self.assertEqual(self.backend1.item_count(), 10241)
self.backend1.stop()
print "done get"
self.backend1.stop()
print "stopped"
self.backend1.start()
print "started"
store = MCStore(self.backend1_addr)
for i in xrange(loop_num):
key = "test%s" % (i)
try:
self.assertEqual(store.get(key), 2)
except Exception, e:
print key, "error", e
return self.fail("fail")
print "done get"
print "check data & hint"
check_data_hint_integrity(self.backend1.db_home, self.backend1.db_depth)
self.assertEqual(self.backend1.item_count(), loop_num + 1)
self.backend1.stop()
print "delete .hint and .htree, should regenerate"
delete_hint_and_htree(self.backend1.db_home, self.backend1.db_depth)
self.backend1.start()
print "check data & hint"
check_data_hint_integrity(self.backend1.db_home, self.backend1.db_depth)
self.assertEqual(self.backend1.item_count(), loop_num + 1)
def tearDown(self):
self.backend1.stop()
class TestGenerateData2(TestGenerateData):
def test_gc(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
ver_key = 'test_version_key'
store.set(ver_key, 1)
store.set(ver_key, 1, rev=3) # will only raise version in htree
self.assertEqual(self._get_version(store, ver_key), 3)
self._gen_data(1)
print "done set data to 1"
time.sleep(10)
self._gen_data(2)
self._gen_data(1, prefix='delete_group')
time.sleep(2)
self.assertEqual(self.backend1.item_count(), 20481)
self._delete_data(prefix='delete_group')
self.assertEqual(self.backend1.item_count(), 10241)
self.assert_(not store.delete('key not exists'))
self.assertEqual(self.backend1.item_count(), 10241)
print "stop beansdb to rotate data file and produce hint"
self.backend1.stop()
self.backend1.start()
print "deleted key should exists in data"
assert locate_key_iterate(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key="delete_group" + "test0",
ver_=1)
assert locate_key_with_hint(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key="delete_group" + "test0",
ver_=-2)
print "done set data to 2"
self._start_gc(0)
print "gc started"
while True:
status = self._gc_status()
if status.find('running') >= 0:
self._check_data(2)
continue
elif status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
else:
self.fail(status)
self._check_data(2)
store = MCStore(self.backend1_addr)
self.assertEqual(self._get_version(store, ver_key),
3) # version 3 should be in data
print "check test key version, old version should not exist"
assert locate_key_with_hint(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key=ver_key,
ver_=3)
assert not locate_key_iterate(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key=ver_key,
ver_=1)
print "check data & hint"
check_data_hint_integrity(self.backend1.db_home,
db_depth=self.backend1.db_depth)
print "deleted key got deleted from data file during gc"
assert not locate_key_iterate(self.backend1.db_home,
db_depth=self.backend1.db_depth,
key="delete_group" + "test0")
self.assertEqual(self.backend1.item_count(), 10241)
self.backend1.stop()
def test_gc_multiple_files(self):
self.backend1.start()
self._gen_data(1, prefix='group1_', loop_num=16 * 1024)
#5M data file 1
self._gen_data(2, prefix='group1_', loop_num=16 * 1024)
print 'group1'
self.backend1.stop()
self.backend1.start()
print "restarted"
self._gen_data(1, prefix='group2_', loop_num=16 * 1024)
self._gen_data(2, prefix='group2_', loop_num=16 * 1024)
self.backend1.stop()
self.backend1.start()
print "restarted"
#5M data file 2
# data file 3
self._gen_data(1, prefix='group3_', loop_num=512)
self._gen_data(2, prefix='group3_', loop_num=512)
self._gen_data(1, prefix='group4_', loop_num=16 * 1024, sector=1)
self._gen_data(2, prefix='group4_', loop_num=16 * 1024, sector=1)
self.assertEqual(self.backend1.item_count(), 32 * 1024 + 512 + 16 * 1024)
sector0_exp = os.path.join(self.backend1.db_home, "0/*.data")
print "sector0 files", glob.glob(sector0_exp)
self.assertEqual(len(glob.glob(sector0_exp)), 3)
self.backend1.stop()
print "test append some junk data to file, simulate incomplete data file got gc"
with open(os.path.join(self.backend1.db_home, "0/001.data"), 'a') as f:
f.write("junkdatasdfsdfsdfdfdf")
buckets_txt_path = os.path.join(self.backend1.db_home, "0/buckets.txt")
if os.path.exists(buckets_txt_path):
print "rm", buckets_txt_path
os.remove(buckets_txt_path)
self.backend1.start()
def check_data():
self._check_data(2, prefix='group1_', loop_num=16 * 1024)
self._check_data(2, prefix='group2_', loop_num=16 * 1024)
self._check_data(2, prefix='group3_', loop_num=512)
time.sleep(1)
self._start_gc(0, bucket="0")
print "gc started"
while True:
status = self._gc_status()
if status.find('running') >= 0:
check_data()
continue
elif status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
else:
self.fail(status)
self.assertEqual(self.backend1.item_count(), 32 * 1024 + 512 + 16 * 1024)
check_data()
for key in self.backend1.generate_key(prefix="group1_", count=2, sector=0):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key, ver_=2))
print "group2 should be not in 000.data, but in 001.data"
for key in self.backend1.generate_key(prefix="group2_", count=2, sector=0):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/001.data"), key, ver_=2))
print "group4 of bucket 1 should be untouched"
for key in self.backend1.generate_key(prefix="group4_", count=2, sector=1):
print key
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/000.data"), key, ver_=2))
print "check data& hint"
check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)
def test_gc_multiple_files(self):
self.backend1.start()
self._gen_data(1, prefix='group1_', loop_num=16 * 1024)
#5M data file 1
self._gen_data(2, prefix='group1_', loop_num=16 * 1024)
print 'group1'
self.backend1.stop()
self.backend1.start()
print "restarted"
self._gen_data(1, prefix='group2_', loop_num=16 * 1024)
self._gen_data(2, prefix='group2_', loop_num=16 * 1024)
self.backend1.stop()
self.backend1.start()
print "restarted"
#5M data file 2
# data file 3
self._gen_data(1, prefix='group3_', loop_num=512)
self._gen_data(2, prefix='group3_', loop_num=512)
self._gen_data(1, prefix='group4_', loop_num=16 * 1024, sector=(1, 0))
self._gen_data(2, prefix='group4_', loop_num=16 * 1024, sector=(1, 0))
self._gen_data(1, prefix='group5_', loop_num=16 * 1024, sector=(0, 1))
self._gen_data(2, prefix='group5_', loop_num=16 * 1024, sector=(0, 1))
self.assertEqual(self.backend1.item_count(), 64 * 1024 + 512)
sector00_exp = os.path.join(self.backend1.db_home, "0/0/0*.data")
sector01_exp = os.path.join(self.backend1.db_home, "0/1/0*.data")
sector10_exp = os.path.join(self.backend1.db_home, "1/0/0*.data")
self.assertEqual(len(glob.glob(sector00_exp)), 3)
self.assertEqual(len(glob.glob(sector01_exp)), 1)
self.assertEqual(len(glob.glob(sector10_exp)), 1)
self.backend1.stop()
def check_data():
self._check_data(2, prefix='group1_', loop_num=16 * 1024)
self._check_data(2, prefix='group2_', loop_num=16 * 1024)
self._check_data(2, prefix='group3_', loop_num=512)
self.backend1.start()
self._start_gc(0, bucket="1")
print "gc @1 started"
while True:
status = self._gc_status()
if status.find('running') >= 0:
check_data()
continue
elif status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
else:
self.fail(status)
print "group4 got gc"
for key in self.backend1.generate_key(prefix="group4_", count=2, sector=(1, 0)):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "1/0/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/0/000.data"), key, ver_=2))
print "group1 should be untouched"
for key in self.backend1.generate_key(prefix="group1_", count=2, sector=(0, 0)):
print key
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=2))
time.sleep(1)
self._start_gc(0, bucket="00")
print "gc @00 started"
while True:
status = self._gc_status()
if status.find('running') >= 0:
check_data()
continue
elif status == 'success':
print "done gc"
break
elif status == 'fail':
return self.fail("optimize_stat = fail")
else:
self.fail(status)
self.assertEqual(self.backend1.item_count(), 64 * 1024 + 512)
check_data()
print "group1 got gc"
for key in self.backend1.generate_key(prefix="group1_", count=2, sector=(0, 0)):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=2))
print "group2 should be not in 000.data, but in 001.data"
for key in self.backend1.generate_key(prefix="group2_", count=2, sector=(0, 0)):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/001.data"), key, ver_=2))
print "group5 of bucket 0/1 should be untouched"
for key in self.backend1.generate_key(prefix="group5_", count=2, sector=(0, 1)):
print key
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/1/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/1/000.data"), key, ver_=2))
print "check data& hint"
check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)
def test_gc_multiple_files(self):
self.backend1.start()
self._gen_data(1, prefix="group1_", loop_num=16 * 1024)
# 5M data file 1
self._gen_data(2, prefix="group1_", loop_num=16 * 1024)
print "group1"
self.backend1.stop()
self.backend1.start()
print "restarted"
self._gen_data(1, prefix="group2_", loop_num=16 * 1024)
self._gen_data(2, prefix="group2_", loop_num=16 * 1024)
self.backend1.stop()
self.backend1.start()
print "restarted"
# 5M data file 2
# data file 3
self._gen_data(1, prefix="group3_", loop_num=512)
self._gen_data(2, prefix="group3_", loop_num=512)
self._gen_data(1, prefix="group4_", loop_num=16 * 1024, sector=1)
self._gen_data(2, prefix="group4_", loop_num=16 * 1024, sector=1)
self.assertEqual(self.backend1.item_count(), 32 * 1024 + 512 + 16 * 1024)
sector0_exp = os.path.join(self.backend1.db_home, "0/*.data")
print "sector0 files", glob.glob(sector0_exp)
self.assertEqual(len(glob.glob(sector0_exp)), 3)
self.backend1.stop()
print "test append some junk data to file, simulate incomplete data file got gc"
with open(os.path.join(self.backend1.db_home, "0/001.data"), "a") as f:
f.write("junkdatasdfsdfsdfdfdf")
buckets_txt_path = os.path.join(self.backend1.db_home, "0/buckets.txt")
if os.path.exists(buckets_txt_path):
print "rm", buckets_txt_path
os.remove(buckets_txt_path)
self.backend1.start()
def check_data():
self._check_data(2, prefix="group1_", loop_num=16 * 1024)
self._check_data(2, prefix="group2_", loop_num=16 * 1024)
self._check_data(2, prefix="group3_", loop_num=512)
time.sleep(1)
self._start_gc(0, bucket="0")
print "gc started"
while True:
status = self._gc_status()
if status.find("running") >= 0:
check_data()
continue
elif status == "success":
print "done gc"
break
elif status == "fail":
return self.fail("optimize_stat = fail")
else:
self.fail(status)
self.assertEqual(self.backend1.item_count(), 32 * 1024 + 512 + 16 * 1024)
check_data()
for key in self.backend1.generate_key(prefix="group1_", count=2, sector=0):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key, ver_=2))
print "group2 should be not in 000.data, but in 001.data"
for key in self.backend1.generate_key(prefix="group2_", count=2, sector=0):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/001.data"), key, ver_=2))
print "group4 of bucket 1 should be untouched"
for key in self.backend1.generate_key(prefix="group4_", count=2, sector=1):
print key
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/000.data"), key, ver_=2))
print "check data& hint"
check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)
def test_gc_multiple_files(self):
self.backend1.start()
self._gen_data(1, prefix="group1_", loop_num=16 * 1024)
# 5M data file 1
self._gen_data(2, prefix="group1_", loop_num=16 * 1024)
print "group1"
self.backend1.stop()
self.backend1.start()
print "restarted"
self._gen_data(1, prefix="group2_", loop_num=16 * 1024)
self._gen_data(2, prefix="group2_", loop_num=16 * 1024)
self.backend1.stop()
self.backend1.start()
print "restarted"
# 5M data file 2
# data file 3
self._gen_data(1, prefix="group3_", loop_num=512)
self._gen_data(2, prefix="group3_", loop_num=512)
self._gen_data(1, prefix="group4_", loop_num=16 * 1024, sector=(1, 0))
self._gen_data(2, prefix="group4_", loop_num=16 * 1024, sector=(1, 0))
self._gen_data(1, prefix="group5_", loop_num=16 * 1024, sector=(0, 1))
self._gen_data(2, prefix="group5_", loop_num=16 * 1024, sector=(0, 1))
self.assertEqual(self.backend1.item_count(), 64 * 1024 + 512)
sector00_exp = os.path.join(self.backend1.db_home, "0/0/0*.data")
sector01_exp = os.path.join(self.backend1.db_home, "0/1/0*.data")
sector10_exp = os.path.join(self.backend1.db_home, "1/0/0*.data")
self.assertEqual(len(glob.glob(sector00_exp)), 3)
self.assertEqual(len(glob.glob(sector01_exp)), 1)
self.assertEqual(len(glob.glob(sector10_exp)), 1)
self.backend1.stop()
def check_data():
self._check_data(2, prefix="group1_", loop_num=16 * 1024)
self._check_data(2, prefix="group2_", loop_num=16 * 1024)
self._check_data(2, prefix="group3_", loop_num=512)
self.backend1.start()
self._start_gc(0, bucket="1")
print "gc @1 started"
while True:
status = self._gc_status()
if status.find("running") >= 0:
check_data()
continue
elif status == "success":
print "done gc"
break
elif status == "fail":
return self.fail("optimize_stat = fail")
else:
self.fail(status)
print "group4 got gc"
for key in self.backend1.generate_key(prefix="group4_", count=2, sector=(1, 0)):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "1/0/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/0/000.data"), key, ver_=2))
print "group1 should be untouched"
for key in self.backend1.generate_key(prefix="group1_", count=2, sector=(0, 0)):
print key
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=2))
time.sleep(1)
self._start_gc(0, bucket="00")
print "gc @00 started"
while True:
status = self._gc_status()
if status.find("running") >= 0:
check_data()
continue
elif status == "success":
print "done gc"
break
elif status == "fail":
return self.fail("optimize_stat = fail")
else:
self.fail(status)
self.assertEqual(self.backend1.item_count(), 64 * 1024 + 512)
check_data()
print "group1 got gc"
for key in self.backend1.generate_key(prefix="group1_", count=2, sector=(0, 0)):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=2))
print "group2 should be not in 000.data, but in 001.data"
for key in self.backend1.generate_key(prefix="group2_", count=2, sector=(0, 0)):
print key
self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/001.data"), key, ver_=2))
print "group5 of bucket 0/1 should be untouched"
for key in self.backend1.generate_key(prefix="group5_", count=2, sector=(0, 1)):
print key
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/1/000.data"), key, ver_=1))
self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/1/000.data"), key, ver_=2))
print "check data& hint"
check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)