def test_gc(self): self.backend1.start() store = MCStore(self.backend1_addr) ver_key = 'test_version_key' store.set(ver_key, 1) store.set(ver_key, 1, rev=3) # will only raise version in htree self.assertEqual(self._get_version(store, ver_key), 3) self._gen_data(1) print "done set data to 1" time.sleep(10) self._gen_data(2) self._gen_data(1, prefix='delete_group') time.sleep(2) self.assertEqual(self.backend1.item_count(), 20481) self._delete_data(prefix='delete_group') self.assertEqual(self.backend1.item_count(), 10241) self.assert_(not store.delete('key not exists')) self.assertEqual(self.backend1.item_count(), 10241) print "stop beansdb to rotate data file and produce hint" self.backend1.stop() self.backend1.start() print "deleted key should exists in data" assert locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=1) assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=-2) print "done set data to 2" self._start_gc(0) print "gc started" while True: status = self._gc_status() if status.find('running') >= 0: self._check_data(2) continue elif status == 'success': print "done gc" break elif status == 'fail': return self.fail("optimize_stat = fail") else: self.fail(status) self._check_data(2) store = MCStore(self.backend1_addr) self.assertEqual(self._get_version(store, ver_key), 3) # version 3 should be in data print "check test key version, old version should not exist" assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=3) assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=1) print "check data & hint" check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth) print "deleted key got deleted from data file during gc" assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0") self.assertEqual(self.backend1.item_count(), 10241) self.backend1.stop()
print "done get" self.backend1.stop() print "stopped" self.backend1.start() print "started" store = MCStore(self.backend1_addr) for i in xrange(loop_num): key = "test%s" % (i) try: self.assertEqual(store.get(key), 2) except Exception, e: print key, "error", e return self.fail("fail") print "done get" print "check data & hint" check_data_hint_integrity(self.backend1.db_home, self.backend1.db_depth) self.assertEqual(self.backend1.item_count(), loop_num + 1) self.backend1.stop() print "delete .hint and .htree, should regenerate" delete_hint_and_htree(self.backend1.db_home, self.backend1.db_depth) self.backend1.start() print "check data & hint" check_data_hint_integrity(self.backend1.db_home, self.backend1.db_depth) self.assertEqual(self.backend1.item_count(), loop_num + 1) def tearDown(self): self.backend1.stop() class TestGenerateData2(TestGenerateData):
def test_gc_multiple_files(self): self.backend1.start() self._gen_data(1, prefix='group1_', loop_num=16 * 1024) #5M data file 1 self._gen_data(2, prefix='group1_', loop_num=16 * 1024) print 'group1' self.backend1.stop() self.backend1.start() print "restarted" self._gen_data(1, prefix='group2_', loop_num=16 * 1024) self._gen_data(2, prefix='group2_', loop_num=16 * 1024) self.backend1.stop() self.backend1.start() print "restarted" #5M data file 2 # data file 3 self._gen_data(1, prefix='group3_', loop_num=512) self._gen_data(2, prefix='group3_', loop_num=512) self._gen_data(1, prefix='group4_', loop_num=16 * 1024, sector=1) self._gen_data(2, prefix='group4_', loop_num=16 * 1024, sector=1) self.assertEqual(self.backend1.item_count(), 32 * 1024 + 512 + 16 * 1024) sector0_exp = os.path.join(self.backend1.db_home, "0/*.data") print "sector0 files", glob.glob(sector0_exp) self.assertEqual(len(glob.glob(sector0_exp)), 3) self.backend1.stop() print "test append some junk data to file, simulate incomplete data file got gc" with open(os.path.join(self.backend1.db_home, "0/001.data"), 'a') as f: f.write("junkdatasdfsdfsdfdfdf") buckets_txt_path = os.path.join(self.backend1.db_home, "0/buckets.txt") if os.path.exists(buckets_txt_path): print "rm", buckets_txt_path os.remove(buckets_txt_path) self.backend1.start() def check_data(): self._check_data(2, prefix='group1_', loop_num=16 * 1024) self._check_data(2, prefix='group2_', loop_num=16 * 1024) self._check_data(2, prefix='group3_', loop_num=512) time.sleep(1) self._start_gc(0, bucket="0") print "gc started" while True: status = self._gc_status() if status.find('running') >= 0: check_data() continue elif status == 'success': print "done gc" break elif status == 'fail': return self.fail("optimize_stat = fail") else: self.fail(status) self.assertEqual(self.backend1.item_count(), 32 * 1024 + 512 + 16 * 1024) check_data() for key in self.backend1.generate_key(prefix="group1_", count=2, sector=0): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key, ver_=2)) print "group2 should be not in 000.data, but in 001.data" for key in self.backend1.generate_key(prefix="group2_", count=2, sector=0): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/001.data"), key, ver_=2)) print "group4 of bucket 1 should be untouched" for key in self.backend1.generate_key(prefix="group4_", count=2, sector=1): print key self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/000.data"), key, ver_=2)) print "check data& hint" check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)
def test_gc_multiple_files(self): self.backend1.start() self._gen_data(1, prefix='group1_', loop_num=16 * 1024) #5M data file 1 self._gen_data(2, prefix='group1_', loop_num=16 * 1024) print 'group1' self.backend1.stop() self.backend1.start() print "restarted" self._gen_data(1, prefix='group2_', loop_num=16 * 1024) self._gen_data(2, prefix='group2_', loop_num=16 * 1024) self.backend1.stop() self.backend1.start() print "restarted" #5M data file 2 # data file 3 self._gen_data(1, prefix='group3_', loop_num=512) self._gen_data(2, prefix='group3_', loop_num=512) self._gen_data(1, prefix='group4_', loop_num=16 * 1024, sector=(1, 0)) self._gen_data(2, prefix='group4_', loop_num=16 * 1024, sector=(1, 0)) self._gen_data(1, prefix='group5_', loop_num=16 * 1024, sector=(0, 1)) self._gen_data(2, prefix='group5_', loop_num=16 * 1024, sector=(0, 1)) self.assertEqual(self.backend1.item_count(), 64 * 1024 + 512) sector00_exp = os.path.join(self.backend1.db_home, "0/0/0*.data") sector01_exp = os.path.join(self.backend1.db_home, "0/1/0*.data") sector10_exp = os.path.join(self.backend1.db_home, "1/0/0*.data") self.assertEqual(len(glob.glob(sector00_exp)), 3) self.assertEqual(len(glob.glob(sector01_exp)), 1) self.assertEqual(len(glob.glob(sector10_exp)), 1) self.backend1.stop() def check_data(): self._check_data(2, prefix='group1_', loop_num=16 * 1024) self._check_data(2, prefix='group2_', loop_num=16 * 1024) self._check_data(2, prefix='group3_', loop_num=512) self.backend1.start() self._start_gc(0, bucket="1") print "gc @1 started" while True: status = self._gc_status() if status.find('running') >= 0: check_data() continue elif status == 'success': print "done gc" break elif status == 'fail': return self.fail("optimize_stat = fail") else: self.fail(status) print "group4 got gc" for key in self.backend1.generate_key(prefix="group4_", count=2, sector=(1, 0)): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "1/0/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/0/000.data"), key, ver_=2)) print "group1 should be untouched" for key in self.backend1.generate_key(prefix="group1_", count=2, sector=(0, 0)): print key self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=2)) time.sleep(1) self._start_gc(0, bucket="00") print "gc @00 started" while True: status = self._gc_status() if status.find('running') >= 0: check_data() continue elif status == 'success': print "done gc" break elif status == 'fail': return self.fail("optimize_stat = fail") else: self.fail(status) self.assertEqual(self.backend1.item_count(), 64 * 1024 + 512) check_data() print "group1 got gc" for key in self.backend1.generate_key(prefix="group1_", count=2, sector=(0, 0)): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=2)) print "group2 should be not in 000.data, but in 001.data" for key in self.backend1.generate_key(prefix="group2_", count=2, sector=(0, 0)): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/001.data"), key, ver_=2)) print "group5 of bucket 0/1 should be untouched" for key in self.backend1.generate_key(prefix="group5_", count=2, sector=(0, 1)): print key self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/1/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/1/000.data"), key, ver_=2)) print "check data& hint" check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)
def test_gc_multiple_files(self): self.backend1.start() self._gen_data(1, prefix="group1_", loop_num=16 * 1024) # 5M data file 1 self._gen_data(2, prefix="group1_", loop_num=16 * 1024) print "group1" self.backend1.stop() self.backend1.start() print "restarted" self._gen_data(1, prefix="group2_", loop_num=16 * 1024) self._gen_data(2, prefix="group2_", loop_num=16 * 1024) self.backend1.stop() self.backend1.start() print "restarted" # 5M data file 2 # data file 3 self._gen_data(1, prefix="group3_", loop_num=512) self._gen_data(2, prefix="group3_", loop_num=512) self._gen_data(1, prefix="group4_", loop_num=16 * 1024, sector=1) self._gen_data(2, prefix="group4_", loop_num=16 * 1024, sector=1) self.assertEqual(self.backend1.item_count(), 32 * 1024 + 512 + 16 * 1024) sector0_exp = os.path.join(self.backend1.db_home, "0/*.data") print "sector0 files", glob.glob(sector0_exp) self.assertEqual(len(glob.glob(sector0_exp)), 3) self.backend1.stop() print "test append some junk data to file, simulate incomplete data file got gc" with open(os.path.join(self.backend1.db_home, "0/001.data"), "a") as f: f.write("junkdatasdfsdfsdfdfdf") buckets_txt_path = os.path.join(self.backend1.db_home, "0/buckets.txt") if os.path.exists(buckets_txt_path): print "rm", buckets_txt_path os.remove(buckets_txt_path) self.backend1.start() def check_data(): self._check_data(2, prefix="group1_", loop_num=16 * 1024) self._check_data(2, prefix="group2_", loop_num=16 * 1024) self._check_data(2, prefix="group3_", loop_num=512) time.sleep(1) self._start_gc(0, bucket="0") print "gc started" while True: status = self._gc_status() if status.find("running") >= 0: check_data() continue elif status == "success": print "done gc" break elif status == "fail": return self.fail("optimize_stat = fail") else: self.fail(status) self.assertEqual(self.backend1.item_count(), 32 * 1024 + 512 + 16 * 1024) check_data() for key in self.backend1.generate_key(prefix="group1_", count=2, sector=0): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key, ver_=2)) print "group2 should be not in 000.data, but in 001.data" for key in self.backend1.generate_key(prefix="group2_", count=2, sector=0): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/000.data"), key)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/001.data"), key, ver_=2)) print "group4 of bucket 1 should be untouched" for key in self.backend1.generate_key(prefix="group4_", count=2, sector=1): print key self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/000.data"), key, ver_=2)) print "check data& hint" check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)
def test_gc_multiple_files(self): self.backend1.start() self._gen_data(1, prefix="group1_", loop_num=16 * 1024) # 5M data file 1 self._gen_data(2, prefix="group1_", loop_num=16 * 1024) print "group1" self.backend1.stop() self.backend1.start() print "restarted" self._gen_data(1, prefix="group2_", loop_num=16 * 1024) self._gen_data(2, prefix="group2_", loop_num=16 * 1024) self.backend1.stop() self.backend1.start() print "restarted" # 5M data file 2 # data file 3 self._gen_data(1, prefix="group3_", loop_num=512) self._gen_data(2, prefix="group3_", loop_num=512) self._gen_data(1, prefix="group4_", loop_num=16 * 1024, sector=(1, 0)) self._gen_data(2, prefix="group4_", loop_num=16 * 1024, sector=(1, 0)) self._gen_data(1, prefix="group5_", loop_num=16 * 1024, sector=(0, 1)) self._gen_data(2, prefix="group5_", loop_num=16 * 1024, sector=(0, 1)) self.assertEqual(self.backend1.item_count(), 64 * 1024 + 512) sector00_exp = os.path.join(self.backend1.db_home, "0/0/0*.data") sector01_exp = os.path.join(self.backend1.db_home, "0/1/0*.data") sector10_exp = os.path.join(self.backend1.db_home, "1/0/0*.data") self.assertEqual(len(glob.glob(sector00_exp)), 3) self.assertEqual(len(glob.glob(sector01_exp)), 1) self.assertEqual(len(glob.glob(sector10_exp)), 1) self.backend1.stop() def check_data(): self._check_data(2, prefix="group1_", loop_num=16 * 1024) self._check_data(2, prefix="group2_", loop_num=16 * 1024) self._check_data(2, prefix="group3_", loop_num=512) self.backend1.start() self._start_gc(0, bucket="1") print "gc @1 started" while True: status = self._gc_status() if status.find("running") >= 0: check_data() continue elif status == "success": print "done gc" break elif status == "fail": return self.fail("optimize_stat = fail") else: self.fail(status) print "group4 got gc" for key in self.backend1.generate_key(prefix="group4_", count=2, sector=(1, 0)): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "1/0/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "1/0/000.data"), key, ver_=2)) print "group1 should be untouched" for key in self.backend1.generate_key(prefix="group1_", count=2, sector=(0, 0)): print key self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=2)) time.sleep(1) self._start_gc(0, bucket="00") print "gc @00 started" while True: status = self._gc_status() if status.find("running") >= 0: check_data() continue elif status == "success": print "done gc" break elif status == "fail": return self.fail("optimize_stat = fail") else: self.fail(status) self.assertEqual(self.backend1.item_count(), 64 * 1024 + 512) check_data() print "group1 got gc" for key in self.backend1.generate_key(prefix="group1_", count=2, sector=(0, 0)): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key, ver_=2)) print "group2 should be not in 000.data, but in 001.data" for key in self.backend1.generate_key(prefix="group2_", count=2, sector=(0, 0)): print key self.assert_(not check_data_with_key(os.path.join(self.backend1.db_home, "0/0/000.data"), key)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/0/001.data"), key, ver_=2)) print "group5 of bucket 0/1 should be untouched" for key in self.backend1.generate_key(prefix="group5_", count=2, sector=(0, 1)): print key self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/1/000.data"), key, ver_=1)) self.assert_(check_data_with_key(os.path.join(self.backend1.db_home, "0/1/000.data"), key, ver_=2)) print "check data& hint" check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth)