def test_0010_restore_changed(self):
"""
We need this to be able to run the tests without reset
"""
lg.debug("")
tid1 = TestDB.idx.nextid-1
tid2 = TestDB.idx.nextid-2
lg.info("Putting back %d and %d" % (tid1, tid2))
oldkey1 = TLV(("key%d" % tid1).encode("ascii"))
newvalue1 = TLV(("value%d" % tid1).encode("ascii"))
t = TestDB.ts.read(tid1)
t.value[oldkey1] = newvalue1
TestDB.ts.update(t)
oldkey2 = TLV(("key%d" % tid2).encode("ascii"))
newvalue2 = TLV(("value%d" % tid2).encode("ascii"))
t = TestDB.ts.read(tid2)
t.value[oldkey2] = newvalue2
TestDB.ts.update(t)
TestDB.ts.vacuum()
t = TestDB.ts.read(tid1)
self.assertEqual(t.value[oldkey1], newvalue1)
t = TestDB.ts.read(tid2)
self.assertEqual(t.value[oldkey2], newvalue2)
def test_0002_read_all(self):
lg.debug("")
for i in range(TestDB.idx.nextid-1, int(TestDB.idx.nextid-TestDB.ITEMS-1), -1):
t = TestDB.ts.read(i)
val = ("value%d" % (i)).encode("ascii")
key = ("key%d" % (i)).encode("ascii")
lg.debug("testing %d: %s" % (i, t))
self.assertEqual(t.value[TLV(key)], TLV(val))
def test_0007_vacuum_confirm(self):
TestDB.ts.vacuum()
tid = TestDB.idx.nextid-1
t = TestDB.ts.read(tid)
oldkey = TLV(("key%d" % tid).encode("ascii"))
newvalue = TLV("-".encode("ascii"))
lg.debug("Updated TLV after Vacuum: %s", t)
self.assertEqual(t.value[oldkey], newvalue)
def test_0009_vacuum_confirm(self):
lg.debug("")
TestDB.ts.vacuum()
tid = TestDB.idx.nextid-2
t = TestDB.ts.read(tid)
oldkey = TLV(("key%d" % tid).encode("ascii"))
newvalue = TLV("I am not fitting for sure!".encode("ascii"))
lg.debug("Updated TLV after Vacuum: %s", t)
self.assertEqual(t.value[oldkey], newvalue)
def _handleEmptying(self, part, oldpos):
"""
Called when something is moved or deleted
"""
if self.backfill is True:
# Log with detail
with self.dfds[part]["lock"]:
tmp_instance = TLV(fd=self.dfds[part]["fd"])
del_size = tmp_instance.size(oldpos)
self.index.setEmpty(part, oldpos, del_size)
else:
# Just log to indicate dirty partition
self.index.setEmpty(part, oldpos, 0)
def test_0008_update_non_fitting(self):
lg.debug("")
tid = TestDB.idx.nextid-2
t = TestDB.ts.read(tid)
oldkey = TLV(("key%d" % tid).encode("ascii"))
newvalue = TLV("I am not fitting for sure!".encode("ascii"))
t.value[oldkey] = newvalue
TestDB.ts.update(t)
lg.debug("Updated TLV: %s", t)
self.assertEqual(t.value[oldkey], newvalue)
def test_0005_read_all2(self):
lg.debug("")
for i in range(TestDB.idx.nextid-1, 0, -1):
try:
t = TestDB.ts.read(i)
except IndexNotFoundError as e:
lg.debug("testing %d: %s" % (i, str(e)))
continue
lg.debug("testing %d: %s" % (i, t))
val = ("value%d" % (i)).encode("ascii")
key = ("key%d" % (i)).encode("ascii")
self.assertEqual(t.value[TLV(key)], TLV(val),
msg="%s != %s" % (t.value[TLV(key)], TLV(val)))
def create(cls, num):
"""
create a 2-key TLV entry
"""
global created
t = TLV({
TLV(rand_str()): TLV(rand_str()),
TLV(rand_str()): TLV(rand_str())
})
lg.info("%d: Creating %s" % (num, t))
tmp = cls.ts.create(t)
with lock:
lg.info("%d: Added: %d" % (num, tmp))
IDS.append(tmp)
cls.created += 1
def test_0001_create(self):
TestDB.ts.beginTransaction()
for i in range(0, TestDB.ITEMS):
t = TLV({
TLV(("key%d" % (TestDB.idx.nextid)).encode("ascii")): TLV(("value%d" % (TestDB.idx.nextid)).encode("ascii"))
})
TestDB.ts.create(t)
TestDB.ts.endTransaction()
num = TestDB.idx.nextid-1
t = TestDB.ts.read(num)
TestDB.headerDump()
val = ("value%d" % (num)).encode("ascii")
key = ("key%d" % (num)).encode("ascii")
self.assertEqual(t.value[TLV(key)], TLV(val))
def test_0003_read(self):
s = time.time()
t2 = TLV(fd=TestTLV.FD)
t2.read(0)
e = time.time()
# lg.info("READ TIME: %f ms" % ((e-s)*1000))
self.assertEqual(TestTLV.t.value[0].value, 16)
self.assertEqual(TestTLV.t.value[1].value, 32)
self.assertEqual(TestTLV.t.value[2].value, b"a")
self.assertEqual(TestTLV.t.value[3].value[TLV("key")], TLV("value"))
def update(cls, num):
"""
update a random entry with another random entry
"""
global updated
with lock:
try:
tmp = random.choice(IDS)
if not tmp:
lg.info("Skipping update... nothing in there")
return
try:
told = cls.ts.read(tmp)
except IndexNotFoundError:
lg.info("Skipping node we cannot read..")
return
except IndexError as e:
return
except Exception as e:
lg.waring("WTF: %s" % str(e))
t = TLV({
TLV(rand_str()): TLV(rand_str()),
TLV(rand_str()): TLV(rand_str())
})
t._tlvdb_id = told._tlvdb_id
lg.info("%d: Updating %d to %s" % (num, tmp, t))
try:
tmp = cls.ts.update(t)
except IndexNotFoundError as e:
lg.debug("Shiiit! Someone was faster: %s" % str(e))
pass
with lock:
IDS.append(tmp)
cls.updated += 1
def vacuum(self, force=False):
"""
Compact the free space in partitions:
- Create a new temp partition
- For every item in the main index
- write it in the new partition
- update the index
- Flush/reset the buffered writer
- Swap the temp partition with the real one
- Flush the index
"""
if self.in_trance:
raise AlreadyInTranceError("In the middle of transaction, vacuum was called!")
# Lock everything
self.lock.acquire()
self.index.lock.acquire()
# Start at the beginning
new_pos = 0
# Iterate, read, write
for part, cont in enumerate(self.index.partitions):
# Lock that partition (index level)
cont["lock"].acquire()
empty = len(cont["empty"])
items = cont["items"]
lg.info("Vacuum: partition %d, status %d/%d" % (part, empty, items))
if (empty == 0 and not force):
lg.info("Skipping ... partition is clean")
cont["lock"].release()
continue
if items and self.vacuum_thres > empty/items:
lg.info("Skipping ... partition less than threshold (thres=%f <> frag=%f)" % (self.vacuum_thres, empty/items))
cont["lock"].release()
continue
# Create swap
swap_part = len(self.dfds)
lg.info("Vacuum: Starting Partition %d" % swap_part)
swap_path = "%s/%s.%d.dat" % (self.dirname, self.basename, swap_part)
self.dfds.append({})
self.dfds[swap_part]["last"] = 0
self.dfds[swap_part]["lock"] = Lock()
self.dfds[swap_part]["fd"] = open(swap_path, "wb")
lg.info(" ... Vacuum: Starting ")
# lock our partition pointers
self.dfds[part]["lock"].acquire()
# Lock swap
self.dfds[swap_part]["lock"].acquire()
for tid, pos in cont["index"].items():
if pos == 0:
lg.warning("Skipping empty/deleted index?")
self.dfds[part]["lock"].release()
continue
tmptlv = TLV(fd=self.dfds[part]["fd"])
# REMEMBER: pos==0 means empty!
lg.debug("Reading from pos=%d" % (pos-1))
data_len = tmptlv.read(pos-1)
lg.debug("Got data length=%d: %s" % (data_len, tmptlv))
self.dfds[swap_part]["fd"].write(tmptlv.pack())
# In memory update of the index
lg.debug("Updating index with %d=>%d (with +1 offset)" % (tid, new_pos + 1))
cont["index"][tid] = new_pos + 1
new_pos += data_len
# Flash whatever remainder
self.dfds[part]["fd"].flush()
# Clean up temp partition
self.dfds[swap_part]["fd"].flush()
self.dfds[swap_part]["fd"].close()
# Release swap... all done for this partions (swap is per-part)
self.dfds[swap_part]["lock"].release()
del self.dfds[swap_part]
# Clean up real partition
self.dfds[part]["fd"].close()
if "last" in self.dfds[part]:
del self.dfds[part]["last"]
# DANGEROUS PART: SHOULD NOT BE INTERUPTED
orig_part = "%s/%s.%d.dat" % (self.dirname, self.basename, part)
# with DelayedInterrupt(signal.SIGINT):
try:
os.rename(swap_path, orig_part)
except:
lg.critical("Failed to move packed parition...")
self.index.reload()
else:
lg.info(" ...Done ")
cont["empty"] = {}
self.index.flush()
finally:
# Reopen real partition
self.dfds[part]["fd"] = util.create_open(orig_part)
# Time to release, this partition is done
cont["lock"].release()
self.dfds[part]["lock"].release()
# Be free...
self.lock.release()
self.index.lock.acquire()
def test_0001_create(self):
TestTLV.t = TLV([
TLV(16),
TLV(32),
TLV("a"),
TLV({TLV("key"): TLV("value")}),
TLV([
TLV(2560),
TLV(25600),
TLV(256000),
TLV(2560000),
TLV(25600000)
]),
TLV(0.00000001),
])
self.assertEqual(TestTLV.t.value[0].value, 16)
self.assertEqual(TestTLV.t.value[1].value, 32)
self.assertEqual(TestTLV.t.value[2].value, b"a")
self.assertEqual(TestTLV.t.value[3].value[TLV("key")], TLV("value"))