def test_0003_read(self):
s = time.time()
t2 = TLV(fd=TestTLV.FD)
t2.read(0)
e = time.time()
# lg.info("READ TIME: %f ms" % ((e-s)*1000))
self.assertEqual(TestTLV.t.value[0].value, 16)
self.assertEqual(TestTLV.t.value[1].value, 32)
self.assertEqual(TestTLV.t.value[2].value, b"a")
self.assertEqual(TestTLV.t.value[3].value[TLV("key")], TLV("value"))
def vacuum(self, force=False):
"""
Compact the free space in partitions:
- Create a new temp partition
- For every item in the main index
- write it in the new partition
- update the index
- Flush/reset the buffered writer
- Swap the temp partition with the real one
- Flush the index
"""
if self.in_trance:
raise AlreadyInTranceError("In the middle of transaction, vacuum was called!")
# Lock everything
self.lock.acquire()
self.index.lock.acquire()
# Start at the beginning
new_pos = 0
# Iterate, read, write
for part, cont in enumerate(self.index.partitions):
# Lock that partition (index level)
cont["lock"].acquire()
empty = len(cont["empty"])
items = cont["items"]
lg.info("Vacuum: partition %d, status %d/%d" % (part, empty, items))
if (empty == 0 and not force):
lg.info("Skipping ... partition is clean")
cont["lock"].release()
continue
if items and self.vacuum_thres > empty/items:
lg.info("Skipping ... partition less than threshold (thres=%f <> frag=%f)" % (self.vacuum_thres, empty/items))
cont["lock"].release()
continue
# Create swap
swap_part = len(self.dfds)
lg.info("Vacuum: Starting Partition %d" % swap_part)
swap_path = "%s/%s.%d.dat" % (self.dirname, self.basename, swap_part)
self.dfds.append({})
self.dfds[swap_part]["last"] = 0
self.dfds[swap_part]["lock"] = Lock()
self.dfds[swap_part]["fd"] = open(swap_path, "wb")
lg.info(" ... Vacuum: Starting ")
# lock our partition pointers
self.dfds[part]["lock"].acquire()
# Lock swap
self.dfds[swap_part]["lock"].acquire()
for tid, pos in cont["index"].items():
if pos == 0:
lg.warning("Skipping empty/deleted index?")
self.dfds[part]["lock"].release()
continue
tmptlv = TLV(fd=self.dfds[part]["fd"])
# REMEMBER: pos==0 means empty!
lg.debug("Reading from pos=%d" % (pos-1))
data_len = tmptlv.read(pos-1)
lg.debug("Got data length=%d: %s" % (data_len, tmptlv))
self.dfds[swap_part]["fd"].write(tmptlv.pack())
# In memory update of the index
lg.debug("Updating index with %d=>%d (with +1 offset)" % (tid, new_pos + 1))
cont["index"][tid] = new_pos + 1
new_pos += data_len
# Flash whatever remainder
self.dfds[part]["fd"].flush()
# Clean up temp partition
self.dfds[swap_part]["fd"].flush()
self.dfds[swap_part]["fd"].close()
# Release swap... all done for this partions (swap is per-part)
self.dfds[swap_part]["lock"].release()
del self.dfds[swap_part]
# Clean up real partition
self.dfds[part]["fd"].close()
if "last" in self.dfds[part]:
del self.dfds[part]["last"]
# DANGEROUS PART: SHOULD NOT BE INTERUPTED
orig_part = "%s/%s.%d.dat" % (self.dirname, self.basename, part)
# with DelayedInterrupt(signal.SIGINT):
try:
os.rename(swap_path, orig_part)
except:
lg.critical("Failed to move packed parition...")
self.index.reload()
else:
lg.info(" ...Done ")
cont["empty"] = {}
self.index.flush()
finally:
# Reopen real partition
self.dfds[part]["fd"] = util.create_open(orig_part)
# Time to release, this partition is done
cont["lock"].release()
self.dfds[part]["lock"].release()
# Be free...
self.lock.release()
self.index.lock.acquire()