def test_0003_read(self): s = time.time() t2 = TLV(fd=TestTLV.FD) t2.read(0) e = time.time() # lg.info("READ TIME: %f ms" % ((e-s)*1000)) self.assertEqual(TestTLV.t.value[0].value, 16) self.assertEqual(TestTLV.t.value[1].value, 32) self.assertEqual(TestTLV.t.value[2].value, b"a") self.assertEqual(TestTLV.t.value[3].value[TLV("key")], TLV("value"))
def vacuum(self, force=False): """ Compact the free space in partitions: - Create a new temp partition - For every item in the main index - write it in the new partition - update the index - Flush/reset the buffered writer - Swap the temp partition with the real one - Flush the index """ if self.in_trance: raise AlreadyInTranceError("In the middle of transaction, vacuum was called!") # Lock everything self.lock.acquire() self.index.lock.acquire() # Start at the beginning new_pos = 0 # Iterate, read, write for part, cont in enumerate(self.index.partitions): # Lock that partition (index level) cont["lock"].acquire() empty = len(cont["empty"]) items = cont["items"] lg.info("Vacuum: partition %d, status %d/%d" % (part, empty, items)) if (empty == 0 and not force): lg.info("Skipping ... partition is clean") cont["lock"].release() continue if items and self.vacuum_thres > empty/items: lg.info("Skipping ... partition less than threshold (thres=%f <> frag=%f)" % (self.vacuum_thres, empty/items)) cont["lock"].release() continue # Create swap swap_part = len(self.dfds) lg.info("Vacuum: Starting Partition %d" % swap_part) swap_path = "%s/%s.%d.dat" % (self.dirname, self.basename, swap_part) self.dfds.append({}) self.dfds[swap_part]["last"] = 0 self.dfds[swap_part]["lock"] = Lock() self.dfds[swap_part]["fd"] = open(swap_path, "wb") lg.info(" ... Vacuum: Starting ") # lock our partition pointers self.dfds[part]["lock"].acquire() # Lock swap self.dfds[swap_part]["lock"].acquire() for tid, pos in cont["index"].items(): if pos == 0: lg.warning("Skipping empty/deleted index?") self.dfds[part]["lock"].release() continue tmptlv = TLV(fd=self.dfds[part]["fd"]) # REMEMBER: pos==0 means empty! lg.debug("Reading from pos=%d" % (pos-1)) data_len = tmptlv.read(pos-1) lg.debug("Got data length=%d: %s" % (data_len, tmptlv)) self.dfds[swap_part]["fd"].write(tmptlv.pack()) # In memory update of the index lg.debug("Updating index with %d=>%d (with +1 offset)" % (tid, new_pos + 1)) cont["index"][tid] = new_pos + 1 new_pos += data_len # Flash whatever remainder self.dfds[part]["fd"].flush() # Clean up temp partition self.dfds[swap_part]["fd"].flush() self.dfds[swap_part]["fd"].close() # Release swap... all done for this partions (swap is per-part) self.dfds[swap_part]["lock"].release() del self.dfds[swap_part] # Clean up real partition self.dfds[part]["fd"].close() if "last" in self.dfds[part]: del self.dfds[part]["last"] # DANGEROUS PART: SHOULD NOT BE INTERUPTED orig_part = "%s/%s.%d.dat" % (self.dirname, self.basename, part) # with DelayedInterrupt(signal.SIGINT): try: os.rename(swap_path, orig_part) except: lg.critical("Failed to move packed parition...") self.index.reload() else: lg.info(" ...Done ") cont["empty"] = {} self.index.flush() finally: # Reopen real partition self.dfds[part]["fd"] = util.create_open(orig_part) # Time to release, this partition is done cont["lock"].release() self.dfds[part]["lock"].release() # Be free... self.lock.release() self.index.lock.acquire()