Ejemplo n.º 1
0
    def test_0010_restore_changed(self):
        """
        We need this to be able to run the tests without reset
        """
        lg.debug("")

        tid1 = TestDB.idx.nextid-1
        tid2 = TestDB.idx.nextid-2
        lg.info("Putting back %d and %d" % (tid1, tid2))

        oldkey1 = TLV(("key%d" % tid1).encode("ascii"))
        newvalue1 = TLV(("value%d" % tid1).encode("ascii"))
        t = TestDB.ts.read(tid1)
        t.value[oldkey1] = newvalue1
        TestDB.ts.update(t)

        oldkey2 = TLV(("key%d" % tid2).encode("ascii"))
        newvalue2 = TLV(("value%d" % tid2).encode("ascii"))
        t = TestDB.ts.read(tid2)
        t.value[oldkey2] = newvalue2
        TestDB.ts.update(t)

        TestDB.ts.vacuum()

        t = TestDB.ts.read(tid1)
        self.assertEqual(t.value[oldkey1], newvalue1)
        t = TestDB.ts.read(tid2)
        self.assertEqual(t.value[oldkey2], newvalue2)
Ejemplo n.º 2
0
 def test_0002_read_all(self):
     lg.debug("")
     for i in range(TestDB.idx.nextid-1, int(TestDB.idx.nextid-TestDB.ITEMS-1), -1):
         t = TestDB.ts.read(i)
         val = ("value%d" % (i)).encode("ascii")
         key = ("key%d" % (i)).encode("ascii")
         lg.debug("testing %d: %s" % (i, t))
         self.assertEqual(t.value[TLV(key)], TLV(val))
Ejemplo n.º 3
0
    def test_0007_vacuum_confirm(self):
        TestDB.ts.vacuum()

        tid = TestDB.idx.nextid-1
        t = TestDB.ts.read(tid)

        oldkey = TLV(("key%d" % tid).encode("ascii"))
        newvalue = TLV("-".encode("ascii"))

        lg.debug("Updated TLV after Vacuum: %s", t)
        self.assertEqual(t.value[oldkey], newvalue)
Ejemplo n.º 4
0
    def test_0009_vacuum_confirm(self):
        lg.debug("")
        TestDB.ts.vacuum()

        tid = TestDB.idx.nextid-2
        t = TestDB.ts.read(tid)

        oldkey = TLV(("key%d" % tid).encode("ascii"))
        newvalue = TLV("I am not fitting for sure!".encode("ascii"))

        lg.debug("Updated TLV after Vacuum: %s", t)
        self.assertEqual(t.value[oldkey], newvalue)
Ejemplo n.º 5
0
    def _handleEmptying(self, part, oldpos):
        """
        Called when something is moved or deleted
        """
        if self.backfill is True:
            # Log with detail
            with self.dfds[part]["lock"]:
                tmp_instance = TLV(fd=self.dfds[part]["fd"])
                del_size = tmp_instance.size(oldpos)

            self.index.setEmpty(part, oldpos, del_size)
        else:
            # Just log to indicate dirty partition
            self.index.setEmpty(part, oldpos, 0)
Ejemplo n.º 6
0
    def test_0008_update_non_fitting(self):
        lg.debug("")

        tid = TestDB.idx.nextid-2
        t = TestDB.ts.read(tid)

        oldkey = TLV(("key%d" % tid).encode("ascii"))
        newvalue = TLV("I am not fitting for sure!".encode("ascii"))

        t.value[oldkey] = newvalue
        TestDB.ts.update(t)

        lg.debug("Updated TLV: %s", t)
        self.assertEqual(t.value[oldkey], newvalue)
Ejemplo n.º 7
0
    def test_0005_read_all2(self):
        lg.debug("")
        for i in range(TestDB.idx.nextid-1, 0, -1):
            try:
                t = TestDB.ts.read(i)
            except IndexNotFoundError as e:
                lg.debug("testing %d: %s" % (i, str(e)))
                continue

            lg.debug("testing %d: %s" % (i, t))
            val = ("value%d" % (i)).encode("ascii")
            key = ("key%d" % (i)).encode("ascii")

            self.assertEqual(t.value[TLV(key)], TLV(val),
                msg="%s != %s" % (t.value[TLV(key)], TLV(val)))
Ejemplo n.º 8
0
def create(cls, num):
    """
    create a 2-key TLV entry
    """
    global created
    t = TLV({
        TLV(rand_str()): TLV(rand_str()),
        TLV(rand_str()): TLV(rand_str())
    })

    lg.info("%d: Creating %s" % (num, t))
    tmp = cls.ts.create(t)
    with lock:
        lg.info("%d: Added: %d" % (num, tmp))
        IDS.append(tmp)
        cls.created += 1
Ejemplo n.º 9
0
    def test_0001_create(self):

        TestDB.ts.beginTransaction()

        for i in range(0, TestDB.ITEMS):
            t = TLV({
                        TLV(("key%d" % (TestDB.idx.nextid)).encode("ascii")): TLV(("value%d" % (TestDB.idx.nextid)).encode("ascii"))
                    })
            TestDB.ts.create(t)

        TestDB.ts.endTransaction()

        num = TestDB.idx.nextid-1
        t = TestDB.ts.read(num)

        TestDB.headerDump()
        val = ("value%d" % (num)).encode("ascii")
        key = ("key%d" % (num)).encode("ascii")
        self.assertEqual(t.value[TLV(key)], TLV(val))
Ejemplo n.º 10
0
    def test_0003_read(self):
        s = time.time()
        t2 = TLV(fd=TestTLV.FD)
        t2.read(0)
        e = time.time()
        # lg.info("READ TIME: %f ms" % ((e-s)*1000))

        self.assertEqual(TestTLV.t.value[0].value, 16)
        self.assertEqual(TestTLV.t.value[1].value, 32)
        self.assertEqual(TestTLV.t.value[2].value, b"a")
        self.assertEqual(TestTLV.t.value[3].value[TLV("key")], TLV("value"))
Ejemplo n.º 11
0
def update(cls, num):
    """
    update a random entry with another random entry
    """
    global updated
    with lock:
        try:
            tmp = random.choice(IDS)

            if not tmp:
                lg.info("Skipping update... nothing in there")
                return
            try:
                told = cls.ts.read(tmp)
            except IndexNotFoundError:
                lg.info("Skipping node we cannot read..")
                return
        except IndexError as e:
            return
        except Exception as e:
            lg.waring("WTF: %s" % str(e))

    t = TLV({
        TLV(rand_str()): TLV(rand_str()),
        TLV(rand_str()): TLV(rand_str())
    })
    t._tlvdb_id = told._tlvdb_id

    lg.info("%d: Updating %d to %s" % (num, tmp, t))
    try:
        tmp = cls.ts.update(t)
    except IndexNotFoundError as e:
        lg.debug("Shiiit! Someone was faster: %s" % str(e))
        pass

    with lock:
        IDS.append(tmp)
        cls.updated += 1
Ejemplo n.º 12
0
    def vacuum(self, force=False):
        """
        Compact the free space in partitions:

        - Create a new temp partition
        - For every item in the main index
            - write it in the new partition
            - update the index
        - Flush/reset the buffered writer
        - Swap the temp partition with the real one
        - Flush the index
        """
        if self.in_trance:
            raise AlreadyInTranceError("In the middle of transaction, vacuum was called!")

        # Lock everything
        self.lock.acquire()
        self.index.lock.acquire()

        # Start at the beginning
        new_pos = 0

        # Iterate, read, write
        for part, cont in enumerate(self.index.partitions):
            # Lock that partition (index level)
            cont["lock"].acquire()

            empty = len(cont["empty"])
            items = cont["items"]
            lg.info("Vacuum: partition %d, status %d/%d" % (part, empty, items))
            if (empty == 0 and not force):
                lg.info("Skipping ... partition is clean")
                cont["lock"].release()
                continue

            if items and self.vacuum_thres > empty/items:
                lg.info("Skipping ... partition less than threshold (thres=%f <> frag=%f)" % (self.vacuum_thres, empty/items))
                cont["lock"].release()
                continue


            # Create swap
            swap_part = len(self.dfds)
            lg.info("Vacuum: Starting Partition %d" % swap_part)
            swap_path = "%s/%s.%d.dat" % (self.dirname, self.basename, swap_part)
            self.dfds.append({})
            self.dfds[swap_part]["last"] = 0
            self.dfds[swap_part]["lock"] = Lock()
            self.dfds[swap_part]["fd"] = open(swap_path, "wb")

            lg.info(" ... Vacuum: Starting ")

            # lock our partition pointers
            self.dfds[part]["lock"].acquire()

            # Lock swap
            self.dfds[swap_part]["lock"].acquire()

            for tid, pos in cont["index"].items():
                if pos == 0:
                    lg.warning("Skipping empty/deleted index?")
                    self.dfds[part]["lock"].release()
                    continue

                tmptlv = TLV(fd=self.dfds[part]["fd"])
                # REMEMBER: pos==0 means empty!
                lg.debug("Reading from pos=%d" % (pos-1))
                data_len = tmptlv.read(pos-1)
                lg.debug("Got data length=%d: %s" % (data_len, tmptlv))

                self.dfds[swap_part]["fd"].write(tmptlv.pack())

                # In memory update of the index
                lg.debug("Updating index with %d=>%d (with +1 offset)" % (tid, new_pos + 1))
                cont["index"][tid] = new_pos + 1

                new_pos += data_len

            # Flash whatever remainder
            self.dfds[part]["fd"].flush()

            # Clean up temp partition
            self.dfds[swap_part]["fd"].flush()
            self.dfds[swap_part]["fd"].close()

            # Release swap... all done for this partions (swap is per-part)
            self.dfds[swap_part]["lock"].release()

            del self.dfds[swap_part]

            # Clean up real partition
            self.dfds[part]["fd"].close()
            if "last" in self.dfds[part]:
                del self.dfds[part]["last"]

            # DANGEROUS PART: SHOULD NOT BE INTERUPTED
            orig_part = "%s/%s.%d.dat" % (self.dirname, self.basename, part)

            # with DelayedInterrupt(signal.SIGINT):
            try:
                os.rename(swap_path, orig_part)
            except:
                lg.critical("Failed to move packed parition...")
                self.index.reload()
            else:
                lg.info(" ...Done ")
                cont["empty"] = {}
                self.index.flush()
            finally:
                # Reopen real partition
                self.dfds[part]["fd"] = util.create_open(orig_part)

            # Time to release, this partition is done
            cont["lock"].release()
            self.dfds[part]["lock"].release()


        # Be free...
        self.lock.release()
        self.index.lock.acquire()
Ejemplo n.º 13
0
    def test_0001_create(self):
        TestTLV.t = TLV([
            TLV(16),
            TLV(32),
            TLV("a"),
            TLV({TLV("key"): TLV("value")}),
            TLV([
                TLV(2560),
                TLV(25600),
                TLV(256000),
                TLV(2560000),
                TLV(25600000)
            ]),
            TLV(0.00000001),
        ])

        self.assertEqual(TestTLV.t.value[0].value, 16)
        self.assertEqual(TestTLV.t.value[1].value, 32)
        self.assertEqual(TestTLV.t.value[2].value, b"a")
        self.assertEqual(TestTLV.t.value[3].value[TLV("key")], TLV("value"))