Example #1
0
 def setUp(self):
     self.database = DB()
     self.database.hasher = HASHERS["c"]
     self.storage = Storage(None, self.database)
     self.changeset = Changeset(self.database)
     self.changeset.add(DummyPage(self.database))  # fill page 0
     self.hlength = self.database.hasher.hashed_length
     self.page = RedislitePageBTree(database=self.database)
Example #2
0
class TestPageString(TestCase):
    def setUp(self):
        self.database = DB()
        self.storage = Storage(None, self.database)
        self.changeset = Changeset(self.database)
        self.changeset.add(DummyPage(self.database))  # fill page 0

    def test_encode_decode_single_page_index(self):
        word = randomword(100)
        page = StringPageIndex(self.database, word, total_length=len(word))
        page_copy = StringPageIndex(self.database)
        page_copy.unserialize(page.serialize())
        self.assertEqual((page.total_length, page.next_page, page.data),
                (page_copy.total_length, page_copy.next_page, page_copy.data))

    def test_encode_decode_single_page_noindex(self):
        word = randomword(100)
        page = StringPage(self.database, word)
        page_copy = StringPage(self.database)
        page_copy.unserialize(page.serialize())
        self.assertEqual((page.next_page, page.data),
                (page_copy.next_page, page_copy.data))

    def test_create_single_page(self):
        word = randomword(100)
        page_number = StringPageIndex.create(self.database, self.changeset,
                word)
        self.changeset.close()
        page = StringPageIndex(self.database)
        page.unserialize(self.storage.read(page_number))
        self.assertEqual((page.total_length, page.next_page, page.data),
                (100, 0, word))

    def test_create_multi_page(self):
        length = 2000
        word = randomword(length)
        page_number = StringPageIndex.create(self.database, self.changeset,
                word)
        self.changeset.close()
        page = StringPageIndex(self.database)
        page.unserialize(self.storage.read(page_number))
        self.assertEqual(page.total_length, length)

        data = page.data
        next_page = page.next_page
        while next_page > 0:
            page = StringPage(self.database)
            page.unserialize(self.storage.read(next_page))
            data += page.data
            next_page = page.next_page
        self.assertEqual(len(data), len(word))
        self.assertEqual(data, word)
Example #3
0
class TestChangeset(TestCase):
    def setUp(self):
        self.database = DB()
        self.storage = Storage(None, self.database)
        self.changeset = Changeset(self.database)

    def test_get_page(self):
        data = {}
        for i in range(0, 2):
            word = randomword(512)
            self.storage.write(((i, word), ))
            data[i] = word

        for i in range(0, 2):
            page = self.changeset.read(i, DummyPage)
            self.assertEqual(page.data, data[i])

    def test_get_page_reversed(self):
        data = {}
        for i in range(0, 2):
            word = randomword(512)
            self.storage.write(((i, word), ))
            data[i] = word

        self.changeset.close()

        for i in reversed(range(0, 2)):
            page = self.changeset.read(i, DummyPage)
            self.assertEqual(page.data, data[i])

    def test_get_page_twice(self):
        data = {}
        for i in range(0, 2):
            word = randomword(512)
            self.storage.write(((i, word), ))
            data[i] = word

        self.changeset.close()

        pages = []
        for i in range(0, 2):
            pages.append(self.changeset.read(i, DummyPage))

        for i in range(0, 2):
            self.assertIs(self.changeset.read(i, DummyPage), pages[i])

    def test_write_page(self):
        data = {}
        for i in range(0, 2):
            word = randomword(512)
            self.changeset.write(i, DummyPage(self.database, word))
            data[i] = word

        self.changeset.close()

        for i in range(0, 2):
            page = self.storage.read(i)
            self.assertEqual(page, data[i])

    def test_add(self):
        data = {}
        for i in range(1, 2):
            word = randomword(512)
            self.changeset.add(DummyPage(self.database, word))
            data[i] = word

        self.changeset.close()

        for i in range(1, 2):
            page = self.storage.read(i)
            self.assertEqual(page, data[i])

    def test_add_db_freelist1(self):
        self.do_test_add_db_freelist(1)

    def test_add_db_freelist2(self):
        self.do_test_add_db_freelist(1)

    def test_add_db_freelist3(self):
        self.do_test_add_db_freelist(1)

    def do_test_add_db_freelist(self, freelist_key):
            data = {}
            for i in range(0, 3):
                word = randomword(512)
                self.changeset.add(DummyPage(self.database, word))
                data[i] = word

            self.database.freelist_item = freelist_key
            word = randomword(512)
            self.changeset.add(DummyPage(self.database, word))
            data[freelist_key] = word

            self.changeset.close()

            for i in range(1, 3):
                page = self.storage.read(i)
                self.assertEqual(page, data[i])

    def test_add_changeset_freelist1(self):
        self.do_test_add_changeset_freelist(1)

    def test_add_changeset_freelist2(self):
        self.do_test_add_changeset_freelist(2)

    def do_test_add_changeset_freelist(self, freelist_key):
        data = {}
        for i in range(0, 3):
            word = randomword(512)
            self.changeset.add(DummyPage(self.database, word))
            data[i] = word

        self.changeset.freelist_item = freelist_key
        word = randomword(512)
        self.changeset.add(DummyPage(self.database, word))
        data[freelist_key] = word

        self.changeset.close()

        for i in range(1, 3):
            page = self.storage.read(i)
            self.assertEqual(page, data[i])
Example #4
0
 def setUp(self):
     self.database = DB()
     self.storage = Storage(None, self.database)
     self.changeset = Changeset(self.database)
Example #5
0
class TestPageBTree(TestCase):
    def setUp(self):
        self.database = DB()
        self.database.hasher = HASHERS["c"]
        self.storage = Storage(None, self.database)
        self.changeset = Changeset(self.database)
        self.changeset.add(DummyPage(self.database))  # fill page 0
        self.hlength = self.database.hasher.hashed_length
        self.page = RedislitePageBTree(database=self.database)

    def test_btree_max_elements(self):
        self.assertEqual(self.page.max_elements, 42)

    def test_serialize_unserialize(self):
        for i in range(0, self.page.max_elements):
            element = RedisliteBTreeElement(database=self.database, hash=randomword(self.hlength))
            self.page.elements.append(element)
        serialized = self.page.serialize()
        new_page = RedislitePageBTree(database=self.database)
        new_page.unserialize(serialized)
        self.assertEqual(new_page.serialize(), serialized)
        self.assertEqual(len(new_page.elements), self.page.max_elements)

    def test_search_empty(self):
        self.assertIs(self.page.search(None, randomword(self.hlength))[0], None)

    def test_search_ok(self):
        words = []
        for i in range(0, self.page.max_elements):
            w = randomword(self.hlength)
            words.append(w)
        words.sort()

        for w in words:
            element = RedisliteBTreeElement(database=self.database, hash=w)
            self.page.elements.append(element)

        for w in words:
            self.assertIsNot(self.page.search(None, w), None)

        self.assertIs(self.page.search(None, randomword(self.hlength))[0], None)
        self.assertIs(self.page.search(None, randomword(self.hlength))[0], None)
        self.assertIs(self.page.search(None, randomword(self.hlength))[0], None)

    def test_search_edges(self):
        words = []
        for i in range(0, self.page.max_elements + 2):
            w = randomword(self.hlength)
            words.append(w)
        words.sort()

        for w in words[1:-1]:
            element = RedisliteBTreeElement(database=self.database, hash=w)
            self.page.elements.append(element)

        self.assertIs(self.page.search(None, words[0])[0], None)
        self.assertIs(self.page.search(None, words[-1])[0], None)

    def test_insert_multipage(self):
        old_max_elements = RedislitePageBTree.max_elements
        try:
            # kids, don't do this at home
            RedislitePageBTree.max_elements = 2
            words = []
            for i in range(0, 6):
                w = randomword(self.hlength)
                w = str(i)
                words.append(w)
                element = RedisliteBTreeElement(database=self.database, hash=w, page_number=1)
                self.page.insert(self.changeset, element)

            for w in words:
                self.assertIsNot(self.page.search(self.changeset, w)[0], None)

            self.assertEqual(
                4, len([item for item in self.changeset.pages_to_write if item[1].__class__ == RedislitePageBTree])
            )
        finally:
            RedislitePageBTree.max_elements = old_max_elements

    def test_update(self):
        words = set()
        for i in range(0, self.page.max_elements * 2):
            w = randomword(self.hlength)
            words.add(w)

        for w in words:
            element = RedisliteBTreeElement(database=self.database, hash=w, page_number=1)
            self.page.insert(self.changeset, element)

        for w in words:
            element = RedisliteBTreeElement(database=self.database, hash=w, page_number=2)
            self.page.insert(self.changeset, element)

        for w in words:
            self.assertEqual(self.page.search(self.changeset, w)[0].page_number, 2)

    def test_force_insert(self):
        words = set()
        for i in range(0, self.page.max_elements * 2):
            w = randomword(self.hlength)
            words.add(w)

        for w in words:
            element = RedisliteBTreeElement(database=self.database, hash=w, page_number=1)
            self.page.insert(self.changeset, element)

        for w in words:
            element = RedisliteBTreeElement(database=self.database, hash=w, page_number=2)
            self.assertFalse(self.page.insert(self.changeset, element, force_insert=True))

        for w in words:
            self.assertEqual(self.page.search(self.changeset, w)[0].page_number, 1)

    def test_insert_multipage_fuzzy(self):
        words = set()
        for i in range(0, self.page.max_elements * 10):
            w = randomword(self.hlength)
            words.add(w)
            element = RedisliteBTreeElement(database=self.database, hash=w, page_number=1)
            self.page.add_element(self.changeset, element)

        for w in words:
            self.assertIsNot(self.page.search(self.changeset, w)[0], None)

        for i in range(0, self.page.max_elements * 10):
            w = randomword(self.hlength)
            self.assertIs(self.page.search(self.changeset, w)[0], None)

    def test_delete(self):
        words = set()
        for i in range(0, self.page.max_elements * 10):
            w = randomword(self.hlength)
            words.add(w)
            element = RedisliteBTreeElement(database=self.database, hash=w, page_number=1)
            self.page.add_element(self.changeset, element)

        for w in words:
            self.page.remove_hash(self.changeset, w)

        for w in words:
            self.assertIs(self.page.search(self.changeset, w)[0], None)
Example #6
0
 def setUp(self):
     self.database = DB()
     self.storage = Storage(None, self.database)
     self.changeset = Changeset(self.database)
     self.changeset.add(DummyPage(self.database))  # fill page 0
Example #7
0
class TestPageList(TestCase):
    def setUp(self):
        self.database = DB()
        self.database.hasher = HASHERS['c']
        self.storage = Storage(None, self.database)
        self.changeset = Changeset(self.database)
        self.changeset.add(DummyPage(self.database))  # fill page 0
        self.hlength = self.database.hasher.hashed_length
        self.page = RedislitePageListRoot(database=self.database)

    def test_serialize_unserialize(self):
        page = RedislitePageList(database=self.database, left_page=123,
                right_page=45)
        for i in range(0, page.max_elements):
            element = RedisliteListElement(database=self.database,
                    hash=randomword(self.hlength))
            page.elements.append(element)
        serialized = page.serialize()
        new_page = RedislitePageList(database=self.database)
        new_page.unserialize(serialized)
        self.assertEqual(new_page.serialize(), serialized)
        self.assertEqual(len(new_page.elements), page.max_elements)
        for attr in ('left_page', 'right_page'):
            self.assertEqual(getattr(new_page, attr), getattr(page, attr))

    def test_serialize_unserialize_root(self):
        self.page.left_page = 124589102
        self.page.right_page = 12904
        self.page.list_size = 123
        for i in range(0, self.page.max_elements):
            element = RedisliteListElement(database=self.database,
                    hash=randomword(self.hlength), page_number=i)
            self.page.elements.append(element)
        serialized = self.page.serialize()
        new_page = RedislitePageListRoot(database=self.database)
        new_page.unserialize(serialized)
        self.assertEqual(new_page.serialize(), serialized)
        self.assertEqual(len(new_page.elements), self.page.max_elements)

        for attr in ('left_page', 'right_page', 'list_size'):
            self.assertEqual(getattr(new_page, attr), getattr(self.page, attr))

    def test_lpush(self):
        words = []
        m = self.page.max_elements
        for i in range(0, m * 3):
            w = randomword(self.hlength)
            element = RedisliteListElement(database=self.database,
                    hash=w, page_number=i)
            self.page.lpush(self.changeset, element)
            words.insert(0, w)

        for i in range(0, m):
            self.assertEqual(words[i], self.page.elements[i].hash)

        right_page = self.changeset.read(self.page.right_page,
                RedislitePageList)
        for i in range(0, m):
            self.assertEqual(words[i + m], right_page.elements[i].hash)

        right_page = self.changeset.read(right_page.right_page,
                RedislitePageList)
        for i in range(0, m):
            self.assertEqual(words[i + m * 2],
                    right_page.elements[i].hash)

    def test_rpush(self):
        words = []
        m = self.page.max_elements * 3
        for i in range(0, m):
            w = randomword(self.hlength)
            element = RedisliteListElement(database=self.database,
                    hash=w, page_number=i)
            self.page.rpush(self.changeset, element)
            words.append(w)

        page = self.page
        start = 0
        for i in range(0, m):
            self.assertEqual(words[i], page.elements[i - start].hash)
            if i - start + 1 == len(page.elements) and i < m - 1:
                start = i + 1
                page = self.changeset.read(page.right_page, RedislitePageList)

    def test_rpush_lpush(self):
        seed(0)
        words = []
        m = self.page.max_elements * 3

        for i in range(0, m):
            w = randomword(self.hlength)
            element = RedisliteListElement(database=self.database,
                    hash=w, page_number=i)
            if randint(0, 1) == 0:
                self.page.rpush(self.changeset, element)
                words.append(w)
            else:
                self.page.lpush(self.changeset, element)
                words.insert(0, w)

        page = self.page
        start = 0
        for i in range(0, m):
            self.assertEqual(words[i], page.elements[i - start].hash)
            if i - start + 1 == len(page.elements) and i < m - 1:
                start = i + 1
                page = self.changeset.read(page.right_page, RedislitePageList)

    def test_search(self):
        words = []
        m = self.page.max_elements * 3
        for i in range(0, m):
            w = randomword(self.hlength)
            element = RedisliteListElement(database=self.database,
                    hash=w, page_number=i)
            self.page.rpush(self.changeset, element)
            words.append(w)

        page = self.page
        for i in range(0, m):
            self.assertEqual(page.search(self.changeset, words[i]), i)
        self.assertEqual(page.search(self.changeset, randomword(self.hlength)),
                -1)