Beispiel #1
0
    def testStringIO(self):
        for blockClass in (NanoBlocks.Index.LeafBlock, NanoBlocks.Index.InteriorBlock):
            for keyType in ("int1", "int2", "int4", "int8", "uint2", "uint8",
                            "float4", "float8"):

                block = blockClass(101, NanoTypes.getType(keyType))
                block2 = blockClass(101, NanoTypes.getType(keyType))
                block.keys = []
                block.addresses = []

                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                block.keys = [1]
                block.addresses = [4]
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                block.keys = range(1, 40, 2)
                block.addresses = [i * 2 for i in range(1, 40, 2)]
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                block.keys = [1] * block.maxKeys
                block.addresses = range(block.maxKeys)
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                self.assertRaises(Exception, NanoBlocks.Index._IndexBlock.fromString, block.toString() + '\x00')

                block.keys = [block.dataType.nullVal] * (block.maxKeys + 1)
                block.addresses = [NanoBlocks.Index.ADDRESS_TYPE.nullVal] * (block.maxKeys + 1)

                self.assertRaises(Exception, block.toString)

        for blockClass in (NanoBlocks.Index.LeafBlock, NanoBlocks.Index.InteriorBlock):
            for keyQuantifier in range(1, 40):
                
                block = blockClass(101, NanoTypes.getType("char%d" % keyQuantifier))
                block2 = blockClass(101, NanoTypes.getType("char%d" % keyQuantifier))
                block.idx = 101
                block.parent = 154
                block.keys = []
                block.addresses = []

                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                block.keys = ['1']
                block.addresses = [4]
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)
                block.keys = ["1" * keyQuantifier] * block.maxKeys
                block.addresses = range(block.maxKeys)
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                self.keys = [str(k) for k in range(block.maxKeys + 1)]
                self.assertRaises(Exception, NanoBlocks.Index._IndexBlock.fromString, block.toString())
Beispiel #2
0
class MemoryMappedTester(MemoryMappedBlock):
    blockSize = 10
    fields = [
        'name',
        'typ',
        'typ2',
    ]
    dataTypes = {
        'name': NanoTypes.Char(4),
        'typ': NanoTypes.Uint(2),
        'typ2': NanoTypes.Uint(4),
    }
Beispiel #3
0
        class MemoryMappedRow(MemoryMappedBlock):
            """
            Class which can be used to get/store rows for this table.  Adds an additional byte flag to the beginning
            of each and every row to indicate whether or not that row is still valid. (When a row is deleted it will
            be set to False).
            """

            blockSize = self.config.rowSize
            fields = ['_valid'] + [col.name for col in self.config.columns]
            dataTypes = {
                col.name: NanoTypes.getType(col.typeString)
                for col in self.config.columns
            }
            dataTypes['_valid'] = NanoTypes.Uint(1)
Beispiel #4
0
 def testGetType(self):
     self.assertEqual(str(NanoTypes.getType("int1")), "Int1")
     self.assertEqual(str(NanoTypes.getType("int2")), "Int2")
     self.assertEqual(str(NanoTypes.getType("int4")), "Int4")
     self.assertEqual(str(NanoTypes.getType("int8")), "Int8")
     self.assertEqual(str(NanoTypes.getType("uint4")), "Uint4")
     self.assertEqual(str(NanoTypes.getType("char10")), "Char10")
     self.assertEqual(str(NanoTypes.getType("float4")), "Float4")
     self.assertEqual(str(NanoTypes.getType("float8")), "Float8")
Beispiel #5
0
 def testGetType(self):
     self.assertEqual(str(NanoTypes.getType("int1")), "Int1")
     self.assertEqual(str(NanoTypes.getType("int2")), "Int2")
     self.assertEqual(str(NanoTypes.getType("int4")), "Int4")
     self.assertEqual(str(NanoTypes.getType("int8")), "Int8")
     self.assertEqual(str(NanoTypes.getType("uint4")), "Uint4")
     self.assertEqual(str(NanoTypes.getType("char10")), "Char10")
     self.assertEqual(str(NanoTypes.getType("float4")), "Float4")
     self.assertEqual(str(NanoTypes.getType("float8")), "Float8")
Beispiel #6
0
 def testMarkDeletedBlock(self):
     indexBlock = NanoBlocks.Index.LeafBlock(5, NanoTypes.Int(4))
     indexBlock.keys = [1]
     indexBlock.addresses = [2]
     self.IndexIO._writeBlockToFile(indexBlock)
     self.IndexIO._markBlockDeleted(indexBlock)
     self.assertEqual(self.IndexIO.delMgr.popRef(), 5)
     self.assertIsNone(self.IndexIO.delMgr.popRef())
Beispiel #7
0
    def setUp(self):
        NanoConfig.num_index_dirty_blocks = 4
        NanoConfig.index_block_size = 29

        self.blocks = []
        self.blocks2 = []
        for i in range(4):
            self.blocks.append(NanoBlocks.Index.LeafBlock(101, NanoTypes.getType('char4')))
            self.blocks[-1].pointers = []
            self.blocks[-1].keys = ["blk%d" % i]
            self.blocks[-1].address = i * NanoConfig.index_block_size
            self.blocks2.append(NanoBlocks.Index.LeafBlock(101, NanoTypes.getType('char4')))
            self.blocks2[-1].pointers = []
            self.blocks2[-1].keys = ["blk%d" % i]
            self.blocks2[-1].address = i * NanoConfig.index_block_size


        self.fd = NanoIO.File.createIndex(self.dbName, self.tableName, self.colName)

        self.dirtyMgr = NanoTools.BlockCacheManager.BlockCacheManager(self.fd)
Beispiel #8
0
    def testChar(self):
        for iVal in range(1, 257):
            c = NanoTypes.getType("char%d" % iVal)

            self.assertEqual(c.isValid(4), False)
            self.assertEqual(c.isValid(c), False)
            self.assertEqual(c.isValid(c.nullVal), True)
            self.assertEqual(c.isValid("a" * iVal), True)
            self.assertEqual(c.isValid("_" * iVal), True)
            self.assertEqual(c.toString(None), c.toString(c.nullVal))
            self.assertEqual(c.fromString(c.nullVal), None)
            self.assertEqual(c.fromString(c.toString(None)), None)
            self.assertRaises(ValueError, c.toString, 54)
Beispiel #9
0
    def testChar(self):
        for iVal in range(1, 257):
            c = NanoTypes.getType("char%d" % iVal)

            self.assertEqual(c.isValid(4), False)
            self.assertEqual(c.isValid(c), False)
            self.assertEqual(c.isValid(c.nullVal), True)
            self.assertEqual(c.isValid("a" * iVal), True)
            self.assertEqual(c.isValid("_" * iVal), True)
            self.assertEqual(c.toString(None), c.toString(c.nullVal))
            self.assertEqual(c.fromString(c.nullVal), None)
            self.assertEqual(c.fromString(c.toString(None)), None)
            self.assertRaises(ValueError, c.toString, 54)
Beispiel #10
0
    def setUp(self):
        NanoConfig.num_index_dirty_blocks = 4
        NanoConfig.index_block_size = 29

        self.blocks = []
        self.blocks2 = []
        for i in range(4):
            self.blocks.append(
                NanoBlocks.Index.LeafBlock(101, NanoTypes.getType('char4')))
            self.blocks[-1].pointers = []
            self.blocks[-1].keys = ["blk%d" % i]
            self.blocks[-1].address = i * NanoConfig.index_block_size
            self.blocks2.append(
                NanoBlocks.Index.LeafBlock(101, NanoTypes.getType('char4')))
            self.blocks2[-1].pointers = []
            self.blocks2[-1].keys = ["blk%d" % i]
            self.blocks2[-1].address = i * NanoConfig.index_block_size

        self.fd = NanoIO.File.createIndex(self.dbName, self.tableName,
                                          self.colName)

        self.dirtyMgr = NanoTools.BlockCacheManager.BlockCacheManager(self.fd)
Beispiel #11
0
    def testFloat(self):
        for iVal in (4, 8):
            f = NanoTypes.getType("float%d" % iVal)

            self.assertEqual(f.isValid('test'), False)
            self.assertEqual(f.isValid(f), False)
            self.assertEqual(f.isValid(f.nullVal), True)
            self.assertEqual(f.isValid(0), True)
            self.assertEqual(f.isValid(0.0), True)
            self.assertEqual(f.isValid(100000.0), True)
            self.assertEqual(f.isValid(23495812980375.123547861235), True)
            self.assertEqual(f.isValid(-129357.12351235), True)
            self.assertEqual(f.toString(f.nullVal), f.toString(None))
            self.assertEqual(f.fromString(f.toString(None)), None)
            self.assertRaises(ValueError, f.toString, 'testing')
Beispiel #12
0
    def testFloat(self):
        for iVal in (4, 8):
            f = NanoTypes.getType("float%d" % iVal)

            self.assertEqual(f.isValid('test'), False)
            self.assertEqual(f.isValid(f), False)
            self.assertEqual(f.isValid(f.nullVal), True)
            self.assertEqual(f.isValid(0), True)
            self.assertEqual(f.isValid(0.0), True)
            self.assertEqual(f.isValid(100000.0), True)
            self.assertEqual(f.isValid(23495812980375.123547861235), True)
            self.assertEqual(f.isValid(-129357.12351235), True)
            self.assertEqual(f.toString(f.nullVal), f.toString(None))
            self.assertEqual(f.fromString(f.toString(None)), None)
            self.assertRaises(ValueError, f.toString, 'testing')
Beispiel #13
0
class Config(VariableMemoryBlock):
    # Class Attributes
    column = None
    unique = None  # TODO implement - raise error if multiple added

    # VariableMemoryMappedBlock definitions
    fields = [
        'column',
        'unique',
    ]
    dataTypes = {
        'column':
        VariableMemoryBlock.SerializableClass(NanoConfig.Column.Config),
        'unique': NanoTypes.Uint(1),
    }
Beispiel #14
0
    def testLookup(self):
        # Leaf block
        block = NanoBlocks.Index.LeafBlock(0, NanoTypes.Int(1))
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 1)

        block.keys = [1]
        block.addresses = [1]
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 0)
        self.assertEqual(block.lookup(1), 1)
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 2)

        block.keys = [1, 3]
        block.addresses = ['a', 'b']
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 0)
        self.assertEqual(block.lookup(1), 'a')
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 2)
        self.assertEqual(block.lookup(3), 'b')
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 4)

        # Interior block
        block = NanoBlocks.Index.InteriorBlock(0, NanoTypes.Int(1))
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 1)

        block.keys = [1]
        block.addresses = [1]
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 0)
        self.assertEqual(block.lookup(1), 1)
        self.assertEqual(block.lookup(2), 1)

        block.keys = [1, 3]
        block.addresses = ['a', 'b']
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 0)
        self.assertEqual(block.lookup(1), 'a')
        self.assertEqual(block.lookup(2), 'a')
        self.assertEqual(block.lookup(3), 'b')
        self.assertEqual(block.lookup(4), 'b')
Beispiel #15
0
    def __init__(self, dbName, tableName, indexConfig):
        self.dbName = dbName
        self.tableName = tableName
        self.indexConfig = indexConfig
        self.indexFD = NanoIO.File.getIndex(self.dbName, self.tableName, self.indexConfig.column.name)

        self.cacheMgr = NanoTools.BlockCacheManager.BlockCacheManager(self.indexFD)
        self.delMgr = NanoTools.DeletedBlockManager.DeletedBlockManager(
            self.dbName, NanoIO.File.indexFileName(self.tableName, self.indexConfig.column.name)
        )
        self.colType = NanoTypes.getType(self.indexConfig.column.typeString)

        # Check if our index is empty.  If it is, create an empty leafblock to start with
        self.indexFD.seek(0, os.SEEK_END)
        if self.indexFD.tell() == 0:
            self.indexFD.write(LeafBlock(0, self.colType).toString())
            self.indexFD.flush()
Beispiel #16
0
    def testUint(self):
        for iVal in (1, 2, 4, 8):
            i = NanoTypes.getType("uint%d" % iVal)

            self.assertEqual(i.isValid('test'), False)
            self.assertEqual(i.isValid(i), False)
            self.assertEqual(i.isValid(i.maxVal + 2), False)
            self.assertEqual(i.isValid(-1), False)
            self.assertEqual(i.isValid(0), True)
            self.assertEqual(i.isValid(i.maxVal), True)
            self.assertEqual(i.toString(1), "\x01" + ("\x00" * (iVal - 1)))
            self.assertEqual(i.toString(16), "\x10" + ("\x00" * (iVal - 1)))
            self.assertEqual(i.fromString("\x10" + ("\x00" * (iVal - 1))), 16)
            self.assertEqual(i.isValid(i.nullVal), True)
            self.assertEqual(i.toString(None), i.toString(i.nullVal))
            self.assertEqual(i.fromString(i.toString(None)), None)
            self.assertEqual(i.fromString(i.toString(i.nullVal)), None)
            self.assertRaises(ValueError, i.toString, i.maxVal + 2)
Beispiel #17
0
    def testUint(self):
        for iVal in (1, 2, 4, 8):
            i = NanoTypes.getType("uint%d" % iVal)

            self.assertEqual(i.isValid('test'), False)
            self.assertEqual(i.isValid(i), False)
            self.assertEqual(i.isValid(i.maxVal + 2), False)
            self.assertEqual(i.isValid(-1), False)
            self.assertEqual(i.isValid(0), True)
            self.assertEqual(i.isValid(i.maxVal), True)
            self.assertEqual(i.toString(1), "\x01" + ("\x00" * (iVal - 1)))
            self.assertEqual(i.toString(16), "\x10" + ("\x00" * (iVal - 1)))
            self.assertEqual(i.fromString("\x10" + ("\x00" * (iVal - 1))), 16)
            self.assertEqual(i.isValid(i.nullVal), True)
            self.assertEqual(i.toString(None), i.toString(i.nullVal))
            self.assertEqual(i.fromString(i.toString(None)), None)
            self.assertEqual(i.fromString(i.toString(i.nullVal)), None)
            self.assertRaises(ValueError, i.toString, i.maxVal + 2)
Beispiel #18
0
    def __init__(self, dbName, tableName, indexConfig):
        self.dbName = dbName
        self.tableName = tableName
        self.indexConfig = indexConfig
        self.indexFD = NanoIO.File.getIndex(self.dbName, self.tableName,
                                            self.indexConfig.column.name)

        self.cacheMgr = NanoTools.BlockCacheManager.BlockCacheManager(
            self.indexFD)
        self.delMgr = NanoTools.DeletedBlockManager.DeletedBlockManager(
            self.dbName,
            NanoIO.File.indexFileName(self.tableName,
                                      self.indexConfig.column.name))
        self.colType = NanoTypes.getType(self.indexConfig.column.typeString)

        # Check if our index is empty.  If it is, create an empty leafblock to start with
        self.indexFD.seek(0, os.SEEK_END)
        if self.indexFD.tell() == 0:
            self.indexFD.write(LeafBlock(0, self.colType).toString())
            self.indexFD.flush()
Beispiel #19
0
    def testGetAndWriteBlockAtIndex(self):
        # Test getting a block from an index with no keys
        self._assertIndexBlockEqual(
            self.IndexIO._getBlockAtAddress(0),
            NanoBlocks.Index.LeafBlock(0, self.IndexIO.colType))

        # Test getting a fresh block
        indexBlock = NanoBlocks.Index.LeafBlock(0, NanoTypes.Int(4))
        self.IndexIO.indexFD.write(indexBlock.toString())

        self.assertFalse(0 in self.IndexIO.cacheMgr)
        self._assertIndexBlockEqual(indexBlock,
                                    self.IndexIO._getBlockAtAddress(0))

        # Test getting a block that should exist in the Block Cache Manager
        indexBlock.keys = [1]
        indexBlock.addresses = [2]
        self.IndexIO._writeBlockToFile(indexBlock)
        self.assertTrue(0 in self.IndexIO.cacheMgr)
        self._assertIndexBlockEqual(indexBlock,
                                    self.IndexIO._getBlockAtAddress(0))
Beispiel #20
0
class Config(VariableMemoryBlock):
    # Class Attributes
    name = None
    columns = None
    indices = None
    rowSize = None

    # VariableMemoryBlock definitions
    fields = [
        'name',
        'rowSize',
        'columns',
        'indices',
    ]
    dataTypes = {
        'name': VariableMemoryBlock.SerializableString,
        'rowSize': NanoTypes.Uint(4),
        'columns': VariableMemoryBlock.SerializableClass(NanoConfig.Column.Config),
        'indices': VariableMemoryBlock.SerializableClass(NanoConfig.Index.Config),
    }
    iterableFields = [
        'columns',
        'indices',
    ]
Beispiel #21
0
    def testStringIO(self):
        for blockClass in (NanoBlocks.Index.LeafBlock,
                           NanoBlocks.Index.InteriorBlock):
            for keyType in ("int1", "int2", "int4", "int8", "uint2", "uint8",
                            "float4", "float8"):

                block = blockClass(101, NanoTypes.getType(keyType))
                block2 = blockClass(101, NanoTypes.getType(keyType))
                block.keys = []
                block.addresses = []

                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                block.keys = [1]
                block.addresses = [4]
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                block.keys = range(1, 40, 2)
                block.addresses = [i * 2 for i in range(1, 40, 2)]
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                block.keys = [1] * block.maxKeys
                block.addresses = range(block.maxKeys)
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                self.assertRaises(Exception,
                                  NanoBlocks.Index._IndexBlock.fromString,
                                  block.toString() + '\x00')

                block.keys = [block.dataType.nullVal] * (block.maxKeys + 1)
                block.addresses = [NanoBlocks.Index.ADDRESS_TYPE.nullVal
                                   ] * (block.maxKeys + 1)

                self.assertRaises(Exception, block.toString)

        for blockClass in (NanoBlocks.Index.LeafBlock,
                           NanoBlocks.Index.InteriorBlock):
            for keyQuantifier in range(1, 40):

                block = blockClass(101,
                                   NanoTypes.getType("char%d" % keyQuantifier))
                block2 = blockClass(
                    101, NanoTypes.getType("char%d" % keyQuantifier))
                block.idx = 101
                block.parent = 154
                block.keys = []
                block.addresses = []

                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                block.keys = ['1']
                block.addresses = [4]
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)
                block.keys = ["1" * keyQuantifier] * block.maxKeys
                block.addresses = range(block.maxKeys)
                block2.fromString(block.toString())
                self.assertBlocksEqual(block, block2)

                self.keys = [str(k) for k in range(block.maxKeys + 1)]
                self.assertRaises(Exception,
                                  NanoBlocks.Index._IndexBlock.fromString,
                                  block.toString())
Beispiel #22
0
    def testDelete(self):
        # Delete from an empty block
        block = NanoBlocks.Index.LeafBlock(101, NanoTypes.getType("int1"))
        block.keys = []
        block.addresses = []
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 1)

        # Delete a non-present key from the block
        block.keys = [5]
        block.addresses = [6]
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 1)

        # Delete from a leaf with 1 key
        block.delete(5)
        self.assertSequenceEqual(block.keys, [])
        self.assertSequenceEqual(block.addresses, [])

        # Delete from the beginning of a half-full leaf
        block.keys = [3, 4, 5, 6, 7]
        block.addresses = [8, 9, 10, 11, 12]
        block.delete(3)
        self.assertSequenceEqual(block.keys, [4, 5, 6, 7])
        self.assertSequenceEqual(block.addresses, [9, 10, 11, 12])

        # Delete from the end of a half-full leaf
        block.delete(7)
        self.assertSequenceEqual(block.keys, [4, 5, 6])
        self.assertSequenceEqual(block.addresses, [9, 10, 11])

        # Delete from the middle of a half-full leaf
        block.delete(5)
        self.assertSequenceEqual(block.keys, [4, 6])
        self.assertSequenceEqual(block.addresses, [9, 11])

        # Delete from a interior with 1 key and 1 address
        block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
        block.keys = [5]
        block.addresses = [6]
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 4)
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 6)
        block.delete(5)
        self.assertSequenceEqual(block.keys, [])
        self.assertSequenceEqual(block.addresses, [])

        # Delete from the beginning of a half-full interior block
        block.keys = [3, 4, 5, 6, 7]
        block.addresses = [8, 9, 10, 11, 12]
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 2)
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 8)
        block.delete(3)
        self.assertSequenceEqual(block.keys, [4, 5, 6, 7])
        self.assertSequenceEqual(block.addresses, [9, 10, 11, 12])

        # Delete from the end of a half-full interior block
        block.delete(7)
        self.assertSequenceEqual(block.keys, [4, 5, 6])
        self.assertSequenceEqual(block.addresses, [9, 10, 11])

        # Delete from the middle of a half-full interior block
        block.delete(5)
        self.assertSequenceEqual(block.keys, [4, 6])
        self.assertSequenceEqual(block.addresses, [9, 11])
Beispiel #23
0
class VariableMemoryBlock:
    """
    Class which provides toString and fromString functionality to subclasses which should be serializable while not
    conforming to strict sizing requirements.

    Stores fields as follows:
        4 bytes: number of bytes required to serialize this field
        X bytes: Serialized version of this field
    """

    # Private Variables

    # DataType which is used to serialize the `numBytes` field preceeding each item in self.fields
    _sizeDataType = NanoTypes.Uint(4)

    # Subclassable Variables

    # List of fields on self that should be serialized in the file
    fields = None
    # Dictionary mapping fieldName to the dataType to convert to from the raw string representation
    dataTypes = None
    # List of fields which are stored as iterables
    iterableFields = None

    ###
    # Variable Memory DataType classes
    #
    # Classes which can be used as dataTypes when constructing new VariableMemoryBlock subclasses
    ###

    class SerializableString:
        """ DataType which can be used to serialize variables as variable lengthed strings. """
        @staticmethod
        def toString(s):
            return str(s) if s else ""

        @staticmethod
        def fromString(s):
            return s

    class SerializableClass:
        """
        DataType which can be used to wrap classes that define toString & fromString methods as types.

        Classes must be instantiable requiring no arguments.
        """

        classType = None

        def __init__(self, classType):
            self.classType = classType

        def toString(self, classInstance):
            return classInstance.toString() if classInstance else ""

        def fromString(self, s):
            classInstance = self.classType()
            if s:
                classInstance.fromString(s)
            return classInstance

    # Private methods
    def __serializeAttribute(self, fieldName, value):
        serializedVal = self.dataTypes[fieldName].toString(value)
        return self._sizeDataType.toString(len(serializedVal)) + serializedVal

    def __getFromBlock(self, fromDataType, idx, numBytes, s):
        """
        Pulls an object out of s.

        Inputs: fromDataType - The dataType to use to fromString the substring of s.
                idx          - The index to begin the substring of s.
                numBytes     - The number of bytes to extract from s.
                s            - The block to extract the substring from.

        Returns: (The Object fromString'd from the block, The index after reading from s)
        """

        if len(s) < idx + numBytes:
            raise Exception(
                "Given string len(%s) is not long enough to unserialize %s" %
                (len(s), self))

        return fromDataType.fromString(s[idx:idx + numBytes]), idx + numBytes

    # Public methods
    def toString(self):
        """
        Returns a string representation of this object suitable for writing to a file.
        """

        # List of serialized fields
        toSerialize = []

        for fieldName in self.fields:
            attrVal = getattr(self, fieldName)

            # If this field has been marked as iterable
            if self.iterableFields and fieldName in self.iterableFields:
                # Ensure this variable is actually iterable
                if not hasattr(attrVal, '__iter__'):
                    toSerialize.append(self._sizeDataType.toString(0))

                else:
                    toSerialize.append("".join(
                        # Serialize the length of the iterable
                        [self._sizeDataType.toString(len(attrVal))] +
                        # Serialize each item of the iterable
                        [
                            self.__serializeAttribute(fieldName, val)
                            for val in attrVal
                        ]))

            # Otherwise serialize the attribute as a single value
            else:
                toSerialize.append(
                    self.__serializeAttribute(fieldName, attrVal))

        return "".join(toSerialize)

    def fromString(self, s):
        """
        Initializes this object from the data found in s.

        Inputs: s - A string containing the data to initialize this instance with.

        Raises an error if s contains insufficient memory to initialize all fields in self.fields, or
        if it contains extra bytes not required to initialize all fields in self.fields.
        """

        # Current index into s
        idx = 0

        for fieldName in self.fields:
            # Get the number of bytes to initialize this field with
            fieldLength, idx = self.__getFromBlock(self._sizeDataType, idx,
                                                   self._sizeDataType.size, s)

            # If this field has been marked as iterable
            if self.iterableFields and fieldName in self.iterableFields:
                setattr(self, fieldName, [])
                for i in range(fieldLength):
                    itemLength, idx = self.__getFromBlock(
                        self._sizeDataType, idx, self._sizeDataType.size, s)
                    fieldVal, idx = self.__getFromBlock(
                        self.dataTypes[fieldName], idx, itemLength, s)
                    getattr(self, fieldName).append(fieldVal)

            # Otherwise unserialize this single value
            else:
                fieldVal, idx = self.__getFromBlock(self.dataTypes[fieldName],
                                                    idx, fieldLength, s)
                setattr(self, fieldName, fieldVal)

        if len(s) > idx:
            raise Exception("Given string len(%s) too long to unserialize %s" %
                            (len(s), self))

        return self
Beispiel #24
0
    def testAdd(self):
        # Add to an empty leaf block
        block = NanoBlocks.Index.LeafBlock(101, NanoTypes.getType("int1"))
        block.keys = []
        block.addresses = []
        block.add(5, 7)
        self.assertSequenceEqual(block.keys, [5])
        self.assertSequenceEqual(block.addresses, [7])

        # Add to a half full leaf block
        block.add(6, 8)
        block.add(4, 5)
        block.add(5, 10)
        self.assertSequenceEqual(block.keys, [4, 5, 5, 6])
        self.assertSequenceEqual(block.addresses, [5, 10, 7, 8])

        # Add to 1 off from full leaf block
        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(1, 1)
        self.assertSequenceEqual(block.keys,
                                 [1, 2] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses,
                                 [1] + ([5] * (block.maxKeys - 1)))

        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(2, 2)
        self.assertSequenceEqual(block.keys,
                                 [2, 2] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses,
                                 [2] + ([5] * (block.maxKeys - 1)))

        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(3, 3)
        self.assertSequenceEqual(block.keys,
                                 [2, 3] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses,
                                 [5, 3] + ([5] * (block.maxKeys - 2)))

        # Add to a full leaf block
        block.keys = [3] * block.maxKeys
        block.addresses = [4] * block.maxKeys
        self.assertRaises(BufferError, block.add, 2, 3)
        self.assertRaises(BufferError, block.add, 3, 3)
        self.assertRaises(BufferError, block.add, 4, 3)

        # Add to an empty interior block
        block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
        block.keys = []
        block.addresses = []
        block.add(5, 7)
        block.add(6, 8)
        self.assertSequenceEqual(block.keys, [5, 6])
        self.assertSequenceEqual(block.addresses, [7, 8])

        # Add to a interior block with 1 key and 1 address
        block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
        block.keys = [4]
        block.addresses = [5]
        block.add(5, 4)
        self.assertSequenceEqual(block.keys, [4, 5])
        self.assertSequenceEqual(block.addresses, [5, 4])

        block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
        block.keys = [4]
        block.addresses = [5]
        block.add(3, 4)
        self.assertSequenceEqual(block.keys, [3, 4])
        self.assertSequenceEqual(block.addresses, [4, 5])

        # Add to a half full interior block
        block.add(6, 8)
        block.add(4, 5)
        block.add(5, 10)
        self.assertSequenceEqual(block.keys, [3, 4, 4, 5, 6])
        self.assertSequenceEqual(block.addresses, [4, 5, 5, 10, 8])

        # Add to 1 off from full interior block
        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(1, 1)
        self.assertSequenceEqual(block.keys,
                                 [1, 2] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses,
                                 [1] + ([5] * (block.maxKeys - 1)))

        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(2, 2)
        self.assertSequenceEqual(block.keys,
                                 [2, 2] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses,
                                 [2] + ([5] * (block.maxKeys - 1)))

        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(3, 3)
        self.assertSequenceEqual(block.keys,
                                 [2, 3] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses,
                                 [5, 3] + ([5] * (block.maxKeys - 2)))

        # Add to a full interior block
        self.assertRaises(BufferError, block.add, 1, 3)
        self.assertRaises(BufferError, block.add, 2, 3)
        self.assertRaises(BufferError, block.add, 3, 3)
        self.assertRaises(BufferError, block.add, 4, 3)
Beispiel #25
0
    def executeQuery(self, conn):
        if self.target == "database":
            NanoIO.File.createDatabase(self.name)

        elif self.target == "table":
            dbName, tableName = conn._parseName(self.name)

            # Assert that the database exists, but the table does not
            NanoIO.File.assertDatabaseExists(dbName)
            if NanoIO.File.checkTableExists(dbName, tableName):
                raise IOError("Cannot create table, table file %s.%s already exists" % (dbName, tableName))
            if NanoIO.File.checkConfigExists(dbName, tableName):
                raise IOError("Cannot create table, config file for table %s.%s already exists" % (dbName, tableName))

            tableConfig = NanoConfig.Table.Config()
            tableConfig.name = tableName

            # Add columns to this table
            colDict = dict()
            tableConfig.columns = []
            rowSize = 1
            for column in self.cols:
                # Sanity check; disallow duplicated column names
                if column['name'] in colDict:
                    raise Exception("Duplicated column name: %s" % column['name'])

                columnConfig = NanoConfig.Column.Config()
                columnConfig.name = column['name']
                columnConfig.typeString = column['type']
                # Validate this type string
                typ = NanoTypes.getType(columnConfig.typeString)
                colDict[column['name']] = columnConfig
                tableConfig.columns.append(columnConfig)
                rowSize += typ.size

            tableConfig.rowSize = rowSize

            # Add indices to this table
            idxSet = set()
            tableConfig.indices = []
            if self.indices:
                for index in self.indices:
                    # Sanity check; disallow duplicated indices
                    if index in idxSet:
                        raise Exception("Duplicated index name: %s" % index)
                    # Ensure this indexed column exists in our table
                    if index not in colDict:
                        raise Exception("Cannot create index on missing column: %s" % index)
                    # Ensure that the index file for this index doesn't somehow already exist
                    if NanoIO.File.checkIndexExists(dbName, tableName, index):
                        raise Exception("Cannot create table %s.%s; index file already exists at %s" % \
                                        (dbName, tableName, NanoIO.File.indexPath(dbName, tableName, index)))

                    idxSet.add(index)

                    indexConfig = NanoConfig.Index.Config()
                    indexConfig.column = colDict[index]
                    tableConfig.indices.append(indexConfig)

                # Create each index for this table
                for index in self.indices:
                    NanoIO.File.createIndex(dbName, tableName, index)

            # Create this table
            configFD = NanoIO.File.createTable(dbName, tableName)
            configFD.write(tableConfig.toString())
            configFD.close()
Beispiel #26
0
class TestIndex(NanoTests.NanoTestCase):
    tableName = "IOTestIndex"
    indexColName = "IOTestIndexedColumn"
    indexConfig = None
    dataType = NanoTypes.getType('int4')

    def setUp(self):
        col = NanoConfig.Column.Config()
        col.name = self.indexColName
        col.typeString = "int4"
        self.indexConfig = NanoConfig.Index.Config()
        self.indexConfig.column = col
        self.indexConfig.unique = False

        NanoIO.File.deleteIndex(self.dbName, self.tableName, self.indexColName)
        NanoIO.File.createIndex(self.dbName, self.tableName,
                                self.indexColName).close()
        self.IndexIO = NanoIO.Index.IndexIO(self.dbName, self.tableName,
                                            self.indexConfig)

    def tearDown(self):
        NanoIO.File.deleteIndex(self.dbName, self.tableName, self.indexColName)

    def _assertIndexBlockEqual(self, indexBlock, indexBlock2):
        self.assertEqual(indexBlock2.isLeaf, indexBlock.isLeaf)
        self.assertEqual(indexBlock2.address, indexBlock.address)
        self.assertEqual(indexBlock2.dataType.quantifier,
                         indexBlock.dataType.quantifier)
        self.assertSequenceEqual(indexBlock2.keys, indexBlock.keys)
        self.assertSequenceEqual(indexBlock2.addresses, indexBlock.addresses)

    def testGetAndWriteBlockAtIndex(self):
        # Test getting a block from an index with no keys
        self._assertIndexBlockEqual(
            self.IndexIO._getBlockAtAddress(0),
            NanoBlocks.Index.LeafBlock(0, self.IndexIO.colType))

        # Test getting a fresh block
        indexBlock = NanoBlocks.Index.LeafBlock(0, NanoTypes.Int(4))
        self.IndexIO.indexFD.write(indexBlock.toString())

        self.assertFalse(0 in self.IndexIO.cacheMgr)
        self._assertIndexBlockEqual(indexBlock,
                                    self.IndexIO._getBlockAtAddress(0))

        # Test getting a block that should exist in the Block Cache Manager
        indexBlock.keys = [1]
        indexBlock.addresses = [2]
        self.IndexIO._writeBlockToFile(indexBlock)
        self.assertTrue(0 in self.IndexIO.cacheMgr)
        self._assertIndexBlockEqual(indexBlock,
                                    self.IndexIO._getBlockAtAddress(0))

    def testMarkDeletedBlock(self):
        indexBlock = NanoBlocks.Index.LeafBlock(5, NanoTypes.Int(4))
        indexBlock.keys = [1]
        indexBlock.addresses = [2]
        self.IndexIO._writeBlockToFile(indexBlock)
        self.IndexIO._markBlockDeleted(indexBlock)
        self.assertEqual(self.IndexIO.delMgr.popRef(), 5)
        self.assertIsNone(self.IndexIO.delMgr.popRef())

    def testGetIdxForNewBlock(self):
        self.assertEqual(self.IndexIO._getAddressForNewBlock(),
                         NanoConfig.index_block_size)
        idxBlock = self.IndexIO._getBlockAtAddress(0)
        idxBlock.address = NanoConfig.index_block_size
        self.IndexIO._writeBlockToFile(idxBlock)
        self.assertEqual(self.IndexIO._getAddressForNewBlock(),
                         NanoConfig.index_block_size * 2)
        self.IndexIO._markBlockDeleted(idxBlock)
        self.assertEqual(self.IndexIO._getAddressForNewBlock(),
                         NanoConfig.index_block_size)

    def testAddLookupDelete(self):
        # Test adding to an empty index
        self.assertRaises(NanoBlocks.Index.KeyNotFound, self.IndexIO.lookup, 1)
        self.IndexIO.add(1, 2)
        self.assertEqual(self.IndexIO.lookup(1), 2)

        # Test adding to an index with a single partially full leaf block
        for i in range(5, 50, 2):
            self.IndexIO.add(i, i + 1)

        self.assertRaises(NanoBlocks.Index.KeyNotFound, self.IndexIO.lookup, 8)
        self.IndexIO.add(8, 9)
        self.assertEqual(self.IndexIO.lookup(8), 9)

        # Test adding values to the beginning of a partially full leaf block
        self.assertRaises(NanoBlocks.Index.KeyNotFound, self.IndexIO.lookup, 0)
        self.IndexIO.add(0, 0)
        self.assertEqual(self.IndexIO.lookup(0), 0)

        # Test adding to an index with a single full leaf block
        self.tearDown()
        self.setUp()
        rootBlock = self.IndexIO._getBlockAtAddress(0)
        rootBlock.keys = []
        rootBlock.addresses = []
        for i in range(rootBlock.maxKeys):
            rootBlock.add(i, i + 1)
        self.IndexIO._writeBlockToFile(rootBlock)

        for i in range(rootBlock.maxKeys):
            self.assertEqual(self.IndexIO.lookup(i), i + 1)
        self.IndexIO.add(20, 20)
        self.IndexIO.add(500, 500)

        for i in range(1, 170):
            self.IndexIO.add(-i, i)

        # Test adding to an index with a root interior block pointing to leaf blocks # TODO

        # Test adding to an index with a full root interior block, pointing to leaf blocks # TODO

        # Test adding to an index with a root interior block pointing to interior blocks # TODO

    def testRandomAddDeleteLookup(self):
        seed = 124
        numPairs = 4000
        keyRange = (0, 10000000)
        random.seed(seed)

        keys = set([random.randint(*keyRange) for i in range(numPairs)])
        pairs = [(k, random.randint(*keyRange)) for k in keys]

        for k, a in pairs:
            self.IndexIO.add(k, a)

        for k, a in reversed(pairs):
            self.assertEqual(self.IndexIO.lookup(k), a)
            self.IndexIO.delete(k)
            self.assertRaises(NanoBlocks.Index.KeyNotFound,
                              self.IndexIO.lookup, k)

        root = self.IndexIO._getBlockAtAddress(0)
        self.assertEqual(len(root.keys), 0)

    def testDelete(self):  # TODO
        pass

    def testLookup(self):  # TODO
        pass

    def testLookupCondition(self):  # TODO
        pass

    def testIterate(self):  # TODO
        pass
Beispiel #27
0
# Standard imports
import os

# Project imports
import NanoTypes
import NanoIO.File

POINTER_TYPE = NanoTypes.Uint(8)


class DeletedBlockManager:
    fd = None
    name = None
    db = None

    def __init__(self, db, name):
        self.db = db
        self.name = name
        self.fd = NanoIO.File.openReadWriteFile(
            NanoIO.File.delMgrPath(db, name))

    def __del__(self):
        try:
            self.close()
        except AttributeError:
            pass

    def truncate(self):
        self.fd.seek(0)
        self.fd.truncate()
Beispiel #28
0
    def testDelete(self):
        # Delete from an empty block
        block = NanoBlocks.Index.LeafBlock(101, NanoTypes.getType("int1"))
        block.keys = []
        block.addresses = []
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 1)

        # Delete a non-present key from the block
        block.keys = [5]
        block.addresses = [6]
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 1)

        # Delete from a leaf with 1 key
        block.delete(5)
        self.assertSequenceEqual(block.keys, [])
        self.assertSequenceEqual(block.addresses, [])

        # Delete from the beginning of a half-full leaf
        block.keys = [3, 4, 5, 6, 7]
        block.addresses = [8, 9, 10, 11, 12]
        block.delete(3)
        self.assertSequenceEqual(block.keys, [4, 5, 6, 7])
        self.assertSequenceEqual(block.addresses, [9, 10, 11, 12])

        # Delete from the end of a half-full leaf
        block.delete(7)
        self.assertSequenceEqual(block.keys, [4, 5, 6])
        self.assertSequenceEqual(block.addresses, [9, 10, 11])

        # Delete from the middle of a half-full leaf
        block.delete(5)
        self.assertSequenceEqual(block.keys, [4, 6])
        self.assertSequenceEqual(block.addresses, [9, 11])

        # Delete from a interior with 1 key and 1 address
        block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
        block.keys = [5]
        block.addresses = [6]
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 4)
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 6)
        block.delete(5)
        self.assertSequenceEqual(block.keys, [])
        self.assertSequenceEqual(block.addresses, [])

        # Delete from the beginning of a half-full interior block
        block.keys = [3, 4, 5, 6, 7]
        block.addresses = [8, 9, 10, 11, 12]
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 2)
        self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 8)
        block.delete(3)
        self.assertSequenceEqual(block.keys, [4, 5, 6, 7])
        self.assertSequenceEqual(block.addresses, [9, 10, 11, 12])

        # Delete from the end of a half-full interior block
        block.delete(7)
        self.assertSequenceEqual(block.keys, [4, 5, 6])
        self.assertSequenceEqual(block.addresses, [9, 10, 11])

        # Delete from the middle of a half-full interior block
        block.delete(5)
        self.assertSequenceEqual(block.keys, [4, 6])
        self.assertSequenceEqual(block.addresses, [9, 11])
Beispiel #29
0
    def testAdd(self):
        # Add to an empty leaf block
        block = NanoBlocks.Index.LeafBlock(101, NanoTypes.getType("int1"))
        block.keys = []
        block.addresses = []
        block.add(5, 7)
        self.assertSequenceEqual(block.keys, [5])
        self.assertSequenceEqual(block.addresses, [7])

        # Add to a half full leaf block
        block.add(6, 8)
        block.add(4, 5)
        block.add(5, 10)
        self.assertSequenceEqual(block.keys, [4, 5, 5, 6])
        self.assertSequenceEqual(block.addresses, [5, 10, 7, 8])

        # Add to 1 off from full leaf block
        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(1, 1)
        self.assertSequenceEqual(block.keys, [1, 2] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses, [1] + ([5] * (block.maxKeys - 1)))

        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(2, 2)
        self.assertSequenceEqual(block.keys, [2, 2] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses, [2] + ([5] * (block.maxKeys - 1)))

        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(3, 3)
        self.assertSequenceEqual(block.keys, [2, 3] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses, [5, 3] + ([5] * (block.maxKeys - 2)))

        # Add to a full leaf block
        block.keys = [3] * block.maxKeys
        block.addresses = [4] * block.maxKeys
        self.assertRaises(BufferError, block.add, 2, 3)
        self.assertRaises(BufferError, block.add, 3, 3)
        self.assertRaises(BufferError, block.add, 4, 3)

        # Add to an empty interior block
        block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
        block.keys = []
        block.addresses = []
        block.add(5, 7)
        block.add(6, 8)
        self.assertSequenceEqual(block.keys, [5, 6])
        self.assertSequenceEqual(block.addresses, [7, 8])

        # Add to a interior block with 1 key and 1 address
        block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
        block.keys = [4]
        block.addresses = [5]
        block.add(5, 4)
        self.assertSequenceEqual(block.keys, [4, 5])
        self.assertSequenceEqual(block.addresses, [5, 4])

        block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
        block.keys = [4]
        block.addresses = [5]
        block.add(3, 4)
        self.assertSequenceEqual(block.keys, [3, 4])
        self.assertSequenceEqual(block.addresses, [4, 5])


        # Add to a half full interior block
        block.add(6, 8)
        block.add(4, 5)
        block.add(5, 10)
        self.assertSequenceEqual(block.keys, [3, 4, 4, 5, 6])
        self.assertSequenceEqual(block.addresses, [4, 5, 5, 10, 8])

        # Add to 1 off from full interior block
        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(1, 1)
        self.assertSequenceEqual(block.keys, [1, 2] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses, [1] + ([5] * (block.maxKeys - 1)))

        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(2, 2)
        self.assertSequenceEqual(block.keys, [2, 2] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses, [2] + ([5] * (block.maxKeys - 1)))

        block.keys = [2] + ([3] * (block.maxKeys - 2))
        block.addresses = [5] * (block.maxKeys - 1)
        block.add(3, 3)
        self.assertSequenceEqual(block.keys, [2, 3] + ([3] * (block.maxKeys - 2)))
        self.assertSequenceEqual(block.addresses, [5, 3] + ([5] * (block.maxKeys - 2)))

        # Add to a full interior block
        self.assertRaises(BufferError, block.add, 1, 3)
        self.assertRaises(BufferError, block.add, 2, 3)
        self.assertRaises(BufferError, block.add, 3, 3)
        self.assertRaises(BufferError, block.add, 4, 3)