def testStringIO(self):
for blockClass in (NanoBlocks.Index.LeafBlock, NanoBlocks.Index.InteriorBlock):
for keyType in ("int1", "int2", "int4", "int8", "uint2", "uint8",
"float4", "float8"):
block = blockClass(101, NanoTypes.getType(keyType))
block2 = blockClass(101, NanoTypes.getType(keyType))
block.keys = []
block.addresses = []
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = [1]
block.addresses = [4]
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = range(1, 40, 2)
block.addresses = [i * 2 for i in range(1, 40, 2)]
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = [1] * block.maxKeys
block.addresses = range(block.maxKeys)
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
self.assertRaises(Exception, NanoBlocks.Index._IndexBlock.fromString, block.toString() + '\x00')
block.keys = [block.dataType.nullVal] * (block.maxKeys + 1)
block.addresses = [NanoBlocks.Index.ADDRESS_TYPE.nullVal] * (block.maxKeys + 1)
self.assertRaises(Exception, block.toString)
for blockClass in (NanoBlocks.Index.LeafBlock, NanoBlocks.Index.InteriorBlock):
for keyQuantifier in range(1, 40):
block = blockClass(101, NanoTypes.getType("char%d" % keyQuantifier))
block2 = blockClass(101, NanoTypes.getType("char%d" % keyQuantifier))
block.idx = 101
block.parent = 154
block.keys = []
block.addresses = []
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = ['1']
block.addresses = [4]
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = ["1" * keyQuantifier] * block.maxKeys
block.addresses = range(block.maxKeys)
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
self.keys = [str(k) for k in range(block.maxKeys + 1)]
self.assertRaises(Exception, NanoBlocks.Index._IndexBlock.fromString, block.toString())
class MemoryMappedTester(MemoryMappedBlock):
blockSize = 10
fields = [
'name',
'typ',
'typ2',
]
dataTypes = {
'name': NanoTypes.Char(4),
'typ': NanoTypes.Uint(2),
'typ2': NanoTypes.Uint(4),
}
class MemoryMappedRow(MemoryMappedBlock):
"""
Class which can be used to get/store rows for this table. Adds an additional byte flag to the beginning
of each and every row to indicate whether or not that row is still valid. (When a row is deleted it will
be set to False).
"""
blockSize = self.config.rowSize
fields = ['_valid'] + [col.name for col in self.config.columns]
dataTypes = {
col.name: NanoTypes.getType(col.typeString)
for col in self.config.columns
}
dataTypes['_valid'] = NanoTypes.Uint(1)
def testGetType(self):
self.assertEqual(str(NanoTypes.getType("int1")), "Int1")
self.assertEqual(str(NanoTypes.getType("int2")), "Int2")
self.assertEqual(str(NanoTypes.getType("int4")), "Int4")
self.assertEqual(str(NanoTypes.getType("int8")), "Int8")
self.assertEqual(str(NanoTypes.getType("uint4")), "Uint4")
self.assertEqual(str(NanoTypes.getType("char10")), "Char10")
self.assertEqual(str(NanoTypes.getType("float4")), "Float4")
self.assertEqual(str(NanoTypes.getType("float8")), "Float8")
def testGetType(self):
self.assertEqual(str(NanoTypes.getType("int1")), "Int1")
self.assertEqual(str(NanoTypes.getType("int2")), "Int2")
self.assertEqual(str(NanoTypes.getType("int4")), "Int4")
self.assertEqual(str(NanoTypes.getType("int8")), "Int8")
self.assertEqual(str(NanoTypes.getType("uint4")), "Uint4")
self.assertEqual(str(NanoTypes.getType("char10")), "Char10")
self.assertEqual(str(NanoTypes.getType("float4")), "Float4")
self.assertEqual(str(NanoTypes.getType("float8")), "Float8")
def testMarkDeletedBlock(self):
indexBlock = NanoBlocks.Index.LeafBlock(5, NanoTypes.Int(4))
indexBlock.keys = [1]
indexBlock.addresses = [2]
self.IndexIO._writeBlockToFile(indexBlock)
self.IndexIO._markBlockDeleted(indexBlock)
self.assertEqual(self.IndexIO.delMgr.popRef(), 5)
self.assertIsNone(self.IndexIO.delMgr.popRef())
def setUp(self):
NanoConfig.num_index_dirty_blocks = 4
NanoConfig.index_block_size = 29
self.blocks = []
self.blocks2 = []
for i in range(4):
self.blocks.append(NanoBlocks.Index.LeafBlock(101, NanoTypes.getType('char4')))
self.blocks[-1].pointers = []
self.blocks[-1].keys = ["blk%d" % i]
self.blocks[-1].address = i * NanoConfig.index_block_size
self.blocks2.append(NanoBlocks.Index.LeafBlock(101, NanoTypes.getType('char4')))
self.blocks2[-1].pointers = []
self.blocks2[-1].keys = ["blk%d" % i]
self.blocks2[-1].address = i * NanoConfig.index_block_size
self.fd = NanoIO.File.createIndex(self.dbName, self.tableName, self.colName)
self.dirtyMgr = NanoTools.BlockCacheManager.BlockCacheManager(self.fd)
def testChar(self):
for iVal in range(1, 257):
c = NanoTypes.getType("char%d" % iVal)
self.assertEqual(c.isValid(4), False)
self.assertEqual(c.isValid(c), False)
self.assertEqual(c.isValid(c.nullVal), True)
self.assertEqual(c.isValid("a" * iVal), True)
self.assertEqual(c.isValid("_" * iVal), True)
self.assertEqual(c.toString(None), c.toString(c.nullVal))
self.assertEqual(c.fromString(c.nullVal), None)
self.assertEqual(c.fromString(c.toString(None)), None)
self.assertRaises(ValueError, c.toString, 54)
def testChar(self):
for iVal in range(1, 257):
c = NanoTypes.getType("char%d" % iVal)
self.assertEqual(c.isValid(4), False)
self.assertEqual(c.isValid(c), False)
self.assertEqual(c.isValid(c.nullVal), True)
self.assertEqual(c.isValid("a" * iVal), True)
self.assertEqual(c.isValid("_" * iVal), True)
self.assertEqual(c.toString(None), c.toString(c.nullVal))
self.assertEqual(c.fromString(c.nullVal), None)
self.assertEqual(c.fromString(c.toString(None)), None)
self.assertRaises(ValueError, c.toString, 54)
def setUp(self):
NanoConfig.num_index_dirty_blocks = 4
NanoConfig.index_block_size = 29
self.blocks = []
self.blocks2 = []
for i in range(4):
self.blocks.append(
NanoBlocks.Index.LeafBlock(101, NanoTypes.getType('char4')))
self.blocks[-1].pointers = []
self.blocks[-1].keys = ["blk%d" % i]
self.blocks[-1].address = i * NanoConfig.index_block_size
self.blocks2.append(
NanoBlocks.Index.LeafBlock(101, NanoTypes.getType('char4')))
self.blocks2[-1].pointers = []
self.blocks2[-1].keys = ["blk%d" % i]
self.blocks2[-1].address = i * NanoConfig.index_block_size
self.fd = NanoIO.File.createIndex(self.dbName, self.tableName,
self.colName)
self.dirtyMgr = NanoTools.BlockCacheManager.BlockCacheManager(self.fd)
def testFloat(self):
for iVal in (4, 8):
f = NanoTypes.getType("float%d" % iVal)
self.assertEqual(f.isValid('test'), False)
self.assertEqual(f.isValid(f), False)
self.assertEqual(f.isValid(f.nullVal), True)
self.assertEqual(f.isValid(0), True)
self.assertEqual(f.isValid(0.0), True)
self.assertEqual(f.isValid(100000.0), True)
self.assertEqual(f.isValid(23495812980375.123547861235), True)
self.assertEqual(f.isValid(-129357.12351235), True)
self.assertEqual(f.toString(f.nullVal), f.toString(None))
self.assertEqual(f.fromString(f.toString(None)), None)
self.assertRaises(ValueError, f.toString, 'testing')
def testFloat(self):
for iVal in (4, 8):
f = NanoTypes.getType("float%d" % iVal)
self.assertEqual(f.isValid('test'), False)
self.assertEqual(f.isValid(f), False)
self.assertEqual(f.isValid(f.nullVal), True)
self.assertEqual(f.isValid(0), True)
self.assertEqual(f.isValid(0.0), True)
self.assertEqual(f.isValid(100000.0), True)
self.assertEqual(f.isValid(23495812980375.123547861235), True)
self.assertEqual(f.isValid(-129357.12351235), True)
self.assertEqual(f.toString(f.nullVal), f.toString(None))
self.assertEqual(f.fromString(f.toString(None)), None)
self.assertRaises(ValueError, f.toString, 'testing')
class Config(VariableMemoryBlock):
# Class Attributes
column = None
unique = None # TODO implement - raise error if multiple added
# VariableMemoryMappedBlock definitions
fields = [
'column',
'unique',
]
dataTypes = {
'column':
VariableMemoryBlock.SerializableClass(NanoConfig.Column.Config),
'unique': NanoTypes.Uint(1),
}
def testLookup(self):
# Leaf block
block = NanoBlocks.Index.LeafBlock(0, NanoTypes.Int(1))
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 1)
block.keys = [1]
block.addresses = [1]
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 0)
self.assertEqual(block.lookup(1), 1)
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 2)
block.keys = [1, 3]
block.addresses = ['a', 'b']
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 0)
self.assertEqual(block.lookup(1), 'a')
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 2)
self.assertEqual(block.lookup(3), 'b')
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 4)
# Interior block
block = NanoBlocks.Index.InteriorBlock(0, NanoTypes.Int(1))
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 1)
block.keys = [1]
block.addresses = [1]
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 0)
self.assertEqual(block.lookup(1), 1)
self.assertEqual(block.lookup(2), 1)
block.keys = [1, 3]
block.addresses = ['a', 'b']
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.lookup, 0)
self.assertEqual(block.lookup(1), 'a')
self.assertEqual(block.lookup(2), 'a')
self.assertEqual(block.lookup(3), 'b')
self.assertEqual(block.lookup(4), 'b')
def __init__(self, dbName, tableName, indexConfig):
self.dbName = dbName
self.tableName = tableName
self.indexConfig = indexConfig
self.indexFD = NanoIO.File.getIndex(self.dbName, self.tableName, self.indexConfig.column.name)
self.cacheMgr = NanoTools.BlockCacheManager.BlockCacheManager(self.indexFD)
self.delMgr = NanoTools.DeletedBlockManager.DeletedBlockManager(
self.dbName, NanoIO.File.indexFileName(self.tableName, self.indexConfig.column.name)
)
self.colType = NanoTypes.getType(self.indexConfig.column.typeString)
# Check if our index is empty. If it is, create an empty leafblock to start with
self.indexFD.seek(0, os.SEEK_END)
if self.indexFD.tell() == 0:
self.indexFD.write(LeafBlock(0, self.colType).toString())
self.indexFD.flush()
def testUint(self):
for iVal in (1, 2, 4, 8):
i = NanoTypes.getType("uint%d" % iVal)
self.assertEqual(i.isValid('test'), False)
self.assertEqual(i.isValid(i), False)
self.assertEqual(i.isValid(i.maxVal + 2), False)
self.assertEqual(i.isValid(-1), False)
self.assertEqual(i.isValid(0), True)
self.assertEqual(i.isValid(i.maxVal), True)
self.assertEqual(i.toString(1), "\x01" + ("\x00" * (iVal - 1)))
self.assertEqual(i.toString(16), "\x10" + ("\x00" * (iVal - 1)))
self.assertEqual(i.fromString("\x10" + ("\x00" * (iVal - 1))), 16)
self.assertEqual(i.isValid(i.nullVal), True)
self.assertEqual(i.toString(None), i.toString(i.nullVal))
self.assertEqual(i.fromString(i.toString(None)), None)
self.assertEqual(i.fromString(i.toString(i.nullVal)), None)
self.assertRaises(ValueError, i.toString, i.maxVal + 2)
def testUint(self):
for iVal in (1, 2, 4, 8):
i = NanoTypes.getType("uint%d" % iVal)
self.assertEqual(i.isValid('test'), False)
self.assertEqual(i.isValid(i), False)
self.assertEqual(i.isValid(i.maxVal + 2), False)
self.assertEqual(i.isValid(-1), False)
self.assertEqual(i.isValid(0), True)
self.assertEqual(i.isValid(i.maxVal), True)
self.assertEqual(i.toString(1), "\x01" + ("\x00" * (iVal - 1)))
self.assertEqual(i.toString(16), "\x10" + ("\x00" * (iVal - 1)))
self.assertEqual(i.fromString("\x10" + ("\x00" * (iVal - 1))), 16)
self.assertEqual(i.isValid(i.nullVal), True)
self.assertEqual(i.toString(None), i.toString(i.nullVal))
self.assertEqual(i.fromString(i.toString(None)), None)
self.assertEqual(i.fromString(i.toString(i.nullVal)), None)
self.assertRaises(ValueError, i.toString, i.maxVal + 2)
def __init__(self, dbName, tableName, indexConfig):
self.dbName = dbName
self.tableName = tableName
self.indexConfig = indexConfig
self.indexFD = NanoIO.File.getIndex(self.dbName, self.tableName,
self.indexConfig.column.name)
self.cacheMgr = NanoTools.BlockCacheManager.BlockCacheManager(
self.indexFD)
self.delMgr = NanoTools.DeletedBlockManager.DeletedBlockManager(
self.dbName,
NanoIO.File.indexFileName(self.tableName,
self.indexConfig.column.name))
self.colType = NanoTypes.getType(self.indexConfig.column.typeString)
# Check if our index is empty. If it is, create an empty leafblock to start with
self.indexFD.seek(0, os.SEEK_END)
if self.indexFD.tell() == 0:
self.indexFD.write(LeafBlock(0, self.colType).toString())
self.indexFD.flush()
def testGetAndWriteBlockAtIndex(self):
# Test getting a block from an index with no keys
self._assertIndexBlockEqual(
self.IndexIO._getBlockAtAddress(0),
NanoBlocks.Index.LeafBlock(0, self.IndexIO.colType))
# Test getting a fresh block
indexBlock = NanoBlocks.Index.LeafBlock(0, NanoTypes.Int(4))
self.IndexIO.indexFD.write(indexBlock.toString())
self.assertFalse(0 in self.IndexIO.cacheMgr)
self._assertIndexBlockEqual(indexBlock,
self.IndexIO._getBlockAtAddress(0))
# Test getting a block that should exist in the Block Cache Manager
indexBlock.keys = [1]
indexBlock.addresses = [2]
self.IndexIO._writeBlockToFile(indexBlock)
self.assertTrue(0 in self.IndexIO.cacheMgr)
self._assertIndexBlockEqual(indexBlock,
self.IndexIO._getBlockAtAddress(0))
class Config(VariableMemoryBlock):
# Class Attributes
name = None
columns = None
indices = None
rowSize = None
# VariableMemoryBlock definitions
fields = [
'name',
'rowSize',
'columns',
'indices',
]
dataTypes = {
'name': VariableMemoryBlock.SerializableString,
'rowSize': NanoTypes.Uint(4),
'columns': VariableMemoryBlock.SerializableClass(NanoConfig.Column.Config),
'indices': VariableMemoryBlock.SerializableClass(NanoConfig.Index.Config),
}
iterableFields = [
'columns',
'indices',
]
def testStringIO(self):
for blockClass in (NanoBlocks.Index.LeafBlock,
NanoBlocks.Index.InteriorBlock):
for keyType in ("int1", "int2", "int4", "int8", "uint2", "uint8",
"float4", "float8"):
block = blockClass(101, NanoTypes.getType(keyType))
block2 = blockClass(101, NanoTypes.getType(keyType))
block.keys = []
block.addresses = []
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = [1]
block.addresses = [4]
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = range(1, 40, 2)
block.addresses = [i * 2 for i in range(1, 40, 2)]
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = [1] * block.maxKeys
block.addresses = range(block.maxKeys)
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
self.assertRaises(Exception,
NanoBlocks.Index._IndexBlock.fromString,
block.toString() + '\x00')
block.keys = [block.dataType.nullVal] * (block.maxKeys + 1)
block.addresses = [NanoBlocks.Index.ADDRESS_TYPE.nullVal
] * (block.maxKeys + 1)
self.assertRaises(Exception, block.toString)
for blockClass in (NanoBlocks.Index.LeafBlock,
NanoBlocks.Index.InteriorBlock):
for keyQuantifier in range(1, 40):
block = blockClass(101,
NanoTypes.getType("char%d" % keyQuantifier))
block2 = blockClass(
101, NanoTypes.getType("char%d" % keyQuantifier))
block.idx = 101
block.parent = 154
block.keys = []
block.addresses = []
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = ['1']
block.addresses = [4]
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
block.keys = ["1" * keyQuantifier] * block.maxKeys
block.addresses = range(block.maxKeys)
block2.fromString(block.toString())
self.assertBlocksEqual(block, block2)
self.keys = [str(k) for k in range(block.maxKeys + 1)]
self.assertRaises(Exception,
NanoBlocks.Index._IndexBlock.fromString,
block.toString())
def testDelete(self):
# Delete from an empty block
block = NanoBlocks.Index.LeafBlock(101, NanoTypes.getType("int1"))
block.keys = []
block.addresses = []
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 1)
# Delete a non-present key from the block
block.keys = [5]
block.addresses = [6]
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 1)
# Delete from a leaf with 1 key
block.delete(5)
self.assertSequenceEqual(block.keys, [])
self.assertSequenceEqual(block.addresses, [])
# Delete from the beginning of a half-full leaf
block.keys = [3, 4, 5, 6, 7]
block.addresses = [8, 9, 10, 11, 12]
block.delete(3)
self.assertSequenceEqual(block.keys, [4, 5, 6, 7])
self.assertSequenceEqual(block.addresses, [9, 10, 11, 12])
# Delete from the end of a half-full leaf
block.delete(7)
self.assertSequenceEqual(block.keys, [4, 5, 6])
self.assertSequenceEqual(block.addresses, [9, 10, 11])
# Delete from the middle of a half-full leaf
block.delete(5)
self.assertSequenceEqual(block.keys, [4, 6])
self.assertSequenceEqual(block.addresses, [9, 11])
# Delete from a interior with 1 key and 1 address
block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
block.keys = [5]
block.addresses = [6]
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 4)
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 6)
block.delete(5)
self.assertSequenceEqual(block.keys, [])
self.assertSequenceEqual(block.addresses, [])
# Delete from the beginning of a half-full interior block
block.keys = [3, 4, 5, 6, 7]
block.addresses = [8, 9, 10, 11, 12]
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 2)
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 8)
block.delete(3)
self.assertSequenceEqual(block.keys, [4, 5, 6, 7])
self.assertSequenceEqual(block.addresses, [9, 10, 11, 12])
# Delete from the end of a half-full interior block
block.delete(7)
self.assertSequenceEqual(block.keys, [4, 5, 6])
self.assertSequenceEqual(block.addresses, [9, 10, 11])
# Delete from the middle of a half-full interior block
block.delete(5)
self.assertSequenceEqual(block.keys, [4, 6])
self.assertSequenceEqual(block.addresses, [9, 11])
class VariableMemoryBlock:
"""
Class which provides toString and fromString functionality to subclasses which should be serializable while not
conforming to strict sizing requirements.
Stores fields as follows:
4 bytes: number of bytes required to serialize this field
X bytes: Serialized version of this field
"""
# Private Variables
# DataType which is used to serialize the `numBytes` field preceeding each item in self.fields
_sizeDataType = NanoTypes.Uint(4)
# Subclassable Variables
# List of fields on self that should be serialized in the file
fields = None
# Dictionary mapping fieldName to the dataType to convert to from the raw string representation
dataTypes = None
# List of fields which are stored as iterables
iterableFields = None
###
# Variable Memory DataType classes
#
# Classes which can be used as dataTypes when constructing new VariableMemoryBlock subclasses
###
class SerializableString:
""" DataType which can be used to serialize variables as variable lengthed strings. """
@staticmethod
def toString(s):
return str(s) if s else ""
@staticmethod
def fromString(s):
return s
class SerializableClass:
"""
DataType which can be used to wrap classes that define toString & fromString methods as types.
Classes must be instantiable requiring no arguments.
"""
classType = None
def __init__(self, classType):
self.classType = classType
def toString(self, classInstance):
return classInstance.toString() if classInstance else ""
def fromString(self, s):
classInstance = self.classType()
if s:
classInstance.fromString(s)
return classInstance
# Private methods
def __serializeAttribute(self, fieldName, value):
serializedVal = self.dataTypes[fieldName].toString(value)
return self._sizeDataType.toString(len(serializedVal)) + serializedVal
def __getFromBlock(self, fromDataType, idx, numBytes, s):
"""
Pulls an object out of s.
Inputs: fromDataType - The dataType to use to fromString the substring of s.
idx - The index to begin the substring of s.
numBytes - The number of bytes to extract from s.
s - The block to extract the substring from.
Returns: (The Object fromString'd from the block, The index after reading from s)
"""
if len(s) < idx + numBytes:
raise Exception(
"Given string len(%s) is not long enough to unserialize %s" %
(len(s), self))
return fromDataType.fromString(s[idx:idx + numBytes]), idx + numBytes
# Public methods
def toString(self):
"""
Returns a string representation of this object suitable for writing to a file.
"""
# List of serialized fields
toSerialize = []
for fieldName in self.fields:
attrVal = getattr(self, fieldName)
# If this field has been marked as iterable
if self.iterableFields and fieldName in self.iterableFields:
# Ensure this variable is actually iterable
if not hasattr(attrVal, '__iter__'):
toSerialize.append(self._sizeDataType.toString(0))
else:
toSerialize.append("".join(
# Serialize the length of the iterable
[self._sizeDataType.toString(len(attrVal))] +
# Serialize each item of the iterable
[
self.__serializeAttribute(fieldName, val)
for val in attrVal
]))
# Otherwise serialize the attribute as a single value
else:
toSerialize.append(
self.__serializeAttribute(fieldName, attrVal))
return "".join(toSerialize)
def fromString(self, s):
"""
Initializes this object from the data found in s.
Inputs: s - A string containing the data to initialize this instance with.
Raises an error if s contains insufficient memory to initialize all fields in self.fields, or
if it contains extra bytes not required to initialize all fields in self.fields.
"""
# Current index into s
idx = 0
for fieldName in self.fields:
# Get the number of bytes to initialize this field with
fieldLength, idx = self.__getFromBlock(self._sizeDataType, idx,
self._sizeDataType.size, s)
# If this field has been marked as iterable
if self.iterableFields and fieldName in self.iterableFields:
setattr(self, fieldName, [])
for i in range(fieldLength):
itemLength, idx = self.__getFromBlock(
self._sizeDataType, idx, self._sizeDataType.size, s)
fieldVal, idx = self.__getFromBlock(
self.dataTypes[fieldName], idx, itemLength, s)
getattr(self, fieldName).append(fieldVal)
# Otherwise unserialize this single value
else:
fieldVal, idx = self.__getFromBlock(self.dataTypes[fieldName],
idx, fieldLength, s)
setattr(self, fieldName, fieldVal)
if len(s) > idx:
raise Exception("Given string len(%s) too long to unserialize %s" %
(len(s), self))
return self
def testAdd(self):
# Add to an empty leaf block
block = NanoBlocks.Index.LeafBlock(101, NanoTypes.getType("int1"))
block.keys = []
block.addresses = []
block.add(5, 7)
self.assertSequenceEqual(block.keys, [5])
self.assertSequenceEqual(block.addresses, [7])
# Add to a half full leaf block
block.add(6, 8)
block.add(4, 5)
block.add(5, 10)
self.assertSequenceEqual(block.keys, [4, 5, 5, 6])
self.assertSequenceEqual(block.addresses, [5, 10, 7, 8])
# Add to 1 off from full leaf block
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(1, 1)
self.assertSequenceEqual(block.keys,
[1, 2] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses,
[1] + ([5] * (block.maxKeys - 1)))
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(2, 2)
self.assertSequenceEqual(block.keys,
[2, 2] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses,
[2] + ([5] * (block.maxKeys - 1)))
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(3, 3)
self.assertSequenceEqual(block.keys,
[2, 3] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses,
[5, 3] + ([5] * (block.maxKeys - 2)))
# Add to a full leaf block
block.keys = [3] * block.maxKeys
block.addresses = [4] * block.maxKeys
self.assertRaises(BufferError, block.add, 2, 3)
self.assertRaises(BufferError, block.add, 3, 3)
self.assertRaises(BufferError, block.add, 4, 3)
# Add to an empty interior block
block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
block.keys = []
block.addresses = []
block.add(5, 7)
block.add(6, 8)
self.assertSequenceEqual(block.keys, [5, 6])
self.assertSequenceEqual(block.addresses, [7, 8])
# Add to a interior block with 1 key and 1 address
block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
block.keys = [4]
block.addresses = [5]
block.add(5, 4)
self.assertSequenceEqual(block.keys, [4, 5])
self.assertSequenceEqual(block.addresses, [5, 4])
block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
block.keys = [4]
block.addresses = [5]
block.add(3, 4)
self.assertSequenceEqual(block.keys, [3, 4])
self.assertSequenceEqual(block.addresses, [4, 5])
# Add to a half full interior block
block.add(6, 8)
block.add(4, 5)
block.add(5, 10)
self.assertSequenceEqual(block.keys, [3, 4, 4, 5, 6])
self.assertSequenceEqual(block.addresses, [4, 5, 5, 10, 8])
# Add to 1 off from full interior block
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(1, 1)
self.assertSequenceEqual(block.keys,
[1, 2] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses,
[1] + ([5] * (block.maxKeys - 1)))
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(2, 2)
self.assertSequenceEqual(block.keys,
[2, 2] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses,
[2] + ([5] * (block.maxKeys - 1)))
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(3, 3)
self.assertSequenceEqual(block.keys,
[2, 3] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses,
[5, 3] + ([5] * (block.maxKeys - 2)))
# Add to a full interior block
self.assertRaises(BufferError, block.add, 1, 3)
self.assertRaises(BufferError, block.add, 2, 3)
self.assertRaises(BufferError, block.add, 3, 3)
self.assertRaises(BufferError, block.add, 4, 3)
def executeQuery(self, conn):
if self.target == "database":
NanoIO.File.createDatabase(self.name)
elif self.target == "table":
dbName, tableName = conn._parseName(self.name)
# Assert that the database exists, but the table does not
NanoIO.File.assertDatabaseExists(dbName)
if NanoIO.File.checkTableExists(dbName, tableName):
raise IOError("Cannot create table, table file %s.%s already exists" % (dbName, tableName))
if NanoIO.File.checkConfigExists(dbName, tableName):
raise IOError("Cannot create table, config file for table %s.%s already exists" % (dbName, tableName))
tableConfig = NanoConfig.Table.Config()
tableConfig.name = tableName
# Add columns to this table
colDict = dict()
tableConfig.columns = []
rowSize = 1
for column in self.cols:
# Sanity check; disallow duplicated column names
if column['name'] in colDict:
raise Exception("Duplicated column name: %s" % column['name'])
columnConfig = NanoConfig.Column.Config()
columnConfig.name = column['name']
columnConfig.typeString = column['type']
# Validate this type string
typ = NanoTypes.getType(columnConfig.typeString)
colDict[column['name']] = columnConfig
tableConfig.columns.append(columnConfig)
rowSize += typ.size
tableConfig.rowSize = rowSize
# Add indices to this table
idxSet = set()
tableConfig.indices = []
if self.indices:
for index in self.indices:
# Sanity check; disallow duplicated indices
if index in idxSet:
raise Exception("Duplicated index name: %s" % index)
# Ensure this indexed column exists in our table
if index not in colDict:
raise Exception("Cannot create index on missing column: %s" % index)
# Ensure that the index file for this index doesn't somehow already exist
if NanoIO.File.checkIndexExists(dbName, tableName, index):
raise Exception("Cannot create table %s.%s; index file already exists at %s" % \
(dbName, tableName, NanoIO.File.indexPath(dbName, tableName, index)))
idxSet.add(index)
indexConfig = NanoConfig.Index.Config()
indexConfig.column = colDict[index]
tableConfig.indices.append(indexConfig)
# Create each index for this table
for index in self.indices:
NanoIO.File.createIndex(dbName, tableName, index)
# Create this table
configFD = NanoIO.File.createTable(dbName, tableName)
configFD.write(tableConfig.toString())
configFD.close()
class TestIndex(NanoTests.NanoTestCase):
tableName = "IOTestIndex"
indexColName = "IOTestIndexedColumn"
indexConfig = None
dataType = NanoTypes.getType('int4')
def setUp(self):
col = NanoConfig.Column.Config()
col.name = self.indexColName
col.typeString = "int4"
self.indexConfig = NanoConfig.Index.Config()
self.indexConfig.column = col
self.indexConfig.unique = False
NanoIO.File.deleteIndex(self.dbName, self.tableName, self.indexColName)
NanoIO.File.createIndex(self.dbName, self.tableName,
self.indexColName).close()
self.IndexIO = NanoIO.Index.IndexIO(self.dbName, self.tableName,
self.indexConfig)
def tearDown(self):
NanoIO.File.deleteIndex(self.dbName, self.tableName, self.indexColName)
def _assertIndexBlockEqual(self, indexBlock, indexBlock2):
self.assertEqual(indexBlock2.isLeaf, indexBlock.isLeaf)
self.assertEqual(indexBlock2.address, indexBlock.address)
self.assertEqual(indexBlock2.dataType.quantifier,
indexBlock.dataType.quantifier)
self.assertSequenceEqual(indexBlock2.keys, indexBlock.keys)
self.assertSequenceEqual(indexBlock2.addresses, indexBlock.addresses)
def testGetAndWriteBlockAtIndex(self):
# Test getting a block from an index with no keys
self._assertIndexBlockEqual(
self.IndexIO._getBlockAtAddress(0),
NanoBlocks.Index.LeafBlock(0, self.IndexIO.colType))
# Test getting a fresh block
indexBlock = NanoBlocks.Index.LeafBlock(0, NanoTypes.Int(4))
self.IndexIO.indexFD.write(indexBlock.toString())
self.assertFalse(0 in self.IndexIO.cacheMgr)
self._assertIndexBlockEqual(indexBlock,
self.IndexIO._getBlockAtAddress(0))
# Test getting a block that should exist in the Block Cache Manager
indexBlock.keys = [1]
indexBlock.addresses = [2]
self.IndexIO._writeBlockToFile(indexBlock)
self.assertTrue(0 in self.IndexIO.cacheMgr)
self._assertIndexBlockEqual(indexBlock,
self.IndexIO._getBlockAtAddress(0))
def testMarkDeletedBlock(self):
indexBlock = NanoBlocks.Index.LeafBlock(5, NanoTypes.Int(4))
indexBlock.keys = [1]
indexBlock.addresses = [2]
self.IndexIO._writeBlockToFile(indexBlock)
self.IndexIO._markBlockDeleted(indexBlock)
self.assertEqual(self.IndexIO.delMgr.popRef(), 5)
self.assertIsNone(self.IndexIO.delMgr.popRef())
def testGetIdxForNewBlock(self):
self.assertEqual(self.IndexIO._getAddressForNewBlock(),
NanoConfig.index_block_size)
idxBlock = self.IndexIO._getBlockAtAddress(0)
idxBlock.address = NanoConfig.index_block_size
self.IndexIO._writeBlockToFile(idxBlock)
self.assertEqual(self.IndexIO._getAddressForNewBlock(),
NanoConfig.index_block_size * 2)
self.IndexIO._markBlockDeleted(idxBlock)
self.assertEqual(self.IndexIO._getAddressForNewBlock(),
NanoConfig.index_block_size)
def testAddLookupDelete(self):
# Test adding to an empty index
self.assertRaises(NanoBlocks.Index.KeyNotFound, self.IndexIO.lookup, 1)
self.IndexIO.add(1, 2)
self.assertEqual(self.IndexIO.lookup(1), 2)
# Test adding to an index with a single partially full leaf block
for i in range(5, 50, 2):
self.IndexIO.add(i, i + 1)
self.assertRaises(NanoBlocks.Index.KeyNotFound, self.IndexIO.lookup, 8)
self.IndexIO.add(8, 9)
self.assertEqual(self.IndexIO.lookup(8), 9)
# Test adding values to the beginning of a partially full leaf block
self.assertRaises(NanoBlocks.Index.KeyNotFound, self.IndexIO.lookup, 0)
self.IndexIO.add(0, 0)
self.assertEqual(self.IndexIO.lookup(0), 0)
# Test adding to an index with a single full leaf block
self.tearDown()
self.setUp()
rootBlock = self.IndexIO._getBlockAtAddress(0)
rootBlock.keys = []
rootBlock.addresses = []
for i in range(rootBlock.maxKeys):
rootBlock.add(i, i + 1)
self.IndexIO._writeBlockToFile(rootBlock)
for i in range(rootBlock.maxKeys):
self.assertEqual(self.IndexIO.lookup(i), i + 1)
self.IndexIO.add(20, 20)
self.IndexIO.add(500, 500)
for i in range(1, 170):
self.IndexIO.add(-i, i)
# Test adding to an index with a root interior block pointing to leaf blocks # TODO
# Test adding to an index with a full root interior block, pointing to leaf blocks # TODO
# Test adding to an index with a root interior block pointing to interior blocks # TODO
def testRandomAddDeleteLookup(self):
seed = 124
numPairs = 4000
keyRange = (0, 10000000)
random.seed(seed)
keys = set([random.randint(*keyRange) for i in range(numPairs)])
pairs = [(k, random.randint(*keyRange)) for k in keys]
for k, a in pairs:
self.IndexIO.add(k, a)
for k, a in reversed(pairs):
self.assertEqual(self.IndexIO.lookup(k), a)
self.IndexIO.delete(k)
self.assertRaises(NanoBlocks.Index.KeyNotFound,
self.IndexIO.lookup, k)
root = self.IndexIO._getBlockAtAddress(0)
self.assertEqual(len(root.keys), 0)
def testDelete(self): # TODO
pass
def testLookup(self): # TODO
pass
def testLookupCondition(self): # TODO
pass
def testIterate(self): # TODO
pass
# Standard imports
import os
# Project imports
import NanoTypes
import NanoIO.File
POINTER_TYPE = NanoTypes.Uint(8)
class DeletedBlockManager:
fd = None
name = None
db = None
def __init__(self, db, name):
self.db = db
self.name = name
self.fd = NanoIO.File.openReadWriteFile(
NanoIO.File.delMgrPath(db, name))
def __del__(self):
try:
self.close()
except AttributeError:
pass
def truncate(self):
self.fd.seek(0)
self.fd.truncate()
def testDelete(self):
# Delete from an empty block
block = NanoBlocks.Index.LeafBlock(101, NanoTypes.getType("int1"))
block.keys = []
block.addresses = []
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 1)
# Delete a non-present key from the block
block.keys = [5]
block.addresses = [6]
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 1)
# Delete from a leaf with 1 key
block.delete(5)
self.assertSequenceEqual(block.keys, [])
self.assertSequenceEqual(block.addresses, [])
# Delete from the beginning of a half-full leaf
block.keys = [3, 4, 5, 6, 7]
block.addresses = [8, 9, 10, 11, 12]
block.delete(3)
self.assertSequenceEqual(block.keys, [4, 5, 6, 7])
self.assertSequenceEqual(block.addresses, [9, 10, 11, 12])
# Delete from the end of a half-full leaf
block.delete(7)
self.assertSequenceEqual(block.keys, [4, 5, 6])
self.assertSequenceEqual(block.addresses, [9, 10, 11])
# Delete from the middle of a half-full leaf
block.delete(5)
self.assertSequenceEqual(block.keys, [4, 6])
self.assertSequenceEqual(block.addresses, [9, 11])
# Delete from a interior with 1 key and 1 address
block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
block.keys = [5]
block.addresses = [6]
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 4)
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 6)
block.delete(5)
self.assertSequenceEqual(block.keys, [])
self.assertSequenceEqual(block.addresses, [])
# Delete from the beginning of a half-full interior block
block.keys = [3, 4, 5, 6, 7]
block.addresses = [8, 9, 10, 11, 12]
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 2)
self.assertRaises(NanoBlocks.Index.KeyNotFound, block.delete, 8)
block.delete(3)
self.assertSequenceEqual(block.keys, [4, 5, 6, 7])
self.assertSequenceEqual(block.addresses, [9, 10, 11, 12])
# Delete from the end of a half-full interior block
block.delete(7)
self.assertSequenceEqual(block.keys, [4, 5, 6])
self.assertSequenceEqual(block.addresses, [9, 10, 11])
# Delete from the middle of a half-full interior block
block.delete(5)
self.assertSequenceEqual(block.keys, [4, 6])
self.assertSequenceEqual(block.addresses, [9, 11])
def testAdd(self):
# Add to an empty leaf block
block = NanoBlocks.Index.LeafBlock(101, NanoTypes.getType("int1"))
block.keys = []
block.addresses = []
block.add(5, 7)
self.assertSequenceEqual(block.keys, [5])
self.assertSequenceEqual(block.addresses, [7])
# Add to a half full leaf block
block.add(6, 8)
block.add(4, 5)
block.add(5, 10)
self.assertSequenceEqual(block.keys, [4, 5, 5, 6])
self.assertSequenceEqual(block.addresses, [5, 10, 7, 8])
# Add to 1 off from full leaf block
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(1, 1)
self.assertSequenceEqual(block.keys, [1, 2] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses, [1] + ([5] * (block.maxKeys - 1)))
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(2, 2)
self.assertSequenceEqual(block.keys, [2, 2] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses, [2] + ([5] * (block.maxKeys - 1)))
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(3, 3)
self.assertSequenceEqual(block.keys, [2, 3] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses, [5, 3] + ([5] * (block.maxKeys - 2)))
# Add to a full leaf block
block.keys = [3] * block.maxKeys
block.addresses = [4] * block.maxKeys
self.assertRaises(BufferError, block.add, 2, 3)
self.assertRaises(BufferError, block.add, 3, 3)
self.assertRaises(BufferError, block.add, 4, 3)
# Add to an empty interior block
block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
block.keys = []
block.addresses = []
block.add(5, 7)
block.add(6, 8)
self.assertSequenceEqual(block.keys, [5, 6])
self.assertSequenceEqual(block.addresses, [7, 8])
# Add to a interior block with 1 key and 1 address
block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
block.keys = [4]
block.addresses = [5]
block.add(5, 4)
self.assertSequenceEqual(block.keys, [4, 5])
self.assertSequenceEqual(block.addresses, [5, 4])
block = NanoBlocks.Index.InteriorBlock(101, NanoTypes.getType("int1"))
block.keys = [4]
block.addresses = [5]
block.add(3, 4)
self.assertSequenceEqual(block.keys, [3, 4])
self.assertSequenceEqual(block.addresses, [4, 5])
# Add to a half full interior block
block.add(6, 8)
block.add(4, 5)
block.add(5, 10)
self.assertSequenceEqual(block.keys, [3, 4, 4, 5, 6])
self.assertSequenceEqual(block.addresses, [4, 5, 5, 10, 8])
# Add to 1 off from full interior block
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(1, 1)
self.assertSequenceEqual(block.keys, [1, 2] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses, [1] + ([5] * (block.maxKeys - 1)))
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(2, 2)
self.assertSequenceEqual(block.keys, [2, 2] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses, [2] + ([5] * (block.maxKeys - 1)))
block.keys = [2] + ([3] * (block.maxKeys - 2))
block.addresses = [5] * (block.maxKeys - 1)
block.add(3, 3)
self.assertSequenceEqual(block.keys, [2, 3] + ([3] * (block.maxKeys - 2)))
self.assertSequenceEqual(block.addresses, [5, 3] + ([5] * (block.maxKeys - 2)))
# Add to a full interior block
self.assertRaises(BufferError, block.add, 1, 3)
self.assertRaises(BufferError, block.add, 2, 3)
self.assertRaises(BufferError, block.add, 3, 3)
self.assertRaises(BufferError, block.add, 4, 3)
