def Load(this, serialization):
keys = this.keys
values = this.values
if len(keys) > 0:
raise BplusTreeException, "cannot load into non-empty bucket"
index = 0
bytecount = len(serialization)
while index < bytecount:
keylength = BufferFile.RetrieveInt(serialization, index)
index += BufferFile.INTSTORAGE
nextindex = index + keylength
keybytes = serialization[index:nextindex]
index = nextindex
(key, dummy) = DECODER(keybytes)
valuelength = BufferFile.RetrieveInt(serialization, index)
index += BufferFile.INTSTORAGE
nextindex = index + valuelength
valuebytes = serialization[index:nextindex]
(value, dummy) = DECODER(valuebytes)
index = nextindex
keys.append(key)
values.append(value)
if index != bytecount:
raise BplusTreeException, "error counting bytes " + repr(
(index, bytecount))
def Dump(this):
L = []
a = L.append
if this.isLeaf:
a(chr(LEAF))
else:
a(chr(NONLEAF))
keys = this.ChildKeys
numbers = this.ChildBufferNumbers
#print "<br>keys", keys
#print "<br>numbers", numbers, "<br>"
maxLength = this.owner.KeyLength
maxKeyPayload = maxLength - BufferFile.SHORTSTORAGE
a(BufferFile.StoreLong(numbers[0]))
for KeyIndex in xrange(this.Size):
# store the key
theKey = keys[KeyIndex]
if theKey is None:
a(BufferFile.StoreShort(-1))
a("X" * maxKeyPayload)
else:
(coded, nchars) = ENCODER(theKey)
lcoded = len(coded)
if (lcoded > maxKeyPayload):
raise BplusTreeException, "too many bytes in key " + repr(
coded)
a(BufferFile.StoreShort(lcoded))
padded = coded + "X" * (maxKeyPayload - lcoded)
a(padded)
# store the seek
a(BufferFile.StoreLong(numbers[KeyIndex + 1]))
result = string.join(L, "")
#print "<br>", this.MyBufferNumber, "raw dump", repr(result), "<br>"
return result
def testBufferFile():
print "testing buffer file"
buffersize = 17
writesize = 10
mstream = newFile("pyBufferFile.dat")
offset = 55
bf = BufferFile.InitializeBufferFileInStream(mstream, buffersize, offset)
testdata = "sample data off the top of my head " * 1000
position = 0
lt = len(testdata)
for i in range(0, lt, writesize):
chunk = testdata[i:i + writesize]
bf.setBuffer(position, chunk)
position += 1
#print position,
lasti = i
#position = 0
print
bf = BufferFile.SetupFromExistingStream(mstream, offset)
#for i in range(0, lt, writesize):
for i in range(lasti, -1, -writesize):
position -= 1
chunksize = min(writesize, lt - i)
chunk = bf.getBuffer(position, chunksize)
#position += 1
#print position,
if chunk != testdata[i:i + writesize]:
raise ValueError, "chunks don't match %s!=%s" % (
repr(chunk), repr(testdata[i:i + writesize]))
print
def Load(this, buffer):
this.Clear()
indicator = ord(buffer[0])
this.isLeaf = (indicator == LEAF)
if indicator not in NONFREE:
raise BplusTreeException, "can't parse buffer not marked non-free"
index = 1
this.ChildBufferNumbers[0] = BufferFile.RetrieveLong(buffer, index)
index += BufferFile.LONGSTORAGE
maxLength = this.owner.KeyLength
maxKeyPayload = maxLength - BufferFile.SHORTSTORAGE
lastkey = ""
keys = this.ChildKeys
numbers = this.ChildBufferNumbers
for KeyIndex in xrange(this.Size):
KeyLength = BufferFile.RetrieveShort(buffer, index)
index += BufferFile.SHORTSTORAGE
key = None
if (KeyLength == 0):
key = ""
elif (KeyLength > 0):
section = buffer[index:index + KeyLength]
(key, nchars) = DECODER(section)
keys[KeyIndex] = key
index += maxKeyPayload
seekPosition = BufferFile.RetrieveLong(buffer, index)
numbers[KeyIndex + 1] = seekPosition
index += BufferFile.LONGSTORAGE
def makeHeader(this):
#return HEADERPREFIX+chr(VERSION)+BufferFile.StoreInt(
L = [
HEADERPREFIX,
chr(VERSION),
BufferFile.StoreInt(this.buffersize),
BufferFile.StoreLong(this.FreeListHead)
]
return string.join(L, "")
def makeHeader(this):
result = [HEADERPREFIX]
result.append(chr(VERSION))
result.append(BufferFile.StoreInt(this.NodeSize))
result.append(BufferFile.StoreInt(this.KeyLength))
result.append(BufferFile.StoreInt(INVARIANTCULTUREID))
result.append(BufferFile.StoreLong(this.rootSeek))
result.append(BufferFile.StoreLong(this.freeHeadSeek))
return string.join(result, "")
def readHeader(this):
f = this.fromfile
f.seek(this.seekStart)
buffer = f.read(this.headersize)
index = len(HEADERPREFIX)
prefix = buffer[:index]
if prefix != HEADERPREFIX:
raise LinkedFileException, "bad header prefix"
remainder = buffer[index + 1:]
this.buffersize = BufferFile.RetrieveInt(remainder, 0)
this.FreeListHead = BufferFile.RetrieveLong(remainder,
BufferFile.INTSTORAGE)
#print "<br>linkedfile readheader buffersize=", this.buffersize, "freehead=", this.FreeListHead
this.headerDirty = False
this.sanityCheck()
def SetupFromExistingStream(file, StartSeek=0):
result = LinkedFile(100, StartSeek)
result.buffers = BufferFile.SetupFromExistingStream(
file, StartSeek + result.headersize)
result.fromfile = file
result.readHeader()
return result
def InitializeLinkedFileInStream(file, buffersize, StartSeek=0):
result = LinkedFile(buffersize, StartSeek)
result.fromfile = file
result.setHeader()
result.buffers = BufferFile.InitializeBufferFileInStream(
file, buffersize + BUFFEROVERHEAD, StartSeek + result.headersize)
return result
def StoreNewChunk(this, fromString):
length = len(fromString)
#print "StoreNewChunk", length
currentBufferNumber = this.AllocateBuffer()
result = currentBufferNumber
CurrentBufferType = HEAD
# store header with length info
firstlength = min(length, this.buffersize - BufferFile.INTSTORAGE)
buffer = BufferFile.StoreInt(length) + fromString[:firstlength]
stored = firstlength
while stored < length:
# store intermediate or head
nextBufferNumber = this.AllocateBuffer()
this.SetBuffer(currentBufferNumber, CurrentBufferType, buffer,
nextBufferNumber)
nextlength = min(this.buffersize, length - stored)
nextstored = stored + nextlength
buffer = fromString[stored:nextstored]
stored = nextstored
currentBufferNumber = nextBufferNumber
CurrentBufferType = BODY
# store tail
this.SetBuffer(currentBufferNumber, CurrentBufferType, buffer,
NULLBUFFERPOINTER)
return result
def GetChunk(this, HeadBufferNumber):
(buffer, buffertype,
nextBufferNumber) = this.ParseBuffer(HeadBufferNumber)
if buffertype != HEAD:
raise LinkedFileException, "head buffer not marked head " + repr(
HeadBufferNumber)
length = BufferFile.RetrieveInt(buffer)
#print "getting", length
#print "buffer=", repr(buffer)
piece = buffer[BufferFile.INTSTORAGE:]
lengthToRead = length
pieces = []
while lengthToRead > 0:
lpiece = len(piece)
if lengthToRead < lpiece:
pieces.append(piece[:lengthToRead])
break
pieces.append(piece)
#print "read", repr(piece)
lengthToRead -= lpiece
#print "length to read", lengthToRead
if (lengthToRead > 0):
(piece, buffertype,
nextBufferNumber) = this.ParseBuffer(nextBufferNumber)
if buffertype != BODY:
raise LinkedFileException, "body buffer not marked body " + repr(
nextBufferNumber)
return string.join(pieces, "")
def SetBuffer(this, bufferNumber, type, thebuffer, NextBufferNumber):
#print "<br>setting", bufferNumber, type, repr(thebuffer), NextBufferNumber
if (this.buffersize < len(thebuffer)):
raise LinkedFileException, "too much data"
fullbuffer = (chr(type) +
BufferFile.StoreLong(NextBufferNumber)) + thebuffer
this.buffers.setBuffer(bufferNumber, fullbuffer)
def parseFreeBuffer(this, buffernumber):
freesize = 1 + BufferFile.LONGSTORAGE
buffer = this.buffers.getBuffer(buffernumber, freesize)
indicator = ord(buffer[0])
if indicator != FREE:
raise BplusTreeException, "free buffer not marked free " + repr(
buffernumber)
return BufferFile.RetrieveLong(buffer, 1)
def dump(this):
allbytes = []
keys = this.keys
values = this.values
for index in xrange(len(this.keys)):
thisKey = keys[index]
thisValue = this.values[index]
keyPrefix = BufferFile.StoreInt(len(thisKey))
(keybytes, dummy) = ENCODER(thisKey)
allbytes.append(keyPrefix)
allbytes.append(keybytes)
valuePrefix = BufferFile.StoreInt(len(thisValue))
(valuebytes, dummy) = ENCODER(thisValue)
allbytes.append(valuePrefix)
allbytes.append(valuebytes)
#print allbytes
return string.join(allbytes, "")
def deallocateBuffer(this, buffernumber):
#freesize = 1+BufferFile.LONGSTORAGE
buffer = this.buffers.getBuffer(buffernumber, 1)
#print "<br> DEALLOCATING", buffernumber, "<br>"
if ord(buffer) == FREE:
raise BplusTreeException, "attempt to free buffer marked free already"
buffer = chr(FREE) + BufferFile.StoreLong(this.freeHeadSeek)
this.buffers.setBuffer(buffernumber, buffer)
this.freeHeadSeek = buffernumber
def ParseBuffer(this, bufferNumber):
if bufferNumber < 0:
raise LinkedFileException, "buffer numbers cannot be negative " + repr(
bufferNumber)
thebuffer = this.buffers.getBuffer(bufferNumber)
typ = ord(thebuffer[0])
nextBufferNumber = BufferFile.RetrieveLong(thebuffer, 1)
realbuffer = thebuffer[BUFFEROVERHEAD:]
#print "<br>got", bufferNumber, typ, repr(realbuffer), nextBufferNumber
return (realbuffer, typ, nextBufferNumber)
def InitializeInStream(fromfile,
KeyLength,
NodeSize,
CultureId=INVARIANTCULTUREID,
StartSeek=0):
result = BplusTreeLong(fromfile, NodeSize, KeyLength, StartSeek, CultureId)
result.setHeader()
result.buffers = BufferFile.InitializeBufferFileInStream(
fromfile, result.buffersize, StartSeek + result.headersize)
return result
def readHeader(this):
f = this.fromfile
f.seek(this.seekStart)
header = f.read(this.headersize)
index = len(HEADERPREFIX)
prefix = header[:index]
if prefix != HEADERPREFIX:
raise BplusTreeException, "bad prefix %s should be %s" % (repr(
header[:index]), repr(HEADERPREFIX))
index += 1 # skip version
this.NodeSize = BufferFile.RetrieveInt(header, index)
index += BufferFile.INTSTORAGE
this.KeyLength = BufferFile.RetrieveInt(header, index)
index += BufferFile.INTSTORAGE
CultureId = BufferFile.RetrieveInt(header, index)
index += BufferFile.INTSTORAGE
this.rootSeek = BufferFile.RetrieveLong(header, index)
index += BufferFile.LONGSTORAGE
this.freeHeadSeek = BufferFile.RetrieveLong(header, index)
this.SanityCheck()
def SetupFromExistingStream(fromfile, StartSeek=0):
result = BplusTreeLong(fromfile, 33, 33, StartSeek)
result.readHeader()
result.buffers = BufferFile.SetupFromExistingStream(
fromfile, StartSeek + result.headersize)
if (result.buffers.buffersize != result.buffersize):
raise BplusTreeException, "inner and outer buffer sizes should match " + repr(
(result.buffers.buffersize, result.buffersize))
if result.rootSeek != NULLBUFFERNUMBER:
result.root = BplusNode(result, None, None, True)
result.root.LoadFromBuffer(result.rootSeek)
return result