def Load(this, serialization): keys = this.keys values = this.values if len(keys) > 0: raise BplusTreeException, "cannot load into non-empty bucket" index = 0 bytecount = len(serialization) while index < bytecount: keylength = BufferFile.RetrieveInt(serialization, index) index += BufferFile.INTSTORAGE nextindex = index + keylength keybytes = serialization[index:nextindex] index = nextindex (key, dummy) = DECODER(keybytes) valuelength = BufferFile.RetrieveInt(serialization, index) index += BufferFile.INTSTORAGE nextindex = index + valuelength valuebytes = serialization[index:nextindex] (value, dummy) = DECODER(valuebytes) index = nextindex keys.append(key) values.append(value) if index != bytecount: raise BplusTreeException, "error counting bytes " + repr( (index, bytecount))
def Dump(this): L = [] a = L.append if this.isLeaf: a(chr(LEAF)) else: a(chr(NONLEAF)) keys = this.ChildKeys numbers = this.ChildBufferNumbers #print "<br>keys", keys #print "<br>numbers", numbers, "<br>" maxLength = this.owner.KeyLength maxKeyPayload = maxLength - BufferFile.SHORTSTORAGE a(BufferFile.StoreLong(numbers[0])) for KeyIndex in xrange(this.Size): # store the key theKey = keys[KeyIndex] if theKey is None: a(BufferFile.StoreShort(-1)) a("X" * maxKeyPayload) else: (coded, nchars) = ENCODER(theKey) lcoded = len(coded) if (lcoded > maxKeyPayload): raise BplusTreeException, "too many bytes in key " + repr( coded) a(BufferFile.StoreShort(lcoded)) padded = coded + "X" * (maxKeyPayload - lcoded) a(padded) # store the seek a(BufferFile.StoreLong(numbers[KeyIndex + 1])) result = string.join(L, "") #print "<br>", this.MyBufferNumber, "raw dump", repr(result), "<br>" return result
def testBufferFile(): print "testing buffer file" buffersize = 17 writesize = 10 mstream = newFile("pyBufferFile.dat") offset = 55 bf = BufferFile.InitializeBufferFileInStream(mstream, buffersize, offset) testdata = "sample data off the top of my head " * 1000 position = 0 lt = len(testdata) for i in range(0, lt, writesize): chunk = testdata[i:i + writesize] bf.setBuffer(position, chunk) position += 1 #print position, lasti = i #position = 0 print bf = BufferFile.SetupFromExistingStream(mstream, offset) #for i in range(0, lt, writesize): for i in range(lasti, -1, -writesize): position -= 1 chunksize = min(writesize, lt - i) chunk = bf.getBuffer(position, chunksize) #position += 1 #print position, if chunk != testdata[i:i + writesize]: raise ValueError, "chunks don't match %s!=%s" % ( repr(chunk), repr(testdata[i:i + writesize])) print
def Load(this, buffer): this.Clear() indicator = ord(buffer[0]) this.isLeaf = (indicator == LEAF) if indicator not in NONFREE: raise BplusTreeException, "can't parse buffer not marked non-free" index = 1 this.ChildBufferNumbers[0] = BufferFile.RetrieveLong(buffer, index) index += BufferFile.LONGSTORAGE maxLength = this.owner.KeyLength maxKeyPayload = maxLength - BufferFile.SHORTSTORAGE lastkey = "" keys = this.ChildKeys numbers = this.ChildBufferNumbers for KeyIndex in xrange(this.Size): KeyLength = BufferFile.RetrieveShort(buffer, index) index += BufferFile.SHORTSTORAGE key = None if (KeyLength == 0): key = "" elif (KeyLength > 0): section = buffer[index:index + KeyLength] (key, nchars) = DECODER(section) keys[KeyIndex] = key index += maxKeyPayload seekPosition = BufferFile.RetrieveLong(buffer, index) numbers[KeyIndex + 1] = seekPosition index += BufferFile.LONGSTORAGE
def makeHeader(this): #return HEADERPREFIX+chr(VERSION)+BufferFile.StoreInt( L = [ HEADERPREFIX, chr(VERSION), BufferFile.StoreInt(this.buffersize), BufferFile.StoreLong(this.FreeListHead) ] return string.join(L, "")
def makeHeader(this): result = [HEADERPREFIX] result.append(chr(VERSION)) result.append(BufferFile.StoreInt(this.NodeSize)) result.append(BufferFile.StoreInt(this.KeyLength)) result.append(BufferFile.StoreInt(INVARIANTCULTUREID)) result.append(BufferFile.StoreLong(this.rootSeek)) result.append(BufferFile.StoreLong(this.freeHeadSeek)) return string.join(result, "")
def readHeader(this): f = this.fromfile f.seek(this.seekStart) buffer = f.read(this.headersize) index = len(HEADERPREFIX) prefix = buffer[:index] if prefix != HEADERPREFIX: raise LinkedFileException, "bad header prefix" remainder = buffer[index + 1:] this.buffersize = BufferFile.RetrieveInt(remainder, 0) this.FreeListHead = BufferFile.RetrieveLong(remainder, BufferFile.INTSTORAGE) #print "<br>linkedfile readheader buffersize=", this.buffersize, "freehead=", this.FreeListHead this.headerDirty = False this.sanityCheck()
def SetupFromExistingStream(file, StartSeek=0): result = LinkedFile(100, StartSeek) result.buffers = BufferFile.SetupFromExistingStream( file, StartSeek + result.headersize) result.fromfile = file result.readHeader() return result
def InitializeLinkedFileInStream(file, buffersize, StartSeek=0): result = LinkedFile(buffersize, StartSeek) result.fromfile = file result.setHeader() result.buffers = BufferFile.InitializeBufferFileInStream( file, buffersize + BUFFEROVERHEAD, StartSeek + result.headersize) return result
def StoreNewChunk(this, fromString): length = len(fromString) #print "StoreNewChunk", length currentBufferNumber = this.AllocateBuffer() result = currentBufferNumber CurrentBufferType = HEAD # store header with length info firstlength = min(length, this.buffersize - BufferFile.INTSTORAGE) buffer = BufferFile.StoreInt(length) + fromString[:firstlength] stored = firstlength while stored < length: # store intermediate or head nextBufferNumber = this.AllocateBuffer() this.SetBuffer(currentBufferNumber, CurrentBufferType, buffer, nextBufferNumber) nextlength = min(this.buffersize, length - stored) nextstored = stored + nextlength buffer = fromString[stored:nextstored] stored = nextstored currentBufferNumber = nextBufferNumber CurrentBufferType = BODY # store tail this.SetBuffer(currentBufferNumber, CurrentBufferType, buffer, NULLBUFFERPOINTER) return result
def GetChunk(this, HeadBufferNumber): (buffer, buffertype, nextBufferNumber) = this.ParseBuffer(HeadBufferNumber) if buffertype != HEAD: raise LinkedFileException, "head buffer not marked head " + repr( HeadBufferNumber) length = BufferFile.RetrieveInt(buffer) #print "getting", length #print "buffer=", repr(buffer) piece = buffer[BufferFile.INTSTORAGE:] lengthToRead = length pieces = [] while lengthToRead > 0: lpiece = len(piece) if lengthToRead < lpiece: pieces.append(piece[:lengthToRead]) break pieces.append(piece) #print "read", repr(piece) lengthToRead -= lpiece #print "length to read", lengthToRead if (lengthToRead > 0): (piece, buffertype, nextBufferNumber) = this.ParseBuffer(nextBufferNumber) if buffertype != BODY: raise LinkedFileException, "body buffer not marked body " + repr( nextBufferNumber) return string.join(pieces, "")
def SetBuffer(this, bufferNumber, type, thebuffer, NextBufferNumber): #print "<br>setting", bufferNumber, type, repr(thebuffer), NextBufferNumber if (this.buffersize < len(thebuffer)): raise LinkedFileException, "too much data" fullbuffer = (chr(type) + BufferFile.StoreLong(NextBufferNumber)) + thebuffer this.buffers.setBuffer(bufferNumber, fullbuffer)
def parseFreeBuffer(this, buffernumber): freesize = 1 + BufferFile.LONGSTORAGE buffer = this.buffers.getBuffer(buffernumber, freesize) indicator = ord(buffer[0]) if indicator != FREE: raise BplusTreeException, "free buffer not marked free " + repr( buffernumber) return BufferFile.RetrieveLong(buffer, 1)
def dump(this): allbytes = [] keys = this.keys values = this.values for index in xrange(len(this.keys)): thisKey = keys[index] thisValue = this.values[index] keyPrefix = BufferFile.StoreInt(len(thisKey)) (keybytes, dummy) = ENCODER(thisKey) allbytes.append(keyPrefix) allbytes.append(keybytes) valuePrefix = BufferFile.StoreInt(len(thisValue)) (valuebytes, dummy) = ENCODER(thisValue) allbytes.append(valuePrefix) allbytes.append(valuebytes) #print allbytes return string.join(allbytes, "")
def deallocateBuffer(this, buffernumber): #freesize = 1+BufferFile.LONGSTORAGE buffer = this.buffers.getBuffer(buffernumber, 1) #print "<br> DEALLOCATING", buffernumber, "<br>" if ord(buffer) == FREE: raise BplusTreeException, "attempt to free buffer marked free already" buffer = chr(FREE) + BufferFile.StoreLong(this.freeHeadSeek) this.buffers.setBuffer(buffernumber, buffer) this.freeHeadSeek = buffernumber
def ParseBuffer(this, bufferNumber): if bufferNumber < 0: raise LinkedFileException, "buffer numbers cannot be negative " + repr( bufferNumber) thebuffer = this.buffers.getBuffer(bufferNumber) typ = ord(thebuffer[0]) nextBufferNumber = BufferFile.RetrieveLong(thebuffer, 1) realbuffer = thebuffer[BUFFEROVERHEAD:] #print "<br>got", bufferNumber, typ, repr(realbuffer), nextBufferNumber return (realbuffer, typ, nextBufferNumber)
def InitializeInStream(fromfile, KeyLength, NodeSize, CultureId=INVARIANTCULTUREID, StartSeek=0): result = BplusTreeLong(fromfile, NodeSize, KeyLength, StartSeek, CultureId) result.setHeader() result.buffers = BufferFile.InitializeBufferFileInStream( fromfile, result.buffersize, StartSeek + result.headersize) return result
def readHeader(this): f = this.fromfile f.seek(this.seekStart) header = f.read(this.headersize) index = len(HEADERPREFIX) prefix = header[:index] if prefix != HEADERPREFIX: raise BplusTreeException, "bad prefix %s should be %s" % (repr( header[:index]), repr(HEADERPREFIX)) index += 1 # skip version this.NodeSize = BufferFile.RetrieveInt(header, index) index += BufferFile.INTSTORAGE this.KeyLength = BufferFile.RetrieveInt(header, index) index += BufferFile.INTSTORAGE CultureId = BufferFile.RetrieveInt(header, index) index += BufferFile.INTSTORAGE this.rootSeek = BufferFile.RetrieveLong(header, index) index += BufferFile.LONGSTORAGE this.freeHeadSeek = BufferFile.RetrieveLong(header, index) this.SanityCheck()
def SetupFromExistingStream(fromfile, StartSeek=0): result = BplusTreeLong(fromfile, 33, 33, StartSeek) result.readHeader() result.buffers = BufferFile.SetupFromExistingStream( fromfile, StartSeek + result.headersize) if (result.buffers.buffersize != result.buffersize): raise BplusTreeException, "inner and outer buffer sizes should match " + repr( (result.buffers.buffersize, result.buffersize)) if result.rootSeek != NULLBUFFERNUMBER: result.root = BplusNode(result, None, None, True) result.root.LoadFromBuffer(result.rootSeek) return result