def next(self):
if self.currentpos == self.fm.blockSize():
self.blk = BlockId(self.blk.fileName(), self.blk.number() - 1)
self.moveToBlock(self.blk)
rec = self.p.getBytes(self.currentpos)
self.currentpos += Integer.BYTES + len(rec)
return rec
def append(self, filename):
newblknum = self.length(filename)
blk = BlockId(filename, newblknum)
b = bytearray(self.blocksize)
try:
f = self.getFile(blk.fileName())
f.seek(blk.number() * self.blocksize)
f.write(b)
except IOError:
raise RuntimeError("cannot append block" + blk)
return blk
def __init__(self, p):
super(SetStringRecord, self).__init__()
tpos = Integer.BYTES
self.txnum = p.getInt(tpos)
fpos = tpos + Integer.BYTES
filename = p.getString(fpos)
bpos = fpos + Page.maxLength(len(filename))
blknum = p.getInt(bpos)
self.blk = BlockId(filename, blknum)
opos = bpos + Integer.BYTES
self.offset = p.getInt(opos)
vpos = opos + Integer.BYTES
self.val = p.getString(vpos)
def main(cls, args):
# db = SimpleDB("recoverytest", 400, 8)
cls.fm = FileMgr(File("recoverytest"), 400)
cls.lm = LogMgr(cls.fm, "simpledb.log")
cls.bm = BufferMgr(cls.fm, cls.lm, 8)
cls.blk0 = BlockId("testfile", 0)
cls.blk1 = BlockId("testfile", 1)
if cls.fm.length("testfile") == 0:
cls.initialize()
cls.modify()
else:
cls.recover()
def beforeFirst(self, searchkey):
self.close()
root = BTreeDir(self.tx, self.rootblk, self.dirLayout)
blknum = root.search(searchkey)
root.close()
leafblk = BlockId(self.leaftbl, blknum)
self.leaf = BTreeLeaf(self.tx, leafblk, self.leafLayout, searchkey)
def __init__(self, tx, idxname, leafLayout):
super(BTreeIndex, self).__init__()
self.tx = tx
# deal with the leaves
self.leaftbl = idxname + "leaf"
self.leafLayout = leafLayout
if tx.size(self.leaftbl) == 0:
blk = tx.append(self.leaftbl)
node = BTPage(tx, blk, leafLayout)
node.format(blk, -1)
# deal with the directory
dirsch = Schema()
dirsch.add("block", leafLayout.schema())
dirsch.add("dataval", leafLayout.schema())
dirtbl = idxname + "dir"
self.dirLayout = Layout(dirsch)
self.rootblk = BlockId(dirtbl, 0)
if tx.size(dirtbl) == 0:
# create new root block
tx.append(dirtbl)
node = BTPage(tx, self.rootblk, self.dirLayout)
node.format(self.rootblk, 0)
# insert initial directory entry
fldtype = dirsch.type("dataval")
minval = Constant(
Integer.MIN_VALUE) if fldtype == INTEGER else Constant("")
node.insertDir(0, minval, 0)
node.close()
class LogIterator:
#
# * Creates an iterator for the records in the log file,
# * positioned after the last log record.
#
def __init__(self, fm, blk):
self.fm = fm
self.blk = blk
self.currentpos = 0
self.boundary = 0
b = bytearray(fm.blockSize())
self.p = Page(b)
self.moveToBlock(blk)
#
# * Determines if the current log record
# * is the earliest record in the log file.
# * @return true if there is an earlier record
#
def hasNext(self):
return self.currentpos < self.fm.blockSize() or self.blk.number() > 0
#
# * Moves to the next log record in the block.
# * If there are no more log records in the block,
# * then move to the previous block
# * and return the log record from there.
# * @return the next earliest log record
#
def next(self):
if self.currentpos == self.fm.blockSize():
self.blk = BlockId(self.blk.fileName(), self.blk.number() - 1)
self.moveToBlock(self.blk)
rec = self.p.getBytes(self.currentpos)
self.currentpos += Integer.BYTES + len(rec)
return rec
#
# * Moves to the specified log block
# * and positions it at the first record in that block
# * (i.e., the most recent one).
#
def moveToBlock(self, blk):
self.fm.read(blk, self.p)
self.boundary = self.p.getInt(0)
self.currentpos = self.boundary
def main(cls, args):
# db = SimpleDB("buffermgrtest", 400, 3) # only 3 buffers
# bm = db.bufferMgr()
fm = FileMgr(File("buffertest"), 400)
lm = LogMgr(fm, "logfile")
bm = BufferMgr(fm, lm, 3)
buff = [None] * 6
buff[0] = bm.pin(BlockId("testfile", 0))
buff[1] = bm.pin(BlockId("testfile", 1))
buff[2] = bm.pin(BlockId("testfile", 2))
bm.unpin(buff[1])
buff[1] = None
buff[3] = bm.pin(BlockId("testfile", 0)) # block 0 pinned twice
buff[4] = bm.pin(BlockId("testfile", 1)) # block 1 repinned
print("Available buffers: " + str(bm.available()))
try:
print("Attempting to pin block 3...")
buff[5] = bm.pin(BlockId("testfile",
3)) # will not work; no buffers left
except BufferAbortException as e:
print("Exception: No available buffers\n")
bm.unpin(buff[2])
buff[2] = None
buff[5] = bm.pin(BlockId("testfile", 3)) # now this works
print("Final Buffer Allocation:")
for i in range(len(buff)):
b = buff[i]
if b is not None:
print("buff[" + str(i) + "] pinned to block " +
b.block().__str__())
def run(self):
try:
txB = Transaction(ConcurrencyTest.fm, ConcurrencyTest.lm, ConcurrencyTest.bm)
blk1 = BlockId("testfile", 1)
blk2 = BlockId("testfile", 2)
txB.pin(blk1)
txB.pin(blk2)
print("Tx B: request xlock 2")
txB.setInt(blk2, 0, 0, False)
print("Tx B: receive xlock 2")
time.sleep(1)
print("Tx B: request slock 1")
txB.getInt(blk1, 0)
print("Tx B: receive slock 1")
txB.commit()
print("Tx B: commit")
except InterruptedError as e:
pass
def tryOverflow(self):
firstkey = self.contents.getDataVal(0)
flag = self.contents.getFlag()
if not self.searchkey == firstkey or flag < 0:
return False
self.contents.close()
nextblk = BlockId(self.filename, flag)
self.contents = BTPage(self.tx, nextblk, self.layout)
self.currentslot = 0
return True
def __init__(self, tx, filename, layout, startbnum, endbnum):
super(ChunkScan, self).__init__()
self.tx = tx
self.filename = filename
self.layout = layout
self.startbnum = startbnum
self.endbnum = endbnum
self.buffs = []
for i in range(startbnum, endbnum + 1):
blk = BlockId(filename, i)
self.buffs.append(RecordPage(tx, blk, layout))
self.moveToBlock(startbnum)
def main(cls, args):
# db = SimpleDB("buffertest", 400, 3) # only 3 buffers
# bm = db.bufferMgr()
fm = FileMgr(File("buffertest"), 400)
lm = LogMgr(fm, "logfile")
bm = BufferMgr(fm, lm, 3)
buff1 = bm.pin(BlockId("testfile", 1))
p = buff1.contents()
n = p.getInt(80)
p.setInt(80, n + 1)
buff1.setModified(1, 0) # placeholder values
print("The new value is " + str(n + 1))
bm.unpin(buff1)
# One of these pins will flush buff1 to disk:
buff2 = bm.pin(BlockId("testfile", 2))
buff3 = bm.pin(BlockId("testfile", 3))
buff4 = bm.pin(BlockId("testfile", 4))
bm.unpin(buff2)
buff2 = bm.pin(BlockId("testfile", 1))
p2 = buff2.contents()
p2.setInt(80, 9999) # This modification
buff2.setModified(1, 0) # won't get written to disk.
def __init__(self, fm, logfile):
self.fm = fm
self.logfile = logfile
b = bytearray(fm.blockSize())
self.logpage = Page(b)
logsize = fm.length(logfile)
if logsize == 0:
self.currentblk = self.appendNewBlock()
else:
self.currentblk = BlockId(logfile, logsize - 1)
self.fm.read(self.currentblk, self.logpage)
self.latestLSN = 0
self.lastSavedLSN = 0
self.lock = threading.Lock() # 同步锁
def main(cls, args):
# db = SimpleDB("studentdb", 400, 8)
fm = FileMgr(File("recoverytest"), 400)
lm = LogMgr(fm, "simpledb.log")
filename = "simpledb.log"
lastblock = fm.length(filename) - 1
blk = BlockId(filename, lastblock)
p = Page(fm.blockSize())
fm.read(blk, p)
iterator = lm.iterator()
while iterator.hasNext():
byte_array = iterator.next()
rec = LogRecord.createLogRecord(byte_array)
print(rec)
def test3():
db = SimpleDB("studentdb")
lm = db.logMgr()
fm = db.fileMgr()
filename = "simpledb.log"
lastblock = fm.length(filename) - 1
blk = BlockId(filename, lastblock)
p = Page(fm.blockSize())
fm.read(blk, p)
iterator = lm.iterator()
while iterator.hasNext():
byte_array = iterator.next()
rec = LogRecord.createLogRecord(byte_array)
print(rec)
def main(cls, args):
# db = SimpleDB("filetest", 400, 8)
# fm = db.fileMgr()
fm = FileMgr(File("filetest"), 400)
blk = BlockId("testfile", 2)
pos1 = 88
p1 = Page(fm.blockSize())
p1.setString(pos1, "abcdefghijklm")
size = Page.maxLength(len("abcdefghijklm"))
pos2 = pos1 + size
p1.setInt(pos2, 345)
fm.write(blk, p1)
p2 = Page(fm.blockSize())
fm.read(blk, p2)
print("offset " + str(pos2) + " contains " + str(p2.getInt(pos2)))
print("offset " + str(pos1) + " contains " + p2.getString(pos1))
def main(cls, args):
# db = SimpleDB("txtest", 400, 8)
fm = FileMgr(File("txtest"), 400)
lm = LogMgr(fm, "simpledb.log")
bm = BufferMgr(fm, lm, 8)
tx1 = Transaction(fm, lm, bm)
blk = BlockId("testfile", 1)
tx1.pin(blk)
# The block initially contains unknown bytes,
# so don't log those values here.
tx1.setInt(blk, 80, 1, False)
tx1.setString(blk, 40, "one", False)
tx1.commit()
tx2 = Transaction(fm, lm, bm)
tx2.pin(blk)
ival = tx2.getInt(blk, 80)
sval = tx2.getString(blk, 40)
print("initial value at location 80 = " + str(ival))
print("initial value at location 40 = " + str(sval))
newival = ival + 1
newsval = sval + "!"
tx2.setInt(blk, 80, newival, True)
tx2.setString(blk, 40, newsval, True)
tx2.commit()
tx3 = Transaction(fm, lm, bm)
tx3.pin(blk)
print("new value at location 80 = " + str(tx3.getInt(blk, 80)))
print("new value at location 40 = " + tx3.getString(blk, 40))
tx3.setInt(blk, 80, 9999, True)
print("pre-rollback value at location 80 = " +
str(tx3.getInt(blk, 80)))
tx3.rollback()
tx4 = Transaction(fm, lm, bm)
tx4.pin(blk)
print("post-rollback at location 80 = " + str(tx4.getInt(blk, 80)))
tx4.commit()
def main(cls, args):
# db = SimpleDB("bufferfiletest", 400, 8)
# bm = db.bufferMgr()
fm = FileMgr(File("buffertest"), 400)
lm = LogMgr(fm, "logfile")
bm = BufferMgr(fm, lm, 8)
blk = BlockId("testfile", 2)
pos1 = 88
b1 = bm.pin(blk)
p1 = b1.contents()
p1.setString(pos1, "abcdefghijklm")
size = Page.maxLength(len("abcdefghijklm"))
pos2 = pos1 + size
p1.setInt(pos2, 345)
b1.setModified(1, 0)
bm.unpin(b1)
b2 = bm.pin(blk)
p2 = b2.contents()
print("offset " + str(pos2) + " contains " + str(p2.getInt(pos2)))
print("offset " + str(pos1) + " contains " + p2.getString(pos1))
bm.unpin(b2)
def findChildBlock(self, searchkey):
slot = self.contents.findSlotBefore(searchkey)
if self.contents.getDataVal(slot + 1) == searchkey:
slot += 1
blknum = self.contents.getChildNum(slot)
return BlockId(self.filename, blknum)
def append(self, filename):
dummyblk = BlockId(filename, self.END_OF_FILE)
self.concurMgr.xLock(dummyblk)
return self.fm.append(filename)
def size(self, filename):
dummyblk = BlockId(filename, self.END_OF_FILE)
self.concurMgr.sLock(dummyblk)
return self.fm.length(filename)
def close(self):
for i in range(len(self.buffs)):
blk = BlockId(self.filename, self.startbnum + i)
self.tx.unpin(blk)
def moveToBlock(self, blknum):
self.close()
blk = BlockId(self.filename, blknum)
self.rp = RecordPage(self.tx, blk, self.layout)
self.currentslot = -1
from simpledb.file.BlockId import BlockId
if __name__ == '__main__':
blk = BlockId("file", 1)
buffers = {blk: 1}
pins = [blk]
# pins.remove(blk)
if blk in pins:
print(1)
print(pins.count(blk))
# print(pins)
# print(buffers.get(blk))
class SetStringRecord(LogRecord):
#
# * Create a new setint log record.
# * @param bb the bytebuffer containing the log values
#
def __init__(self, p):
super(SetStringRecord, self).__init__()
tpos = Integer.BYTES
self.txnum = p.getInt(tpos)
fpos = tpos + Integer.BYTES
filename = p.getString(fpos)
bpos = fpos + Page.maxLength(len(filename))
blknum = p.getInt(bpos)
self.blk = BlockId(filename, blknum)
opos = bpos + Integer.BYTES
self.offset = p.getInt(opos)
vpos = opos + Integer.BYTES
self.val = p.getString(vpos)
def op(self):
return LogRecord.SETSTRING
def txNumber(self):
return self.txnum
def __str__(self):
return "<SETSTRING " + str(self.txnum) + " " + self.blk.__str__() + " " + str(self.offset) + " " \
+ str(self.val) + ">"
#
# * Replace the specified data value with the value saved in the log record.
# * The method pins a buffer to the specified block,
# * calls setInt to restore the saved value,
# * and unpins the buffer.
# * @see LogRecord#undo(int)
#
def undo(self, tx):
tx.pin(self.blk)
tx.setString(self.blk, self.offset, self.val,
False) # don't log the undo!
tx.unpin(self.blk)
#
# * A static method to write a setInt record to the log.
# * This log record contains the SETINT operator,
# * followed by the transaction id, the filename, number,
# * and offset of the modified block, and the previous
# * integer value at that offset.
# * @return the LSN of the last log value
#
@staticmethod
def writeToLog(lm, txnum, blk, offset, val):
tpos = Integer.BYTES
fpos = tpos + Integer.BYTES
bpos = fpos + Page.maxLength(len(blk.fileName()))
opos = bpos + Integer.BYTES
vpos = opos + Integer.BYTES
reclen = vpos + Page.maxLength(len(val))
rec = bytearray(reclen)
p = Page(rec)
p.setInt(0, LogRecord.SETSTRING)
p.setInt(tpos, txnum)
p.setString(fpos, blk.fileName())
p.setInt(bpos, blk.number())
p.setInt(opos, offset)
p.setString(vpos, val)
return lm.append(rec)
def moveToRid(self, rid):
self.close()
blk = BlockId(self.filename, rid.blockNumber())
self.rp = RecordPage(self.tx, blk, self.layout)
self.currentslot = rid.slot()
