class XidLruCache: PAGES_PER_SEGMENT = 32 def __init__(self, datadir, pageclass, buffsz): self.datadir = datadir ctrlfile = ControlFile(datadir) self.blcksz = ctrlfile.blcksz self.buffer = LruCache(pageclass, self.blcksz, buffsz) def getfilename(self, segno): pass def readfromdisk(self, pageno): blocksz = self.blcksz segno = pageno / self.PAGES_PER_SEGMENT filename = self.getfilename(segno) try: with open(filename) as file: file.seek(blocksz * pageno) block = file.read(blocksz) except: raise UPgException('error in reading xid') if len(block) != blocksz: raise UPgException('error in reading xid') return block def getlrupage(self, pageno): lrupage = self.buffer.get_and_visit(pageno) if not lrupage: block = self.readfromdisk(pageno) lrupage = self.buffer.put(pageno, block) return lrupage
class HeapBuffer: def __init__(self, pgdatadir, catalog_class): self.pgdatadir = pgdatadir self.catalog_class = catalog_class ctrlfile = ControlFile(pgdatadir) self.blocksz = ctrlfile.blcksz self.segfilesz = ctrlfile.relseg_size self.catalog_version_no = ctrlfile.catalog_version_no self.pg_majorversion = catalog_class.loadpgversion(pgdatadir) self.heapbuffer = LruCache(HeapBufferPage, self.blocksz, HEAP_BUFFER_COUNT) ''' 计算buffer tag。 ''' def __getbuftag(self, relfilenode, forknum, blocknum): return struct.pack('5I', relfilenode.space_node, relfilenode.db_node, relfilenode.rel_node, forknum, blocknum) ''' 根据relfilenode得到文件名。根据global tablespace,default table space,还有指定了table space,文件名 规则不一样。 ''' def getrelationpath(self, relfilenode, forknum): assert forknum == MAIN_FORKNUM or forknum == FSM_FORKNUM \ or forknum == VISIBILITYMAP_FORKNUM or forknum == INIT_FORKNUM pgdatadir = self.pgdatadir forkNames = ('', 'fsm', 'vm', 'init') if forknum == MAIN_FORKNUM: filename = relfilenode.rel_node else: filename = '%u_%s' % (relfilenode.rel_node, forkNames[forknum]) if relfilenode.space_node == self.catalog_class.GLOBALTABLESPACE_OID: return '%s/global/%s' % (pgdatadir, filename) elif relfilenode.space_node == self.catalog_class.DEFAULTTABLESPACE_OID: return '%s/base/%u/%s' % (pgdatadir, relfilenode.db_node, filename) else: tablespacedir = "PG_%s_%u" % (self.pg_majorversion, self.catalog_version_no) return '%s/pg_tblspc/%u/%s/%u/%s' % ( pgdatadir, relfilenode.space_node, tablespacedir, relfilenode.db_node, filename) def __loadbuffer(self, relfilenode, forknum, blocknum): try: blocksz = self.blocksz segsz = self.segfilesz filepath = self.getrelationpath(relfilenode, forknum) segno = blocknum / segsz if segno > 0: filepath = '%s.%u' % (filepath, segno) blockoff = blocksz * (blocknum % segsz) with open(filepath) as file: file.seek(blockoff) block = file.read(blocksz) if len(block) != blocksz: logger.error('could not read block %u in file \"%s\": %m' % (blocknum, filepath)) raise UPgException('could not read block in file') return block except IOError: logger.error('could not read block %u in file \"%s\": %m' % (blocknum, filepath)) raise UPgException('could not read block in file') def readpage(self, relfilenode, forknum, blocknum): tag = self.__getbuftag(relfilenode, forknum, blocknum) buffpage = self.heapbuffer.get_and_visit(tag) if not buffpage: buffdata = self.__loadbuffer(relfilenode, forknum, blocknum) buffpage = self.heapbuffer.put(tag, buffdata) return buffpage def getblocknums(self, relfilenode, forknum): blocksz = self.blocksz segsz = self.segfilesz filepath = self.getrelationpath(relfilenode, forknum) segno = 0 while True: if segno > 0: filepath = '%s.%u' % (filepath, segno) if not os.path.exists(filepath): return segno * segsz filesize = os.path.getsize(filepath) / blocksz if filesize > segsz: raise UPgException('could not read block in file') elif filesize < segsz: return segno * segsz + filesize else: segno += 1
class HeapBuffer: def __init__(self, pgdatadir, catalog_class): self.pgdatadir = pgdatadir self.catalog_class = catalog_class ctrlfile = ControlFile(pgdatadir) self.blocksz = ctrlfile.blcksz self.segfilesz = ctrlfile.relseg_size self.catalog_version_no = ctrlfile.catalog_version_no self.pg_majorversion = catalog_class.loadpgversion(pgdatadir) self.heapbuffer = LruCache(HeapBufferPage, self.blocksz, HEAP_BUFFER_COUNT) ''' 计算buffer tag。 ''' def __getbuftag(self, relfilenode, forknum, blocknum): return struct.pack('5I', relfilenode.space_node, relfilenode.db_node, relfilenode.rel_node, forknum, blocknum) ''' 根据relfilenode得到文件名。根据global tablespace,default table space,还有指定了table space,文件名 规则不一样。 ''' def getrelationpath(self, relfilenode, forknum): assert forknum == MAIN_FORKNUM or forknum == FSM_FORKNUM \ or forknum == VISIBILITYMAP_FORKNUM or forknum == INIT_FORKNUM pgdatadir = self.pgdatadir forkNames = ('', 'fsm', 'vm', 'init') if forknum == MAIN_FORKNUM: filename = relfilenode.rel_node else: filename = '%u_%s'%(relfilenode.rel_node, forkNames[forknum]) if relfilenode.space_node == self.catalog_class.GLOBALTABLESPACE_OID: return '%s/global/%s'%(pgdatadir, filename) elif relfilenode.space_node == self.catalog_class.DEFAULTTABLESPACE_OID: return '%s/base/%u/%s'%(pgdatadir, relfilenode.db_node, filename) else: tablespacedir = "PG_%s_%u"%(self.pg_majorversion, self.catalog_version_no) return '%s/pg_tblspc/%u/%s/%u/%s'%(pgdatadir, relfilenode.space_node, tablespacedir, relfilenode.db_node, filename) def __loadbuffer(self, relfilenode, forknum, blocknum): try: blocksz = self.blocksz segsz = self.segfilesz filepath = self.getrelationpath(relfilenode, forknum) segno = blocknum/segsz if segno > 0: filepath = '%s.%u'%(filepath, segno) blockoff = blocksz * (blocknum%segsz) with open(filepath) as file: file.seek(blockoff) block = file.read(blocksz) if len(block) != blocksz: logger.error('could not read block %u in file \"%s\": %m'%(blocknum, filepath)) raise UPgException('could not read block in file') return block except IOError: logger.error('could not read block %u in file \"%s\": %m'%(blocknum, filepath)) raise UPgException('could not read block in file') def readpage(self, relfilenode, forknum, blocknum): tag = self.__getbuftag(relfilenode, forknum, blocknum) buffpage = self.heapbuffer.get_and_visit(tag) if not buffpage: buffdata = self.__loadbuffer(relfilenode, forknum, blocknum) buffpage = self.heapbuffer.put(tag, buffdata) return buffpage def getblocknums(self, relfilenode, forknum): blocksz = self.blocksz segsz = self.segfilesz filepath = self.getrelationpath(relfilenode, forknum) segno = 0 while True: if segno > 0: filepath = '%s.%u' % (filepath, segno) if not os.path.exists(filepath): return segno * segsz filesize = os.path.getsize(filepath)/blocksz if filesize > segsz: raise UPgException('could not read block in file') elif filesize < segsz: return segno * segsz + filesize else: segno += 1
class LruRepl(cmd.Cmd): """LruRepl: The LruCache shell (Read Eval Print loop) """ intro = "Welcome to LruCache shell.\n\nType help or ? to list commands.\nCtrl-d or exit to exit.\n" prompt = "lrucache> " cache = None def precmd(self, line): """accept command as upper or lower case""" line = line.lower() return line def default(self, line): """overrides the default error message.""" print 'ERROR' return False def command_preproc(self, line, expected_len): if self.cache == None: print "Cache not initialized. Use SIZE command first." return None args = line.split() if len(args) != expected_len: print "Syntax Error. Expected %d arguments, got %d" % ( expected_len, len(args)) return None return args def do_size(self, line): """size n - sets the maximum size of the cache to n objects""" if self.cache == None: args = line.split() if len(args) != 1: print "Syntax Error. Expected 1 arguments, got %d" % len(args) return False self.cache = LruCache(int(args[0])) print("SIZE OK") else: print("ERROR") def do_set(self, line): """set key value - puts the key/value pair into the cache if the key does not exist or overwrites the value of an existing pair """ args = self.command_preproc(line, 2) if args != None: self.cache.put(args[0], args[1]) print("SET OK") def do_get(self, line): """get key -- prints the value associated the the key if found""" args = self.command_preproc(line, 1) if args != None: v = self.cache.fetch(args[0]) if v == None: print("NOTFOUND") else: print("GOT %s" % v) def do_stats(self, line): """stats - print the cache stats""" if self.cache == None: print "Cache not initialized. Use SIZE command first." return False print("Cache max size: %d" % self.cache.max_size) print("Current size: %d" % self.cache.current_size) def do_dump(self, line): """dump - dump the cache""" if self.cache == None: print "Cache not initialized. Use SIZE command first." return False self.cache.data.dump() def do_eof(self, line): """Ctrl-D to exit""" print '\n' return True def do_exit(self, line): """exits shell""" print "\n" return True