class Queue(object): """ Simple queue with PyDbLite backend. """ def __init__(self, queue_type=0, config=None): if config is not None: # copy config pass else: self.commit = False self.db_filename = "/tmp/queue.pydb" self.mode = "override" self.queue = Base(self.db_filename) def create_queue(self): self.queue.create('id', 'item', mode = self.mode) self.queue.create_index('id') return None def push(self, item): self.queue.insert(self.length(), item) return None def pop(self): if not self.is_empty(): id = self.length() - 1 r = self.queue._id[id] self.queue.delete(r) return r else: return None def list(self): return self.queue.records def length(self): return len(self.queue) def is_empty(self): return self.length() == 0 def commit(self): if self.commit is True: self.queue.commit() return None
class ArticleDB: """class for persistent storage of articles. what is stored from each Article object is defined in Article.TO_SAVE """ def __init__(self, dbfile, mode = 'open', autocommit = False): self.db = Base(dbfile) self.db.create(*Article.TO_SAVE, **{'mode': mode}) self.db.create_index(*INDEX_ON) self.autocommit = autocommit def insertArticle(self, art): """insert article into database, with id consistency check""" present = [] if art.id_int != None: present.extend(self.db._id_int[art.id_int]) # if art.idZBL != None: # present.extend(self.db._idZBL[art.idZBL]) # if art.idMR != None: # present.extend(self.db._idMR[art.idMR]) ids = list(set([rec['__id__'] for rec in present])) # unique ids present = [self.db[id] for id in ids] # remove duplicate identical entries (coming from matches on more than one id on the same article) new = art for old in present: # FIXME HACK turns off consistency checking try: new.mergeWith(Article.Article(record = old)) # article already present in database -- check if ids are consistent, update it with new info from art except Exception, e: # logging.error('inconsistent database contents (%i overlapping records); leaving database unchanged' % (len(present))) #logging.info('inconsistency between \n%s\n%s' % (new, Article.Article(old))) logging.warning('inconsistency between %s and %s' % (new, Article.Article(old))) # return False if len(present) == 0: # logging.debug('inserting a new article') pass else: # logging.debug('replacing %i old (consistent) record(s) for %s' % (len(present), new)) pass self.db.delete(present) id = self.db.insert(**new.__dict__) if self.autocommit: self.commit() return True
"""Create or open the page database Save this file as wikiBase.py to use a PyDBLite database """ import os from PyDbLite import Base db = Base(os.path.join(CONFIG.data_dir,'pages.pdl')) db.create('name','content','admin','nbvisits','created', 'version','lastmodif',mode="open") db.create_index('name')
"""Create or open the page database """ from PyDbLite import Base db = Base('pages.pdl').create('name', 'content', 'admin', 'nbvisits', 'created', 'version', 'lastmodif', mode="open") db.create_index('name')
for i in range(ln): res += random.choice(string.letters) return res def sentence(n, m): ln = random.randint(1, n) res = [] for i in range(ln): res.append(word(m)) return " ".join(res) os.remove("blog") db = Base("blog").create("parent", "title", "author", "text", "date") db.create_index("parent") nbthreads = 200 for i in range(nbthreads): # generate thread author = "pierre" title = sentence(10, 10) text = sentence(100, 10) date = datetime.datetime( random.randint(2004, 2006), random.randint(1, 12), random.randint(1, 28), random.randint(0, 23), random.randint(0, 59), random.randint(0, 59), )
def word(m): res = '' ln = random.randint(1,m) for i in range(ln): res += random.choice(string.letters) return res def sentence(n,m): ln = random.randint(1,n) res = [] for i in range(ln): res.append(word(m)) return ' '.join(res) db = Base('blog.pdl').create('parent','title','author','text','date',mode="override") db.create_index('parent') nbthreads = 200 for i in range(nbthreads): # generate thread author = 'pierre' title = sentence(10,10) text = sentence(100,10) date = datetime.datetime(random.randint(2006,2008),random.randint(1,12), random.randint(1,28),random.randint(0,23),random.randint(0,59), random.randint(0,59)) thread_id = db.insert(parent=-1,author=author,title=title,text=text,date=date) # generate comments nbcomments = random.randint(0,5) for i in range(nbcomments):
for i in range(ln): res += random.choice(string.letters) return res def sentence(n, m): ln = random.randint(1, n) res = [] for i in range(ln): res.append(word(m)) return ' '.join(res) os.remove('blog') db = Base('blog').create('parent', 'title', 'author', 'text', 'date') db.create_index('parent') nbthreads = 200 for i in range(nbthreads): # generate thread author = 'pierre' title = sentence(10, 10) text = sentence(100, 10) date = datetime.datetime(random.randint(2004, 2006), random.randint(1, 12), random.randint(1, 28), random.randint(0, 23), random.randint(0, 59), random.randint(0, 59)) thread_id = db.insert(parent=-1, author=author, title=title, text=text, date=date)
class TagOrganizer(Organizer): def __init__(self, cache, category=None): self.tags = None self.category = category Organizer.__init__(self, cache, False) def reset(self): if not self.tags: self.tags = Base(DB_FILE_TAGS) self.tags.create('realpath', 'category', 'tag', mode = 'override') self.tags.create_index('realpath') self.tags.create_index('category') Organizer.reset(self) def updatecache(self): self._generatetags() Organizer.updatecache(self) def _deletefromcache(self, path): realpath = self.realpath(path) logger.debug("_deletefromcache(%s)" % realpath) for tag in self.tags.get_index('realpath')[realpath]: self.tags.delete(tag) def deletefromcache(self, path): self._deletefromcache(path) Organizer.deletefromcache(self, path) def addtocache(self, path): self._deletefromcache(path) self.generatetags(self.realpath(path)) Organizer.addtocache(self, path) def generatepaths(self, realpath): for record in self.tags.get_index('realpath')[realpath]: yield os.path.join(os.sep, record['tag'], os.path.basename(realpath)) def dirlist(self, path): if path == '/': return self.taglist(self.category) else: return [] ############################################ # Tag functions def _generatetags(self): for filename in filter(util.ignoretag, #IGNORE:W0141 self.cache.filelist()): self.generatetags(filename) def generatetags(self, filename): pass def tag(self, realpath, category, tag): logger.debug('tag(%s, %s, %s)' % (realpath, category, tag)) if not tag == None and not tag == '': self.tags.insert(realpath, category, tag) def filelistbytags(self, category, tags): self.refreshcache() for record in self.tags.get_index('category')[category]: if record['tag'] in tags: yield os.path.basename(record['realpath']) def taglist(self, category): self.refreshcache() return util.unique([record['tag'] for record in self.tags.get_index('category')[category]])
class Organizer(Cacheable): """ This is the base class for organizers """ def __init__(self, cache, recursive=True): Cacheable.__init__(self) self.cache = cache self.recursive = recursive self.transformed = None # Do not call reset here, it is called from fs.py when the fs is # already started def reset(self): if not self.transformed: self.transformed = Base(DB_TRANSFORMED) self.transformed.create('realpath', 'path', 'dirname', mode='override') self.transformed.create_index('realpath') self.transformed.create_index('path') self.transformed.create_index('dirname') self.cache.reset() Cacheable.reset(self) def updatecache(self): self.generateallpaths() def deletefromcache(self, path): realpath = self.realpath(path) logger.debug("deletefromcache(%s)" % realpath) self.cache.deletefromcache(realpath) for item in self.transformed.get_index('realpath')[realpath]: self.transformed.delete(item) def addtocache(self, path): if not self.transformed.get_index('path')[path]: realpath = self.realpath(path) self.cache.addtocache(realpath) self.addfile(realpath) ############################################ # Overwritable functions def dirlist(self, path): #IGNORE:W0613 """ Returns a list of (non-existent, generated, virtual) directories for a given path. Default implementation. """ return [] def generatepaths(self, realpath): """ Generates paths for a given real path. A file can have more than one transformed path. Default implementation. """ yield util.addtrailingslash(util.removeroot(realpath, self.cache.filter.root)) def generaterealpath(self, path): """ Generates a real path for a inexistent path. Default implementation. """ return os.path.join(self.cache.filter.root, path[1:]) ############################################ # General functions def generateallpaths(self): """ Generates paths for all the files given by the cache and stores them in self.transformed """ for realpath in self.cache.filelist(): if self.recursive: # Add all sub-directories first currentpath = self.cache.filter.root for pathpart in util.pathparts(util.removeroot(realpath, self.cache.filter.root)): currentpath = os.path.join(currentpath, pathpart) self.addfile(currentpath) else: self.addfile(realpath) def addfile(self, realpath): """ Stores a file in self.transformed if not there already and returns the paths for that file in the proxy file system """ logger.debug('addfile(%s)' % realpath) if not util.ignoretag(util.removeroot(realpath, self.cache.filter.root)): return [] self.refreshcache() transformed = self.transformed.get_index('realpath')[realpath] if transformed: return (record['path'] for record in transformed) else: paths = [] for path in self.paths(realpath): while self.transformed.get_index('path')[path]: path = self.increasefilename(path) dirname = os.path.dirname(path) logger.debug('addfile(%s, %s, %s)' % (realpath, path, dirname)) self.transformed.insert(realpath=realpath, path=path, dirname=dirname) paths.append(path) return paths def increasefilename(self, filename): """ Returns a new filename in sequence. Called if the current filename already exists. This default implementation adds a "(1)" to the end if not present or increases that number by one. """ root, ext = os.path.splitext(filename) num = 1 matches = _INCREASE_REGEX.match(root) if not matches is None: num = int(matches.group(2)) + 1 filename = matches.group(1) return '%s(%i)%s' % (root, num, ext) ############################################ # General functions that read the cache def filelist(self, path): """ Returns a list of directories and filenames in a list from cache """ logger.debug('filelist(%s)' % path) self.refreshcache() for dirname in self.dirlist(path): yield dirname for record in self.transformed.get_index('dirname')[path]: yield os.path.basename(record['path']) def paths(self, realpath): """ Generates or returns paths from cache for a given real path """ self.refreshcache() paths = self.transformed.get_index('realpath')[realpath] if paths: return (path['path'] for path in paths) else: return (path for path in self.generatepaths(realpath)) def realpath(self, path): """ Returns the real path for a file given the path in the file system. """ logger.debug('realpath(%s)' % path) self.refreshcache() realpaths = [r['realpath'] for r in self.transformed.get_index('path')[path]] realpath = None if realpaths: realpath = realpaths[0] elif path == '/': realpath = self.cache.filter.root elif path == util.addtrailingslash(util.ORIGINAL_DIR): realpath = '.' elif util.isspecial(path, 'original', True): realpath = os.path.join('.', os.sep.join(util.pathparts(path)[2:])) elif util.isspecial(path, 'root', True): realpath = os.path.join(self.cache.filter.root, os.sep.join(util.pathparts(path)[2:])) elif util.isspecial(path, 'commands'): realpath = '.' elif util.iscommand(path): realpath = getserver().tempfile.name else: realpath = self.generaterealpath(path) logger.debug('realpath(%s) = %s' % (path, realpath)) return realpath ############################################ # File system functions def getattr(self, path): dirname = os.path.dirname(path) if util.removeroot(path, os.sep) in self.dirlist(dirname): return self.cache.getattr(self.realpath(dirname)) else: return self.cache.getattr(self.realpath(path)) def readdir(self, path, offset): #IGNORE:W0613 for filename in util.getbasefilelist(): yield fuse.Direntry(filename) for filename in self._filelist(path): yield fuse.Direntry(filename) def _filelist(self, path): filelist = [] if path == util.addtrailingslash(util.ORIGINAL_DIR): filelist = ['original', 'root', 'commands'] elif util.isspecial(path, 'root', True): filelist = os.listdir(self.realpath(path)) elif util.isspecial(path, 'original', True): filelist = os.listdir(self.realpath(path)) elif util.isspecial(path, 'commands'): filelist = CommandHandler.COMMANDS else: filelist = self.filelist(path) for filename in filelist: yield filename
class TagOrganizer(Organizer): def __init__(self, cache, category=None): self.tags = None self.category = category Organizer.__init__(self, cache, False) def reset(self): if not self.tags: self.tags = Base(DB_FILE_TAGS) self.tags.create('realpath', 'category', 'tag', mode='override') self.tags.create_index('realpath') self.tags.create_index('category') Organizer.reset(self) def updatecache(self): self._generatetags() Organizer.updatecache(self) def _deletefromcache(self, path): realpath = self.realpath(path) logger.debug("_deletefromcache(%s)" % realpath) for tag in self.tags.get_index('realpath')[realpath]: self.tags.delete(tag) def deletefromcache(self, path): self._deletefromcache(path) Organizer.deletefromcache(self, path) def addtocache(self, path): self._deletefromcache(path) self.generatetags(self.realpath(path)) Organizer.addtocache(self, path) def generatepaths(self, realpath): for record in self.tags.get_index('realpath')[realpath]: yield os.path.join(os.sep, record['tag'], os.path.basename(realpath)) def dirlist(self, path): if path == '/': return self.taglist(self.category) else: return [] ############################################ # Tag functions def _generatetags(self): for filename in filter( util.ignoretag, #IGNORE:W0141 self.cache.filelist()): self.generatetags(filename) def generatetags(self, filename): pass def tag(self, realpath, category, tag): logger.debug('tag(%s, %s, %s)' % (realpath, category, tag)) if not tag == None and not tag == '': self.tags.insert(realpath, category, tag) def filelistbytags(self, category, tags): self.refreshcache() for record in self.tags.get_index('category')[category]: if record['tag'] in tags: yield os.path.basename(record['realpath']) def taglist(self, category): self.refreshcache() return util.unique([ record['tag'] for record in self.tags.get_index('category')[category] ])
class Organizer(Cacheable): """ This is the base class for organizers """ def __init__(self, cache, recursive=True): Cacheable.__init__(self) self.cache = cache self.recursive = recursive self.transformed = None # Do not call reset here, it is called from fs.py when the fs is # already started def reset(self): if not self.transformed: self.transformed = Base(DB_TRANSFORMED) self.transformed.create('realpath', 'path', 'dirname', mode='override') self.transformed.create_index('realpath') self.transformed.create_index('path') self.transformed.create_index('dirname') self.cache.reset() Cacheable.reset(self) def updatecache(self): self.generateallpaths() def deletefromcache(self, path): realpath = self.realpath(path) logger.debug("deletefromcache(%s)" % realpath) self.cache.deletefromcache(realpath) for item in self.transformed.get_index('realpath')[realpath]: self.transformed.delete(item) def addtocache(self, path): if not self.transformed.get_index('path')[path]: realpath = self.realpath(path) self.cache.addtocache(realpath) self.addfile(realpath) ############################################ # Overwritable functions def dirlist(self, path): #IGNORE:W0613 """ Returns a list of (non-existent, generated, virtual) directories for a given path. Default implementation. """ return [] def generatepaths(self, realpath): """ Generates paths for a given real path. A file can have more than one transformed path. Default implementation. """ yield util.addtrailingslash( util.removeroot(realpath, self.cache.filter.root)) def generaterealpath(self, path): """ Generates a real path for a inexistent path. Default implementation. """ return os.path.join(self.cache.filter.root, path[1:]) ############################################ # General functions def generateallpaths(self): """ Generates paths for all the files given by the cache and stores them in self.transformed """ for realpath in self.cache.filelist(): if self.recursive: # Add all sub-directories first currentpath = self.cache.filter.root for pathpart in util.pathparts( util.removeroot(realpath, self.cache.filter.root)): currentpath = os.path.join(currentpath, pathpart) self.addfile(currentpath) else: self.addfile(realpath) def addfile(self, realpath): """ Stores a file in self.transformed if not there already and returns the paths for that file in the proxy file system """ logger.debug('addfile(%s)' % realpath) if not util.ignoretag(util.removeroot(realpath, self.cache.filter.root)): return [] self.refreshcache() transformed = self.transformed.get_index('realpath')[realpath] if transformed: return (record['path'] for record in transformed) else: paths = [] for path in self.paths(realpath): while self.transformed.get_index('path')[path]: path = self.increasefilename(path) dirname = os.path.dirname(path) logger.debug('addfile(%s, %s, %s)' % (realpath, path, dirname)) self.transformed.insert(realpath=realpath, path=path, dirname=dirname) paths.append(path) return paths def increasefilename(self, filename): """ Returns a new filename in sequence. Called if the current filename already exists. This default implementation adds a "(1)" to the end if not present or increases that number by one. """ root, ext = os.path.splitext(filename) num = 1 matches = _INCREASE_REGEX.match(root) if not matches is None: num = int(matches.group(2)) + 1 filename = matches.group(1) return '%s(%i)%s' % (root, num, ext) ############################################ # General functions that read the cache def filelist(self, path): """ Returns a list of directories and filenames in a list from cache """ logger.debug('filelist(%s)' % path) self.refreshcache() for dirname in self.dirlist(path): yield dirname for record in self.transformed.get_index('dirname')[path]: yield os.path.basename(record['path']) def paths(self, realpath): """ Generates or returns paths from cache for a given real path """ self.refreshcache() paths = self.transformed.get_index('realpath')[realpath] if paths: return (path['path'] for path in paths) else: return (path for path in self.generatepaths(realpath)) def realpath(self, path): """ Returns the real path for a file given the path in the file system. """ logger.debug('realpath(%s)' % path) self.refreshcache() realpaths = [ r['realpath'] for r in self.transformed.get_index('path')[path] ] realpath = None if realpaths: realpath = realpaths[0] elif path == '/': realpath = self.cache.filter.root elif path == util.addtrailingslash(util.ORIGINAL_DIR): realpath = '.' elif util.isspecial(path, 'original', True): realpath = os.path.join('.', os.sep.join(util.pathparts(path)[2:])) elif util.isspecial(path, 'root', True): realpath = os.path.join(self.cache.filter.root, os.sep.join(util.pathparts(path)[2:])) elif util.isspecial(path, 'commands'): realpath = '.' elif util.iscommand(path): realpath = getserver().tempfile.name else: realpath = self.generaterealpath(path) logger.debug('realpath(%s) = %s' % (path, realpath)) return realpath ############################################ # File system functions def getattr(self, path): dirname = os.path.dirname(path) if util.removeroot(path, os.sep) in self.dirlist(dirname): return self.cache.getattr(self.realpath(dirname)) else: return self.cache.getattr(self.realpath(path)) def readdir(self, path, offset): #IGNORE:W0613 for filename in util.getbasefilelist(): yield fuse.Direntry(filename) for filename in self._filelist(path): yield fuse.Direntry(filename) def _filelist(self, path): filelist = [] if path == util.addtrailingslash(util.ORIGINAL_DIR): filelist = ['original', 'root', 'commands'] elif util.isspecial(path, 'root', True): filelist = os.listdir(self.realpath(path)) elif util.isspecial(path, 'original', True): filelist = os.listdir(self.realpath(path)) elif util.isspecial(path, 'commands'): filelist = CommandHandler.COMMANDS else: filelist = self.filelist(path) for filename in filelist: yield filename