Esempio n. 1
0
    def __init__(self, request, **kw):
        log.debug("shelve graphdb init")
        GraphDataBase.__init__(self, request, **kw)

        gddir = os.path.join(request.cfg.data_dir, 'graphdata')
        if not os.path.isdir(gddir):
            os.mkdir(gddir)
        self.graphshelve = os.path.join(gddir, 'graphdata.shelve')

        self.use_sq_dict = getattr(request.cfg, 'use_sq_dict', False)
        if self.use_sq_dict:
            import sq_dict
            self.shelveopen = sq_dict.shelve
        else:
            self.shelveopen = shelve.open

        # XXX (falsely) assumes shelve.open creates file with same name;
        # it happens to work with the bsddb backend.
        if not os.path.exists(self.graphshelve):
            db = self.shelveopen(self.graphshelve, 'c')
            db.close()

        self.db = None
        self.cache = dict()
        self.out = dict()

        lock_path = os.path.join(gddir, "graphdata-lock")
        self._lock_timeout = getattr(request.cfg, 'graphdata_lock_timeout', None)
        self._readlock = _Lock(lock_path, exclusive=False)
        self._writelock = _Lock(lock_path, exclusive=True)
Esempio n. 2
0
    def __init__(self, request, **kw):
        log.debug("shelve graphdb init")
        GraphDataBase.__init__(self, request, **kw)

        gddir = os.path.join(request.cfg.data_dir, 'graphdata')
        if not os.path.isdir(gddir):
            os.mkdir(gddir)
        self.graphshelve = os.path.join(gddir, 'graphdata.shelve')

        self.use_sq_dict = getattr(request.cfg, 'use_sq_dict', False)
        if self.use_sq_dict:
            import sq_dict
            self.shelveopen = sq_dict.shelve
        else:
            self.shelveopen = shelve.open

        # XXX (falsely) assumes shelve.open creates file with same name;
        # it happens to work with the bsddb backend.
        if not os.path.exists(self.graphshelve):
            db = self.shelveopen(self.graphshelve, 'c')
            db.close()

        self.db = None
        self.cache = dict()
        self.out = dict()

        lock_path = os.path.join(gddir, "graphdata-lock")
        self._lock_timeout = getattr(request.cfg, 'graphdata_lock_timeout',
                                     None)
        self._readlock = _Lock(lock_path, exclusive=False)
        self._writelock = _Lock(lock_path, exclusive=True)
 def commit(self):
     # Ha, puny gullible humans think I do transactions
     # .. instead, I abuse this metod for rehash mode & bulk update
     if self.doing_rehash:
         log.debug("commit in rehash mode, doing bulk update")
         for success, docid, rev_or_exc in self.couch_db.update(self.modified_pages.values()):
             if not success:
                 raise DbError(
                     "at least one update failed while writing updated docs at end of rehash. first exception docid: %s exception: %s" % (docid, rev_or_exc))
             
         self.modified_pages = {}
Esempio n. 4
0
    def close(self):
        if self.out:
            self.writelock()

            for key, value in self.out.items():
                if value is self.UNDEFINED:
                    self.db.pop(key, None)
                else:
                    self.db[key] = value

            self.out = dict()

        self.cache.clear()

        if self.db is not None:
            self.db.close()
            self.db = None

        if self._writelock.release():
            log.debug("released a write lock for %r" % (self.graphshelve,))
        else:
            log.debug("did not release any write locks for %r" % (self.graphshelve,))

        if self._readlock.release():
            log.debug("released a read lock for %r" % (self.graphshelve,))
        else:
            log.debug("did not released any read locks for %r" % (self.graphshelve,))
Esempio n. 5
0
    def close(self):
        if self.out:
            self.writelock()

            for key, value in self.out.items():
                if value is self.UNDEFINED:
                    self.db.pop(key, None)
                else:
                    self.db[key] = value

            self.out = dict()

        self.cache.clear()

        if self.db is not None:
            self.db.close()
            self.db = None

        if self._writelock.release():
            log.debug("released a write lock for %r" % (self.graphshelve, ))
        else:
            log.debug("did not release any write locks for %r" %
                      (self.graphshelve, ))

        if self._readlock.release():
            log.debug("released a read lock for %r" % (self.graphshelve, ))
        else:
            log.debug("did not released any read locks for %r" %
                      (self.graphshelve, ))
Esempio n. 6
0
    def readlock(self):
        if self._writelock.is_locked():
            return
        if self._readlock.is_locked():
            return

        log.debug("getting a read lock for %r" % (self.graphshelve,))
        try:
            self._readlock.acquire(self._lock_timeout)
        except LockTimeout:
            items = self.graphshelve, self._lock_timeout
            log.error("getting a read lock for %r timed out after %.02fs" % items)
            raise
        log.debug("got a read lock for %r" % (self.graphshelve,))

        self.db = self.shelveopen(self.graphshelve, "r")
Esempio n. 7
0
    def readlock(self):
        if self._writelock.is_locked():
            return
        if self._readlock.is_locked():
            return

        log.debug("getting a read lock for %r" % (self.graphshelve, ))
        try:
            self._readlock.acquire(self._lock_timeout)
        except LockTimeout:
            items = self.graphshelve, self._lock_timeout
            log.error("getting a read lock for %r timed out after %.02fs" %
                      items)
            raise
        log.debug("got a read lock for %r" % (self.graphshelve, ))

        self.db = self.shelveopen(self.graphshelve, "r")
    def __init__(self, request, dbname="gwiki", couchurl=None):
        log.debug("couchdb graphdb init")
        GraphDataBase.__init__(self, request)

        self.dbname = dbname
        if couchurl:
            self.couch_server = couchdb.Server(couchurl)
        else:
            self.couch_server = couchdb.Server()

        self.make_pagemeta_class()
        
        # we could really use db connection recycling/pooling...
        self.init_db()

        # use write cache/bulk update workaround for rehash slowness,
        # unsafe otherwise
        self.doing_rehash = False
        self.modified_pages = {}
Esempio n. 9
0
    def writelock(self):
        if self._writelock.is_locked():
            return

        if self._readlock.is_locked():
            if self.db is not None:
                self.db.close()
                self.db = None
            self._readlock.release()
            log.debug("released a write lock for %r" % (self.graphshelve,))

        log.debug("getting a write lock for %r" % (self.graphshelve,))
        try:
            self._writelock.acquire(self._lock_timeout)
        except LockTimeout:
            items = self.graphshelve, self._lock_timeout
            log.error("getting a write lock for %r timed out after %.02fs" % items)
            raise
        log.debug("got a write lock for %r" % (self.graphshelve,))

        self.db = self.shelveopen(self.graphshelve, "c")
Esempio n. 10
0
    def writelock(self):
        if self._writelock.is_locked():
            return

        if self._readlock.is_locked():
            if self.db is not None:
                self.db.close()
                self.db = None
            self._readlock.release()
            log.debug("released a write lock for %r" % (self.graphshelve, ))

        log.debug("getting a write lock for %r" % (self.graphshelve, ))
        try:
            self._writelock.acquire(self._lock_timeout)
        except LockTimeout:
            items = self.graphshelve, self._lock_timeout
            log.error("getting a write lock for %r timed out after %.02fs" %
                      items)
            raise
        log.debug("got a write lock for %r" % (self.graphshelve, ))

        self.db = self.shelveopen(self.graphshelve, "c")
Esempio n. 11
0
    def delpage(self, pagename):
        log.debug("delpage %s" % (repr(pagename),))
        page = encode_page(pagename)

        self.out[page] = self.UNDEFINED
        self.cache.pop(page, None)
Esempio n. 12
0
    def savepage(self, pagename, pagedict):
        log.debug("savepage %s = %s" % (repr(pagename), repr(pagedict)))
        page = encode_page(pagename)

        self.out[page] = pagedict
        self.cache.pop(page, None)
Esempio n. 13
0
    def delpage(self, pagename):
        log.debug("delpage %s" % (repr(pagename), ))
        page = encode_page(pagename)

        self.out[page] = self.UNDEFINED
        self.cache.pop(page, None)
Esempio n. 14
0
    def savepage(self, pagename, pagedict):
        log.debug("savepage %s = %s" % (repr(pagename), repr(pagedict)))
        page = encode_page(pagename)

        self.out[page] = pagedict
        self.cache.pop(page, None)
Esempio n. 15
0
 def dump_db(self):
     log.debug("db dump:")
     log.debug(dict(self.dbroot.items()))
 def clear_metas(self):
     log.debug("deleting db from couchdb: ", self.dbname)
     del self.couch_server[self.dbname]
     self.init_db()