Ejemplo n.º 1
0
    def writerThread(self, d, howMany, writerNum):
        name = currentThread().getName()
        start = howMany * writerNum
        stop = howMany * (writerNum + 1) - 1
        if verbose:
            print "%s: creating records %d - %d" % (name, start, stop)

        # create a bunch of records
        for x in xrange(start, stop):
            key = '%04d' % x
            dbutils.DeadlockWrap(d.put,
                                 key,
                                 self.makeData(key),
                                 max_retries=12)

            if verbose and x % 100 == 0:
                print "%s: records %d - %d finished" % (name, start, x)

            # do a bit or reading too
            if random() <= 0.05:
                for y in xrange(start, x):
                    key = '%04d' % x
                    data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
                    self.assertEqual(data, self.makeData(key))

        # flush them
        try:
            dbutils.DeadlockWrap(d.sync, max_retries=12)
        except db.DBIncompleteError, val:
            if verbose:
                print "could not complete sync()..."
Ejemplo n.º 2
0
 def _iteritems(self):
     try:
         return dbutils.DeadlockWrap(self._data.iteritems,
                                     max_retries=MAX_RETRIES)
     except:
         print_exc()
         print >> sys.stderr, "bsdcachedb: bsdcachedb.BasicDB._iteritems error", self.__class__.__name__
Ejemplo n.º 3
0
    def writerThread(self, d, howMany, writerNum):
        #time.sleep(0.01 * writerNum + 0.01)
        name = currentThread().getName()
        start = howMany * writerNum
        stop = howMany * (writerNum + 1) - 1
        if verbose:
            print "%s: creating records %d - %d" % (name, start, stop)

        for x in range(start, stop):
            key = '%04d' % x
            dbutils.DeadlockWrap(d.put,
                                 key,
                                 self.makeData(key),
                                 max_retries=12)
            if verbose and x % 100 == 0:
                print "%s: records %d - %d finished" % (name, start, x)

        if verbose:
            print "%s: finished creating records" % name


##         # Each write-cursor will be exclusive, the only one that can update the DB...
##         if verbose: print "%s: deleting a few records" % name
##         c = d.cursor(flags = db.DB_WRITECURSOR)
##         for x in range(10):
##             key = int(random() * howMany) + start
##             key = '%04d' % key
##             if d.has_key(key):
##                 c.set(key)
##                 c.delete()

##         c.close()
        if verbose:
            print "%s: thread finished" % name
Ejemplo n.º 4
0
 def _size(self):
     try:
         return dbutils.DeadlockWrap(len,
                                     self._data,
                                     max_retries=MAX_RETRIES)
         #return len(self._data)
     except:
         print_exc()
         print >> sys.stderr, "bsdcachedb: bsdcachedb.BasicDB._size error", self.__class__.__name__
         return 0
Ejemplo n.º 5
0
 def _has_key(self, key):  # find a key
     try:
         return dbutils.DeadlockWrap(self._data.has_key,
                                     key,
                                     max_retries=MAX_RETRIES)
         #return self._data.has_key(key)
     except Exception, e:
         print >> sys.stderr, "bsdcachedb: _has_key EXCEPTION BY", currentThread(
         ).getName(), Exception, e, self.db_name, ` key `
         return False
Ejemplo n.º 6
0
 def _keys(self):
     try:
         return dbutils.DeadlockWrap(self._data.keys,
                                     max_retries=MAX_RETRIES)
         #return self._data.keys()
     except Exception, e:
         print >> sys.stderr, "bsdcachedb: _keys EXCEPTION BY", currentThread(
         ).getName(), self.db_name
         #print_exc()
         self.report_exception(e)
         return []
Ejemplo n.º 7
0
    def readerThread(self, d, readerNum):
        time.sleep(0.01 * readerNum)
        name = currentThread().getName()

        for loop in range(5):
            c = d.cursor()
            count = 0
            rec = dbutils.DeadlockWrap(c.first, max_retries=10)
            while rec:
                count += 1
                key, data = rec
                self.assertEqual(self.makeData(key), data)
                rec = dbutils.DeadlockWrap(c.next, max_retries=10)
            if verbose:
                print "%s: found %d records" % (name, count)
            c.close()
            time.sleep(0.05)

        if verbose:
            print "%s: thread finished" % name
Ejemplo n.º 8
0
    def _sync(self):  # write data from mem to disk
        try:
            dbutils.DeadlockWrap(self._data.sync, max_retries=MAX_RETRIES)


#        except db.DBRunRecoveryError, e:
#            print >> sys.stderr, "bsdcachedb: Sorry, meet DBRunRecoveryError at sync, have to remove the whole database", self.db_name
#            self.report_exception(e)
#            self._recover_db()    # have to clear the whole database
        except Exception, e:
            #print >> sys.stderr, "bsdcachedb: synchronize db error", self.db_name, Exception, e
            self.report_exception(e)
Ejemplo n.º 9
0
 def close(self):
     if DEBUG:
         print >> sys.stderr, "bsdcachedb: Closing database", self.db_name, currentThread(
         ).getName()
     if self.opened:
         try:
             self._sync()
             dbutils.DeadlockWrap(self._data.close, max_retries=MAX_RETRIES)
             if DEBUG:
                 print >> sys.stderr, "bsdcachedb: Done waiting for database close", self.db_name, currentThread(
                 ).getName()
             #self._data.close()
         except:
             print_exc()
     self.opened = False
Ejemplo n.º 10
0
    def _delete(self, key):
        try:
            if DEBUG:
                name = currentThread().getName()
                if name not in self.threadnames:
                    self.threadnames[name] = 0
                self.threadnames[name] += 1
                print >> sys.stderr, "bsdcachedb: del", len(self.threadnames), name, \
                    self.threadnames[name], time(), self.__class__.__name__

            dbutils.DeadlockWrap(self._data.delete,
                                 key,
                                 max_retries=MAX_RETRIES)
            #self._data.delete(key)
        except:
            pass
Ejemplo n.º 11
0
    def _get(self, key, value=None):  # read
        try:
            return dbutils.DeadlockWrap(self._data.get,
                                        key,
                                        value,
                                        max_retries=MAX_RETRIES)
            #return self._data.get(key, value)
#        except db.DBRunRecoveryError, e:
#            print >> sys.stderr, "bsdcachedb: Sorry, meet DBRunRecoveryError at get, have to remove the whole database", self.db_name
#            self.report_exception(e)
#            self._recover_db()    # have to clear the whole database
        except Exception, e:
            print >> sys.stderr, "bsdcachedb: _get EXCEPTION BY", currentThread(
            ).getName(), Exception, e, self.db_name, ` key `, value
            if value is not None:
                return value
            self.report_exception(e)
            return None
Ejemplo n.º 12
0
    def _put(self, key, value):  # write
        try:
            if DEBUG:
                name = currentThread().getName()
                if name not in self.threadnames:
                    self.threadnames[name] = 0
                self.threadnames[name] += 1
                print >> sys.stderr, "bsdcachedb: put", len(self.threadnames), name, \
                    self.threadnames[name], time(), self.__class__.__name__
            if not value and type(value) == dict:
                raise Exception(
                    'Warning someone tries to insert empty data in db: %s:%s' %
                    (key, value))

            dbutils.DeadlockWrap(self._data.put,
                                 key,
                                 value,
                                 max_retries=MAX_RETRIES)
            #self._data.put(key, value)
        except:
            pass
Ejemplo n.º 13
0
 def _items(self):
     return dbutils.DeadlockWrap(self._data.items, max_retries=MAX_RETRIES)
Ejemplo n.º 14
0
 def _values(self):
     return dbutils.DeadlockWrap(self._data.values, max_retries=MAX_RETRIES)
Ejemplo n.º 15
0
 def _clear(self):
     dbutils.DeadlockWrap(self._data.clear, max_retries=MAX_RETRIES)
Ejemplo n.º 16
0
class SimpleThreadedBase(BaseThreadedTestCase):
    dbopenflags = db.DB_THREAD
    envflags = db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK
    readers = 5
    writers = 3
    records = 1000

    def setEnvOpts(self):
        self.env.set_lk_detect(db.DB_LOCK_DEFAULT)

    def test02_SimpleLocks(self):
        if verbose:
            print '\n', '-=' * 30
            print "Running %s.test02_SimpleLocks..." % self.__class__.__name__

        threads = []
        for x in range(self.writers):
            wt = Thread(
                target=self.writerThread,
                args=(self.d, self.records, x),
                name='writer %d' % x,
            )  #verbose = verbose)
            threads.append(wt)
        for x in range(self.readers):
            rt = Thread(
                target=self.readerThread,
                args=(self.d, x),
                name='reader %d' % x,
            )  #verbose = verbose)
            threads.append(rt)

        for t in threads:
            t.start()
        for t in threads:
            t.join()

    def writerThread(self, d, howMany, writerNum):
        name = currentThread().getName()
        start = howMany * writerNum
        stop = howMany * (writerNum + 1) - 1
        if verbose:
            print "%s: creating records %d - %d" % (name, start, stop)

        # create a bunch of records
        for x in xrange(start, stop):
            key = '%04d' % x
            dbutils.DeadlockWrap(d.put,
                                 key,
                                 self.makeData(key),
                                 max_retries=12)

            if verbose and x % 100 == 0:
                print "%s: records %d - %d finished" % (name, start, x)

            # do a bit or reading too
            if random() <= 0.05:
                for y in xrange(start, x):
                    key = '%04d' % x
                    data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
                    self.assertEqual(data, self.makeData(key))

        # flush them
        try:
            dbutils.DeadlockWrap(d.sync, max_retries=12)
        except db.DBIncompleteError, val:
            if verbose:
                print "could not complete sync()..."

        # read them back, deleting a few
        for x in xrange(start, stop):
            key = '%04d' % x
            data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
            if verbose and x % 100 == 0:
                print "%s: fetched record (%s, %s)" % (name, key, data)
            self.assertEqual(data, self.makeData(key))
            if random() <= 0.10:
                dbutils.DeadlockWrap(d.delete, key, max_retries=12)
                if verbose:
                    print "%s: deleted record %s" % (name, key)

        if verbose:
            print "%s: thread finished" % name