def writerThread(self, d, howMany, writerNum):
name = currentThread().getName()
start = howMany * writerNum
stop = howMany * (writerNum + 1) - 1
if verbose:
print "%s: creating records %d - %d" % (name, start, stop)
# create a bunch of records
for x in xrange(start, stop):
key = '%04d' % x
dbutils.DeadlockWrap(d.put,
key,
self.makeData(key),
max_retries=12)
if verbose and x % 100 == 0:
print "%s: records %d - %d finished" % (name, start, x)
# do a bit or reading too
if random() <= 0.05:
for y in xrange(start, x):
key = '%04d' % x
data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
self.assertEqual(data, self.makeData(key))
# flush them
try:
dbutils.DeadlockWrap(d.sync, max_retries=12)
except db.DBIncompleteError, val:
if verbose:
print "could not complete sync()..."
def _iteritems(self):
try:
return dbutils.DeadlockWrap(self._data.iteritems,
max_retries=MAX_RETRIES)
except:
print_exc()
print >> sys.stderr, "bsdcachedb: bsdcachedb.BasicDB._iteritems error", self.__class__.__name__
def writerThread(self, d, howMany, writerNum):
#time.sleep(0.01 * writerNum + 0.01)
name = currentThread().getName()
start = howMany * writerNum
stop = howMany * (writerNum + 1) - 1
if verbose:
print "%s: creating records %d - %d" % (name, start, stop)
for x in range(start, stop):
key = '%04d' % x
dbutils.DeadlockWrap(d.put,
key,
self.makeData(key),
max_retries=12)
if verbose and x % 100 == 0:
print "%s: records %d - %d finished" % (name, start, x)
if verbose:
print "%s: finished creating records" % name
## # Each write-cursor will be exclusive, the only one that can update the DB...
## if verbose: print "%s: deleting a few records" % name
## c = d.cursor(flags = db.DB_WRITECURSOR)
## for x in range(10):
## key = int(random() * howMany) + start
## key = '%04d' % key
## if d.has_key(key):
## c.set(key)
## c.delete()
## c.close()
if verbose:
print "%s: thread finished" % name
def _size(self):
try:
return dbutils.DeadlockWrap(len,
self._data,
max_retries=MAX_RETRIES)
#return len(self._data)
except:
print_exc()
print >> sys.stderr, "bsdcachedb: bsdcachedb.BasicDB._size error", self.__class__.__name__
return 0
def _has_key(self, key): # find a key
try:
return dbutils.DeadlockWrap(self._data.has_key,
key,
max_retries=MAX_RETRIES)
#return self._data.has_key(key)
except Exception, e:
print >> sys.stderr, "bsdcachedb: _has_key EXCEPTION BY", currentThread(
).getName(), Exception, e, self.db_name, ` key `
return False
def _keys(self):
try:
return dbutils.DeadlockWrap(self._data.keys,
max_retries=MAX_RETRIES)
#return self._data.keys()
except Exception, e:
print >> sys.stderr, "bsdcachedb: _keys EXCEPTION BY", currentThread(
).getName(), self.db_name
#print_exc()
self.report_exception(e)
return []
def readerThread(self, d, readerNum):
time.sleep(0.01 * readerNum)
name = currentThread().getName()
for loop in range(5):
c = d.cursor()
count = 0
rec = dbutils.DeadlockWrap(c.first, max_retries=10)
while rec:
count += 1
key, data = rec
self.assertEqual(self.makeData(key), data)
rec = dbutils.DeadlockWrap(c.next, max_retries=10)
if verbose:
print "%s: found %d records" % (name, count)
c.close()
time.sleep(0.05)
if verbose:
print "%s: thread finished" % name
def _sync(self): # write data from mem to disk
try:
dbutils.DeadlockWrap(self._data.sync, max_retries=MAX_RETRIES)
# except db.DBRunRecoveryError, e:
# print >> sys.stderr, "bsdcachedb: Sorry, meet DBRunRecoveryError at sync, have to remove the whole database", self.db_name
# self.report_exception(e)
# self._recover_db() # have to clear the whole database
except Exception, e:
#print >> sys.stderr, "bsdcachedb: synchronize db error", self.db_name, Exception, e
self.report_exception(e)
def close(self):
if DEBUG:
print >> sys.stderr, "bsdcachedb: Closing database", self.db_name, currentThread(
).getName()
if self.opened:
try:
self._sync()
dbutils.DeadlockWrap(self._data.close, max_retries=MAX_RETRIES)
if DEBUG:
print >> sys.stderr, "bsdcachedb: Done waiting for database close", self.db_name, currentThread(
).getName()
#self._data.close()
except:
print_exc()
self.opened = False
def _delete(self, key):
try:
if DEBUG:
name = currentThread().getName()
if name not in self.threadnames:
self.threadnames[name] = 0
self.threadnames[name] += 1
print >> sys.stderr, "bsdcachedb: del", len(self.threadnames), name, \
self.threadnames[name], time(), self.__class__.__name__
dbutils.DeadlockWrap(self._data.delete,
key,
max_retries=MAX_RETRIES)
#self._data.delete(key)
except:
pass
def _get(self, key, value=None): # read
try:
return dbutils.DeadlockWrap(self._data.get,
key,
value,
max_retries=MAX_RETRIES)
#return self._data.get(key, value)
# except db.DBRunRecoveryError, e:
# print >> sys.stderr, "bsdcachedb: Sorry, meet DBRunRecoveryError at get, have to remove the whole database", self.db_name
# self.report_exception(e)
# self._recover_db() # have to clear the whole database
except Exception, e:
print >> sys.stderr, "bsdcachedb: _get EXCEPTION BY", currentThread(
).getName(), Exception, e, self.db_name, ` key `, value
if value is not None:
return value
self.report_exception(e)
return None
def _put(self, key, value): # write
try:
if DEBUG:
name = currentThread().getName()
if name not in self.threadnames:
self.threadnames[name] = 0
self.threadnames[name] += 1
print >> sys.stderr, "bsdcachedb: put", len(self.threadnames), name, \
self.threadnames[name], time(), self.__class__.__name__
if not value and type(value) == dict:
raise Exception(
'Warning someone tries to insert empty data in db: %s:%s' %
(key, value))
dbutils.DeadlockWrap(self._data.put,
key,
value,
max_retries=MAX_RETRIES)
#self._data.put(key, value)
except:
pass
def _items(self):
return dbutils.DeadlockWrap(self._data.items, max_retries=MAX_RETRIES)
def _values(self):
return dbutils.DeadlockWrap(self._data.values, max_retries=MAX_RETRIES)
def _clear(self):
dbutils.DeadlockWrap(self._data.clear, max_retries=MAX_RETRIES)
class SimpleThreadedBase(BaseThreadedTestCase):
dbopenflags = db.DB_THREAD
envflags = db.DB_THREAD | db.DB_INIT_MPOOL | db.DB_INIT_LOCK
readers = 5
writers = 3
records = 1000
def setEnvOpts(self):
self.env.set_lk_detect(db.DB_LOCK_DEFAULT)
def test02_SimpleLocks(self):
if verbose:
print '\n', '-=' * 30
print "Running %s.test02_SimpleLocks..." % self.__class__.__name__
threads = []
for x in range(self.writers):
wt = Thread(
target=self.writerThread,
args=(self.d, self.records, x),
name='writer %d' % x,
) #verbose = verbose)
threads.append(wt)
for x in range(self.readers):
rt = Thread(
target=self.readerThread,
args=(self.d, x),
name='reader %d' % x,
) #verbose = verbose)
threads.append(rt)
for t in threads:
t.start()
for t in threads:
t.join()
def writerThread(self, d, howMany, writerNum):
name = currentThread().getName()
start = howMany * writerNum
stop = howMany * (writerNum + 1) - 1
if verbose:
print "%s: creating records %d - %d" % (name, start, stop)
# create a bunch of records
for x in xrange(start, stop):
key = '%04d' % x
dbutils.DeadlockWrap(d.put,
key,
self.makeData(key),
max_retries=12)
if verbose and x % 100 == 0:
print "%s: records %d - %d finished" % (name, start, x)
# do a bit or reading too
if random() <= 0.05:
for y in xrange(start, x):
key = '%04d' % x
data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
self.assertEqual(data, self.makeData(key))
# flush them
try:
dbutils.DeadlockWrap(d.sync, max_retries=12)
except db.DBIncompleteError, val:
if verbose:
print "could not complete sync()..."
# read them back, deleting a few
for x in xrange(start, stop):
key = '%04d' % x
data = dbutils.DeadlockWrap(d.get, key, max_retries=12)
if verbose and x % 100 == 0:
print "%s: fetched record (%s, %s)" % (name, key, data)
self.assertEqual(data, self.makeData(key))
if random() <= 0.10:
dbutils.DeadlockWrap(d.delete, key, max_retries=12)
if verbose:
print "%s: deleted record %s" % (name, key)
if verbose:
print "%s: thread finished" % name