def taskrunner(self):
'''Continuously process tasks that may block'''
tid = threadid()
self.log.log(self.loglevels['threading'], 'Thread %i started' % tid)
self.taskrunnerinit()
self.activetaskrunners += 1
try:
while self.activetaskrunners < self.numtaskrunners + 1 and not self.exittaskrunner:
self.waitingtaskrunners += 1
try:
self.log.log(self.loglevels['threading'],
'Thread %i waiting for tasks' % tid)
task = self.tasks.get(block=True)
strtask = str(task)
self.log.log(self.loglevels['threading'],
'Thread %i got task %s' % (tid, strtask))
finally:
self.waitingtaskrunners -= 1
try:
self.handletask(task)
except:
self.log.exception(
'Serious error in thread %i while handling task %s' %
(tid, strtask))
else:
self.log.log(
self.loglevels['threading'],
'Thread %i executed task %s without raising an exception'
% (tid, strtask))
finally:
self.activetaskrunners -= 1
self.taskrunnerclose()
self.log.log(self.loglevels['threading'],
'Thread %i exiting' % tid)
def execsql(self, sql, commit = True, fetch = False, raiseerrors = False):
'''Execute SQL code on the database
Commits the changes by default (the only time this shouldn't be used is
if you want to send multiple commands in a single transaction, with the
ability to make changes to later queries based on the results of
earlier queries).
Fetches the resutls if requested. In general only SELECT statements
should set fetch. If fetch is True, it fetchs the entire result set,
otherwise if it is a integer, it fetchs that number of rows.
fetch=True and fetch=1 are two different commands.
'''
tid = threadid()
connection = self.threaddata[tid]['dbconn']
cursor = self.threaddata[tid]['dbcursor']
self.log.log(self.loglevels['sql'], 'Executing SQL: %r' % sql)
# apsw uses Unicode internally, so if you have a user with a
# non-ascii nick (common with users from European countries),
# it will complain, bitterly, if you give it a string that isn't
# 7-bit ascii. pysqlite is different and will happily stick Latin-1
# formatted strings into the database. In order to be able to
# switch between apsw and pysqlite easily, you need to store everything
# as Unicode strings, which necessitates decoding them before
# executing them and encoding them after getting back the results.
sql = sql.decode('latin1')
return getattr(self, 'execsql_%s' % self.dbtype)(connection, cursor, sql, commit, fetch, raiseerrors)
def taskrunner(self):
'''Continuously process tasks that may block'''
tid = threadid()
self.log.log(self.loglevels['threading'], 'Thread %i started' % tid)
self.taskrunnerinit()
self.activetaskrunners += 1
try:
while self.activetaskrunners < self.numtaskrunners + 1 and not self.exittaskrunner:
self.waitingtaskrunners += 1
try:
self.log.log(self.loglevels['threading'], 'Thread %i waiting for tasks' % tid)
task = self.tasks.get(block=True)
strtask = str(task)
self.log.log(self.loglevels['threading'], 'Thread %i got task %s' % (tid, strtask))
finally:
self.waitingtaskrunners -= 1
try:
self.handletask(task)
except:
self.log.exception('Serious error in thread %i while handling task %s' % (tid, strtask))
else:
self.log.log(self.loglevels['threading'], 'Thread %i executed task %s without raising an exception' % (tid, strtask))
finally:
self.activetaskrunners -= 1
self.taskrunnerclose()
self.log.log(self.loglevels['threading'], 'Thread %i exiting' % tid)
def dbconnect_pysqlite2(self, connect):
'''Connect to the database using pysqlite'''
tid = threadid()
if connect:
self.threaddata[tid]['dbconn'] = sqlite2.connect(self.dbfile, timeout=5)
self.threaddata[tid]['dbcursor'] = self.threaddata[tid]['dbconn'].cursor()
else:
self.threaddata[tid]['dbcursor'].close()
self.threaddata[tid]['dbconn'].close()
def dbconnect(self, connect = True):
'''Connect to the database
This function is typically run once per thread, since threads may not
share database connections in pysqlite.
'''
tid = threadid()
type = connect and 'Connecting to' or 'Disconnecting from'
self.log.log(self.loglevels['threading'], 'Thread %i %s database' % (tid, type))
self.threaddata[tid]['dbtype'] = self.dbtype
getattr(self, 'dbconnect_%s' % self.threaddata[tid]['dbtype'])(connect)
def dbconnect_apsw(self, connect):
'''Connect to the database using apsw
apsw doesn't have close functions for their connections and cursors,
so I assume that closing those resources happens when they are garbage
collected.
'''
tid = threadid()
if connect:
self.threaddata[tid]['dbconn'] = apsw.Connection(self.dbfile)
self.threaddata[tid]['dbcursor'] = self.threaddata[tid]['dbconn'].cursor()
self.threaddata[tid]['dbconn'].setbusytimeout(5000)
else:
del self.threaddata[tid]['dbcursor']
del self.threaddata[tid]['dbconn']
def setupdefaults(self, **kwargs):
'''Setup thread related defaults'''
super(ThreadedDCHub, self).setupdefaults(**kwargs)
self.supers['ThreadedDCHub'] = super(ThreadedDCHub, self)
self.reloadmodules.append('ThreadedDCHub')
self.threaddata = {threadid():{}}
self.emptytask = emptytask
self.cleanuptime = 5
self.activetaskrunners = 0
self.exittaskrunner = False
self.waitingtaskrunners = 0
self.numtaskrunners = 5
self.tasks = Queue(0)
self.rlock = threading.RLock()
self.loglevels['threading'] = 8
self.nonreloadableattrs.union_update('exittaskrunner waitingtaskrunners activetaskrunners tasks threaddata'.split())
def setupdefaults(self, **kwargs):
'''Setup thread related defaults'''
super(ThreadedDCHub, self).setupdefaults(**kwargs)
self.supers['ThreadedDCHub'] = super(ThreadedDCHub, self)
self.reloadmodules.append('ThreadedDCHub')
self.threaddata = {threadid(): {}}
self.emptytask = emptytask
self.cleanuptime = 5
self.activetaskrunners = 0
self.exittaskrunner = False
self.waitingtaskrunners = 0
self.numtaskrunners = 5
self.tasks = Queue(0)
self.rlock = threading.RLock()
self.loglevels['threading'] = 8
self.nonreloadableattrs.union_update(
'exittaskrunner waitingtaskrunners activetaskrunners tasks threaddata'
.split())
def taskrunnerinit(self):
'''Preform the necessary actions on thread startup'''
self.threaddata[threadid()] = {}
def taskrunnerclose(self):
'''Preform the necessary actions on thread shutdown'''
del self.threaddata[threadid()]
def taskrunnerinit(self):
'''Preform the necessary actions on thread startup'''
self.threaddata[threadid()] = {}
def taskrunnerclose(self):
'''Preform the necessary actions on thread shutdown'''
del self.threaddata[threadid()]
