except KeyboardInterrupt:
logger.logMessage(level="INFO",
message="Ending process, stopping worker threads.")
watchdogThread.stop()
for t in threadList:
if t is not None:
t.stop()
dataEvent.set()
for t in threadList:
if t is not None:
t.join()
watchdogThread.join()
logger.logMessage("Process ended, all threads stopped")
exit
except Exception as e:
logger.logException(message="Unexpected exception caught")
logger.logMessage(level="CRITICAL",
message="Unexpected error, trying to close...")
watchdogThread.stop()
for t in threadList:
if t is not None:
t.stop()
dataEvent.set()
for t in threadList:
if t is not None:
t.join()
watchdogThread.join()
raise
class WeatherQueue(object):
"""
Weather measurements queue.
Implemented on a sqlite3 database
"""
def __init__(self,dbdir):
"""
Initialize the queue database connection and, if necessary,
create the database. Also create the lock object that will
be used to synchronize access
"""
self.logger = WLogger()
self.theLock = threading.Lock()
self.curDay = 0
self.curTSA = 0
ini_file = pkg_resources.resource_filename(__name__,'./database/wQueue.ini')
config = configparser.ConfigParser()
config.read([ini_file])
tableDDL = config['queueDatabase']['table']
tsasDDL = config['queueDatabase']['control']
indexESDDL = config['queueDatabase']['indexES']
indexDBDDL = config['queueDatabase']['indexDB']
dbFile = os.path.join(dbdir,'wQueue.db')
try:
self.theConn = sqlite3.connect(dbFile,check_same_thread=False)
self.theConn.isolation_level = 'IMMEDIATE'
self.theConn.execute(tableDDL)
self.theConn.execute(indexESDDL)
self.theConn.execute(indexDBDDL)
self.theConn.execute(tsasDDL)
self.theConn.commit()
self.logger.logMessage(level="INFO",message="Queue database opened at {0:s}".format(dbFile))
except:
self.logger.logException('Error initializing queue database')
def pushLine(self,line):
"""
Push a line into the queue.
This function blocks until the database is not locked
"""
stamp,_,_,_,_,_,_,_,_,_,_,_ = parseLine(line)
datestamp = calendar.timegm(stamp.date().timetuple())
theTsa = 1
with self.theLock:
try:
result = self.theConn.execute(_SELECT_TSA, [datestamp])
resCol = result.fetchone()
if resCol == None:
self.theConn.execute(_INSERT_DAY, [datestamp])
else:
theTsa = resCol[0] + 1
self.theConn.execute(_UPDATE_TSA, [theTsa, datestamp])
fullTsa = (stamp.year * 10000 +
stamp.month * 100 +
stamp.day) * 1000000 + theTsa
self.theConn.execute(_INSERT_QUEUE, [fullTsa,line])
self.theConn.commit()
except:
self.logger.logException('Error inserting line into the queue database')
self.theConn.rollback()
def getDbQueue(self):
"""
Get al the queue lines NOT marked as inserted into the database.
(isDB == 0)
"""
with self.theLock:
try:
result = self.theConn.execute(_SELECT_DB)
queueContent = result.fetchall()
return queueContent
except:
self.logger.logException('Error fetching DB queue')
self.theConn.rollback()
return None
def markDbQueue(self, theId):
"""
Mark a queue entry as inserted into the database
Parameters:
- theId: row identifier to mark
"""
with self.theLock:
with self.theConn:
self.theConn.execute(_UPDATE_DB, [theId])
self.theConn.commit()
self.logger.logMessage(level='DEBUG',
message = 'Queue entry {0} marked as DB-done'.format(theId))
def getESQueue(self):
"""
Get al the queue lines NOT marked as indexed in elasticserch.
(isES == 0)
"""
with self.theLock:
try:
result = self.theConn.execute(_SELECT_ES)
queueContent = result.fetchall()
return queueContent
except:
self.logger.logException('Error fetching ES queue')
self.theConn.rollback()
return None
def markESQueue(self, theId):
"""
Mark a queue entry as indexed in elasticsearch
Parameters:
- theId: row identifier to mark
"""
with self.theLock:
with self.theConn:
self.theConn.execute(_UPDATE_ES, [theId])
self.theConn.commit()
self.logger.logMessage(level='DEBUG',
message = 'Queue entry {0} marked as ES-done'.format(theId))
def purgeQueue(self):
with self.theLock:
with self.theConn as conn:
result = conn.execute(_COUNT_QUEUE)
r = result.fetchone()
count = r[0]
self.logger.logMessage(message="About to purge {0} queue entries.".format(count))
conn.execute(_PURGE_QUEUE)
conn.commit()
self.logger.logMessage(message="Queue purged.")