def __init__(self,dbdir):
"""
Initialize the queue database connection and, if necessary,
create the database. Also create the lock object that will
be used to synchronize access
"""
self.logger = WLogger()
self.theLock = threading.Lock()
self.curDay = 0
self.curTSA = 0
ini_file = pkg_resources.resource_filename(__name__,'./database/wQueue.ini')
config = configparser.ConfigParser()
config.read([ini_file])
tableDDL = config['queueDatabase']['table']
tsasDDL = config['queueDatabase']['control']
indexESDDL = config['queueDatabase']['indexES']
indexDBDDL = config['queueDatabase']['indexDB']
dbFile = os.path.join(dbdir,'wQueue.db')
try:
self.theConn = sqlite3.connect(dbFile,check_same_thread=False)
self.theConn.isolation_level = 'IMMEDIATE'
self.theConn.execute(tableDDL)
self.theConn.execute(indexESDDL)
self.theConn.execute(indexDBDDL)
self.theConn.execute(tsasDDL)
self.theConn.commit()
self.logger.logMessage(level="INFO",message="Queue database opened at {0:s}".format(dbFile))
except:
self.logger.logException('Error initializing queue database')
class WeatherES(object):
"""
WeatherES: Class to manage the connection to the ElasticSearch cluster
Currently its only function is to initialize the ElasticSearch client
"""
_logger = WLogger()
def __init__(self, hosts, retryDelay=5, elTimeout=10):
"""
Initialize the WeatherEL object.
Parameters:
- hosts: list of elasticsearch ingest nodes to try to connect to
- retryDelay: delay between reconnection attempts
- elTimeout: timeout for elasticsearch operations
"""
self.theHosts = hosts
self.theDelay = retryDelay
self.theTimeOut = elTimeout
numHosts = len(self.theHosts)
hostlist = [{
'host': h,
'port': 9200,
'timeout': self.theTimeOut
} for h in self.theHosts]
self.theClient = elasticsearch.Elasticsearch(hosts=hostlist,
max_retries=numHosts)
class QueueJanitorThread(threading.Thread):
"""
Class to implement a thread to do maintenance tasks in the queue
database.
It will awake itself periodically to delete the queue elements
which have already been processed.
"""
_logger = WLogger()
def __init__(self,queue,period=60):
super(QueueJanitorThread, self).__init__()
self.theQueue = queue
self.thePeriod = period
self._stopSwitch = False
self.name = 'QueueJanitorThread'
self._pending = False
QueueJanitorThread._logger.logMessage("Janitor configured to run every {0} seconds".format(period))
def stop(self):
self._stopSwitch = True
def run(self):
"""
Run method.
It creates a timer object and schedules it according to the configured
perdiod.
The method runs an infinite loop with 1-second delays to check if the
termination flag (_stopSwitch) has been raised. In this case it cancels
the timer request (if pending) and ends.
"""
theTimer = None
self._pending = False
QueueJanitorThread._logger.logMessage("Starting thread {0}.".format(self.getName()), level="INFO")
while not self._stopSwitch:
if not self._pending:
theTimer = threading.Timer(self.thePeriod,self.doCleanup)
theTimer.name = "JanitorTimer"
self._pending = True
theTimer.start()
sleep(1)
theTimer.cancel()
QueueJanitorThread._logger.logMessage("Thread {0} stopped by request.".format(self.getName()), level="INFO")
def doCleanup(self):
"""
This method is scheduled inside a Timer object by the run() loop.
"""
self.theQueue.purgeQueue()
self._pending = False
from weatherLib.weatherDoc import WeatherData
from weatherLib.weatherUtil import WLogger
__INSERT_OBS = "insert into weather_work " + \
"(tsa, time, temperature, humidity, pressure, " + \
"light, fwVersion, swVersion, version, " + \
"isThermometer, isBarometer, isHygrometer, isClock) " + \
"values (%(tsa)s, %(time)s, %(temperature)s, %(humidity)s, %(pressure)s, " + \
"%(light)s, %(fwVersion)s, %(swVersion)s, %(version)s, " + \
"%(isThermometer)s, %(isBarometer)s, %(isHygrometer)s, %(isClock)s); "
host = 'localhost'
user = '******'
password = '******'
database = 'weather'
logger = WLogger(loggerName='weather.tools')
logger.logMessage("Starting...")
hostlist = [{'host:': 'localhost', 'port': 9200}]
#hostlist = [
# {'host':'elastic00','port':9200},
# {'host':'elastic01','port':9200},
# {'host':'elastic02','port':9200},
# ]
def scanIndex(indexName, filtered):
doc = WeatherData(using=client)
s_filt = doc.search(using=client,index=indexName).\
filter('range', **{'tsa': {'lt':20180916001872}})
s_all = doc.search(using=client, index=indexName)
class WeatherDB(object):
"""
WeatherDB: Class to manage the storage of observations into a postgresql
database.
The class contains methods to reconnect to the database and to do the
insertion of the rows containing the observations.
"""
_logger = WLogger()
def __init__(self, host, user, password, database):
"""
Establish a postgresql connection and
ready the WeatherDB object.
It tries to connect once. If the connection is not posible it
doesn't abort; the connection object is set to None it can be
retried afterwards.
Parameters:
- host: machine hosting the pgsql instalce
- user: connection username
- password: connection password
- database: database name
"""
try:
cur = None
self.theConn = None
self._theHost = host
self._theUser = user
self._thePassword = password
self._theDatabase = database
self.theConn = pg.connect(host=host,
user=user,
password=password,
database=database)
cur = self.theConn.cursor()
cur.execute('set search_path to \'weather\';')
WeatherDB._logger.logMessage(level="INFO",
message="Connection to database {0:s} on host {1:s} established." \
.format(database,host))
except:
WeatherDB._logger.logException(message="Connection to database {0:s} on host {1:s} failed." \
.format(database,host))
finally:
if cur is not None:
cur.close()
def close(self):
"""
Close the connection.
"""
if self.theConn is not None:
self.theConn.close()
self.theConn = None
def reconnect(self):
"""
reconnect: try to connect to the postgres database
Returns:
True if the connection was made
False otherwise
"""
try:
cur = None
self.theConn = pg.connect(host=self._theHost,
user=self._theUser,
password=self._thePassword,
database=self._theDatabase)
cur = self.theConn.cursor()
cur.execute('set search_path to \'WEATHER\';')
WeatherDB._logger.logMessage(level="INFO",
message="Connection to database {0:s} on host {1:s} established." \
.format(self._theDatabase,self._theHost))
return True
except:
WeatherDB._logger.logException(message="Connection to database {0:s} on host {1:s} failed." \
.format(self._theDatabase,self._theHost))
return False
finally:
if cur is not None:
cur.close()
def insertObs(self, theObservation):
if self.theConn is not None:
with self.theConn as conn:
with self.theConn.cursor() as c:
dic = theObservation.to_dict()
dic['esDocId'] = theObservation.meta.id
if dic['temperature'] == -999:
dic['temperature'] = None
if dic['pressure'] == -999:
dic['pressure'] = None
if dic['humidity'] == -999:
dic['humidity'] = None
c.execute(_INSERT_OBS, dic)
conn.commit()
WeatherDB._logger.logMessage(
level="DEBUG",
message="Inserted row: {0}".format(theObservation.tsa))
else:
raise pg.InterfaceError()
class WeatherDBThread(threading.Thread):
"""
Database updating thread
"""
_logger = WLogger()
def __init__(self, weatherQueue, weatherDb, event, retryInterval=5):
super(WeatherDBThread, self).__init__()
self.theDb = weatherDb
self.theQueue = weatherQueue
self.theEvent = event
self.name = 'WeatherDBThread'
self._stopSwitch = False
self.theRetryInterval = retryInterval
def stop(self):
self._stopSwitch = True
def run(self):
WeatherDBThread._logger.logMessage("Starting thread {0}.".format(
self.getName()),
level="INFO")
while not self._stopSwitch:
self.theEvent.wait()
q = self.theQueue.getDbQueue()
WeatherDBThread._logger.logMessage(
level='DEBUG',
message="{0} items to insert in database".format(len(q)))
for item in q:
line = item[1]
newTsa = item[0]
stamp,temp,humt,pres,lght,firmware,hardware,devName,clock,\
thermometer,hygrometer,barometer = parseLine(line)
doc = WeatherData()
doc.init(_tsa=newTsa,
_time=stamp,
_temperature=temp,
_humidity=humt,
_pressure=pres,
_light=lght,
_fwVersion=firmware,
_hwVersion=hardware,
_devName=devName,
_isBarometer=barometer,
_isClock=clock,
_isThermometer=thermometer,
_isHygrometer=hygrometer)
try:
self.theDb.insertObs(doc)
self.theQueue.markDbQueue(newTsa)
except pg.IntegrityError as ie:
WeatherDBThread._logger.logMessage(
level="ERROR",
message="Can't store tsa {0}: {1}".format(newTsa, ie))
except pg.InterfaceError as ex:
WeatherDBThread._logger.logException(
message="Can't talk to postgresql ({0})".format(ex))
connected = False
while not self._stopSwitch and not connected:
WeatherDB._logger.logMessage(
level="INFO",
message="Waiting {0} seconds to retry".format(
self.theRetryInterval))
time.sleep(self.theRetryInterval)
connected = self.theDb.reconnect()
except:
WeatherDBThread._logger.logException(
'Exception trying to store observation {0}'.format(
newTsa))
self.theEvent.clear()
WeatherDBThread._logger.logMessage(
"Thread {0} stopped by request.".format(self.getName()),
level="INFO")
script_path = os.path.abspath(os.path.dirname(__file__))
sys.path.append(script_path)
from weatherLib.weatherQueue import WeatherQueue, QueueJanitorThread
from weatherLib.weatherBT import WeatherBTThread
from weatherLib.weatherUtil import WLogger
from weatherLib.weatherDB import WeatherDB, WeatherDBThread
from weatherLib.weatherES import WeatherES, WeatherESThread
from weatherLib.watchdog import WatchdogThread
dbThread = None
esThread = None
janitorThread = None
watchdogThread = None
logger = WLogger()
dataEvent = threading.Event()
dataEvent.clear()
config = configparser.ConfigParser()
cf = config.read([
'/etc/weartherClient.ini', '/usr/local/etc/weatherClient.ini',
'weatherClient.ini'
])
logger.logMessage(
level="INFO",
message="Configuration loaded from configuration files [{l}]".format(l=cf))
data_dir = config['data']['directory']
w_address = config['bluetooth']['address']
class WeatherQueue(object):
"""
Weather measurements queue.
Implemented on a sqlite3 database
"""
def __init__(self,dbdir):
"""
Initialize the queue database connection and, if necessary,
create the database. Also create the lock object that will
be used to synchronize access
"""
self.logger = WLogger()
self.theLock = threading.Lock()
self.curDay = 0
self.curTSA = 0
ini_file = pkg_resources.resource_filename(__name__,'./database/wQueue.ini')
config = configparser.ConfigParser()
config.read([ini_file])
tableDDL = config['queueDatabase']['table']
tsasDDL = config['queueDatabase']['control']
indexESDDL = config['queueDatabase']['indexES']
indexDBDDL = config['queueDatabase']['indexDB']
dbFile = os.path.join(dbdir,'wQueue.db')
try:
self.theConn = sqlite3.connect(dbFile,check_same_thread=False)
self.theConn.isolation_level = 'IMMEDIATE'
self.theConn.execute(tableDDL)
self.theConn.execute(indexESDDL)
self.theConn.execute(indexDBDDL)
self.theConn.execute(tsasDDL)
self.theConn.commit()
self.logger.logMessage(level="INFO",message="Queue database opened at {0:s}".format(dbFile))
except:
self.logger.logException('Error initializing queue database')
def pushLine(self,line):
"""
Push a line into the queue.
This function blocks until the database is not locked
"""
stamp,_,_,_,_,_,_,_,_,_,_,_ = parseLine(line)
datestamp = calendar.timegm(stamp.date().timetuple())
theTsa = 1
with self.theLock:
try:
result = self.theConn.execute(_SELECT_TSA, [datestamp])
resCol = result.fetchone()
if resCol == None:
self.theConn.execute(_INSERT_DAY, [datestamp])
else:
theTsa = resCol[0] + 1
self.theConn.execute(_UPDATE_TSA, [theTsa, datestamp])
fullTsa = (stamp.year * 10000 +
stamp.month * 100 +
stamp.day) * 1000000 + theTsa
self.theConn.execute(_INSERT_QUEUE, [fullTsa,line])
self.theConn.commit()
except:
self.logger.logException('Error inserting line into the queue database')
self.theConn.rollback()
def getDbQueue(self):
"""
Get al the queue lines NOT marked as inserted into the database.
(isDB == 0)
"""
with self.theLock:
try:
result = self.theConn.execute(_SELECT_DB)
queueContent = result.fetchall()
return queueContent
except:
self.logger.logException('Error fetching DB queue')
self.theConn.rollback()
return None
def markDbQueue(self, theId):
"""
Mark a queue entry as inserted into the database
Parameters:
- theId: row identifier to mark
"""
with self.theLock:
with self.theConn:
self.theConn.execute(_UPDATE_DB, [theId])
self.theConn.commit()
self.logger.logMessage(level='DEBUG',
message = 'Queue entry {0} marked as DB-done'.format(theId))
def getESQueue(self):
"""
Get al the queue lines NOT marked as indexed in elasticserch.
(isES == 0)
"""
with self.theLock:
try:
result = self.theConn.execute(_SELECT_ES)
queueContent = result.fetchall()
return queueContent
except:
self.logger.logException('Error fetching ES queue')
self.theConn.rollback()
return None
def markESQueue(self, theId):
"""
Mark a queue entry as indexed in elasticsearch
Parameters:
- theId: row identifier to mark
"""
with self.theLock:
with self.theConn:
self.theConn.execute(_UPDATE_ES, [theId])
self.theConn.commit()
self.logger.logMessage(level='DEBUG',
message = 'Queue entry {0} marked as ES-done'.format(theId))
def purgeQueue(self):
with self.theLock:
with self.theConn as conn:
result = conn.execute(_COUNT_QUEUE)
r = result.fetchone()
count = r[0]
self.logger.logMessage(message="About to purge {0} queue entries.".format(count))
conn.execute(_PURGE_QUEUE)
conn.commit()
self.logger.logMessage(message="Queue purged.")
script_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir))
sys.path.append(script_path)
from weatherLib.weatherDoc import WeatherData
from weatherLib.weatherUtil import WLogger
_UPDATE_DOC = 'update weather set esDocId = %(esDocId)s where tsa = %(tsa)s;'
_SELECT_DOC = 'select tsa,time from weather where esDocId is null order by tsa;'
host = 'localhost'
user = '******'
password = '******'
database = 'weather'
logger = WLogger(loggerName='weather.tools')
logger.logMessage("Starting...")
hostlist = [
{
'host': 'elastic00',
'port': 9200
},
{
'host': 'elastic01',
'port': 9200
},
{
'host': 'elastic02',
'port': 9200
},
class WeatherBT(object):
_logger = WLogger()
def __init__(self, addr, serv):
"""
Setup the bluetooth connection object
Parameters:
- address: BT address of the device, in hex form (XX:XX:XX:XX:XX:XX)
- service: UUID of the RFCOMM service in the device
"""
WeatherBT._logger.logMessage(
level="DEBUG",
message="Service: {0:s}, address:{1:s}".format(serv, addr))
srvlist = bt.find_service(uuid=serv, address=addr)
if len(srvlist) == 0:
msg = "BT service not available."
WeatherBT._logger.logMessage(level="WARNING", message=msg)
raise ConnError
else:
srv = srvlist[0]
port = srv["port"]
name = srv["name"]
host = srv["host"]
sock = bt.BluetoothSocket(bt.RFCOMM)
sock.connect((host, port))
self.theSocket = sock
self.theName = name
def getLine(self):
"""
Read a line from a socket connection.
It reads characters from a socket until it gets a CR+LF combination. The
CR+LF is *not* returned as part of the read line.
If a line does not include the LF (the terminator is just a CR, it's
discarded.
Returns:
Received string
"""
line = ""
onLoop = True # End of loop switch
while onLoop:
byte = self.theSocket.recv(1) # Get byte from socket
if byte == b'\r': # Carriage return?
byte = self.theSocket.recv(1) # Consume LF
if byte != b'\n': # IF not LF, big trouble: discard line
line = ""
else:
onLoop = False # End of loop, line ready
else:
try:
line = line + byte.decode(
) # Add character to current working line
except UnicodeDecodeError as e:
msg = "Error decoding received byte: {0:s}".format(repr(e))
WeatherBT._logger.logMessage(level="WARNING", message=msg)
return line # The line is complete
def send(self, line):
"""
Send a string to the underlying BT socket
Parameters:
- line: string to send
"""
self.theSocket.send(line)
self.theSocket.send("\r\n")
def waitAnswer(self, answer, retries=5):
"""
Wait for a specific answer, discarding all the read lines until that
answer is read or the number of retries is exhausted.
Parameters:
- answer: text (6 characters) to expect
- retries: Number of lines to read until leaving
Returns:
Boolean (true = anwer found, false = retries exhausted)
"""
answ = ""
remain = retries
while answ != answer[0:6] and remain > 0:
line = self.getLine()
self._logger.logMessage(level="INFO", message=line)
answ = line[0:6]
remain -= 1
return answ == answer[0:6]
def close(self):
self.theSocket.close()
self.theName = None
class WeatherBTThread(threading.Thread):
"""
This class implements a thread to read the data coming from the Bluetooth
device.
"""
_logger = WLogger()
def __init__(self,
address,
service,
queue,
event,
directory,
pollInterval=15):
super(WeatherBTThread, self).__init__()
self.name = 'WeatherBTThread'
self.theDirectory = directory
self.theEvent = event
self.theAddress = address
self.theService = service
self.thePollInterval = pollInterval
self.theQueue = queue
self._stopSwitch = False
def stop(self):
self._stopSwitch = True
def connect_wait(self, times=10):
"""
Connect (create) a BT object, with timed retries
Parameters:
- address: BT address of the device, in hex form (XX:XX:XX:XX:XX:XX)
- service: UUID of the RFCOMM service in the device
Returns:
The created object
"""
numTries = 0
delay = 5
while numTries < times and not self._stopSwitch:
try:
theBT = WeatherBT(addr=self.theAddress, serv=self.theService)
WeatherBT._logger.logMessage(level="INFO",message="Connected to weather service at {0:s} : {1:s}" \
.format(theBT.theName,self.theAddress))
return theBT
except:
self._logger.logMessage(
level="WARNING",
message="BT Connection atempt {0} failed".format(numTries +
1))
tm.sleep(delay)
numTries += 1
return None
def run(self):
self._logger.logMessage("Starting thread {0}.".format(self.getName()),
level="INFO")
gizmo = None
gizmo = self.connect_wait()
if gizmo == None and not self._stopSwitch:
raise ConnError
currentDay = datetime.strptime(
'1970-01-01', '%Y-%m-%d').date() # Initialize to "zero" date
f = None
self._logger.logMessage(message="Start weather processing.",
level="INFO")
while not self._stopSwitch:
# Check date change and open new file if necessary
thisDay = datetime.utcnow().date()
if thisDay != currentDay:
currentDay = thisDay
if (f != None): f.close()
f = openFile(self.theDirectory)
self._logger.logMessage(
f'Opened raw log file for date {thisDay} : {f.name}')
try:
line = gizmo.getLine()
cmd = line[0:5]
if cmd == "DATA ": # It is a data line so...
f.write(line + '\n') # ... write it!
f.flush() # Don't wait, write now!
self.theQueue.pushLine(line)
self.theEvent.set() # Send event: data received
elif cmd == "DEBUG":
self._logger.logMessage(level="DEBUG", message=line)
elif cmd == "INFO:":
self._logger.logMessage(level="INFO", message=line)
elif cmd == "ERROR":
self._logger.logMessage(
level="WARNING",
message="Error in firmware/hardware: {0:s}".format(
line))
elif cmd == "HARDW":
self._logger.logMessage(level="CRITICAL", message=line)
elif cmd == "TIME?":
self._logger.logMessage(
level="INFO",
message="Setting time as requested by the device.")
now = tm.gmtime() # So send current time to set RTC...
timcmd = "TIME " + tm.strftime("%Y%m%d%H%M%S", now)
self._logger.logMessage(
level="INFO",
message="Setting time, command: {0:s}".format(timcmd))
gizmo.send(timcmd)
gizmo.waitAnswer("OK-000")
elif cmd == "BEGIN":
now = tm.gmtime() # So send current time to set RTC...
timcmd = "TIME " + tm.strftime("%Y%m%d%H%M%S", now)
self._logger.logMessage(
level="INFO",
message="Setting time, command: {0:s}".format(timcmd))
gizmo.send(timcmd)
gizmo.waitAnswer("OK-000")
dlycmd = f"DLAY {self.thePollInterval:d}"
self._logger.logMessage(
(f"Setting the poll interval, command: {dlycmd}"))
gizmo.send(dlycmd)
gizmo.waitAnswer("OK-000")
else:
self._logger.logMessage(
level="WARNING",
message="Non-processable line: {0:s}".format(line))
except:
self._logger.logMessage(level="CRITICAL",
message="Error while reading gizmo")
raise
if not gizmo == None:
gizmo.send(b'BYE ')
gizmo.waitAnswer("OK-BYE")
gizmo.close()
if self._stopSwitch:
self._logger.logMessage("Thread {0} stopped by request.".format(
self.getName()),
level="INFO")
'host': 'elastic01',
'port': 9200
},
{
'host': 'elastic02',
'port': 9200
},
]
script_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir))
sys.path.append(script_path)
from weatherLib.weatherUtil import WLogger
logger = WLogger(loggerName='weather.tools')
def step010():
"""
Dump the elasticsearch index into a file
"""
client = es.Elasticsearch(hostlist)
search = {
"_source": {
"includes": ["time", "tsa"]
},
"query": {
"exists": {
"field": "tsa"
}
class WeatherESThread(threading.Thread):
_logger = WLogger()
def __init__(self, weatherQueue, weatherES, event):
super(WeatherESThread, self).__init__()
self.theES = weatherES
self.theQueue = weatherQueue
self.theEvent = event
self.name = 'WeatherESThread'
self._stopSwitch = False
def stop(self):
self._stopSwitch = True
def run(self):
WeatherESThread._logger.logMessage("Starting thread {0}.".format(
self.getName()),
level="INFO")
templname = 'weather-' + VERSION + '-*'
while not self._stopSwitch:
try:
ic = elasticsearch.client.IndicesClient(self.theES.theClient)
if not ic.exists_template(templname):
templFileName = 'weather-' + VERSION + '-template.json'
with open(templFileName) as templFile:
templateBody = templFile.read()
ic.put_template(name=templname, body=templateBody)
WeatherESThread._logger.logMessage(
level="INFO",
message="Template {0} created.".format(templname))
else:
WeatherESThread._logger.logMessage(
level="INFO",
message="Template {0} already exists.".format(
templname))
break
except:
WeatherESThread._logger.logException(
message='Error trying to create the document template.')
tm.sleep(5)
if self._stopSwitch:
WeatherESThread._logger.logMessage(
"Thread {0} stopped by request.".format(self.getName()),
level="INFO")
else:
WeatherESThread._logger.logMessage(
level="INFO", message="ES template established.")
while not self._stopSwitch:
self.theEvent.wait()
q = self.theQueue.getESQueue()
WeatherESThread._logger.logMessage(
level='DEBUG', message="{0} docs to index".format(len(q)))
for item in q:
line = item[1]
newTsa = item[0]
stamp, temp, humt, pres, lght, firmware, hardware, devName, clock, thermometer, hygrometer, barometer = parseLine(
line)
doc = WeatherData()
doc.init(_tsa=newTsa,
_time=stamp,
_temperature=temp,
_humidity=humt,
_pressure=pres,
_light=lght,
_fwVersion=firmware,
_hwVersion=hardware,
_devName=devName,
_isBarometer=barometer,
_isClock=clock,
_isThermometer=thermometer,
_isHygrometer=hygrometer)
try:
doc.save(client=self.theES.theClient)
WeatherES._logger.logMessage(
level="DEBUG",
message="Indexed doc: {0}".format(doc.tsa))
self.theQueue.markESQueue(newTsa)
except:
WeatherESThread._logger.logException(
'Exception trying to push observation {0}'.format(
newTsa))
self.theEvent.clear()
WeatherESThread._logger.logMessage(
"Thread {0} stopped by request.".format(self.getName()),
level="INFO")