Can be used after airfields.json got extended by new locations.
"""
from time import sleep
from airfieldManager import AirfieldManager
from configuration import dbConnectionInfo
from db.DbSource import DbSource
from db.DbThread import DbThread
if __name__ == '__main__':
afm = AirfieldManager()
dbt = DbThread(dbConnectionInfo=dbConnectionInfo)
dbt.start()
dbs = DbSource(dbConnectionInfo=dbConnectionInfo)
strSql = 'select id, takeoff_icao, takeoff_lat, takeoff_lon, landing_icao, landing_lat, landing_lon from logbook_entries where takeoff_icao is null or landing_icao is null;'
cur = dbs.getConnection().cursor()
cur.execute(strSql)
numUpdatedRecords = 0
for row in cur:
(id, takeoffIcao, takeoffLat, takeoffLon, landingIcao, landingLat,
landingLon) = row
if not takeoffIcao and takeoffLat and takeoffLon:
takeoffIcao = afm.getNearest(takeoffLat, takeoffLon)
class RedisReaper(object):
RUN_INTERVAL = 5 * 60 # [s]
REDIS_STALE_INTERVAL_1 = 20 * 60 # [s]
REDIS_STALE_INTERVAL_2 = 30 * 60 # [s]
REDIS_TTL_LIMIT = REDIS_RECORD_EXPIRATION - REDIS_STALE_INTERVAL_1
GS_THRESHOLD = getGroundSpeedThreshold(1, 'L')
def __init__(self):
print(
f"[INFO] RedisReaper(lite) scheduled to run every {self.RUN_INTERVAL}s."
)
self.dbt = DbThread(dbConnectionInfo=dbConnectionInfo)
self.dbt.start()
self.redis = StrictRedis(**redisConfig)
self.influx = InfluxDbThread(dbName=INFLUX_DB_NAME,
host=INFLUX_DB_HOST)
self.airfieldManager = AirfieldManager()
def doWork(self):
airborne = []
keys = self.redis.keys('*status')
for key in keys:
key = key.decode('ascii')
ttl = self.redis.ttl(key)
value = self.redis.get(key)
if value: # entries could have been deleted in the mean while..
status = int(value.decode('ascii').split(';')[0])
if status == 1: # 1 = airborne
addr = key.split(
'-'
)[0] # in fact addressTypeStr + addr (e.g. I123456, F123456, O123456, ..)
airborne.append(addr)
numLanded = 0
for addr in airborne:
prefix = addr[:1]
addr = addr[1:]
addrType = REVERSE_ADDRESS_TYPE.get(prefix, None)
addrPrefixLong = ADDRESS_TYPE_PREFIX.get(addrType, None)
if not addrPrefixLong:
continue
rs = self.influx.client.query(
f"SELECT * FROM pos WHERE addr='{addrPrefixLong}{addr}' ORDER BY time DESC LIMIT 1;"
)
for res in rs:
# print('res:', res)
time = res[0]['time']
ts = int(
datetime.strptime(
time, '%Y-%m-%dT%H:%M:%SZ').timestamp()) # UTC ts
agl = res[0]['agl'] if res[0]['agl'] else 0
alt = res[0]['alt'] if res[0]['alt'] else 0
gs = res[0]['gs']
lat = res[0]['lat']
lon = res[0]['lon']
landingSuspected = False
if 0 < agl < 100 and gs < self.GS_THRESHOLD:
landingSuspected = True
else:
lastBeaconAge = datetime.utcnow().timestamp() - ts
if lastBeaconAge > self.REDIS_STALE_INTERVAL_2:
landingSuspected = True
if landingSuspected:
icaoLocation = self.airfieldManager.getNearest(lat, lon)
# if not icaoLocation: # no outlandings yet..
# continue
# print(f"addr: {addr}; dt: {dt / 60:.0f}min ; agl: {agl:.0f}m near {icaoLocation}")
# set status as Landed in redis (or delete?):
self.redis.set(
f"{prefix}{addr}-status", '0;0'
) # 0 = on-ground; ts=0 to indicate forced landing
self.redis.expire(key, REDIS_RECORD_EXPIRATION)
# look-up related takeoff data:
logbookItem: LogbookItem = findMostRecentTakeoff(
addr, addrType)
# create a LANDING logbook_event -> a stored procedure then creates a logbook_entry:
flightTime = ts - logbookItem.takeoff_ts
if flightTime < 0:
flightTime = 0
icaoLocationVal = f"'{icaoLocation}'" if icaoLocation else 'null'
strSql = f"INSERT INTO logbook_events " \
f"(ts, address, address_type, aircraft_type, event, lat, lon, location_icao, flight_time) " \
f"VALUES " \
f"({ts}, '{addr}', {logbookItem.address_type}, '{logbookItem.aircraft_type}', " \
f"'L', {lat:.5f}, {lon:.5f}, {icaoLocationVal}, {flightTime});"
# print('strSql:', strSql)
self.dbt.addStatement(strSql)
numLanded += 1
if numLanded > 0:
print(f"[INFO] RedisReaper: cleared {numLanded} stale records")
def stop(self):
self.dbt.stop()
class BeaconProcessor(object):
redis = StrictRedis(**redisConfig)
rawQueueOGN = Queue(
maxsize=666666666
) # 0 ~ infinite (according to docs).. but apparently not
rawQueueFLR = Queue(maxsize=666666666)
rawQueueICA = Queue(maxsize=666666666)
queues = (rawQueueOGN, rawQueueFLR, rawQueueFLR, rawQueueICA
) # one worker's performance on current CPU is 35k/min
queueIds = ('ogn', 'flarm1', 'flarm2', 'icao1')
# TODO there shall be separate queues for each worker and traffic shall be split/shaped evenly for every worker of the same kind..
workers = list()
def __init__(self):
# restore unprocessed data from redis:
numRead = 0
for key, queue in zip(self.queueIds, self.queues):
while True:
item = self.redis.lpop(key)
if not item:
break
queue.put(item)
numRead += 1
print(f"[INFO] Loaded {numRead} raw message(s) from redis.")
self.dbThread = DbThread(dbConnectionInfo)
self.dbThread.start()
self.influxDb = InfluxDbThread(dbName=INFLUX_DB_NAME,
host=INFLUX_DB_HOST)
self.influxDb.start()
for id, queue in zip(self.queueIds, self.queues):
rawWorker = RawWorker(id=id,
dbThread=self.dbThread,
rawQueue=queue,
influxDb=self.influxDb)
rawWorker.start()
self.workers.append(rawWorker)
self.timer = PeriodicTimer(60, self._processStats)
self.timer.start()
def stop(self):
for worker in self.workers:
worker.stop()
# store all unprocessed data into redis:
n = 0
for key, queue in zip(self.queueIds, self.queues):
n += queue.qsize()
for item in list(queue.queue):
self.redis.rpush(key, item)
print(f"[INFO] Flushed {n} rawQueueX items into redis.")
self.dbThread.stop()
self.influxDb.stop()
self.timer.stop()
print('[INFO] BeaconProcessor terminated.')
startTime = time.time()
numEnquedTasks = 0
def _processStats(self):
now = time.time()
tDiff = now - self.startTime
numTasksPerMin = self.numEnquedTasks / tDiff * 60
numQueuedTasks = self.rawQueueOGN.qsize() + self.rawQueueFLR.qsize(
) + self.rawQueueICA.qsize()
print(
f"[INFO] Beacon rate: {numTasksPerMin:.0f}/min. {numQueuedTasks} queued."
)
traffic = dict()
for worker in self.workers:
traffic[worker.id] = worker.numProcessed
worker.numProcessed = 0
if not DEBUG and numTasksPerMin >= 10:
cmd = f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/rate -m '{round(numTasksPerMin)}'; " \
f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/queued -m '{round(numQueuedTasks)}'; " \
f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/ogn -m '{traffic['ogn']}'; " \
f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/flarm -m '{traffic['flarm1'] + traffic['flarm2']}'; " \
f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/icao -m '{traffic['icao1']}';"
os.system(cmd)
self.numEnquedTasks = 0
self.startTime = now
def enqueueForProcessing(self, raw_message: str):
prefix = raw_message[:3]
if prefix == 'OGN':
self.rawQueueOGN.put(raw_message)
elif prefix == 'FLR':
self.rawQueueFLR.put(raw_message)
elif prefix == 'ICA':
self.rawQueueICA.put(raw_message)
else:
print(f'[WARN] Worker for "{prefix}" not implemented!',
raw_message,
file=sys.stderr)
return
self.numEnquedTasks += 1
class BeaconProcessor(object):
redis = StrictRedis(**redisConfig)
rawQueueOGN = Queue(maxsize=0) # 0 ~ infinite (according to docs)
rawQueueFLR = Queue(maxsize=0)
rawQueueICA = Queue(maxsize=0)
queues = (rawQueueOGN, rawQueueFLR, rawQueueICA)
queueIds = ('ogn', 'flarm', 'icao')
workers = list()
def __init__(self):
# restore unprocessed data from redis:
numRead = 0
for key, queue in zip(self.queueIds, self.queues):
while True:
item = self.redis.lpop(key)
if not item:
break
queue.put(item)
numRead += 1
print(f"[INFO] Loaded {numRead} raw message(s) from redis.")
self.dbThread = DbThread(dbConnectionInfo)
self.dbThread.start()
self.influxDb = InfluxDbThread(dbName=INFLUX_DB_NAME,
host=INFLUX_DB_HOST)
self.influxDb.start()
for id, queue in zip(self.queueIds, self.queues):
rawWorker = RawWorker(id=id,
dbThread=self.dbThread,
rawQueue=queue,
influxDb=self.influxDb)
rawWorker.start()
self.workers.append(rawWorker)
self.timer = PeriodicTimer(60, self._processStats)
self.timer.start()
def stop(self):
for worker in self.workers:
worker.stop()
# store all unprocessed data into redis:
n = 0
for key, queue in zip(self.queueIds, self.queues):
n += queue.qsize()
for item in list(queue.queue):
self.redis.rpush(key, item)
print(f"[INFO] Flushed {n} rawQueueX items into redis.")
self.dbThread.stop()
self.influxDb.stop()
self.timer.stop()
print('[INFO] BeaconProcessor terminated.')
startTime = time.time()
numEnquedTasks = 0
def _processStats(self):
now = time.time()
tDiff = now - self.startTime
numTasksPerMin = self.numEnquedTasks / tDiff * 60
numQueuedTasks = self.rawQueueOGN.qsize() + self.rawQueueFLR.qsize(
) + self.rawQueueICA.qsize()
print(
f"[INFO] Beacon rate: {numTasksPerMin:.0f}/min. {numQueuedTasks} queued."
)
traffic = dict()
for worker in self.workers:
traffic[worker.id] = worker.numProcessed
worker.numProcessed = 0
if not debugMode and numTasksPerMin >= 400:
cmd = f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/rate -m '{round(numTasksPerMin)}'; " \
f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/queued -m '{round(numQueuedTasks)}'; " \
f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/ogn -m '{traffic['ogn']}'; " \
f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/flarm -m '{traffic['flarm']}'; " \
f"mosquitto_pub -h {MQ_HOST} -p {MQ_PORT} -u {MQ_USER} -P {MQ_PASSWORD} -t ognLogbook/icao -m '{traffic['icao']}';"
os.system(cmd)
self.numEnquedTasks = 0
self.startTime = now
def enqueueForProcessing(self, raw_message: str):
prefix = raw_message[:3]
if prefix == 'OGN':
self.rawQueueOGN.put(raw_message)
elif prefix == 'FLR':
self.rawQueueFLR.put(raw_message)
else: # 'ICA'
self.rawQueueICA.put(raw_message)
self.numEnquedTasks += 1
class BeaconProcessor(object):
redis = StrictRedis(**redisConfig)
rawQueueOGN = Queue()
rawQueueFLR = Queue()
rawQueueICA = Queue()
queues = (rawQueueOGN, rawQueueFLR, rawQueueICA)
queueKeys = ('rawQueueOGN', 'rawQueueFLR', 'rawQueueICA')
workers = list()
def __init__(self):
# restore unprocessed data from redis:
numRead = 0
for key, queue in zip(self.queueKeys, self.queues):
while True:
item = self.redis.lpop(key)
if not item:
break
queue.put(item)
numRead += 1
print(f"[INFO] Loaded {numRead} raw message(s) from redis.")
self.dbThread = DbThread(dbConnectionInfo)
self.dbThread.start()
for i, queue in enumerate(self.queues):
rawWorker = RawWorker(index=i,
dbThread=self.dbThread,
rawQueue=queue)
rawWorker.start()
self.workers.append(rawWorker)
def stop(self):
for worker in self.workers:
worker.stop()
# store all unprocessed data into redis:
n = 0
for key, queue in zip(self.queueKeys, self.queues):
n += queue.qsize()
for item in list(queue.queue):
self.redis.rpush(key, item)
print(f"[INFO] Flushed {n} rawQueueX items into redis.")
self.dbThread.stop()
print('[INFO] BeaconProcessor terminated.')
startTime = time.time()
numEnquedTasks = 0
def _printStats(self):
now = time.time()
tDiff = now - self.startTime
if tDiff >= 60:
numTasksPerMin = self.numEnquedTasks / tDiff * 60
numQueuedTasks = self.rawQueueOGN.qsize() + self.rawQueueFLR.qsize(
) + self.rawQueueICA.qsize()
print(
f"Beacon rate: {numTasksPerMin:.0f}/min. {numQueuedTasks} queued."
)
self.numEnquedTasks = 0
self.startTime = now
def enqueueForProcessing(self, raw_message: str):
self._printStats()
prefix = raw_message[:3]
if prefix == 'OGN':
self.rawQueueOGN.put(raw_message)
elif prefix == 'FLR':
self.rawQueueFLR.put(raw_message)
else: # 'ICA'
self.rawQueueICA.put(raw_message)
self.numEnquedTasks += 1