def ws_restart_mod(self):
"""
Places all module statuses into queue when websocket handler is started/restarted
"""
# This method is only called when websocket is operating
# Cycle through all lanes
for addr_ln in self.stat_ln:
# Only place module statuses into MPQ_WS for operational lanes
if not addr_ln['status']:
# Get stat_lane from CouchDB modconfig database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='get_view',
cdb_name='modconfig',
cdb_doc='stat_lane{0}'.format(addr_ln['addr_ln']),
logfile=logfile)
if not stat_cdb:
# Cycle through all modules and place status values into MPQ_WS
for dict_mod in data_cdb_out:
try:
MPQ_WS.put([
'module', addr_ln['addr_ln'], dict_mod['key'],
dict_mod['value']['status']
])
except queue.Full:
log = 'Can not place item in module status queue, queue is full.'
logger.exception(log)
def ws_restart_ch(self):
"""
Places all lane statuses into queue when websocket handler is started/restarted
"""
# This method is only called when websocket is operating
# Cycle through all lanes and place lane values into MPQ_WS
for addr_ln in self.stat_ln:
try:
MPQ_WS.put([
'lane', addr_ln['addr_ln'], addr_ln['status'],
addr_ln['last_module'], addr_ln['poll'],
addr_ln['last_dtg'], addr_ln['setup_id']
])
except queue.Full:
log = 'Can not place item in lane status queue, queue is full.'
logger.exception(log)
def act_listener(self, ):
"""
Places activity messages into websocket queue as they are received.
mpq_record[0] = dtg
mpq_record[1] = status
mpq_record[2] = message
"""
log_level = 'DEBUG'
pid_websocket = False
while not self.stat_list:
# All JanusESS status information enters this point
if not MPQ_ACT_CMD.empty():
mpq_record = MPQ_ACT_CMD.get()
if mpq_record[0] == 'log_level':
if log_level != mpq_record[1]:
log_level = mpq_record[1]
if mpq_record[0] == 'websocket':
if pid_websocket != mpq_record[1]:
pid_websocket = mpq_record[1]
# Only retrieve MPQ_ACT if websocket handler is functioning.
if pid_websocket:
if not MPQ_ACT.empty():
mpq_record = MPQ_ACT.get()
# Compare activity logging level with activity statement, if activity
# statement has higher level, place it onto queue
if LOG_LVL[mpq_record[1]] >= LOG_LVL[log_level]:
try:
MPQ_WS.put([
'activity', mpq_record[0][:19] + ' ' +
mpq_record[1] + ': ' + mpq_record[2]
])
except queue.Full:
log = 'Can not place item in activity queue, malformed record or queue is full.'
logger.exception(log)
time.sleep(0.02)
def websocket(
self,
):
"""
Starts websocket multiprocessing queue listener
mpq_record[0] = type
mpq_record[1] to mpq_record[?] = data
"""
while True:
# Check for loop stop/exit condition
if not self.mp_ws_listener_stop.empty():
message = self.mp_ws_listener_stop.get()
if message is None:
break
# Get record from queue
if not MPQ_WS.empty():
mpq_record = MPQ_WS.get()
# Build message for websocket
message = ''
for field in mpq_record:
message = message + str(field) + "/"
message = message[:-1]
# Write message to websocket and trap any errors
try:
self.ws.send(message)
except geventwebsocket.WebSocketError:
MPQ_WS.put(mpq_record)
log = 'Can not put message on websocket due to socket closed error: key {0}, status {1}.'. \
format(mpq_record[0], mpq_record[1])
logger.exception(log)
print(log)
break
time.sleep(0.02)
def poll(self, mpq_record: list):
"""
Places polling values into websocket and SNMP queues as they are received.
:param mpq_record: list
mpq_record[0][0] = module id
mpq_record[0][1] = customer id
mpq_record[0][2] = base unit id
mpq_record[0][3] = module location
mpq_record[0][4] = lane address
mpq_record[0][5] = module address
mpq_record[1][0] = sensor address
mpq_record[1][1] = alert type
mpq_record[1][2] = alert enabled
mpq_record[1][3] = sensor type
mpq_record[1][4] = sensor value
mpq_record[1][5] = sensor value dtg
mpq_record[1][6] = sensor value unit
mpq_record[1][7] = trigger
mpq_record[1][8] = trigger interval
mpq_record[1][9] = trigger step
"""
from shared.messaging.smtp import send_mail
uid_mod = mpq_record[0][0]
mod_loc = mpq_record[0][3]
addr_ln = mpq_record[0][4]
addr_mod = mpq_record[0][5]
addr_s = mpq_record[1][0]
alert_type = mpq_record[1][1]
s_type = mpq_record[1][3]
converted_val = mpq_record[1][4]
data_dtg = mpq_record[1][5]
unit = mpq_record[1][6]
trigger = mpq_record[1][7]
trig_int = mpq_record[1][8]
trig_step = mpq_record[1][9]
# Cycle through poll_value list to search for matching module and sensor
# If found process alert value
mod_found = False
message = False
msg_type = None
msg_args = None
# Get lane_status document from CouchDB lanes database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='get_doc',
cdb_name='config',
cdb_doc='dataunits',
logfile=logfile)
# Convert sensor value to user-selected unit
s_value, s_unit = self.unit_convert.convert(converted_val, unit,
data_cdb_out[s_type])
# Convert low-threshold value to user-selected unit
t_value, t_unit = self.unit_convert.convert(trigger, unit,
data_cdb_out[s_type])
for dict_alert in self.poll_val:
if dict_alert['mod_uid'] == uid_mod:
# A new alert period is triggered
if (dict_alert['sensors'][addr_s]['type']
== 'off') and (alert_type != 'off'):
dict_alert_hist = {
'trigger': t_value,
'value': s_value,
'unit': s_unit,
'data_dtg': data_dtg
}
dict_alert['sensors'][addr_s]['prev_value'] = s_value
dict_alert['sensors'][addr_s]['value'] = s_value
dict_alert['sensors'][addr_s]['type'] = alert_type
dict_alert['sensors'][addr_s]['data_dtg'] = data_dtg
dict_alert['sensors'][addr_s]['history'].append(
dict_alert_hist)
# Email settings for reporting new alert period
msg_type = 'alert_new'
msg_args = [[], 'polling']
message = True
# Service existing low alert
elif (dict_alert['sensors'][addr_s]['type']
== 'low') and (alert_type == 'low'):
# Existing low alert is stable
if (dict_alert['sensors'][addr_s]['value'] >
(dict_alert['sensors'][addr_s]['prev_value'] - trig_step)) and \
(dict_alert['sensors'][addr_s]['value'] <
(dict_alert['sensors'][addr_s]['prev_value'] + trig_step)):
# Email setting for reporting stable low alert
msg_type = 'alert_stable'
# Existing low alert with increased value
elif dict_alert['sensors'][addr_s]['value'] >= \
(dict_alert['sensors'][addr_s]['prev_value'] + trig_step):
dict_alert['sensors'][addr_s]['prev_value'] = s_value
msg_type = 'alert_increased'
# Existing low alert with decreased value
elif dict_alert['sensors'][addr_s]['value'] <= \
(dict_alert['sensors'][addr_s]['prev_value'] - trig_step):
dict_alert['sensors'][addr_s]['prev_value'] = s_value
msg_type = 'alert_decreased'
# If trigger interval has elapsed, conduct reporting
if data_dtg >= (dict_alert['sensors'][addr_s]['data_dtg'] +
trig_int):
dict_alert_hist = {
'trigger': t_value,
'value': s_value,
'unit': s_unit,
'data_dtg': data_dtg
}
dict_alert['sensors'][addr_s]['value'] = s_value
dict_alert['sensors'][addr_s]['type'] = alert_type
dict_alert['sensors'][addr_s]['data_dtg'] = data_dtg
dict_alert['sensors'][addr_s]['history'].append(
dict_alert_hist)
msg_args = [[],
dict_alert['sensors'][addr_s]['history'],
'polling']
message = True
# Service existing high alert
elif (dict_alert['sensors'][addr_s]['type']
== 'high') and (alert_type == 'high'):
# Existing high alert is stable
if (dict_alert['sensors'][addr_s]['value'] >
(dict_alert['sensors'][addr_s]['prev_value'] - trig_step)) and \
(dict_alert['sensors'][addr_s]['value'] <
(dict_alert['sensors'][addr_s]['prev_value'] + trig_step)):
# Email setting for reporting stable high alert
msg_type = 'alert_stable'
# Existing high alert with increased value
elif dict_alert['sensors'][addr_s]['value'] >= \
(dict_alert['sensors'][addr_s]['prev_value'] + trig_step):
dict_alert['sensors'][addr_s]['prev_value'] = s_value
msg_type = 'alert_increased'
# Existing high alert with decreased value
elif dict_alert['sensors'][addr_s]['value'] <= \
(dict_alert['sensors'][addr_s]['prev_value'] - trig_step):
dict_alert['sensors'][addr_s]['prev_value'] = s_value
msg_type = 'alert_decreased'
# If trigger interval has elapsed, conduct reporting
if data_dtg >= (dict_alert['sensors'][addr_s]['data_dtg'] +
trig_int):
dict_alert_hist = {
'trigger': t_value,
'value': s_value,
'unit': s_unit,
'data_dtg': data_dtg
}
dict_alert['sensors'][addr_s]['value'] = s_value
dict_alert['sensors'][addr_s]['type'] = alert_type
dict_alert['sensors'][addr_s]['data_dtg'] = data_dtg
dict_alert['sensors'][addr_s]['history'].append(
dict_alert_hist)
msg_args = [[],
dict_alert['sensors'][addr_s]['history'],
'polling']
message = True
# Existing alert period is ended
elif (dict_alert['sensors'][addr_s]['type'] !=
'off') and (alert_type == 'off'):
dict_alert['sensors'][addr_s]['prev_value'] = None
dict_alert['sensors'][addr_s]['value'] = None
dict_alert['sensors'][addr_s]['type'] = alert_type
dict_alert['sensors'][addr_s]['data_dtg'] = None
dict_alert['sensors'][addr_s]['history'] = []
# Report end of alert period
msg_type = 'alert_cancel'
msg_args = [[], 'polling']
message = True
mod_found = True
break
# A new alert period is issued for a new module
if not mod_found:
dict_mod = {'mod_uid': uid_mod, 'sensors': []}
# Cycle through all possible sensors to build alert structure
# Some sensors are either not installed or may never issue alert data
for sensor_id in range(0, 17):
dict_alert = {
'data_dtg': None,
'type': 'off',
'prev_value': None,
'value': None,
'unit': None,
'history': []
}
dict_mod['sensors'].append(dict_alert)
# Update data for specific sensor number
if alert_type != 'off':
dict_mod['sensors'][addr_s]['data_dtg'] = data_dtg
dict_mod['sensors'][addr_s]['type'] = alert_type
dict_mod['sensors'][addr_s]['prev_value'] = s_value
dict_mod['sensors'][addr_s]['value'] = s_value
dict_history = {
'trigger': t_value,
'value': s_value,
'unit': s_unit,
'data_dtg': data_dtg
}
dict_mod['sensors'][addr_s]['history'].append(dict_history)
# Email settings for reporting new alert period
msg_type = 'alert_new'
msg_args = [mpq_record, 'polling']
message = True
self.poll_val.append(dict_mod)
# If changes take place in alert status, send messaging message
if message:
msg_args[0] = [
uid_mod,
mod_loc,
addr_ln,
addr_mod,
addr_s,
alert_type,
s_type,
s_value,
data_dtg,
s_unit,
t_value,
]
send_mail(
msg_type=msg_type,
args=msg_args,
)
# Only place into MPQs if processes/threads are functioning.
try:
if self.pid_websocket:
MPQ_WS.put([
'poll',
addr_ln,
addr_mod,
addr_s,
mod_loc,
s_type,
s_value,
s_unit,
strftime("%H:%M:%S", time.localtime(float(data_dtg))),
alert_type,
t_value,
])
if self.tid_snmp_agent:
self.MPQ_SNMPA5.put([
uid_mod,
addr_ln,
addr_mod,
addr_s,
s_type,
s_value,
s_unit,
str(ctime(float(data_dtg))),
alert_type,
t_value,
])
if self.tid_snmp_notify and message:
self.MPQ_SNMPN5.put([
uid_mod,
addr_ln,
addr_mod,
addr_s,
s_type,
s_value,
s_unit,
str(ctime(float(data_dtg))),
alert_type,
t_value,
])
except queue.Full:
log = 'Can not place item in poll value queue, queue is full.'
logger.exception(log)
def module(self, mpq_record: list):
"""
Places module statues into websocket and SNMP queues if status has changed.
mpq_record[0] = module id
mpq_record[1] = lane address
mpq_record[2] = module address
mpq_record[3] = module status
:param mpq_record: list
"""
uid_mod = mpq_record[0]
addr_ln = mpq_record[1]
addr_mod = mpq_record[2]
stat_mod = mpq_record[3]
# Only process for valid module ids, a module with '0000' failed I2C write
# during setup
#
# NOTE: Do not change '0000' here without also changing application.setup.module.setup()
if (uid_mod != '0000000000000000000000000000000000000000000000000000000000000000') and \
(uid_mod is not None):
print('HEARTBEAT MODULE: {0}'.format(mpq_record))
# Get stat_lane view from CouchDB modconfig database
# This determines if lane status should be downgraded if all modules error.
data0_cdb_out, stat0_cdb, http0_cdb = dbase.cdb_request(
cdb_cmd='get_view',
cdb_name='modconfig',
cdb_doc='stat_lane{0}'.format(addr_ln),
logfile=logfile)
ln_reset_flag = False
if not stat0_cdb and not isinstance(data0_cdb_out, int):
# Cycle through modules until module is located
for dict_mod in data0_cdb_out:
if (uid_mod == dict_mod['id']) and (addr_mod
== dict_mod['key']):
type_mod = dict_mod['value']['mod_type']
ver_mod = dict_mod['value']['mod_ver']
loc_mod = dict_mod['value']['loc']
num_sensors = dict_mod['value']['num_sensors']
# Initialize module database entry
mod_errno = dict_mod['value']['errno']
data_cdb_in = {'status': stat_mod}
# If module is operational, ensure module config errno field is reset
if stat_mod == STAT_LVL['op']:
mod_errno = 0
data_cdb_in['errno'] = mod_errno
# Modules do not report statuses above STAT_LVL['op_err'], this allows
# for heartbeat to initiate module recover processes
#
# If module reports an operational error,
# increment module config errno field
if stat_mod == STAT_LVL['op_err']:
mod_errno += 1
data_cdb_in['errno'] = mod_errno
log = 'Lane {0} module {1} id {2} error incremented to {3}'.\
format(addr_ln, addr_mod, uid_mod, mod_errno)
print(log)
logger.warning(log)
# If three attempts are made to communicate with module, set status to
# STAT_LVL['crit']
if mod_errno == 3:
print('Pre Auto lane reset var: {0}'.format(
AUTO_LNRST))
if AUTO_LNRST[addr_ln] <= 3:
AUTO_LNRST[addr_ln] += 1
ln_reset_flag = True
else:
ln_reset_flag = False
print('Post Auto lane reset var: {0}'.format(
AUTO_LNRST))
data_cdb_in['status'] = STAT_LVL['crit']
stat_mod = STAT_LVL['crit']
log = 'Lane {0} module {1} '.format(addr_ln, addr_mod, uid_mod) +\
'id {0} error incremented to {1}, '.format(uid_mod, mod_errno) +\
'now considered failed.'
logger.critical(log)
# Update module document in CouchDB modconfig database.
# This document must be updated prior to any module
# recovery actions are executed.
data1_cdb_out, stat1_cdb, http1_cdb = dbase.cdb_request(
cdb_cmd='upd_doc',
cdb_name='modconfig',
cdb_doc=uid_mod,
data_cdb_in=data_cdb_in,
logfile=logfile)
# Execute lane/module recovery routine if module failed
#
# TODO: When designed, add module re-setup function here
if (not stat1_cdb) and (mod_errno == 3) and \
(stat_mod == STAT_LVL['crit']):
# MPQ_CMD3.put_no_wait([
# addr_ln
# ])
print('Reset lane called.')
if ln_reset_flag and not FLAG_LNRST[addr_ln]:
MPQ_CMD3.put_nowait([addr_ln])
# Only place into MPQs if processes/threads are functioning.
try:
if self.pid_websocket:
MPQ_WS.put(
['module', addr_ln, addr_mod, stat_mod])
if addr_mod is not None:
if self.tid_snmp_agent:
self.MPQ_SNMPA4.put([
addr_ln, addr_mod, type_mod, ver_mod,
loc_mod, stat_mod, num_sensors
])
if self.tid_snmp_notify:
self.MPQ_SNMPN4.put([
addr_ln, addr_mod, type_mod, ver_mod,
loc_mod, stat_mod, num_sensors
])
except queue.Full:
log = 'Can not place item in module status queue, queue is full.'
logger.exception(log)
break
# Cycle through modules to determine if lane is inop
stat_ln = STAT_LVL['crit']
for dict_mod in data0_cdb_out:
if dict_mod['value']['status'] <= STAT_LVL['op_err']:
stat_ln = STAT_LVL['op']
break
if stat_ln:
self.stat_ln[addr_ln]['status'] = stat_ln
try:
if self.pid_websocket:
MPQ_WS.put([
'lane', addr_ln,
self.stat_ln[addr_ln]['status'],
self.stat_ln[addr_ln]['last_module'],
self.stat_ln[addr_ln]['poll'],
self.stat_ln[addr_ln]['last_dtg'],
self.stat_ln[addr_ln]['setup_id']
])
if self.tid_snmp_agent:
self.MPQ_SNMPA3.put([
addr_ln, self.stat_ln[addr_ln]['status'],
self.stat_ln[addr_ln]['last_module'],
self.stat_ln[addr_ln]['poll'],
self.stat_ln[addr_ln]['last_dtg']
])
if self.tid_snmp_notify:
self.MPQ_SNMPN3.put([
addr_ln, self.stat_ln[addr_ln]['status'],
self.stat_ln[addr_ln]['last_module'],
self.stat_ln[addr_ln]['poll'],
self.stat_ln[addr_ln]['last_dtg']
])
except queue.Full:
log = 'Can not place item in lane status queue, queue is full.'
logger.exception(log)
else:
MPQ_CMD3.put_no_wait([addr_ln])
print('Reset lane called.')
if uid_mod is None:
if self.tid_snmp_agent:
self.MPQ_SNMPA4.put(
[addr_ln, addr_mod, 'Null', 'Null', 'Null', stat_mod, 0])
def lane(self, mpq_record: list):
"""
Places lane statues into websocket and SNMP queues if status has changed.
mpq_record[0] = lane address
mpq_record[1] = lane dictionary
:param mpq_record: list
"""
stat_change = False
setup_change = False
addr_ln = mpq_record[0]
dict_ln = mpq_record[1]
# Only process record if key values have changed
for key in dict_ln.keys():
if dict_ln[key] != self.stat_ln[addr_ln][key]:
self.stat_ln[addr_ln][key] = dict_ln[key]
stat_change = True
if key == 'setup_id':
setup_change = True
# If any key value has changed, process record
if stat_change:
# Update lane document in CouchDB lanes database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='upd_doc',
cdb_name='lanes',
cdb_doc='lane{0}_status'.format(addr_ln),
data_cdb_in=self.stat_ln[addr_ln],
logfile=logfile)
if stat_cdb == STAT_LVL['crit']:
log = 'Failed to update lane{0}_status document in CouchDB lanes database.'.\
format(addr_ln)
logger.critical(log)
if setup_change:
MPQ_LN_SETUP.put_nowait(
[addr_ln, self.stat_ln[addr_ln]['status']])
if self.stat_ln[addr_ln]['poll']:
MPQ_STAT.put_nowait(
['base', ['poll_data', STAT_LVL['not_cfg']]])
# Only place into MPQs if processes/threads are functioning.
try:
if self.pid_websocket:
MPQ_WS.put([
'lane', addr_ln, self.stat_ln[addr_ln]['status'],
self.stat_ln[addr_ln]['last_module'],
self.stat_ln[addr_ln]['poll'],
self.stat_ln[addr_ln]['last_dtg'],
self.stat_ln[addr_ln]['setup_id']
])
if self.tid_snmp_agent:
self.MPQ_SNMPA3.put([
addr_ln, self.stat_ln[addr_ln]['status'],
self.stat_ln[addr_ln]['last_module'],
self.stat_ln[addr_ln]['poll'],
self.stat_ln[addr_ln]['last_dtg']
])
if self.tid_snmp_notify:
self.MPQ_SNMPN3.put([
addr_ln, self.stat_ln[addr_ln]['status'],
self.stat_ln[addr_ln]['last_module'],
self.stat_ln[addr_ln]['poll'],
self.stat_ln[addr_ln]['last_dtg']
])
except queue.Full:
log = 'Can not place item in lane status queue, queue is full.'
logger.exception(log)