def snmp_agent_start():
"""
Starts SNMP agent
"""
from server.snmp import agent
# SNMP agent must be placed in thread rather than
# multiprocess process because dynamic variables are
# shared between main process and the agent
obj_snmpa = agent.SNMPAgent()
try:
th_snmpa = threading.Thread(
target=obj_snmpa.engine,
args=()
)
th_snmpa.start()
MPQ_STAT.put_nowait([
'snmp_notify',
th_snmpa.ident
])
except threading.ThreadError:
log = 'Could not start SNMP agent due to threading error.'
logger.exception(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
MPQ_STAT.put_nowait([
'snmp_notify',
False
])
def on_close(
self,
reason
):
"""
Processes to execute when websocket closes
"""
# Ensure that multiprocess process is terminated
if self.mp_ws_listener is not None:
if self.mp_ws_listener.is_alive():
self.mp_ws_listener_stop.put_nowait(None)
time.sleep(0.1)
if self.mp_ws_listener.is_alive():
self.mp_ws_listener.join()
# Update base_pid document in CouchDB config database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='upd_doc',
cdb_name='config',
cdb_doc='base_pid',
data_cdb_in={'websocket_handler': False},
logfile=logfile
)
if stat_cdb:
log = 'Could not update websocket PID values in process status document due to CouchDB error.'
logger.warning(log)
MPQ_STAT.put_nowait([
'websocket',
False
])
def notify(self, mib_identity: str, mib_object: dict, notify_msg: dict):
"""
Dispatches prebuilt mib object
:param mib_identity: str
:param mib_object: dict
:param notify_msg: dict
"""
log = notify_msg['start']
logger.debug(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
error_indication, error_status, error_index, var_binds = next(
sendNotification(
SnmpEngine(), CommunityData(self.host_community),
UdpTransportTarget((self.host_ip, self.host_port)),
ContextData(), 'inform',
NotificationType(ObjectIdentity(
'JANUSESS-MIB', mib_identity + 'Notify').addMibSource(
'/opt/Janus/ESS/python3/server/snmp/mibs'),
objects=mib_object)))
if error_indication:
log = notify_msg['error'] + str(error_indication)
logger.warning(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'WARNING', log])
MPQ_STAT.put_nowait(['base', ['snmp_notify', STAT_LVL['op_err']]])
else:
log = notify_msg['end']
logger.info(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['snmp_notify', STAT_LVL['op']]])
def poll_clear(self):
"""
Clears operation polling CouchDB
self.args[0] = addr_ln
"""
logfile = 'polling'
logger = logging.getLogger(logfile)
log = 'Attempting to clear lane {0} '.format(self.args[0]) +\
'poll data from polling database.'
logger.debug(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'DEBUG',
log
])
# Build and issue InfluxDB command to delete pertinent data
data_idb_in = 'q=DROP SERIES WHERE "chan"=\'{0}\''.format(self.args[0])
http_resp = requests.post(
'http://localhost:8086/query?db=JanusESS',
headers={'Content-type': 'application/x-www-form-urlencoded'},
data=data_idb_in
)
# Determine result of command and issue status
if http_resp.status_code == 200:
log = 'Clear lane {0} poll data successful.'.\
format(self.args[0])
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
MPQ_STAT.put_nowait([
'base',
[
'influxdb',
STAT_LVL['op']
]
])
else:
log = 'Could not clear lane {0} poll data due '.format(self.args[0]) +\
'to InfluxDB query error.'
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
MPQ_STAT.put_nowait([
'base',
[
'influxdb',
STAT_LVL['op_err']
]
])
def i2c_lane_set(self,
addr_ln: int,
stat_en: bool = True,
attempts: int = 3):
"""
Sets lane for I2C network
:param addr_ln: int
:param stat_en: bool
:param attempts: int
:return stat_iface: STAT_LVL['op'] or STAT_LVL['crit']
"""
attempt = 0
err_iface = False
stat_iface = STAT_LVL['op']
# Cycle through attempts
if addr_ln != self.addr_ln:
for attempt in range(1, (attempts + 1)):
err_iface = self.obj_janus.i2c_set_lane(addr_ln=addr_ln)
if err_iface:
err_iface = True
if stat_en:
MPQ_STAT.put_nowait(
['base', ['interface', STAT_LVL['op_err']]])
# Only log warning on last attempt, keeps log clean
if attempt == attempts:
log = 'Attempt {0} of {1} to '.format(attempt, attempts) + \
'set interface to lane {0} failed.'.format(addr_ln)
logger.warning(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'WARNING', log])
else:
err_iface = False
self.addr_ln = addr_ln
break
if err_iface:
stat_iface = STAT_LVL['op_err']
if stat_en:
MPQ_STAT.put_nowait(['base', ['interface', stat_iface]])
log = 'Set lane {0} on four port interface failed.'.format(addr_ln)
logger.critical(log)
else:
log = 'Set lane {0} on four port interface succeeded after {1} attempts.'. \
format(addr_ln, attempt)
logger.info(log)
return stat_iface
def interrupt_check_flag(self, attempts: int = 2, stat_en: bool = True):
"""
Checks GPIO interrupt
:param attempts: int
:param stat_en: bool
:return data_iface_out: int (0/1 if STAT_LVL['op'])
:return stat_iface: STAT_LVL['op'] or STAT_LVL['crit']
"""
attempt = 0
err_iface = True
stat_iface = STAT_LVL['op']
data_iface_out = None
# Cycle through attempts
for attempt in range(1, (attempts + 1)):
# There is only one interrupt flag for all four lanes. Therefore
# no method exists to isolate which of the four GPIOs were triggered.
data_iface_out, err_iface = self.obj_janus.interrupt_check_flag()
if err_iface:
MPQ_STAT.put_nowait(
['base', ['interface', STAT_LVL['op_err']]])
# Only log warning on last attempt, keeps log clean
if attempt == attempts:
log = 'Attempt {0} of {1} to '.format(attempt, attempts) + \
'check GPIO interrupt flag failed.'
logger.warning(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'WARNING', log])
else:
break
if err_iface:
log = 'General IO failure to check GPIO interrupt flag.'
logger.critical(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'CRITICAL', log])
stat_iface = STAT_LVL['crit']
print(log)
else:
log = 'Successfully checked GPIO interrupt flag after {0} attempts.'.\
format(attempt)
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
if stat_en:
MPQ_STAT.put_nowait(['base', ['interface', stat_iface]])
return data_iface_out, stat_iface
def interrupt_clear_flag(self, attempts: int = 3, stat_en: bool = True):
"""
Clears GPIO interrupt
:param attempts: int
:param stat_en: bool
:return stat_iface: STAT_LVL['op'] or STAT_LVL['crit']
"""
attempt = 0
err_iface = True
stat_iface = STAT_LVL['op']
# Cycle through attempts
for attempt in range(1, (attempts + 1)):
err_iface = self.obj_janus.interrupt_clear_flag()
if err_iface:
MPQ_STAT.put_nowait(
['base', ['interface', STAT_LVL['op_err']]])
# Only log warning on last attempt, keeps log clean
if attempt == attempts:
log = 'Attempt {0} of {1} to '.format(attempt, attempts) + \
'clear GPIO interrupt flag failed.'
logger.warning(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'WARNING', log])
else:
break
if err_iface:
log = 'General IO failure to clear GPIO interrupt flag.'
logger.critical(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'CRITICAL', log])
stat_iface = STAT_LVL['crit']
print(log)
else:
log = 'Successfully cleared GPIO interrupt flag after {0} attempts.'. \
format(attempt)
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
if stat_en:
MPQ_STAT.put_nowait(['base', ['interface', stat_iface]])
return stat_iface
def store():
"""
Stores core time in file
:return stat_time: STAT_LVL['op'] or STAT_LVL['op_err']
:return stat_cdb: STAT_LVL['op'] or STAT_LVL['crit']
"""
stat_time = STAT_LVL['op']
# Update time document in CouchDB config database
data_cdb_out, stat_cdb, code_cdb = dbase.cdb_request(
cdb_cmd='upd_doc',
cdb_name='config',
cdb_doc='time',
data_cdb_in={'time': str(time.time())},
logfile=logfile
)
if stat_cdb:
log = 'Could not save time due to CouchDB document update error.'
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
MPQ_STAT.put_nowait([
'base',
[
'couchdb',
STAT_LVL['op_err']
]
])
log = 'Time storage complete.'
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
if not stat_time:
MPQ_STAT.put_nowait([
'base',
[
'tasks',
stat_time
]
])
def update(interval: int):
"""
Produces timed status reports
:param interval: int
"""
log = 'Conducting {0}-hour system status check.'.format(interval)
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
# Only issue messaging if recent network check shows it is up
send_mail(
msg_type='status_dispatch',
args=[],
)
log = '{0}-hour status check completed.'.format(interval)
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['tasks', STAT_LVL['op']]])
def snmp_notify_start(self):
"""
Starts SNMP notify
self.args[0] = host ip
self.args[1] = host port
self.args[1] = host community
"""
from server.snmp import notification
# SNMP notifier must be placed in thread rather than
# multiprocess process because dynamic variables are shared
# between main process and the agent
obj_snmpn = notification.SNMPNotify(
self.args[0],
self.args[1],
self.args[2]
)
try:
th_snmpn = threading.Thread(
target=obj_snmpn.listener,
args=()
)
th_snmpn.start()
MPQ_STAT.put_nowait([
'snmp_notify',
th_snmpn.ident
])
except threading.ThreadError:
log = 'Could not start SNMP notifier due to threading error.'
logger.exception(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
MPQ_STAT.put_nowait([
'snmp_notify',
False
])
def clear():
"""
Produces timed status reports
"""
# Clear queues at most every 24 hours to prevent filling up.
# There is no better way to clear a multiprocessing queue other than run a loop.
log = 'Attempting to clear activity and status queues.'
logger.debug(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
while not MPQ_STAT.empty():
MPQ_STAT.get()
time.sleep(0.001)
while not MPQ_ACT.empty():
MPQ_ACT.get()
time.sleep(0.001)
log = 'Activity and status queues cleared.'
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
def dispatcher(self):
# Register an imaginary never-ending job to keep I/O dispatcher running forever
self.eng_snmp.transportDispatcher.jobStarted(1)
# Run I/O dispatcher which would receive queries and send responses
try:
log = 'SNMP agent dispatcher started.'
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['snmp_agent', STAT_LVL['op']]])
self.eng_snmp.transportDispatcher.runDispatcher()
except queue.Full:
log = 'SNMP agent dispatcher experienced critical error.'
logger.critical(log)
print(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'CRITICAL', log])
MPQ_STAT.put_nowait(['base', ['snmp_agent', STAT_LVL['crit']]])
self.eng_snmp.tr0ansportDispatcher.closeDispatcher()
raise
def mdb_check():
logfile = 'janusess'
logger = logging.getLogger(logfile)
check_time = 3.0
log = 'Checking MariaDB every {0} sec until operational.'.format(
check_time)
logger.debug(log)
count = 1
while True:
try:
mdb_conn = mysql.connector.connect(
user='******',
password='******',
host='127.0.0.1',
database='aurora',
unix_socket='/run/mysqld/mysqld.sock')
mdb_conn.close()
stat_mdb = STAT_LVL['op']
log = 'MariaDB is operational.'
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['mariadb', stat_mdb]])
except mysql.connector.Error:
stat_mdb = STAT_LVL['not_cfg']
log = 'Local MariaDB server did not respond to check. Trying again.'
logger.warning(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'WARNING', log])
MPQ_STAT.put_nowait(['base', ['mariadb', stat_mdb]])
if stat_mdb < STAT_LVL['not_cfg']:
break
count += count
time.sleep(check_time)
def listener(
self
):
"""
Listener to open when a page websocket connection is established
"""
try:
self.mp_ws_listener = multiprocessing.Process(
target=self.websocket,
args=()
)
self.mp_ws_listener.start()
pid_handler = self.mp_ws_listener.pid
log = 'Flask websocket handler opened, pid: {0}.'.format(pid_handler)
logger.info(log)
MPQ_STAT.put_nowait([
'websocket',
self.mp_ws_listener.pid
])
except multiprocessing.ProcessError:
pid_handler = False
log = 'Can not dispatch heartbeat channel encoded messages ' + \
'due to multiprocessing error.'
logger.exception(log)
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='upd_doc',
cdb_name='config',
cdb_doc='base_pid',
data_cdb_in={'websocket_handler': pid_handler},
logfile=logfile
)
if stat_cdb:
log = 'Could not update websocket PID values in process status document due to CouchDB error.'
logger.warning(log)
def cdb_check():
"""
Checks CouchDB if ready to accept transactions
"""
logfile = 'janusess'
logger = logging.getLogger(logfile)
check_time = 0.5
log = 'Checking CouchDB every {0} sec until operational.'.format(
check_time)
logger.debug(log)
count = 1
while True:
# Issue CouchDB GET request and process result
http_resp = requests.get('http://127.0.0.1:5984/')
# Successful GET request
if http_resp.status_code == 200:
log = 'CouchDB is operational.'
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['couchdb', STAT_LVL['op']]])
break
# All GET errors
else:
log = 'CouchDB is not operational, failed with http ' +\
'response {0}. Making another attempt.'.format(http_resp.status_code)
logger.warning(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'WARNING', log])
MPQ_STAT.put_nowait(['base', ['couchdb', STAT_LVL['cfg_err']]])
count += count
time.sleep(check_time)
def listener(self):
"""
Listens for notification activity and assigns to appropriate process
"""
log = 'SNMP notification listener started.'
logger.info(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['snmp_notify', STAT_LVL['op']]])
# Poll SNMP agent queues for values to update variables
while True:
if not MPQ_SNMPN_STOP.empty():
MPQ_SNMPN_STOP.get()
break
if not MPQ_SNMPN2.empty():
mpq_record = MPQ_SNMPN2.get()
self.base(mpq_record=mpq_record)
if not MPQ_SNMPN3.empty():
mpq_record = MPQ_SNMPN3.get()
self.lane(mpq_record=mpq_record)
if not MPQ_SNMPN4.empty():
mpq_record = MPQ_SNMPN4.get()
self.module(mpq_record=mpq_record)
if not MPQ_SNMPN5.empty():
mpq_record = MPQ_SNMPN5.get()
self.poll_val(mpq_record=mpq_record)
time.sleep(0.1)
MPQ_STAT.put_nowait(['snmp_notify', False])
log = 'SNMP notification listener stopped.'
logger.info(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['snmp_notify', STAT_LVL['not_cfg']]])
def listener():
"""
Listener for commands to execute
"""
from application.polling import flag, poll
# Initialize operations and Janus interface libraries
# Enable interrupts immediately
obj_cmd = Command()
obj_iface = Interface()
obj_iface.interrupts_enable()
# Send message to JanusESS main to proceed with JanusESS startup procedures
MPQ_IFACE_SETUP.put_nowait(obj_iface.error_iface())
stat_cmd_prev = STAT_LVL['not_cfg']
# This while loop has no exit, JanusESS will not function without this
# ongoing loop to check the following command queues in priority order:
#
# MPQ_CMD0: User-initiated command requests
# MPQ_CMD1: Checks for neighbor-bus triggers
# MPQ_CMD2: User-initiated module sensor polling
# MPQ_CMD3: Lane/module initialization and setup routine
# MPQ_CMD4: Upload module configuration to module
# MPQ_CMD5: Recurring module sensor polling
while True:
stat_cmd = STAT_LVL['op']
# User-initiated command requests
if not MPQ_CMD0.empty():
data_cmd0_in = MPQ_CMD0.get()
log = 'Priority 0 command, command #{0} request received.'.\
format(data_cmd0_in['command'])
logger.debug(log)
log = 'Command {0} called.'.format(data_cmd0_in['command'])
logger.info(log)
if data_cmd0_in['command'] == 'log_level':
if data_cmd0_in['args'][0] == 'DEBUG':
logger.setLevel(logging.DEBUG)
elif data_cmd0_in['args'][0] == 'INFO':
logger.setLevel(logging.INFO)
elif data_cmd0_in['args'][0] == 'ERROR':
logger.setLevel(logging.ERROR)
elif data_cmd0_in['args'][0] == 'WARNING':
logger.setLevel(logging.WARNING)
elif data_cmd0_in['args'][0] == 'CRITICAL':
logger.setLevel(logging.CRITICAL)
else:
try:
th_cmd0 = threading.Thread(
target=obj_cmd.exec_cmd,
args=(
data_cmd0_in['command'],
data_cmd0_in['args'],
data_cmd0_in['data'],
)
)
th_cmd0.start()
except threading.ThreadError:
stat_cmd = STAT_LVL['op_err']
log = 'Could not start user-initiated command due to threading error.'
logger.exception(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
log = 'Priority 0 interface request concluded.'
logger.info(log)
# Checks for neighbor-bus triggers
#
# This command is only executed if a trigger flag is discovered during
# recurring module sensor polling
elif not MPQ_CMD1.empty():
MPQ_CMD1.get()
log = 'Priority 1 interface request received.'
logger.debug(log)
# Get all documents from CouchDB lanes database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='get_all',
cdb_name='lanes',
logfile=logfile
)
if not stat_cdb:
# Cycle through lanes, if lane and lane
# polling are operational, then continue procedure
# to check module triggers
for addr_ln in range(0, 4):
if (data_cdb_out[addr_ln]['status'] < STAT_LVL['crit']) \
and (data_cdb_out[addr_ln]['poll'] <
STAT_LVL['crit']):
# Set four-port interface lane. This function
# ignores single-port interface devices.
stat_iface = obj_iface.i2c_lane_set(addr_ln=addr_ln)
if not stat_iface:
# Get stat_chan view from CouchDB
# modconfig database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='get_view',
cdb_name='modconfig',
cdb_doc='stat_chan{0}'.format(addr_ln),
logfile=logfile,
)
# Cycle through each non-failed module connected
# to the lane
if not stat_cdb:
for dict_mod in data_cdb_out:
if dict_mod['value']['status'] < STAT_LVL['crit']:
# Call function to check an
# individual module'strigger status
evt_byte = MMAP[dict_mod['value']['mem_map_ver']]['M_EVT']
stat_mod = flag.interrupt(
obj_iface=obj_iface,
addr_ln=addr_ln,
addr_mod=dict_mod['key'],
evt_byte=evt_byte
)
MPQ_STAT.put_nowait([
'module', [
dict_mod['id'],
addr_ln,
dict_mod['key'],
stat_mod
]
])
else:
log = 'Could not check module interrupt flag due to CouchDB error.'
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
stat_cmd = STAT_LVL['op_err']
else:
stat_cmd = STAT_LVL['op_err']
log = 'Could not complete priority 1 interface request ' + \
'on lane {0} due to i2c lane '.format(addr_ln) +\
'set error.'
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
obj_iface.interrupt_clear_flag()
log = 'Priority 1 interface request concluded.'
logger.info(log)
else:
log = 'Could not complete priority 1 interface request due to CouchDB error.'
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
stat_cmd = STAT_LVL['op_err']
# User-initiated module sensor polling
#
# This command only polls one module per request
elif not MPQ_CMD2.empty():
data_cmd2_in = MPQ_CMD2.get()
uid_mod = data_cmd2_in[0]
addr_ln = data_cmd2_in[1]
addr_mod = data_cmd2_in[2]
log = 'Lane {0} module {1} priority 2 interface request received.'.format(addr_ln, addr_mod)
logger.info(log)
# Set four-port interface lane. This function ignores
# single-port interface devices.
stat_iface = obj_iface.i2c_lane_set(addr_ln=addr_ln)
if not stat_iface:
stat_poll_data, uid_mod_i2c = poll.get_data(
obj_iface=obj_iface,
uid_mod=uid_mod,
addr_ln=addr_ln,
addr_mod=addr_mod
)
if not stat_poll_data:
MPQ_STAT.put_nowait([
'base',
[
'poll_data',
STAT_LVL['op']
]
])
stat_iface, flag = obj_iface.interrupt_check_flag()
if flag:
MPQ_CMD1.put(True)
if not stat_iface:
log = 'Lane {0} module {1} on-demand poll completed.'.format(addr_ln, addr_mod)
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
# USB reset interface board if bad data returned
#
# TODO: Need higher level tracking of this error
# TODO: Do not wish to reset device more than once
# TODO: If reset after first time fails, all
# TODO: related commands will be bypassed
elif (stat_poll_data == STAT_LVL['op_err']) and \
(uid_mod_i2c != uid_mod):
obj_iface.setup()
obj_iface.interrupts_enable()
stat_cmd = STAT_LVL['op_err']
log = 'Resetting interface due to mismatch in module id: ' + \
'requested={0} vs polled={1}.'.format(uid_mod, uid_mod_i2c)
logger.warning(log)
else:
log = 'Could not complete priority 2 interface request on ' + \
'lane {0} module {1} '.format(addr_ln, addr_mod) +\
'due to i2c lane set error.'
logger.critical(log)
stat_cmd = STAT_LVL['op_err']
log = 'Lane {0} module '.format(addr_ln) +\
'{0} priority 2 interface request concluded.'.format(addr_mod)
logger.info(log)
# Lane/module initialization and setup routine
elif not MPQ_CMD3.empty():
data_cmd3_in = MPQ_CMD3.get()
addr_ln = data_cmd3_in[0]
# FLAG_LNRST[addr_ln] = True
log = 'Lane {0} priority 3 interface request received.'. \
format(addr_ln)
logger.debug(log)
log = 'Begin lane {0} network reset and initialization.'.\
format(addr_ln)
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
# Call lane reset command to toggle GPIO pins
stat_iface = lane.reset(
obj_iface=obj_iface,
addr_ln=addr_ln
)
if not stat_iface:
# Call lane init command to setup any modules
# connected to the lane
stat_ch, stat_cdb = lane.init(
obj_iface=obj_iface,
addr_ln=addr_ln
)
if not stat_ch:
# Ensure that all interrupt flags are cleared prior
# to any other lane activity. GPIO interrupts may
# get triggered during lane setup routines.
stat_iface = obj_iface.interrupt_clear_flag()
if not stat_iface:
log = 'Interrupt flags successfully cleared.'
logger.debug(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'DEBUG',
log
])
if not stat_cdb:
log = 'Lane {0} network reset and initialization complete.'.format(addr_ln)
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
else:
stat_cmd = STAT_LVL['op_err']
log = 'Lane {0} network reset and '.format(addr_ln) + \
'initialization complete with CouchDB errors.'
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
else:
stat_cmd = STAT_LVL['op_err']
log = 'Could not clear interrupt flags from interface due to interface error.'
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
else:
stat_cmd = STAT_LVL['op_err']
log = 'Lane {0} network reset and initialization failed to complete.'.format(addr_ln)
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
else:
stat_cmd = STAT_LVL['op_err']
log = 'Could not initialize lane {0} network due to Neighbor Bus reset error.'.format(addr_ln)
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
# FLAG_LNRST[addr_ln] = False
log = 'Lane {0} priority 3 interface request concluded.'.format(addr_ln)
logger.info(log)
# Upload module configuration to module
elif not MPQ_CMD4.empty():
while not MPQ_CMD4.empty():
data_cmd4_in = MPQ_CMD4.get()
uid_mod = data_cmd4_in[0]
addr_ln = data_cmd4_in[1]
addr_mod = data_cmd4_in[2]
addr_mem = data_cmd4_in[3]
data_iface_in = data_cmd4_in[4]
log = 'Lane {0} module '.format(addr_ln) +\
'{0} priority 4 interface request received.'.format(addr_mod)
logger.debug(log)
stat_mod = STAT_LVL['op']
# Set four-port interface lane. This function ignores
# single-port interface devices.
stat_iface = obj_iface.i2c_lane_set(addr_ln=addr_ln)
if not stat_iface:
stat_iface = obj_iface.i2c_write(
addr_ln=addr_ln,
addr_mod=addr_mod,
addr_mem=addr_mem,
data_iface_in=data_iface_in
)
if stat_iface:
stat_mod = STAT_LVL['crit']
stat_cmd = STAT_LVL['op_err']
log = 'Upload of module settings to lane {0} '.format(addr_ln) +\
'module {0} unsuccessful.'.format(addr_mod)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
else:
log = 'Upload of module settings to lane {0} '.format(addr_ln) +\
'module {0} successful.'.format(addr_mod)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
logger.log(
logging.INFO if not stat_iface else logging.CRITICAL,
log
)
print(log)
MPQ_STAT.put_nowait([
'module',
[
uid_mod,
addr_ln,
addr_mod,
stat_mod
]
])
else:
stat_cmd = STAT_LVL['op_err']
log = 'Could not complete priority 4 interface request on ' + \
'lane {0}, '.format(addr_ln) +\
'module {0} due to i2c lane set error.'.format(addr_mod)
logger.critical(log)
log = 'Lane {0} module '.format(addr_ln) +\
'{0} priority 4 interface request concluded.'.format(addr_mod)
logger.info(log)
# Recurring module sensor polling
#
# While this command algorithm is essentially identical
# to MPQ_CMD2 algorithm, it remains separate so that any
# user-initiated polling request upon an individual
# module will receive a much higher priority so that
# execution takes place more quickly.
elif not MPQ_CMD5.empty():
time_a = time.time()
data_cmd5_in = MPQ_CMD5.get()
uid_mod = data_cmd5_in[0]
addr_ln = data_cmd5_in[1]
addr_mod = data_cmd5_in[2]
log = 'Lane {0} module {1} '.format(addr_ln, addr_mod) +\
'priority 5 interface request received.'
logger.info(log)
# Set four-port interface lane. This function ignores
# single-port interface devices.
stat_iface = obj_iface.i2c_lane_set(addr_ln=addr_ln)
if not stat_iface:
time_b = time.time()
stat_poll_data, uid_mod_i2c = poll.get_data(
obj_iface=obj_iface,
uid_mod=uid_mod,
addr_ln=addr_ln,
addr_mod=addr_mod
)
print('Lane {0} module {1} priority 5 get_data time: {2}'.
format(addr_ln, addr_mod, round((time.time() - time_b), 3)))
# Check for any interrupts on all lanes before polling again
if not stat_poll_data:
MPQ_STAT.put_nowait([
'base',
[
'poll_data',
STAT_LVL['op']
]
])
stat_iface, flag = obj_iface.interrupt_check_flag()
if flag:
MPQ_CMD1.put(True)
# USB reset interface board if bad data returned
#
# TODO: Need higher level tracking of this error
# TODO: Do not wish to reset device more than once
# TODO: If reset after first time fails, all
# TODO: related commands will be bypassed
elif (stat_poll_data == STAT_LVL['op_err']) and \
(uid_mod_i2c != uid_mod):
obj_iface.setup()
obj_iface.interrupts_enable()
stat_cmd = STAT_LVL['op_err']
log = 'Resetting interface due to mismatch in module id: ' + \
'requested={0} vs polled={1}.'.format(uid_mod, uid_mod_i2c)
logger.warning(log)
MPQ_POLL_COMPLETE.put_nowait([
addr_ln,
addr_mod
])
else:
stat_cmd = STAT_LVL['op_err']
log = 'Could not complete priority 5 interface request on lane ' + \
'{0} module {1} '.format(addr_ln, addr_mod) +\
'due to i2c lane set error.'
logger.critical(log)
log = 'Lane {0} module {1} '.format(addr_ln, addr_mod) +\
'priority 5 interface request concluded.'
logger.info(log)
print('Lane {0} module {1} priority 5 time: {2}'.
format(addr_ln, addr_mod, round((time.time() - time_a), 3)))
time.sleep(0.05)
# If command listener status has changed, send heartbeat update
if stat_cmd != stat_cmd_prev:
stat_cmd_prev = stat_cmd
MPQ_STAT.put_nowait([
'base',
[
'command_listener',
stat_cmd
]
])
'activity', 'janusess', 'command', 'conversion', 'heartbeat',
'interface', 'polling', 'server', 'setup', 'tasks'
]
for log_file in logs:
logger = logging.getLogger(log_file)
for i in range(1, 6):
logger.info('')
log = 'JanusESS logging started'
logger.info(log)
# Set log file for JanusESS start sequence actions
logfile = 'janusess'
logger = logging.getLogger(logfile)
MPQ_STAT.put_nowait(['base', ['logging', STAT_LVL['op']]])
# Check to determine if CouchDB is operating
dbase.cdb_check()
dbase.mdb_check()
# A simple check for corrupted primary CouchDB databases
# This check is not a guarantee that databases are corruption-free
# If any cannot be restored, then prevent JanusESS start
db_list = ['config', 'lanes', 'modules', 'modconfig']
stat_cdb_dbases = dbase.recover(db_list=db_list)
appdbase.compact()
# Get log document from CouchDB config database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(cdb_cmd='get_doc',
cdb_name='config',
def gpio_read(self,
addr_ln: int,
mode: bool = True,
stat_en: bool = True,
attempts: int = 2):
"""
Reads GPIO pin value
:param addr_ln: int
:param mode: bool
:param stat_en: bool
:param attempts: int
:return data_iface_out: int (0/1 if STAT_LVL['op'])
:return stat_iface: STAT_LVL['op'] or STAT_LVL['crit']
"""
attempt = 0
err_iface = True
data_iface_out = None
stat_iface = STAT_LVL['op']
# Cycle through attempts
for attempt in range(1, (attempts + 1)):
# If mode flag is set, set GPIO mode to 'IN' prior to GPIO read
if mode:
err_iface = self.obj_janus.gpio_set_mode(pin=addr_ln,
mode='IN')
if not err_iface:
data_iface_out, err_iface = self.obj_janus.gpio_read(
pin=addr_ln)
else:
data_iface_out, err_iface = self.obj_janus.gpio_read(
pin=addr_ln)
if err_iface:
MPQ_STAT.put_nowait(
['base', ['interface', STAT_LVL['op_err']]])
# Only log warning on last attempt, keeps log clean
if attempt == attempts:
log = 'Attempt {0} of {1} to '.format(attempt, attempts) + \
'read GPIO pin {0} failed.'.format(addr_ln)
logger.warning(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'WARNING', log])
else:
break
if err_iface:
log = 'General IO failure to read GPIO pin {0}.'.format(addr_ln)
logger.critical(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'CRITICAL', log])
stat_iface = STAT_LVL['crit']
print(log)
else:
log = 'Successfully read GPIO pin {0} after {1} attempts.'.\
format(addr_ln, attempt)
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
if stat_en:
MPQ_STAT.put_nowait(['base', ['interface', stat_iface]])
return data_iface_out, stat_iface
def send_mail(
msg_type: str,
args: list,
):
"""
Sends messaging message
:param msg_type: str
:param args: list
"""
# Get base_status document from CouchDB config database
data0_cdb_out, stat0_cdb, http0_cdb = dbase.cdb_request(
cdb_cmd='get_doc',
cdb_name='config',
cdb_doc='base_status',
logfile=logfile)
# Get email document from CouchDB config database
data1_cdb_out, stat1_cdb, http1_cdb = dbase.cdb_request(
cdb_cmd='get_doc',
cdb_name='config',
cdb_doc='email',
logfile=logfile,
)
# Get sms document from CouchDB config database
data2_cdb_out, stat2_cdb, http2_cdb = dbase.cdb_request(
cdb_cmd='get_doc',
cdb_name='config',
cdb_doc='sms',
logfile=logfile,
)
# If messaging is enabled by user and network is operational,
# then build template and send messaging
if not stat0_cdb and not stat1_cdb and not stat2_cdb:
if data1_cdb_out['smtp_enable'] and not data0_cdb_out['network']:
dict_msg, stat_msg_temp = templates.message_templates(
sms_enable=data2_cdb_out['sms_enable'],
msg_type=msg_type,
args=args)
# Uncomment to test messaging text formatting
# print("\n\n\n")
# print(dict_msg['smtp_subject'])
# print(dict_msg['smtp_body'])
# print("\n\n\n")
# print(dict_msg['sms_subject'])
# print(dict_msg['sms_body'])
# print("\n\n\n")
# Comment this block to test message text formatting
if not stat_msg_temp:
try:
email_mp = multiprocessing.Process(target=send,
args=(data1_cdb_out,
data2_cdb_out,
dict_msg))
email_mp.start()
except multiprocessing.ProcessError:
log = 'Can not send email due to multiprocessing error.'
logger.exception(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'ERROR', log])
MPQ_STAT.put_nowait(['base', ['email', STAT_LVL['crit']]])
else:
log = 'Email not attempted, email template build failed.'
logger.warning(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'WARNING', log])
MPQ_STAT.put_nowait(['base', ['email', stat_msg_temp]])
elif not data1_cdb_out['smtp_enable']:
log = 'Email disabled by user.'
logger.debug(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
MPQ_STAT.put_nowait(['base', ['email', STAT_LVL['not_cfg']]])
else:
log = 'Email not attempted, latest network check shows network ' + \
'disconnected.'
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['email', STAT_LVL['not_cfg']]])
else:
log = 'Email not attempted due to CouchDB error.'
logger.warning(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'WARNING', log])
MPQ_STAT.put_nowait(['base', ['email', STAT_LVL['not_cfg']]])
def get_data(
obj_iface: TYPE_INTERFACE,
uid_mod: str,
addr_ln: int,
addr_mod: int
):
"""
Retrieves and publishes module sensor data
:param obj_iface: Interface Object
:param uid_mod: str
:param addr_ln: int
:param addr_mod: int
"""
# Change logging level since this operates in multiprocess
# Cycle to last entry for most current log setting
while not MPQ_POLL_LOG_DATA.empty():
mpq_record = MPQ_POLL_LOG_DATA.get()
if mpq_record[0] == 'DEBUG':
logger.setLevel(logging.DEBUG)
elif mpq_record[0] == 'INFO':
logger.setLevel(logging.INFO)
elif mpq_record[0] == 'ERROR':
logger.setLevel(logging.ERROR)
elif mpq_record[0] == 'WARNING':
logger.setLevel(logging.WARNING)
elif mpq_record[0] == 'CRITICAL':
logger.setLevel(logging.CRITICAL)
time_a = time.time()
log = 'Retrieving lane {0} module {1} id {2} data.'.format(addr_ln, addr_mod, uid_mod)
logger.info(log)
stat_poll_data = STAT_LVL['op']
uid_mod_i2c = ''
uid_mod_i2c_print = ''
# Retrieve memory map version of module with I2C address of 0x7F.
# If module responds, proceed to module setup actions, otherwise
# mod_last_found flag is set.
high_mmap = len(MMAP) - 1
addr_mem = MMAP[high_mmap]['M_CFG_ALL'][0]
data_len = MMAP[high_mmap]['M_CFG_ALL'][1]
data0_iface_out, stat0_iface = obj_iface.i2c_read(
addr_ln=addr_ln,
addr_mod=addr_mod,
addr_mem=addr_mem,
data_len=data_len,
stat_en=False
)
print('RAW POLL DATA: {0}'.format(data0_iface_out))
print('Lane {0} module {1} get_data i2c config time: {2}'.
format(addr_ln, addr_mod, round((time.time() - time_a), 3)))
if stat0_iface:
log = 'Lane {0} module {1} poll can '.format(addr_ln, addr_mod) + \
'not be completed due to I2C interface error.'
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
print(log)
MPQ_STAT.put_nowait([
'base',
[
'poll_data',
STAT_LVL['op_err']
]
])
MPQ_STAT.put_nowait([
'module',
[
uid_mod,
addr_ln,
addr_mod,
STAT_LVL['op_err']
]
])
stat_poll_data = STAT_LVL['op_err']
else:
# Build module id string from I2C data
mod_uid_end = MMAP[data0_iface_out[2]]['M_UID'][0] + \
MMAP[data0_iface_out[2]]['M_UID'][1] - 1
mod_uid_begin = MMAP[data0_iface_out[2]]['M_UID'][0] - 1
for addr_mem in range(mod_uid_end, mod_uid_begin, -1):
uidmod_i2c = str(hex(data0_iface_out[addr_mem]))[2:]
if len(uidmod_i2c) == 1:
uidmod_i2c = '0' + uidmod_i2c
uid_mod_i2c += uidmod_i2c
# Check that module ids match, then proceed with data collection and reporting
uid_mod_print = ''.join(char for char in uid_mod.strip() if isprint(char))
uid_mod_i2c_print = ''.join(char for char in uid_mod_i2c.strip() if isprint(char))
if uid_mod_i2c_print == uid_mod_print:
time_b = time.time()
# Get module document from CouchDB modconfig database
data0_cdb_out, stat0_cdb, http0_cdb = dbase.cdb_request(
cdb_cmd='get_doc',
cdb_name='modconfig',
cdb_doc=uid_mod_print,
logfile=logfile
)
# Get cloud document from CouchDB config database
data1_cdb_out, stat1_cdb, http1_cdb = dbase.cdb_request(
cdb_cmd='get_doc',
cdb_name='config',
cdb_doc='cloud',
logfile=logfile
)
# Get core document from CouchDB config database
data2_cdb_out, stat2_cdb, http2_cdb = dbase.cdb_request(
cdb_cmd='get_doc',
cdb_name='config',
cdb_doc='core',
logfile=logfile
)
print(
'Lane {0} module {1} get_data database time: {2}'.
format(addr_ln, addr_mod, round((time.time() - time_b), 3)))
if not stat0_cdb and not stat1_cdb and not stat2_cdb:
module_stat = STAT_LVL['op']
poll_data_mod = []
poll_head_mod = [
uid_mod_i2c_print,
data2_cdb_out['customer'],
data2_cdb_out['name'],
data0_cdb_out['loc'],
addr_ln,
addr_mod,
]
# Retrieves sensor polling value from module
time_c = time.time()
data_iface_out, stat_iface = obj_iface.i2c_read(
addr_ln=addr_ln,
addr_mod=addr_mod,
addr_mem=MMAP[data0_iface_out[2]]['S_ALL_VAL'][0],
data_len=MMAP[data0_iface_out[2]]['S_ALL_VAL'][1]
)
print('Lane {0} module {1} get_data i2c sensor time: {2}'.
format(addr_ln, addr_mod, round((time.time() - time_c), 3)))
print(data_iface_out)
if not stat_iface:
# Cycle through all sensors installed on the module
led_ctl = Control()
for addr_s in range(0, int(data0_cdb_out['num_sensors'])):
sensor = 'S{0}'.format(addr_s)
log = 'Retrieving lane {0} module {1} sensor {2} data.'.\
format(addr_ln, addr_mod, addr_s)
logger.debug(log)
# Initialize polling data packet
data_dtg = time.time()
poll_data_s = [addr_s]
# Convert raw values to floating point number, and add to packet
val_raw = struct.pack(
'BBBB',
int(data_iface_out[3 + (addr_s * 4)]),
int(data_iface_out[2 + (addr_s * 4)]),
int(data_iface_out[1 + (addr_s * 4)]),
int(data_iface_out[0 + (addr_s * 4)])
)
val_convert = round(
struct.unpack('>f', val_raw)[0],
data0_cdb_out[sensor]['precision']
)
if (val_convert >= data0_cdb_out[sensor]['min']) or \
(val_convert <= data0_cdb_out[sensor]['max']):
trig_low = round(
float(data0_cdb_out[sensor]['trig_low']),
data0_cdb_out[sensor]['precision']
)
trig_high = round(
float(data0_cdb_out[sensor]['trig_high']),
data0_cdb_out[sensor]['precision']
)
# Determine triggers
if val_convert < trig_low:
poll_data_s.append('low')
poll_data_s.append(True)
trigger = trig_low
module_stat = STAT_LVL['s_evt']
led_ctl.effect(
'sensor_low',
uid_mod_print,
addr_ln,
addr_mod
)
elif val_convert > trig_high:
poll_data_s.append('high')
poll_data_s.append(True)
trigger = trig_high
module_stat = STAT_LVL['s_evt']
led_ctl.effect(
'sensor_high',
uid_mod_print,
addr_ln,
addr_mod
)
else:
poll_data_s.append('off')
poll_data_s.append(False)
trigger = 0.0
poll_data_s.append(data0_cdb_out[sensor]['type'])
poll_data_s.append(val_convert)
poll_data_s.append(data_dtg)
poll_data_s.append(data0_cdb_out[sensor]['unit'])
poll_data_s.append(trigger)
poll_data_s.append(data0_cdb_out[sensor]['trig_int'])
poll_data_s.append(data0_cdb_out[sensor]['trig_step'])
poll_data_mod.append(poll_data_s)
MPQ_STAT.put_nowait([
'poll',
[
poll_head_mod,
poll_data_s
]
])
time_e = time.time()
store_data(data1_cdb_out, poll_head_mod, poll_data_mod)
print('Lane {0} module {1} get_data store data time: {2}'.
format(addr_ln, addr_mod, round((time.time() - time_e), 3)))
else:
log = 'Lane {0} module {1} '.format(addr_ln, addr_mod) + \
'data not added to storage queue due to I2C errors.'
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
stat_poll_data = STAT_LVL['op_err']
module_stat = STAT_LVL['op_err']
else:
log = 'Lane {0} module {1} '.format(addr_ln, addr_mod) + \
'data not added to storage queue due to CouchDB errors.'
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
stat_poll_data = STAT_LVL['op_err']
module_stat = STAT_LVL['op_err']
log = 'Completed lane {0} module {1} poll.'.format(addr_ln, addr_mod)
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'DEBUG',
log
])
MPQ_STAT.put_nowait([
'base',
[
'poll_data',
stat_poll_data
]
])
MPQ_STAT.put_nowait([
'module',
[
uid_mod_print,
addr_ln,
addr_mod,
module_stat
]
])
else:
stat_poll_data = STAT_LVL['op_err']
log = 'Lane {0} module {1} poll can '.format(addr_ln, addr_mod) + \
'not be completed due to mismatch in module id: ' + \
'requested={0} vs polled={1}.'.format(uid_mod_print, uid_mod_i2c_print)
logger.critical(log)
print(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
MPQ_STAT.put_nowait([
'base',
[
'poll_data',
stat_poll_data
]
])
MPQ_STAT.put_nowait([
'module',
[
uid_mod_print,
addr_ln,
addr_mod,
STAT_LVL['op_err']
]
])
return stat_poll_data, uid_mod_i2c_print
def request(file_cmd: str,
file_name: str,
data_loc: int,
num_bytes: int,
data_file_in: str,
search_field: str = '',
replace_line: str = '',
logfile: str = 'janusess',
attempts: int = 3):
"""
Executes file transaction
:param file_cmd: str
:param file_name: str
:param data_loc: int
:param num_bytes: int
:param search_field: str,
:param replace_line: str,
:param data_file_in: str
:param logfile: str
:param attempts: int
:return read_err: bool
:return data: 0 (read_err = True)
:return data: str (read_err = False)
"""
logger = logging.getLogger(logfile)
stat_file = STAT_LVL['op']
data_file_out = 0
attempt = 1
file_mode = ''
field_dict = {}
temp_path = None
file_hdlr = None
file_temp = None
log_oserror = ''
log_no_oserror = ''
if file_cmd == 'data_read':
file_mode = 'r'
log_no_oserror = 'to retrieve data from file {0} succeeded.'.format(
file_name)
log_oserror = 'to retrieve data from file {0} failed.'.format(
file_name)
elif file_cmd == 'line_app':
file_mode = 'a+'
log_no_oserror = 'to append line to file {0} succeeded.'.format(
file_name)
log_oserror = 'to append line to file {0} failed.'.format(file_name)
elif file_cmd == 'fld_read':
file_mode = 'r'
log_no_oserror = 'to read field from file {0} succeeded.'.format(
file_name)
log_oserror = 'to read field from file {0} failed.'.format(file_name)
elif file_cmd == 'fld_read_all':
file_mode = 'r'
log_no_oserror = 'to read all fields from file {0} succeeded.'.format(
file_name)
log_oserror = 'to read all fields from file {0} failed.'.format(
file_name)
elif file_cmd == 'fld_edit':
file_mode = 'r'
log_no_oserror = 'to edit field in file {0} succeeded.'.format(
file_name)
log_oserror = 'to edit field in file {0} failed.'.format(file_name)
elif file_cmd == 'data_wrt':
file_mode = 'r+'
log_no_oserror = 'to write data to file {0} succeeded.'.format(
file_name)
log_oserror = 'to write data to file {0} failed.'.format(file_name)
elif file_cmd == 'file_replace':
file_mode = 'w'
log_no_oserror = 'to replace contents in file {0} succeeded.'.format(
file_name)
log_oserror = 'to replace contents in file {0} failed.'.format(
file_name)
# Cycle through attempts
for attempt in range(1, (attempts + 1)):
try:
# Open file in specified mode, utf-8
file_open = open(file_name, mode=file_mode, encoding='utf-8')
if file_cmd == 'data_read':
file_open.seek(data_loc)
data_file_out = file_open.read(num_bytes)
elif file_cmd == 'line_app':
file_open.write(data_file_in + '\n')
file_open.flush()
elif file_cmd == 'fld_read':
for line in file_open:
line = line.split('\n')[0]
key = line.split('=')[0]
if key == search_field:
data_file_out = line.split('=')[1]
elif file_cmd == 'fld_read_all':
for line in file_open:
line = line.split('\n')[0]
key, val = line.split('=')
field_dict[key] = str(val)
elif file_cmd == 'fld_edit':
file_hdlr, temp_path = mkstemp()
file_temp = open(temp_path, mode='w', encoding='utf-8')
found_field = False
for line in file_temp:
if search_field in line:
file_temp.write(replace_line + '\n')
found_field = True
else:
file_temp.write(line)
if not found_field:
file_temp.write(replace_line)
file_temp.flush()
elif file_cmd == 'data_wrt':
file_open.seek(data_loc)
file_open.write(data_file_in)
file_open.flush()
elif file_cmd == 'file_replace':
file_open.seek(0)
file_open.write(data_file_in)
file_open.flush()
# Close file
file_open.close()
if file_cmd == 'fld_edit':
remove(file_name)
move(temp_path, file_name)
file_temp.close()
close(file_hdlr)
log = 'Attempt {0} of {1} '.format(attempt,
(attempts - 1)) + log_no_oserror
logger.debug(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
stat_file = STAT_LVL['op']
break
except OSError:
stat_file = STAT_LVL['op_err']
MPQ_STAT.put_nowait(['base', ['file', stat_file]])
if attempt == (attempts - 1):
log = 'Attempt {0} of {1} '.format(
attempt, (attempts - 1)) + log_oserror
logger.exception(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'ERROR', log])
time.sleep(0.1 * randint(0, 9) * attempt)
log_success = ''
log_failure = ''
if file_cmd == 'data_read':
log_success = 'Successfully read data from file {0} after {1} attempts.'.\
format(file_name, attempt)
log_failure = 'General failure to read data from file {0}.'.format(
file_name)
elif file_cmd == 'line_app':
log_success = 'Successfully appended line to file {0} after {1} attempts.'.\
format(file_name, attempt)
log_failure = 'General failure to append line to file {0}.'.format(
file_name)
elif file_cmd == 'fld_read':
log_success = 'Successfully read from file {0} after {1} attempts.'.\
format(file_name, attempt)
log_failure = 'General failure to read field from file {0}.'.format(
file_name)
elif file_cmd == 'fld_read_all':
log_success = 'Successfully read all fields from file {0} after {1} attempts.'.\
format(file_name, attempt)
log_failure = 'General failure to read all fields from file {0}.'.format(
file_name)
elif file_cmd == 'fld_edit':
log_success = 'Successfully edited field in file {0} after {1} attempts.'.\
format(file_name, attempt)
log_failure = 'General failure to edit field in file {0}.'.format(
file_name)
elif file_cmd == 'data_wrt':
log_success = 'Successfully wrote data to file {0} after {1} attempts.'.\
format(file_name, attempt)
log_failure = 'General failure to write data to file {0}.'.format(
file_name)
elif file_cmd == 'file_replace':
log_success = 'Successfully replaced contents in file {0} after {1} attempts.'.\
format(file_name, attempt)
log_failure = 'General failure to replace contents in file {0}.'.format(
file_name)
if not stat_file:
log = log_success
activity_status = 'DEBUG'
else:
log = log_failure
activity_status = 'CRITICAL'
stat_file = STAT_LVL['crit']
logger.log(logging.INFO if not stat_file else logging.CRITICAL, log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), activity_status, log])
MPQ_STAT.put_nowait(['base', ['file', stat_file]])
return data_file_out, stat_file
def send(
data_cdb_smtp: dict,
data_cdb_sms: dict,
dict_msg: dict,
):
"""
Sends email message
:param data_cdb_smtp: dict
:param data_cdb_sms: dict
:param dict_msg: dict
"""
stat_smtp = STAT_LVL['op']
for attempt in range(1, 4):
# Prepare message body, default is plain text
smtp_email_msg = MIMEText(dict_msg['smtp_body'])
sms_email_msg = None
# Prepare recipients lists
#
# p = operational polling
# s = system status updates
# e = system errors
smtp_recipients = None
sms_recipients = None
if dict_msg['smtp_distribution'] == 'p':
smtp_email_msg['To'] = ', '.join(data_cdb_smtp['smtp_list_alert'])
smtp_recipients = data_cdb_smtp['smtp_list_alert']
elif dict_msg['smtp_distribution'] == 's':
smtp_email_msg['To'] = ', '.join(data_cdb_smtp['smtp_list_status'])
smtp_recipients = data_cdb_smtp['smtp_list_status']
elif dict_msg['smtp_distribution'] == 'e':
smtp_email_msg['To'] = ', '.join(data_cdb_smtp['smtp_list_error'])
smtp_recipients = data_cdb_smtp['smtp_list_error']
# Prepare message sender
smtp_email_msg['From'] = data_cdb_smtp['smtp_from']
# Prepare message subject line
smtp_email_msg['Subject'] = Header(dict_msg['smtp_subject'])
# If SMS is enabled prepare second message
if data_cdb_sms['sms_enable']:
sms_email_msg = MIMEText(dict_msg['sms_body'])
sms_recipients = None
if dict_msg['sms_distribution'] == 'p':
sms_email_msg['To'] = ', '.join(data_cdb_sms['sms_list_alert'])
sms_recipients = data_cdb_sms['sms_list_alert']
elif dict_msg['sms_distribution'] == 's':
sms_email_msg['To'] = ', '.join(
data_cdb_sms['sms_list_status'])
sms_recipients = data_cdb_sms['sms_list_status']
elif dict_msg['sms_distribution'] == 'e':
sms_email_msg['To'] = ', '.join(data_cdb_sms['sms_list_error'])
sms_recipients = data_cdb_sms['sms_list_error']
sms_email_msg['From'] = data_cdb_smtp['smtp_from']
sms_email_msg['Subject'] = Header(dict_msg['sms_subject'])
pymail = None
# Build SMTP server connection
try:
pymail = smtplib.SMTP(host=data_cdb_smtp['smtp_server'],
port=data_cdb_smtp['smtp_port'],
timeout=data_cdb_smtp['smtp_timeout'])
except socket.gaierror:
stat_smtp = STAT_LVL['crit']
log = 'Attempt to send email failed due to unknown server.'
logger.exception(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'ERROR', log])
except socket.timeout:
stat_smtp = STAT_LVL['crit']
log = 'Attempt to send email failed due to connection timeout.'
logger.exception(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'ERROR', log])
except smtplib.SMTPConnectError:
stat_smtp = STAT_LVL['crit']
log = 'Attempt to connect to email server reached timeout ' +\
'of {0} sec.'.format(data_cdb_smtp['timeout'])
logger.exception(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'ERROR', log])
except smtplib.SMTPServerDisconnected:
stat_smtp = STAT_LVL['crit']
log = 'Attempt to send email failed because client and ' +\
'server inadvertently disconnected.'
logger.exception(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'ERROR', log])
except smtplib.SMTPException:
stat_smtp = STAT_LVL['crit']
log = 'Attempt to send email failed due to unspecified error.'
logger.exception(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'ERROR', log])
if (not stat_smtp) and (data_cdb_smtp['smtp_password'] is not None):
# Authenticate with smtp server
try:
pymail.login(user=data_cdb_smtp['smtp_from'],
password=data_cdb_smtp['smtp_password'])
except smtplib.SMTPAuthenticationError:
stat_smtp = STAT_LVL['crit']
log = 'Attempt to login to email server failed due to ' +\
'invalid credentials.'
logger.exception(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'ERROR', log])
except smtplib.SMTPException:
stat_smtp = STAT_LVL['crit']
log = 'Attempt to send email failed due to unspecified error.'
logger.exception(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'ERROR', log])
if not stat_smtp:
try:
# Send message packets through SMTP server
pymail.sendmail(from_addr=data_cdb_smtp['smtp_from'],
to_addrs=smtp_recipients,
msg=smtp_email_msg.as_string())
if data_cdb_sms['sms_enable']:
if len(sms_recipients) >= 1:
pymail.sendmail(from_addr=data_cdb_smtp['smtp_from'],
to_addrs=sms_recipients,
msg=sms_email_msg.as_string())
pymail.quit()
log = 'Emails successfully sent.'
logger.debug(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'DEBUG', log])
MPQ_STAT.put_nowait(['base', ['email', stat_smtp]])
break
except smtplib.SMTPException:
stat_smtp = STAT_LVL['crit']
log = 'Attempt to send emails failed due to unspecified error.'
logger.debug(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'ERROR', log])
time.sleep(5 * attempt)
if stat_smtp:
log = 'General failure to send emails.'
logger.warning(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'WARNING', log])
MPQ_STAT.put_nowait(['base', ['email', stat_smtp]])
def gpio_write(self,
addr_ln: int,
data_iface_in: int,
mode: bool = True,
stat_en: bool = True,
attempts: int = 2):
"""
Sets GPIO pin to output LOW signal
:param addr_ln: int
:param data_iface_in: int (0 or 1)
:param mode: bool
:param stat_en: bool
:param attempts: int
:return stat_iface: STAT_LVL['op'] or STAT_LVL['crit']
"""
attempt = 0
err_iface = True
stat_iface = STAT_LVL['op']
# Cycle through attempts
for attempt in range(1, (attempts + 1)):
# If mode flag is set, set GPIO mode to 'OUT' prior to GPIO write
if mode:
err_iface = self.obj_janus.gpio_set_mode(pin=addr_ln,
mode='OUT')
if not err_iface:
err_iface = self.obj_janus.gpio_write(pin=addr_ln,
value=data_iface_in)
else:
err_iface = self.obj_janus.gpio_write(pin=addr_ln,
value=data_iface_in)
if err_iface:
MPQ_STAT.put_nowait(
['base', ['interface', STAT_LVL['op_err']]])
# Only log warning on last attempt, keeps log clean
if attempt == attempts:
log = 'Attempt {0} of {1} to '.format(attempt, attempts) + \
'set GPIO pin {0} to value {1} failed.'.format(addr_ln, data_iface_in)
logger.warning(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'WARNING', log])
else:
break
if err_iface:
log = 'General IO failure to set GPIO pin {0} to value {1}.'.\
format(addr_ln, data_iface_in)
logger.critical(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'CRITICAL', log])
stat_iface = STAT_LVL['crit']
print(log)
else:
log = 'Successfully set GPIO pin {0} to value {1} after {2} attempts.'.\
format(addr_ln, data_iface_in, attempt)
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
if stat_en:
MPQ_STAT.put_nowait(['base', ['interface', stat_iface]])
return stat_iface
def engine(self):
"""
Setup SNMP engine and context
"""
log = 'SNMP agent engine initialization sequence begun. ' +\
'This may take a minute or two.'
logger.info(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
# Create SNMP engine with auto generated engineID and pre-bound
# to socket transport dispatcher
self.eng_snmp = engine.SnmpEngine()
# Transport setup
# UDP over IPv4 at 0.0.0.0:8900
config.addTransport(
self.eng_snmp, udp.domainName,
udp.UdpTransport().openServerMode(iface=('', 8900)))
# UDP over IPv6 at [::]:8900
config.addTransport(
self.eng_snmp, udp6.domainName,
udp6.Udp6Transport().openServerMode(iface=('::', 8900)))
# SNMPv2c setup
# SecurityName <-> CommunityName mapping.
config.addV1System(snmpEngine=self.eng_snmp,
communityIndex='agent',
communityName='janusess')
# Allow full MIB access for this user / securityModels at VACM
# MIB 1.3.6.1.4.1.9934 refers to Janus Research Group
# MIB 1.3.6.1.4.1.9934.0 refers to JanusESS Project
config.addVacmUser(snmpEngine=self.eng_snmp,
securityModel=2,
securityName='agent',
securityLevel='noAuthNoPriv',
readSubTree=(1, 3, 6, 1, 4, 1, 9934, 0))
# Get default SNMP context this SNMP engine serves
self.ctx_snmp = context.SnmpContext(snmpEngine=self.eng_snmp)
# Create custom Managed Object Instance
self.mib_builder = self.ctx_snmp.getMibInstrum().getMibBuilder()
mib_sources = self.mib_builder.getMibSources() + \
(
builder.DirMibSource('/opt/Janus/ESS/python3/server/snmp/mibs'),
)
self.mib_builder.setMibSources(*mib_sources)
# JANUS-MIB defines and locates all Janus Research Group projects
# JANUSESS-MIB defines all JanusESS project entries
self.mib_builder.loadModules('JANUS-MIB', 'JANUSESS-MIB')
self.config_base()
self.config_lane()
self.config_module()
self.config_sensor()
# Register SNMP Applications at the SNMP engine for particular SNMP context
cmdrsp.GetCommandResponder(self.eng_snmp, self.ctx_snmp)
cmdrsp.NextCommandResponder(self.eng_snmp, self.ctx_snmp)
cmdrsp.BulkCommandResponder(self.eng_snmp, self.ctx_snmp)
dispatcher = threading.Thread(target=self.dispatcher, args=())
dispatcher.start()
MPQ_STAT.put_nowait(['snmp_agent', dispatcher.ident])
log = 'SNMP agent engine sequence concluded.'
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['snmp_agent', STAT_LVL['op']]])
log = 'SNMP agent listener started.'
logger.info(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
# Poll SNMP agent queues for values to update variables
while True:
if not MPQ_SNMPA_STOP.empty():
MPQ_SNMPA_STOP.get()
self.eng_snmp.transportDispatcher.closeDispatcher()
break
if not MPQ_SNMPA2.empty():
mpq_record = MPQ_SNMPA2.get()
self.base(mpq_record=mpq_record)
if not MPQ_SNMPA3.empty():
mpq_record = MPQ_SNMPA3.get()
self.lane(mpq_record=mpq_record)
if not MPQ_SNMPA4.empty():
mpq_record = MPQ_SNMPA4.get()
self.module(mpq_record=mpq_record)
if not MPQ_SNMPA5.empty():
mpq_record = MPQ_SNMPA5.get()
self.sensor(mpq_record=mpq_record)
time.sleep(0.1)
MPQ_STAT.put_nowait(['snmp_agent', False])
log = 'SNMP agent dispatcher stopped.'
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
log = 'SNMP agent listener stopped.'
logger.info(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
MPQ_STAT.put_nowait(['base', ['snmp_agent', STAT_LVL['not_cfg']]])
def init(obj_iface: TYPE_INTERFACE, addr_ln: int):
"""
Initializes lane
:param obj_iface: Interface Object
:param addr_ln: int
:return mod_last: 0 (if STAT_LVL['crit'])
:return mod_last: int (if STAT_LVL['op'])
:return stat_ln: json (if STAT_LVL['op'] or STAT_LVL['not_cfg'])
:return stat_iface: STAT_LVL['op'] or STAT_LVL['crit']
:return stat_cdb: STAT_LVL['op'] or STAT_LVL['crit']
"""
# Change logging level since this operates in multiprocess
# Cycle to last entry for most current log setting
while not MPQ_SETUP_LOG_INIT.empty():
mpq_record = MPQ_SETUP_LOG_INIT.get()
if mpq_record[0] == 'DEBUG':
logger.setLevel(logging.DEBUG)
elif mpq_record[0] == 'INFO':
logger.setLevel(logging.INFO)
elif mpq_record[0] == 'ERROR':
logger.setLevel(logging.ERROR)
elif mpq_record[0] == 'WARNING':
logger.setLevel(logging.WARNING)
elif mpq_record[0] == 'CRITICAL':
logger.setLevel(logging.CRITICAL)
stat_ln = STAT_LVL['op']
count_mod = 0
# Each module with an entry in CouchDB modconfig database
# has a setup_id field that uniquely identifies the latest
# time that it was placed on a lane and setup.
setup_id = time.time()
# Set lane GPIO pin to READ mode so that interrupts can
# be captured. This also places pin into HIGH state.
data_iface_out, stat_iface = obj_iface.gpio_read(addr_ln=addr_ln,
mode=True,
stat_en=True)
if not stat_iface:
mod_last_found = False
# Sets the lane on a four-lane interface, default
# return of operational if interface is single lane
stat_iface = obj_iface.i2c_lane_set(addr_ln=addr_ln)
if not stat_iface:
# This loop cycles through each connected module with address
# of 0x7F until mod_last_found flag is set
while not mod_last_found and (count_mod <= 126):
# Retrieve memory to end of mod_uid from module with I2C address of 0x7F.
# If module responds, proceed to module setup actions, otherwise
# mod_last_found flag is set.
time_a = time.time()
high_mmap = len(MMAP) - 1
addr_mem = MMAP[high_mmap]['M_CFG_ALL'][0]
data_len = MMAP[high_mmap]['M_CFG_ALL'][1]
data_iface_out, stat_iface = obj_iface.i2c_read(
addr_ln=addr_ln,
addr_mod=0x7F,
addr_mem=addr_mem,
data_len=data_len,
stat_en=False)
print('lane {0} module {1} i2c_read mod_config: {2}'.format(
addr_ln, (count_mod + 1), round((time.time() - time_a),
3)))
print(data_iface_out)
# Check for proper memory map version
if not stat_iface and (data_iface_out[2] <= high_mmap):
# Call module setup routine and return module id and status
module.setup(obj_iface=obj_iface,
setup_id=setup_id,
cfg_bytes=data_iface_out,
addr_ln=addr_ln,
addr_mod=(count_mod + 1))
# Increment counter before moving to another module
count_mod += 1
# Skip assigning I2C address #70 to module, on four-port interface
# this address used to set the lane
if count_mod == 70:
count_mod += 1
print('Full lane {0} module {1} setup: {2}'.format(
addr_ln, (count_mod - 1),
round((time.time() - time_a), 3)))
# If module has improper memory map version, or if module throws error
# on I2C read, halt lane setup routine
#
# Impossible to differentiate from error thrown from reading connected module
# or error thrown by reading non-existent module, treat both as the same and
# end lane setup routine on this module.
else:
mod_last_found = True
else:
log = 'Could not setup lane due to interface and/or CouchDB error.'
logger.warning(log)
if count_mod >= 1:
log = 'Lane {0} initialized.'.format(addr_ln)
logger.info(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
log = 'Lane {0} last module is {1}.'.format(addr_ln, count_mod)
logger.info(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'INFO', log])
else:
log = 'No modules found for lane {0}.'.format(addr_ln)
logger.warning(log)
print(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'WARNING', log])
stat_ln = STAT_LVL['not_cfg']
else:
log = 'Could not complete priority 3 interface request on ' + \
'lane {0} due to i2c lane set error.'.format(addr_ln)
logger.critical(log)
stat_ln = STAT_LVL['not_cfg']
MPQ_STAT.put_nowait([
'lane',
[
addr_ln, {
'status': stat_ln,
'last_module': count_mod,
'setup_id': setup_id
}
]
])
# Get stat_lane view from CouchDB modconfig database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='get_view',
cdb_name='modconfig',
cdb_doc='stat_lane{0}'.format(addr_ln),
logfile=logfile)
mdb_sql = """
SELECT
*
FROM
aurora.lanes
WHERE
lane={0}
""".format(addr_ln)
data_mdb_out, stat_mdb, mdb_err = dbase.mdb_request(mdb_sql=mdb_sql,
logfile=logfile)
print(stat_mdb)
print(mdb_err)
print(data_mdb_out)
if not stat_cdb:
# Iterate through modules in view, determine which modules were
# previously connected to this lane but are no longer connected.
# Set their lane and module addresses to NULL and their status
# to unconfigured.
for dict_mod in data_cdb_out:
if dict_mod['value']['setup_id'] != setup_id:
data_cdb_in = {
'lane_addr': None,
'mod_addr': None,
'status': STAT_LVL['not_cfg'],
'errno': 0
}
# Update module document in CouchDB modconfig database
data_cdb_out, stat_cdb, http_cdb = dbase.cdb_request(
cdb_cmd='upd_doc',
cdb_name='modconfig',
cdb_doc=dict_mod['id'],
data_cdb_in=data_cdb_in,
logfile=logfile)
if stat_cdb:
log = 'Could not update module configurations due to CouchDB error.'
logger.warning(log)
print('UPDATED STATUS ON NON-EXISTENT MODULE: {0}'.format(
dict_mod['id']))
else:
log = 'Could not update module configurations due to CouchDB error.'
logger.warning(log)
return stat_ln, stat_cdb
def dispatcher():
"""
Automatically dispatches polling commands into MPQ_CMD5 priority queue
"""
stat_cdb = STAT_LVL['op']
stat_cdb_prev = STAT_LVL['not_cfg']
cfg_poll = [
STAT_LVL['not_cfg'],
STAT_LVL['not_cfg'],
STAT_LVL['not_cfg'],
STAT_LVL['not_cfg']
]
while not stat_cdb:
# Cycle through all lanes
for addr_ln in range(0, 4):
# Change logging level since this operates in multiprocess
# Cycle to last entry for most current log setting
while not MPQ_POLL_LOG_DISP.empty():
mpq_record = MPQ_POLL_LOG_DISP.get()
if mpq_record[0] == 'DEBUG':
logger.setLevel(logging.DEBUG)
elif mpq_record[0] == 'INFO':
logger.setLevel(logging.INFO)
elif mpq_record[0] == 'ERROR':
logger.setLevel(logging.ERROR)
elif mpq_record[0] == 'WARNING':
logger.setLevel(logging.WARNING)
elif mpq_record[0] == 'CRITICAL':
logger.setLevel(logging.CRITICAL)
# Set polling status flag for this lane if start command is issued
# on MPQ_POLL_START
if not MPQ_POLL_START.empty():
mpq_record = MPQ_POLL_START.get()
cfg_poll[mpq_record[0]] = STAT_LVL['op']
MPQ_STAT.put_nowait([
'lane',
[
mpq_record[0],
{'poll': cfg_poll[mpq_record[0]]}
]
])
# Send polling start messaging with timeout in seconds
send_mail(
msg_type='poll_start',
args=[mpq_record[0]],
)
# If polling status flag is set, execute polling for lane
if not cfg_poll[addr_ln]:
# Get lane_status document from CouchDB lanes database
data0_cdb_out, stat0_cdb, http0_cdb = dbase.cdb_request(
cdb_cmd='get_doc',
cdb_name='lanes',
cdb_doc='lane{0}_status'.format(addr_ln),
logfile=logfile
)
# Check that both lane status and polling have not failed
if (not stat0_cdb) and (data0_cdb_out['status'] < STAT_LVL['crit']) and \
(data0_cdb_out['poll'] < STAT_LVL['crit']):
# Get stat_lane view from CouchDB modconfig database
data1_cdb_out, stat1_cdb, http1_cdb = dbase.cdb_request(
cdb_cmd='get_view',
cdb_name='modconfig',
cdb_doc='stat_lane{0}'.format(addr_ln),
logfile=logfile
)
if not stat1_cdb:
# Cycle through all modules connected to lane
# If a module has not failed, initiate polling actions
for dict_mod in data1_cdb_out:
if dict_mod['value']['status'] < STAT_LVL['crit']:
# Developer code to check for speed
print('priority 5 interface added')
time_m = time.time()
# Immediate stop polling for this lane
# if stop command is issued on MPQ_POLL_STOP
if not MPQ_POLL_STOP.empty():
mpq_record = MPQ_POLL_STOP.get()
cfg_poll[mpq_record] = STAT_LVL['not_cfg']
log = 'Lane {0} polling exit command executed.'. \
format(mpq_record)
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
MPQ_STAT.put_nowait([
'lane',
[
mpq_record,
{'poll': cfg_poll[mpq_record]}
]
])
# Send polling stop messaging with timeout in seconds
send_mail(
msg_type='poll_stop',
args=[mpq_record],
)
if mpq_record == addr_ln:
break
# Issue request to poll this module
MPQ_CMD5.put([
dict_mod['id'],
addr_ln,
dict_mod['key']
])
# Immediate stop polling for this lane
# if stop command is issued on MPQ_POLL_STOP.
# Second check ensures quicker response time.
if not MPQ_POLL_STOP.empty():
mpq_record = MPQ_POLL_STOP.get()
cfg_poll[mpq_record] = STAT_LVL['not_cfg']
log = 'Lane {0} polling exit command executed.'.format(mpq_record)
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
MPQ_STAT.put_nowait([
'lane',
[
mpq_record,
{'poll': cfg_poll[mpq_record]}
]
])
# Send polling stop messaging with timeout in seconds
send_mail(
msg_type='poll_stop',
args=[mpq_record],
)
if mpq_record == addr_ln:
break
log = 'Lane {0} module {1} poll added to job queue.'. \
format(addr_ln, dict_mod['key'])
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'DEBUG',
log
])
# Determine if all module sensor data processing is
# complete prior to proceeding. This pause is here
# to prevent the poll dispatcher from getting too far
# ahead of module sensor data processing.
while True:
if not MPQ_POLL_COMPLETE.empty():
mpq_record = MPQ_POLL_COMPLETE.get()
if (mpq_record[0] == addr_ln) and \
(mpq_record[1] == dict_mod['key']):
log = 'Lane {0} module {1} automated poll completed.'.\
format(addr_ln, dict_mod['key'])
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'INFO',
log
])
break
time.sleep(0.02)
# Developer code to check for speed
print('Module {0} cycle complete: {1}'.
format(dict_mod['key'], round((time.time() - time_m), 3)))
log = 'Lane {0} module {1} poll dispatch cycle complete.'. \
format(addr_ln, dict_mod['key'])
logger.info(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'DEBUG',
log
])
else:
stat_cdb = stat1_cdb
last_dtg = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time())
)
MPQ_STAT.put_nowait([
'lane',
[
addr_ln,
{'last_dtg': last_dtg}
]
])
time.sleep(15)
else:
stat_cdb = stat0_cdb
time.sleep(1)
# Determine any changes in CouchDB status and report for both
# CouchDB and poll dispatcher. CouchDB is only cause for change in
# poll_dispatcher status
if stat_cdb != stat_cdb_prev:
stat_cdb_prev = stat_cdb
MPQ_STAT.put_nowait([
'base',
[
'couchdb',
stat_cdb
]
])
MPQ_STAT.put_nowait([
'base',
[
'poll_dispatch',
stat_cdb
]
])
log = 'Polling dispatcher failed due to CouchDB error.'
logger.critical(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
def i2c_read(self,
addr_ln: int,
addr_mod: int,
addr_mem: int,
data_len: int,
stat_en: bool = True,
attempts: int = 1):
"""
Reads block of bytes from I2C network
:param addr_ln: int
:param addr_mod: int
:param addr_mem: int
:param data_len: int
:param stat_en: bool
:param attempts: int
:return data_iface_out: list (if STAT_LVL['op'])
:return data_iface_out: 0 (if STAT_LVL['crit'])
:return stat_iface: STAT_LVL['op'] or STAT_LVL['crit']
"""
# Janus interfaces allow for 64 byte transfers, including 5 or 6 bytes of
# addressing header information. Modules use an memory map that is
# 256 or 512 bytes in length, therefore breaking data into 32 byte blocks
# is more convenient to track and error-check than 58- or 59-byte blocks.
packet_length = 48
number_packets = data_len // packet_length
number_bytes = data_len % packet_length
attempt = 0
data_iface_out = []
err_iface = False
stat_iface = STAT_LVL['op']
# Cycle through all whole blocks
for packet in range(0, (number_packets + 1)):
data_addr = ((packet * packet_length) + addr_mem)
if packet < number_packets:
data_length = packet_length
else:
data_length = number_bytes
# Cycle through attempts
for attempt in range(1, (attempts + 1)):
# Module 0x7F read begins at later failure stage and uses
# less corrective actions on failure to reduce setup time.
if addr_mod == 127:
data_out, err_iface = self.obj_janus.i2c_read(
addr_i2c=addr_mod,
addr_mem=data_addr,
data_len=data_length,
reset=False)
else:
data_out, err_iface = self.obj_janus.i2c_read(
addr_i2c=addr_mod,
addr_mem=data_addr,
data_len=data_length)
if err_iface:
# Only process errors on attempts for non-0x7F I2C addresses
if addr_mod != 127:
MPQ_STAT.put_nowait(
['base', ['interface', STAT_LVL['op_err']]])
# Only log warning on last attempt, keeps log clean
if attempt == attempts:
log = 'Attempt {0} of {1} to '.format(attempt, attempts) + \
'read packet {0} from I2C '.format(packet) +\
'link {0} module {1} failed.'.format(addr_ln, addr_mod)
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '), 'WARNING', log
])
else:
break
else:
# Add retrieved data to the end of the Python list variable
data_iface_out.extend(data_out)
break
if err_iface:
break
if err_iface:
data_iface_out = []
# Only process errors on attempts for non-0x7F I2C addresses
if addr_mod != 127:
log = 'General IO failure to read data from lane {0} module {1}.'.\
format(addr_ln, addr_mod)
logger.critical(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'CRITICAL', log])
print(log)
stat_iface = STAT_LVL['crit']
else:
stat_iface = STAT_LVL['op_err']
else:
log = 'Successfully read data from lane {0} '.format(addr_ln) + \
'module {0} memory {1} length {2} '.format(addr_mod, addr_mem, data_len) +\
'after {0} attempts.'.format(attempt)
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
if stat_en:
MPQ_STAT.put_nowait(['base', ['interface', stat_iface]])
return data_iface_out, stat_iface
def store_data(
cloud: dict,
poll_head: list,
poll_data: list
):
"""
Stores module poll data in CouchDB
:param cloud: dict
:param poll_head: list
:param poll_data: list
"""
loc_mod = poll_head[3]
loc_mod = loc_mod.replace(' ', '\ ')
loc_mod = loc_mod.replace(',\ ', '\,\ ')
addr_ln = poll_head[4]
addr_mod = poll_head[5]
data_idb_in = ''
data_idb_in_head = '{0},'.format(str(poll_head[0])) + \
'customer={0},'.format(poll_head[1]) + \
'base={0},'.format(poll_head[2]) + \
'location={0},'.format(loc_mod) + \
'lane={0},'.format(poll_head[4]) + \
'module={0},'.format(poll_head[5])
for poll_data_s in poll_data:
s_type = poll_data_s[3]
s_type = s_type.replace(' ', '\ ')
s_type = s_type.replace(',\ ', '\,\ ')
data_idb_in = data_idb_in + data_idb_in_head + \
'sensor={0},'.format(poll_data_s[0]) + \
'alert_type={0},'.format(poll_data_s[1]) + \
'alert={0} '.format(poll_data_s[2]) + \
'{0}={1} '.format(s_type, str(poll_data_s[4])) + \
'{0}'.format(int(poll_data_s[5] * 1000000000)) +\
"\n"
# Store data packet to local InfluxDB JanusESS database
try:
http0_resp = requests.post(
'http://localhost:8086/write?db=JanusESS',
headers={'Content-type': 'application/octet-stream'},
data=data_idb_in,
timeout=2.0 # Set higher for portability to rPi v3
)
# Returns HTTP status codes
if http0_resp.status_code == 204:
log = 'Uploaded of lane {0} module {1} '.format(addr_ln, addr_mod) +\
'data to local InfluxDB server successful.'
logger.debug(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'DEBUG',
log
])
MPQ_STAT.put_nowait([
'base',
[
'influxdb',
STAT_LVL['op']
]
])
else:
log = 'Could not upload lane {0} module {1} '.format(addr_ln, addr_mod) + \
'data to local InfluxDB server due to query error.'
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
MPQ_STAT.put_nowait([
'base',
[
'influxdb',
STAT_LVL['op_err']
]
])
except requests.exceptions.ConnectionError:
log = 'Local InfluxDB server did not respond to request.'
logger.critical(log)
print(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'CRITICAL',
log
])
MPQ_STAT.put_nowait([
'base',
[
'influxdb',
STAT_LVL['crit']
]
])
except requests.exceptions.ReadTimeout:
log = 'Local InfluxDB server timed out on request.'
logger.warning(log)
print(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
MPQ_STAT.put_nowait([
'base',
[
'influxdb',
STAT_LVL['op_err']
]
])
# If cloud storage is enabled, store data to cloud InfluxDB JanusESS database
if cloud['enable']:
# Store data packet to cloud InfluxDB JanusESS database
server = 'http://' + cloud['url'] + ':8086/write?db=JanusESS'
try:
http1_resp = requests.post(
server,
headers={'Content-type': 'application/octet-stream'},
data=data_idb_in,
timeout=1
)
# Returns HTTP status codes
if http1_resp.status_code == 204:
log = 'Uploaded of lane {0} module {1} '.format(addr_ln, addr_mod) + \
'data to remote InfluxDB server {0} successful.'.format(cloud['url'])
logger.debug(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'DEBUG',
log
])
else:
log = 'Could not upload lane {0} module {1} '.format(addr_ln, addr_mod) + \
'data to remote InfluxDB server due to query error.'
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
except requests.exceptions.ConnectionError:
log = 'Remote InfluxDB server {0} did not respond to request.'.format(cloud['url'])
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
except requests.exceptions.ReadTimeout:
log = 'Remote InfluxDB server {0} timed out on request.'.format(cloud['url'])
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '),
'WARNING',
log
])
log = 'Completed storage of lane {0} module {1} data.'.format(addr_ln, addr_mod)
logger.info(log)
def i2c_write(self,
addr_ln: int,
addr_mod: int,
addr_mem: int,
data_iface_in: list,
stat_en: bool = True,
attempts: int = 1):
"""
Writes block of bytes to I2C network
:param addr_ln: int
:param addr_mod: int
:param addr_mem: int
:param data_iface_in: list
:param stat_en: bool
:param attempts: int
:return stat_iface: STAT_LVL['op'] or STAT_LVL['crit']
"""
data_len = len(data_iface_in)
packet_length = 24
number_packets = data_len // packet_length
attempt = 0
err_iface = False
stat_iface = STAT_LVL['op']
# Cycle through all whole blocks
for packet in range(0, (number_packets + 1)):
if packet < number_packets:
data_in = data_iface_in[(packet * packet_length):packet_length]
else:
data_in = data_iface_in[(packet * packet_length):]
# Cycle through attempts
for attempt in range(1, (attempts + 1)):
# Module 0x7F write begins at later failure stage and uses
# less corrective actions on failure to reduce setup time.
if addr_mod == 127:
data_out, err_iface = self.obj_janus.i2c_write(
addr_i2c=addr_mod,
addr_mem=((packet * packet_length) + addr_mem),
data=data_in,
reset=False)
else:
data_out, err_iface = self.obj_janus.i2c_write(
addr_i2c=addr_mod,
addr_mem=((packet * packet_length) + addr_mem),
data=data_in)
if err_iface:
# Only process errors on attempts for non-0x7F I2C addresses
if addr_mod != 127:
MPQ_STAT.put_nowait(
['base', ['interface', STAT_LVL['op_err']]])
# Only log warning on last attempt, keeps log clean
if attempt == attempts:
log = 'Attempt {0} of {1} to '.format(attempt, attempts) + \
'write packet {0} to I2C '.format(packet) + \
'link {0} module {1} failed.'.format(addr_ln, addr_mod)
logger.warning(log)
MPQ_ACT.put_nowait([
datetime.now().isoformat(' '), 'WARNING', log
])
else:
break
else:
break
if err_iface:
break
if err_iface:
# Only process errors on attempts for non-0x7F I2C addresses
if addr_mod != 127:
log = 'General IO failure to write data to lane {0} module {1}.'. \
format(addr_ln, addr_mod)
logger.critical(log)
MPQ_ACT.put_nowait(
[datetime.now().isoformat(' '), 'CRITICAL', log])
print(log)
stat_iface = STAT_LVL['crit']
else:
stat_iface = STAT_LVL['op_err']
else:
log = 'Successfully write data to lane {0} '.format(addr_ln) + \
'module {0} memory {1} length {2} '.format(addr_mod, addr_mem, data_len) + \
'after {0} attempts.'.format(attempt)
logger.info(log)
MPQ_ACT.put_nowait([datetime.now().isoformat(' '), 'DEBUG', log])
if stat_en:
MPQ_STAT.put_nowait(['base', ['interface', stat_iface]])
return stat_iface