def processactions(**kwargs): # Read database to get our actions from cupid import pilib settings = { 'debug':False } settings.update(kwargs) if settings['debug']: pilib.set_debug() actiondicts = pilib.dbs.control.read_table('actions') for actiondict in actiondicts: # if we only want to process one action, skip others. if 'name' in kwargs: if actiondict['name'] != kwargs['name']: continue actiondict['log_path'] = pilib.dirs.logs.actions thisaction = action(actiondict) thisaction.process() thisaction.publish()
def monitor(**kwargs): settings = { 'port': None, 'baudrate': 115200, 'timeout': 1, 'checkstatus': True, 'printmessages': False, 'debug': True } settings.update(kwargs) if not settings['port']: settings['port'] = getsystemserialport() import serial from iiutilities import datalib, dblib from time import mktime, localtime from time import sleep motes_db = pilib.dbs.motes system_db = pilib.dbs.system if settings['debug']: pilib.set_debug() if settings['printmessages']: print('Message printing is enabled.') data = [] stringmessage = '' seriallog = True if seriallog: print('serial logging is enabled.') logfile = open(pilib.dirs.logs.serial, 'a', 1) logfile.write('\n' + datalib.gettimestring() + ": Initializing serial log\n") if settings['checkstatus']: systemstatus = system_db.read_table_row('systemstatus')[0] runhandler = systemstatus['serialhandlerenabled'] checktime = mktime(localtime()) checkfrequency = 15 # seconds if runhandler: utility.log( pilib.dirs.logs.io, "Starting monitoring of serial port based on check status", 1, pilib.loglevels.io) else: utility.log( pilib.dirs.logs.io, "Not starting monitoring of serial port. How did I get here?", 1, pilib.loglevels.serial) else: runhandler = True if runhandler: ser = serial.Serial(port=settings['port'], baudrate=settings['baudrate'], timeout=settings['timeout']) utility.log( pilib.dirs.logs.io, "Monitoring serial port {}, settings {}/{}".format( ser.name, settings['baudrate'], settings['timeout']), 1, pilib.loglevels.serial) else: utility.log(pilib.dirs.logs.io, 'not monitoring serial port ', 1, pilib.loglevels.serial) while runhandler: # This reading has to happen faster than the messages come, or they will all be stuck together try: ch = ser.read(1).decode('utf-8') # if ch == '\x0D': # print('carriage return') # elif ch == '\x00': # print('null character') if len(ch) == 0 or ch == '\x0D': utility.log(pilib.dirs.logs.io, 'Time to process message ', 5, pilib.loglevels.serial) # rec'd nothing print all if len( data ) > 1: # This will avoid processing endline characters and other trash. s = '' for x in data: s += '%s' % x # ord(x) # clear data data = [] # Here for diagnostics # print '%s [len = %d]' % (s, len(data)) # now process data # print(s) # print(s.split('\n')) try: utility.log( pilib.dirs.logs.serial, 'processing datadict from serial message of length {}' .format(len(data)), 3, pilib.loglevels.serial) datadicts, messages = processserialdata(s) except: import traceback message = "An exception of occurred (line 99): {}".format( traceback.format_exc()) utility.log(pilib.dirs.logs.serial, message, 1, pilib.loglevels.serial) else: for datadict, message in zip(datadicts, messages): if datadict: if (settings['printmessages']): print("datadict: ") print(datadict) # print("message: ") # print(message) # publish = False # for k in datadict: # # print(k + datadict[k]) # if k not in ['nodeid','RX_RSSI']: # pass # if 'cmd' in datadict: publish = True if publish: if (settings['printmessages']): print('publishing message: ') print(message) lograwmessages(message) motes_db.size_table('read', **{'size': 1000}) try: processremotedata(datadict, message) except: import traceback message = "An exception of occurred (line 184): {}".format( traceback.format_exc()) utility.log(pilib.dirs.logs.serial, message, 1, pilib.loglevels.serial) else: if message and settings['printmessage']: print('message: \n{}'.format(message)) print(message) # Log message if seriallog: try: logfile.write(datalib.gettimestring() + ' : ' + message + '\n') except: import traceback message = "An exception of occurred (line 198): {}".format( traceback.format_exc()) utility.log(pilib.dirs.logs.serial, message, 1, pilib.loglevels.serial) else: # no data, let's see if we should send message utility.log(pilib.dirs.logs.serial, 'No data, try sending', 1, pilib.loglevels.serial) # print('CLEARING DATA !!!') data = [] # try: # utility.log(pilib.dirs.logs.serial, "Attempting send routine", 4, pilib.loglevels.serial) # except Exception as e: # template = "An exception of type {0} occured while doing some serial logging. Arguments:\n{1!r}" # message = template.format(type(ex).__name__, ex.args) # print message # See if there are messages to send. # print('LET US TRY SEND HANDLER') try: queue_commands() except: import traceback print('ERROR IN QUEUE COMMANDS \n {}'.format( traceback.format_exc())) try: runsendhandler(ser) except: import traceback template = "An exception of in runsendhandler (line 142): {} .".format( traceback.format_exc()) utility.log(pilib.dirs.logs.serial, "Error in send routine: {}".format(template), 1, 1) # print('SEND HANDLER DONE') # # template = "An exception of type {0} occured. Arguments:\n{1!r}" # message = template.format(type(ex).__name__, ex.args) # pilib.log(pilib.dirs.logs.serial, message, 1, 1) else: # print('DATA NOT ZERO') # print(ch) data.append(ch) stringmessage += str(ch) if settings['checkstatus']: print('checking status') thetime = mktime(localtime()) if thetime - checktime > checkfrequency: print('checking control status') systemstatus = dblib.readonedbrow(pilib.dirs.dbs.control, 'systemstatus')[0] runserialhandler = systemstatus['serialhandlerenabled'] if runserialhandler: checktime = thetime utility.log( pilib.dirs.logs.io, 'Continuing serialhandler based on status check', 3, pilib.loglevels.io) else: runhandler = False utility.log( pilib.dirs.logs.io, 'Aborting serialhandler based on status check', 3, pilib.loglevels.io) except KeyboardInterrupt: print('\n Exiting on keyboard interrupt\n') logfile.close() return except: # print('no characters available!') sleep(0.5) # return #runsendhandler(ser) logfile.close() ser.close() return
def runconfig(**kwargs): from iiutilities import utility from cupid import pilib from iiutilities.datalib import gettimestring from iiutilities import dblib """ Interfaces and modes Interfaces: eth0 | wlan0 | wlan1 Modes: dhcp| ok | -- | -- static| ok | -- | -- station| -- | ok | ok ap| -- | ok | ok """ settings = { 'debug': False, 'onboot': False, 'config_all': False, 'ifaces_to_configure': ['wlan0'], 'config': { 'eth0': { 'enabled': True, 'mode': 'dhcp' }, 'wlan0': { 'enabled': True, 'mode': 'station', 'config': { 'network_select': ['name', 'strongest'], 'network': 'leHouse' } } }, 'use_default': False } settings.update(kwargs) if settings['debug']: pilib.set_debug() # Get from database # TODO : Used passed configuration data. if not settings['use_default']: import json netiface_config = pilib.dbs.system.read_table('netifaceconfig') if not netiface_config: message = 'netifaceconfig table empty or not found ! ' utility.log(pilib.dirs.logs.network, message, 0, pilib.loglevels.network) return {'status': 1, 'status_message': message} settings['config'] = {} for iface_config in netiface_config: settings['config'][iface_config['name']] = iface_config # unpack json dump of config details try: settings['config'][ iface_config['name']]['config'] = json.loads( iface_config['config']) except: message = 'Config entry for interface {} is empty or cannot be unpacked as json: {}. '.format( iface_config['name'], iface_config['config']) # print(settings['config'][iface_config['name']]) utility.log(pilib.dirs.logs.network, message, 3, pilib.loglevels.network) utility.log(pilib.dirs.logs.network, 'Updating ifconfig file. ', 0, pilib.loglevels.network) print('MAKING CONFIG FILE WITH CONFIG') print(settings['config']) make_ifconfig_file(config=settings['config']) # For now, we are going to assume that we are only using one wireless interface at most as a network station station_interface = None for interface_name, interface in settings['config'].items(): if interface['mode'] == 'station': station_interface = interface['name'] utility.log(pilib.dirs.logs.network, 'Running network reconfig (setting lastnetreconfig). ', 0, pilib.loglevels.network) dblib.setsinglevalue(pilib.dirs.dbs.system, 'netstatus', 'lastnetreconfig', gettimestring()) try: netconfigdata = dblib.readonedbrow(pilib.dirs.dbs.system, 'netconfig')[0] if settings['debug']: print("NETCONFIG:\n{}".format(netconfigdata)) except: utility.log(pilib.dirs.logs.network, 'Error reading netconfig data. ', 0, pilib.loglevels.network) return { 'status': 1, 'status_message': 'Error reading netconfig data. ' } else: utility.log(pilib.dirs.logs.network, 'Successfully read netconfig data', 3, pilib.loglevels.network) dblib.setsinglevalue(pilib.dirs.dbs.system, 'netstatus', 'mode', netconfigdata['mode']) utility.log(pilib.dirs.logs.network, 'Netconfig is enabled', 3, pilib.loglevels.network) # This will grab the specified SSID and the credentials and update # the wpa_supplicant file. At the moment, it also looks to see if the network is available. if station_interface: utility.log(pilib.dirs.logs.network, 'Updating wpa_supplicant', 3, pilib.loglevels.network) updatewpasupplicant(station_interface=station_interface) if settings['config_all']: utility.log(pilib.dirs.logs.network, 'Configuring all interfaces. ', 3, pilib.loglevels.network) settings['ifaces_to_configure'] = [ interface_name for interface_name in settings['config'] ] for interface_name in settings['ifaces_to_configure']: utility.log(pilib.dirs.logs.network, 'Configuring interface: {}'.format(interface_name), 3, pilib.loglevels.network) if interface_name not in settings['config']: message = 'Configuration not present for interface {}. '.format( interface_name) utility.log(pilib.dirs.logs.network, message, 1, pilib.loglevels.network) continue this_config = settings['config'][interface_name] if settings['debug']: print('CONFIG: \n{}'.format(this_config)) if this_config['mode'] == 'ap': killapservices() reset_net_iface(interface=interface_name) startapservices(interface_name) else: reset_net_iface(interface=interface_name) # Bridges require ipv4 being enabled in /etc/sysctl.conf # Here we are going to auto-bridge, but I think we should probably manually specify that the bridge should exist mode = None if all(interface in settings['config'] for interface in ['eth0', 'wlan0']): if settings['config']['wlan0']['mode'] == 'ap': mode = 'eth0wlan0bridge' if all(interface in settings['config'] for interface in ['wlan0', 'wlan1']): if settings['config']['wlan0']['mode'] == 'dhcp' and settings[ 'config']['wlan1']['mode'] == 'ap': mode = 'wlan0wlan1bridge' if all(interface in settings['config'] for interface in ['wlan0', 'wlan1']): if settings['config']['wlan1']['mode'] == 'dhcp' and settings[ 'config']['wlan0']['mode'] == 'ap': mode = 'wlan1wlan0bridge' if mode: utility.log(pilib.dirs.logs.network, 'Setting bridge for mode {}'.format(mode), 1, pilib.loglevels.network) runIPTables(mode)
def run_system_status(**kwargs): import pilib import time from iiutilities import utility from iiutilities import dblib from iiutilities import datalib from iiutilities import gitupdatelib from iiutilities import data_agent settings = {'debug': False, 'quiet': False, 'force': True} settings.update(kwargs) if settings['debug']: print('DEBUG MODE') settings['quiet'] = False pilib.set_debug() # This doesn't update git libraries. It checks current versions and updates the database try: utility.log(pilib.dirs.logs.system, 'Checking git versions', 3, pilib.loglevels.system) gitupdatelib.updaterepoversion(pilib.dirs.baselib, pilib.dirs.dbs.system, versiontablename='versions') gitupdatelib.updaterepoversion(pilib.dirs.web, pilib.dirs.dbs.system, versiontablename='versions') except: utility.log(pilib.dirs.logs.system, 'Error in git version check', 0, pilib.loglevels.system) else: utility.log(pilib.dirs.logs.system, 'Git version check complete', 3, pilib.loglevels.system) systemstatus = pilib.dbs.system.read_table_row('systemstatus')[0] # Get hardware info updatehardwareinfo() ## Read wireless config via iwconfig # this is breaking systemstatus for some reason # updateiwstatus() ## Read current netstatus lastnetstatus = {} try: lastnetstatus = dblib.readonedbrow(pilib.dirs.dbs.system, 'netstatus')[0] except: utility.log(pilib.dirs.logs.network, 'Error reading network status. ', 1, pilib.loglevels.network) else: utility.log(pilib.dirs.logs.system, 'Completed network status. ', 3, pilib.loglevels.network) # # Poll netstatus and return data # allnetstatus = updatenetstatus(lastnetstatus, quiet=settings['quiet']) # Keep reading system status? while systemstatus['systemstatusenabled'] or settings['force']: # Run notifications pilib.process_notifications_queue() try: data_agent.run_data_agent() except: utility.log(pilib.dirs.logs.system, 'Error running data agent. ', 1, pilib.loglevels.network) else: utility.log(pilib.dirs.logs.system, 'Data agent run successfully. ', 3, pilib.loglevels.network) currenttime = datalib.gettimestring() dblib.setsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'lastsystemstatuspoll', datalib.gettimestring()) starttime = time.time() utility.log(pilib.dirs.logs.system, 'System status routine is starting. ', 3, pilib.loglevels.system) """ Check all network statuses. The goal here is to totally decouple status read and reconfigure When we need to check all status data, we'll have it either in a dict or dict array, or in a database table This sub will read config and status and give both overall and granular interface statuses. Then, if status is not 'ok', we will reconfigure interface. """ if systemstatus['netstatusenabled']: utility.log(pilib.dirs.logs.system, 'Beginning network routines. ', 3, pilib.loglevels.system) # Update network interfaces statuses for all interfaces, in database tables as well # Check on wpa supplicant status as well. Function returns wpastatusdict try: utility.log(pilib.dirs.logs.system, 'Running updateifacestatus. ', 4, pilib.loglevels.system) utility.log(pilib.dirs.logs.network, 'Running updateifacestatus', 4, pilib.loglevels.network) allnetstatus = update_net_status(lastnetstatus, quiet=settings['quiet']) except: utility.log(pilib.dirs.logs.network, 'Exception in updateifacestatus. ') else: utility.log(pilib.dirs.logs.network, 'Updateifacestatus completed. ') utility.log(pilib.dirs.logs.system, 'Completed net status update. ', 4, pilib.loglevels.system) """ End network configuration status """ """ Do we want to autoconfig the network? If so, we analyze our netstatus data against what should be going on, and translate this into a network status. We have a list of ifaceconfigs and a list if ifacestatus """ if systemstatus['netconfigenabled'] and systemstatus[ 'netstatusenabled']: # No need to get this fresh. We have it stored. netconfig_data = allnetstatus['netconfig_data'] # We are going to hack in a jumper that sets AP configuration. This isn't the worst thing ever. # if netconfig_data['apoverride']: # result = processapoverride(21) ''' Now we check network status depending on the configuration we have selected ''' utility.log(pilib.dirs.logs.system, 'Running interface configuration watchdog. ', 4, pilib.loglevels.system) utility.log( pilib.dirs.logs.network, 'Running interface configuration. Mode: {}'.format( netconfig_data['mode']), 4, pilib.loglevels.network) # print('ALL IFACE STATUS') # print(allnetstatus['ifaces_status']) result = watchdognetstatus(allnetstatus=allnetstatus) else: utility.log(pilib.dirs.logs.system, 'Netconfig or netstatus disabled. ', 1, pilib.loglevels.system) dblib.setsinglevalue(pilib.dirs.dbs.system, 'netstatus', 'mode', 'manual') dblib.setsinglevalue(pilib.dirs.dbs.system, 'netstatus', 'statusmsg', 'netconfig is disabled') """ Check Hamachi """ if systemstatus['checkhamachistatus']: utility.log(pilib.dirs.logs.system, 'Hamachi watchdog is enabled', 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.network, 'Hamachi watchdog is enabled. ', 3, pilib.loglevels.network) # Only watchdog haamchi if we are connected to the network. netstatus = dblib.readonedbrow(pilib.dirs.dbs.system, 'netstatus')[0] if netstatus['WANaccess']: utility.log( pilib.dirs.logs.system, 'We appear to be online. Checking Hamachi Status. ', 3, pilib.loglevels.system) utility.log( pilib.dirs.logs.network, 'We appear to be online. Checking Hamachi Status. ', 3, pilib.loglevels.network) watchdoghamachi(pingip='25.11.87.7') utility.log(pilib.dirs.logs.system, 'Completed checking Hamachi Status. ', 3, pilib.loglevels.network) utility.log(pilib.dirs.logs.system, 'Completed checking Hamachi Status. ', 3, pilib.loglevels.network) else: utility.log( pilib.dirs.logs.system, 'We appear to be offline. Not checking Hamachi Status, but setting to 0. ', 3, pilib.loglevels.system) utility.log( pilib.dirs.logs.network, 'We appear to be offline. Not checking Hamachi Status, but setting to 0. ', 3, pilib.loglevels.network) dblib.setsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'hamachistatus', 0) else: utility.log(pilib.dirs.logs.system, 'Hamachi watchdog is disabled', 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.system, 'Finished interface configuration. ', 4, pilib.loglevels.system) # pilib.writedatedlogmsg(pilib.dirs.logs.network, statusmsg) utility.log(pilib.dirs.logs.system, 'Running updateifacestatus. ', 4, pilib.loglevels.system) update_net_status() utility.log(pilib.dirs.logs.system, 'Completed updateifacestatus. ', 4, pilib.loglevels.system) utility.log(pilib.dirs.logs.system, 'Network routines complete. ', 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.system, 'Checking system flags. ', 3, pilib.loglevels.system) processsystemflags() utility.log(pilib.dirs.logs.system, 'System flags complete. ', 3, pilib.loglevels.system) # Get system status again systemstatus = pilib.dbs.system.read_table('systemstatus')[0] elapsedtime = int(time.time() - starttime) utility.log( pilib.dirs.logs.system, 'Status routines complete. Elapsed time: {}'.format( str(elapsedtime)), 3, pilib.loglevels.system) utility.log( pilib.dirs.logs.system, 'System status is sleeping for {} .'.format( systemstatus['systemstatusfreq']), 3, pilib.loglevels.system) # print('enabled: ' , systemstatus['systemstatusenabled']) if 'runonce' in kwargs and kwargs['runonce']: break time.sleep(systemstatus['systemstatusfreq']) else: utility.log(pilib.dirs.logs.system, 'System status is disabled. Exiting. ', 0, pilib.loglevels.system)
def watchdoghamachi(pingip='self', threshold=3000, debug=False, restart=True): from iiutilities.netfun import runping, restarthamachi, killhamachi, gethamachistatusdata from iiutilities import utility from iiutilities import dblib from cupid import pilib if debug: pilib.set_debug() try: # if this throws an error, it means that there is not a valid status hamachistatusdata = gethamachistatusdata() except: import traceback error_message = traceback.format_exc() print(error_message) utility.log( pilib.dirs.logs.network, 'Error checking Hamachi via gethamachistatusdata with message: {}' .format(error_message), 1, pilib.loglevels.network) hamachistatusdata = {} # We are having some issue with return on hamachi status check. This is not always a reason to restart hamachi # We will carry on below and try to ping. if we can ping, we are good. if we need to self ping, this will still # Throw errors. BUT, main point is that if we can ping our chosen hamachi address, we are good. numpings = 10 try: print('I am here, doing the pings') utility.log( pilib.dirs.logs.network, 'Trying to ping hamachi gateway ({} pings) ... '.format(numpings), 1, pilib.loglevels.network) # So instead, we are going to test with a ping to another member on the network that # should always be online. This of course means that we have to make sure that it is, in fact, always # online if pingip in ['self', 'Self']: pingip = hamachistatusdata['address'] pingtimes = runping(pingip, numpings=numpings, quiet=False) pingmax = max(pingtimes) pingmin = min(pingtimes) pingave = sum(pingtimes) / len(pingtimes) # if hamachistatusdata['status'] not in ['logged in']: if pingave <= 0 or pingave > threshold: utility.log(pilib.dirs.logs.network, 'Pingtime unacceptable: ' + str(pingave) + '. ', 1, pilib.loglevels.network) dblib.setsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'hamachistatus', 0) utility.log(pilib.dirs.logs.network, 'Restarting Hamachi. ', 1, pilib.loglevels.network) killhamachi() restarthamachi() utility.log(pilib.dirs.logs.network, 'Completed restarting Hamachi. ', 1, pilib.loglevels.network) else: if pingmax > threshold or pingmin <= 0: utility.log(pilib.dirs.logs.system, 'Hamachi lives, with issues: ' + str(pingtimes), 3, pilib.loglevels.system) else: dblib.setsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'hamachistatus', 1) utility.log(pilib.dirs.logs.network, 'Hamachi appears fine. ', 3, pilib.loglevels.network) except: import traceback error_message = traceback.format_exc() utility.log( pilib.dirs.logs.network, 'Error checking Hamachi (second stage, pings) with message: {}'. format(error_message), 1, pilib.loglevels.network) if restart: utility.log(pilib.dirs.logs.network, 'Restarting hamachi: {}'.format(error_message), 1, pilib.loglevels.network) killhamachi() restarthamachi()
def rundaemon(**kwargs): """ First thing we are going to do is check to see if code is working. We do this first to minimize what we have to import to test this -- the script should not crash out before we do this. So we need dblib to function to read from the database to see whether we are going to email someone if things are broken. We need datalib to parse options on the notifications We also need utility to send an email """ settings = { 'startall':False, 'debug':False, 'daemon_freq': 60, 'unit_test_frequency': 3600, # Once per hour 'runonce':False } settings.update(kwargs) FNULL = open(os.devnull, 'w') try: import socket hostname = socket.gethostname() except: hostname = 'unknown (?!)' import importlib try: import simplejson as json except: import json testmodules = ['iiutilities.dblib', 'iiutilities.utility', 'iiutilities.datalib', 'cupid.pilib'] # these are the libraries we will need to send notifications that things aren't working. # To do this, however, we need some libraries. failures = '' for testmodule in testmodules: try: tempmodule = importlib.import_module(testmodule) except: failures += testmodule + ', ' if failures: # Send an email to indicate that things are horribly broken subject = 'Hostname: ' + hostname + ' things are broken.' message = 'Test import of module(s) ' + failures[:-2] + ' failed. ' em_gmail = gmail(subject=subject, message=message) em_gmail.send() from iiutilities import dblib, utility, datalib from cupid import pilib if settings['debug']: print('** DEBUG MODE ** ') pilib.set_debug() last_unittests = '' # Get notifications so we know when to notify system_database = pilib.dbs.system notification_database = pilib.dbs.notifications while True: notifications = system_database.read_table('notifications') currenttime = datalib.gettimestring() run_unit_tests = False if not last_unittests: run_unit_tests = True elif datalib.timestringtoseconds(currenttime) - datalib.timestringtoseconds(last_unittests) > settings['unit_test_frequency']: run_unit_tests = True if run_unit_tests: utility.log(pilib.dirs.logs.daemon, 'Running unit tests. ', 2, pilib.loglevels.daemon) handle_unit_tests() last_unittests = datalib.gettimestring() from subprocess import Popen, PIPE from time import sleep """ Set up list of enabled statuses (whether to restart if we find that the process is not currently running from iiutilities import dblib, utility, datalib """ system_status_options = system_database.read_table_row('systemstatus')[0] # print('systemstatusoptions') # print(system_status_options) item_enabled_dict = {'updateio':int(system_status_options['updateioenabled']), 'picontrol':int(system_status_options['picontrolenabled']), 'systemstatus':int(system_status_options['systemstatusenabled']), 'sessioncontrol':int(system_status_options['sessioncontrolenabled']), 'serialhandler':int(system_status_options['serialhandlerenabled']) } # updateio_enabled = int(system_status_options['updateioenabled']) # picontrol_enabled = int(system_status_options['picontrolenabled']) # systemstatus_enabled = int(system_status_options['systemstatusenabled']) # sessioncontrol_enabled = int(system_status_options['sessioncontrolenabled']) # serialhandler_enabled =int( system_status_options['serialhandlerenabled']) # enableditemlist = [(int(updateio_enabled)), (int(picontrolenabled)), int(systemstatusenabled), int(sessioncontrolenabled), int(serialhandlerenabled)] # These are hard-coded and must match up for now. This should be cleaned up to be more easily modified. itemstatuses = utility.find_proc_statuses(pilib.daemonprocs) item_status_dict = {} for proc_name, status in zip(pilib.daemonprocnames, itemstatuses): item_status_dict[proc_name] = status """ Here we check to see if things are running properly and not hung. First here is systemstatus """ if item_enabled_dict['systemstatus'] and item_status_dict['systemstatus']['count'] == 1: lastsystemstatus = dblib.getsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'lastsystemstatuspoll') currenttime = datalib.gettimestring() timesincelastsystemstatus = datalib.timestringtoseconds(currenttime) - datalib.timestringtoseconds(lastsystemstatus) timecriterion = 90 if timesincelastsystemstatus > timecriterion: utility.log(pilib.dirs.logs.daemon, 'Killing systemstatus because it has not run in ' + str(timesincelastsystemstatus) + 's', 1,pilib.loglevels.daemon) # utility.log(pilib.dirs.logs.system, 'Killing systemstatus because it has not run in ' + str(timesincelastsystemstatus) + 's',1,1, pilib.loglevels.system) killnotify = next((item for item in notifications if item['item'] == 'daemonkillproc' and int(item['enabled'])), None) if killnotify: options = datalib.parseoptions(killnotify['options']) if 'type' in options: if 'type' == 'email' and 'email' in options: # Queue a message indicating we had to restart the systemstatus daemon message = 'Systemstatus is being killed on ' + hostname + ' because it has not run in ' + \ str(timesincelastsystemstatus) + 's with a criteria of ' + \ str(timecriterion) + '. This occurred at ' + currenttime subject = 'CuPID : ' + hostname + ' : killnotify' notification_database.insert('queuednotifications', {'type': 'email', 'message': message, 'options': 'email:' + options['email'] + ',subject:' + subject, 'queuedtime': currenttime}) utility.kill_proc_by_name('systemstatus') # Also kill hamachi, since this is almost always the culprit utility.kill_proc_by_name('hamachi') # These are hard-coded and must match up for now. This should be cleaned up to be more easily modified. hamachi_status = utility.find_proc_statuses(['hamachi'])[0] if hamachi_status['count'] > 1: utility.log(pilib.dirs.logs.daemon, 'Killing hamachi with proc count of {}'.format(hamachi_status['count']), 0, pilib.loglevels.daemon) utility.kill_proc_by_name('hamachi') # Set system message systemstatusmsg = '' for name in pilib.daemonprocnames: systemincmessage = name + ' - Enabled: ' + str(item_enabled_dict[name]) + ' Status: ' + str(item_status_dict[name]['count']) + '. ' systemstatusmsg += systemincmessage utility.log(pilib.dirs.logs.daemon, 'Item status message: ' + systemincmessage, 0, pilib.loglevels.daemon) system_database.set_single_value('systemstatus', 'systemmessage', systemstatusmsg) # Set up list of itemnames in the systemstatus table that # we assign the values to when we detect if the process # is running or not for name, process in zip(pilib.daemonprocnames, pilib.daemonprocs): # set status if item_status_dict[name]['count'] == 1: # Set status variable by name. This is static based on schema system_database.set_single_value('systemstatus', name + 'status', 1) if pilib.loglevels.daemon > 0: utility.log(pilib.dirs.logs.daemon, 'Process is running: ' + pilib.dirs.baselib + process, 4, pilib.loglevels.daemon) elif item_status_dict[name]['count'] > 1: # multiple instances are running. This is bad. system_database.set_single_value('systemstatus', name + 'status', 0) if pilib.loglevels.daemon > 0: utility.log(pilib.dirs.logs.daemon, 'Multple instances of process {} are running ({}): '.format(pilib.dirs.baselib + process, item_status_dict[name]['count']), 2, pilib.loglevels.daemon) utility.kill_proc_by_name(process) # Now fire up if we need to. if item_status_dict[name]['count'] != 1: system_database.set_single_value('systemstatus', name + 'status', 0) if pilib.loglevels.daemon > 0: utility.log(pilib.dirs.logs.daemon, 'Process is not running: ' + pilib.dirs.baselib + process, 2, pilib.loglevels.daemon) # run if set to enable if item_enabled_dict[name]: # print(pilib.dirs.baselib + pilib.daemonprocs[index]) if pilib.loglevels.daemon > 0: utility.log(pilib.dirs.logs.daemon, 'Starting ' + pilib.dirs.baselib + process, 2, pilib.loglevels.daemon) # procresult = Popen([pilib.dirs.baselib + process], stdout=PIPE, stderr=PIPE) procresult = Popen([pilib.dirs.baselib + process, '&'], stdout=FNULL, stderr=FNULL) # if pilib.loglevels.daemon > 0: # pilib.writedatedlogmsg(pilib.dirs.logs.daemonproc, procresult.stdout.read()) # Time to let things start up sleep(3) # Refresh after set itemstatuses = utility.find_proc_statuses(pilib.daemonprocs) item_status_dict = {} for name, status in zip(pilib.daemonprocnames, itemstatuses): item_status_dict[name] = status for name in pilib.daemonprocnames: # set status if item_status_dict[name]: system_database.set_single_value('systemstatus', name + 'status', 1) else: system_database.set_single_value('systemstatus', name + 'status', 0) """ Process Actions. Careful here. This does not carry out things like indicators, which are set from picontrol. A bit wonky, as we would like the indicators to update immediately. On the other hand, we want picontrol to be the master controller of IO. """ from cupid.actions import processactions utility.log(pilib.dirs.logs.daemon, 'Processing actions', 2, pilib.loglevels.daemon) processactions() utility.log(pilib.dirs.logs.daemon, 'Done processing actions', 2, pilib.loglevels.daemon) systemstatusmsg = '' for name in pilib.daemonprocnames: systemincmessage = name + ' - Enabled: ' + str(item_enabled_dict[name]) + ' Status: ' + json.dumps( item_status_dict[name]) + '. ' systemstatusmsg += systemincmessage if pilib.loglevels.daemon > 0: utility.log(pilib.dirs.logs.daemon, 'Item status message: ' + systemincmessage, 2, pilib.loglevels.daemon) # print(systemstatusmsg) system_database.set_single_value('systemstatus', 'systemmessage', systemstatusmsg) # Rotate all logs utility.log(pilib.dirs.logs.daemon, 'Rotating logs. ') pilib.rotate_all_logs() if settings['runonce']: return
def run_system_status(**kwargs): import pilib import time from iiutilities import utility from iiutilities import dblib from iiutilities import datalib from iiutilities import gitupdatelib from iiutilities import data_agent settings = { 'debug':False, 'quiet':False, 'force':True } settings.update(kwargs) if settings['debug']: print('DEBUG MODE') settings['quiet']=False pilib.set_debug() # This doesn't update git libraries. It checks current versions and updates the database try: utility.log(pilib.dirs.logs.system, 'Checking git versions', 3, pilib.loglevels.system) gitupdatelib.updaterepoversion(pilib.dirs.baselib, pilib.dirs.dbs.system, versiontablename='versions') gitupdatelib.updaterepoversion(pilib.dirs.web, pilib.dirs.dbs.system, versiontablename='versions') except: utility.log(pilib.dirs.logs.system, 'Error in git version check', 0, pilib.loglevels.system) else: utility.log(pilib.dirs.logs.system, 'Git version check complete', 3, pilib.loglevels.system) systemstatus = pilib.dbs.system.read_table_row('systemstatus')[0] # Get hardware info updatehardwareinfo() ## Read wireless config via iwconfig # this is breaking systemstatus for some reason # updateiwstatus() ## Read current netstatus lastnetstatus={} try: lastnetstatus = dblib.readonedbrow(pilib.dirs.dbs.system, 'netstatus')[0] except: utility.log(pilib.dirs.logs.network, 'Error reading network status. ', 1, pilib.loglevels.network) else: utility.log(pilib.dirs.logs.system, 'Completed network status. ', 3, pilib.loglevels.network) # # Poll netstatus and return data # allnetstatus = updatenetstatus(lastnetstatus, quiet=settings['quiet']) # Keep reading system status? while systemstatus['systemstatusenabled'] or settings['force']: # Run notifications pilib.process_notifications_queue() try: data_agent.run_data_agent() except: utility.log(pilib.dirs.logs.system, 'Error running data agent. ', 1, pilib.loglevels.network) else: utility.log(pilib.dirs.logs.system, 'Data agent run successfully. ', 3, pilib.loglevels.network) currenttime = datalib.gettimestring() dblib.setsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'lastsystemstatuspoll', datalib.gettimestring()) starttime = time.time() utility.log(pilib.dirs.logs.system, 'System status routine is starting. ', 3, pilib.loglevels.system) """ Check all network statuses. The goal here is to totally decouple status read and reconfigure When we need to check all status data, we'll have it either in a dict or dict array, or in a database table This sub will read config and status and give both overall and granular interface statuses. Then, if status is not 'ok', we will reconfigure interface. """ if systemstatus['netstatusenabled']: utility.log(pilib.dirs.logs.system, 'Beginning network routines. ', 3, pilib.loglevels.system) # Update network interfaces statuses for all interfaces, in database tables as well # Check on wpa supplicant status as well. Function returns wpastatusdict try: utility.log(pilib.dirs.logs.system, 'Running updateifacestatus. ', 4, pilib.loglevels.system) utility.log(pilib.dirs.logs.network, 'Running updateifacestatus', 4, pilib.loglevels.network) allnetstatus = update_net_status(lastnetstatus, quiet=settings['quiet']) except: utility.log(pilib.dirs.logs.network, 'Exception in updateifacestatus. ') else: utility.log(pilib.dirs.logs.network, 'Updateifacestatus completed. ') utility.log(pilib.dirs.logs.system, 'Completed net status update. ', 4, pilib.loglevels.system) """ End network configuration status """ """ Do we want to autoconfig the network? If so, we analyze our netstatus data against what should be going on, and translate this into a network status. We have a list of ifaceconfigs and a list if ifacestatus """ if systemstatus['netconfigenabled'] and systemstatus['netstatusenabled']: # No need to get this fresh. We have it stored. netconfig_data = allnetstatus['netconfig_data'] # We are going to hack in a jumper that sets AP configuration. This isn't the worst thing ever. # if netconfig_data['apoverride']: # result = processapoverride(21) ''' Now we check network status depending on the configuration we have selected ''' utility.log(pilib.dirs.logs.system, 'Running interface configuration watchdog. ', 4, pilib.loglevels.system) utility.log(pilib.dirs.logs.network, 'Running interface configuration. Mode: {}'.format(netconfig_data['mode']), 4, pilib.loglevels.network) # print('ALL IFACE STATUS') # print(allnetstatus['ifaces_status']) result = watchdognetstatus(allnetstatus=allnetstatus) else: utility.log(pilib.dirs.logs.system, 'Netconfig or netstatus disabled. ', 1, pilib.loglevels.system) dblib.setsinglevalue(pilib.dirs.dbs.system, 'netstatus', 'mode', 'manual') dblib.setsinglevalue(pilib.dirs.dbs.system, 'netstatus', 'statusmsg', 'netconfig is disabled') """ Check Hamachi """ if systemstatus['checkhamachistatus']: utility.log(pilib.dirs.logs.system, 'Hamachi watchdog is enabled', 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.network, 'Hamachi watchdog is enabled. ', 3, pilib.loglevels.network) # Only watchdog haamchi if we are connected to the network. netstatus = dblib.readonedbrow(pilib.dirs.dbs.system, 'netstatus')[0] if netstatus['WANaccess']: utility.log(pilib.dirs.logs.system, 'We appear to be online. Checking Hamachi Status. ', 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.network, 'We appear to be online. Checking Hamachi Status. ', 3, pilib.loglevels.network) watchdoghamachi(pingip='25.11.87.7') utility.log(pilib.dirs.logs.system, 'Completed checking Hamachi Status. ', 3, pilib.loglevels.network) utility.log(pilib.dirs.logs.system, 'Completed checking Hamachi Status. ', 3, pilib.loglevels.network) else: utility.log(pilib.dirs.logs.system, 'We appear to be offline. Not checking Hamachi Status, but setting to 0. ', 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.network, 'We appear to be offline. Not checking Hamachi Status, but setting to 0. ', 3, pilib.loglevels.network) dblib.setsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'hamachistatus', 0) else: utility.log(pilib.dirs.logs.system, 'Hamachi watchdog is disabled', 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.system, 'Finished interface configuration. ', 4, pilib.loglevels.system) # pilib.writedatedlogmsg(pilib.dirs.logs.network, statusmsg) utility.log(pilib.dirs.logs.system, 'Running updateifacestatus. ', 4, pilib.loglevels.system) update_net_status() utility.log(pilib.dirs.logs.system, 'Completed updateifacestatus. ', 4, pilib.loglevels.system) utility.log(pilib.dirs.logs.system, 'Network routines complete. ', 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.system, 'Checking system flags. ', 3, pilib.loglevels.system) processsystemflags() utility.log(pilib.dirs.logs.system, 'System flags complete. ', 3, pilib.loglevels.system) # Get system status again systemstatus = pilib.dbs.system.read_table('systemstatus')[0] elapsedtime = int(time.time() - starttime) utility.log(pilib.dirs.logs.system, 'Status routines complete. Elapsed time: {}'.format(str(elapsedtime)), 3, pilib.loglevels.system) utility.log(pilib.dirs.logs.system, 'System status is sleeping for {} .'.format(systemstatus['systemstatusfreq']), 3, pilib.loglevels.system) # print('enabled: ' , systemstatus['systemstatusenabled']) if 'runonce' in kwargs and kwargs['runonce']: break time.sleep(systemstatus['systemstatusfreq']) else: utility.log(pilib.dirs.logs.system, 'System status is disabled. Exiting. ', 0, pilib.loglevels.system)
def watchdoghamachi(pingip='self', threshold=3000, debug=False, restart=True): from iiutilities.netfun import runping, restarthamachi, killhamachi, gethamachistatusdata from iiutilities import utility from iiutilities import dblib from cupid import pilib if debug: pilib.set_debug() try: # if this throws an error, it means that there is not a valid status hamachistatusdata = gethamachistatusdata() except: import traceback error_message = traceback.format_exc() print(error_message) utility.log(pilib.dirs.logs.network, 'Error checking Hamachi via gethamachistatusdata with message: {}'.format(error_message), 1, pilib.loglevels.network) hamachistatusdata = {} # We are having some issue with return on hamachi status check. This is not always a reason to restart hamachi # We will carry on below and try to ping. if we can ping, we are good. if we need to self ping, this will still # Throw errors. BUT, main point is that if we can ping our chosen hamachi address, we are good. numpings = 10 try: print('I am here, doing the pings') utility.log(pilib.dirs.logs.network, 'Trying to ping hamachi gateway ({} pings) ... '.format(numpings), 1, pilib.loglevels.network) # So instead, we are going to test with a ping to another member on the network that # should always be online. This of course means that we have to make sure that it is, in fact, always # online if pingip in ['self', 'Self']: pingip = hamachistatusdata['address'] pingtimes = runping(pingip, numpings=numpings, quiet=False) pingmax = max(pingtimes) pingmin = min(pingtimes) pingave = sum(pingtimes)/len(pingtimes) # if hamachistatusdata['status'] not in ['logged in']: if pingave <= 0 or pingave > threshold: utility.log(pilib.dirs.logs.network, 'Pingtime unacceptable: ' + str(pingave) + '. ', 1, pilib.loglevels.network) dblib.setsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'hamachistatus', 0) utility.log(pilib.dirs.logs.network, 'Restarting Hamachi. ', 1, pilib.loglevels.network) killhamachi() restarthamachi() utility.log(pilib.dirs.logs.network, 'Completed restarting Hamachi. ', 1, pilib.loglevels.network) else: if pingmax > threshold or pingmin <= 0: utility.log(pilib.dirs.logs.system, 'Hamachi lives, with issues: ' + str(pingtimes), 3, pilib.loglevels.system) else: dblib.setsinglevalue(pilib.dirs.dbs.system, 'systemstatus', 'hamachistatus', 1) utility.log(pilib.dirs.logs.network, 'Hamachi appears fine. ', 3, pilib.loglevels.network) except: import traceback error_message = traceback.format_exc() utility.log(pilib.dirs.logs.network, 'Error checking Hamachi (second stage, pings) with message: {}'.format(error_message), 1, pilib.loglevels.network) if restart: utility.log(pilib.dirs.logs.network, 'Restarting hamachi: {}'.format(error_message), 1, pilib.loglevels.network) killhamachi() restarthamachi()
def runperiodicio(**kwargs): settings = {'force_run': False, 'run_once': False, 'debug': False, 'quiet':True, 'logerrors':True} settings.update(kwargs) systemdb = pilib.cupidDatabase(pilib.dirs.dbs.system, **settings) logdb = pilib.cupidDatabase(pilib.dirs.dbs.log, **settings) controldb = pilib.cupidDatabase(pilib.dirs.dbs.control, **settings) if settings['force_run']: updateioenabled = True else: updateioenabled = systemdb.get_single_value('systemstatus', 'updateioenabled') if settings['debug']: pilib.set_debug() settings['quiet'] = False if updateioenabled: import pigpio pi = pigpio.pi() io_objects = {} first_run = True # print("quiet : {}, {}, {} ".format(systemdb.settings['quiet'], logdb.settings['quiet'], controldb.settings['quiet'])) while updateioenabled: utility.log(pilib.dirs.logs.io, 'Running periodicupdateio', 3, pilib.loglevels.io) utility.log(pilib.dirs.logs.system, 'Running periodicupdateio', 3, pilib.loglevels.system) # Set last run time systemdb.set_single_value('systemstatus', 'lastupdateiopoll', datalib.gettimestring()) systemdb.set_single_value('systemstatus', 'updateiostatus', '1') # Read and record everything as specified in dirs.dbs.control # Update database of inputs with read data # DEBUG runupdate = True if runupdate: try: reply = updateio.updateiodata(piobject=pi, io_objects=io_objects, first_run=first_run, settings=settings) except: import traceback formatted_lines = traceback.format_exc().splitlines() error_mail = utility.gmail() import socket hostname = socket.gethostname() error_mail.subject = 'Error running ioupdate on host ' + hostname + '. ' error_mail.message = error_mail.subject + ' Traceback: ' + '\n'.join(formatted_lines) error_mail.send() else: utility.log(pilib.dirs.logs.io, 'DEBUG: Update IO disabled', 1, pilib.loglevels.io) utility.log(pilib.dirs.log.system, 'DEBUG: Update IO disabled', 1, pilib.loglevels.system) systemsdict = systemdb.read_table_row('systemstatus')[0] readtime = systemsdict['updateiofreq'] """ Defaults. TODO: We need to get these from a db entry that the user can set on the main control panel. These will live in the 'defaults' table. Imagine that. Then, we can set the logpoints for each input and channel. We'll store them in the ioinfo table """ plotpoints = 20 logpoints = 1000 try: logsettings = logdb.read_table('logsettings') for setting in logsettings: if setting['item'] == 'defaultlogpoints': logpoints = int(setting['value']) # print('logpoints found and set to ' + str(logpoints)) except: pass # print('not found or other error. oops. ') """ Update process_values in channels """ channels = controldb.read_table('channels') for channel in channels: # Get pv_input for each channel channelname = channel['name'] pv_input = channel['pv_input'] # Get the input for the name from inputs info # Then get the value and readtime from the input if it # can be found if pv_input and pv_input not in ['none', 'None']: values = controldb.get_values('inputs', ['value', 'polltime'], condition="name='" + pv_input + "'") if values: process_value = values['value'] process_value_time = values['polltime'] # Only update channel value if value was found # print('process_value: ', process_value) if process_value or process_value == 0: # print('control value for channel ' + channelname + ' = ' + str(process_value)) controldb.set_single_value('channels', 'process_value', str(process_value), "pv_input='" + pv_input + "'", queue=True) # pilib.sqlitequery(pilib.dirs.dbs.control, 'update channels set process_value=' + str( # process_value) + ' where pv_input = ' + "'" + pv_input + "'") controldb.set_single_value('channels', 'process_value_time', str(process_value_time), "pv_input='" + pv_input + "'", queue=True) # pilib.sqlitequery(pilib.dirs.dbs.control, # 'update channels set process_valuetime=\'' + controltime + '\' where pv_input = ' + "'" + pv_input + "'") else: print('input not found. ') else: # input is empty controldb.set_single_value('channels', 'statusmessage', 'No pv_input found', "name='" + channelname + "'", queue=True) # disable channel #pilib.sqlitequery(dirs.dbs.control,"update channels set enabled=0 where pv_input = \'" + pv_input + "'") if channel['sv_input'] and channel['sv_input'] not in ['none', 'None']: value = controldb.set_single_value('inputs', 'value', "name='" + channel['sv_input'] + "'", queue=True) # Only update channel value if value was found if value or value==0: # print('control value for channel ' + channelname + ' = ' + str(process_value)) controldb.set_single_value('channels', 'setpoint_value', str(value), "sv_input='" + channel['sv_input'] + "'", queue=True) if channel['enabled_input'] and channel['enabled_input'] not in ['none', 'None']: value = controldb.set_single_value('inputs', 'value', "name='" + channel['enabled_input'] + "'", queue=True) # Only update channel value if value was found if value or value==0: # print('control value for channel ' + channelname + ' = ' + str(process_value)) controldb.set_single_value('channels', 'enabled', str(value), "enabled_input='" + channel['enabled_input'] + "'", queue=True) if controldb.queued_queries: controldb.execute_queue() """ Log value into tabled log Get data for all sensors online """ inputsdata = controldb.read_table('inputs') for inputrow in inputsdata: logtablename = 'input_' + inputrow['id'] + '_log' utility.log(pilib.dirs.logs.io, 'Logging: ' + logtablename, 5, pilib.loglevels.io) # print( 'Logging: ' + logtablename, 5, pilib.loglevels.io) if datalib.isvalidtimestring(inputrow['polltime']): # Create table if it doesn't exist # query = 'create table if not exists \'' + logtablename + '\' ( value real, time text primary key)' # print('we are logging') log_db = pilib.cupidDatabase(pilib.dirs.dbs.log, **settings) # Includes 'if not exists' , so will not overwrite log_db.create_table(logtablename, pilib.schema.standard_datalog, dropexisting=False, queue=True) # dblib.sqlitequery(pilib.dirs.dbs.log, query) # Enter row insert = {'time':inputrow['polltime'], 'value':inputrow['value']} log_db.insert(logtablename, insert, queue=True) # query = dblib.makesqliteinsert(logtablename, [inputrow['value'], inputrow['polltime']],['value', 'time']) # dblib.sqlitequery(pilib.dirs.dbs.log, query) # print(log_db.queued_queries) log_db.execute_queue() # Clean log log_db.clean_log(logtablename) # Size log based on specified size log_options = datalib.parseoptions(inputrow['log_options']) log_db.size_table(logtablename, **log_options) else: pass # print('invalid poll time') """ log metadata """ pilib.get_and_set_logdb_metadata(pilib.dirs.dbs.log) if not settings['run_once']: utility.log(pilib.dirs.logs.io, 'Sleeping for ' + str(readtime), 1, pilib.loglevels.io) sleep(readtime) else: break if not settings['force_run']: # Read from systemstatus to make sure we should be running updateioenabled = systemdb.get_single_value('systemstatus', 'updateioenabled') # Signal to io reads that we just started. first_run = False systemdb.set_single_value('systemstatus', 'updateiostatus', '0')