def heartbeat():
print "start heartbeat check", " " , datetime.now()
logger.info('Start heartbeat routine %s', datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
connectedssid=networkmod.connectedssid()
connected=False
if len(connectedssid)==0:
logger.warning('Heartbeat check , no network connected -------------- try to connect')
print 'Heartbeat check , no network connected -------------- try to connect'
connected=networkmod.connect_network()
else:
logger.info('Heartbeat check , Connected Wifi Network: %s ', connectedssid[0])
if connectedssid[0]==networkmod.localwifisystem:
logger.info('Heartbeat check , Configured as wifi access point, check if possible to connect to wifi network')
connected=networkmod.connect_network()
else:
reachgoogle=networkmod.check_internet_connection(1)
if not reachgoogle:
logger.warning('Heartbeat check , test ping not able to reach Google -------------- No action')
print 'Heartbeat check , no IP connection-------------- No action'
#connected=networkmod.connect_network() # use this in case you require the system to try connect wifi again in case no internet is reached
connected=False
else:
logger.info('Heartbeat check , wifi connection OK')
print 'Heartbeat check , wifi connection OK'
connected=True
if connected:
# Check if remote IP address is changed compared to previous communication and in such case resend the mail
ipext=networkmod.get_external_ip()
logger.info('Heartbeat check , Check IP address change -%s- and previously sent -%s-', ipext , emailmod.IPEXTERNALSENT)
if (ipext!=""):
if (emailmod.IPEXTERNALSENT!=""):
if ipext!=emailmod.IPEXTERNALSENT:
print "Heartbeat check, IP address change detected. Send email with updated IP address"
logger.info('Heartbeat check, IP address change detected. Send email with updated IP address')
emailmod.sendallmail("alert","System detected IP address change, below the updated links")
else:
logger.info('Heartbeat check, IP address unchanged')
else:
# first mail has not been sent succesfully of IPEXTERNALSENT was not available by the time
print "System has been reconnected"
logger.info("System has been reconnected, IPEXTERNALSENT was empty")
emailmod.sendallmail("alert","System has been reconnected")
# Check current time is less than 60 second different from NTP information
# try to get the clock from network
print "check system clock"
logger.info('Heartbeat check, check clock')
networktime=clockmod.getNTPTime()
logger.info('Heartbeat check , Network time NTP: %s ', networktime)
systemtime=clockmod.readsystemdatetime()
logger.info('Heartbeat check , System time NTP: %s ', systemtime)
if not networktime=='':
diffsec=clockmod.timediffinsec(networktime, systemtime)
logger.info('Heartbeat check , difference between system time and network time, diffsec = %d ', diffsec)
if diffsec>60:
print "Heartbeat check , warning difference between system time and network time >60 sec, diffsec = " , diffsec
logger.warning('Heartbeat check , warning difference between system time and network time >60 sec, diffsec = %d ', diffsec)
print "Heartbeat check , Apply network datetime to system"
logger.warning('Heartbeat check , Apply network datetime to system ')
clockmod.setHWclock(networktime)
clockmod.setsystemclock(networktime)
else:
print "Heartbeat check , Clock OK"
logger.info('Heartbeat check , Clock OK')
else:
print "not able to get network time"
logger.warning('Heartbeat check , not able to get network time')
else:
print "not able to establish an internet connection"
logger.warning("not able to establish an internet connection")
# check master job has a next run"
isok, datenextrun = SchedulerMod.get_next_run_time("master")
if isok:
datenow=datetime.utcnow()
datenextrun = datenextrun.replace(tzinfo=None)
print "Master Scheduler Next run " , datenextrun , " Now (UTC) ", datenow
if datenextrun>datenow:
print "Masterschedule next RUN confirmed"
logger.info('Heartbeat check , Master Scheduler OK')
else:
isok=False
if not isok:
print "No next run for master scheduler"
logger.warning('Heartbeat check , Master Scheduler Interrupted')
emailmod.sendallmail("alert","Master Scheduler has been interrupted, try to restart scheduler")
resetmastercallback()
# check if there have been errors in Syslog
if DEBUGMODE:
Errortextlist=debuggingmod.searchsyslogkeyword("error")
if Errortextlist:
print "found error in syslog"
logger.warning("ERROR: found error in syslog -------------------------")
#send notification mail
if debuggingmod.SENTERRORTEXT!=Errortextlist[0]:
emailmod.sendallmail("alert","Error found in Syslog",Errortextlist)
debuggingmod.SENTERRORTEXT=Errortextlist[0]
else:
print "No error found in syslog"
logger.info('Heartbeat check , SYSLOG ok')
# check if there have been errors in Schedulerlog
if DEBUGMODE:
filename="logfiles/apscheduler_hydrosystem.log"
MYPATH=hardwaremod.get_path()
filenameandpath=os.path.join(MYPATH, filename)
Errortextlist=debuggingmod.searchLOGkeyword(filenameandpath,"error")
if Errortextlist:
print "found error in LOG ",filename
logger.warning("ERROR: found error in LOG , %s -------------------------",filename)
#send notification mail
if debuggingmod.SENTERRORTEXT!=Errortextlist[0]:
emailmod.sendallmail("alert","Error found in LOG",Errortextlist)
debuggingmod.SENTERRORTEXT=Errortextlist[0]
else:
print "No error found in LOG", filename
logger.info('Heartbeat check , LOG ok')
return True
def heartbeat():
print("start heartbeat check", " ", datetime.now())
logger.info('Start heartbeat routine %s',
datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
# check wifi connection
connectedssid = networkmod.connectedssid()
connected = False
if len(connectedssid) == 0:
if not networkmod.WIFIENDIS == "Disabled":
logger.warning(
'Heartbeat check , no network connected -------------- try to connect'
)
print(
'Heartbeat check , no network connected -------------- try to connect'
)
connected = networkmod.connect_network()
else:
logger.info('Heartbeat check , wifi disabled')
else:
logger.info('Heartbeat check , Connected Wifi Network: %s ',
connectedssid[0])
if connectedssid[0] == networkmod.localwifisystem:
logger.info(
'Heartbeat check , Configured as wifi access point, check if possible to connect to wifi network'
)
connected = networkmod.connect_network()
networkmod.DHCP_COUNTER = 0
else: # Connected to wifi network
reachgoogle = networkmod.check_internet_connection(1)
if not reachgoogle:
logger.warning(
'Heartbeat check wifi SSID ok, but test ping not able to reach Google'
)
print('Heartbeat check , no IP connection')
#connected=networkmod.connect_network() # use this in case you require the system to try connect wifi again in case no internet is reached
#logger.warning('Heartbeat check , DHCP reset counter %s' , str(networkmod.DHCP_COUNTER))
# DHCP reset
#if (networkmod.DHCP_COUNTER % 16)==0: # try to reset every 16x15min =4 hours
# try to reset DHCP
#logger.warning('Heartbeat check , reset DHCP')
#print 'Heartbeat check , reset DHCP'
#networkmod.resetDHCP()
#networkmod.DHCP_COUNTER=networkmod.DHCP_COUNTER+1
goON, GWipaddr = networkmod.checkGWsubnet("wlan0")
if GWipaddr == "":
logger.info(
'Gateway IP address NOT found, back to AP mode')
# back to AP mode! kind of risky business
networkmod.connect_AP()
else:
logger.info('Gateway IP address found %s', GWipaddr)
connected = False
else:
logger.info('Heartbeat check , wifi connection OK')
print('Heartbeat check , wifi connection OK')
#networkmod.DHCP_COUNTER=0
connected = True
if connected:
# Check if remote IP address is changed compared to previous communication and in such case resend the mail
ipext = networkmod.get_external_ip()
logger.info(
'Heartbeat check , Check IP address change -%s- and previously sent -%s-',
ipext, emailmod.IPEXTERNALSENT)
if (ipext != ""):
if (emailmod.IPEXTERNALSENT != ""):
if ipext != emailmod.IPEXTERNALSENT:
print(
"Heartbeat check, IP address change detected. Send email with updated IP address"
)
logger.info(
'Heartbeat check, IP address change detected. Send email with updated IP address'
)
emailmod.sendallmail(
"alert",
"System detected IP address change, below the updated links"
)
else:
logger.info('Heartbeat check, IP address unchanged')
else:
# first mail has not been sent succesfully of IPEXTERNALSENT was not available by the time
print("System has been reconnected")
logger.info(
"System has been reconnected, IPEXTERNALSENT was empty")
emailmod.sendallmail("alert", "System has been reconnected")
else:
print("not able to establish an internet connection")
logger.warning("not able to establish an internet connection")
# check clock with NTP and reset master scheduler in case of clock change
if CheckNTPandAdjustClockandResetSched():
return True
#check the static IP address
currentipaddr = networkmod.get_local_ip_raw()
logger.info('Target IP address= %s. Current access point IP addresses= %s',
networkmod.IPADDRESS, currentipaddr)
if networkmod.IPADDRESS not in currentipaddr:
#set IP address
logger.warning(
'Local Static IP address not in the list, Set Target IP address')
networkmod.addIP("wlan0")
else:
logger.info('Local Statip IP address OK')
# check master job has a next run"
isok, datenextrun = SchedulerMod.get_next_run_time("master")
if isok:
datenow = datetime.utcnow()
datenextrun = datenextrun.replace(tzinfo=None)
print("Master Scheduler Next run ", datenextrun, " Now (UTC) ",
datenow)
if datenextrun > datenow:
print("Masterschedule next RUN confirmed")
logger.info('Heartbeat check , Master Scheduler OK')
else:
isok = False
if not isok:
print("No next run for master scheduler")
logger.warning('Heartbeat check , Master Scheduler Interrupted')
#emailmod.sendallmail("alert","Master Scheduler has been interrupted, try to restart scheduler")
resetmastercallback()
return True
# check if there have been errors in Syslog
if DEBUGMODE:
logger.info('Heartbeat check , check errors in Syslog file')
Errortextlist = debuggingmod.searchsyslogkeyword("error")
if Errortextlist:
print("found error in syslog")
logger.warning(
"ERROR: found error in syslog -------------------------")
#send notification mail
if debuggingmod.SENTERRORTEXT != Errortextlist[0]:
emailmod.sendallmail("alert", "Error found in Syslog",
Errortextlist)
debuggingmod.SENTERRORTEXT = Errortextlist[0]
else:
print("No error found in syslog")
logger.info('Heartbeat check , SYSLOG ok')
# check if there have been errors in Schedulerlog
if DEBUGMODE:
logger.info('Heartbeat check , check errors in Sched log file')
filename = "logfiles/apscheduler_hydrosystem.log"
MYPATH = hardwaremod.get_path()
filenameandpath = os.path.join(MYPATH, filename)
Errortextlist = debuggingmod.searchLOGkeyword(filenameandpath, "error")
if Errortextlist:
print("found error in LOG ", filename)
logger.warning(
"ERROR: found error in LOG , %s -------------------------",
filename)
#send notification mail
if debuggingmod.SENTERRORTEXT != Errortextlist[0]:
emailmod.sendallmail("alert", "Error found in LOG",
Errortextlist)
debuggingmod.SENTERRORTEXT = Errortextlist[0]
else:
print("No error found in LOG", filename)
logger.info('Heartbeat check , LOG ok')
return True
def heartbeat():
print "start heartbeat check", " ", datetime.now()
logger.info('Start heartbeat routine %s',
datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
connectedssid = networkmod.connectedssid()
connected = False
if len(connectedssid) == 0:
logger.warning(
'Heartbeat check , no network connected -------------- try to connect'
)
print 'Heartbeat check , no network connected -------------- try to connect'
connected = networkmod.connect_network()
else:
logger.info('Heartbeat check , Connected Wifi Network: %s ',
connectedssid[0])
if connectedssid[0] == networkmod.localwifisystem:
logger.info(
'Heartbeat check , Configured as wifi access point, check if possible to connect to wifi network'
)
connected = networkmod.connect_network()
else:
reachgoogle = networkmod.check_internet_connection(1)
if not reachgoogle:
logger.warning(
'Heartbeat check , test ping not able to reach Google -------------- No action'
)
print 'Heartbeat check , no IP connection-------------- No action'
#connected=networkmod.connect_network() # use this in case you require the system to try connect wifi again in case no internet is reached
connected = False
else:
logger.info('Heartbeat check , wifi connection OK')
print 'Heartbeat check , wifi connection OK'
connected = True
if connected:
# Check if remote IP address is changed compared to previous communication and in such case resend the mail
ipext = networkmod.get_external_ip()
logger.info(
'Heartbeat check , Check IP address change -%s- and previously sent -%s-',
ipext, emailmod.IPEXTERNALSENT)
if (ipext != ""):
if (emailmod.IPEXTERNALSENT != ""):
if ipext != emailmod.IPEXTERNALSENT:
print "Heartbeat check, IP address change detected. Send email with updated IP address"
logger.info(
'Heartbeat check, IP address change detected. Send email with updated IP address'
)
emailmod.sendallmail(
"alert",
"System detected IP address change, below the updated links"
)
else:
logger.info('Heartbeat check, IP address unchanged')
else:
# first mail has not been sent succesfully of IPEXTERNALSENT was not available by the time
print "System has been reconnected"
logger.info(
"System has been reconnected, IPEXTERNALSENT was empty")
emailmod.sendallmail("alert", "System has been reconnected")
# Check current time is less than 60 second different from NTP information
# try to get the clock from network
print "check system clock"
logger.info('Heartbeat check, check clock')
networktime = clockmod.getNTPTime()
logger.info('Heartbeat check , Network time NTP: %s ', networktime)
systemtime = clockmod.readsystemdatetime()
logger.info('Heartbeat check , System time NTP: %s ', systemtime)
if not networktime == '':
diffsec = clockmod.timediffinsec(networktime, systemtime)
logger.info(
'Heartbeat check , difference between system time and network time, diffsec = %d ',
diffsec)
if diffsec > 60:
print "Heartbeat check , warning difference between system time and network time >60 sec, diffsec = ", diffsec
logger.warning(
'Heartbeat check , warning difference between system time and network time >60 sec, diffsec = %d ',
diffsec)
print "Heartbeat check , Apply network datetime to system"
logger.warning(
'Heartbeat check , Apply network datetime to system ')
clockmod.setHWclock(networktime)
clockmod.setsystemclock(networktime)
else:
print "Heartbeat check , Clock OK"
logger.info('Heartbeat check , Clock OK')
else:
print "not able to get network time"
logger.warning('Heartbeat check , not able to get network time')
else:
print "not able to establish an internet connection"
logger.warning("not able to establish an internet connection")
# check master job has a next run"
isok, datenextrun = SchedulerMod.get_next_run_time("master")
if isok:
datenow = datetime.utcnow()
datenextrun = datenextrun.replace(tzinfo=None)
print "Master Scheduler Next run ", datenextrun, " Now (UTC) ", datenow
if datenextrun > datenow:
print "Masterschedule next RUN confirmed"
logger.info('Heartbeat check , Master Scheduler OK')
else:
isok = False
if not isok:
print "No next run for master scheduler"
logger.warning('Heartbeat check , Master Scheduler Interrupted')
emailmod.sendallmail(
"alert",
"Master Scheduler has been interrupted, try to restart scheduler")
resetmastercallback()
# check if there have been errors in Syslog
if DEBUGMODE:
Errortextlist = debuggingmod.searchsyslogkeyword("error")
if Errortextlist:
print "found error in syslog"
logger.warning(
"ERROR: found error in syslog -------------------------")
#send notification mail
if debuggingmod.SENTERRORTEXT != Errortextlist[0]:
emailmod.sendallmail("alert", "Error found in Syslog",
Errortextlist)
debuggingmod.SENTERRORTEXT = Errortextlist[0]
else:
print "No error found in syslog"
logger.info('Heartbeat check , SYSLOG ok')
# check if there have been errors in Schedulerlog
if DEBUGMODE:
filename = "logfiles/apscheduler_hydrosystem.log"
MYPATH = hardwaremod.get_path()
filenameandpath = os.path.join(MYPATH, filename)
Errortextlist = debuggingmod.searchLOGkeyword(filenameandpath, "error")
if Errortextlist:
print "found error in LOG ", filename
logger.warning(
"ERROR: found error in LOG , %s -------------------------",
filename)
#send notification mail
if debuggingmod.SENTERRORTEXT != Errortextlist[0]:
emailmod.sendallmail("alert", "Error found in LOG",
Errortextlist)
debuggingmod.SENTERRORTEXT = Errortextlist[0]
else:
print "No error found in LOG", filename
logger.info('Heartbeat check , LOG ok')
return True