def CheckNTPandAdjustClockandResetSched(timediffsec=60):
# try to get the clock from network
print("check system clock")
logger.info('Check system clock vs NTP (Network Time Protocol)')
networktime = clockmod.getNTPTime()
logger.info('Network time NTP: %s ', networktime)
systemtime = clockmod.readsystemdatetime()
logger.info('System time NTP: %s ', systemtime)
if not networktime == '':
diffsec = clockmod.timediffinsec(networktime, systemtime)
logger.info(
'Difference between system time and network time, diffsec = %d ',
diffsec)
if diffsec > timediffsec:
print("Warning difference between system time and network time >",
timediffsec, " sec, diffsec = ", diffsec)
logger.warning(
'Warning difference between system time and network time >%d sec, diffsec = %d ',
timediffsec, diffsec)
print("Apply network datetime to system")
logger.warning('Apply network datetime to system ')
clockmod.setHWclock(networktime)
clockmod.setsystemclock(networktime)
# reset scheduling
logger.warning('Reset master Scheduler due to time change')
resetmastercallback()
return True
else:
print("Clock OK")
logger.info('Clock OK')
else:
print("not able to get network time")
logger.warning('Not able to get network time')
return False
def heartbeat():
print "start heartbeat check", " " , datetime.now()
logger.info('Start heartbeat routine %s', datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
connectedssid=networkmod.connectedssid()
connected=False
if len(connectedssid)==0:
logger.warning('Heartbeat check , no network connected -------------- try to connect')
print 'Heartbeat check , no network connected -------------- try to connect'
connected=networkmod.connect_network()
else:
logger.info('Heartbeat check , Connected Wifi Network: %s ', connectedssid[0])
if connectedssid[0]==networkmod.localwifisystem:
logger.info('Heartbeat check , Configured as wifi access point, check if possible to connect to wifi network')
connected=networkmod.connect_network()
else:
reachgoogle=networkmod.check_internet_connection(1)
if not reachgoogle:
logger.warning('Heartbeat check , test ping not able to reach Google -------------- No action')
print 'Heartbeat check , no IP connection-------------- No action'
#connected=networkmod.connect_network() # use this in case you require the system to try connect wifi again in case no internet is reached
connected=False
else:
logger.info('Heartbeat check , wifi connection OK')
print 'Heartbeat check , wifi connection OK'
connected=True
if connected:
# Check if remote IP address is changed compared to previous communication and in such case resend the mail
ipext=networkmod.get_external_ip()
logger.info('Heartbeat check , Check IP address change -%s- and previously sent -%s-', ipext , emailmod.IPEXTERNALSENT)
if (ipext!=""):
if (emailmod.IPEXTERNALSENT!=""):
if ipext!=emailmod.IPEXTERNALSENT:
print "Heartbeat check, IP address change detected. Send email with updated IP address"
logger.info('Heartbeat check, IP address change detected. Send email with updated IP address')
emailmod.sendallmail("alert","System detected IP address change, below the updated links")
else:
logger.info('Heartbeat check, IP address unchanged')
else:
# first mail has not been sent succesfully of IPEXTERNALSENT was not available by the time
print "System has been reconnected"
logger.info("System has been reconnected, IPEXTERNALSENT was empty")
emailmod.sendallmail("alert","System has been reconnected")
# Check current time is less than 60 second different from NTP information
# try to get the clock from network
print "check system clock"
logger.info('Heartbeat check, check clock')
networktime=clockmod.getNTPTime()
logger.info('Heartbeat check , Network time NTP: %s ', networktime)
systemtime=clockmod.readsystemdatetime()
logger.info('Heartbeat check , System time NTP: %s ', systemtime)
if not networktime=='':
diffsec=clockmod.timediffinsec(networktime, systemtime)
logger.info('Heartbeat check , difference between system time and network time, diffsec = %d ', diffsec)
if diffsec>60:
print "Heartbeat check , warning difference between system time and network time >60 sec, diffsec = " , diffsec
logger.warning('Heartbeat check , warning difference between system time and network time >60 sec, diffsec = %d ', diffsec)
print "Heartbeat check , Apply network datetime to system"
logger.warning('Heartbeat check , Apply network datetime to system ')
clockmod.setHWclock(networktime)
clockmod.setsystemclock(networktime)
else:
print "Heartbeat check , Clock OK"
logger.info('Heartbeat check , Clock OK')
else:
print "not able to get network time"
logger.warning('Heartbeat check , not able to get network time')
else:
print "not able to establish an internet connection"
logger.warning("not able to establish an internet connection")
# check master job has a next run"
isok, datenextrun = SchedulerMod.get_next_run_time("master")
if isok:
datenow=datetime.utcnow()
datenextrun = datenextrun.replace(tzinfo=None)
print "Master Scheduler Next run " , datenextrun , " Now (UTC) ", datenow
if datenextrun>datenow:
print "Masterschedule next RUN confirmed"
logger.info('Heartbeat check , Master Scheduler OK')
else:
isok=False
if not isok:
print "No next run for master scheduler"
logger.warning('Heartbeat check , Master Scheduler Interrupted')
emailmod.sendallmail("alert","Master Scheduler has been interrupted, try to restart scheduler")
resetmastercallback()
# check if there have been errors in Syslog
if DEBUGMODE:
Errortextlist=debuggingmod.searchsyslogkeyword("error")
if Errortextlist:
print "found error in syslog"
logger.warning("ERROR: found error in syslog -------------------------")
#send notification mail
if debuggingmod.SENTERRORTEXT!=Errortextlist[0]:
emailmod.sendallmail("alert","Error found in Syslog",Errortextlist)
debuggingmod.SENTERRORTEXT=Errortextlist[0]
else:
print "No error found in syslog"
logger.info('Heartbeat check , SYSLOG ok')
# check if there have been errors in Schedulerlog
if DEBUGMODE:
filename="logfiles/apscheduler_hydrosystem.log"
MYPATH=hardwaremod.get_path()
filenameandpath=os.path.join(MYPATH, filename)
Errortextlist=debuggingmod.searchLOGkeyword(filenameandpath,"error")
if Errortextlist:
print "found error in LOG ",filename
logger.warning("ERROR: found error in LOG , %s -------------------------",filename)
#send notification mail
if debuggingmod.SENTERRORTEXT!=Errortextlist[0]:
emailmod.sendallmail("alert","Error found in LOG",Errortextlist)
debuggingmod.SENTERRORTEXT=Errortextlist[0]
else:
print "No error found in LOG", filename
logger.info('Heartbeat check , LOG ok')
return True
def heartbeat():
print "start heartbeat check", " ", datetime.now()
logger.info('Start heartbeat routine %s',
datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
connectedssid = networkmod.connectedssid()
connected = False
if len(connectedssid) == 0:
logger.warning(
'Heartbeat check , no network connected -------------- try to connect'
)
print 'Heartbeat check , no network connected -------------- try to connect'
connected = networkmod.connect_network()
else:
logger.info('Heartbeat check , Connected Wifi Network: %s ',
connectedssid[0])
if connectedssid[0] == networkmod.localwifisystem:
logger.info(
'Heartbeat check , Configured as wifi access point, check if possible to connect to wifi network'
)
connected = networkmod.connect_network()
else:
reachgoogle = networkmod.check_internet_connection(3)
if not reachgoogle:
logger.warning(
'Heartbeat check , test ping not able to reach Google -------------- try to connect'
)
print 'Heartbeat check , no IP connection-------------- try to connect'
connected = networkmod.connect_network()
else:
logger.info('Heartbeat check , wifi connection OK')
print 'Heartbeat check , wifi connection OK'
connected = True
if connected:
# Check if remote IP address is changed compared to previous communication and in such case resend the mail
ipext = networkmod.get_external_ip()
logger.info('Heartbeat check , Check IP address change %s', ipext)
if ipext != "":
if ipext != emailmod.IPEXTERNALSENT:
print "Heartbeat check, IP address change detected. Send email with updated IP address"
logger.info(
'Heartbeat check, IP address change detected. Send email with updated IP address'
)
emailmod.sendallmail(
"alert",
"System detected IP address change, below the updated links"
)
# Check current time is less than 60 second different from NTP information
# try to get the clock from network
print "check system clock"
logger.info('Heartbeat check, check clock')
networktime = clockmod.getNTPTime()
logger.info('Heartbeat check , Network time NTP: %s ', networktime)
systemtime = clockmod.readsystemdatetime()
logger.info('Heartbeat check , System time NTP: %s ', systemtime)
if not networktime == '':
diffsec = clockmod.timediffinsec(networktime, systemtime)
logger.info(
'Heartbeat check , difference between system time and network time, diffsec = %d ',
diffsec)
if diffsec > 60:
print "Heartbeat check , warning difference between system time and network time >60 sec, diffsec = ", diffsec
logger.warning(
'Heartbeat check , warning difference between system time and network time >60 sec, diffsec = %d ',
diffsec)
print "Heartbeat check , Apply network datetime to system"
logger.warning(
'Heartbeat check , Apply network datetime to system ')
clockmod.setHWclock(networktime)
clockmod.setsystemclock(networktime)
else:
print "Heartbeat check , Clock OK"
logger.info('Heartbeat check , Clock OK')
else:
print "not able to get network time"
logger.warning('Heartbeat check , not able to get network time')
else:
print "not able to establish an internet connection"
logger.warning("not able to establish an internet connection")
# check master job has a next run"
isok, datenextrun = SchedulerMod.get_next_run_time("master")
if isok:
datenow = datetime.utcnow()
datenextrun = datenextrun.replace(tzinfo=None)
print "Master Scheduler Next run ", datenextrun, " Now (UTC) ", datenow
if datenextrun > datenow:
print "Masterschedule next RUN confirmed"
logger.info('Heartbeat check , Master Scheduler OK')
else:
isok = False
if not isok:
print "No next run for master scheduler"
logger.warning('Heartbeat check , Master Scheduler Interrupted')
emailmod.sendallmail(
"alert",
"Master Scheduler has been interrupted, try to restart scheduler")
setmastercallback()
return True
def heartbeat():
print "start heartbeat check", " ", datetime.now()
logger.info('Start heartbeat routine %s',
datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
connectedssid = networkmod.connectedssid()
connected = False
if len(connectedssid) == 0:
logger.warning(
'Heartbeat check , no network connected -------------- try to connect'
)
print 'Heartbeat check , no network connected -------------- try to connect'
connected = networkmod.connect_network()
else:
logger.info('Heartbeat check , Connected Wifi Network: %s ',
connectedssid[0])
if connectedssid[0] == networkmod.localwifisystem:
logger.info(
'Heartbeat check , Configured as wifi access point, check if possible to connect to wifi network'
)
connected = networkmod.connect_network()
else:
reachgoogle = networkmod.check_internet_connection(1)
if not reachgoogle:
logger.warning(
'Heartbeat check , test ping not able to reach Google -------------- No action'
)
print 'Heartbeat check , no IP connection-------------- No action'
#connected=networkmod.connect_network() # use this in case you require the system to try connect wifi again in case no internet is reached
connected = False
else:
logger.info('Heartbeat check , wifi connection OK')
print 'Heartbeat check , wifi connection OK'
connected = True
if connected:
# Check if remote IP address is changed compared to previous communication and in such case resend the mail
ipext = networkmod.get_external_ip()
logger.info(
'Heartbeat check , Check IP address change -%s- and previously sent -%s-',
ipext, emailmod.IPEXTERNALSENT)
if (ipext != ""):
if (emailmod.IPEXTERNALSENT != ""):
if ipext != emailmod.IPEXTERNALSENT:
print "Heartbeat check, IP address change detected. Send email with updated IP address"
logger.info(
'Heartbeat check, IP address change detected. Send email with updated IP address'
)
emailmod.sendallmail(
"alert",
"System detected IP address change, below the updated links"
)
else:
logger.info('Heartbeat check, IP address unchanged')
else:
# first mail has not been sent succesfully of IPEXTERNALSENT was not available by the time
print "System has been reconnected"
logger.info(
"System has been reconnected, IPEXTERNALSENT was empty")
emailmod.sendallmail("alert", "System has been reconnected")
# Check current time is less than 60 second different from NTP information
# try to get the clock from network
print "check system clock"
logger.info('Heartbeat check, check clock')
networktime = clockmod.getNTPTime()
logger.info('Heartbeat check , Network time NTP: %s ', networktime)
systemtime = clockmod.readsystemdatetime()
logger.info('Heartbeat check , System time NTP: %s ', systemtime)
if not networktime == '':
diffsec = clockmod.timediffinsec(networktime, systemtime)
logger.info(
'Heartbeat check , difference between system time and network time, diffsec = %d ',
diffsec)
if diffsec > 60:
print "Heartbeat check , warning difference between system time and network time >60 sec, diffsec = ", diffsec
logger.warning(
'Heartbeat check , warning difference between system time and network time >60 sec, diffsec = %d ',
diffsec)
print "Heartbeat check , Apply network datetime to system"
logger.warning(
'Heartbeat check , Apply network datetime to system ')
clockmod.setHWclock(networktime)
clockmod.setsystemclock(networktime)
else:
print "Heartbeat check , Clock OK"
logger.info('Heartbeat check , Clock OK')
else:
print "not able to get network time"
logger.warning('Heartbeat check , not able to get network time')
else:
print "not able to establish an internet connection"
logger.warning("not able to establish an internet connection")
# check master job has a next run"
isok, datenextrun = SchedulerMod.get_next_run_time("master")
if isok:
datenow = datetime.utcnow()
datenextrun = datenextrun.replace(tzinfo=None)
print "Master Scheduler Next run ", datenextrun, " Now (UTC) ", datenow
if datenextrun > datenow:
print "Masterschedule next RUN confirmed"
logger.info('Heartbeat check , Master Scheduler OK')
else:
isok = False
if not isok:
print "No next run for master scheduler"
logger.warning('Heartbeat check , Master Scheduler Interrupted')
emailmod.sendallmail(
"alert",
"Master Scheduler has been interrupted, try to restart scheduler")
resetmastercallback()
# check if there have been errors in Syslog
if DEBUGMODE:
Errortextlist = debuggingmod.searchsyslogkeyword("error")
if Errortextlist:
print "found error in syslog"
logger.warning(
"ERROR: found error in syslog -------------------------")
#send notification mail
if debuggingmod.SENTERRORTEXT != Errortextlist[0]:
emailmod.sendallmail("alert", "Error found in Syslog",
Errortextlist)
debuggingmod.SENTERRORTEXT = Errortextlist[0]
else:
print "No error found in syslog"
logger.info('Heartbeat check , SYSLOG ok')
# check if there have been errors in Schedulerlog
if DEBUGMODE:
filename = "logfiles/apscheduler_hydrosystem.log"
MYPATH = hardwaremod.get_path()
filenameandpath = os.path.join(MYPATH, filename)
Errortextlist = debuggingmod.searchLOGkeyword(filenameandpath, "error")
if Errortextlist:
print "found error in LOG ", filename
logger.warning(
"ERROR: found error in LOG , %s -------------------------",
filename)
#send notification mail
if debuggingmod.SENTERRORTEXT != Errortextlist[0]:
emailmod.sendallmail("alert", "Error found in LOG",
Errortextlist)
debuggingmod.SENTERRORTEXT = Errortextlist[0]
else:
print "No error found in LOG", filename
logger.info('Heartbeat check , LOG ok')
return True