def index():
'''
add the logs to the services and send them to the client so that the monitoring tool can be displayed.
'''
services = WebServiceMonitor.objects()[:]
services_to_send = []
log_string = ""
for service in services:
for log in Logs.objects(web_server_ID=service.web_server_ip + ':' +
str(service.web_server_port)):
log_string += ' '.join([
datetime.strftime(log.log_timestamp, "%Y-%m-%d_%Hh%Mm%Ss"),
log.log_type, log.log_content
])
service_dict = dict(service_id=service.web_server_ip + ':' +
str(service.web_server_port),
status_monitor=service.status_monitor,
status_service=service.status_service,
status_through_lb=service.status_through_lb,
response_time=service.response_time,
last_excpt_raised=service.last_excpt_raised,
logs=log_string)
services_to_send.append(service_dict)
return render_template("index.html", services=services_to_send)
class UdpProtocol(protocol.DatagramProtocol):
def __init__(self, heartbeats):
self.heartbeats = heartbeats
def datagramReceived(self, data, (host, port)):
if data.startswith("logs"):
_, host, port, logs = data.split('#')
logs_list = json.loads(logs)
for log in logs_list:
date, type_log, content = parseLog(log)
log_to_save = Logs(web_server_ID=(host + ":%d" % int(port)),
log_timestamp=date,
log_type=type_log,
log_content=content)
log_to_save.save()
elif data.startswith("last_exception"):
_, host, port, exception = data.split('#')
ws = WebServiceMonitor.objects(web_server_ip=host,
web_server_port=port).first()
ws.last_excpt_raised = exception
ws.save()
#load_balancer sends the list of servers
elif data.startswith("list_servers") and (host, port) == (
config.load_bal_monitor.ip, config.load_bal_monitor.port_udp):
cust_logger.info("received list servers")
_, list_json = data.split('#')
heartbeats = [{
(server['ip'], server['port'], server['monitor_port']):
time.time()
} for server in json.loads(list_json)]
def checksDaemon(heartbeats):
'''
check directly the connexion http with the web service, without going through load balancing.
Check of the load balancer must be added. Update the status of the web services depending on
the success of the connexion.
'''
while (True):
for url_to_check, ws in [("%s:%d" % (ws[0], ws[1]), ws)
for ws in heartbeats]:
ws_db = WebServiceMonitor.objects(web_server_ip=ws[0],
web_server_port=ws[1]).first()
oldStatus = ws_db.status_service
if not ws_db:
continue
try:
connexion = httplib.HTTPConnection(url_to_check, timeout=6)
connexion.request("GET", "/fortune")
time_before = time.time()
res = connexion.getresponse()
response_time = time.time() - time_before
if oldStatus != res.status:
cust_logger.info(
"%s status changed. old: %s new: %s" %
(url_to_check, ws_db.status_service, res.status))
#if status changed and is now 200, we remove old logs
if res.status == 200:
list_logs = Logs.objects(web_server_ID=(ws[0] +
":%d" % ws[1]))
for log in list_logs:
log.delete()
else: #status is the same
cust_logger.info("%s remains the same with status %d" %
(url_to_check, res.status))
ws_db.status_service = res.status
ws_db.response_time = response_time
ws_db.successfull_calls += 1
ws_db.save()
#in case the connection is not possible, status is -1
except httplib.BadStatusLine:
ws_db.status_service = -1
ws_db.save()
cust_logger.warning("connexion ended with BAD STATUS")
except StandardError as e:
ws_db.status_service = -1
ws_db.save()
cust_logger.warning("connexion ended with standard Error: " +
str(e))
#When status is not 200 but the web service monitor is alive, we ask the logs
if oldStatus != ws_db.status_service and ws_db.status_service != 200 and ws_db.status_monitor == StatusWebService.STATUS_BEATING:
cust_logger.info("Requesting logs on %s:%d" % (ws[0], ws[2]))
sendMessageUdpFromTo("request_logs", "localhost",
config.health_monitor.port_udp_process,
ws[0], ws[2])
time.sleep(PERIOD_CHECK_STATUS)
def run(self):
while self.event_thread.isSet():
try:
datagram, sender = self.listen_socket.recvfrom(SIZE_BUFFER_HB)
sender_formatted = sender[0]
if sender[0]=="127.0.0.1":
sender_formatted = "localhost"
if datagram.startswith(HB_DATAGRAM) :
cust_logger.info("received Datagram from web service")
_,port_sender, port_monitor = datagram.split('#')
self.heartbeats[(sender_formatted, int(port_sender), int(port_monitor))] = time.time()
web_service = WebServiceMonitor.objects(web_server_ip=sender_formatted, web_server_port = int(port_sender)).first()
if not web_service :
web_service = WebServiceMonitor(web_server_ip=sender_formatted, web_server_port=int(port_sender), monitor_port=int(port_monitor))
web_service.status_monitor = StatusWebService.STATUS_BEATING
web_service.save()
except socket.timeout:
pass
def checksDaemon(heartbeats):
'''
check directly the connexion http with the web service, without going through load balancing.
Check of the load balancer must be added. Update the status of the web services depending on
the success of the connexion.
'''
while(True):
for url_to_check,ws in [ ("%s:%d"%(ws[0],ws[1]),ws) for ws in heartbeats]:
ws_db = WebServiceMonitor.objects(web_server_ip=ws[0], web_server_port=ws[1]).first()
oldStatus = ws_db.status_service
if not ws_db:
continue
try:
connexion = httplib.HTTPConnection(url_to_check,timeout=6)
connexion.request("GET", "/fortune")
time_before = time.time()
res = connexion.getresponse()
response_time = time.time()-time_before
if oldStatus != res.status:
cust_logger.info("%s status changed. old: %s new: %s" % (url_to_check, ws_db.status_service, res.status))
#if status changed and is now 200, we remove old logs
if res.status == 200:
list_logs = Logs.objects(web_server_ID=(ws[0]+":%d"%ws[1]))
for log in list_logs:
log.delete()
else: #status is the same
cust_logger.info("%s remains the same with status %d"%(url_to_check,res.status))
ws_db.status_service = res.status
ws_db.response_time = response_time
ws_db.successfull_calls +=1
ws_db.save()
#in case the connection is not possible, status is -1
except httplib.BadStatusLine:
ws_db.status_service = -1
ws_db.save()
cust_logger.warning("connexion ended with BAD STATUS")
except StandardError as e:
ws_db.status_service = -1
ws_db.save()
cust_logger.warning("connexion ended with standard Error: "+str(e))
#When status is not 200 but the web service monitor is alive, we ask the logs
if oldStatus != ws_db.status_service and ws_db.status_service != 200 and ws_db.status_monitor == StatusWebService.STATUS_BEATING:
cust_logger.info("Requesting logs on %s:%d"%(ws[0],ws[2]))
sendMessageUdpFromTo("request_logs","localhost", config.health_monitor.port_udp_process, ws[0], ws[2])
time.sleep(PERIOD_CHECK_STATUS)
def heartBeatDaemon(heartbeats):
'''
Thread handler which update the status of the nodes if they don't send any heartbeat
'''
event_thread = threading.Event()
event_thread.set()
listener = ListenerHeartBeat(event_thread, heartbeats)
listener.start()
try:
while True:
list_timed_out = heartbeats.listTimedOut
for dic in list_timed_out:
web_service = WebServiceMonitor.objects(web_server_ip=dic['ip'], web_server_port = dic['port']).first()
if web_service.status_monitor != StatusWebService.STATUS_LOST:
cust_logger.info("%s:%d has timedout"%(dic['ip'],dic['port']))
web_service.status_monitor=StatusWebService.STATUS_LOST
web_service.save()
time.sleep(PERIOD_CHECK_HB)
except ValueError as e:
cust_logger.error("HeartBeatDeamon stopped: "+ str(e))
pass
def heartBeatDaemon(heartbeats):
'''
Thread handler which update the status of the nodes if they don't send any heartbeat
'''
event_thread = threading.Event()
event_thread.set()
listener = ListenerHeartBeat(event_thread, heartbeats)
listener.start()
try:
while True:
list_timed_out = heartbeats.listTimedOut
for dic in list_timed_out:
web_service = WebServiceMonitor.objects(
web_server_ip=dic['ip'],
web_server_port=dic['port']).first()
if web_service.status_monitor != StatusWebService.STATUS_LOST:
cust_logger.info("%s:%d has timedout" %
(dic['ip'], dic['port']))
web_service.status_monitor = StatusWebService.STATUS_LOST
web_service.save()
time.sleep(PERIOD_CHECK_HB)
except ValueError as e:
cust_logger.error("HeartBeatDeamon stopped: " + str(e))
pass
def run(self):
while self.event_thread.isSet():
try:
datagram, sender = self.listen_socket.recvfrom(SIZE_BUFFER_HB)
sender_formatted = sender[0]
if sender[0] == "127.0.0.1":
sender_formatted = "localhost"
if datagram.startswith(HB_DATAGRAM):
cust_logger.info("received Datagram from web service")
_, port_sender, port_monitor = datagram.split('#')
self.heartbeats[(sender_formatted, int(port_sender),
int(port_monitor))] = time.time()
web_service = WebServiceMonitor.objects(
web_server_ip=sender_formatted,
web_server_port=int(port_sender)).first()
if not web_service:
web_service = WebServiceMonitor(
web_server_ip=sender_formatted,
web_server_port=int(port_sender),
monitor_port=int(port_monitor))
web_service.status_monitor = StatusWebService.STATUS_BEATING
web_service.save()
except socket.timeout:
pass