def __init__(self,
src_file,
avg_trafic_threshold=10,
csv_start_date=1549573860):
"""
Parameters
----------
src_file : string
Path to the csv file containing the logs.
avg_trafic_threshold : int
Average traffic threshold to trigger alerts (in requests per second).
If the average traffic on a slidding window of 2 minutes is above this
threshold, an alert is triggered.
"""
# Threshold on the average traffic
self.avg_trafic_threshold = avg_trafic_threshold
# Log generator object
self.stream = LogGenerator(src_file, csv_start_date=csv_start_date)
# buffer that contains the logs not already processed
self.buffer = []
# variable that stores the total number of requests
# on a sliding window of 2 minutes.
self.total_traffic_120 = 0
# we store the number of requests of each second in a fixed sized queue.
self.sliding_requests_number = deque([], 120)
# Variable that contains the reports to show on the console
self.sections_stats_report = ""
self.avg_total_traffic_report = ""
self.alert_report = ""
# We define a thread to update the screen of the console app
self.print_thread = Thread(target=self.print_reports)
self.run_printing = True
# We define a thread to retrive the new logs and process them
self.updater_thread = Thread(target=self.updater)
self.run_updater = True
# Variables for alerts
self.alert = False
self.alert_start_time = 0
self.alert_list = []
def test_LogGenerator(self):
"""Tests if the log generator works"""
print("test_LogGenerator")
logPath = "test.log"
rate = 100
log_Generator = LogGenerator(logPath, rate)
log_Generator.start()
time.sleep(1)
log_Generator.stop()
log_Generator.join()
with open(logPath, "r") as f:
lines = f.readlines()
self.assertTrue(len(lines) > 0)
def test_alarm(self):
monitor = HTTPMonitor(verbosity=False, duration=30)
generator = LogGenerator("http.log")
p = Thread(target=monitor.start, daemon=True)
p2 = Process(target=generator.generate_logs,
kwargs={
"duration": 10,
"interval": 0.05
},
daemon=True)
p2.start()
p.start()
time.sleep(15)
assert (monitor.alarm_triggered) #check if the alarm was triggered
p.join()
self.assertFalse(
monitor.alarm_triggered) #check if http traffic is back to normal
p2.terminate()
def test_high_traffic(self):
monitor = HTTPMonitor(verbosity=False, duration=30)
generator = LogGenerator("http.log")
p = Thread(target=monitor.start)
p2 = Process(target=generator.generate_logs,
kwargs={
"duration": 40,
"interval": 0.08
},
daemon=True)
p2.start()
p.start()
while p.is_alive():
if len(monitor.flow_rates) > 0:
assert (monitor.alarm_triggered
) #check if the alarm was triggered
time.sleep(1)
p2.terminate()
total = 0
for log in logs:
total += 1
section = get_section_from_route(log["request"]["route"])
method = log["request"]["method"]
status = log["status"]
stats[section]["count_status"][str(status)] += 1
stats[section]["nb_requests"] += 1
stats[section]["count_methods"][method] += 1
return total, stats
if __name__ == "__main__":
filename = "sample_csv.txt"
lg = LogGenerator(filename, csv_start_date=None)
delta_update = 2
lg.start()
while True:
update_console(lg.empty_buffer())
# faux, faire un delta
time.sleep(delta_update)
#! /usr/bin/env python3
import configparser
from time import sleep
from LogGenerator import LogGenerator
from Monitoring import Monitoring
# reads configuration
configuration = configparser.ConfigParser()
configuration.read("parameters.cfg")
# parsing the parameters:
logPath = str(configuration.get("Simulation", "logPath"))
refresh = float(configuration.get("Simulation", "refresh"))
alertThreshold = float(configuration.get("Simulation", "alertThreshold"))
monitoringRate = float(configuration.get("Simulation", "monitoringRate"))
rate = float(configuration.get("Simulation", "generationRate"))
# defines loggenerator and monitor
log_generator = LogGenerator(logPath, rate)
monitor = Monitoring(logPath, refresh, alertThreshold, monitoringRate)
# starts the simulation and monitoring
log_generator.start()
sleep(1)
monitor.start()
# waits until monitor finishes to end generation of lines
monitor.join()
log_generator.stop()
log_generator.join()
class ConsoleApp:
"""
Class that describes a console application for our project.
"""
def __init__(self,
src_file,
avg_trafic_threshold=10,
csv_start_date=1549573860):
"""
Parameters
----------
src_file : string
Path to the csv file containing the logs.
avg_trafic_threshold : int
Average traffic threshold to trigger alerts (in requests per second).
If the average traffic on a slidding window of 2 minutes is above this
threshold, an alert is triggered.
"""
# Threshold on the average traffic
self.avg_trafic_threshold = avg_trafic_threshold
# Log generator object
self.stream = LogGenerator(src_file, csv_start_date=csv_start_date)
# buffer that contains the logs not already processed
self.buffer = []
# variable that stores the total number of requests
# on a sliding window of 2 minutes.
self.total_traffic_120 = 0
# we store the number of requests of each second in a fixed sized queue.
self.sliding_requests_number = deque([], 120)
# Variable that contains the reports to show on the console
self.sections_stats_report = ""
self.avg_total_traffic_report = ""
self.alert_report = ""
# We define a thread to update the screen of the console app
self.print_thread = Thread(target=self.print_reports)
self.run_printing = True
# We define a thread to retrive the new logs and process them
self.updater_thread = Thread(target=self.updater)
self.run_updater = True
# Variables for alerts
self.alert = False
self.alert_start_time = 0
self.alert_list = []
def start(self):
"""
Function to start all the threads
"""
self.print_thread.start()
self.updater_thread.start()
self.stream.start()
def stop(self):
"""
Function to stop the simulation
"""
self.stream.stop()
self.run_printing = False
self.run_updater = False
self.print_thread.join()
self.updater_thread.join()
self.stream.join()
def updater(self):
"""
Function that triggers update events in the right time
and retrieves logs from buffer.
"""
stats_period = 10 # seconds
request_period = 1 # second
next_total_requests_update = time.time() - request_period
next_stats_update = time.time() - stats_period
nb_logs = 0
while self.run_updater:
# get new logs
new_logs = self.stream.empty_buffer()
nb_logs += len(new_logs)
# store logs in buffer
self.buffer.extend(new_logs)
# trigger nb request update every 1s
if time.time() - next_total_requests_update >= request_period:
# update avg traffic value
self.update_total_request_report(nb_logs)
# reset nb_logs
nb_logs = 0
# mark the update date
next_total_requests_update += request_period
# trigger stats computation every 10s
if time.time() - next_stats_update >= stats_period:
# update the sections' report
self.update_sections_stats_report()
# mark the update date
next_stats_update += stats_period
time.sleep(0.1)
def print_reports(self):
"""
Function that updates the console with the reports every 0.5 seconds
"""
self.update_alert_report()
while self.run_printing:
# prepare report
total_update = f" (last update: {format_time(time.time()+self.stream.get_offset_ms())})"
report = f"HTTP log monitoring console program {total_update}\n\n"
report += self.sections_stats_report
report += "\n"
report += self.avg_total_traffic_report
report += "\n"
report += self.alert_report
# print report
os.system('clear')
print(report)
# wait some time
time.sleep(0.5)
def update_total_request_report(self, nb_logs):
"""
Function that computes the report for the average total traffic.
The function manage also alerts if any.
Parameters
----------
nb_logs : int
number of new logs that appeared since last update
"""
# update the total traffic using a fixed size queue
if len(self.sliding_requests_number) == 120:
self.total_traffic_120 -= self.sliding_requests_number.popleft()
self.sliding_requests_number.append(nb_logs)
self.total_traffic_120 += nb_logs
# compute the average traffic
avg_total_traffic = self.total_traffic_120 / 120
# create report and update traffic report
report = f"Average total taffic: {avg_total_traffic:.2f} req/s over 2min-sliding window"
if True:
report += f" (last update: {format_time(time.time()+self.stream.get_offset_ms())})"
report += "\n"
self.avg_total_traffic_report = report
# Handle alerts
if self.alert:
# resolve alert if the condition is satisfied
if avg_total_traffic < self.avg_trafic_threshold:
self.alert = False
alert_time = time.time() + self.stream.get_offset_ms()
self.alert_list[-1].resolve(alert_time)
self.update_alert_report()
else:
# trigger and alert if the threshold was exceeded
if avg_total_traffic >= self.avg_trafic_threshold:
self.alert = True
alert_time = time.time() + self.stream.get_offset_ms()
self.alert_list.append(Alert(alert_time, avg_total_traffic))
self.update_alert_report()
def update_sections_stats_report(self, verbose=False):
"""
Function that updates the report that deals with statistics
every 10 seconds and about each sections.
Parameters
----------
verbose : bool
Define the level of detail of the stats, default to False (low level of details)
"""
# compute stats
total, stats = compute_stats(self.buffer)
self.buffer = []
# sort sections by number of requests
infos = sorted(
((stats[st]["nb_requests"], st, stats[st]) for st in stats),
reverse=True)
# create a report
report = f"Top {3} websites:"
if True:
report += f" (last update: {format_time(time.time()+self.stream.get_offset_ms())})"
report += "\n"
nb_requests = 0
# for each section, format stats and append it to the report
for i, (count_req, section, section_stats) in enumerate(infos):
nb_requests = section_stats["nb_requests"]
ratio_requests = nb_requests / total if total else 0
nb_post = section_stats["count_methods"]["POST"]
nb_get = section_stats["count_methods"]["GET"]
nb_404 = section_stats["count_status"]["404"]
nb_500 = section_stats["count_status"]["500"]
report += f"{i+1}: Section {section}: \t traffic: {100*ratio_requests:.1f}%"
report += f", requests: {nb_requests}"
if verbose:
report += f" (GET:{nb_get}, POST:{nb_post})"
report += f", errors: {nb_404+nb_500} (404: {nb_404}, 500: {nb_500})"
report += "\n"
# finally update the section report
self.sections_stats_report = report
def update_alert_report(self):
"""
Function that update the alert report
"""
# custom report if no alert ever triggered
if not self.alert_list:
self.alert_report = "No alert triggered"
return
# create the report
report = "List of alerts:\n"
# add a description line for each alert
for alert in self.alert_list:
report += alert.report()
# finally update the alert report
self.alert_report = report