def calculate_urls(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password,
self.bothound_tools.es_host, self.bothound_tools.es_port)
incident_urls = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
urls = es_handler.get_banned_urls(incident['start'], incident['stop'], incident['target'])
urls_list = []
for key, value in urls.iteritems():
temp = [key,value]
urls_list.append(temp)
urls_sorted = sorted(urls_list, key=lambda k: k[1], reverse=True)
num_most = len(urls_sorted) if len(urls_sorted) < 3 else 3
incident_urls.append(urls_sorted[0:num_most])
f1=open('urls.txt', 'w+')
for urls in incident_urls:
print "incident", i
for url in urls:
print url[1], url[0]
print >> f1, url[1], url[0]
f1.close()
def calculate_responses(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
res = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
rates = es_handler.get_banned_responses(incident['start'],
incident['stop'],
incident['target'])
res_list = []
for key, value in rates.iteritems():
temp = [key, value]
res_list.append(temp)
res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True)
num_most = len(res_sorted) if len(res_sorted) < 3 else 3
res.append(res_sorted[0:num_most])
i = 1
for incident in res:
print "incident", i
i = i + 1
for r in incident:
print r[1], r[0]
def calculate_user_agents(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
res = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
res_dict = es_handler.get_banned_user_agents(
incident['start'], incident['stop'], incident['target'])
res_list = []
for key, value in res_dict.iteritems():
temp = [key, value]
res_list.append(temp)
res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True)
num_most = len(res_sorted) if len(res_sorted) < 50 else 50
print "incident", i, "winner", res_sorted[0]
res.append(res_sorted[0:num_most])
i = 1
f1 = open('user_agents.txt', 'w+')
for incident in res:
print >> f1, "incident", i
print "incident", i
i = i + 1
for r in incident:
print >> f1, r[0], r[1]
print r[0], r[1]
f1.close()
def calculate_urls(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
incident_urls = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
urls = es_handler.get_banned_urls(incident['start'],
incident['stop'],
incident['target'])
urls_list = []
for key, value in urls.iteritems():
temp = [key, value]
urls_list.append(temp)
urls_sorted = sorted(urls_list, key=lambda k: k[1], reverse=True)
num_most = len(urls_sorted) if len(urls_sorted) < 3 else 3
incident_urls.append(urls_sorted[0:num_most])
f1 = open('urls.txt', 'w+')
for urls in incident_urls:
print "incident", i
for url in urls:
print url[1], url[0]
print >> f1, url[1], url[0]
f1.close()
def calculate_user_agents(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password,
self.bothound_tools.es_host, self.bothound_tools.es_port)
res = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
res_dict = es_handler.get_banned_user_agents(incident['start'], incident['stop'], incident['target'])
res_list = []
for key, value in res_dict.iteritems():
temp = [key,value]
res_list.append(temp)
res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True)
num_most = len(res_sorted) if len(res_sorted) < 50 else 50
print "incident", i, "winner", res_sorted[0]
res.append(res_sorted[0:num_most])
i = 1
f1=open('user_agents.txt', 'w+')
for incident in res:
print >>f1, "incident", i
print "incident", i
i = i + 1
for r in incident:
print >> f1, r[0], r[1]
print r[0], r[1]
f1.close()
def calculate_pingback_domains(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
index = 1
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
res_dict = es_handler.get_pingback_domains(incident['start'],
incident['stop'],
incident['target'])
res_list = []
for key, value in res_dict.iteritems():
temp = [key, value]
res_list.append(temp)
res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True)
f1 = open('pingback_domains_incident_{}({}).txt'.format(index, i),
'w+')
for r in res_sorted:
print >> f1, r[0]
f1.close()
index = index + 1
def calculate_incident_intersection_plus_ua(self, incidents1, incidents2):
# Calculating common Banned Ips for two sets of incidents
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
print "processing..."
ips1 = []
for i in incidents1:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target'])
for key, value in banned_ips.iteritems():
ips1.append([key, value])
ips2 = []
for i in incidents2:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target'])
for key, value in banned_ips.iteritems():
ips2.append([key, value])
intersection = 0
processed_ips = {}
for ip1 in ips1:
for ip2 in ips2:
if ip1[0] == ip2[0]:
if ip1[0] in processed_ips:
break
found = False
for ua1 in ip1[1]['ua'].keys():
for ua2 in ip2[1]['ua'].keys():
if (ua1 == ua2):
found = True
break
if found:
break
if found:
intersection = intersection + 1
processed_ips[ip1[0]] = 1
num = intersection
d = [
len(ips1),
len(ips2), num, num * 100.0 / min(len(ips1), len(ips2))
]
print "group1", incidents1
print "group2", incidents2
s = "{},{},{},{:.1f}%".format(d[0], d[1], d[2], d[3])
print s
def calculate_unique_ips(self, incidents): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) ips = [] for i in incidents: incident = self.bothound_tools.get_incident(i)[0] cur_ips = es_handler.get_deflect_unique_ips(incident['start'], incident['stop'], incident['target']) ips.append(set(cur_ips.keys())) for i in range(0, len(ips)): print i, "Num unique IPs:", len(ips[i])
def __init__(self, bothound_tools):
"""
store the exp config in self's attribute.
"""
Thread.__init__(self)
self.daemon = True
utc_datetime = datetime.utcnow()
self.bothound_tools = bothound_tools
self.es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
bothound_tools.es_host,
self.bothound_tools.es_port)
def find_intersections(self, id_incidents,
date_from,
date_to,
file_name = "intersection_report.txt",
title = "",
target_domain_no_www = None, #domain without "www."
window_size_in_hours = 24,
threshold_in_percentage = 10.0) :
es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password,
self.bothound_tools.es_host, self.bothound_tools.es_port)
ips = []
start = date_from
f1=open(file_name, 'w+')
print >>f1, "Intersection Report:", title
print >>f1, "\nIncidents:"
for id_incident in id_incidents:
incident = self.bothound_tools.get_incident(id_incident)[0]
ips = ips + es_handler.get_banjax(incident['start'], incident['stop'], incident['target']).keys()
print >>f1, "Incident {}, target domain = {}, Start:{}, Stop:{}".format(
id_incident, incident['target'], incident['start'], incident['stop'])
ips = set(ips)
print >>f1, "\nNotal number of banned IPs:", len(ips)
print >>f1, "\nFinding intersection with data from {} to {}".format(date_from, date_to)
print >>f1, "Threshold = {}%".format(threshold_in_percentage)
print >>f1, "Target domain = {}".format(target_domain_no_www if target_domain_no_www != None else "all domains")
print >>f1, "Sliding window size : {} hours".format(window_size_in_hours)
print >>f1, "\nIntersections found:"
while start < date_to:
stop = start + timedelta(hours=window_size_in_hours)
print "processing {}, + {} hours...".format(start, window_size_in_hours)
ips_to_check = es_handler.get_banjax(start, stop, target_domain_no_www).keys()
ips_to_check = set(ips_to_check)
intersection = len(ips.intersection(ips_to_check))
percentage1 = intersection * 100.0 / len(ips)
percentage2 = intersection * 100.0 / len(ips_to_check)
if percentage1 > threshold_in_percentage or percentage2 > threshold_in_percentage:
print >> f1, "Original:{:.2f}%, Window:{:.2f}%, Total: {}, start : {} + {} hours".format(
percentage1, percentage2, intersection, start, window_size_in_hours)
start = start + timedelta(hours=window_size_in_hours/2.0)
print >>f1, "End"
f1.close()
def get_banned_ips(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password,
self.bothound_tools.es_host, self.bothound_tools.es_port)
index = 1
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
ips = es_handler.get_banjax(incident['start'], incident['stop'], incident['target'])
f1=open('incident_{}({}).txt'.format(index, i), 'w+')
for ip in ips:
#pdb.set_trace()
print >> f1, ip
f1.close()
index = index + 1
def calculate_unique_ips(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
ips = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
cur_ips = es_handler.get_deflect_unique_ips(
incident['start'], incident['stop'], incident['target'])
ips.append(set(cur_ips.keys()))
for i in range(0, len(ips)):
print i, "Num unique IPs:", len(ips[i])
def __init__(self, bothound_tools): """ store the exp config in self's attribute. """ utc_datetime = datetime.utcnow() self.bothound_tools = bothound_tools self.es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, bothound_tools.es_host, self.bothound_tools.es_port)
def calculate_incident_intersection_plus_ua(self, incidents1, incidents2):
# Calculating common Banned Ips for two sets of incidents
es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password,
self.bothound_tools.es_host, self.bothound_tools.es_port)
print "processing..."
ips1 = []
for i in incidents1:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'], incident['stop'], incident['target'])
for key, value in banned_ips.iteritems():
ips1.append([key, value])
ips2 = []
for i in incidents2:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'], incident['stop'], incident['target'])
for key, value in banned_ips.iteritems():
ips2.append([key, value])
intersection = 0
processed_ips = {}
for ip1 in ips1:
for ip2 in ips2:
if ip1[0] == ip2[0]:
if ip1[0] in processed_ips:
break
found = False
for ua1 in ip1[1]['ua'].keys():
for ua2 in ip2[1]['ua'].keys():
if(ua1 == ua2):
found = True
break
if found:
break
if found:
intersection = intersection + 1
processed_ips[ip1[0]] = 1
num = intersection
d = [len(ips1), len(ips2), num, num * 100.0 / min(len(ips1), len(ips2))]
print "group1", incidents1
print "group2", incidents2
s = "{},{},{},{:.1f}%".format(d[0], d[1], d[2], d[3])
print s
def get_banned_ips(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
index = 1
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
ips = es_handler.get_banjax(incident['start'], incident['stop'],
incident['target'])
f1 = open('incident_{}({}).txt'.format(index, i), 'w+')
for ip in ips:
#pdb.set_trace()
print >> f1, ip
f1.close()
index = index + 1
def calculate_pingback_domains(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password,
self.bothound_tools.es_host, self.bothound_tools.es_port)
index = 1
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
res_dict = es_handler.get_pingback_domains(incident['start'], incident['stop'], incident['target'])
res_list = []
for key, value in res_dict.iteritems():
temp = [key,value]
res_list.append(temp)
res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True)
f1=open('pingback_domains_incident_{}({}).txt'.format(index,i), 'w+')
for r in res_sorted:
print >> f1, r[0]
f1.close()
index = index + 1
def calculate_responses(self, incidents): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) res = [] for i in incidents: incident = self.bothound_tools.get_incident(i)[0] rates = es_handler.get_banned_responses(incident['start'], incident['stop'], incident['target']) res_list = [] for key, value in rates.iteritems(): temp = [key,value] res_list.append(temp) res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True) num_most = len(res_sorted) if len(res_sorted) < 3 else 3 res.append(res_sorted[0:num_most]) i = 1 for incident in res: print "incident", i i = i + 1 for r in incident: print r[1], r[0]
def calculate_cross_table_banjax(self, incidents):
# Calculating common Banned Ips for a set of incidents
es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password,
self.bothound_tools.es_host, self.bothound_tools.es_port)
common = -1
print "processing..."
result = []
groups = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'], incident['stop'], incident['target'])
ips = []
for p in banned_ips.keys():
ips.append(p)
if(common<0):
common = set(ips)
else:
common = common.intersection(ips)
result.append([len(ips), len(common)])
groups.append(ips)
for i in range(0, len(groups)):
print "Incident", i, len(groups[i])
ip_counts = {}
for g in groups:
for ip in g:
if(ip in ip_counts):
ip_counts[ip] = ip_counts[ip] + 1
else:
ip_counts[ip] = 1
for i in range(1, len(incidents)+1):
cur_count = 0
for ip in ip_counts:
if(ip_counts[ip] == i):
cur_count = cur_count + 1
print cur_count, i
"""
#calculate moving intersection
print "moving intersection"
for i in range(0, len(incidents)-1):
ips1 = set(groups[i])
ips2 = set(groups[i+1])
print i+1, i+2, len(ips1.intersection(ips2))
"""
print "cross table"
cross_table = []
for i in range(0, len(incidents)):
for j in range(i+1, len(incidents)):
ips1 = set(groups[i])
ips2 = set(groups[j])
num = len(ips1.intersection(ips2))
cross_table.append((i+1, j+1, len(ips1), len(ips2), num, num * 100.0 / min(len(ips1), len(ips2))))
sorted_cross_table = sorted(cross_table, key=lambda k: k[5], reverse=True)
f1=open('cross_table.txt', 'w+')
for d in sorted_cross_table:
s = "{},{},{},{},{},{:.1f}%".format(d[0], d[1], d[2], d[3], d[4], d[5])
print s
print >> f1, s
f1.close()
def __init__(self, bothound_tools):
self.bothound_tools = bothound_tools
self.es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
bothound_tools.es_host,
self.bothound_tools.es_port)
def calculate_cross_table_banjax(self, incidents):
# Calculating common Banned Ips for a set of incidents
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
common = -1
print "processing..."
result = []
groups = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target'])
ips = []
for p in banned_ips.keys():
ips.append(p)
if (common < 0):
common = set(ips)
else:
common = common.intersection(ips)
result.append([len(ips), len(common)])
groups.append(ips)
for i in range(0, len(groups)):
print "Incident", i, len(groups[i])
ip_counts = {}
for g in groups:
for ip in g:
if (ip in ip_counts):
ip_counts[ip] = ip_counts[ip] + 1
else:
ip_counts[ip] = 1
for i in range(1, len(incidents) + 1):
cur_count = 0
for ip in ip_counts:
if (ip_counts[ip] == i):
cur_count = cur_count + 1
print cur_count, i
"""
#calculate moving intersection
print "moving intersection"
for i in range(0, len(incidents)-1):
ips1 = set(groups[i])
ips2 = set(groups[i+1])
print i+1, i+2, len(ips1.intersection(ips2))
"""
print "cross table"
cross_table = []
for i in range(0, len(incidents)):
for j in range(i + 1, len(incidents)):
ips1 = set(groups[i])
ips2 = set(groups[j])
num = len(ips1.intersection(ips2))
cross_table.append((i + 1, j + 1, len(ips1), len(ips2), num,
num * 100.0 / min(len(ips1), len(ips2))))
sorted_cross_table = sorted(cross_table,
key=lambda k: k[5],
reverse=True)
f1 = open('cross_table.txt', 'w+')
for d in sorted_cross_table:
s = "{},{},{},{},{},{:.1f}%".format(d[0], d[1], d[2], d[3], d[4],
d[5])
print s
print >> f1, s
f1.close()
class SessionExtractor():
"""
This class read the db for the time of the attack from the database
and compute the sessions for the chosen incidents
"""
def __init__(self, bothound_tools):
"""
store the exp config in self's attribute.
"""
utc_datetime = datetime.utcnow()
self.bothound_tools = bothound_tools
self.es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password,
bothound_tools.es_host, self.bothound_tools.es_port)
def process_incident(self, incident):
"""
get the incident time from the db and gathers all features
INPUT:
log_files: the logs that we went through it.
"""
if(incident is None):
return
# get the logs from ES
banned_ips = self.es_handler.get_banjax(incident['start'], incident['stop'], incident['target'])
# get the logs from ES
ats_records = self.es_handler.get(incident['start'], incident['stop'], incident['target'])
# calculate IP dictionary with ATS records
ip_sieve = IPSieve()
ip_records = ip_sieve.process_ats_records(ats_records)
# calculate features
ip_feature_db = {}
#At this stage it is only a peliminary list we might lose features
#due to 0 variance
self._active_feature_list = []
#do a dry run on all features just to gather the indeces of all available
#features
for CurentFeature in Learn2BanFeature.__subclasses__():
f = CurentFeature(ip_records, ip_feature_db)
self._active_feature_list.append(f._FEATURE_INDEX)
for CurentFeature in Learn2BanFeature.__subclasses__():
f = CurentFeature(ip_records, ip_feature_db)
#logging.info("Computing feature %i..."% f._FEATURE_INDEX)
print "Computing feature %i..."% f._FEATURE_INDEX
f.compute()
# post process the features
ip_feature_db = self.bothound_tools.post_process(ip_feature_db)
# delete the old sessions for thie incidend
self.bothound_tools.delete_sessions(incident['id'])
#print ip_feature_db
self.bothound_tools.add_sessions(incident['id'], ip_feature_db, banned_ips)
self.bothound_tools.set_incident_process(incident['id'], False)
print "Incident {} processed.".format(incident['id'])
return ip_feature_db
def extract(self):
"""
check all incidents which needs to be processed and compute the features on them
finally store the sessions in the db
"""
#this make more sense to happens in the constructor however,
for incident in bothound_tools.get_incidents(process = True):
cur_session_feature_db = self.process_incident(incident)
def store_results(self, session_feature_db):
# Add the result to the database
for cur_sesion in session_feature_db:
db_tools.store(cur_session)
def find_intersections(
self,
id_incidents,
date_from,
date_to,
file_name="intersection_report.txt",
title="",
target_domain_no_www=None, #domain without "www."
window_size_in_hours=24,
threshold_in_percentage=10.0):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
ips = []
start = date_from
f1 = open(file_name, 'w+')
print >> f1, "Intersection Report:", title
print >> f1, "\nIncidents:"
for id_incident in id_incidents:
incident = self.bothound_tools.get_incident(id_incident)[0]
ips = ips + es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target']).keys()
print >> f1, "Incident {}, target domain = {}, Start:{}, Stop:{}".format(
id_incident, incident['target'], incident['start'],
incident['stop'])
ips = set(ips)
print >> f1, "\nNotal number of banned IPs:", len(ips)
print >> f1, "\nFinding intersection with data from {} to {}".format(
date_from, date_to)
print >> f1, "Threshold = {}%".format(threshold_in_percentage)
print >> f1, "Target domain = {}".format(
target_domain_no_www
if target_domain_no_www != None else "all domains")
print >> f1, "Sliding window size : {} hours".format(
window_size_in_hours)
print >> f1, "\nIntersections found:"
while start < date_to:
stop = start + timedelta(hours=window_size_in_hours)
print "processing {}, + {} hours...".format(
start, window_size_in_hours)
ips_to_check = es_handler.get_banjax(start, stop,
target_domain_no_www).keys()
ips_to_check = set(ips_to_check)
intersection = len(ips.intersection(ips_to_check))
percentage1 = intersection * 100.0 / len(ips)
percentage2 = intersection * 100.0 / len(ips_to_check)
if percentage1 > threshold_in_percentage or percentage2 > threshold_in_percentage:
print >> f1, "Original:{:.2f}%, Window:{:.2f}%, Total: {}, start : {} + {} hours".format(
percentage1, percentage2, intersection, start,
window_size_in_hours)
start = start + timedelta(hours=window_size_in_hours / 2.0)
print >> f1, "End"
f1.close()
class SessionComputer(Thread):
"""
This class read the db for the time of the attack from the database
and compute the sessions for the chosen incidents
"""
def __init__(self, bothound_tools):
"""
store the exp config in self's attribute.
"""
Thread.__init__(self)
self.daemon = True
utc_datetime = datetime.utcnow()
self.bothound_tools = bothound_tools
self.es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
bothound_tools.es_host,
self.bothound_tools.es_port)
def process_incident(self, incident):
"""
get the incident time from the db and gathers all features
INPUT:
log_files: the logs that we went through it.
"""
if (incident is None):
return
ip_sieve = IPSieve()
ip_records = {}
banned_ips = []
if (incident["file_name"] is None) or (len(incident["file_name"])
== 0):
# get the logs from ES
# get the logs from ES
banned_ips = self.es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target'])
ats_records = self.es_handler.get(incident['start'],
incident['stop'],
incident['target'])
# calculate IP dictionary with ATS records
ip_records = ip_sieve.process_ats_records(ats_records)
else:
# read the sessions from the log file
ip_sieve.add_log_file(incident["file_name"])
ip_records = ip_sieve.parse_log("nginx")
# calculate features
ip_feature_db = {}
#At this stage it is only a peliminary list we might lose features
#due to 0 variance
self._active_feature_list = []
#do a dry run on all features just to gather the indeces of all available
#features
for CurentFeature in Learn2BanFeature.__subclasses__():
f = CurentFeature(ip_records, ip_feature_db)
self._active_feature_list.append(f._FEATURE_INDEX)
for CurentFeature in Learn2BanFeature.__subclasses__():
f = CurentFeature(ip_records, ip_feature_db)
#logging.info("Computing feature %i..."% f._FEATURE_INDEX)
print "Computing feature %i..." % f._FEATURE_INDEX
f.compute()
# post process the features
ip_feature_db = self.bothound_tools.post_process(ip_feature_db)
# delete the old sessions for thie incidend
self.bothound_tools.delete_sessions(incident['id'])
#print ip_feature_db
self.bothound_tools.add_sessions(incident['id'], ip_feature_db,
banned_ips)
self.bothound_tools.set_incident_process(incident['id'], False)
print "Incident {} processed.".format(incident['id'])
return ip_feature_db
def compute_incidents(self):
"""
check all incidents which needs to be processed and compute the features on them
finally store the sessions in the db
"""
#this make more sense to happens in the constructor however,
for incident in self.bothound_tools.get_incidents(process=True):
cur_session_feature_db = self.process_incident(incident)
def store_results(self, session_feature_db):
# Add the result to the database
for cur_sesion in session_feature_db:
db_tools.store(cur_session)
def run(self):
print "Running SessionComputer..."
while True:
self.compute_incidents()
time.sleep(10)
print "Exit session computer."