def calculate_pingback_domains(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
index = 1
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
res_dict = es_handler.get_pingback_domains(incident['start'],
incident['stop'],
incident['target'])
res_list = []
for key, value in res_dict.iteritems():
temp = [key, value]
res_list.append(temp)
res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True)
f1 = open('pingback_domains_incident_{}({}).txt'.format(index, i),
'w+')
for r in res_sorted:
print >> f1, r[0]
f1.close()
index = index + 1
def calculate_user_agents(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
res = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
res_dict = es_handler.get_banned_user_agents(
incident['start'], incident['stop'], incident['target'])
res_list = []
for key, value in res_dict.iteritems():
temp = [key, value]
res_list.append(temp)
res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True)
num_most = len(res_sorted) if len(res_sorted) < 50 else 50
print "incident", i, "winner", res_sorted[0]
res.append(res_sorted[0:num_most])
i = 1
f1 = open('user_agents.txt', 'w+')
for incident in res:
print >> f1, "incident", i
print "incident", i
i = i + 1
for r in incident:
print >> f1, r[0], r[1]
print r[0], r[1]
f1.close()
def calculate_responses(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
res = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
rates = es_handler.get_banned_responses(incident['start'],
incident['stop'],
incident['target'])
res_list = []
for key, value in rates.iteritems():
temp = [key, value]
res_list.append(temp)
res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True)
num_most = len(res_sorted) if len(res_sorted) < 3 else 3
res.append(res_sorted[0:num_most])
i = 1
for incident in res:
print "incident", i
i = i + 1
for r in incident:
print r[1], r[0]
def calculate_urls(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
incident_urls = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
urls = es_handler.get_banned_urls(incident['start'],
incident['stop'],
incident['target'])
urls_list = []
for key, value in urls.iteritems():
temp = [key, value]
urls_list.append(temp)
urls_sorted = sorted(urls_list, key=lambda k: k[1], reverse=True)
num_most = len(urls_sorted) if len(urls_sorted) < 3 else 3
incident_urls.append(urls_sorted[0:num_most])
f1 = open('urls.txt', 'w+')
for urls in incident_urls:
print "incident", i
for url in urls:
print url[1], url[0]
print >> f1, url[1], url[0]
f1.close()
def calculate_incident_intersection_plus_ua(self, incidents1, incidents2):
# Calculating common Banned Ips for two sets of incidents
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
print "processing..."
ips1 = []
for i in incidents1:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target'])
for key, value in banned_ips.iteritems():
ips1.append([key, value])
ips2 = []
for i in incidents2:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target'])
for key, value in banned_ips.iteritems():
ips2.append([key, value])
intersection = 0
processed_ips = {}
for ip1 in ips1:
for ip2 in ips2:
if ip1[0] == ip2[0]:
if ip1[0] in processed_ips:
break
found = False
for ua1 in ip1[1]['ua'].keys():
for ua2 in ip2[1]['ua'].keys():
if (ua1 == ua2):
found = True
break
if found:
break
if found:
intersection = intersection + 1
processed_ips[ip1[0]] = 1
num = intersection
d = [
len(ips1),
len(ips2), num, num * 100.0 / min(len(ips1), len(ips2))
]
print "group1", incidents1
print "group2", incidents2
s = "{},{},{},{:.1f}%".format(d[0], d[1], d[2], d[3])
print s
def __init__(self, bothound_tools):
"""
store the exp config in self's attribute.
"""
Thread.__init__(self)
self.daemon = True
utc_datetime = datetime.utcnow()
self.bothound_tools = bothound_tools
self.es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
bothound_tools.es_host,
self.bothound_tools.es_port)
def calculate_unique_ips(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
ips = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
cur_ips = es_handler.get_deflect_unique_ips(
incident['start'], incident['stop'], incident['target'])
ips.append(set(cur_ips.keys()))
for i in range(0, len(ips)):
print i, "Num unique IPs:", len(ips[i])
def get_banned_ips(self, incidents):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
index = 1
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
ips = es_handler.get_banjax(incident['start'], incident['stop'],
incident['target'])
f1 = open('incident_{}({}).txt'.format(index, i), 'w+')
for ip in ips:
#pdb.set_trace()
print >> f1, ip
f1.close()
index = index + 1
def find_intersections(
self,
id_incidents,
date_from,
date_to,
file_name="intersection_report.txt",
title="",
target_domain_no_www=None, #domain without "www."
window_size_in_hours=24,
threshold_in_percentage=10.0):
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
ips = []
start = date_from
f1 = open(file_name, 'w+')
print >> f1, "Intersection Report:", title
print >> f1, "\nIncidents:"
for id_incident in id_incidents:
incident = self.bothound_tools.get_incident(id_incident)[0]
ips = ips + es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target']).keys()
print >> f1, "Incident {}, target domain = {}, Start:{}, Stop:{}".format(
id_incident, incident['target'], incident['start'],
incident['stop'])
ips = set(ips)
print >> f1, "\nNotal number of banned IPs:", len(ips)
print >> f1, "\nFinding intersection with data from {} to {}".format(
date_from, date_to)
print >> f1, "Threshold = {}%".format(threshold_in_percentage)
print >> f1, "Target domain = {}".format(
target_domain_no_www
if target_domain_no_www != None else "all domains")
print >> f1, "Sliding window size : {} hours".format(
window_size_in_hours)
print >> f1, "\nIntersections found:"
while start < date_to:
stop = start + timedelta(hours=window_size_in_hours)
print "processing {}, + {} hours...".format(
start, window_size_in_hours)
ips_to_check = es_handler.get_banjax(start, stop,
target_domain_no_www).keys()
ips_to_check = set(ips_to_check)
intersection = len(ips.intersection(ips_to_check))
percentage1 = intersection * 100.0 / len(ips)
percentage2 = intersection * 100.0 / len(ips_to_check)
if percentage1 > threshold_in_percentage or percentage2 > threshold_in_percentage:
print >> f1, "Original:{:.2f}%, Window:{:.2f}%, Total: {}, start : {} + {} hours".format(
percentage1, percentage2, intersection, start,
window_size_in_hours)
start = start + timedelta(hours=window_size_in_hours / 2.0)
print >> f1, "End"
f1.close()
def calculate_cross_table_banjax(self, incidents):
# Calculating common Banned Ips for a set of incidents
es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
self.bothound_tools.es_host,
self.bothound_tools.es_port)
common = -1
print "processing..."
result = []
groups = []
for i in incidents:
incident = self.bothound_tools.get_incident(i)[0]
#pdb.set_trace()
banned_ips = es_handler.get_banjax(incident['start'],
incident['stop'],
incident['target'])
ips = []
for p in banned_ips.keys():
ips.append(p)
if (common < 0):
common = set(ips)
else:
common = common.intersection(ips)
result.append([len(ips), len(common)])
groups.append(ips)
for i in range(0, len(groups)):
print "Incident", i, len(groups[i])
ip_counts = {}
for g in groups:
for ip in g:
if (ip in ip_counts):
ip_counts[ip] = ip_counts[ip] + 1
else:
ip_counts[ip] = 1
for i in range(1, len(incidents) + 1):
cur_count = 0
for ip in ip_counts:
if (ip_counts[ip] == i):
cur_count = cur_count + 1
print cur_count, i
"""
#calculate moving intersection
print "moving intersection"
for i in range(0, len(incidents)-1):
ips1 = set(groups[i])
ips2 = set(groups[i+1])
print i+1, i+2, len(ips1.intersection(ips2))
"""
print "cross table"
cross_table = []
for i in range(0, len(incidents)):
for j in range(i + 1, len(incidents)):
ips1 = set(groups[i])
ips2 = set(groups[j])
num = len(ips1.intersection(ips2))
cross_table.append((i + 1, j + 1, len(ips1), len(ips2), num,
num * 100.0 / min(len(ips1), len(ips2))))
sorted_cross_table = sorted(cross_table,
key=lambda k: k[5],
reverse=True)
f1 = open('cross_table.txt', 'w+')
for d in sorted_cross_table:
s = "{},{},{},{},{},{:.1f}%".format(d[0], d[1], d[2], d[3], d[4],
d[5])
print s
print >> f1, s
f1.close()
def __init__(self, bothound_tools):
self.bothound_tools = bothound_tools
self.es_handler = ESHandler(self.bothound_tools.es_user,
self.bothound_tools.es_password,
bothound_tools.es_host,
self.bothound_tools.es_port)
