def calculate_pingback_domains(self, incidents): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) index = 1 for i in incidents: incident = self.bothound_tools.get_incident(i)[0] res_dict = es_handler.get_pingback_domains(incident['start'], incident['stop'], incident['target']) res_list = [] for key, value in res_dict.iteritems(): temp = [key, value] res_list.append(temp) res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True) f1 = open('pingback_domains_incident_{}({}).txt'.format(index, i), 'w+') for r in res_sorted: print >> f1, r[0] f1.close() index = index + 1
def calculate_user_agents(self, incidents): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) res = [] for i in incidents: incident = self.bothound_tools.get_incident(i)[0] res_dict = es_handler.get_banned_user_agents( incident['start'], incident['stop'], incident['target']) res_list = [] for key, value in res_dict.iteritems(): temp = [key, value] res_list.append(temp) res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True) num_most = len(res_sorted) if len(res_sorted) < 50 else 50 print "incident", i, "winner", res_sorted[0] res.append(res_sorted[0:num_most]) i = 1 f1 = open('user_agents.txt', 'w+') for incident in res: print >> f1, "incident", i print "incident", i i = i + 1 for r in incident: print >> f1, r[0], r[1] print r[0], r[1] f1.close()
def calculate_responses(self, incidents): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) res = [] for i in incidents: incident = self.bothound_tools.get_incident(i)[0] rates = es_handler.get_banned_responses(incident['start'], incident['stop'], incident['target']) res_list = [] for key, value in rates.iteritems(): temp = [key, value] res_list.append(temp) res_sorted = sorted(res_list, key=lambda k: k[1], reverse=True) num_most = len(res_sorted) if len(res_sorted) < 3 else 3 res.append(res_sorted[0:num_most]) i = 1 for incident in res: print "incident", i i = i + 1 for r in incident: print r[1], r[0]
def calculate_urls(self, incidents): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) incident_urls = [] for i in incidents: incident = self.bothound_tools.get_incident(i)[0] urls = es_handler.get_banned_urls(incident['start'], incident['stop'], incident['target']) urls_list = [] for key, value in urls.iteritems(): temp = [key, value] urls_list.append(temp) urls_sorted = sorted(urls_list, key=lambda k: k[1], reverse=True) num_most = len(urls_sorted) if len(urls_sorted) < 3 else 3 incident_urls.append(urls_sorted[0:num_most]) f1 = open('urls.txt', 'w+') for urls in incident_urls: print "incident", i for url in urls: print url[1], url[0] print >> f1, url[1], url[0] f1.close()
def calculate_incident_intersection_plus_ua(self, incidents1, incidents2): # Calculating common Banned Ips for two sets of incidents es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) print "processing..." ips1 = [] for i in incidents1: incident = self.bothound_tools.get_incident(i)[0] #pdb.set_trace() banned_ips = es_handler.get_banjax(incident['start'], incident['stop'], incident['target']) for key, value in banned_ips.iteritems(): ips1.append([key, value]) ips2 = [] for i in incidents2: incident = self.bothound_tools.get_incident(i)[0] #pdb.set_trace() banned_ips = es_handler.get_banjax(incident['start'], incident['stop'], incident['target']) for key, value in banned_ips.iteritems(): ips2.append([key, value]) intersection = 0 processed_ips = {} for ip1 in ips1: for ip2 in ips2: if ip1[0] == ip2[0]: if ip1[0] in processed_ips: break found = False for ua1 in ip1[1]['ua'].keys(): for ua2 in ip2[1]['ua'].keys(): if (ua1 == ua2): found = True break if found: break if found: intersection = intersection + 1 processed_ips[ip1[0]] = 1 num = intersection d = [ len(ips1), len(ips2), num, num * 100.0 / min(len(ips1), len(ips2)) ] print "group1", incidents1 print "group2", incidents2 s = "{},{},{},{:.1f}%".format(d[0], d[1], d[2], d[3]) print s
def __init__(self, bothound_tools): """ store the exp config in self's attribute. """ Thread.__init__(self) self.daemon = True utc_datetime = datetime.utcnow() self.bothound_tools = bothound_tools self.es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, bothound_tools.es_host, self.bothound_tools.es_port)
def calculate_unique_ips(self, incidents): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) ips = [] for i in incidents: incident = self.bothound_tools.get_incident(i)[0] cur_ips = es_handler.get_deflect_unique_ips( incident['start'], incident['stop'], incident['target']) ips.append(set(cur_ips.keys())) for i in range(0, len(ips)): print i, "Num unique IPs:", len(ips[i])
def get_banned_ips(self, incidents): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) index = 1 for i in incidents: incident = self.bothound_tools.get_incident(i)[0] ips = es_handler.get_banjax(incident['start'], incident['stop'], incident['target']) f1 = open('incident_{}({}).txt'.format(index, i), 'w+') for ip in ips: #pdb.set_trace() print >> f1, ip f1.close() index = index + 1
def find_intersections( self, id_incidents, date_from, date_to, file_name="intersection_report.txt", title="", target_domain_no_www=None, #domain without "www." window_size_in_hours=24, threshold_in_percentage=10.0): es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) ips = [] start = date_from f1 = open(file_name, 'w+') print >> f1, "Intersection Report:", title print >> f1, "\nIncidents:" for id_incident in id_incidents: incident = self.bothound_tools.get_incident(id_incident)[0] ips = ips + es_handler.get_banjax(incident['start'], incident['stop'], incident['target']).keys() print >> f1, "Incident {}, target domain = {}, Start:{}, Stop:{}".format( id_incident, incident['target'], incident['start'], incident['stop']) ips = set(ips) print >> f1, "\nNotal number of banned IPs:", len(ips) print >> f1, "\nFinding intersection with data from {} to {}".format( date_from, date_to) print >> f1, "Threshold = {}%".format(threshold_in_percentage) print >> f1, "Target domain = {}".format( target_domain_no_www if target_domain_no_www != None else "all domains") print >> f1, "Sliding window size : {} hours".format( window_size_in_hours) print >> f1, "\nIntersections found:" while start < date_to: stop = start + timedelta(hours=window_size_in_hours) print "processing {}, + {} hours...".format( start, window_size_in_hours) ips_to_check = es_handler.get_banjax(start, stop, target_domain_no_www).keys() ips_to_check = set(ips_to_check) intersection = len(ips.intersection(ips_to_check)) percentage1 = intersection * 100.0 / len(ips) percentage2 = intersection * 100.0 / len(ips_to_check) if percentage1 > threshold_in_percentage or percentage2 > threshold_in_percentage: print >> f1, "Original:{:.2f}%, Window:{:.2f}%, Total: {}, start : {} + {} hours".format( percentage1, percentage2, intersection, start, window_size_in_hours) start = start + timedelta(hours=window_size_in_hours / 2.0) print >> f1, "End" f1.close()
def calculate_cross_table_banjax(self, incidents): # Calculating common Banned Ips for a set of incidents es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, self.bothound_tools.es_host, self.bothound_tools.es_port) common = -1 print "processing..." result = [] groups = [] for i in incidents: incident = self.bothound_tools.get_incident(i)[0] #pdb.set_trace() banned_ips = es_handler.get_banjax(incident['start'], incident['stop'], incident['target']) ips = [] for p in banned_ips.keys(): ips.append(p) if (common < 0): common = set(ips) else: common = common.intersection(ips) result.append([len(ips), len(common)]) groups.append(ips) for i in range(0, len(groups)): print "Incident", i, len(groups[i]) ip_counts = {} for g in groups: for ip in g: if (ip in ip_counts): ip_counts[ip] = ip_counts[ip] + 1 else: ip_counts[ip] = 1 for i in range(1, len(incidents) + 1): cur_count = 0 for ip in ip_counts: if (ip_counts[ip] == i): cur_count = cur_count + 1 print cur_count, i """ #calculate moving intersection print "moving intersection" for i in range(0, len(incidents)-1): ips1 = set(groups[i]) ips2 = set(groups[i+1]) print i+1, i+2, len(ips1.intersection(ips2)) """ print "cross table" cross_table = [] for i in range(0, len(incidents)): for j in range(i + 1, len(incidents)): ips1 = set(groups[i]) ips2 = set(groups[j]) num = len(ips1.intersection(ips2)) cross_table.append((i + 1, j + 1, len(ips1), len(ips2), num, num * 100.0 / min(len(ips1), len(ips2)))) sorted_cross_table = sorted(cross_table, key=lambda k: k[5], reverse=True) f1 = open('cross_table.txt', 'w+') for d in sorted_cross_table: s = "{},{},{},{},{},{:.1f}%".format(d[0], d[1], d[2], d[3], d[4], d[5]) print s print >> f1, s f1.close()
def __init__(self, bothound_tools): self.bothound_tools = bothound_tools self.es_handler = ESHandler(self.bothound_tools.es_user, self.bothound_tools.es_password, bothound_tools.es_host, self.bothound_tools.es_port)