def report(yesterday, start_datetime, end_datetime):
# Check report_dir
report_dir = config.report_dir + str(yesterday) + "/"
if not os.path.exists(report_dir):
os.makedirs(report_dir)
nf = nflow.NetFlow()
iplists = iplist.IPList()
ip_proto = ""
#for ipv4, ipv6
for ip in [4, 6]:
for orderby in ["flows", "packets", "bytes"]:
if ip != 4:
ip_proto = "6"
#for ICMP
absname = report_dir + str(
yesterday) + "_ICMP" + ip_proto + "_" + orderby
nf.readLog(start_datetime, end_datetime, 1,
options=["-s", "srcip/"+orderby, "-n", \
"0", "proto icmp%s" % ip_proto],
file_name=absname)
if ip != 4:
ip_proto = "v6"
# For port list
for port in [
21, 22, 23, 25, 53, 80, 135, 139, 443, 445, 3128, 3389
]:
absname = "%s%s_port%d%s_%s" % \
(report_dir, str(yesterday), port, ip_proto, orderby)
nf.readLog(start_datetime, end_datetime, 1,
options=["-s", "srcip/"+orderby, "-n", "0", \
"dst port %s and (%s)" % \
(str(port), iplists.getNetList("src",ip))],
file_name=absname)
# For uplink
absname = report_dir + str(
yesterday) + "_uplink" + ip_proto + "_" + orderby
nf.readLog(start_datetime, end_datetime, 1,
options=["-s", "srcip/"+orderby, "-n", "0", "%s" % \
iplists.getNetList("src",ip)],
file_name=absname)
# For downlink
absname = "%s%s_downlink%s_%s" % \
(report_dir, str(yesterday), ip_proto, orderby)
nf.readLog(start_datetime, end_datetime, 1,
options=["-s", "dstip/"+orderby, "-n", "0", "%s" % \
iplists.getNetList("dst",ip)],
file_name=absname)
def writeAnomalyFile(self, events):
iplist_handler = iplist.IPList()
file_handler = open(config.anomaly_log, "aw")
file_handler.write("%s -> %s\n" % \
(str(self.start_datetime),
str(self.end_datetime))
)
for event in events:
# see if in whitelist
if iplist_handler.inWhiteList([event[1], event[3]]):
continue
file_handler.write("%s: %s:%s -> %s:%s (%d)\n" % \
(event[0],event[1],event[2],event[3],event[4],event[5]))
file_handler.close()
def writeAnomalyDB(self, events):
iplist_handler = iplist.IPList()
# Writing into database
local_db = database.DB("local")
local_db.Execute("SET NAMES 'utf8'")
# For each event
for event in events:
# update blacklist
if len(event[1]) > 2:
iplist_handler.addBlackList(event[1], event[0])
# Update anomaly_log table
timestamp = ""
sql = "select * from anomaly_log where stop_time='%s'" \
" and event_type='%s' and src_ip='%s' and src_port='%s'" \
" and dst_ip='%s' and dst_port='%s';" % \
(self.start_datetime, event[0], event[1], \
event[2], event[3], event[4])
result = local_db.Select(sql)
# If the log exists
if result != ():
timestamp = str(result[0][1])[:-3]
sql = "update anomaly_log set stop_time='%s'," \
" attack_count=%d where stop_time='%s' and" \
" event_type='%s' and src_ip='%s' and src_port='%s'" \
" and dst_ip='%s' and dst_port='%s';" % \
(self.end_datetime, event[5]+int(result[0][8]), \
self.start_datetime, event[0], event[1], event[2], \
event[3], event[4])
# If this is a new log
else:
timestamp = self.start_datetime
sql = "insert into anomaly_log (start_time, stop_time," \
" event_type, src_ip, src_port, dst_ip, dst_port," \
" attack_count) values" \
" ('%s','%s','%s','%s','%s','%s','%s',%d);" % \
(self.start_datetime, self.end_datetime, event[0], \
event[1], event[2], event[3], event[4], event[5])
local_db.Execute(sql)
local_db.CloseDB()
def plots(start_datetime, end_datetime):
nf = nflow.NetFlow()
iplists = iplist.IPList()
src_plain = nf.readLog(
start_datetime,
end_datetime,
1,
options=["-a", "-A", "srcip4/16",
iplists.getNetList("src")])
src = nf.parseSummary(src_plain)
srcbytes, srcflows, srcpackets = src['flows'], src['bytes'], src['packets']
dst_plain = nf.readLog(
start_datetime,
end_datetime,
1,
options=["-a", "-A", "dstip4/16",
iplists.getNetList("dst")])
dst = nf.parseSummary(dst_plain)
dstbytes, dstflows, dstpackets = dst['flows'], dst['bytes'], dst['packets']
outputer = output.Output(end_datetime=end_datetime)
outputer.writePlot(srcbytes, srcflows, srcpackets, "src")
outputer.writePlot(dstbytes, dstflows, dstpackets, "dst")
minute_interval = 1
# Get rules
rule_handler = rule.Rules()
rules = rule_handler.getRulesFromDB()
ssrcip_min_thres = rule_handler.getSsrcipMinThres()
# Ignore ACK response
ACK = [22, 23, 53, 80, 110, 443, 1433, 3306]
P2P = [4672, 51413, 6881, 17788] #eMule, BT, BT, PPS
EXCEPT = [6633] #SDN
# Brute Force port
BF_ports = [22, 110, 3306, 3389]
# Get white list
iplists = iplist.IPList()
whitelist = ",".join(iplists.whitelist)
def getDconnPort(start_datetime, end_datetime, minute_interval):
DCONN = []
ignore_port = []
ignore_port.extend(ACK)
outputer = output.Output(start_datetime, end_datetime)
nf = nflow.NetFlow()
# get the set of dst port where (srcip,dstip,dstport) over threshold
plain = nf.readLog(start_datetime, end_datetime, minute_interval, \
options=["-A", "srcip,dstip,dstport", "-s", \
"record/flows", "-n", "20", "-N"], \
mode="fmt:%sa,%da,%dp,%fl")
for line in nf.parseLogLine(plain, mode="fmt:%sa,%da,%dp,%fl"):
#!/usr/bin/env python
import os
import sys
# The module we need
directory = os.path.dirname(os.path.abspath(__file__))
sys.path.append(directory + "/..")
import date
import output
import nflow
import iplist
nf = nflow.NetFlow()
ip_list = iplist.IPList()
unusedIP_list = ip_list.unusedIP
d = date.Date()
date_list = d.datetimeLastMonth()
outputer = output.Output()
rule_filter = []
for ip in unusedIP_list:
rule_filter.append("dst ip %s" % ip)
rule_filter = "not src net 140.116.0.0/16 and ( " + " or ".join(
rule_filter) + " )"
# CMD can't have such a long rules
f = open("rule.list", "w")
f.write(rule_filter)
f.close()
start_date = date_list[0]
# Add the flag to notify the end of datalist