def getDconnPort(start_datetime, end_datetime, minute_interval):
DCONN = []
ignore_port = []
ignore_port.extend(ACK)
outputer = output.Output(start_datetime, end_datetime)
nf = nflow.NetFlow()
# get the set of dst port where (srcip,dstip,dstport) over threshold
plain = nf.readLog(start_datetime, end_datetime, minute_interval, \
options=["-A", "srcip,dstip,dstport", "-s", \
"record/flows", "-n", "20", "-N"], \
mode="fmt:%sa,%da,%dp,%fl")
for line in nf.parseLogLine(plain, mode="fmt:%sa,%da,%dp,%fl"):
if int(line["fl"]) > 1000:
ignore_port.append(int(line["dp"]))
# get the (dstip,dstport) without wellknown ports and the port below
plain = nf.readLog(start_datetime, end_datetime, minute_interval, \
options=["-A", "dstip,dstport", "-s", "record", "-n", \
"20", "-N", "not port in %s" % ignore_port], \
mode="fmt:%da,%dp,%fl")
for line in nf.parseLogLine(plain, mode="fmt:%da,%dp,%fl"):
if int(line["fl"]) > 1000:
DCONN.append(int(line["dp"]))
outputer.writeDconn(line["da"], line["dp"], line["fl"])
return DCONN
def monitorBlacklistIP(start_datetime, end_datetime):
# Set up the filter rules (blacklist IP)
# Well Known Ports (0 – 1023)
# Registered Ports (1024 – 49151)
# Dynamic or Private Ports (49152 – 65535)
# We think port 0-10000 as system reversed ports
blackConn_list = []
nf = nflow.NetFlow()
plain = nf.readLog(start_datetime, end_datetime, minute_interval,
options=["dst ip in [%s] and src port > 10000" \
% ",".join(black_list)],
mode="csv")
for e in nf.parseLogLine(plain, mode="csv"):
blackConn_list.append({
'src_ip': e['sa'],
'dst_ip': e['da'],
'src_port': e['sp'],
'dst_port': e['dp'],
'date': e['te']
})
# Output to DB
output_handler = output.Output(start_datetime, end_datetime)
output_handler.writeBlackList(blackConn_list)
def spamDetect(start_datetime, end_datetime):
output_handler = output.Output(start_datetime)
nf = nflow.NetFlow()
plain = nf.readLog(start_datetime,
end_datetime,
1,
options=["-N", "-s", "srcip/flows", "dst port 25"])
for line in nf.parseLogLine(plain):
if int(line["fl"]) > 1000:
output_handler.writeSpam(line["val"], int(line["fl"]))
def report(yesterday, start_datetime, end_datetime):
# Check report_dir
report_dir = config.report_dir + str(yesterday) + "/"
if not os.path.exists(report_dir):
os.makedirs(report_dir)
nf = nflow.NetFlow()
iplists = iplist.IPList()
ip_proto = ""
#for ipv4, ipv6
for ip in [4, 6]:
for orderby in ["flows", "packets", "bytes"]:
if ip != 4:
ip_proto = "6"
#for ICMP
absname = report_dir + str(
yesterday) + "_ICMP" + ip_proto + "_" + orderby
nf.readLog(start_datetime, end_datetime, 1,
options=["-s", "srcip/"+orderby, "-n", \
"0", "proto icmp%s" % ip_proto],
file_name=absname)
if ip != 4:
ip_proto = "v6"
# For port list
for port in [
21, 22, 23, 25, 53, 80, 135, 139, 443, 445, 3128, 3389
]:
absname = "%s%s_port%d%s_%s" % \
(report_dir, str(yesterday), port, ip_proto, orderby)
nf.readLog(start_datetime, end_datetime, 1,
options=["-s", "srcip/"+orderby, "-n", "0", \
"dst port %s and (%s)" % \
(str(port), iplists.getNetList("src",ip))],
file_name=absname)
# For uplink
absname = report_dir + str(
yesterday) + "_uplink" + ip_proto + "_" + orderby
nf.readLog(start_datetime, end_datetime, 1,
options=["-s", "srcip/"+orderby, "-n", "0", "%s" % \
iplists.getNetList("src",ip)],
file_name=absname)
# For downlink
absname = "%s%s_downlink%s_%s" % \
(report_dir, str(yesterday), ip_proto, orderby)
nf.readLog(start_datetime, end_datetime, 1,
options=["-s", "dstip/"+orderby, "-n", "0", "%s" % \
iplists.getNetList("dst",ip)],
file_name=absname)
def aflowDetect(start_datetime, end_datetime):
output_handler = output.Output(start_datetime)
nf = nflow.NetFlow()
plain = nf.readLog(start_datetime, end_datetime, 1,
options=["-N", "-A", "proto,srcip,dstip,dstport",
"-s", "record/bytes", "-L", "5G",
"inet and proto udp and not net 10.0.0.0/8" \
" and not ip 140.116.49.6 and" \
" not port in [3389]"],
mode="fmt:%sa,%da,%dp,%byt,%fl")
for line in nf.parseLogLine(plain, mode="fmt:%sa,%da,%dp,%byt,%fl"):
if int(line["fl"]) > 5:
output_handler.writeAflow(start_datetime, line["sa"], line["da"],
line["dp"], line["byt"], line["fl"])
def monitorUnusedIP(start_datetime, end_datetime):
monIP_list = []
nf = nflow.NetFlow()
plain = nf.readLog(start_datetime,
end_datetime,
minute_interval,
options=["dst ip in [%s]" % ",".join(unused_list)],
mode="csv")
for e in nf.parseLogLine(plain, mode="csv"):
monIP_list.append({
'source': e['sa'],
'srcport': e['sp'],
'target': e['da'],
'dstport': e['dp'],
'date': e['te']
})
# Output to DB
output_handler = output.Output(start_datetime, end_datetime)
output_handler.writeMonIPLog(monIP_list)
def scanDetect(start_datetime, end_datetime):
"""
We only detect the following scan: 22,23,139,445,3389
"""
port_statistics = {}
output_handler = output.Output(start_datetime)
nf = nflow.NetFlow()
plain = nf.readLog(start_datetime, end_datetime, 1,
options=["-N", "-s", "record/flows", "-n", "0", \
"-A", "srcip,dstip,dstport", \
"inet and not net 10.0.0.0/8 and " \
" port in [22,23,139,445,3389]"], \
mode="fmt:%sa,%dp")
for line in nf.parseLogLine(plain, mode="fmt:%sa,%dp"):
item = (line["sa"], line["dp"])
if item in port_statistics:
port_statistics[item] += 1
else:
port_statistics[item] = 0
for tuples, count in port_statistics.iteritems():
if count > 300:
output_handler.writeScan(tuples[0], tuples[1], count)
def main():
output = Output(start_datetime, end_datetime)
checkIP = IPchecker()
nf = nflow.NetFlow()
for rule in rules:
plain = nf.readLog(
start_datetime,
real_end_datetime,
options=["-a", "-L",
"%d" % rule[2],
"port %s" % rule[1]])
for line in nf.parseLogLine(plain):
if checkIP.inWhiteList([line['sa'], line['da']]):
continue
elif checkIP.AllInternal([line['sa'], line['da']]):
continue
srcIP, srcport, dstIP, dstport = line['sa'], line['sp'], \
line['da'], line['dp']
bytes = int(line['obyt']) + int(line['ibyt'])
protocol = line['pr']
output.writeTraffic(srcIP, srcport, \
dstIP, dstport, \
protocol, bytes)
def plots(start_datetime, end_datetime):
nf = nflow.NetFlow()
iplists = iplist.IPList()
src_plain = nf.readLog(
start_datetime,
end_datetime,
1,
options=["-a", "-A", "srcip4/16",
iplists.getNetList("src")])
src = nf.parseSummary(src_plain)
srcbytes, srcflows, srcpackets = src['flows'], src['bytes'], src['packets']
dst_plain = nf.readLog(
start_datetime,
end_datetime,
1,
options=["-a", "-A", "dstip4/16",
iplists.getNetList("dst")])
dst = nf.parseSummary(dst_plain)
dstbytes, dstflows, dstpackets = dst['flows'], dst['bytes'], dst['packets']
outputer = output.Output(end_datetime=end_datetime)
outputer.writePlot(srcbytes, srcflows, srcpackets, "src")
outputer.writePlot(dstbytes, dstflows, dstpackets, "dst")
def scanDetect(start_datetime, end_datetime):
nf = nflow.NetFlow()
# event_name, srcip, srcport, dstip, dstport, count, pattern
events = []
# Get Dconn port result
DCONN = getDconnPort(start_datetime, end_datetime, minute_interval)
plain = nf.readLog(start_datetime, end_datetime, minute_interval, \
options=[" %s and not inet6 and not ip in [%s]" \
" and not port in %s and not src port in %s" \
" and (proto tcp or proto udp)" \
% (iplists.getIgnoreNet(), whitelist, \
P2P+DCONN+EXCEPT, ACK), \
"-A", "srcip,dstip,dstport,proto"],
mode="fmt:%sa,%da,%dp,%fl,%pr")
srcip_dict = {}
for line in nf.parseLogLine(plain, mode="%fmt:%sa,%da,%dp,%fl,%pr"):
# Count the number of srcip
try:
srcip_dict[line['sa']].append((line['da'], line['dp'], line['pr']))
except:
srcip_dict[line['sa']] = [(line['da'], line['dp'], line['pr'])]
# Checking "TXTT"
if int(line['fl']) > rules["TXTT"]["threshold"]:
events.append([rules["TXTT"]["event"], line['sa'], "X", \
line['da'], line['dp'], int(line['fl']), "TXTT"])
elif int(line["fl"]) > rules["TXTL"]["threshold"] \
and int(line["dp"]) in BF_ports:
# Start with 140.116. in both src, dst IP
if line['sa'][0:8] == line['da'][0:8]:
continue
events.append([rules["TXTL"]["event"], line['sa'], "X", \
line['da'], line['dp'], int(line['fl']), "TXTL"])
# Counting vertical and horizontal scan
for ip in srcip_dict:
# Get the list of dst IP
ip_list = [(i[0], i[2]) for i in srcip_dict[ip]]
# Get the list of dst port
port_list = [(i[1], i[2]) for i in srcip_dict[ip]]
# Check whether there is "TXFT" (horizontal scan)
tmp_list = list(set(port_list))
if len(port_list) - len(tmp_list) > rules["TXFT"]["threshold"]:
for tmp in tmp_list:
# To avoid P2P connection, check the distribution of dst IP
# the # of max classA should be bigger than half of dst IP
if "140.116." in ip:
classA_list = [i[0].split('.')[0] \
for i in srcip_dict[ip] if i[1]==tmp[0]]
if len(classA_list)/2 > \
max([classA_list.count(i) for i in set(classA_list)]):
continue
# Count the # of port
num = port_list.count(tmp)
if num > rules["TXFT"]["threshold"]:
events.append([
rules["TXFT"]["event"], ip, "X", "F", tmp[0], num,
"TXFT"
])
# Check whether there is "TXTF" (vertical scan)
tmp_list = list(set(ip_list))
if len(ip_list) - len(tmp_list) > rules["TXTF"]["threshold"]:
for tmp in tmp_list:
# Count the # of IP
num = ip_list.count(tmp)
if num > rules["TXTF"]["threshold"]:
# Not in Taiwan and
# No 1/10 of dst port is smaller than 2000
if not iplists.inForeign([ip, tmp[0]]) and \
(num/10) > len([i for i in srcip_dict[ip] \
if i[0]==tmp[0] and int(i[1])<2000]):
continue
events.append([
rules["TXTF"]["event"], ip, "X", tmp[0], "F", num,
"TXTF"
])
return events
#!/usr/bin/env python
import os
import sys
# The module we need
directory = os.path.dirname(os.path.abspath(__file__))
sys.path.append(directory + "/..")
import date
import output
import nflow
import iplist
nf = nflow.NetFlow()
ip_list = iplist.IPList()
unusedIP_list = ip_list.unusedIP
d = date.Date()
date_list = d.datetimeLastMonth()
outputer = output.Output()
rule_filter = []
for ip in unusedIP_list:
rule_filter.append("dst ip %s" % ip)
rule_filter = "not src net 140.116.0.0/16 and ( " + " or ".join(
rule_filter) + " )"
# CMD can't have such a long rules
f = open("rule.list", "w")
f.write(rule_filter)
f.close()
start_date = date_list[0]
# Add the flag to notify the end of datalist
def DRDoSDetect(start_datetime, end_datetime):
for port in port_list:
if port == 53:
proto_filter = dns_filter
else:
proto_filter = ""
# format: {(vul_ip,target_ip): [#flows, total_bytes]}
statistics_to_port = {}
statistics_from_port = {}
nf = nflow.NetFlow()
plain = nf.readLog(start_datetime, end_datetime, 1,
options=["port %d and not ip 239.255.255.250" \
" and not (src port %d and dst port %d)" \
" and inet and proto udp %s" \
% (port, port, port, proto_filter)])
for line in nf.parseLogLine(plain):
if line["sp"] == str(port):
stat_tuple = (line["sa"], line["da"])
if stat_tuple not in statistics_from_port \
and iplists.allInternal([line["sa"]]):
statistics_from_port.update({stat_tuple: \
[1,int(line["ibyt"])+int(line["obyt"])]})
elif iplists.allInternal([line["sa"]]):
statistics_from_port[stat_tuple][0] += 1
statistics_from_port[stat_tuple][1] += \
int(line["ibyt"])+int(line["obyt"])
elif line["dp"] == str(port):
stat_tuple = (line["da"], line["sa"])
if stat_tuple not in statistics_to_port \
and iplists.allInternal([line["da"]]):
statistics_to_port.update({stat_tuple: \
[1,int(line["ibyt"])+int(line["obyt"])]})
elif iplists.allInternal([line["da"]]):
statistics_to_port[stat_tuple][0] += 1
statistics_to_port[stat_tuple][1] += \
int(line["ibyt"])+int(line["obyt"])
# set
from_port = set(statistics_from_port.keys())
to_port = set(statistics_to_port.keys())
# and set
and_set = from_port & to_port
vul_iplist = set()
# suspicious_vulnerable_ip: target_ip
sus_vul_ips = {}
# suspicious_target_ip: vulnerable_ip
sus_target_ips = {}
for item in and_set:
# ratio
from_byte = statistics_from_port[item][1]
from_flow = statistics_from_port[item][0]
to_byte = statistics_to_port[item][1]
to_flow = statistics_to_port[item][0]
from_ratio = from_byte / from_flow
to_ratio = to_byte / to_flow
if (from_ratio / to_ratio) > 3:
# for vul ip
if item[0] not in sus_vul_ips:
sus_vul_ips[item[0]] = [item[1]]
else:
sus_vul_ips[item[0]].append(item[1])
# for target ip
if item[1] not in sus_target_ips:
sus_target_ips[item[1]] = [item[0]]
else:
sus_target_ips[item[1]].append(item[0])
for ip in sus_vul_ips:
if len(sus_vul_ips[ip]) > 1:
vul_iplist.add(ip)
for ip in sus_target_ips:
if len(sus_target_ips[ip]) > 1:
vul_iplist |= set(sus_target_ips[ip])
output_handler = output.Output(start_datetime=start_datetime)
for IP in vul_iplist:
output_handler.writeDRDoS(IP, port)