def _portsVary(self, sPortVarying):
"""If key is not in sortingDict put key-value pairs in sortingDict and
tablesDict,
else merge value to existing value.
sIPs/dIPs exist in key and value, because they could be the same ("any")
as the one already in the hashmap and still contain different elements.
To provide a correct elements list they needs to be extended.
"""
self.sortingDict.clear()
self.tablesDict.clear()
newR = RuleSet()
for r in self.ruleSet:
if sPortVarying:
keys = (r.direction, r.sIPs, r.dIPs, r.dPorts, r.interface, r.action)
value = (r.sPorts, r.timeStamp, r.flag, r.sIPs, r.dIPs)
else:
keys = (r.direction, r.sIPs, r.sPorts, r.dIPs, r.interface, r.action)
value = (r.dPorts, r.timeStamp, r.flag, r.sIPs, r.dIPs)
self._sortIntoDicts(keys, value)
for key, value in self.tablesDict.iteritems():
if sPortVarying:
r = Rule(key[0], value[3], value[0], value[4], key[3], self.proto, key[4], key[5], self.style, value[1], value[2])
else:
r = Rule(key[0], value[3], key[2], value[4], value[0], self.proto, key[4], key[5], self.style, value[1], value[2])
newR.insert(r)
self.ruleSet = newR
del newR
def _filterRules(self, rules, checkList, proto, direction):
filteredRules = RuleSet()
for s in checkList:
if s[2] == proto or s[2] == "any":
for r in rules.values():
for d in direction:
if r.direction == d \
and (s[0] in r.dIPs or s[0] == "any") \
and (s[1] in r.dPorts or s[1] == "any"):
rules.remove(r)
filteredRules.insert(r)
return filteredRules
def testRules(connections, rules, verbose=False):
filtertConns = RuleSet()
for c in connections:
ruled = False
for r in rules:
if c.interface == r.interface and c.proto == r.proto:
if _testPorts(c, r):
if _testIPs(c, r):
ruled = True
break
if not ruled:
if verbose:
print "Was not matched"
print c
filtertConns.insert(c)
return filtertConns
def _checkTcpForConns(self):
"""Checks tcp-set for connections."""
print "Checking for Connections"
tcpRuleSetConnsOnly = RuleSet()
if self.p.linktype == self.PFLOG_DUMP:
tcpRuleSetConnsOnly = self.tcpRuleSet
else:
for r in self.tcpRuleSet:
connection = False
for r2 in self.tcpRuleSet:
if (r == r2 and (r.flag[0]=="SYN") and (r2.flag[0]=="FIN")):
connection = True
break
if connection:
tcpRuleSetConnsOnly.insert(r)
self.tcpRuleSet = tcpRuleSetConnsOnly
class PcapParser():
"""
PcapParser parses a pcap-file with dpkt.
Every ip-paket is put into either the tcp or udp or icmp ruleset.
Rulesets are sets, they contain every item only once.
Concerning tcp: only full connections are kept (there must be a syn
and a fin paket).
Non-ip-pakets are not concerned.
"""
def __init__(self, pcapFile, action, style, direction, maxNumPortsAny, \
connectionChecking, portscanChecking, printElements, \
innerNetworks=None, ddosDetection=True):
self.p = pcapFile
self.direction = direction
self.action = action
self.style = style
self.interface = "None"
self.tcpRuleSet = RuleSet()
self.udpRuleSet = RuleSet()
self.icmpRuleSet = RuleSet()
self.tcpRuleList = []
self.udpRuleList = []
self.icmpRuleList = []
self.MAX_NUM_PORTS_ANY = maxNumPortsAny
self.connectionChecking = connectionChecking
self.ddosDetection = ddosDetection
self.saveElements = portscanChecking or printElements
self.PFLOG_DUMP = 117
self.innerNetworks = innerNetworks
def parsePcapFile(self):
"""Parse a pcap file"""
for ts, buf in self.p:
self._packetHandler(buf, ts)
if self.connectionChecking:
self._checkTcpForConns()
return self.tcpRuleSet, self.udpRuleSet, self.icmpRuleSet, \
self.tcpRuleList, self.udpRuleList, self.icmpRuleList
def _packetHandler(self, buf, ts):
"""Parse a pcap buffer"""
SINGLE_IP_SLASHSIZE = 32
try:
if (self.p.datalink() == self.PFLOG_DUMP):
#pkt = dpkt.pflog.Pflog(buf)
pkt = Pflog(buf)
self.interface = pkt.interfaceName
self.direction = "in" if pkt.direction == 1 else "out"
subpkt = pkt.data
try:
subpkt = dpkt.ip.IP(subpkt)
except:
#skip non IP packets
return
else:
pkt = dpkt.ethernet.Ethernet(buf)
subpkt = pkt.data
if not isinstance(subpkt, dpkt.ip.IP):
#skip non IP packets
return
proto = subpkt.p
shost = socket.inet_ntoa(subpkt.src)
dhost = socket.inet_ntoa(subpkt.dst)
shostInner = False
dhostInner = False
if self.innerNetworks != None:
for n in self.innerNetworks:
slashSize = str(n).split("/")[1]
if IPv4Network(str(shost)+"/"+str(slashSize)) == n:
shostInner = True
if IPv4Network(dhost+"/"+slashSize) == n:
dhostInner = True
if shostInner and dhostInner:
self.direction = "none-inner"
elif shostInner and not dhostInner:
self.direction = "out"
elif not shostInner and dhostInner:
self.direction = "in"
else:
self.direction = "none-outer"
except dpkt.Error:
#skip non-ethernet packages
return
try:
if proto == socket.IPPROTO_TCP:
try:
tcp = subpkt.data
flag = tcp.flags
if self.connectionChecking:
rightFlag = ((flag == dpkt.tcp.TH_SYN) \
or (flag & dpkt.tcp.TH_FIN != 0))
else:
rightFlag = (flag == dpkt.tcp.TH_SYN)
dport = tcp.dport
sport = tcp.sport
sIP = IPsMap(self.saveElements)
sIP.insert(IPv4Address(shost))
sPort = PortsMap(self.MAX_NUM_PORTS_ANY, self.saveElements)
sPort.insert(Port(sport, True))
dIP = IPsMap(self.saveElements)
dIP.insert(IPv4Address(dhost))
dPort = PortsMap(self.MAX_NUM_PORTS_ANY, self.saveElements)
dPort.insert(Port(dport, False))
if flag == dpkt.tcp.TH_SYN:
flag = "SYN"
else:
flag = "FIN"
r = Rule(self.direction, sIP, sPort, dIP, dPort, "tcp", \
self.interface, self.action, self.style, [ts, ], [flag, ])
if self.p.datalink() == self.PFLOG_DUMP or rightFlag:
self.tcpRuleSet.insert(r)
if self.ddosDetection:
self.tcpRuleList.append((IPv4Address(shost), ts))
except AttributeError:
#skip broken packages
return
elif proto == socket.IPPROTO_UDP:
udp = subpkt.data
dport = udp.dport
sport = udp.sport
sIP = IPsMap(self.saveElements)
sIP.insert(IPv4Address(shost))
sPort = PortsMap(self.MAX_NUM_PORTS_ANY, self.saveElements)
sPort.insert(Port(sport, True))
dIP = IPsMap(self.saveElements)
dIP.insert(IPv4Address(dhost))
dPort = PortsMap(self.MAX_NUM_PORTS_ANY, self.saveElements)
dPort.insert(Port(dport, False))
r = Rule(self.direction, sIP, sPort, dIP, dPort, "udp", \
self.interface, self.action, self.style, [ts, ])
self.udpRuleSet.insert(r)
if self.ddosDetection:
self.udpRuleList.append((IPv4Address(shost), ts))
elif proto == socket.IPPROTO_ICMP:
sIP = IPsMap(self.saveElements)
sIP.insert(IPv4Address(shost))
sPort = PortsMap(self.MAX_NUM_PORTS_ANY)
sPort.insert(Port(-1, True))
dIP = IPsMap(self.saveElements)
dIP.insert(IPv4Address(dhost))
dPort = PortsMap(self.MAX_NUM_PORTS_ANY)
dPort.insert(Port(-1, False))
r = Rule(self.direction, sIP, sPort, dIP, dPort, "icmp", \
self.interface, self.action, self.style, [ts, ])
self.icmpRuleSet.insert(r)
if self.ddosDetection:
self.icmpRuleList.append((IPv4Address(shost), ts))
except dpkt.Error:
return
def _checkTcpForConns(self):
"""Checks tcp-set for connections."""
print "Checking for Connections"
tcpRuleSetConnsOnly = RuleSet()
if self.p.linktype == self.PFLOG_DUMP:
tcpRuleSetConnsOnly = self.tcpRuleSet
else:
for r in self.tcpRuleSet:
connection = False
for r2 in self.tcpRuleSet:
if (r == r2 and (r.flag[0]=="SYN") and (r2.flag[0]=="FIN")):
connection = True
break
if connection:
tcpRuleSetConnsOnly.insert(r)
self.tcpRuleSet = tcpRuleSetConnsOnly
class InfectedHostsDetector:
"""
InfectedHostsDetector takes found portscans and policy violations
and tries to find indected hosts.
"""
def __init__(self, tcpPortscans, udpPortscans, policyViolations, suspPolVio):
self.tcpPortscans = tcpPortscans
self.udpPortscans = udpPortscans
self.policyViolations = policyViolations
self.portscans = RuleSet()
self.doubleIPs = IPsMap()
self.portscanIPs = IPsMap()
self.violationIPs = IPsMap()
self.clearedPolicyViolations = RuleSet()
self.suspPolVio = suspPolVio.split(":")
def detectInfectedHosts(self):
for ps in self.tcpPortscans.portscanSet:
if ps.direction == "out" or ps.direction == "none-inner":
self.portscans.insert(ps)
for ps in self.udpPortscans.portscanSet:
if ps.direction == "out" or ps.direction == "none-inner":
self.portscans.insert(ps)
for pv in self.policyViolations:
if pv.direction == "out" or pv.direction == "none-inner":
if str(pv.dPorts.values()[0].portNumber) in self.suspPolVio:
self.clearedPolicyViolations.insert(pv)
for portscan in self.portscans:
for violation in self.clearedPolicyViolations:
if portscan.sIPs == violation.sIPs:
self.doubleIPs.insert(portscan.sIPs)
for doubleIP in self.doubleIPs:
for policyVio in self.clearedPolicyViolations.values():
if policyVio.sIPs == doubleIP:
try:
self.clearedPolicyViolations.remove(policyVio)
except:
# already removed
pass
for doubleIP in self.doubleIPs:
for portscan in self.portscans.values():
if portscan.sIPs == doubleIP:
try:
self.portscans.remove(portscan)
except:
# already removed
pass
for violation in self.clearedPolicyViolations:
self.violationIPs.insert(violation.sIPs)
for portscan in self.portscans:
self.portscanIPs.insert(portscan.sIPs)
def printInfectedHosts(self):
for host in self.doubleIPs:
print "%s might be infected! Warning signals include: portscans and policy violations" % str(host)
for host in self.portscanIPs:
print "%s might be infected! Warning signals include: portscans" % str(host)
for host in self.violationIPs:
print "%s might be infected! Warning signals include: policy violations" % str(host)
class PortscanDetector(RuleGenerator):
"""
PortscanDetector takes a ruleSet and tries to find portscans.
"""
def __init__(self, ruleSet, proto, style, numChecksP, numChecksIP, \
numAnyP, numAnyIP, distanceRange, slashSize, ipsPerSlash, \
numPortsPortscan, numIPsPortscan):
RuleGenerator.__init__(self, ruleSet, proto, style, numChecksP, numChecksIP, \
numAnyP, numAnyIP, distanceRange, \
slashSize, ipsPerSlash)
self.MAX_NUM_PORTS_PORTSCAN = numPortsPortscan
self.MAX_NUM_IPS_PORTSCAN = numIPsPortscan
self.MAX_DIST_RANGE_SCAN = 65535
self.portscanSet = RuleSet()
def detectPortscans(self):
self.generateRules()
self._checkForPortscans()
self.portscanSet.checkTables(self.NUM_CHECKS_P, self.MAX_DIST_RANGE_SCAN,\
self.NUM_ANY_P, self.NUM_CHECKS_IP, \
self.NUM_EXP, self.NUM_ANY_IP, \
self.SLASH_SIZE, self.MIN_IPS_PER_SLASH)
def _checkForPortscans(self):
ruleList = self.ruleSet.values()
for r in ruleList:
if (len(r.dPorts.elements) > self.MAX_NUM_PORTS_PORTSCAN) and \
(len(r.sIPs) == 1) or \
(len(r.dIPs.elements) > self.MAX_NUM_IPS_PORTSCAN and \
len(r.sIPs) == 1) and (len(r.dPorts.elements) == 1)\
and not r.direction == "out":
psr = PortscanRule(r.direction, r.sIPs, r.sPorts, r.dIPs, \
r.dPorts, r.proto, r.interface, "block", \
self.style, r.timeStamp, r.flag)
self.portscanSet.insert(psr)
def removeRulesContainingPortscanners(self, ruleSet):
for r in ruleSet.values():
for pr in self.portscanSet.values():
for e in pr.sIPs.elements.values():
if r.sIPs.values()[0] == e \
or r.dIPs.values()[0] == e:
try:
ruleSet.remove(r)
except KeyError:
#Is already removed
pass
break
def generateRules(self):
self.generateTables()
del self.sortingDict
del self.tablesDict
def printRules(self):
l = len(self.portscanSet)
if l > 0:
if l == 1:
print "%d rule which results from traffic which looks like a portscan\n" % l
else:
print "%d rules which result from traffic which looks like portscans\n" % l
print self.portscanSet
def generateTables(self):
slashSizeBackup = self.SLASH_SIZE
self.SLASH_SIZE = 32
if not self.proto == "icmp":
# 2. sport is the varying part
sPort = True
self._portsVary(sPort)
# 3. dport is the varying part
sPort = False
self._portsVary(sPort)
# 4. dIP is the varying part
sIP = False
self._ipsVary(sIP)
self.SLASH_SIZE = slashSizeBackup
def _ipsVaryIteration(self, ruleSet, sIPvarying):
"""If sIP=True, the checks are made assuming sIP ist the varying part
else, the check are made assuming dIP is the varying part.
The checks work like this:
1. The key for a rule are the ports.
2. If the key (ports) is not in sortingDict, the rule is added to
sortingDict and to tablesDict.
3. If the key (ports) is already added, it is checked if the new rule
is belonging to the rule already added.
Belonging to is defined as that:
3.1 If, by putting the rule together, they would have the same IP
on both sides (source/destination) they don't belong together.
3.2 If the IPs don't belong into the same size slashSize networks,
the rules don't belong together.
4. If the rule can be put together by making the existing IPs into
size slashsize networks, the IPs are made into networks.
5. If a rule has its key already in the dict, but can not be matched
to the existing rule, it is put into the restSet.
sPorts/dPorts exist in key and value, because they could be the same ("any")
as the one already in the hashmap and still contain different elements.
To provide a correct elements list they needs to be extended.
"""
self.sortingDict.clear()
self.tablesDict.clear()
newR = RuleSet()
restSet = RuleSet()
for r in ruleSet:
if sIPvarying:
value = [r.sIPs, r.dIPs, r.sPorts, r.dPorts, r.timeStamp, r.flag]
else:
value = [r.dIPs, r.sIPs, r.sPorts, r.dPorts, r.timeStamp, r.flag]
key1 = copy.deepcopy(r.dPorts)
key2 = copy.deepcopy(r.sPorts)
keys = (r.direction, key1, r.interface, r.action, key2)
if keys not in self.sortingDict:
self.sortingDict[keys] = value
self.tablesDict[keys] = value
else:
tmpIPs1 = IPsMap()
tmpIPs1.extend(value[0])
tmpIPs1.extend(self.tablesDict[keys][0])
tmpIPs2 = IPsMap()
tmpIPs2.extend(value[1])
tmpIPs2.extend(self.tablesDict[keys][1])
sPortsRandom = value[2].isRandomized and\
self.tablesDict[keys][2].isRandomized \
or not value[2].isRandomized and not \
self.tablesDict[keys][2].isRandomized
if not tmpIPs1.ipInBoth(tmpIPs2) and sPortsRandom \
and self.tablesDict[keys][1].checkForJoinedNetwork(value[1], self.SLASH_SIZE):
self.tablesDict[keys][0].extend(value[0])
self.tablesDict[keys][2].extend(value[2])
self.tablesDict[keys][3].extend(value[3])
if self.proto == "tcp":
self.tablesDict[keys][5].extend(value[5])
else:
if sIPvarying:
restSet.insert(Rule(keys[0], value[0], value[2], \
value[1], value[3], self.proto, keys[2], keys[3], self.style, value[4], value[5]))
else:
restSet.insert(Rule(keys[0], value[1], value[2], \
value[0], value[3], self.proto, keys[2], keys[3], self.style, value[4], value[5]))
for key, value in self.tablesDict.iteritems():
if sIPvarying:
newR.insert(Rule(key[0], value[0], value[2], value[1], \
value[3], self.proto, key[2], key[3], self.style, value[4], value[5]))
else:
newR.insert(Rule(key[0], value[1], value[2], value[0], \
value[3], self.proto, key[2], key[3], self.style, value[4], value[5]))
return newR, restSet