def test_depends_ab_ba(self):
rs = RuleSet()
rs.addDep("a", "b")
rs.addDep("b", "a")
self.assertTrue(rs.isCoherent(), "rs.isCoherent failed")
def _portsVary(self, sPortVarying):
"""If key is not in sortingDict put key-value pairs in sortingDict and
tablesDict,
else merge value to existing value.
sIPs/dIPs exist in key and value, because they could be the same ("any")
as the one already in the hashmap and still contain different elements.
To provide a correct elements list they needs to be extended.
"""
self.sortingDict.clear()
self.tablesDict.clear()
newR = RuleSet()
for r in self.ruleSet:
if sPortVarying:
keys = (r.direction, r.sIPs, r.dIPs, r.dPorts, r.interface, r.action)
value = (r.sPorts, r.timeStamp, r.flag, r.sIPs, r.dIPs)
else:
keys = (r.direction, r.sIPs, r.sPorts, r.dIPs, r.interface, r.action)
value = (r.dPorts, r.timeStamp, r.flag, r.sIPs, r.dIPs)
self._sortIntoDicts(keys, value)
for key, value in self.tablesDict.iteritems():
if sPortVarying:
r = Rule(key[0], value[3], value[0], value[4], key[3], self.proto, key[4], key[5], self.style, value[1], value[2])
else:
r = Rule(key[0], value[3], key[2], value[4], value[0], self.proto, key[4], key[5], self.style, value[1], value[2])
newR.insert(r)
self.ruleSet = newR
del newR
def __init__(self):
ruleset_id = 'ga_ws_pub_sdts'
ruleset_name = 'GA Web Service Publication Standards'
passed = True
rules_results = []
# make a Graph from the string or file
#
# Run all the rules
#
service_pass = r'http://services.ga.gov.au/gis/services/NEXIS_Building_Exposure/MapServer/WFSServer?request=GetCapabilities'
service_fail = r'http://www.ga.gov.au/gis/services/topography/National_Ferry_Terminals/MapServer/WFSServer?request=GetCapabilities'
#rules_results.append(GaTitle(service_pass).get_result())
rules_results.append(WebServiceTitle(service_pass).get_result())
# calculate if RuleSet passed
for rule in rules_results:
if not rule['passed']:
passed = False
#
# Call the base RuleSet constructor
#
RuleSet.__init__(self,
ruleset_id,
ruleset_name,
'*****@*****.**',
rules_results,
passed)
def test_ruleset(self):
with open('people.json') as f1:
people_data = json.load(f1)
results = list()
for person in people_data['people']:
with open('rules.json') as f2:
rules_data = json.load(f2)
rs = RuleSet(rules_data)
results.append(rs.evaluate(person))
for result in results:
if result['name'] == 'FRED':
self.assertEqual(result['status'], 'COOL')
elif result['name'] == 'GINGER':
self.assertEqual(result['status'], 'OLD')
elif result['name'] == 'SID':
self.assertEqual(result['status'], 'ANCIENT')
elif result['name'] == 'NANCY':
self.assertEqual(result['status'], 'VERY OLD')
elif result['name'] == 'JOE':
self.assertEqual(result['status'], 'YOUNG')
elif result['name'] == 'FLO':
self.assertEqual(result['status'], 'VERY OLD')
elif result['name'] == 'REG':
self.assertEqual(result['status'], 'WEIRD')
elif result['name'] == 'BRIAN':
self.assertEqual(result['status'], 'DEAD')
elif result['name'] == 'ELSIE':
self.assertEqual(result['status'], 'OLD HIPPY')
self.assertEqual(result['nickname'], 'Hippy')
def __init__(self):
ruleset_id = 'ga_ws_pub_sdts'
ruleset_name = 'GA Web Service Publication Standards'
passed = True
rules_results = []
# make a Graph from the string or file
#
# Run all the rules
#
service_pass = r'http://services.ga.gov.au/gis/services/NEXIS_Building_Exposure/MapServer/WFSServer?request=GetCapabilities'
service_fail = r'http://www.ga.gov.au/gis/services/topography/National_Ferry_Terminals/MapServer/WFSServer?request=GetCapabilities'
#rules_results.append(GaTitle(service_pass).get_result())
rules_results.append(WebServiceTitle(service_pass).get_result())
# calculate if RuleSet passed
for rule in rules_results:
if not rule['passed']:
passed = False
#
# Call the base RuleSet constructor
#
RuleSet.__init__(self, ruleset_id, ruleset_name,
'*****@*****.**', rules_results, passed)
def filterWithExistingRules(existRulesFile, style, tcpRules, udpRules, icmpRules):
existingRuleset = RuleSet()
existingRuleset.initializeFromExistingRuleset(existRulesFile, "pass", style)
print "Filtering Rules with existing ruleset"
tcpRules = testRules(tcpRules, existingRuleset)
udpRules = testRules(udpRules, existingRuleset)
icmpRules = testRules(icmpRules, existingRuleset)
return tcpRules, udpRules, icmpRules
def __init__(self, tcpPortscans, udpPortscans, policyViolations, suspPolVio):
self.tcpPortscans = tcpPortscans
self.udpPortscans = udpPortscans
self.policyViolations = policyViolations
self.portscans = RuleSet()
self.doubleIPs = IPsMap()
self.portscanIPs = IPsMap()
self.violationIPs = IPsMap()
self.clearedPolicyViolations = RuleSet()
self.suspPolVio = suspPolVio.split(":")
def __init__(self, id, ranges, rules, depth, partitions, manual_partition):
self.id = id
self.partitions = list(partitions or [])
self.manual_partition = manual_partition
self.ranges = ranges
self.rules = RuleSet(rules) if isinstance(rules, list) else rules
self.depth = depth
self.children = []
self.action = None
self.pushup_rules = None
self.num_rules = len(self.rules)
def _filterRules(self, rules, checkList, proto, direction):
filteredRules = RuleSet()
for s in checkList:
if s[2] == proto or s[2] == "any":
for r in rules.values():
for d in direction:
if r.direction == d \
and (s[0] in r.dIPs or s[0] == "any") \
and (s[1] in r.dPorts or s[1] == "any"):
rules.remove(r)
filteredRules.insert(r)
return filteredRules
def test_ab_bc_toggle(self):
rs = RuleSet()
rs.addDep("a", "b")
rs.addDep("b", "c")
self.assertTrue(rs.isCoherent(), "rs.isCoherent failed")
opts = Options(rs)
opts.toggle("c")
self.assertSetEqual(
opts.selection(),
set(["c"]),
"toggle expected (c) got %s" % opts.selection(),
)
def __init__(self, tcpRules, udpRules, icmpRules, innerServicesList, outerServicesList, \
forbiddenInnerServicesList, restrictedNetworks):
self.tcpRules = tcpRules
self.udpRules = udpRules
self.icmpRules = icmpRules
self.innerServicesList = self._parseServiceList(innerServicesList)
self.outerServicesList = self._parseServiceList(outerServicesList)
self.forbiddenInnerServicesList = self._parseServiceList(forbiddenInnerServicesList)
self.restrictedNetworksList = self._parseNetworkList(restrictedNetworks)
self.filteredRules = RuleSet()
self.innerServicesFiltered = RuleSet()
self.outerServicesFiltered = RuleSet()
self.forbiddenInnerServicesFiltered = RuleSet()
self.restrictedNetworksFiltered = RuleSet()
def __init__(
self,
rules,
leaf_threshold,
refinements={
"node_merging": False,
"rule_overlay": False,
"region_compaction": False,
"rule_pushup": False,
"equi_dense": False
}):
# hyperparameters
self.leaf_threshold = leaf_threshold
self.refinements = refinements
self.rules = RuleSet(rules) if isinstance(rules, list) else rules
self.root = self.create_node(
0, [0, 2**32, 0, 2**32, 0, 2**16, 0, 2**16, 0, 2**8], rules, 1,
None, None)
if (self.refinements["region_compaction"]):
self.refinement_region_compaction(self.root)
self.current_node = self.root
self.nodes_to_cut = [self.root]
self.depth = 1
self.node_count = 1
def testRules(connections, rules, verbose=False):
filtertConns = RuleSet()
for c in connections:
ruled = False
for r in rules:
if c.interface == r.interface and c.proto == r.proto:
if _testPorts(c, r):
if _testIPs(c, r):
ruled = True
break
if not ruled:
if verbose:
print "Was not matched"
print c
filtertConns.insert(c)
return filtertConns
def _checkTcpForConns(self):
"""Checks tcp-set for connections."""
print "Checking for Connections"
tcpRuleSetConnsOnly = RuleSet()
if self.p.linktype == self.PFLOG_DUMP:
tcpRuleSetConnsOnly = self.tcpRuleSet
else:
for r in self.tcpRuleSet:
connection = False
for r2 in self.tcpRuleSet:
if (r == r2 and (r.flag[0]=="SYN") and (r2.flag[0]=="FIN")):
connection = True
break
if connection:
tcpRuleSetConnsOnly.insert(r)
self.tcpRuleSet = tcpRuleSetConnsOnly
def main():
'''Main entry point for module.'''
config = load_config('test.conf')
myrules = RuleSet()
myrules.load_rules('syntax.yml')
myresult = []
for rule in myrules.rules:
result = rule.apply(config)
for element in result:
if element.result == False:
print "Rule evaluation failed:\n Cfgline: '{}'\n Rule: {}".format(
element.cfgline.text,
element.rule)
myresult.append(result)
return myresult
def test_exclusive_ab(self):
rs = RuleSet()
rs.addDep("a", "b")
rs.addConflict("a", "b")
self.assertFalse(rs.isCoherent(), "rs.isCoherent failed")
def __init__(self, pcapFile, action, style, direction, maxNumPortsAny, \
connectionChecking, portscanChecking, printElements, \
innerNetworks=None, ddosDetection=True):
self.p = pcapFile
self.direction = direction
self.action = action
self.style = style
self.interface = "None"
self.tcpRuleSet = RuleSet()
self.udpRuleSet = RuleSet()
self.icmpRuleSet = RuleSet()
self.tcpRuleList = []
self.udpRuleList = []
self.icmpRuleList = []
self.MAX_NUM_PORTS_ANY = maxNumPortsAny
self.connectionChecking = connectionChecking
self.ddosDetection = ddosDetection
self.saveElements = portscanChecking or printElements
self.PFLOG_DUMP = 117
self.innerNetworks = innerNetworks
def __init__(self, ruleSet, proto, style, numChecksP, numChecksIP, \
numAnyP, numAnyIP, distanceRange, slashSize, ipsPerSlash, \
numPortsPortscan, numIPsPortscan):
RuleGenerator.__init__(self, ruleSet, proto, style, numChecksP, numChecksIP, \
numAnyP, numAnyIP, distanceRange, \
slashSize, ipsPerSlash)
self.MAX_NUM_PORTS_PORTSCAN = numPortsPortscan
self.MAX_NUM_IPS_PORTSCAN = numIPsPortscan
self.MAX_DIST_RANGE_SCAN = 65535
self.portscanSet = RuleSet()
def createRuleSet(self, name):
logging.info("Creating RuleSet: " + name)
if (name is None or name == ""):
raise ValueError("RuleSet name cannot be none or empty")
rs = RuleSet(name)
self.ruleSets[name] = rs
logging.info("Created RuleSet: " + name)
return rs
def test_ab_ac(self):
rs = RuleSet()
rs.addDep("a", "b")
rs.addDep("a", "c")
rs.addConflict("b", "d")
rs.addConflict("b", "e")
self.assertTrue(rs.isCoherent(), "rs.isCoherent failed")
opts = Options(rs)
opts.toggle("d")
opts.toggle("e")
opts.toggle("a")
self.assertSetEqual(
opts.selection(),
set(["a", "c", "b"]),
"toggle expected (a, c, b) got %s" % opts.selection(),
)
def detectViolations(self):
if self.innerServicesList != None:
self.innerServicesFiltered.extend(self._filterRules(self.tcpRules, self.innerServicesList, "tcp", ["in",]))
self.innerServicesFiltered.extend(self._filterRules(self.udpRules, self.innerServicesList, "udp", ["in",]))
self.innerServicesFiltered.extend(self._filterRules(self.icmpRules, self.innerServicesList, "icmp", ["in",]))
if self.outerServicesList != None:
self.outerServicesFiltered.extend(self._filterRules(self.tcpRules, self.outerServicesList, "tcp", ["out",]))
self.outerServicesFiltered.extend(self._filterRules(self.udpRules, self.outerServicesList, "udp", ["out",]))
self.outerServicesFiltered.extend(self._filterRules(self.icmpRules, self.outerServicesList, "icmp", ["out",]))
if self.forbiddenInnerServicesList != None:
self.forbiddenInnerServicesFiltered.extend(self._filterRules(self.tcpRules, self.forbiddenInnerServicesList, "tcp", ["none-inner", "in"]))
self.forbiddenInnerServicesFiltered.extend(self._filterRules(self.udpRules, self.forbiddenInnerServicesList, "udp", ["none-inner", "in"]))
self.forbiddenInnerServicesFiltered.extend(self._filterRules(self.icmpRules, self.forbiddenInnerServicesList, "icmp", ["none-inner", "in"]))
if self.restrictedNetworksList != None:
self.restrictedNetworksFiltered.extend(self._filterRules(self.tcpRules, self.restrictedNetworksList, "tcp", ["none-inner", "in"]))
self.restrictedNetworksFiltered.extend(self._filterRules(self.udpRules, self.restrictedNetworksList, "udp", ["none-inner", "in"]))
self.restrictedNetworksFiltered.extend(self._filterRules(self.icmpRules, self.restrictedNetworksList, "icmp", ["none-inner", "in"]))
mergedViolations = RuleSet()
mergedViolations.extend(self.innerServicesFiltered)
mergedViolations.extend(self.outerServicesFiltered)
mergedViolations.extend(self.forbiddenInnerServicesFiltered)
mergedViolations.extend(self.restrictedNetworksFiltered)
return mergedViolations
class Node:
def __init__(self, id, ranges, rules, depth, partitions, manual_partition):
self.id = id
self.partitions = list(partitions or [])
self.manual_partition = manual_partition
self.ranges = ranges
self.rules = RuleSet(rules) if isinstance(rules, list) else rules
self.depth = depth
self.children = []
self.action = None
self.pushup_rules = None
self.num_rules = len(self.rules)
def is_partition(self):
"""Returns if node was partitioned."""
if not self.action:
return False
elif self.action[0] == "partition":
return True
elif self.action[0] == "cut":
return False
else:
return False
def match(self, packet):
if self.is_partition():
matches = []
for c in self.children:
match = c.match(packet)
if match:
matches.append(match)
if matches:
matches.sort(key=lambda r: self.rules.index(r))
return matches[0]
return None
elif self.children:
for n in self.children:
if n.contains(packet):
return n.match(packet)
return None
else:
for r in self.rules:
if r.matches(packet):
return r
def is_intersect_multi_dimension(self, ranges):
for i in range(5):
if ranges[i*2] >= self.ranges[i*2+1] or \
ranges[i*2+1] <= self.ranges[i*2]:
return False
return True
def contains(self, packet):
assert len(packet) == 5, packet
return self.is_intersect_multi_dimension([
packet[0] + 0, # src ip
packet[0] + 1,
packet[1] + 0, # dst ip
packet[1] + 1,
packet[2] + 0, # src port
packet[2] + 1,
packet[3] + 0, # dst port
packet[3] + 1,
packet[4] + 0, # protocol
packet[4] + 1
])
def is_useless(self):
if not self.children:
return False
return max(len(c.rules) for c in self.children) == len(self.rules)
def pruned_rules(self):
return self.rules.prune(self.ranges)
def get_state(self):
state = []
state.extend(to_bits(self.ranges[0], 32))
state.extend(to_bits(self.ranges[1] - 1, 32))
state.extend(to_bits(self.ranges[2], 32))
state.extend(to_bits(self.ranges[3] - 1, 32))
assert len(state) == 128, len(state)
state.extend(to_bits(self.ranges[4], 16))
state.extend(to_bits(self.ranges[5] - 1, 16))
state.extend(to_bits(self.ranges[6], 16))
state.extend(to_bits(self.ranges[7] - 1, 16))
assert len(state) == 192, len(state)
state.extend(to_bits(self.ranges[8], 8))
state.extend(to_bits(self.ranges[9] - 1, 8))
assert len(state) == 208, len(state)
if self.manual_partition is None:
# 0, 6 -> 0-64%
# 6, 7 -> 64-100%
partition_state = [
0,
7, # [>=min, <max) -- 0%, 2%, 4%, 8%, 16%, 32%, 64%, 100%
0,
7,
0,
7,
0,
7,
0,
7,
]
for (smaller, part_dim, part_size) in self.partitions:
if smaller:
partition_state[part_dim * 2 + 1] = min(
partition_state[part_dim * 2 + 1], part_size + 1)
else:
partition_state[part_dim * 2] = max(
partition_state[part_dim * 2], part_size + 1)
state.extend(onehot_encode(partition_state, 7))
else:
partition_state = [0] * 70
partition_state[self.manual_partition] = 1
state.extend(partition_state)
state.append(self.num_rules)
return np.array(state)
def __str__(self):
result = "ID:%d\tAction:%s\tDepth:%d\tRange:\t%s\nChildren: " % (
self.id, str(self.action), self.depth, str(self.ranges))
for child in self.children:
result += str(child.id) + " "
result += "\nRules:\n"
for rule in self.rules:
result += str(rule) + "\n"
if self.pushup_rules != None:
result += "Pushup Rules:\n"
for rule in self.pushup_rules:
result += str(rule) + "\n"
return result
class PcapParser():
"""
PcapParser parses a pcap-file with dpkt.
Every ip-paket is put into either the tcp or udp or icmp ruleset.
Rulesets are sets, they contain every item only once.
Concerning tcp: only full connections are kept (there must be a syn
and a fin paket).
Non-ip-pakets are not concerned.
"""
def __init__(self, pcapFile, action, style, direction, maxNumPortsAny, \
connectionChecking, portscanChecking, printElements, \
innerNetworks=None, ddosDetection=True):
self.p = pcapFile
self.direction = direction
self.action = action
self.style = style
self.interface = "None"
self.tcpRuleSet = RuleSet()
self.udpRuleSet = RuleSet()
self.icmpRuleSet = RuleSet()
self.tcpRuleList = []
self.udpRuleList = []
self.icmpRuleList = []
self.MAX_NUM_PORTS_ANY = maxNumPortsAny
self.connectionChecking = connectionChecking
self.ddosDetection = ddosDetection
self.saveElements = portscanChecking or printElements
self.PFLOG_DUMP = 117
self.innerNetworks = innerNetworks
def parsePcapFile(self):
"""Parse a pcap file"""
for ts, buf in self.p:
self._packetHandler(buf, ts)
if self.connectionChecking:
self._checkTcpForConns()
return self.tcpRuleSet, self.udpRuleSet, self.icmpRuleSet, \
self.tcpRuleList, self.udpRuleList, self.icmpRuleList
def _packetHandler(self, buf, ts):
"""Parse a pcap buffer"""
SINGLE_IP_SLASHSIZE = 32
try:
if (self.p.datalink() == self.PFLOG_DUMP):
#pkt = dpkt.pflog.Pflog(buf)
pkt = Pflog(buf)
self.interface = pkt.interfaceName
self.direction = "in" if pkt.direction == 1 else "out"
subpkt = pkt.data
try:
subpkt = dpkt.ip.IP(subpkt)
except:
#skip non IP packets
return
else:
pkt = dpkt.ethernet.Ethernet(buf)
subpkt = pkt.data
if not isinstance(subpkt, dpkt.ip.IP):
#skip non IP packets
return
proto = subpkt.p
shost = socket.inet_ntoa(subpkt.src)
dhost = socket.inet_ntoa(subpkt.dst)
shostInner = False
dhostInner = False
if self.innerNetworks != None:
for n in self.innerNetworks:
slashSize = str(n).split("/")[1]
if IPv4Network(str(shost)+"/"+str(slashSize)) == n:
shostInner = True
if IPv4Network(dhost+"/"+slashSize) == n:
dhostInner = True
if shostInner and dhostInner:
self.direction = "none-inner"
elif shostInner and not dhostInner:
self.direction = "out"
elif not shostInner and dhostInner:
self.direction = "in"
else:
self.direction = "none-outer"
except dpkt.Error:
#skip non-ethernet packages
return
try:
if proto == socket.IPPROTO_TCP:
try:
tcp = subpkt.data
flag = tcp.flags
if self.connectionChecking:
rightFlag = ((flag == dpkt.tcp.TH_SYN) \
or (flag & dpkt.tcp.TH_FIN != 0))
else:
rightFlag = (flag == dpkt.tcp.TH_SYN)
dport = tcp.dport
sport = tcp.sport
sIP = IPsMap(self.saveElements)
sIP.insert(IPv4Address(shost))
sPort = PortsMap(self.MAX_NUM_PORTS_ANY, self.saveElements)
sPort.insert(Port(sport, True))
dIP = IPsMap(self.saveElements)
dIP.insert(IPv4Address(dhost))
dPort = PortsMap(self.MAX_NUM_PORTS_ANY, self.saveElements)
dPort.insert(Port(dport, False))
if flag == dpkt.tcp.TH_SYN:
flag = "SYN"
else:
flag = "FIN"
r = Rule(self.direction, sIP, sPort, dIP, dPort, "tcp", \
self.interface, self.action, self.style, [ts, ], [flag, ])
if self.p.datalink() == self.PFLOG_DUMP or rightFlag:
self.tcpRuleSet.insert(r)
if self.ddosDetection:
self.tcpRuleList.append((IPv4Address(shost), ts))
except AttributeError:
#skip broken packages
return
elif proto == socket.IPPROTO_UDP:
udp = subpkt.data
dport = udp.dport
sport = udp.sport
sIP = IPsMap(self.saveElements)
sIP.insert(IPv4Address(shost))
sPort = PortsMap(self.MAX_NUM_PORTS_ANY, self.saveElements)
sPort.insert(Port(sport, True))
dIP = IPsMap(self.saveElements)
dIP.insert(IPv4Address(dhost))
dPort = PortsMap(self.MAX_NUM_PORTS_ANY, self.saveElements)
dPort.insert(Port(dport, False))
r = Rule(self.direction, sIP, sPort, dIP, dPort, "udp", \
self.interface, self.action, self.style, [ts, ])
self.udpRuleSet.insert(r)
if self.ddosDetection:
self.udpRuleList.append((IPv4Address(shost), ts))
elif proto == socket.IPPROTO_ICMP:
sIP = IPsMap(self.saveElements)
sIP.insert(IPv4Address(shost))
sPort = PortsMap(self.MAX_NUM_PORTS_ANY)
sPort.insert(Port(-1, True))
dIP = IPsMap(self.saveElements)
dIP.insert(IPv4Address(dhost))
dPort = PortsMap(self.MAX_NUM_PORTS_ANY)
dPort.insert(Port(-1, False))
r = Rule(self.direction, sIP, sPort, dIP, dPort, "icmp", \
self.interface, self.action, self.style, [ts, ])
self.icmpRuleSet.insert(r)
if self.ddosDetection:
self.icmpRuleList.append((IPv4Address(shost), ts))
except dpkt.Error:
return
def _checkTcpForConns(self):
"""Checks tcp-set for connections."""
print "Checking for Connections"
tcpRuleSetConnsOnly = RuleSet()
if self.p.linktype == self.PFLOG_DUMP:
tcpRuleSetConnsOnly = self.tcpRuleSet
else:
for r in self.tcpRuleSet:
connection = False
for r2 in self.tcpRuleSet:
if (r == r2 and (r.flag[0]=="SYN") and (r2.flag[0]=="FIN")):
connection = True
break
if connection:
tcpRuleSetConnsOnly.insert(r)
self.tcpRuleSet = tcpRuleSetConnsOnly
class InfectedHostsDetector:
"""
InfectedHostsDetector takes found portscans and policy violations
and tries to find indected hosts.
"""
def __init__(self, tcpPortscans, udpPortscans, policyViolations, suspPolVio):
self.tcpPortscans = tcpPortscans
self.udpPortscans = udpPortscans
self.policyViolations = policyViolations
self.portscans = RuleSet()
self.doubleIPs = IPsMap()
self.portscanIPs = IPsMap()
self.violationIPs = IPsMap()
self.clearedPolicyViolations = RuleSet()
self.suspPolVio = suspPolVio.split(":")
def detectInfectedHosts(self):
for ps in self.tcpPortscans.portscanSet:
if ps.direction == "out" or ps.direction == "none-inner":
self.portscans.insert(ps)
for ps in self.udpPortscans.portscanSet:
if ps.direction == "out" or ps.direction == "none-inner":
self.portscans.insert(ps)
for pv in self.policyViolations:
if pv.direction == "out" or pv.direction == "none-inner":
if str(pv.dPorts.values()[0].portNumber) in self.suspPolVio:
self.clearedPolicyViolations.insert(pv)
for portscan in self.portscans:
for violation in self.clearedPolicyViolations:
if portscan.sIPs == violation.sIPs:
self.doubleIPs.insert(portscan.sIPs)
for doubleIP in self.doubleIPs:
for policyVio in self.clearedPolicyViolations.values():
if policyVio.sIPs == doubleIP:
try:
self.clearedPolicyViolations.remove(policyVio)
except:
# already removed
pass
for doubleIP in self.doubleIPs:
for portscan in self.portscans.values():
if portscan.sIPs == doubleIP:
try:
self.portscans.remove(portscan)
except:
# already removed
pass
for violation in self.clearedPolicyViolations:
self.violationIPs.insert(violation.sIPs)
for portscan in self.portscans:
self.portscanIPs.insert(portscan.sIPs)
def printInfectedHosts(self):
for host in self.doubleIPs:
print "%s might be infected! Warning signals include: portscans and policy violations" % str(host)
for host in self.portscanIPs:
print "%s might be infected! Warning signals include: portscans" % str(host)
for host in self.violationIPs:
print "%s might be infected! Warning signals include: policy violations" % str(host)
def test_deep_deps(self):
rs = RuleSet()
rs.addDep("a", "b")
rs.addDep("b", "c")
rs.addDep("c", "d")
rs.addDep("d", "e")
rs.addDep("a", "f")
rs.addConflict("e", "f")
self.assertFalse(rs.isCoherent(), "rs.isCoherent failed")
class PortscanDetector(RuleGenerator):
"""
PortscanDetector takes a ruleSet and tries to find portscans.
"""
def __init__(self, ruleSet, proto, style, numChecksP, numChecksIP, \
numAnyP, numAnyIP, distanceRange, slashSize, ipsPerSlash, \
numPortsPortscan, numIPsPortscan):
RuleGenerator.__init__(self, ruleSet, proto, style, numChecksP, numChecksIP, \
numAnyP, numAnyIP, distanceRange, \
slashSize, ipsPerSlash)
self.MAX_NUM_PORTS_PORTSCAN = numPortsPortscan
self.MAX_NUM_IPS_PORTSCAN = numIPsPortscan
self.MAX_DIST_RANGE_SCAN = 65535
self.portscanSet = RuleSet()
def detectPortscans(self):
self.generateRules()
self._checkForPortscans()
self.portscanSet.checkTables(self.NUM_CHECKS_P, self.MAX_DIST_RANGE_SCAN,\
self.NUM_ANY_P, self.NUM_CHECKS_IP, \
self.NUM_EXP, self.NUM_ANY_IP, \
self.SLASH_SIZE, self.MIN_IPS_PER_SLASH)
def _checkForPortscans(self):
ruleList = self.ruleSet.values()
for r in ruleList:
if (len(r.dPorts.elements) > self.MAX_NUM_PORTS_PORTSCAN) and \
(len(r.sIPs) == 1) or \
(len(r.dIPs.elements) > self.MAX_NUM_IPS_PORTSCAN and \
len(r.sIPs) == 1) and (len(r.dPorts.elements) == 1)\
and not r.direction == "out":
psr = PortscanRule(r.direction, r.sIPs, r.sPorts, r.dIPs, \
r.dPorts, r.proto, r.interface, "block", \
self.style, r.timeStamp, r.flag)
self.portscanSet.insert(psr)
def removeRulesContainingPortscanners(self, ruleSet):
for r in ruleSet.values():
for pr in self.portscanSet.values():
for e in pr.sIPs.elements.values():
if r.sIPs.values()[0] == e \
or r.dIPs.values()[0] == e:
try:
ruleSet.remove(r)
except KeyError:
#Is already removed
pass
break
def generateRules(self):
self.generateTables()
del self.sortingDict
del self.tablesDict
def printRules(self):
l = len(self.portscanSet)
if l > 0:
if l == 1:
print "%d rule which results from traffic which looks like a portscan\n" % l
else:
print "%d rules which result from traffic which looks like portscans\n" % l
print self.portscanSet
def generateTables(self):
slashSizeBackup = self.SLASH_SIZE
self.SLASH_SIZE = 32
if not self.proto == "icmp":
# 2. sport is the varying part
sPort = True
self._portsVary(sPort)
# 3. dport is the varying part
sPort = False
self._portsVary(sPort)
# 4. dIP is the varying part
sIP = False
self._ipsVary(sIP)
self.SLASH_SIZE = slashSizeBackup
class ViolationDetector():
"""
ViolationDetector takes a ruleSet and tries to find policy violations.
"""
def __init__(self, tcpRules, udpRules, icmpRules, innerServicesList, outerServicesList, \
forbiddenInnerServicesList, restrictedNetworks):
self.tcpRules = tcpRules
self.udpRules = udpRules
self.icmpRules = icmpRules
self.innerServicesList = self._parseServiceList(innerServicesList)
self.outerServicesList = self._parseServiceList(outerServicesList)
self.forbiddenInnerServicesList = self._parseServiceList(forbiddenInnerServicesList)
self.restrictedNetworksList = self._parseNetworkList(restrictedNetworks)
self.filteredRules = RuleSet()
self.innerServicesFiltered = RuleSet()
self.outerServicesFiltered = RuleSet()
self.forbiddenInnerServicesFiltered = RuleSet()
self.restrictedNetworksFiltered = RuleSet()
def _parseNetworkList(self, networkList):
if networkList != "None":
networkList = networkList.split(",")
tmpList = []
for n in networkList:
tmp = n.split("/")
tmp[0] = IPv4Network(tmp[0]+"/"+tmp[1])
tmp[1] = "any"
tmp.append("any")
tmpList.append(tmp)
return tmpList
return None
def _parseServiceList(self, serviceList):
if serviceList != "None":
servicesList = serviceList.split(",")
tmpList = []
for s in servicesList:
tmp = s.split(":")
tmp[0] = ("any" if tmp[0]=="any" else IPv4Address(tmp[0]))
tmp[1] = ("any" if tmp[1]=="any" else Port(int(tmp[1])))
tmpList.append(tmp)
return tmpList
return None
def printViolations(self):
if len(self.innerServicesFiltered) > 0:
print "Violations detected for services, which should only be accessed by ips from the inner networks:"
print self.innerServicesFiltered
if len(self.outerServicesFiltered) > 0:
print "Violations detected for services, which should not be accessed by ips from the inner networks:"
print self.outerServicesFiltered
if len(self.forbiddenInnerServicesFiltered) > 0:
print "Violations detected for services, which should not be used in the inner networks:"
print self.forbiddenInnerServicesFiltered
if len(self.restrictedNetworksFiltered) > 0:
print "Violations detected for networks, which should have no incoming traffic:"
print self.restrictedNetworksFiltered
def detectViolations(self):
if self.innerServicesList != None:
self.innerServicesFiltered.extend(self._filterRules(self.tcpRules, self.innerServicesList, "tcp", ["in",]))
self.innerServicesFiltered.extend(self._filterRules(self.udpRules, self.innerServicesList, "udp", ["in",]))
self.innerServicesFiltered.extend(self._filterRules(self.icmpRules, self.innerServicesList, "icmp", ["in",]))
if self.outerServicesList != None:
self.outerServicesFiltered.extend(self._filterRules(self.tcpRules, self.outerServicesList, "tcp", ["out",]))
self.outerServicesFiltered.extend(self._filterRules(self.udpRules, self.outerServicesList, "udp", ["out",]))
self.outerServicesFiltered.extend(self._filterRules(self.icmpRules, self.outerServicesList, "icmp", ["out",]))
if self.forbiddenInnerServicesList != None:
self.forbiddenInnerServicesFiltered.extend(self._filterRules(self.tcpRules, self.forbiddenInnerServicesList, "tcp", ["none-inner", "in"]))
self.forbiddenInnerServicesFiltered.extend(self._filterRules(self.udpRules, self.forbiddenInnerServicesList, "udp", ["none-inner", "in"]))
self.forbiddenInnerServicesFiltered.extend(self._filterRules(self.icmpRules, self.forbiddenInnerServicesList, "icmp", ["none-inner", "in"]))
if self.restrictedNetworksList != None:
self.restrictedNetworksFiltered.extend(self._filterRules(self.tcpRules, self.restrictedNetworksList, "tcp", ["none-inner", "in"]))
self.restrictedNetworksFiltered.extend(self._filterRules(self.udpRules, self.restrictedNetworksList, "udp", ["none-inner", "in"]))
self.restrictedNetworksFiltered.extend(self._filterRules(self.icmpRules, self.restrictedNetworksList, "icmp", ["none-inner", "in"]))
mergedViolations = RuleSet()
mergedViolations.extend(self.innerServicesFiltered)
mergedViolations.extend(self.outerServicesFiltered)
mergedViolations.extend(self.forbiddenInnerServicesFiltered)
mergedViolations.extend(self.restrictedNetworksFiltered)
return mergedViolations
def _filterRules(self, rules, checkList, proto, direction):
filteredRules = RuleSet()
for s in checkList:
if s[2] == proto or s[2] == "any":
for r in rules.values():
for d in direction:
if r.direction == d \
and (s[0] in r.dIPs or s[0] == "any") \
and (s[1] in r.dPorts or s[1] == "any"):
rules.remove(r)
filteredRules.insert(r)
return filteredRules
def _ipsVaryIteration(self, ruleSet, sIPvarying):
"""If sIP=True, the checks are made assuming sIP ist the varying part
else, the check are made assuming dIP is the varying part.
The checks work like this:
1. The key for a rule are the ports.
2. If the key (ports) is not in sortingDict, the rule is added to
sortingDict and to tablesDict.
3. If the key (ports) is already added, it is checked if the new rule
is belonging to the rule already added.
Belonging to is defined as that:
3.1 If, by putting the rule together, they would have the same IP
on both sides (source/destination) they don't belong together.
3.2 If the IPs don't belong into the same size slashSize networks,
the rules don't belong together.
4. If the rule can be put together by making the existing IPs into
size slashsize networks, the IPs are made into networks.
5. If a rule has its key already in the dict, but can not be matched
to the existing rule, it is put into the restSet.
sPorts/dPorts exist in key and value, because they could be the same ("any")
as the one already in the hashmap and still contain different elements.
To provide a correct elements list they needs to be extended.
"""
self.sortingDict.clear()
self.tablesDict.clear()
newR = RuleSet()
restSet = RuleSet()
for r in ruleSet:
if sIPvarying:
value = [r.sIPs, r.dIPs, r.sPorts, r.dPorts, r.timeStamp, r.flag]
else:
value = [r.dIPs, r.sIPs, r.sPorts, r.dPorts, r.timeStamp, r.flag]
key1 = copy.deepcopy(r.dPorts)
key2 = copy.deepcopy(r.sPorts)
keys = (r.direction, key1, r.interface, r.action, key2)
if keys not in self.sortingDict:
self.sortingDict[keys] = value
self.tablesDict[keys] = value
else:
tmpIPs1 = IPsMap()
tmpIPs1.extend(value[0])
tmpIPs1.extend(self.tablesDict[keys][0])
tmpIPs2 = IPsMap()
tmpIPs2.extend(value[1])
tmpIPs2.extend(self.tablesDict[keys][1])
sPortsRandom = value[2].isRandomized and\
self.tablesDict[keys][2].isRandomized \
or not value[2].isRandomized and not \
self.tablesDict[keys][2].isRandomized
if not tmpIPs1.ipInBoth(tmpIPs2) and sPortsRandom \
and self.tablesDict[keys][1].checkForJoinedNetwork(value[1], self.SLASH_SIZE):
self.tablesDict[keys][0].extend(value[0])
self.tablesDict[keys][2].extend(value[2])
self.tablesDict[keys][3].extend(value[3])
if self.proto == "tcp":
self.tablesDict[keys][5].extend(value[5])
else:
if sIPvarying:
restSet.insert(Rule(keys[0], value[0], value[2], \
value[1], value[3], self.proto, keys[2], keys[3], self.style, value[4], value[5]))
else:
restSet.insert(Rule(keys[0], value[1], value[2], \
value[0], value[3], self.proto, keys[2], keys[3], self.style, value[4], value[5]))
for key, value in self.tablesDict.iteritems():
if sIPvarying:
newR.insert(Rule(key[0], value[0], value[2], value[1], \
value[3], self.proto, key[2], key[3], self.style, value[4], value[5]))
else:
newR.insert(Rule(key[0], value[1], value[2], value[0], \
value[3], self.proto, key[2], key[3], self.style, value[4], value[5]))
return newR, restSet
def test_exclusive_ab_bc_ca_de(self):
rs = RuleSet()
rs.addDep("a", "b")
rs.addDep("b", "c")
rs.addDep("c", "a")
rs.addDep("d", "e")
rs.addConflict("c", "e")
self.assertTrue(rs.isCoherent(), "rs.isCoherent failed")
opts = Options(rs)
opts.toggle("a")
self.assertSetEqual(
opts.selection(),
set(["a", "c", "b"]),
"toggle expected (a, c, b) got %s" % opts.selection(),
)
rs.addDep("f", "f")
opts.toggle("f")
self.assertSetEqual(
opts.selection(),
set(["a", "c", "b", "f"]),
"toggle expected (a, c, b, f) got %s" % opts.selection(),
)
opts.toggle("e")
self.assertSetEqual(
opts.selection(),
set(["e", "f"]),
"toggle expected (e, f) got %s" % opts.selection(),
)
opts.toggle("b")
self.assertSetEqual(
opts.selection(),
set(["a", "c", "b", "f"]),
"toggle expected (a, c, b, f) got %s" % opts.selection(),
)
rs.addDep("b", "g")
opts.toggle("g")
opts.toggle("b")
self.assertSetEqual(
opts.selection(),
set(["g", "f"]),
"toggle expected (g, f) got %s" % opts.selection(),
)