def test13(self): """ Verify the functionatliy of the Evidence manager with IPv6 and UDP """ def domain_callback(flow): self.called_callback += 1 flow.evidence = True d = pyaiengine.DomainName("Google domain", ".google.com") dm = pyaiengine.DomainNameManager() d.callback = domain_callback dm.add_domain_name(d) self.s.set_domain_name_manager(dm, "DNSProtocol") with pyaiengine.PacketDispatcher( "../pcapfiles/ipv6_google_dns.pcap") as pd: pd.evidences = True pd.stack = self.s pd.run() self.assertEqual(self.called_callback, 1) self.assertEqual(d.matchs, 1) """ verify the integrity of the new file created """ files = glob.glob("evidences.*.pcap") os.remove(files[0])
def setUp(self): self.s = pyaiengine.StackLan() self.dis = pyaiengine.PacketDispatcher() self.dis.stack = self.s self.s.tcp_flows = 2048 self.s.udp_flows = 1024 self.f = pyaiengine.FrequencyGroup()
def test29(self): """ Verify the functionatliy of the Evidence manager """ def domain_callback(flow): self.called_callback += 1 flow.evidence = True d = pyaiengine.DomainName("Wired domain", ".wired.com") dm = pyaiengine.DomainNameManager() d.callback = domain_callback dm.add_domain_name(d) self.s.set_domain_name_manager(dm, "HTTPProtocol") with pyaiengine.PacketDispatcher( "../pcapfiles/two_http_flows_noending.pcap") as pd: pd.evidences = True pd.stack = self.s pd.run() self.assertEqual(self.called_callback, 1) self.assertEqual(d.matchs, 1) """ verify the integrity of the new file created """ files = glob.glob("evidences.*.pcap") os.remove(files[0])
def test8(self): """ Test the functionality of make graphs of regex, for complex detecctions """ rmbase = pyaiengine.RegexManager() rm2 = pyaiengine.RegexManager() r1 = pyaiengine.Regex("r1", b"^(No hacker should visit Las Vegas).*$") rmbase.add_regex(r1) r1.next_regex_manager = rm2 r2 = pyaiengine.Regex("r2", b"(this can not match)") r3 = pyaiengine.Regex("r3", b"^\x90\x90\x90\x90.*$") rm2.add_regex(r2) rm2.add_regex(r3) self.s.tcp_regex_manager = rmbase with pyaiengine.PacketDispatcher( "../pcapfiles/generic_exploit_ipv6_defcon20.pcap") as pd: pd.stack = self.s pd.run() self.assertEqual(r1.matchs, 1) self.assertEqual(r2.matchs, 0) self.assertEqual(r3.matchs, 1)
def setUp(self): self.s = pyaiengine.StackLanIPv6() self.dis = pyaiengine.PacketDispatcher() self.dis.stack = self.s self.s.tcp_flows = 2048 self.s.udp_flows = 1024 self.called_callback = 0
def test19(self): """ Verify SMTP traffic with domain callback """ self.from_correct = False def domain_callback(flow): s = flow.smtp_info if (s): if (str(s.mail_from) == "*****@*****.**"): self.from_correct = True self.called_callback += 1 d = pyaiengine.DomainName("Some domain", ".patriots.in") d.callback = domain_callback dm = pyaiengine.DomainNameManager() dm.add_domain_name(d) self.s.set_domain_name_manager(dm, "SMTPProtocol") oldstack = None with pyaiengine.PacketDispatcher("../pcapfiles/smtp.pcap") as pd: pd.stack = self.s pd.run() oldstack = pd.stack self.assertEqual(oldstack, self.s) self.assertEqual(d.matchs, 1) self.assertEqual(self.called_callback, 1) self.assertEqual(self.from_correct, True)
def test24(self): """ Verify the property of the PacketDispatcher.stack """ p = pyaiengine.PacketDispatcher() self.assertEqual(p.stack, None) # p.stack = p self.dis.stack = None self.assertEqual(self.dis.stack, None)
def test11(self): """ Verify the correctness of the HTTP Protocol on IPv6 """ with pyaiengine.PacketDispatcher( "../pcapfiles/http_over_ipv6.pcap") as pd: pd.stack = self.s pd.run() c = self.s.get_counters("HTTPProtocol") self.assertEqual(c["requests"], 11) self.assertEqual(c["responses"], 11)
def test20(self): """ Test the chains of regex with RegexManagers """ rlist = [ pyaiengine.Regex("expression %d" % x, "some regex %d" % x) for x in xrange(0, 5) ] rmbase = pyaiengine.RegexManager() rm1 = pyaiengine.RegexManager() rm2 = pyaiengine.RegexManager() rm3 = pyaiengine.RegexManager() [rmbase.add_regex(r) for r in rlist] r1 = pyaiengine.Regex("smtp1", "^AUTH LOGIN") r1.next_regex_manager = rm1 rmbase.add_regex(r1) r2 = pyaiengine.Regex("smtp2", "^NO MATCHS") r3 = pyaiengine.Regex("smtp3", "^MAIL FROM") rm1.add_regex(r2) rm1.add_regex(r3) r3.next_regex_manager = rm2 r4 = pyaiengine.Regex("smtp4", "^NO MATCHS") r5 = pyaiengine.Regex("smtp5", "^DATA") rm2.add_regex(r4) rm2.add_regex(r5) r5.next_regex_manager = rm3 r6 = pyaiengine.Regex("smtp6", "^QUIT") rm3.add_regex(r6) self.s.tcp_regex_manager = rmbase self.s.enable_nids_engine = True with pyaiengine.PacketDispatcher("../pcapfiles/smtp.pcap") as pd: pd.stack = self.s pd.run() for r in rlist: self.assertEqual(r.matchs, 0) self.assertEqual(r1.matchs, 1) self.assertEqual(r2.matchs, 0) self.assertEqual(r3.matchs, 1) self.assertEqual(r4.matchs, 0) self.assertEqual(r5.matchs, 1) self.assertEqual(r6.matchs, 1)
def test28(self): """ Verify the correctness of the HTTP Protocol """ """ The filter tcp and port 49503 will filter just one HTTP flow that contains exactly 39 requests and 38 responses """ with pyaiengine.PacketDispatcher( "../pcapfiles/two_http_flows_noending.pcap") as pd: pd.pcap_filter = "tcp and port 49503" pd.stack = self.s pd.run() c = self.s.get_counters("HTTPProtocol") self.assertEqual(c["requests"], 3) self.assertEqual(c["responses"], 3)
def test2(self): """ Test the with statement of the PacketDispatcher """ rm = pyaiengine.RegexManager() r = pyaiengine.Regex("Bin directory", b"^\x26\x01") rm.add_regex(r) self.s.tcp_regex_manager = rm with pyaiengine.PacketDispatcher("../pcapfiles/openflow.pcap") as pd: pd.stack = self.s pd.run() self.assertEqual(r.matchs, 1)
def test10(self): """ Verify the functionality of the getCache method """ with pyaiengine.PacketDispatcher( "../pcapfiles/ipv6_google_dns.pcap") as pd: pd.stack = self.s pd.run() d = self.s.get_cache("DNSProtocol") self.assertEqual(len(self.s.get_cache("DNSProtocol")), 1) self.assertEqual(len(self.s.get_cache("DNSProtocolNoExists")), 0) self.s.release_cache("DNSProtocol") self.assertEqual(len(self.s.get_cache("DNSProtocol")), 0) self.assertEqual(len(self.s.get_cache("HTTPProtocol")), 0) self.assertEqual(len(self.s.get_cache("SSLProtocol")), 0)
def test25(self): """ Verify the functionatliy of the SSDP Protocol """ with pyaiengine.PacketDispatcher("../pcapfiles/ssdp_flow.pcap") as pd: pd.stack = self.s pd.run() fu = self.s.udp_flow_manager for flow in fu: s = flow.ssdp_info if (s): self.assertEqual(s.uri, "*") self.assertEqual(s.host_name, "239.255.255.250:1900") else: self.assertFalse(False)
def threadHandler(netmask): # Load an instance of a Network Stack on Lan Network st = pyaiengine.StackLan() st.tcp_flows = 327680 st.udp_flows = 163840 with pyaiengine.PacketDispatcher("re0") as pd: pd.stack = st pd.pcap_filter = netmask pd.run() st.stats_level = 5 f = open("statistics.log.%d" % os.getpid(), "w") f.write(str(st)) f.close()
def test26(self): """ Verify the functionatliy of the RegexManager on the HTTP Protocol for analise inside the l7 payload of HTTP """ def callback_domain(flow): self.called_callback += 1 pass def callback_regex(flow): self.called_callback += 1 self.assertEqual(flow.packets, 11) self.assertEqual(flow.packets_layer7, 4) d = pyaiengine.DomainName("Wired domain", ".wired.com") rm = pyaiengine.RegexManager() r1 = pyaiengine.Regex("Regex for analysing the content of HTTP", b"^\x1f\x8b\x08\x00\x00\x00\x00.*$") r2 = pyaiengine.Regex("Regex for analysing the content of HTTP", b"^.{3}\xcd\x9c\xc0\x0a\x34.*$") r3 = pyaiengine.Regex("Regex for analysing the content of HTTP", b"^.*\x44\x75\x57\x0c\x22\x7b\xa7\x6d$") r2.next_regex = r3 r1.next_regex = r2 rm.add_regex(r1) r3.callback = callback_regex """ So the flows from wired.com will be analise the regexmanager attached """ d.regex_manager = rm dm = pyaiengine.DomainNameManager() d.callback = callback_domain dm.add_domain_name(d) self.s.set_domain_name_manager(dm, "http") with pyaiengine.PacketDispatcher( "../pcapfiles/two_http_flows_noending.pcap") as pd: pd.stack = self.s pd.run() self.assertEqual(self.called_callback, 2) self.assertEqual(r1.matchs, 1) self.assertEqual(r2.matchs, 1) self.assertEqual(r3.matchs, 1) self.assertEqual(d.matchs, 1)
def test31(self): """ Verify the functionatliy of the RegexManager on the IPSets """ def regex_callback(flow): r = flow.regex i = flow.ip_set self.assertEqual(flow.dstip, "95.100.96.10") self.assertEqual(r.name, "generic http") self.assertEqual(i.name, "Generic set") self.called_callback += 1 def ipset_callback(flow): r = flow.regex i = flow.ip_set self.assertNotEqual(i, None) self.assertEqual(i.name, "Generic set") self.assertEqual(r, None) self.called_callback += 1 rm = pyaiengine.RegexManager() i = pyaiengine.IPSet("Generic set") i.add_ip_address("95.100.96.10") i.regexmanager = None i.callback = ipset_callback im = pyaiengine.IPSetManager() im.add_ip_set(i) self.s.tcp_ip_set_manager = im r = pyaiengine.Regex("generic http", "^GET.*HTTP") r.callback = regex_callback rm.add_regex(r) self.s.enable_nids_engine = True with pyaiengine.PacketDispatcher( "../pcapfiles/two_http_flows_noending.pcap") as pd: pd.stack = self.s pd.run() self.assertEqual(self.called_callback, 1) self.assertEqual(i.lookups_in, 1) self.assertEqual(r.matchs, 0)
def test22(self): """ Verify the functionatliy of the HTTPUriSets with the callbacks """ self.uset = pyaiengine.HTTPUriSet() def domain_callback(flow): self.called_callback += 1 def uri_callback(flow): self.assertEqual(self.uset.uris, 1) self.assertEqual(self.uset.lookups, 39) self.assertEqual(self.uset.lookups_in, 1) self.assertEqual(self.uset.lookups_out, 38) self.called_callback += 1 d = pyaiengine.DomainName("Wired domain", ".wired.com") dm = pyaiengine.DomainNameManager() d.callback = domain_callback dm.add_domain_name(d) self.uset.add_uri( "/images_blogs/gadgetlab/2013/08/AP090714043057-60x60.jpg") self.uset.callback = uri_callback d.http_uri_set = self.uset self.s.set_domain_name_manager(dm, "HTTPProtocol") with pyaiengine.PacketDispatcher( "../pcapfiles/two_http_flows_noending.pcap") as pd: pd.stack = self.s pd.run() self.assertEqual(d.http_uri_set, self.uset) self.assertEqual(self.uset.uris, 1) self.assertEqual(self.uset.lookups, 39) self.assertEqual(self.uset.lookups_in, 1) self.assertEqual(self.uset.lookups_out, 38) self.assertEqual(self.called_callback, 2)
def test9(self): """ Another test for the functionality of make graphs of regex, for complex detecctions """ rm1 = pyaiengine.RegexManager() rm2 = pyaiengine.RegexManager() rm3 = pyaiengine.RegexManager() r1 = pyaiengine.Regex("r1", b"^(No hacker should visit Las Vegas).*$") r1.next_regex_manager = rm2 rm1.add_regex(r1) r2 = pyaiengine.Regex("r2", b"(this can not match)") r3 = pyaiengine.Regex("r3", b"^\x90\x90\x90\x90.*$") rm2.add_regex(r2) rm2.add_regex(r3) r3.next_regex_manager = rm3 r4 = pyaiengine.Regex("r4", b"^Upgrade.*$") r5 = pyaiengine.Regex("r5", b"(this can not match)") rm3.add_regex(r4) rm3.add_regex(r5) self.s.tcp_regex_manager = rm1 oldstack = None with pyaiengine.PacketDispatcher( "../pcapfiles/generic_exploit_ipv6_defcon20.pcap") as pd: pd.stack = self.s pd.run() oldstack = self.s self.assertEqual(self.s, oldstack) self.assertEqual(r1.matchs, 1) self.assertEqual(r2.matchs, 0) self.assertEqual(r3.matchs, 1) self.assertEqual(r4.matchs, 1)
def test23(self): """ Verify the functionatliy of the HTTPUriSets with the callbacks """ self.uset = pyaiengine.HTTPUriSet() def domain_callback(flow): self.called_callback += 1 def uri_callback(flow): self.assertEqual(self.uset.uris, 1) self.assertEqual(self.uset.lookups, 4) self.assertEqual(self.uset.lookups_in, 1) self.assertEqual(self.uset.lookups_out, 3) self.called_callback += 1 d = pyaiengine.DomainName("Wired domain", ".wired.com") dm = pyaiengine.DomainNameManager() d.callback = domain_callback dm.add_domain_name(d) # This uri is the thrid of the wired.com flow self.uset.add_uri("/js/scrolldock/scrolldock.css?v=20121120a") self.uset.callback = uri_callback d.http_uri_set = self.uset self.s.set_domain_name_manager(dm, "HTTPProtocol") with pyaiengine.PacketDispatcher( "../pcapfiles/two_http_flows_noending.pcap") as pd: pd.stack = self.s pd.run() self.assertEqual(self.uset.uris, 1) self.assertEqual(self.uset.lookups, 39) self.assertEqual(self.uset.lookups_in, 1) self.assertEqual(self.uset.lookups_out, 38) self.assertEqual(self.called_callback, 2)
def domain_names(self): """Fit the pcap as input file to pyaiengine.PacketDispatcher, Gets the total number of packets process by the PacketDispatcher. Returns ------- self : dictionary object or raise PcapFileNotFoundError Returns domains. if pcap is not None: :return domains else: :return raise PcapFileNotFoundError """ try: if self.pcap is not None: with pyaiengine.PacketDispatcher(self.pcap) as pd: pd.stack = self.st pd.run() # Get the UDP flows processed self.flows = self.st.udp_flow_manager for flow in self.flows: if flow.bytes > 0: if flow.dns_info: name = flow.dns_info.domain_name if not self.domains.has_key(name): self.domains[name] = 1 else: self.domains[name] += 1 raise PcapFileNotFoundError('Pcap file not found : {}'.format(self.pcap)) except Exception as excep: # print('I/O Exception occured: ', error.message) pass return self.domains
def test12(self): """ Verify the functionatliy of the RegexManager on the HTTP Protocol for analise inside the l7 payload of HTTP on IPv6 traffic """ def callback_domain(flow): self.called_callback += 1 def callback_regex(flow): self.called_callback += 1 self.assertEqual(flow.regex.name, "Regex for analysing the content of HTTP") self.assertEqual(flow.http_info.host_name, "media.us.listen.com") d = pyaiengine.DomainName("Music domain", ".us.listen.com") rm = pyaiengine.RegexManager() r1 = pyaiengine.Regex("Regex for analysing the content of HTTP", b"^\x89\x50\x4e\x47\x0d\x0a\x1a\x0a.*$") rm.add_regex(r1) r1.callback = callback_regex """ So the flows from listen.com will be analise the regexmanager attached """ d.regex_manager = rm dm = pyaiengine.DomainNameManager() d.callback = callback_domain dm.add_domain_name(d) self.s.set_domain_name_manager(dm, "HTTPProtocol") with pyaiengine.PacketDispatcher( "../pcapfiles/http_over_ipv6.pcap") as pd: pd.stack = self.s pd.run() self.assertEqual(self.called_callback, 2) self.assertEqual(r1.matchs, 1) self.assertEqual(d.matchs, 1)
st = pyaiengine.StackLan() dm = pyaiengine.DomainNameManager() st.tcp_flows = 200000 st.udp_flows = 100000 d1 = pyaiengine.DomainName("Generic com", ".com") d2 = pyaiengine.DomainName("Generic org", ".com") d3 = pyaiengine.DomainName("Generic net", ".org") d1.callback = random_callback_name d2.callback = random_callback_name d3.callback = random_callback_name dm.add_domain_name(d1) dm.add_domain_name(d2) dm.add_domain_name(d3) st.set_domain_name_manager(dm, "DNSProtocol") st.set_domain_name_manager(dm, "SSLProtocol") st.set_domain_name_manager(dm, "HTTPProtocol") with pyaiengine.PacketDispatcher("enp0s25") as pd: pd.stack = st pd.run() dm.show() sys.exit(0)
if __name__ == '__main__': # Load an instance of a Network Stack on Lan network st = pyaiengine.StackLan() dm = pyaiengine.DomainNameManager() dom = pyaiengine.DomainName("Service to analyze", "marca.com") dom.callback = callback_host dm.add_domain_name(dom) st.set_domain_name_manager(dm, "HTTPProtocol") st.tcp_flows = 327680 st.udp_flows = 163840 with pyaiengine.PacketDispatcher("eth0") as pd: # Plug the stack on the PacketDispatcher pd.stack = st pd.run() # Dump on file the statistics of the stack st.stats_level = 5 f = open("statistics.log", "w") f.write(str(st)) f.close() print(top_ips) sys.exit(0)
output_path = "/aiengine_vol/" filename = "aiengine_pkt_results.json" # scan every file in the /pcap/ dir json_output = {} for filename in os.listdir("pcap"): print("[AIEngine] Scanning: " + filename) packet_dump_filename = "aiengine_packet_scans_{}_{}.json".format( filename, time.strftime('%H-%M-%S', time.localtime())) # update tcp & udp traffic every 16 packets db = fileAdaptor(output_path + packet_dump_filename) st.set_udp_database_adaptor(db, 16) st.set_tcp_database_adaptor(db, 16) with pyaiengine.PacketDispatcher("/pcap/" + filename) as pd: pd.stack = st pd.run() old_stdout_fd = os.dup(1) fd = redirect_stdout_to_file("/stdout_temp.txt") # parse redirected summary messages json_output[filename] = { "protocol_statistics": parse_protocol_summary(st, fd), "flow_statistics": parse_flows_summary(st, fd), "anomaly_statistics": parse_anomaly_summary(st, fd) } # put stdout back os.dup2(old_stdout_fd, 1)
con = dict() for f in ssh_connections: """ Normally connections between 4000 and 3000 encrypted bytes are fail logins """ if (4000 > f.ssh_info.encrypted_bytes > 3000): """ Store the source IP address of the SSH connection """ if (f.src_ip not in con): con[f.src_ip] = 0 con[f.src_ip] += 1 """ Check the IPs that have been abuse """ for k, v in con.iteritems(): if (v > 5): print("IP %s is bruting force the SSH service" % k) if __name__ == '__main__': # Load an instance of a Network Stack on a LAN st = pyaiengine.StackLan() st.tcp_flows = 327680 st.udp_flows = 163840 with pyaiengine.PacketDispatcher("lo") as pd: pd.stack = st pd.add_timer(timer_5seconds, 5) pd.run() sys.exit(0)
total_ips = dict() print("NTP DDoS Checker") c = st.get_counters("NTPProtocol") # Count the number different ips of the NTP flows for flow in st.udp_flow_manager: if (flow.l7_protocol_name == "NTPProtocol"): total_ips[flow.src_ip] = 1 if (total_ips.len() == len(fu)): print("System under a NTP DDoS attack") if __name__ == '__main__': # Load an instance of a Network Stack Lan st = pyaiengine.StackLan() st.tcp_flows = 327680 st.udp_flows = 163840 # Create a instace of a PacketDispatcher with pyaiengine.PacketDispatcher("ens7") as pd: pd.stack = st # Sets a handler method that will be call # every 5 seconds for check the values pd.set_scheduler(scheduler_handler_tcp, 5) pd.run() sys.exit(0)
ipstats = st.get_counters("IP") current_ip_packets = ipstats["packets"] current_fragmented = ipstats["fragmented packets"] print("\033[34m" + "INFO: " + str(ipstats) + "\033[0m") if (current_fragmented > previous_fragments + delta): print("\033[31m" + "ALERT: IP Fragment attack on the network" + "\033[0m") previous_ip_packets = current_ip_packets previous_fragments = current_fragmented if __name__ == '__main__': st = pyaiengine.StackLan() st.tcp_flows = 327680 st.udp_flows = 163840 source = "enp0s31f6" with pyaiengine.PacketDispatcher(source) as pd: pd.stack = st pd.add_timer(timer_5seconds, 5) pd.run() sys.exit(0)
if __name__ == '__main__': # Load an instance of a Network Stack on a Lan network st1 = pyaiengine.StackLan() st1.tcp_flows = 327680 st1.udp_flows = 163840 """ Generate two instances of the FrequencyGroup and LearnerEngine """ learn = pyaiengine.LearnerEngine() freq = pyaiengine.FrequencyGroup() """ Tell the stack that should store the payloads """ st1.enable_frequency_engine = True """ Open the pcapfile and process """ with pyaiengine.PacketDispatcher("unknown_traffic.pcap") as pd: pd.stack = st1 pd.run() """ Use the method most suitable for your case """ freq.add_flows_by_destination_port(st1.tcp_flow_manager) freq.compute() flow_list = freq.get_reference_flows() learn.agregate_flows(flow_list) learn.compute() """ Get the generated regex """ r_candidate = learn.regex print("Generated Regex:", r_candidate) rm = pyaiengine.RegexManager() r = pyaiengine.Regex("Unknown attack/traffic", r_candidate)
# The URI is encoded, so decode for analysis try: urie = base64.b64decode(uri) print "Original:", uri print "Decoded:", urie except: pass if __name__ == '__main__': # Load an instance of a Network Stack on mobile network st = pyaiengine.StackMobile() # Create a instace of a PacketDispatcher pdis = pyaiengine.PacketDispatcher() # Plug the stack on the PacketDispatcher pdis.stack = st dm = pyaiengine.DomainNameManager() dom = pyaiengine.DomainName("Service to analyze", "facebook.com") dom.callback = callback_uri dm.add_domain_name(dom) st.set_domain_name_manager(dm, "http") st.tcp_flows = 327680 st.udp_flows = 163840
r_mng = pyaiengine.RegexManager() reg_head = pyaiengine.Regex("First regex", "mining.subscribe") reg_tail = pyaiengine.Regex( "Second regex", "c4758493e4f9804beeb784b4ff0be019b03678952ea8bb6f5c5365b2b76438a7") reg_head.next_regex = reg_tail reg_tail.callback = callback r_mng.add_regex(reg_head) st.tcp_regex_manager = r_mng st.tcp_flows = 327680 st.udp_flows = 163840 with pyaiengine.PacketDispatcher( "/home/luis/pcapfiles/bitcoinminer.pcap") as pd: pd.stack = st pd.run() st.show_flows() # Dump on file the statistics of the stack st.stats_level = 5 f = open("statistics.log", "w") f.write(str(st)) f.close() sys.exit(0)