def test29(self):
""" Verify the functionatliy of the Evidence manager """
def domain_callback(flow):
self.called_callback += 1
flow.evidence = True
d = pyaiengine.DomainName("Wired domain", ".wired.com")
dm = pyaiengine.DomainNameManager()
d.callback = domain_callback
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "HTTPProtocol")
with pyaiengine.PacketDispatcher(
"../pcapfiles/two_http_flows_noending.pcap") as pd:
pd.evidences = True
pd.stack = self.s
pd.run()
self.assertEqual(self.called_callback, 1)
self.assertEqual(d.matchs, 1)
""" verify the integrity of the new file created """
files = glob.glob("evidences.*.pcap")
os.remove(files[0])
def test19(self):
""" Verify SMTP traffic with domain callback """
self.from_correct = False
def domain_callback(flow):
s = flow.smtp_info
if (s):
if (str(s.mail_from) == "*****@*****.**"):
self.from_correct = True
self.called_callback += 1
d = pyaiengine.DomainName("Some domain", ".patriots.in")
d.callback = domain_callback
dm = pyaiengine.DomainNameManager()
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "SMTPProtocol")
oldstack = None
with pyaiengine.PacketDispatcher("../pcapfiles/smtp.pcap") as pd:
pd.stack = self.s
pd.run()
oldstack = pd.stack
self.assertEqual(oldstack, self.s)
self.assertEqual(d.matchs, 1)
self.assertEqual(self.called_callback, 1)
self.assertEqual(self.from_correct, True)
def test10(self):
""" Attach a database to the engine and domain name"""
def domain_callback(flow):
self.called_callback += 1
self.assertEqual(str(flow.ssl_info.server_name),
"0.drive.google.com")
self.assertEqual(flow.l7_protocol_name, "SSLProtocol")
d = pyaiengine.DomainName("Google All", ".google.com")
dm = pyaiengine.DomainNameManager()
d.callback = domain_callback
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "SSLProtocol")
db = databaseTestAdaptor()
self.s.set_tcp_database_adaptor(db, 16)
self.dis.open("../pcapfiles/sslflow.pcap")
self.dis.run()
self.dis.close()
self.assertEqual(db.getInserts(), 1)
self.assertEqual(db.getUpdates(), 5)
self.assertEqual(db.getRemoves(), 0)
self.assertEqual(d.matchs, 1)
self.assertEqual(self.called_callback, 1)
def test6(self):
""" Verify SSL traffic with domain callback"""
def domain_callback(flow):
self.called_callback += 1
d = pyaiengine.DomainName("Google Drive Cert", ".drive.google.com")
d.callback = domain_callback
dm = pyaiengine.DomainNameManager()
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "SSLProtocol")
self.dis.open("../pcapfiles/sslflow.pcap")
self.dis.run()
self.dis.close()
self.assertEqual(len(dm), 1)
self.assertEqual(d.matchs, 1)
self.assertEqual(self.called_callback, 1)
""" check also the integrity of the ssl cache and counters """
cc = self.s.get_counters("SSLProtocol")
ca = self.s.get_cache("SSLProtocol")
self.assertEqual(len(ca), 1)
self.assertEqual(cc['server hellos'], 1)
def test7(self):
""" Verify SSL traffic with domain callback and IPset"""
def ipset_callback(flow):
self.ip_called_callback += 1
def domain_callback(flow):
self.called_callback += 1
ip = pyaiengine.IPSet("Specific IP address")
ip.add_ip_address("74.125.24.189")
ip.callback = ipset_callback
ipm = pyaiengine.IPSetManager()
ipm.add_ip_set(ip)
d = pyaiengine.DomainName("Google All", ".google.com")
d.callback = domain_callback
dm = pyaiengine.DomainNameManager()
dm.add_domain_name(d)
self.s.tcp_ip_set_manager = ipm
self.s.set_domain_name_manager(dm, "SSLProtocol")
self.dis.open("../pcapfiles/sslflow.pcap")
self.dis.run()
self.dis.close()
self.assertEqual(d.matchs, 1)
self.assertEqual(self.called_callback, 1)
self.assertEqual(self.ip_called_callback, 1)
def test13(self):
""" Verify the functionatliy of the Evidence manager with IPv6 and UDP """
def domain_callback(flow):
self.called_callback += 1
flow.evidence = True
d = pyaiengine.DomainName("Google domain", ".google.com")
dm = pyaiengine.DomainNameManager()
d.callback = domain_callback
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "DNSProtocol")
with pyaiengine.PacketDispatcher(
"../pcapfiles/ipv6_google_dns.pcap") as pd:
pd.evidences = True
pd.stack = self.s
pd.run()
self.assertEqual(self.called_callback, 1)
self.assertEqual(d.matchs, 1)
""" verify the integrity of the new file created """
files = glob.glob("evidences.*.pcap")
os.remove(files[0])
def test16(self):
""" Verify that ban domains dont take memory """
d = pyaiengine.DomainName("Wired domain", ".wired.com")
dm = pyaiengine.DomainNameManager()
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "HTTPProtocol", False)
self.dis.open("../pcapfiles/two_http_flows_noending.pcap")
self.dis.pcap_filter = "tcp"
self.dis.run()
self.dis.close()
self.assertEqual(d.matchs, 1)
ft = self.s.tcp_flow_manager
self.assertEqual(len(ft), 2)
self.assertEqual(self.dis.pcap_filter, "tcp")
# Only the first flow is the banned
for flow in ft:
info = flow.http_info
self.assertEqual(info.host_name, '')
self.assertEqual(info.user_agent, '')
self.assertEqual(info.uri, '')
break
def test26(self):
""" Verify the functionatliy of the RegexManager on the HTTP Protocol for analise
inside the l7 payload of HTTP """
def callback_domain(flow):
self.called_callback += 1
pass
def callback_regex(flow):
self.called_callback += 1
self.assertEqual(flow.packets, 11)
self.assertEqual(flow.packets_layer7, 4)
d = pyaiengine.DomainName("Wired domain", ".wired.com")
rm = pyaiengine.RegexManager()
r1 = pyaiengine.Regex("Regex for analysing the content of HTTP",
b"^\x1f\x8b\x08\x00\x00\x00\x00.*$")
r2 = pyaiengine.Regex("Regex for analysing the content of HTTP",
b"^.{3}\xcd\x9c\xc0\x0a\x34.*$")
r3 = pyaiengine.Regex("Regex for analysing the content of HTTP",
b"^.*\x44\x75\x57\x0c\x22\x7b\xa7\x6d$")
r2.next_regex = r3
r1.next_regex = r2
rm.add_regex(r1)
r3.callback = callback_regex
""" So the flows from wired.com will be analise the regexmanager attached """
d.regex_manager = rm
dm = pyaiengine.DomainNameManager()
d.callback = callback_domain
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "http")
with pyaiengine.PacketDispatcher(
"../pcapfiles/two_http_flows_noending.pcap") as pd:
pd.stack = self.s
pd.run()
self.assertEqual(self.called_callback, 2)
self.assertEqual(r1.matchs, 1)
self.assertEqual(r2.matchs, 1)
self.assertEqual(r3.matchs, 1)
self.assertEqual(d.matchs, 1)
def test7(self):
""" Extract IPv6 address from a DomainName matched """
def dns_callback(flow):
for ip in flow.dns_info:
if (ip == "2607:f8b0:4001:c05::6a"):
self.called_callback += 1
d = pyaiengine.DomainName("Google test", ".google.com")
d.callback = dns_callback
dm = pyaiengine.DomainNameManager()
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "dns")
self.dis.open("../pcapfiles/ipv6_google_dns.pcap")
self.dis.run()
self.dis.close()
self.assertEqual(self.called_callback, 1)
def test22(self):
""" Verify the functionatliy of the HTTPUriSets with the callbacks """
self.uset = pyaiengine.HTTPUriSet()
def domain_callback(flow):
self.called_callback += 1
def uri_callback(flow):
self.assertEqual(self.uset.uris, 1)
self.assertEqual(self.uset.lookups, 39)
self.assertEqual(self.uset.lookups_in, 1)
self.assertEqual(self.uset.lookups_out, 38)
self.called_callback += 1
d = pyaiengine.DomainName("Wired domain", ".wired.com")
dm = pyaiengine.DomainNameManager()
d.callback = domain_callback
dm.add_domain_name(d)
self.uset.add_uri(
"/images_blogs/gadgetlab/2013/08/AP090714043057-60x60.jpg")
self.uset.callback = uri_callback
d.http_uri_set = self.uset
self.s.set_domain_name_manager(dm, "HTTPProtocol")
with pyaiengine.PacketDispatcher(
"../pcapfiles/two_http_flows_noending.pcap") as pd:
pd.stack = self.s
pd.run()
self.assertEqual(d.http_uri_set, self.uset)
self.assertEqual(self.uset.uris, 1)
self.assertEqual(self.uset.lookups, 39)
self.assertEqual(self.uset.lookups_in, 1)
self.assertEqual(self.uset.lookups_out, 38)
self.assertEqual(self.called_callback, 2)
def test23(self):
""" Verify the functionatliy of the HTTPUriSets with the callbacks """
self.uset = pyaiengine.HTTPUriSet()
def domain_callback(flow):
self.called_callback += 1
def uri_callback(flow):
self.assertEqual(self.uset.uris, 1)
self.assertEqual(self.uset.lookups, 4)
self.assertEqual(self.uset.lookups_in, 1)
self.assertEqual(self.uset.lookups_out, 3)
self.called_callback += 1
d = pyaiengine.DomainName("Wired domain", ".wired.com")
dm = pyaiengine.DomainNameManager()
d.callback = domain_callback
dm.add_domain_name(d)
# This uri is the thrid of the wired.com flow
self.uset.add_uri("/js/scrolldock/scrolldock.css?v=20121120a")
self.uset.callback = uri_callback
d.http_uri_set = self.uset
self.s.set_domain_name_manager(dm, "HTTPProtocol")
with pyaiengine.PacketDispatcher(
"../pcapfiles/two_http_flows_noending.pcap") as pd:
pd.stack = self.s
pd.run()
self.assertEqual(self.uset.uris, 1)
self.assertEqual(self.uset.lookups, 39)
self.assertEqual(self.uset.lookups_in, 1)
self.assertEqual(self.uset.lookups_out, 38)
self.assertEqual(self.called_callback, 2)
def test17(self):
""" Verify the ban functionatly on the fly with a callback """
def domain_callback(flow):
self.called_callback += 1
info = flow.http_info
url = info.uri
# Some URI analsys on the first request could be done here
if (url == "/css/global.css?v=20121120a"):
info.banned = True
d = pyaiengine.DomainName("Wired domain", ".wired.com")
dm = pyaiengine.DomainNameManager()
d.callback = domain_callback
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "HTTPProtocol")
self.dis.open("../pcapfiles/two_http_flows_noending.pcap")
self.dis.run()
self.dis.close()
self.assertEqual(self.called_callback, 1)
ft = self.s.tcp_flow_manager
self.assertEqual(len(ft), 2)
# Only the first flow is the banned and released
for flow in self.s.tcp_flow_manager:
inf = flow.http_info
self.assertNotEqual(inf, None)
self.assertEqual(inf.uri, '')
self.assertEqual(inf.user_agent, '')
self.assertEqual(inf.host_name, '')
break
def loadBadDomains():
dm = pyaiengine.DomainNameManager()
# List from http://www.malwaredomainlist.com/hostslist/hosts.txt
# Or https://zeustracker.abuse.ch/blocklist.php?download=baddomains for ZeusDomains
# Parse the file and add the domains.
f = open("zeus.dat", "r")
lines = f.readlines()
i = 0
for line in lines:
if (line[0] != "#"):
domain = line.replace("\r\n", "").split(" ")
if (len(domain) > 2):
name = "Bad domain %d" % i
i = i + 1
dom = pyaiengine.DomainName(name, domain[2])
dom.callback = callback_domain
dm.add_domain_name(dom)
f.close()
return dm
def test12(self):
""" Verify the functionatliy of the RegexManager on the HTTP Protocol for analise
inside the l7 payload of HTTP on IPv6 traffic """
def callback_domain(flow):
self.called_callback += 1
def callback_regex(flow):
self.called_callback += 1
self.assertEqual(flow.regex.name,
"Regex for analysing the content of HTTP")
self.assertEqual(flow.http_info.host_name, "media.us.listen.com")
d = pyaiengine.DomainName("Music domain", ".us.listen.com")
rm = pyaiengine.RegexManager()
r1 = pyaiengine.Regex("Regex for analysing the content of HTTP",
b"^\x89\x50\x4e\x47\x0d\x0a\x1a\x0a.*$")
rm.add_regex(r1)
r1.callback = callback_regex
""" So the flows from listen.com will be analise the regexmanager attached """
d.regex_manager = rm
dm = pyaiengine.DomainNameManager()
d.callback = callback_domain
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "HTTPProtocol")
with pyaiengine.PacketDispatcher(
"../pcapfiles/http_over_ipv6.pcap") as pd:
pd.stack = self.s
pd.run()
self.assertEqual(self.called_callback, 2)
self.assertEqual(r1.matchs, 1)
self.assertEqual(d.matchs, 1)
if (idx > -1):
value = arg[idx + 1:]
if (len(value) > 64):
""" Ummmm a value of an argument bigger than 64 """
decode = value.replace("%", "").decode("hex")
if (decode.find("Runtime.getRuntime().exec") > -1):
""" Somebody is executing remote commands """
print("[WARNING]: Jboss exploit detected")
print("[WARNING]: argument value:%s" % decode)
if __name__ == '__main__':
st = pyaiengine.StackLan()
http_mng = pyaiengine.DomainNameManager()
dom = pyaiengine.DomainName("My jboss host", "52.90.136.228:8080")
re = pyaiengine.Regex("Set regex for Jboss uris",
"^(/jmx-console/HtmlAdaptor).*")
rm = pyaiengine.RegexManager()
rm.add_regex(re)
http_mng.add_domain_name(dom)
dom.http_uri_regex_manager = rm
re.callback = callback_uri
""" Plug the DomainNameManager on the HTTPProtocol """
st.set_domain_name_manager(http_mng, "HTTPProtocol")
st.tcp_flows = 50000
def test13(self):
""" Verify all the URIs of an HTTP flow """
def domain_callback(flow):
urls = (
"/css/global.css?v=20121120a", "/js/jquery.hoverIntent.js",
"/js/ecom/ecomPlacement.js",
"/js/scrolldock/scrolldock.css?v=20121120a",
"/images_blogs/gadgetlab/2013/07/MG_9640edit-200x100.jpg",
"/images_blogs/underwire/2013/08/Back-In-Time-200x100.jpg",
"/images_blogs/thisdayintech/2013/03/set.jpg",
"/js/scrolldock/i/sub_righttab.gif",
"/images/global_header/new/Marriott_217x109.jpg",
"/images/global_header/subscribe/gh_flyout_failsafe.jpg",
"/images/global_header/new/the-connective.jpg",
"/images/covers/120x164.jpg",
"/images/subscribe/xrail_headline.gif",
"/images_blogs/gadgetlab/2013/08/bb10-bg.jpg",
"/images_blogs/autopia/2013/08/rescuer_cam06_110830-200x100.jpg",
"/images_blogs/wiredscience/2013/08/earth-ring-200x100.jpg",
"/images_blogs/underwire/2013/08/breaking-bad-small-200x100.png",
"/insights/wp-content/uploads/2013/08/dotcombubble_660-200x100.jpg",
"/geekdad/wp-content/uploads/2013/03/wreck-it-ralph-title1-200x100.png",
"/wiredenterprise/wp-content/uploads/2013/08/apple-logo-pixels-200x100.jpg",
"/images_blogs/threatlevel/2013/08/drone-w.jpg",
"/images_blogs/rawfile/2013/08/CirculationDesk-200x100.jpg",
"/images_blogs/magazine/2013/07/theoptimist_wired-200x100.jpg",
"/images_blogs/underwire/2013/08/Back-In-Time-w.jpg",
"/design/wp-content/uploads/2013/08/dyson-w.jpg",
"/images_blogs/threatlevel/2013/08/aaron_swartz-w.jpg",
"/images_blogs/threatlevel/2013/08/aaron_swartz-w.jpg",
"/images_blogs/wiredscience/2013/08/NegativelyRefracting-w.jpg",
"/images_blogs/wiredscience/2013/08/bee-w.jpg",
"/gadgetlab/2013/08/blackberry-failures/",
"/gadgetlab/wp-content/themes/wired-global/style.css?ver=20121114",
"/css/global.css?ver=20121114",
"/js/cn-fe-common/jquery-1.7.2.min.js?ver=1.7.2",
"/js/cn.minified.js?ver=20121114",
"/js/videos/MobileCompatibility.js?ver=20121114",
"/images_blogs/gadgetlab/2013/06/internets.png",
"/gadgetlab/wp-content/themes/wired-responsive/i/design-sprite.png",
"/images_blogs/gadgetlab/2013/08/Blackberry8820.jpg",
"/images_blogs/gadgetlab/2013/08/vsapple-60x60.jpg",
"/images_blogs/gadgetlab/2013/08/AP090714043057-60x60.jpg")
self.called_callback += 1
sw = False
for url in urls:
if (str(flow.http_info.uri) == url):
sw = True
self.assertEqual(sw, True)
self.assertEqual(str(flow.http_info.host_name), "www.wired.com")
self.assertEqual(flow.l7_protocol_name, "HTTPProtocol")
d = pyaiengine.DomainName("Wired domain", ".wired.com")
dm = pyaiengine.DomainNameManager()
d.callback = domain_callback
dm.add_domain_name(d)
self.s.set_domain_name_manager(dm, "HTTPProtocol")
self.dis.open("../pcapfiles/two_http_flows_noending.pcap")
self.dis.run()
self.dis.close()
self.assertEqual(self.called_callback, 1)
if (name.startswith("www.")):
name = name[4:]
if (name):
value = gib_detect_train.avg_transition_prob(name,
model_mat) > threshold
if (value == False):
print("WARNING:%s:%s Unknown malware detected" %
(flow.l7_protocol_name, name))
if __name__ == '__main__':
st = pyaiengine.StackLan()
dm = pyaiengine.DomainNameManager()
st.tcp_flows = 200000
st.udp_flows = 100000
d1 = pyaiengine.DomainName("Generic com", ".com")
d2 = pyaiengine.DomainName("Generic org", ".com")
d3 = pyaiengine.DomainName("Generic net", ".org")
d1.callback = random_callback_name
d2.callback = random_callback_name
d3.callback = random_callback_name
dm.add_domain_name(d1)
dm.add_domain_name(d2)
dm.add_domain_name(d3)
udp_rm = pyaiengine.RegexManager()
""" Put here your code for load regexs
>>> tcp_rm.add_regex(pyaiengine.Regex("some regex", "\x00\x0a\x0b")
>>> tcp_rm.add_regex(pyaiengine.Regex("some regex", "^\x00\x0a.*exe", callback)
"""
tcp_set = pyaiengine.IPSetManager()
udp_set = pyaiengine.IPSetManager()
tcp_ipset = pyaiengine.IPSet()
udp_ipset = pyaiengine.IPSet()
""" Put here your code with your IP lists
>>> tcp_ipset.add_ip_address("192.158.1.1")
"""
http_names = pyaiengine.DomainNameManager()
ssl_names = pyaiengine.DomainNameManager()
dns_names = pyaiengine.DomainNameManager()
""" Put here your code with your domains for matching """
http_names.add_domain_name(pyaiengine.DomainName("Fedora", ".fedora.com"))
ssl_names.add_domain_name(pyaiengine.DomainName("Google", ".google.com"))
dns_names.add_domain_name(pyaiengine.DomainName("Google DNS", ".google.com"))
st.set_domain_name_manager(dns_names, "DNSProtocol")
st.set_domain_name_manager(http_names, "HTTPProtocol")
st.set_domain_name_manager(ssl_names, "SSLProtocol")
tcp_set.add_ip_set(tcp_ipset)
udp_set.add_ip_set(udp_ipset)
st.tcp_ip_set_manager = tcp_set
