def getFeature(self, src_path): FILEPATH = src_path print("Start: " + FILEPATH) print(datetime.datetime.now()) FILE_EXTRACT_PATH = self.get_extract_path(FILEPATH) pcap = pyshark.FileCapture(FILEPATH) featureTotal = "" featureSet = set() featureSetFinal = set() packetList = [] packetListStt = [] index = 1 for packet in pcap: print("Index packet %d" % index) index += 1 protocol = self.get_protocol(packet) if protocol != "other": src_ip = self.get_src_ip(packet) dst_ip = self.get_dst_ip(packet) src_port = self.get_src_port(packet, protocol) dst_port = self.get_dst_port(packet, protocol) service = self.get_service_http(protocol, dst_port) flag = self.get_flag(packet, protocol) sequence_number = self.get_seq_number(packet, protocol) acknowledgment_number = self.get_ackn_number(packet, protocol) packetList.append( Packet(src_ip, dst_ip, src_port, dst_port, protocol, service, flag, sequence_number, acknowledgment_number)) pcap.close() ind = 1 for packet in packetList: print("Index packetList %d" % ind) ind += 1 status_conn = self.is_connection_success(ind, packet, packetList) packetListStt.append( PacketStt(packet.src_ip, packet.dst_ip, packet.src_port, packet.dst_port, packet.protocol, packet.service, packet.flag, packet.sequence_number, packet.acknowledgment_number, status_conn)) i = 0 for packet in packetListStt: print("Index packetListStt %d" % i) i += 1 featureStrTuple = "" featureStr = "" featureStrTuple += packet.src_ip + "," featureStrTuple += packet.dst_ip + "," featureStrTuple += packet.src_port + "," featureStrTuple += packet.dst_port + "," featureStrTuple += packet.protocol + "," featureStrTuple += packet.service + "," # featureStrTuple += packet.flag + "," # featureStrTuple += packet.sequence_number + "," # featureStrTuple += packet.acknowledgment_number + "," featureStrTuple += packet.status_conn if featureStrTuple not in featureSet: featureSet.add(featureStrTuple) else: continue calcu_feature = self.get_calculate_feature(packet, packetListStt) featureStr += packet.src_ip + "," featureStr += packet.dst_ip + "," featureStr += packet.src_port + "," featureStr += packet.dst_port + "," featureStr += packet.protocol + "," featureStr += calcu_feature + "," featureStr += packet.service + "," #flag_s0 if packet.status_conn == "False": featureStr += "1" + "," else: featureStr += "0" + "," #flag_sf if packet.status_conn == "True": featureStr += "1" + "," else: featureStr += "0" featureStr += "\n" if featureStr not in featureSetFinal: featureSetFinal.add(featureStr) else: continue featureTotal += featureStr f = open(FILE_EXTRACT_PATH, "w+") f.write(featureTotal) f.close() print("Done: " + FILEPATH) print(datetime.datetime.now())
def process_pcap(fname, proto, port, outdir): """ Process pcap file :param fname: Input pcap file name :param proto: Protocol to analyze :param port: TCP/UDP Port used by this protocol :return: """ # global packets_array # print('Opening {}...'.format(fname)) if proto == 'gsm_map': filtered_cap = pyshark.FileCapture(fname, display_filter='gsm_map', only_summaries=False) filtered_cap.apply_on_packets(process_packet_gsm_map, timeout=10000) print_results( dict_ips, nodes_csv, proto, '{}_{}.pdf'.format( os.path.splitext(os.path.basename(fname))[0], 'ips')) print_results( dict_pcs, nodes_csv, proto, '{}_{}.pdf'.format( os.path.splitext(os.path.basename(fname))[0], 'point-codes')) # print_results(dict_gts, proto) elif proto == 'diameter': if port: decode_as = {'tcp.port=={}'.format(port): 'diameter'} filtered_cap = pyshark.FileCapture(fname, decode_as=decode_as, display_filter='diameter', only_summaries=False) else: filtered_cap = pyshark.FileCapture(fname, display_filter='diameter', only_summaries=False) filtered_cap.apply_on_packets(process_packet_diameter, timeout=10000) print_results( dict_ips, nodes_csv, proto, '{}_{}.pdf'.format( os.path.splitext(os.path.basename(fname))[0], 'ips')) # print_results(dict_nodes, proto, '{}_{}.pdf'.format(os.path.splitext(os.path.basename(fname))[0], 'nodes')) elif proto == 'gtp': # This option needs to cover both GTP and GTPv2 filtered_cap = pyshark.FileCapture(fname, display_filter='gtp', only_summaries=False) filtered_cap.apply_on_packets(process_packet_gtp, timeout=10000) print_results( dict_ips, nodes_csv, proto, '{}_{}.pdf'.format( os.path.splitext(os.path.basename(fname))[0], 'ips'), outdir) # print_results(dict_nodes, proto, '{}_{}.pdf'.format(os.path.splitext(os.path.basename(fname))[0], 'nodes'), outdir) print_results( dict_vlans, nodes_csv, proto, '{}_{}.pdf'.format( os.path.splitext(os.path.basename(fname))[0], 'vlans'), outdir) elif proto == 'gtpv2': filtered_cap = pyshark.FileCapture(fname, display_filter='gtpv2', only_summaries=False) filtered_cap.apply_on_packets(process_packet_gtp, timeout=10000) print_results( dict_ips, nodes_csv, proto, '{}_{}.pdf'.format( os.path.splitext(os.path.basename(fname))[0], 'ips'), outdir) # print_results(dict_nodes, proto, '{}_{}.pdf'.format(os.path.splitext(os.path.basename(fname))[0], 'nodes'), outdir) print_results( dict_vlans, nodes_csv, proto, '{}_{}.pdf'.format( os.path.splitext(os.path.basename(fname))[0], 'vlans'), outdir)
import pyshark cap = pyshark.FileCapture('mult2.pcap') for pkt in cap: try: print(pkt.bootp.field_names) print(pkt.bootp.option_hostname) #print(pkt.bootp['id']) except AttributeError: pass
def check_app(app, fullpacket=False, force=False): ''' Check application based on app name in Tapioca results ''' dnscacheloaded = False targetscacheloaded = False largewarned = False # load local network from config net.set_local() # Get pcap file location if app.endswith('.pcap'): pcapfile = app if os.path.exists(pcapfile): sys.stdout = Logger('%s.%s' % (pcapfile, report_output)) else: pcapfile = os.path.join('results', app, 'tcpdump.pcap') if os.path.exists(pcapfile): sys.stdout = Logger(os.path.join('results', app, report_output)) if os.path.exists(pcapfile): pcapdir = os.path.dirname(pcapfile) dnspkl = os.path.join(pcapdir, '.dnsmap.pkl') targetspkl = os.path.join(pcapdir, '.targets.pkl') eprint(color.bright('Checking app %s...' % color.cyan(app))) if os.path.exists(dnspkl) and not force: eprint('Loading cached DNS info...') with open(dnspkl, 'rb') as pklhandle: try: net.dnsmap = pickle.load(pklhandle) net.dnsreqs = pickle.load(pklhandle) dnscacheloaded = True except: pass if not dnscacheloaded: if os.path.getsize(pcapfile) > 100000000: # Over 100MB eprint( color.bright(color.yellow('Warning: capture size is large. Please be patient.'))) largewarned = True # Get captured DNS info for IP addresses eprint('Getting DNS info...') dnspackets = pyshark.FileCapture( pcapfile, keep_packets=False, display_filter='dns') dnspackets.apply_on_packets(net.get_dns_info, timeout=1000) with open(dnspkl, 'wb') as pklhandle: pickle.dump( net.dnsmap, pklhandle, protocol=pickle.HIGHEST_PROTOCOL) pickle.dump( net.dnsreqs, pklhandle, protocol=pickle.HIGHEST_PROTOCOL) # if os.path.exists(targetspkl) and not force: # eprint('Loading cached targets...') # with open(targetspkl, 'rb') as pklhandle: # try: # net.targets = pickle.load(pklhandle) # targetscacheloaded = True # except: # pass if not targetscacheloaded: if fullpacket: packets = pyshark.FileCapture( pcapfile, keep_packets=False) # Get hosts contacted eprint('Getting hosts contacted...') packets.apply_on_packets( net.get_hosts_contacted_fullpacket, timeout=1000) else: packets = pyshark.FileCapture( pcapfile, keep_packets=False, only_summaries=True) # Get hosts contacted eprint('Getting hosts contacted...') packets.apply_on_packets(net.get_hosts_contacted, timeout=1000) # with open(targetspkl, 'wb') as pklhandle: # pickle.dump( # net.targets, pklhandle, protocol=pickle.HIGHEST_PROTOCOL) # Print report generate_report(app, fullpacket=fullpacket, pcapfile=pcapfile) # Reset globals net.clear()
def _insert_msql(self, file_path, app_package, app_type): import time startTime = time.time() print "Start inserting", app_package, file_path dbdao = SqlDao() appInfo = self.apps.get(app_package, None) if appInfo == None: if app_type == consts.IOS: print 'Error!! Can not find', app_package packages = pyshark.FileCapture(file_path, display_filter='http') timeStampTwo = time.time() print 'Get appInfo', timeStampTwo - startTime dns_info = {} comunicate_host = set() if appInfo is not None: app_package = appInfo.package app_name = appInfo.name app_category = appInfo.category app_company = appInfo.company else: app_name = None app_category = None app_company = None pkgInfos = [] while True: try: p = packages.next() if hasattr(p, 'http'): pkgInfo = self._parse_http_package(p) if pkgInfo[ETLConsts.HOST] == None or len( pkgInfo[ETLConsts.HOST] ) == 0 or '.' not in pkgInfo[ETLConsts.HOST]: ip = pkgInfo[ETLConsts.DESTINATION] if ip in dns_info and len(dns_info[ip]) == 1: host = dns_info[ip].pop() pkgInfo[ETLConsts.HOST] = host dns_info[ip].add(host) pkgInfo['app_name'] = app_name pkgInfo['app_category'] = app_category pkgInfo['app_company'] = app_company pkgInfos.append(pkgInfo) if len(pkgInfos) > 500: break if time.time() - timeStampTwo > 5: print "Spent more than 10s. Stop" break else: print app_name, app_category, app_company print 'ERROR WRONG PACKAGE TYPE' except StopIteration: break except: print 'ERROR', sys.exc_info()[0] try: packages.close() except: pass timeStampThree = time.time() print 'Parsing pcaps', timeStampThree - timeStampTwo params = [] for pkgInfo in pkgInfos: app_name = pkgInfo['app_name'] app_category = pkgInfo['app_category'] app_company = pkgInfo['app_company'] params.append( (app_package, pkgInfo[ETLConsts.SOURCE], pkgInfo[ETLConsts.DESTINATION], pkgInfo[ETLConsts.TIME], pkgInfo[ETLConsts.ADD_HEADER], pkgInfo[ETLConsts.HOST], pkgInfo[ETLConsts.PATH], pkgInfo[ETLConsts.ACCEPT], pkgInfo[ETLConsts.AGENT], pkgInfo[ETLConsts.REFER], pkgInfo[ETLConsts.AUTHOR], pkgInfo[ETLConsts.CONTENT_LENGTH], pkgInfo[ETLConsts.CONTENT_TYPE], pkgInfo[ETLConsts.METHOD], pkgInfo[ETLConsts.SIZE], pkgInfo[ETLConsts.HTTP_TYPE], app_name, app_category, app_company, pkgInfo[ETLConsts.RAW])) comunicate_host.add(pkgInfo[ETLConsts.HOST]) dbdao.executeBatch(self.INSERT_PACKAGES, params) dbdao.close() print 'inserting', time.time() - timeStampThree print "Finish", app_package, "Package:", len(params), len(pkgInfos)
def get_trace_controller(self, tracepath): try: cap = pyshark.FileCapture(tracepath, only_summaries=True) except: sys.stderr.write("Importing file failed.\n") return cap
def __getContent(self): cap = pyshark.FileCapture( self.capture, display_filter= '(tcp or udp) and (not mdns) and (not tls) and (not ssdp) and (not gryphon)' ) srcList = [] dstList = [] self.udpCounter = 0 self.udpData = set() self.udpTime = [] for packet in cap: payload = packet transportLayer = packet.transport_layer if str(packet[0].src) != 'ff:ff:ff:ff:ff:ff' and str( packet[0].dst) != 'ff:ff:ff:ff:ff:ff': if str(packet[0].src) not in srcList: srcMac = str(packet[0].src) srcList.append(srcMac) devicekey = { srcMac: { 'TCP Payload': [], 'TCP Payload Length': [], 'Window Size': [], 'Rogue': False } } try: srcVendor = MacLookup().lookup(srcMac) except: srcVendor = 'Unknown' if srcVendor not in self.VendorDeviceDict: srckey = {srcVendor: {}} self.VendorDeviceDict.update(srckey) if srcMac not in self.VendorDeviceDict[srcVendor]: self.VendorDeviceDict[srcVendor].update(devicekey) if packet[0].dst not in dstList: dstMac = str(packet[0].dst) dstList.append(dstMac) devicekey = { dstMac: { 'TCP Payload': [], 'TCP Payload Length': [], 'Window Size': [], 'Rogue': False } } try: dstVendor = MacLookup().lookup(dstMac) except: dstVendor = 'Unknown' if dstVendor not in self.VendorDeviceDict: dstkey = {dstVendor: {}} self.VendorDeviceDict.update(dstkey) if dstMac not in self.VendorDeviceDict[dstVendor]: self.VendorDeviceDict[dstVendor].update(devicekey) if transportLayer == 'TCP': self.udpCounter = 0 self.udpTime = [] self.udpData = [] try: windowSize = packet[2].window_size_value except: continue self.VendorDeviceDict[srcVendor][srcMac][ 'Window Size'].append(windowSize) for layer in packet: if 'DATA' in str(layer): try: payload = str(packet.data.data) payloadLen = int(packet.data.len) self.VendorDeviceDict[srcVendor][srcMac][ 'TCP Payload'].append(payload) self.VendorDeviceDict[srcVendor][srcMac][ 'TCP Payload Length'].append(payloadLen) except: continue if transportLayer == 'UDP': self.__udpCheck(srcVendor, packet, srcMac) self.pCount = self.pCount + 1 cap.close() self.finish = time.perf_counter()
def print_user(): count = 0 capture = pyshark.FileCapture('sample.pcap', display_filter='http contains user') for packet in capture: count = count + 1 print(count)
def data_data_dump(self): cap = pyshark.FileCapture('../data_file/zhanbao.pcap') new_order_len = 121 # ๆฐ่ฎขๅๅป้คๅคด้จ็้ฟๅบฆ order_confirm_len = 201 # ๅงๆ็กฎ่ฎค/ๆคๅๆๅๅๅบๅป้คๅคด้จ็้ฟๅบฆ cancel_order_len = 98 # ๆคๅ่ฏทๆฑๅป้คๅคด้จ็้ฟๅบฆ cancel_order_failed_len = 127 # ๆคๅๅคฑ่ดฅๅๅบๆๅๅป้คๅคด้จ็้ฟๅบฆ match_order_len = 165 # ๅงๆๆไบคๅป้คๅคด้จ้ฟๅบฆ for i in cap: if 'data' in i: # ๅคๆญi้้ขๆฏๅฆๆdataๆฐๆฎ len_data = len(i.data.data) if i.ip.src_host == self.ors_ip and i.ip.dst_host == self.tgw_ip: # ๆๅฎๆบIPๅฐๅๅ็ฎๆ ipๅฐๅ """ๆฐ่ฎขๅ๏ผORS->TGW""" if len_data % (2 * new_order_len) == 0: # ๆฐ่ฎขๅ count = 0 multi = int(len_data / (2 * new_order_len)) while count < multi: self.dump_data_list.append( i.data.data[count * (2 * new_order_len): (2 * new_order_len) * (count + 1)]) self.dump_timestamp.append(i.sniff_timestamp) count += 1 elif i.ip.src_host == self.tgw_ip and i.ip.dst_host == self.ors_ip: """ๅงๆ็กฎ่ฎคๅๆคๅๆๅๅๅบ๏ผTGW->ORS""" if len_data % (2 * order_confirm_len) == 0: # ๅงๆ็กฎ่ฎคๅๆคๅๆๅๅๅบ count = 0 multi = int(len_data / (2 * order_confirm_len)) while count < multi: self.dump_data_list.append( i.data.data[count * (2 * order_confirm_len): (2 * order_confirm_len) * (count + 1)]) self.dump_timestamp.append(i.sniff_timestamp) count += 1 elif len_data % (2 * cancel_order_len) == 0: # ๆคๅ่ฏทๆฑ """ๆคๅ่ฏทๆฑ๏ผTGW->ORS""" print(len_data) count = 0 multi = int(len_data / (2 * cancel_order_len)) while count < multi: self.dump_data_list.append( i.data.data[count * (2 * cancel_order_len): (2 * cancel_order_len) * (count + 1)]) self.dump_timestamp.append(i.sniff_timestamp) count += 1 elif len_data % (2 * cancel_order_failed_len) == 0: # ๆคๅๅคฑ่ดฅๅๅบ """ๆคๅๅๅบๅคฑ่ดฅ๏ผTGW->ORS""" count = 0 multi = int(len_data / (2 * cancel_order_failed_len)) while count < multi: self.dump_data_list.append( i.data.data[count * (2 * cancel_order_failed_len): (2 * cancel_order_failed_len) * (count + 1)]) self.dump_timestamp.append(i.sniff_timestamp) count += 1 elif len_data % (2 * match_order_len) == 0: # ๅงๆๆไบค """ๅงๆๆไบค๏ผTGW->ORS""" count = 0 multi = int(len_data / (2 * match_order_len)) while count < multi: self.dump_data_list.append( i.data.data[count * (2 * match_order_len): (2 * match_order_len) * (count + 1)]) self.dump_timestamp.append(i.sniff_timestamp) count += 1 else: continue else: continue # print(self.dump_data) # print(self.dump_timestamp) cap.close()
# encoding: utf-8 import pyshark import json import gzip, zlib # ๆๅผๅญๅจ็ๆ่ทๆไปถ #cap = pyshark.FileCapture('/Users/jiajia/Develop/workspace/PacketProcess/py_capature_pkt/pcap/mysql_complete.pcap') #cap = pyshark.FileCapture('/Users/jiajia/Develop/workspace/PacketProcess/py_capature_pkt/pcap/pgsql.cap') #cap = pyshark.FileCapture('/Users/jiajia/Develop/workspace/PacketProcess/py_capature_pkt/pcap/ms-sql-tds-rpc-requests.cap') #cap = pyshark.FileCapture('/Users/jiajia/Develop/workspace/PacketProcess/py_capature_pkt/pcap/TNS_Oracle1.pcap') cap = pyshark.FileCapture('/Users/jiajia/Develop/workspace/PacketProcess/py_capature_pkt/pcap/http_pkt.pcap') cap.load_packets() # ไป็ฝ็ปๆฅๅฃไธ่ฟ่กๆ่ท #cap = pyshark.LiveCapture(interface='en0') #cap.sniff(packet_count=1000) #layer_dict = {} data = {} for pkt in cap: #layer_list = [] #data['highest_layer'] = pkt.highest_layer #if not pkt.number in ['574']: # continue #if not pkt.number in ['839', '841', '857', '859']: # continue if pkt.highest_layer != 'HTTP': continue print "Number:" + pkt.number
import subprocess import pyshark f = pyshark.FileCapture('hackpad.pcap', display_filter='http') print('done parsing') secret = '3ed2e01c1d1248125c67ac637384a22d997d9369c74c82abba4cc3b1bfc65f026c957ff0feef61b161cfe3373c2d9b905639aa3688659566d9acc93bb72080f7e5ebd643808a0e50e1fc3d16246afcf688dfedf02ad4ae84fd92c5c53bbd98f08b21d838a3261874c4ee3ce8fbcb96628d5706499dd985ec0c13573eeee03766f7010a867edfed92c33233b17a9730eb4a82a6db51fa6124bfc48ef99d669e21740d12656f597e691bbcbaa67abe1a09f02afc37140b167533c7536ab2ecd4ed37572fc9154d23aa7d8c92b84b774702632ed2737a569e4dfbe01338fcbb2a77ddd6990ce169bb4f48e1ca96d30eced23b6fe5b875ca6481056848be0fbc26bcbffdfe966da4221103408f459ec1ef12c72068bc1b96df045d3fa12cc2a9dcd162ffdf876b3bc3a3ed2373559bcbe3f470a8c695bf54796bfe471cd34b463e9876212df912deef882b657954d7dada47' for i in xrange(8+256,8+256*1000,2): # blargh c = f[i]['urlencoded-form'] s = f[i+1]['data-text-lines'] if 'Internal Server Error' in str(s): continue print(i) print(c) print(s) # from pwn import xor; print(xor(data,16,prev_block)) ''' 3ed2e01c1d1248125c67ac637384a22d In cryptography, 997d9369c74c82abba4cc3b1bfc65f02 a padding oracl 6c957ff0feef61b161cfe3373c2d9b90 e attack is an a 5639aa3688659566d9acc93bb72080f7 ttack which is p e5ebd643808a0e50e1fc3d16246afcf6 erformed using t 88dfedf02ad4ae84fd92c5c53bbd98f0 he padding of a 8b21d838a3261874c4ee3ce8fbcb9662 8d5706499dd985ec0c13573eeee03766 hitcon{H4 f7010a867edfed92c33233b17a9730eb cked by a de1ici 4a82a6db51fa6124bfc48ef99d669e21 0us pudding '3'} 740d12656f597e691bbcbaa67abe1a09 f02afc37140b167533c7536ab2ecd4ed
# -*- coding=utf-8 -*- import pyshark pkt_list = [] cap = pyshark.FileCapture('cap1.pcap', keep_packets=False, display_filter='http') method_dict = {} def print_highest_layer(pkt): # ๆฌไปฃ็ ็ไธป่ฆไปปๅก: ๅฏนHTTPๆต้่ฟ่กๅๆ,ๆพๅฐ็นๅฎmethod็่ฏทๆฑๆฐ้ try: # ๅญๅ ธๆฐๆฎ็ปๆๅฆไธ # ้ฎไธบmethod, ๅผไธบๆฐ้ counts = method_dict.get(pkt.http.request_method, 0) counts += 1 method_dict[pkt.http.request_method] = counts except: pass # ๅบ็จๅฝๆฐๅฐๆฐๆฎๅ cap.apply_on_packets(print_highest_layer) print(method_dict) if __name__ == '__main__': # ไฝฟ็จmatplot่ฟ่กๅพๅฝขๅๅฑ็คบ
import pyshark from collections import defaultdict from statistics import mean cap = pyshark.FileCapture('./mpi_exp1.pcapng') syn_list = [] syn_ack_dict = defaultdict(list) ack_dict = defaultdict(list) fin_ack_dict = defaultdict(list) conn_time = [] # TODO: setup discard of retrans SYN or SYN/ACK packets. # Probably by only allowing first relative time per stream for pkt in cap: if 'TCP' in pkt: if pkt.tcp.flags == '0x00000002': # SYN packet, 1 per stream syn_list.append(pkt) if pkt.tcp.flags == '0x00000012': # SYN/ACK packet, 1 per stream syn_ack_dict[pkt.tcp.stream].append(pkt) if pkt.tcp.flags == '0x00000010': # 1st ACK packet in stream if pkt.tcp.seq == '1' and pkt.tcp.ack == '1': ack_dict[pkt.tcp.stream].append(pkt) if pkt.tcp.flags == '0x00000011': # Fin/ACK packet, 2 per stream fin_ack_dict[pkt.tcp.stream].append(pkt) for pkt in syn_list: ack = ack_dict[pkt.tcp.stream].pop() syn_time = float(pkt.tcp.time_relative) ack_time = float(ack.tcp.time_relative)
##cap = pyshark.FileCapture('data/2018-07-31-15-15-09-192.168.100.113.pcap') ##device_IP = "192.168.100.113" ##results_file = "192.168.100.113-1" ##analyze_packet(cap,device_IP,results_file) ##cap = pyshark.FileCapture('data/2018-10-03-15-22-32-192.168.100.113.pcap') ##device_IP = "192.168.100.113" ##results_file = "192.168.100.113-2" ##analyze_packet(cap,device_IP,results_file) for i in range(1, 10): file = ( "data/2018-07-20-17-31-20-192.168.100.108_0000{0}.pcapng".format(i)) cap = pyshark.FileCapture(file) device_IP = "192.168.100.108" results_file = ("192.168.100.108-1_{0}".format(i)) analyze_packet(cap, device_IP, results_file) for i in range(10, 100): file = ("data/2018-07-20-17-31-20-192.168.100.108_000{0}.pcapng".format(i)) cap = pyshark.FileCapture(file) device_IP = "192.168.100.108" results_file = ("192.168.100.108-1_{0}".format(i)) analyze_packet(cap, device_IP, results_file) ##cap = pyshark.FileCapture('data/2019-01-10-14-34-38-192.168.1.197.pcap') ##device_IP = "192.168.1.197" ##results_file = "192.168.1.197-1" ##analyze_packet(cap,device_IP,results_file)
if len(sys.argv) < 4 or not parse_args(sys.argv): print( "Usage: python3 from_pcap.py <input file> [-out output_file] [-es Elasticsearch_URI] [--debug]" ) exit(1) pcapfile = sys.argv[1] subnet = load_config("./subnet.config") tcp_sessions = {} udp_sessions = {} first_udp = {} if debug: print("[*] Reading pcap file...") packets = pyshark.FileCapture(pcapfile, display_filter="ip && (tcp || udp)") if debug: print("[*] Parsing sessions...") i = 0 for packet in packets: if 'TCP' in packet: # Session beign if packet.tcp.flags_syn == '1' and packet.tcp.flags_ack == '1' and in_subnet( packet.ip.src, subnet) and not in_subnet(packet.ip.dst, subnet): tcp_sessions[packet.tcp.stream] = { 'client': packet.ip.dst, 'server': packet.ip.src, 'client_port': packet.tcp.dstport, 'server_port': packet.tcp.srcport, 'protocol': packet.highest_layer,
def FeatureExtraction(pcapPath, csvPath): caps = rdpcap(pcapPath) filteredCaps = pyshark.FileCapture(pcapPath, display_filter="tcp") FlowSource = None FlowDestination = None #print(caps.sessions().keys()) totalFwdPackets = 0 totalBwdPackets = 0 TotalLengthofFwdPackets = 0 TotalLengthofBwdPackets = 0 TotalLengthofFwdHeaders = 0 TotalLengthofBwdHeaders = 0 flowDuration = 0 Act_data_pkt_forward = 0 lengthofFwdPackets = [] lengthofBwdPackets = [] lengthofFlowPackets = [] lengthofFwdHeaders = [] lengthofBwdHeaders = [] deltaFwd = [] deltaBwd = [] deltaFlow = [] FwdCaps = [] bwdCaps = [] FilterdFwdCaps = [] FilterdBwdCaps = [] flowActive = [] flowIdle = [] FINFlagCount = 0 SYNFlagCount = 0 RSTFlagCount = 0 PSHFlagCount = 0 ACKFlagCount = 0 URGFlagCount = 0 CWEFlagCount = 0 ECEFlagCount = 0 fwdPSHFlags = 0 bwdPSHFlags = 0 fwdURGFlags = 0 bwdURGFlags = 0 sfLastPacketTS = -1 sfCount = 0 sfAcHelper = -1 threshold = 5000000 FIN = 0x01 SYN = 0x02 RST = 0x04 PSH = 0x08 ACK = 0x10 URG = 0x20 ECE = 0x40 CWE = 0x80 protocol = 6 flowPacketNumber = 0 fbulkDuration = 0 fbulkPacketCount = 0 fbulkSizeTotal = 0 fbulkStateCount = 0 fbulkPacketCountHelper = 0 fbulkStartHelper = 0 fbulkSizeHelper = 0 flastBulkTS = 0 bbulkDuration = 0 bbulkPacketCount = 0 bbulkSizeTotal = 0 bbulkStateCount = 0 bbulkPacketCountHelper = 0 bbulkStartHelper = 0 bbulkSizeHelper = 0 blastBulkTS = 0 bAvgBytesPerBulk = 0 bAvgPacketsPerBulk = 0 bAvgBulkRate = 0 fAvgBytesPerBulk = 0 fAvgPacketsPerBulk = 0 fAvgBulkRate = 0 time.strftime("%I:%M") time.strftime("%d/%m/%y") print(time.strftime("%I:%M")) flag = True flagBwd = True for i, cap in enumerate(caps): if TCP in cap: if i == 0: FwdSource = cap['IP'].src FwdDestination = cap['IP'].dst FlowSource = FwdSource FlowDestination = FwdDestination flowStartTime = cap.time * 1000000 flowLastSeen = cap.time * 1000000 startActive = cap.time * 1000000 endActiveTime = cap.time * 1000000 x = cap.time * 1000000 # timeStamp = datetime.datetime.strptime(str(x/1000), "%a %b %d %H:%M:%S %Y") if (sfLastPacketTS == -1): sfLastPacketTS = cap.time * 1000000 sfAcHelper = cap.time * 1000000 if ((cap.time - (sfLastPacketTS) / 1000000.0) > 1.0): sfCount += 1 lastSFduration = (cap.time - sfAcHelper) * 1000000 currentTime = (cap.time * 1000000 - sfLastPacketTS) if (currentTime - endActiveTime > threshold): if (endActiveTime - startActive > 0): flowActive.append(endActiveTime - startActive) flowIdle.append(currentTime - endActiveTime) endActiveTime = currentTime startActive = currentTime else: endActiveTime = currentTime sfAcHelper = cap.time * 1000000 sfLastPacketTS = cap.time * 1000000 #forwardBytes+=packet.getPayloadBytes() else: if (cap['IP'].src == FwdSource and cap['IP'].dst == FwdDestination): FwdCaps.append(cap) if (cap['IP'].dst == FwdSource and cap['IP'].src == FwdDestination): bwdCaps.append(cap) deltaFlow.append(abs(cap.time * 1000000 - flowLastSeen)) flowLastSeen = cap.time * 1000000 flowPacketNumber += 1 for i, filteredCap in enumerate(filteredCaps): lengthofFlowPackets.append(int(filteredCaps[i].length)) #FwdFlow if filteredCap['ip'].src == FwdSource and filteredCap[ 'ip'].dst == FwdDestination: if (i == 0): FlowSourcePort = filteredCap[ filteredCap.transport_layer].srcport FlowDestinationPort = filteredCap[ filteredCap.transport_layer].dstport FilterdFwdCaps.append(filteredCap) #BwdFlow if filteredCap['ip'].src == FwdDestination and filteredCap[ 'ip'].dst == FwdSource: FilterdBwdCaps.append(filteredCap) flowIATTotal = sum(deltaFlow) flowIATMean = numpy.mean(deltaFlow) flowIATStd = numpy.std(deltaFlow, ddof=1) flowIATMax = max(deltaFlow) flowIATMin = min(deltaFlow) flowDuration = (flowLastSeen - flowStartTime) for i, filteredCap in enumerate(FilterdFwdCaps): lengthofFwdPackets.append(int(FilterdFwdCaps[i].length)) if int(FilterdFwdCaps[i].length) >= 1: Act_data_pkt_forward += 1 for i, cap in enumerate(FwdCaps): if TCP in cap: size = lengthofFwdPackets[i] if flag == True: if (blastBulkTS > fbulkStartHelper): fbulkStartHelper = 0 if (size <= 0): flag = False lengthofFwdPackets[i] += 1 if (fbulkStartHelper == 0): fbulkStartHelper = cap.time * 1000000 fbulkPacketCountHelper = 1 fbulkSizeHelper = size flastBulkTS = cap.time * 1000000 else: if ((cap.time * 1000000 - flastBulkTS) / 1000000.0 > 1.0): fbulkStartHelper = cap.time * 1000000 flastBulkTS = cap.time * 1000000 fbulkPacketCountHelper = 1 fbulkSizeHelper = size else: fbulkPacketCountHelper += 1 fbulkSizeHelper += size if (fbulkPacketCountHelper == 4): fbulkStateCount += 1 fbulkPacketCount += fbulkPacketCountHelper fbulkSizeTotal += fbulkSizeHelper fbulkDuration += cap.time * 1000000 - fbulkStartHelper if (fbulkPacketCountHelper > 4): fbulkPacketCount += 1 fbulkSizeTotal += size fbulkDuration += cap.time * 1000000 - flastBulkTS flastBulkTS = cap.time * 1000000 if (i == 0): fwdWindowSize = cap[TCP].window F = cap['TCP'].flags if F & PSH: fwdPSHFlags += 1 if F & URG: fwdURGFlags += 1 if i != 0: forwardLastSeen = FwdCaps[i - 1].time * 1000000 deltaFwd.append(abs(cap.time * 1000000 - forwardLastSeen)) F = cap['TCP'].flags if F & FIN: FINFlagCount += 1 if F & SYN: SYNFlagCount += 1 if F & RST: RSTFlagCount += 1 if F & PSH: PSHFlagCount += 1 if F & ACK: ACKFlagCount += 1 if F & URG: URGFlagCount += 1 if F & ECE: ECEFlagCount += 1 if F & CWE: CWEFlagCount += 1 if (sfLastPacketTS == -1): sfLastPacketTS = cap.time * 1000000 sfAcHelper = cap.time * 1000000 if ((cap.time - (sfLastPacketTS) / 1000000.0) > 1.0): sfCount += 1 lastSFduration = (cap.time - sfAcHelper) * 1000000 currentTime = (cap.time * 1000000 - sfLastPacketTS) if (currentTime - endActiveTime > threshold): if (endActiveTime - startActive > 0): flowActive.append(endActiveTime - startActive) flowIdle.append(currentTime - endActiveTime) endActiveTime = currentTime startActive = currentTime else: endActiveTime = currentTime sfAcHelper = cap.time * 1000000 sfLastPacketTS = cap.time * 1000000 c = cap.show(dump=True) word = "len" index = c.find(word) if c[index + 12:index + 16] != ' ': lengthofFwdHeaders.append(int(c[index + 12:index + 16])) TotalLengthofFwdHeaders += int(c[index + 12:index + 16]) totalFwdPackets += 1 else: lengthofFwdHeaders.append(0) if i == 0: min_seg_size_forward = int(c[index + 12:index + 16]) else: min_seg_size_forward = min(min_seg_size_forward, int(c[index + 12:index + 16])) fwdPacketsPerSecond = totalFwdPackets / (flowDuration / 1000000) subFlowFwdPackets = totalFwdPackets / sfCount FwdIATTotal = sum(deltaFwd) FwdIATMean = numpy.mean(deltaFwd) FwdIATStd = numpy.std(deltaFwd, ddof=1) FwdIATMax = max(deltaFwd) FwdIATMin = min(deltaFwd) for i, filteredCap in enumerate(FilterdBwdCaps): lengthofBwdPackets.append(int(FilterdBwdCaps[i].length)) for i, cap in enumerate(bwdCaps): if TCP in cap: size = lengthofBwdPackets[i] if flagBwd == True: if (flastBulkTS > bbulkStartHelper): bbulkStartHelper = 0 if (size <= 0): flagBwd = False lengthofBwdPackets[i] += 1 if (bbulkStartHelper == 0): bbulkStartHelper = cap.time * 1000000 bbulkPacketCountHelper = 1 bbulkSizeHelper = size blastBulkTS = cap.time * 1000000 else: if ((cap.time * 1000000 - blastBulkTS) / 1000000.0 > 1.0): bbulkStartHelper = cap.time * 1000000 blastBulkTS = cap.time * 1000000 bbulkPacketCountHelper = 1 bbulkSizeHelper = size else: bbulkPacketCountHelper += 1 bbulkSizeHelper += size if (bbulkPacketCountHelper == 4): bbulkStateCount += 1 bbulkPacketCount += bbulkPacketCountHelper bbulkSizeTotal += bbulkSizeHelper bbulkDuration += cap.time * 1000000 - bbulkStartHelper if (bbulkPacketCountHelper > 4): bbulkPacketCount += 1 bbulkSizeTotal += size bbulkDuration += cap.time * 1000000 - flastBulkTS flastBulkTS = cap.time * 1000000 if (i == 0): bwdWindowSize = cap[TCP].window F = cap['TCP'].flags if F & PSH: fwdPSHFlags += 1 if F & URG: fwdURGFlags += 1 if i != 0: backwardLastSeen = bwdCaps[i - 1].time * 1000000 deltaBwd.append(abs(cap.time * 1000000 - backwardLastSeen)) F = cap['TCP'].flags if F & FIN: FINFlagCount += 1 if F & SYN: SYNFlagCount += 1 if F & RST: RSTFlagCount += 1 if F & PSH: PSHFlagCount += 1 bwdPSHFlags += 1 if F & ACK: ACKFlagCount += 1 if F & URG: URGFlagCount += 1 bwdURGFlags += 1 if F & ECE: ECEFlagCount += 1 if F & CWE: CWEFlagCount += 1 if (sfLastPacketTS == -1): sfLastPacketTS = cap.time * 1000000 sfAcHelper = cap.time * 1000000 if ((cap.time - (sfLastPacketTS) / 1000000.0) > 1.0): sfCount += 1 lastSFduration = (cap.time - sfAcHelper) * 1000000 currentTime = (cap.time * 1000000 - sfLastPacketTS) if (currentTime - endActiveTime > threshold): if (endActiveTime - startActive > 0): flowActive.append(endActiveTime - startActive) flowIdle.append(currentTime - endActiveTime) endActiveTime = currentTime startActive = currentTime else: endActiveTime = currentTime sfAcHelper = cap.time * 1000000 sfLastPacketTS = cap.time * 1000000 c = cap.show(dump=True) word = "len" index = c.find(word) if c[index + 12:index + 16] != ' ': lengthofFwdHeaders.append(int(c[index + 12:index + 16])) TotalLengthofBwdHeaders += int(c[index + 12:index + 16]) totalBwdPackets += 1 if (fbulkStateCount != 0): fAvgBytesPerBulk = fbulkSizeTotal / fbulkStateCount if (fbulkStateCount != 0): fAvgPacketsPerBulk = fbulkPacketCount / fbulkStateCount if (fbulkDuration != 0): fAvgBulkRate = fbulkSizeTotal / (fbulkDuration / 1000000) if (bbulkStateCount != 0): bAvgBytesPerBulk = bbulkSizeTotal / bbulkStateCount if (bbulkStateCount != 0): bAvgPacketsPerBulk = bbulkPacketCount / bbulkStateCount if (bbulkDuration != 0): bAvgBulkRate = bbulkSizeTotal / (bbulkDuration / 1000000) bwdPacketsPerSecond = totalBwdPackets / (flowDuration / 1000000) bwdIATTotal = sum(deltaBwd) bwdIATMean = numpy.mean(deltaBwd) bwdIATStd = numpy.std(deltaBwd, ddof=1) bwdIATMax = max(deltaBwd) bwdIATMin = min(deltaBwd) subFlowBwdPackets = totalBwdPackets / sfCount downUpRatio = totalBwdPackets / totalFwdPackets minPacketLength = min(lengthofFlowPackets) maxPacketLength = max(lengthofFlowPackets) packetLengthMean = numpy.mean(lengthofFlowPackets) packetLengthStd = numpy.std(lengthofFlowPackets, ddof=1) packetLengthvar = numpy.var(lengthofFlowPackets, ddof=1) avgPacketSize = sum(lengthofFlowPackets) / flowPacketNumber TotalLengthofFwdPackets = sum(lengthofFwdPackets) MaxLengthofFwdPackets = max(lengthofFwdPackets) MinLengthofFwdPackets = min(lengthofFwdPackets) FwdPacketLengthMean = numpy.mean(lengthofFwdPackets) FwdPacketLengthStd = numpy.std(lengthofFwdPackets, ddof=1) fwdAvgSegmentSize = TotalLengthofFwdPackets / totalFwdPackets subFlowFwdBytes = TotalLengthofFwdPackets / sfCount TotalLengthofBwdPackets = sum(lengthofBwdPackets) MaxLengthofBwdPackets = max(lengthofBwdPackets) MinLengthofBwdPackets = min(lengthofBwdPackets) bwdPacketLengthMean = numpy.mean(lengthofBwdPackets) bwdPacketLengthStd = numpy.std(lengthofBwdPackets, ddof=1) bwdAvgSegmentSize = TotalLengthofBwdPackets / totalBwdPackets subFlowBwdBytes = TotalLengthofBwdPackets / sfCount FlowID = FlowSource + '-' + FlowDestination + '-' + FlowSourcePort + '-' + FlowDestinationPort + '-' + str( protocol) packetdPerSecond = flowPacketNumber / flowDuration bytePerSecond = (TotalLengthofFwdHeaders + TotalLengthofBwdHeaders) / flowDuration if len(flowActive) != 0: activeMean = numpy.mean(flowActive) activeStd = numpy.std(flowActive, ddof=1) activeMax = max(flowActive) activeMin = min(flowActive) else: activeMax = 0 activeMean = 0 activeMin = 0 activeStd = 0 if len(flowIdle) != 0: idleMean = numpy.mean(flowIdle) idleStd = numpy.std(flowIdle, ddof=1) idleMax = max(flowIdle) idleMin = min(flowIdle) else: idleMax = 0 idleMean = 0 idleMin = 0 idleStd = 0 with open(csvPath, 'w') as myfile: wr = csv.writer(myfile) fieldnames = [ 'Flow ID', 'Source IP', 'Source Port', 'Destination IP', 'Destination Port', 'Protocol', 'Flow Duration', 'Total Fwd Packets', 'Total Bwd Packets', 'Total Length of Fwd Packets', 'Total Length of Bwd Packets', 'Fwd Packet Length Max', 'Fwd Packet Length Min', 'Fwd Packet Length Mean', 'Fwd Packet Length Std', 'Bwd Packet Length Max', 'Bwd Packet Length Min', 'Bwd Packet Length Mean', 'Bwd Packet Length Std', 'Fwd Packets/s', 'Bwd Packets/s', 'Flow Bytes/s', 'Flow Packets/s', 'Flow IAT Mean', 'Flow IAT Std', 'Flow IAT Max', 'Flow IAT Min', 'Fwd IAT Total', 'Fwd IAT Mean', 'Fwd IAT Std', 'Fwd IAT Max', 'Fwd IAT Min', 'Bwd IAT Total', 'Bwd IAT Mean', 'Bwd IAT Std', 'Bwd IAT Max', 'Bwd IAT Min', 'Fwd PSH Flags', 'Bwd PSH Flags', 'Fwd URG Flags', 'Bwd URG Flags', 'Fwd Header Length', 'Bwd Header Length', "Pkt Len Min", "Pkt Len Max", "Pkt Len Mean", "Pkt Len Std", "Pkt Len Var", 'FIN Flag Count', 'SYN Flag Count', 'RST Flag Count', 'PSH Flag Count', 'ACK Flag Count', 'URG Flag Count', 'CWE Flag Count', 'ECE Flag Count', 'Down/Up Ratio', 'Average Packet Size', ' Avg Fwd Segment Size', ' Avg Bwd Segment Size', 'Fwd Avg Bytes/Bulk', ' Fwd Avg Packets/Bulk', ' Fwd Avg Bulk Rate', ' Bwd Avg Bytes/Bulk', ' Bwd Avg Packets/Bulk', 'Bwd Avg Bulk Rate', 'Subflow Fwd Packets', ' Subflow Fwd Bytes', ' Subflow Bwd Packets', ' Subflow Bwd Bytes', 'Init_Win_bytes_forward', ' Init_Win_bytes_backward', ' act_data_pkt_fwd', ' min_seg_size_forward', 'Active Mean', ' Active Std', ' Active Max', ' Active Min', 'Idle Mean', ' Idle Std', ' Idle Max', ' Idle Min' ] writer = csv.DictWriter(myfile, fieldnames=fieldnames) writer.writeheader() writer.writerow({ 'Flow ID': FlowID, 'Source IP': FlowSource, 'Source Port': FlowSourcePort, 'Destination IP': FlowDestination, 'Destination Port': FlowDestinationPort, 'Protocol': protocol, 'Flow Duration': flowDuration, 'Total Fwd Packets': totalFwdPackets, 'Total Bwd Packets': totalBwdPackets, 'Total Length of Fwd Packets': TotalLengthofFwdPackets, 'Total Length of Bwd Packets': TotalLengthofBwdPackets, 'Fwd Packet Length Max': MaxLengthofFwdPackets, 'Fwd Packet Length Min': MinLengthofFwdPackets, 'Fwd Packet Length Mean': FwdPacketLengthMean, 'Fwd Packet Length Std': FwdPacketLengthStd, 'Bwd Packet Length Max': MaxLengthofBwdPackets, 'Bwd Packet Length Min': MinLengthofBwdPackets, 'Bwd Packet Length Mean': bwdPacketLengthMean, 'Bwd Packet Length Std': bwdPacketLengthStd, 'Flow Bytes/s': bytePerSecond, 'Flow Packets/s': packetdPerSecond, 'Flow IAT Mean': flowIATMean, 'Flow IAT Std': flowIATStd, 'Flow IAT Max': flowIATMax, 'Flow IAT Min': flowIATMin, 'Fwd IAT Total': FwdIATTotal, 'Fwd IAT Mean': FwdIATMean, 'Fwd IAT Std': FwdIATStd, 'Fwd IAT Max': FwdIATMax, 'Fwd IAT Min': FwdIATMin, 'Bwd IAT Total': bwdIATTotal, 'Bwd IAT Mean': bwdIATMean, 'Bwd IAT Std': bwdIATStd, 'Bwd IAT Max': bwdIATMax, 'Bwd IAT Min': bwdIATMin, 'Fwd PSH Flags': fwdPSHFlags, 'Bwd PSH Flags': bwdPSHFlags, 'Fwd URG Flags': fwdURGFlags, 'Bwd URG Flags': bwdURGFlags, 'Fwd Header Length': TotalLengthofFwdHeaders, 'Bwd Header Length': TotalLengthofBwdHeaders, 'Fwd Packets/s': fwdPacketsPerSecond, 'Bwd Packets/s': bwdPacketsPerSecond, "Pkt Len Min": minPacketLength, "Pkt Len Max": maxPacketLength, "Pkt Len Mean": packetLengthMean, "Pkt Len Std": packetLengthStd, "Pkt Len Var": packetLengthvar, 'FIN Flag Count': FINFlagCount, 'SYN Flag Count': SYNFlagCount, 'RST Flag Count': RSTFlagCount, 'PSH Flag Count': PSHFlagCount, 'ACK Flag Count': ACKFlagCount, 'URG Flag Count': URGFlagCount, 'CWE Flag Count': CWEFlagCount, 'ECE Flag Count': ECEFlagCount, 'Down/Up Ratio': downUpRatio, 'Average Packet Size': avgPacketSize, ' Avg Fwd Segment Size': fwdAvgSegmentSize, ' Avg Bwd Segment Size': bwdAvgSegmentSize, 'Fwd Avg Bytes/Bulk': fAvgBytesPerBulk, ' Fwd Avg Packets/Bulk': fAvgPacketsPerBulk, ' Fwd Avg Bulk Rate': fAvgBulkRate, ' Bwd Avg Bytes/Bulk': bAvgBytesPerBulk, ' Bwd Avg Packets/Bulk': bAvgPacketsPerBulk, 'Bwd Avg Bulk Rate': bAvgBulkRate, 'Subflow Fwd Packets': subFlowFwdPackets, ' Subflow Fwd Bytes': subFlowFwdBytes, ' Subflow Bwd Packets': subFlowBwdPackets, ' Subflow Bwd Bytes': subFlowBwdBytes, 'Init_Win_bytes_forward': fwdWindowSize, ' Init_Win_bytes_backward': bwdWindowSize, ' act_data_pkt_fwd': Act_data_pkt_forward, ' min_seg_size_forward': min_seg_size_forward, 'Active Mean': activeMean, ' Active Std': activeStd, ' Active Max': activeMax, ' Active Min': activeMin, 'Idle Mean': idleMean, ' Idle Std': idleStd, ' Idle Max': idleMax, ' Idle Min': idleMin }) features = numpy.array([ flowDuration, totalFwdPackets, totalBwdPackets, TotalLengthofFwdPackets, TotalLengthofBwdPackets, MaxLengthofFwdPackets, MinLengthofFwdPackets, FwdPacketLengthMean, FwdPacketLengthStd, MaxLengthofBwdPackets, MinLengthofBwdPackets, bwdPacketLengthMean, bwdPacketLengthStd, bytePerSecond, packetdPerSecond, flowIATMean, flowIATStd, flowIATMax, flowIATMin, FwdIATTotal, FwdIATMean, FwdIATStd, FwdIATMax, FwdIATMin, bwdIATTotal, bwdIATMean, bwdIATStd, bwdIATMax, bwdIATMin, fwdPSHFlags, bwdPSHFlags, fwdURGFlags, bwdURGFlags, TotalLengthofFwdHeaders, TotalLengthofBwdHeaders, fwdPacketsPerSecond, bwdPacketsPerSecond, minPacketLength, maxPacketLength, packetLengthMean, packetLengthStd, packetLengthvar, FINFlagCount, SYNFlagCount, RSTFlagCount, PSHFlagCount, ACKFlagCount, URGFlagCount, CWEFlagCount, ECEFlagCount, downUpRatio, avgPacketSize, fwdAvgSegmentSize, bwdAvgSegmentSize, fAvgBytesPerBulk, fAvgPacketsPerBulk, fAvgBulkRate, bAvgBytesPerBulk, bAvgPacketsPerBulk, bAvgBulkRate, subFlowFwdPackets, subFlowFwdBytes, subFlowBwdPackets, subFlowBwdBytes, fwdWindowSize, bwdWindowSize, Act_data_pkt_forward, min_seg_size_forward, activeMean, activeStd, activeMax, activeMin, idleMean, idleStd, idleMax, idleMin ]) return features
#Find hosts attempting a Directory Traversal import pyshark #First read the input source file. #Next we want to filter out only those packets that use http as the application protocol #and out of these packets we further filter out only those http requests where the host attempts a directory traversal. input_trace = pyshark.FileCapture( "project2.trace", display_filter='http and http.request.uri contains "../.."') #Create an empty set src_host = set() #Next we iterate over these packets and store the source IP address of all source hosts attempting a directory traversal. for packet in input_trace: src_host.add(packet.ip.src) print(sorted(src_host))
import pyshark import base64 cap = pyshark.FileCapture('somepang.pcap', display_filter='icmp.type==8') # filter by echo data = '' for packet in cap: data += packet['icmp'].data.decode('hex')[ -2:] # get last 2 chars from data f = open('flag.jpg', 'wb') f.write(base64.b64decode(data)) print 'done'
streams.add(pkt.tcp.stream) def decrypt_callback(pkt): global send global recv if hasattr(pkt, 'data'): if pkt.tcp.dstport == server: send += unhexlify(pkt.data.data) else: recv += unhexlify(pkt.data.data) shark = pyshark.FileCapture( 'misc_chall.pcapng', display_filter="tcp.port == 25565 and ip.addr == 123.207.121.32") shark.apply_on_packets(stream_callback) print(streams) for i in streams: send = b'' recv = b'' shark = pyshark.FileCapture('misc_chall.pcapng', display_filter=f"tcp.stream eq {i}") shark.apply_on_packets(decrypt_callback) decryptor = OpenSSLCrypto('aes-256-cfb', password, recv[:16], 0) data = decryptor.update(recv[16:])
plt.xticks(rotation=45) plt.xlabel("Time") plt.ylabel("KBytes") plt.show() # MAIN print("") printGreen("********************************************") printGreen("************* DEMO LIVECAPTURE *************") printGreen("********************************************") ''' file = inputYellow("Inserisci il nome del file di cattura: ") cap = pyshark.FileCapture(file) ''' cap = pyshark.FileCapture("prova.pcap") print("") # Lettura file di cattura nums = [] dates = [] count = 0 for pkt in cap: dates.append(float(pkt.frame_info.time_epoch)) nums.append(int(pkt.length) / 1000) count = count + 1 print("\r\033[F\033[K-- Pacchetti esaminati: " + "#" + str(count) + " " + str(int(pkt.length))) print(f"-- Il numero di pacchetti รจ: {len(nums)}") # Aggregazione dati su intervalli
if (os.path.isfile("{}{}_protocols.csv".format(pcapdir, file))): print("Skipping {}; pcap is already analyzed".format(file)) continue # Debug fileID += 1 print("Analyzing {} ({}/{} ({}%))".format(file, fileID, totalFiles, round(100/totalFiles*fileID))) # Setting up protocolCounters = {} protocolBytes = {} destinationIPCounters = {} # Reading packets in dir try: all_packets = pyshark.FileCapture(pcapdir + file, only_summaries=True) except: print("Error: Failed to read pcap file!") continue # Analyze packets print("Analyzing packets that were read...") i = 0 try: for packet in all_packets: # Read packet data packet = str(packet).split(" ") protocol = packet[4] bytes = int(packet[5]) s_ip = packet[2] # Find destination IP
def extract_attributes(src, dst, attacker_ip, split_flags=False, include_validation_attributes=False): pcap = pyshark.FileCapture(src_file_name, keep_packets=False) first = True with open(dst_file_name, "a") as csv_file: for packet in pcap: entry = {} if include_validation_attributes: for key in validation_attributes: entry[key] = '' for key in attributes: if 'flag_' in key and split_flags == False: continue entry[key] = '' try: entry['is_attack'] = 0 if include_validation_attributes: entry['timestamp'] = packet.sniff_time.strftime( '%m/%d/%Y, %H:%M:%S:%f') entry['protocol'] = packet.highest_layer if 'ip' in packet: if include_validation_attributes: entry['src_ip'] = packet.ip.src entry['dst_ip'] = packet.ip.dst if packet.ip.src == attacker_ip or packet.ip.dst == attacker_ip: entry['is_attack'] = 1 entry['ttl'] = packet.ip.ttl entry['ip_len'] = packet.ip.len if split_flags: entry['ip_flag_df'] = packet.ip.flags_df entry['ip_flag_mf'] = packet.ip.flags_mf entry['ip_flag_rb'] = packet.ip.flags_rb else: entry['ip_flags'] = packet.ip.flags if 'udp' in packet: entry['src_port'] = packet.udp.srcport entry['dst_port'] = packet.udp.dstport elif 'tcp' in packet: entry['src_port'] = packet.tcp.srcport entry['dst_port'] = packet.tcp.dstport if split_flags: entry['tcp_flag_res'] = packet.tcp.flags_res entry['tcp_flag_ns'] = packet.tcp.flags_ns entry['tcp_flag_cwr'] = packet.tcp.flags_cwr entry['tcp_flag_ecn'] = packet.tcp.flags_ecn entry['tcp_flag_urg'] = packet.tcp.flags_urg entry['tcp_flag_ack'] = packet.tcp.flags_ack entry['tcp_flag_push'] = packet.tcp.flags_push entry['tcp_flag_reset'] = packet.tcp.flags_reset entry['tcp_flag_syn'] = packet.tcp.flags_syn entry['tcp_flag_fin'] = packet.tcp.flags_fin else: entry['tcp_flags'] = packet.tcp.flags else: continue if 'mqtt' in packet: entry['mqtt_messagetype'] = packet.mqtt.msgtype entry['mqtt_messagelength'] = packet.mqtt.len if 'conflags' in packet.mqtt.field_names: if split_flags: entry[ 'mqtt_flag_uname'] = packet.mqtt.conflag_uname entry[ 'mqtt_flag_passwd'] = packet.mqtt.conflag_passwd entry[ 'mqtt_flag_retain'] = packet.mqtt.conflag_retain entry['mqtt_flag_qos'] = packet.mqtt.conflag_qos entry[ 'mqtt_flag_willflag'] = packet.mqtt.conflag_willflag entry[ 'mqtt_flag_clean'] = packet.mqtt.conflag_cleansess entry[ 'mqtt_flag_reserved'] = packet.mqtt.conflag_reserved else: entry['mqtt_flags'] = packet.mqtt.conflags writer = csv.DictWriter(csv_file, list(entry.keys()), delimiter=',') if first: writer.writeheader() first = False writer.writerow(entry) except Exception: traceback.print_exc() break pcap.close()
def main(): all_router_multicast = "ff02::1" first_iteration = True packet_type = "ICMPV6" time_window = 2.000000 path_to_pcap_file = '/root/final_RA.pcap' pcap = pyshark.FileCapture(path_to_pcap_file) prefix_list = [] source_IP_list = [] destination_IP_list = [] # create logger with 'spam_application' logger = logging.getLogger('RA_IDS_flow_based_time') logger.setLevel(logging.DEBUG) # create file handler which logs even debug messages fh = logging.FileHandler('RA_IDS_flow_based_time.log') fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logging.INFO) # create formatter and add it to the handlers formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) start_time = pcap[0].sniff_time for pkt in pcap: current_time = pkt.sniff_time elapsed_time = current_time - start_time if pkt.highest_layer == packet_type: logger.debug('Packet is type ICMPv6') if pkt.icmpv6.type == "134": logger.debug('Packet is type ICMPv6 Router Advertisement 134') var_prefix = str(pkt.icmpv6.opt_prefix) prefix_list.append(var_prefix.replace(":", "")) if pkt.ipv6.dst == all_router_multicast: logger.debug('Packet Flow to destination {}'.format( all_router_multicast)) var_dst = pkt.ipv6.dst destination_IP_list.append(var_dst.replace(":", "")) var_src = pkt.ipv6.src source_IP_list.append(var_src.replace(":", "")) else: pass logger.debug('Packet is NOT type ICMPv6') if elapsed_time.total_seconds() >= time_window: start_time = pkt.sniff_time logger.info('---------- Window Starts ----------') logger.info('Start time: {}'.format(start_time)) logger.info('Current time: {}'.format(current_time)) logger.info('Elapsed time: {}'.format(elapsed_time)) logger.info( 'Calculating Shannon\'s entropy for flow based - source IP') logger.info('Source_IP count: {}'.format(len(source_IP_list))) entropy_source_IP = numpy_shannon_entropy(source_IP_list) logger.info('Entropy for flow based - source IP u: {}\n'.format( entropy_source_IP)) logger.info( 'Calculating Shannon\'s entropy for flow based - destination IP' ) logger.info('Source_IP count: {}'.format(len(destination_IP_list))) entropy_destination_IP = numpy_shannon_entropy(destination_IP_list) logger.info( 'Entropy for flow based - destination IP u: {}\n'.format( entropy_destination_IP)) if first_iteration == True: logger.debug('In first iteration - calculating EWMA threshold') source_IP_threshold = ewma(ewma_lambda, entropy_source_IP, entropy_source_IP) previous_source_IP_threshold = source_IP_threshold logger.info( 'source_IP_threshold: {}\n'.format(source_IP_threshold)) first_iteration = False else: logger.debug( 'Not in first Iteration - calculating EWMA threshold') source_IP_threshold = ewma(ewma_lambda, entropy_source_IP, previous_source_IP_threshold) previous_source_IP_threshold = source_IP_threshold logger.info( 'source_IP_threshold: {}\n'.format(source_IP_threshold)) if entropy_source_IP > source_IP_threshold: logger.info( 'RA Flooding Detected - entropy of source IP > threshold\n' ) logger.info( 'entropy_source_IP: {}'.format(entropy_source_IP)) logger.info( 'prefix_threshold: {}\n'.format(source_IP_threshold)) # flush list prefix_list = [] source_IP_list = [] destination_IP_list = [] logger.info("---------- Window Ends ----------")
#Identify host(s) attempting a directory traversal. import pyshark #Input the trace file #The filter is meant to accept packets for the udp protocol with no application layer payload #Essentially, protocols like DNS also use UDP as a transport level protocol but we don't want to consider these packets. #Since traceroute uses UDP but does not use any transport level packets, want only those packets that use udp as a transport #but with not application layer payload. trace_input = pyshark.FileCapture("project2.trace", display_filter='udp and not dns and not browser and not mdns and not cups and not nbns and not auto_rp and not smb_netlogon') #declare an Empty dictionary. This structure will store the IP addresses of the source host and destination hosts ip_dict = {} #Now iterate over the packets from the trace_input. #We extract the source IP address and build a Dictionary whose each tuple if of the form (<source_ip>, <destination_ip>):<list_of_ttl_values> #Here the key is a UDP connection endpoint (<source_ip>, <destination_ip) while the value is a list of all ttl values. for packet in trace_input: ip_src = packet.ip.src ip_dst = packet.ip.dst #Get the IP endpoints for the trace route. ip_endpoints = (ip_src, ip_dst) ttl_dict = set() if ip_endpoints in ip_dict: ttl_dict = ip_dict[ip_endpoints] ttl = int(packet.ip.ttl) if ttl not in ttl_dict: ttl_dict.add(ttl) else: ttl_dict.add(int(packet.ip.ttl)) #Store the ttl value in the dictionary ip_dict[ip_endpoints] = ttl_dict
def perform_analysis(pid, idx, files_len, pcap_file): print("P%s (%s/%s): Processing pcap file \"%s\"..." % (pid, idx, files_len, pcap_file)) cap = pyshark.FileCapture(pcap_file, use_json=True) Utils.sysUsage("PCAP file loading") cap.close() base_ts = 0 try: if args.no_time_shift: cap[0] else: base_ts = float(cap[0].frame_info.time_epoch) except KeyError: print(c.NO_PCKT % pcap_file, file=sys.stderr) return node_id = Node.NodeId(args.mac_addr, args.ip_addr) node_stats = Node.NodeStats(node_id, base_ts, devices) print(" P%s: Processing packets..." % pid) try: for packet in cap: node_stats.processPacket(packet) except: print( " %sP%s: Error: There is something wrong with \"%s\". Skipping file.%s" % (RED, pid, pcap_file, END), file=sys.stderr) return del cap Utils.sysUsage("Packets processed") print(" P%s: Mapping IP to host..." % pid) ip_map = IP.IPMapping() if args.hosts_dir != "": host_file = args.hosts_dir + "/" + os.path.basename( pcap_file)[:-4] + "txt" ip_map.extractFromFile(pcap_file, host_file) else: ip_map.extractFromFile(pcap_file) ip_map.loadOrgMapping(IP_TO_ORG) ip_map.loadCountryMapping(IP_TO_COUNTRY) Utils.sysUsage("TShark hosts loaded") print(" P%s: Generating CSV output..." % pid) de = DataPresentation.DomainExport(node_stats.stats.stats, ip_map, GEO_DB_CITY, GEO_DB_COUNTRY) de.loadDiffIPFor("eth") if args.find_diff else de.loadIPFor("eth") de.loadDomains(args.dev, args.lab, args.experiment, args.network, pcap_file, str(base_ts)) de.exportDataRows(args.out_file) print(" P%s: Analyzed data from \"%s\" successfully written to \"%s\"" % (pid, pcap_file, args.out_file)) Utils.sysUsage("Data exported") if len(plots) != 0: print(" P%s: Generating plots..." % pid) pm = DataPresentation.PlotManager(node_stats.stats.stats, plots) pm.ipMap = ip_map pm.generatePlot(pid, pcap_file, args.fig_dir, GEO_DB_CITY, GEO_DB_COUNTRY) Utils.sysUsage("Plots generated")
def __init__(self, pcap_file_name, pcap_parser_engine="scapy"): """ Init function imports libraries based on the parser engine selected Return: * packetDB ==> Full Duplex Packet Streams - Used while finally plotting streams as graph - dump packets during report generation * lan_hosts ==> Private IP (LAN) list - device details * destination_hosts ==> Destination Hosts - communication details - tor identification - malicious identification """ # Initialize Data Structures memory.packet_db = {} memory.lan_hosts = {} memory.destination_hosts = {} memory.possible_mal_traffic = [] memory.possible_tor_traffic = [] # Set Pcap Engine self.engine = pcap_parser_engine # Import library for pcap parser engine selected if pcap_parser_engine == "scapy": try: from scapy.all import rdpcap except: logging.error("Cannot import selected pcap engine: Scapy!") sys.exit() try: from scapy.all import load_layer global tls_view_feature tls_view_feature = True logging.info("tls view feature enabled") except: logging.info("tls view feature not enabled") if tls_view_feature: load_layer("tls") # Scapy sessions and other types use more O(N) iterations so just # - use rdpcap + our own iteration (create full duplex streams) self.packets = rdpcap(pcap_file_name) elif pcap_parser_engine == "pyshark": try: import pyshark except: logging.error("Cannot import selected pcap engine: PyShark!") sys.exit() self.packets = pyshark.FileCapture(pcap_file_name, include_raw=True, use_json=True) #self.packets.load_packets() #self.packets.apply_on_packets(self.analyse_packet_data, timeout=100) # Analyse capture to populate data self.analyse_packet_data()
def count_packets(): cap = pyshark.FileCapture('http.cap', keep_packets=False) cap.apply_on_packets(counter, timeout=10000) return len(packets_array)
def load_packets(filename): pack=pyshark.FileCapture(filename,display_filter='dns') return pack
import pyshark cap = pyshark.FileCapture("mbti.pcap") for i in cap: if 'ssl' in i: if 'app_data' in dir(i.ssl): data = i.ssl.app_data.replace(':', '').decode('hex') print "APP DATA ", i.ip.src, len(data)
def extract_features(file, time_interval): # open the capture file and set up the parameters for the sliding time window td = datetime.timedelta( seconds=time_interval ) # set the timedelta of the interval for window sliding cap = pyshark.FileCapture(file) startTime = cap[0].sniff_time endTime = cap[0].sniff_time + td #print(startTime.strftime("%A, %d. %B %Y %I:%M.%S%p")) #print(endTime.strftime("%A, %d. %B %Y %I:%M.%S%p")) # prepare the feature dictionary and list of the features at every time window startCount = 1 endCount = 1 startWindow = cap[0].sniff_time + datetime.timedelta(seconds=START_WINDOW) endWindow = cap[-1].sniff_time - datetime.timedelta(seconds=END_WINDOW) # percentage calculations totalCount = 0 TLSCount = 0 STUNCount = 0 TCPCount = 0 ICMPCount = 0 UDPCount = 0 # host number calculations UDPHosts = [] TLSHosts = [] STUNHosts = [] TCPHosts = [] ftrs = Features(startTime, time_interval) featureArray = [] # start processing the packets while True: # get the next packet from capture try: currpkt = cap.next() except StopIteration: break try: _ = currpkt.ip except AttributeError: continue t = currpkt.sniff_time totalCount += 1 # check the protocol of the packet, increment counts or save hosts if 'SSL' in currpkt: TLSCount += 1 host = currpkt.ip.host if host not in TLSHosts: TLSHosts.append(host) if 'TCP' in currpkt: TCPCount += 1 host = currpkt.ip.host if host not in TCPHosts: TCPHosts.append(host) if 'UDP' in currpkt: UDPCount += 1 host = currpkt.ip.host if host not in UDPHosts: UDPHosts.append(host) if 'STUN' in currpkt: STUNCount += 1 host = currpkt.ip.host if host not in STUNHosts: STUNHosts.append(host) if 'HTTP' in currpkt: ftrs.HTTPExists = 1 if 'DTLS' in currpkt: ftrs.DTLSExists = 1 if 'ICMP' in currpkt: ftrs.ICMPExists = 1 ICMPCount += 1 # check atStart and atEnd conditions if t < startWindow: if 'DNS' in currpkt: ftrs.DNSAtStart = 1 if 'STUN' in currpkt: ftrs.firstSTUNMessageTime = 1 elif t > endWindow: if 'DNS' in currpkt: ftrs.DNSAtEnd = 1 # check the time window if endTime < t: # update startTime and endTime startTime = endTime endTime = startTime + td # finalize the feature list, prepare a new list # calculate numberOfXHosts fields for HMM length = len(UDPHosts) if length < 11: ftrs.numOfUDPHosts = 0 elif length < 21: ftrs.numOfUDPHosts = 1 else: ftrs.numOfUDPHosts = 2 length = len(TLSHosts) if length < 11: ftrs.numOfTLSHosts = 0 elif length < 21: ftrs.numOfTLSHosts = 1 else: ftrs.numOfTLSHosts = 2 length = len(TCPHosts) if length < 11: ftrs.numOfTCPHosts = 0 elif length < 21: ftrs.numOfTCPHosts = 1 else: ftrs.numOfTCPHosts = 2 length = len(STUNHosts) if length < 11: ftrs.numOfSTUNHosts = 0 elif length < 21: ftrs.numOfSTUNHosts = 1 else: ftrs.numOfSTUNHosts = 2 # calculate percentages perc = TLSCount / totalCount if perc < 1: ftrs.TLSMessagePercentage = 0 elif perc < 4: ftrs.TLSMessagePercentage = 1 else: ftrs.TLSMessagePercentage = 2 perc = STUNCount / totalCount if perc < 1: ftrs.STUNMessagePercentage = 0 elif perc < 4: ftrs.STUNMessagePercentage = 1 else: ftrs.STUNMessagePercentage = 2 perc = TCPCount / totalCount if perc < 1: ftrs.TCPMessagePercentage = 0 elif perc < 4: ftrs.TCPMessagePercentage = 1 else: ftrs.TCPMessagePercentage = 2 rate = STUNCount / (t - cap[0].sniff_time).total_seconds() rate /= 1000 if rate < 0.1: ftrs.STUNMessageRate = 0 elif rate < 0.2: ftrs.STUNMessageRate = 1 else: ftrs.STUNMessageRate = 2 rate = ICMPCount / (t - cap[0].sniff_time).total_seconds() rate /= 1000 if rate < 0.1: ftrs.ICMPMessageRate = 0 elif rate < 0.2: ftrs.ICMPMessageRate = 1 else: ftrs.ICMPMessageRate = 2 rate = UDPCount / (t - cap[0].sniff_time).total_seconds() rate /= 1000 if rate < 0.1: ftrs.UDPDatagramRate = 0 elif rate < 0.2: ftrs.UDPDatagramRate = 1 else: ftrs.UDPDatagramRate = 2 featureArray.append(ftrs) ftrs = Features(startTime, time_interval) try: # set the features that will be carried on for all iterations oldftrs = featureArray[-1] if oldftrs.DNSAtStart: ftrs.DNSAtStart = 1 if oldftrs.DNSAtEnd: ftrs.DNSAtEnd = 1 if oldftrs.DTLSExists: ftrs.DTLSExists = 1 if oldftrs.HTTPExists: ftrs.HTTPExists = 1 if oldftrs.ICMPExists: ftrs.ICMPExists = 1 if oldftrs.firstSTUNMessageTime: ftrs.firstSTUNMessageTime = 1 except IndexError: pass print("Packets {}-{} processed".format(startCount, endCount)) startCount = currpkt.frame_info.number endCount += 1 #print(a.sniff_time.strftime("%A, %d. %B %Y %I:%M.%S%p")) print("Extraction finished.") return featureArray