def parse_pcap_files(self, pcapFiles, quite=True):
"""
Take one more more (list, or tuple) of pcap files and parse them
into the engine.
"""
if not hasattr(pcapFiles, '__iter__'):
if isinstance(pcapFiles, str):
pcapFiles = [pcapFiles]
else:
return
for i in range(0, len(pcapFiles)):
pcap = pcapFiles[i]
pcapName = os.path.split(pcap)[1]
if not quite:
sys.stdout.write("Reading PCap File: {0}\r".format(pcapName))
sys.stdout.flush()
if not os.path.isfile(pcap):
if not quite:
sys.stdout.write("Skipping File {0}: File Not Found\n".format(pcap))
sys.stdout.flush()
continue
elif not os.access(pcap, os.R_OK):
if not quite:
sys.stdout.write("Skipping File {0}: Permissions Issue\n".format(pcap))
sys.stdout.flush()
continue
pcapr = PcapReader(pcap) # pylint: disable=no-value-for-parameter
packet = pcapr.read_packet()
i = 1
try:
while packet:
if not quite:
sys.stdout.write('Parsing File: ' + pcap + ' Packets Done: ' + str(i) + '\r')
sys.stdout.flush()
self.parse_wireless_packet(packet)
packet = pcapr.read_packet()
i += 1
i -= 1
if not quite:
sys.stdout.write((' ' * len('Parsing File: ' + pcap + ' Packets Done: ' + str(i))) + '\r')
sys.stdout.write('Done With File: ' + pcap + ' Read ' + str(i) + ' Packets\n')
sys.stdout.flush()
except KeyboardInterrupt:
if not quite:
sys.stdout.write("Skipping File {0} Due To Ctl+C\n".format(pcap))
sys.stdout.flush()
except: # pylint: disable=bare-except
if not quite:
sys.stdout.write("Skipping File {0} Due To Scapy Exception\n".format(pcap))
sys.stdout.flush()
self.fragment_buffer = {}
pcapr.close()
def simulate(self, pcap_file_name):
print("Simulation starts!")
print("Reading pcap file: " + pcap_file_name)
pcap_reader = PcapReader(pcap_file_name)
statistic_stop_time = 1
pkt_time_base = -1
stop_time = 0
while True:
pkt = pcap_reader.read_packet()
if pkt is None:
break
if not pkt.haslayer('Ether') or not pkt.haslayer(
'IP') or not pkt.haslayer('TCP'):
continue
if pkt_time_base == -1:
pkt_time_base = pkt.time
pkt_time = pkt.time - pkt_time_base
# print("pkt.time %f" % pkt_time)
while pkt_time > stop_time:
for i in range(self.device_count):
(self.entry_num_per_sec[i]).append(
len(self.ip2hc_tables[i]))
print("%d len: %d" % (i, len(self.entry_num_per_sec[i])))
print("stop time %d" % stop_time)
stop_time += 1
self.__learn_host(pkt['IP'].src, pkt['IP'].dst, pkt['TCP'].sport,
pkt['TCP'].dport, pkt['IP'].ttl)
# print("time:%f length:%d " % (pkt_time, len(self.entry_num_per_sec[0])))
pcap_reader.close()
return self.entry_num_per_sec
def process(csvfile, filename, DATA_LENGTH):
#计数
count = 0
#定义csv写入器
writer = csv.writer(csvfile)
writer.writerow(["data"])
#定义pcap读入器
pr = PcapReader(filename)
#循环一行一行数据读入，直到为空
while True:
pcap_data = pr.read_packet()
#空则退出
if pcap_data is None:
break
else:
ether_data = np.fromstring(pcap_data.original, np.uint8, count=14)
type = ether_data[12:14]
# 去除Ethernet包头，取IP包
ip_data = pcap_data.payload
#转换成0-255的数
ipdata = np.fromstring(ip_data.original, np.uint8)
#求数据长度
length = len(ipdata)
#去IP包头
communication_data = ip_data.payload
# 定义一个tmp数据变量，用于存储扩展udp包头与Tcp包头一样长后的udp数据，定义为1500长度，并且填充0，用不补0
tmp_data = np.zeros(shape=[1500], dtype=np.uint8)
# 查找UDP包头的pcap
if communication_data.name == 'UDP':
#ip包长+udp包头长=28字节，因此取前28
tmp_data[0:27] = ipdata[0:27]
#后面补12个0，使其长度与IP+TCP的40字节一致
#后面接上，UDP包长度超过1488 = 1500 - 12的部分去掉，因为补12个0与TCP包头对齐
if length > DATA_LENGTH - 12:
tmp_data[28:] = ipdata[28:DATA_LENGTH - 12]
else:
tmp_data[28:length] = ipdata[28:length]
else:
# TCP保持不变，超过1500的部分去掉
if length > DATA_LENGTH:
tmp_data = ipdata[0:DATA_LENGTH]
else:
tmp_data[0:length] = ipdata[0:length]
#用规范模型对数据进行规范化
X_train = model.transform(np.reshape(tmp_data, newshape=[-1, 1]))
# 写入时写入一行，因此transpose
writer.writerow(np.transpose(X_train)[0])
count = count + 1
if count == 100:
break
# 关闭scapy读入器，防止内存泄露
pr.close()
#反馈文件中的数据总数
return count
def _run_on_pcap(self, args):
reader = PcapReader(args.input_file)
try:
while True:
self.handle_packet(reader.read_packet())
except EOFError:
pass
reader.close()
def modify_traffic(self, output_filename, fake_ttl=False,
src_mac=None, dst_mac=None, src_ip_addr=None, dst_ip_addr=None):
if fake_ttl is True and len(self.src_hosts_with_fake_ttl) == 0:
print("Please extract ip2hc table before modify traffic with fake ttl.")
# show modify request
request = "Generating " + output_filename + " with\n"
if fake_ttl:
request += " fake ttl\n"
if src_mac is not None:
request += " src mac:" + src_mac + "\n"
if dst_mac is not None:
request += " dst mac:" + dst_mac + "\n"
if src_ip_addr is not None:
request += " src ip addr:" + src_ip_addr + "\n"
if dst_ip_addr is not None:
request += " dst ip addr:" + dst_ip_addr + "\n"
print(request + "\n")
pcap_reader = PcapReader(self.pcap_filename)
pcap_writer = PcapWriter(output_filename)
counter = 0
while True:
pkt = pcap_reader.read_packet()
if pkt is None:
break
if not pkt.haslayer('Ether') or not pkt.haslayer('IP'):
continue
# ipv4 packets
counter += 1
ip_int = self.__ip_str2int(pkt['IP'].src)
if fake_ttl:
pkt['IP'].ttl = self.src_hosts_with_fake_ttl[ip_int]
if src_mac is not None:
pkt['Ethernet'].src = src_mac
if dst_mac is not None:
pkt['Ethernet'].dst = dst_mac
if src_ip_addr is not None:
pkt['IP'].src = src_ip_addr
if dst_ip_addr is not None:
pkt['IP'].dst = dst_ip_addr
pcap_writer.write(pkt)
if counter % 10000 == 0:
print("%d packets have been processed\n" % counter)
pcap_writer.flush()
pcap_writer.close()
pcap_reader.close()
def arp_analyse(self, files_path):
# count = 0
for i in range(0, len(files_path)):
try:
packets = PcapReader(files_path[i])
while True:
packet = packets.read_packet()
if packet is None:
break
else:
# count = count+1
print(repr(packet))
packets.close()
except Scapy_Exception as e:
print(e)
def analyse(self, files_path):
count = 0
for i in range(0, len(files_path)):
try:
packets = PcapReader(files_path[i])
while True:
packet = packets.read_packet()
if packet is None:
break
else:
count = count + 1
print(repr(packet))
# print(packet['DNS'].id)
# for packet in packets:
# print(repr(packet))
# #print(type(packet))
# #print(packet['CookedLinux'].src)
# break
print(count)
packets.close()
except Scapy_Exception as e:
print(e)
def extract_ip2hc_table(self):
self.counter = 0
print("Reading pcap file: " + self.pcap_filename)
# src_hosts is a dict
pcap_reader = PcapReader(self.pcap_filename)
while True:
pkt = pcap_reader.read_packet()
if pkt is None:
break
if not pkt.haslayer('Ether') or not pkt.haslayer('IP'):
continue
self.counter += 1
# progress indicator
if self.counter % 10000 == 0:
print(str(self.counter) + "packets have been processed")
# ipv4 packets
ip_int = self.__ip_str2int(pkt['IP'].src)
if ip_int in self.src_hosts:
continue
# unrecorded source host
hc = self.__ttl2hc(pkt['IP'].ttl)
fake_ttl = self.__gen_fake_ttl(pkt['IP'].ttl)
if hc is not None and fake_ttl is not None:
self.src_hosts[ip_int] = hc
self.src_hosts_with_fake_ttl[ip_int] = fake_ttl
pcap_reader.close()
print(str(len(self.src_hosts)) + " source hosts extracted")
print("Writing ip,hc dict into " + self.ip_hc_filename + "...")
with open(self.ip_hc_filename, "w") as f:
json.dump(self.src_hosts, f)
f.close()
def extract_url_from_pcap(input_f, output_file):
st = time.time()
print('process ... \'%s\'' % input_f, flush=True)
# Step 1. read from pcap and do not return a list of packets
try:
load_layer("tls") # enable scapy can parse tls
myreader = PcapReader(
input_f
) # iterator, please use it to process large file, such as more than 4 GB
except MemoryError as me:
print('memory error ', me)
return -1
except FileNotFoundError as fnfe:
print('file not found ', fnfe)
return -2
except:
print('other exceptions')
return -10
res_dict = {}
while True:
pkt = myreader.read_packet()
if pkt is None:
break
cnt = 0
# step 1. parse "Ethernet" firstly
if pkt.name == "Ethernet":
if pkt.payload.name.upper() in ['IP', 'IPV4']:
if pkt.payload.payload.name.upper() in ["TCP", "UDP"]:
if pkt.payload.payload.payload.name.upper() in 'TLS' and "Client Hello" in \
pkt.payload.payload.payload.msg[0].name:
five_tuple = pkt.payload.src + ':' + str(
pkt.payload.payload.sport
) + '-' + pkt.payload.dst + ':' + str(
pkt.payload.payload.dport
) + '-' + pkt.payload.payload.name.upper()
try:
if 'servernames' in pkt.payload.payload.payload.msg[
0].ext[00].fields.keys():
url_str = pkt.payload.payload.payload.msg[
0].ext[00].fields['servernames'][0].fields[
'servername']
print('packet[0] info: "%s:%d-%s:%d-%s"+%s' % (
pkt.payload.src, pkt.payload.payload.sport,
pkt.payload.dst, pkt.payload.payload.dport,
pkt.payload.payload.name, url_str))
if url_str not in res_dict.keys():
res_dict[url_str] = [five_tuple]
else:
res_dict[url_str].append(five_tuple)
print(
f'\'{url_str}\' appears: {len(res_dict[url_str])} times'
)
except TypeError as e:
print(f'{five_tuple}')
else: # step 2. if this pkt can not be recognized as "Ethernet", then try to parse it as (IP,IPv4)
pkt = IP(
pkt
) # without ethernet header, then try to parse it as (IP,IPv4)
if pkt.payload.name.upper() in ["TCP", "UDP"]:
if "Client Hello" in pkt.payload.payload.msg[0].name:
five_tuple = pkt.payload.src + ':' + str(
pkt.payload.payload.sport
) + '-' + pkt.payload.dst + ':' + str(
pkt.payload.payload.dport
) + '-' + pkt.payload.payload.name.upper()
url_str = pkt.payload.payload.msg[0].ext[00].fields[
'servernames'][0].fields['servername']
print('packet[0] info: "%s:%d-%s:%d-%s"+%s' %
(pkt.src, pkt.payload.sport, pkt.dst,
pkt.payload.dport, pkt.payload.name, url_str))
if url_str not in res_dict.keys():
res_dict[url_str] = [five_tuple]
else:
res_dict[url_str].append(five_tuple)
print(
f'\'{url_str}\' appears: {len(res_dict[url_str])} times'
)
save_to_file(output_file, input_f, res_dict)
def read_file(filename, interval_in_s):
start_time = -1
stop_time = interval
sender_reader = PcapReader(filename)
SYN_pps = []
ACK_pps = []
UDP_pps = []
normal_pps = []
SYN_count_in_interval = 0
ACK_count_in_interval = 0
UDP_count_in_interval = 0
normal_count_in_interval = 0
while True:
pkt = sender_reader.read_packet()
if pkt is None:
break
if not pkt.haslayer('Ether') or not pkt.haslayer('IP'):
continue
if start_time == -1:
start_time = pkt.time
time = pkt.time - start_time
while time > stop_time:
if stop_time != interval:
if SYN_pps[-1] \
+ ACK_pps[-1] \
+ UDP_pps[-1] == 0 \
and \
SYN_count_in_interval \
+ ACK_count_in_interval \
+ UDP_count_in_interval != 0:
print("time:%f, stop_time: %f" % (time, stop_time))
print("SYN %d ACK %d UDP %d normal %d" %
(SYN_count_in_interval, ACK_count_in_interval,
UDP_count_in_interval, normal_count_in_interval))
elif SYN_pps[-1] \
+ ACK_pps[-1] \
+ UDP_pps[-1] != 0 \
and \
SYN_count_in_interval \
+ ACK_count_in_interval \
+ UDP_count_in_interval == 0:
print("time:%f, stop_time: %f" % (time, stop_time))
print("SYN %d ACK %d UDP %d normal %d" %
(SYN_count_in_interval, ACK_count_in_interval,
UDP_count_in_interval, normal_count_in_interval))
normal_pps.append(normal_count_in_interval / 10.0)
SYN_pps.append(SYN_count_in_interval / 10.0)
ACK_pps.append(ACK_count_in_interval / 10.0)
UDP_pps.append(UDP_count_in_interval / 10.0)
stop_time += interval
SYN_count_in_interval = 0
ACK_count_in_interval = 0
UDP_count_in_interval = 0
normal_count_in_interval = 0
if pkt['Ether'].src == "00:00:00:00:00:11":
# spoofed!
if pkt.haslayer('UDP'):
UDP_count_in_interval += 1
elif pkt.haslayer('TCP'):
if pkt['TCP'].flags == 0x02:
SYN_count_in_interval += 1
elif pkt['TCP'].flags == 0x10:
ACK_count_in_interval += 1
else:
# unspoofed!
normal_count_in_interval += 1
sender_reader.close()
return (SYN_pps, ACK_pps, UDP_pps, normal_pps), stop_time - interval
def main(cli_arguments: argparse.Namespace) -> None:
"""
Connects to multiple hosts and reads the output from their tcpdump processes
:param cli_arguments: arguments passed to the main function
"""
log_level = logging.DEBUG if cli_arguments.debug else logging.INFO
logging.basicConfig(level=log_level)
remote_processes = []
for host in cli_arguments.hosts:
parent_pipe, child_pipe = multiprocessing.Pipe()
process = RemoteTcpDump(
cli_arguments.filter,
host,
child_pipe,
log_level=log_level,
username=cli_arguments.username,
pem_file=cli_arguments.pem_file,
echo_only=cli_arguments.echo_only,
sudo=cli_arguments.sudo,
pcap=True if cli_arguments.pcap else False,
)
remote_processes.append((process, parent_pipe, child_pipe))
process.start()
# Close it here so that it is closed on both ends
child_pipe.close()
pcap_file = io.BytesIO()
pcap_reader = None
pcap_file_header_length = 24
pcap_reader_exit_pos = False
while True:
try:
for process, parent_pipe, _ in remote_processes:
if cli_arguments.pcap:
for integer in parent_pipe.recv_bytes():
byte = int(integer).to_bytes(1, byteorder='big')
pcap_file.write(byte)
pcap_file_write_pos = pcap_file.tell()
# Read the headers and start the pcap reader instance
if pcap_file_write_pos > pcap_file_header_length and not pcap_reader_exit_pos:
pcap_file.seek(0)
pcap_reader = PcapReader(pcap_file)
# Keep this for later
pcap_reader_exit_pos = pcap_file.tell()
if pcap_reader_exit_pos:
# Put the file in state that pcap reader expects
pcap_file.seek(pcap_reader_exit_pos)
packet = pcap_reader.read_packet()
if packet:
# Only on successful packet, store position for next iteration
pcap_reader_exit_pos = pcap_file.tell()
print(packet.time)
# Return to original write position for next iteration
pcap_file.seek(pcap_file_write_pos)
else:
for line in parent_pipe.recv_bytes().decode("utf-8").split(
"\n"):
print(f'{process.remote_host}: {line}')
except KeyboardInterrupt:
logging.debug("Parent: Received CTRL+C, stopping processing")
break
for process, _, _ in remote_processes:
process.join()
if pcap_file:
pcap_file.close()
#!/usr/bin/env python
# coding=utf-8
from scapy.utils import PcapReader, PcapWriter
from scapy.all import *
import sys
filename = sys.argv[1]
print("Reading " + filename)
pkt_reader = PcapReader(filename)
# pcap_reader = PcapReader("./p1p1_500w.pcap")
ip_pool = []
count = 0
while True:
pkt = pkt_reader.read_packet()
if pkt is None:
break
if not pkt.haslayer('Ether') or not pkt.haslayer('IP'):
continue
if pkt['IP'].src not in ip_pool:
ip_pool.append(pkt['IP'].src)
count = count + 1
if count % 10000 == 0:
print("Reading line " + str(count))
print("File " + filename + " has " + str(len(ip_pool)) + " distice IPs.")
def pcap2_parser(input_f):
"""
:param input_f:
:return:
"""
try:
# pkts_lst = rdpcap(input_f) # this will read all packets in memory at once, please don't use it directly.
# input_f = '/home/kun/PycharmProjects/pcap_process_scapy/pcaps_data/vpn_hangouts_audio2.pcap' #
myreader = PcapReader(input_f) # iterator, please use it to process large file, such as more than 4 GB
except MemoryError as me:
print('memory error ', me)
return -1
except FileNotFoundError as fnfe:
print('file not found ', fnfe)
return -2
except:
print('other exceptions')
return -10
# Step 2. achieve all the session in pcap.
# input_data.stats
pkts_stats = {'non_Ether_IPv4_pkts': 0, 'non_IPv4_pkts': 0, 'non_TCP_UDP_pkts': 0, 'TCP_pkts': 0,
'UDP_pkts': 0}
cnt = 0
sess_dict = {}
first_print_flg = True
while True:
pkt = myreader.read_packet()
if pkt is None:
break
# step 1. parse "Ethernet" firstly
if pkt.name == "Ethernet":
if first_print_flg:
first_print_flg = False
print('\'%s\' encapsulated by "Ethernet Header" directly' % input_f)
if pkt.payload.name.upper() in ['IP', 'IPV4']:
if pkt.payload.payload.name.upper() in ["TCP", "UDP"]:
if cnt == 0:
print('packet[0] info: "%s:%d-%s:%d-%s"+%s' % (
pkt.payload.src, pkt.payload.payload.sport, pkt.payload.dst, pkt.payload.payload.dport,
pkt.payload.payload.name, pkt.payload.payload.payload))
### ...
if pkt.payload.payload.name.upper() == "TCP":
pkts_stats['TCP_pkts'] += 1
else:
pkts_stats['UDP_pkts'] += 1
else:
pkts_stats['non_TCP_UDP_pkts'] += 1
# pkts_stats['IPv4_pkts'] += 1
else:
pkts_stats['non_IPv4_pkts'] += 1
else: # step 2. if this pkt can not be recognized as "Ethernet", then try to parse it as (IP,IPv4)
pkt = IP(pkt) # without ethernet header, then try to parse it as (IP,IPv4)
if first_print_flg:
first_print_flg = False
print('\'%s\' encapsulated by "IP Header" directly, without "Ethernet Header" ' % input_f)
if pkt.name.upper() in ['IP', 'IPV4']:
if pkt.payload.name.upper() in ["TCP", "UDP"]:
if cnt == 0:
print('packet[0] info: "%s:%d-%s:%d-%s"+%s' % (
pkt.src, pkt.payload.sport, pkt.dst, pkt.payload.dport,
pkt.payload.name, pkt.payload.payload))
### ...
if pkt.payload.name.upper() == "TCP":
pkts_stats['TCP_pkts'] += 1
else:
pkts_stats['UDP_pkts'] += 1
else:
pkts_stats['non_TCP_UDP_pkts'] += 1
# pkts_stats['IPv4_pkts'] += 1
else:
pkts_stats['non_IPv4_pkts'] += 1
# print('unknown packets type!',pkt.name)
pkts_stats['non_Ether_IPv4_pkts'] += 1
print('packet info:"srcIP:srcPort-dstIP:dstPort-prtcl" + IP_payload')
# -*- coding:utf-8 -*-
import time
from scapy.all import *
from scapy.utils import PcapReader, PcapWriter
if __name__ == '__main__':
time1 = time.time()
s1 = PcapReader(r'C:\Users\hasee\Desktop\tcppcap.pcap')
fpwrite = open(r'resultbyscapy.txt', 'w')
count = 0
while (True):
count += 1
data = s1.read_packet()
if (data == None):
break
if (data['Ethernet'].type == 0x0800):
if (data['IP'].proto == 6):
fpwrite.write(
'序号:%d\nIP协议版本:4\nIP源地址:%s\nIP目的地址:%s\n源端口:%d\n目的端口:%d\n\n'
% (count, data['IP'].src, data['IP'].dst,
data['TCP'].sport, data['TCP'].dport))
fpwrite.close()
s1.close()
time2 = time.time()
print(time2 - time1)
