def load_data(self):
import os
import os.path as path
import numpy as np
import scapy.all as sp
import h5py as hp
from tqdm import tqdm
if self.h5db_path is None:
with sp.PcapReader(self.pcap_path) as reader:
return reader.read_all(self.max_packets)
if path.exists(self.h5db_path):
if self.verbose:
print(f"Loading: pcap='{self.pcap_path}' h5='{self.h5db_path}'")
self.h5db = hp.File(self.h5db_path, "r")
return self.h5db[self.h5ds_name]
dirpath = path.dirname(self.h5db_path)
if len(dirpath) == 0:
dirpath = "."
os.makedirs(dirpath, exist_ok=True)
with hp.File(self.h5db_path, "w") as db, sp.PcapReader(self.pcap_path) as packets:
if self.verbose:
print(f"Caching: pcap='{self.pcap_path}' h5='{self.h5db_path}'")
cache = None
processed_packets = 0
iterable = enumerate(packets)
if self.verbose:
iterable = tqdm(iterable)
for idx, packet in iterable:
if self.max_packets and processed_packets >= self.max_packets:
break
packet_numpy = self.preprocess(packet)
if packet_numpy is None:
continue
processed_packets += 1
if cache is None:
dims = packet_numpy.shape
cache = db.create_dataset(self.h5ds_name, shape=(1, *dims), maxshape=(None, *dims), dtype=np.int, chunks=True)
if processed_packets >= len(cache):
cache.resize(processed_packets + 1, axis=0)
cache[processed_packets] = packet_numpy
self.h5db = hp.File(self.h5db_path, "r")
return self.h5db[self.h5ds_name]
def _pcap_reader_instance(self, bpf_filter):
if sys.platform == "darwin":
self._warning_logger("Need tcpdump from Brew for filter to work")
if bpf_filter:
return scapy.PcapReader(
scapy.tcpdump(self._pcap_filepath,
args=["-w", "-", bpf_filter],
getfd=True))
else:
return scapy.PcapReader(self._pcap_filepath)
def compare(x, y):
xr = scapy.PcapReader(str(x))
yr = scapy.PcapReader(str(y))
n = 0
xp = xr.read_packet()
yp = yr.read_packet()
while xp and yp:
packet_crawl(xp, yp, n)
xp = xr.read_packet()
yp = yr.read_packet()
n = +1
assert not xp and not yp, "Unequal pcap length"
def export_signal_strength(self, src_ip, save_path):
is_all = src_ip == 'all'
with open(save_path, 'w') as f:
f.write(
"Src IP,Transmit Mac, Transmit IP, Dst IP,"
"Signal Strength, Time of the day, Raw time (seconds), Micros\n"
)
for packet in sp.PcapReader(self.path):
dels_sec = packet.time - self.exp_day
dels_mins = dels_sec / 60
secs = dels_sec % 60
micro = int((secs - int(secs)) * 10**6)
secs = int(secs)
mins = int(dels_mins % 60) + self.ist_mins
hours = int(dels_mins / 60) + self.ist_hour
timestamp = dt.datetime(year=2018,
month=10,
day=13,
hour=hours,
minute=mins,
second=int(secs),
microsecond=micro)
if packet.haslayer(sp.IP) and packet.haslayer(sp.RadioTap):
if packet[sp.Dot11].fields['addr2'] == self.IP_Mac_Map[
src_ip] or is_all:
f.write(
packet[sp.IP].fields['src'] + "," +
packet[sp.Dot11].fields['addr2'] + "," + src_ip +
"," + packet[sp.IP].fields['dst'] + "," +
str(packet[sp.RadioTap].fields['dBm_AntSignal']) +
"," + str(timestamp) + "," + str(packet.time) +
"," + str(int(micro)) + "\n")
def export_inter_arrival_time(self, save_path):
old = 0
with open(save_path, 'w') as f:
f.write(
"Time (milli), Time of the day, Raw time (seconds), Micros\n")
for packet in sp.PcapReader(self.path):
dels_sec = packet.time - self.exp_day
dels_mins = dels_sec / 60
secs = dels_sec % 60
micro = int((secs - int(secs)) * 10**6)
secs = int(secs)
mins = int(dels_mins % 60) + self.ist_mins
hours = int(dels_mins / 60) + self.ist_hour
timestamp = dt.datetime(year=2018,
month=10,
day=13,
hour=hours,
minute=mins,
second=int(secs),
microsecond=micro)
delta = packet.time - old
delta = delta * 1000
old = packet.time
f.write(
str(delta) + "," + str(timestamp) + "," +
str(packet.time) + "," + str(int(micro)) + "\n")
def analyse_pcap(pcap_file, ip_addr):
prev = 0
for packet in scapy.PcapReader(pcap_file):
# Weed out everything which is not a TCP SYN segment.
if (not scapy.TCP in packet) or (not packet[scapy.TCP].flags == 2):
continue
# Add timestamp and port for our reference IP address.
if packet[scapy.IP].src == ip_addr:
if prev == 0: prev = packet.time
decoy_conns[packet[scapy.TCP].dport] = packet.time
print int(packet.time) - int(prev)
prev = packet.time
continue
# Figure out time delta.
time = decoy_conns.get(packet[scapy.TCP].dport)
if time is not None:
delta = packet.time - time
del decoy_conns[packet[scapy.TCP].dport]
time_deltas[packet[scapy.TCP].dport] = delta
# Dump CSV to stdout.
print "delay, port"
ports = time_deltas.keys()
for port in ports:
print "%.3f, %d" % (time_deltas[port], port)
def extract_data(path):
print('------ extracting data:')
raw_datas = []
with scapy.PcapReader(path) as pcap_reader:
i = 0
for p in pcap_reader:
ip = None
if p.haslayer(scapy.IP):
ip = p.getlayer(scapy.IP)
item = {
"no": i + 1,
"time": p.time,
"src": (ip and ip.src) or p.src,
"sport": 0,
"dst": (ip and ip.dst) or p.dst,
"dport": 0
# "len": len(scapy.corrupt_bytes(p)),
# "protocal": p.summary().split('/')[2].split(' ')[1]
}
if ip and (p.haslayer(scapy.TCP) or p.haslayer(scapy.UDP)):
item["sport"] = ip.sport
item["dport"] = ip.dport
raw_datas.append(item)
i = i + 1
if (i % 10000 == 1):
print('extracting count: ', i)
return np.array(raw_datas)
def export_arrival_rate(self, save_path):
count = 0
old_time = 0
with open(save_path, 'w') as f:
f.write("Start Time,Rate\n")
for packet in sp.PcapReader(self.path):
dels_sec = packet.time - self.exp_day
dels_mins = dels_sec / 60
secs = dels_sec % 60
micro = int((secs - int(secs)) * 10**6)
secs = int(secs)
mins = int(dels_mins % 60) + self.ist_mins
hours = int(dels_mins / 60) + self.ist_hour
timestamp = dt.datetime(year=2018,
month=10,
day=13,
hour=hours,
minute=mins,
second=secs,
microsecond=micro)
if old_time == 0:
old_time = timestamp
count = 1
continue
else:
if (timestamp - old_time) > dt.timedelta(seconds=1):
f.write(str(old_time) + "," + str(count) + "\n")
old_time = timestamp
count = 1
else:
count += 1
def split_pcap(filename, session_list, parent_thread = None):
protocol_map = {"00000006" : "tcp", "00000011" : "udp"}
files = {}
cap_dir = "captures"
if not os.path.exists(cap_dir):
os.makedirs(cap_dir)
for session_tuple in session_list:
sip = hex_to_ipv4(session_tuple[0])
sp = hex_to_port(session_tuple[1])
dip = hex_to_ipv4(session_tuple[2])
dp = hex_to_port(session_tuple[3])
prot = protocol_map[session_tuple[4]]
files[session_tuple] = [cap_dir, f"{prot}_{sip}.{sp}-{dip}.{dp}.pcap"]
size_counter = 24 # PCAP file header
pcap_size = os.stat(filename).st_size
analyzed_percent = 0
with scapy.PcapReader(filename) as packets:
for packet in packets:
size_counter += 16 # PCAP packet header
size_counter += len(packet)
if parent_thread:
current_percent = int(size_counter * 100 // pcap_size)
if current_percent > analyzed_percent:
analyzed_percent = current_percent
parent_thread.update_progress(analyzed_percent)
if packet.haslayer(scapy.TCP) and packet.haslayer(scapy.IP):
ip1 = ipv4_to_hex(packet[scapy.IP].src)
ip2 = ipv4_to_hex(packet[scapy.IP].dst)
p1 = port_to_hex(packet[scapy.TCP].sport)
p2 = port_to_hex(packet[scapy.TCP].dport)
proto = "00000006"
session1 = (ip1, p1, ip2, p2, proto)
session2 = (ip2, p2, ip1, p1, proto)
if session1 in files.keys():
filename = pathlib.PurePath(files[session1][0], files[session1][1])
scapy.wrpcap(filename, packet, append=True)
elif session2 in files.keys():
filename = pathlib.PurePath(files[session2][0], files[session2][1])
scapy.wrpcap(filename, packet, append=True)
elif packet.haslayer(scapy.UDP) and packet.haslayer(scapy.IP):
ip1 = ipv4_to_hex(packet[scapy.IP].src)
ip2 = ipv4_to_hex(packet[scapy.IP].dst)
p1 = port_to_hex(packet[scapy.UDP].sport)
p2 = port_to_hex(packet[scapy.UDP].dport)
proto = "00000011"
session1 = (ip1, p1, ip2, p2, proto)
session2 = (ip2, p2, ip1, p1, proto)
if session1 in files.keys():
filename = pathlib.PurePath(files[session1][0], files[session1][1])
scapy.wrpcap(filename, packet, append=True)
elif session2 in files.keys():
filename = pathlib.PurePath(files[session2][0], files[session2][1])
scapy.wrpcap(filename, packet, append=True)
return files
def read_pcap(self):
"""Load a pcap. The state is set to scapy
Returns:
Chepy: The Chepy object.
"""
pcap_path = str(self._abs_path(self.state))
self._pcap_read = scapy.PcapReader(pcap_path)
pcap_file = scapy.rdpcap(pcap_path)
self.state = GREEN("Pcap loaded")
self._pcap_sessions = pcap_file.sessions(full_duplex)
return self
def ip_scrape(pcap, outfile):
"""
Scrapes pcap and outputs data in yaml format
:param pcap: path to pcap
:type pcap: Path
:param outfile: path to output yaml
:type outfile: Path
"""
pcap=str(pcap)
## Build up crawlers
ipd = IPDetector()
mpd = MacAssociations()
_George = Bush()
_George.fs += [ipd, mpd] ## Crawlers like to hide in bushes
_TCPnator = TCPTimestampMapper()
packets = scapy.PcapReader(pcap)
# packet = packets.read_packet() # read next packet
pnum=0
for packet in packets: # empty packet == None
_TCPnator(packet,pnum)
crawl(packet, _George, pnum=pnum) ## Crawl
# packet = packets.read_packet() # read next packet
_George.next() ## cleanup
pnum+=1
packets.close()
## Filter the ignored addresses
for m in ignored_macs:
mpd.mac_ip_map.pop(m, None)
for i in ignored_ips:
ipd.ip_protocol_map.pop(i, None)
## Build up output
ipd.ips = list(ipd.ip_protocol_map.keys()) ## keys are set of all ips
output = {
'ip.groups' : {
'source' : []
,'intermediate' : ipd.ips
, 'destination' : []
}
, 'tcp.timestamp.min' : [{'ip':key, 'min':val} for key,val in _TCPnator.min_map.items()]
, 'ip.searched_protocols' : [ {'ip':key, 'protocols':list(val)} for key,val in ipd.ip_protocol_map.items() ]
#### Black magic to move key from {key:value} into into value
, 'ip.occurrences' : [val for key, val in ipd.ip_pktnum_map.items() if (lambda x,y : x.update(ip=y))(val, key) is None ]
, 'mac.associations' : [{'mac' : key, 'ips' : list(val)} for key, val in mpd.mac_ip_map.items()]
# , 'mac.orphaned' : list(set(ipd.orphaned_mac))
}
with outfile.open('w') as ff:
yaml.dump(output, ff)
def read_pcap(path, lfilter=None):
"""Read pcap file and yields packets.
Args:
path (str): path to pcap file
lfilter (function, optional): function to filter returned packets. By
default all packets will be returned.
Yields:
obj: packet
"""
with scapy.PcapReader(path) as reader:
for packet in filter(lfilter, reader):
yield packet
def analyse_pcap(pcap_file):
print "x, y"
for packet in scapy.PcapReader(pcap_file):
# Weed out SYN retransmissions.
if is_retransmission(packet):
continue
for opt_name, opt_val in packet[scapy.TCP].options:
if opt_name == "Timestamp":
print packet.time, opt_val[0]
def main():
try:
pcap_file = sys.argv[1]
except Exception:
return 'Argument: [pcap_file]'
sys.exit(1)
reader = sc.PcapReader(pcap_file)
for pkt in reader:
parsed = parse_packet(pkt)
if parsed:
print json.dumps(parsed)
def main():
itr_list = []
for pcap_file in sys.argv[1:]:
itr_list.append(sc.PcapReader(pcap_file))
while itr_list:
itr = itr_list.pop(0)
try:
pkt = itr.next()
except StopIteration:
continue
itr_list.append(itr)
rewrite_pkt(pkt)
def pcap_usb_keyboard(self, layout: str = "qwerty"):
"""Decode usb keyboard pcap
Args:
layout (str, optional): Layout of the keyboard. Defaults to "qwerty".
Raises:
TypeError: If layout is not qwerty or dvorak
Returns:
Chepy: The Chepy object.
"""
if layout == "qwerty":
key_map = PcapUSB.qwerty_map
shift_modifier = PcapUSB.qwerty_modifier
elif layout == "dvorak": # pragma: no cover
key_map = PcapUSB.dvorak
shift_modifier = PcapUSB.dvorak_modifier
else: # pragma: no cover
raise TypeError("Valid layouts are qwerty and dvorak")
packets = scapy.PcapReader(self._pcap_filepath)
hold = []
for packet in packets:
if not scapy.Raw in packet: # pragma: no cover
continue
load = packet.getlayer("Raw").load
key_press = binascii.hexlify(load)[-16:]
if key_press == "0000000000000000": # pragma: no cover
continue
shift, _, key = re.findall(b".{2}", key_press)[0:3]
shift_pressed = bool(shift == b"02")
pressed = key_map.get(key.decode())
if shift_pressed:
special = shift_modifier.get(key.decode())
if special:
hold.append(special)
elif pressed:
hold.append(pressed.upper())
elif pressed:
hold.append(pressed)
self.state = "".join(hold)
return self
def pcap_to_X(pcapFile):
pkts = scapy.rdpcap(pcapFile)
X_tmp = np.empty((len(pkts), 230, 90))
i = 0
with scapy.PcapReader(pcapFile) as pcap_reader:
for pkt in pcap_reader:
http_payload = get_http_payload(pkt)
http_payload.remove_payload()
tmp = str(bytes(http_payload), encoding='utf-8')
# 不足230个字符的以空格填充
tmp = tmp.ljust(230, ' ')
# 取前230个字符
tmp = tmp[:230]
encoded = onehot_encode(tmp)
np_encoded = np.array(encoded)
X_tmp[i] = np_encoded
i += 1
print(i)
return X_tmp
def split_pcap(self, filename, rate):
package_pr = scapy.PcapReader(filename)
package_one = []
i = 1
while (True):
package = package_pr.read_packet()
if package is None:
break
package_one.append(package)
package_pr.close()
length = len(package_one)
i = 0
final_len = int(length * length)
t_writer = PcapWriter('/home/wxw/data/Ethernetip/' + 'modbus_pure' +
'.pcap',
append=True)
for p in package_one[0:final_len]:
t_writer.write(p)
t_writer.flush()
t_writer.close()
def get_youtube_ips(pcap_file):
with sp.PcapReader(pcap_file) as trace:
for packet in trace:
# DNS Packet
if packet.haslayer(sp.UDP) and packet[sp.UDP].sport == 53:
# Get DNS data
raw = sp.raw(packet[sp.UDP].payload)
# Process the DNS query
dns = dnslib.DNSRecord.parse(raw)
# Iterate over answers
for a in dns.rr:
# Check if it's a domain of interest (domain.com)
question = str(a.rname)
if any(s in question for s in YT_DOMAINS):
# Check if it's an answer
if a.rtype == 1 or a.rtype == 28:
print(
"Query {} is a Youtube one. Appending IP {} to Youtube IPs"
.format(question, a.rdata))
youtube_ips.append(str(a.rdata))
# youtube_ips object:
print("Youtube IPs: {}".format(youtube_ips))
return youtube_ips
def analyse_pcap(pcap_file):
old_port = 0
for packet in scapy.PcapReader(pcap_file):
# Weed out SYN retransmissions.
if (not scapy.TCP in packet) or (not packet[scapy.TCP].flags == 2):
continue
if packet[scapy.IP].src == "211.155.86.135":
continue
print packet[scapy.IP].src
continue
if is_retransmission(packet):
continue
t = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(packet.time))
#diff = packet[scapy.TCP].sport - old_port
#if diff < 0:
# diff = 65535 - abs(diff)
#print diff
#old_port = packet[scapy.TCP].sport
#for opt_name, opt_val in packet[scapy.TCP].options:
# if opt_name == "Timestamp":
#print packet.time, opt_val[0]
if packet[scapy.IP].src == "211.155.86.135":
print "%s, %d, 1" % (t, packet[scapy.TCP].dport)
else:
print "%s, %d, 0" % (t, packet[scapy.TCP].dport)
def get(self):
"""
:param bad_filename: 数据包存储位置，需要是csv格式
:param bad_pcap_filename: pcap包读取位置
:param num: 读取数据包的数量
:return: None
"""
PD = PcapDecode()
scapy.load_layer('tls')
with open(self.bad_filename, 'a') as f:
with scapy.PcapReader(self.bad_pcap_filename) as packets:
for i, pkt in enumerate(packets):
data = PD.ether_decode(pkt)
[
f.write("{},".format(value))
for key, value in data.items()
]
# [f.write("{}:{}, ".format(key, value)) for key, value in data.items()]
f.write("bad\n")
# if (i % 200 == 0):
# print("目前已处理{0}个数据包.".format(i))
if i == self.num:
print("已经处理{}个恶意文件数据包".format(i))
return True
def noise_remove(self, filename, protocolname, t_lo):
package_two = FileCapture(filename)
package_pr = scapy.PcapReader(filename)
package_one = []
i = 1
while (True):
package = package_pr.read_packet()
if package is None:
break
package_one.append(package)
package_pr.close()
package_three = []
length = len(package_one)
i = 0
while (i < length):
if package_two[i].layers[t_lo].layer_name == protocolname:
package_three.append(package_one[i])
i = i + 1
t_writer = PcapWriter('/home/wxw/data/' + 'modbus_pure' + '.pcap',
append=True)
for p in package_three:
t_writer.write(p)
t_writer.flush()
t_writer.close()
def get_flow_traffic_corrected(pcap_file, youtube_ips):
traffic_data = []
interval = 1.0
with sp.PcapReader(pcap_file) as trace:
start_time = 0
end_time = -1
slot_dict = {"start": start_time, "end": end_time, "flows": {}}
print("Processing slot {}-{}".format(start_time, end_time))
for packet in trace:
if not packet.haslayer(sp.IP):
continue
if packet[sp.IP].time > end_time:
#reset your countes
traffic_data.append(slot_dict)
if start_time == 0:
start_time = packet[sp.IP].time
else:
start_time = end_time
end_time = start_time + interval
slot_dict = {"start": start_time, "end": end_time, "flows": {}}
print("Processing slot {}-{}".format(start_time, end_time))
# If it belongs to Youtube's traffic
if packet.haslayer(sp.TCP) and (packet[sp.IP].src in youtube_ips or
packet[sp.IP].dst in youtube_ips):
key = ''
# identify the direction
if packet[sp.IP].src in youtube_ips:
dir = 1
key = "{}:{}:{}:{}:TCP".format(packet[sp.IP].src,
packet[sp.TCP].sport,
packet[sp.IP].dst,
packet[sp.TCP].dport)
elif packet[sp.IP].dst in youtube_ips:
dir = 0
key = "{}:{}:{}:{}:TCP".format(packet[sp.IP].dst,
packet[sp.TCP].dport,
packet[sp.IP].src,
packet[sp.TCP].sport)
# UPD traffic:
if packet.haslayer(sp.UDP) and (packet[sp.IP].src in youtube_ips or
packet[sp.IP].dst in youtube_ips):
key = ''
# identify the direction
if packet[sp.IP].src in youtube_ips:
dir = 1
key = "{}:{}:{}:{}:UDP".format(packet[sp.IP].src,
packet[sp.UDP].sport,
packet[sp.IP].dst,
packet[sp.UDP].dport)
elif packet[sp.IP].dst in youtube_ips:
dir = 0
key = "{}:{}:{}:{}:UDP".format(packet[sp.IP].dst,
packet[sp.UDP].dport,
packet[sp.IP].src,
packet[sp.UDP].sport)
if key not in slot_dict["flows"]:
slot_dict["flows"][key] = counters()
if dir == 1:
slot_dict["flows"][key]['in_pkts'] += 1
slot_dict["flows"][key]['in_bytes'] += packet[sp.IP].len
else:
slot_dict["flows"][key]['out_pkts'] += 1
slot_dict["flows"][key]['out_bytes'] += packet[sp.IP].len
return traffic_data
def rewrapping(pcap, res_path, param_dict, rewrap, timestamp_next_pkt,
timestamp):
"""
Parsing and rewrapping (and writing) of attack pcap.
:param attack: Mix
:param param_dict: parsed config, dict
:param rewrap: Rewrapper
"""
## check for readwrite mode
readwrite = None
rw = param_dict.get('read.write')
if rw:
readwrite = rw
else: ## default
readwrite = 'sequence'
## Create empty starting file
scapy.wrpcap(res_path, [])
## read & write all at once
pkt_num = 0
pkt_end = 0
pkt_ts = []
if readwrite == 'bulk':
## read all packets
packets = scapy.rdpcap(pcap)
res_packets = []
## timestamp shift based on first packet and input param
if timestamp is None:
rewrap.set_timestamp_shift(timestamp_next_pkt - packets[0].time)
else:
rewrap.set_timestamp_shift(timestamp_next_pkt - timestamp)
## rewrapp packets
for packet in packets:
try:
rewrap.digest(packet, recursive=True)
except Exception as e:
print('Error while digesting packet num {}'.format(pkt_num))
raise e
res_packets.append(packet)
pkt_num += 1
pkt_num = len(res_packets)
pkt_end = res_packets[-1].time
pkt_ts = [i.time for i in res_packets]
scapy.wrpcap(res_path, res_packets)
## read & write packet by packet
elif readwrite == 'sequence':
## create packet reader
packets = scapy.PcapReader(pcap)
## temporary list, avoid recreating lists for writing
tmp_l = [0]
pkt_num = 0
# packet = packets.read_packet() # read next packet
pktdump = None
pkt_ts = []
for packet in packets: # empty packet == None
tmp_l[0] = packet # store current packet for writing
try:
if pkt_num == 0: # first packet
if timestamp is None:
rewrap.set_timestamp_shift(timestamp_next_pkt -
packet.time)
else:
rewrap.set_timestamp_shift(timestamp_next_pkt -
timestamp)
rewrap.digest(packet, recursive=True)
## Create new pcap
scapy.wrpcap(res_path, packet)
else:
rewrap.digest(packet, recursive=True)
## Apend to existing pcap
scapy.wrpcap(res_path, packet, append=True)
except Exception as e:
print('Error while digesting packet num {}'.format(pkt_num))
raise e
pkt_num += 1
pkt_end = packet.time
pkt_ts.append(pkt_end)
# packet = packets.read_packet() # read next packet
return {
'packets.count': pkt_num,
'packets.start': 0,
'packets.end': pkt_end
# , 'packets.timestamps' : pkt_ts
}
def processCap(fileName, filename2):
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
rawTuple_read = open(filename2, "r")
rawTuple_dic = json.load(rawTuple_read)
#print("CCCCCCCC")
global check_dic
global length
myreader = scapy.PcapReader(fileName)
#pkts = scapy.rdpcap(fileName)
for key in rawTuple_dic:
dic_ = {}
dic_["client_sessionID"] = None
dic_["server_sessionID"] = None
dic_["sessionID_eq"] = None
dic_["ssl_state"] = rawTuple_dic[key]["ssl_dic"]["sslConnection"]
dic_["conn_state"] = rawTuple_dic[key]["conn_dic"]["connectionState"]
dic_["certificate"] = "F"
dic_["serverKey"] = "F"
dic_["serverChangeCipherSpec"] = "F"
check_dic[key] = dic_
print("************************************************")
#only process client hello packet
#for pkt in pkts:
i = 0
while True:
try:
pkt = myreader.read_packet()
if pkt is None:
break
# i = i + 1
# print(i)
if pkt.haslayer("IP"):
srcIP = pkt["IP"].src
dstIP = pkt["IP"].dst
if srcIP == "10.0.2.15":
if pkt.haslayer("TCP"):
sport = pkt["TCP"].sport
dport = pkt["TCP"].dport
key = srcIP + "/" + str(
sport) + "/" + dstIP + "/" + str(dport)
if pkt.haslayer(TLSClientHello):
clienthello = pkt.getlayer(TLSClientHello)
id_length = clienthello.session_id_length
if id_length != 0:
check_dic[key][
"client_sessionID"] = clienthello.session_id
elif dstIP == "10.0.2.15":
srcIP = pkt["IP"].src
dstIP = pkt["IP"].dst
if dstIP == "10.0.2.15":
if pkt.haslayer(TCP):
sport = pkt["TCP"].sport
dport = pkt["TCP"].dport
key = dstIP + "/" + str(
dport) + "/" + srcIP + "/" + str(sport)
if pkt.haslayer(TLSRecord):
layer1 = pkt.getlayer(TLSCiphertext)
layer1.display()
# print(layer2)
print("AAAAA")
handshakes = pkt.getlayer(TLSHandshakes)
for handshake in handshakes:
if handshake.haslayer(TLSServerHello):
print("aaa")
serverhello = pkt.getlayer(
TLSServerHello)
id_length = serverhello.session_id_length
if id_length != 0:
check_dic[key][
"server_sessionID"] = serverhello.session_id
elif pkt.haslayer(TLSCertificate):
print("bbbb")
check_dic[key]["certificate"] = "T"
elif pkt.haslayer(TLSServerKeyExchange):
print("cccccccc")
check_dic[key]["serverKey"] = "T"
elif pkt.haslayer(TLSChangeCipherSpec):
check_dic[key][
"serverChangeCipherSpec"] = "T"
else:
pass
else:
pass
except:
break
def rewrapping(attack, param_dict, rewrap, timestamp_next_pkt):
"""
Parsing and rewrapping (and writing) of attack pcap.
:param attack: Mix
:param param_dict: parsed config, dict
:param rewrap: Rewrapper
"""
## check for readwrite mode
rw = param_dict.get('read.write')
if rw:
attack.readwrite = rw
else: ## default
attack.readwrite = 'sequence'
## read & write all at once
if attack.readwrite == 'bulk':
## read all packets
packets = scapy.rdpcap(attack.attack_file)
## timestamp shift based on first packet and input param
rewrap.set_timestamp_shift(timestamp_next_pkt - packets[0].time)
## rewrapp packets
for packet in packets:
rewrap.digest(packet)
attack.packets.append(packet)
attack.pkt_num = len(attack.packets)
## read & write packet by packet
elif attack.readwrite == 'sequence':
## create packet reader
packets = scapy.PcapReader(attack.attack_file)
## temporary list, avoid recreating lists for writing
tmp_l = [0]
attack.pkt_num = 0
# packet = packets.read_packet() # read next packet
for packet in packets: # empty packet == None
tmp_l[0] = packet # store current packet for writing
if attack.pkt_num == 0: # first packet
rewrap.set_timestamp_shift(timestamp_next_pkt - packet.time)
rewrap.digest(packet)
attack.attack_start_utime = packet.time
## Create new pcap
attack.path_attack_pcap = attack.write_attack_pcap(
packets=tmp_l)
else:
rewrap.digest(packet)
attack.attack_end_utime = packet.time
## Apend to existing pcap
attack.write_attack_pcap(
packets=tmp_l,
append_flag=True,
destination_path=attack.path_attack_pcap)
attack.pkt_num += 1
def get_session_list(filename, parent_thread = None):
log = logging.getLogger("splitter.analyze")
log.info(f"Trying to analyze {filename} file")
sessions = {}
size_counter = 24 # PCAP file header
pcap_size = os.stat(filename).st_size
log.info(f"{pcap_size} bytes received")
analyzed_percent = 0
with scapy.PcapReader(filename) as packets:
for packet in packets:
size_counter += 16 # PCAP packet header
size_counter += len(packet)
if parent_thread:
current_percent = int(size_counter * 100 // pcap_size)
if current_percent > analyzed_percent:
analyzed_percent = current_percent
parent_thread.update_progress(analyzed_percent)
if packet.haslayer(scapy.TCP) and packet.haslayer(scapy.IP):
payload_size = len(packet[scapy.TCP].payload)
packet_size = len(packet.payload)
ip1 = ipv4_to_hex(packet[scapy.IP].src)
ip2 = ipv4_to_hex(packet[scapy.IP].dst)
p1 = port_to_hex(packet[scapy.TCP].sport)
p2 = port_to_hex(packet[scapy.TCP].dport)
proto = "00000006" # TCP
flags = "{0:012b}".format(packet[scapy.TCP].flags)
if int(flags[10]) and not int(flags[7]): # SYN with no ACK
session = (ip1, p1, ip2, p2, proto)
if session in sessions.keys():
if sessions[session][2:4] == [True, True]:
sessions[session][5] = True
elif sessions[session][4]:
sessions[session][5] = True
else:
sessions[session] = [True, False, False, False, False, False, packet_size, 1, payload_size]
elif int(flags[10]) and int(flags[7]): # SYN-ACK
session = (ip2, p2, ip1, p1, proto)
if session in sessions.keys():
if sessions[session][1] or sessions[session][2] or sessions[session][3]:
if sessions[session][1:5] == [True, False, False, False]:
pass
else:
print(f"SYN-ACK {sessions[session]}")
else:
sessions[session][1] = True
sessions[session][6] += packet_size
sessions[session][7] += 1
sessions[session][8] += payload_size
elif int(flags[11]): # FIN
session1 = (ip1, p1, ip2, p2, proto)
session2 = (ip2, p2, ip1, p1, proto)
if session1 in sessions.keys():
sessions[session1][6] += packet_size
sessions[session1][7] += 1
sessions[session1][8] += payload_size
if not sessions[session1][2]:
sessions[session1][2] = True
elif session2 in sessions.keys():
sessions[session2][6] += packet_size
sessions[session2][7] += 1
sessions[session2][8] += payload_size
if not sessions[session2][3]:
sessions[session2][3] = True
elif int(flags[9]): # RST
session1 = (ip1, p1, ip2, p2, proto)
session2 = (ip2, p2, ip1, p1, proto)
if session1 in sessions.keys():
sessions[session1][6] += packet_size
sessions[session1][7] += 1
sessions[session1][8] += payload_size
sessions[session1][4] = True
elif session2 in sessions.keys():
sessions[session2][6] += packet_size
sessions[session2][7] += 1
sessions[session2][8] += payload_size
sessions[session2][4] = True
else:
session1 = (ip1, p1, ip2, p2, proto)
session2 = (ip2, p2, ip1, p1, proto)
if session1 in sessions.keys():
sessions[session1][6] += packet_size
sessions[session1][7] += 1
sessions[session1][8] += payload_size
elif session2 in sessions.keys():
sessions[session2][6] += packet_size
sessions[session2][7] += 1
sessions[session2][8] += payload_size
if packet.haslayer(scapy.UDP) and packet.haslayer(scapy.IP):
payload_size = len(packet[scapy.UDP].payload)
packet_size = len(packet.payload)
ip1 = ipv4_to_hex(packet[scapy.IP].src)
ip2 = ipv4_to_hex(packet[scapy.IP].dst)
p1 = port_to_hex(packet[scapy.UDP].sport)
p2 = port_to_hex(packet[scapy.UDP].dport)
proto = "00000011" # UDP
session1 = (ip1, p1, ip2, p2, proto)
session2 = (ip2, p2, ip1, p1, proto)
if session1 in sessions.keys():
sessions[session1][6] += packet_size
sessions[session1][7] += 1
sessions[session1][8] += payload_size
elif session2 in sessions.keys():
sessions[session2][6] += packet_size
sessions[session2][7] += 1
sessions[session2][8] += payload_size
else:
sessions[session1] = [False, False, False, False, False, False, packet_size, 1, payload_size]
else:
pass
valid_sessions = {}
for session, details in sessions.items():
if session[4] == "00000006": # TCP
if details[0:4] == [True, True, True, True]:
valid_sessions[session] = details
elif details[4] and details[0] and details[1]:
valid_sessions[session] = details
elif session[4] == "00000011": #UDP
valid_sessions[session] = details
return valid_sessions
def sniff(self):
if self.options.pcap_file == 'None':
self.logger.info("Using network interface: %s" %
(self.options.listen_interface))
try:
scapy.sniff(iface=self.options.listen_interface,
prn=self.pkt_callback,
lfilter=self.pkt_check,
filter=self.options.filter,
store=0)
except socket.error, e:
self.logger.error("Sniffer error on %s: %s" %
(self.options.listen_interface, e))
else:
self.logger.info("Using pcap file: %s" % (self.options.pcap_file))
pcapr = scapy.PcapReader(self.options.pcap_file)
while 1:
try:
pkt = pcapr.next()
if self.pkt_check(pkt):
self.pkt_callback(pkt)
except scapy.StopIteration, e:
break
def pkt_is_last(self, pkt):
if pkt['TCP'].flags == 17 or pkt['TCP'].flags == 25:
return True
return False
def pkt_hash(self, pkt):
sdip = (pkt['IP'].src, pkt['TCP'].sport, pkt['IP'].dst,
#!/usr/bin/env python
# File dumpUDP.py
import scapy.all as scapy
import sys
pcap_file = sys.argv[1]
mac = sys.argv[2]
print(''.join(p[scapy.UDP].load for p in scapy.PcapReader(pcap_file)
if p[scapy.Ether].src == mac and scapy.UDP in p))
try:
pcap_in = sys.argv[1]
except IndexError as e:
print('Usage: create_pcap_stats.py filename.pcap')
sys.exit(-1)
if not pcap_in.endswith('.pcap'):
print('input filename should end with ".pcap"')
sys.exit(-1)
stats_out = pcap_in.replace('.pcap', '-stat.txt')
def write_stats(stats):
with open(stats_out, 'w') as f:
for item in stats.items():
f.write("%s %s\n" % item)
dst = {}
for i, pkt in enumerate(scapy.PcapReader(pcap_in)):
try:
ip = pkt[scapy.IP]
except IndexError:
print("pkt #%d has no IP header" % i)
dst[ip.dst] = dst.get(ip.dst, 0) + 1
if i % 50000 == 0:
print(i)
write_stats(dst)
print('bye')
