def remove_flags(input, nflags=5):
output = f'{input}_tmp'
writer = ppcap.Writer(filename=output)
try:
reader = pcap.pcap(input)
for timestamp, raw in reader:
try:
pkt = ethernet.Ethernet(raw)
if pkt[ip.IP] is not None:
ip_body = pkt[ip.IP]
if ip_body[tcp.TCP] is not None:
tcp_body = ip_body[tcp.TCP]
print(tcp_body.flags)
flags = decode_tcp_flags_value(tcp_body.flags,
nflags)[::-1]
flags = int(''.join([str(i) for i in flags]), 2)
print(flags)
pkt[ip.IP][tcp.TCP].flags = flags
writer.write(pkt.bin(), ts=timestamp * 1e9)
except Exception as e:
print(e)
except Exception as e:
print(e)
writer.close()
shutil.copyfile(output, input)
os.remove(output)
def split_by_label(input, labeler, meta_fpath, nulify_dscp=True):
# meta
try:
with open(meta_fpath, 'r') as jf:
meta = json.load(jf)
if 'labels' not in meta.keys():
meta['labels'] = []
except:
meta = {'labels': []}
# read and write
labels = []
pwriters = []
try:
reader = pcap.pcap(input)
for ts, raw in reader:
eth = ethernet.Ethernet(raw)
if eth[ethernet.Ethernet, ip.IP] is not None:
src = eth[ip.IP].src_s
dst = eth[ip.IP].dst_s
if eth[tcp.TCP] is not None:
sport = eth[tcp.TCP].sport
dport = eth[tcp.TCP].dport
elif eth[udp.UDP] is not None:
sport = eth[udp.UDP].sport
dport = eth[udp.UDP].dport
else:
sport = 0
dport = 0
label, description = labeler(ts, src, dst, sport, dport)
if label in labels:
idx = labels.index(label)
else:
labels.append(label)
pwriters.append(
ppcap.Writer(filename=f'{input}_label:{label}'))
idx = -1
if nulify_dscp:
eth[ip.IP].tos = 0
pwriters[idx].write(eth.bin(), ts=ts * 1e9)
except Exception as e:
print(e)
os.remove(input)
for pwriter in pwriters:
pwriter.close()
meta['labels'] += labels
meta['labels'] = np.unique(meta['labels']).tolist()
with open(meta_fpath, 'w') as jf:
json.dump(meta, jf)
def readndump_capfile(infile_name="bugpackets.pcap"):
outfile_name = infile_name + "_buggyagain.pcap"
#print("will store bug-packets to: %s" % outfile_name)
pcap_in = ppcap.Reader(filename=infile_name)
pcap_out = ppcap.Writer(filename=outfile_name)
for ts, bts in pcap_in:
pass_or_dump(bts, pcap_out)
pcap_in.close()
pcap_out.close()
def readndump_network(file_name="bugpackets.pcap", iface_name="lo"):
#print("will store bug-packets to: %s" % file_name)
pcap_writer = ppcap.Writer(filename=file_name)
psock = psocket.SocketHndl(iface_name=iface_name, timeout=999999)
try:
for bts in psock:
pass_or_dump(bts, pcap_writer)
except KeyboardInterrupt:
pass
pcap_writer.close()
psock.close()
def do_checkparsefail(self, _):
pktwrite = ppcap.Writer(filename="parsefail.cap")
cnt = 0
for bts in self._logic._packet_collector:
if cnt > 10000:
break
cnt += 1
try:
pkt = Radiotap(bts)
pkt.dissect_full()
for layer in pkt:
if layer.dissect_error:
logger.warning("error while parsing (in layer): %r" %
ex)
pktwrite.write(bts)
break
except Exception as ex:
logger.warning("error while parsing: %r" % ex)
pktwrite.write(bts)
pktwrite.close()
logger.warning("parsefail checking finished")
import socket
from pypacker import psocket, ppcap, utils
from pypacker.layer12 import ethernet, linuxcc, radiotap
from pypacker.layer3 import ip
from pypacker.layer4 import tcp
from pypacker.layer567 import http
iface = sys.argv[1]
#print("opening (wlan?) interface: %s" % iface)
#utils.set_wlan_monmode(iface)
sockhndl = psocket.SocketHndl(iface_name=iface, timeout=99999)
#pcapwriter = ppcap.Writer(filename="parsefail.pcap", linktype=ppcap.DLT_IEEE802_11_RADIO)
pcapwriter = ppcap.Writer(filename="parsefail.pcap")
raw_bytes = b""
cnt = 0
time_start = time.time()
for bts in sockhndl:
if cnt % 1000 == 0:
print("%d pps" % (cnt / (time.time() - time_start)))
time_start = time.time()
cnt = 0
cnt += 1
try:
#pkt = radiotap.Radiotap(bts)
pkt = ethernet.Ethernet(bts)
"""
import sys
import time
import socket
from pypacker import psocket, ppcap, utils
from pypacker.layer12 import ethernet, linuxcc, radiotap
from pypacker.layer3 import ip
iface = sys.argv[1]
print("opening (wlan?) interface: %s" % iface)
utils.set_wlan_monmode(iface)
sockhndl = psocket.SocketHndl(iface_name=iface)
pcapwriter = ppcap.Writer(filename="parsefail.pcap",
linktype=ppcap.DLT_IEEE802_11_RADIO)
# pcapwriter = ppcap.Writer(filename="parsefail.pcap")
raw_bytes = b""
cnt = 0
time_start = time.time()
while True:
if cnt % 1000 == 0:
print("%d pps" % (cnt / (time.time() - time_start)))
cnt = 0
time_start = time.time()
cnt += 1
try:
raw_bytes = sockhndl.recv()
pkt = radiotap.Radiotap(raw_bytes)
def split_by_label_and_extract_flow_features(input,
fdir,
sdir,
dname,
meta_fpath,
labeler,
tstep,
stages,
splits,
nnewpkts_min=0,
lasttime_min=1.0,
nulify_dscp=True,
remove_flags=True):
src_ip_idx = 0
src_port_idx = 1
dst_ip_idx = 2
dst_port_idx = 3
proto_idx = 4
flow_ids = []
flow_objects = []
flow_labels = []
flow_features = []
flow_feature_labels = []
tstart = None
ttotal = 0
npkts = 0
nflows = 0
nvectors = 0
if type(tstep) == tuple or type(tstep) == list:
assert len(
tstep) == 4, 'There should be 4 parameters: mu, std, min and max'
get_next_tstep = lambda: np.clip(
np.abs(tstep[0] + np.random.rand() * tstep[1]), tstep[2], tstep[3])
tstep_str = '-'.join([str(item) for item in tstep])
else:
get_next_tstep = lambda: tstep
tstep_str = str(tstep)
# load meta
try:
with open(meta_fpath, 'r') as jf:
meta = json.load(jf)
if 'labels' not in meta.keys():
meta['labels'] = []
except:
meta = {'labels': []}
try:
nwindows = meta['nwindows']
nfeatures = meta['nfeatures']
xmin = meta['xmin']
xmax = meta['xmax']
except:
nwindows = None
nfeatures = None
xmin = None
xmax = None
# main read loop
labels = []
pwriters = []
try:
reader = pcap.pcap(input)
for timestamp, raw in reader:
# read pkt
id, features, flags, ether, tos = read_pkt(raw)
if id is not None:
# label the packet
src = id[src_ip_idx]
dst = id[dst_ip_idx]
sport = id[src_port_idx]
dport = id[dst_port_idx]
proto = id[proto_idx]
label, description = labeler(timestamp, src, dst, sport, dport)
# nulify tos field because it will be used to mark flows
if nulify_dscp:
ether[ip.IP].tos = 0
# remove last 3 tcp flags
if np.sum(flags) > 0 and remove_flags:
ether[ip.IP][tcp.TCP].flags = int(
''.join([str(i) for i in flags[::-1]]), 2)
# write the packet
if label in labels:
label_idx = labels.index(label)
else:
labels.append(label)
pwriters.append(
ppcap.Writer(filename=f'{input}_label:{label}'))
label_idx = -1
pwriters[label_idx].write(ether.bin(), ts=timestamp * 1e9)
# time start
if tstart is None:
tstart = int(timestamp)
seconds = get_next_tstep()
# add packets to flows
reverse_id = [dst, dport, src, sport, proto]
if timestamp > (tstart + seconds):
# remove old flows
tmp_ids = []
tmp_objects = []
tmp_labels = []
for i, o, l in zip(flow_ids, flow_objects, flow_labels):
if o.is_active:
tmp_ids.append(i)
tmp_objects.append(o)
tmp_labels.append(l)
flow_ids = list(tmp_ids)
flow_objects = list(tmp_objects)
flow_labels = list(tmp_labels)
# calculate_features
flow_features_t = []
flow_labels_t = []
for flow_id, flow_object, flow_label in zip(
flow_ids, flow_objects, flow_labels):
if flow_object.nnewpkts > nnewpkts_min or (
timestamp -
flow_object.lasttime) > lasttime_min:
t_calc_start = time()
_features = flow_object.get_features()
ttotal += time() - t_calc_start
flow_features_t.append(_features)
flow_labels_t.append(flow_label)
flow_features.extend(flow_features_t)
flow_feature_labels.extend(flow_labels_t)
# update time
seconds += get_next_tstep()
# add packets
if id in flow_ids:
direction = 1
idx = flow_ids.index(id)
flow_objects[idx].append(timestamp, features, flags,
direction)
elif reverse_id in flow_ids:
direction = -1
idx = flow_ids.index(reverse_id)
flow_objects[idx].append(timestamp, features, flags,
direction)
else:
flow_ids.append(id)
flow_objects.append(Flow(timestamp, id, features, flags))
flow_labels.append(label)
nflows += 1
npkts += 1
# lists to arrays
flow_features = np.array(flow_features, dtype=np.float)
flow_feature_labels = np.array(flow_feature_labels, dtype=np.float)
assert flow_features.shape[0] == len(flow_feature_labels)
# update meta
nvectors = flow_features.shape[0]
if nvectors > 0:
if nfeatures is None:
nwindows = flow_features.shape[1]
nfeatures = flow_features.shape[2]
xmin = np.min(flow_features[:, -1, :], axis=0)
xmax = np.max(flow_features[:, -1, :], axis=0)
else:
assert nwindows == flow_features.shape[1]
assert nfeatures == flow_features.shape[2]
xmin = np.min(np.vstack([xmin, flow_features[:, -1, :]]),
axis=0)
xmax = np.max(np.vstack([xmax, flow_features[:, -1, :]]),
axis=0)
# split and save features
for label in labels:
features_label_dir = osp.join(fdir, str(label))
stats_label_dir = osp.join(sdir, str(label))
for _dir in [features_label_dir, stats_label_dir]:
if not osp.isdir(_dir):
os.mkdir(_dir)
output_f = osp.join(features_label_dir, dname)
stats_f = osp.join(stats_label_dir, dname)
idx = np.where(flow_feature_labels == label)[0]
if len(idx) > 0:
values_l = np.hstack([
flow_features[idx, :].reshape(len(idx),
nwindows * nfeatures),
flow_feature_labels[idx, None]
])
inds = np.arange(len(values_l))
inds_splitted = [[] for _ in stages]
np.random.shuffle(inds)
val, remaining = np.split(inds,
[int(splits[1] * len(inds))])
tr, te = np.split(remaining,
[int(splits[0] * len(remaining))])
inds_splitted[0] = tr
inds_splitted[1] = te
inds_splitted[2] = val
for fi, stage in enumerate(stages):
fname = '{0}_{1}_{2}'.format(output_f, tstep_str,
stage)
pandas.DataFrame(
values_l[inds_splitted[fi], :]).to_csv(
fname, header=False, mode='a', index=False)
# save stats
pandas.DataFrame([[input] + [nflows, npkts]
]).to_csv(stats_f,
header=False,
mode='a',
index=False)
# save meta
meta['labels'] += labels
meta['labels'] = np.unique(meta['labels']).tolist()
meta['nwindows'] = nwindows
meta['nfeatures'] = nfeatures
meta['xmin'] = xmin.tolist()
meta['xmax'] = xmax.tolist()
with open(meta_fpath, 'w') as jf:
json.dump(meta, jf)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(e, fname, exc_tb.tb_lineno)
# close writers
os.remove(input)
for pwriter in pwriters:
pwriter.close()
return nvectors, ttotal
type=auto_addr,
help=
"""Specific advertisting address to follow (mode 1 or 2). If not set all connections fill be followed""",
default=None)
CMD_BTLE_MODES = {
0: BTLE_MODE_FIXEDCHANNEL,
1: BTLE_MODE_FOLLOW_STATIC,
2: BTLE_MODE_FOLLOW_DYNAMIC
}
parsed_args = parser.parse_args()
parsed_args.fh_pcap = None
parsed_args.btle_mode = CMD_BTLE_MODES[parsed_args.mode]
parsed_args.crc_init = 0xAAAAAA
if parsed_args.btle_mode == BTLE_MODE_FOLLOW_DYNAMIC and parsed_args.advaddr is None:
logger.warning("Parameter advaddr NOT set in mode 2!")
sys.exit(1)
if parsed_args.pcapfile is not None:
parsed_args.fh_pcap = ppcap.Writer(
filename=parsed_args.pcapfile,
linktype=ppcap.LINKTYPE_BLUETOOTH_LE_LL_WITH_PHDR)
scan_btle(parsed_args)
if parsed_args.fh_pcap is not None:
parsed_args.fh_pcap.close()