def write_csv(path, data, compression):
f = open_compressed_file('{}{}'.format(
path, COMPRESSION_EXTENSIONS[compression]),
write=True)
connections = []
max_length = 0
columns = 1
for d in data:
connections.append(data[d])
columns = len(data[d])
max_length = max(max_length, len(data[d][0]))
for col in range(0, columns):
f.write('{};'.format(d))
f.write('\n')
for i in range(max_length):
for j in connections:
if i < len(j[0]):
for col in range(0, columns):
f.write('{};'.format(j[col][i]))
else:
for col in range(0, columns):
f.write(';')
f.write('\n')
f.close()
def read_csv(path, columns_per_connection=2):
output = {}
file_path = find_file(path)
if file_path is None:
raise IOError('File not found {}'.format(path))
f = open_compressed_file(file_path)
first_line = f.readline().split(';')[:-1]
for line in f:
split = line.split(';')
for i in range(0, len(first_line), columns_per_connection):
if split[i] == '':
continue
try:
index = int(first_line[i])
except ValueError:
index = first_line[i]
if index not in output:
output[index] = tuple(
[[] for _ in range(0, columns_per_connection)])
for column in range(0, columns_per_connection):
output[index][column].append(float(split[i + column]))
f.close()
return output
def parse_buffer_backlog(path):
output = {}
paths = glob.glob(os.path.join(path, '*.{}*'.format(BUFFER_FILE_EXTENSION)))
for i, file_path in enumerate(paths):
output[i] = ([], [])
f = open_compressed_file(file_path)
for line in f:
split = line.split(';')
timestamp = parse_timestamp(split[0])
size = split[1].replace('b\n', '')
if 'K' in size:
size = float(size.replace('K', '')) * 1000
elif 'M' in size:
size = float(size.replace('M', '')) * 1000000
elif 'G' in size:
size = float(size.replace('G', '')) * 1000000000
output[i][0].append(timestamp)
output[i][1].append(float(size) * 8)
f.close()
return output
def parse_bbr_and_cwnd_values(path):
bbr_values = {}
cwnd_values = {}
paths = glob.glob(os.path.join(path, '*.{}*'.format(FLOW_FILE_EXTENSION)))
all_files = sorted(paths)
for i, file_path in enumerate(all_files):
bbr_values[i] = ([], [], [], [], [], [])
cwnd_values[i] = ([], [], [])
f = open_compressed_file(file_path)
for line in f:
split = map(lambda x: x.strip(), line.split(';'))
timestamp = parse_timestamp(split[0])
cwnd, ssthresh = 0, 0
if split[1] != '':
cwnd = int(split[1])
if split[2] != '':
ssthresh = int(split[2])
cwnd_values[i][0].append(timestamp)
cwnd_values[i][1].append(cwnd)
cwnd_values[i][2].append(ssthresh)
if split[3] != '':
bbr = split[3].replace('bw:', '')\
.replace('mrtt:','')\
.replace('pacing_gain:', '')\
.replace('cwnd_gain:', '')
bbr = bbr.split(',')
if len(bbr) < 4:
pacing_gain = 0
cwnd_gain = 0
else:
pacing_gain = float(bbr[2])
cwnd_gain = float(bbr[3])
if 'Mbps' in bbr[0]:
bw = float(bbr[0].replace('Mbps', '')) * 1000000
elif 'Kbps' in bbr[0]:
bw = float(bbr[0].replace('Kbps', '')) * 1000
elif 'bps' in bbr[0]:
bw = float(bbr[0].replace('bps', ''))
else:
bw = 0
rtt = float(bbr[1])
bbr_values[i][0].append(timestamp)
bbr_values[i][1].append(bw)
bbr_values[i][2].append(rtt)
bbr_values[i][3].append(pacing_gain)
bbr_values[i][4].append(cwnd_gain)
bbr_values[i][5].append(bw * rtt / 1000)
f.close()
return bbr_values, cwnd_values
def parse_pcap(path, delta_t):
# Find correct .pcap files
pcap1 = glob.glob(os.path.join(path, PCAP1 + '*'))[0]
pcap2 = glob.glob(os.path.join(path, PCAP2 + '*'))[0]
total_packets = len(list(dpkt.pcap.Reader(open_compressed_file(pcap1)))) + \
len(list(dpkt.pcap.Reader(open_compressed_file(pcap1))))
print(' Found {} frames.'.format(colorize(total_packets, 'green')))
processed_packets = 0
f = open_compressed_file(pcap1)
pcap = dpkt.pcap.Reader(f)
connections = []
active_connections = []
round_trips = {}
inflight = {}
sending_rate = {}
avg_rtt = {}
inflight_seq = {}
inflight_ack = {}
sending_rate_data_size = {}
inflight_avg = {}
avg_rtt_samples = {}
total_throughput = ([], [])
total_sending_rate = ([], [])
retransmissions = {}
retransmission_counter = {}
packet_counter = {}
retransmissions_interval = {}
total_retransmisions = ([], [], [])
start_seq = {}
t = 0
ts_vals = {}
seqs = {}
start_ts = -1
print('Connections:')
for ts, buf in pcap:
if start_ts < 0:
start_ts = ts
t = start_ts + delta_t
processed_packets += 1
if processed_packets % 500 == 0:
print_progress(processed_packets, total_packets)
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.data
tcp = ip.data
src_ip = socket.inet_ntoa(ip.src)
dst_ip = socket.inet_ntoa(ip.dst)
src_port = tcp.sport
dst_port = tcp.dport
# identify a connection always as (client port, server port)
if src_port > dst_port:
tcp_tuple = (src_ip, src_port, dst_ip, dst_port)
else:
tcp_tuple = (dst_ip, dst_port, src_ip, src_port)
while ts >= t:
total_sending_rate[0].append(t)
total_sending_rate[1].append(0)
total_retransmisions[0].append(t)
total_retransmisions[1].append(0)
total_retransmisions[2].append(0)
for i, c in enumerate(connections):
if c not in active_connections:
continue
tp = float(sending_rate_data_size[i]) / delta_t
sending_rate[i][0].append(t)
sending_rate[i][1].append(tp)
sending_rate_data_size[i] = 0
total_sending_rate[1][-1] += tp
retransmissions_interval[i][0].append(t)
retransmissions_interval[i][1].append(
retransmission_counter[i])
retransmissions_interval[i][2].append(packet_counter[i])
total_retransmisions[1][-1] += retransmission_counter[i]
total_retransmisions[2][-1] += packet_counter[i]
retransmission_counter[i] = 0
packet_counter[i] = 0
inflight[i][0].append(t)
if len(inflight_avg[i]) > 0:
inflight[i][1].append(
sum(inflight_avg[i]) / len(inflight_avg[i]))
else:
inflight[i][1].append(0)
inflight_avg[i] = []
if len(avg_rtt_samples[i]) > 0:
avg_rt = sum(avg_rtt_samples[i]) / len(avg_rtt_samples[i])
avg_rtt[i][0].append(t)
avg_rtt[i][1].append(avg_rt)
avg_rtt_samples[i] = []
t += delta_t
if tcp.flags & 0x02 and tcp_tuple not in connections:
connections.append(tcp_tuple)
active_connections.append(tcp_tuple)
connection_index = connections.index(tcp_tuple)
start_seq[connection_index] = tcp.seq
round_trips[connection_index] = ([], [])
inflight[connection_index] = ([], [])
avg_rtt[connection_index] = ([], [])
sending_rate[connection_index] = ([], [])
ts_vals[connection_index] = ([], [])
seqs[connection_index] = []
inflight_seq[connection_index] = 0
inflight_ack[connection_index] = 0
inflight_avg[connection_index] = []
sending_rate_data_size[connection_index] = 0
avg_rtt_samples[connection_index] = []
retransmissions[connection_index] = ([], )
retransmission_counter[connection_index] = 0
packet_counter[connection_index] = 0
retransmissions_interval[connection_index] = ([], [], [])
print(' [SYN] {}:{} -> {}:{}'.format(tcp_tuple[0], tcp_tuple[1],
tcp_tuple[2], tcp_tuple[3]))
if tcp.flags & 0x01:
if tcp_tuple in active_connections:
active_connections.remove(tcp_tuple)
print(' [FIN] {}:{} -> {}:{}'.format(tcp_tuple[0],
tcp_tuple[1],
tcp_tuple[2],
tcp_tuple[3]))
continue
connection_index = connections.index(tcp_tuple)
ts_val = None
ts_ecr = None
options = dpkt.tcp.parse_opts(tcp.opts)
for opt in options:
if opt[0] == dpkt.tcp.TCP_OPT_TIMESTAMP:
ts_val = reduce(lambda x, r: (x << 8) + r,
map(ord, opt[1][:4]))
ts_ecr = reduce(lambda x, r: (x << 8) + r,
map(ord, opt[1][4:]))
if src_port > dst_port:
# client -> server
tcp_seq = tcp.seq - start_seq[connection_index]
if tcp_seq < 0:
tcp_seq += 2**32
packet_counter[connection_index] += 1
inflight_seq[connection_index] = max(
tcp_seq, inflight_seq[connection_index])
sending_rate_data_size[connection_index] += ip.len * 8
if tcp_seq in seqs[connection_index]:
retransmissions[connection_index][0].append(ts)
retransmission_counter[connection_index] += 1
else:
seqs[connection_index].append(tcp_seq)
if ts_val is not None:
ts_vals[connection_index][0].append(ts)
ts_vals[connection_index][1].append(ts_val)
else:
# server -> client
tcp_ack = tcp.ack - start_seq[connection_index]
if tcp_ack < 0:
tcp_ack += 2**32
inflight_ack[connection_index] = max(
tcp_ack, inflight_ack[connection_index])
seqs[connection_index] = [
x for x in seqs[connection_index] if x >= tcp_ack
]
if ts_ecr in ts_vals[connection_index][1]:
index = ts_vals[connection_index][1].index(ts_ecr)
rtt = (ts - ts_vals[connection_index][0][index]) * 1000
ts_vals[connection_index][0].pop(index)
ts_vals[connection_index][1].pop(index)
avg_rtt_samples[connection_index].append(rtt)
round_trips[connection_index][0].append(ts)
round_trips[connection_index][1].append(rtt)
inflight_data = max(
0, inflight_seq[connection_index] - inflight_ack[connection_index])
inflight_avg[connection_index].append(inflight_data * 8)
f.close()
# Compute throughput after the bottleneck
f = open_compressed_file(pcap2)
pcap = dpkt.pcap.Reader(f)
connections = []
active_connections = []
throughput = {}
throughput_data_size = {}
t = start_ts + delta_t
for ts, buf in pcap:
processed_packets += 1
if processed_packets % 500 == 0:
print_progress(processed_packets, total_packets)
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.data
tcp = ip.data
src_ip = socket.inet_ntoa(ip.src)
dst_ip = socket.inet_ntoa(ip.dst)
src_port = tcp.sport
dst_port = tcp.dport
# identify a connection always as (client port, server port)
if src_port > dst_port:
tcp_tuple = (src_ip, src_port, dst_ip, dst_port)
else:
tcp_tuple = (dst_ip, dst_port, src_ip, src_port)
while ts >= t:
total_throughput[0].append(t)
total_throughput[1].append(0)
for i, c in enumerate(connections):
if c not in active_connections:
continue
tp = float(throughput_data_size[i]) / delta_t
throughput[i][0].append(t)
throughput[i][1].append(tp)
total_throughput[1][-1] += tp
throughput_data_size[i] = 0
t += delta_t
if tcp.flags & 0x02 and tcp_tuple not in connections:
connections.append(tcp_tuple)
active_connections.append(tcp_tuple)
connection_index = connections.index(tcp_tuple)
throughput[connection_index] = ([], [])
throughput_data_size[connection_index] = 0
if tcp.flags & 0x01:
if tcp_tuple in active_connections:
active_connections.remove(tcp_tuple)
continue
connection_index = connections.index(tcp_tuple)
if src_port > dst_port:
# client -> server
throughput_data_size[connection_index] += ip.len * 8
print(' 100.00%')
fairness_troughput = compute_fairness(throughput, delta_t)
fairness_sending_rate = compute_fairness(sending_rate, delta_t)
fairness = {
'Throughtput': fairness_troughput,
'Sending Rate': fairness_sending_rate
}
bbr_values, cwnd_values = parse_bbr_and_cwnd_values(path)
bbr_total_values, sync_phases, sync_duration = compute_total_values(
bbr_values)
buffer_backlog = parse_buffer_backlog(path)
data_info = DataInfo(sync_duration=sync_duration, sync_phases=sync_phases)
throughput[len(throughput)] = total_throughput
sending_rate[len(sending_rate)] = total_sending_rate
retransmissions_interval[len(
retransmissions_interval)] = total_retransmisions
return PcapData(rtt=round_trips,
inflight=inflight,
throughput=throughput,
fairness=fairness,
avg_rtt=avg_rtt,
sending_rate=sending_rate,
bbr_values=bbr_values,
bbr_total_values=bbr_total_values,
cwnd_values=cwnd_values,
retransmissions=retransmissions,
retransmissions_interval=retransmissions_interval,
buffer_backlog=buffer_backlog,
data_info=data_info)