def plot_accumulated_data_netflix(input_file, home_ip, website, platform, y_limit, show=False):
data = generate_data(input_file, home_ip, website=website, protocols=['TCP'])
all_stream_data = data['all_streams']
# all_stream_data['time'] = filter(lambda x: x[0] < 50, all_stream_data['time'])
if platform == 'android':
first_stream_id, first_stream_data = sorted(data.items(), cmp=lambda x, y: x[1]['total'] - y[1]['total'], reverse=True)[1]
second_stream_id, second_stream_data = sorted(data.items(), cmp=lambda x, y: x[1]['total'] - y[1]['total'], reverse=True)[2]
third_stream_id, third_stream_data = sorted(data.items(), cmp=lambda x, y: x[1]['total'] - y[1]['total'], reverse=True)[3]
fig, ax = plt.subplots()
ax.set_ylabel('accumulated data')
ax.set_xlabel('time, s')
if y_limit is not None:
fn, tix = make_storage_ticks(filter(lambda x: x < y_limit, map(lambda x: x[1], all_stream_data['time'])))
else:
fn, tix = make_storage_ticks(map(lambda x: x[1], all_stream_data['time']))
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
red_line = plt.plot(map(lambda x: x[1], all_stream_data['time']), color='red')
if platform == 'android':
blue_line = plt.plot(map(lambda x: x[1], first_stream_data['time']), color='blue')
green_line = plt.plot(map(lambda x: x[1], second_stream_data['time']), color='green')
orange_line = plt.plot(map(lambda x: x[1], third_stream_data['time']), color='orange')
plt.legend([red_line[0], blue_line[0], green_line[0], orange_line[0]], map(pretty_name(website), ['{} combined traffic', '{} first stream', '{} second stream', '{} third stream']))
else:
plt.legend([red_line[0]], map(pretty_name(website), ['{} combined traffic']))
if y_limit is not None:
plt.ylim(0, y_limit)
plt.savefig('{}_{}_accumulated_data{}.svg'.format(website, platform, '_limited' if y_limit is not None else ''))
if show:
plt.show()
def plot_accumulated_data(input_file, home_ip, website, platform, show=False):
data = generate_data(input_file, home_ip, website=website, protocols=["TCP"])
all_stream_data = data["all_streams"]
video_stream_id, video_stream_data = sorted(
data.items(), cmp=lambda x, y: x[1]["total"] - y[1]["total"], reverse=True
)[1]
audio_stream_id, audio_stream_data = sorted(
data.items(), cmp=lambda x, y: x[1]["total"] - y[1]["total"], reverse=True
)[2]
fig, ax = plt.subplots()
ax.set_ylabel("accumulated data")
ax.set_xlabel("time, s")
fn, tix = make_storage_ticks(map(lambda x: x[1], all_stream_data["time"]))
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
red_line = plt.plot(map(lambda x: x[1], all_stream_data["time"]), color="red")
blue_line = plt.plot(map(lambda x: x[1], video_stream_data["time"]), color="blue")
green_line = plt.plot(map(lambda x: x[1], audio_stream_data["time"]), color="green")
plt.legend(
[red_line[0], blue_line[0], green_line[0]],
map(pretty_name(website), ["{} combined traffic", "{} video traffic", "{} audio traffic"]),
loc=4,
)
plt.savefig("{}_{}_accumulated_data.svg".format(website, platform))
if show:
plt.show()
def netflix_chunks_over_time(show=False):
android_file = 'netflix_chunks.csv'
android_chunks = netflix_android_get_chunk_size_data(android_file)
android_chunks = filter(lambda x: 100 < x[1] < 400, android_chunks)
android_x_values = map(lambda x: x[1], android_chunks)
android_y_values = map(lambda x: x[0], android_chunks)
chrome_file = 'hannibal_dump/chrome.csv'
chrome_chunks = netflix_chrome_get_chunk_size_data(chrome_file)
chrome_chunks = filter(lambda x: 100 < x[1] < 400, chrome_chunks)
chrome_x_values = map(lambda x: x[1], chrome_chunks)
chrome_y_values = map(lambda x: x[0], chrome_chunks)
fig, ax = plt.subplots()
ax.set_ylabel('chunk size')
ax.set_xlabel('time, s')
fn, tix = make_storage_ticks(android_y_values + chrome_y_values)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
chrome_line = plt.plot(chrome_x_values, chrome_y_values, 'r-')
plt.plot(chrome_x_values, chrome_y_values, 'ro')
android_line = plt.plot(android_x_values, android_y_values, 'b-')
plt.legend([android_line[0], chrome_line[0]], ['Netflix Android', 'Netflix Chrome'])
plt.savefig('netflix_chunks_over_time.svg')
if show:
plt.show()
plt.clf()
def top_flows_chart(chrome_file,
android_file,
chrome_ip,
android_ip,
website,
is_incoming,
chrome_color='red',
android_color='blue',
ax=None):
"""
Draws a plot of how much traffic transferred per flow.
:return:
"""
chrome_data, chrome_shortened = generate_data(chrome_file, chrome_ip,
website, is_incoming)
chrome_data = map(lambda x: x[1], chrome_data)
android_data, android_shortened = generate_data(android_file, android_ip,
website, is_incoming)
android_data = map(lambda x: x[1], android_data)
# android_data = scale_data(android_data, sum(chrome_data), 1)
num_values = max(len(chrome_data), len(android_data))
shortened = chrome_shortened if len(chrome_data) > len(
android_data) else android_shortened
labels = ['Conn. {0}'.format(i + 1) for i in xrange(num_values)]
if shortened:
labels[-1] = 'Rest'
ind = np.array(
map(lambda x: x + 0.15,
np.arange(num_values))) # the x locations for the groups
width = 0.35 # the width of the bars
if ax is None:
fig, ax = plt.subplots()
print chrome_data
chrome_bars = ax.bar(ind, chrome_data, width, color=chrome_color)
print android_data
android_bars = ax.bar(ind + width,
android_data,
width,
color=android_color)
ax.legend((chrome_bars[0], android_bars[0]),
map(pretty_name(website), ['{} Chrome', '{} Android']))
# add some text for labels, title and axes ticks
ax.set_ylabel('Data transferred')
ax.set_xlabel('Connection')
fn, tix = make_storage_ticks(chrome_data)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
ax.set_xticks(ind + width)
ax.set_xticklabels(labels)
plt.tight_layout()
def plot_accumulated_data_netflix(input_file, home_ip, website, platform, y_limit, show=False):
data = generate_data(input_file, home_ip, website=website, protocols=["TCP"])
all_stream_data = data["all_streams"]
# all_stream_data['time'] = filter(lambda x: x[0] < 50, all_stream_data['time'])
if platform == "android":
first_stream_id, first_stream_data = sorted(
data.items(), cmp=lambda x, y: x[1]["total"] - y[1]["total"], reverse=True
)[1]
second_stream_id, second_stream_data = sorted(
data.items(), cmp=lambda x, y: x[1]["total"] - y[1]["total"], reverse=True
)[2]
third_stream_id, third_stream_data = sorted(
data.items(), cmp=lambda x, y: x[1]["total"] - y[1]["total"], reverse=True
)[3]
fig, ax = plt.subplots()
ax.set_ylabel("accumulated data")
ax.set_xlabel("time, s")
if y_limit is not None:
fn, tix = make_storage_ticks(filter(lambda x: x < y_limit, map(lambda x: x[1], all_stream_data["time"])))
else:
fn, tix = make_storage_ticks(map(lambda x: x[1], all_stream_data["time"]))
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
red_line = plt.plot(map(lambda x: x[1], all_stream_data["time"]), color="red")
if platform == "android":
blue_line = plt.plot(map(lambda x: x[1], first_stream_data["time"]), color="blue")
green_line = plt.plot(map(lambda x: x[1], second_stream_data["time"]), color="green")
orange_line = plt.plot(map(lambda x: x[1], third_stream_data["time"]), color="orange")
plt.legend(
[red_line[0], blue_line[0], green_line[0], orange_line[0]],
map(
pretty_name(website), ["{} combined traffic", "{} first stream", "{} second stream", "{} third stream"]
),
)
else:
plt.legend([red_line[0]], map(pretty_name(website), ["{} combined traffic"]))
if y_limit is not None:
plt.ylim(0, y_limit)
plt.savefig("{}_{}_accumulated_data{}.svg".format(website, platform, "_limited" if y_limit is not None else ""))
if show:
plt.show()
def save_scatter(x_values, y_values, threshold, is_chrome, show=False):
fig, ax = plt.subplots()
ax.set_ylabel('chunk size')
ax.set_xlabel('time, s')
fn, tix = make_storage_ticks(y_values)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
plt.plot(x_values, y_values, 'bo')
plt.savefig('scatter-fig-{}0ms-{}.svg'.format(int(threshold * 100), 'chrome' if is_chrome else 'android'))
if show:
plt.show()
plt.clf()
def save_scatter(x_values, y_values, threshold, is_chrome, show=False):
fig, ax = plt.subplots()
ax.set_ylabel('chunk size')
ax.set_xlabel('time, s')
fn, tix = make_storage_ticks(y_values)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
plt.plot(x_values, y_values, 'bo')
plt.savefig('scatter-fig-{}0ms-{}.svg'.format(
int(threshold * 100), 'chrome' if is_chrome else 'android'))
if show:
plt.show()
plt.clf()
def top_charts_mixed_protocols(input_file, home_ip, website, is_incoming=True):
num_values = 5
quic_data, shortened = generate_data(input_file,
home_ip,
website=website,
is_incoming=True,
num_values=num_values,
protocols=['TCP', 'UDP'])
# separate data and protocol names
top_labels = map(lambda x: x[0], quic_data)
data = map(lambda x: x[1], quic_data)
# make labels
labels = ['Conn. {0}'.format(i + 1) for i in xrange(num_values)]
if shortened:
labels[-1] = 'Rest'
ind = np.array(map(
lambda x: x, np.arange(num_values))) # the x locations for the groups
width = 0.5 # the width of the bars
fig, ax = plt.subplots()
chrome_bars = ax.bar(ind, data, width, color='red')
# add some text for labels, title and axes ticks
ax.set_ylabel('Data transferred')
ax.set_xlabel('Connection')
fn, tix = make_storage_ticks(data)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
ax.set_xticks(ind + width)
ax.set_xticklabels(labels)
def autolabel(bars, data):
# attach some text labels
for j, bar in enumerate(bars):
height = bar.get_height()
ax.text(bar.get_x() + bar.get_width() / 2.,
height,
data[j],
ha='center',
va='bottom')
autolabel(chrome_bars, top_labels)
plt.tight_layout()
def top_flows_chart(chrome_file, android_file, chrome_ip, android_ip, website, is_incoming,
chrome_color='red', android_color='blue', ax=None):
"""
Draws a plot of how much traffic transferred per flow.
:return:
"""
chrome_data, chrome_shortened = generate_data(chrome_file, chrome_ip, website, is_incoming)
chrome_data = map(lambda x: x[1], chrome_data)
android_data, android_shortened = generate_data(android_file, android_ip, website, is_incoming)
android_data = map(lambda x: x[1], android_data)
# android_data = scale_data(android_data, sum(chrome_data), 1)
num_values = max(len(chrome_data), len(android_data))
shortened = chrome_shortened if len(chrome_data) > len(android_data) else android_shortened
labels = ['Conn. {0}'.format(i + 1) for i in xrange(num_values)]
if shortened:
labels[-1] = 'Rest'
ind = np.array(map(lambda x: x + 0.15, np.arange(num_values))) # the x locations for the groups
width = 0.35 # the width of the bars
if ax is None:
fig, ax = plt.subplots()
print chrome_data
chrome_bars = ax.bar(ind, chrome_data, width, color=chrome_color)
print android_data
android_bars = ax.bar(ind + width, android_data, width, color=android_color)
ax.legend((chrome_bars[0], android_bars[0]), map(pretty_name(website), ['{} Chrome', '{} Android']))
# add some text for labels, title and axes ticks
ax.set_ylabel('Data transferred')
ax.set_xlabel('Connection')
fn, tix = make_storage_ticks(chrome_data)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
ax.set_xticks(ind + width)
ax.set_xticklabels(labels)
plt.tight_layout()
def plot_accumulated_data(input_file, home_ip, website, platform, show=False):
data = generate_data(input_file, home_ip, website=website, protocols=['TCP'])
all_stream_data = data['all_streams']
video_stream_id, video_stream_data = sorted(data.items(), cmp=lambda x, y: x[1]['total'] - y[1]['total'], reverse=True)[1]
audio_stream_id, audio_stream_data = sorted(data.items(), cmp=lambda x, y: x[1]['total'] - y[1]['total'], reverse=True)[2]
fig, ax = plt.subplots()
ax.set_ylabel('accumulated data')
ax.set_xlabel('time, s')
fn, tix = make_storage_ticks(map(lambda x: x[1], all_stream_data['time']))
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
red_line = plt.plot(map(lambda x: x[1], all_stream_data['time']), color='red')
blue_line = plt.plot(map(lambda x: x[1], video_stream_data['time']), color='blue')
green_line = plt.plot(map(lambda x: x[1], audio_stream_data['time']), color='green')
plt.legend([red_line[0], blue_line[0], green_line[0]], map(pretty_name(website), ['{} combined traffic', '{} video traffic', '{} audio traffic']), loc=4)
plt.savefig('{}_{}_accumulated_data.svg'.format(website, platform))
if show:
plt.show()
def top_charts_mixed_protocols(input_file, home_ip, website, is_incoming=True):
num_values = 5
quic_data, shortened = generate_data(input_file, home_ip, website=website, is_incoming=True, num_values=num_values, protocols=['TCP', 'UDP'])
# separate data and protocol names
top_labels = map(lambda x: x[0], quic_data)
data = map(lambda x: x[1], quic_data)
# make labels
labels = ['Conn. {0}'.format(i + 1) for i in xrange(num_values)]
if shortened:
labels[-1] = 'Rest'
ind = np.array(map(lambda x: x, np.arange(num_values))) # the x locations for the groups
width = 0.5 # the width of the bars
fig, ax = plt.subplots()
chrome_bars = ax.bar(ind, data, width, color='red')
# add some text for labels, title and axes ticks
ax.set_ylabel('Data transferred')
ax.set_xlabel('Connection')
fn, tix = make_storage_ticks(data)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
ax.set_xticks(ind + width)
ax.set_xticklabels(labels)
def autolabel(bars, data):
# attach some text labels
for j, bar in enumerate(bars):
height = bar.get_height()
ax.text(bar.get_x() + bar.get_width() / 2., height,
data[j], ha='center', va='bottom')
autolabel(chrome_bars, top_labels)
plt.tight_layout()
def plot_thing(input_file,
home_ip,
plot_all=True,
plot_first=False,
plot_second=False,
plot_rest=False,
website='youtube.com',
is_incoming=True,
colors=None,
chart=None,
names=None,
sizes=True,
protocols=None,
window=1):
if protocols is None:
protocols = ['TCP']
if names is None:
names = ['All streams', 'Top stream', 'Rest']
if colors is None:
colors = ['r', 'g', 'b']
streams = {}
with open(input_file, 'rb') as csv_file:
data_reader = csv.DictReader(csv_file, delimiter=',')
for row in data_reader:
if website in row['website'] and (row['protocol'] in protocols):
if row['dst' if is_incoming else 'src'] == home_ip and row[
'is_ack'] == 'False':
stream_id = make_stream_id(row)
try:
streams[stream_id].append(
(float(row['timestamp']), int(row['len'])))
except KeyError:
streams[stream_id] = [(float(row['timestamp']),
int(row['len']))]
# sort list of lists by first value of each list
sorted_streams = sorted(streams.values(), key=lambda tuples: tuples[0][0])
first_timestamp = sorted_streams[0][0][0]
# subtract first timestamp from all
sorted_streams = map(
lambda stream: map(lambda tup:
(tup[0] - first_timestamp, tup[1]), stream),
sorted_streams)
# merge list of lists
all_streams = reduce(lambda x, y: x + y, sorted_streams, [])
def put_in_bins(time_stamps, size=True):
"""
Puts packets into time bins (e. g. packets of same second)
:param time_stamps: list of tuples (time_stamp, packet_length)
:param size: (optional) if True, sums up lengths, otherwise number of packets
:return:
"""
time_bins = []
for time_stamp, length in time_stamps:
bin_index = int(time_stamp / window)
try:
time_bins[bin_index] += length if size else 1
except IndexError:
while len(time_bins) != bin_index:
time_bins.append(0)
time_bins.append(length if size else 1)
return time_bins
plot_line_list = []
plot_name_list = []
if chart is None:
fig, ax = plt.subplots()
if sizes:
ax.set_ylabel('size in KB')
else:
ax.set_ylabel('# of packets')
ax.set_xlabel('time, s')
else:
fig, ax, plot_line_list, plot_name_list = chart
top_stream = sorted(sorted_streams,
key=lambda stream: len(stream),
reverse=True)[0]
second_stream = sorted(sorted_streams,
key=lambda stream: len(stream),
reverse=True)[1]
rest_start_index = 2 if plot_second else 1
rest_streams = reduce(
lambda x, y: x + y,
sorted(sorted_streams, key=lambda stream: len(stream),
reverse=True)[rest_start_index:], [])
i = 0
all_values = []
if plot_all:
values = put_in_bins(all_streams, sizes)
all_values.extend(values)
red_line = plt.plot(values, colors[i] + '-')
plot_line_list.append(red_line[0])
plot_name_list.append(names[i])
i += 1
if plot_first:
values = put_in_bins(top_stream, sizes)
all_values.extend(values)
blue_line = plt.plot(values, colors[i] + '-')
plot_line_list.append(blue_line[0])
plot_name_list.append(names[i])
i += 1
if plot_second:
values = put_in_bins(second_stream, sizes)
all_values.extend(values)
magenta_line = plt.plot(values, colors[i] + '-')
plot_line_list.append(magenta_line[0])
plot_name_list.append(names[i])
i += 1
if plot_rest:
values = put_in_bins(rest_streams, sizes)
all_values.extend(values)
green_line = plt.plot(values, colors[i] + '-')
plot_line_list.append(green_line[0])
plot_name_list.append(names[i])
i += 1
plt.legend(plot_line_list, plot_name_list)
if sizes:
fn, tix = make_storage_ticks(all_values)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
ax.set_xticklabels([int(x * window) for x in ax.get_xticks()])
return fig, ax, plot_line_list, plot_name_list
def plot_thing(input_file, home_ip, plot_all=True, plot_first=False, plot_second=False, plot_rest=False, website='youtube.com', is_incoming=True, colors=None, chart=None, names=None, sizes=True, protocols=None, window=1):
if protocols is None:
protocols = ['TCP']
if names is None:
names = ['All streams', 'Top stream', 'Rest']
if colors is None:
colors = ['r', 'g', 'b']
streams = {}
with open(input_file, 'rb') as csv_file:
data_reader = csv.DictReader(csv_file, delimiter=',')
for row in data_reader:
if website in row['website'] and (row['protocol'] in protocols):
if row['dst' if is_incoming else 'src'] == home_ip and row['is_ack'] == 'False':
stream_id = make_stream_id(row)
try:
streams[stream_id].append((float(row['timestamp']), int(row['len'])))
except KeyError:
streams[stream_id] = [(float(row['timestamp']), int(row['len']))]
# sort list of lists by first value of each list
sorted_streams = sorted(streams.values(), key=lambda tuples: tuples[0][0])
first_timestamp = sorted_streams[0][0][0]
# subtract first timestamp from all
sorted_streams = map(lambda stream: map(lambda tup: (tup[0] - first_timestamp, tup[1]), stream), sorted_streams)
# merge list of lists
all_streams = reduce(lambda x, y: x + y, sorted_streams, [])
def put_in_bins(time_stamps, size=True):
"""
Puts packets into time bins (e. g. packets of same second)
:param time_stamps: list of tuples (time_stamp, packet_length)
:param size: (optional) if True, sums up lengths, otherwise number of packets
:return:
"""
time_bins = []
for time_stamp, length in time_stamps:
bin_index = int(time_stamp / window)
try:
time_bins[bin_index] += length if size else 1
except IndexError:
while len(time_bins) != bin_index:
time_bins.append(0)
time_bins.append(length if size else 1)
return time_bins
plot_line_list = []
plot_name_list = []
if chart is None:
fig, ax = plt.subplots()
if sizes:
ax.set_ylabel('size in KB')
else:
ax.set_ylabel('# of packets')
ax.set_xlabel('time, s')
else:
fig, ax, plot_line_list, plot_name_list = chart
top_stream = sorted(sorted_streams, key=lambda stream: len(stream), reverse=True)[0]
second_stream = sorted(sorted_streams, key=lambda stream: len(stream), reverse=True)[1]
rest_start_index = 2 if plot_second else 1
rest_streams = reduce(lambda x, y: x + y, sorted(sorted_streams, key=lambda stream: len(stream), reverse=True)[rest_start_index:], [])
i = 0
all_values = []
if plot_all:
values = put_in_bins(all_streams, sizes)
all_values.extend(values)
red_line = plt.plot(values, colors[i] + '-')
plot_line_list.append(red_line[0])
plot_name_list.append(names[i])
i += 1
if plot_first:
values = put_in_bins(top_stream, sizes)
all_values.extend(values)
blue_line = plt.plot(values, colors[i] + '-')
plot_line_list.append(blue_line[0])
plot_name_list.append(names[i])
i += 1
if plot_second:
values = put_in_bins(second_stream, sizes)
all_values.extend(values)
magenta_line = plt.plot(values, colors[i] + '-')
plot_line_list.append(magenta_line[0])
plot_name_list.append(names[i])
i += 1
if plot_rest:
values = put_in_bins(rest_streams, sizes)
all_values.extend(values)
green_line = plt.plot(values, colors[i] + '-')
plot_line_list.append(green_line[0])
plot_name_list.append(names[i])
i += 1
plt.legend(plot_line_list, plot_name_list)
if sizes:
fn, tix = make_storage_ticks(all_values)
ax.yaxis.set_major_formatter(fn)
plt.yticks(tix)
ax.set_xticklabels([int(x * window) for x in ax.get_xticks()])
return fig, ax, plot_line_list, plot_name_list
