def plot_latency_v0(ms_files, labels, plot_file, plot_dir):
colors = plot_util.COLORLISTS[len(ms_files)]
markers = plot_util.MARKERS
for i, (ms_file, label, m,
c) in enumerate(zip(ms_files, labels, markers, colors)):
xs, ys, errs = data_util.get_performance_data(ms_file)
plt.plot(xs,
ys,
label=label,
lw=4,
markersize=8,
alpha=0.5,
marker=m,
color=c)
#plt.errorbar(xs, ys, yerr=errs, label=label,
# lw=4,
# markersize=8,
# marker=m,
# color=c)
filename = os.path.join(plot_dir, plot_file)
plot_util.format_plot("Num Apps", "Average Latency (us)", 15)
plt.tight_layout()
plt.yscale('log')
plt.legend(loc=0, fontsize=15)
plt.savefig(filename + ".pdf")
plt.clf()
def plot_by_num_apps_v0(ms_files, labels, num_setups, plot_file, plot_dir):
colors = plot_util.COLORLISTS[len(ms_files)]
markers = plot_util.MARKERS
metric = "F1"
for j in range(2):
lines = []
all_pts = []
for i, (ms_file, label, m,
c) in enumerate(zip(ms_files, labels, markers, colors)):
metrics, fpses = data_util.get_scheduler_data(ms_file)
xs = []
ys = []
errs = []
for num_apps in sorted(metrics.keys()):
if num_setups > len(metrics[num_apps]):
continue
xs.append(num_apps)
ys.append(np.average(metrics[num_apps]))
errs.append(np.std(metrics[num_apps]))
plt.plot(xs,
ys,
label=label,
lw=4,
markersize=8,
alpha=0.5,
marker=m,
color=c)
#plt.errorbar(xs, ys, yerr=errs, label=label,
# lw=4,
# markersize=8,
# marker=m,
# color=c)
plot_util.format_plot("Num Apps", "Average Event " + metric)
plt.title(str(num_setups) + " setups per point", fontsize=20)
plt.ylim(0, 1)
filename = os.path.join(plot_dir, plot_file + "-" + metric)
plt.legend(loc=0, fontsize=15)
plt.savefig(filename + ".pdf")
plt.clf()
def plot_precision(ms_files,
max_files,
min_files,
plot_files,
titles,
plot_dir,
errbars=True):
for i in range(2):
for ms_file, max_file, min_file, plot_file, title \
in zip(ms_files, max_files, min_files, plot_files, titles):
xs1, ys1, errs1, _, _ = get_precision_data(ms_file)
xs2, ys2, errs2, _, _ = get_precision_data(max_file)
xs3, ys3, errs3, _, _ = get_precision_data(min_file)
if not errbars:
errs3 = None
errs2 = None
errs1 = None
plt.errorbar(xs3,
ys3,
yerr=errs3,
lw=4,
markersize=8,
marker=plot_util.NO_SHARING['marker'],
color=plot_util.NO_SHARING['color'],
label=plot_util.NO_SHARING['label'])
plt.errorbar(xs2,
ys2,
yerr=errs2,
lw=4,
markersize=8,
marker=plot_util.MAX_SHARING['marker'],
color=plot_util.MAX_SHARING['color'],
label=plot_util.MAX_SHARING['label'])
plt.errorbar(xs1,
ys1,
yerr=errs1,
lw=4,
markersize=8,
marker=plot_util.MAINSTREAM['marker'],
color=plot_util.MAINSTREAM['color'],
label=plot_util.MAINSTREAM['label'])
plot_util.format_plot("Number of concurrent apps",
"Event Precision")
plt.xlim(max(min(xs1), 2), max(xs1))
plt.ylim(0, 1)
plt.savefig(plot_dir + "/" + plot_file + "-precision.pdf")
plt.clf()
def plot_recall(ms_files,
max_files,
min_files,
plot_files,
titles,
plot_dir,
version=0):
for i in range(2):
for ms_file, max_file, min_file, plot_file, title \
in zip(ms_files, max_files, min_files, plot_files, titles):
xs1, ys1, errs1, losses1, fpses1 = get_recall_data(
ms_file, version)
xs2, ys2, errs2, losses2, fpses2 = get_recall_data(
max_file, version)
xs3, ys3, errs3, losses3, fpses3 = get_recall_data(
min_file, version)
plt.errorbar(xs3,
ys3,
yerr=errs3,
lw=4,
markersize=8,
marker=plot_util.NO_SHARING['marker'],
color=plot_util.NO_SHARING['color'],
label=plot_util.NO_SHARING['label'])
plt.errorbar(xs2,
ys2,
yerr=errs2,
lw=4,
markersize=8,
marker=plot_util.MAX_SHARING['marker'],
color=plot_util.MAX_SHARING['color'],
label=plot_util.MAX_SHARING['label'])
plt.errorbar(xs1,
ys1,
yerr=errs1,
lw=4,
markersize=8,
marker=plot_util.MAINSTREAM['marker'],
color=plot_util.MAINSTREAM['color'],
label=plot_util.MAINSTREAM['label'])
plot_util.format_plot("Number of concurrent apps", "Event Recall")
plt.xlim(max(min(xs1), 2), max(xs1))
plt.ylim(0, 1)
plt.savefig(plot_dir + "/" + plot_file + "-recall.pdf")
plt.clf()
def plot_bar_v0(ms_files, label_prefixes, plot_file, plot_dir, limit=20):
colors = plot_util.COLORLISTS[len(ms_files)]
metric = "F1"
title = "F1"
width = 0.3
for j in range(2):
lines = []
all_pts = []
for i, (ms_file, label_prefix,
c) in enumerate(zip(ms_files, label_prefixes, colors)):
metrics, fpses = data_util.get_scheduler_data(ms_file)
ys = []
avg_fpses = []
for num_apps in sorted(metrics.keys()):
ys += metrics[num_apps]
avg_fpses += fpses[num_apps]
xs = range(len(ys))
xs_bar = [x + (i * width) for x in xs]
plt.bar(xs_bar[:limit],
ys[:limit],
width,
color=c,
alpha=0.5,
label=label_prefix)
plot_util.format_plot("Config ID", "Event " + title)
plt.ylim(0, 1)
filename = os.path.join(plot_dir, plot_file + "-" + metric)
plt.legend(loc=3, fontsize=15)
plt.savefig(filename + ".pdf")
plt.clf()
def plot_correlation(ms_files, labels, plot_file, plot_dir, version=0):
colors = plot_util.COLORLISTS[len(ms_files)]
markers = plot_util.MARKERS[:len(ms_files)]
for i in range(2):
for ms_file, label, c, m in zip(ms_files, labels, colors, markers):
xs, ys, errs, losses, fpses = get_recall_data(ms_file, version)
plt.errorbar(xs,
ys,
yerr=errs,
label=label,
lw=4,
markersize=8,
marker=m,
color=c)
plot_util.format_plot("Number of concurrent apps", "Recall")
plt.xlim(max(min(xs), 2), max(xs))
plt.ylim(0, 1)
plt.savefig(plot_dir + "/" + plot_file + "-correlation.pdf")
plt.clf()
def run(exponent_range, num_frozen, plot_dir):
#exponents = range(8, exponent_range + 2, 2)
#c0s = np.arange(0, 0.2, 0.05)
epses = np.arange(0, 0.4, 0.1)
max_accs = np.arange(0.4, 1, 0.2)
exponent = 10
c0 = 0.1
#epses = [epses[1]]
#max_accs = [max_accs[1]]
for eps in epses:
for max_acc in max_accs:
plot_range(exponent, c0, eps, max_acc, num_frozen)
plot_file = os.path.join(plot_dir, "variants")
plot_util.format_plot("Number of frozen layers", "Top-1 Accuracy")
plt.ylim(0, 1)
plt.savefig(plot_file)
plt.clf()
def plot_x_voting(ms_files,
labels,
plot_file,
plot_dir,
dual=False,
frontier=False):
colors = plot_util.COLORLISTS[4]
colors = [colors[0], colors[3], colors[1], colors[2]]
markers = plot_util.MARKERS[:len(ms_files)]
metrics = ["f1", "recall", "precision"]
titles = ["F1-score", "Recall", "Precision"]
for metric, title in zip(metrics, titles):
for i in range(2):
fig, ax1 = plt.subplots()
if dual:
ax2 = ax1.twinx()
lines = []
all_pts = []
for ms_file, label, c, m in zip(ms_files, labels, colors, markers):
if metric == 'f1':
xs, ys, errs, losses, fpses = get_f1_data(ms_file)
elif metric == 'recall':
xs, ys, errs, losses, fpses = get_recall_data(ms_file, 1)
elif metric == 'precision':
xs, ys, errs, losses, fpses = get_precision_data(ms_file)
if frontier:
if label == "7-voting":
lines.append(
ax1.errorbar(xs,
ys,
yerr=errs,
label=label,
lw=2,
markersize=6,
marker=m,
color=c))
else:
lines.append(
ax1.errorbar(xs,
ys,
yerr=errs,
label=label,
lw=2,
markersize=4,
marker=m,
color=c))
# lines.append(ax1.scatter(xs, ys, label=label, s=50,
# edgecolor='black', marker=m, color=c))
all_pts += list(zip(xs, ys))
else:
lines.append(
ax1.errorbar(xs,
ys,
yerr=errs,
label=label,
lw=4,
markersize=8,
marker=m,
color=c))
if dual:
ax2.plot(xs,
fpses,
lw=2,
markersize=8,
alpha=0.2,
marker=m,
color=c)
if frontier:
if plot_file == 'voting-train-500-f1' and metric == 'f1':
xss, ys = plot_util.frontier(all_pts, True)
else:
xss, ys = plot_util.frontier(all_pts, False)
lines += ax1.plot(xss, ys, '--', label='Pareto Frontier', lw=7)
ax1.set_ylim(0, 1)
if dual:
plot_util.format_plot_dual(ax1, ax2,
"Number of concurrent apps",
"Event " + title, "Average FPS")
else:
plot_util.format_plot("Number of concurrent apps",
"Event " + title)
ax1.set_xlim(max(min(xs), 2), max(xs))
filename = plot_dir + "/" + plot_file + "-" + metric
if dual:
filename += "-dual"
if frontier:
filename += "-frontier"
labels = [l.get_label() for l in lines]
leg = ax1.legend(lines, labels, loc=0, fontsize=15)
leg.get_frame().set_alpha(0.5)
plt.savefig(filename + ".pdf")
def plot_f1(ms_files,
max_files,
min_files,
plot_files,
titles,
plot_dir,
errbars=True,
xlim=None,
annotations=[],
ms_variant_files=[],
ms_variant_name=None,
legend=True,
legend_xargs={}):
# import matplotlib
# matplotlib.use('ps')
# matplotlib.rc('text', usetex=True)
# matplotlib.rc('text.latex', preamble='\usepackage{xcolor}')
for i in range(2):
for i, (ms_file, max_file, min_file, plot_file, title) \
in enumerate(zip(ms_files, max_files, min_files, plot_files, titles)):
ms_variant_file = ms_variant_files[i] if ms_variant_files else None
xs1, ys1, errs1, losses1, fpses1 = get_f1_data(ms_file)
xs2, ys2, errs2, losses2, fpses2 = get_f1_data(max_file)
xs3, ys3, errs3, losses3, fpses3 = get_f1_data(min_file)
if ms_variant_file:
xs4, ys4, errs4, losses4, fpses4 = get_f1_data(ms_variant_file)
plot_util.format_plot("Number of concurrent apps",
"Event F1-score")
if xlim:
plt.xlim(*xlim)
else:
plt.xlim(max(min(xs1), 2), max(xs1))
plt.ylim(0, 1)
if not errbars:
errs3 = None
errs2 = None
errs1 = None
plt.errorbar(xs3,
ys3,
yerr=errs3,
lw=4,
markersize=8,
marker=plot_util.NO_SHARING['marker'],
color=plot_util.NO_SHARING['color'],
label=plot_util.NO_SHARING['label'])
plt.errorbar(xs2,
ys2,
yerr=errs2,
lw=4,
markersize=8,
marker=plot_util.MAX_SHARING['marker'],
color=plot_util.MAX_SHARING['color'],
label=plot_util.MAX_SHARING['label'])
plt.errorbar(xs1,
ys1,
yerr=errs1,
lw=4,
markersize=8,
marker=plot_util.MAINSTREAM['marker'],
color=plot_util.MAINSTREAM['color'],
label=plot_util.MAINSTREAM['label'])
# To get calculation for atc2018.
diffs = [[], []]
for x1, y1, x2, y2, x3, y3 in zip(xs1, ys1, xs2, ys2, xs3, ys3):
# print "should be same", x1, x2, x3
assert x1 == x2 == x3
print x1, y1, y2, y3, "%", y1 / y2, y1 / y3
diffs[0].append(y1 / y2)
diffs[1].append(y1 / y3)
# print "difference", 0., y1 - y2, y1 - y3
print "max", max(diffs[0]), max(diffs[1])
if ms_variant_file:
plt.errorbar(xs4,
ys4,
yerr=errs4,
lw=4,
markersize=8,
marker=plot_util.MAINSTREAM_VARIANT['marker'],
color=plot_util.MAINSTREAM_VARIANT['color'],
label=ms_variant_name if ms_variant_name else
plot_util.MAINSTREAM_VARIANT['label'])
if legend:
leg = plt.legend(**legend_xargs)
leg.get_frame().set_alpha(0.5)
plt.savefig(plot_dir + "/" + plot_file + "-f1.pdf")
if len(annotations) > 0:
assert len(annotations) == 6
annotations_params = [
{
'xy': (-30, 30),
'src': 1,
'name': 'a'
},
{
'xy': (-280, 40),
'src': 1,
'name': 'b'
},
{
'xy': (15, -70),
'src': 2,
'name': 'e'
},
{
'xy': (-130, -40),
'src': 2,
'name': 'd'
},
{
'xy': (35, -25),
'src': 3,
'name': 'c'
},
{
'xy': (-170, 30),
'src': 3,
'name': 'f'
},
]
for annotation, params in zip(annotations, annotations_params):
if params['src'] == 1:
(x, y, loss,
fps) = (xs1[annotation], ys1[annotation],
losses1[annotation], fpses1[annotation])
elif params['src'] == 2:
(x, y, loss,
fps) = (xs2[annotation], ys2[annotation],
losses2[annotation], fpses2[annotation])
elif params['src'] == 3:
(x, y, loss,
fps) = (xs3[annotation], ys3[annotation],
losses3[annotation], fpses3[annotation])
# plt.annotate("\\textcolor{{red}}{{({})}} Frame Acc: {}, FPS: {}".format(params["name"], 1-loss, fps), fpses3[annotation])
plt.annotate("({}) Frame Acc: {}, FPS: {}".format(
params["name"], 1 - loss, fps),
xy=(x, y),
xytext=params['xy'],
xycoords='data',
fontsize=15,
textcoords='offset points',
arrowprops=dict(arrowstyle="->"))
plt.savefig(plot_dir + "/" + plot_file + "-f1-annotated.pdf")
plt.clf()