def main(pkl_list, name_list, optimum=0, title="", log=False, save="", y_max=0,
y_min=0, cut=sys.maxint, linewidth=1,
linestyles=plot_util.get_single_linestyle(), colors=None,
markers=plot_util.get_empty_iterator(),
ylabel=None, xlabel=None):
if colors is None:
colors= plot_util.get_plot_colors()
trial_list = list()
for i in range(len(pkl_list)):
if len(pkl_list[i]) != 1:
raise ValueError("%s is more than <onePickle>!" % str(pkl_list))
fh = open(pkl_list[i][0])
trl = cPickle.load(fh)
fh.close()
trial_list.append(plot_util.extract_results(trl))
sys.stdout.write("Plotting trace\n")
plot_optimization_trace(trial_list=trial_list, name_list=name_list, optimum=optimum,
title=title, log=log, save=save, y_max=y_max,
y_min=y_min, cut=cut, linewidth=linewidth,
linestyles=linestyles, colors=colors,
markers=markers, ylabel=ylabel, xlabel=xlabel)
if save != "":
sys.stdout.write("Saved plot to " + save + "\n")
else:
sys.stdout.write("..Done\n")
def main(pkl_list,
name_list,
optimum=0,
title="",
log=False,
save="",
y_max=0,
y_min=0,
cut=sys.maxint,
linewidth=1,
linestyles=plot_util.get_single_linestyle(),
colors=None,
markers=plot_util.get_empty_iterator(),
ylabel=None,
xlabel=None):
if colors is None:
colors = plot_util.get_plot_colors()
trial_list = list()
for i in range(len(pkl_list)):
if len(pkl_list[i]) != 1:
raise ValueError("%s is more than <onePickle>!" % str(pkl_list))
fh = open(pkl_list[i][0])
trl = cPickle.load(fh)
fh.close()
trial_list.append(plot_util.extract_results(trl))
sys.stdout.write("Plotting trace\n")
plot_optimization_trace(trial_list=trial_list,
name_list=name_list,
optimum=optimum,
title=title,
log=log,
save=save,
y_max=y_max,
y_min=y_min,
cut=cut,
linewidth=linewidth,
linestyles=linestyles,
colors=colors,
markers=markers,
ylabel=ylabel,
xlabel=xlabel)
if save != "":
sys.stdout.write("Saved plot to " + save + "\n")
else:
sys.stdout.write("..Done\n")
def plot_ranking(rankings, optimum=0, title="", log=False,
save="", y_min=0, y_max=0, figsize=(16, 6), legend_ncols=4,
colors=None, markers=None, markersize=6, linewidth=3):
# check if all optimizers have the same number of runs
# if np.mean([name[1] for name in name_list]) != name_list[0][1]:
# raise Exception("All optimizers must have the same numbers of "
# "experiment runs! %s" % name_list)
fig = plt.figure(dpi=600, figsize=figsize)
ax = plt.subplot(111)
if colors is None:
colors = plot_util.get_plot_colors()
if markers is None:
markers = plot_util.get_empty_iterator()
for i, optimizer in enumerate(rankings):
ax.plot(range(0, rankings[optimizer].shape[0]),
rankings[optimizer],
marker=markers.next(), markersize=markersize,
color=colors.next(), linewidth=linewidth,
label=optimizer.replace("\\", "").
replace("learned", "d_c").replace("l1", "d_p"))
ax.legend(loc='upper center', #bbox_to_anchor=(0.5, -0.05),
fancybox=True, shadow=True, ncol=legend_ncols,
labelspacing=0.25, fontsize=12)
ax.set_xlabel("#Function evaluations")
ax.set_ylabel("Average rank")
box = ax.get_position()
#ax.set_position([box.x0, box.y0 + box.height * 0.1,
# box.width, box.height * 0.9])
if save != "":
plt.savefig(save, facecolor='w', edgecolor='w',
orientation='portrait', papertype=None, format=None,
transparent=False, bbox_inches="tight", pad_inches=0.1)
else:
plt.show()
plt.close(fig)
def main(pkl_list, name_list, autofill, optimum=0, maxvalue=sys.maxint,
save="", title="", log=False,
y_min=0, y_max=0, linewidth=1, linestyles=plot_util.get_single_linestyle(),
colors=None, markers=plot_util.get_empty_iterator(),
markersize=6, ylabel=None, xlabel=None):
if colors is None:
colors = plot_util.get_plot_colors()
trial_list = list()
for i in range(len(pkl_list)):
tmp_trial_list = list()
max_len = -sys.maxint
for pkl in pkl_list[i]:
fh = open(pkl, "r")
trials = cPickle.load(fh)
fh.close()
trace = plot_util.get_Trace_cv(trials, maxvalue=maxvalue)
tmp_trial_list.append(trace)
max_len = max(max_len, len(trace))
trial_list.append(list())
for tr in tmp_trial_list:
# if len(tr) < max_len:
# tr.extend([tr[-1] for idx in range(abs(max_len - len(tr)))])
trial_list[-1].append(np.array(tr))
plot_optimization_trace_cv(trial_list, name_list, optimum, title=title, log=log,
save=save, y_min=y_min, y_max=y_max,
linewidth=linewidth, linestyles=linestyles,
colors=colors, markers=markers,
markersize=markersize, ylabel=ylabel, xlabel=xlabel)
if save != "":
sys.stdout.write("Saved plot to " + save + "\n")
else:
sys.stdout.write("..Done\n")
def plot_time_trace(time_dict,
name_list,
title="",
log=True,
save="",
y_max=0,
y_min=0):
colors = plot_util.get_plot_colors()
markers = plot_util.get_plot_markers()
linestyles = itertools.cycle(['-'])
size = 5
ratio = 5
gs = matplotlib.gridspec.GridSpec(ratio, 1)
fig = plt.figure(1, dpi=100)
fig.suptitle(title, fontsize=16)
ax1 = plt.subplot(gs[0:ratio, :])
ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
min_val = sys.maxint
max_val = -sys.maxint
max_trials = 0
trial_list_means = list()
trial_list_std = list()
num_runs_list = list()
# Get mean and std for all times and optimizers
for entry in name_list:
k = entry[0]
trial_list_std.append(np.std(np.array(time_dict[k]), axis=0))
if log:
trial_list_means.append(
np.log10(np.mean(np.array(time_dict[k]), axis=0)))
else:
trial_list_means.append(np.mean(np.array(time_dict[k]), axis=0))
num_runs_list.append(len(time_dict[k]))
for k in range(len(name_list)):
# Plot mean and std for optimizer duration
c = colors.next()
m = markers.next()
x = range(len(trial_list_means[k]))
l = linestyles.next()
ax1.fill_between(x,
trial_list_means[k] - trial_list_std[k],
trial_list_means[k] + trial_list_std[k],
facecolor=c,
alpha=0.3,
edgecolor=c)
ax1.plot(x,
trial_list_means[k],
color=c,
linewidth=size,
label=name_list[k][0],
linestyle=l,
marker=m)
# Plot number of func evals for this experiment
if min(trial_list_means[k] - trial_list_std[k]) < min_val:
min_val = min(trial_list_means[k] - trial_list_std[k])
if max(trial_list_means[k] + trial_list_std[k]) > max_val:
max_val = max(trial_list_means[k] + trial_list_std[k])
if len(trial_list_means[k]) > max_trials:
max_trials = len(trial_list_means[k])
# Descript and label the stuff
fig.suptitle(title, fontsize=16)
leg = ax1.legend(loc='best', fancybox=True)
leg.get_frame().set_alpha(0.5)
if log:
ax1.set_ylabel("log10(Optimizer time in [sec])")
else:
ax1.set_ylabel("Optimizer time in [sec]")
if y_max == y_min:
ax1.set_ylim([min_val - 2, max_val + 2])
else:
ax1.set_ylim([y_min, y_max])
ax1.set_xlim([0, max_trials])
plt.tight_layout()
plt.subplots_adjust(top=0.85)
if save != "":
plt.savefig(save,
dpi=100,
facecolor='w',
edgecolor='w',
orientation='portrait',
papertype=None,
format=None,
transparent=False,
bbox_inches="tight",
pad_inches=0.1)
else:
plt.show()
def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=False,
save="", y_min=0, y_max=0, cut=sys.maxint,
linewidth=1, linestyles=plot_util.get_single_linestyle(),
colors=None, markers=plot_util.get_empty_iterator(),
ylabel=None, xlabel=None):
if colors is None:
colors= plot_util.get_plot_colors()
size = 1
# get handles
ratio = 5
gs = matplotlib.gridspec.GridSpec(ratio, 1)
fig = figure(1, dpi=100)
fig.suptitle(title, fontsize=16)
ax = subplot(gs[0:ratio, :])
ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
min_val = sys.maxint
max_val = -sys.maxint
max_trials = 0
# This might not do what we actually want; Ideally it would only take the
# ones that where actually run according to instance_order
for i in range(len(name_list)):
print cut, len(trial_list[i])
num_plotted_trials = np.min([cut, len(trial_list[i])])
print num_plotted_trials
x = range(num_plotted_trials)
y = np.zeros((num_plotted_trials))
line = np.zeros((num_plotted_trials))
for j, inst_res in enumerate(trial_list[i][:num_plotted_trials]):
if j >= len(y):
break
if type(inst_res) == np.ndarray and not np.isfinite(inst_res).any():
inst_res[np.isnan(inst_res)] = 1
elif type(inst_res) != np.ndarray and np.isnan(inst_res):
inst_res = 1
tmp = sc.nanmean(np.array([inst_res, inst_res]).flat) # Black Magic
if log:
y[j] = np.log(tmp - optimum)
line[j] = np.min(y[:j + 1])
else:
y[j] = tmp - optimum
line[j] = np.min(y[:j + 1])
# Plot the stuff
marker = markers.next()
color = colors.next()
line_style_ = linestyles.next()
ax.scatter(np.argmin(line), min(line), facecolor="w", edgecolor=color,
s=size*10*15, marker=marker)
ax.scatter(x, y, color=color, marker=marker, s=size*15)
ax.plot(x, line, color=color, label=name_list[i][0], linestyle=line_style_,
linewidth=linewidth)
if min(y) < min_val:
min_val = min(y)
if max(y) > max_val:
max_val = max(y)
if num_plotted_trials > max_trials:
max_trials = num_plotted_trials
# Describe and label the stuff
if xlabel is None:
xlabel = "#Function evaluations"
ax.set_xlabel(xlabel)
if ylabel is None:
if log:
ylabel = "log10(Minfunction value)"
else:
ylabel = "Minfunction value"
ax.set_ylabel(ylabel)
if y_min == y_max:
ax.set_ylim([min_val - 0.1, max_val + 0.1])
else:
ax.set_ylim([y_min, y_max])
ax.set_xlim([0, max_trials])
leg = ax.legend(loc='best', fancybox=True)
leg.get_frame().set_alpha(0.5)
tight_layout()
subplots_adjust(top=0.85)
if save != "":
savefig(save, dpi=100, facecolor='w', edgecolor='w',
orientation='portrait', papertype=None, format=None,
transparent=False, bbox_inches="tight", pad_inches=0.1)
else:
show()
def plot_optimization_trace(trial_list, name_list, times_list, optimum=0, title="",
log=True, save="", y_max=0, y_min=0, scale_std=1):
markers = plot_util.get_plot_markers()
colors = plot_util.get_plot_colors()
linestyles = itertools.cycle(['-'])
size = 1
ratio = 5
gs = matplotlib.gridspec.GridSpec(ratio, 1)
fig = figure(1, dpi=100)
fig.suptitle(title, fontsize=16)
ax1 = subplot(gs[0:ratio, :])
ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
min_val = sys.maxint
max_val = -sys.maxint
max_trials = 0
trial_list_means = list()
trial_list_std = list()
# One trialList represents all runs from one optimizer
for i in range(len(trial_list)):
if log:
trial_list_means.append(np.log10(np.mean(np.array(trial_list[i]), axis=0)))
else:
trial_list_means.append(np.mean(np.array(trial_list[i]), axis=0))
trial_list_std.append(np.std(np.array(trial_list[i]), axis=0)*scale_std)
times_list[i] = np.array(times_list[i])
fig.suptitle(title, fontsize=16)
# Plot the average error and std
for i in range(len(trial_list_means)):
x = times_list[i]
y = trial_list_means[i] - optimum
# m = markers.next()
c = colors.next()
l = linestyles.next()
std_up = y + trial_list_std[i]
std_down = y - trial_list_std[i]
ax1.fill_between(x, std_down, std_up,
facecolor=c, alpha=0.3, edgecolor=c)
ax1.plot(x, y, color=c, linewidth=size*2,
label=name_list[i][0] + "(" + str(len(trial_list[i])) + ")",
linestyle=l, marker="")
if min(std_down) < min_val:
min_val = min(std_down)
if max(y + std_up) > max_val:
max_val = max(std_up)
if max(times_list[i]) > max_trials:
max_trials = max(times_list[i])
# Maybe plot on logscale
if scale_std != 1:
ylabel = ", %s * std" % scale_std
else:
ylabel = ""
if log:
ax1.set_ylabel("log10(Minfunction value)" + ylabel)
else:
ax1.set_ylabel("Minfunction value" + ylabel)
# Descript and label the stuff
leg = ax1.legend(loc='best', fancybox=True)
leg.get_frame().set_alpha(0.5)
ax1.set_xlabel("Duration [sec] ")
if y_max == y_min:
# Set axes limit
ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))])
else:
ax1.set_ylim([y_min, y_max])
ax1.set_xlim([0, max_trials])
tight_layout()
subplots_adjust(top=0.85)
if save != "":
savefig(save, dpi=100, facecolor='w', edgecolor='w',
orientation='portrait', papertype=None, format=None,
transparent=False, bbox_inches="tight", pad_inches=0.1)
else:
show()
def plot_ranking(rankings,
optimum=0,
title="",
log=False,
save="",
y_min=0,
y_max=0,
figsize=(16, 6),
legend_ncols=4,
colors=None,
markers=None,
markersize=6,
linewidth=3):
# check if all optimizers have the same number of runs
# if np.mean([name[1] for name in name_list]) != name_list[0][1]:
# raise Exception("All optimizers must have the same numbers of "
# "experiment runs! %s" % name_list)
fig = plt.figure(dpi=600, figsize=figsize)
ax = plt.subplot(111)
if colors is None:
colors = plot_util.get_plot_colors()
if markers is None:
markers = plot_util.get_empty_iterator()
for i, optimizer in enumerate(rankings):
ax.plot(range(0, rankings[optimizer].shape[0]),
rankings[optimizer],
marker=markers.next(),
markersize=markersize,
color=colors.next(),
linewidth=linewidth,
label=optimizer.replace("\\", "").replace("learned",
"d_c").replace(
"l1", "d_p"))
ax.legend(
loc='upper center', #bbox_to_anchor=(0.5, -0.05),
fancybox=True,
shadow=True,
ncol=legend_ncols,
labelspacing=0.25,
fontsize=12)
ax.set_xlabel("#Function evaluations")
ax.set_ylabel("Average rank")
box = ax.get_position()
#ax.set_position([box.x0, box.y0 + box.height * 0.1,
# box.width, box.height * 0.9])
if save != "":
plt.savefig(save,
facecolor='w',
edgecolor='w',
orientation='portrait',
papertype=None,
format=None,
transparent=False,
bbox_inches="tight",
pad_inches=0.1)
else:
plt.show()
plt.close(fig)
def plot_summed_wins_of_optimizers(trial_list_per_dataset,
name_list_per_dataset,
save="",
cut=sys.maxint,
figsize=(16, 4),
legend_ncols=3,
colors=None,
linewidth=3,
markers=None,
markersize=6):
# TODO colors should be a function handle which returns an Iterable!
# This is a hack
cut = 50
optimizers, summed_wins_of_optimizer = get_summed_wins_of_optimizers(
trial_list_per_dataset, name_list_per_dataset)
# Make a copy of the colors iterator, because we need it more than once!
if colors is not None:
if not isinstance(colors, itertools.cycle):
raise TypeError()
else:
color_values = list()
for i in range(10):
r, g, b = colors.next()
color_values.append((r, g, b))
if markers is not None:
if not isinstance(markers, itertools.cycle):
raise TypeError()
else:
marker_values = list()
for i in range(10):
marker_values.append(markers.next())
################################################################################
# Plot statistics
for opt1_idx, key in enumerate(optimizers):
fig, (ax0, ax1) = plt.subplots(nrows=2,
sharex=True,
dpi=600,
figsize=figsize)
if colors is None:
colors_ = plot_util.get_plot_colors()
else:
colors_ = itertools.cycle(color_values)
if markers is None:
markers_ = plot_util.get_empty_iterator()
else:
markers_ = itertools.cycle(marker_values)
y_max = 0.
for opt2_idx, key2 in enumerate(optimizers):
if opt1_idx == opt2_idx:
continue
y = []
y1 = []
for i in range(0, cut + 1):
y.append(summed_wins_of_optimizer[i][opt1_idx, opt2_idx] /
len(trial_list_per_dataset) * 100)
y1.append(-summed_wins_of_optimizer[i][opt2_idx, opt1_idx] /
len(trial_list_per_dataset) * 100)
y_max_tmp = max(np.max(y), np.max(np.abs(y1)))
y_max_tmp = np.ceil(y_max_tmp * 10) / 10.
y_max = max(y_max_tmp, y_max)
label = "%s vs %s" % (key, key2)
label = label.replace("learned", "d_c").replace("l1", "d_p")
color = colors_.next()
marker = markers_.next()
ax0.plot(range(0, cut + 1),
y,
color=color,
label=label,
linewidth=linewidth,
marker=marker,
markersize=markersize)
ax1.plot(range(0, cut + 1),
y1,
color=color,
label=label,
linewidth=linewidth,
marker=marker,
markersize=markersize)
#handles, labels = ax1.get_legend_handles_labels()
#fig.legend(handles, labels, loc="upper center", fancybox=True,
# ncol=legend_ncols, shadow=True)
ax0.set_xlim((0, cut))
ax0.set_ylim((0, y_max))
ax0.set_ylabel("Significant wins (%)")
ax1.set_xlim((0, cut))
ax1.set_ylim((-y_max, 0))
ax1.set_ylabel("Significant losses (%)")
yticklabels = ax1.get_yticks().tolist()
#print yticklabels, [item.get_text() for item in yticklabels]
ax1.set_yticklabels([-int(item) for item in yticklabels])
ax1.legend(loc="best", fancybox=True, ncol=legend_ncols, shadow=True)
plt.tight_layout()
#plt.subplots_adjust(top=0.85)
plt.xlabel("#Function evaluations")
if save != "":
plt.savefig(save % key,
facecolor='w',
edgecolor='w',
orientation='portrait',
papertype=None,
format=None,
transparent=False,
bbox_inches="tight",
pad_inches=0.1)
else:
plt.show()
plt.close(fig)
def plot_optimization_trace(trial_list,
name_list,
optimum=0,
title="",
log=False,
save="",
y_min=0,
y_max=0,
cut=sys.maxint):
markers = plot_util.get_plot_markers()
colors = plot_util.get_plot_colors()
linestyles = itertools.cycle(['-'])
size = 1
# get handles
ratio = 5
gs = matplotlib.gridspec.GridSpec(ratio, 1)
fig = figure(1, dpi=100)
fig.suptitle(title, fontsize=16)
ax = subplot(gs[0:ratio, :])
ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
min_val = sys.maxint
max_val = -sys.maxint
max_trials = 0
# This might not do what we actually want; Ideally it would only take the
# ones that where actually run according to instance_order
for i in range(len(name_list)):
print cut, len(trial_list[i])
num_plotted_trials = np.min([cut, len(trial_list[i])])
print num_plotted_trials
x = range(num_plotted_trials)
y = np.zeros((num_plotted_trials))
line = np.zeros((num_plotted_trials))
for j, inst_res in enumerate(trial_list[i][:num_plotted_trials]):
if j >= len(y):
break
if type(inst_res
) == np.ndarray and not np.isfinite(inst_res).any():
inst_res[np.isnan(inst_res)] = 1
elif type(inst_res) != np.ndarray and np.isnan(inst_res):
inst_res = 1
tmp = sc.nanmean(np.array([inst_res,
inst_res]).flat) # Black Magic
if log:
y[j] = np.log(tmp - optimum)
line[j] = np.min(y[:j + 1])
else:
y[j] = tmp - optimum
line[j] = np.min(y[:j + 1])
# Plot the stuff
marker = markers.next()
color = colors.next()
l = linestyles.next()
ax.scatter(np.argmin(line),
min(line),
facecolor="w",
edgecolor=color,
s=size * 10 * 15,
marker=marker)
ax.scatter(x, y, color=color, marker=marker, s=size * 15)
ax.plot(x,
line,
color=color,
label=name_list[i][0],
linestyle=l,
linewidth=size)
if min(y) < min_val:
min_val = min(y)
if max(y) > max_val:
max_val = max(y)
if num_plotted_trials > max_trials:
max_trials = num_plotted_trials
# Describe and label the stuff
ax.set_xlabel("#Function evaluations")
if log:
ax.set_ylabel("log10(Minfunction value)")
else:
ax.set_ylabel("Minfunction value")
if y_min == y_max:
ax.set_ylim([min_val - 0.1, max_val + 0.1])
else:
ax.set_ylim([y_min, y_max])
ax.set_xlim([0, max_trials])
leg = ax.legend(loc='best', fancybox=True)
leg.get_frame().set_alpha(0.5)
tight_layout()
subplots_adjust(top=0.85)
if save != "":
savefig(save,
dpi=100,
facecolor='w',
edgecolor='w',
orientation='portrait',
papertype=None,
format=None,
transparent=False,
bbox_inches="tight",
pad_inches=0.1)
else:
show()
def plot_time_trace(time_dict, name_list, title="", log=True, save="", y_max=0, y_min=0):
colors = plot_util.get_plot_colors()
markers = plot_util.get_plot_markers()
linestyles = itertools.cycle(['-'])
size = 5
ratio = 5
gs = matplotlib.gridspec.GridSpec(ratio, 1)
fig = plt.figure(1, dpi=100)
fig.suptitle(title, fontsize=16)
ax1 = plt.subplot(gs[0:ratio, :])
ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
min_val = sys.maxint
max_val = -sys.maxint
max_trials = 0
trial_list_means = list()
trial_list_std = list()
num_runs_list = list()
# Get mean and std for all times and optimizers
for entry in name_list:
k = entry[0]
trial_list_std.append(np.std(np.array(time_dict[k]), axis=0))
if log:
trial_list_means.append(np.log10(np.mean(np.array(time_dict[k]), axis=0)))
else:
trial_list_means.append(np.mean(np.array(time_dict[k]), axis=0))
num_runs_list.append(len(time_dict[k]))
for k in range(len(name_list)):
# Plot mean and std for optimizer duration
c = colors.next()
m = markers.next()
x = range(len(trial_list_means[k]))
l = linestyles.next()
ax1.fill_between(x, trial_list_means[k] - trial_list_std[k],
trial_list_means[k] + trial_list_std[k],
facecolor=c, alpha=0.3, edgecolor=c)
ax1.plot(x, trial_list_means[k], color=c, linewidth=size, label=name_list[k][0], linestyle=l, marker=m)
# Plot number of func evals for this experiment
if min(trial_list_means[k] - trial_list_std[k]) < min_val:
min_val = min(trial_list_means[k] - trial_list_std[k])
if max(trial_list_means[k] + trial_list_std[k]) > max_val:
max_val = max(trial_list_means[k] + trial_list_std[k])
if len(trial_list_means[k]) > max_trials:
max_trials = len(trial_list_means[k])
# Descript and label the stuff
fig.suptitle(title, fontsize=16)
leg = ax1.legend(loc='best', fancybox=True)
leg.get_frame().set_alpha(0.5)
if log:
ax1.set_ylabel("log10(Optimizer time in [sec])")
else:
ax1.set_ylabel("Optimizer time in [sec]")
if y_max == y_min:
ax1.set_ylim([min_val-2, max_val+2])
else:
ax1.set_ylim([y_min, y_max])
ax1.set_xlim([0, max_trials])
plt.tight_layout()
plt.subplots_adjust(top=0.85)
if save != "":
plt.savefig(save, dpi=100, facecolor='w', edgecolor='w',
orientation='portrait', papertype=None, format=None,
transparent=False, bbox_inches="tight", pad_inches=0.1)
else:
plt.show()
def plot_summed_wins_of_optimizers(trial_list_per_dataset,
name_list_per_dataset,
save="", cut=sys.maxint,
figsize=(16, 4), legend_ncols=3,
colors=None, linewidth=3,
markers=None, markersize=6):
# TODO colors should be a function handle which returns an Iterable!
# This is a hack
cut = 50
optimizers, summed_wins_of_optimizer = get_summed_wins_of_optimizers(
trial_list_per_dataset, name_list_per_dataset)
# Make a copy of the colors iterator, because we need it more than once!
if colors is not None:
if not isinstance(colors, itertools.cycle):
raise TypeError()
else:
color_values = list()
for i in range(10):
r, g, b = colors.next()
color_values.append((r, g, b))
if markers is not None:
if not isinstance(markers, itertools.cycle):
raise TypeError()
else:
marker_values = list()
for i in range(10):
marker_values.append(markers.next())
################################################################################
# Plot statistics
for opt1_idx, key in enumerate(optimizers):
fig, (ax0, ax1) = plt.subplots(nrows=2, sharex=True, dpi=600,
figsize=figsize)
if colors is None:
colors_ = plot_util.get_plot_colors()
else:
colors_ = itertools.cycle(color_values)
if markers is None:
markers_ = plot_util.get_empty_iterator()
else:
markers_ = itertools.cycle(marker_values)
y_max = 0.
for opt2_idx, key2 in enumerate(optimizers):
if opt1_idx == opt2_idx:
continue
y = []
y1 = []
for i in range(0, cut+1):
y.append(summed_wins_of_optimizer[i][opt1_idx, opt2_idx]
/ len(trial_list_per_dataset) * 100)
y1.append(- summed_wins_of_optimizer[i][opt2_idx, opt1_idx]
/ len(trial_list_per_dataset) * 100)
y_max_tmp = max(np.max(y), np.max(np.abs(y1)))
y_max_tmp = np.ceil(y_max_tmp * 10) / 10.
y_max = max(y_max_tmp, y_max)
label = "%s vs %s" % (key, key2)
label = label.replace("learned", "d_c").replace("l1", "d_p")
color = colors_.next()
marker = markers_.next()
ax0.plot(range(0, cut+1), y, color=color, label=label,
linewidth=linewidth, marker=marker, markersize=markersize)
ax1.plot(range(0, cut+1), y1, color=color, label=label,
linewidth=linewidth, marker=marker, markersize=markersize)
#handles, labels = ax1.get_legend_handles_labels()
#fig.legend(handles, labels, loc="upper center", fancybox=True,
# ncol=legend_ncols, shadow=True)
ax0.set_xlim((0, cut))
ax0.set_ylim((0, y_max))
ax0.set_ylabel("Significant wins (%)")
ax1.set_xlim((0, cut))
ax1.set_ylim((-y_max, 0))
ax1.set_ylabel("Significant losses (%)")
yticklabels = ax1.get_yticks().tolist()
#print yticklabels, [item.get_text() for item in yticklabels]
ax1.set_yticklabels([-int(item) for item in yticklabels])
ax1.legend(loc="best", fancybox=True, ncol=legend_ncols, shadow=True)
plt.tight_layout()
#plt.subplots_adjust(top=0.85)
plt.xlabel("#Function evaluations")
if save != "":
plt.savefig(save % key, facecolor='w', edgecolor='w',
orientation='portrait', papertype=None, format=None,
transparent=False, bbox_inches="tight", pad_inches=0.1)
else:
plt.show()
plt.close(fig)
def plot_optimization_trace_cv(trial_list, name_list, optimum=0, title="",
log=True, save="", y_max=0, y_min=0):
markers =plot_util.get_plot_markers()
colors = plot_util.get_plot_colors()
linestyles = itertools.cycle(['-'])
size = 1
ratio = 5
gs = matplotlib.gridspec.GridSpec(ratio, 1)
fig = figure(1, dpi=100)
fig.suptitle(title, fontsize=16)
ax1 = subplot(gs[0:ratio, :])
ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
min_val = sys.maxint
max_val = -sys.maxint
max_trials = 0
fig.suptitle(title, fontsize=16)
# Plot the average error and std
for i in range(len(name_list)):
m = markers.next()
c = colors.next()
l = linestyles.next()
leg = False
for tr in trial_list[i]:
if log:
tr = np.log10(tr)
x = range(1, len(tr)+1)
y = tr
if not leg:
ax1.plot(x, y, color=c, linewidth=size, linestyle=l, label=name_list[i][0])
leg = True
ax1.plot(x, y, color=c, linewidth=size, linestyle=l)
min_val = min(min_val, min(tr))
max_val = max(max_val, max(tr))
max_trials = max(max_trials, len(tr))
# Maybe plot on logscale
ylabel = ""
if log:
ax1.set_ylabel("log10(Minfunction value)" + ylabel)
else:
ax1.set_ylabel("Minfunction value" + ylabel)
# Descript and label the stuff
leg = ax1.legend(loc='best', fancybox=True)
leg.get_frame().set_alpha(0.5)
ax1.set_xlabel("#Function evaluations")
if y_max == y_min:
# Set axes limits
ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))])
else:
ax1.set_ylim([y_min, y_max])
ax1.set_xlim([0, max_trials + 1])
tight_layout()
subplots_adjust(top=0.85)
if save != "":
savefig(save, dpi=100, facecolor='w', edgecolor='w',
orientation='portrait', papertype=None, format=None,
transparent=False, bbox_inches="tight", pad_inches=0.1)
else:
show()
def plot_optimization_trace(trial_list,
name_list,
optimum=0,
title="",
log=True,
save="",
y_max=0,
y_min=0,
scale_std=1,
xmax=100,
every=10):
markers = plot_util.get_plot_markers()
colors = plot_util.get_plot_colors()
linestyles = itertools.cycle(['-'])
size = 1
ratio = 5
gs = matplotlib.gridspec.GridSpec(ratio, 1)
fig = figure(1, dpi=100)
fig.suptitle(title, fontsize=16)
ax1 = subplot(gs[0:ratio, :])
ax1.grid(True,
linestyle=':',
which='major',
color='grey',
alpha=0.4,
zorder=0)
min_val = sys.maxint
max_val = -sys.maxint
max_trials = 0
trial_list_means = list()
trial_list_std = list()
# One trialList represents all runs from one optimizer
for i in range(len(trial_list)):
if log:
trial_list_means.append(
np.log10(np.mean(np.array(trial_list[i]), axis=0)))
else:
trial_list_means.append(np.mean(np.array(trial_list[i]), axis=0))
trial_list_std.append(
np.std(np.array(trial_list[i]), axis=0) * scale_std)
fig.suptitle(title, fontsize=16)
# Plot the average error and std
for i in range(len(trial_list_means)):
x = range(1, len(trial_list_means[i]) + 1)
y = trial_list_means[i] - optimum
m = markers.next()
c = colors.next()
l = linestyles.next()
std_up = y + trial_list_std[i]
std_down = y - trial_list_std[i]
ax1.fill_between(x,
std_down,
std_up,
facecolor=c,
alpha=0.3,
edgecolor=c)
# ax1.plot(x, y, color=c, linewidth=size,
# label=name_list[i][0] + "(" + str(len(trial_list[i])) + ")",
# linestyle=l, marker=m)
ax1.plot(x,
y,
color=c,
linewidth=size * 2,
label=name_list[i][0],
linestyle=l,
marker=m,
markevery=every)
if min(std_down) < min_val:
min_val = min(std_down)
if max(std_up) > max_val:
max_val = max(std_up)
if len(trial_list_means[i]) > max_trials:
max_trials = len(trial_list_means[i])
# Maybe plot on logscale
if scale_std != 1:
ylabel = ", %s * std" % scale_std
else:
ylabel = ""
ylabel = ''
if log:
ax1.set_ylabel("log10(Minfunction value)" + ylabel)
else:
ax1.set_ylabel("Test error rate (best so far)" + ylabel)
# Descript and label the stuff
leg = ax1.legend(loc='best', fancybox=True)
leg.get_frame().set_alpha(0.5)
ax1.set_xlabel("Number of pipeline runs")
if y_max == y_min:
# Set axes limits
ax1.set_ylim([
min_val - 0.1 * abs((max_val - min_val)), max_val + 0.1 * abs(
(max_val - min_val))
])
else:
ax1.set_ylim([y_min, y_max])
# ax1.set_xlim([0, max_trials + 1])
ax1.set_xlim([0, xmax])
tight_layout()
subplots_adjust(top=0.85)
if save != "":
savefig(save,
dpi=100,
facecolor='w',
edgecolor='w',
orientation='portrait',
papertype=None,
format=None,
transparent=False,
bbox_inches="tight",
pad_inches=0.1)
else:
show()
