def smooth_valacc(self, step=None):
if step is None:
step = self.print_every
return smooth(self.val_accs, step=step)
def smooth_loss(self, step=None):
if step is None:
step = self.print_every
return smooth(self.all_losses, step=step)
def plot_best(pattern,
smooth_window=1,
title=None,
ylim=None,
xlim=None,
methods=None,
method2label=None,
desired_settings=None,
report_result=False,
start=0,
xlabel=None,
ylabel=None,
end=-1,
default_setting=None,
q_low=25,
q_high=75):
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("whitegrid")
methods_settings = {}
accs = {}
for fn in sorted(glob.glob(pattern)):
method_setting = fn.split('/')[-1].split('^')[0].split('-')
method, setting = method_setting[0], '-'.join(method_setting[1:])
if desired_settings is not None and setting not in desired_settings:
continue
if methods is not None and method not in methods:
continue
if method not in methods_settings:
methods_settings[method] = {}
accs[method] = {}
if setting not in methods_settings[method]:
methods_settings[method][setting] = []
accs[method][setting] = []
tr_loss = smooth(joblib.load(fn), smooth_window)
methods_settings[method][setting].append(tr_loss)
accs[method][setting].append(float(fn.split('/')[-1].split('^')[1]))
if len(methods_settings.keys()
) > 1: # plot the best one, smallest area under the curve
for method, setting_dict in methods_settings.items():
setting_areas = []
for setting, v in setting_dict.items():
# find the smallest area
max_len = max([len(a) for a in v])
v = [a for a in v if len(a) == max_len]
#min_len = min([len(a) for a in v])
#end_ = min_len if end == -1 else min(end, min_len)
#area = np.mean([a[start:end_] for a in v])
area = np.mean([a[start:end] for a in v])
setting_areas.append((setting, area))
setting = sorted(setting_areas, key=lambda x: x[1])[0][0]
v = setting_dict[setting]
max_len = max([len(a) for a in v])
v = [a for a in v if len(a) == max_len]
print('{}: {:.2f}% ({:.2f}) {} runs'.format(
method, np.mean(accs[method][setting]),
np.std(accs[method][setting]), len(v)))
if method2label is None:
label = method + '-' + setting
else:
label = method2label.get(method, method)
#p = plt.plot(sum(v) / len(v), label=method + '-' + setting)#, c=colors[i])
p = plt.plot(np.percentile(v, 50, 0), label=label, ls='--')
plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
np.percentile(v, q_low, 0),
np.percentile(v, q_high, 0),
alpha=0.1,
color=p[-1].get_color())
if default_setting and method in default_setting and default_setting[
method] in setting_dict:
v = setting_dict[default_setting[method]]
max_len = max([len(a) for a in v])
v = [a for a in v if len(a) == max_len]
p = plt.plot(np.percentile(v, 50, 0),
ls='-',
c=p[-1].get_color())
plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
np.percentile(v, q_low, 0),
np.percentile(v, q_high, 0),
alpha=0.1,
color=p[-1].get_color())
else: # plot all
for method, setting_dict in methods_settings.items():
for setting, v in setting_dict.items():
max_len = max([len(a) for a in v])
v = [a for a in v if len(a) == max_len]
print('{}: {:.2f}% ({:.2f}) {} runs'.format(
method, np.mean(accs[method][setting]),
np.std(accs[method][setting]), len(v)))
if method2label is None:
label = method + '-' + setting
else:
label = method2label.get(method, method) + '-' + setting
p = plt.plot(np.percentile(v, 50, 0), label=label)
plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
np.percentile(v, q_low, 0),
np.percentile(v, q_high, 0),
alpha=0.1,
color=p[-1].get_color())
plt.legend()
if title:
plt.title(title, fontsize=15)
plt.ylim(ylim)
plt.xlim(xlim)
ylabel = ylabel or pattern.split("*")[-1]
xlabel = xlabel or "Epoch"
plt.ylabel(ylabel, fontsize=15)
plt.xlabel(xlabel, fontsize=15)
plt.grid(color='gray', alpha=0.5)
plt.show()
def smooth_valloss(self, step=None):
if step is None:
step = self.print_every
return smooth(np.sqrt(self.val_losses), step=step)
def plot_best_test(train_pattern,
smooth_window=1,
title=None,
ylim=None,
xlim=None,
methods=None,
method2label=None,
desired_settings=None,
start=0,
end=-1,
default_setting=None,
ylabel=None,
xlabel=None,
q_low=25,
q_high=75,
early_stop=False,
early_stop_patience=None):
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style("whitegrid")
assert 'train_losses' in train_pattern, 'train_losses must appear in train_pattern'
methods_settings = {}
val_settings = {}
test_settings = {}
accs = {} # accs are based on best validation performance
best_val_test_error = {}
for fn in sorted(glob.glob(train_pattern)):
method_setting = fn.split('/')[-1].split('^')[0].split('-')
method, setting = method_setting[0], '-'.join(method_setting[1:])
if desired_settings is not None and setting not in desired_settings:
continue
if methods is not None and method not in methods:
continue
if method not in methods_settings:
methods_settings[method] = {}
val_settings[method] = {}
test_settings[method] = {}
accs[method] = {}
if setting not in methods_settings[method]:
methods_settings[method][setting] = []
val_settings[method][setting] = []
test_settings[method][setting] = []
accs[method][setting] = []
tr_loss = smooth(joblib.load(fn), smooth_window)
val_error = joblib.load(fn.replace('train_losses', 'val_errors'))
test_error = joblib.load(fn.replace('train_losses', 'test_errors'))
if early_stop:
index_list = early_stop_curve(val_error, early_stop_patience)
val_error = np.array(val_error)[index_list]
test_error = np.array(test_error)[index_list]
val_error = smooth(val_error, smooth_window)
test_error = smooth(test_error, smooth_window)
methods_settings[method][setting].append(tr_loss)
val_settings[method][setting].append(val_error)
test_settings[method][setting].append(test_error)
# change the following two to see different evaluation
best_error_index = np.argmin(val_error)
# best_error_index = -1
min_val_error = val_error[best_error_index]
accs[method][setting].append(test_error[best_error_index])
if best_val_test_error.get(method) is None:
best_val_test_error[method] = (min_val_error,
test_error[best_error_index], fn)
else:
val_best_error, test_best_error, fn_best = best_val_test_error[
method]
if val_best_error >= min_val_error:
val_best_error = min_val_error
test_best_error = test_error[best_error_index]
best_val_test_error[method] = (val_best_error, test_best_error,
fn)
for method, setting_dict in val_settings.items():
val_best_error, test_best_error, fn_best = best_val_test_error[method]
print('{} test: {:.3f} val: {:.3f} fn:{}'.format(
method, test_best_error, val_best_error, fn_best))
setting_areas = []
for setting, v in setting_dict.items():
# find the smallest area
max_len = max([len(a) for a in v])
v = [a for a in v if len(a) == max_len]
area = np.mean([a[start:end] for a in v])
setting_areas.append((setting, area))
setting = sorted(setting_areas, key=lambda x: x[1])[0][0]
def plot_setting():
v = test_settings[method][setting]
max_len = max([len(a) for a in v])
v = [a for a in v if len(a) == max_len]
# print('{}: {:.3f} ({:.2f}) {} runs'.format(method,
# np.mean(accs[method][setting]),
# np.std(accs[method][setting]),
# len(v)))
if method2label is None:
label = method + '-' + setting
else:
label = method2label.get(method, method)
#p = plt.plot(sum(v) / len(v), label=method + '-' + setting)#, c=colors[i]
p = plt.plot(np.percentile(v, 50, 0), label=label, ls='--')
plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
np.percentile(v, q_low, 0),
np.percentile(v, q_high, 0),
alpha=0.1,
color=p[-1].get_color())
if default_setting and method in default_setting and\
default_setting[method] in test_settings[method]:
v = test_settings[method][default_setting[method]]
max_len = max([len(a) for a in v])
v = [a for a in v if len(a) == max_len]
p = plt.plot(np.percentile(v, 50, 0),
ls='-',
c=p[-1].get_color())
plt.fill_between(np.arange(len(np.percentile(v, q_low, 0))),
np.percentile(v, q_low, 0),
np.percentile(v, q_high, 0),
alpha=0.1,
color=p[-1].get_color())
plot_setting()
plt.legend()
if title:
plt.title(title, fontsize=15)
plt.ylim(ylim)
plt.xlim(xlim)
ylabel = ylabel or "error"
xlabel = xlabel or "Epoch"
plt.ylabel('test {}'.format(ylabel), fontsize=15)
plt.xlabel(xlabel, fontsize=15)
plt.grid(color='gray', alpha=0.5)
plt.show()
