def run_expe(da_matrix,
meta_learners,
name_expe=None,
results_dir='../results',
with_once_random=False,
ylim=None,
show_legend=False,
figsize=(5, 3)):
"""Use run_meta_validation to run experiment."""
if with_once_random:
fig = run_once_random(da_matrix)
else:
fig = None
fig = run_meta_validation(meta_learners,
da_matrix,
fig=fig,
ylim=ylim,
show_legend=show_legend,
figsize=figsize)
# Create directory for the experiment
expe_dir = os.path.join(results_dir, str(name_expe))
os.makedirs(expe_dir, exist_ok=True)
# Save performance matrix and the figure
save_perfs(da_matrix.perfs.astype(int), name_expe=name_expe)
save_fig(fig, name_expe=name_expe)
def plot_meta_learner_comparison(da_tr,
da_te,
meta_learners,
repeat=10,
save=True):
n_algos = len(da_tr.algos)
means = []
stds = []
kss = []
for i, meta_learner in enumerate(meta_learners):
mean, std, ks = get_meta_learner_avg_rank(da_tr,
da_te,
meta_learner,
repeat=10)
means.append(mean)
stds.append(std)
kss.append(ks)
x_pos = np.arange(len(meta_learners))
# Build the plot
fig, ax = plt.subplots()
ax.bar(x_pos,
means,
yerr=stds,
align='center',
alpha=0.5,
ecolor='black',
capsize=10)
ax.set_ylabel('Average rank in percentage')
ax.set_xticks(x_pos)
names = [meta_learner.name for meta_learner in meta_learners]
ax.set_xticklabels(names)
for i in range(len(meta_learners)):
ks = kss[i]
kmean = np.mean(ks)
kstd = np.std(ks)
if kstd == 0:
s = "k={}".format(kmean)
else:
s = "k={:.1f}±{:.1f}".format(kmean, kstd)
x = x_pos[i] - 0.2
y = means[i] * 0.9 - 1
plt.text(x, y, s)
da_name = da_tr.name[:-11]
title = "Meta-learner comparison on {} (n_algos={})".format(
da_name, n_algos)
ax.set_title(title)
# Save the figure and show
plt.tight_layout()
plt.show()
name_expe = 'meta-learner-comparison'
filename = '{}.png'.format(da_name.lower())
if save:
save_fig(fig, name_expe=name_expe, filename=filename)
def plot_full_meta_learner_comparison(
da_matrices,
meta_learners,
name_expe='full-meta-learner-comparison',
show=True,
ylabel=None,
**kwargs,
):
if not 'metric' in kwargs or kwargs['metric'] is None:
kwargs['metric'] = ArgmaxMeanMetric(name='emp-acc')
# Names of meta-learners
names_ml = [ml.name for ml in meta_learners]
task_names = []
participant_names = []
means = []
stds = []
for da_matrix in da_matrices:
ms, ss = plot_meta_learner_comparison_sample_meta_test(
da_matrix,
meta_learners,
show=False,
**kwargs,
)
for i in range(len(meta_learners)):
task_names.append(da_matrix.name)
participant_names.append(meta_learners[i].name)
means.append(ms[i])
stds.append(ss[i])
agg_df = pd.DataFrame({
'task_name': task_names,
'participant_name': participant_names,
'mean': means,
'std': stds,
})
save = kwargs['save'] if 'save' in kwargs else True
fig = show_score_per_task_with_error_bars(
agg_df,
legend_aside=False,
ylabel=ylabel,
)
if save:
save_fig(fig, name_expe=name_expe)
if show:
fig.show()
return fig
def run_leave_one_out_on_real_datasets():
dataset_dirs = get_real_datasets_dataset_dirs()
meta_learners = get_the_meta_learners(exclude_optimal=True)[1:]
for dataset_dir in dataset_dirs:
name_expe = "LOO-{}".format(os.path.basename(dataset_dir))
da_matrix = get_da_matrix_from_real_dataset_dir(dataset_dir)
fig = run_once_random(da_matrix, leave_one_out=True)
fig = run_leave_one_out(meta_learners,
da_matrix,
use_all=True,
fig=fig)
save_fig(fig, name_expe=name_expe)
save_perfs(da_matrix.perfs, name_expe=name_expe)
def plot_overfit_curve_DFP(Ds,
Fs,
Ps,
da_name=None,
save=True,
name_expe=None):
"""
Args:
Ds, Fs, Ps: list of permutations
"""
assert len(Ds) == len(Fs)
assert len(Fs) == len(Ps)
num_trials = len(Ds)
eps = np.finfo(float).eps # Machine precision
m = len(Ds[0])
TR = np.zeros((num_trials, m))
TE = np.zeros((num_trials, m))
C = np.zeros((num_trials, ))
G = np.arange(m)
for t, (D, F, P) in enumerate(zip(Ds, Fs, Ps)):
Tr, Te = get_ofc_P(D, F, P) ### This is new
TR[t, :] = Tr
TE[t, :] = Te
C[t] = c = pearsonr(D, P)[0]
##### Isabelle's code #####
Correl = np.mean(C)
Tr = np.mean(TR, axis=0)
Te = np.mean(TE, axis=0)
STr = np.std(TR, axis=0)
#print(STr)
Stderr = np.mean(STr)
STe = np.std(TE, axis=0)
#print(STe)
STre = 2 * STr / np.sqrt(num_trials)
STee = 2 * STe / np.sqrt(num_trials)
Gap = np.abs(Te - Tr)
#Tr_pred = Tr[0]*1/(1+np.arange(m))
Tr_pred = np.zeros(Tr.shape)
K = 1. * Tr[0] / (STr[0] + eps)
for mm in np.arange(m):
Tr_pred[mm] = K * 1. / np.sum(1. / (STr[0:mm + 1] + eps))
#s = np.sqrt(np.arange(m))
#A = Tr[0]*(1-np.sqrt(m-1))/(eps+Tr[0]-Gap[1]*np.sqrt(m-1))
#B = A-1
#Gap_pred = (A * Gap[1] * s) / (eps + B + s)
Gap_pred = Gap[1] * np.sqrt(np.arange(m))
# Te_pred = Tr + Gap_pred
Te_pred = Tr_pred + Gap_pred
kopt = np.round((2 * Tr[0] / (eps + Gap_pred[1]))**(2 / 3))
# Correction: the number of participants should start at 1
mx = 6
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot(G + 1, Tr, 'ro')
ax.plot(G + 1, Tr, 'r-', label='Meta-training error')
ax.fill_between(G + 1, (Tr - STre), (Tr + STre), color='red', alpha=0.1)
ax.plot(G + 1, Tr_pred, 'mo')
ax.plot(G + 1, Tr_pred, 'm-', label='Predicted meta-training error')
ax.plot(G + 1, Gap, 'go')
ax.plot(G + 1, Gap, 'g-', label='Generalization gap')
ax.plot(G[0:mx] + 1, Gap_pred[0:mx], 'co')
ax.plot(G[0:mx] + 1,
Gap_pred[0:mx],
'c-',
label='Predicted generalization gap')
ax.plot(G + 1, Te, 'bo')
ax.plot(G + 1, Te, 'b-', label='Meta-test error')
ax.fill_between(G + 1, (Te - STee), (Te + STee), color='blue', alpha=0.1)
ax.plot(G[0:mx] + 1, Te_pred[0:mx], 'ko')
ax.plot(G[0:mx] + 1,
Te_pred[0:mx],
'k-',
label='Predicted meta-test error')
ax.set_xlabel('Number of Final phase participants')
ax.set_ylabel('Average error of final phase winner')
ax.legend(loc='best')
ax.set_title('%s - Ebar=2SE; <C>=%5.2f; <SE>=%5.2f; k-opt=%d' %
(da_name, Correl, Stderr, kopt.astype(int)))
#########################
if m >= 70:
plt.legend(loc='lower right')
plt.xscale('log')
else:
plt.legend(loc='best')
# Save the figure and show
plt.tight_layout()
plt.show()
filename = '{}.png'.format(da_name.lower())
if save:
save_fig(fig, name_expe=name_expe, filename=filename)
def plot_score_vs_n_algos_with_error_bars(repeat=100,
datasets_dir="../datasets",
dataset_names=None,
log_scale=False,
save=False,
max_ticks=50,
shuffling=False,
**kwargs):
"""
Args:
repeat: int, number of repetitions for sampling meta-training examples
datasets_dir: str, path to directory containing all (meta-)datasets
dataset_names: list of str, list of dataset names to carry out the plot
log_scale: boolean. If True, x-axis and y-axis will be in log-scale
save: boolean. If True, figures will be saved
max_ticks: int, maximum number of ticks/points for the plot
shuffling: boolean, whether with shuffling for (meta-)train-test split
Returns:
Plots or saves several figures.
"""
score_names = {
'artificial_r50c20r20': 'Performance',
'AutoDL': 'ALC',
'AutoML': 'BAC or R2',
'OpenML-Alors': 'Accuracy',
'Statlog': 'Error rate',
}
if dataset_names is None:
ds = os.listdir(datasets_dir)
else:
ds = [d for d in os.listdir(datasets_dir) if d in set(dataset_names)]
for d in ds:
if True:
dataset_dir = os.path.join(datasets_dir, d)
if os.path.isdir(dataset_dir):
da_matrix = get_da_matrix_from_real_dataset_dir(dataset_dir)
meta_learner = MeanMetaLearner()
name_expe = "alc-vs-n_algos"
if d == 'AutoDL':
n_meta_train = 5
else:
n_meta_train = da_matrix.perfs.shape[0] // 2
n_meta_test = da_matrix.perfs.shape[0] - n_meta_train
curves = get_meta_scores_vs_n_algos(da_matrix,
meta_learner,
n_meta_train=n_meta_train,
repeat=repeat,
max_ticks=max_ticks,
shuffling=shuffling,
**kwargs)
ticks = curves[4]
score_name = score_names[
d] if d in score_names else 'Performance'
total_n_algos = len(da_matrix.algos)
fig = plot_curve_with_error_bars(curves[0],
curves[1],
xs=ticks,
label='meta-train',
marker='o',
markersize=2)
fig = plot_curve_with_error_bars(curves[2],
curves[3],
fig=fig,
xs=ticks,
label='meta-test',
marker='o',
markersize=2)
plt.legend()
plt.xlabel(
"Number of algos " +
"(|Dtr|={}, |Dte|={}, ".format(n_meta_train, n_meta_test) +
"total #algos={})".format(total_n_algos))
plt.ylabel("Average {} score".format(score_name))
if log_scale:
plt.xscale('log')
plt.yscale('log')
# Use another figure
fig2 = plt.figure()
ax = fig2.add_subplot(1, 1, 1)
ax.xaxis.set_major_locator(
matplotlib.ticker.MaxNLocator(integer=True))
# Meta-train - meta-test
diff_curve = curves[0] - curves[2]
ax.plot(ticks,
diff_curve,
label='meta-train - meta-test',
marker='o',
markersize=2)
# Theoretical bounds
n_T = n_meta_train
n_B = total_n_algos
error_bars_the = [
get_theoretical_error_bar(n_T, i, delta=0.05)
for i in ticks
]
ax.plot(ticks,
error_bars_the,
label='Theoretical error bar',
marker='o',
markersize=2)
if d == 'OpenML-Alors':
plt.xscale('log')
plt.xlabel(
"Number of algos " +
"(|Dtr|={}, |Dte|={}, ".format(n_meta_train, n_meta_test) +
"total #algos={})".format(total_n_algos))
plt.ylabel("Average {} score".format(score_name))
if log_scale:
plt.xscale('log')
plt.yscale('log')
# Title
fig.axes[0].set_title("{} - {} VS #algos".format(
d, score_name))
fig2.axes[0].set_title("{} - {} diff VS #algos".format(
d, score_name))
plt.legend()
plt.show()
if save:
save_fig(fig,
name_expe=name_expe,
filename="{}-alc-vs-n_algos.jpg".format(d))
save_fig(fig2,
name_expe=name_expe,
filename="{}-alc-diff-vs-n_algos.jpg".format(d))
def inspect_da_matrix(
da_matrix,
results_dir="../results",
save=False,
perfs_corr=False,
algos_corr=False,
tasks_corr=False,
sort_algos=True,
):
"""Inspect DA matrix. Plot the mean and std of the performance of each
algorithm. Plot cluster map for:
- perfomance correlation
- algorithm correlation
- task correlation
if the corresponding argument is `True`.
"""
if results_dir is None:
results_dir = get_default_results_dir()
perfs = np.array(da_matrix.perfs)
li_mean = np.mean(perfs, axis=0)
if sort_algos:
argsort = li_mean.argsort()
li_mean = li_mean[argsort]
perfs = perfs[:, argsort]
li_std = np.std(perfs, axis=0)
fig = plot_curve_with_error_bars(li_mean, li_std)
name = da_matrix.name
n_datasets = len(da_matrix.datasets)
n_algos = len(da_matrix.algos)
assert n_datasets == perfs.shape[0]
assert n_algos == perfs.shape[1]
title = "{} (n_datasets={}, n_algos={})".format(name, n_datasets, n_algos)
plt.title(title)
name_expe = 'inspect-da-matrix'
if save:
filename = "mean-std-algos-{}".format(name)
save_fig(fig,
name_expe=name_expe,
results_dir=results_dir,
filename=filename)
if perfs_corr:
heatmap = sns.clustermap(perfs, metric='correlation')
heatmap.fig.suptitle(name)
if save:
heatmap.fig.savefig(os.path.join(results_dir, name_expe, name))
if algos_corr:
cov = np.corrcoef(perfs.T)
hm_cov = sns.clustermap(cov)
title = name + " algos correlation"
hm_cov.fig.suptitle(title)
if save:
hm_cov.fig.savefig(os.path.join(results_dir, name_expe, title))
if tasks_corr:
cov = np.corrcoef(perfs)
hm_cov = sns.clustermap(cov)
title = name + " tasks correlation"
hm_cov.fig.suptitle(title)
if save:
hm_cov.fig.savefig(os.path.join(results_dir, name_expe, title))
plt.show()
def plot_score_vs_n_algos_per_matrix(da_matrix,
meta_learner,
repeat=100,
log_scale=False,
save=False,
max_ticks=50,
n_meta_train=None,
name_expe="alc-vs-n_algos",
score_name="Performance",
shuffling=False,
**kwargs):
"""Given DA matrix `da_matrix` and meta-learn `meta_learner`, plot a score
vs n_algos figure.
Following procedures are adopted:
- Runs are repeated experiments for computing the mean and the std in each
settings. The learning curves typically plot these mean and std;
- Random meta-train-test split (matrix -> meta-train reservoir, meta-test)
was done once for all runs in the first version. If `shuffling`,
we do it in each run;
- Random meta-train-valid split: a random subset of meta-train reservoir is
used for real meta-training. The remaining tasks in the meta-train
reservoir are used for meta-validation;
- Gamma-level algorithm: chooses only one (beta-)algorithm during
meta-training. We choose the algorithm with best column mean (i.e. the
algorithm with the highest mean performance over tasks in meta-train)
among `n_algos` algorithms, which are chosen randomly.
- Meta-test: the chosen (beta-)algorithm during meta-training is used for
meta-test, where the column mean of this algorithm among the meta-test
set is used as final score. (Thus if the meta-test set is fixed, then the
final scores only have a very finite set of possibilities);
Args:
da_matrix: `mlt.data.DAMatrix` object
meta_learner: `mlt.meta_learner.MetaLearner` object
repeat: int, number of repetitions for sampling meta-training examples
log_scale: boolean. If True, x-axis and y-axis will be in log-scale
save: boolean. If True, figures will be saved
max_ticks: int, maximum number of ticks/points for the plot
shuffling: boolean, whether with shuffling for (meta-)train-test split
n_meta_train: int, number of examples used for meta-training. If `None`,
half of the examples are used
name_expe: str, name of the experiment. Used for naming the resulting
figures
score_name: str, name of the score. Used in the figures' title
kwargs: dict of other arguments, which is passed to the function
`get_meta_scores_vs_n_algos`.
Returns:
list of curves: [mtr_mean, mtr_std, mte_mean, mte_std]
"""
if n_meta_train is None:
n_meta_train = da_matrix.perfs.shape[0] // 2
n_meta_test = da_matrix.perfs.shape[0] - n_meta_train
curves = get_meta_scores_vs_n_algos(da_matrix,
meta_learner,
n_meta_train=n_meta_train,
repeat=repeat,
max_ticks=max_ticks,
shuffling=shuffling,
**kwargs)
ticks = curves[4]
total_n_algos = len(da_matrix.algos)
fig = plot_curve_with_error_bars(curves[0],
curves[1],
xs=ticks,
label='meta-train',
marker='o',
markersize=2)
fig = plot_curve_with_error_bars(curves[2],
curves[3],
fig=fig,
xs=ticks,
label='meta-test',
marker='o',
markersize=2)
plt.legend()
plt.xlabel("Number of algos " +
"(|Dtr|={}, |Dte|={}, ".format(n_meta_train, n_meta_test) +
"total #algos={})".format(total_n_algos))
plt.ylabel("Average {} score".format(score_name))
if log_scale:
plt.xscale('log')
plt.yscale('log')
# Use another figure
fig2 = plt.figure()
ax = fig2.add_subplot(1, 1, 1)
ax.xaxis.set_major_locator(matplotlib.ticker.MaxNLocator(integer=True))
# Meta-train - meta-test
diff_curve = curves[0] - curves[2]
ax.plot(ticks,
diff_curve,
label='meta-train - meta-test',
marker='o',
markersize=2)
# Theoretical bounds
n_T = n_meta_train
n_B = total_n_algos
error_bars_the = [
get_theoretical_error_bar(n_T, i, delta=0.05) for i in ticks
]
ax.plot(ticks,
error_bars_the,
label='Theoretical error bar',
marker='o',
markersize=2)
# Figure's
plt.xlabel("Number of algos " +
"(|Dtr|={}, |Dte|={}, ".format(n_meta_train, n_meta_test) +
"total #algos={})".format(total_n_algos))
plt.ylabel("Average {} score".format(score_name))
if log_scale:
plt.xscale('log')
plt.yscale('log')
# Title
d = da_matrix.name
fig.axes[0].set_title("{} - {} VS #algos".format(d, score_name))
fig2.axes[0].set_title("{} - {} diff VS #algos".format(d, score_name))
plt.legend()
plt.show()
if save:
save_fig(fig,
name_expe=name_expe,
filename="{}-alc-vs-n_algos.jpg".format(d))
save_fig(fig2,
name_expe=name_expe,
filename="{}-alc-diff-vs-n_algos.jpg".format(d))
return curves
def show_score_per_task_with_error_bars(
agg_df,
figsize=(12, 5),
bar_width=1.0,
sep_width=2.0,
save=False,
task_names=None,
participant_names=None,
xlabel_rotation=0,
legend_aside=True,
avg_ranks=None,
xlabel='Meta-datasets',
ylabel='empirical accuracy (EmpAcc)',
filename='score_per_task_with_error_bars.jpg',
name_expe=None,
):
"""Generate histograms with error bars.
Args:
agg_df: a pandas.DataFrame object, should contain columns:
participant_name, task_name, mean, std
phase: a tuple of (challenge, phase)
avg_ranks: dict, participant_name: avg_rank.
"""
fig, ax = plt.subplots(figsize=figsize)
if task_names is None:
task_names = agg_df['task_name'].unique()
n_tasks = len(task_names)
if participant_names is None:
participant_names = agg_df['participant_name'].unique()
n_participants = len(participant_names)
participant_style = get_style(participant_names)
df = agg_df.set_index(['participant_name', 'task_name'])
df = df[['mean', 'std']]
# Width for each task
width_per_task = bar_width * n_participants + sep_width
# Begin location of each participant on each task
begins = [[p * bar_width + width_per_task * t for t in range(n_tasks)]
for p in range(n_participants)]
# Show legend once per participant
seen_labels = set()
# Draw a bar for each participant on each task
for p, participant_name in enumerate(participant_names):
for t, task_name in enumerate(task_names):
key = (participant_name, task_name)
if key in df.index:
alc_score = df.loc[key]['mean']
std = df.loc[key]['std']
begin = begins[p][t]
if participant_name not in seen_labels:
if avg_ranks and participant_name in avg_ranks:
ar = avg_ranks[participant_name]
label = "{} - {:.2f}".format(participant_name, ar)
else:
label = participant_name
seen_labels.add(participant_name)
else:
label = None
color = participant_style[participant_name]['color']
ax.bar(
begin,
alc_score,
label=label,
color=color,
)
ax.errorbar(begin,
alc_score,
yerr=std,
ecolor='black',
elinewidth=1,
capsize=1.5)
# Set ticks for x-axis
ax.set_xticks(begins[n_participants // 2])
ax.set_xticklabels(task_names, rotation=xlabel_rotation)
# Axis labels
plt.xlabel(xlabel)
plt.ylabel(ylabel)
# Show legends
if legend_aside:
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
else:
plt.legend(loc='best')
# Save figure
if save:
filename = filename if filename is not None\
else 'score_per_task_with_error_bars.jpg'
save_fig(fig, name_expe=name_expe, filename=filename)
return fig
def plot_meta_learner_comparison_sample_meta_test(
da_matrix,
meta_learners,
metric=None,
repeat=25,
train_size=0.5,
save=False,
show=True,
name_expe='meta-learner-comparison-sample-test',
):
"""Plot comparison histogram of `meta_learners` on `da_matrix` for the `metric`."""
if metric is None:
metric = ArgmaxMeanMetric()
n_algos = len(da_matrix.algos)
means = []
stds = []
for i, meta_learner in enumerate(meta_learners):
scores = []
for j in range(repeat):
da_tr, da_te = da_matrix.train_test_split(train_size=train_size,
shuffling=True)
meta_learner.meta_fit(da_tr)
dist_pred = meta_learner.rec_algo()
score = metric(dist_pred, da_te)
scores.append(score)
mean = np.mean(scores)
std = np.std(scores)
means.append(mean)
stds.append(std)
da_name = da_matrix.name
x_pos = np.arange(len(meta_learners))
# Build the plot
fig, ax = plt.subplots()
stds = np.array(stds) / np.sqrt(repeat)
ax.bar(x_pos,
means,
yerr=stds,
align='center',
alpha=0.5,
ecolor='black',
capsize=10)
ylabel = metric.name
ax.set_ylabel(ylabel)
ax.set_xticks(x_pos)
names = [meta_learner.name for meta_learner in meta_learners]
ax.set_xticklabels(names)
for i in range(len(meta_learners)):
x = x_pos[i]
y = means[i] * 0.9
s = '{:.3f}±{:.3f}'.format(means[i], stds[i])
plt.text(x, y, s)
title = "Comparison on {} (n_algos={}) - Ebar: 1 sigma".format(
da_name, n_algos)
ax.set_title(title)
# Save the figure and show
plt.tight_layout()
if show:
plt.show()
filename = '{}.png'.format(da_name.lower())
if save:
save_fig(fig, name_expe=name_expe, filename=filename)
return means, stds