def plot_mondrian_kernel_vs_mondrian_forest(lifetime_max, res):
""" Plots training and test set error of Mondrian kernel and Mondrian forest based on the same set of M Mondrian samples.
This procedure takes as input a dictionary res, returned by the evaluate_all_lifetimes procedure in mondrian_kernel.py.
"""
times = res['times']
forest_train = res['forest_train']
forest_test = res['forest_test']
kernel_train = res['kernel_train']
kernel_test = res['kernel_test']
# set up test error plot
fig = plt.figure(figsize=(7, 4))
ax = fig.add_subplot('111')
remove_chartjunk(ax)
ax.set_xlabel('lifetime $\lambda$')
ax.set_ylabel('relative error [\%]')
ax.yaxis.grid(b=True, which='major', linestyle='dotted', lw=0.5, color='black', alpha=0.3)
ax.set_xscale('log')
ax.set_xlim((1e-8, lifetime_max))
ax.set_ylim((0, 25))
rasterized = False
ax.plot(times, forest_test, drawstyle="steps-post", ls='-', lw=2, color=tableau20(6), label='"M. forest" (test)', rasterized=rasterized)
ax.plot(times, forest_train, drawstyle="steps-post", ls='-', color=tableau20(7), label='"M. forest" (train)', rasterized=rasterized)
ax.plot(times, kernel_test, drawstyle="steps-post", ls='-', lw=2, color=tableau20(4), label='M. kernel (test)', rasterized=rasterized)
ax.plot(times, kernel_train, drawstyle="steps-post", ls='-', color=tableau20(5), label='M. kernel (train)', rasterized=rasterized)
ax.legend(bbox_to_anchor=[1.15, 1.05], frameon=False)
def experiment_synthetic():
""" Simulates data from a Gaussian process prior with a Laplace kernel of known lifetime (inverse width). Then the
Mondrian kernel procedure for evaluating all lifetimes from 0 up to a terminal lifetime is run on this simulated
dataset and the results are plotted, showing how accurately the ground truth inverse kernel width could be recovered.
"""
# synthetize data from Laplace kernel
D = 2
lifetime = 10.00
noise_var = 0.01 ** 2
N_train = 1000
N_validation = 1000
N_test = 1000
X, y, X_test, y_test = construct_data_synthetic_Laplacian(D, lifetime, noise_var, N_train, N_validation + N_test)
# Mondrian kernel lifetime sweep parameters
M = 50
lifetime_max = lifetime * 2
delta = noise_var # prior variance 1
res = evaluate_all_lifetimes(X, y, X_test, y_test, M, lifetime_max, delta, mondrian_kernel=True, validation=True)
lifetimes = res['times']
error_train = res['kernel_train']
error_validation = res['kernel_validation']
error_test = res['kernel_test']
# set up plot
fig = plt.figure(figsize=(7, 4))
ax = fig.add_subplot('111')
remove_chartjunk(ax)
ax.set_title('$M = %d$, $\mathcal{D}$ = synthetic ($D = 2$, $N = N_{val} = N_{test}=%d$)' % (M, 1000))
ax.set_xlabel('lifetime $\lambda$')
ax.set_ylabel('relative error [\%]')
ax.set_xscale('log')
ax.yaxis.grid(b=True, which='major', linestyle='dotted', lw=0.5, color='black', alpha=0.3)
ax.plot(lifetimes, error_train, drawstyle="steps-post", ls='-', color=tableau20(15), label='train')
ax.plot(lifetimes, error_validation, drawstyle="steps-post", ls='-', lw=2, color=tableau20(2), label='validation')
ax.plot(lifetimes, error_test, drawstyle="steps-post", ls='-', color=tableau20(4), label='test')
# plot ground truth and estimate
ax.axvline(x=10, ls=':', color='black')
i = np.argmin(error_validation)
lifetime_hat = lifetimes[i]
print 'lifetime_hat = %.3f' % lifetime_hat
ax.plot([lifetime_hat, lifetime_hat], [0, error_validation[i]], ls='dashed', lw=2, color=tableau20(2))
fig.canvas.draw()
labels = [item.get_text() for item in ax.get_xticklabels()]
labels[5] = '$\lambda_0$'
labels.append('\hat{\lambda}')
ax.set_xticks(list(ax.get_xticks()) + [lifetime_hat])
ax.set_xticklabels(labels)
ax.set_xlim((1e-2, lifetime_max))
ax.legend(bbox_to_anchor=[0.35, 0.35], frameon=False)
def plot_kernel_vs_forest_weights(y, res):
""" Plots the weights learned by Mondrian kernel and Mondrian forest based on the same set of M Mondrian samples.
This procedure takes as input a dictionary res, returned by the evaluate_all_lifetimes procedure in mondrian_kernel.py.
"""
w_forest = res['w_forest']
w_kernel = res['w_kernel']
# plot weights against each other
fig1 = plt.figure(figsize=(8, 4))
ax1 = fig1.add_subplot('121')
ax1.set_xlabel('weights learned by "Mondrian forest"')
ax1.set_ylabel('weights learned by Mondrian kernel')
ax1.scatter(w_forest, w_kernel, marker='.', color=tableau20(16))
xl = ax1.get_xlim()
yl = ax1.get_ylim()
lims = [
np.min([xl, yl]), # min of both axes
np.max([xl, yl]), # max of both axes
]
ax1.plot(lims, lims, '--', color='black', alpha=0.75, zorder=0)
ax1.set_xlim(xl)
#ax1.set_ylim(yl)
ax1.set_ylim((-60, 60))
# plot histogram of weight values (and training targets)
ax2 = fig1.add_subplot('122')
ax2.set_xlabel('values')
ax2.set_ylabel('value frequency')
bins = np.linspace(-100, 20, 50)
ax2.hist(w_forest,
bins=bins,
histtype='stepfilled',
normed=True,
color=tableau20(6),
alpha=0.5,
label='M. forest weights $\mathbf{w}$')
ax2.hist(w_kernel,
bins=bins,
histtype='stepfilled',
normed=True,
color=tableau20(4),
alpha=0.5,
label='M. kernel weights $\mathbf{w}$')
ax2.hist(y - np.mean(y),
bins=bins,
histtype='stepfilled',
normed=True,
color=tableau20(8),
alpha=0.5,
label='training targets $\mathbf{y}$')
ax2.set_ylim((0.0, 0.16))
ax2.legend(frameon=False, loc='upper left')
fig1.tight_layout()
def main(args_dict):
# Extract configuration from command line arguments
MK = args_dict['MK']
Kmin = args_dict['Kmin']
# Load data
data = json_load('data/astar_rbr_MK%d.json' % (MK))
lnZ = data['lnZ']
MAPs = np.array(data['MAPs'])
print('Loaded %d MAP samples from A* sampling' % (len(MAPs)))
# Estimate MSE of lnZ estimators from Gumbel and Exponential tricks
MSEs_Gumb = []
MSEs_Expo = []
Ms = xrange(1, MK / Kmin)
for M in Ms:
# Computation with M samples, repeated K >= Kmin times with a new set every time
K = MK / M
myMAPs = np.reshape(MAPs[:(K * M)], (K, M))
# Compute unbiased estimators of ln(Z)
lnZ_Gumb = np.mean(myMAPs, axis=1)
lnZ_Expo = EULER - np.log(np.mean(np.exp(-myMAPs),
axis=1)) - (np.log(M) - digamma(M))
# Save MSE estimates
MSEs_Gumb.append(np.mean((lnZ_Gumb - lnZ)**2))
MSEs_Expo.append(np.mean((lnZ_Expo - lnZ)**2))
# Set up plot
matplotlib_configure_as_notebook()
fig, ax = plt.subplots(1, 1, facecolor='w', figsize=(4.25, 3.25))
ax.set_xscale('log')
ax.set_xlabel('desired MSE (lower to the right)')
ax.set_ylabel('required number of samples $M$')
ax.grid(b=True,
which='both',
linestyle='dotted',
lw=0.5,
color='black',
alpha=0.3)
# Plot MSEs
ax.plot(MSEs_Gumb, Ms, color=tableau20(0), label='Gumbel')
ax.plot(MSEs_Expo, Ms, color=tableau20(2), label='Exponential')
# Finalize plot
ax.set_xlim((1e-2, 2))
ax.invert_xaxis()
lgd = ax.legend(loc='upper left')
save_plot(fig, 'figures/fig3a', (lgd, ))
def plot_kernel_vs_forest_weights(y, res):
""" Plots the weights learned by Mondrian kernel and Mondrian forest based on the same set of M Mondrian samples.
This procedure takes as input a dictionary res, returned by the evaluate_all_lifetimes procedure in mondrian_kernel.py.
"""
w_forest = res['w_forest']
w_kernel = res['w_kernel']
# plot weights against each other
fig1 = plt.figure(figsize=(8, 4))
ax1 = fig1.add_subplot('121')
ax1.set_xlabel('weights learned by "Mondrian forest"')
ax1.set_ylabel('weights learned by Mondrian kernel')
ax1.scatter(w_forest, w_kernel, marker='.', color=tableau20(16))
xl = ax1.get_xlim()
yl = ax1.get_ylim()
lims = [
np.min([xl, yl]), # min of both axes
np.max([xl, yl]), # max of both axes
]
ax1.plot(lims, lims, '--', color='black', alpha=0.75, zorder=0)
ax1.set_xlim(xl)
#ax1.set_ylim(yl)
ax1.set_ylim((-60, 60))
# plot histogram of weight values (and training targets)
ax2 = fig1.add_subplot('122')
ax2.set_xlabel('values')
ax2.set_ylabel('value frequency')
bins = np.linspace(-100, 20, 50)
ax2.hist(w_forest, bins=bins, histtype='stepfilled', normed=True, color=tableau20(6), alpha=0.5,
label='M. forest weights $\mathbf{w}$')
ax2.hist(w_kernel, bins=bins, histtype='stepfilled', normed=True, color=tableau20(4), alpha=0.5,
label='M. kernel weights $\mathbf{w}$')
ax2.hist(y - np.mean(y), bins=bins, histtype='stepfilled', normed=True, color=tableau20(8), alpha=0.5,
label='training targets $\mathbf{y}$')
ax2.set_ylim((0.0, 0.16))
ax2.legend(frameon=False, loc='upper left')
fig1.tight_layout()
def experiment_CPU():
""" Plots test error performance as a function of computational time on the CPU data set. For the Mondrian kernel,
all lifetime values from 0 up to a terminal lifetime are swept through. For Fourier features and random binning
a binary search procedure is employed to find good lifetime parameter values, with an initial expansion phase.
"""
# fix random seed
np.random.seed(9879846)
# load data
X, y, X_test, y_test = load_CPU()
# set parameters
M = 350 # number of Mondrian trees to use
lifetime_max = 1*1e-6 # terminal lifetime
delta = 0.0001 # ridge regression delta
# set up plot
fig = plt.figure(figsize=(8, 4))
ax = fig.add_subplot(1, 1, 1)
remove_chartjunk(ax)
ax.yaxis.grid(b=True, which='major', linestyle='dotted', lw=0.5, color='black', alpha=0.3)
ax.set_xlabel('computational time [s]')
ax.set_ylabel('validation set relative error [\%]')
ax.set_xscale('log')
ax.set_ylim((0, 25))
# Fourier features
runtimes, errors = select_width(X, y, X_test, y_test, M, delta, evaluate_fourier_features, 100)
ax.scatter(runtimes, errors, marker='^', color=tableau20(0), label='Fourier features')
# random binning
runtimes, errors = select_width(X, y, X_test, y_test, M, delta, evaluate_random_binning, 50)
ax.scatter(runtimes, errors, marker='o', color=tableau20(6), label='random binning')
# Mondrian kernel
res = evaluate_all_lifetimes(X, y, X_test, y_test, M, lifetime_max, delta, mondrian_kernel=True)
ax.scatter(res['runtimes'], res['kernel_test'], marker='.', color=tableau20(4), label='Mondrian kernel')
ax.legend(bbox_to_anchor=[0.42, 0.35], frameon=False, ncol=1)
def main(args_dict):
# Set up plot
matplotlib_configure_as_notebook()
fig, ax = plt.subplots(1, 2, facecolor='w', figsize=(9.25, 3.25))
# Estimating Z
Ms = np.arange(3, args_dict['M'] + 1)
ax[0].set_xlabel('number of samples $M$')
ax[0].set_ylabel('MSE of $\hat{Z}$, in units of $Z^2$')
ax[0].set_xlim((np.min(Ms), np.max(Ms)))
ax[0].set_xscale('log')
ax[0].set_yscale('log')
ax[0].grid(b=True,
which='major',
linestyle='dotted',
lw=.5,
color='black',
alpha=0.5)
ax[0].plot(Ms,
Z_Gumbel_MSE(Ms),
linestyle='-',
color=tableau20(0),
label='Gumbel: MSE')
ax[0].plot(Ms,
Z_Gumbel_var(Ms),
linestyle='dashed',
color=tableau20(0),
label='Gumbel: var')
ax[0].plot(Ms,
Z_Exponential_MSE(Ms),
linestyle='-',
color=tableau20(2),
label='Exponential: MSE')
ax[0].plot(Ms,
Z_Exponential_var(Ms),
linestyle='dashed',
color=tableau20(2),
label='Exponential: var')
# Estimating ln Z
Ms = np.arange(1, args_dict['M'] + 1)
ax[1].set_xlabel('number of samples $M$')
ax[1].set_ylabel('MSE of $\widehat{\ln Z}$, in units of $1$')
ax[1].set_xlim((np.min(Ms), np.max(Ms)))
ax[1].set_xscale('log')
ax[1].set_yscale('log')
ax[1].grid(b=True,
which='major',
linestyle='dotted',
lw=0.5,
color='black',
alpha=0.5)
ax[1].plot(Ms,
lnZ_Gumbel_MSE(Ms),
linestyle='-',
color=tableau20(0),
label='Gumbel: MSE')
ax[1].plot(Ms,
lnZ_Exponential_MSE(Ms),
linestyle='-',
color=tableau20(2),
label='Exponential: MSE')
ax[1].plot(Ms,
lnZ_Exponential_var(Ms),
linestyle='dashed',
color=tableau20(2),
label='Exponential: var')
# Finalize plot
lgd0 = ax[0].legend()
lgd1 = ax[1].legend()
plt.tight_layout()
save_plot(fig, 'figures/fig1', (
lgd0,
lgd1,
))
def plot_mondrian_kernel_vs_mondrian_forest(lifetime_max, res):
""" Plots training and test set error of Mondrian kernel and Mondrian forest based on the same set of M Mondrian samples.
This procedure takes as input a dictionary res, returned by the evaluate_all_lifetimes procedure in mondrian_kernel.py.
"""
times = res['times']
forest_train = res['forest_train']
forest_test = res['forest_test']
kernel_train = res['kernel_train']
kernel_test = res['kernel_test']
# set up test error plot
fig = plt.figure(figsize=(7, 4))
ax = fig.add_subplot('111')
remove_chartjunk(ax)
ax.set_xlabel('lifetime $\lambda$')
ax.set_ylabel('relative error [\%]')
ax.yaxis.grid(b=True,
which='major',
linestyle='dotted',
lw=0.5,
color='black',
alpha=0.3)
ax.set_xscale('log')
ax.set_xlim((1e-8, lifetime_max))
ax.set_ylim((0, 25))
rasterized = False
ax.plot(times,
forest_test,
drawstyle="steps-post",
ls='-',
lw=2,
color=tableau20(6),
label='"M. forest" (test)',
rasterized=rasterized)
ax.plot(times,
forest_train,
drawstyle="steps-post",
ls='-',
color=tableau20(7),
label='"M. forest" (train)',
rasterized=rasterized)
ax.plot(times,
kernel_test,
drawstyle="steps-post",
ls='-',
lw=2,
color=tableau20(4),
label='M. kernel (test)',
rasterized=rasterized)
ax.plot(times,
kernel_train,
drawstyle="steps-post",
ls='-',
color=tableau20(5),
label='M. kernel (train)',
rasterized=rasterized)
ax.legend(bbox_to_anchor=[1.15, 1.05], frameon=False)