def compare_kernels_experiment():
kernel1 = fk.Carls_Mauna_kernel()
kernel2 = ( fk.SqExpKernel(-0.7, -1.3) + fk.SqExpKernel(4.8, 2.3) ) * \
( fk.SqExpKernel(3.0, 0.5) + fk.SqExpPeriodicKernel(0.4, -0.0, -0.9) )
#kernel2 = ( SqExp(ell=-0.8, sf=-1.4) + Periodic(ell=0.5, p=-0.3, sf=-1.1) + RQ(ell=1.9, sf=1.6, a=0.2) + ( SqExp(ell=4.5, sf=1.0) x Periodic(ell=0.6, p=-0.0, sf=0.1) ) )
X, y = load_mauna_original()
N_orig = X.shape[0] # subsample data.
X = X[:N_orig // 5, :]
y = y[:N_orig // 5, :]
print "Carl's kernel"
print kernel1.pretty_print()
kernel_hypers1, nll1 = gpml.optimize_params(kernel1.gpml_kernel_expression(), kernel1.param_vector(), \
X, y, noise=np.log(0.19), iters=100 )
k1_opt = kernel1.family().from_param_vector(kernel_hypers1)
print k1_opt.pretty_print()
print "Carl's NLL =", nll1
print "Our kernel"
print kernel2.pretty_print()
kernel_hypers2, nll2 = gpml.optimize_params(kernel2.gpml_kernel_expression(), kernel2.param_vector(), \
X, y, noise=np.log(0.19), iters=100)
k2_opt = kernel2.family().from_param_vector(kernel_hypers2)
print k2_opt.pretty_print()
print "Our NLL =", nll2
def simple_gef_load_experiment(verbose=True):
'''A first version of an experiment learning kernels'''
seed_kernels = [
fk.MaskKernel(2, 0, fk.SqExpKernel(0, 0)),
fk.MaskKernel(2, 1, fk.SqExpKernel(0, 0))
]
X, y = load_simple_gef_load()
# subsample data.
X = X[0:99, :]
y = y[0:99, :]
max_depth = 5
k = 2 # Expand k best
nll_key = 1
BIC_key = 2
active_key = BIC_key
results = []
for dummy in range(max_depth):
new_results = structure_search.try_expanded_kernels(
X, y, D=2, seed_kernels=seed_kernels, verbose=verbose)
results = results + new_results
print
results = sorted(results, key=lambda p: p[active_key], reverse=True)
for kernel, nll, BIC in results:
print nll, BIC, kernel.pretty_print()
seed_kernels = [
r[0] for r in sorted(new_results, key=lambda p: p[active_key])[0:k]
]
def plot_our_kernel():
kernel = ( fk.SqExpKernel(-0.7, -1.3) + fk.SqExpKernel(4.8, 2.3) ) * \
( fk.SqExpKernel(3.0, 0.5) + fk.SqExpPeriodicKernel(0.4, -0.0, -0.9) )
X = np.linspace(0, 10, 1000)
sigma = gpml.plot_kernel(kernel, X)
pylab.figure()
pylab.plot(X, sigma)
pylab.title('Our kernel')
def sample_mauna_best():
# This kernel was chosen from a run of Mauna datapoints.
kernel = ( fk.SqExpKernel(-0.7, -1.3) + fk.SqExpKernel(4.8, 2.3) ) * \
( fk.SqExpKernel(3.0, 0.5) + fk.SqExpPeriodicKernel(0.4, -0.0, -0.9) )
X = np.linspace(0, 50, 500)
# Todo: set random seed.
sample = gpml.sample_from_gp_prior(kernel, X)
pylab.figure()
pylab.plot(X, sample)
pylab.title(
'( SqExp(ell=-0.7, sf=-1.3) + SqExp(ell=4.8, sf=2.3) ) \n x ( SqExp(ell=3.0, sf=0.5) + Periodic(ell=0.4, p=-0.0, sf=-0.9) )'
)
def expand_test2():
k1 = fk.MaskKernel(2, 0, fk.SqExpKernel(1, 1))
k2 = fk.MaskKernel(2, 1, fk.SqExpPeriodicKernel(2, 2, 2))
e = fk.SumKernel([k1, k2])
g = grammar.MultiDGrammar(2)
print ''
for f in grammar.expand(e, g):
print f.pretty_print()
print grammar.canonical(f).pretty_print()
print
print ' ***** duplicates removed *****'
print
kernels = grammar.expand(e, g)
for f in grammar.remove_duplicates(kernels):
print f.pretty_print()
print
print '%d originally, %d without duplicates' % (
len(kernels), len(grammar.remove_duplicates(kernels)))
print 'expand_test complete'
def simple_mauna_experiment():
'''A first version of an experiment learning kernels'''
seed_kernels = [fk.SqExpKernel(0, 0)]
X, y = load_mauna_original()
N_orig = X.shape[0] # subsample data.
X = X[:N_orig // 3, :]
y = y[:N_orig // 3, :]
max_depth = 4
k = 4 # Expand k best
nll_key = 1
laplace_key = 2
results = []
for dummy in range(max_depth):
new_results = structure_search.try_expanded_kernels(
X, y, D=2, seed_kernels=seed_kernels, verbose=False)
results = results + new_results
print
results = sorted(results, key=lambda p: p[nll_key], reverse=True)
for kernel, nll, laplace in results:
print nll, laplace, kernel.pretty_print()
seed_kernels = [
r[0] for r in sorted(new_results, key=lambda p: p[nll_key])[0:k]
]
def not_tet_num_params():
'''The number of hyper-parameters passed in to gpml should be the same as the number returned.'''
k = fk.SqExpKernel(0, 0)
init_params = k.param_vector()
X, y = load_mauna()
optimized_hypers, _ = gpml.optimize_params(k.gpml_kernel_expression(),
init_params, X, y)
assert optimized_hypers.size == init_params.size
def call_gpml_test():
np.random.seed(0)
k = fk.SumKernel([fk.SqExpKernel(0, 0), fk.SqExpKernel(0, 0)])
print k.gpml_kernel_expression()
print k.pretty_print()
print '[%s]' % k.param_vector()
X, y = load_mauna()
N_orig = X.shape[0]
X = X[:N_orig // 3, :]
y = y[:N_orig // 3, :]
results = []
pylab.figure()
for i in range(15):
init_params = np.random.normal(size=k.param_vector().size)
#kernel_hypers, nll, nlls = gpml.optimize_params(k.gpml_kernel_expression(), k.param_vector(), X, y, return_all=True)
kernel_hypers, nll, nlls = gpml.optimize_params(
k.gpml_kernel_expression(), init_params, X, y, return_all=True)
print "kernel_hypers =", kernel_hypers
print "nll =", nll
k_opt = k.family().from_param_vector(kernel_hypers)
print k_opt.gpml_kernel_expression()
print k_opt.pretty_print()
print '[%s]' % k_opt.param_vector()
pylab.semilogx(range(1, nlls.size + 1), nlls)
results.append((kernel_hypers, nll))
pylab.draw()
print
print
results = sorted(results, key=lambda p: p[1])
for kernel_hypers, nll in results:
print nll, kernel_hypers
print "done"
def fear_experiment(data_file, results_filename, y_dim=1, subset=None, max_depth=2, k=2, verbose=True, sleep_time=60, n_sleep_timeout=20, re_submit_wait=60, \
description=''):
'''Recursively search for the best kernel'''
X, y, D = fear_load_mat(data_file, y_dim)
# Subset if necessary
if not subset is None:
X = X[subset, :]
y = y[subset]
##### This should be abstracted
seed_kernels = [fk.MaskKernel(D, i, fk.SqExpKernel(0., 0.)) for i in range(D)] + \
[fk.MaskKernel(D, i, fk.SqExpPeriodicKernel(0., 0., 0.)) for i in range(D)] + \
[fk.MaskKernel(D, i, fk.RQKernel(0., 0., 0.)) for i in range(D)]
nll_key = 1
laplace_key = 2
BIC_key = 3
active_key = BIC_key
results = []
results_sequence = []
for r in range(max_depth):
if r == 0:
new_results = fear_run_experiments(seed_kernels, X, y, verbose=verbose, \
sleep_time=sleep_time, n_sleep_timeout=n_sleep_timeout, re_submit_wait=re_submit_wait)
else:
new_results = fear_run_experiments(fear_expand_kernels(D, seed_kernels, verbose=verbose), X, y, verbose=verbose, \
sleep_time=sleep_time, n_sleep_timeout=n_sleep_timeout, re_submit_wait=re_submit_wait)
results = results + new_results
print
results = sorted(results, key=lambda p: p[active_key], reverse=True)
for kernel, nll, laplace, BIC in results:
print nll, laplace, BIC, kernel.pretty_print()
seed_kernels = [
r[0] for r in sorted(new_results, key=lambda p: p[active_key])[0:k]
]
results_sequence.append(results)
# Write results to a file
results = sorted(results, key=lambda p: p[active_key], reverse=True)
with open(results_filename, 'w') as outfile:
outfile.write(
'Experiment results for\n datafile = %s\n y_dim = %d\n subset = %s\n max_depth = %f\n k = %f\n Description = %s\n\n'
% (data_file, y_dim, subset, max_depth, k, description))
for (i, results) in enumerate(results_sequence):
outfile.write('\n%%%%%%%%%% Level %d %%%%%%%%%%\n\n' % i)
for kernel, nll, laplace, BIC in results:
outfile.write('nll=%f, laplace=%f, BIC=%f, kernel=%s\n' %
(nll, laplace, BIC, kernel.__repr__()))
def full_gef_load_experiment(zone=1, max_depth=5, verbose=True):
'''Round 2'''
# seed_kernels = [fk.MaskKernel(2, 0, fk.SqExpKernel(0, 0)),
# fk.MaskKernel(2, 1, fk.SqExpKernel(0, 0))]
seed_kernels = [fk.MaskKernel(12, i, fk.SqExpKernel(0., 0.)) for i in range(12)] + \
[fk.MaskKernel(12, i, fk.SqExpPeriodicKernel(0., 0., 0.)) for i in range(12)] + \
[fk.MaskKernel(12, i, fk.RQKernel(0., 0., 0.)) for i in range(12)]
X, y = load_full_gef_load()
# subsample data.
X = X[0:299, :]
y = y[0:299, zone - 1]
# max_depth = 5
k = 2 # Expand k best
nll_key = 1
BIC_key = 2
active_key = BIC_key
results = []
for i in range(max_depth):
if i:
expand = True
else:
expand = False
new_results = structure_search.try_expanded_kernels(
X,
y,
D=12,
seed_kernels=seed_kernels,
expand=expand,
verbose=verbose)
results = results + new_results
print
results = sorted(results, key=lambda p: p[active_key], reverse=True)
for kernel, nll, BIC in results:
print nll, BIC, kernel.pretty_print()
seed_kernels = [
r[0] for r in sorted(new_results, key=lambda p: p[active_key])[0:k]
]
def plot_gef_load_Z01():
# This kernel was chosen from a run of gef_load datapoints.
# kernel = eval(ProductKernel([ covMask(ndim=12, active_dimension=0, base_kernel=RQKernel(lengthscale=0.268353, output_variance=-0.104149, alpha=-2.105742)), covMask(ndim=12, active_dimension=9, base_kernel=SqExpKernel(lengthscale=1.160242, output_variance=0.004344)), SumKernel([ covMask(ndim=12, active_dimension=0, base_kernel=SqExpPeriodicKernel(lengthscale=-0.823413, period=0.000198, output_variance=-0.917064)), covMask(ndim=12, active_dimension=0, base_kernel=RQKernel(lengthscale=-0.459219, output_variance=-0.077250, alpha=-2.212718)) ]) ]))
X, y, D = fear_load_mat('../data/gef_load_full_Xy.mat', 1)
kernel = fk.MaskKernel(D, 0, fk.RQKernel(0.268353, -0.104149, -2.105742)) * fk.MaskKernel(D, 9, fk.SqExpKernel(1.160242, 0.004344)) * \
(fk.MaskKernel(D, 0, fk.SqExpPeriodicKernel(-0.823413, 0.000198, -0.917064)) + fk.MaskKernel(D, 0, fk.RQKernel(-0.459219, -0.077250, -2.212718)))
# Todo: set random seed.
sample = gpml.sample_from_gp_prior(kernel, X[0:499, :])
pylab.figure()
pylab.plot(X[0:499, 0], y[0:499])
pylab.title('GEFCom2012 Z01 and T09 - first 500 data points')
pylab.xlabel('Time')
pylab.ylabel('Load')
def plot_gef_load_Z01_smooth_2d_mean():
X, y, D = fear_load_mat('../data/gef_load_full_Xy.mat', 1)
kernel = fk.MaskKernel(D, 0, fk.RQKernel(0.268353, -0.104149, -2.105742)) * fk.MaskKernel(D, 9, fk.SqExpKernel(1.160242, 0.004344)) * \
(fk.MaskKernel(D, 0, fk.SqExpPeriodicKernel(-0.823413, 0.000198, -0.917064)) + fk.MaskKernel(D, 0, fk.RQKernel(-0.459219, -0.077250, -2.212718)))
kernel_1 = fk.MaskKernel(D, 0, fk.RQKernel(0.268353, -0.104149, -2.105742)) * fk.MaskKernel(D, 9, fk.SqExpKernel(1.160242, 0.004344)) * \
fk.MaskKernel(D, 0, fk.SqExpPeriodicKernel(-0.823413, 0.000198, -0.917064))
kernel_2 = fk.MaskKernel(D, 0, fk.RQKernel(0.268353, -0.104149, -2.105742)) * fk.MaskKernel(D, 9, fk.SqExpKernel(1.160242, 0.004344)) * \
fk.MaskKernel(D, 0, fk.RQKernel(-0.459219, -0.077250, -2.212718))
min_T = -3.0
max_T = 1.0
N_T = 10
temps = np.repeat(np.linspace(min_T, max_T, N_T), 499)
input = np.tile(X[0:499, :], (N_T, 1))
input[:, 9] = temps
posterior_mean = gpml.posterior_mean(kernel,
kernel_2,
X[0:499, :],
y[0:499],
input,
iters=300)
X_plt = X[0:499, 0]
Y_plt = np.linspace(min_T, max_T, N_T)
Z_plt = np.reshape(posterior_mean, (N_T, 499), 'A')
data = {'X': X_plt, 'Y': Y_plt, 'Z': Z_plt, 'post_mean': posterior_mean}
scipy.io.savemat('temp_data.mat', data)
def plot_gef_load_Z01_split_mean_temp():
X, y, D = fear_load_mat('../data/gef_load_full_Xy.mat', 1)
kernel = fk.MaskKernel(D, 0, fk.RQKernel(0.268353, -0.104149, -2.105742)) * fk.MaskKernel(D, 9, fk.SqExpKernel(1.160242, 0.004344)) * \
(fk.MaskKernel(D, 0, fk.SqExpPeriodicKernel(-0.823413, 0.000198, -0.917064)) + fk.MaskKernel(D, 0, fk.RQKernel(-0.459219, -0.077250, -2.212718)))
kernel_1 = fk.MaskKernel(D, 0, fk.RQKernel(0.268353, -0.104149, -2.105742)) * fk.MaskKernel(D, 9, fk.SqExpKernel(1.160242, 0.004344)) * \
fk.MaskKernel(D, 0, fk.SqExpPeriodicKernel(-0.823413, 0.000198, -0.917064))
posterior_mean_1 = gpml.posterior_mean(kernel,
kernel_1,
X[0:499, :],
y[0:499],
iters=10)
kernel_2 = fk.MaskKernel(D, 0, fk.RQKernel(0.268353, -0.104149, -2.105742)) * fk.MaskKernel(D, 9, fk.SqExpKernel(1.160242, 0.004344)) * \
fk.MaskKernel(D, 0, fk.RQKernel(-0.459219, -0.077250, -2.212718))
posterior_mean_2 = gpml.posterior_mean(kernel,
kernel_2,
X[0:499, :],
y[0:499],
iters=10)
plt.figure()
host = host_subplot(111, axes_class=AA.Axes)
plt.subplots_adjust(right=0.85)
par1 = host.twinx()
# host.set_xlim(0, 2)
# host.set_ylim(0, 2)
host.set_xlabel("Temperature (T09)")
# par1.set_ylabel("Periodic component")
plt.title('Posterior mean function')
host.set_ylabel("Load posterior mean")
p2, = host.plot(X[0:499, 9], y[0:499], 'o', alpha=0.5)
p1, = host.plot(X[0:499, 9], posterior_mean_2, 'o')
# par1.set_ylim(0, 4)
host.legend()
host.axis["left"].label.set_color(p1.get_color())
# par1.axis["right"].label.set_color(p2.get_color())
plt.draw()
plt.show()
def kernel_test():
k = fk.MaskKernel(4, 3, fk.SqExpKernel(0, 0))
print k.gpml_kernel_expression()
print k.pretty_print()
print '[%s]' % k.param_vector()
print 'kernel_test complete'
