def test_lml_improving():
""" Test that hyperparameter-tuning improves log-marginal likelihood. """
for kernel in kernels:
if kernel == fixed_kernel: continue
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(kernel.theta))
def test_lml_improving():
""" Test that hyperparameter-tuning improves log-marginal likelihood. """
for kernel in kernels:
if kernel == fixed_kernel: continue
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(kernel.theta))
def test_lml_without_cloning_kernel(kernel):
# Test that clone_kernel=False has side-effects of kernel.theta.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
input_theta = np.ones(gpc.kernel_.theta.shape, dtype=np.float64)
gpc.log_marginal_likelihood(input_theta, clone_kernel=False)
assert_almost_equal(gpc.kernel_.theta, input_theta, 7)
def test_custom_optimizer():
""" Test that GPC can use externally defined optimizers. """
# Define a dummy optimizer that simply tests 50 random hyperparameters
def optimizer(obj_func, initial_theta, bounds):
rng = np.random.RandomState(0)
theta_opt, func_min = \
initial_theta, obj_func(initial_theta, eval_gradient=False)
for _ in range(50):
theta = np.atleast_1d(
rng.uniform(np.maximum(-2, bounds[:, 0]),
np.minimum(1, bounds[:, 1])))
f = obj_func(theta, eval_gradient=False)
if f < func_min:
theta_opt, func_min = theta, f
return theta_opt, func_min
for kernel in kernels:
if kernel == fixed_kernel:
continue
gpc = GaussianProcessClassifier(kernel=kernel, optimizer=optimizer)
gpc.fit(X, y_mc)
# Checks that optimizer improved marginal likelihood
assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(kernel.theta))
def test_lml_gradient(kernel):
# Compare analytic and numeric gradient of log marginal likelihood.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
lml, lml_gradient = gpc.log_marginal_likelihood(kernel.theta, True)
lml_gradient_approx = approx_fprime(
kernel.theta, lambda theta: gpc.log_marginal_likelihood(theta, False), 1e-10
)
assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
def test_lml_gradient(kernel):
# Compare analytic and numeric gradient of log marginal likelihood.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
lml, lml_gradient = gpc.log_marginal_likelihood(kernel.theta, True)
lml_gradient_approx = \
approx_fprime(kernel.theta,
lambda theta: gpc.log_marginal_likelihood(theta,
False),
1e-10)
assert_almost_equal(lml_gradient, lml_gradient_approx, 3)
def test_converged_to_local_maximum(kernel):
# Test that we are in local maximum after hyperparameter-optimization.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
lml, lml_gradient = gpc.log_marginal_likelihood(gpc.kernel_.theta, True)
assert np.all((np.abs(lml_gradient) < 1e-4)
| (gpc.kernel_.theta == gpc.kernel_.bounds[:, 0])
| (gpc.kernel_.theta == gpc.kernel_.bounds[:, 1]))
def test_converged_to_local_maximum(kernel):
# Test that we are in local maximum after hyperparameter-optimization.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
lml, lml_gradient = \
gpc.log_marginal_likelihood(gpc.kernel_.theta, True)
assert np.all((np.abs(lml_gradient) < 1e-4) |
(gpc.kernel_.theta == gpc.kernel_.bounds[:, 0]) |
(gpc.kernel_.theta == gpc.kernel_.bounds[:, 1]))
def test_custom_optimizer(kernel):
# Test that GPC can use externally defined optimizers.
# Define a dummy optimizer that simply tests 50 random hyperparameters
def optimizer(obj_func, initial_theta, bounds):
rng = np.random.RandomState(0)
theta_opt, func_min = \
initial_theta, obj_func(initial_theta, eval_gradient=False)
for _ in range(50):
theta = np.atleast_1d(rng.uniform(np.maximum(-2, bounds[:, 0]),
np.minimum(1, bounds[:, 1])))
f = obj_func(theta, eval_gradient=False)
if f < func_min:
theta_opt, func_min = theta, f
return theta_opt, func_min
gpc = GaussianProcessClassifier(kernel=kernel, optimizer=optimizer)
gpc.fit(X, y_mc)
# Checks that optimizer improved marginal likelihood
assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(kernel.theta))
def test_converged_to_local_maximum():
""" Test that we are in local maximum after hyperparameter-optimization."""
for kernel in kernels:
if kernel == fixed_kernel: continue
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
lml, lml_gradient = \
gpc.log_marginal_likelihood(gpc.kernel_.theta, True)
assert_true(
np.all((np.abs(lml_gradient) < 1e-4)
| (gpc.kernel_.theta == gpc.kernel_.bounds[:, 0])
| (gpc.kernel_.theta == gpc.kernel_.bounds[:, 1])))
def test_random_starts():
# Test that an increasing number of random-starts of GP fitting only
# increases the log marginal likelihood of the chosen theta.
n_samples, n_features = 25, 2
rng = np.random.RandomState(0)
X = rng.randn(n_samples, n_features) * 2 - 1
y = (np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1)) > 0
kernel = C(1.0, (1e-2, 1e2)) * RBF(
length_scale=[1e-3] * n_features, length_scale_bounds=[(1e-4, 1e2)] * n_features
)
last_lml = -np.inf
for n_restarts_optimizer in range(5):
gp = GaussianProcessClassifier(
kernel=kernel, n_restarts_optimizer=n_restarts_optimizer, random_state=0
).fit(X, y)
lml = gp.log_marginal_likelihood(gp.kernel_.theta)
assert lml > last_lml - np.finfo(np.float32).eps
last_lml = lml
def test_random_starts():
# Test that an increasing number of random-starts of GP fitting only
# increases the log marginal likelihood of the chosen theta.
n_samples, n_features = 25, 2
rng = np.random.RandomState(0)
X = rng.randn(n_samples, n_features) * 2 - 1
y = (np.sin(X).sum(axis=1) + np.sin(3 * X).sum(axis=1)) > 0
kernel = C(1.0, (1e-2, 1e2)) \
* RBF(length_scale=[1e-3] * n_features,
length_scale_bounds=[(1e-4, 1e+2)] * n_features)
last_lml = -np.inf
for n_restarts_optimizer in range(5):
gp = GaussianProcessClassifier(
kernel=kernel, n_restarts_optimizer=n_restarts_optimizer,
random_state=0).fit(X, y)
lml = gp.log_marginal_likelihood(gp.kernel_.theta)
assert_greater(lml, last_lml - np.finfo(np.float32).eps)
last_lml = lml
def test_lml_improving(kernel):
# Test that hyperparameter-tuning improves log-marginal likelihood.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert gpc.log_marginal_likelihood(gpc.kernel_.theta) > gpc.log_marginal_likelihood(
kernel.theta
)
X = rng.randn(200, 2)
Y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0)
# fit the model
plt.figure(figsize=(10, 5))
kernels = [1.0 * RBF(length_scale=1.0), 1.0 * DotProduct(sigma_0=1.0)**2]
for i, kernel in enumerate(kernels):
clf = GaussianProcessClassifier(kernel=kernel, warm_start=True).fit(X, Y)
# plot the decision function for each datapoint on the grid
Z = clf.predict_proba(np.vstack((xx.ravel(), yy.ravel())).T)[:, 1]
Z = Z.reshape(xx.shape)
plt.subplot(1, 2, i + 1)
image = plt.imshow(Z, interpolation='nearest',
extent=(xx.min(), xx.max(), yy.min(), yy.max()),
aspect='auto', origin='lower', cmap=plt.cm.PuOr_r)
contours = plt.contour(xx, yy, Z, levels=[0], linewidths=2,
linetypes='--')
plt.scatter(X[:, 0], X[:, 1], s=30, c=Y, cmap=plt.cm.Paired)
plt.xticks(())
plt.yticks(())
plt.axis([-3, 3, -3, 3])
plt.colorbar(image)
plt.title("%s\n Log-Marginal-Likelihood:%.3f"
% (clf.kernel_, clf.log_marginal_likelihood(clf.kernel_.theta)),
fontsize=12)
plt.tight_layout()
plt.show()
def main():
# mpl.style.use('classic')
mpl.rcParams['font.size'] = FONT_SIZE
mpl.rcParams['lines.markersize'] = MARKER_SIZE
mpl.rcParams['lines.linewidth'] = LINE_WIDTH
if len(sys.argv) < 2:
print('USAGE: python analyze-log.py <path to log file> [gpr|1d|easy] [shift|alt|dist|lat|orient]')
sys.exit()
mode = sys.argv[2]
measure = 'std'
if len(sys.argv) > 3:
sc = 1
if sys.argv[3] == 'shift':
target = 'rel shift error (m/km)'
elif sys.argv[3] == 'alt':
target = 'altitude error'
sc = 1000
elif sys.argv[3] == 'dist':
target = 'dist error (m/km)'
elif sys.argv[3] == 'lat':
target = 'lat error (m/km)'
elif sys.argv[3] == 'orient':
target = 'rot error'
else:
assert False, 'unknown target: %s' % sys.argv[3]
if 'prct' in sys.argv[3:]:
measure = 'p99.73'
else:
target = 'rel shift error (m/km)' # 'shift error km' #if not one_d_only else 'dist error'
predictors = (
'sol elong', # solar elongation
'total dev angle', # total angle between initial estimate and actual relative orientation
'distance', # distance of object
'visible', # esimate of % visible because of camera view edge
)
predictor_labels = (
'Phase Angle (deg)',
'Initial orientation error (deg)',
'Distance (km)',
'In camera view (%)',
)
data = []
system_models = {}
for logfile in sys.argv[1].split(" "):
mission = logfile.split('-')[0]
sm = get_system_model(mission)
system_models[mission] = sm
# read data
X, Y, yc, labels = read_data(sm, os.path.join(LOG_DIR, logfile), predictors, [target])
#X[:, 1] = np.abs(tools.wrap_degs(X[:, 1]))
data.append((logfile, X, yc, Y.flatten(), labels))
title_map = {
'didy2w': 'D2, WAC',
'didy2n': 'D2, NAC',
'didy1w': 'D1, WAC',
'didy1n': 'D1, NAC',
}
if mode in ('1d', 'easy'):
n_groups = 6
# yr = yr/1000
# idxs = (0, 1, 2, 3)
idxs = (2,)
for idx in idxs:
fig, axs = plt.subplots(len(data), 1, figsize=FIG_SIZE, sharex=True)
for i, (logfile, X, yc, yr, labels) in enumerate(data):
if mode == 'easy':
q997 = np.percentile(np.abs(yr), 99.73)
tmp = tuple((X[:, k] >= EASY_LIMITS[k][0], X[:, k] <= EASY_LIMITS[k][1]) for k in idxs if k != idx)
# concatenate above & take logical and, also remove worst 0.3%
I = np.logical_and.reduce(
sum(tmp, ()) + (np.logical_or(np.abs(yr) < q997, yr == FAIL_ERRS[target]),))
else:
I = np.ones((X.shape[0],), dtype='bool')
xmin, xmax = np.min(X[I, idx]), np.max(X[I, idx])
ax = axs[i] if len(data) > 1 else axs
line, = ax.plot(X[I, idx], yr[I], 'x')
if n_groups:
# calc means and stds in bins
# x = [1/v for v in np.linspace(1/xmin, 1/xmax, n_groups+1)]
x = np.linspace(xmin, xmax, n_groups + 1)
y_grouped = [yr[np.logical_and.reduce((
I,
np.logical_not(yc),
X[:, idx] > x[i],
X[:, idx] < x[i + 1],
))] for i in range(n_groups)]
# means = [np.percentile(yg, 50) for yg in y_grouped]
means = np.array([np.mean(yg) for yg in y_grouped])
# stds = np.subtract([np.percentile(yg, 68) for yg in y_grouped], means)
stds = np.array([np.std(yg) for yg in y_grouped])
x = x.reshape((-1, 1))
stds = stds.reshape((-1, 1))
means = means.reshape((-1, 1))
xstep = np.concatenate((x, x), axis=1).flatten()[1:-1]
sstep = np.concatenate((stds, stds), axis=1).flatten()
mstep = np.concatenate((means, means), axis=1).flatten()
ax.plot(xstep, sstep, '-')
ax.plot(xstep, mstep, '-')
# bar_width = (xmax - xmin)/n_groups * 0.2
# rects1 = ax.bar((x[1:] + x[:-1]) * 0.5, stds, width=bar_width, bottom=means-stds/2,
# alpha=0.4, color='b', yerr=stds, error_kw={'ecolor': '0.3'}, label='error')
else:
# filtered means, stds
xt = np.linspace(xmin, xmax, 100)
if False:
# exponential weight
weight_fun = lambda d: 0.01 ** abs(d / (xmax - xmin))
else:
# gaussian weight
from scipy.stats import norm
from scipy.interpolate import interp1d
interp = interp1d(xt - xmin, norm.pdf(xt - xmin, 0, (xmax - xmin) / 10))
weight_fun = lambda d: interp(abs(d))
if False:
# use smoothed mean for std calc
yma = tools.smooth1d(X[I, idx], X[I, idx], yr[I], weight_fun)
else:
# use global mean for std calc (fast)
yma = np.mean(yr[I])
ym = tools.smooth1d(xt, X[I, idx], yr[I], weight_fun)
ystd = tools.smooth1d(xt, X[I, idx], (yr[I] - yma) ** 2, weight_fun) ** (1 / 2)
ax.plot(xt, ym, '-')
ax.plot(xt, ystd, '-')
# ax.set_title('%s: %s by %s' % (logfile, target, predictor_labels[idx]))
ax.set_title('%s' % title_map[logfile.split('-')[0]])
if i == len(data)-1:
ax.set_xlabel(predictor_labels[idx])
ax.set_ylabel(target)
ax.set_yticks(range(-200, 201, 50))
ax.hlines(range(-200, 201, 10), xmin, xmax, '0.95', '--')
ax.hlines(range(-200, 201, 50), xmin, xmax, '0.7', '-')
plt.setp(ax.get_xticklabels(), rotation=45)
# plt.setp(ax.get_yticklabels())
tools.hover_annotate(fig, ax, line, np.array(labels)[I])
# ax.set_xticks((x[1:] + x[:-1]) * 0.5)
# ax.set_xticklabels(['%.2f-%.2f' % (x[i], x[i+1]) for i in range(n_groups)])
# ax.legend()
# operation zones for didymos mission
if mission[:4] == 'didy' and idx == 2:
ax.set_xticks(np.arange(0.1, 10.5, 0.2))
if i == 0:
ax.axvspan(1.1, 1.3, facecolor='cyan', alpha=0.3)
ax.axvspan(3.8, 4.2, facecolor='orange', alpha=0.3)
elif i == 1:
ax.axvspan(0.15, 0.3, facecolor='pink', alpha=0.5)
ax.axvspan(1.1, 1.3, facecolor='cyan', alpha=0.3)
elif i == 2:
ax.axvspan(3.8, 4.2, facecolor='orange', alpha=0.3)
elif i == 3:
ax.axvspan(1.1, 1.3, facecolor='cyan', alpha=0.3)
ax.axvspan(2.8, 5.2, facecolor='orange', alpha=0.3)
plt.tight_layout()
plt.subplots_adjust(top=0.94, hspace=0.35)
while (plt.waitforbuttonpress() == False):
pass
elif mode == '2d':
scatter = False
# 0: solar elong
# 1: initial deviation angle
# 2: distance
# 3: ratio in view
idxs = tuple(range(4))
pairs = (
(2, 0),
# (1, 3),
# (2, 3),
# (0, 1),
)
# titles = ['ORB', 'AKAZE', 'SURF', 'SIFT']
# titles = [d[0][:-4] for d in data]
titles = [title_map[d[0].split('-')[0]] for d in data]
nd = len(data)
r, c = {
1: (1, 1),
2: (1, 2),
3: (3, 1),
4: (2, 2),
}[nd]
if scatter:
fig, axs = plt.subplots(r, c * len(pairs), figsize=FIG_SIZE)
else:
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import Normalize
fig = plt.figure(figsize=FIG_SIZE)
fig2 = plt.figure(figsize=FIG_SIZE)
for j, (logfile, X, yc, yr, labels) in enumerate(data):
# samples belonging to operational/easy domain
mission = logfile.split('-')[0]
sm = system_models[mission]
EASY_LIMITS[predictors.index('distance')] = (sm.min_distance, sm.max_distance)
E = np.logical_and.reduce(sum(tuple(
(X[:, k] >= EASY_LIMITS[k][0], X[:, k] <= EASY_LIMITS[k][1]) for k in idxs
), ()))
print('%s: overall 99.73%% perf: %.3f m/km (n=%d)' % (mission,
np.percentile(yr[np.logical_and(E, np.logical_not(yc))], 99.73),
np.sum(np.logical_and(E, np.logical_not(yc)))))
for i, (i0, i1) in enumerate(pairs):
if scatter:
tmp = tuple(
(X[:, k] >= EASY_LIMITS[k][0], X[:, k] <= EASY_LIMITS[k][1]) for k in idxs if k not in (i0, i1))
I = np.logical_and.reduce(sum(tmp, ()))
else:
I = np.ones((X.shape[0],), dtype='bool')
# add some offset if ratio in view is one so that they dont all stack in same place
offsets = (X[I, 3] == 1) * np.random.uniform(0, 0.2, (np.sum(I),))
off0 = 0 if i0 != 3 else offsets
off1 = 0 if i1 != 3 else offsets
if scatter:
ax = axs.flatten()[j * len(pairs) + i]
line = ax.scatter(X[I, i0] + off0, X[I, i1] + off1, s=60, c=yc[I], cmap=plt.cm.Paired,
alpha=0.5) # edgecolors=(0, 0, 0))
tools.hover_annotate(fig, ax, line, np.array(labels)[I])
else:
ax = fig.add_subplot(r, c * len(pairs), j * len(pairs) + i + 1, projection='3d')
ax2 = fig2.add_subplot(r, c * len(pairs), j * len(pairs) + i + 1, projection='3d')
xmin, xmax = np.min(X[I, i0]), np.max(X[I, i0])
ymin, ymax = np.min(X[I, i1]), np.max(X[I, i1])
x = np.linspace(xmin, xmax, 7)
y = np.linspace(ymin, ymax, 7) if i1 != 0 else np.array([0, 22.5, 45, 67.5, 90, 112.5, 135])
xx, yy = np.meshgrid(x, y)
y_grouped = [[yr[np.logical_and.reduce((
I,
np.logical_not(yc),
X[:, i0] > x[i],
X[:, i0] < x[i + 1],
X[:, i1] > y[j],
X[:, i1] < y[j + 1],
))] for i in range(len(x) - 1)] for j in range(len(y) - 1)]
# std when we assume zero mean, remove lowest 0.1% and highest 0.1% before calculating stats
stds = np.array(
[[np.sqrt(tools.robust_mean(y_grouped[j][i]**2, 0.1)) for i in range(len(x) - 1)] for j in range(len(y) - 1)])
samples = np.array([[len(y_grouped[j][i]) for i in range(len(x) - 1)] for j in range(len(y) - 1)])
d_coefs, pa_coefs, mod_stds = model_stds2(stds, samples)
R2 = (1 - np.sum((stds - mod_stds)**2 * samples) / np.sum(stds**2 * samples)) * 100
print_model(titles[j], x, d_coefs, y, pa_coefs, R2)
xstep = np.kron(xx, np.ones((2, 2)))[1:-1, 1:-1]
ystep = np.kron(yy, np.ones((2, 2)))[1:-1, 1:-1]
# mstep = np.kron(means, np.ones((2, 2)))
sstep = np.kron(stds, np.ones((2, 2)))
msstep = np.kron(mod_stds, np.ones((2, 2)))
scalarMap = plt.cm.ScalarMappable(norm=Normalize(vmin=np.min(sstep), vmax=np.max(sstep)), cmap=plt.cm.PuOr_r)
ax.plot_surface(xstep, ystep, sstep, facecolors=scalarMap.to_rgba(sstep), antialiased=True)
ax.view_init(30, -60)
ax2.plot_surface(xstep, ystep, msstep, facecolors=scalarMap.to_rgba(msstep), antialiased=True)
ax2.view_init(30, -60)
# ax.tick_params(labelsize=18)
ax.set_xlabel(predictor_labels[i0])
ax.set_ylabel(predictor_labels[i1])
ax2.set_xlabel(predictor_labels[i0])
ax2.set_ylabel(predictor_labels[i1])
if i == 0:
col, row = j % c, j // c
fig.text(0.36 + col * 0.5, 0.96 - row * 0.5, titles[j], horizontalalignment='center')
fig2.text(0.36 + col * 0.5, 0.96 - row * 0.5, titles[j]+' (R2=%.1f%%)'%R2, horizontalalignment='center')
# ax.set_xbound(xmin, xmax)
# ax.set_ybound(ymin, ymax)
plt.tight_layout()
plt.subplots_adjust(top=0.94, hspace=0.3, wspace=0.25)
plt.show()
elif mode == 'gpr':
logfile, X, yc, yr, labels = data[0]
pairs = (
(0, 1),
(0, 2),
(1, 2),
# (0,3),(1,3),(2,3),
)
for pair in pairs:
xmin, xmax = np.min(X[:, pair[0]]), np.max(X[:, pair[0]])
ymin, ymax = np.min(X[:, pair[1]]), np.max(X[:, pair[1]])
xx, yy = np.meshgrid(np.linspace(xmin, xmax, 50), np.linspace(ymin, ymax, 50))
kernel = 0.01 * RBF(length_scale=((xmax - xmin) * 2, (ymax - ymin) * 2))
if False:
y = yc
# fit hyper parameters
kernel += 0.1 * WhiteKernel(noise_level=0.001)
gpc = GaussianProcessClassifier(kernel=kernel, warm_start=True).fit(X[:, pair], yc)
# hyper parameter results
res = gpc.kernel_, gpc.log_marginal_likelihood(gpc.kernel_.theta)
# classify on each grid point
P = gpc.predict_proba(np.vstack((xx.ravel(), yy.ravel())).T)[:, 1]
else:
y = yr
# fit hyper parameters
kernel += 4.0 * WhiteKernel(noise_level=4.0)
gpr = GaussianProcessRegressor(kernel=kernel, alpha=0, normalize_y=True).fit(X[:, pair], yr)
# hyper parameter results
res = gpr.kernel_, gpr.log_marginal_likelihood(gpr.kernel_.theta)
# regress on each grid point
P = gpr.predict(np.vstack((xx.ravel(), yy.ravel())).T)
P = P.reshape(xx.shape)
# plot classifier output
fig = plt.figure(figsize=FIG_SIZE)
if True:
print('%s' % ((np.min(P), np.max(P), np.min(y), np.max(y)),))
image = plt.imshow(P, interpolation='nearest', extent=(xmin, xmax, ymin, ymax),
aspect='auto', origin='lower', cmap=plt.cm.PuOr_r)
plt.scatter(X[:, pair[0]], X[:, pair[1]], s=30, c=y, cmap=plt.cm.Paired, edgecolors=(0, 0, 0))
cb = plt.colorbar(image)
ax = fig.gca()
else:
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import Normalize
ax = fig.gca(projection='3d')
scalarMap = plt.cm.ScalarMappable(norm=Normalize(vmin=np.min(P), vmax=np.max(P)),
cmap=plt.cm.PuOr_r)
ax.plot_surface(xx, yy, P, rstride=1, cstride=1, facecolors=scalarMap.to_rgba(P), antialiased=True)
# cb.ax.tick_params(labelsize=18)
# ax.tick_params(labelsize=18)
plt.xlabel(predictors[pair[0]])
plt.ylabel(predictors[pair[1]])
plt.axis([xmin, xmax, ymin, ymax])
# plt.title("%s\n Log-Marginal-Likelihood:%.3f" % res, fontsize=12)
plt.tight_layout()
plt.show()
elif mode == '3d':
from mpl_toolkits.mplot3d import Axes3D # noqa: F401 unused import
logfile, X, yc, yr, labels = data[0]
xmin, xmax = np.min(X[:, 0]), np.max(X[:, 0])
ymin, ymax = np.min(X[:, 1]), np.max(X[:, 1])
zmin, zmax = np.min(X[:, 2]), np.max(X[:, 2])
fig = plt.figure(figsize=FIG_SIZE)
ax = fig.add_subplot(111, projection='3d')
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=yr, cmap=plt.cm.Paired, edgecolors=(0, 0, 0))
# cb = plt.colorbar(image)
# cb.ax.tick_params(labelsize=18)
# ax.tick_params(labelsize=18)
ax.set_xlabel(predictors[0])
ax.set_ylabel(predictors[1])
ax.set_zlabel(predictors[2])
ax.set_xbound(xmin, xmax)
ax.set_ybound(ymin, ymax)
ax.set_zbound(zmin, zmax)
plt.tight_layout()
plt.show()
else:
assert False, 'wrong mode'
clf = GaussianProcessClassifier(kernel=kernel, warm_start=True).fit(X, Y)
# plot the decision function for each datapoint on the grid
Z = clf.predict_proba(np.vstack((xx.ravel(), yy.ravel())).T)[:, 1]
Z = Z.reshape(xx.shape)
plt.subplot(1, 2, i + 1)
image = plt.imshow(Z,
interpolation='nearest',
extent=(xx.min(), xx.max(), yy.min(), yy.max()),
aspect='auto',
origin='lower',
cmap=plt.cm.PuOr_r)
contours = plt.contour(xx, yy, Z, levels=[0.5], linewidths=2, colors=['k'])
plt.scatter(X[:, 0],
X[:, 1],
s=30,
c=Y,
cmap=plt.cm.Paired,
edgecolors=(0, 0, 0))
plt.xticks(())
plt.yticks(())
plt.axis([-3, 3, -3, 3])
plt.colorbar(image)
plt.title("%s\n Log-Marginal-Likelihood:%.3f" %
(clf.kernel_, clf.log_marginal_likelihood(clf.kernel_.theta)),
fontsize=12)
plt.tight_layout()
plt.show()
# Generate data
train_size = 50
rng = np.random.RandomState(0)
X = rng.uniform(0, 5, 100)[:, np.newaxis]
y = np.array(X[:, 0] > 2.5, dtype=int)
# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_fix = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0),
optimizer=None)
gp_fix.fit(X[:train_size], y[:train_size])
gp_opt = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0))
gp_opt.fit(X[:train_size], y[:train_size])
print("Log Marginal Likelihood (initial): %.3f" %
gp_fix.log_marginal_likelihood(gp_fix.kernel_.theta))
print("Log Marginal Likelihood (optimized): %.3f" %
gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
print("Accuracy: %.3f (initial) %.3f (optimized)" %
(accuracy_score(y[:train_size], gp_fix.predict(X[:train_size])),
accuracy_score(y[:train_size], gp_opt.predict(X[:train_size]))))
print("Log-loss: %.3f (initial) %.3f (optimized)" %
(log_loss(y[:train_size],
gp_fix.predict_proba(X[:train_size])[:, 1]),
log_loss(y[:train_size],
gp_opt.predict_proba(X[:train_size])[:, 1])))
# Plot posteriors
plt.figure()
plt.scatter(X[:train_size, 0],
def test_lml_precomputed():
# Test that lml of optimized kernel is stored correctly.
for kernel in kernels:
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_almost_equal(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(), 7)
elif 2 < pts[0] < 3 and 0 < pts[1] < 1:
training_Y.append(1)
elif 4 < pts[0] < 5 and 2 < pts[1] < 3:
training_Y.append(1)
else:
training_Y.append(0)
# for idx, pt in enumerate(training_X):
# print("vertex ({},{}), the occupancy is {}".format(pt[0],pt[1],training_Y[idx]))
# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_opt = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1))
gp_opt.fit(training_X, training_Y)
print("The trained hyperparameter are {}".format((gp_opt.kernel_.theta)))
print("Log Marginal Likelihood (optimized): %.3f" %
gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
# # Plot posteriors
# plt.figure(0)
# plt.scatter(X[:train_size][0], X[:train_size][1],y[:train_size], c='k', label="Train data",
# edgecolors=(0, 0, 0))
# plt.scatter(X[train_size:][0], X[train_size:][1],y[train_size:], c='g', label="Test data",
# edgecolors=(0, 0, 0))
# X1_ = np.linspace(1, 5, 5)
# X2_ = np.linspace(1, 5, 5)
# # print(X1_)
#
# X_ = np.asanyarray([[row,col] for row in X1_ for col in X2_])
#
#
#
# Generate data
train_size = 50
rng = np.random.RandomState(0)
X = rng.uniform(0, 5, 100)[:, np.newaxis]
y = np.array(X[:, 0] > 2.5, dtype=int)
# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_fix = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0),
optimizer=None)
gp_fix.fit(X[:train_size], y[:train_size])
gp_opt = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0))
gp_opt.fit(X[:train_size], y[:train_size])
print("Log Marginal Likelihood (initial): %.3f"
% gp_fix.log_marginal_likelihood(gp_fix.kernel_.theta))
print("Log Marginal Likelihood (optimized): %.3f"
% gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
print("Accuracy: %.3f (initial) %.3f (optimized)"
% (accuracy_score(y[:train_size], gp_fix.predict(X[:train_size])),
accuracy_score(y[:train_size], gp_opt.predict(X[:train_size]))))
print("Log-loss: %.3f (initial) %.3f (optimized)"
% (log_loss(y[:train_size], gp_fix.predict_proba(X[:train_size])[:, 1]),
log_loss(y[:train_size], gp_opt.predict_proba(X[:train_size])[:, 1])))
# Plot posteriors
plt.figure(0)
plt.scatter(X[:train_size, 0], y[:train_size], c='k', label="Train data",
edgecolors=(0, 0, 0))
def test_lml_precomputed(kernel):
# Test that lml of optimized kernel is stored correctly.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_almost_equal(
gpc.log_marginal_likelihood(gpc.kernel_.theta), gpc.log_marginal_likelihood(), 7
)
def plot(df, options):
UNIQ_GROUPS = df.group.unique()
UNIQ_GROUPS.sort()
sns.set_style("white")
grppal = sns.color_palette("Set2", len(UNIQ_GROUPS))
print '# UNIQ GROUPS', UNIQ_GROUPS
cent_stats = df.groupby(
['position', 'group', 'side']).apply(stats_per_group)
cent_stats.reset_index(inplace=True)
import time
from sklearn import preprocessing
from sklearn.gaussian_process import GaussianProcessRegressor, GaussianProcessClassifier
from sklearn.gaussian_process.kernels import Matern, WhiteKernel, ExpSineSquared, ConstantKernel, RBF
ctlDF = cent_stats[ cent_stats['group'] == 0 ]
TNRightDF = cent_stats[ cent_stats['group'] != 0]
TNRightDF = TNRightDF[TNRightDF['side'] == 'right']
dataDf = pd.concat([ctlDF, TNRightDF], ignore_index=True)
print dataDf
yDf = dataDf['group'] == 0
yDf = yDf.astype(int)
y = yDf.values
print y
print y.shape
XDf = dataDf[['position', 'values']]
X = XDf.values
X = preprocessing.scale(X)
print X
print X.shape
# kernel = ConstantKernel() + Matern(length_scale=mean, nu=3 / 2) + \
# WhiteKernel(noise_level=1e-10)
kernel = 1**2 * Matern(length_scale=1, nu=1.5) + \
WhiteKernel(noise_level=0.1)
figure = plt.figure(figsize=(10, 6))
stime = time.time()
gp = GaussianProcessClassifier(kernel)
gp.fit(X, y)
print gp.kernel_
print gp.log_marginal_likelihood()
print("Time for GPR fitting: %.3f" % (time.time() - stime))
# create a mesh to plot in
h = 0.1
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h))
plt.figure(figsize=(10, 5))
# Plot the predicted probabilities. For that, we will assign a color to
# each point in the mesh [x_min, m_max]x[y_min, y_max].
Z = gp.predict_proba(np.c_[xx.ravel(), yy.ravel()])
Z = Z[:,1]
print Z
print Z.shape
# Put the result into a color plot
Z = Z.reshape((xx.shape[0], xx.shape[1]))
print Z.shape
plt.imshow(Z, extent=(x_min, x_max, y_min, y_max), origin="lower")
# Plot also the training points
plt.scatter(X[:, 0], X[:, 1], c=np.array(["r", "g"])[y])
plt.xlabel('position')
plt.ylabel('normalized val')
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.xticks(())
plt.yticks(())
plt.title("%s, LML: %.3f" %
("TN vs. Control", gp.log_marginal_likelihood(gp.kernel_.theta)))
plt.tight_layout()
if options.title:
plt.suptitle(options.title)
if options.output:
plt.savefig(options.output, dpi=150)
if options.is_show:
plt.show()
# Generate data
train_size = 50
rng = np.random.RandomState(0)
X = rng.uniform(0, 5, 100)[:, np.newaxis]
y = np.array(X[:, 0] > 2.5, dtype=int)
# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_fix = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0), optimizer=None)
gp_fix.fit(X[:train_size], y[:train_size])
gp_opt = GaussianProcessClassifier(kernel=1.0 * RBF(length_scale=1.0))
gp_opt.fit(X[:train_size], y[:train_size])
print("Log Marginal Likelihood (initial): %.3f" % gp_fix.log_marginal_likelihood(gp_fix.kernel_.theta))
print("Log Marginal Likelihood (optimized): %.3f" % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
print(
"Accuracy: %.3f (initial) %.3f (optimized)"
% (
accuracy_score(y[:train_size], gp_fix.predict(X[:train_size])),
accuracy_score(y[:train_size], gp_opt.predict(X[:train_size])),
)
)
print(
"Log-loss: %.3f (initial) %.3f (optimized)"
% (
log_loss(y[:train_size], gp_fix.predict_proba(X[:train_size])[:, 1]),
log_loss(y[:train_size], gp_opt.predict_proba(X[:train_size])[:, 1]),
)
xmin, xmax = np.min(X[:, pair[0]]), np.max(X[:, pair[0]])
ymin, ymax = np.min(X[:, pair[1]]), np.max(X[:, pair[1]])
xx, yy = np.meshgrid(np.linspace(xmin, xmax, 50),
np.linspace(ymin, ymax, 50))
kernel = 0.01 * RBF(length_scale=((xmax - xmin) * 2,
(ymax - ymin) * 2))
if False:
y = yc
# fit hyper parameters
kernel += 0.1 * WhiteKernel(noise_level=0.001)
gpc = GaussianProcessClassifier(kernel=kernel,
warm_start=True).fit(
X[:, pair], yc)
# hyper parameter results
res = gpc.kernel_, gpc.log_marginal_likelihood(
gpc.kernel_.theta)
# classify on each grid point
P = gpc.predict_proba(np.vstack((xx.ravel(), yy.ravel())).T)[:,
1]
else:
y = yr
# fit hyper parameters
kernel += 4.0 * WhiteKernel(noise_level=4.0)
gpr = GaussianProcessRegressor(kernel=kernel,
alpha=0,
normalize_y=True).fit(
X[:, pair], yr)
# hyper parameter results
res = gpr.kernel_, gpr.log_marginal_likelihood(
gpr.kernel_.theta)
# regress on each grid point
def test_lml_improving(kernel):
# Test that hyperparameter-tuning improves log-marginal likelihood.
gpc = GaussianProcessClassifier(kernel=kernel).fit(X, y)
assert_greater(gpc.log_marginal_likelihood(gpc.kernel_.theta),
gpc.log_marginal_likelihood(kernel.theta))
class GPR_kernelfit(object):
'''
Class that is used for fitting Gaussian Kernels.
This uses an implementation from Scikit that uses the
Laplace approximation to expand around the posterior
'''
X = [] # x-data
y = [] # y-data
kernel = 0.1 * RBF([25.0]) # Implicitly sets the initial values
fit = 0
def __init__(self, x, y, kernel=0):
'''
Constructor
'''
self.X = x
self.y = y
if kernel != 0:
self.kernel = kernel
def run_fit(self):
'''Runs a fit for the Gaussian Process Classifier'''
print("Starting Run...")
print(self.X)
print(self.y)
start = time()
self.fit = GaussianProcessClassifier(kernel=self.kernel).fit(
self.X, self.y)
print("Run time: %.4f" % (time() - start))
print("Theta-Estimates: ")
print(self.fit.kernel_)
print(self.fit.kernel_.theta)
print("Finished Run!!")
def show_fit(self):
'''Visualizes the fit'''
# create a mesh to plot in
h = 5
x_min, x_max = self.X[:, 0].min() - 1, self.X[:, 0].max() + 1
y_min, y_max = self.X[:, 1].min() - 1, self.X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h))
# Put the result into a color plot
Z = self.fit.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]
Z = Z.reshape(xx.shape)
plt.figure()
plt.imshow(Z, extent=(x_min, x_max, y_min, y_max), origin="lower")
image = plt.imshow(Z,
interpolation='nearest',
extent=(xx.min(), xx.max(), yy.min(), yy.max()),
aspect='auto',
origin='lower',
cmap=plt.cm.PuOr_r) # @UndefinedVariable
contours = plt.contour(
xx,
yy,
Z,
levels=[0],
linewidths=2, # @UnusedVariable
linetypes='--')
plt.colorbar(image)
# Plot also the training points
plt.scatter(self.X[:, 0],
self.X[:, 1],
s=30,
c=self.y,
cmap=plt.cm.Paired) # @UndefinedVariable
plt.xlabel('x-axis')
plt.ylabel('y-axis')
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.xticks(())
plt.yticks(())
plt.title("%s, LML: %.3f" %
("Log Likelihood Surface",
self.fit.log_marginal_likelihood(self.fit.kernel_.theta)))
plt.show()
def plot_loglike_sf(self):
'''Plots the log likelihood surface'''
theta0 = np.logspace(-2, 0, 20) # Manually set values
theta1 = np.logspace(0, 2, 20) # Manually set values
Theta0, Theta1 = np.meshgrid(theta0, theta1)
LML = [[
self.fit.log_marginal_likelihood(
np.log([Theta0[i, j], Theta1[i, j]]))
for i in range(Theta0.shape[0])
] for j in range(Theta0.shape[1])]
LML = np.array(LML).T
plt.plot(np.exp(self.fit.kernel_.theta)[0],
np.exp(self.fit.kernel_.theta)[1],
'ko',
zorder=10)
plt.plot(np.exp(self.fit.kernel_.theta)[0],
np.exp(self.fit.kernel_.theta)[1],
'ko',
zorder=10)
plt.pcolor(Theta0, Theta1, LML)
plt.xscale("log")
plt.yscale("log")
plt.colorbar()
plt.xlabel("Magnitude")
plt.ylabel("Length-scale")
plt.title("Log-marginal-likelihood")
plt.show()
from sklearn.gaussian_process.kernels import Matern
# This file is used to test can GP represent certain occupancy grid map with given sensors position and measurements
x = np.arange(0.5,10.5,1.0)
y = np.arange(0.5,10.5,1.0)
training_X = [[0.5,6.5],[1.5,7.5],[1.5,6.5],[1.5,5.5],[2.5,6.5],[2.5,4.5],[3.5,5.5],[3.5,4.5],[3.5,3.5],[4.5,4.5]]
training_Y = np.array([1,1,1,1,0,0,0,0,1,0])
# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_opt = GaussianProcessClassifier(RBF(length_scale=1.0))
gp_opt.fit(training_X,training_Y)
print("The trained hyperparameter are {}".format((gp_opt.kernel_.theta)))
print("===========================================")
lml,gradient = gp_opt.log_marginal_likelihood(theta=[0.44992595],eval_gradient=True)
print("The log likelihood is {}".format(lml))
print("The gradient is {}".format(gradient))
# print("Log Marginal Likelihood (optimized): %.3f"
# % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
fig = plt.figure()
ZZ = np.empty([10,10])
for idx1, row in enumerate(x):
for idx2,col in enumerate(y):
K = [row,col]
# print("({},{})".format(row,col))
# print("({},{})".format(idx1,idx2))
p_occ = gp_opt.predict_proba(np.reshape(K,(-1,2)))[:,1]
