def update_plot(val=None):
global omega, u
print("Updating plot")
lmbda = 2**s_lmbda.val
sigma = 2**s_sigma.val
b = compute_b(Z, omega, u)
C = compute_C(Z, omega, u)
theta = score_matching_sym(Z, lmbda, omega, u, b, C)
J = objective(Z, theta, lmbda, omega, u, b, C)
J_xval = np.mean(
xvalidate(Z, lmbda, omega, u, n_folds=5, num_repetitions=3))
logq_est = lambda x: np.dot(theta, feature_map_single(x, omega, u))
dlogq_est = lambda x: np.dot(theta, feature_map_grad_single(x, omega, u))
description = "N=%d, sigma: %.2f, lambda: %.2f, m=%.d, J=%.2f, J_xval=%.2f" % \
(N, sigma, lmbda, m, J, J_xval)
if plot_pdf:
D = evaluate_density_grid(Xs, Ys, logq_est)
description = "log-pdf: " + description
else:
D = evaluate_density_grad_grid(Xs, Ys, dlogq_est)
description = "norm-grad-log-pdf: " + description
ax.clear()
ax.plot(Z[:, 0], Z[:, 1], 'bx')
plot_array(Xs, Ys, D, ax, plot_contour=True)
ax.set_title(description)
fig.canvas.draw_idle()
def optimise_sigma_surface(val):
global gamma
print("sigma")
log2_lmbda = s_lmbda.val
lmbda = 2**log2_lmbda
log2_sigmas = np.linspace(s_sigma.valmin, s_sigma.valmax, 50)
Js = np.zeros(len(log2_sigmas))
for i, log2_sigma in enumerate(log2_sigmas):
s_sigma.val = log2_sigma
update_basis(val)
Js[i] = np.mean(
xvalidate(Z, lmbda, omega, u, n_folds=5, num_repetitions=3))
log2_sigma_min = log2_sigmas[Js.argmin()]
log_Js = np.log(Js - (Js.min() if Js.min() < 0 else 0) + 1)
# update slider
s_sigma.set_val(log2_sigma_min)
update_plot()
plt.figure()
plt.plot(log2_sigmas, log_Js)
plt.plot([log2_sigma_min, log2_sigma_min],
[log_Js.min(), log_Js.max()], 'r')
plt.title(
r"$\sigma$ surface for $\log_2 \lambda=%.2f$, best value of $J(\alpha)=%.2f$ at $\log_2 \sigma=%.2f$"
% (log2_lmbda, Js.min(), log2_sigma_min))
plt.show()
def plot_lmbda_surface(val):
print("lambda")
log2_sigma = s_sigma.val
log2_lambdas = np.linspace(s_lmbda.valmin, s_lmbda.valmax, 50)
Js = np.array([
np.mean(
xvalidate(Z, 2**log2_lmbda, omega, u, n_folds=5,
num_repetitions=3)) for log2_lmbda in log2_lambdas
])
log2_lambda_min = log2_lambdas[Js.argmin()]
log_Js = np.log(Js - (Js.min() if Js.min() < 0 else 0) + 1)
# update slider
s_lmbda.set_val(log2_lambda_min)
update_plot()
plt.figure()
plt.plot(log2_lambdas, log_Js)
plt.plot([log2_lambda_min, log2_lambda_min],
[log_Js.min(), log_Js.max()], 'r')
plt.title(
r"$\lambda$ surface for $\log_2 \sigma=%.2f$, best value of $J(\alpha)=%.2f$ at $\log_2 \lambda=%.2f$"
% (log2_sigma, Js.min(), log2_lambda_min))
plt.show()
def sigma_objective(log2_sigma, lmbda):
sigma = 2**log2_sigma
folds = np.zeros(num_repetitions)
for i in range(num_repetitions):
gamma = 0.5 * (sigma**2)
omega, u = sample_basis(D, m, gamma)
folds[i] = np.mean(
xvalidate(Z, lmbda, omega, u, num_folds, num_repetitions=1))
result = np.mean(folds)
logger.info("xvalidation, sigma: %.2f, lambda: %.2f, J=%.3f" % \
(sigma, lmbda, result))
return result
def lmbda_objective(log2_lmbda, sigma):
lmbda = 2**log2_lmbda
gamma = 0.5 * (sigma**2)
omega, u = sample_basis(D, m, gamma)
folds = xvalidate(Z,
lmbda,
omega,
u,
num_folds,
num_repetitions=num_repetitions)
result = np.mean(folds)
logger.info("xvalidation, sigma: %.2f, lambda: %.2f, J=%.3f" % \
(sigma, lmbda, result))
return result
def fun(sigma_lmbda, m=100, num_repetitions=3):
log2_sigma = sigma_lmbda[0]
log2_lmbda = sigma_lmbda[1]
sigma = 2**log2_sigma
gamma = 0.5 * (sigma**2)
lmbda = 2**log2_lmbda
omega, u = sample_basis(D, m, gamma)
folds = [xvalidate(Z, lmbda, omega, u) for _ in range(num_repetitions)]
J = np.mean(folds)
J_std = np.std(folds)
print("fun: log2_sigma=%.2f, log_lmbda=%.2f, J(a)=%.2f" %
(log2_sigma, log2_lmbda, J))
return J, J_std
def multicore_fun(log2_sigma, log2_lmbda, num_repetitions, num_folds, Z, m):
D = Z.shape[1]
lmbda = 2**log2_lmbda
sigma = 2**log2_sigma
gamma = 0.5 * (sigma**2)
folds = np.zeros(num_repetitions)
for j in range(num_repetitions):
logger.debug("xvalidation repetition %d/%d" % (j + 1, num_repetitions))
omega, u = sample_basis(D, m, gamma)
folds[j] = np.mean(
xvalidate(Z, lmbda, omega, u, num_folds, num_repetitions=1))
result = np.mean(folds)
logger.info("cma particle, sigma: %.2f, lambda: %.6f, J=%.4f" % \
(sigma, lmbda, result))
return result
def _f(log2_sigma):
sigma = 2**log2_sigma
folds = np.zeros(num_repetitions)
for i in range(num_repetitions):
gamma = 0.5 * (sigma**2)
omega, u = sample_basis(D, m, gamma)
folds[i] = np.mean(
xvalidate(Z, lmbda, omega, u, num_folds, num_repetitions=1))
result = np.mean(folds)
print("xvalidation, sigma: %.2f, lambda: %.6f, J=%.3f" % \
(sigma, lmbda, result))
# transform to log space to make GP work.
# since unbounded, add constant
# also pybo maximises so invert sign
result = -np.log(result + 100)
return result
def select_sigma_grid(Z,
m,
num_folds=5,
num_repetitions=3,
log2_sigma_min=-3,
log2_sigma_max=10,
resolution_sigma=25,
lmbda=0.0001,
plot_surface=False):
D = Z.shape[1]
sigmas = 2**np.linspace(log2_sigma_min, log2_sigma_max, resolution_sigma)
Js = np.zeros(len(sigmas))
for i, sigma in enumerate(sigmas):
gamma = 0.5 * (sigma**2)
folds = np.zeros(num_repetitions)
for j in range(num_repetitions):
# re-sample basis every repetition
omega, u = sample_basis(D, m, gamma)
folds[j] = xvalidate(Z,
lmbda,
omega,
u,
n_folds=num_folds,
num_repetitions=1)
Js[i] = np.mean(folds)
logger.info("fold %d/%d, sigma: %.2f, lambda: %.2f, J=%.3f" % \
(i + 1, len(sigmas), sigma, lmbda, Js[i]))
# if plot_surface:
# plt.figure()
# plt.plot(np.log2(sigmas), Js)
return sigmas[Js.argmin()]
logq = lambda x: log_banana_pdf(x, compute_grad=False)
dlogq = lambda x: log_banana_pdf(x, compute_grad=True)
# estimate density in rkhs
N = 200
mu = np.zeros(D)
Z = sample_banana(N, D)
lmbda = 1.
sigma = select_sigma_grid(Z, lmbda=lmbda)
K = gaussian_kernel(Z, sigma=sigma)
b = _compute_b_sym(Z, K, sigma)
C = _compute_C_sym(Z, K, sigma)
a = score_matching_sym(Z, sigma, lmbda, K, b, C)
J = _objective_sym(Z, sigma, lmbda, a, K, b, C)
J_xval = np.mean(xvalidate(Z, 5, sigma, lmbda, K))
print("N=%d, sigma: %.2f, lambda: %.2f, J(a)=%.2f, XJ(a)=%.2f" % \
(N, sigma, lmbda, J, J_xval))
kernel = lambda X, Y=None: gaussian_kernel(X, Y, sigma=sigma)
kernel_grad = lambda x, X=None: gaussian_kernel_grad(x, X, sigma)
logq_est = lambda x: log_pdf_estimate(x, a, Z, kernel)
dlogq_est = lambda x: log_pdf_estimate_grad(x, a, Z, kernel_grad)
# momentum
Sigma_p = np.eye(D) * .1
L_p = np.linalg.cholesky(Sigma_p)
logp = lambda x: log_gaussian_pdf(
x, Sigma=L_p, compute_grad=False, is_cholesky=True)
dlogp = lambda x: log_gaussian_pdf(
x, Sigma=L_p, compute_grad=True, is_cholesky=True)
N = 20
D = 2
Z = np.random.randn(N, D)
m = N
# print np.sum(Z) * np.std(Z) * np.sum(Z**2) * np.std(Z**2)
num_folds = 5
num_repetitions = 3
cma_opts = {'tolfun': 0.1, 'maxiter': 1, 'verb_disp': 1}
num_threads = 2
# D = 2
sigma0 = 0.51
lmbda0 = 0.0000081
sigma, lmbda, es = select_sigma_lambda_cma(Z,
m,
num_threads,
num_folds,
num_repetitions,
sigma0,
lmbda0,
cma_opts,
return_cma=True)
gamma = 0.5 * (sigma**2)
omega, u = sample_basis(D, m, gamma)
print(sigma, lmbda,
np.mean(xvalidate(Z, lmbda, omega, u, n_folds=5, num_repetitions=3)))