def get_static_surrogate(D):
N = 200
X = np.random.randn(N, D)
est = KernelExpLiteGaussian(sigma=1, lmbda=.1, D=D, N=N)
est.fit(X)
return est
def get_static_surrogate(D):
N = 200
X = np.random.randn(N, D)
est = KernelExpLiteGaussian(sigma=1, lmbda=.1, D=D, N=N)
est.fit(X)
return est
def test_third_order_derivative_tensor_execute():
if not theano_available:
raise SkipTest("Theano not available.")
sigma = 1.
lmbda = 1.
N = 100
D = 2
X = np.random.randn(N, D)
est = KernelExpLiteGaussian(sigma, lmbda, D, N)
est.fit(X)
est.third_order_derivative_tensor(X[0])
def test_hessian_execute():
if not theano_available:
raise SkipTest("Theano not available.")
sigma = 1.
lmbda = 1.
N = 100
D = 2
X = np.random.randn(N, D)
est = KernelExpLiteGaussian(sigma, lmbda, D, N)
est.fit(X)
est.hessian(X[0])
class KernelExpLiteGaussianSurrogate(StaticSurrogate):
def __init__(self, ndim, sigma, lmbda, N):
self.surrogate = KernelExpLiteGaussian(sigma=sigma,
lmbda=lmbda,
N=N,
D=ndim)
def train(self, samples):
self.surrogate.fit(samples)
def log_pdf_gradient(self, x):
return self.surrogate.grad(x)
def get_kmc_static_kernel():
num_steps_min, num_steps_max, step_size_min, step_size_max = get_hmc_parameters()
target, momentum = get_target_momentum()
N = 200
X = np.random.randn(N, momentum.D)
est = KernelExpLiteGaussian(sigma=1, lmbda=.1, D=momentum.D, N=N)
est.fit(X)
surrogate = get_static_surrogate(momentum.D)
kmc = KMCStatic(surrogate, target, momentum, num_steps_min, num_steps_max, step_size_min, step_size_max)
return kmc
def __init__(self,
sigma,
lmbda,
D,
N,
eta=0.1,
cg_tol=1e-3,
cg_maxiter=None):
KernelExpLiteGaussian.__init__(self, sigma, lmbda, D, N)
self.eta = eta
self.cg_tol = cg_tol
self.cg_maxiter = cg_maxiter
def get_kmc_static_kernel():
num_steps_min, num_steps_max, step_size_min, step_size_max = get_hmc_parameters(
)
target, momentum = get_target_momentum()
N = 200
X = np.random.randn(N, momentum.D)
est = KernelExpLiteGaussian(sigma=1, lmbda=.1, D=momentum.D, N=N)
est.fit(X)
surrogate = get_static_surrogate(momentum.D)
kmc = KMCStatic(surrogate, target, momentum, num_steps_min, num_steps_max,
step_size_min, step_size_max)
return kmc
def test_third_order_derivative_tensor_execute():
if not theano_available:
raise SkipTest("Theano not available.")
sigma = 1.
lmbda = 1.
N = 100
D = 2
X = np.random.randn(N, D)
est = KernelExpLiteGaussian(sigma, lmbda, D, N)
est.fit(X)
est.third_order_derivative_tensor(X[0])
def test_hessian_execute():
if not theano_available:
raise SkipTest("Theano not available.")
sigma = 1.
lmbda = 1.
N = 100
D = 2
X = np.random.randn(N, D)
est = KernelExpLiteGaussian(sigma, lmbda, D, N)
est.fit(X)
est.hessian(X[0])
estimator, based on a Bayesian optimisation black-box optimiser.
Note that this optimiser can be "hot-started", i.e. it can be reset, but using
the previous model as initialiser for the new optimisation, which is useful
when the objective function changes slightly, e.g. when a new data was added
to the kernel exponential family model.
"""
N = 200
D = 2
# fit model to samples from a standard Gaussian
X = np.random.randn(N, D)
# use any of the below models, might have to change parameter bounds
estimators = [
KernelExpFiniteGaussian(sigma=1., lmbda=1., m=N, D=D),
KernelExpLiteGaussian(sigma=1., lmbda=.001, D=D, N=N),
]
for est in estimators:
print(est.__class__.__name__)
est.fit(X)
# specify bounds of parameters to search for
param_bounds = {
# 'lmbda': [-5,0], # fixed lmbda, uncomment to include in search
'sigma': [-2, 3],
}
# oop interface for optimising and using results
# objective is not put through log here, if it is, might want to bound away from zero
def _instantiate(self, D):
return KernelExpLiteGaussian(self.sigma, self.lmbda, D, self.m,
reg_f_norm=True, reg_alpha_norm=True)
def __init__(self, ndim, sigma, lmbda, N):
self.surrogate = KernelExpLiteGaussian(sigma=sigma,
lmbda=lmbda,
N=N,
D=ndim)
# for D=2, the fitted log-density is plotted, otherwise trajectory only
D = 2
N = 1000
# target is banana density, fallback to Gaussian if theano is not present
if banana_available:
target = Banana(D=D)
X = sample_banana(N, D)
else:
target = IsotropicZeroMeanGaussian(D=D)
X = sample_gaussian(N=N)
# plot trajectories for both KMC lite and finite, parameters are chosen for D=2
for surrogate in [
KernelExpFiniteGaussian(sigma=2, lmbda=0.001, m=N, D=D),
KernelExpLiteGaussian(sigma=20., lmbda=0.001, D=D, N=N),
KernelExpLiteGaussianLowRank(sigma=20,
lmbda=0.1,
D=D,
N=N,
cg_tol=0.01),
]:
# try uncommenting this line to illustrate KMC's ability to mix even
# when no (or incomplete) samples from the target are available
surrogate.fit(X)
# HMC parameters, fixed here, use oracle mean variance to set momentum
momentum = IsotropicZeroMeanGaussian(D=D,
sigma=np.sqrt(
np.mean(np.var(X, 0))))
num_steps_min = 10
# fit model to samples from a standard Gaussian
X = np.random.randn(N, D)
# create grid over sigma parameters, fixed regulariser
log_sigmas = np.linspace(-5, 10, 20)
lmbda = 0.001
# evaluate objective function over all those parameters
O = np.zeros(len(log_sigmas))
O_lower = np.zeros(len(log_sigmas))
O_upper = np.zeros(len(log_sigmas))
# grid search
for i, sigma in enumerate(log_sigmas):
est = KernelExpLiteGaussian(np.exp(sigma), lmbda, D, N)
# this is an array num_repetitions x num_folds, each containing a objective
xval_result = est.xvalidate_objective(X, num_folds=5, num_repetitions=2)
O[i] = np.mean(xval_result)
O_lower[i] = np.percentile(xval_result, 10)
O_upper[i] = np.percentile(xval_result, 90)
# best parameter
best_log_sigma = log_sigmas[np.argmin(O)]
# visualisation
plt.figure()
plt.plot([best_log_sigma, best_log_sigma], [np.min(O), np.max(O)], 'r')
plt.plot(log_sigmas, O, 'b-')
plt.plot(log_sigmas, O_lower, 'b--')
def get_KernelExpLiteGaussian_instance(D, N):
# arbitrary choice of parameters here
sigma = 1.
lmbda = 0.01
return KernelExpLiteGaussian(sigma, lmbda, D, N)
def __init__(self, sigma, lmbda, D, N, eta=0.1, cg_tol=1e-3, cg_maxiter=None):
KernelExpLiteGaussian.__init__(self, sigma, lmbda, D, N)
self.eta = eta
self.cg_tol = cg_tol
self.cg_maxiter = cg_maxiter
sigma = np.median(squareform(pdist(X))**2) / np.log(N + 1.0) * 2
M = 200
if N < M:
start_samples = np.tile(
X, [int(M / N) + 1, 1])[:M] + np.random.randn(M, 2) * 2
else:
start_samples = X[:M] + np.random.randn(M, 2) * 2
#start_samples[:, 0] *= 4; start_samples[:, 1] *= 2
# plot trajectories for both KMC lite and finite, parameters are chosen for D=2
results = []
num_steps = 2000
step_size = 0.1
for surrogate in [
KernelExpLiteGaussian(sigma=25 * sigma, lmbda=0.01, D=D, N=N),
KernelExpStein(sigma=16 * sigma, lmbda=0.01, D=D, N=N),
KernelExpSteinNonparam(sigma=9 * sigma, lmbda=0.01, D=D, N=N)
]:
surrogate.fit(X)
# HMC parameters
momentum = IsotropicZeroMeanGaussian(D=D, sigma=1.0)
# kmc sampler instance
kmc = KMCStatic(surrogate, target, momentum, num_steps, num_steps,
step_size, step_size)
# simulate trajectory from starting point, note _proposal_trajectory is a "hidden" method
Qs_total = []
acc_probs_total = []
def get_instace_KernelExpLiteGaussian(N):
sigma = 2.
lmbda = 1.
D = 2
return KernelExpLiteGaussian(sigma, lmbda, D, N)
Ns_fit = np.array([5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2000, 5000])
sigma = 1
lmbda = 0.01
grad = lambda x: est.grad(np.array([x]))[0]
s = GaussianQuadraticTest(grad)
num_bootstrap = 200
result_fname = os.path.splitext(os.path.basename(__file__))[0] + ".txt"
num_repetitions = 150
for _ in range(num_repetitions):
for N in Ns_fit:
est = KernelExpLiteGaussian(sigma, lmbda, D, N)
X_test = np.random.randn(N_test, D)
X = np.random.randn(N, D)
est.fit(X)
U_matrix, stat = s.get_statistic_multiple(X_test[:,0])
bootsraped_stats = np.empty(num_bootstrap)
for i in range(num_bootstrap):
W = np.sign(np.random.randn(N_test))
WW = np.outer(W, W)
st = np.mean(U_matrix * WW)
bootsraped_stats[i] = N_test * st
p_value = np.mean(bootsraped_stats>stat)
