def __init__(self,
mu=array([0, 0]),
Sigma=eye(2),
is_cholesky=False,
ell=None):
Distribution.__init__(self, len(Sigma))
assert (len(shape(mu)) == 1)
assert (max(shape(Sigma)) == len(mu))
self.mu = mu
self.ell = ell
if is_cholesky:
self.L = Sigma
if ell == None:
assert (shape(Sigma)[0] == shape(Sigma)[1])
else:
assert (shape(Sigma)[1] == ell)
else:
assert (shape(Sigma)[0] == shape(Sigma)[1])
if ell is not None:
self.L, _, _ = MatrixTools.low_rank_approx(Sigma, ell)
self.L = self.L.T
assert (shape(self.L)[1] == ell)
else:
try:
self.L = cholesky(Sigma)
except LinAlgError:
# some really crude check for PSD (which only corrects for orunding errors
self.L = cholesky(Sigma + eye(len(Sigma)) * 1e-5)
def log_pdf(self, X):
assert (len(shape(X)) == 2)
assert (shape(X)[1] == self.dimension)
# compute all norms
norms = array([norm(x) for x in X])
# compute angles (second component first first)
angles = arctan2(X[:, 1], X[:, 0])
# gaussian parameters
mu = self.radius + self.amplitude * cos(self.frequency * angles)
log_pdf2d = zeros(len(X))
gaussian = Gaussian(array([mu[0]]), array([[self.variance]]))
for i in range(len(X)):
gaussian.mu = mu[i]
log_pdf2d[i] = gaussian.log_pdf(array([[norms[i]]]))
if self.dimension > 2:
remain_dims = Gaussian(zeros([self.dimension - 2]),
eye(self.dimension - 2))
log_pdfoverall = log_pdf2d + remain_dims.log_pdf(
X[:, 2:self.dimension])
else:
log_pdfoverall = log_pdf2d
return log_pdfoverall
def find_mode_newton(self, return_full=False):
"""
Newton search for mode of p(y|f)p(f)
from GP book, algorithm 3.1, added step size
"""
K = self.gp.K
if self.newton_start is None:
f = zeros(len(K))
else:
f = self.newton_start
if return_full:
steps = [f]
iteration = 0
norm_difference = inf
objective_value = -inf
while iteration < self.newton_max_iterations and norm_difference > self.newton_epsilon:
# from GP book, algorithm 3.1, added step size
# scale log_lik_grad_vector and K^-1 f = a
w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f)
w_sqrt = sqrt(w)
# diag(w_sqrt).dot(K.dot(diag(w_sqrt))) == (K.T*w_sqrt).T*w_sqrt
L = cholesky(eye(len(K)) + (K.T * w_sqrt).T * w_sqrt)
b = f * w + self.newton_step * self.gp.likelihood.log_lik_grad_vector(self.gp.y, f)
# a=b-diag(w_sqrt).dot(inv(eye(len(K)) + (K.T*w_sqrt).T*w_sqrt).dot(diag(w_sqrt).dot(K.dot(b))))
a = w_sqrt * (K.dot(b))
a = solve_triangular(L, a, lower=True)
a = solve_triangular(L.T, a, lower=False)
a = w_sqrt * a
a = b - a
f_new = K.dot(self.newton_step * a)
# convergence stuff and next iteration
objective_value_new = -0.5 * a.T.dot(f) + sum(self.gp.likelihood.log_lik_vector(self.gp.y, f))
norm_difference = norm(f - f_new)
if objective_value_new > objective_value:
f = f_new
if return_full:
steps.append(f)
else:
self.newton_step /= 2
iteration += 1
objective_value = objective_value_new
self.computed = True
if return_full:
return f, L, asarray(steps)
else:
return f
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
posterior = OzonePosterior(prior, logdet_alg="scikits",
solve_method="scikits")
proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=5000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_average_serial"])
store_chain_output = StoreChainOutput(folder)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
posterior = OzonePosterior(prior,
logdet_alg="scikits",
solve_method="scikits")
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=5000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_average_serial"])
store_chain_output = StoreChainOutput(folder)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def log_pdf(self, X):
assert(len(shape(X)) == 2)
assert(shape(X)[1] == self.dimension)
transformed = X.copy()
transformed[:, 1] = X[:, 1] - self.bananicity * ((X[:, 0] ** 2) - self.V)
transformed[:, 0] = X[:, 0] / sqrt(self.V)
phi = Gaussian(zeros([self.dimension]), eye(self.dimension))
return phi.log_pdf(transformed)
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)):
norms = array([norm(x) for x in X])
angles = arctan2(X[:, 1], X[:, 0])
mu = self.radius + self.amplitude * cos(self.frequency * angles)
transformed = hstack((array([norms - mu]).T, X[:, 2:self.dimension]))
cov = eye(self.dimension - 1)
cov[0, 0] = self.variance
gaussian = Gaussian(zeros([self.dimension - 1]), cov)
return gaussian.emp_quantiles(transformed)
def log_pdf(self, X):
assert (len(shape(X)) == 2)
assert (shape(X)[1] == self.dimension)
transformed = X.copy()
transformed[:, 1] = X[:, 1] - self.bananicity * ((X[:, 0]**2) - self.V)
transformed[:, 0] = X[:, 0] / sqrt(self.V)
phi = Gaussian(zeros([self.dimension]), eye(self.dimension))
return phi.log_pdf(transformed)
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)):
norms = array([norm(x) for x in X])
angles = arctan2(X[:, 1], X[:, 0])
mu = self.radius + self.amplitude * cos(self.frequency * angles)
transformed = hstack((array([norms-mu]).T, X[:,2:self.dimension]))
cov=eye(self.dimension-1)
cov[0,0]=self.variance
gaussian=Gaussian(zeros([self.dimension-1]), cov)
return gaussian.emp_quantiles(transformed)
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 1000
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"])
# cluster admin set project jump for me to exclusively allocate nodes
parameter_prefix = "" # #$ -P jump"
cluster_parameters = BatchClusterParameters(
foldername=folder,
memory=7.8,
loglevel=logging.DEBUG,
parameter_prefix=parameter_prefix,
max_walltime=60 * 60 * 24 - 1)
computation_engine = SGEComputationEngine(cluster_parameters,
check_interval=10)
rr_instance = RussianRoulette(1e-3, block_size=400)
posterior = OzonePosteriorRREngine(rr_instance=rr_instance,
computation_engine=computation_engine,
num_estimates=num_estimates,
prior=prior)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.55, -10.1])
mcmc_params = MCMCParams(start=start, num_iterations=5000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
# chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1))
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
store_chain_output = StoreChainOutput(folder, lag=1)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def __init__(self, distribution, scale=None, cov=None):
MCMCSampler.__init__(self, distribution)
if scale is None:
self.scale = (2.38 ** 2) / distribution.dimension
else:
self.scale = scale
if cov is None:
self.cov = eye(distribution.dimension)
else:
self.cov = cov
def __init__(self, distribution, scale=None, cov=None):
MCMCSampler.__init__(self, distribution)
if scale is None:
self.scale = (2.38**2) / distribution.dimension
else:
self.scale = scale
if cov is None:
self.cov = eye(distribution.dimension)
else:
self.cov = cov
def main():
Log.set_loglevel(logging.DEBUG)
modulename = "sample_ozone_posterior_average_slurm"
if not FileSystem.cmd_exists("sbatch"):
engine = SerialComputationEngine()
else:
johns_slurm_hack = "#SBATCH --partition=intel-ivy,wrkstn,compute"
johns_slurm_hack = "#SBATCH --partition=intel-ivy,compute"
folder = os.sep + os.sep.join(["nfs", "data3", "ucabhst", modulename])
batch_parameters = BatchClusterParameters(
foldername=folder,
max_walltime=24 * 60 * 60,
resubmit_on_timeout=False,
memory=3,
parameter_prefix=johns_slurm_hack)
engine = SlurmComputationEngine(batch_parameters,
check_interval=1,
do_clean_up=True)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 100
posterior = OzonePosteriorAverageEngine(computation_engine=engine,
num_estimates=num_estimates,
prior=prior)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=2000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
home = expanduser("~")
folder = os.sep.join([home, modulename])
store_chain_output = StoreChainOutput(folder)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def __init__(self, mu=asarray([0, 0]), Sigma=eye(2), is_cholesky=False):
DensityFunction.__init__(self, len(Sigma))
assert(len(shape(mu)) == 1)
assert(max(shape(Sigma)) == len(mu))
self.mu = mu
if is_cholesky:
self.L = Sigma
else:
assert(shape(Sigma)[0] == shape(Sigma)[1])
self.L = cholesky(Sigma)
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 1000
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"])
# cluster admin set project jump for me to exclusively allocate nodes
parameter_prefix = "" # #$ -P jump"
cluster_parameters = BatchClusterParameters(foldername=folder,
memory=7.8,
loglevel=logging.DEBUG,
parameter_prefix=parameter_prefix,
max_walltime=60 * 60 * 24 - 1)
computation_engine = SGEComputationEngine(cluster_parameters, check_interval=10)
rr_instance = RussianRoulette(1e-3, block_size=400)
posterior = OzonePosteriorRREngine(rr_instance=rr_instance,
computation_engine=computation_engine,
num_estimates=num_estimates,
prior=prior)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.55, -10.1])
mcmc_params = MCMCParams(start=start, num_iterations=5000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
# chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1))
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
store_chain_output = StoreChainOutput(folder, lag=1)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def __init__(self, distribution, num_eigen=2, \
mean_est=array([-2.0, -2.0]), cov_est=0.05 * eye(2), \
sample_discard=500, sample_lag=10, accstar=0.234):
AdaptiveMetropolis.__init__(self, distribution=distribution, \
mean_est=mean_est, cov_est=cov_est, \
sample_discard=sample_discard, sample_lag=sample_lag, accstar=accstar)
assert (num_eigen <= distribution.dimension)
self.num_eigen = num_eigen
self.dwscale = self.globalscale * ones([self.num_eigen])
u, s, _ = svd(self.cov_est)
self.eigvalues = s[0:self.num_eigen]
self.eigvectors = u[:, 0:self.num_eigen]
def main():
Log.set_loglevel(logging.DEBUG)
modulename = "sample_ozone_posterior_average_slurm"
if not FileSystem.cmd_exists("sbatch"):
engine = SerialComputationEngine()
else:
johns_slurm_hack = "#SBATCH --partition=intel-ivy,wrkstn,compute"
johns_slurm_hack = "#SBATCH --partition=intel-ivy,compute"
folder = os.sep + os.sep.join(["nfs", "data3", "ucabhst", modulename])
batch_parameters = BatchClusterParameters(foldername=folder, max_walltime=24 * 60 * 60,
resubmit_on_timeout=False, memory=3,
parameter_prefix=johns_slurm_hack)
engine = SlurmComputationEngine(batch_parameters, check_interval=1,
do_clean_up=True)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 100
posterior = OzonePosteriorAverageEngine(computation_engine=engine,
num_estimates=num_estimates,
prior=prior)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=2000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
home = expanduser("~")
folder = os.sep.join([home, modulename])
store_chain_output = StoreChainOutput(folder)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def __init__(self, dimension=2, num_components=2, components=None, mixing_proportion=None):
Distribution.__init__(self, dimension)
self.num_components = num_components
if (components == None):
self.components = [Gaussian(mu=zeros(self.dimension),Sigma=eye(self.dimension)) for _ in range(self.num_components)]
else:
assert(len(components)==self.num_components)
self.components=components
if (mixing_proportion == None):
self.mixing_proportion=Discrete((1.0/num_components)*ones([num_components]))
else:
assert(num_components==mixing_proportion.num_objects)
self.mixing_proportion = mixing_proportion
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)):
assert(len(shape(X)) == 2)
assert(shape(X)[1] == self.dimension)
substract=self.bananicity * ((X[:, 0] ** 2) - self.V)
divide=sqrt(self.V)
X[:, 1] -= substract
X[:, 0] /= divide
phi = Gaussian(zeros([self.dimension]), eye(self.dimension))
quantiles=phi.emp_quantiles(X, quantiles)
# undo changes to X
X[:, 0] *= divide
X[:, 1] += substract
return quantiles
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)):
assert (len(shape(X)) == 2)
assert (shape(X)[1] == self.dimension)
substract = self.bananicity * ((X[:, 0]**2) - self.V)
divide = sqrt(self.V)
X[:, 1] -= substract
X[:, 0] /= divide
phi = Gaussian(zeros([self.dimension]), eye(self.dimension))
quantiles = phi.emp_quantiles(X, quantiles)
# undo changes to X
X[:, 0] *= divide
X[:, 1] += substract
return quantiles
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 2
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"])
computation_engine = SerialComputationEngine()
rr_instance = RussianRoulette(1e-3, block_size=10)
posterior = OzonePosteriorRREngine(rr_instance=rr_instance,
computation_engine=computation_engine,
num_estimates=num_estimates,
prior=prior)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=200)
chain = MCMCChain(mcmc_sampler, mcmc_params)
# chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1))
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
store_chain_output = StoreChainOutput(folder, lag=50)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 2
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"])
computation_engine = SerialComputationEngine()
rr_instance = RussianRoulette(1e-3, block_size=10)
posterior = OzonePosteriorRREngine(
rr_instance=rr_instance, computation_engine=computation_engine, num_estimates=num_estimates, prior=prior
)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=200)
chain = MCMCChain(mcmc_sampler, mcmc_params)
# chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1))
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
store_chain_output = StoreChainOutput(folder, lag=50)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def construct_proposal(self, y):
"""
proposal is a mixture of normals,
centred at y and with covariance gamma^2 I + nu^2 MHaa'HM',
where a are the eigenvectors of centred kernel matrix Kc=HKH
"""
assert (len(shape(y)) == 1)
m = MixtureDistribution(self.distribution.dimension, self.num_eigen)
m.mixing_proportion = Discrete(
(self.eigvalues + 1) / (sum(self.eigvalues) + self.num_eigen))
# print "current mixing proportion: ", m.mixing_proportion.omega
M = 2 * self.kernel.gradient(y, self.Z)
H = Kernel.centring_matrix(len(self.Z))
for ii in range(self.num_eigen):
Sigma = self.gamma ** 2 * eye(len(y)) + \
self.nu2 * (M.T).dot(H.dot(outer(self.eigvectors[:, ii], self.eigvectors[:, ii]).dot(H.dot(M))))
m.components[ii] = Gaussian(y, Sigma)
return m
def construct_proposal(self, y):
"""
proposal is a mixture of normals,
centred at y and with covariance gamma^2 I + nu^2 MHaa'HM',
where a are the eigenvectors of centred kernel matrix Kc=HKH
"""
assert len(shape(y)) == 1
m = MixtureDistribution(self.distribution.dimension, self.num_eigen)
m.mixing_proportion = Discrete((self.eigvalues + 1) / (sum(self.eigvalues) + self.num_eigen))
# print "current mixing proportion: ", m.mixing_proportion.omega
M = 2 * self.kernel.gradient(y, self.Z)
H = Kernel.centring_matrix(len(self.Z))
for ii in range(self.num_eigen):
Sigma = self.gamma ** 2 * eye(len(y)) + self.nu2 * (M.T).dot(
H.dot(outer(self.eigvectors[:, ii], self.eigvectors[:, ii]).dot(H.dot(M)))
)
m.components[ii] = Gaussian(y, Sigma)
return m
def TheLeastSquareMethod(X, Y):
"""
最小二乘法
"""
regula = eye(ORDER + 1)
X_matrix = []
for i in range(ORDER + 1):
X_matrix.append(array(X)**i)
X_matrix = mat(X_matrix).T
Y_matrix = array(Y).reshape((len(Y), 1))
X_matrix_T = X_matrix.T
#print dot(X_matrix_T,X_matrix)
B = dot(dot(dot(X_matrix_T, X_matrix).I, X_matrix_T), Y_matrix)
B1 = dot(dot((dot(X_matrix_T, X_matrix) + lamda * regula).I, X_matrix_T),
Y_matrix)
result = dot(X_matrix, B)
result_reg = dot(X_matrix, B1)
return X_matrix, Y_matrix, B, result, result_reg, B1
def __init__(self,
dimension=2,
num_components=2,
components=None,
mixing_proportion=None):
Distribution.__init__(self, dimension)
self.num_components = num_components
if (components == None):
self.components = [
Gaussian(mu=zeros(self.dimension), Sigma=eye(self.dimension))
for _ in range(self.num_components)
]
else:
assert (len(components) == self.num_components)
self.components = components
if (mixing_proportion == None):
self.mixing_proportion = Discrete(
(1.0 / num_components) * ones([num_components]))
else:
assert (num_components == mixing_proportion.num_objects)
self.mixing_proportion = mixing_proportion
def predict(self, X_test, f_mode=None):
"""
Predictions for GP with Laplace approximation.
from GP book, algorithm 3.2,
"""
if f_mode is None:
f_mode = self.find_mode_newton()
predictions = zeros(len(X_test))
K = self.gp.K
K_train_test = self.gp.covariance.compute(self.gp.X, X_test)
w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f_mode)
w_sqrt = sqrt(w)
# diag(w_sqrt).dot(K.dot(diag(w_sqrt))) == (K.T*w_sqrt).T*w_sqrt
L = cholesky(eye(len(K)) + (K.T * w_sqrt).T * w_sqrt)
# iterator for all testing points
for i in range(len(X_test)):
k = K_train_test[:, i]
k_self = self.gp.covariance.compute([X_test[i]], [X_test[i]])[0]
f_mean = k.dot(
self.gp.likelihood.log_lik_grad_vector(self.gp.y, f_mode))
v = solve_triangular(L, w_sqrt * k, lower=True)
f_var = k_self - v.T.dot(v)
predictions[i] = integrate.quad(
lambda x: norm.pdf(x, f_mean, f_var), -inf, inf)[0]
# # integrate over Gaussian using some crude numerical integration
# samples=randn(1000)*sqrt(f_var) + f_mean
#
# log_liks=self.gp.likelihood.log_lik_vector(1.0, samples)
# predictions[i]=1.0/len(samples)*GPTools.log_sum_exp(log_liks)
return predictions
def predict(self, X_test, f_mode=None):
"""
Predictions for GP with Laplace approximation.
from GP book, algorithm 3.2,
"""
if f_mode is None:
f_mode = self.find_mode_newton()
predictions = zeros(len(X_test))
K = self.gp.K
K_train_test = self.gp.covariance.compute(self.gp.X, X_test)
w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f_mode)
w_sqrt = sqrt(w)
# diag(w_sqrt).dot(K.dot(diag(w_sqrt))) == (K.T*w_sqrt).T*w_sqrt
L = cholesky(eye(len(K)) + (K.T * w_sqrt).T * w_sqrt)
# iterator for all testing points
for i in range(len(X_test)):
k = K_train_test[:, i]
k_self = self.gp.covariance.compute([X_test[i]], [X_test[i]])[0]
f_mean = k.dot(self.gp.likelihood.log_lik_grad_vector(self.gp.y, f_mode))
v = solve_triangular(L, w_sqrt * k, lower=True)
f_var = k_self - v.T.dot(v)
predictions[i] = integrate.quad(lambda x: norm.pdf(x, f_mean, f_var), -inf, inf)[0]
# # integrate over Gaussian using some crude numerical integration
# samples=randn(1000)*sqrt(f_var) + f_mean
#
# log_liks=self.gp.likelihood.log_lik_vector(1.0, samples)
# predictions[i]=1.0/len(samples)*GPTools.log_sum_exp(log_liks)
return predictions
def test_simple_array_operations(self):
a = array([[1.,2.],
[3.,4.]])
numpy.testing.assert_array_equal(a.transpose(), array([[1.,3.],
[2.,4.]]))
numpy.testing.assert_array_almost_equal(trace(a), 5)
inv_a = inv(a)
b = array([[-2.,1.],
[1.5,-.5]])
self.assertTrue(numpy.allclose(inv_a,b))
i = dot(a,inv_a)
numpy.testing.assert_array_almost_equal(i, eye(2), 1)
numpy.testing.assert_array_almost_equal(inv_a, b)
# system of linear equations
a = array([[3,2,-1],
[2,-2,4],
[-1,0.5,-1]])
b = array([1,-2,0])
c = solve(a,b)
d = dot(a,c)
numpy.testing.assert_array_almost_equal(b, d, 1)
a = array([[.8,.3],
[.2,.7]])
eigen_values, eigen_vectors = eig(a)
lambda_1 = eigen_values[0]
x_1 = eigen_vectors[:,0]
lambda_2 = eigen_values[1]
x_2 = eigen_vectors[:,1]
def log_pdf(self, X):
assert(len(shape(X)) == 2)
assert(shape(X)[1] == self.dimension)
# compute all norms
norms = array([norm(x) for x in X])
# compute angles (second component first first)
angles = arctan2(X[:, 1], X[:, 0])
# gaussian parameters
mu = self.radius + self.amplitude * cos(self.frequency * angles)
log_pdf2d = zeros(len(X))
gaussian = Gaussian(array([mu[0]]), array([[self.variance]]))
for i in range(len(X)):
gaussian.mu = mu[i]
log_pdf2d[i] = gaussian.log_pdf(array([[norms[i]]]))
if self.dimension>2:
remain_dims=Gaussian(zeros([self.dimension-2]), eye(self.dimension-2))
log_pdfoverall=log_pdf2d+remain_dims.log_pdf(X[:,2:self.dimension])
else:
log_pdfoverall=log_pdf2d
return log_pdfoverall
def find_mode_newton(self, return_full=False):
"""
Newton search for mode of p(y|f)p(f)
from GP book, algorithm 3.1, added step size
"""
K = self.gp.K
if self.newton_start is None:
f = zeros(len(K))
else:
f = self.newton_start
if return_full:
steps = [f]
iteration = 0
norm_difference = inf
objective_value = -inf
while iteration < self.newton_max_iterations and norm_difference > self.newton_epsilon:
# from GP book, algorithm 3.1, added step size
# scale log_lik_grad_vector and K^-1 f = a
w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f)
w_sqrt = sqrt(w)
# diag(w_sqrt).dot(K.dot(diag(w_sqrt))) == (K.T*w_sqrt).T*w_sqrt
L = cholesky(eye(len(K)) + (K.T * w_sqrt).T * w_sqrt)
b = f * w + self.newton_step * \
self.gp.likelihood.log_lik_grad_vector(self.gp.y, f)
# a=b-diag(w_sqrt).dot(inv(eye(len(K)) + (K.T*w_sqrt).T*w_sqrt).dot(diag(w_sqrt).dot(K.dot(b))))
a = (w_sqrt * (K.dot(b)))
a = solve_triangular(L, a, lower=True)
a = solve_triangular(L.T, a, lower=False)
a = w_sqrt * a
a = b - a
f_new = K.dot(self.newton_step * a)
# convergence stuff and next iteration
objective_value_new = -0.5 * a.T.dot(f) + \
sum(self.gp.likelihood.log_lik_vector(self.gp.y, f))
norm_difference = norm(f - f_new)
if objective_value_new > objective_value:
f = f_new
if return_full:
steps.append(f)
else:
self.newton_step /= 2
iteration += 1
objective_value = objective_value_new
self.computed = True
if return_full:
return f, L, asarray(steps)
else:
return f
# store index sets at source and target of every edge
index_sets={}
print "precomputing systems for messages from observed nodes"
graphlab_lines.lines.append(os.linesep + "# edges with observed targets")
for node, observation in observations.items():
for out_message in graph[node]:
edge=(out_message, node)
graphlab_lines.new_edge_observed_target(node, out_message)
data_source=data[edge][0]
data_target=data[edge][1]
Ks_chol, Is, Rs, Ws=incomplete_cholesky(data_source, kernel, eta)
Kt_chol, It, Rt, Wt=incomplete_cholesky(data_target, kernel, eta)
Qs,Rs,Ps=qr(Ws.dot(Ws.T)+Ks_chol+eye(shape(Ks_chol)[0])*reg_lambda, pivoting=True)
Qt,Rt,Pt=qr(Wt.dot(Wt.T)+Kt_chol+eye(shape(Kt_chol)[0])*reg_lambda, pivoting=True)
savetxt(graphlab_lines.add_edge(node, out_message,"Q_s"), Qs)
savetxt(graphlab_lines.add_edge(node, out_message,"R_s"), Rs)
savetxt(graphlab_lines.add_edge(node, out_message,"P_s"), Ps)
savetxt(graphlab_lines.add_edge(node, out_message,"Q_t"), Qt)
savetxt(graphlab_lines.add_edge(node, out_message,"R_t"), Rt)
savetxt(graphlab_lines.add_edge(node, out_message,"P_t"), Pt)
savetxt(graphlab_lines.add_edge(node, out_message,"W"), Ws.dot(Wt.T))
print "precomputing systems for messages from non-observed nodes"
graphlab_lines.lines.append(os.linesep + "# edges with non-observed targets")
for edge in edges:
#coding:utf-8 # ------------------------------------------------------------ # 简介 : 机器学习基础 numpy # # 更新 : 2015年1月7日@ # ------------------------------------------------------------ from numpy import random from numpy.matrixlib.defmatrix import mat from numpy.lib.twodim_base import eye #4*4随机数组 random.rand(4, 4) #数组转矩阵 randmat = mat(random.rand(4, 4)) print(randmat) #.I专为逆矩阵 print(randmat.I) randmatv = randmat.I #矩阵和逆矩阵相乘，对角为1 其他为0 -eye(4)为单位矩阵 相减后看到有误差 print(randmat * randmatv - eye(4))
experiment_dir = experiment_dir_base + os.sep + str(
os.path.abspath(sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep
print "running experiments", n, "times at base", experiment_dir
for i in range(n):
distribution = Ring()
mcmc_samplers = []
kernel = GaussianKernel(sigma=1)
# mcmc_samplers.append(KameleonWindow(distribution, kernel))
mcmc_samplers.append(KameleonWindowLearnScale(distribution, kernel))
mean_est = array([-2.0, -2.0])
cov_est = 0.05 * eye(2)
# mcmc_samplers.append(AdaptiveMetropolis(distribution, mean_est=mean_est, cov_est=cov_est))
mcmc_samplers.append(
AdaptiveMetropolisLearnScale(distribution,
mean_est=mean_est,
cov_est=cov_est))
num_eigen = 2
mcmc_samplers.append(
AdaptiveMetropolisPCA(distribution,
num_eigen=num_eigen,
mean_est=mean_est,
cov_est=cov_est))
mcmc_samplers.append(StandardMetropolis(distribution))
experiment_dir = experiment_dir_base + str(os.path.abspath(
sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep
print "running experiments", n, "times at base", experiment_dir
# load data
data, labels = GPData.get_glass_data()
# normalise and whiten dataset
data -= mean(data, 0)
L = cholesky(cov(data.T))
data = solve_triangular(L, data.T, lower=True).T
dim = shape(data)[1]
# prior on theta and posterior target estimate
theta_prior = Gaussian(mu=0 * ones(dim), Sigma=eye(dim) * 5)
distribution=PseudoMarginalHyperparameterDistribution(data, labels, \
n_importance=100, prior=theta_prior, \
ridge=1e-3)
sigma = 23.0
print "using sigma", sigma
kernel = GaussianKernel(sigma=sigma)
for i in range(n):
mcmc_samplers = []
burnin = 20000
num_iterations = 100000
# throw away some data
n=250
seed(1)
idx=permutation(len(data))
idx=idx[:n]
data=data[idx]
labels=labels[idx]
# normalise and whiten dataset
data-=mean(data, 0)
L=cholesky(cov(data.T))
data=solve_triangular(L, data.T, lower=True).T
dim=shape(data)[1]
# prior on theta and posterior target estimate
theta_prior=Gaussian(mu=0*ones(dim), Sigma=eye(dim)*5)
target=PseudoMarginalHyperparameterDistribution(data, labels, \
n_importance=100, prior=theta_prior, \
ridge=1e-3)
# create sampler
burnin=10000
num_iterations=burnin+300000
kernel = GaussianKernel(sigma=23.0)
sampler=KameleonWindowLearnScale(target, kernel, stop_adapt=burnin)
# sampler=AdaptiveMetropolisLearnScale(target)
# sampler=StandardMetropolis(target)
# posterior mode derived by initial tests
start=zeros(target.dimension)
params = MCMCParams(start=start, num_iterations=num_iterations, burnin=burnin)
__all__ = ['matrix_power', 'solve', 'tensorsolve', 'tensorinv', 'inv',
'cholesky', 'eigvals', 'eigvalsh', 'pinv', 'slogdet', 'det',
'svd', 'eig', 'eigh', 'lstsq', 'norm', 'qr', 'cond', 'matrix_rank',
'LinAlgError', 'multi_dot']; import functools
import operator
import warnings; from numpy.core import (
array, asarray, zeros, empty, empty_like, intc, single, double,
csingle, cdouble, inexact, complexfloating, newaxis, all, Inf, dot,
add, multiply, sqrt, fastCopyAndTranspose, sum, isfinite,
finfo, errstate, geterrobj, moveaxis, amin, amax, product, abs,
atleast_2d, intp, asanyarray, object_, matmul,
swapaxes, divide, count_nonzero, isnan, sign
); from numpy.core.multiarray import normalize_axis_index; from numpy.core.overrides import set_module; from numpy.core import overrides; from numpy.lib.twodim_base import triu, eye; from numpy.linalg import lapack_lite, _umath_linalg; array_function_dispatch = functools.partial(
overrides.array_function_dispatch, module='numpy.linalg'); _N = b'N'; _V = b'V'; _A = b'A'; _S = b'S'; _L = b'L' fortran_int = intc @set_module('numpy.linalg') class LinAlgError(Exception): def _determine_error_states(): errobj = geterrobj() bufsize = errobj[0] with errstate(invalid='call', over='ignore', divide='ignore', under='ignore'): invalid_call_errmask = geterrobj()[1] return [bufsize, invalid_call_errmask, None]; _linalg_error_extobj = _determine_error_states(); del _determine_error_states; def _raise_linalgerror_singular(err, flag): raise LinAlgError("Singular matrix"); def _raise_linalgerror_nonposdef(err, flag): raise LinAlgError("Matrix is not positive definite"); def _raise_linalgerror_eigenvalues_nonconvergence(err, flag): raise LinAlgError("Eigenvalues did not converge"); def _raise_linalgerror_svd_nonconvergence(err, flag): raise LinAlgError("SVD did not converge"); def _raise_linalgerror_lstsq(err, flag): raise LinAlgError("SVD did not converge in Linear Least Squares"); def get_linalg_error_extobj(callback): extobj = list(_linalg_error_extobj); extobj[2] = callback; return extobj; def _makearray(a): new = asarray(a); wrap = getattr(a, "__array_prepare__", new.__array_wrap__); return new, wrap; def isComplexType(t): return issubclass(t, complexfloating); _real_types_map = {single: single,; double: double,; csingle: single,; cdouble: double}; _complex_types_map = {single: csingle,; double: cdouble,; csingle: csingle,; cdouble: cdouble}; def _realType(t, default=double): return _real_types_map.get(t, default); def _complexType(t, default=cdouble): return _complex_types_map.get(t, default); def _linalgRealType(t): """Cast the type t to either double or cdouble."""; return double; def _commonType(*arrays): result_type = single; is_complex = False; for a in arrays: if issubclass(a.dtype.type, inexact): if isComplexType(a.dtype.type): is_complex = True; rt = _realType(a.dtype.type, default=None); if rt is None: raise TypeError("array type %s is unsupported in linalg" %; (a.dtype.name,)); else: rt = double; if rt is double: result_type = double; if is_complex: t = cdouble; result_type = _complex_types_map[result_type]; else: t = double; return t, result_type; _fastCT = fastCopyAndTranspose; def _to_native_byte_order(*arrays): ret = []; for arr in arrays: if arr.dtype.byteorder not in ('=', '|'): ret.append(asarray(arr, dtype=arr.dtype.newbyteorder('='))); else: ret.append(arr); if len(ret) == 1: return ret[0]; else: return ret; def _fastCopyAndTranspose(type, *arrays): cast_arrays = (); for a in arrays: if a.dtype.type is type: cast_arrays = cast_arrays + (_fastCT(a),); else: cast_arrays = cast_arrays + (_fastCT(a.astype(type)),); if len(cast_arrays) == 1: return cast_arrays[0]; else: return cast_arrays; def _assert_2d(*arrays): for a in arrays: if a.ndim != 2: raise LinAlgError('%d-dimensional array given. Array must be '; 'two-dimensional' % a.ndim); def _assert_stacked_2d(*arrays): for a in arrays: if a.ndim < 2: raise LinAlgError('%d-dimensional array given. Array must be '; 'at least two-dimensional' % a.ndim); def _assert_stacked_square(*arrays): for a in arrays: m, n = a.shape[-2:]; if m != n: raise LinAlgError('Last 2 dimensions of the array must be square'); def _assert_finite(*arrays): for a in arrays: if not isfinite(a).all(): raise LinAlgError("Array must not contain infs or NaNs"); def _is_empty_2d(arr): return arr.size == 0 and product(arr.shape[-2:]) == 0; def transpose(a): a, wrap = _makearray(a); b = asarray(b); an = a.ndim; if axes is not None: allaxes = list(range(0, an)); for k in axes: allaxes.remove(k); allaxes.insert(an, k); a = a.transpose(allaxes); oldshape = a.shape[-(an-b.ndim):]; prod = 1; for k in oldshape: prod *= k; a = a.reshape(-1, prod); b = b.ravel(); res = wrap(solve(a, b)); res.shape = oldshape; return res; def _solve_dispatcher(a, b): return (a, b); @array_function_dispatch(_solve_dispatcher); def solve(a, b): a, _ = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); b, wrap = _makearray(b); t, result_t = _commonType(a, b); if b.ndim == a.ndim - 1: gufunc = _umath_linalg.solve1; else: gufunc = _umath_linalg.solve; signature = 'DD->D' if isComplexType(t) else 'dd->d'; extobj = get_linalg_error_extobj(_raise_linalgerror_singular); r = gufunc(a, b, signature=signature, extobj=extobj); return wrap(r.astype(result_t, copy=False)); def _tensorinv_dispatcher(a, ind=None): return (a,); @array_function_dispatch(_tensorinv_dispatcher); def tensorinv(a, ind=2): a = asarray(a); oldshape = a.shape; prod = 1; if ind > 0: invshape = oldshape[ind:] + oldshape[:ind]; for k in oldshape[ind:]: prod *= k; else: raise ValueError("Invalid ind argument."); a = a.reshape(prod, -1); ia = inv(a); return ia.reshape(*invshape); def _unary_dispatcher(a): return (a,); @array_function_dispatch(_unary_dispatcher); def inv(a): a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); signature = 'D->D' if isComplexType(t) else 'd->d'; extobj = get_linalg_error_extobj(_raise_linalgerror_singular); ainv = _umath_linalg.inv(a, signature=signature, extobj=extobj); return wrap(ainv.astype(result_t, copy=False)); def _matrix_power_dispatcher(a, n): return (a,); @array_function_dispatch(_matrix_power_dispatcher); def matrix_power(a, n): a = asanyarray(a); _assert_stacked_2d(a); _assert_stacked_square(a); try: n = operator.index(n); except TypeError: raise TypeError("exponent must be an integer"); if a.dtype != object: fmatmul = matmul; elif a.ndim == 2: fmatmul = dot; else: raise NotImplementedError(; "matrix_power not supported for stacks of object arrays"); if n == 0: a = empty_like(a); a[...] = eye(a.shape[-2], dtype=a.dtype); return a; elif n < 0: a = inv(a); n = abs(n); if n == 1: return a; elif n == 2: return fmatmul(a, a); elif n == 3: return fmatmul(fmatmul(a, a), a); z = result = None; while n > 0: z = a if z is None else fmatmul(z, z); n, bit = divmod(n, 2); if bit: result = z if result is None else fmatmul(result, z); return result; @array_function_dispatch(_unary_dispatcher); def cholesky(a): extobj = get_linalg_error_extobj(_raise_linalgerror_nonposdef); gufunc = _umath_linalg.cholesky_lo; a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); signature = 'D->D' if isComplexType(t) else 'd->d'; r = gufunc(a, signature=signature, extobj=extobj); return wrap(r.astype(result_t, copy=False)); def _qr_dispatcher(a, mode=None): return (a,); @array_function_dispatch(_qr_dispatcher); def qr(a, mode='reduced'): if mode not in ('reduced', 'complete',x 'r', 'raw'): if mode in ('f', 'full'): msg = "".join((; "The 'full' option is deprecated in favor of 'reduced'.\n",; "For backward compatibility let mode default.")); warnings.warn(msg, DeprecationWarning, stacklevel=3); mode = 'reduced'; elif mode in ('e', 'economic'): msg = "The 'economic' option is deprecated."; warnings.warn(msg, DeprecationWarning, stacklevel=3); mode = 'economic'; else: raise ValueError("Unrecognized mode '%s'" % mode); a, wrap = _makearray(a); _assert_2d(a); m, n = a.shape; t, result_t = _commonType(a); a = _fastCopyAndTranspose(t, a); a = _to_native_byte_order(a); mn = min(m, n); tau = zeros((mn,), t); if isComplexType(t): lapack_routine = lapack_lite.zgeqrf; routine_name = 'zgeqrf'; else: lapack_routine = lapack_lite.dgeqrf; routine_name = 'dgeqrf'; lwork = 1; work = zeros((lwork,), t); results = lapack_routine(m, n, a, max(1, m), tau, work, -1, 0); if results['info'] != 0: raise LinAlgError('%s returns %d' % (routine_name, results['info'])); lwork = max(1, n, int(abs(work[0]))); work = zeros((lwork,), t); results = lapack_routine(m, n, a, max(1, m), tau, work, lwork, 0); if results['info'] != 0: raise LinAlgError('%s returns %d' % (routine_name, results['info'])); if mode == 'r': r = _fastCopyAndTranspose(result_t, a[:, :mn]); return wrap(triu(r)); if mode == 'raw': return a, tau; if mode == 'economic': if t != result_t : a = a.astype(result_t, copy=False); return wrap(a.T); if mode == 'complete' and m > n: mc = m; q = empty((m, m), t); else: mc = mn; q = empty((n, m), t); q[:n] = a; if isComplexType(t): lapack_routine = lapack_lite.zungqr; routine_name = 'zungqr'; else: lapack_routine = lapack_lite.dorgqr; routine_name = 'dorgqr'; lwork = 1; work = zeros((lwork,), t); results = lapack_routine(m, mc, mn, q, max(1, m), tau, work, -1, 0); if results['info'] != 0: raise LinAlgError('%s returns %d' % (routine_name, results['info'])); lwork = max(1, n, int(abs(work[0]))); work = zeros((lwork,), t); results = lapack_routine(m, mc, mn, q, max(1, m), tau, work, lwork, 0); if results['info'] != 0: raise LinAlgError('%s returns %d' % (routine_name, results['info'])); q = _fastCopyAndTranspose(result_t, q[:mc]); r = _fastCopyAndTranspose(result_t, a[:, :mc]); return wrap(q), wrap(triu(r)); @array_function_dispatch(_unary_dispatcher); def eigvals(a): a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); _assert_finite(a); t, result_t = _commonType(a); extobj = get_linalg_error_extobj(; _raise_linalgerror_eigenvalues_nonconvergence); signature = 'D->D' if isComplexType(t) else 'd->D'; w = _umath_linalg.eigvals(a, signature=signature, extobj=extobj); if not isComplexType(t): if all(w.imag == 0): w = w.real; result_t = _realType(result_t); else: result_t = _complexType(result_t); return w.astype(result_t, copy=False); def _eigvalsh_dispatcher(a, UPLO=None): return (a,); @array_function_dispatch(_eigvalsh_dispatcher); def eigvalsh(a, UPLO='L'): UPLO = UPLO.upper(); if UPLO not in ('L', 'U'): raise ValueError("UPLO argument must be 'L' or 'U'"); extobj = get_linalg_error_extobj(; _raise_linalgerror_eigenvalues_nonconvergence); if UPLO == 'L': gufunc = _umath_linalg.eigvalsh_lo; else: gufunc = _umath_linalg.eigvalsh_up; a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); signature = 'D->d' if isComplexType(t) else 'd->d'; w = gufunc(a, signature=signature, extobj=extobj); return w.astype(_realType(result_t), copy=False); def _convertarray(a): t, result_t = _commonType(a); a = _fastCT(a.astype(t)); return a, t, result_t; def eig(a): a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); _assert_finite(a); t, result_t = _commonType(a); extobj = get_linalg_error_extobj(; _raise_linalgerror_eigenvalues_nonconvergence); signature = 'D->DD' if isComplexType(t) else 'd->DD'; w, vt = _umath_linalg.eig(a, signature=signature, extobj=extobj); if not isComplexType(t) and all(w.imag == 0.0): w = w.real; vt = vt.real; result_t = _realType(result_t); else: result_t = _complexType(result_t); vt = vt.astype(result_t, copy=False); return w.astype(result_t, copy=False), wrap(vt); @array_function_dispatch(_eigvalsh_dispatcher); def eigh(a, UPLO='L'): UPLO = UPLO.upper(); if UPLO not in ('L', 'U'): raise ValueError("UPLO argument must be 'L' or 'U'"); a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); extobj = get_linalg_error_extobj(; _raise_linalgerror_eigenvalues_nonconvergence); if UPLO == 'L': gufunc = _umath_linalg.eigh_lo; else: gufunc = _umath_linalg.eigh_up; signature = 'D->dD' if isComplexType(t) else 'd->dd'; w, vt = gufunc(a, signature=signature, extobj=extobj); w = w.astype(_realType(result_t), copy=False); vt = vt.astype(result_t, copy=False); return w, wrap(vt); def _svd_dispatcher(a, full_matrices=None, compute_uv=None, hermitian=None): return (a,); @array_function_dispatch(_svd_dispatcher); def svd(a, full_matrices=True, compute_uv=True, hermitian=False): a, wrap = _makearray(a); if hermitian: if compute_uv: s, u = eigh(a); s = s[..., ::-1]; u = u[..., ::-1]; vt = transpose(u * sign(s)[..., None, :]).conjugate(); s = abs(s); return wrap(u), s, wrap(vt); else: s = eigvalsh(a); s = s[..., ::-1]; s = abs(s); return s; _assert_stacked_2d(a); t, result_t = _commonType(a); extobj = get_linalg_error_extobj(_raise_linalgerror_svd_nonconvergence); m, n = a.shape[-2:]; if compute_uv: if full_matrices: if m < n: gufunc = _umath_linalg.svd_m_f; else: gufunc = _umath_linalg.svd_n_f; else: if m < n: gufunc = _umath_linalg.svd_m_s; else: gufunc = _umath_linalg.svd_n_s; signature = 'D->DdD' if isComplexType(t) else 'd->ddd'; u, s, vh = gufunc(a, signature=signature, extobj=extobj); u = u.astype(result_t, copy=False); s = s.astype(_realType(result_t), copy=False); vh = vh.astype(result_t, copy=False); return wrap(u), s, wrap(vh); else: if m < n: gufunc = _umath_linalg.svd_m; else: gufunc = _umath_linalg.svd_n; signature = 'D->d' if isComplexType(t) else 'd->d'; s = gufunc(a, signature=signature, extobj=extobj); s = s.astype(_realType(result_t), copy=False); return s; def _cond_dispatcher(x, p=None): return (x,); @array_function_dispatch(_cond_dispatcher); def cond(x, p=None): x = asarray(x); if _is_empty_2d(x): raise LinAlgError("cond is not defined on empty arrays"); if p is None or p == 2 or p == -2: s = svd(x, compute_uv=False); with errstate(all='ignore'): if p == -2: r = s[..., -1] / s[..., 0]; else: r = s[..., 0] / s[..., -1]; else: _assert_stacked_2d(x); _assert_stacked_square(x); t, result_t = _commonType(x); signature = 'D->D' if isComplexType(t) else 'd->d'; with errstate(all='ignore'): invx = _umath_linalg.inv(x, signature=signature); r = norm(x, p, axis=(-2, -1)) * norm(invx, p, axis=(-2, -1)); r = r.astype(result_t, copy=False); r = asarray(r); nan_mask = isnan(r); if nan_mask.any(): nan_mask &= ~isnan(x).any(axis=(-2, -1)); if r.ndim > 0: r[nan_mask] = Inf; elif nan_mask: r[()] = Inf; if r.ndim == 0: r = r[()]; return r; def _matrix_rank_dispatcher(M, tol=None, hermitian=None): return (M,); @array_function_dispatch(_matrix_rank_dispatcher); def matrix_rank(M, tol=None, hermitian=False): M = asarray(M); if M.ndim < 2: return int(not all(M==0)); S = svd(M, compute_uv=False, hermitian=hermitian); if tol is None: tol = S.max(axis=-1, keepdims=True) * max(M.shape[-2:]) * finfo(S.dtype).eps; else: tol = asarray(tol)[..., newaxis]; return count_nonzero(S > tol, axis=-1); def pinv(a, rcond=1e-15, hermitian=False): a, wrap = _makearray(a); rcond = asarray(rcond); if _is_empty_2d(a): m, n = a.shape[-2:]; res = empty(a.shape[:-2] + (n, m), dtype=a.dtype); return wrap(res); a = a.conjugate(); u, s, vt = svd(a, full_matrices=False, hermitian=hermitian); cutoff = rcond[..., newaxis] * amax(s, axis=-1, keepdims=True); large = s > cutoff; s = divide(1, s, where=large, out=s); s[~large] = 0; res = matmul(transpose(vt), multiply(s[..., newaxis], transpose(u))); return wrap(res); def slogdet(a): a = asarray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); real_t = _realType(result_t); signature = 'D->Dd' if isComplexType(t) else 'd->dd'; sign, logdet = _umath_linalg.slogdet(a, signature=signature); sign = sign.astype(result_t, copy=False); logdet = logdet.astype(real_t, copy=False); return sign, logdet; @array_function_dispatch(_unary_dispatcher); def det(a): a = asarray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); signature = 'D->D' if isComplexType(t) else 'd->d'; r = _umath_linalg.det(a, signature=signature); r = r.astype(result_t, copy=False); return r; def lstsq(a, b, rcond="warn"): a, _ = _makearray(a); b, wrap = _makearray(b); is_1d = b.ndim == 1; if is_1d: b = b[:, newaxis]; _assert_2d(a, b); m, n = a.shape[-2:]; m2, n_rhs = b.shape[-2:]; if m != m2: raise LinAlgError('Incompatible dimensions'); t, result_t = _commonType(a, b); real_t = _linalgRealType(t); result_real_t = _realType(result_t); if rcond == "warn": warnings.warn("`rcond` parameter will change to the default of "; "machine precision times ``max(M, N)`` where M and N "; "are the input matrix dimensions.\n"; "To use the future default and silence this warning "; "we advise to pass `rcond=None`, to keep using the old, "; "explicitly pass `rcond=-1`.",; FutureWarning, stacklevel=3); rcond = -1; if rcond is None: rcond = finfo(t).eps * max(n, m); if m <= n: gufunc = _umath_linalg.lstsq_m; else: gufunc = _umath_linalg.lstsq_n; signature = 'DDd->Ddid' if isComplexType(t) else 'ddd->ddid'; extobj = get_linalg_error_extobj(_raise_linalgerror_lstsq); if n_rhs == 0: b = zeros(b.shape[:-2] + (m, n_rhs + 1), dtype=b.dtype); x, resids, rank, s = gufunc(a, b, rcond, signature=signature, extobj=extobj); if m == 0: x[...] = 0; if n_rhs == 0: x = x[..., :n_rhs]; resids = resids[..., :n_rhs]; if is_1d: x = x.squeeze(axis=-1); if rank != n or m <= n: resids = array([], result_real_t); s = s.astype(result_real_t, copy=False); resids = resids.astype(result_real_t, copy=False); x = x.astype(result_t, copy=True); return wrap(x), wrap(resids), rank, s; def _multi_svd_norm(x, row_axis, col_axis, op): y = moveaxis(x, (row_axis, col_axis), (-2, -1)); result = op(svd(y, compute_uv=False), axis=-1); return result; def _norm_dispatcher(x, ord=None, axis=None, keepdims=None): return (x,); @array_function_dispatch(_norm_dispatcher); def norm(x, ord=None, axis=None, keepdims=False): x = asarray(x); if not issubclass(x.dtype.type, (inexact, object_)): x = x.astype(float); if axis is None: ndim = x.ndim; if ((ord is None) or; (ord in ('f', 'fro') and ndim == 2) or; (ord == 2 and ndim == 1)): x = x.ravel(order='K'); if isComplexType(x.dtype.type): sqnorm = dot(x.real, x.real) + dot(x.imag, x.imag); else: sqnorm = dot(x, x); ret = sqrt(sqnorm); if keepdims: ret = ret.reshape(ndim*[1]); return ret; nd = x.ndim; if axis is None: axis = tuple(range(nd)); elif not isinstance(axis, tuple): try: axis = int(axis); except Exception: raise TypeError("'axis' must be None, an integer or a tuple of integers"); axis = (axis,); if len(axis) == 1: if ord == Inf: return abs(x).max(axis=axis, keepdims=keepdims); elif ord == -Inf: return abs(x).min(axis=axis, keepdims=keepdims); elif ord == 0: return (x != 0).astype(x.real.dtype).sum(axis=axis, keepdims=keepdims); elif ord == 1: return add.reduce(abs(x), axis=axis, keepdims=keepdims); elif ord is None or ord == 2: s = (x.conj() * x).real; return sqrt(add.reduce(s, axis=axis, keepdims=keepdims)); else: try: ord + 1; except TypeError: raise ValueError("Invalid norm order for vectors."); absx = abs(x); absx **= ord; ret = add.reduce(absx, axis=axis, keepdims=keepdims); ret **= (1 / ord); return ret; elif len(axis) == 2: row_axis, col_axis = axis; row_axis = normalize_axis_index(row_axis, nd); col_axis = normalize_axis_index(col_axis, nd); if row_axis == col_axis: raise ValueError('Duplicate axes given.'); if ord == 2: ret =_multi_svd_norm(x, row_axis, col_axis, amax); elif ord == -2: ret = _multi_svd_norm(x, row_axis, col_axis, amin); elif ord == 1: if col_axis > row_axis: col_axis -= 1; ret = add.reduce(abs(x), axis=row_axis).max(axis=col_axis); elif ord == Inf: if row_axis > col_axis: row_axis -= 1; ret = add.reduce(abs(x), axis=col_axis).max(axis=row_axis); elif ord == -1: if col_axis > row_axis: col_axis -= 1; ret = add.reduce(abs(x), axis=row_axis).min(axis=col_axis); elif ord == -Inf: if row_axis > col_axis: row_axis -= 1; ret = add.reduce(abs(x), axis=col_axis).min(axis=row_axis); elif ord in [None, 'fro', 'f']: ret = sqrt(add.reduce((x.conj() * x).real, axis=axis)); elif ord == 'nuc': ret = _multi_svd_norm(x, row_axis, col_axis, sum); else: raise ValueError("Invalid norm order for matrices."); if keepdims: ret_shape = list(x.shape); ret_shape[axis[0]] = 1; ret_shape[axis[1]] = 1; ret = ret.reshape(ret_shape); return ret; else: raise ValueError("Improper number of dimensions to norm."); def multi_dot(arrays): n = len(arrays); if n < 2: raise ValueError("Expecting at least two arrays."); elif n == 2: return dot(arrays[0], arrays[1]); arrays = [asanyarray(a) for a in arrays]; ndim_first, ndim_last = arrays[0].ndim, arrays[-1].ndim; if arrays[0].ndim == 1: arrays[0] = atleast_2d(arrays[0]); if arrays[-1].ndim == 1: arrays[-1] = atleast_2d(arrays[-1]).T; _assert_2d(*arrays)
if n == 3: result = _multi_dot_three(arrays[0], arrays[1], arrays[2])
else: order = _multi_dot_matrix_chain_order(arrays)
result = _multi_dot(arrays, order, 0, n - 1)
if ndim_first == 1 and ndim_last == 1: return result[0, 0]
elif ndim_first == 1 or ndim_last == 1: return result.ravel()
else: return result; def _multi_dot_three(A, B, C): a0, a1b0 = A.shape
b1c0, c1 = C.shape; cost1 = a0 * b1c0 * (a1b0 + c1); cost2 = a1b0 * c1 * (a0 + b1c0); if cost1 < cost2: return dot(dot(A, B), C); else: return dot(A, dot(B, C)); def _multi_dot_matrix_chain_order(arrays, return_costs=False): n = len(arrays); p = [a.shape[0] for a in arrays] + [arrays[-1].shape[1]]; m = zeros((n, n), dtype=double); s = empty((n, n), dtype=intp); for l in range(1, n): for i in range(n - l): j = i + l; m[i, j] = Inf; for k in range(i, j): q = m[i, k] + m[k+1, j] + p[i]*p[k+1]*p[j+1]; if q < m[i, j]: m[i, j] = q; s[i, j] = k; if i == j: return arrays[i]; else: return dot(_multi_dot(arrays, order, i, order[i, j]),; _multi_dot(arrays, order, order[i, j] + 1, j))
KameleonWindowLearnScale
from kameleon_mcmc.mcmc.samplers.StandardMetropolis import StandardMetropolis
if __name__ == '__main__':
experiment_dir = str(os.path.abspath(sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep
distribution = Flower(amplitude=6, frequency=6, variance=1, radius=10, dimension=8)
sigma = 5
kernel = GaussianKernel(sigma=sigma)
burnin = 60000
num_iterations = 120000
#mcmc_sampler = KameleonWindowLearnScale(distribution, kernel, stop_adapt=burnin)
mean_est = zeros(distribution.dimension, dtype="float64")
cov_est = 1.0 * eye(distribution.dimension)
#mcmc_sampler = AdaptiveMetropolisLearnScale(distribution, mean_est=mean_est, cov_est=cov_est)
#mcmc_sampler = AdaptiveMetropolis(distribution, mean_est=mean_est, cov_est=cov_est)
mcmc_sampler = StandardMetropolis(distribution)
start = zeros(distribution.dimension, dtype="float64")
mcmc_params = MCMCParams(start=start, num_iterations=num_iterations, burnin=burnin)
mcmc_chain = MCMCChain(mcmc_sampler, mcmc_params)
mcmc_chain.append_mcmc_output(StatisticsOutput())
experiment = SingleChainExperiment(mcmc_chain, experiment_dir)
experiment.run()
from distribution.Gaussian import Gaussian
from distribution.SampleObject import SampleObject
from mcmc.chain.Chain import Chain
from mcmc.chain.ChainParams import ChainParams
from mcmc.sampler.MetropolisHastings import MetropolisHastings
from numpy.lib.twodim_base import eye
from numpy.ma.core import zeros
if __name__ == '__main__':
start=SampleObject(zeros(2))
target=Gaussian(mu=zeros(2), Sigma=eye(2))
sampler=MetropolisHastings(target, start)
params=ChainParams(1000)
chain=Chain(params, sampler)
chain.run()
# throw away some data
n = 250
seed(1)
idx = permutation(len(data))
idx = idx[:n]
data = data[idx]
labels = labels[idx]
# normalise and whiten dataset
data -= mean(data, 0)
L = cholesky(cov(data.T))
data = solve_triangular(L, data.T, lower=True).T
dim = shape(data)[1]
# prior on theta and posterior target estimate
theta_prior = Gaussian(mu=0 * ones(dim), Sigma=eye(dim) * 5)
target=PseudoMarginalHyperparameterDistribution(data, labels, \
n_importance=100, prior=theta_prior, \
ridge=1e-3)
# create sampler
burnin = 10000
num_iterations = burnin + 300000
kernel = GaussianKernel(sigma=23.0)
sampler = KameleonWindowLearnScale(target, kernel, stop_adapt=burnin)
# sampler=AdaptiveMetropolisLearnScale(target)
# sampler=StandardMetropolis(target)
# posterior mode derived by initial tests
start = zeros(target.dimension)
params = MCMCParams(start=start,
# loop over parameters here
experiment_dir=experiment_dir_base +os.sep + str(os.path.abspath(sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep
print "running experiments", n, "times at base", experiment_dir
for i in range(n):
distribution = Ring()
mcmc_samplers = []
kernel = GaussianKernel(sigma=1)
# mcmc_samplers.append(KameleonWindow(distribution, kernel))
mcmc_samplers.append(KameleonWindowLearnScale(distribution, kernel))
mean_est = array([-2.0, -2.0])
cov_est = 0.05 * eye(2)
# mcmc_samplers.append(AdaptiveMetropolis(distribution, mean_est=mean_est, cov_est=cov_est))
mcmc_samplers.append(AdaptiveMetropolisLearnScale(distribution, mean_est=mean_est, cov_est=cov_est))
num_eigen = 2
mcmc_samplers.append(AdaptiveMetropolisPCA(distribution, num_eigen=num_eigen, mean_est=mean_est, cov_est=cov_est))
mcmc_samplers.append(StandardMetropolis(distribution))
start = array([-2.0, -2.0])
mcmc_params = MCMCParams(start=start, num_iterations=num_iterations, burnin=burnin)
mcmc_chains = [MCMCChain(mcmc_sampler, mcmc_params) for mcmc_sampler in mcmc_samplers]
for mcmc_chain in mcmc_chains:
mcmc_chain.append_mcmc_output(StatisticsOutput())
def precompute(self):
# collect lines for Graphlab graph definition file for full rank case
graphlab_lines=GraphlabLines(output_filename=self.output_filename)
# compute all non-symmetric kernels for incoming messages at a node
print "precomputing (non-symmetric) kernels for incoming messages at a node"
graphlab_lines.lines.append("# non-observed nodes")
for node in self.graph:
added_node=False
for in_message in self.graph[node]:
for out_message in self.graph[node]:
if in_message==out_message:
continue
# dont add nodes which have no kernels, and only do once if they have
if not added_node:
graphlab_lines.new_non_observed_node(node)
added_node=True
edge_in_message=(node, in_message)
edge_out_message=(out_message, node)
lhs=self.data[edge_in_message][0]
rhs=self.data[edge_out_message][1]
lhs=reshape(lhs, (len(lhs),1))
rhs=reshape(rhs, (len(rhs),1))
K=self.kernel.kernel(lhs,rhs)
graphlab_lines.add_non_observed_node(node, out_message, in_message, K)
print "precomputing kernel (vectors) at observed nodes"
graphlab_lines.lines.append(os.linesep + "# observed nodes")
for node, observation in self.observations.items():
graphlab_lines.new_observed_node(node)
for out_message in self.graph[node]:
edge=(out_message, node)
lhs=self.data[edge][1]
lhs=reshape(lhs, (len(lhs), 1))
rhs=[[observation]]
K=self.kernel.kernel(lhs, rhs)
graphlab_lines.add_observed_node(node, out_message, K)
# now precompute systems for inference
print "precomputing systems for messages from observed nodes"
graphlab_lines.lines.append(os.linesep + "# edges with observed targets")
for node, observation in self.observations.items():
for out_message in self.graph[node]:
edge=(out_message, node)
graphlab_lines.new_edge_observed_target(node, out_message)
data_source=self.data[edge][0]
data_source=reshape(data_source, (len(data_source), 1))
data_target=self.data[edge][1]
data_target=reshape(data_target, (len(data_target), 1))
Ks=self.kernel.kernel(data_source)
Kt=self.kernel.kernel(data_target)
Ls=cholesky(Ks+eye(shape(Ks)[0])*self.reg_lambda)
Lt=cholesky(Kt+eye(shape(Kt)[0])*self.reg_lambda)
graphlab_lines.add_edge(node, out_message,"L_s", Ls)
graphlab_lines.add_edge(node, out_message,"L_t", Lt)
print "precomputing systems for messages from non-observed nodes"
graphlab_lines.lines.append(os.linesep + "# edges with non-observed targets")
for edge in self.edges:
# exclude edges which involve observed nodes
is_edge_target_observed=len(Set(self.observations.keys()).intersection(Set(edge)))>0
if not is_edge_target_observed:
graphlab_lines.new_edge_observed_target(edge[1], edge[0])
data_source=self.data[edge][0]
data_source=reshape(data_source, (len(data_source), 1))
Ks=self.kernel.kernel(data_source)
Ls=cholesky(Ks+eye(shape(Ks)[0])*self.reg_lambda)
graphlab_lines.add_edge(edge[1], edge[0],"L_s", Ls)
# write graph definition file to disc
graphlab_lines.flush()
# loop over parameters here
experiment_dir = experiment_dir_base + str(os.path.abspath(sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep
print "running experiments", n, "times at base", experiment_dir
# load data
data,labels=GPData.get_glass_data()
# normalise and whiten dataset
data-=mean(data, 0)
L=cholesky(cov(data.T))
data=solve_triangular(L, data.T, lower=True).T
dim=shape(data)[1]
# prior on theta and posterior target estimate
theta_prior=Gaussian(mu=0*ones(dim), Sigma=eye(dim)*5)
distribution=PseudoMarginalHyperparameterDistribution(data, labels, \
n_importance=100, prior=theta_prior, \
ridge=1e-3)
sigma = 23.0
print "using sigma", sigma
kernel = GaussianKernel(sigma=sigma)
for i in range(n):
mcmc_samplers = []
burnin=20000
num_iterations=100000
distribution = Banana(dimension=8, bananicity=0.03, V=100)
sigma = GaussianKernel.get_sigma_median_heuristic(
distribution.sample(1000).samples)
sigma = 10
print "using sigma", sigma
kernel = GaussianKernel(sigma=sigma)
burnin = 20000
num_iterations = 40000
mcmc_sampler = KameleonWindowLearnScale(distribution,
kernel,
stop_adapt=burnin)
mean_est = zeros(distribution.dimension, dtype="float64")
cov_est = 1.0 * eye(distribution.dimension)
cov_est[0, 0] = distribution.V
#mcmc_sampler = AdaptiveMetropolisLearnScale(distribution, mean_est=mean_est, cov_est=cov_est)
#mcmc_sampler = AdaptiveMetropolis(distribution, mean_est=mean_est, cov_est=cov_est)
#mcmc_sampler = StandardMetropolis(distribution)
start = zeros(distribution.dimension, dtype="float64")
mcmc_params = MCMCParams(start=start,
num_iterations=num_iterations,
burnin=burnin)
mcmc_chain = MCMCChain(mcmc_sampler, mcmc_params)
mcmc_chain.append_mcmc_output(StatisticsOutput())
experiment = SingleChainExperiment(mcmc_chain, experiment_dir)
experiment.run()
def __init__(self, mu=array([0, 0]), Sigma=eye(2), is_cholesky=False, ell=None):
Distribution.__init__(self, len(Sigma))
assert(len(shape(mu)) == 1)
assert(max(shape(Sigma)) == len(mu))
self.mu = mu
self.ell = ell
if is_cholesky:
self.L = Sigma
if ell == None:
assert(shape(Sigma)[0] == shape(Sigma)[1])
else:
assert(shape(Sigma)[1] == ell)
else:
assert(shape(Sigma)[0] == shape(Sigma)[1])
if ell is not None:
self.L, _, _ = MatrixTools.low_rank_approx(Sigma, ell)
self.L = self.L.T
assert(shape(self.L)[1] == ell)
else:
try:
self.L = cholesky(Sigma)
except LinAlgError:
# some really crude check for PSD (which only corrects for orunding errors
self.L = cholesky(Sigma+eye(len(Sigma))*1e-5)
