def log_pdf(self, X):
assert (len(shape(X)) == 2)
assert (shape(X)[1] == self.dimension)
transformed = X.copy()
transformed[:, 1] = X[:, 1] - self.bananicity * ((X[:, 0]**2) - self.V)
transformed[:, 0] = X[:, 0] / sqrt(self.V)
phi = Gaussian(zeros([self.dimension]), eye(self.dimension))
return phi.log_pdf(transformed)
def log_pdf(self, X):
assert(len(shape(X)) == 2)
assert(shape(X)[1] == self.dimension)
transformed = X.copy()
transformed[:, 1] = X[:, 1] - self.bananicity * ((X[:, 0] ** 2) - self.V)
transformed[:, 0] = X[:, 0] / sqrt(self.V)
phi = Gaussian(zeros([self.dimension]), eye(self.dimension))
return phi.log_pdf(transformed)
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)):
norms = array([norm(x) for x in X])
angles = arctan2(X[:, 1], X[:, 0])
mu = self.radius + self.amplitude * cos(self.frequency * angles)
transformed = hstack((array([norms-mu]).T, X[:,2:self.dimension]))
cov=eye(self.dimension-1)
cov[0,0]=self.variance
gaussian=Gaussian(zeros([self.dimension-1]), cov)
return gaussian.emp_quantiles(transformed)
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)):
norms = array([norm(x) for x in X])
angles = arctan2(X[:, 1], X[:, 0])
mu = self.radius + self.amplitude * cos(self.frequency * angles)
transformed = hstack((array([norms - mu]).T, X[:, 2:self.dimension]))
cov = eye(self.dimension - 1)
cov[0, 0] = self.variance
gaussian = Gaussian(zeros([self.dimension - 1]), cov)
return gaussian.emp_quantiles(transformed)
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)):
assert(len(shape(X)) == 2)
assert(shape(X)[1] == self.dimension)
substract=self.bananicity * ((X[:, 0] ** 2) - self.V)
divide=sqrt(self.V)
X[:, 1] -= substract
X[:, 0] /= divide
phi = Gaussian(zeros([self.dimension]), eye(self.dimension))
quantiles=phi.emp_quantiles(X, quantiles)
# undo changes to X
X[:, 0] *= divide
X[:, 1] += substract
return quantiles
def emp_quantiles(X, bananicity=0.03, V=100, quantiles=np.arange(0.1, 1, 0.1)):
assert (len(X.shape) == 2)
D = X.shape[1]
substract = bananicity * ((X[:, 0]**2) - V)
divide = np.sqrt(V)
X[:, 1] -= substract
X[:, 0] /= divide
phi = Gaussian(np.zeros(D), np.eye(D))
quantiles = phi.emp_quantiles(X, quantiles)
# undo changes to X
X[:, 0] *= divide
X[:, 1] += substract
return quantiles
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)):
assert (len(shape(X)) == 2)
assert (shape(X)[1] == self.dimension)
substract = self.bananicity * ((X[:, 0]**2) - self.V)
divide = sqrt(self.V)
X[:, 1] -= substract
X[:, 0] /= divide
phi = Gaussian(zeros([self.dimension]), eye(self.dimension))
quantiles = phi.emp_quantiles(X, quantiles)
# undo changes to X
X[:, 0] *= divide
X[:, 1] += substract
return quantiles
def fit_gmm(self, samples):
"""
Runs a couple of em instances on random starting points and returns
internal GMM representation of best instance
"""
features = RealFeatures(samples.T)
gmms = []
log_likelihoods = zeros(self.num_runs_em)
for i in range(self.num_runs_em):
# set up Shogun's GMM class and run em (corresponds to random
# initialisation)
gmm = GMM(self.num_components)
gmm.set_features(features)
log_likelihoods[i] = gmm.train_em()
gmms.append(gmm)
max_idx = log_likelihoods.argmax()
# construct Gaussian mixture components in internal representation
components = []
for i in range(self.num_components):
mu = gmms[max_idx].get_nth_mean(i)
Sigma = gmms[max_idx].get_nth_cov(i)
components.append(Gaussian(mu, Sigma))
# construct a Gaussian mixture model based on the best EM run
pie = gmms[max_idx].get_coef()
proposal = MixtureDistribution(components[0].dimension,
self.num_components, components,
Discrete(pie))
return proposal
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
posterior = OzonePosterior(prior,
logdet_alg="scikits",
solve_method="scikits")
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=5000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_average_serial"])
store_chain_output = StoreChainOutput(folder)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def emp_quantiles(X, bananicity=0.03, V=100, quantiles=np.arange(0.1, 1, 0.1)):
assert(len(X.shape) == 2)
D = X.shape[1]
substract=bananicity * ((X[:, 0] ** 2) - V)
divide=np.sqrt(V)
X[:, 1] -= substract
X[:, 0] /= divide
phi = Gaussian(np.zeros(D), np.eye(D))
quantiles=phi.emp_quantiles(X, quantiles)
# undo changes to X
X[:, 0] *= divide
X[:, 1] += substract
return quantiles
def main():
# covariance has stretched Eigenvalues, and rotated basis
Sigma = eye(2)
Sigma[0, 0] = 30
Sigma[1, 1] = 1
theta = -pi / 4
U = MatrixTools.rotation_matrix(theta)
Sigma = U.T.dot(Sigma).dot(U)
gaussian = Gaussian(Sigma=Sigma)
distribution = GaussianFullConditionals(gaussian, [0., 0.])
mcmc_sampler = Gibbs(distribution)
start = zeros(distribution.dimension)
mcmc_params = MCMCParams(start=start, num_iterations=20000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
chain.append_mcmc_output(StatisticsOutput(plot_times=True, lag=100))
chain.append_mcmc_output(
PlottingOutput(distribution=gaussian,
plot_from=1,
colour_by_likelihood=False,
num_samples_plot=0,
lag=100))
chain.run()
def construct_proposal(self, y):
"""
Returns the proposal distribution at point y given the current history
"""
assert(len(shape(y))==1)
mu, L_R = self.compute_constants(y)
return Gaussian(mu, L_R, is_cholesky=True)
def log_pdf(self, X):
assert (len(shape(X)) == 2)
assert (shape(X)[1] == self.dimension)
# compute all norms
norms = array([norm(x) for x in X])
# compute angles (second component first first)
angles = arctan2(X[:, 1], X[:, 0])
# gaussian parameters
mu = self.radius + self.amplitude * cos(self.frequency * angles)
log_pdf2d = zeros(len(X))
gaussian = Gaussian(array([mu[0]]), array([[self.variance]]))
for i in range(len(X)):
gaussian.mu = mu[i]
log_pdf2d[i] = gaussian.log_pdf(array([[norms[i]]]))
if self.dimension > 2:
remain_dims = Gaussian(zeros([self.dimension - 2]),
eye(self.dimension - 2))
log_pdfoverall = log_pdf2d + remain_dims.log_pdf(
X[:, 2:self.dimension])
else:
log_pdfoverall = log_pdf2d
return log_pdfoverall
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 1000
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"])
# cluster admin set project jump for me to exclusively allocate nodes
parameter_prefix = "" # #$ -P jump"
cluster_parameters = BatchClusterParameters(
foldername=folder,
memory=7.8,
loglevel=logging.DEBUG,
parameter_prefix=parameter_prefix,
max_walltime=60 * 60 * 24 - 1)
computation_engine = SGEComputationEngine(cluster_parameters,
check_interval=10)
rr_instance = RussianRoulette(1e-3, block_size=400)
posterior = OzonePosteriorRREngine(rr_instance=rr_instance,
computation_engine=computation_engine,
num_estimates=num_estimates,
prior=prior)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.55, -10.1])
mcmc_params = MCMCParams(start=start, num_iterations=5000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
# chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1))
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
store_chain_output = StoreChainOutput(folder, lag=1)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def main():
Log.set_loglevel(logging.DEBUG)
modulename = "sample_ozone_posterior_average_slurm"
if not FileSystem.cmd_exists("sbatch"):
engine = SerialComputationEngine()
else:
johns_slurm_hack = "#SBATCH --partition=intel-ivy,wrkstn,compute"
johns_slurm_hack = "#SBATCH --partition=intel-ivy,compute"
folder = os.sep + os.sep.join(["nfs", "data3", "ucabhst", modulename])
batch_parameters = BatchClusterParameters(
foldername=folder,
max_walltime=24 * 60 * 60,
resubmit_on_timeout=False,
memory=3,
parameter_prefix=johns_slurm_hack)
engine = SlurmComputationEngine(batch_parameters,
check_interval=1,
do_clean_up=True)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 100
posterior = OzonePosteriorAverageEngine(computation_engine=engine,
num_estimates=num_estimates,
prior=prior)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=2000)
chain = MCMCChain(mcmc_sampler, mcmc_params)
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
home = expanduser("~")
folder = os.sep.join([home, modulename])
store_chain_output = StoreChainOutput(folder)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def construct_proposal(self, y):
assert(len(shape(y)) == 1)
m = MixtureDistribution(self.distribution.dimension, self.num_eigen)
m.mixing_proportion = Discrete((self.eigvalues + 1) / (sum(self.eigvalues) + self.num_eigen))
# print "current mixing proportion: ", m.mixing_proportion.omega
for ii in range(self.num_eigen):
L = sqrt(self.dwscale[ii] * self.eigvalues[ii]) * reshape(self.eigvectors[:, ii], (self.distribution.dimension, 1))
m.components[ii] = Gaussian(y, L, is_cholesky=True, ell=1)
# Z=m.sample(1000).samples
# Visualise.plot_data(Z)
return m
def main():
# covariance has stretched Eigenvalues, and rotated basis
Sigma1 = eye(2)
Sigma1[0, 0] = 30.0
Sigma1[1, 1] = 1.0
Sigma2 = Sigma1
Sigma2[0, 0] = 20.0
theta = -pi / 4
U = MatrixTools.rotation_matrix(theta)
Sigma1 = U.T.dot(Sigma1).dot(U)
Sigma2 = U.T.dot(Sigma2).dot(U)
gaussian1 = Gaussian(Sigma=Sigma1)
gaussian2 = Gaussian(mu=array([1., 0.]), Sigma=Sigma1)
numTrials=500
vanillap=empty((numTrials,2,2))
blockp=empty((numTrials,2,2))
wildp=empty((numTrials,2,2))
# f = open("/home/dino/git/test_results.dat", "r")
# vanillarej=load(f)
# blockrej=load(f)
# wildrej=load(f)
# f.close()
#
for i in range(numTrials):
print 'trial', i
vanillap[i],blockp[i],wildp[i]=all_tests(gaussian1,gaussian2,n=200)
f = open("/nfs/home2/dino/git/test_results.dat", "w")
vanillarej=sum(vanillap<0.05,0)/numTrials
blockrej=sum(blockp<0.05,0)/numTrials
wildrej=sum(wildp<0.05,0)/numTrials
dump(vanillarej, f)
dump(blockrej, f)
dump(wildrej, f)
f.close()
def __init__(self, distribution, n=200, kernel=GaussianKernel(3), nu2=0.1, \
gamma=0.1, ell=15, nXs=100, nYs=100):
self.kernel = kernel
self.distribution = distribution
self.nu2 = nu2
self.gamma = gamma
self.ell = ell
# fix some samples
self.Z = self.distribution.sample(n).samples
# evaluate and center kernel and scale
self.K = self.kernel.kernel(self.Z, None)
self.K = Kernel.center_kernel_matrix(self.K)
# sample beta
self.rkhs_gaussian = Gaussian(mu=zeros(len(self.Z)), Sigma=self.K, is_cholesky=False, \
ell=self.ell)
self.beta = self.rkhs_gaussian.sample().samples
# plotting resolution
[(xmin, xmax), (ymin, ymax)] = self.distribution.get_plotting_bounds()
self.Xs = linspace(xmin, xmax, nXs)
self.Ys = linspace(ymin, ymax, nYs)
def log_pdf(self, X):
assert(len(shape(X)) == 2)
assert(shape(X)[1] == self.dimension)
# compute all norms
norms = array([norm(x) for x in X])
# compute angles (second component first first)
angles = arctan2(X[:, 1], X[:, 0])
# gaussian parameters
mu = self.radius + self.amplitude * cos(self.frequency * angles)
log_pdf2d = zeros(len(X))
gaussian = Gaussian(array([mu[0]]), array([[self.variance]]))
for i in range(len(X)):
gaussian.mu = mu[i]
log_pdf2d[i] = gaussian.log_pdf(array([[norms[i]]]))
if self.dimension>2:
remain_dims=Gaussian(zeros([self.dimension-2]), eye(self.dimension-2))
log_pdfoverall=log_pdf2d+remain_dims.log_pdf(X[:,2:self.dimension])
else:
log_pdfoverall=log_pdf2d
return log_pdfoverall
def plot_proposal(self, ys):
# evaluate density itself
Visualise.visualise_distribution(self.distribution, Z=self.Z, Xs=self.Xs, Ys=self.Ys)
# precompute constants of proposal
mcmc_hammer = Kameleon(self.distribution, self.kernel, self.Z, \
self.nu2, self.gamma)
# plot proposal around each y
for y in ys:
mu, L_R = mcmc_hammer.compute_constants(y)
gaussian = Gaussian(mu, L_R, is_cholesky=True)
hold(True)
Visualise.contour_plot_density(gaussian)
hold(False)
draw()
def main():
Log.set_loglevel(logging.DEBUG)
prior = Gaussian(Sigma=eye(2) * 100)
num_estimates = 2
home = expanduser("~")
folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"])
computation_engine = SerialComputationEngine()
rr_instance = RussianRoulette(1e-3, block_size=10)
posterior = OzonePosteriorRREngine(rr_instance=rr_instance,
computation_engine=computation_engine,
num_estimates=num_estimates,
prior=prior)
posterior.logdet_method = "shogun_estimate"
proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02])
mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov)
start = asarray([-11.35, -13.1])
mcmc_params = MCMCParams(start=start, num_iterations=200)
chain = MCMCChain(mcmc_sampler, mcmc_params)
# chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1))
chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1))
store_chain_output = StoreChainOutput(folder, lag=50)
chain.append_mcmc_output(store_chain_output)
loaded = store_chain_output.load_last_stored_chain()
if loaded is None:
logging.info("Running chain from scratch")
else:
logging.info("Running chain from iteration %d" % loaded.iteration)
chain = loaded
chain.run()
f = open(folder + os.sep + "final_chain", "w")
dump(chain, f)
f.close()
def get_gaussian(self, f=None, L=None):
if f is None or L is None:
f, L, _ = self.find_mode_newton(return_full=True)
w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f)
w_sqrt = sqrt(w)
K = self.gp.K
# gp book 3.27, matrix inversion lemma on
# (K^-1 +W)^-1 = K -KW^0.5 B^-1 W^0.5 K
C = (K.T * w_sqrt).T
C = solve_triangular(L, C, lower=True)
C = solve_triangular(L.T, C, lower=False)
C = (C.T * w_sqrt).T
C = K.dot(C)
C = K - C
return Gaussian(f, C, is_cholesky=False)
def construct_proposal(self, y):
"""
proposal is a mixture of normals,
centred at y and with covariance gamma^2 I + nu^2 MHaa'HM',
where a are the eigenvectors of centred kernel matrix Kc=HKH
"""
assert (len(shape(y)) == 1)
m = MixtureDistribution(self.distribution.dimension, self.num_eigen)
m.mixing_proportion = Discrete(
(self.eigvalues + 1) / (sum(self.eigvalues) + self.num_eigen))
# print "current mixing proportion: ", m.mixing_proportion.omega
M = 2 * self.kernel.gradient(y, self.Z)
H = Kernel.centring_matrix(len(self.Z))
for ii in range(self.num_eigen):
Sigma = self.gamma ** 2 * eye(len(y)) + \
self.nu2 * (M.T).dot(H.dot(outer(self.eigvectors[:, ii], self.eigvectors[:, ii]).dot(H.dot(M))))
m.components[ii] = Gaussian(y, Sigma)
return m
def __init__(self,
dimension=2,
num_components=2,
components=None,
mixing_proportion=None):
Distribution.__init__(self, dimension)
self.num_components = num_components
if (components == None):
self.components = [
Gaussian(mu=zeros(self.dimension), Sigma=eye(self.dimension))
for _ in range(self.num_components)
]
else:
assert (len(components) == self.num_components)
self.components = components
if (mixing_proportion == None):
self.mixing_proportion = Discrete(
(1.0 / num_components) * ones([num_components]))
else:
assert (num_components == mixing_proportion.num_objects)
self.mixing_proportion = mixing_proportion
def construct_proposal(self, y):
assert(len(shape(y))==1)
return Gaussian(mu=y, Sigma=self.globalscale * self.cov_est, is_cholesky=False)
def main():
numTrials = 500
n=200
Sigma1 = eye(2)
Sigma1[0, 0] = 30.0
Sigma1[1, 1] = 1.0
theta = - pi / 4
U = MatrixTools.rotation_matrix(theta)
Sigma1 = U.T.dot(Sigma1).dot(U)
print Sigma1
gaussian1 = Gaussian(Sigma=Sigma1)
gaussian2 = Gaussian(mu=array([1., 0.]), Sigma=Sigma1)
oracle_samples1 = gaussian1.sample(n=n).samples
oracle_samples2 = gaussian2.sample(n=n).samples
print 'mean1:', mean(oracle_samples1,0)
print 'mean2:', mean(oracle_samples2,0)
plot(oracle_samples1[:,0],oracle_samples1[:,1],'b*')
plot(oracle_samples2[:,0],oracle_samples2[:,1],'r*')
show()
distribution1 = GaussianFullConditionals(gaussian1, list(gaussian1.mu))
distribution2 = GaussianFullConditionals(gaussian2, list(gaussian2.mu))
H0_samples = zeros(numTrials)
HA_samples = zeros(numTrials)
mcmc_sampler1 = Gibbs(distribution1)
mcmc_sampler2 = Gibbs(distribution2)
burnin = 9000
thin = 5
start = zeros(2)
mcmc_params = MCMCParams(start=start, num_iterations=burnin+thin*n, burnin=burnin)
sigma = GaussianKernel.get_sigma_median_heuristic(concatenate((oracle_samples1,oracle_samples2),axis=0))
print 'using bandwidth: ', sigma
kernel = GaussianKernel(sigma=sigma)
for ii in arange(numTrials):
start =time.time()
print 'trial:', ii
oracle_samples1 = gaussian1.sample(n=n).samples
oracle_samples1a = gaussian1.sample(n=n).samples
oracle_samples2 = gaussian2.sample(n=n).samples
# chain1 = MCMCChain(mcmc_sampler1, mcmc_params)
# chain1.run()
# gibbs_samples1 = chain1.get_samples_after_burnin()
# gibbs_samples1 = gibbs_samples1[thin*arange(n)]
#
# chain1a = MCMCChain(mcmc_sampler1, mcmc_params)
# chain1a.run()
# gibbs_samples1a = chain1a.get_samples_after_burnin()
# gibbs_samples1a = gibbs_samples1a[thin*arange(n)]
#
# chain2 = MCMCChain(mcmc_sampler2, mcmc_params)
# chain2.run()
# gibbs_samples2 = chain2.get_samples_after_burnin()
# gibbs_samples2 = gibbs_samples2[thin*arange(n)]
# H0_samples[ii]=kernel.estimateMMD(gibbs_samples1,gibbs_samples1a)
# HA_samples[ii]=kernel.estimateMMD(gibbs_samples1,gibbs_samples2)
#
H0_samples[ii]=kernel.estimateMMD(oracle_samples1,oracle_samples1a)
HA_samples[ii]=kernel.estimateMMD(oracle_samples1,oracle_samples2)
end=time.time()
print 'time elapsed: ', end-start
f = open("/home/dino/git/mmdIIDTrueSamples.dat", "w")
dump(H0_samples, f)
dump(HA_samples, f)
dump(gaussian1, f)
dump(gaussian2, f)
f.close()
return None
def get_gp_prior(self):
"""
Returns GP prior N(0,K), only possible do if K is psd
"""
return Gaussian(zeros(len(self.K)), self.K, is_cholesky=False)
def main():
numTrials = 500
n = 200
Sigma1 = eye(2)
Sigma1[0, 0] = 30.0
Sigma1[1, 1] = 1.0
theta = -pi / 4
U = MatrixTools.rotation_matrix(theta)
Sigma1 = U.T.dot(Sigma1).dot(U)
print Sigma1
gaussian1 = Gaussian(Sigma=Sigma1)
gaussian2 = Gaussian(mu=array([1., 0.]), Sigma=Sigma1)
oracle_samples1 = gaussian1.sample(n=n).samples
oracle_samples2 = gaussian2.sample(n=n).samples
print 'mean1:', mean(oracle_samples1, 0)
print 'mean2:', mean(oracle_samples2, 0)
plot(oracle_samples1[:, 0], oracle_samples1[:, 1], 'b*')
plot(oracle_samples2[:, 0], oracle_samples2[:, 1], 'r*')
show()
distribution1 = GaussianFullConditionals(gaussian1, list(gaussian1.mu))
distribution2 = GaussianFullConditionals(gaussian2, list(gaussian2.mu))
H0_samples = zeros(numTrials)
HA_samples = zeros(numTrials)
mcmc_sampler1 = Gibbs(distribution1)
mcmc_sampler2 = Gibbs(distribution2)
burnin = 9000
thin = 5
start = zeros(2)
mcmc_params = MCMCParams(start=start,
num_iterations=burnin + thin * n,
burnin=burnin)
sigma = GaussianKernel.get_sigma_median_heuristic(
concatenate((oracle_samples1, oracle_samples2), axis=0))
print 'using bandwidth: ', sigma
kernel = GaussianKernel(sigma=sigma)
for ii in arange(numTrials):
start = time.time()
print 'trial:', ii
oracle_samples1 = gaussian1.sample(n=n).samples
oracle_samples1a = gaussian1.sample(n=n).samples
oracle_samples2 = gaussian2.sample(n=n).samples
# chain1 = MCMCChain(mcmc_sampler1, mcmc_params)
# chain1.run()
# gibbs_samples1 = chain1.get_samples_after_burnin()
# gibbs_samples1 = gibbs_samples1[thin*arange(n)]
#
# chain1a = MCMCChain(mcmc_sampler1, mcmc_params)
# chain1a.run()
# gibbs_samples1a = chain1a.get_samples_after_burnin()
# gibbs_samples1a = gibbs_samples1a[thin*arange(n)]
#
# chain2 = MCMCChain(mcmc_sampler2, mcmc_params)
# chain2.run()
# gibbs_samples2 = chain2.get_samples_after_burnin()
# gibbs_samples2 = gibbs_samples2[thin*arange(n)]
# H0_samples[ii]=kernel.estimateMMD(gibbs_samples1,gibbs_samples1a)
# HA_samples[ii]=kernel.estimateMMD(gibbs_samples1,gibbs_samples2)
#
H0_samples[ii] = kernel.estimateMMD(oracle_samples1, oracle_samples1a)
HA_samples[ii] = kernel.estimateMMD(oracle_samples1, oracle_samples2)
end = time.time()
print 'time elapsed: ', end - start
f = open("/home/dino/git/mmdIIDTrueSamples.dat", "w")
dump(H0_samples, f)
dump(HA_samples, f)
dump(gaussian1, f)
dump(gaussian2, f)
f.close()
return None
class GFunction(object):
def __init__(self, distribution, n=200, kernel=GaussianKernel(3), nu2=0.1, \
gamma=0.1, ell=15, nXs=100, nYs=100):
self.kernel = kernel
self.distribution = distribution
self.nu2 = nu2
self.gamma = gamma
self.ell = ell
# fix some samples
self.Z = self.distribution.sample(n).samples
# evaluate and center kernel and scale
self.K = self.kernel.kernel(self.Z, None)
self.K = Kernel.center_kernel_matrix(self.K)
# sample beta
self.rkhs_gaussian = Gaussian(mu=zeros(len(self.Z)), Sigma=self.K, is_cholesky=False, \
ell=self.ell)
self.beta = self.rkhs_gaussian.sample().samples
# plotting resolution
[(xmin, xmax), (ymin, ymax)] = self.distribution.get_plotting_bounds()
self.Xs = linspace(xmin, xmax, nXs)
self.Ys = linspace(ymin, ymax, nYs)
def resample_beta(self):
self.beta = self.rkhs_gaussian.sample().samples
def compute(self, x, y, Z, beta):
"""
Given two points x and y, a set of samples Z, and a vector beta,
and a kernel function, this computes the g function
g(x,beta,Z)=||k(x,.)-f|| for f=k(.,y)+sum_i beta_i*k(.,z_i)
=k(x,x) -2k(x,y) -2sum_i beta_i*k(x,z_i) +C
Constant C is not computed
"""
first = self.kernel.kernel(x, x)
second = -2 * self.kernel.kernel(x, y)
third = -2 * self.kernel.kernel(x, Z).dot(beta.T)
return first + second + third
def compute_gradient(self, x, y, Z, beta):
"""
Given two points x and y, a set of samples Z, and a vector beta,
and a kernel gradient, this computes the g function's gradient
\nabla_x g(x,beta,Z)=\nabla_x k(x,x) -2k(x,y) -2sum_i beta_i*k(x,z_i)
"""
x_2d = reshape(x, (1, len(x)))
first = self.kernel.gradient(x, x_2d)
second = -2 * self.kernel.gradient(x, y)
# compute sum_i beta_i \nabla_x k(x,z_i) and beta is a row vector
gradients = self.kernel.gradient(x, Z)
third = -2 * beta.dot(gradients)
return first + second + third
def plot(self, y=array([[-2, -2]]), gradient_scale=None, plot_data=False):
# where to evaluate G?
G = zeros((len(self.Ys), len(self.Xs)))
# for plotting the gradient field, each U and V are one dimension of gradient
if gradient_scale is not None:
GXs2 = linspace(self.Xs.min(), self.Xs.max(), 30)
GYs2 = linspace(self.Ys.min(), self.Ys.max(), 20)
X, Y = meshgrid(GXs2, GYs2)
U = zeros(shape(X))
V = zeros(shape(Y))
# evaluate g at a set of points in Xs and Ys
for i in range(len(self.Xs)):
# print i, "/", len(self.Xs)
for j in range(len(self.Ys)):
x_2d = array([[self.Xs[i], self.Ys[j]]])
y_2d = reshape(y, (1, len(y)))
G[j, i] = self.compute(x_2d, y_2d, self.Z, self.beta)
# gradient at lower resolution
if gradient_scale is not None:
for i in range(len(GXs2)):
# print i, "/", len(GXs2)
for j in range(len(GYs2)):
x_1d = array([GXs2[i], GYs2[j]])
y_2d = reshape(y, (1, len(y)))
G_grad = self.compute_gradient(x_1d, y_2d, self.Z, self.beta)
U[j, i] = -G_grad[0, 0]
V[j, i] = -G_grad[0, 1]
# plot g and Z points and y
y_2d = reshape(y, (1, len(y)))
Visualise.plot_array(self.Xs, self.Ys, G)
if gradient_scale is not None:
hold(True)
quiver(X, Y, U, V, color='y', scale=gradient_scale)
hold(False)
if plot_data:
hold(True)
Visualise.plot_data(self.Z, y_2d)
hold(False)
def plot_proposal(self, ys):
# evaluate density itself
Visualise.visualise_distribution(self.distribution, Z=self.Z, Xs=self.Xs, Ys=self.Ys)
# precompute constants of proposal
mcmc_hammer = Kameleon(self.distribution, self.kernel, self.Z, \
self.nu2, self.gamma)
# plot proposal around each y
for y in ys:
mu, L_R = mcmc_hammer.compute_constants(y)
gaussian = Gaussian(mu, L_R, is_cholesky=True)
hold(True)
Visualise.contour_plot_density(gaussian)
hold(False)
draw()
def construct_proposal(self, y):
return Gaussian(y, self.scale * self.cov)
# throw away some data
n = 250
seed(1)
idx = permutation(len(data))
idx = idx[:n]
data = data[idx]
labels = labels[idx]
# normalise and whiten dataset
data -= mean(data, 0)
L = cholesky(cov(data.T))
data = solve_triangular(L, data.T, lower=True).T
dim = shape(data)[1]
# prior on theta and posterior target estimate
theta_prior = Gaussian(mu=0 * ones(dim), Sigma=eye(dim) * 5)
target=PseudoMarginalHyperparameterDistribution(data, labels, \
n_importance=100, prior=theta_prior, \
ridge=1e-3)
# create sampler
burnin = 10000
num_iterations = burnin + 300000
kernel = GaussianKernel(sigma=23.0)
sampler = KameleonWindowLearnScale(target, kernel, stop_adapt=burnin)
# sampler=AdaptiveMetropolisLearnScale(target)
# sampler=StandardMetropolis(target)
# posterior mode derived by initial tests
start = zeros(target.dimension)
params = MCMCParams(start=start,