def test_metropolis_hastings(self):
pdf = lambda x: normal.p(x, 0, 1) + normal.p(x, 6, 3)
log_pdf = lambda x: np.log(normal.p(x, 0, 1) + normal.p(x, 6, 3))
samples = metropolis_hastings(50000,
pdf,
normal.med(sigma=1.0),
initial=-5)
fast_samples = fast_metropolis_hastings(
500000,
pdf,
initial=(np.random.rand(100) - 0.5) * 10.0,
energy=1.0)
log_fast_samples = fast_metropolis_hastings_log_space(
500000,
log_pdf,
initial=(np.random.rand(100) - 0.5) * 10.0,
energy=1.0)
samples_mean = samples[10000:].mean()
fast_samples_mean = fast_samples[100000:].mean()
log_fast_samples = log_fast_samples[100000:].mean()
for i in [samples_mean, fast_samples_mean, log_fast_samples]:
for j in [samples_mean, fast_samples_mean, log_fast_samples]:
self.assertAlmostEqual(i, j, delta=0.5)
def test_running_urbk(self):
prior_rv = normal.med(mu=0.5, sigma=1.0)
n = normal.fast_p
prior = jit.jit_probability(prior_rv)
@numba.jit(fastmath=True, nopython=True, forceobj=False)
def likelihood(y, w):
return n(y - w, mu=0.0, sigma=1.0)
data = normal.sample(mu=3.0, sigma=1.0, size=100)
log_likelihood, log_prior = jit_log_probabilities((data,), likelihood, prior)
samples, densities = search_posterior_estimation(
size=300, log_likelihood=log_likelihood,
log_prior=log_prior,
initial=prior_rv.sample(size=10),
energy=0.1,
volume=100
)
density = URBK(variance=5.0, verbose=True)
density.fit(samples, densities)
lb, ub = -6, 6
n = 2000
x = np.linspace(lb, ub, n)
y = density.p(x)
self.assertEqual(y.size, 2000)
fast_p = density.get_fast_p()
fast_p(x) # is slower than normal p.. but numba need numba functions
def test_metropolis(self):
pdf = lambda x: normal.p(x, 0, 1) + normal.p(x, 6, 3) + normal.p(
x, -6, 0.5)
samples = metropolis(size=100000,
pdf=pdf,
proposal=normal.med(mu=0, sigma=10),
M=30.0)
self.assertAlmostEqual(samples.mean(), 0.0, delta=0.1)
def posterior(likelihood: RandomVariable,
priors: Tuple[RandomVariable]) -> RandomVariable:
prior = priors[0]
prior_mu = prior.parameters[normal.mu].value
prior_sigma = prior.parameters[normal.sigma].value
likelihood_sigma = likelihood.parameters[normal.sigma].value
return normal.med(mu=prior_mu, sigma=prior_sigma + likelihood_sigma)
def posterior(data: np.ndarray, likelihood: RandomVariable,
prior: RandomVariable) -> RandomVariable:
data = np.array(data[0])
n = data.size
prior_sigma = prior.parameters[normal.sigma].value
prior_mu = prior.parameters[normal.mu].value
sigma = likelihood.parameters[normal.sigma].value
n_sigma = n / sigma
_inv_prior_sigma = 1 / prior_sigma
posterior_sigma = 1 / (n_sigma + _inv_prior_sigma)
posterior_mu = posterior_sigma * ((prior_mu / prior_sigma) +
(data.sum() / sigma))
return normal.med(mu=posterior_mu, sigma=posterior_sigma)
def test_metropolis_hastings_visual(self):
pdf = lambda x: normal.p(x, 0, 1) + normal.p(x, 6, 3) + normal.p(
x, -6, 0.5)
x = np.linspace(-10, 10, 1000)
plt.figure(figsize=(10, 6))
plt.subplot(2, 1, 1)
sb.lineplot(x, pdf(x))
plt.subplot(2, 1, 2)
samples = metropolis_hastings(50000,
pdf,
normal.med(sigma=1.0),
initial=-5)
samples = samples[1000:]
sb.distplot(samples)
plt.xlim([-10, 10])
plt.savefig("../../images/metropolis-hastings.png",
bbox_inches="tight")
plt.show()
def test_metropolis_visual(self):
pdf = lambda x: normal.p(x, 0, 1) + normal.p(x, 6, 3) + normal.p(
x, -6, 0.5)
x = np.linspace(-10, 10, 1000)
plt.figure(figsize=(10, 6))
plt.subplot(2, 1, 1)
plt.title("True", fontsize=18)
sb.lineplot(x, pdf(x))
plt.subplot(2, 1, 2)
plt.title("Estimated with 1000000 samples", fontsize=18)
samples = metropolis(size=1000000,
pdf=pdf,
proposal=normal.med(mu=0, sigma=10),
M=30.0)
sb.distplot(samples)
plt.tight_layout()
plt.xlim([-10, 10])
plt.savefig("../../images/metropolis.png", bbox_inches="tight")
plt.show()
def test_parameter_posterior_mcmc(self):
prior_rv = normal.med(mu=0.5, sigma=1.0)
n = normal.fast_p
prior = jit.jit_probability(prior_rv)
@numba.jit(fastmath=True, nopython=True, forceobj=False)
def likelihood(y, w):
return n(y - w, mu=0.0, sigma=1.0)
data = normal.sample(mu=3.0, sigma=1.0, size=100)
log_likelihood, log_prior = jit_log_probabilities((data, ), likelihood,
prior)
result = fast_metropolis_hastings_log_space_parameter_posterior_estimation(
size=2000,
log_likelihood=log_likelihood,
log_prior=log_prior,
initial=prior_rv.sample(size=10),
energy=0.1)
self.assertAlmostEqual(result.mean(), 3.0, delta=1.0)
def test_uniform_importance_sampling_visual(self):
f = lambda x: -np.square(x) + 3
plt.figure(figsize=(10, 6))
for sz in np.logspace(3, 5, 3):
sz = int(sz)
results = []
for i in range(100):
results.append(
uniform_importance_sampling(size=sz,
function=f,
domain=(-2, 2),
proposal=normal.med(mu=0,
sigma=2)))
sb.distplot(results, label=f"size {sz}")
plt.legend(fontsize=16)
plt.xticks(fontsize=16)
plt.yticks(fontsize=16)
plt.tight_layout()
plt.savefig("../../images/uniform_importance_sampling.png",
bbox_inches="tight")
plt.show()
def test_parameter_posterior_search(self):
prior_rv = normal.med(mu=0.5, sigma=1.0)
n = normal.fast_p
prior = jit.jit_probability(prior_rv)
@numba.jit(fastmath=True, nopython=True, forceobj=False)
def likelihood(y, w):
return n(y - w, mu=0.0, sigma=1.0)
data = normal.sample(mu=3.0, sigma=1.0, size=100)
log_likelihood, log_prior = jit_log_probabilities((data, ), likelihood,
prior)
points, densities = search_posterior_estimation(
size=1000,
log_likelihood=log_likelihood,
log_prior=log_prior,
initial=prior_rv.sample(size=10),
energy=0.1,
volume=1000)
self.assertTrue(True)
def univariate_normal_matcher(samples: np.ndarray):
mean = samples.mean()
variance = samples.var()
return normal.med(mu=mean, sigma=variance)
def test_search(self):
def _run_test(prior=None, likelihood=None, data=None, correct=None):
posterior = parameter_posterior(data,
likelihood=likelihood,
prior=prior,
samples=20,
batch=5,
energy=0.3,
mode="search")
if correct is not None:
pass # TODO
print(posterior.cls, pp.mode(posterior))
mu_prior = np.zeros(2)
sigma_prior = np.eye(2)
variables = np.array([2, 1])
x = np.linspace(-2, 2, 30)
y = unilinear.sample(x=x, variables=variables, sigma=0.3)
a = np.zeros(2)
b = np.ones(2)
norm_prior = normal.med(mu=1.0, sigma=1.0)
exp_prior = exponential.med(lam=1.0)
gam_prior = gamma.med(a=1.0, b=1.0)
beta_prior = beta.med(a=1.0, b=1.0)
logistic_x = np.linspace(-5, 5, 50).reshape(-1, 1)
logistic_y = (logistic_x > 0).astype(np.float).flatten()
@numba.jit(nopython=True)
def sigmoid(x):
return 1 / (1 + np.exp(-x))
fast_n = normal.fast_p
def _custom_likelihood(y, x, w):
return fast_n(y - sigmoid(x * w[0] + w[1]), mu=0.0, sigma=0.5)[0]
tests = (
{
"prior": exponential.med(lam=1.0),
"likelihood": normal.med(sigma=2.0),
"data": normal.sample(mu=3.0, sigma=2.0, size=30),
"correct": None
},
{
"prior": exponential.med(lam=1.0),
"likelihood": normal.med(mu=1.0),
"data": normal.sample(mu=1.0, sigma=2.0, size=30),
"correct": None
},
{
"prior":
multivariate_uniform.med(a=a, b=b),
"likelihood":
multivariate_normal.med(sigma=sigma_prior),
"data":
multivariate_normal.sample(mu=mu_prior,
sigma=sigma_prior,
size=30),
"correct":
None
},
{
"prior": multivariate_uniform.med(a=a, b=b),
"likelihood": unilinear.med(sigma=1.0),
"data": (y, x),
"correct": None
},
{
"prior": multivariate_normal.med(mu=mu_prior,
sigma=sigma_prior),
"likelihood": _custom_likelihood,
"data": (logistic_y, logistic_x),
"correct": None
},
{
"prior": exp_prior,
"likelihood": exponential.med(),
"data": exp_prior.sample(size=50),
"correct": None
},
{
"prior": exp_prior,
"likelihood": normal.med(sigma=1.0),
"data": norm_prior.sample(size=50),
"correct": None
},
#{
# "priors": gam_prior,
# "likelihood": bernoulli.med(),
# "data": bernoulli.sample(probability=0.6, size=30),
# "correct": None
#},
#{
# "priors": gam_prior,
# "likelihood": bernoulli.med(),
# "data": bernoulli.sample(probability=0.6, size=30),
# "correct": None
#},
#{
# "priors": (exp_prior, exp_prior),
# "likelihood": beta.med(),
# "data": beta_prior.sample(size=30),
# "correct": None
#},
## {
## "priors": (exp_prior, exp_prior),
## "likelihood": binomial.med(),
## "data": binomial.sample(n=3, probability=0.5, size=30),
## "correct": None
## }
#{
# "priors": multivariate_uniform.med(a=np.zeros(3), b=np.ones(3)),
# "likelihood": categorical.med(),
# "data": categorical.med(probabilities=np.ones(3) / 3).sample(size=30),
# "correct": None
#},
## {
## "priors": multivariate_uniform.med(a=np.zeros(3), b=np.ones(3)),
# "likelihood": dirichlet.med(),
# "data": dirichlet.med(alpha=np.ones(3)).sample(size=30),
# "correct": None
# }
#{
# "priors": (exp_prior, exp_prior),
# "likelihood": gamma.med(),
# "data": gamma.med(a=1.0, b=1.0).sample(size=30),
# "correct": None
#},
#{
# "priors": gam_prior,
# "likelihood": geometric.med(),
# "data": geometric.sample(probability=0.1, size=30),
# "correct": None
#},
## {
# "priors": (exp_prior, exp_prior, exp_prior),
# "likelihood": hypergeometric.med(),
# "data": hypergeometric.sample(N=6, K=3, n=2, size=30),
# "correct": None
# },
# {
# "priors": (exp_prior, multivariate_normal.med(mu=np.ones(3), sigma=np.eye(3))),
# "likelihood": multinomial.med(),
# "data": multinomial.sample(n=3, probabilities=np.ones(3) / 3, size=30),
# "correct": None
# },
# {
# "priors": (exp_prior, exp_prior, exp_prior, exp_prior),
# "likelihood": normal_inverse_gamma.med(),
# "data": normal_inverse_gamma.sample(mu=1.0, lam=1.0, a=2.0, b=2.0, size=30),
# "correct": None
# },
# {
# "priors": exp_prior,
# "likelihood": poisson.med(),
# "data": poisson.sample(lam=2.0, size=30),
# "correct": None
# },
# {
# "priors": (exp_prior, exp_prior),
# "likelihood": uniform.med(),
# "data": uniform.sample(a=0, b=1, size=30),
# "correct": None
# },
# {
# "priors": (multivariate_normal.med(mu=np.zeros(2), sigma=np.eye(2)),
# multivariate_normal.med(mu=np.ones(2), sigma=np.eye(2))),
# "likelihood": multivariate_uniform.med(),
# "data": multivariate_uniform.sample(a=np.zeros(2), b=np.ones(2), size=30),
# "correct": None
# },
)
for test in tests:
_run_test(**test)
def test_mcmc_moment_matching(self):
def _run_test(prior=None,
likelihood=None,
data=None,
match=None,
correct=None):
posterior = parameter_posterior(data,
likelihood=likelihood,
mode="mcmc",
prior=prior,
samples=500,
batch=40,
match_moments_for=match)
if correct is not None:
pass # TODO
print(posterior)
print("\n\n")
fast_n = normal.fast_p
def _custom_likelihood(y, x, w):
return fast_n(y - (x * w[0] + w[1]), mu=0.0, sigma=0.3)
a, b = np.ones(2) * -2, np.ones(2) * 2
sigma = np.eye(2)
mu = np.zeros(2)
x = np.linspace(-1, 1, 100)
y = x * 2 + 0.5 + normal.sample(mu=0.0, sigma=0.3, size=100)
tests = [{
"prior": multivariate_normal.med(mu=mu, sigma=sigma),
"likelihood": _custom_likelihood,
"data": (y, x),
"match": multivariate_normal,
"correct": None
}, {
"prior": exponential.med(lam=0.6),
"likelihood": normal.med(sigma=1.0),
"data": normal.sample(mu=3.0, sigma=2.0, size=200),
"match": normal,
"correct": None
}, {
"prior":
multivariate_uniform.med(a=a, b=b),
"likelihood":
multivariate_normal.med(sigma=sigma),
"data":
multivariate_normal.sample(mu=mu, sigma=sigma, size=100),
"match":
multivariate_normal,
"correct":
None,
}, {
"prior": multivariate_uniform.med(a=np.zeros(2), b=np.ones(2)),
"likelihood": unilinear.med(sigma=1.0),
"data": (y, x),
"match": (multivariate_normal, exponential),
"correct": None
}]
for test in tests:
_run_test(**test)
def test_conjugates(self):
def _run_test(prior=None, likelihood=None, data=None, correct=None):
posterior = parameter_posterior(data,
likelihood=likelihood,
prior=prior,
samples=1000)
if correct is not None:
pass # TODO
mu_prior, sigma_prior = np.ones(2), np.eye(2)
probabilities = np.array([0.3, 0.2, 0.2, 0.2, 0.1])
x = np.linspace(0, 1, 10).reshape(-1, 1)
y = x.flatten() + 1 + normal.sample(mu=0.0, sigma=1e-1, size=10)
tests = [
{
"prior": normal.med(mu=1.0, sigma=1.0),
"likelihood": normal.med(sigma=2.0),
"data": normal.med(mu=-2.0, sigma=2.0).sample(size=200),
"correct": None
},
{
"prior": normal_inverse_gamma.med(mu=1.0,
lam=2.0,
a=3.0,
b=3.0),
"likelihood": normal.med(),
"data": normal.med(mu=-2.0, sigma=2.0).sample(size=200),
"correct": None
},
{
"prior":
multivariate_normal.med(mu=mu_prior, sigma=sigma_prior),
"likelihood":
multivariate_normal.med(sigma=sigma_prior),
"data":
multivariate_normal.med(mu=mu_prior,
sigma=sigma_prior).sample(size=200),
"correct":
None
},
{
"prior": beta.med(a=1.0, b=3.0),
"likelihood": bernoulli.med(),
"data": bernoulli.med(probability=0.6).sample(size=200),
"correct": None
},
{
"prior":
dirichlet.med(alpha=np.ones(5)),
"likelihood":
categorical.med(categories=5),
"data":
categorical.med(probabilities=probabilities).sample(size=200),
"correct":
None
},
{
"prior": gamma.med(a=9, b=2),
"likelihood": exponential.med(),
"data": exponential.med(lam=1.0).sample(size=200),
"correct": None
},
{
"prior": beta.med(a=6.0, b=3.0),
"likelihood": binomial.med(n=5),
"data": binomial.med(n=5, probability=0.7).sample(size=200),
"correct": None
},
{
"prior":
dirichlet.med(alpha=np.ones(3)),
"likelihood":
multinomial.med(n=3),
"data":
multinomial.med(n=3,
probabilities=np.ones(3) / 3).sample(size=100),
"correct":
None
},
{
"prior": gamma.med(a=9.0, b=2.0),
"likelihood": poisson.med(),
"data": poisson.med(lam=2).sample(size=200),
"correct": None
},
{
"prior": beta.med(a=6.0, b=3.0),
"likelihood": geometric.med(),
"data": geometric.med(probability=0.7).sample(size=200),
"correct": None
},
{
"prior": multivariate_normal.med(mu=mu_prior,
sigma=sigma_prior),
"likelihood": unilinear.med(sigma=0.1),
"data": (y, x),
"correct": None
},
{
"prior": multivariate_normal.med(mu=mu_prior,
sigma=sigma_prior),
"likelihood": unilinear.med(sigma=0.1),
"data": (1.0, 2.0),
"correct": None
},
{
"prior": normal.med(mu=1.0, sigma=1.0),
"likelihood": normal.med(sigma=2.0),
"data": 2.0,
"correct": None
},
{
"prior": normal_inverse_gamma.med(mu=1.0,
lam=2.0,
a=3.0,
b=3.0),
"likelihood": normal.med(),
"data": 2.0,
"correct": None
},
{
"prior": multivariate_normal.med(mu=mu_prior,
sigma=sigma_prior),
"likelihood": multivariate_normal.med(sigma=sigma_prior),
"data": np.zeros(2),
"correct": None
},
{
"prior": beta.med(a=1.0, b=3.0),
"likelihood": bernoulli.med(),
"data": 1,
"correct": None
},
{
"prior": dirichlet.med(alpha=np.ones(5)),
"likelihood": categorical.med(categories=5),
"data": 3,
"correct": None
},
{
"prior": gamma.med(a=9, b=2),
"likelihood": exponential.med(),
"data": 5.1,
"correct": None
},
{
"prior": beta.med(a=6.0, b=3.0),
"likelihood": binomial.med(n=5),
"data": 4,
"correct": None
},
{
"prior": dirichlet.med(alpha=np.ones(3)),
"likelihood": multinomial.med(n=3),
"data": [1, 1, 1],
"correct": None
},
{
"prior": gamma.med(a=9.0, b=2.0),
"likelihood": poisson.med(),
"data": 7,
"correct": None
},
{
"prior": beta.med(a=6.0, b=3.0),
"likelihood": geometric.med(),
"data": 6,
"correct": None
},
]
for test in tests:
_run_test(**test)