def test_uniform_importance_multivariate_visual(self): f = lambda x: -np.square(x[:, 0]) + np.square(x[:, 1]) lower_bound = np.array([0, 0]) upper_bound = np.array([4, 2]) plt.figure(figsize=(10, 6)) for sz in np.logspace(3, 5, 3): sz = int(sz) results = [] for i in range(100): results.append( uniform_importance_sampling( size=sz, function=f, domain=(lower_bound, upper_bound), proposal=multivariate_normal.med(mu=np.zeros(2), sigma=np.eye(2) * 2))) sb.distplot(results, label=f"size {sz}") plt.legend(fontsize=16) plt.xticks(fontsize=16) plt.yticks(fontsize=16) plt.tight_layout() plt.savefig( "../../images/uniform_importance_sampling_multivariate.png", bbox_inches="tight") plt.show()
def posterior(data: np.ndarray, likelihood: RandomVariable, prior: RandomVariable) -> RandomVariable: y, x = data y, x = np.array(y), np.array(x) if y.ndim != 1: y = y.reshape(-1) if x.ndim == 0: x = x.reshape(1, 1) if x.ndim == 1: x = x[None, :] x_samples = x.shape[0] x = np.concatenate([x, np.ones((x_samples, 1))], axis=1) # Add bias term likelihood_sigma = np.eye(x_samples) / likelihood.parameters[ unilinear.sigma].value prior_mu = prior.parameters[multivariate_normal.mu].value prior_sigma_inv = np.linalg.inv( prior.parameters[multivariate_normal.sigma].value) x_squiggle = x sigma_posterior = np.linalg.inv( x_squiggle.T @ likelihood_sigma @ x_squiggle + prior_sigma_inv) mu_posterior = (sigma_posterior @ x_squiggle.T) @ likelihood_sigma @ y \ + sigma_posterior @ prior_sigma_inv @ prior_mu return multivariate_normal.med(mu=mu_posterior, sigma=sigma_posterior)
def posterior(likelihood: RandomVariable, priors: Tuple[RandomVariable]) -> RandomVariable: prior = priors[0] prior_mu = prior.parameters[normal.mu].value prior_sigma = prior.parameters[normal.sigma].value likelihood_sigma = likelihood.parameters[normal.sigma].value return multivariate_normal.med(mu=prior_mu, sigma=prior_sigma + likelihood_sigma)
def posterior(data: np.ndarray, likelihood: RandomVariable, prior: RandomVariable) -> RandomVariable: data = np.array(data[0]) if data.ndim == 1: data = data.reshape(1, -1) n = data.shape[0] inv_prior_sigma = np.linalg.inv( prior.parameters[multivariate_normal.sigma].value) prior_mu = prior.parameters[normal.mu].value inv_sigma = np.linalg.inv( likelihood.parameters[multivariate_normal.sigma].value) posterior_sigma = np.linalg.inv(inv_prior_sigma + n * inv_sigma) posterior_mu = posterior_sigma @ (inv_prior_sigma @ prior_mu + n * inv_sigma @ data.mean(axis=0)) return multivariate_normal.med(mu=posterior_mu, sigma=posterior_sigma)
def multivariate_normal_matcher(samples: np.ndarray): mean = samples.mean(axis=0) covariance = np.cov(samples, rowvar=False) return multivariate_normal.med(mu=mean, sigma=covariance)
def test_search(self): def _run_test(prior=None, likelihood=None, data=None, correct=None): posterior = parameter_posterior(data, likelihood=likelihood, prior=prior, samples=20, batch=5, energy=0.3, mode="search") if correct is not None: pass # TODO print(posterior.cls, pp.mode(posterior)) mu_prior = np.zeros(2) sigma_prior = np.eye(2) variables = np.array([2, 1]) x = np.linspace(-2, 2, 30) y = unilinear.sample(x=x, variables=variables, sigma=0.3) a = np.zeros(2) b = np.ones(2) norm_prior = normal.med(mu=1.0, sigma=1.0) exp_prior = exponential.med(lam=1.0) gam_prior = gamma.med(a=1.0, b=1.0) beta_prior = beta.med(a=1.0, b=1.0) logistic_x = np.linspace(-5, 5, 50).reshape(-1, 1) logistic_y = (logistic_x > 0).astype(np.float).flatten() @numba.jit(nopython=True) def sigmoid(x): return 1 / (1 + np.exp(-x)) fast_n = normal.fast_p def _custom_likelihood(y, x, w): return fast_n(y - sigmoid(x * w[0] + w[1]), mu=0.0, sigma=0.5)[0] tests = ( { "prior": exponential.med(lam=1.0), "likelihood": normal.med(sigma=2.0), "data": normal.sample(mu=3.0, sigma=2.0, size=30), "correct": None }, { "prior": exponential.med(lam=1.0), "likelihood": normal.med(mu=1.0), "data": normal.sample(mu=1.0, sigma=2.0, size=30), "correct": None }, { "prior": multivariate_uniform.med(a=a, b=b), "likelihood": multivariate_normal.med(sigma=sigma_prior), "data": multivariate_normal.sample(mu=mu_prior, sigma=sigma_prior, size=30), "correct": None }, { "prior": multivariate_uniform.med(a=a, b=b), "likelihood": unilinear.med(sigma=1.0), "data": (y, x), "correct": None }, { "prior": multivariate_normal.med(mu=mu_prior, sigma=sigma_prior), "likelihood": _custom_likelihood, "data": (logistic_y, logistic_x), "correct": None }, { "prior": exp_prior, "likelihood": exponential.med(), "data": exp_prior.sample(size=50), "correct": None }, { "prior": exp_prior, "likelihood": normal.med(sigma=1.0), "data": norm_prior.sample(size=50), "correct": None }, #{ # "priors": gam_prior, # "likelihood": bernoulli.med(), # "data": bernoulli.sample(probability=0.6, size=30), # "correct": None #}, #{ # "priors": gam_prior, # "likelihood": bernoulli.med(), # "data": bernoulli.sample(probability=0.6, size=30), # "correct": None #}, #{ # "priors": (exp_prior, exp_prior), # "likelihood": beta.med(), # "data": beta_prior.sample(size=30), # "correct": None #}, ## { ## "priors": (exp_prior, exp_prior), ## "likelihood": binomial.med(), ## "data": binomial.sample(n=3, probability=0.5, size=30), ## "correct": None ## } #{ # "priors": multivariate_uniform.med(a=np.zeros(3), b=np.ones(3)), # "likelihood": categorical.med(), # "data": categorical.med(probabilities=np.ones(3) / 3).sample(size=30), # "correct": None #}, ## { ## "priors": multivariate_uniform.med(a=np.zeros(3), b=np.ones(3)), # "likelihood": dirichlet.med(), # "data": dirichlet.med(alpha=np.ones(3)).sample(size=30), # "correct": None # } #{ # "priors": (exp_prior, exp_prior), # "likelihood": gamma.med(), # "data": gamma.med(a=1.0, b=1.0).sample(size=30), # "correct": None #}, #{ # "priors": gam_prior, # "likelihood": geometric.med(), # "data": geometric.sample(probability=0.1, size=30), # "correct": None #}, ## { # "priors": (exp_prior, exp_prior, exp_prior), # "likelihood": hypergeometric.med(), # "data": hypergeometric.sample(N=6, K=3, n=2, size=30), # "correct": None # }, # { # "priors": (exp_prior, multivariate_normal.med(mu=np.ones(3), sigma=np.eye(3))), # "likelihood": multinomial.med(), # "data": multinomial.sample(n=3, probabilities=np.ones(3) / 3, size=30), # "correct": None # }, # { # "priors": (exp_prior, exp_prior, exp_prior, exp_prior), # "likelihood": normal_inverse_gamma.med(), # "data": normal_inverse_gamma.sample(mu=1.0, lam=1.0, a=2.0, b=2.0, size=30), # "correct": None # }, # { # "priors": exp_prior, # "likelihood": poisson.med(), # "data": poisson.sample(lam=2.0, size=30), # "correct": None # }, # { # "priors": (exp_prior, exp_prior), # "likelihood": uniform.med(), # "data": uniform.sample(a=0, b=1, size=30), # "correct": None # }, # { # "priors": (multivariate_normal.med(mu=np.zeros(2), sigma=np.eye(2)), # multivariate_normal.med(mu=np.ones(2), sigma=np.eye(2))), # "likelihood": multivariate_uniform.med(), # "data": multivariate_uniform.sample(a=np.zeros(2), b=np.ones(2), size=30), # "correct": None # }, ) for test in tests: _run_test(**test)
def test_mcmc_moment_matching(self): def _run_test(prior=None, likelihood=None, data=None, match=None, correct=None): posterior = parameter_posterior(data, likelihood=likelihood, mode="mcmc", prior=prior, samples=500, batch=40, match_moments_for=match) if correct is not None: pass # TODO print(posterior) print("\n\n") fast_n = normal.fast_p def _custom_likelihood(y, x, w): return fast_n(y - (x * w[0] + w[1]), mu=0.0, sigma=0.3) a, b = np.ones(2) * -2, np.ones(2) * 2 sigma = np.eye(2) mu = np.zeros(2) x = np.linspace(-1, 1, 100) y = x * 2 + 0.5 + normal.sample(mu=0.0, sigma=0.3, size=100) tests = [{ "prior": multivariate_normal.med(mu=mu, sigma=sigma), "likelihood": _custom_likelihood, "data": (y, x), "match": multivariate_normal, "correct": None }, { "prior": exponential.med(lam=0.6), "likelihood": normal.med(sigma=1.0), "data": normal.sample(mu=3.0, sigma=2.0, size=200), "match": normal, "correct": None }, { "prior": multivariate_uniform.med(a=a, b=b), "likelihood": multivariate_normal.med(sigma=sigma), "data": multivariate_normal.sample(mu=mu, sigma=sigma, size=100), "match": multivariate_normal, "correct": None, }, { "prior": multivariate_uniform.med(a=np.zeros(2), b=np.ones(2)), "likelihood": unilinear.med(sigma=1.0), "data": (y, x), "match": (multivariate_normal, exponential), "correct": None }] for test in tests: _run_test(**test)
def test_conjugates(self): def _run_test(prior=None, likelihood=None, data=None, correct=None): posterior = parameter_posterior(data, likelihood=likelihood, prior=prior, samples=1000) if correct is not None: pass # TODO mu_prior, sigma_prior = np.ones(2), np.eye(2) probabilities = np.array([0.3, 0.2, 0.2, 0.2, 0.1]) x = np.linspace(0, 1, 10).reshape(-1, 1) y = x.flatten() + 1 + normal.sample(mu=0.0, sigma=1e-1, size=10) tests = [ { "prior": normal.med(mu=1.0, sigma=1.0), "likelihood": normal.med(sigma=2.0), "data": normal.med(mu=-2.0, sigma=2.0).sample(size=200), "correct": None }, { "prior": normal_inverse_gamma.med(mu=1.0, lam=2.0, a=3.0, b=3.0), "likelihood": normal.med(), "data": normal.med(mu=-2.0, sigma=2.0).sample(size=200), "correct": None }, { "prior": multivariate_normal.med(mu=mu_prior, sigma=sigma_prior), "likelihood": multivariate_normal.med(sigma=sigma_prior), "data": multivariate_normal.med(mu=mu_prior, sigma=sigma_prior).sample(size=200), "correct": None }, { "prior": beta.med(a=1.0, b=3.0), "likelihood": bernoulli.med(), "data": bernoulli.med(probability=0.6).sample(size=200), "correct": None }, { "prior": dirichlet.med(alpha=np.ones(5)), "likelihood": categorical.med(categories=5), "data": categorical.med(probabilities=probabilities).sample(size=200), "correct": None }, { "prior": gamma.med(a=9, b=2), "likelihood": exponential.med(), "data": exponential.med(lam=1.0).sample(size=200), "correct": None }, { "prior": beta.med(a=6.0, b=3.0), "likelihood": binomial.med(n=5), "data": binomial.med(n=5, probability=0.7).sample(size=200), "correct": None }, { "prior": dirichlet.med(alpha=np.ones(3)), "likelihood": multinomial.med(n=3), "data": multinomial.med(n=3, probabilities=np.ones(3) / 3).sample(size=100), "correct": None }, { "prior": gamma.med(a=9.0, b=2.0), "likelihood": poisson.med(), "data": poisson.med(lam=2).sample(size=200), "correct": None }, { "prior": beta.med(a=6.0, b=3.0), "likelihood": geometric.med(), "data": geometric.med(probability=0.7).sample(size=200), "correct": None }, { "prior": multivariate_normal.med(mu=mu_prior, sigma=sigma_prior), "likelihood": unilinear.med(sigma=0.1), "data": (y, x), "correct": None }, { "prior": multivariate_normal.med(mu=mu_prior, sigma=sigma_prior), "likelihood": unilinear.med(sigma=0.1), "data": (1.0, 2.0), "correct": None }, { "prior": normal.med(mu=1.0, sigma=1.0), "likelihood": normal.med(sigma=2.0), "data": 2.0, "correct": None }, { "prior": normal_inverse_gamma.med(mu=1.0, lam=2.0, a=3.0, b=3.0), "likelihood": normal.med(), "data": 2.0, "correct": None }, { "prior": multivariate_normal.med(mu=mu_prior, sigma=sigma_prior), "likelihood": multivariate_normal.med(sigma=sigma_prior), "data": np.zeros(2), "correct": None }, { "prior": beta.med(a=1.0, b=3.0), "likelihood": bernoulli.med(), "data": 1, "correct": None }, { "prior": dirichlet.med(alpha=np.ones(5)), "likelihood": categorical.med(categories=5), "data": 3, "correct": None }, { "prior": gamma.med(a=9, b=2), "likelihood": exponential.med(), "data": 5.1, "correct": None }, { "prior": beta.med(a=6.0, b=3.0), "likelihood": binomial.med(n=5), "data": 4, "correct": None }, { "prior": dirichlet.med(alpha=np.ones(3)), "likelihood": multinomial.med(n=3), "data": [1, 1, 1], "correct": None }, { "prior": gamma.med(a=9.0, b=2.0), "likelihood": poisson.med(), "data": 7, "correct": None }, { "prior": beta.med(a=6.0, b=3.0), "likelihood": geometric.med(), "data": 6, "correct": None }, ] for test in tests: _run_test(**test)