# The data data = np.load('data_concentrations_left_corners.npy') # The forward model diff_model = ContaminantTransportModelLeft() print 'Num_input' print str(diff_model.num_input) + '\n' # The isotropic Likelihood IsotropicL = IsotropicGaussianLikelihood(data[:], diff_model) # The joint log_p = Joint(IsotropicL, prior) print 'Target:' print str(log_p) # The approximating distribution comp = [MultivariateNormal(np.random.gamma(10,1,num_dim))]#, MultivariateNormal(np.random.gamma(10,1,num_dim))] log_q = MixtureOfMultivariateNormals(comp) log_q.comp[0].mu = np.ones(log_q.comp[0].mu.shape) * 0.5 #log_q.comp[1].mu = np.ones(log_q.comp[0].mu.shape) * 0.5 log_q.comp[0].C = np.eye(num_dim) * 1e-4 #log_q.comp[1].C = np.eye(num_dim) * 1e-4 print 'Initial:' print log_q # Pick an entropy approximation entropy = FirstOrderEntropyApproximation() # Pick an approximation for the expectation of the joint expectation_functional = ThirdOrderExpectationFunctional(log_p) # Restrictions for mu mu_bounds = (tuple((0., 1.) for i in xrange(log_q.num_dim - 1)) + ((1e-6, None), )) C_bounds = tuple((1e-32, None) for i in xrange(log_q.num_comp * log_q.num_dim))
y = data[:, 1:] y = y.reshape((1,y.shape[0] * y.shape[1])) # The forward model catal_model = CatalysisModel() print 'Num_input' print str(catal_model.num_input) + '\n' # The isotropic Likelihood IsotropicL = IsotropicGaussianLikelihood(y[0,:], catal_model) # The joint log_p = Joint(IsotropicL, prior) print 'Target:' print str(log_p) # The approximating distribution comp = [MultivariateNormal(np.random.gamma(10,1,num_dim))]#, MultivariateNormal(np.random.gamma(10,1,num_dim))] log_q = MixtureOfMultivariateNormals(comp) log_q.mu = np.ones(log_q.mu.shape) * 1e-4 log_q.comp[0].C = np.eye(num_dim) * 0.5 print 'Initial:' print log_q # Pick an entropy approximation entropy = FirstOrderEntropyApproximation() # Pick an approximation for the expectation of the joint expectation_functional = ThirdOrderExpectationFunctional(log_p) # Restrictions for mu mu_bounds = (tuple((1e-6, 1) for i in xrange(log_q.num_dim - 1)) + ((1e-6, None), )) C_bounds = tuple((1e-32, None) for i in xrange(log_q.num_comp * log_q.num_dim)) # Build the ELBO elbo = EvidenceLowerBound(entropy, expectation_functional)
data = np.load('data_concentrations_upperleft_corner.npy') # The forward model diff_model = ContaminantTransportModelUpperLeft() print 'Num_input' print str(diff_model.num_input) + '\n' # The isotropic Likelihood IsotropicL = IsotropicGaussianLikelihood(data[:], diff_model) # The joint log_p = Joint(IsotropicL, prior) print 'Target:' print str(log_p) # The approximating distribution comp = [MultivariateNormal(np.random.gamma(10,1,num_dim)), MultivariateNormal(np.random.gamma(10,1,num_dim))] # MultivariateNormal(np.random.gamma(10,1,num_dim))]#, MultivariateNormal(np.random.gamma(10,1,num_dim))] log_q = MixtureOfMultivariateNormals(comp) log_q.comp[0].mu = np.ones(log_q.comp[0].mu.shape) * 0.25 log_q.comp[1].mu = np.ones(log_q.comp[0].mu.shape) * 0.75 #log_q.comp[2].mu = np.ones(log_q.comp[2].mu.shape) * 0.4 #log_q.comp[3].mu = np.ones(log_q.comp[3].mu.shape) * 0.6 log_q.comp[0].C = np.eye(num_dim) * 1e-4 log_q.comp[1].C = np.eye(num_dim) * 1e-4 #log_q.comp[2].C = np.eye(num_dim) * 1e-4 #log_q.comp[3].C = np.eye(num_dim) * 1e-4 print 'Initial:' print log_q # Pick an entropy approximation entropy = FirstOrderEntropyApproximation() # Pick an approximation for the expectation of the joint expectation_functional = ThirdOrderExpectationFunctional(log_p)
num_comp = 1 # The model model = TestModel0() # The prior log_p_x = MultivariateNormal(mu=[0]) log_p_z_fake = FlatPDF(model.num_output) log_p_x_ext = PDFCollection([log_p_x, log_p_z_fake]) # The isotropic Likelihood log_p_z_given_x = UncertaintyPropagationLikelihood(model, alpha=100.) # The joint log_p = Joint(log_p_z_given_x, log_p_x_ext) # The approximating distribution log_q = MixtureOfMultivariateNormals.create(log_p.num_dim, num_comp) # Build the ELBO # Pick an entropy approximation entropy = FirstOrderEntropyApproximation() # Pick an approximation for the expectation of the joint expectation_functional = ThirdOrderExpectationFunctional(log_p) # Build the ELBO elbo = EvidenceLowerBound(entropy, expectation_functional) print 'ELBO:' print str(elbo) # Optimize the elbo optimizer = Optimizer(elbo) C_bounds = tuple((1e-32, None) for i in xrange(log_q.num_comp * log_q.num_dim))
num_comp = 5 # The model model = TestModel1() # The prior log_p_x = MultivariateNormal(mu=[0]) log_p_z_fake = FlatPDF(model.num_output) log_p_x_ext = PDFCollection([log_p_x, log_p_z_fake]) # The isotropic Likelihood log_p_z_given_x = UncertaintyPropagationLikelihood(model, alpha=1e-1) # The joint log_p = Joint(log_p_z_given_x, log_p_x_ext) # The approximating distribution log_q = MixtureOfMultivariateNormals.create(log_p.num_dim, num_comp) # Build the ELBO # Pick an entropy approximation entropy = FirstOrderEntropyApproximation() # Pick an approximation for the expectation of the joint expectation_functional = ThirdOrderExpectationFunctional(log_p) # Build the ELBO elbo = EvidenceLowerBound(entropy, expectation_functional) print 'ELBO:' print str(elbo) # Optimize the elbo optimizer = FullOptimizer(elbo) C_bounds = tuple((1e-32, None) for i in xrange(log_q.num_comp * log_q.num_dim))
import numpy as np import matplotlib.pyplot as plt import sys import os sys.path.insert(0, os.path.abspath('.')) from vuq import MixtureOfMultivariateNormals from vuq import MonteCarloEntropyApproximation from vuq import FirstOrderEntropyApproximation from vuq import EntropyLowerBound # Number of components for the test num_dim = 1 # Create a random multivariate normal q = MixtureOfMultivariateNormals.create(num_dim, 1) # With one component first # Create the entropy approximation S0 = FirstOrderEntropyApproximation() # Create a monte carlo approximation to the entropy Smc = MonteCarloEntropyApproximation(num_samples=100) # The lower bound to the entropy Sl = EntropyLowerBound() # The first order approximation to the entropy print 'S0[q] =', S0(q)['S'] # The lower bound to the entropy print 'Sl[q] =', Sl.eval(q)
if k != j: C_new[i, k, j] += h q.C = C_new Sph = S0.eval(q)[0] J[i, j, k] = (Sph - S) / h if k != j: J[i, j, k] *= 0.5 q.C = C return J # Number of components for the test num_dim = 1 # Create a random multivariate normal q = MixtureOfMultivariateNormals.create(num_dim, 2) # With one component first # Create the entropy approximation #S0 = FirstOrderEntropyApproximation() S0 = EntropyLowerBound() # Create a monte carlo approximation to the entropy Smc = MonteCarloEntropyApproximation(num_samples=100) # Try it out with more components print 'Doing it with two components...' q = MixtureOfMultivariateNormals.create(num_dim, 2) #q.comp[0].C = np.array([[2]]) #q.w = np.array([0.3, 0.7]) print str(q) # Evaluate the entropy
def main(options): """ The main function. """ # DATA AND MODEL data = load_dimensionless_catalysis_data() model = catalysis_dimensionless_model.make_model() catal_model = model['catal_model'] # VARIATIONAL PLOTS with open(options.var_file, 'rb') as fd: var_results = pickle.load(fd) log_q = var_results['log_q'] if var_results.has_key('output_samples') and not options.force: Y = var_results['output_samples'] else: Y = [] for i in xrange(options.num_samples): print 'taking sample', i + 1 omega = log_q.sample().flatten() x = omega[:5] sigma = np.exp(omega[5]) y = catal_model(x)['f'] Y.append(y + sigma * np.random.randn(*y.shape)) Y = np.vstack(Y) var_results['output_samples'] = Y with open(options.var_file, 'wb') as fd: pickle.dump(var_results, fd, protocol=pickle.HIGHEST_PROTOCOL) var_out_fig = os.path.abspath(os.path.splitext(options.var_file)[0] + '_output.png') plot_catalysis_output(var_out_fig, Y, legend=True, title='VAR. (L=%d)' % log_q.num_comp) # MCMC PLOTS f = tb.open_file(options.mcmc_file, mode='r') chain = options.chain if chain is None: chain_counter = f.get_node('/mcmc/chain_counter') chain = chain_counter[-1][-1] chain_data = getattr(f.root.mcmc.data, chain) mcmc_step = chain_data.cols.step[:] omega = chain_data.cols.params[:] x = omega[:, :5] sigma = np.exp(omega[:, 5]) # Do not do the following step unless this is a forced calculation mcmc_file_prefix = os.path.abspath(os.path.splitext(options.mcmc_file)[0]) +\ '_ch=' + str(chain) mcmc_output_file = mcmc_file_prefix + '_output.pcl' if os.path.exists(mcmc_output_file) and not options.force: print '- I found', mcmc_output_file with open(mcmc_output_file, 'rb') as fd: Y_mcmc = pickle.load(fd) elif not options.skip_mcmc_output: print '- computing the output of the model for each one of the MCMC samples' print '- be patient' Y_mcmc = catal_model(x)['f'] with open(mcmc_output_file, 'wb') as fd: pickle.dump(Y_mcmc, fd, protocol=pickle.HIGHEST_PROTOCOL) if not options.skip_mcmc_output: Y_mcmc = Y_mcmc[options.skip::options.thin, :] sigma = sigma[options.skip::options.thin] Y_mcmc += sigma[:, None] * np.random.randn(*Y_mcmc.shape) mcmc_out_fig = mcmc_file_prefix + '_output.png' plot_catalysis_output(mcmc_out_fig, Y_mcmc, title='MCMC (MALA)') # Plot the histograms or_omega = omega omega = np.exp(omega[options.skip::options.thin, :]) w = log_q.w mu =log_q.mu C = log_q.C c = np.vstack([np.diag(C[i, :, :]) for i in xrange(log_q.num_comp)]) for i in xrange(omega.shape[1]): fig = plt.figure() ax = fig.add_subplot(111) ax.hist(omega[:, i], alpha=0.5, normed=True) #ax.hist(smc_samples[:, i], weights=smc_weights, alpha=0.5, normed=True) x_min = omega[:, i].min() x_max = omega[:, i].max() x_i = np.linspace(x_min, x_max, 100) # Create a 1D mixture comp = [MultivariateNormal([[mu[j, i]]], C=[[c[j, i]]]) for j in xrange(log_q.num_comp)] log_q_i = MixtureOfMultivariateNormals(comp) y_i = np.exp(log_q_i(np.log(x_i[:, None]))) / x_i mu_i = 0. if i < 5 else -1. x_p_max = min(np.exp(mu_i + 2.), x_i[-1]) x_p_i = np.linspace(np.exp(mu_i - 2.), x_p_max, 100) ax.plot(x_p_i, stats.norm.pdf(np.log(x_p_i), mu_i, scale=1.) / x_p_i, 'g--', linewidth=2) ax.plot(x_i, y_i, 'r-', linewidth=2) #ax.hist(samples_var[:, i], alpha=0.5, normed=True) name = '\kappa_%d' % (i + 1) if i < 5 else '\sigma' xlabel = '$' + name + '$' ylabel = '$p(' + name + '|y)$' ax.set_xlabel(xlabel, fontsize=26) ax.set_ylabel(ylabel, fontsize=26) plt.setp(ax.get_xticklabels(), fontsize=26) plt.setp(ax.get_yticklabels(), fontsize=26) if i == 0: leg = plt.legend(['Prior', 'VAR ($L=%d$)' % log_q.num_comp, 'MCMC (MALA)'], loc='best') plt.setp(leg.get_texts(), fontsize=26) if i == 5: ax.xaxis.set_major_locator(plt.MaxNLocator(6)) plt.tight_layout() png_file = os.path.abspath(os.path.splitext(options.var_file)[0] + '_input_' + str(i) + '.png') print '- writing', png_file plt.savefig(png_file) del fig # Draw the error of MCMC as a function of the number of evaluations # Compute the MCMC mean #x_m_mcmc = pd.rolling_mean(omega, options.rolling) #x_s_mcmc = pd.rolling_std(omega, options.rolling) omega = or_omega[50:, :] mcmc_step = mcmc_step[50:] x_m_mcmc = np.cumsum(omega, axis=0) / np.arange(1, omega.shape[0] + 1)[:, None] x_m_mcmc_2 = np.cumsum(omega ** 2., axis=0) / np.arange(1, omega.shape[0] + 1)[:, None] x_s_mcmc = np.sqrt(x_m_mcmc_2 - x_m_mcmc ** 2.) x_m_var = np.mean(w * mu, axis=0) samples_var = log_q.sample(options.var_samples) x_s_var = np.std(samples_var, axis=0) # We will compute the mean and the std of MCMC as accurately as possible t_m = np.mean(or_omega[700:, :], axis=0) t_s = np.std(or_omega[700:, :], axis=0) #t_m = x_m_mcmc[-1, :] #t_s = x_s_mcmc[-1, :] e_m = x_m_mcmc - t_m m_rms = np.sqrt(np.mean(e_m ** 2., axis=1)) / np.linalg.norm(t_m) e_s = x_s_mcmc - t_s s_rms = np.sqrt(np.mean(e_s ** 2., axis=1)) / np.linalg.norm(t_s) # Variational error v_m_rms = np.linalg.norm(mu - t_m) / np.linalg.norm(t_m) v_s_rms = np.linalg.norm(np.sqrt(c) - t_s) / np.linalg.norm(t_s) fig = plt.figure() ax = fig.add_subplot(111) ax.plot(mcmc_step, m_rms, '-b', linewidth=2) ax.plot(mcmc_step, np.ones(mcmc_step.shape[0]) * v_m_rms, '--b', linewidth=2) ax.plot(mcmc_step, s_rms, '.-g', linewidth=2) ax.plot(mcmc_step, np.ones(mcmc_step.shape[0]) * v_s_rms, ':g', linewidth=2) ax.set_xlim([0, 70000]) ax.set_yscale('log') png_file = 'rms.png' plt.savefig(png_file)