def plot_mix(i): plt.clf() plt.title('Proposal density - iteration %i' %i) plot_mixture(vb.make_mixture(), 1, 2, visualize_weights=True, cmap='jet') plt.colorbar() plt.xlabel('$x_2$') plt.ylabel('$x_3$') plt.xlim(-30,10) plt.ylim(-30,30) plt.draw() return i
short_patches = pypmc.tools.patch_data(stacked_data, L=100) # run hierarchical clustering hc = pypmc.mix_adapt.hierarchical.Hierarchical(short_patches, long_patches, verbose=True) hc.run() hcmix = hc.g # run variational bayes with mixture --> use the long patches as initial guess merge = pypmc.mix_adapt.variational.VBMerge(short_patches, len(stacked_data), initial_guess=long_patches) merge.run(1000, abs_tol=1e-5, rel_tol=1e-10, prune=.5*len(vb.data)/vb.K, verbose=True) # Note: ``prune`` should be ``vb.N/vb.K`` but ``.5*len(vb.data)/vb.K`` is smaller (due to taking every 50th sample only) # and still merge tends to prune too much --> overall message remains valid mergemix = merge.make_mixture() # plot the mixtures from these algorithms plt.figure(); plt.title('Hierarchical'); plot_mixture(hc.g) plt.figure(); plt.title('VB samples'); plot_mixture(vbmix) plt.figure(); plt.title('VB mixtures'); plot_mixture(mergemix) plt.draw() # define an importance sampler for each proposal and run 10**5 steps hc_sampler = pypmc.sampler.importance_sampling.DeterministicIS(log_target, hcmix, std_weights=True) vb_sampler = pypmc.sampler.importance_sampling.DeterministicIS(log_target, vbmix, std_weights=True) merge_sampler = pypmc.sampler.importance_sampling.DeterministicIS(log_target, mergemix, std_weights=True) hc_sampler.run(10**5) vb_sampler.run(10**5) merge_sampler.run(10**5) # plot importance samples plt.figure(); plt.title('Hierarchical') plt.hist2d(hc_sampler.history[:][:,1], hc_sampler.history[:][:,2], weights=hc_sampler.history[:][:,0], cmap='gray_r', bins=100)
# data points N = 500 data = np.random.normal(size=2 * N).reshape(N, 2) # maximum number of components in mixture K = 6 vb = GaussianInference(data, components=K, alpha=10 * np.ones(K), nu=3 * np.ones(K)) # plot data and initial guess plt.subplot(1, 2, 1) plt.scatter(data[:, 0], data[:, 1], color='gray') initial_mix = vb.make_mixture() plot_mixture(initial_mix, cmap='gist_rainbow') x_range = (-4, 4) y_range = x_range plt.xlim(x_range) plt.ylim(y_range) plt.gca().set_aspect('equal') plt.title('Initial') # compute variational Bayes posterior vb.run(prune=0.5 * len(data) / K, verbose=True) # obtain most probable mixture and plot it mix = vb.make_mixture() plt.subplot(1, 2, 2) plt.scatter(data[:, 0], data[:, 1], color='gray') plt.xlim(x_range)
# form the "long_patches" long_patches = pypmc.mix_adapt.r_value.make_r_gaussmix(mcmc_data) # form first proposal with PMC print 'running PMC' pmcmix = pypmc.mix_adapt.pmc.gaussian_pmc(stacked_data[::100], long_patches, copy=True) pmcmix.prune(.5/len(long_patches)) for i in range(1000-1): print i pypmc.mix_adapt.pmc.gaussian_pmc(stacked_data[::100], pmcmix, copy=False) pmcmix.prune(.5/len(long_patches)) plt.figure() plot_mixture(pmcmix) plt.xlim(-6,6) plt.ylim(-6,6) plt.xlabel('$x_1$') plt.ylabel('$x_2$') plt.savefig('first_prop_pmc_7500.pdf') print 'PMC done' # form first proposal with Variational Bayes # run variational bayes with samples --> use the long patches as initial guess vb = pypmc.mix_adapt.variational.GaussianInference(stacked_data[::100], initial_guess=long_patches) print 'running VB...' vb.run(1000, abs_tol=1e-5, rel_tol=1e-10, prune=.5*len(vb.data)/vb.K, verbose=True) print 'VB done'
plt.title('Markov Chain 2') plt.hexbin(mcmc_data[1][:,1], mcmc_data[1][:,2], cmap='gray_r', extent=(-30,10,-30,30)) plt.xlabel('$x_2$') plt.ylabel('$x_3$') plt.draw() # form the "long_patches" long_patches = pypmc.mix_adapt.r_value.make_r_gaussmix(mcmc_data) # run variational bayes with samples --> use the long patches as initial guess vb = pypmc.mix_adapt.variational.GaussianInference(stacked_data[::50], initial_guess=long_patches) vb.run(1000, abs_tol=1e-5, rel_tol=1e-10, prune=.5*len(vb.data)/vb.K, verbose=True) vbmix = vb.make_mixture() plt.figure() plt.title('Proposal density') plot_mixture(vbmix, 1, 2) plt.xlabel('$x_2$') plt.ylabel('$x_3$') plt.xlim(-30,10) plt.ylim(-30,30) plt.draw() # define an importance sampler for each proposal and run 10**5 steps vb_sampler = pypmc.sampler.importance_sampling.DeterministicIS(log_target, vbmix, std_weights=True) vb_sampler.run(10**5) # plot importance samples plt.figure() plt.title('Importance Samples') plt.hist2d(vb_sampler.history[:][:,2], vb_sampler.history[:][:,3], weights=vb_sampler.std_weights[:][:,0], cmap='gray_r', bins=100) plt.xlabel('$x_2$')
mcmcmix_components.append(pypmc.density.gauss.Gauss(mean, cov)) except np.linalg.LinAlgError: print "Could not create component:" print 'mean:', mean print 'cov:', cov statusfile.write("Could not create component:\n") statusfile.write("mean: %s\n" %mean) statusfile.write("cov: %s\n" %cov) # delete components with negative det if mcmcmix_components[-1].det_sigma <= 0.: mcmcmix_components.pop() mcmcmix = pypmc.density.mixture.MixtureDensity(mcmcmix_components) plt.figure() plt.title('from chains') plot_mixture(mcmcmix, 0,1) plotfile.savefig() # --------------------------- end of markov chain --------------------------------------- # --------------------------- chain grouping -------------------------------------------- chain_groups = pypmc.mix_adapt.r_value.r_group([np.mean(mc.history[:], axis=0) for mc in mcs], [np.cov(mc.history[:], rowvar=0) for mc in mcs], len(val), critical_r) statusfile.write('found %i chain grous\n' %len(chain_groups) ) long_patches_means = [] long_patches_covs = [] for group in chain_groups: