def test_slice_theta_mm(): N = 100 data = np.array([(np.random.random() < 0.8, ) for _ in xrange(N)], dtype=[('', bool)]) defn = model_definition(N, [bbnc]) r = rng() prior = {'alpha': 1.0, 'beta': 9.0} view = numpy_dataview(data) s = initialize(defn, view, cluster_hp={ 'alpha': 1., 'beta': 9. }, feature_hps=[prior], r=r, assignment=[0] * N) heads = len([1 for y in data if y[0]]) tails = N - heads alpha1 = prior['alpha'] + heads beta1 = prior['beta'] + tails bs = bind(s, view) params = {0: {'p': 0.05}} def sample_fn(): theta(bs, r, tparams=params) return s.get_suffstats(0, 0)['p'] rv = beta(alpha1, beta1) assert_1d_cont_dist_approx_sps(sample_fn, rv, nsamples=50000)
def _test_convergence_bb_cxx(N, D, kernel, preprocess_data_fn=None, nonconj=False, burnin_niters=10000, skip=10, ntries=50, nsamples=1000, kl_places=2): r = rng() cluster_hp = {'alpha': 2.0} feature_hps = [{'alpha': 1.0, 'beta': 1.0}] * D defn = model_definition(N, [bb] * D) nonconj_defn = model_definition(N, [bbnc] * D) Y, posterior = data_with_posterior( defn, cluster_hp, feature_hps, preprocess_data_fn) data = numpy_dataview(Y) s = initialize(nonconj_defn if nonconj else defn, data, cluster_hp=cluster_hp, feature_hps=feature_hps, r=r) bs = bind(s, data) wrapped_kernel = lambda s: kernel(s, r) _test_convergence(bs, posterior, wrapped_kernel, burnin_niters, skip, ntries, nsamples, kl_places)
def _test_convergence_bb_cxx(N, D, kernel, preprocess_data_fn=None, nonconj=False, burnin_niters=10000, skip=10, ntries=50, nsamples=1000, kl_places=2): r = rng() cluster_hp = {'alpha': 2.0} feature_hps = [{'alpha': 1.0, 'beta': 1.0}] * D defn = model_definition(N, [bb] * D) nonconj_defn = model_definition(N, [bbnc] * D) Y, posterior = data_with_posterior(defn, cluster_hp, feature_hps, preprocess_data_fn) data = numpy_dataview(Y) s = initialize(nonconj_defn if nonconj else defn, data, cluster_hp=cluster_hp, feature_hps=feature_hps, r=r) bs = bind(s, data) wrapped_kernel = lambda s: kernel(s, r) _test_convergence(bs, posterior, wrapped_kernel, burnin_niters, skip, ntries, nsamples, kl_places)
def run(self, r, niters=10000): """Run the specified mixturemodel kernel for `niters`, in a single thread. Parameters ---------- r : random state niters : int """ validator.validate_type(r, rng, param_name='r') validator.validate_positive(niters, param_name='niters') model = bind(self._latent, self._view) for _ in xrange(niters): for name, config in self._kernel_config: if name == 'assign': gibbs.assign(model, r) elif name == 'assign_resample': gibbs.assign_resample(model, config['m'], r) elif name == 'grid_feature_hp': gibbs.hp(model, config, r) elif name == 'slice_feature_hp': slice.hp(model, r, hparams=config['hparams']) elif name == 'slice_cluster_hp': slice.hp(model, r, cparam=config['cparam']) elif name == 'theta': slice.theta(model, r, tparams=config['tparams']) else: assert False, "should not be reach"
def test_slice_theta_mm(): N = 100 data = np.array( [(np.random.random() < 0.8,) for _ in xrange(N)], dtype=[('', bool)]) defn = model_definition(N, [bbnc]) r = rng() prior = {'alpha': 1.0, 'beta': 9.0} view = numpy_dataview(data) s = initialize( defn, view, cluster_hp={'alpha': 1., 'beta': 9.}, feature_hps=[prior], r=r, assignment=[0] * N) heads = len([1 for y in data if y[0]]) tails = N - heads alpha1 = prior['alpha'] + heads beta1 = prior['beta'] + tails bs = bind(s, view) params = {0: {'p': 0.05}} def sample_fn(): theta(bs, r, tparams=params) return s.get_suffstats(0, 0)['p'] rv = beta(alpha1, beta1) assert_1d_cont_dist_approx_sps(sample_fn, rv, nsamples=50000)
def _test_scalar_hp_inference(view, prior_fn, w, grid_min, grid_max, grid_n, likelihood_model, scalar_hp_key, burnin=1000, nsamples=1000, every=10, trials=100, places=2): """ view must be 1D """ r = rng() hparams = {0: {scalar_hp_key: (prior_fn, w)}} def score_fn(scalar): d = latent.get_feature_hp(0) prev_scalar = d[scalar_hp_key] d[scalar_hp_key] = scalar latent.set_feature_hp(0, d) score = prior_fn(scalar) + latent.score_data(0, None, r) d[scalar_hp_key] = prev_scalar latent.set_feature_hp(0, d) return score defn = model_definition(len(view), [likelihood_model]) latent = initialize(defn, view, r=r) model = bind(latent, view) def sample_fn(): for _ in xrange(every): slice_hp(model, r, hparams=hparams) return latent.get_feature_hp(0)[scalar_hp_key] for _ in xrange(burnin): slice_hp(model, r, hparams=hparams) print 'finished burnin of', burnin, 'iterations' print 'grid_min', grid_min, 'grid_max', grid_max assert_1d_cont_dist_approx_emp(sample_fn, score_fn, grid_min, grid_max, grid_n, trials, nsamples, places)
def latent(groups, entities_per_group, features, r): N = groups * entities_per_group defn = model_definition(N, [bb] * features) # generate fake data Y = np.random.random(size=(N, features)) <= 0.5 view = numpy_dataview( np.array([tuple(y) for y in Y], dtype=[('', bool)] * features)) # assign entities to their respective groups assignment = [[g] * entities_per_group for g in xrange(groups)] assignment = list(it.chain.from_iterable(assignment)) latent = bind(initialize(defn, view, r, assignment=assignment), view) latent.create_group(r) # perftest() doesnt modify group assignments return latent
def _test_multivariate_models(initialize_fn, dataview, bind, gibbs_assign, R): # XXX: this test only checks that the operations don't crash mu = np.ones(3) kappa = 0.3 Q = random_orthonormal_matrix(3) psi = np.dot(Q, np.dot(np.diag([1.0, 0.5, 0.2]), Q.T)) nu = 6 N = 10 def genrow(): return tuple([ np.random.choice([False, True]), [np.random.uniform(-3.0, 3.0) for _ in xrange(3)] ]) X = np.array([genrow() for _ in xrange(N)], dtype=[('', bool), ('', float, (3, ))]) view = dataview(X) defn = model_definition(N, [bb, niw(3)]) s = initialize_fn(defn, view, cluster_hp={'alpha': 2.}, feature_hps=[{ 'alpha': 2., 'beta': 2. }, { 'mu': mu, 'kappa': kappa, 'psi': psi, 'nu': nu }], r=R) bound_s = bind(s, view) for _ in xrange(10): gibbs_assign(bound_s, R)
def _test_multivariate_models(initialize_fn, dataview, bind, gibbs_assign, R): # XXX: this test only checks that the operations don't crash mu = np.ones(3) kappa = 0.3 Q = random_orthonormal_matrix(3) psi = np.dot(Q, np.dot(np.diag([1.0, 0.5, 0.2]), Q.T)) nu = 6 N = 10 def genrow(): return tuple( [np.random.choice([False, True]), [np.random.uniform(-3.0, 3.0) for _ in xrange(3)]]) X = np.array([genrow() for _ in xrange(N)], dtype=[('', bool), ('', float, (3,))]) view = dataview(X) defn = model_definition(N, [bb, niw(3)]) s = initialize_fn( defn, view, cluster_hp={'alpha': 2.}, feature_hps=[ {'alpha': 2., 'beta': 2.}, {'mu': mu, 'kappa': kappa, 'psi': psi, 'nu': nu} ], r=R) bound_s = bind(s, view) for _ in xrange(10): gibbs_assign(bound_s, R)
def test_mnist_supervised(): mnist_dataset = _get_mnist_dataset() classes = range(10) classmap = {c: i for i, c in enumerate(classes)} train_data, test_data = [], [] for c in classes: Y = mnist_dataset['data'][ np.where(mnist_dataset['target'] == float(c))[0]] Y_train, Y_test = train_test_split(Y, test_size=0.01) train_data.append(Y_train) test_data.append(Y_test) sample_size_max = 10000 def mk_class_data(c, Y): n, D = Y.shape print 'number of digit', c, 'in training is', n dtype = [('', bool)] * D + [('', int)] inds = np.random.permutation(Y.shape[0])[:sample_size_max] Y = np.array([tuple(list(y) + [classmap[c]]) for y in Y[inds]], dtype=dtype) return Y Y_train = np.hstack([mk_class_data(c, y) for c, y in zip(classes, train_data)]) Y_train = Y_train[np.random.permutation(np.arange(Y_train.shape[0]))] n, = Y_train.shape D = len(Y_train.dtype) print 'training data is', n, 'examples' print 'image dimension is', (D - 1), 'pixels' view = numpy_dataview(Y_train) defn = model_definition(n, [bb] * (D - 1) + [dd(len(classes))]) r = rng() s = initialize(defn, view, cluster_hp={'alpha': 0.2}, feature_hps=[{'alpha': 1., 'beta': 1.}] * (D - 1) + [{'alphas': [1. for _ in classes]}], r=r) bound_s = bind(s, view) indiv_prior_fn = log_exponential(1.2) hparams = { i: { 'alpha': (indiv_prior_fn, 1.5), 'beta': (indiv_prior_fn, 1.5), } for i in xrange(D - 1)} hparams[D - 1] = { 'alphas[{}]'.format(idx): (indiv_prior_fn, 1.5) for idx in xrange(len(classes)) } def print_prediction_results(): results = [] for c, Y_test in zip(classes, test_data): for y in Y_test: query = ma.masked_array( np.array([tuple(y) + (0,)], dtype=[('', bool)] * (D - 1) + [('', int)]), mask=[(False,) * (D - 1) + (True,)])[0] samples = [ s.sample_post_pred(query, r)[1][0][-1] for _ in xrange(30)] samples = np.bincount(samples, minlength=len(classes)) prediction = np.argmax(samples) results.append((classmap[c], prediction, samples)) print 'finished predictions for class', c Y_actual = np.array([a for a, _, _ in results], dtype=np.int) Y_pred = np.array([b for _, b, _ in results], dtype=np.int) print 'accuracy:', accuracy_score(Y_actual, Y_pred) print 'confusion matrix:' print confusion_matrix(Y_actual, Y_pred) # AUROC for one vs all (each class) for i, clabel in enumerate(classes): Y_true = np.copy(Y_actual) # treat class c as the "positive" example positive_examples = Y_actual == i negative_examples = Y_actual != i Y_true[positive_examples] = 1 Y_true[negative_examples] = 0 Y_prob = np.array([float(c[i]) / c.sum() for _, _, c in results]) cls_auc = roc_auc_score(Y_true, Y_prob) print 'class', clabel, 'auc=', cls_auc #import matplotlib.pylab as plt #Y_prob = np.array([c for _, _, c in results]) #fpr, tpr, thresholds = roc_curve(Y_actual, Y_prob, pos_label=0) #plt.plot(fpr, tpr) #plt.show() def kernel(rid): start0 = time.time() assign(bound_s, r) sec0 = time.time() - start0 start1 = time.time() hp(bound_s, r, hparams=hparams) sec1 = time.time() - start1 print 'rid=', rid, 'nclusters=', s.ngroups(), \ 'iter0=', sec0, 'sec', 'iter1=', sec1, 'sec' sec_per_post_pred = sec0 / (float(view.size()) * (float(s.ngroups()))) print ' time_per_post_pred=', sec_per_post_pred, 'sec' # print group size breakdown sizes = [(gid, s.groupsize(gid)) for gid in s.groups()] sizes = sorted(sizes, key=lambda x: x[1], reverse=True) print ' group_sizes=', sizes print_prediction_results() # save state mkdirp("mnist-states") fname = os.path.join("mnist-states", "state-iter{}.ser".format(rid)) with open(fname, "w") as fp: fp.write(s.serialize()) # training iters = 30 for rid in xrange(iters): kernel(rid)
def test_mnist(): import matplotlib.pylab as plt from PIL import Image, ImageOps mnist_dataset = _get_mnist_dataset() Y_2 = mnist_dataset['data'][np.where(mnist_dataset['target'] == 2.)[0]] Y_3 = mnist_dataset['data'][np.where(mnist_dataset['target'] == 3.)[0]] print 'number of twos:', Y_2.shape[0] print 'number of threes:', Y_3.shape[0] _, D = Y_2.shape W = int(math.sqrt(D)) assert W * W == D dtype = [('', bool)] * D Y = np.vstack([Y_2, Y_3]) Y = np.array( [tuple(y) for y in Y[np.random.permutation(np.arange(Y.shape[0]))]], dtype=dtype) view = numpy_dataview(Y) defn = model_definition(Y.shape[0], [bb] * D) r = rng() s = initialize( defn, view, cluster_hp={'alpha': 0.2}, feature_hps=[{'alpha': 1., 'beta': 1.}] * D, r=r) bound_s = bind(s, view) indiv_prior_fn = log_exponential(1.2) hparams = { i: { 'alpha': (indiv_prior_fn, 1.5), 'beta': (indiv_prior_fn, 1.5), } for i in xrange(D)} def plot_clusters(s, fname, scalebysize=False): hps = [s.get_feature_hp(i) for i in xrange(D)] def prior_prob(hp): return hp['alpha'] / (hp['alpha'] + hp['beta']) def data_for_group(gid): suffstats = [s.get_suffstats(gid, i) for i in xrange(D)] def prob(hp, ss): top = hp['alpha'] + ss['heads'] bot = top + hp['beta'] + ss['tails'] return top / bot probs = [prob(hp, ss) for hp, ss in zip(hps, suffstats)] return np.array(probs) def scale(d, weight): im = d.reshape((W, W)) newW = max(int(weight * W), 1) im = Image.fromarray(im) im = im.resize((newW, newW)) im = ImageOps.expand(im, border=(W - newW) / 2) im = np.array(im) a, b = im.shape #print 'a,b:', a, b if a < W: im = np.append(im, np.zeros(b)[np.newaxis, :], axis=0) elif a > W: im = im[:W, :] assert im.shape[0] == W if b < W: #print 'current:', im.shape im = np.append(im, np.zeros(W)[:, np.newaxis], axis=1) elif b > W: im = im[:, :W] assert im.shape[1] == W return im.flatten() data = [(data_for_group(g), cnt) for g, cnt in groupsbysize(s)] largest = max(cnt for _, cnt in data) data = [scale(d, cnt / float(largest)) if scalebysize else d for d, cnt in data] digits_per_row = 12 rem = len(data) % digits_per_row if rem: fill = digits_per_row - rem for _ in xrange(fill): data.append(np.zeros(D)) assert not (len(data) % digits_per_row) #rows = len(data) / digits_per_row data = np.vstack([np.hstack([d.reshape((W, W)) for d in data[i:i + digits_per_row]]) for i in xrange(0, len(data), digits_per_row)]) #print 'saving figure', fname plt.imshow(data, cmap=plt.cm.binary, interpolation='nearest') plt.savefig(fname) plt.close() def plot_hyperparams(s, fname): hps = [s.get_feature_hp(i) for i in xrange(D)] alphas = np.array([hp['alpha'] for hp in hps]) betas = np.array([hp['beta'] for hp in hps]) data = np.hstack([alphas.reshape((W, W)), betas.reshape((W, W))]) plt.imshow(data, interpolation='nearest') plt.colorbar() plt.savefig(fname) plt.close() def kernel(rid): start0 = time.time() assign(bound_s, r) sec0 = time.time() - start0 start1 = time.time() hp(bound_s, r, hparams=hparams) sec1 = time.time() - start1 print 'rid=', rid, 'nclusters=', s.ngroups(), \ 'iter0=', sec0, 'sec', 'iter1=', sec1, 'sec' sec_per_post_pred = sec0 / (float(view.size()) * (float(s.ngroups()))) print ' time_per_post_pred=', sec_per_post_pred, 'sec' return s.score_joint(r) # burnin burnin = 20 for rid in xrange(burnin): print 'score:', kernel(rid) print 'finished burnin' plot_clusters(s, 'mnist_clusters.pdf') plot_clusters(s, 'mnist_clusters_bysize.pdf', scalebysize=True) plot_hyperparams(s, 'mnist_hyperparams.pdf') print 'groupcounts:', groupcounts(s) # posterior predictions present = D / 2 absent = D - present queries = [tuple(Y_2[i]) for i in np.random.permutation(Y_2.shape[0])[:4]] + \ [tuple(Y_3[i]) for i in np.random.permutation(Y_3.shape[0])[:4]] queries_masked = ma.masked_array( np.array(queries, dtype=[('', bool)] * D), mask=[(False,) * present + (True,) * absent]) def postpred_sample(y_new): Y_samples = [s.sample_post_pred(y_new, r)[1] for _ in xrange(1000)] Y_samples = np.array([list(y) for y in np.hstack(Y_samples)]) Y_avg = Y_samples.mean(axis=0) return Y_avg queries_masked = [postpred_sample(y) for y in queries_masked] data0 = np.hstack([q.reshape((W, W)) for q in queries_masked]) data1 = np.hstack( [np.clip(np.array(q, dtype=np.float), 0., 1.).reshape((W, W)) for q in queries]) data = np.vstack([data0, data1]) plt.imshow(data, cmap=plt.cm.binary, interpolation='nearest') plt.savefig('mnist_predict.pdf') plt.close()
def test_mnist_supervised(n): mnist_dataset = _get_mnist_dataset() classes = range(10) classmap = {c: i for i, c in enumerate(classes)} train_data, test_data = [], [] for c in classes: Y = mnist_dataset['data'][np.where( mnist_dataset['target'] == float(c))[0]] Y_train, Y_test = train_test_split(Y, test_size=0.01) train_data.append(Y_train) test_data.append(Y_test) sample_size_max = n def mk_class_data(c, Y): n, D = Y.shape print 'number of digit', c, 'in training is', n dtype = [('', bool)] * D + [('', int)] inds = np.random.permutation(Y.shape[0])[:sample_size_max] Y = np.array([tuple(list(y) + [classmap[c]]) for y in Y[inds]], dtype=dtype) return Y Y_train = np.hstack( [mk_class_data(c, y) for c, y in zip(classes, train_data)]) Y_train = Y_train[np.random.permutation(np.arange(Y_train.shape[0]))] n, = Y_train.shape D = len(Y_train.dtype) print 'training data is', n, 'examples' print 'image dimension is', (D - 1), 'pixels' view = numpy_dataview(Y_train) defn = model_definition(n, [bb] * (D - 1) + [dd(len(classes))]) r = rng() s = initialize(defn, view, cluster_hp={'alpha': 0.2}, feature_hps=[{ 'alpha': 1., 'beta': 1. }] * (D - 1) + [{ 'alphas': [1. for _ in classes] }], r=r) bound_s = bind(s, view) indiv_prior_fn = log_exponential(1.2) hparams = { i: { 'alpha': (indiv_prior_fn, 1.5), 'beta': (indiv_prior_fn, 1.5), } for i in xrange(D - 1) } hparams[D - 1] = { 'alphas[{}]'.format(idx): (indiv_prior_fn, 1.5) for idx in xrange(len(classes)) } def print_prediction_results(): results = [] for c, Y_test in zip(classes, test_data): for y in Y_test: query = ma.masked_array( np.array([tuple(y) + (0, )], dtype=[('', bool)] * (D - 1) + [('', int)]), mask=[(False, ) * (D - 1) + (True, )])[0] samples = [ s.sample_post_pred(query, r)[1][0][-1] for _ in xrange(30) ] samples = np.bincount(samples, minlength=len(classes)) prediction = np.argmax(samples) results.append((classmap[c], prediction, samples)) print 'finished predictions for class', c Y_actual = np.array([a for a, _, _ in results], dtype=np.int) Y_pred = np.array([b for _, b, _ in results], dtype=np.int) print 'accuracy:', accuracy_score(Y_actual, Y_pred) print 'confusion matrix:' print confusion_matrix(Y_actual, Y_pred) # AUROC for one vs all (each class) for i, clabel in enumerate(classes): Y_true = np.copy(Y_actual) # treat class c as the "positive" example positive_examples = Y_actual == i negative_examples = Y_actual != i Y_true[positive_examples] = 1 Y_true[negative_examples] = 0 Y_prob = np.array([float(c[i]) / c.sum() for _, _, c in results]) cls_auc = roc_auc_score(Y_true, Y_prob) print 'class', clabel, 'auc=', cls_auc #import matplotlib.pylab as plt #Y_prob = np.array([c for _, _, c in results]) #fpr, tpr, thresholds = roc_curve(Y_actual, Y_prob, pos_label=0) #plt.plot(fpr, tpr) #plt.show() def kernel(rid): start0 = time.time() assign(bound_s, r) sec0 = time.time() - start0 start1 = time.time() hp(bound_s, r, hparams=hparams) sec1 = time.time() - start1 print 'rid=', rid, 'nclusters=', s.ngroups(), \ 'iter0=', sec0, 'sec', 'iter1=', sec1, 'sec' sec_per_post_pred = sec0 / (float(view.size()) * (float(s.ngroups()))) print ' time_per_post_pred=', sec_per_post_pred, 'sec' # training iters = 30 for rid in xrange(iters): kernel(rid) # print group size breakdown sizes = [(gid, s.groupsize(gid)) for gid in s.groups()] sizes = sorted(sizes, key=lambda x: x[1], reverse=True) print ' group_sizes=', sizes #print_prediction_results() # save state mkdirp("mnist-states") fname = os.path.join("mnist-states", "state-iter{}.ser".format(rid)) with open(fname, "w") as fp: fp.write(s.serialize())
def _test_nonconj_inference(initialize_fn, dataview, bind, assign_nonconj_fn, slice_theta_fn, R, ntries, nsamples, tol): N, D = 1000, 5 defn = model_definition(N, [bbnc] * D) cluster_hp = {'alpha': 0.2} feature_hps = [{'alpha': 1.0, 'beta': 1.0}] * D while True: Y_clustered, cluster_samplers = sample( defn, cluster_hp, feature_hps, R) if len(Y_clustered) == 2: break dominant = np.argmax(map(len, Y_clustered)) truth = np.array([s.p for s in cluster_samplers[dominant]]) print 'truth:', truth # see if we can learn the p-values for each of the two clusters. we proceed # by running gibbs_assign_nonconj, followed by slice sampling on the # posterior p(\theta | Y). we'll "cheat" a little by bootstrapping the # DP with the correct assignment (but not with the correct p-values) Y, assignment = data_with_assignment(Y_clustered) view = dataview(Y) s = initialize_fn( defn, view, cluster_hp=cluster_hp, feature_hps=feature_hps, assignment=assignment, r=R) bs = bind(s, view) def mkparam(): return {'p': 0.1} thetaparams = {fi: mkparam() for fi in xrange(D)} def kernel(): assign_nonconj_fn(bs, 10, R) slice_theta_fn(bs, R, tparams=thetaparams) def inference(niters): for _ in xrange(niters): kernel() groups = s.groups() inferred_dominant = groups[ np.argmax([s.groupsize(gid) for gid in groups])] inferred = [s.get_suffstats(inferred_dominant, d)['p'] for d in xrange(D)] inferred = np.array(inferred) yield inferred posterior = [] while ntries: samples = list(inference(nsamples)) posterior.extend(samples) inferred = sum(posterior) / len(posterior) diff = np.linalg.norm(truth - inferred) print 'inferred:', inferred print 'diff:', diff if diff <= tol: return ntries -= 1 print 'tries left:', ntries assert False, 'did not converge'
def _test_nonconj_inference(initialize_fn, dataview, bind, assign_nonconj_fn, slice_theta_fn, R, ntries, nsamples, tol): N, D = 1000, 5 defn = model_definition(N, [bbnc] * D) cluster_hp = {'alpha': 0.2} feature_hps = [{'alpha': 1.0, 'beta': 1.0}] * D while True: Y_clustered, cluster_samplers = sample(defn, cluster_hp, feature_hps, R) if len(Y_clustered) == 2: break dominant = np.argmax(map(len, Y_clustered)) truth = np.array([s.p for s in cluster_samplers[dominant]]) print 'truth:', truth # see if we can learn the p-values for each of the two clusters. we proceed # by running gibbs_assign_nonconj, followed by slice sampling on the # posterior p(\theta | Y). we'll "cheat" a little by bootstrapping the # DP with the correct assignment (but not with the correct p-values) Y, assignment = data_with_assignment(Y_clustered) view = dataview(Y) s = initialize_fn(defn, view, cluster_hp=cluster_hp, feature_hps=feature_hps, assignment=assignment, r=R) bs = bind(s, view) def mkparam(): return {'p': 0.1} thetaparams = {fi: mkparam() for fi in xrange(D)} def kernel(): assign_nonconj_fn(bs, 10, R) slice_theta_fn(bs, R, tparams=thetaparams) def inference(niters): for _ in xrange(niters): kernel() groups = s.groups() inferred_dominant = groups[np.argmax( [s.groupsize(gid) for gid in groups])] inferred = [ s.get_suffstats(inferred_dominant, d)['p'] for d in xrange(D) ] inferred = np.array(inferred) yield inferred posterior = [] while ntries: samples = list(inference(nsamples)) posterior.extend(samples) inferred = sum(posterior) / len(posterior) diff = np.linalg.norm(truth - inferred) print 'inferred:', inferred print 'diff:', diff if diff <= tol: return ntries -= 1 print 'tries left:', ntries assert False, 'did not converge'
def test_mnist(): import matplotlib.pylab as plt from PIL import Image, ImageOps mnist_dataset = _get_mnist_dataset() Y_2 = mnist_dataset['data'][np.where(mnist_dataset['target'] == 2.)[0]] Y_3 = mnist_dataset['data'][np.where(mnist_dataset['target'] == 3.)[0]] print 'number of twos:', Y_2.shape[0] print 'number of threes:', Y_3.shape[0] _, D = Y_2.shape W = int(math.sqrt(D)) assert W * W == D dtype = [('', bool)] * D Y = np.vstack([Y_2, Y_3]) Y = np.array( [tuple(y) for y in Y[np.random.permutation(np.arange(Y.shape[0]))]], dtype=dtype) view = numpy_dataview(Y) defn = model_definition(Y.shape[0], [bb] * D) r = rng() s = initialize(defn, view, cluster_hp={'alpha': 0.2}, feature_hps=[{ 'alpha': 1., 'beta': 1. }] * D, r=r) bound_s = bind(s, view) indiv_prior_fn = log_exponential(1.2) hparams = { i: { 'alpha': (indiv_prior_fn, 1.5), 'beta': (indiv_prior_fn, 1.5), } for i in xrange(D) } def plot_clusters(s, fname, scalebysize=False): hps = [s.get_feature_hp(i) for i in xrange(D)] def prior_prob(hp): return hp['alpha'] / (hp['alpha'] + hp['beta']) def data_for_group(gid): suffstats = [s.get_suffstats(gid, i) for i in xrange(D)] def prob(hp, ss): top = hp['alpha'] + ss['heads'] bot = top + hp['beta'] + ss['tails'] return top / bot probs = [prob(hp, ss) for hp, ss in zip(hps, suffstats)] return np.array(probs) def scale(d, weight): im = d.reshape((W, W)) newW = max(int(weight * W), 1) im = Image.fromarray(im) im = im.resize((newW, newW)) im = ImageOps.expand(im, border=(W - newW) / 2) im = np.array(im) a, b = im.shape #print 'a,b:', a, b if a < W: im = np.append(im, np.zeros(b)[np.newaxis, :], axis=0) elif a > W: im = im[:W, :] assert im.shape[0] == W if b < W: #print 'current:', im.shape im = np.append(im, np.zeros(W)[:, np.newaxis], axis=1) elif b > W: im = im[:, :W] assert im.shape[1] == W return im.flatten() data = [(data_for_group(g), cnt) for g, cnt in groupsbysize(s)] largest = max(cnt for _, cnt in data) data = [ scale(d, cnt / float(largest)) if scalebysize else d for d, cnt in data ] digits_per_row = 12 rem = len(data) % digits_per_row if rem: fill = digits_per_row - rem for _ in xrange(fill): data.append(np.zeros(D)) assert not (len(data) % digits_per_row) #rows = len(data) / digits_per_row data = np.vstack([ np.hstack([d.reshape((W, W)) for d in data[i:i + digits_per_row]]) for i in xrange(0, len(data), digits_per_row) ]) #print 'saving figure', fname plt.imshow(data, cmap=plt.cm.binary, interpolation='nearest') plt.savefig(fname) plt.close() def plot_hyperparams(s, fname): hps = [s.get_feature_hp(i) for i in xrange(D)] alphas = np.array([hp['alpha'] for hp in hps]) betas = np.array([hp['beta'] for hp in hps]) data = np.hstack([alphas.reshape((W, W)), betas.reshape((W, W))]) plt.imshow(data, interpolation='nearest') plt.colorbar() plt.savefig(fname) plt.close() def kernel(rid): start0 = time.time() assign(bound_s, r) sec0 = time.time() - start0 start1 = time.time() hp(bound_s, r, hparams=hparams) sec1 = time.time() - start1 print 'rid=', rid, 'nclusters=', s.ngroups(), \ 'iter0=', sec0, 'sec', 'iter1=', sec1, 'sec' sec_per_post_pred = sec0 / (float(view.size()) * (float(s.ngroups()))) print ' time_per_post_pred=', sec_per_post_pred, 'sec' return s.score_joint(r) # burnin burnin = 20 for rid in xrange(burnin): print 'score:', kernel(rid) print 'finished burnin' plot_clusters(s, 'mnist_clusters.pdf') plot_clusters(s, 'mnist_clusters_bysize.pdf', scalebysize=True) plot_hyperparams(s, 'mnist_hyperparams.pdf') print 'groupcounts:', groupcounts(s) # posterior predictions present = D / 2 absent = D - present queries = [tuple(Y_2[i]) for i in np.random.permutation(Y_2.shape[0])[:4]] + \ [tuple(Y_3[i]) for i in np.random.permutation(Y_3.shape[0])[:4]] queries_masked = ma.masked_array(np.array(queries, dtype=[('', bool)] * D), mask=[(False, ) * present + (True, ) * absent]) def postpred_sample(y_new): Y_samples = [s.sample_post_pred(y_new, r)[1] for _ in xrange(1000)] Y_samples = np.array([list(y) for y in np.hstack(Y_samples)]) Y_avg = Y_samples.mean(axis=0) return Y_avg queries_masked = [postpred_sample(y) for y in queries_masked] data0 = np.hstack([q.reshape((W, W)) for q in queries_masked]) data1 = np.hstack([ np.clip(np.array(q, dtype=np.float), 0., 1.).reshape((W, W)) for q in queries ]) data = np.vstack([data0, data1]) plt.imshow(data, cmap=plt.cm.binary, interpolation='nearest') plt.savefig('mnist_predict.pdf') plt.close()