s_rng = RandomStreams(123) nr_words = 4 nr_topics = 2 alpha = 0.8 beta = 1. # Topic distribution per document doc_mixture = memoized(lambda doc_id: s_rng.dirichlet([alpha/nr_topics]*nr_topics)) # Word distribution per topic topic_mixture = memoized(lambda top_id: s_rng.dirichlet([beta/nr_words]*nr_words)) # For each word in the document, draw a topic according to multinomial with document specific prior # TODO, see comment below: topics = memoized(lambda doc_id, nr: s_rng.multinomial(1, doc_mixture[doc_id], draw_shape=(nr,))) topics = memoized(lambda doc_id, nr: s_rng.binomial(1, doc_mixture(doc_id)[0], draw_shape=(nr,))) # Draw words for a specific topic word_topic = lambda top_id: s_rng.multinomial(1, topic_mixture(top_id)) # TODO: memoized only works on the pre-compiled graph. This makes it fail in the case where we have to map # a vector of topics to individual multinomials with as priors the different topics. In the case of two topics # we can hack around this by using a binomial topic distribution and using a switch statement here: word_topic_mapper = lambda top_id: tensor.switch(top_id, word_topic(0), word_topic(1)) # Maps topics to words # TODO, see comment above: get_words = memoized(lambda doc_id, nr: theano.map(word_topic, topics(doc_id, nr))[0]) get_words = memoized(lambda doc_id, nr: theano.map(word_topic_mapper, topics(doc_id, nr))[0]) # Define training 'documents' document_1 = numpy.asarray([[1,0,0,0],
from sample import hybridmc_sample from rv import full_log_likelihood from max_lik import likelihood_gradient s_rng = RandomStreams(3424) # Weight prior: w = s_rng.normal(0, 2, draw_shape=(3,)) # Linear model: x = tensor.matrix('x') y = tensor.nnet.sigmoid(tensor.dot(x, w)) # Bernouilli observation model: t = s_rng.binomial(p=y, draw_shape=(4,)) # Some data: X_data = numpy.asarray([[-1.5, -0.4, 1.3, 2.2], [-1.1, -2.2, 1.3, 0], [1., 1., 1., 1.]], dtype=theano.config.floatX).T Y_data = numpy.asarray([1., 1., 0., 0.], dtype=theano.config.floatX) # Compute gradient updates: observations = dict([(t, Y_data)]) params, updates, log_likelihood = likelihood_gradient(observations) # Compile training function and assign input data as givens: givens = dict([(x, X_data)]) train = theano.function([], [log_likelihood], givens=givens, updates=updates) # Run 100 epochs of training: for i in range(100):
nr_words = 4 nr_topics = 2 alpha = 0.8 beta = 1. # Topic distribution per document doc_mixture = memoized( lambda doc_id: s_rng.dirichlet([alpha / nr_topics] * nr_topics)) # Word distribution per topic topic_mixture = memoized( lambda top_id: s_rng.dirichlet([beta / nr_words] * nr_words)) # For each word in the document, draw a topic according to multinomial with document specific prior # TODO, see comment below: topics = memoized(lambda doc_id, nr: s_rng.multinomial(1, doc_mixture[doc_id], draw_shape=(nr,))) topics = memoized(lambda doc_id, nr: s_rng.binomial( 1, doc_mixture(doc_id)[0], draw_shape=(nr, ))) # Draw words for a specific topic word_topic = lambda top_id: s_rng.multinomial(1, topic_mixture(top_id)) # TODO: memoized only works on the pre-compiled graph. This makes it fail in the case where we have to map # a vector of topics to individual multinomials with as priors the different topics. In the case of two topics # we can hack around this by using a binomial topic distribution and using a switch statement here: word_topic_mapper = lambda top_id: tensor.switch(top_id, word_topic(0), word_topic(1)) # Maps topics to words # TODO, see comment above: get_words = memoized(lambda doc_id, nr: theano.map(word_topic, topics(doc_id, nr))[0]) get_words = memoized( lambda doc_id, nr: theano.map(word_topic_mapper, topics(doc_id, nr))[0])
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from rv import full_log_likelihood s_rng = RandomStreams(3424) p = s_rng.dirichlet(numpy.asarray([1, 1]))[0] m1 = s_rng.uniform(low=-5, high=5) m2 = s_rng.uniform(low=-5, high=5) v = s_rng.uniform(low=0, high=1) C = s_rng.binomial(1, p, draw_shape=(4,)) m = tensor.switch(C, m1, m2) D = s_rng.normal(m, v, draw_shape=(4,)) D_data = numpy.asarray([1, 1.2, 3, 3.4], dtype=theano.config.floatX) givens = dict([(D, D_data)]) sampler = mh2_sample(s_rng, [p, m1, m2, v], givens) samples = sampler(200, 1000, 100) print samples[0].mean(), samples[1].mean(), samples[2].mean(), samples[3].mean()
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from for_theano import evaluate from rv import full_log_likelihood s_rng = RandomStreams(23424) fair_prior = 0.999 coin_weight = tensor.switch( s_rng.binomial(1, fair_prior) > 0.5, 0.5, s_rng.dirichlet([1, 1])[0]) make_coin = lambda p, size: s_rng.binomial(1, p, draw_shape=(size, )) coin = lambda size: make_coin(coin_weight, size) for size in [1, 3, 6, 10, 20, 30, 50, 70, 100]: data = evaluate(make_coin(0.9, size)) sampler = mh2_sample(s_rng, [coin_weight], {coin(size): data}) print "nr of examples", size, ", estimated probability", sampler( nr_samples=400, burnin=20000, lag=10)[0].mean()
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from for_theano import evaluate from rv import full_log_likelihood s_rng = RandomStreams(23424) fair_prior = 0.999 coin_weight = tensor.switch(s_rng.binomial(1, fair_prior) > 0.5, 0.5, s_rng.dirichlet([1, 1])[0]) make_coin = lambda p, size: s_rng.binomial(1, p, draw_shape=(size,)) coin = lambda size: make_coin(coin_weight, size) for size in [1, 3, 6, 10, 20, 30, 50, 70, 100]: data = evaluate(make_coin(0.9, size)) sampler = mh2_sample(s_rng, [coin_weight], {coin(size) : data}) print "nr of examples", size, ", estimated probability", sampler(nr_samples=400, burnin=20000, lag=10)[0].mean()
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from rv import full_log_likelihood s_rng = RandomStreams(3424) p = s_rng.dirichlet(numpy.asarray([1, 1]))[0] m1 = s_rng.uniform(low=-5, high=5) m2 = s_rng.uniform(low=-5, high=5) v = s_rng.uniform(low=0, high=1) C = s_rng.binomial(1, p, draw_shape=(4, )) m = tensor.switch(C, m1, m2) D = s_rng.normal(m, v, draw_shape=(4, )) D_data = numpy.asarray([1, 1.2, 3, 3.4], dtype=theano.config.floatX) givens = dict([(D, D_data)]) sampler = mh2_sample(s_rng, [p, m1, m2, v], givens) samples = sampler(200, 1000, 100) print samples[0].mean(), samples[1].mean(), samples[2].mean(), samples[3].mean( )