def test_dirichlet(): R = RandomStreams(234) n = R.dirichlet(alpha=numpy.ones(10, ), draw_shape=(5, )) f = theano.function([], n) assert f().shape == (5, 10)
def test_multinomial(): R = RandomStreams(234) n = R.multinomial(5, numpy.ones(5, ) / 5, draw_shape=(2, )) f = theano.function([], n) assert f().shape == (2, 5)
def test_multinomial(): R = RandomStreams(234) n = R.multinomial(5, numpy.ones(5,)/5, draw_shape=(2,)) f = theano.function([], n) assert f().shape == (2, 5)
def test_dirichlet(): R = RandomStreams(234) n = R.dirichlet(alpha=numpy.ones(10,), draw_shape=(5,)) f = theano.function([], n) assert f().shape == (5, 10)
def setUp(self): R = RandomStreams(234) weights = tensor.dvector() mus = tensor.dvector() sigmas = tensor.dvector() draw_shape = tensor.ivector() xsca = R.GMM1(weights, mus, sigmas, draw_shape=draw_shape, ndim=0) xvec = R.GMM1(weights, mus, sigmas, draw_shape=draw_shape, ndim=1) xmat = R.GMM1(weights, mus, sigmas, draw_shape=draw_shape, ndim=2) self.__dict__.update(locals()) del self.self
def test_normal_nonscalar(): s_rng = RandomStreams(234) n = s_rng.normal() data = numpy.asarray([1, 2, 3, 4, 5]) p_data = rv.lpdf(n, data) f = theano.function([], [p_data]) pvals = f() targets = numpy.log(numpy.exp(-0.5 * (data**2)) / numpy.sqrt(2*numpy.pi)) assert numpy.allclose(pvals,targets), (pvals, targets)
def test_normal_nonscalar(): s_rng = RandomStreams(234) n = s_rng.normal() data = numpy.asarray([1, 2, 3, 4, 5]) p_data = rv.lpdf(n, data) f = theano.function([], [p_data]) pvals = f() targets = numpy.log(numpy.exp(-0.5 * (data**2)) / numpy.sqrt(2 * numpy.pi)) assert numpy.allclose(pvals, targets), (pvals, targets)
def test_uniform_w_params(): s_rng = RandomStreams(234) u = s_rng.uniform(low=-0.999, high=9.001) p0 = rv.lpdf(u, 0) p1 = rv.lpdf(u, 2) p05 = rv.lpdf(u, -1.5) pn1 = rv.lpdf(u, 10) f = theano.function([], [p0, p1, p05, pn1]) pvals = f() targets = numpy.log(numpy.asarray([.1, .1, 0, 0])) assert numpy.allclose(pvals, targets), (pvals, targets)
def test_uniform_w_params(): s_rng = RandomStreams(234) u = s_rng.uniform(low=-0.999, high=9.001) p0 = rv.lpdf(u, 0) p1 = rv.lpdf(u, 2) p05 = rv.lpdf(u, -1.5) pn1 = rv.lpdf(u, 10) f = theano.function([], [p0, p1, p05, pn1]) pvals = f() targets = numpy.log(numpy.asarray([.1, .1, 0, 0])) assert numpy.allclose(pvals,targets), (pvals, targets)
def test_uniform_simple(): s_rng = RandomStreams(234) u = s_rng.uniform() p0 = rv.lpdf(u, 0) p1 = rv.lpdf(u, 1) p05 = rv.lpdf(u, 0.5) pn1 = rv.lpdf(u, -1) f = theano.function([], [p0, p1, p05, pn1]) pvals = f() targets = numpy.log(numpy.asarray([1.0, 1.0, 1.0, 0.0])) assert numpy.allclose(pvals,targets), (pvals, targets)
def test_uniform_simple(): s_rng = RandomStreams(234) u = s_rng.uniform() p0 = rv.lpdf(u, 0) p1 = rv.lpdf(u, 1) p05 = rv.lpdf(u, 0.5) pn1 = rv.lpdf(u, -1) f = theano.function([], [p0, p1, p05, pn1]) pvals = f() targets = numpy.log(numpy.asarray([1.0, 1.0, 1.0, 0.0])) assert numpy.allclose(pvals, targets), (pvals, targets)
def setUp(self): s_rng = self.s_rng = RandomStreams(23424) p = 0.5 self.A = s_rng.binomial(1, p) self.B = s_rng.binomial(1, p) self.C = s_rng.binomial(1, p) self.D = self.A + self.B + self.C self.condition = tensor.ge(self.D, 2)
def test_normal_simple(): s_rng = RandomStreams(23) n = s_rng.normal() p0 = rv.lpdf(n, 0) p1 = rv.lpdf(n, 1) pn1 = rv.lpdf(n, -1) f = theano.function([], [p0, p1, pn1]) pvals = f() targets = numpy.asarray([ numpy.log(1.0 / numpy.sqrt(2*numpy.pi)), numpy.log(numpy.exp(-0.5) / numpy.sqrt(2*numpy.pi)), numpy.log(numpy.exp(-0.5) / numpy.sqrt(2*numpy.pi)), ]) assert numpy.allclose(pvals,targets), (pvals, targets)
def test_normal_simple(): s_rng = RandomStreams(23) n = s_rng.normal() p0 = rv.lpdf(n, 0) p1 = rv.lpdf(n, 1) pn1 = rv.lpdf(n, -1) f = theano.function([], [p0, p1, pn1]) pvals = f() targets = numpy.asarray([ numpy.log(1.0 / numpy.sqrt(2 * numpy.pi)), numpy.log(numpy.exp(-0.5) / numpy.sqrt(2 * numpy.pi)), numpy.log(numpy.exp(-0.5) / numpy.sqrt(2 * numpy.pi)), ]) assert numpy.allclose(pvals, targets), (pvals, targets)
def setUp(self): s_rng = self.s_rng = RandomStreams(23424) self.fair_prior = 0.999 self.fair_coin = s_rng.binomial(1, self.fair_prior) make_coin = lambda x: s_rng.binomial((4, ), 1, x) self.coin = make_coin(tensor.switch(self.fair_coin > 0.5, 0.5, 0.95)) self.data = tensor.as_tensor_variable([[1, 1, 1, 1]])
def test_normal_w_params(): s_rng = RandomStreams(23) n = s_rng.normal(mu=2, sigma=3) p0 = rv.lpdf(n, 0) p1 = rv.lpdf(n, 2) pn1 = rv.lpdf(n, -1) f = theano.function([], [p0, p1, pn1]) pvals = f() targets = numpy.asarray([ numpy.log(numpy.exp(-0.5 * ((2.0/3.0)**2)) / numpy.sqrt(2*numpy.pi*9.0)), numpy.log(numpy.exp(0) / numpy.sqrt(2*numpy.pi*9)), numpy.log(numpy.exp(-0.5 * ((3.0/3.0)**2)) / numpy.sqrt(2*numpy.pi*9.0)), ]) assert numpy.allclose(pvals,targets), (pvals, targets)
def setUp(self): s_rng = self.s_rng = RandomStreams(23424) self.p = tensor.scalar() self.m1 = tensor.scalar() self.m2 = tensor.scalar() self.v = tensor.scalar() self.C = s_rng.binomial(1, p) self.m = tensor.switch(self.C, self.m1, self.m2) self.D = s_rng.normal(self.m, self.v) self.D_data = tensor.as_tensor_variable([1, 1.2, 3, 3.4])
def setUp(self): s_rng = self.s_rng = RandomStreams(23424) a = 0.0 b = 1.0 c = 1.5 d = 2.0 self.M = s_rng.normal(a, b) self.V = s_rng.normal(c, d) self.V_ = abs(self.V) + .1 self.X = s_rng.normal((4, ), self.M, self.V_) self.X_data = tensor.as_tensor_variable([1, 2, 3, 2.4])
def test_normal_w_params(): s_rng = RandomStreams(23) n = s_rng.normal(mu=2, sigma=3) p0 = rv.lpdf(n, 0) p1 = rv.lpdf(n, 2) pn1 = rv.lpdf(n, -1) f = theano.function([], [p0, p1, pn1]) pvals = f() targets = numpy.asarray([ numpy.log( numpy.exp(-0.5 * ((2.0 / 3.0)**2)) / numpy.sqrt(2 * numpy.pi * 9.0)), numpy.log(numpy.exp(0) / numpy.sqrt(2 * numpy.pi * 9)), numpy.log( numpy.exp(-0.5 * ((3.0 / 3.0)**2)) / numpy.sqrt(2 * numpy.pi * 9.0)), ]) assert numpy.allclose(pvals, targets), (pvals, targets)
def test_dag_condition_bottom(): """ Test test of conditioning an upper node on a lower one """ with RandomStreams(234) as _: mu = normal(10, .1) x = normal(mu, sigma=1) post_mu = rv.condition([mu], {x: -7}) theano.printing.debugprint(post_mu) f = theano.function([], post_mu) f()
def test_dag_condition_top(): """ Easy test of conditioning """ with RandomStreams(234) as _: mu = normal(10, .1) x = normal(mu, sigma=1) post_x = rv.condition([x], {mu: -7}) theano.printing.debugprint(post_x) f = theano.function([], post_x) r = [f() for i in range(10)] assert numpy.allclose(numpy.mean(r), -7.4722755432)
def setUp(self): s_rng = self.s_rng = RandomStreams(23424) self.nr_states = 5 self.nr_obs = 3 self.observation_model = memoized( lambda state: s_rng.dirichlet([1] * self.nr_obs)) self.transition_model = memoized( lambda state: s_rng.dirichlet([1] * self.nr_states)) self.transition = lambda state: s_rng.multinomial( 1, self.tranisition_model(state)) self.observation = lambda state: s_rng.multinomial( 1, self.observation_model(state)) def transition(obs, state): return [self.observation(state), self.transition(state) ], {}, until(state == numpy.asarray([0, 0, 0, 0, 1])) [self.sampled_words, self.sampled_states], updates = scan([], [obs, state])
class Fitting1D(unittest.TestCase): def setUp(self): self.obs = tensor.as_tensor_variable( numpy.asarray([0.0, 1.01, 0.7, 0.65, 0.3])) self.rstream = RandomStreams(234) self.n = self.rstream.normal() self.u = self.rstream.uniform() def test_normal_ml(self): up = self.rstream.ml(self.n, self.obs) p = self.rstream.params(self.n) f = theano.function([], [up[p[0]], up[p[1]]]) m,v = f() assert numpy.allclose([m,v], [.532, 0.34856276335]) def test_uniform_ml(self): up = self.rstream.ml(self.u, self.obs) p = self.rstream.params(self.u) f = theano.function([], [up[p[0]], up[p[1]]]) l,h = f() assert numpy.allclose([l,h], [0.0, 1.01])
class Fitting1D(unittest.TestCase): def setUp(self): self.obs = tensor.as_tensor_variable( numpy.asarray([0.0, 1.01, 0.7, 0.65, 0.3])) self.rstream = RandomStreams(234) self.n = self.rstream.normal() self.u = self.rstream.uniform() def test_normal_ml(self): up = self.rstream.ml(self.n, self.obs) p = self.rstream.params(self.n) f = theano.function([], [up[p[0]], up[p[1]]]) m, v = f() assert numpy.allclose([m, v], [.532, 0.34856276335]) def test_uniform_ml(self): up = self.rstream.ml(self.u, self.obs) p = self.rstream.params(self.u) f = theano.function([], [up[p[0]], up[p[1]]]) l, h = f() assert numpy.allclose([l, h], [0.0, 1.01])
def setUp(self): s_rng = self.s_rng = RandomStreams(23424) self.repetitions = 100 self.coin_weight = s_rng.uniform(low=0, high=1) self.coin = s_rng.binomial((self.repetitions, ), 1, self.coin_weight)
def setUp(self): self.obs = tensor.as_tensor_variable( numpy.asarray([0.0, 1.01, 0.7, 0.65, 0.3])) self.rstream = RandomStreams(234) self.n = self.rstream.normal() self.u = self.rstream.uniform()
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from rv import full_log_likelihood from for_theano import evaluate s_rng = RandomStreams(3424) def poly_expansion(x, order): x = x.T result, updates = theano.scan(fn=lambda prior_result, x: prior_result * x, outputs_info=tensor.ones_like(x), non_sequences=x, n_steps=order) return tensor.concatenate([tensor.ones([x.shape[1],1]), tensor.reshape(result.T, (x.shape[1], x.shape[0]*order))], axis=1) # Define priors to be inverse gamma distributions alpha = 1/s_rng.gamma(1., 2.) beta = 1/s_rng.gamma(1., .1) # Order of the model # TODO: this currently has to be fixed, would be nice if this could also be a RV! m = 7 #s_rng.random_integers(1, 10) w = s_rng.normal(0, beta, draw_shape=(m+1,)) # Input variable used for training x = tensor.matrix('x')
import numpy, pylab import theano from rstreams import RandomStreams import distributions from sample import mh2_sample from for_theano import memoized s_rng = RandomStreams(23424) phi = s_rng.dirichlet(numpy.asarray([1, 1, 1, 1, 1])) alpha = s_rng.gamma(2., 2.) prototype = phi*alpha bag_prototype = memoized(lambda bag: s_rng.dirichlet(prototype)) draw_marbles = lambda bag, nr: s_rng.multinomial(1, bag_prototype(bag), draw_shape=(nr,)) marbles_bag_1 = numpy.asarray([[1,1,1,1,1,1], [0,0,0,0,0,0], [0,0,0,0,0,0], [0,0,0,0,0,0], [0,0,0,0,0,0]], dtype=theano.config.floatX).T marbles_bag_2 = numpy.asarray([[0,0,0,0,0,0], [1,1,1,1,1,1], [0,0,0,0,0,0], [0,0,0,0,0,0], [0,0,0,0,0,0]], dtype=theano.config.floatX).T marbles_bag_3 = numpy.asarray([[0,0,0,0,0,0], [0,0,0,0,0,0], [0,0,0,0,0,0], [1,1,1,1,1,1], [0,0,0,0,0,0]], dtype=theano.config.floatX).T
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from for_theano import evaluate from rv import full_log_likelihood s_rng = RandomStreams(23424) fair_prior = 0.999 coin_weight = tensor.switch(s_rng.binomial(1, fair_prior) > 0.5, 0.5, s_rng.dirichlet([1, 1])[0]) make_coin = lambda p, size: s_rng.binomial(1, p, draw_shape=(size,)) coin = lambda size: make_coin(coin_weight, size) for size in [1, 3, 6, 10, 20, 30, 50, 70, 100]: data = evaluate(make_coin(0.9, size)) sampler = mh2_sample(s_rng, [coin_weight], {coin(size) : data}) print "nr of examples", size, ", estimated probability", sampler(nr_samples=400, burnin=20000, lag=10)[0].mean()
import numpy import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample, mh_sample from for_theano import memoized, evaluate s_rng = RandomStreams(123) nr_words = 4 nr_topics = 2 alpha = 0.8 beta = 1. # Topic distribution per document doc_mixture = memoized(lambda doc_id: s_rng.dirichlet([alpha/nr_topics]*nr_topics)) # Word distribution per topic topic_mixture = memoized(lambda top_id: s_rng.dirichlet([beta/nr_words]*nr_words)) # For each word in the document, draw a topic according to multinomial with document specific prior # TODO, see comment below: topics = memoized(lambda doc_id, nr: s_rng.multinomial(1, doc_mixture[doc_id], draw_shape=(nr,))) topics = memoized(lambda doc_id, nr: s_rng.binomial(1, doc_mixture(doc_id)[0], draw_shape=(nr,))) # Draw words for a specific topic word_topic = lambda top_id: s_rng.multinomial(1, topic_mixture(top_id)) # TODO: memoized only works on the pre-compiled graph. This makes it fail in the case where we have to map # a vector of topics to individual multinomials with as priors the different topics. In the case of two topics # we can hack around this by using a binomial topic distribution and using a switch statement here:
import numpy import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample, mh_sample from for_theano import memoized, evaluate s_rng = RandomStreams(123) nr_words = 4 nr_topics = 2 alpha = 0.8 beta = 1. # Topic distribution per document doc_mixture = memoized( lambda doc_id: s_rng.dirichlet([alpha / nr_topics] * nr_topics)) # Word distribution per topic topic_mixture = memoized( lambda top_id: s_rng.dirichlet([beta / nr_words] * nr_words)) # For each word in the document, draw a topic according to multinomial with document specific prior # TODO, see comment below: topics = memoized(lambda doc_id, nr: s_rng.multinomial(1, doc_mixture[doc_id], draw_shape=(nr,))) topics = memoized(lambda doc_id, nr: s_rng.binomial( 1, doc_mixture(doc_id)[0], draw_shape=(nr, ))) # Draw words for a specific topic word_topic = lambda top_id: s_rng.multinomial(1, topic_mixture(top_id))
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from rv import full_log_likelihood from for_theano import evaluate s_rng = RandomStreams(3424) def poly_expansion(x, order): x = x.T result, updates = theano.scan(fn=lambda prior_result, x: prior_result * x, outputs_info=tensor.ones_like(x), non_sequences=x, n_steps=order) return tensor.concatenate([ tensor.ones([x.shape[1], 1]), tensor.reshape(result.T, (x.shape[1], x.shape[0] * order)) ], axis=1) # Define priors to be inverse gamma distributions alpha = 1 / s_rng.gamma(1., 2.) beta = 1 / s_rng.gamma(1., .1) # Order of the model
import numpy, pylab import theano from rstreams import RandomStreams import distributions from sample import mh2_sample from for_theano import memoized s_rng = RandomStreams(23424) # Define data marbles_bag_1 = numpy.asarray([[1,1,1,1,1,1], [0,0,0,0,0,0], [0,0,0,0,0,0], [0,0,0,0,0,0], [0,0,0,0,0,0]], dtype=theano.config.floatX).T marbles_bag_2 = numpy.asarray([[0,0,0,0,0,0], [1,1,1,1,1,1], [0,0,0,0,0,0], [0,0,0,0,0,0], [0,0,0,0,0,0]], dtype=theano.config.floatX).T marbles_bag_3 = numpy.asarray([[0,0,0,0,0,0], [0,0,0,0,0,0], [0,0,0,0,0,0], [1,1,1,1,1,1], [0,0,0,0,0,0]], dtype=theano.config.floatX).T marbles_bag_4 = numpy.asarray([[0],[0],[0],[0],[1]], dtype=theano.config.floatX).T # Define flat model bag_prototype = memoized(lambda bag: s_rng.dirichlet(numpy.asarray([1, 1, 1, 1, 1])*5))
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import hybridmc_sample from rv import full_log_likelihood s_rng = RandomStreams(3424) # Define model w = s_rng.normal(0, 4, draw_shape=(2,)) x = tensor.matrix("x") y = tensor.nnet.sigmoid(tensor.dot(x, w)) t = s_rng.binomial(p=y, draw_shape=(4,)) # Define data X_data = numpy.asarray([[-1.5, -0.4, 1.3, 2.2], [-1.1, -2.2, 1.3, 0]], dtype=theano.config.floatX).T Y_data = numpy.asarray([1.0, 1.0, 0.0, 0.0], dtype=theano.config.floatX) # Plot full likelihood function RVs = dict([(t, Y_data)]) lik = full_log_likelihood(RVs) givens = dict([(x, X_data)]) lik_func = theano.function([w], lik, givens=givens, allow_input_downcast=True) delta = 0.1 x_range = numpy.arange(-10.0, 10.0, delta)
def setUp(self): s_rng = self.s_rng = RandomStreams(23424) self.weights = tensor.dvector() self.mus = tensor.dvector() self.sigmas = tensor.dvector()
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from for_theano import evaluate from rv import full_log_likelihood s_rng = RandomStreams(23424) fair_prior = 0.999 coin_weight = tensor.switch( s_rng.binomial(1, fair_prior) > 0.5, 0.5, s_rng.dirichlet([1, 1])[0]) make_coin = lambda p, size: s_rng.binomial(1, p, draw_shape=(size, )) coin = lambda size: make_coin(coin_weight, size) for size in [1, 3, 6, 10, 20, 30, 50, 70, 100]: data = evaluate(make_coin(0.9, size)) sampler = mh2_sample(s_rng, [coin_weight], {coin(size): data}) print "nr of examples", size, ", estimated probability", sampler( nr_samples=400, burnin=20000, lag=10)[0].mean()
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from rv import full_log_likelihood s_rng = RandomStreams(3424) p = s_rng.dirichlet(numpy.asarray([1, 1]))[0] m1 = s_rng.uniform(low=-5, high=5) m2 = s_rng.uniform(low=-5, high=5) v = s_rng.uniform(low=0, high=1) C = s_rng.binomial(1, p, draw_shape=(4,)) m = tensor.switch(C, m1, m2) D = s_rng.normal(m, v, draw_shape=(4,)) D_data = numpy.asarray([1, 1.2, 3, 3.4], dtype=theano.config.floatX) givens = dict([(D, D_data)]) sampler = mh2_sample(s_rng, [p, m1, m2, v], givens) samples = sampler(200, 1000, 100) print samples[0].mean(), samples[1].mean(), samples[2].mean(), samples[3].mean()
import numpy import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import hybridmc_sample from rv import full_log_likelihood from max_lik import likelihood_gradient s_rng = RandomStreams(3424) # Weight prior: w = s_rng.normal(0, 2, draw_shape=(3,)) # Linear model: x = tensor.matrix('x') y = tensor.nnet.sigmoid(tensor.dot(x, w)) # Bernouilli observation model: t = s_rng.binomial(p=y, draw_shape=(4,)) # Some data: X_data = numpy.asarray([[-1.5, -0.4, 1.3, 2.2], [-1.1, -2.2, 1.3, 0], [1., 1., 1., 1.]], dtype=theano.config.floatX).T Y_data = numpy.asarray([1., 1., 0., 0.], dtype=theano.config.floatX) # Compute gradient updates: observations = dict([(t, Y_data)]) params, updates, log_likelihood = likelihood_gradient(observations) # Compile training function and assign input data as givens:
import numpy, pylab import theano from theano import tensor from rstreams import RandomStreams import distributions from sample import mh2_sample from rv import full_log_likelihood s_rng = RandomStreams(3424) p = s_rng.dirichlet(numpy.asarray([1, 1]))[0] m1 = s_rng.uniform(low=-5, high=5) m2 = s_rng.uniform(low=-5, high=5) v = s_rng.uniform(low=0, high=1) C = s_rng.binomial(1, p, draw_shape=(4, )) m = tensor.switch(C, m1, m2) D = s_rng.normal(m, v, draw_shape=(4, )) D_data = numpy.asarray([1, 1.2, 3, 3.4], dtype=theano.config.floatX) givens = dict([(D, D_data)]) sampler = mh2_sample(s_rng, [p, m1, m2, v], givens) samples = sampler(200, 1000, 100) print samples[0].mean(), samples[1].mean(), samples[2].mean(), samples[3].mean( )