def test_dirichlet(self):
     with self.test_session():
         self._test(Dirichlet(tf.zeros(3)), [], [], [3])
         self._test(Dirichlet(tf.zeros([2, 3])), [], [2], [3])
         self._test(Dirichlet(tf.zeros(3), sample_shape=1), [1], [], [3])
         self._test(Dirichlet(tf.zeros(3), sample_shape=[2, 1]), [2, 1], [],
                    [3])
Ejemplo n.º 2
0
def _test(alpha, n):
  rv = Dirichlet(alpha=alpha)
  rv_sample = rv.sample(n)
  x = rv_sample.eval()
  x_tf = tf.constant(x, dtype=tf.float32)
  alpha = alpha.eval()
  assert np.allclose(rv.log_prob(x_tf).eval(),
                     dirichlet_logpdf_vec(x, alpha), atol=1e-3)
Ejemplo n.º 3
0
def _test(alpha, n):
  rv = Dirichlet(alpha=alpha)
  rv_sample = rv.sample(n)
  x = rv_sample.eval()
  x_tf = tf.constant(x, dtype=tf.float32)
  alpha = alpha.eval()
  assert np.allclose(rv.log_prob(x_tf).eval(),
                     dirichlet_logpdf_vec(x, alpha))
Ejemplo n.º 4
0
 def __init__(self, K, D, N, nu, use_param=False):
     self.K = K  # number of topics
     self.D = D  # number of documents
     self.N = N  # number of words of each document
     self.nu = nu
     self.alpha = alpha = tf.zeros([K]) + 0.1
     self.sigmasq = InverseGamma(tf.ones(nu), tf.ones(nu), sample_shape=K)
     self.sigma = sigma = tf.sqrt(self.sigmasq)
     self.mu = mu = Normal(tf.zeros(nu), tf.ones(nu), sample_shape=K)
     self.theta = theta = [None] * D
     self.z = z = [None] * D
     self.w = w = [None] * D
     for d in range(D):
         theta[d] = Dirichlet(alpha)
         if use_param:
             w[d] = ParamMixture(mixing_weights=theta[d],
                                 component_params={
                                     'loc': mu,
                                     'scale_diag': sigma
                                 },
                                 component_dist=MultivariateNormalDiag,
                                 sample_shape=N[d])
             z[d] = w[d].cat
         else:
             z[d] = Categorical(probs=theta[d], sample_shape=N[d])
             components = [
                 MultivariateNormalDiag(loc=tf.gather(mu, k),
                                        scale_diag=tf.gather(self.sigma, k),
                                        sample_shape=N[d]) for k in range(K)
             ]
             w[d] = Mixture(cat=z[d],
                            components=components,
                            sample_shape=N[d])
Ejemplo n.º 5
0
 def test_simplex(self):
     with self.test_session():
         x = Dirichlet([1.1, 1.2, 1.3, 1.4])
         y = ed.transform(x)
         self.assertIsInstance(y, TransformedDistribution)
         sample = y.sample(10, seed=1).eval()
         self.assertSamplePosNeg(sample)
Ejemplo n.º 6
0
    def run(self, adj_mat, n_iter=1000):
        assert adj_mat.shape[0] == adj_mat.shape[1]
        n_node = adj_mat.shape[0]

        # model
        gamma = Dirichlet(concentration=tf.ones([self.n_cluster]))
        Pi = Beta(concentration0=tf.ones([self.n_cluster, self.n_cluster]),
                  concentration1=tf.ones([self.n_cluster, self.n_cluster]))
        Z = Multinomial(total_count=1., probs=gamma, sample_shape=n_node)
        X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))

        # inference (point estimation)
        qgamma = PointMass(params=tf.nn.softmax(
            tf.Variable(tf.random_normal([self.n_cluster]))))
        qPi = PointMass(params=tf.nn.sigmoid(
            tf.Variable(tf.random_normal([self.n_cluster, self.n_cluster]))))
        qZ = PointMass(params=tf.nn.softmax(
            tf.Variable(tf.random_normal([n_node, self.n_cluster]))))

        # map estimation
        inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: adj_mat})
        inference.initialize(n_iter=n_iter)

        tf.global_variables_initializer().run()

        for _ in range(inference.n_iter):
            info_dict = inference.update()
            inference.print_progress(info_dict)
        inference.finalize()
        return qZ.mean().eval().argmax(axis=1)
Ejemplo n.º 7
0
 def __init__(self, num_data, num_cluster, vector_dim, num_mcmc_sample):
     self.K = num_cluster
     self.D = vector_dim
     self.N = num_data
     self.pi = Dirichlet(tf.ones(self.K))
     self.mu = Normal(tf.zeros(self.D),
                      tf.ones(self.D),
                      sample_shape=self.K)
     self.sigmasq = InverseGamma(tf.ones(self.D),
                                 tf.ones(self.D),
                                 sample_shape=self.K)
     self.x = ParamMixture(self.pi, {
         'loc': self.mu,
         'scale_diag': tf.sqrt(self.sigmasq)
     },
                           MultivariateNormalDiag,
                           sample_shape=self.N)
     self.z = self.x.cat
     self.T = num_mcmc_sample  # number of MCMC samples
     self.qpi = Empirical(tf.Variable(tf.ones([self.T, self.K]) / self.K))
     self.qmu = Empirical(tf.Variable(tf.zeros([self.T, self.K, self.D])))
     self.qsigmasq = Empirical(
         tf.Variable(tf.ones([self.T, self.K, self.D])))
     self.qz = Empirical(
         tf.Variable(tf.zeros([self.T, self.N], dtype=tf.int32)))
Ejemplo n.º 8
0
def _test(shape, n):
    K = shape[-1]
    rv = Dirichlet(shape, alpha=tf.constant(1.0 / K, shape=shape))
    rv_sample = rv.sample(n)
    x = rv_sample.eval()
    x_tf = tf.constant(x, dtype=tf.float32)
    alpha = rv.alpha.eval()
    if len(shape) == 1:
        assert np.allclose(
            rv.log_prob_idx((), x_tf).eval(),
            dirichlet_logpdf_vec(x[:, :], alpha[:]))
    elif len(shape) == 2:
        for i in range(shape[0]):
            assert np.allclose(
                rv.log_prob_idx((i, ), x_tf).eval(),
                dirichlet_logpdf_vec(x[:, i, :], alpha[i, :]))
    else:
        assert False
def _test(shape, n):
    K = shape[-1]
    rv = Dirichlet(shape, alpha=tf.constant(1.0/K, shape=shape))
    rv_sample = rv.sample(n)    
    x = rv_sample.eval()
    x_tf = tf.constant(x, dtype=tf.float32)
    alpha = rv.alpha.eval()
    if len(shape) == 1:
        assert np.allclose(
            rv.log_prob_idx((), x_tf).eval(),
            dirichlet_logpdf_vec(x[:, :], alpha[:]))
    elif len(shape) == 2:
        for i in range(shape[0]):
            assert np.allclose(
                rv.log_prob_idx((i, ), x_tf).eval(),
                dirichlet_logpdf_vec(x[:, i, :], alpha[i, :]))
    else:
        assert False
Ejemplo n.º 10
0
def model_stationary_dirichlet_categorical_edward(n_states, chain_len,
                                                  batch_size):
    """ Models a stationary Dirichlet-Categorical Markov Chain in Edward """
    tf.reset_default_graph()

    # create default starting state probability vector
    pi_0 = Dirichlet(tf.ones(n_states))
    x_0 = Categorical(probs=pi_0, sample_shape=batch_size)

    # transition matrix
    pi_T = Dirichlet(tf.ones([n_states, n_states]))

    x = []
    for _ in range(chain_len):
        x_tm1 = x[-1] if x else x_0
        x_t = Categorical(probs=tf.gather(pi_T, x_tm1))
        x.append(x_t)

    return x, pi_0, pi_T
Ejemplo n.º 11
0
def main(_):
  # DATA
  pi_true = np.random.dirichlet(np.array([20.0, 30.0, 10.0, 10.0]))
  z_data = np.array([np.random.choice(FLAGS.K, 1, p=pi_true)[0]
                     for n in range(FLAGS.N)])
  print("pi: {}".format(pi_true))

  # MODEL
  pi = Dirichlet(tf.ones(4))
  z = Categorical(probs=pi, sample_shape=FLAGS.N)

  # INFERENCE
  qpi = Dirichlet(tf.nn.softplus(
      tf.get_variable("qpi/concentration", [FLAGS.K])))

  inference = ed.KLqp({pi: qpi}, data={z: z_data})
  inference.run(n_iter=1500, n_samples=30)

  sess = ed.get_session()
  print("Inferred pi: {}".format(sess.run(qpi.mean())))
Ejemplo n.º 12
0
 def __init__(self, K, V, D, N):
     self.K = K  # number of topics
     self.V = V  # vocabulary size
     self.D = D  # number of documents
     self.N = N  # number of words of each document
     self.alpha = alpha = tf.zeros([K]) + 0.1
     self.yita = yita = tf.zeros([V]) + 0.01
     self.theta = [None] * D
     self.beta = Dirichlet(yita, sample_shape=K)
     self.z = [None] * D
     self.w = [None] * D
     temp = self.beta
     for d in range(D):
         self.theta[d] = Dirichlet(alpha)
         self.w[d] = ParamMixture(mixing_weights=self.theta[d],
                                  component_params={'probs': temp},
                                  component_dist=Categorical,
                                  sample_shape=N[d],
                                  validate_args=False)
         self.z[d] = self.w[d].cat
Ejemplo n.º 13
0
def main(_):
    # DATA
    pi_true = np.random.dirichlet(np.array([20.0, 30.0, 10.0, 10.0]))
    z_data = np.array(
        [np.random.choice(FLAGS.K, 1, p=pi_true)[0] for n in range(FLAGS.N)])
    print("pi: {}".format(pi_true))

    # MODEL
    pi = Dirichlet(tf.ones(4))
    z = Categorical(probs=pi, sample_shape=FLAGS.N)

    # INFERENCE
    qpi = Dirichlet(
        tf.nn.softplus(tf.get_variable("qpi/concentration", [FLAGS.K])))

    inference = ed.KLqp({pi: qpi}, data={z: z_data})
    inference.run(n_iter=1500, n_samples=30)

    sess = ed.get_session()
    print("Inferred pi: {}".format(sess.run(qpi.mean())))
Ejemplo n.º 14
0
def model_non_stationary_dirichlet_categorical(n_states, chain_len,
                                               batch_size):
    """ Models a non-stationary Dirichlet-Categorical
    Markov Chain in Edward """
    tf.reset_default_graph()

    # create default starting state probability vector
    pi_0 = Dirichlet(tf.ones(n_states))
    x_0 = Categorical(probs=pi_0, sample_shape=batch_size)

    pi_T, x = [], []
    for _ in range(chain_len):
        x_tm1 = x[-1] if x else x_0
        # transition matrix, one per position in the chain:
        # i.e. we now condition both on previous state and age of the loan
        pi_T_t = Dirichlet(tf.ones([n_states, n_states]))
        x_t = Categorical(probs=tf.gather(pi_T_t, x_tm1))
        pi_T.append(pi_T_t)
        x.append(x_t)

    return x, pi_0, pi_T
Ejemplo n.º 15
0
def model_stationary_dirichlet_categorical_tfp(n_states, chain_len,
                                               batch_size):
    """ Models a stationary Dirichlet-Categorical Markov Chain
    in TensorFlow Probability/Edward2 """
    tf.reset_default_graph()

    # create default starting state probability vector
    pi_0 = Dirichlet(tf.ones(n_states))
    x_0 = Categorical(probs=pi_0, sample_shape=batch_size)

    # transition matrix
    pi_T = Dirichlet(tf.ones([n_states, n_states]))
    transition_distribution = Categorical(probs=pi_T)

    pi_E = np.eye(n_states, dtype=np.float32)  # identity matrix
    emission_distribution = Categorical(probs=pi_E)

    model = HiddenMarkovModel(initial_distribution=x_0,
                              transition_distribution=transition_distribution,
                              observation_distribution=emission_distribution,
                              num_steps=chain_len,
                              sample_shape=batch_size)

    return model, pi_0, pi_T
Ejemplo n.º 16
0
def mmsb(N, K, data):
    # sparsity
    rho = 0.3
    # MODEL
    # probability of belonging to each of K blocks for each node
    gamma = Dirichlet(concentration=tf.ones([K]))
    # block connectivity
    Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
    # probability of belonging to each of K blocks for all nodes
    Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N)
    # adjacency
    X = Bernoulli(probs=(1 - rho) *
                  tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))

    # INFERENCE (EM algorithm)
    qgamma = PointMass(
        params=tf.nn.softmax(tf.Variable(tf.random_normal([K]))))
    qPi = PointMass(
        params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K]))))
    qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K]))))

    #qgamma = Normal(loc=tf.get_variable("qgamma/loc", [K]),
    #                scale=tf.nn.softplus(
    #                        tf.get_variable("qgamma/scale", [K])))
    #qPi = Normal(loc=tf.get_variable("qPi/loc", [K, K]),
    #                scale=tf.nn.softplus(
    #                        tf.get_variable("qPi/scale", [K, K])))
    #qZ = Normal(loc=tf.get_variable("qZ/loc", [N, K]),
    #                scale=tf.nn.softplus(
    #                        tf.get_variable("qZ/scale", [N, K])))

    #inference = ed.KLqp({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data})
    inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data})

    #inference.run()
    n_iter = 6000
    inference.initialize(optimizer=tf.train.AdamOptimizer(learning_rate=0.01),
                         n_iter=n_iter)

    tf.global_variables_initializer().run()

    for _ in range(inference.n_iter):
        info_dict = inference.update()
        inference.print_progress(info_dict)

    inference.finalize()
    print('qgamma after: ', qgamma.mean().eval())
    return qZ.mean().eval(), qPi.eval()
Ejemplo n.º 17
0
def model_stationary_dirichlet_multinomial(n_states, chain_len,
                                           total_counts_per_month):
    """ Models a stationary Dirichlet-Multinomial Markov Chain in Edward """
    tf.reset_default_graph()

    # create default starting state probability vector
    pi_list = [Dirichlet(tf.ones(n_states)) for _ in range(chain_len)]
    # now instead of sample_shape we use total_count which
    # is how many times we sample from each categorical
    # i.e. number of accounts
    counts = [
        Multinomial(probs=pi, total_count=float(total_counts_per_month))
        for pi in pi_list
    ]

    return pi_list, counts
Ejemplo n.º 18
0
def main(_):
    ed.set_seed(42)

    # Prior on scalar hyperparameter to Dirichlet.
    alpha = Gamma(1.0, 1.0)

    # Prior on size of Dirichlet.
    n = 1 + tf.cast(Exponential(0.5), tf.int32)

    # Build a vector of ones whose size is n; multiply it by alpha.
    p = Dirichlet(tf.ones([n]) * alpha)

    sess = ed.get_session()
    print(sess.run(p))
    # [ 0.01012419  0.02939712  0.05036638  0.51287931  0.31020424  0.0485355
    #   0.0384932 ]
    print(sess.run(p))
Ejemplo n.º 19
0
 def __init__(self, K, D, N, nu, use_param=False):
     self.K = K  # number of topics
     self.D = D  # number of documents
     self.N = N  # number of words of each document
     self.nu = nu
     self.alpha = alpha = tf.zeros([K]) + 0.1
     mu0 = tf.constant([0.0] * nu)
     sigma0 = tf.eye(nu)
     self.sigma = sigma = WishartCholesky(
         df=nu,
         scale=sigma0,
         cholesky_input_output_matrices=True,
         sample_shape=K)
     # sigma_inv = tf.matrix_inverse(sigma)
     self.mu = mu = Normal(mu0, tf.ones(nu), sample_shape=K)
     self.theta = theta = [None] * D
     self.z = z = [None] * D
     self.w = w = [None] * D
     for d in range(D):
         theta[d] = Dirichlet(alpha)
         if use_param:
             w[d] = ParamMixture(mixing_weights=theta[d],
                                 component_params={
                                     'loc': mu,
                                     'scale_tril': sigma
                                 },
                                 component_dist=MultivariateNormalTriL,
                                 sample_shape=N[d])
             z[d] = w[d].cat
         else:
             z[d] = Categorical(probs=theta[d], sample_shape=N[d])
             components = [
                 MultivariateNormalTriL(loc=tf.gather(mu, k),
                                        scale_tril=tf.gather(sigma, k),
                                        sample_shape=N[d]) for k in range(K)
             ]
             w[d] = Mixture(cat=z[d],
                            components=components,
                            sample_shape=N[d])
Ejemplo n.º 20
0
def main(_):
    ed.set_seed(42)

    # DATA
    X_data, Z_true = karate("~/data")
    N = X_data.shape[0]  # number of vertices
    K = 2  # number of clusters

    # MODEL
    gamma = Dirichlet(concentration=tf.ones([K]))
    Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
    Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N)
    X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))

    # INFERENCE (EM algorithm)
    qgamma = PointMass(tf.nn.softmax(tf.get_variable("qgamma/params", [K])))
    qPi = PointMass(tf.nn.sigmoid(tf.get_variable("qPi/params", [K, K])))
    qZ = PointMass(tf.nn.softmax(tf.get_variable("qZ/params", [N, K])))

    inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data})
    inference.initialize(n_iter=250)

    tf.global_variables_initializer().run()

    for _ in range(inference.n_iter):
        info_dict = inference.update()
        inference.print_progress(info_dict)

    # CRITICISM
    Z_pred = qZ.mean().eval().argmax(axis=1)
    print("Result (label flip can happen):")
    print("Predicted")
    print(Z_pred)
    print("True")
    print(Z_true)
    print("Adjusted Rand Index =", adjusted_rand_score(Z_pred, Z_true))
Ejemplo n.º 21
0
plt.scatter(x_train[:, 0], x_train[:, 1])
plt.axis([-3, 3, -3, 3])
plt.title("Simulated dataset")
plt.show()

K = 2
D = 2
model = MixtureGaussian(K, D)

qpi_alpha = tf.nn.softplus(tf.Variable(tf.random_normal([K])))
qmu_mu = tf.Variable(tf.random_normal([K * D]))
qmu_sigma = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qsigma_alpha = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qsigma_beta = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))

qpi = Dirichlet(alpha=qpi_alpha)
qmu = Normal(mu=qmu_mu, sigma=qmu_sigma)
qsigma = InverseGamma(alpha=qsigma_alpha, beta=qsigma_beta)

data = {'x': x_train}
inference = ed.KLqp({'pi': qpi, 'mu': qmu, 'sigma': qsigma}, data, model)
inference.run(n_iter=2500, n_samples=10, n_minibatch=20)

# Average per-cluster and per-data point likelihood over many posterior samples.
log_liks = []
for s in range(100):
  zrep = {'pi': qpi.sample(()),
          'mu': qmu.sample(()),
          'sigma': qsigma.sample(())}
  log_liks += [model.predict(data, zrep)]
Ejemplo n.º 22
0
        log_prior = dirichlet.logpdf(pi, self.alpha)
        log_prior += tf.reduce_sum(norm.logpdf(mus, 0, np.sqrt(self.c)), 1)
        log_prior += tf.reduce_sum(invgamma.logpdf(sigmas, self.a, self.b), 1)

        # Loop over each mini-batch zs[b,:]
        log_lik = []
        n_minibatch = get_dims(zs[0])[0]
        for s in range(n_minibatch):
            log_lik_z = N*tf.reduce_sum(tf.log(pi), 1)
            for k in range(self.K):
                log_lik_z += tf.reduce_sum(multivariate_normal.logpdf(xs,
                    mus[s, (k*self.D):((k+1)*self.D)],
                    sigmas[s, (k*self.D):((k+1)*self.D)]))

            log_lik += [log_lik_z]

        return log_prior + tf.pack(log_lik)

ed.set_seed(42)
x = np.loadtxt('data/mixture_data.txt', dtype='float32', delimiter=',')
data = ed.Data(tf.constant(x, dtype=tf.float32))

model = MixtureGaussian(K=2, D=2)
variational = Variational()
variational.add(Dirichlet(model.K))
variational.add(Normal(model.K*model.D))
variational.add(InvGamma(model.K*model.D))

inference = ed.MFVI(model, variational, data)
inference.run(n_iter=500, n_minibatch=5, n_data=5)
Ejemplo n.º 23
0
true_mu = np.array([-1.0, 0.0, 1.0], np.float32) * 10
true_sigmasq = np.array([1.0**2, 2.0**2, 3.0**2], np.float32)
true_pi = np.array([0.2, 0.3, 0.5], np.float32)
N = 10000
K = len(true_mu)
true_z = np.random.choice(np.arange(K), size=N, p=true_pi)
x_data = true_mu[true_z] + np.random.randn(N) * np.sqrt(true_sigmasq[true_z])

# Prior hyperparameters
pi_alpha = np.ones(K, dtype=np.float32)
mu_sigma = np.std(true_mu)
sigmasq_alpha = 1.0
sigmasq_beta = 2.0

# Model
pi = Dirichlet(pi_alpha)
mu = Normal(0.0, mu_sigma, sample_shape=K)
sigmasq = InverseGamma(sigmasq_alpha, sigmasq_beta, sample_shape=K)
x = ParamMixture(pi, {
    'mu': mu,
    'sigma': tf.sqrt(sigmasq)
},
                 Normal,
                 sample_shape=N)
z = x.cat

# Conditionals
mu_cond = ed.complete_conditional(mu)
sigmasq_cond = ed.complete_conditional(sigmasq)
pi_cond = ed.complete_conditional(pi)
z_cond = ed.complete_conditional(z)
Ejemplo n.º 24
0
    x = np.zeros((N, 2), dtype=np.float32)
    for n in range(N):
        k = np.argmax(np.random.multinomial(1, pi))
        x[n, :] = np.random.multivariate_normal(mus[k], np.diag(stds[k]))

    return x


N = 500  # number of data points
K = 2  # number of components
D = 2  # dimensionality of data
ed.set_seed(42)

x_train = build_toy_dataset(N)

pi = Dirichlet(tf.ones(K))
mu = Normal(tf.zeros(D), tf.ones(D), sample_shape=K)
sigmasq = InverseGamma(tf.ones(D), tf.ones(D), sample_shape=K)
x = ParamMixture(pi, {
    'loc': mu,
    'scale_diag': tf.sqrt(sigmasq)
},
                 MultivariateNormalDiag,
                 sample_shape=N)
z = x.cat
T = 500  # number of MCMC samples
qpi = Empirical(
    tf.get_variable("qpi/params", [T, K],
                    initializer=tf.constant_initializer(1.0 / K)))
qmu = Empirical(
    tf.get_variable("qmu/params", [T, K, D],
Ejemplo n.º 25
0
def _test(alpha, n):
  x = Dirichlet(alpha=alpha)
  val_est = get_dims(x.sample(n))
  val_true = n + get_dims(alpha)
  assert val_est == val_true
Ejemplo n.º 26
0
    X[src, dst] = 1

  return X, Z


ed.set_seed(42)

# DATA
label_filepath = 'data/karate_labels.txt'
graph_filepath = 'data/karate_edgelist.txt'
X_data, Z_true = build_dataset(label_filepath, graph_filepath)
N = X_data.shape[0]  # number of vertices
K = 2  # number of clusters

# MODEL
gamma = Dirichlet(concentration=tf.ones([K]))
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
Z = Multinomial(total_count=1., probs=gamma, sample_shape=N)
X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))

# INFERENCE (EM algorithm)
qgamma = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([K]))))
qPi = PointMass(params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K]))))
qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K]))))

inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data})

n_iter = 100
inference.initialize(n_iter=n_iter)

tf.global_variables_initializer().run()
Ejemplo n.º 27
0
def _test(shape, alpha, size):
    x = Dirichlet(shape, alpha)
    val_est = tuple(get_dims(x.sample(size=size)))
    val_true = (size, ) + shape
    assert val_est == val_true
Ejemplo n.º 28
0
        k = np.argmax(np.random.multinomial(1, pi))
        x[n, :] = np.random.multivariate_normal(mus[k], np.diag(stds[k]))

    return x


N = 500  # num data points
K = 2  # num components
D = 2  # dimensionality of data
ed.set_seed(42)

# DATA
x_data = build_toy_dataset(N)

# MODEL
pi = Dirichlet(alpha=tf.constant([1.0] * K))
mu = Normal(mu=tf.zeros([K, D]), sigma=tf.ones([K, D]))
sigma = InverseGamma(alpha=tf.ones([K, D]), beta=tf.ones([K, D]))
c = Categorical(logits=ed.tile(ed.logit(pi), [N, 1]))
x = Normal(mu=tf.gather(mu, c), sigma=tf.gather(sigma, c))

# INFERENCE
T = 5000
qpi = Empirical(params=tf.Variable(tf.ones([T, K]) / K))
qmu = Empirical(params=tf.Variable(tf.zeros([T, K, D])))
qsigma = Empirical(params=tf.Variable(tf.ones([T, K, D])))
qc = Empirical(params=tf.Variable(tf.zeros([T, N], dtype=tf.int32)))

gpi = Dirichlet(alpha=tf.constant([1.4, 1.6]))
gmu = Normal(mu=tf.constant([[1.0, 1.0], [-1.0, -1.0]]),
             sigma=tf.constant([[0.5, 0.5], [0.5, 0.5]]))
Ejemplo n.º 29
0
    x = np.zeros((N, 2))
    for n in range(N):
        k = np.argmax(np.random.multinomial(1, pi))
        x[n, :] = np.random.multivariate_normal(mus[k], np.diag(stds[k]))
    return x


N = 1000  # Number of data points
K = 2  # Number of components
D = 2  # Dimensionality of data

# DATA
x_data = build_toy_dataset(N)

# MODEL
pi = Dirichlet(concentration=tf.constant([1.0] * K))
mu = Normal(loc=tf.zeros([K, D]), scale=tf.ones([K, D]))
sigma = InverseGamma(concentration=tf.ones([K, D]), rate=tf.ones([K, D]))
c = Categorical(logits=tf.tile(tf.reshape(ed.logit(pi), [1, K]), [N, 1]))
x = Normal(loc=tf.gather(mu, c), scale=tf.gather(sigma, c))

# INFERENCE
qpi = Dirichlet(
    concentration=tf.nn.softplus(tf.Variable(tf.random_normal([K]))))
qmu = Normal(loc=tf.Variable(tf.random_normal([K, D])),
             scale=tf.nn.softplus(tf.Variable(tf.random_normal([K, D]))))
qsigma = InverseGamma(concentration=tf.nn.softplus(
    tf.Variable(tf.random_normal([K, D]))),
                      rate=tf.nn.softplus(tf.Variable(tf.random_normal([K,
                                                                        D]))))
qc = Categorical(logits=tf.Variable(tf.zeros([N, K])))
Ejemplo n.º 30
0
img_no = 61060
TRAIN_DIR = "../data/BSR/BSDS500/data/images/train/"
img, train_img = load_image_matrix(img_no, TRAIN_DIR)

# Hyperparameters
N = train_img.shape[0]
K = 15
D = train_img.shape[1]
T = 800  # number of MCMC samples
M = 300  # number of posterior samples sampled

ed.set_seed(67)

with tf.name_scope("model"):
    pi = Dirichlet(concentration=tf.constant([1.0] * K, name="pi/weights"),
                   name="pi")
    mu = Normal(loc=tf.zeros(D, name="centroids/loc"),
                scale=tf.ones(D, name="centroids/scale"),
                sample_shape=K,
                name="centroids")
    sigma = InverseGamma(concentration=tf.ones(
        D, name="variability/concentration"),
                         rate=tf.ones(D, name="variability/rate"),
                         sample_shape=K,
                         name="variability")

    x = ParamMixture(pi, {
        'loc': mu,
        'scale_diag': tf.sqrt(sigma)
    },
                     MultivariateNormalDiag,
Ejemplo n.º 31
0
plt.scatter(x_train[:, 0], x_train[:, 1])
plt.axis([-3, 3, -3, 3])
plt.title("Simulated dataset")
plt.show()

K = 2
D = 2
model = MixtureGaussian(K, D)

qpi_alpha = tf.nn.softplus(tf.Variable(tf.random_normal([K])))
qmu_mu = tf.Variable(tf.random_normal([K * D]))
qmu_sigma = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qsigma_alpha = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qsigma_beta = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))

qpi = Dirichlet(alpha=qpi_alpha)
qmu = Normal(mu=qmu_mu, sigma=qmu_sigma)
qsigma = InverseGamma(alpha=qsigma_alpha, beta=qsigma_beta)

data = {'x': x_train}
inference = ed.KLqp({'pi': qpi, 'mu': qmu, 'sigma': qsigma}, data, model)
inference.run(n_iter=2500, n_samples=10, n_minibatch=20)

# Average per-cluster and per-data point likelihood over many posterior samples.
log_liks = []
for s in range(100):
    zrep = {
        'pi': qpi.sample(()),
        'mu': qmu.sample(()),
        'sigma': qsigma.sample(())
    }
Ejemplo n.º 32
0
We build a random variable whose size depends on a sample from another
random variable.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import edward as ed
import tensorflow as tf

from edward.models import Exponential, Dirichlet, Gamma

ed.set_seed(42)

# Prior on scalar hyperparameter to Dirichlet.
alpha = Gamma(alpha=1.0, beta=1.0)

# Prior on size of Dirichlet.
n = 1 + tf.cast(Exponential(lam=0.5), tf.int32)

# Build a vector of ones whose size is n; multiply it by alpha.
p = Dirichlet(alpha=tf.ones([n]) * alpha)

sess = ed.get_session()
print(sess.run(p.value()))
# [ 0.01012419  0.02939712  0.05036638  0.51287931  0.31020424  0.0485355
#   0.0384932 ]
print(sess.run(p.value()))
# [ 0.12836078  0.23335715  0.63828212]
from edward.models import Normal, Exponential
import tensorflow as tf

# 都是RandomVariable衍生出来
# 一元分布
Normal(loc=tf.constant(0.0), scale=tf.constant(1.0))
Normal(loc=tf.zeros(5), scale=tf.ones(5))
Exponential(rate=tf.ones([2, 3]))

# 多元分布
from edward.models import Dirichlet, MultivariateNormalTriL
K = 3
Dirichlet(concentration=tf.constant([0.1] * K))  # K为Dirichlet分布
MultivariateNormalTriL(loc=tf.zeros([5, K]),
                       scale_tril=tf.ones([5, K, K]))  # loc的最后一位表示维数
MultivariateNormalTriL(loc=tf.zeros([2, 5, K]),
                       scale_tril=tf.ones([2, 5, K, K]))

# 每个RandomVariable有方法log_prob(),mean(),sample(),且与计算图上的一个张量对应
# 可以支持诸多运算
from edward.models import Normal

x = Normal(loc=tf.zeros(10), scale=tf.ones(10))
y = tf.constant(5.0)
x + y, x - y, x * y, x / y
tf.tanh(x * y)
tf.gather(x, 2)
print(x[2])

# 有向图模型
from edward.models import Bernoulli, Beta
Ejemplo n.º 34
0
true_mu = np.array([-3.0, 0.0, 3.0], np.float32)
true_sigma = np.array([1.0, 1.0, 1.0], np.float32)

true_pi = np.array([.2, .3, .5], np.float32)
K = len(true_pi)
true_z = np.random.choice(np.arange(K), p=true_pi, size=N)
x = true_mu[true_z] + np.random.randn(N) * true_sigma[true_z]

# plt.hist(x, bins=200)
# plt.show()

# we like to calculate posterior p(\theta|x) where \theta=[mu_1,..., mu_3, sigma_1,...,sigma_3, z_1,...,z_3]

# Model
pi = Dirichlet(np.ones(K, np.float32))
mu = Normal(0.0, 9.0, sample_shape=[K])
sigma = InverseGamma(1.0, 1.0, sample_shape=[K])

c = Categorical(logits=tf.log(pi) - tf.log(1.0 - pi), sample_shape=N)
ed_x = Normal(loc=tf.gather(mu, c), scale=tf.gather(sigma, c))

# parameters
q_pi = Dirichlet(
    tf.nn.softplus(
        tf.get_variable("qpi", [K],
                        initializer=tf.constant_initializer(1.0 / K))))
q_mu = Normal(loc=tf.get_variable("qmu", [K]), scale=1.0)
q_sigma = Normal(loc=tf.nn.softplus(tf.get_variable("qsigma", [K])), scale=1.0)

inference = ed.KLqp(latent_vars={
Ejemplo n.º 35
0
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import edward as ed
import numpy as np
import tensorflow as tf

from edward.models import Categorical, Dirichlet

N = 1000
K = 4

# DATA
pi_true = np.random.dirichlet(np.array([20.0, 30.0, 10.0, 10.0]))
z_data = np.array([np.random.choice(K, 1, p=pi_true)[0] for n in range(N)])
print('pi={}'.format(pi_true))

# MODEL
pi = Dirichlet(alpha=tf.ones(4))
z = Categorical(p=tf.ones([N, 1]) * pi)

# INFERENCE
qpi = Dirichlet(alpha=tf.nn.softplus(tf.Variable(tf.random_normal([K]))))

inference = ed.KLqp({pi: qpi}, data={z: z_data})
inference.run(n_iter=1500, n_samples=30)

sess = ed.get_session()
print('Inferred pi={}'.format(sess.run(qpi.mean())))
Ejemplo n.º 36
0
def _test(shape, alpha, n):
    x = Dirichlet(shape, alpha)
    val_est = tuple(get_dims(x.sample(n)))
    val_true = (n, ) + shape
    assert val_est == val_true