def test_dirichlet(self):
with self.test_session():
self._test(Dirichlet(tf.zeros(3)), [], [], [3])
self._test(Dirichlet(tf.zeros([2, 3])), [], [2], [3])
self._test(Dirichlet(tf.zeros(3), sample_shape=1), [1], [], [3])
self._test(Dirichlet(tf.zeros(3), sample_shape=[2, 1]), [2, 1], [],
[3])
def _test(alpha, n):
rv = Dirichlet(alpha=alpha)
rv_sample = rv.sample(n)
x = rv_sample.eval()
x_tf = tf.constant(x, dtype=tf.float32)
alpha = alpha.eval()
assert np.allclose(rv.log_prob(x_tf).eval(),
dirichlet_logpdf_vec(x, alpha), atol=1e-3)
def _test(alpha, n):
rv = Dirichlet(alpha=alpha)
rv_sample = rv.sample(n)
x = rv_sample.eval()
x_tf = tf.constant(x, dtype=tf.float32)
alpha = alpha.eval()
assert np.allclose(rv.log_prob(x_tf).eval(),
dirichlet_logpdf_vec(x, alpha))
def __init__(self, K, D, N, nu, use_param=False):
self.K = K # number of topics
self.D = D # number of documents
self.N = N # number of words of each document
self.nu = nu
self.alpha = alpha = tf.zeros([K]) + 0.1
self.sigmasq = InverseGamma(tf.ones(nu), tf.ones(nu), sample_shape=K)
self.sigma = sigma = tf.sqrt(self.sigmasq)
self.mu = mu = Normal(tf.zeros(nu), tf.ones(nu), sample_shape=K)
self.theta = theta = [None] * D
self.z = z = [None] * D
self.w = w = [None] * D
for d in range(D):
theta[d] = Dirichlet(alpha)
if use_param:
w[d] = ParamMixture(mixing_weights=theta[d],
component_params={
'loc': mu,
'scale_diag': sigma
},
component_dist=MultivariateNormalDiag,
sample_shape=N[d])
z[d] = w[d].cat
else:
z[d] = Categorical(probs=theta[d], sample_shape=N[d])
components = [
MultivariateNormalDiag(loc=tf.gather(mu, k),
scale_diag=tf.gather(self.sigma, k),
sample_shape=N[d]) for k in range(K)
]
w[d] = Mixture(cat=z[d],
components=components,
sample_shape=N[d])
def test_simplex(self):
with self.test_session():
x = Dirichlet([1.1, 1.2, 1.3, 1.4])
y = ed.transform(x)
self.assertIsInstance(y, TransformedDistribution)
sample = y.sample(10, seed=1).eval()
self.assertSamplePosNeg(sample)
def run(self, adj_mat, n_iter=1000):
assert adj_mat.shape[0] == adj_mat.shape[1]
n_node = adj_mat.shape[0]
# model
gamma = Dirichlet(concentration=tf.ones([self.n_cluster]))
Pi = Beta(concentration0=tf.ones([self.n_cluster, self.n_cluster]),
concentration1=tf.ones([self.n_cluster, self.n_cluster]))
Z = Multinomial(total_count=1., probs=gamma, sample_shape=n_node)
X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# inference (point estimation)
qgamma = PointMass(params=tf.nn.softmax(
tf.Variable(tf.random_normal([self.n_cluster]))))
qPi = PointMass(params=tf.nn.sigmoid(
tf.Variable(tf.random_normal([self.n_cluster, self.n_cluster]))))
qZ = PointMass(params=tf.nn.softmax(
tf.Variable(tf.random_normal([n_node, self.n_cluster]))))
# map estimation
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: adj_mat})
inference.initialize(n_iter=n_iter)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
inference.finalize()
return qZ.mean().eval().argmax(axis=1)
def __init__(self, num_data, num_cluster, vector_dim, num_mcmc_sample):
self.K = num_cluster
self.D = vector_dim
self.N = num_data
self.pi = Dirichlet(tf.ones(self.K))
self.mu = Normal(tf.zeros(self.D),
tf.ones(self.D),
sample_shape=self.K)
self.sigmasq = InverseGamma(tf.ones(self.D),
tf.ones(self.D),
sample_shape=self.K)
self.x = ParamMixture(self.pi, {
'loc': self.mu,
'scale_diag': tf.sqrt(self.sigmasq)
},
MultivariateNormalDiag,
sample_shape=self.N)
self.z = self.x.cat
self.T = num_mcmc_sample # number of MCMC samples
self.qpi = Empirical(tf.Variable(tf.ones([self.T, self.K]) / self.K))
self.qmu = Empirical(tf.Variable(tf.zeros([self.T, self.K, self.D])))
self.qsigmasq = Empirical(
tf.Variable(tf.ones([self.T, self.K, self.D])))
self.qz = Empirical(
tf.Variable(tf.zeros([self.T, self.N], dtype=tf.int32)))
def _test(shape, n):
K = shape[-1]
rv = Dirichlet(shape, alpha=tf.constant(1.0 / K, shape=shape))
rv_sample = rv.sample(n)
x = rv_sample.eval()
x_tf = tf.constant(x, dtype=tf.float32)
alpha = rv.alpha.eval()
if len(shape) == 1:
assert np.allclose(
rv.log_prob_idx((), x_tf).eval(),
dirichlet_logpdf_vec(x[:, :], alpha[:]))
elif len(shape) == 2:
for i in range(shape[0]):
assert np.allclose(
rv.log_prob_idx((i, ), x_tf).eval(),
dirichlet_logpdf_vec(x[:, i, :], alpha[i, :]))
else:
assert False
def _test(shape, n):
K = shape[-1]
rv = Dirichlet(shape, alpha=tf.constant(1.0/K, shape=shape))
rv_sample = rv.sample(n)
x = rv_sample.eval()
x_tf = tf.constant(x, dtype=tf.float32)
alpha = rv.alpha.eval()
if len(shape) == 1:
assert np.allclose(
rv.log_prob_idx((), x_tf).eval(),
dirichlet_logpdf_vec(x[:, :], alpha[:]))
elif len(shape) == 2:
for i in range(shape[0]):
assert np.allclose(
rv.log_prob_idx((i, ), x_tf).eval(),
dirichlet_logpdf_vec(x[:, i, :], alpha[i, :]))
else:
assert False
def model_stationary_dirichlet_categorical_edward(n_states, chain_len,
batch_size):
""" Models a stationary Dirichlet-Categorical Markov Chain in Edward """
tf.reset_default_graph()
# create default starting state probability vector
pi_0 = Dirichlet(tf.ones(n_states))
x_0 = Categorical(probs=pi_0, sample_shape=batch_size)
# transition matrix
pi_T = Dirichlet(tf.ones([n_states, n_states]))
x = []
for _ in range(chain_len):
x_tm1 = x[-1] if x else x_0
x_t = Categorical(probs=tf.gather(pi_T, x_tm1))
x.append(x_t)
return x, pi_0, pi_T
def main(_):
# DATA
pi_true = np.random.dirichlet(np.array([20.0, 30.0, 10.0, 10.0]))
z_data = np.array([np.random.choice(FLAGS.K, 1, p=pi_true)[0]
for n in range(FLAGS.N)])
print("pi: {}".format(pi_true))
# MODEL
pi = Dirichlet(tf.ones(4))
z = Categorical(probs=pi, sample_shape=FLAGS.N)
# INFERENCE
qpi = Dirichlet(tf.nn.softplus(
tf.get_variable("qpi/concentration", [FLAGS.K])))
inference = ed.KLqp({pi: qpi}, data={z: z_data})
inference.run(n_iter=1500, n_samples=30)
sess = ed.get_session()
print("Inferred pi: {}".format(sess.run(qpi.mean())))
def __init__(self, K, V, D, N):
self.K = K # number of topics
self.V = V # vocabulary size
self.D = D # number of documents
self.N = N # number of words of each document
self.alpha = alpha = tf.zeros([K]) + 0.1
self.yita = yita = tf.zeros([V]) + 0.01
self.theta = [None] * D
self.beta = Dirichlet(yita, sample_shape=K)
self.z = [None] * D
self.w = [None] * D
temp = self.beta
for d in range(D):
self.theta[d] = Dirichlet(alpha)
self.w[d] = ParamMixture(mixing_weights=self.theta[d],
component_params={'probs': temp},
component_dist=Categorical,
sample_shape=N[d],
validate_args=False)
self.z[d] = self.w[d].cat
def main(_):
# DATA
pi_true = np.random.dirichlet(np.array([20.0, 30.0, 10.0, 10.0]))
z_data = np.array(
[np.random.choice(FLAGS.K, 1, p=pi_true)[0] for n in range(FLAGS.N)])
print("pi: {}".format(pi_true))
# MODEL
pi = Dirichlet(tf.ones(4))
z = Categorical(probs=pi, sample_shape=FLAGS.N)
# INFERENCE
qpi = Dirichlet(
tf.nn.softplus(tf.get_variable("qpi/concentration", [FLAGS.K])))
inference = ed.KLqp({pi: qpi}, data={z: z_data})
inference.run(n_iter=1500, n_samples=30)
sess = ed.get_session()
print("Inferred pi: {}".format(sess.run(qpi.mean())))
def model_non_stationary_dirichlet_categorical(n_states, chain_len,
batch_size):
""" Models a non-stationary Dirichlet-Categorical
Markov Chain in Edward """
tf.reset_default_graph()
# create default starting state probability vector
pi_0 = Dirichlet(tf.ones(n_states))
x_0 = Categorical(probs=pi_0, sample_shape=batch_size)
pi_T, x = [], []
for _ in range(chain_len):
x_tm1 = x[-1] if x else x_0
# transition matrix, one per position in the chain:
# i.e. we now condition both on previous state and age of the loan
pi_T_t = Dirichlet(tf.ones([n_states, n_states]))
x_t = Categorical(probs=tf.gather(pi_T_t, x_tm1))
pi_T.append(pi_T_t)
x.append(x_t)
return x, pi_0, pi_T
def model_stationary_dirichlet_categorical_tfp(n_states, chain_len,
batch_size):
""" Models a stationary Dirichlet-Categorical Markov Chain
in TensorFlow Probability/Edward2 """
tf.reset_default_graph()
# create default starting state probability vector
pi_0 = Dirichlet(tf.ones(n_states))
x_0 = Categorical(probs=pi_0, sample_shape=batch_size)
# transition matrix
pi_T = Dirichlet(tf.ones([n_states, n_states]))
transition_distribution = Categorical(probs=pi_T)
pi_E = np.eye(n_states, dtype=np.float32) # identity matrix
emission_distribution = Categorical(probs=pi_E)
model = HiddenMarkovModel(initial_distribution=x_0,
transition_distribution=transition_distribution,
observation_distribution=emission_distribution,
num_steps=chain_len,
sample_shape=batch_size)
return model, pi_0, pi_T
def mmsb(N, K, data):
# sparsity
rho = 0.3
# MODEL
# probability of belonging to each of K blocks for each node
gamma = Dirichlet(concentration=tf.ones([K]))
# block connectivity
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
# probability of belonging to each of K blocks for all nodes
Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N)
# adjacency
X = Bernoulli(probs=(1 - rho) *
tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# INFERENCE (EM algorithm)
qgamma = PointMass(
params=tf.nn.softmax(tf.Variable(tf.random_normal([K]))))
qPi = PointMass(
params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K]))))
qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K]))))
#qgamma = Normal(loc=tf.get_variable("qgamma/loc", [K]),
# scale=tf.nn.softplus(
# tf.get_variable("qgamma/scale", [K])))
#qPi = Normal(loc=tf.get_variable("qPi/loc", [K, K]),
# scale=tf.nn.softplus(
# tf.get_variable("qPi/scale", [K, K])))
#qZ = Normal(loc=tf.get_variable("qZ/loc", [N, K]),
# scale=tf.nn.softplus(
# tf.get_variable("qZ/scale", [N, K])))
#inference = ed.KLqp({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data})
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data})
#inference.run()
n_iter = 6000
inference.initialize(optimizer=tf.train.AdamOptimizer(learning_rate=0.01),
n_iter=n_iter)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
inference.finalize()
print('qgamma after: ', qgamma.mean().eval())
return qZ.mean().eval(), qPi.eval()
def model_stationary_dirichlet_multinomial(n_states, chain_len,
total_counts_per_month):
""" Models a stationary Dirichlet-Multinomial Markov Chain in Edward """
tf.reset_default_graph()
# create default starting state probability vector
pi_list = [Dirichlet(tf.ones(n_states)) for _ in range(chain_len)]
# now instead of sample_shape we use total_count which
# is how many times we sample from each categorical
# i.e. number of accounts
counts = [
Multinomial(probs=pi, total_count=float(total_counts_per_month))
for pi in pi_list
]
return pi_list, counts
def main(_):
ed.set_seed(42)
# Prior on scalar hyperparameter to Dirichlet.
alpha = Gamma(1.0, 1.0)
# Prior on size of Dirichlet.
n = 1 + tf.cast(Exponential(0.5), tf.int32)
# Build a vector of ones whose size is n; multiply it by alpha.
p = Dirichlet(tf.ones([n]) * alpha)
sess = ed.get_session()
print(sess.run(p))
# [ 0.01012419 0.02939712 0.05036638 0.51287931 0.31020424 0.0485355
# 0.0384932 ]
print(sess.run(p))
def __init__(self, K, D, N, nu, use_param=False):
self.K = K # number of topics
self.D = D # number of documents
self.N = N # number of words of each document
self.nu = nu
self.alpha = alpha = tf.zeros([K]) + 0.1
mu0 = tf.constant([0.0] * nu)
sigma0 = tf.eye(nu)
self.sigma = sigma = WishartCholesky(
df=nu,
scale=sigma0,
cholesky_input_output_matrices=True,
sample_shape=K)
# sigma_inv = tf.matrix_inverse(sigma)
self.mu = mu = Normal(mu0, tf.ones(nu), sample_shape=K)
self.theta = theta = [None] * D
self.z = z = [None] * D
self.w = w = [None] * D
for d in range(D):
theta[d] = Dirichlet(alpha)
if use_param:
w[d] = ParamMixture(mixing_weights=theta[d],
component_params={
'loc': mu,
'scale_tril': sigma
},
component_dist=MultivariateNormalTriL,
sample_shape=N[d])
z[d] = w[d].cat
else:
z[d] = Categorical(probs=theta[d], sample_shape=N[d])
components = [
MultivariateNormalTriL(loc=tf.gather(mu, k),
scale_tril=tf.gather(sigma, k),
sample_shape=N[d]) for k in range(K)
]
w[d] = Mixture(cat=z[d],
components=components,
sample_shape=N[d])
def main(_):
ed.set_seed(42)
# DATA
X_data, Z_true = karate("~/data")
N = X_data.shape[0] # number of vertices
K = 2 # number of clusters
# MODEL
gamma = Dirichlet(concentration=tf.ones([K]))
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N)
X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# INFERENCE (EM algorithm)
qgamma = PointMass(tf.nn.softmax(tf.get_variable("qgamma/params", [K])))
qPi = PointMass(tf.nn.sigmoid(tf.get_variable("qPi/params", [K, K])))
qZ = PointMass(tf.nn.softmax(tf.get_variable("qZ/params", [N, K])))
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data})
inference.initialize(n_iter=250)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
# CRITICISM
Z_pred = qZ.mean().eval().argmax(axis=1)
print("Result (label flip can happen):")
print("Predicted")
print(Z_pred)
print("True")
print(Z_true)
print("Adjusted Rand Index =", adjusted_rand_score(Z_pred, Z_true))
plt.scatter(x_train[:, 0], x_train[:, 1])
plt.axis([-3, 3, -3, 3])
plt.title("Simulated dataset")
plt.show()
K = 2
D = 2
model = MixtureGaussian(K, D)
qpi_alpha = tf.nn.softplus(tf.Variable(tf.random_normal([K])))
qmu_mu = tf.Variable(tf.random_normal([K * D]))
qmu_sigma = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qsigma_alpha = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qsigma_beta = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qpi = Dirichlet(alpha=qpi_alpha)
qmu = Normal(mu=qmu_mu, sigma=qmu_sigma)
qsigma = InverseGamma(alpha=qsigma_alpha, beta=qsigma_beta)
data = {'x': x_train}
inference = ed.KLqp({'pi': qpi, 'mu': qmu, 'sigma': qsigma}, data, model)
inference.run(n_iter=2500, n_samples=10, n_minibatch=20)
# Average per-cluster and per-data point likelihood over many posterior samples.
log_liks = []
for s in range(100):
zrep = {'pi': qpi.sample(()),
'mu': qmu.sample(()),
'sigma': qsigma.sample(())}
log_liks += [model.predict(data, zrep)]
log_prior = dirichlet.logpdf(pi, self.alpha)
log_prior += tf.reduce_sum(norm.logpdf(mus, 0, np.sqrt(self.c)), 1)
log_prior += tf.reduce_sum(invgamma.logpdf(sigmas, self.a, self.b), 1)
# Loop over each mini-batch zs[b,:]
log_lik = []
n_minibatch = get_dims(zs[0])[0]
for s in range(n_minibatch):
log_lik_z = N*tf.reduce_sum(tf.log(pi), 1)
for k in range(self.K):
log_lik_z += tf.reduce_sum(multivariate_normal.logpdf(xs,
mus[s, (k*self.D):((k+1)*self.D)],
sigmas[s, (k*self.D):((k+1)*self.D)]))
log_lik += [log_lik_z]
return log_prior + tf.pack(log_lik)
ed.set_seed(42)
x = np.loadtxt('data/mixture_data.txt', dtype='float32', delimiter=',')
data = ed.Data(tf.constant(x, dtype=tf.float32))
model = MixtureGaussian(K=2, D=2)
variational = Variational()
variational.add(Dirichlet(model.K))
variational.add(Normal(model.K*model.D))
variational.add(InvGamma(model.K*model.D))
inference = ed.MFVI(model, variational, data)
inference.run(n_iter=500, n_minibatch=5, n_data=5)
true_mu = np.array([-1.0, 0.0, 1.0], np.float32) * 10
true_sigmasq = np.array([1.0**2, 2.0**2, 3.0**2], np.float32)
true_pi = np.array([0.2, 0.3, 0.5], np.float32)
N = 10000
K = len(true_mu)
true_z = np.random.choice(np.arange(K), size=N, p=true_pi)
x_data = true_mu[true_z] + np.random.randn(N) * np.sqrt(true_sigmasq[true_z])
# Prior hyperparameters
pi_alpha = np.ones(K, dtype=np.float32)
mu_sigma = np.std(true_mu)
sigmasq_alpha = 1.0
sigmasq_beta = 2.0
# Model
pi = Dirichlet(pi_alpha)
mu = Normal(0.0, mu_sigma, sample_shape=K)
sigmasq = InverseGamma(sigmasq_alpha, sigmasq_beta, sample_shape=K)
x = ParamMixture(pi, {
'mu': mu,
'sigma': tf.sqrt(sigmasq)
},
Normal,
sample_shape=N)
z = x.cat
# Conditionals
mu_cond = ed.complete_conditional(mu)
sigmasq_cond = ed.complete_conditional(sigmasq)
pi_cond = ed.complete_conditional(pi)
z_cond = ed.complete_conditional(z)
x = np.zeros((N, 2), dtype=np.float32)
for n in range(N):
k = np.argmax(np.random.multinomial(1, pi))
x[n, :] = np.random.multivariate_normal(mus[k], np.diag(stds[k]))
return x
N = 500 # number of data points
K = 2 # number of components
D = 2 # dimensionality of data
ed.set_seed(42)
x_train = build_toy_dataset(N)
pi = Dirichlet(tf.ones(K))
mu = Normal(tf.zeros(D), tf.ones(D), sample_shape=K)
sigmasq = InverseGamma(tf.ones(D), tf.ones(D), sample_shape=K)
x = ParamMixture(pi, {
'loc': mu,
'scale_diag': tf.sqrt(sigmasq)
},
MultivariateNormalDiag,
sample_shape=N)
z = x.cat
T = 500 # number of MCMC samples
qpi = Empirical(
tf.get_variable("qpi/params", [T, K],
initializer=tf.constant_initializer(1.0 / K)))
qmu = Empirical(
tf.get_variable("qmu/params", [T, K, D],
def _test(alpha, n): x = Dirichlet(alpha=alpha) val_est = get_dims(x.sample(n)) val_true = n + get_dims(alpha) assert val_est == val_true
X[src, dst] = 1
return X, Z
ed.set_seed(42)
# DATA
label_filepath = 'data/karate_labels.txt'
graph_filepath = 'data/karate_edgelist.txt'
X_data, Z_true = build_dataset(label_filepath, graph_filepath)
N = X_data.shape[0] # number of vertices
K = 2 # number of clusters
# MODEL
gamma = Dirichlet(concentration=tf.ones([K]))
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
Z = Multinomial(total_count=1., probs=gamma, sample_shape=N)
X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# INFERENCE (EM algorithm)
qgamma = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([K]))))
qPi = PointMass(params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K]))))
qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K]))))
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data})
n_iter = 100
inference.initialize(n_iter=n_iter)
tf.global_variables_initializer().run()
def _test(shape, alpha, size):
x = Dirichlet(shape, alpha)
val_est = tuple(get_dims(x.sample(size=size)))
val_true = (size, ) + shape
assert val_est == val_true
k = np.argmax(np.random.multinomial(1, pi))
x[n, :] = np.random.multivariate_normal(mus[k], np.diag(stds[k]))
return x
N = 500 # num data points
K = 2 # num components
D = 2 # dimensionality of data
ed.set_seed(42)
# DATA
x_data = build_toy_dataset(N)
# MODEL
pi = Dirichlet(alpha=tf.constant([1.0] * K))
mu = Normal(mu=tf.zeros([K, D]), sigma=tf.ones([K, D]))
sigma = InverseGamma(alpha=tf.ones([K, D]), beta=tf.ones([K, D]))
c = Categorical(logits=ed.tile(ed.logit(pi), [N, 1]))
x = Normal(mu=tf.gather(mu, c), sigma=tf.gather(sigma, c))
# INFERENCE
T = 5000
qpi = Empirical(params=tf.Variable(tf.ones([T, K]) / K))
qmu = Empirical(params=tf.Variable(tf.zeros([T, K, D])))
qsigma = Empirical(params=tf.Variable(tf.ones([T, K, D])))
qc = Empirical(params=tf.Variable(tf.zeros([T, N], dtype=tf.int32)))
gpi = Dirichlet(alpha=tf.constant([1.4, 1.6]))
gmu = Normal(mu=tf.constant([[1.0, 1.0], [-1.0, -1.0]]),
sigma=tf.constant([[0.5, 0.5], [0.5, 0.5]]))
x = np.zeros((N, 2))
for n in range(N):
k = np.argmax(np.random.multinomial(1, pi))
x[n, :] = np.random.multivariate_normal(mus[k], np.diag(stds[k]))
return x
N = 1000 # Number of data points
K = 2 # Number of components
D = 2 # Dimensionality of data
# DATA
x_data = build_toy_dataset(N)
# MODEL
pi = Dirichlet(concentration=tf.constant([1.0] * K))
mu = Normal(loc=tf.zeros([K, D]), scale=tf.ones([K, D]))
sigma = InverseGamma(concentration=tf.ones([K, D]), rate=tf.ones([K, D]))
c = Categorical(logits=tf.tile(tf.reshape(ed.logit(pi), [1, K]), [N, 1]))
x = Normal(loc=tf.gather(mu, c), scale=tf.gather(sigma, c))
# INFERENCE
qpi = Dirichlet(
concentration=tf.nn.softplus(tf.Variable(tf.random_normal([K]))))
qmu = Normal(loc=tf.Variable(tf.random_normal([K, D])),
scale=tf.nn.softplus(tf.Variable(tf.random_normal([K, D]))))
qsigma = InverseGamma(concentration=tf.nn.softplus(
tf.Variable(tf.random_normal([K, D]))),
rate=tf.nn.softplus(tf.Variable(tf.random_normal([K,
D]))))
qc = Categorical(logits=tf.Variable(tf.zeros([N, K])))
img_no = 61060
TRAIN_DIR = "../data/BSR/BSDS500/data/images/train/"
img, train_img = load_image_matrix(img_no, TRAIN_DIR)
# Hyperparameters
N = train_img.shape[0]
K = 15
D = train_img.shape[1]
T = 800 # number of MCMC samples
M = 300 # number of posterior samples sampled
ed.set_seed(67)
with tf.name_scope("model"):
pi = Dirichlet(concentration=tf.constant([1.0] * K, name="pi/weights"),
name="pi")
mu = Normal(loc=tf.zeros(D, name="centroids/loc"),
scale=tf.ones(D, name="centroids/scale"),
sample_shape=K,
name="centroids")
sigma = InverseGamma(concentration=tf.ones(
D, name="variability/concentration"),
rate=tf.ones(D, name="variability/rate"),
sample_shape=K,
name="variability")
x = ParamMixture(pi, {
'loc': mu,
'scale_diag': tf.sqrt(sigma)
},
MultivariateNormalDiag,
plt.scatter(x_train[:, 0], x_train[:, 1])
plt.axis([-3, 3, -3, 3])
plt.title("Simulated dataset")
plt.show()
K = 2
D = 2
model = MixtureGaussian(K, D)
qpi_alpha = tf.nn.softplus(tf.Variable(tf.random_normal([K])))
qmu_mu = tf.Variable(tf.random_normal([K * D]))
qmu_sigma = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qsigma_alpha = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qsigma_beta = tf.nn.softplus(tf.Variable(tf.random_normal([K * D])))
qpi = Dirichlet(alpha=qpi_alpha)
qmu = Normal(mu=qmu_mu, sigma=qmu_sigma)
qsigma = InverseGamma(alpha=qsigma_alpha, beta=qsigma_beta)
data = {'x': x_train}
inference = ed.KLqp({'pi': qpi, 'mu': qmu, 'sigma': qsigma}, data, model)
inference.run(n_iter=2500, n_samples=10, n_minibatch=20)
# Average per-cluster and per-data point likelihood over many posterior samples.
log_liks = []
for s in range(100):
zrep = {
'pi': qpi.sample(()),
'mu': qmu.sample(()),
'sigma': qsigma.sample(())
}
We build a random variable whose size depends on a sample from another random variable. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function import edward as ed import tensorflow as tf from edward.models import Exponential, Dirichlet, Gamma ed.set_seed(42) # Prior on scalar hyperparameter to Dirichlet. alpha = Gamma(alpha=1.0, beta=1.0) # Prior on size of Dirichlet. n = 1 + tf.cast(Exponential(lam=0.5), tf.int32) # Build a vector of ones whose size is n; multiply it by alpha. p = Dirichlet(alpha=tf.ones([n]) * alpha) sess = ed.get_session() print(sess.run(p.value())) # [ 0.01012419 0.02939712 0.05036638 0.51287931 0.31020424 0.0485355 # 0.0384932 ] print(sess.run(p.value())) # [ 0.12836078 0.23335715 0.63828212]
from edward.models import Normal, Exponential
import tensorflow as tf
# 都是RandomVariable衍生出来
# 一元分布
Normal(loc=tf.constant(0.0), scale=tf.constant(1.0))
Normal(loc=tf.zeros(5), scale=tf.ones(5))
Exponential(rate=tf.ones([2, 3]))
# 多元分布
from edward.models import Dirichlet, MultivariateNormalTriL
K = 3
Dirichlet(concentration=tf.constant([0.1] * K)) # K为Dirichlet分布
MultivariateNormalTriL(loc=tf.zeros([5, K]),
scale_tril=tf.ones([5, K, K])) # loc的最后一位表示维数
MultivariateNormalTriL(loc=tf.zeros([2, 5, K]),
scale_tril=tf.ones([2, 5, K, K]))
# 每个RandomVariable有方法log_prob()，mean()，sample()，且与计算图上的一个张量对应
# 可以支持诸多运算
from edward.models import Normal
x = Normal(loc=tf.zeros(10), scale=tf.ones(10))
y = tf.constant(5.0)
x + y, x - y, x * y, x / y
tf.tanh(x * y)
tf.gather(x, 2)
print(x[2])
# 有向图模型
from edward.models import Bernoulli, Beta
true_mu = np.array([-3.0, 0.0, 3.0], np.float32)
true_sigma = np.array([1.0, 1.0, 1.0], np.float32)
true_pi = np.array([.2, .3, .5], np.float32)
K = len(true_pi)
true_z = np.random.choice(np.arange(K), p=true_pi, size=N)
x = true_mu[true_z] + np.random.randn(N) * true_sigma[true_z]
# plt.hist(x, bins=200)
# plt.show()
# we like to calculate posterior p(\theta|x) where \theta=[mu_1,..., mu_3, sigma_1,...,sigma_3, z_1,...,z_3]
# Model
pi = Dirichlet(np.ones(K, np.float32))
mu = Normal(0.0, 9.0, sample_shape=[K])
sigma = InverseGamma(1.0, 1.0, sample_shape=[K])
c = Categorical(logits=tf.log(pi) - tf.log(1.0 - pi), sample_shape=N)
ed_x = Normal(loc=tf.gather(mu, c), scale=tf.gather(sigma, c))
# parameters
q_pi = Dirichlet(
tf.nn.softplus(
tf.get_variable("qpi", [K],
initializer=tf.constant_initializer(1.0 / K))))
q_mu = Normal(loc=tf.get_variable("qmu", [K]), scale=1.0)
q_sigma = Normal(loc=tf.nn.softplus(tf.get_variable("qsigma", [K])), scale=1.0)
inference = ed.KLqp(latent_vars={
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import edward as ed
import numpy as np
import tensorflow as tf
from edward.models import Categorical, Dirichlet
N = 1000
K = 4
# DATA
pi_true = np.random.dirichlet(np.array([20.0, 30.0, 10.0, 10.0]))
z_data = np.array([np.random.choice(K, 1, p=pi_true)[0] for n in range(N)])
print('pi={}'.format(pi_true))
# MODEL
pi = Dirichlet(alpha=tf.ones(4))
z = Categorical(p=tf.ones([N, 1]) * pi)
# INFERENCE
qpi = Dirichlet(alpha=tf.nn.softplus(tf.Variable(tf.random_normal([K]))))
inference = ed.KLqp({pi: qpi}, data={z: z_data})
inference.run(n_iter=1500, n_samples=30)
sess = ed.get_session()
print('Inferred pi={}'.format(sess.run(qpi.mean())))
def _test(shape, alpha, n):
x = Dirichlet(shape, alpha)
val_est = tuple(get_dims(x.sample(n)))
val_true = (n, ) + shape
assert val_est == val_true
