def test_ar_mle(self):
# set up test data: a random walk
T = 100
z_true = np.zeros(T)
r = 0.95
sig = 0.01
eta = 0.01
for t in range(1, 100):
z_true[t] = r * z_true[t - 1] + sig * np.random.randn()
x_data = (z_true + eta * np.random.randn(T)).astype(np.float32)
# use scipy to find max likelihood
def cost(z):
initial = z[0]**2 / sig**2
ar = np.sum((z[1:] - r * z[:-1])**2) / sig**2
data = np.sum((x_data - z)**2) / eta**2
return initial + ar + data
mle = minimize(cost, np.zeros(T)).x
with self.test_session() as sess:
z = AutoRegressive(T, r, sig)
x = Normal(loc=z, scale=eta)
qz = PointMass(params=tf.Variable(tf.zeros(T)))
inference = ed.MAP({z: qz}, data={x: x_data})
inference.run(n_iter=500)
self.assertAllClose(qz.eval(), mle, rtol=1e-3, atol=1e-3)
def __init__(self, latent_vars=None, data=None, model_wrapper=None):
"""
Parameters
----------
latent_vars : list of RandomVariable or
dict of RandomVariable to RandomVariable
Collection of random variables to perform inference on. If
list, each random variable will be implictly optimized
using a ``PointMass`` random variable that is defined
internally (with unconstrained support). If dictionary, each
random variable must be a ``PointMass`` random variable.
Examples
--------
Most explicitly, ``MAP`` is specified via a dictionary:
>>> qpi = PointMass(params=ed.to_simplex(tf.Variable(tf.zeros(K-1))))
>>> qmu = PointMass(params=tf.Variable(tf.zeros(K*D)))
>>> qsigma = PointMass(params=tf.nn.softplus(tf.Variable(tf.zeros(K*D))))
>>> MAP({pi: qpi, mu: qmu, sigma: qsigma}, data)
We also automate the specification of ``PointMass`` distributions,
so one can pass in a list of latent variables instead:
>>> MAP([beta], data)
>>> MAP([pi, mu, sigma], data)
Currently, ``MAP`` can only instantiate ``PointMass`` random variables
with unconstrained support. To constrain their support, one must
manually pass in the ``PointMass`` family.
"""
if isinstance(latent_vars, list):
with tf.variable_scope("posterior"):
if model_wrapper is None:
latent_vars = {
rv: PointMass(params=tf.Variable(
tf.random_normal(rv.batch_shape())))
for rv in latent_vars
}
elif len(latent_vars) == 1:
latent_vars = {
latent_vars[0]:
PointMass(params=tf.Variable(
tf.squeeze(tf.random_normal([model_wrapper.n_vars
]))))
}
elif len(latent_vars) == 0:
latent_vars = {}
else:
raise NotImplementedError(
"A list of more than one element is "
"not supported. See documentation.")
elif isinstance(latent_vars, dict):
for qz in six.itervalues(latent_vars):
if not isinstance(qz, PointMass):
raise TypeError(
"Posterior approximation must consist of only "
"PointMass random variables.")
super(MAP, self).__init__(latent_vars, data, model_wrapper)
def run(self, adj_mat, n_iter=1000):
assert adj_mat.shape[0] == adj_mat.shape[1]
n_node = adj_mat.shape[0]
# model
gamma = Dirichlet(concentration=tf.ones([self.n_cluster]))
Pi = Beta(concentration0=tf.ones([self.n_cluster, self.n_cluster]),
concentration1=tf.ones([self.n_cluster, self.n_cluster]))
Z = Multinomial(total_count=1., probs=gamma, sample_shape=n_node)
X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# inference (point estimation)
qgamma = PointMass(params=tf.nn.softmax(
tf.Variable(tf.random_normal([self.n_cluster]))))
qPi = PointMass(params=tf.nn.sigmoid(
tf.Variable(tf.random_normal([self.n_cluster, self.n_cluster]))))
qZ = PointMass(params=tf.nn.softmax(
tf.Variable(tf.random_normal([n_node, self.n_cluster]))))
# map estimation
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: adj_mat})
inference.initialize(n_iter=n_iter)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
inference.finalize()
return qZ.mean().eval().argmax(axis=1)
def _test(params, n):
rv = PointMass(params=params)
rv_sample = rv.sample(n)
x = rv_sample.eval()
x_tf = tf.constant(x, dtype=tf.float32)
params = params.eval()
assert np.allclose(
rv.log_prob(x_tf).eval(), pointmass_logpmf_vec(x, params))
def __init__(self, latent_vars, data=None, model_wrapper=None):
"""
Parameters
----------
latent_vars : list of RandomVariable or
dict of RandomVariable to RandomVariable
Collection of random variables to perform inference on. If
list, each random variable will be implictly optimized
using a ``PointMass`` distribution that is defined
internally (with support matching each random variable).
Examples
--------
Most explicitly, MAP is specified via a dictionary:
>>> qpi = PointMass(params=ed.to_simplex(tf.Variable(tf.zeros(K-1))))
>>> qmu = PointMass(params=tf.Variable(tf.zeros(K*D)))
>>> qsigma = PointMass(params=tf.nn.softplus(tf.Variable(tf.zeros(K*D))))
>>> MAP({pi: qpi, mu: qmu, sigma: qsigma}, data)
We also automate the specification of ``PointMass`` distributions
(with matching support), so one can pass in a list of latent
variables instead:
>>> MAP([beta], {X: np.array(), y: np.array()})
>>> MAP([pi, mu, sigma], {x: np.array()}
However, for model wrappers, the list can only have one element:
>>> MAP(['z'], data, model_wrapper)
For example, the following is not supported:
>>> MAP(['pi', 'mu', 'sigma'], data, model_wrapper)
This is because internally with model wrappers, we have no way
of knowing the dimensions in which to optimize each
distribution; further, we do not know their support. For more
than one random variable, or for constrained support, one must
explicitly pass in the point mass distributions.
"""
if isinstance(latent_vars, list):
with tf.variable_scope("variational"):
if model_wrapper is None:
latent_vars = {rv: PointMass(
params=tf.Variable(tf.random_normal(rv.batch_shape())))
for rv in latent_vars}
elif len(latent_vars) == 1:
latent_vars = {latent_vars[0]: PointMass(
params=tf.Variable(
tf.squeeze(tf.random_normal([model_wrapper.n_vars]))))}
elif len(latent_vars) == 0:
latent_vars = {}
else:
raise NotImplementedError("A list of more than one element is "
"not supported. See documentation.")
super(MAP, self).__init__(latent_vars, data, model_wrapper)
def __init__(self, model, data=Data(), transform=tf.identity):
if hasattr(model, 'num_vars'):
variational = Variational()
variational.add(PointMass(model.num_vars, transform))
else:
variational = Variational()
variational.add(PointMass(0, transform))
VariationalInference.__init__(self, model, variational, data)
def _test(shape, n):
rv = PointMass(shape, params=tf.zeros(shape)+0.5)
rv_sample = rv.sample(n)
x = rv_sample.eval()
x_tf = tf.constant(x, dtype=tf.float32)
params = rv.params.eval()
for idx in range(shape[0]):
assert np.allclose(
rv.log_prob_idx((idx, ), x_tf).eval(),
pointmass_logpmf_vec(x[:, idx], params[idx]))
def __init__(self, model, data=None, params=None):
with tf.variable_scope("variational"):
if hasattr(model, 'n_vars'):
variational = Variational()
variational.add(PointMass(model.n_vars, params))
else:
variational = Variational()
variational.add(PointMass(0))
super(MAP, self).__init__(model, variational, data)
def test_normalnormal_run(self):
with self.test_session() as sess:
x_data = np.array([0.0] * 50, dtype=np.float32)
mu = Normal(mu=0.0, sigma=1.0)
x = Normal(mu=tf.ones(50) * mu, sigma=1.0)
qmu = PointMass(params=tf.Variable(1.0))
# analytic solution: N(mu=0.0, sigma=\sqrt{1/51}=0.140)
inference = ed.MAP({mu: qmu}, data={x: x_data})
inference.run(n_iter=1000)
self.assertAllClose(qmu.mean().eval(), 0)
def test_normalnormal_run(self):
with self.test_session() as sess:
x_data = np.array([0.0] * 50, dtype=np.float32)
mu = Normal(loc=0.0, scale=1.0)
x = Normal(loc=mu, scale=1.0, sample_shape=50)
qmu = PointMass(params=tf.Variable(1.0))
# analytic solution: N(loc=0.0, scale=\sqrt{1/51}=0.140)
inference = ed.MAP({mu: qmu}, data={x: x_data})
inference.run(n_iter=1000)
self.assertAllClose(qmu.mean().eval(), 0)
def initialize(self, *args, **kwargs):
# Store latent variables in a temporary attribute; MAP will
# optimize ``PointMass`` random variables, which subsequently
# optimizes mean parameters of the normal approximations.
latent_vars_normal = self.latent_vars.copy()
self.latent_vars = {z: PointMass(params=qz.loc)
for z, qz in six.iteritems(latent_vars_normal)}
super(Laplace, self).initialize(*args, **kwargs)
hessians = tf.hessians(self.loss, list(six.itervalues(self.latent_vars)))
self.finalize_ops = []
for z, hessian in zip(six.iterkeys(self.latent_vars), hessians):
qz = latent_vars_normal[z]
if isinstance(qz, (MultivariateNormalDiag, Normal)):
scale_var = get_variables(qz.variance())[0]
scale = 1.0 / tf.diag_part(hessian)
else: # qz is MultivariateNormalTriL
scale_var = get_variables(qz.covariance())[0]
scale = tf.matrix_inverse(tf.cholesky(hessian))
self.finalize_ops.append(scale_var.assign(scale))
self.latent_vars = latent_vars_normal.copy()
del latent_vars_normal
def __init__(self, latent_vars=None, data=None):
"""Create an inference algorithm.
Args:
latent_vars: list of RandomVariable or
dict of RandomVariable to RandomVariable.
Collection of random variables to perform inference on. If
list, each random variable will be implictly optimized
using a `PointMass` random variable that is defined
internally (with unconstrained support). If dictionary, each
value in the dictionary must be a `PointMass` random variable.
"""
if isinstance(latent_vars, list):
with tf.variable_scope(None, default_name="posterior"):
latent_vars = {
rv: PointMass(
params=tf.Variable(tf.random_normal(rv.batch_shape)))
for rv in latent_vars
}
elif isinstance(latent_vars, dict):
for qz in six.itervalues(latent_vars):
if not isinstance(qz, PointMass):
raise TypeError(
"Posterior approximation must consist of only "
"PointMass random variables.")
super(MAP, self).__init__(latent_vars, data)
def train(self, n_iter=1000):
D = len(self.team_num_map.keys())
N = self.xs.shape[0]
with tf.name_scope('model'):
self.X = tf.placeholder(tf.float32, [N, D])
self.w1 = Normal(loc=tf.zeros(D), scale=tf.ones(D))
# self.b1 = Normal(loc=tf.zeros(1), scale=tf.ones(1))
self.y1 = Poisson(rate=tf.exp(ed.dot(self.X, self.w1)))
with tf.name_scope('posterior'):
if self.inf_type == 'Var':
self.qw1 = Normal(loc=tf.get_variable("qw1_ll/loc", [D]),
scale=tf.nn.softplus(
tf.get_variable("qw1_ll/scale", [D])))
# self.qb1 = Normal(loc=tf.get_variable("qb1/loc", [1]),
# scale=tf.nn.softplus(tf.get_variable("qb1/scale",
# [1])))
elif self.inf_type == 'MAP':
self.qw1 = PointMass(
Normal(loc=tf.get_variable("qw1_ll/loc", [D]),
scale=tf.nn.softplus(
tf.get_variable("qw1_ll/scale", [D]))))
if self.inf_type == 'Var':
inference = ed.ReparameterizationKLqp({self.w1: self.qw1},
data={
self.X: self.xs,
self.y1: self.ys
})
elif self.inf_type == 'MAP':
inference = ed.MAP({self.w1: self.qw1},
data={
self.X: self.xs,
self.y1: self.ys
})
inference.initialize(optimizer=tf.train.AdamOptimizer(
learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08),
n_iter=n_iter)
tf.global_variables_initializer().run()
self.loss = np.empty(n_iter, dtype=np.float32)
for i in range(n_iter):
info_dict = inference.update()
self.loss[i] = info_dict["loss"]
inference.print_progress(info_dict)
self._trained = True
graph = tf.get_default_graph()
self.team_skill = graph.get_tensor_by_name("qw1_ll/loc:0").eval()
self.perf_variance = graph.get_tensor_by_name("qw1_ll/scale:0").eval()
# self.bias = (graph.get_tensor_by_name("qb1/loc:0").eval(),
# graph.get_tensor_by_name("qb2/loc:0").eval())
self.y_post = ed.copy(self.y1, {self.w1: self.qw1})
return
def mmsb(N, K, data):
# sparsity
rho = 0.3
# MODEL
# probability of belonging to each of K blocks for each node
gamma = Dirichlet(concentration=tf.ones([K]))
# block connectivity
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
# probability of belonging to each of K blocks for all nodes
Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N)
# adjacency
X = Bernoulli(probs=(1 - rho) *
tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# INFERENCE (EM algorithm)
qgamma = PointMass(
params=tf.nn.softmax(tf.Variable(tf.random_normal([K]))))
qPi = PointMass(
params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K]))))
qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K]))))
#qgamma = Normal(loc=tf.get_variable("qgamma/loc", [K]),
# scale=tf.nn.softplus(
# tf.get_variable("qgamma/scale", [K])))
#qPi = Normal(loc=tf.get_variable("qPi/loc", [K, K]),
# scale=tf.nn.softplus(
# tf.get_variable("qPi/scale", [K, K])))
#qZ = Normal(loc=tf.get_variable("qZ/loc", [N, K]),
# scale=tf.nn.softplus(
# tf.get_variable("qZ/scale", [N, K])))
#inference = ed.KLqp({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data})
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data})
#inference.run()
n_iter = 6000
inference.initialize(optimizer=tf.train.AdamOptimizer(learning_rate=0.01),
n_iter=n_iter)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
inference.finalize()
print('qgamma after: ', qgamma.mean().eval())
return qZ.mean().eval(), qPi.eval()
def initialize(self, var_list=None, *args, **kwargs):
# Store latent variables in a temporary attribute; MAP will
# optimize ``PointMass`` random variables, which subsequently
# optimizes mean parameters of the normal approximations.
self.latent_vars_normal = self.latent_vars.copy()
self.latent_vars = {
z: PointMass(params=qz.mu)
for z, qz in six.iteritems(self.latent_vars_normal)
}
super(Laplace, self).initialize(var_list, *args, **kwargs)
def test_export_meta_graph(self):
with self.test_session() as sess:
x_data = np.array([0.0] * 50, dtype=np.float32)
mu = Normal(loc=0.0, scale=1.0)
x = Normal(loc=mu, scale=1.0, sample_shape=50)
qmu = PointMass(params=tf.Variable(1.0))
inference = ed.MAP({mu: qmu}, data={x: x_data})
inference.run(n_iter=10)
saver = tf.train.Saver()
saver.export_meta_graph("/tmp/test_saver.meta")
def test_normalnormal_regularization(self):
with self.test_session() as sess:
x_data = np.array([5.0] * 50, dtype=np.float32)
mu = Normal(loc=0.0, scale=1.0)
x = Normal(loc=mu, scale=1.0, sample_shape=50)
qmu = PointMass(params=tf.Variable(1.0))
inference = ed.MAP({mu: qmu}, data={x: x_data})
inference.run(n_iter=1000)
mu_val = qmu.mean().eval()
# regularized solution
regularizer = tf.contrib.layers.l2_regularizer(scale=1.0)
mu_reg = tf.get_variable("mu_reg", shape=[],
regularizer=regularizer)
x_reg = Normal(loc=mu_reg, scale=1.0, sample_shape=50)
inference_reg = ed.MAP(None, data={x_reg: x_data})
inference_reg.run(n_iter=1000)
mu_reg_val = mu_reg.eval()
self.assertAllClose(mu_val, mu_reg_val)
def test_map_custom(self):
with self.test_session() as sess:
x = Gamma(2.0, 0.5)
qx = PointMass(tf.nn.softplus(tf.Variable(0.5)))
inference = ed.MAP({x: qx})
inference.initialize(auto_transform=True, n_iter=500)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
# Check approximation on constrained space has same mode as
# target distribution.
stats = sess.run([x.mode(), qx])
self.assertAllClose(stats[0], stats[1], rtol=1e-5, atol=1e-5)
def test_save(self):
with self.test_session() as sess:
x_data = np.array([0.0] * 50, dtype=np.float32)
mu = Normal(loc=0.0, scale=1.0)
x = Normal(loc=mu, scale=1.0, sample_shape=50)
with tf.variable_scope("posterior"):
qmu = PointMass(params=tf.Variable(1.0))
inference = ed.MAP({mu: qmu}, data={x: x_data})
inference.run(n_iter=10)
saver = tf.train.Saver()
saver.save(sess, "/tmp/test_saver")
def main(_):
ed.set_seed(42)
# DATA
X_data, Z_true = karate("~/data")
N = X_data.shape[0] # number of vertices
K = 2 # number of clusters
# MODEL
gamma = Dirichlet(concentration=tf.ones([K]))
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N)
X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# INFERENCE (EM algorithm)
qgamma = PointMass(tf.nn.softmax(tf.get_variable("qgamma/params", [K])))
qPi = PointMass(tf.nn.sigmoid(tf.get_variable("qPi/params", [K, K])))
qZ = PointMass(tf.nn.softmax(tf.get_variable("qZ/params", [N, K])))
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data})
inference.initialize(n_iter=250)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
# CRITICISM
Z_pred = qZ.mean().eval().argmax(axis=1)
print("Result (label flip can happen):")
print("Predicted")
print(Z_pred)
print("True")
print(Z_true)
print("Adjusted Rand Index =", adjusted_rand_score(Z_pred, Z_true))
def main(_):
ed.set_seed(42)
# DATA
X_data, Z_true = karate("~/data")
N = X_data.shape[0] # number of vertices
K = 2 # number of clusters
# MODEL
gamma = Dirichlet(concentration=tf.ones([K]))
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N)
X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# INFERENCE (EM algorithm)
qgamma = PointMass(tf.nn.softmax(tf.get_variable("qgamma/params", [K])))
qPi = PointMass(tf.nn.sigmoid(tf.get_variable("qPi/params", [K, K])))
qZ = PointMass(tf.nn.softmax(tf.get_variable("qZ/params", [N, K])))
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data})
inference.initialize(n_iter=250)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
# CRITICISM
Z_pred = qZ.mean().eval().argmax(axis=1)
print("Result (label flip can happen):")
print("Predicted")
print(Z_pred)
print("True")
print(Z_true)
print("Adjusted Rand Index =", adjusted_rand_score(Z_pred, Z_true))
def test_restore(self):
with self.test_session() as sess:
x_data = np.array([0.0] * 50, dtype=np.float32)
mu = Normal(loc=0.0, scale=1.0)
x = Normal(loc=mu, scale=1.0, sample_shape=50)
with tf.variable_scope("posterior"):
qmu = PointMass(params=tf.Variable(1.0))
inference = ed.MAP({mu: qmu}, data={x: x_data})
saver = tf.train.Saver()
saver.restore(sess, "tests/data/test_saver")
qmu_variable = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope="posterior")[0]
self.assertNotEqual(qmu_variable.eval(), 1.0)
def __init__(self, latent_vars=None, data=None):
"""Create an inference algorithm.
Args:
latent_vars: list of RandomVariable or
dict of RandomVariable to RandomVariable.
Collection of random variables to perform inference on. If
list, each random variable will be implictly optimized using a
`PointMass` random variable that is defined internally with
constrained support, has unconstrained free parameters, and is
initialized using standard normal draws. If dictionary, each
value in the dictionary must be a `PointMass` random variable
with the same support as the key.
"""
if isinstance(latent_vars, list):
with tf.variable_scope(None, default_name="posterior"):
latent_vars_dict = {}
for z in latent_vars:
# Define point masses to have constrained support and
# unconstrained free parameters.
batch_event_shape = z.batch_shape.concatenate(
z.event_shape)
params = tf.Variable(tf.random_normal(batch_event_shape))
if hasattr(z, 'support'):
z_transform = transform(z)
if hasattr(z_transform, 'bijector'):
params = z_transform.bijector.inverse(params)
latent_vars_dict[z] = PointMass(params=params)
latent_vars = latent_vars_dict
del latent_vars_dict
elif isinstance(latent_vars, dict):
for qz in six.itervalues(latent_vars):
if not isinstance(qz, PointMass):
raise TypeError(
"Posterior approximation must consist of only "
"PointMass random variables.")
super(MAP, self).__init__(latent_vars, data)
def mmsb(N, K, data):
# sparsity
rho = 0.3
# MODEL
# probability of belonging to each of K blocks for each node
gamma = Dirichlet(concentration=tf.ones([K]))
# block connectivity
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
# probability of belonging to each of K blocks for all nodes
Z = Multinomial(total_count=1.0, probs=gamma, sample_shape=N)
# adjacency
X = Bernoulli(probs=(1 - rho) * tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# INFERENCE (EM algorithm)
qgamma = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([K]))))
qPi = PointMass(params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K]))))
qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K]))))
#qgamma = Normal(loc=tf.get_variable("qgamma/loc", [K]),
# scale=tf.nn.softplus(
# tf.get_variable("qgamma/scale", [K])))
#qPi = Normal(loc=tf.get_variable("qPi/loc", [K, K]),
# scale=tf.nn.softplus(
# tf.get_variable("qPi/scale", [K, K])))
#qZ = Normal(loc=tf.get_variable("qZ/loc", [N, K]),
# scale=tf.nn.softplus(
# tf.get_variable("qZ/scale", [N, K])))
#inference = ed.KLqp({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data})
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: data})
#inference.run()
n_iter = 6000
inference.initialize(optimizer=tf.train.AdamOptimizer(learning_rate=0.01), n_iter=n_iter)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
inference.finalize()
print('qgamma after: ', qgamma.mean().eval())
return qZ.mean().eval(), qPi.eval()
if not (self.interaction == "additive"
or self.prior == "Lognormal"):
raise NotImplementedError(
"Rate of Poisson has to be nonnegatve.")
log_lik = tf.reduce_sum(poisson.logpmf(xs['x'], xp))
else:
raise NotImplementedError("likelihood not available.")
return log_lik + log_prior
ed.set_seed(42)
x_train = np.load('data/celegans_brain.npy')
K = 3
model = MatrixFactorization(K, n_rows=x_train.shape[0])
qz = PointMass(
params=tf.nn.softplus(tf.Variable(tf.random_normal([model.n_vars]))))
data = {'x': x_train}
inference = ed.MAP({'z': qz}, data, model)
# Alternatively, run
# qz_mu = tf.Variable(tf.random_normal([model.n_vars]))
# qz_sigma = tf.nn.softplus(tf.Variable(tf.random_normal([model.n_vars])))
# qz = Normal(mu=qz_mu, sigma=qz_sigma)
# inference = ed.MFVI({'z': qz}, data, model)
inference.run(n_iter=2500)
#!/usr/bin/env python
"""Generate `test_saver`."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import edward as ed
import numpy as np
import tensorflow as tf
from edward.models import Normal, PointMass
x_data = np.array([0.0] * 50, dtype=np.float32)
mu = Normal(loc=0.0, scale=1.0)
x = Normal(loc=mu, scale=1.0, sample_shape=50)
with tf.variable_scope("posterior"):
qmu = PointMass(params=tf.Variable(1.0))
inference = ed.MAP({mu: qmu}, data={x: x_data})
inference.run(n_iter=10)
sess = ed.get_session()
saver = tf.train.Saver()
saver.save(sess, "test_saver")
def bayes_mult_cmd(table_file, metadata_file, formula, output_file):
#metadata = _type_cast_to_float(metadata.copy())
metadata = pd.read_table(metadata_file, index_col=0)
G_data = dmatrix(formula, metadata, return_type='dataframe')
table = load_table(table_file)
# basic filtering parameters
soil_filter = lambda val, id_, md: id_ in metadata.index
read_filter = lambda val, id_, md: np.sum(val) > 10
#sparse_filter = lambda val, id_, md: np.mean(val > 0) > 0.1
sample_filter = lambda val, id_, md: np.sum(val) > 1000
table = table.filter(soil_filter, axis='sample')
table = table.filter(sample_filter, axis='sample')
table = table.filter(read_filter, axis='observation')
#table = table.filter(sparse_filter, axis='observation')
print(table.shape)
y_data = pd.DataFrame(np.array(table.matrix_data.todense()).T,
index=table.ids(axis='sample'),
columns=table.ids(axis='observation'))
y_data, G_data = y_data.align(G_data, axis=0, join='inner')
psi = _gram_schmidt_basis(y_data.shape[1])
G_data = G_data.values
y_data = y_data.values
N, D = y_data.shape
p = G_data.shape[1] # number of covariates
r = G_data.shape[1] # rank of covariance matrix
psi = tf.convert_to_tensor(psi, dtype=tf.float32)
n = tf.convert_to_tensor(y_data.sum(axis=1), dtype=tf.float32)
# hack to get multinomial working
def _sample_n(self, n=1, seed=None):
# define Python function which returns samples as a Numpy array
def np_sample(p, n):
return multinomial.rvs(p=p, n=n, random_state=seed).astype(np.float32)
# wrap python function as tensorflow op
val = tf.py_func(np_sample, [self.probs, n], [tf.float32])[0]
# set shape from unknown shape
batch_event_shape = self.batch_shape.concatenate(self.event_shape)
shape = tf.concat(
[tf.expand_dims(n, 0), tf.convert_to_tensor(batch_event_shape)], 0)
val = tf.reshape(val, shape)
return val
Multinomial._sample_n = _sample_n
# dummy variable for gradient
G = tf.placeholder(tf.float32, [N, p])
b = Exponential(rate=1.0)
B = Normal(loc=tf.zeros([p, D-1]),
scale=tf.ones([p, D-1]) )
# Factorization of covariance matrix
# http://edwardlib.org/tutorials/klqp
l = Exponential(rate=1.0)
L = Normal(loc=tf.zeros([p, D-1]),
scale=tf.ones([p, D-1]) )
z = Normal(loc=tf.zeros([N, p]),
scale=tf.ones([N, p]))
# Cholesky trick to get multivariate normal
v = tf.matmul(G, B) + tf.matmul(z, L)
# get clr transformed values
eta = tf.matmul(v, psi)
Y = Multinomial(total_count=n, logits=eta)
T = 100000 # the number of mixin samples from MCMC sampling
qb = PointMass(params=tf.Variable(tf.random_normal([])))
qB = PointMass(params=tf.Variable(tf.random_normal([p, D-1])))
qz = Empirical(params=tf.Variable(tf.random_normal([T, N, p])))
ql = PointMass(params=tf.Variable(tf.random_normal([])))
qL = PointMass(params=tf.Variable(tf.random_normal([p, D-1])))
# Imputation
inference_z = ed.SGLD(
{z: qz},
data={G: G_data, Y: y_data, B: qB, L: qL}
)
# Maximization
inference_BL = ed.MAP(
{B: qB, L: qL, b: qb, l: ql},
data={G: G_data, Y: y_data, z: qz}
)
inference_z.initialize(step_size=1e-10)
inference_BL.initialize(n_iter=1000)
sess = ed.get_session()
saver = tf.train.Saver()
tf.global_variables_initializer().run()
for i in range(inference_BL.n_iter):
inference_z.update() # e-step
# will need to compute the expectation of z
info_dict = inference_BL.update() # m-step
inference_BL.print_progress(info_dict)
save_path = saver.save(sess, output_file)
print("Model saved in file: %s" % save_path)
pickle.dump({'qB': sess.run(qB.mean()),
'qL': sess.run(qL.mean()),
'qz': sess.run(qz.mean())},
open(output_file + '.params.pickle', 'wb')
)
reconstr_cau_ph = tf.placeholder(tf.float32, [M, N])
U = Gamma(0.3 * tf.ones([M, K]), 0.3 * tf.ones([M, K]))
V = Gamma(0.3 * tf.ones([N, K]), 0.3 * tf.ones([N, K]))
gamma = Gamma(tf.ones([M, 1]), tf.ones([M, 1]))
beta0 = Gamma(0.3 * tf.ones([1, 1]), 0.3 * tf.ones([1, 1]))
x = Poisson(tf.add(tf.matmul(U, V, transpose_b=True),\
tf.multiply(tf.matmul(gamma, tf.ones([1, N])), \
reconstr_cau_ph)) + beta0)
qU_variables = [tf.Variable(tf.random_uniform([D, K])), \
tf.Variable(tf.random_uniform([D, K]))]
qU = PointMass(
params=tf.nn.softplus(tf.gather(qU_variables[0], idx_ph)))
qV_variables = [tf.Variable(tf.random_uniform([N, K])), \
tf.Variable(tf.random_uniform([N, K]))]
qV = PointMass(params=tf.nn.softplus(qV_variables[0]))
qgamma_variables = [tf.Variable(tf.random_uniform([D, 1])), \
tf.Variable(tf.nn.softplus(tf.random_uniform([D, 1])))]
qgamma = PointMass(
params=tf.nn.softplus(tf.gather(qgamma_variables[0], idx_ph)))
qbeta0_variables = [tf.Variable(tf.random_uniform([1, 1])), \
idx_ph = tf.placeholder(tf.int32, M)
cau_ph = tf.placeholder(tf.float32, [M, N])
sd_ph = tf.placeholder(tf.float32, [M, N])
U = Normal(loc=tf.zeros([M, K]), scale=pri_U * tf.ones([M, K]))
V = Normal(loc=tf.zeros([N, K]), scale=pri_V * tf.ones([N, K]))
x = Normal(loc=tf.multiply(cau_ph, tf.matmul(U,
V,
transpose_b=True)),
scale=tf.ones([M, N]))
qU_variables = [tf.Variable(tf.random_uniform([D, K])), \
tf.Variable(tf.random_uniform([D, K]))]
qU = PointMass(params=tf.gather(qU_variables[0], idx_ph))
qV_variables = [tf.Variable(tf.random_uniform([N, K])), \
tf.Variable(tf.random_uniform([N, K]))]
qV = PointMass(params=qV_variables[0])
x_ph = tf.placeholder(tf.float32, [M, N])
optimizer = tf.train.RMSPropOptimizer(5e-5)
scale_factor = float(D) / M
inference_U = ed.MAP({U: qU}, \
data={x: x_ph, V: qV})
def _test(shape, params, size):
x = PointMass(shape, params)
val_est = tuple(get_dims(x.sample(size=size)))
val_true = (size, ) + shape
assert val_est == val_true
return log_prior + log_lik
def build_toy_dataset(N):
pi = np.array([0.4, 0.6])
mus = [[1, 1], [-1, -1]]
stds = [[0.1, 0.1], [0.1, 0.1]]
x = np.zeros((N, 2), dtype=np.float32)
for n in range(N):
k = np.argmax(np.random.multinomial(1, pi))
x[n, :] = np.random.multivariate_normal(mus[k], np.diag(stds[k]))
return x
ed.set_seed(42)
x_train = build_toy_dataset(500)
K = 2
D = 2
model = MixtureGaussian(K, D)
qpi = PointMass(params=ed.to_simplex(tf.Variable(tf.random_normal([K - 1]))))
qmu = PointMass(params=tf.Variable(tf.random_normal([K * D])))
qsigma = PointMass(params=tf.exp(tf.Variable(tf.random_normal([K * D]))))
data = {'x': x_train}
inference = ed.MAP({'pi': qpi, 'mu': qmu, 'sigma': qsigma}, data, model)
inference.run(n_iter=500, n_minibatch=10)
def _test(shape, params, n):
x = PointMass(shape, params)
val_est = tuple(get_dims(x.sample(n)))
val_true = (n,) + shape
assert val_est == val_true
#!/usr/bin/env python
"""A simple coin flipping example. Inspired by Stan's toy example.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import edward as ed
import numpy as np
import tensorflow as tf
from edward.models import Bernoulli, Beta, PointMass
ed.set_seed(42)
# DATA
x_data = np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1])
# MODEL
p = Beta(a=1.0, b=1.0)
x = Bernoulli(p=tf.ones(10) * p)
# INFERENCE
qp_params = tf.nn.sigmoid(tf.Variable(tf.random_normal([])))
qp = PointMass(params=qp_params)
data = {x: x_data}
inference = ed.MAP({p: qp}, data)
inference.run(n_iter=50)
noise_proc = tf.constant(0.1)
# InverseGamma(alpha=1.0, beta=1.0)
noise_obs = tf.constant(0.1)
# InverseGamma(alpha=1.0, beta=1.0)
x = [0] * T
for n in range(p):
x[n] = Normal(loc=mu, scale=10.0) # fat prior on x
for n in range(p, T):
mu_ = mu
for j in range(p):
mu_ += beta[j] * x[n - j - 1]
x[n] = Normal(loc=mu_, scale=noise_proc)
print("setting up distributions")
qmu = PointMass(params=tf.Variable(0.))
qbeta = [PointMass(params=tf.Variable(0.)) for i in range(p)]
print("constructing inference object")
vdict = {mu: qmu}
vdict.update({b: qb for b, qb in zip(beta, qbeta)})
inference = ed.MAP(vdict, data={xt: xt_true for xt, xt_true in zip(x, x_true)})
print("running inference")
inference.run()
print("parameter estimates:")
print("beta: ", [qb.value().eval() for qb in qbeta])
print("mu: ", qmu.value().eval())
print("setting up variational distributions")
qmu = Normal(loc=tf.Variable(0.), scale=tf.nn.softplus(tf.Variable(0.)))
qbeta = [
label_filepath = 'data/karate_labels.txt'
graph_filepath = 'data/karate_edgelist.txt'
X_data, Z_true = build_dataset(label_filepath, graph_filepath)
N = X_data.shape[0] # number of vertices
K = 2 # number of clusters
# MODEL
gamma = Dirichlet(concentration=tf.ones([K]))
Pi = Beta(concentration0=tf.ones([K, K]), concentration1=tf.ones([K, K]))
Z = Multinomial(total_count=1., probs=gamma, sample_shape=N)
X = Bernoulli(probs=tf.matmul(Z, tf.matmul(Pi, tf.transpose(Z))))
# INFERENCE (EM algorithm)
qgamma = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([K]))))
qPi = PointMass(params=tf.nn.sigmoid(tf.Variable(tf.random_normal([K, K]))))
qZ = PointMass(params=tf.nn.softmax(tf.Variable(tf.random_normal([N, K]))))
inference = ed.MAP({gamma: qgamma, Pi: qPi, Z: qZ}, data={X: X_data})
n_iter = 100
inference.initialize(n_iter=n_iter)
tf.global_variables_initializer().run()
for _ in range(inference.n_iter):
info_dict = inference.update()
inference.print_progress(info_dict)
inference.finalize()
# CRITICISM
Z_pred = qZ.mean().eval().argmax(axis=1)
def pointmass_q(shape, name=None):
with tf.variable_scope(name, default_name="pointmass_q"):
min_mean = 1e-3
mean = tf.get_variable("mean", shape)
rv = PointMass(tf.maximum(tf.nn.softplus(mean), min_mean))
return rv
