def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
# MODEL
# Define a subgraph of the full model, corresponding to a minibatch of
# size M.
z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]),
scale=tf.ones([FLAGS.M, FLAGS.d]))
hidden = tf.layers.dense(z, 256, activation=tf.nn.relu)
x = Bernoulli(logits=tf.layers.dense(hidden, 28 * 28))
# INFERENCE
# Define a subgraph of the variational model, corresponding to a
# minibatch of size M.
x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28])
hidden = tf.layers.dense(tf.cast(x_ph, tf.float32),
256,
activation=tf.nn.relu)
qz = Normal(loc=tf.layers.dense(hidden, FLAGS.d),
scale=tf.layers.dense(hidden,
FLAGS.d,
activation=tf.nn.softplus))
# Bind p(x, z) and q(z | x) to the same TensorFlow placeholder for x.
inference = ed.KLqp({z: qz}, data={x: x_ph})
optimizer = tf.train.RMSPropOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)
tf.global_variables_initializer().run()
n_iter_per_epoch = x_train.shape[0] // FLAGS.M
for epoch in range(1, FLAGS.n_epoch + 1):
print("Epoch: {0}".format(epoch))
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss /= n_iter_per_epoch
avg_loss /= FLAGS.M
print("-log p(x) <= {:0.3f}".format(avg_loss))
# Prior predictive check.
images = x.eval()
for m in range(FLAGS.M):
imsave(
os.path.join(FLAGS.out_dir, '%d.png') % m,
images[m].reshape(28, 28))
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
# MODEL
# Define a subgraph of the full model, corresponding to a minibatch of
# size M.
z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]),
scale=tf.ones([FLAGS.M, FLAGS.d]))
hidden = tf.layers.dense(z, 256, activation=tf.nn.relu)
x = Bernoulli(logits=tf.layers.dense(hidden, 28 * 28))
# INFERENCE
# Define a subgraph of the variational model, corresponding to a
# minibatch of size M.
x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28])
hidden = tf.layers.dense(tf.cast(x_ph, tf.float32), 256,
activation=tf.nn.relu)
qz = Normal(loc=tf.layers.dense(hidden, FLAGS.d),
scale=tf.layers.dense(
hidden, FLAGS.d, activation=tf.nn.softplus))
# Bind p(x, z) and q(z | x) to the same TensorFlow placeholder for x.
inference = ed.KLqp({z: qz}, data={x: x_ph})
optimizer = tf.train.RMSPropOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)
tf.global_variables_initializer().run()
n_iter_per_epoch = x_train.shape[0] // FLAGS.M
for epoch in range(1, FLAGS.n_epoch + 1):
print("Epoch: {0}".format(epoch))
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss /= n_iter_per_epoch
avg_loss /= FLAGS.M
print("-log p(x) <= {:0.3f}".format(avg_loss))
# Prior predictive check.
images = x.eval()
for m in range(FLAGS.M):
imsave(os.path.join(FLAGS.out_dir, '%d.png') % m,
images[m].reshape(28, 28))
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
# MODEL
z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]),
scale=tf.ones([FLAGS.M, FLAGS.d]))
logits = generative_network(z)
x = Bernoulli(logits=logits)
# INFERENCE
x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28])
loc, scale = inference_network(tf.cast(x_ph, tf.float32))
qz = Normal(loc=loc, scale=scale)
# Bind p(x, z) and q(z | x) to the same placeholder for x.
inference = ed.KLqp({z: qz}, data={x: x_ph})
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)
hidden_rep = tf.sigmoid(logits)
tf.global_variables_initializer().run()
n_iter_per_epoch = x_train.shape[0] // FLAGS.M
for epoch in range(1, FLAGS.n_epoch + 1):
print("Epoch: {0}".format(epoch))
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss /= n_iter_per_epoch
avg_loss /= FLAGS.M
print("-log p(x) <= {:0.3f}".format(avg_loss))
# Visualize hidden representations.
images = hidden_rep.eval()
for m in range(FLAGS.M):
imsave(
os.path.join(FLAGS.out_dir, '%d.png') % m,
images[m].reshape(28, 28))
def main(_):
ed.set_seed(42)
# DATA. MNIST batches are fed at training time.
(x_train, _), (x_test, _) = mnist(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.M)
# MODEL
z = Normal(loc=tf.zeros([FLAGS.M, FLAGS.d]),
scale=tf.ones([FLAGS.M, FLAGS.d]))
logits = generative_network(z)
x = Bernoulli(logits=logits)
# INFERENCE
x_ph = tf.placeholder(tf.int32, [FLAGS.M, 28 * 28])
loc, scale = inference_network(tf.cast(x_ph, tf.float32))
qz = Normal(loc=loc, scale=scale)
# Bind p(x, z) and q(z | x) to the same placeholder for x.
inference = ed.KLqp({z: qz}, data={x: x_ph})
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer)
hidden_rep = tf.sigmoid(logits)
tf.global_variables_initializer().run()
n_iter_per_epoch = x_train.shape[0] // FLAGS.M
for epoch in range(1, FLAGS.n_epoch + 1):
print("Epoch: {0}".format(epoch))
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss /= n_iter_per_epoch
avg_loss /= FLAGS.M
print("-log p(x) <= {:0.3f}".format(avg_loss))
# Visualize hidden representations.
images = hidden_rep.eval()
for m in range(FLAGS.M):
imsave(os.path.join(FLAGS.out_dir, '%d.png') % m,
images[m].reshape(28, 28))
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer, use_prettytensor=True)
hidden_rep = tf.sigmoid(logits)
init = tf.global_variables_initializer()
init.run()
n_epoch = 100
n_iter_per_epoch = 1000
for epoch in range(n_epoch):
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_train, _ = mnist.train.next_batch(M)
info_dict = inference.update(feed_dict={x_ph: x_train})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss = avg_loss / n_iter_per_epoch
avg_loss = avg_loss / M
print("log p(x) >= {:0.3f}".format(avg_loss))
# Visualize hidden representations.
imgs = hidden_rep.eval()
for m in range(M):
imsave(os.path.join(IMG_DIR, '%d.png') % m, imgs[m].reshape(28, 28))
class Inference(object):
"""Abstract base class for inference. All inference algorithms in
Edward inherit from ``Inference``, sharing common methods and
properties via a class hierarchy.
Specific algorithms typically inherit from other subclasses of
``Inference`` rather than ``Inference`` directly. For example, one
might inherit from the abstract classes ``MonteCarlo`` or
``VariationalInference``.
To build an algorithm inheriting from ``Inference``, one must at the
minimum implement ``initialize`` and ``update``: the former builds
the computational graph for the algorithm; the latter runs the
computational graph for the algorithm.
To reset inference (e.g., internal variable counters incremented
over training), fetch inference's reset ops from session with
``sess.run(inference.reset)``.
"""
def __init__(self, latent_vars=None, data=None):
"""Initialization.
Parameters
----------
latent_vars : dict, optional
Collection of latent variables (of type ``RandomVariable`` or
``tf.Tensor``) to perform inference on. Each random variable is
binded to another random variable; the latter will infer the
former conditional on data.
data : dict, optional
Data dictionary which binds observed variables (of type
``RandomVariable`` or ``tf.Tensor``) to their realizations (of
type ``tf.Tensor``). It can also bind placeholders (of type
``tf.Tensor``) used in the model to their realizations; and
prior latent variables (of type ``RandomVariable``) to posterior
latent variables (of type ``RandomVariable``).
Examples
--------
>>> mu = Normal(loc=tf.constant(0.0), scale=tf.constant(1.0))
>>> x = Normal(loc=tf.ones(50) * mu, scale=tf.constant(1.0))
>>>
>>> qmu_loc = tf.Variable(tf.random_normal([]))
>>> qmu_scale = tf.nn.softplus(tf.Variable(tf.random_normal([])))
>>> qmu = Normal(loc=qmu_loc, scale=qmu_scale)
>>>
>>> inference = ed.Inference({mu: qmu}, data={x: tf.zeros(50)})
"""
sess = get_session()
if latent_vars is None:
latent_vars = {}
if data is None:
data = {}
check_latent_vars(latent_vars)
self.latent_vars = latent_vars
check_data(data)
self.data = {}
for key, value in six.iteritems(data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
self.data[key] = value
elif isinstance(key, (RandomVariable, tf.Tensor)):
if isinstance(value, (RandomVariable, tf.Tensor)):
self.data[key] = value
elif isinstance(
value, (float, list, int, np.ndarray, np.number, str)):
# If value is a Python type, store it in the graph.
# Assign its placeholder with the key's data type.
with tf.variable_scope("data"):
ph = tf.placeholder(key.dtype, np.shape(value))
var = tf.Variable(ph, trainable=False, collections=[])
sess.run(var.initializer, {ph: value})
self.data[key] = var
def run(self, variables=None, use_coordinator=True, *args, **kwargs):
"""A simple wrapper to run inference.
1. Initialize algorithm via ``initialize``.
2. (Optional) Build a TensorFlow summary writer for TensorBoard.
3. (Optional) Initialize TensorFlow variables.
4. (Optional) Start queue runners.
5. Run ``update`` for ``self.n_iter`` iterations.
6. While running, ``print_progress``.
7. Finalize algorithm via ``finalize``.
8. (Optional) Stop queue runners.
To customize the way inference is run, run these steps
individually.
Parameters
----------
variables : list, optional
A list of TensorFlow variables to initialize during inference.
Default is to initialize all variables (this includes
reinitializing variables that were already initialized). To
avoid initializing any variables, pass in an empty list.
use_coordinator : bool, optional
Whether to start and stop queue runners during inference using a
TensorFlow coordinator. For example, queue runners are necessary
for batch training with file readers.
*args
Passed into ``initialize``.
**kwargs
Passed into ``initialize``.
"""
self.initialize(*args, **kwargs)
if variables is None:
init = tf.global_variables_initializer()
else:
init = tf.variables_initializer(variables)
# Feed placeholders in case initialization depends on them.
feed_dict = {}
for key, value in six.iteritems(self.data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
feed_dict[key] = value
init.run(feed_dict)
if use_coordinator:
# Start input enqueue threads.
self.coord = tf.train.Coordinator()
self.threads = tf.train.start_queue_runners(coord=self.coord)
for _ in range(self.n_iter):
info_dict = self.update()
self.print_progress(info_dict)
self.finalize()
if use_coordinator:
# Ask threads to stop.
self.coord.request_stop()
self.coord.join(self.threads)
@abc.abstractmethod
def initialize(self,
n_iter=1000,
n_print=None,
scale=None,
logdir=None,
log_timestamp=True,
log_vars=None,
debug=False):
"""Initialize inference algorithm. It initializes hyperparameters
and builds ops for the algorithm's computation graph.
Any derived class of ``Inference`` **must** implement this method.
No methods which build ops should be called outside ``initialize()``.
Parameters
----------
n_iter : int, optional
Number of iterations for algorithm.
n_print : int, optional
Number of iterations for each print progress. To suppress print
progress, then specify 0. Default is ``int(n_iter / 100)``.
scale : dict of RandomVariable to tf.Tensor, optional
A tensor to scale computation for any random variable that it is
binded to. Its shape must be broadcastable; it is multiplied
element-wise to the random variable. For example, this is useful
for mini-batch scaling when inferring global variables, or
applying masks on a random variable.
logdir : str, optional
Directory where event file will be written. For details,
see ``tf.summary.FileWriter``. Default is to log nothing.
log_timestamp : bool, optional
If True (and ``logdir`` is specified), create a subdirectory of
``logdir`` to save the specific run results. The subdirectory's
name is the current UTC timestamp with format 'YYYYMMDD_HHMMSS'.
log_vars : list, optional
Specifies the list of variables to log after each ``n_print``
steps. If None, will log all variables. If ``[]``, no variables
will be logged. ``logdir`` must be specified for variables to be
logged.
debug : bool, optional
If True, add checks for ``NaN`` and ``Inf`` to all computations
in the graph. May result in substantially slower execution
times.
"""
self.n_iter = n_iter
if n_print is None:
self.n_print = int(n_iter / 100)
else:
self.n_print = n_print
self.progbar = Progbar(self.n_iter)
self.t = tf.Variable(0, trainable=False, name="iteration")
self.increment_t = self.t.assign_add(1)
if scale is None:
scale = {}
elif not isinstance(scale, dict):
raise TypeError("scale must be a dict object.")
self.scale = scale
if logdir is not None:
self.logging = True
if log_timestamp:
logdir = os.path.join(
logdir,
datetime.strftime(datetime.utcnow(), "%Y%m%d_%H%M%S"))
self._set_log_variables(log_vars)
self.train_writer = tf.summary.FileWriter(logdir,
tf.get_default_graph())
self.summarize = tf.summary.merge_all()
else:
self.logging = False
self.debug = debug
if self.debug:
self.op_check = tf.add_check_numerics_ops()
# Store reset ops which user can call. Subclasses should append
# any ops needed to reset internal variables in inference.
self.reset = [tf.variables_initializer([self.t])]
@abc.abstractmethod
def update(self, feed_dict=None):
"""Run one iteration of inference.
Any derived class of ``Inference`` **must** implement this method.
Parameters
----------
feed_dict : dict, optional
Feed dictionary for a TensorFlow session run. It is used to feed
placeholders that are not fed during initialization.
Returns
-------
dict
Dictionary of algorithm-specific information.
"""
if feed_dict is None:
feed_dict = {}
for key, value in six.iteritems(self.data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
feed_dict[key] = value
sess = get_session()
t = sess.run(self.increment_t)
if self.debug:
sess.run(self.op_check, feed_dict)
if self.logging and self.n_print != 0:
if t == 1 or t % self.n_print == 0:
summary = sess.run(self.summarize, feed_dict)
self.train_writer.add_summary(summary, t)
return {'t': t}
def print_progress(self, info_dict):
"""Print progress to output.
Parameters
----------
info_dict : dict
Dictionary of algorithm-specific information.
"""
if self.n_print != 0:
t = info_dict['t']
if t == 1 or t % self.n_print == 0:
self.progbar.update(t)
def finalize(self):
"""Function to call after convergence.
"""
if self.logging:
self.train_writer.close()
def _set_log_variables(self, log_vars=None):
"""Log variables to TensorBoard.
For each variable in ``log_vars``, forms a ``tf.summary.scalar``if
the variable has scalar shape; otherwise forms a``tf.summary.histogram``.
Parameters
----------
log_vars : list, optional
Specifies the list of variables to log after each ``n_print``
steps. If None, will log all variables. If ``[]``, no variables
will be logged.
"""
summary_key = 'summaries_' + str(id(self))
if log_vars is None:
log_vars = []
for key in six.iterkeys(self.data):
log_vars += get_variables(key)
for key, value in six.iteritems(self.latent_vars):
log_vars += get_variables(key)
log_vars += get_variables(value)
log_vars = set(log_vars)
for var in log_vars:
# replace colons which are an invalid character
var_name = var.name.replace(':', '/')
# Log all scalars.
if len(var.shape) == 0:
tf.summary.scalar("parameter/{}".format(var_name),
var,
collections=[summary_key])
elif len(var.shape) == 1 and var.shape[0] == 1:
tf.summary.scalar("parameter/{}".format(var_name),
var[0],
collections=[summary_key])
else:
# If var is multi-dimensional, log a histogram of its values.
tf.summary.histogram("parameter/{}".format(var_name),
var,
collections=[summary_key])
def main(_):
ed.set_seed(42)
# DATA
(x_train, _), (x_test, _), (x_valid, _) = caltech101_silhouettes(
FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.batch_size)
x_ph = tf.placeholder(tf.int32, [None, 28 * 28])
# MODEL
zs = [0] * len(FLAGS.hidden_sizes)
for l in reversed(range(len(FLAGS.hidden_sizes))):
if l == len(FLAGS.hidden_sizes) - 1:
logits = tf.zeros([tf.shape(x_ph)[0], FLAGS.hidden_sizes[l]])
else:
logits = tf.layers.dense(tf.cast(zs[l + 1], tf.float32),
FLAGS.hidden_sizes[l], activation=None)
zs[l] = Bernoulli(logits=logits)
x = Bernoulli(logits=tf.layers.dense(tf.cast(zs[0], tf.float32),
28 * 28, activation=None))
# INFERENCE
# Define variational model with reverse ordering as probability model:
# if p is 15-100-300 from top-down, q is 300-100-15 from bottom-up.
qzs = [0] * len(FLAGS.hidden_sizes)
for l in range(len(FLAGS.hidden_sizes)):
if l == 0:
logits = tf.layers.dense(tf.cast(x_ph, tf.float32),
FLAGS.hidden_sizes[l], activation=None)
else:
logits = tf.layers.dense(tf.cast(qzs[l - 1], tf.float32),
FLAGS.hidden_sizes[l], activation=None)
qzs[l] = Bernoulli(logits=logits)
inference = ed.KLqp({z: qz for z, qz in zip(zs, qzs)}, data={x: x_ph})
optimizer = tf.train.AdamOptimizer(FLAGS.step_size)
inference.initialize(optimizer=optimizer, n_samples=FLAGS.n_train_samples)
# Build tensor for log-likelihood given one variational sample to run
# on test data.
x_post = ed.copy(x, {z: qz for z, qz in zip(zs, qzs)})
x_neg_log_prob = (-tf.reduce_sum(x_post.log_prob(x_ph)) /
tf.cast(tf.shape(x_ph)[0], tf.float32))
sess = ed.get_session()
tf.global_variables_initializer().run()
for epoch in range(FLAGS.n_epoch):
print("Epoch {}".format(epoch))
train_loss = 0.0
pbar = Progbar(FLAGS.n_iter_per_epoch)
for t in range(1, FLAGS.n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
train_loss += info_dict['loss']
# Print per-data point loss, averaged over training epoch.
train_loss /= FLAGS.n_iter_per_epoch
train_loss /= FLAGS.batch_size
print("Training negative log-likelihood: {:0.3f}".format(train_loss))
test_loss = [sess.run(x_neg_log_prob, {x_ph: x_test})
for _ in range(FLAGS.n_test_samples)]
test_loss = np.mean(test_loss)
print("Test negative log-likelihood: {:0.3f}".format(test_loss))
# Prior predictive check.
images = sess.run(x, {x_ph: x_batch}) # feed ph to determine sample size
for m in range(FLAGS.batch_size):
imsave("{}/{}.png".format(out_dir, m), images[m].reshape(28, 28))
class Inference(object):
"""Abstract base class for inference. All inference algorithms in
Edward inherit from `Inference`, sharing common methods and
properties via a class hierarchy.
Specific algorithms typically inherit from other subclasses of
`Inference` rather than `Inference` directly. For example, one
might inherit from the abstract classes `MonteCarlo` or
`VariationalInference`.
To build an algorithm inheriting from `Inference`, one must at the
minimum implement `initialize` and `update`: the former builds
the computational graph for the algorithm; the latter runs the
computational graph for the algorithm.
To reset inference (e.g., internal variable counters incremented
over training), fetch inference's reset ops from session with
`sess.run(inference.reset)`.
#### Examples
```python
mu = Normal(loc=tf.constant(0.0), scale=tf.constant(1.0))
x = Normal(loc=tf.ones(50) * mu, scale=tf.constant(1.0))
qmu_loc = tf.Variable(tf.random_normal([]))
qmu_scale = tf.nn.softplus(tf.Variable(tf.random_normal([])))
qmu = Normal(loc=qmu_loc, scale=qmu_scale)
inference = ed.Inference({mu: qmu}, data={x: tf.zeros(50)})
```
"""
def __init__(self, latent_vars=None, data=None):
"""Create an inference algorithm.
Args:
latent_vars: dict, optional.
Collection of latent variables (of type `RandomVariable` or
`tf.Tensor`) to perform inference on. Each random variable is
binded to another random variable; the latter will infer the
former conditional on data.
data: dict, optional.
Data dictionary which binds observed variables (of type
`RandomVariable` or `tf.Tensor`) to their realizations (of
type `tf.Tensor`). It can also bind placeholders (of type
`tf.Tensor`) used in the model to their realizations; and
prior latent variables (of type `RandomVariable`) to posterior
latent variables (of type `RandomVariable`).
"""
sess = get_session()
if latent_vars is None:
latent_vars = {}
if data is None:
data = {}
check_latent_vars(latent_vars)
self.latent_vars = latent_vars
check_data(data)
self.data = {}
for key, value in six.iteritems(data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
self.data[key] = value
elif isinstance(key, (RandomVariable, tf.Tensor)):
if isinstance(value, (RandomVariable, tf.Tensor)):
self.data[key] = value
elif isinstance(value, (float, list, int, np.ndarray, np.number, str)):
# If value is a Python type, store it in the graph.
# Assign its placeholder with the key's data type.
with tf.variable_scope("data"):
ph = tf.placeholder(key.dtype, np.shape(value))
var = tf.Variable(ph, trainable=False, collections=[])
sess.run(var.initializer, {ph: value})
self.data[key] = var
def run(self, variables=None, use_coordinator=True, *args, **kwargs):
"""A simple wrapper to run inference.
1. Initialize algorithm via `initialize`.
2. (Optional) Build a TensorFlow summary writer for TensorBoard.
3. (Optional) Initialize TensorFlow variables.
4. (Optional) Start queue runners.
5. Run `update` for `self.n_iter` iterations.
6. While running, `print_progress`.
7. Finalize algorithm via `finalize`.
8. (Optional) Stop queue runners.
To customize the way inference is run, run these steps
individually.
Args:
variables: list, optional.
A list of TensorFlow variables to initialize during inference.
Default is to initialize all variables (this includes
reinitializing variables that were already initialized). To
avoid initializing any variables, pass in an empty list.
use_coordinator: bool, optional.
Whether to start and stop queue runners during inference using a
TensorFlow coordinator. For example, queue runners are necessary
for batch training with file readers.
*args:
Passed into `initialize`.
**kwargs:
Passed into `initialize`.
"""
self.initialize(*args, **kwargs)
if variables is None:
init = tf.global_variables_initializer()
else:
init = tf.variables_initializer(variables)
# Feed placeholders in case initialization depends on them.
feed_dict = {}
for key, value in six.iteritems(self.data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
feed_dict[key] = value
init.run(feed_dict)
if use_coordinator:
# Start input enqueue threads.
self.coord = tf.train.Coordinator()
self.threads = tf.train.start_queue_runners(coord=self.coord)
for _ in range(self.n_iter):
info_dict = self.update()
self.print_progress(info_dict)
self.finalize()
if use_coordinator:
# Ask threads to stop.
self.coord.request_stop()
self.coord.join(self.threads)
@abc.abstractmethod
def initialize(self, n_iter=1000, n_print=None, scale=None, logdir=None,
log_timestamp=True, log_vars=None, debug=False):
"""Initialize inference algorithm. It initializes hyperparameters
and builds ops for the algorithm's computation graph.
Any derived class of `Inference` **must** implement this method.
No methods which build ops should be called outside `initialize()`.
Args:
n_iter: int, optional.
Number of iterations for algorithm when calling `run()`.
Alternatively if controlling inference manually, it is the
expected number of calls to `update()`; this number determines
tracking information during the print progress.
n_print: int, optional.
Number of iterations for each print progress. To suppress print
progress, then specify 0. Default is `int(n_iter / 100)`.
scale: dict of RandomVariable to tf.Tensor, optional.
A tensor to scale computation for any random variable that it is
binded to. Its shape must be broadcastable; it is multiplied
element-wise to the random variable. For example, this is useful
for mini-batch scaling when inferring global variables, or
applying masks on a random variable.
logdir: str, optional.
Directory where event file will be written. For details,
see `tf.summary.FileWriter`. Default is to log nothing.
log_timestamp: bool, optional.
If True (and `logdir` is specified), create a subdirectory of
`logdir` to save the specific run results. The subdirectory's
name is the current UTC timestamp with format 'YYYYMMDD_HHMMSS'.
log_vars: list, optional.
Specifies the list of variables to log after each `n_print`
steps. If None, will log all variables. If `[]`, no variables
will be logged. `logdir` must be specified for variables to be
logged.
debug: bool, optional.
If True, add checks for `NaN` and `Inf` to all computations
in the graph. May result in substantially slower execution
times.
"""
self.n_iter = n_iter
if n_print is None:
self.n_print = int(n_iter / 100)
else:
self.n_print = n_print
self.progbar = Progbar(self.n_iter)
self.t = tf.Variable(0, trainable=False, name="iteration")
self.increment_t = self.t.assign_add(1)
if scale is None:
scale = {}
elif not isinstance(scale, dict):
raise TypeError("scale must be a dict object.")
self.scale = scale
if logdir is not None:
self.logging = True
if log_timestamp:
logdir = os.path.join(
logdir, datetime.strftime(datetime.utcnow(), "%Y%m%d_%H%M%S"))
self._set_log_variables(log_vars)
self.train_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())
self.summarize = tf.summary.merge_all()
else:
self.logging = False
self.debug = debug
if self.debug:
self.op_check = tf.add_check_numerics_ops()
# Store reset ops which user can call. Subclasses should append
# any ops needed to reset internal variables in inference.
self.reset = [tf.variables_initializer([self.t])]
@abc.abstractmethod
def update(self, feed_dict=None):
"""Run one iteration of inference.
Any derived class of `Inference` **must** implement this method.
Args:
feed_dict: dict, optional.
Feed dictionary for a TensorFlow session run. It is used to feed
placeholders that are not fed during initialization.
Returns:
dict.
Dictionary of algorithm-specific information.
"""
if feed_dict is None:
feed_dict = {}
for key, value in six.iteritems(self.data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
feed_dict[key] = value
sess = get_session()
t = sess.run(self.increment_t)
if self.debug:
sess.run(self.op_check, feed_dict)
if self.logging and self.n_print != 0:
if t == 1 or t % self.n_print == 0:
summary = sess.run(self.summarize, feed_dict)
self.train_writer.add_summary(summary, t)
return {'t': t}
def print_progress(self, info_dict):
"""Print progress to output.
Args:
info_dict: dict.
Dictionary of algorithm-specific information.
"""
if self.n_print != 0:
t = info_dict['t']
if t == 1 or t % self.n_print == 0:
self.progbar.update(t)
def finalize(self):
"""Function to call after convergence.
"""
if self.logging:
self.train_writer.close()
def _set_log_variables(self, log_vars=None):
"""Log variables to TensorBoard.
For each variable in `log_vars`, forms a `tf.summary.scalar` if
the variable has scalar shape; otherwise forms a `tf.summary.histogram`.
Args:
log_vars: list, optional.
Specifies the list of variables to log after each `n_print`
steps. If None, will log all variables. If `[]`, no variables
will be logged.
"""
summary_key = 'summaries_' + str(id(self))
if log_vars is None:
log_vars = []
for key in six.iterkeys(self.data):
log_vars += get_variables(key)
for key, value in six.iteritems(self.latent_vars):
log_vars += get_variables(key)
log_vars += get_variables(value)
log_vars = set(log_vars)
for var in log_vars:
# replace colons which are an invalid character
var_name = var.name.replace(':', '/')
# Log all scalars.
if len(var.shape) == 0:
tf.summary.scalar("parameter/{}".format(var_name),
var, collections=[summary_key])
elif len(var.shape) == 1 and var.shape[0] == 1:
tf.summary.scalar("parameter/{}".format(var_name),
var[0], collections=[summary_key])
else:
# If var is multi-dimensional, log a histogram of its values.
tf.summary.histogram("parameter/{}".format(var_name),
var, collections=[summary_key])
def main(_):
ed.set_seed(42)
# DATA
x_train, metadata = nips(FLAGS.data_dir)
documents = metadata['columns']
words = metadata['rows']
# Subset to documents in 2011 and words appearing in at least two
# documents and have a total word count of at least 10.
doc_idx = [
i for i, document in enumerate(documents)
if document.startswith('2011')
]
documents = [documents[doc] for doc in doc_idx]
x_train = x_train[:, doc_idx]
word_idx = np.logical_and(
np.sum(x_train != 0, 1) >= 2,
np.sum(x_train, 1) >= 10)
words = [word for word, idx in zip(words, word_idx) if idx]
x_train = x_train[word_idx, :]
x_train = x_train.T
N = x_train.shape[0] # number of documents
D = x_train.shape[1] # vocabulary size
# MODEL
W2 = Gamma(0.1, 0.3, sample_shape=[FLAGS.K[2], FLAGS.K[1]])
W1 = Gamma(0.1, 0.3, sample_shape=[FLAGS.K[1], FLAGS.K[0]])
W0 = Gamma(0.1, 0.3, sample_shape=[FLAGS.K[0], D])
z3 = Gamma(0.1, 0.1, sample_shape=[N, FLAGS.K[2]])
z2 = Gamma(FLAGS.shape, FLAGS.shape / tf.matmul(z3, W2))
z1 = Gamma(FLAGS.shape, FLAGS.shape / tf.matmul(z2, W1))
x = Poisson(tf.matmul(z1, W0))
# INFERENCE
qW2 = pointmass_q(W2.shape)
qW1 = pointmass_q(W1.shape)
qW0 = pointmass_q(W0.shape)
if FLAGS.q == 'gamma':
qz3 = gamma_q(z3.shape)
qz2 = gamma_q(z2.shape)
qz1 = gamma_q(z1.shape)
else:
qz3 = lognormal_q(z3.shape)
qz2 = lognormal_q(z2.shape)
qz1 = lognormal_q(z1.shape)
# We apply variational EM with E-step over local variables
# and M-step to point estimate the global weight matrices.
inference_e = ed.KLqp({
z1: qz1,
z2: qz2,
z3: qz3
},
data={
x: x_train,
W0: qW0,
W1: qW1,
W2: qW2
})
inference_m = ed.MAP({
W0: qW0,
W1: qW1,
W2: qW2
},
data={
x: x_train,
z1: qz1,
z2: qz2,
z3: qz3
})
optimizer_e = tf.train.RMSPropOptimizer(FLAGS.lr)
optimizer_m = tf.train.RMSPropOptimizer(FLAGS.lr)
kwargs = {
'optimizer': optimizer_e,
'n_print': 100,
'logdir': FLAGS.logdir,
'log_timestamp': False
}
if FLAGS.q == 'gamma':
kwargs['n_samples'] = 30
inference_e.initialize(**kwargs)
inference_m.initialize(optimizer=optimizer_m)
sess = ed.get_session()
tf.global_variables_initializer().run()
n_epoch = 20
n_iter_per_epoch = 10000
for epoch in range(n_epoch):
print("Epoch {}".format(epoch))
nll = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
info_dict_e = inference_e.update()
info_dict_m = inference_m.update()
nll += info_dict_e['loss']
# Compute perplexity averaged over a number of training iterations.
# The model's negative log-likelihood of data is upper bounded by
# the variational objective.
nll /= n_iter_per_epoch
perplexity = np.exp(nll / np.sum(x_train))
print("Negative log-likelihood <= {:0.3f}".format(nll))
print("Perplexity <= {:0.3f}".format(perplexity))
# Print top 10 words for first 10 topics.
qW0_vals = sess.run(qW0)
for k in range(10):
top_words_idx = qW0_vals[k, :].argsort()[-10:][::-1]
top_words = " ".join([words[i] for i in top_words_idx])
print("Topic {}: {}".format(k, top_words))
def main(_):
ed.set_seed(42)
# DATA
(x_train, _), (x_test, _), (x_valid,
_) = caltech101_silhouettes(FLAGS.data_dir)
x_train_generator = generator(x_train, FLAGS.batch_size)
x_ph = tf.placeholder(tf.int32, [None, 28 * 28])
# MODEL
zs = [0] * len(FLAGS.hidden_sizes)
for l in reversed(range(len(FLAGS.hidden_sizes))):
if l == len(FLAGS.hidden_sizes) - 1:
logits = tf.zeros([tf.shape(x_ph)[0], FLAGS.hidden_sizes[l]])
else:
logits = tf.layers.dense(tf.cast(zs[l + 1], tf.float32),
FLAGS.hidden_sizes[l],
activation=None)
zs[l] = Bernoulli(logits=logits)
x = Bernoulli(logits=tf.layers.dense(
tf.cast(zs[0], tf.float32), 28 * 28, activation=None))
# INFERENCE
# Define variational model with reverse ordering as probability model:
# if p is 15-100-300 from top-down, q is 300-100-15 from bottom-up.
qzs = [0] * len(FLAGS.hidden_sizes)
for l in range(len(FLAGS.hidden_sizes)):
if l == 0:
logits = tf.layers.dense(tf.cast(x_ph, tf.float32),
FLAGS.hidden_sizes[l],
activation=None)
else:
logits = tf.layers.dense(tf.cast(qzs[l - 1], tf.float32),
FLAGS.hidden_sizes[l],
activation=None)
qzs[l] = Bernoulli(logits=logits)
inference = ed.KLqp({z: qz for z, qz in zip(zs, qzs)}, data={x: x_ph})
optimizer = tf.train.AdamOptimizer(FLAGS.step_size)
inference.initialize(optimizer=optimizer, n_samples=FLAGS.n_train_samples)
# Build tensor for log-likelihood given one variational sample to run
# on test data.
x_post = ed.copy(x, {z: qz for z, qz in zip(zs, qzs)})
x_neg_log_prob = (-tf.reduce_sum(x_post.log_prob(x_ph)) /
tf.cast(tf.shape(x_ph)[0], tf.float32))
sess = ed.get_session()
tf.global_variables_initializer().run()
for epoch in range(FLAGS.n_epoch):
print("Epoch {}".format(epoch))
train_loss = 0.0
pbar = Progbar(FLAGS.n_iter_per_epoch)
for t in range(1, FLAGS.n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(x_train_generator)
info_dict = inference.update(feed_dict={x_ph: x_batch})
train_loss += info_dict['loss']
# Print per-data point loss, averaged over training epoch.
train_loss /= FLAGS.n_iter_per_epoch
train_loss /= FLAGS.batch_size
print("Training negative log-likelihood: {:0.3f}".format(train_loss))
test_loss = [
sess.run(x_neg_log_prob, {x_ph: x_test})
for _ in range(FLAGS.n_test_samples)
]
test_loss = np.mean(test_loss)
print("Test negative log-likelihood: {:0.3f}".format(test_loss))
# Prior predictive check.
images = sess.run(x,
{x_ph: x_batch}) # feed ph to determine sample size
for m in range(FLAGS.batch_size):
imsave("{}/{}.png".format(out_dir, m), images[m].reshape(28, 28))
def main(_):
ed.set_seed(42)
# DATA
x_train, _, x_test = text8(FLAGS.data_dir)
vocab = string.ascii_lowercase + ' '
vocab_size = len(vocab)
encoder = dict(zip(vocab, range(vocab_size)))
decoder = {v: k for k, v in encoder.items()}
data = generator(x_train, FLAGS.batch_size, FLAGS.timesteps, encoder)
# MODEL
x_ph = tf.placeholder(tf.int32, [None, FLAGS.timesteps])
with tf.variable_scope("language_model"):
# Shift input sequence to right by 1, [0, x[0], ..., x[timesteps - 2]].
x_ph_shift = tf.pad(x_ph, [[0, 0], [1, 0]])[:, :-1]
x = language_model(x_ph_shift, vocab_size)
with tf.variable_scope("language_model", reuse=True):
x_gen = language_model_gen(5, vocab_size)
imb = range(0, len(x_test) - FLAGS.timesteps, FLAGS.timesteps)
encoded_x_test = np.asarray(
[[encoder[c] for c in x_test[i:(i + FLAGS.timesteps)]] for i in imb],
dtype=np.int32)
test_size = encoded_x_test.shape[0]
print("Test set shape: {}".format(encoded_x_test.shape))
test_nll = -tf.reduce_sum(x.log_prob(x_ph))
# INFERENCE
inference = ed.MAP({}, {x: x_ph})
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.lr)
inference.initialize(optimizer=optimizer,
logdir=FLAGS.log_dir,
log_timestamp=False)
print("Number of sets of parameters: {}".format(
len(tf.trainable_variables())))
print("Number of parameters: {}".format(
np.sum([np.prod(v.shape.as_list())
for v in tf.trainable_variables()])))
for v in tf.trainable_variables():
print(v)
sess = ed.get_session()
tf.global_variables_initializer().run()
# Double n_epoch and print progress every half an epoch.
n_iter_per_epoch = len(x_train) // (FLAGS.batch_size * FLAGS.timesteps * 2)
epoch = 0.0
for _ in range(FLAGS.n_epoch * 2):
epoch += 0.5
print("Epoch: {0}".format(epoch))
avg_nll = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(data)
info_dict = inference.update({x_ph: x_batch})
avg_nll += info_dict['loss']
# Print average bits per character over epoch.
avg_nll /= (n_iter_per_epoch * FLAGS.batch_size * FLAGS.timesteps *
np.log(2))
print("Train average bits/char: {:0.8f}".format(avg_nll))
# Print per-data point log-likelihood on test set.
avg_nll = 0.0
for start in range(0, test_size, batch_size):
end = min(test_size, start + batch_size)
x_batch = encoded_x_test[start:end]
avg_nll += sess.run(test_nll, {x_ph: x_batch})
avg_nll /= test_size
print("Test average NLL: {:0.8f}".format(avg_nll))
# Generate samples from model.
samples = sess.run(x_gen)
samples = [''.join([decoder[c] for c in sample]) for sample in samples]
print("Samples:")
for sample in samples:
print(sample)
class Inference(object):
"""Base class for Edward inference methods.
Attributes
----------
latent_vars : dict
Collection of latent variables (of type ``RandomVariable`` or
``tf.Tensor``) to perform inference on. Each random variable is
binded to another random variable; the latter will infer the
former conditional on data.
data : dict
Data dictionary which binds observed variables (of type
``RandomVariable`` or ``tf.Tensor``) to their realizations (of
type ``tf.Tensor``).
"""
def __init__(self, latent_vars=None, data=None):
"""Initialization.
Parameters
----------
latent_vars : dict, optional
Collection of latent variables (of type ``RandomVariable`` or
``tf.Tensor``) to perform inference on. Each random variable is
binded to another random variable; the latter will infer the
former conditional on data.
data : dict, optional
Data dictionary which binds observed variables (of type
``RandomVariable`` or ``tf.Tensor``) to their realizations (of
type ``tf.Tensor``). It can also bind placeholders (of type
``tf.Tensor``) used in the model to their realizations; and
prior latent variables (of type ``RandomVariable``) to posterior
latent variables (of type ``RandomVariable``).
Notes
-----
If ``data`` is not passed in, the dictionary is empty.
Three options are available for batch training:
1. internally if user passes in data as a dictionary of NumPy
arrays;
2. externally if user passes in data as a dictionary of
TensorFlow placeholders (and manually feeds them);
3. externally if user passes in data as TensorFlow tensors
which are the outputs of data readers.
Examples
--------
>>> mu = Normal(mu=tf.constant(0.0), sigma=tf.constant(1.0))
>>> x = Normal(mu=tf.ones(N) * mu, sigma=tf.constant(1.0))
>>>
>>> qmu_mu = tf.Variable(tf.random_normal([1]))
>>> qmu_sigma = tf.nn.softplus(tf.Variable(tf.random_normal([1])))
>>> qmu = Normal(mu=qmu_mu, sigma=qmu_sigma)
>>>
>>> Inference({mu: qmu}, {x: tf.constant([0.0] * N)})
"""
sess = get_session()
if latent_vars is None:
latent_vars = {}
elif not isinstance(latent_vars, dict):
raise TypeError("latent_vars must have type dict.")
for key, value in six.iteritems(latent_vars):
if not isinstance(key, (RandomVariable, tf.Tensor)):
raise TypeError("Latent variable key has an invalid type.")
elif not isinstance(value, (RandomVariable, tf.Tensor)):
raise TypeError("Latent variable value has an invalid type.")
elif not key.get_shape().is_compatible_with(value.get_shape()):
raise TypeError("Latent variable bindings do not have same shape.")
self.latent_vars = latent_vars
if data is None:
data = {}
elif not isinstance(data, dict):
raise TypeError("data must have type dict.")
self.data = {}
for key, value in six.iteritems(data):
if isinstance(key, RandomVariable) or \
(isinstance(key, tf.Tensor) and "Placeholder" not in key.op.type):
if isinstance(value, tf.Tensor):
if not key.get_shape().is_compatible_with(value.get_shape()):
raise TypeError("Observed variable bindings do not have same "
"shape.")
self.data[key] = tf.cast(value, tf.float32)
elif isinstance(value, RandomVariable):
if not key.get_shape().is_compatible_with(value.get_shape()):
raise TypeError("Observed variable bindings do not have same "
"shape.")
self.data[key] = value
elif isinstance(value, np.ndarray):
if not key.get_shape().is_compatible_with(value.shape):
raise TypeError("Observed variable bindings do not have same "
"shape.")
# If value is a np.ndarray, store it in the graph. Assign its
# placeholder to an appropriate data type.
if np.issubdtype(value.dtype, np.float):
ph_type = tf.float32
elif np.issubdtype(value.dtype, np.int):
ph_type = tf.int32
else:
raise TypeError("Data value has an unsupported type.")
ph = tf.placeholder(ph_type, value.shape)
var = tf.Variable(ph, trainable=False, collections=[])
self.data[key] = var
sess.run(var.initializer, {ph: value})
elif isinstance(value, np.number):
if np.issubdtype(value.dtype, np.float):
ph_type = tf.float32
elif np.issubdtype(value.dtype, np.int):
ph_type = tf.int32
else:
raise TypeError("Data value as an invalid type.")
ph = tf.placeholder(ph_type, value.shape)
var = tf.Variable(ph, trainable=False, collections=[])
self.data[key] = var
sess.run(var.initializer, {ph: value})
elif isinstance(value, float):
ph_type = tf.float32
ph = tf.placeholder(ph_type, ())
var = tf.Variable(ph, trainable=False, collections=[])
self.data[key] = var
sess.run(var.initializer, {ph: value})
elif isinstance(value, int):
ph_type = tf.int32
ph = tf.placeholder(ph_type, ())
var = tf.Variable(ph, trainable=False, collections=[])
self.data[key] = var
# handle if value is `bool` which this case catches
sess.run(var.initializer, {ph: int(value)})
else:
raise TypeError("Data value has an invalid type.")
elif isinstance(key, tf.Tensor):
if isinstance(value, RandomVariable):
raise TypeError("Data placeholder cannot be bound to a "
"RandomVariable.")
self.data[key] = value
else:
raise TypeError("Data key has an invalid type.")
def run(self, variables=None, use_coordinator=True, *args, **kwargs):
"""A simple wrapper to run inference.
1. Initialize algorithm via ``initialize``.
2. (Optional) Build a TensorFlow summary writer for TensorBoard.
3. (Optional) Initialize TensorFlow variables.
4. (Optional) Start queue runners.
5. Run ``update`` for ``self.n_iter`` iterations.
6. While running, ``print_progress``.
7. Finalize algorithm via ``finalize``.
8. (Optional) Stop queue runners.
To customize the way inference is run, run these steps
individually.
Parameters
----------
variables : list, optional
A list of TensorFlow variables to initialize during inference.
Default is to initialize all variables (this includes
reinitializing variables that were already initialized). To
avoid initializing any variables, pass in an empty list.
use_coordinator : bool, optional
Whether to start and stop queue runners during inference using a
TensorFlow coordinator. For example, queue runners are necessary
for batch training with file readers.
*args
Passed into ``initialize``.
**kwargs
Passed into ``initialize``.
"""
self.initialize(*args, **kwargs)
if variables is None:
init = tf.global_variables_initializer()
else:
init = tf.variables_initializer(variables)
# Feed placeholders in case initialization depends on them.
feed_dict = {}
for key, value in six.iteritems(self.data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
feed_dict[key] = value
init.run(feed_dict)
if use_coordinator:
# Start input enqueue threads.
self.coord = tf.train.Coordinator()
self.threads = tf.train.start_queue_runners(coord=self.coord)
for _ in range(self.n_iter):
info_dict = self.update()
self.print_progress(info_dict)
self.finalize()
if use_coordinator:
# Ask threads to stop.
self.coord.request_stop()
self.coord.join(self.threads)
def initialize(self, n_iter=1000, n_print=None, scale=None, logdir=None,
debug=False):
"""Initialize inference algorithm.
Parameters
----------
n_iter : int, optional
Number of iterations for algorithm.
n_print : int, optional
Number of iterations for each print progress. To suppress print
progress, then specify 0. Default is ``int(n_iter / 10)``.
scale : dict of RandomVariable to tf.Tensor, optional
A tensor to scale computation for any random variable that it is
binded to. Its shape must be broadcastable; it is multiplied
element-wise to the random variable. For example, this is useful
for mini-batch scaling when inferring global variables, or
applying masks on a random variable.
logdir : str, optional
Directory where event file will be written. For details,
see ``tf.summary.FileWriter``. Default is to write nothing.
debug : bool, optional
If True, add checks for ``NaN`` and ``Inf`` to all computations
in the graph. May result in substantially slower execution
times.
"""
self.n_iter = n_iter
if n_print is None:
self.n_print = int(n_iter / 10)
else:
self.n_print = n_print
self.progbar = Progbar(self.n_iter)
self.t = tf.Variable(0, trainable=False)
self.increment_t = self.t.assign_add(1)
if scale is None:
scale = {}
elif not isinstance(scale, dict):
raise TypeError()
self.scale = scale
if logdir is not None:
self.logging = True
self.train_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())
self.summarize = tf.summary.merge_all()
else:
self.logging = False
self.debug = debug
if self.debug:
self.op_check = tf.add_check_numerics_ops()
def update(self, feed_dict=None):
"""Run one iteration of inference.
Parameters
----------
feed_dict : dict, optional
Feed dictionary for a TensorFlow session run. It is used to feed
placeholders that are not fed during initialization.
Returns
-------
dict
Dictionary of algorithm-specific information.
"""
if feed_dict is None:
feed_dict = {}
for key, value in six.iteritems(self.data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
feed_dict[key] = value
sess = get_session()
t = sess.run(self.increment_t)
if self.debug:
sess.run(self.op_check)
if self.logging and self.n_print != 0:
if t == 1 or t % self.n_print == 0:
summary = sess.run(self.summarize, feed_dict)
self.train_writer.add_summary(summary, t)
return {'t': t}
def print_progress(self, info_dict):
"""Print progress to output.
Parameters
----------
info_dict : dict
Dictionary of algorithm-specific information.
"""
if self.n_print != 0:
t = info_dict['t']
if t == 1 or t % self.n_print == 0:
self.progbar.update(t)
def finalize(self):
"""Function to call after convergence.
"""
if self.logging:
self.train_writer.close()
for j in range(n_eval):
U_sample = U_post.sample().eval()
V_sample = V_post.sample().eval()
holdoutmean_sample = np.multiply(U_sample.dot(V_sample.T), holdout_mask)
obs_ll.append(
np.mean(np.ma.masked_invalid(
stats.poisson.logpmf(np.array(x_vad, dtype=int),
holdoutmean_sample)),
axis=1))
rep_ll.append(
np.mean(np.ma.masked_invalid(
stats.poisson.logpmf(holdout_gen, holdoutmean_sample)),
axis=2))
pbar.update(j)
obs_ll_per_zi, rep_ll_per_zi = np.mean(np.array(obs_ll),
axis=0), np.mean(np.array(rep_ll),
axis=0)
pvals = np.array([
np.mean(rep_ll_per_zi[:, i] < obs_ll_per_zi[i])
for i in range(len(obs_ll_per_zi))
])
holdout_subjects = np.unique(holdout_row)
overall_pval = np.mean(pvals[holdout_subjects])
print("Predictive check p-values", overall_pval)
# In[12]:
def main(_):
ed.set_seed(42)
# DATA
x_train, metadata = nips(FLAGS.data_dir)
documents = metadata['columns']
words = metadata['rows']
# Subset to documents in 2011 and words appearing in at least two
# documents and have a total word count of at least 10.
doc_idx = [i for i, document in enumerate(documents)
if document.startswith('2011')]
documents = [documents[doc] for doc in doc_idx]
x_train = x_train[:, doc_idx]
word_idx = np.logical_and(np.sum(x_train != 0, 1) >= 2,
np.sum(x_train, 1) >= 10)
words = [word for word, idx in zip(words, word_idx) if idx]
x_train = x_train[word_idx, :]
x_train = x_train.T
N = x_train.shape[0] # number of documents
D = x_train.shape[1] # vocabulary size
# MODEL
W2 = Gamma(0.1, 0.3, sample_shape=[FLAGS.K[2], FLAGS.K[1]])
W1 = Gamma(0.1, 0.3, sample_shape=[FLAGS.K[1], FLAGS.K[0]])
W0 = Gamma(0.1, 0.3, sample_shape=[FLAGS.K[0], D])
z3 = Gamma(0.1, 0.1, sample_shape=[N, FLAGS.K[2]])
z2 = Gamma(FLAGS.shape, FLAGS.shape / tf.matmul(z3, W2))
z1 = Gamma(FLAGS.shape, FLAGS.shape / tf.matmul(z2, W1))
x = Poisson(tf.matmul(z1, W0))
# INFERENCE
qW2 = pointmass_q(W2.shape)
qW1 = pointmass_q(W1.shape)
qW0 = pointmass_q(W0.shape)
if FLAGS.q == 'gamma':
qz3 = gamma_q(z3.shape)
qz2 = gamma_q(z2.shape)
qz1 = gamma_q(z1.shape)
else:
qz3 = lognormal_q(z3.shape)
qz2 = lognormal_q(z2.shape)
qz1 = lognormal_q(z1.shape)
# We apply variational EM with E-step over local variables
# and M-step to point estimate the global weight matrices.
inference_e = ed.KLqp({z1: qz1, z2: qz2, z3: qz3},
data={x: x_train, W0: qW0, W1: qW1, W2: qW2})
inference_m = ed.MAP({W0: qW0, W1: qW1, W2: qW2},
data={x: x_train, z1: qz1, z2: qz2, z3: qz3})
optimizer_e = tf.train.RMSPropOptimizer(FLAGS.lr)
optimizer_m = tf.train.RMSPropOptimizer(FLAGS.lr)
kwargs = {'optimizer': optimizer_e,
'n_print': 100,
'logdir': FLAGS.logdir,
'log_timestamp': False}
if FLAGS.q == 'gamma':
kwargs['n_samples'] = 30
inference_e.initialize(**kwargs)
inference_m.initialize(optimizer=optimizer_m)
sess = ed.get_session()
tf.global_variables_initializer().run()
n_epoch = 20
n_iter_per_epoch = 10000
for epoch in range(n_epoch):
print("Epoch {}".format(epoch))
nll = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
info_dict_e = inference_e.update()
info_dict_m = inference_m.update()
nll += info_dict_e['loss']
# Compute perplexity averaged over a number of training iterations.
# The model's negative log-likelihood of data is upper bounded by
# the variational objective.
nll /= n_iter_per_epoch
perplexity = np.exp(nll / np.sum(x_train))
print("Negative log-likelihood <= {:0.3f}".format(nll))
print("Perplexity <= {:0.3f}".format(perplexity))
# Print top 10 words for first 10 topics.
qW0_vals = sess.run(qW0)
for k in range(10):
top_words_idx = qW0_vals[k, :].argsort()[-10:][::-1]
top_words = " ".join([words[i] for i in top_words_idx])
print("Topic {}: {}".format(k, top_words))
class Inference(object):
"""Abstract base class for inference. All inference algorithms in
Edward inherit from `Inference`, sharing common methods and
properties via a class hierarchy.
Specific algorithms typically inherit from other subclasses of
`Inference` rather than `Inference` directly. For example, one
might inherit from the abstract classes `MonteCarlo` or
`VariationalInference`.
To build an algorithm inheriting from `Inference`, one must at the
minimum implement `initialize` and `update`: the former builds
the computational graph for the algorithm; the latter runs the
computational graph for the algorithm.
To reset inference (e.g., internal variable counters incremented
over training), fetch inference's reset ops from session with
`sess.run(inference.reset)`.
#### Examples
```python
# Set up probability model.
mu = Normal(loc=0.0, scale=1.0)
x = Normal(loc=mu, scale=1.0, sample_shape=50)
# Set up posterior approximation.
qmu_loc = tf.Variable(tf.random_normal([]))
qmu_scale = tf.nn.softplus(tf.Variable(tf.random_normal([])))
qmu = Normal(loc=qmu_loc, scale=qmu_scale)
inference = ed.Inference({mu: qmu}, data={x: tf.zeros(50)})
```
"""
def __init__(self, latent_vars=None, data=None):
"""Create an inference algorithm.
Args:
latent_vars: dict, optional.
Collection of latent variables (of type `RandomVariable` or
`tf.Tensor`) to perform inference on. Each random variable is
binded to another random variable; the latter will infer the
former conditional on data.
data: dict, optional.
Data dictionary which binds observed variables (of type
`RandomVariable` or `tf.Tensor`) to their realizations (of
type `tf.Tensor`). It can also bind placeholders (of type
`tf.Tensor`) used in the model to their realizations; and
prior latent variables (of type `RandomVariable`) to posterior
latent variables (of type `RandomVariable`).
"""
sess = get_session()
if latent_vars is None:
latent_vars = {}
if data is None:
data = {}
check_latent_vars(latent_vars)
self.latent_vars = latent_vars
check_data(data)
self.data = {}
for key, value in six.iteritems(data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
self.data[key] = value
elif isinstance(key, (RandomVariable, tf.Tensor)):
if isinstance(value, (RandomVariable, tf.Tensor)):
self.data[key] = value
elif isinstance(
value, (float, list, int, np.ndarray, np.number, str)):
# If value is a Python type, store it in the graph.
# Assign its placeholder with the key's data type.
with tf.variable_scope(None, default_name="data"):
ph = tf.placeholder(key.dtype, np.shape(value))
var = tf.Variable(ph, trainable=False, collections=[])
sess.run(var.initializer, {ph: value})
self.data[key] = var
def run(self, variables=None, use_coordinator=True, *args, **kwargs):
"""A simple wrapper to run inference.
1. Initialize algorithm via `initialize`.
2. (Optional) Build a TensorFlow summary writer for TensorBoard.
3. (Optional) Initialize TensorFlow variables.
4. (Optional) Start queue runners.
5. Run `update` for `self.n_iter` iterations.
6. While running, `print_progress`.
7. Finalize algorithm via `finalize`.
8. (Optional) Stop queue runners.
To customize the way inference is run, run these steps
individually.
Args:
variables: list, optional.
A list of TensorFlow variables to initialize during inference.
Default is to initialize all variables (this includes
reinitializing variables that were already initialized). To
avoid initializing any variables, pass in an empty list.
use_coordinator: bool, optional.
Whether to start and stop queue runners during inference using a
TensorFlow coordinator. For example, queue runners are necessary
for batch training with file readers.
*args, **kwargs:
Passed into `initialize`.
"""
self.initialize(*args, **kwargs)
if variables is None:
init = tf.global_variables_initializer()
else:
init = tf.variables_initializer(variables)
# Feed placeholders in case initialization depends on them.
feed_dict = {}
for key, value in six.iteritems(self.data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
feed_dict[key] = value
init.run(feed_dict)
if use_coordinator:
# Start input enqueue threads.
self.coord = tf.train.Coordinator()
self.threads = tf.train.start_queue_runners(coord=self.coord)
for _ in range(self.n_iter):
info_dict = self.update()
self.print_progress(info_dict)
self.finalize()
if use_coordinator:
# Ask threads to stop.
self.coord.request_stop()
self.coord.join(self.threads)
@abc.abstractmethod
def initialize(self,
n_iter=1000,
n_print=None,
scale=None,
auto_transform=True,
logdir=None,
log_timestamp=True,
log_vars=None,
debug=False):
"""Initialize inference algorithm. It initializes hyperparameters
and builds ops for the algorithm's computation graph.
Any derived class of `Inference` **must** implement this method.
No methods which build ops should be called outside `initialize()`.
Args:
n_iter: int, optional.
Number of iterations for algorithm when calling `run()`.
Alternatively if controlling inference manually, it is the
expected number of calls to `update()`; this number determines
tracking information during the print progress.
n_print: int, optional.
Number of iterations for each print progress. To suppress print
progress, then specify 0. Default is `int(n_iter / 100)`.
scale: dict of RandomVariable to tf.Tensor, optional.
A tensor to scale computation for any random variable that it is
binded to. Its shape must be broadcastable; it is multiplied
element-wise to the random variable. For example, this is useful
for mini-batch scaling when inferring global variables, or
applying masks on a random variable.
auto_transform: bool, optional.
Whether to automatically transform continuous latent variables
of unequal support to be on the unconstrained space. It is
only applied if the argument is `True`, the latent variable
pair are `ed.RandomVariable`s with the `support` attribute,
the supports are both continuous and unequal.
logdir: str, optional.
Directory where event file will be written. For details,
see `tf.summary.FileWriter`. Default is to log nothing.
log_timestamp: bool, optional.
If True (and `logdir` is specified), create a subdirectory of
`logdir` to save the specific run results. The subdirectory's
name is the current UTC timestamp with format 'YYYYMMDD_HHMMSS'.
log_vars: list, optional.
Specifies the list of variables to log after each `n_print`
steps. If None, will log all variables. If `[]`, no variables
will be logged. `logdir` must be specified for variables to be
logged.
debug: bool, optional.
If True, add checks for `NaN` and `Inf` to all computations
in the graph. May result in substantially slower execution
times.
"""
self.n_iter = n_iter
if n_print is None:
self.n_print = int(n_iter / 100)
else:
self.n_print = n_print
self.progbar = Progbar(self.n_iter)
self.t = tf.Variable(0, trainable=False, name="iteration")
self.increment_t = self.t.assign_add(1)
if scale is None:
scale = {}
elif not isinstance(scale, dict):
raise TypeError("scale must be a dict object.")
self.scale = scale
# map from original latent vars to unconstrained versions
self.transformations = {}
if auto_transform:
latent_vars = self.latent_vars.copy()
# latent_vars maps original latent vars to constrained Q's.
# latent_vars_unconstrained maps unconstrained vars to unconstrained Q's.
self.latent_vars = {}
self.latent_vars_unconstrained = {}
for z, qz in six.iteritems(latent_vars):
if hasattr(z, 'support') and hasattr(qz, 'support') and \
z.support != qz.support and qz.support != 'point':
# transform z to an unconstrained space
z_unconstrained = transform(z)
self.transformations[z] = z_unconstrained
# make sure we also have a qz that covers the unconstrained space
if qz.support == "points":
qz_unconstrained = qz
else:
qz_unconstrained = transform(qz)
self.latent_vars_unconstrained[
z_unconstrained] = qz_unconstrained
# additionally construct the transformation of qz
# back into the original constrained space
if z_unconstrained != z:
qz_constrained = transform(
qz_unconstrained,
bijectors.Invert(z_unconstrained.bijector))
try: # attempt to pushforward the params of Empirical distributions
qz_constrained.params = z_unconstrained.bijector.inverse(
qz_unconstrained.params)
except: # qz_unconstrained is not an Empirical distribution
pass
else:
qz_constrained = qz_unconstrained
self.latent_vars[z] = qz_constrained
else:
self.latent_vars[z] = qz
self.latent_vars_unconstrained[z] = qz
del latent_vars
if logdir is not None:
self.logging = True
if log_timestamp:
logdir = os.path.expanduser(logdir)
logdir = os.path.join(
logdir,
datetime.strftime(datetime.utcnow(), "%Y%m%d_%H%M%S"))
self._summary_key = tf.get_default_graph().unique_name("summaries")
self._set_log_variables(log_vars)
self.train_writer = tf.summary.FileWriter(logdir,
tf.get_default_graph())
else:
self.logging = False
self.debug = debug
if self.debug:
self.op_check = tf.add_check_numerics_ops()
# Store reset ops which user can call. Subclasses should append
# any ops needed to reset internal variables in inference.
self.reset = [tf.variables_initializer([self.t])]
@abc.abstractmethod
def update(self, feed_dict=None):
"""Run one iteration of inference.
Any derived class of `Inference` **must** implement this method.
Args:
feed_dict: dict, optional.
Feed dictionary for a TensorFlow session run. It is used to feed
placeholders that are not fed during initialization.
Returns:
dict.
Dictionary of algorithm-specific information.
"""
if feed_dict is None:
feed_dict = {}
for key, value in six.iteritems(self.data):
if isinstance(key, tf.Tensor) and "Placeholder" in key.op.type:
feed_dict[key] = value
sess = get_session()
t = sess.run(self.increment_t)
if self.debug:
sess.run(self.op_check, feed_dict)
if self.logging and self.n_print != 0:
if t == 1 or t % self.n_print == 0:
summary = sess.run(self.summarize, feed_dict)
self.train_writer.add_summary(summary, t)
return {'t': t}
def print_progress(self, info_dict):
"""Print progress to output.
Args:
info_dict: dict.
Dictionary of algorithm-specific information.
"""
if self.n_print != 0:
t = info_dict['t']
if t == 1 or t % self.n_print == 0:
self.progbar.update(t)
def finalize(self):
"""Function to call after convergence.
"""
if self.logging:
self.train_writer.close()
def _set_log_variables(self, log_vars=None):
"""Log variables to TensorBoard.
For each variable in `log_vars`, forms a `tf.summary.scalar` if
the variable has scalar shape; otherwise forms a `tf.summary.histogram`.
Args:
log_vars: list, optional.
Specifies the list of variables to log after each `n_print`
steps. If None, will log all variables. If `[]`, no variables
will be logged.
"""
if log_vars is None:
log_vars = []
for key in six.iterkeys(self.data):
log_vars += get_variables(key)
for key, value in six.iteritems(self.latent_vars):
log_vars += get_variables(key)
log_vars += get_variables(value)
log_vars = set(log_vars)
for var in log_vars:
# replace colons which are an invalid character
var_name = var.name.replace(':', '/')
# Log all scalars.
if len(var.shape) == 0:
tf.summary.scalar("parameter/{}".format(var_name),
var,
collections=[self._summary_key])
elif len(var.shape) == 1 and var.shape[0] == 1:
tf.summary.scalar("parameter/{}".format(var_name),
var[0],
collections=[self._summary_key])
else:
# If var is multi-dimensional, log a histogram of its values.
tf.summary.histogram("parameter/{}".format(var_name),
var,
collections=[self._summary_key])
inference = ed.ReparameterizationKLKLqp({z: qz}, data)
optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0)
inference.initialize(optimizer=optimizer, use_prettytensor=True)
hidden_rep = tf.sigmoid(logits)
tf.global_variables_initializer().run()
n_epoch = 100
n_iter_per_epoch = 1000
for epoch in range(n_epoch):
avg_loss = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_train, _ = mnist.train.next_batch(M)
x_train = np.random.binomial(1, x_train)
info_dict = inference.update(feed_dict={x_ph: x_train})
avg_loss += info_dict['loss']
# Print a lower bound to the average marginal likelihood for an
# image.
avg_loss = avg_loss / n_iter_per_epoch
avg_loss = avg_loss / M
print("log p(x) >= {:0.3f}".format(avg_loss))
# Visualize hidden representations.
imgs = hidden_rep.eval()
for m in range(M):
imsave(os.path.join(IMG_DIR, '%d.png') % m, imgs[m].reshape(28, 28))
def main(_):
ed.set_seed(42)
# DATA
x_train, _, x_test = text8(FLAGS.data_dir)
vocab = string.ascii_lowercase + ' '
vocab_size = len(vocab)
encoder = dict(zip(vocab, range(vocab_size)))
decoder = {v: k for k, v in encoder.items()}
data = generator(x_train, FLAGS.batch_size, FLAGS.timesteps, encoder)
# MODEL
x_ph = tf.placeholder(tf.int32, [None, FLAGS.timesteps])
with tf.variable_scope("language_model"):
# Shift input sequence to right by 1, [0, x[0], ..., x[timesteps - 2]].
x_ph_shift = tf.pad(x_ph, [[0, 0], [1, 0]])[:, :-1]
x = language_model(x_ph_shift, vocab_size)
with tf.variable_scope("language_model", reuse=True):
x_gen = language_model_gen(5, vocab_size)
imb = range(0, len(x_test) - FLAGS.timesteps, FLAGS.timesteps)
encoded_x_test = np.asarray(
[[encoder[c] for c in x_test[i:(i + FLAGS.timesteps)]] for i in imb],
dtype=np.int32)
test_size = encoded_x_test.shape[0]
print("Test set shape: {}".format(encoded_x_test.shape))
test_nll = -tf.reduce_sum(x.log_prob(x_ph))
# INFERENCE
inference = ed.MAP({}, {x: x_ph})
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.lr)
inference.initialize(optimizer=optimizer,
logdir=FLAGS.log_dir,
log_timestamp=False)
print("Number of sets of parameters: {}".format(
len(tf.trainable_variables())))
print("Number of parameters: {}".format(
np.sum([np.prod(v.shape.as_list()) for v in tf.trainable_variables()])))
for v in tf.trainable_variables():
print(v)
sess = ed.get_session()
tf.global_variables_initializer().run()
# Double n_epoch and print progress every half an epoch.
n_iter_per_epoch = len(x_train) // (FLAGS.batch_size * FLAGS.timesteps * 2)
epoch = 0.0
for _ in range(FLAGS.n_epoch * 2):
epoch += 0.5
print("Epoch: {0}".format(epoch))
avg_nll = 0.0
pbar = Progbar(n_iter_per_epoch)
for t in range(1, n_iter_per_epoch + 1):
pbar.update(t)
x_batch = next(data)
info_dict = inference.update({x_ph: x_batch})
avg_nll += info_dict['loss']
# Print average bits per character over epoch.
avg_nll /= (n_iter_per_epoch * FLAGS.batch_size * FLAGS.timesteps *
np.log(2))
print("Train average bits/char: {:0.8f}".format(avg_nll))
# Print per-data point log-likelihood on test set.
avg_nll = 0.0
for start in range(0, test_size, batch_size):
end = min(test_size, start + batch_size)
x_batch = encoded_x_test[start:end]
avg_nll += sess.run(test_nll, {x_ph: x_batch})
avg_nll /= test_size
print("Test average NLL: {:0.8f}".format(avg_nll))
# Generate samples from model.
samples = sess.run(x_gen)
samples = [''.join([decoder[c] for c in sample]) for sample in samples]
print("Samples:")
for sample in samples:
print(sample)