def _build(self, X):
"""Build the graph of this layer."""
n_samples, (height, width, channels) = self._get_X_dims(X)
W_shape, b_shape = self._weight_shapes(channels)
W_init = initialise_weights(W_shape, self.init_fn)
W = tf.Variable(W_init, name="W_map")
summary_histogram(W)
Net = tf.map_fn(
lambda x: tf.nn.conv2d(x, W,
padding=self.padding,
strides=self.strides), X)
# Regularizers
penalty = self.l2 * tf.nn.l2_loss(W) + self.l1 * _l1_loss(W)
# Optional Bias
if self.use_bias:
b_init = initialise_weights(b_shape, self.init_fn)
b = tf.Variable(b_init, name="b_map")
summary_histogram(b)
Net = tf.nn.bias_add(Net, b)
penalty += self.l2 * tf.nn.l2_loss(b) + self.l1 * _l1_loss(b)
return Net, penalty
def _build(self, X):
"""Build the graph of this layer."""
n_samples, input_shape = self._get_X_dims(X)
Wdim = input_shape + [self.output_dim]
W_init = initialise_weights(Wdim, self.init_fn)
W = tf.Variable(W_init, name="W_map")
summary_histogram(W)
# Tiling W is much faster than mapping (tf.map_fn) the matmul
Net = tf.matmul(X, _tile2samples(n_samples, W))
# Regularizers
penalty = self.l2 * tf.nn.l2_loss(W) + self.l1 * _l1_loss(W)
# Optional Bias
if self.use_bias is True:
b_init = initialise_weights((1, self.output_dim), self.init_fn)
b = tf.Variable(b_init, name="b_map")
summary_histogram(b)
Net += b
penalty += self.l2 * tf.nn.l2_loss(b) + self.l1 * _l1_loss(b)
return Net, penalty
def gaus_posterior(dim, std0, suffix=None):
"""Initialise a posterior Gaussian distribution with a diagonal covariance.
Even though this is initialised with a diagonal covariance, a full
covariance will be learned, using a lower triangular Cholesky
parameterisation.
Parameters
----------
dim : tuple or list
the dimension of this distribution.
std0 : float
the initial (unoptimized) diagonal standard deviation of this
distribution.
suffix : str
suffix to add to the names of the variables of the parameters of this
distribution.
Returns
-------
Q : tf.contrib.distributions.MultivariateNormalTriL
the initialised posterior Gaussian object.
Note
----
This will make tf.Variables on the mean and covariance of the posterior.
The initialisation of the mean is zero, and the initialisation of the
(lower triangular of the) covariance is from diagonal matrices with
diagonal elements taking the value of `std0`.
"""
o, i = dim
# Optimize only values in lower triangular
u, v = np.tril_indices(i)
indices = (u * i + v)[:, np.newaxis]
l0 = (np.tile(np.eye(i, dtype=np.float32) * std0, [o, 1, 1])[:, u, v].T)
lflat = tf.Variable(l0, name=_add_suffix("W_cov_q", suffix))
Lt = tf.transpose(tf.scatter_nd(indices, lflat, shape=(i * i, o)))
L = tf.reshape(Lt, (o, i, i))
mu_0 = tf.zeros((o, i))
mu = tf.Variable(mu_0, name=_add_suffix("W_mu_q", suffix))
summary_histogram(mu)
summary_histogram(lflat)
Q = tfp.distributions.MultivariateNormalTriL(mu, L)
return Q
def _init_lenscale(given_lenscale, learn_lenscale, input_dim):
"""Provide the lenscale variable and its initial value."""
given_lenscale = (np.sqrt(1.0 / input_dim) if given_lenscale is None
else np.array(given_lenscale).squeeze()).astype(
np.float32)
if learn_lenscale:
lenscale = pos_variable(given_lenscale, name="kernel_lenscale")
if np.size(given_lenscale) == 1:
summary_scalar(lenscale)
else:
summary_histogram(lenscale)
else:
lenscale = given_lenscale
lenscale_vec = tf.ones(input_dim, dtype=tf.float32) * lenscale
init_lenscale = given_lenscale * np.ones(input_dim, dtype=np.float32)
return lenscale_vec, init_lenscale
def _build(self, X):
"""Build the graph of this layer."""
n_samples, (input_dim,) = self._get_X_dims(X)
Wdim = (self.n_categories, self.output_dim)
n_batch = tf.shape(X)[1]
W_init = initialise_weights(Wdim, self.init_fn)
W = tf.Variable(W_init, name="W_map")
summary_histogram(W)
# Index into the relevant weights rather than using sparse matmul
features = tf.gather(W, X, axis=0)
f_dims = int(np.prod(features.shape[2:])) # need this for placeholders
Net = tf.reshape(features, [n_samples, n_batch, f_dims])
# Regularizers
penalty = self.l2 * tf.nn.l2_loss(W) + self.l1 * _l1_loss(W)
return Net, penalty
def norm_posterior(dim, std0, suffix=None):
"""Initialise a posterior (diagonal) Normal distribution.
Parameters
----------
dim : tuple or list
the dimension of this distribution.
std0 : float, np.array
the initial (unoptimized) standard deviation of this distribution.
Must be a scalar or have the same shape as dim.
suffix : str
suffix to add to the names of the variables of the parameters of this
distribution.
Returns
-------
Q : tf.distributions.Normal
the initialised posterior Normal object.
Note
----
This will make tf.Variables on the mean standard deviation of the
posterior. The initialisation of the mean is zero and the initialisation of
the standard deviation is simply ``std0`` for each element.
"""
assert (np.ndim(std0) == 0) or (np.shape(std0) == dim)
mu_0 = tf.zeros(dim)
mu = tf.Variable(mu_0, name=_add_suffix("W_mu_q", suffix))
if np.ndim(std0) == 0:
std0 = tf.ones(dim) * std0
std = pos_variable(std0, name=_add_suffix("W_std_q", suffix))
summary_histogram(mu)
summary_histogram(std)
Q = tf.distributions.Normal(loc=mu, scale=std)
return Q