def _materialised_conv_layer_dual_objective(w, b, padding, strides, lam_in,
mu_out, lb, ub):
"""Materialised version of `conv_layer_dual_objective`."""
# Flatten the inputs, as the materialised convolution will have no
# spatial structure.
mu_out_flat = snt.BatchFlatten(preserve_dims=2)(mu_out)
# Materialise the convolution as a (sparse) fully connected linear layer.
w_flat, b_flat = layer_utils.materialise_conv(w,
b,
lb.shape[1:].as_list(),
padding=padding,
strides=strides)
activation_coeffs = -tf.tensordot(
mu_out_flat, tf.transpose(w_flat), axes=1)
dual_obj_bias = -tf.tensordot(mu_out_flat, b_flat, axes=1)
# Flatten the inputs, as the materialised convolution will have no
# spatial structure.
if lam_in is not None:
lam_in = snt.FlattenTrailingDimensions(2)(lam_in)
lb = snt.BatchFlatten()(lb)
ub = snt.BatchFlatten()(ub)
return standard_layer_calcs.linear_dual_objective(lam_in,
activation_coeffs,
dual_obj_bias, lb, ub)
def _materialised_conv_simplex_bounds(w, b, padding, strides, bounds_in):
"""Calculates naive bounds on output of an N-D convolution layer.
The calculation is performed by first materialising the convolution as a
(sparse) fully-connected linear layer. Doing so will affect performance, but
may be useful for investigating numerical stability issues.
The layer inputs and the vertices are assumed to be (N-D) sequences in an
embedding space. The input domain is taken to be the simplex of perturbations
of the centres (true inputs) towards the given vertices.
Specifically, the input domain is the convex hull of this set of vertices::
{ (1-r)*centres + r*vertices[j] : j<num_vertices }
Args:
w: (N+2)D tensor of shape (kernel_length, input_channels, output_channels)
containing weights for the convolution.
b: 1D tensor of shape (output_channels) containing biases for the
convolution, or `None` if no bias.
padding: `"VALID"` or `"SAME"`, the convolution's padding algorithm.
strides: Integer list of length N: `[vertical_stride, horizontal_stride]`.
bounds_in: bounds of shape (batch_size, input_length, input_channels)
containing bounds on the inputs to the convolution layer.
Returns:
bounds of shape (batch_size, output_length, output_channels)
with bounds on the outputs of the convolution layer.
Raises:
ValueError: if an unsupported convolution dimensionality is encountered.
"""
# Flatten the inputs, as the materialised convolution will have no
# spatial structure.
bounds_in_flat = bounds_in.apply_batch_reshape(None, [-1])
# Materialise the convolution as a (sparse) fully connected linear layer.
input_shape = bounds_in.shape[1:]
w_lin, b_lin = layer_utils.materialise_conv(w,
b,
input_shape,
padding=padding,
strides=strides)
bounds_out_flat = bounds_in_flat.apply_linear(None, w_lin, b_lin)
# Unflatten the output bounds.
output_shape = layer_utils.conv_output_shape(input_shape, w, padding,
strides)
return bounds_out_flat.apply_batch_reshape(None, output_shape)
def _materialised_conv_bounds(w, b, padding, strides, bounds_in):
"""Calculates bounds on output of an N-D convolution layer.
The calculation is performed by first materialising the convolution as a
(sparse) fully-connected linear layer. Doing so will affect performance, but
may be useful for investigating numerical stability issues.
Args:
w: (N+2)D tensor of shape (kernel_height, kernel_width, input_channels,
output_channels) containing weights for the convolution.
b: 1D tensor of shape (output_channels) containing biases for the
convolution, or `None` if no bias.
padding: `"VALID"` or `"SAME"`, the convolution's padding algorithm.
strides: Integer list of length N: `[vertical_stride, horizontal_stride]`.
bounds_in: bounds of shape (batch_size, input_height, input_width,
input_channels) containing bounds on the inputs to the
convolution layer.
Returns:
bounds of shape (batch_size, output_height, output_width,
output_channels) with bounds on the outputs of the
convolution layer.
Raises:
ValueError: if an unsupported convolution dimensionality is encountered.
"""
# Flatten the inputs, as the materialised convolution will have no
# spatial structure.
bounds_in_flat = bounds_in.apply_batch_reshape(None, [-1])
# Materialise the convolution as a (sparse) fully connected linear layer.
input_shape = bounds_in.shape[1:]
w_lin, b_lin = layer_utils.materialise_conv(w,
b,
input_shape,
padding=padding,
strides=strides)
bounds_out_flat = bounds_in_flat.apply_linear(None, w_lin, b_lin)
# Unflatten the output bounds.
output_shape = layer_utils.conv_output_shape(input_shape, w, padding,
strides)
return bounds_out_flat.apply_batch_reshape(None, output_shape)
def conv_weighted_gram_abs_projection_slow(w, d, beta, padding, strides):
"""Calculates a projection of | W^T d W | for an N-D convolution W.
Computes beta_i^{-1} sum_j |Q_ij| beta_j where Q = W^T d W is the
weighted Gram matrix for the convolution.
The convolution exploits sparsity of the convolution, thereby managing
to run in O(K^2 M C^3 + K^3 M C^2) time per example, for C channels,
spatial size M, and kernel size K. By comparison, working with
a fully materialised MCxMC matrix would require O(M^3 C^3) time.
Args:
w: (N+2)D tensor of shape (kernel_height, kernel_width,
input_channels, output_channels) containing the convolutional kernel.
d: (N+3)D tensor of shape (num_targets, batch_size,
output_height, output_width, output_channels), interpreted as a
diagonal weight matrix.
beta: (N+3)D tensor of shape (num_targets, batch_size,
input_height, input_width, input_channels) specifying the projection.
padding: `"VALID"` or `"SAME"`, the convolution's padding algorithm.
strides: Integer list of `[vertical_stride, horizontal_stride]`.
Returns:
(N+3)D tensor of shape (num_targets, batch_size,
input_height, input_width, input_channels) containing | W^T d W | beta.
"""
input_shape = beta.shape[2:].as_list()
flatten = snt.BatchFlatten(preserve_dims=2)
unflatten = snt.BatchReshape(input_shape, preserve_dims=2)
w_lin, _ = layer_utils.materialise_conv(w, None, input_shape, padding,
strides)
return unflatten(
linear_weighted_gram_abs_projection_slow(w_lin, flatten(d),
flatten(beta)))
def flatten(self):
return layer_utils.materialise_conv(self._w,
self._b,
input_shape=self.input_shape,
padding=self._padding,
strides=self._strides)
def conv_weighted_gram_matrix(w, d, input_shape, padding, strides, w_s=None):
"""Calculates W^T d W for an N-D convolution W, exploiting sparsity.
Args:
w: (N+2)D tensor of shape (kernel_height, kernel_width,
input_channels, output_channels) containing the convolutional kernel.
d: (N+3)D tensor of shape (num_targets, batch_size,
output_height, output_width, output_channels), interpreted as a
diagonal weight matrix.
input_shape: List of length N+1 specifying
[input_height, input_width, input_channels].
padding: `"VALID"` or `"SAME"`, the convolution's padding algorithm.
strides: Integer list of `[vertical_stride, horizontal_stride]`.
w_s: Optional (N+2)D tensor of shape (kernel_height, kernel_width,
input_slice_channels, output_channels) containing a slice of `w`
(over input_channels) if it is desired to build the Gram matrix a few
columns at a time; defaults to `w` to build the Gram matrix in full.
Returns:
(2N+4)D tensor of shape (num_targets, batch_size,
input_height, input_width, input_slice_channels,
2*kernel_height-1, 2*kernel_width-1, input_channels)
expressing W^T d W in a sheared form to exploit sparsity.
"""
w_s = w_s if w_s is not None else w
num_targets = d.shape[0].value
batch_size = tf.shape(d)[1]
n = w.shape.ndims - 2
kernel_shape = w.shape[:-2].as_list()
input_channels = w.shape[-2].value
input_slice_channels = w_s.shape[-2].value
output_channels = w.shape[-1].value
enlarged_kernel_shape = [2 * s - 1 for s in kernel_shape]
# We wish to combine W with itself at different kernel offsets,
# from -kernel_size to +kernel_size (exclusive).
# Achieve this by considering W (kernel) as a new stride-1 deconvolution.
w_offset, _ = layer_utils.materialise_conv(
tf.reverse(w, axis=list(range(n))),
None,
input_shape=(enlarged_kernel_shape + [-1]),
padding='VALID',
strides=(n * [1]))
# The above materialises it as a 2D tensor with shape
# (enlarged_kernel_shape*input_channels,
# kernel_height*kernel_width*output_channels).
w_offset = tf.reshape(
w_offset,
shape=([1] + enlarged_kernel_shape + [input_channels] + kernel_shape +
[output_channels]))
w_offset = tf.transpose(tf.reverse(w_offset, axis=list(range(1, n + 1))),
perm=(list(range(n + 2, 2 * n + 2)) +
list(range(n + 2)) + [2 * n + 2]))
# W_offset is now a (2N+3)D tensor with shape
# (kernel_height, kernel_width, 1,
# 2*kernel_height-1, 2*kernel_width-1, input_channels, output_channels).
# Take all relevant pair-wise products of W with W_offset.
wtw = w_offset * tf.reshape(w_s,
shape=(kernel_shape + [input_slice_channels] +
(n * [1]) + [1, output_channels]))
# WTW is a (2N+3)D tensor with shape
# (kernel_height, kernel_width, input_slice_channels,
# 2*kernel_height-1, 2*kernel_width-1, input_channels, output_channels).
# Combine with d, by performing a deconvolution.
wtw = tf.reshape(
wtw,
shape=(kernel_shape + [
input_slice_channels * np.prod(enlarged_kernel_shape) *
input_channels, output_channels
]))
result = common.conv_transpose(d, wtw,
input_shape[:-1] + [wtw.shape[n].value],
padding, strides)
# Output from common.conv_transpose is of shape:
# (num_targets, batch_size, input_height, input_width,
# input_slice_channels*enlarged_kernel_shape*input_channels).
result = tf.reshape(result,
shape=([num_targets, batch_size] + input_shape[:-1] +
[input_slice_channels] + enlarged_kernel_shape +
[input_channels]))
# Return a (2N+4)D tensor of shape (num_targets, batch_size,
# input_height, input_width, input_slice_channels,
# 2*kernel_height-1, 2*kernel_width-1, input_channels).
return result