def call(self, inputs):
outputs = nn.convolution(
input=inputs,
filter=self.kernel,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=utils.convert_data_format(self.data_format, self.rank + 2))
if self.bias is not None:
if self.rank != 2 and self.data_format == 'channels_first':
# bias_add does not support channels_first for non-4D inputs.
if self.rank == 1:
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
if self.rank == 3:
bias = array_ops.reshape(self.bias, (1, self.filters, 1, 1))
outputs += bias
else:
outputs = nn.bias_add(
outputs,
self.bias,
data_format=utils.convert_data_format(self.data_format, 4))
# Note that we passed rank=4 because bias_add will only accept
# NHWC and NCWH even if the rank of the inputs is 3 or 5.
if self.activation is not None:
return self.activation(outputs)
return outputs
def call(self, inputs):
outputs = nn.convolution(
input=inputs,
filter=self.masked_kernel,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=utils.convert_data_format(self.data_format, self.rank + 2))
if self.bias is not None:
if self.data_format == 'channels_first':
if self.rank == 1:
# nn.bias_add does not accept a 1D input tensor.
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
outputs += bias
if self.rank == 2:
outputs = nn.bias_add(outputs, self.bias, data_format='NCHW')
if self.rank == 3:
# As of Mar 2017, direct addition is significantly slower than
# bias_add when computing gradients. To use bias_add, we collapse Z
# and Y into a single dimension to obtain a 4D input tensor.
outputs_shape = outputs.shape.as_list()
outputs_4d = array_ops.reshape(outputs, [
outputs_shape[0], outputs_shape[1],
outputs_shape[2] * outputs_shape[3], outputs_shape[4]
])
outputs_4d = nn.bias_add(outputs_4d, self.bias, data_format='NCHW')
outputs = array_ops.reshape(outputs_4d, outputs_shape)
else:
outputs = nn.bias_add(outputs, self.bias, data_format='NHWC')
if self.activation is not None:
return self.activation(outputs)
return outputs
def call(self, inputs):
outputs = nn.convolution(
input=inputs,
filter=self.kernel,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=utils.convert_data_format(self.data_format, self.rank + 2))
if self.bias is not None:
if self.rank != 2 and self.data_format == 'channels_first':
# bias_add does not support channels_first for non-4D inputs.
if self.rank == 1:
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
if self.rank == 3:
bias = array_ops.reshape(self.bias, (1, self.filters, 1, 1))
outputs += bias
else:
outputs = nn.bias_add(
outputs,
self.bias,
data_format=utils.convert_data_format(self.data_format, 4))
# Note that we passed rank=4 because bias_add will only accept
# NHWC and NCWH even if the rank of the inputs is 3 or 5.
if self.activation is not None:
return self.activation(outputs)
return outputs
def atrous_conv2d(value, filters, rate, padding, name=None, **kwargs):
if kwargs['quantizer'] is not None:
return q2dconvolution(input=value,
filter=filters,
quantizer=kwargs['quantizer'],
padding=padding,
dilation_rate=np.broadcast_to(rate, (2, )),
name=name)
else:
return nn.convolution(input=value,
filter=filters,
padding=padding,
dilation_rate=np.broadcast_to(rate, (2, )),
name=name)
def my_graph(inp, bias):
with ops.device(
"/device:IPU:0"), variable_scope.variable_scope(
"vs", use_resource=True, reuse=False):
weights = variable_scope.get_variable(
"weights", [8] * ndims + [M, N * K])
output = nn.convolution(inp,
weights,
strides=[1] + [4] * ndims + [1],
padding="VALID",
name='cnv')
output = nn.bias_add(output, bias, name='bias_add')
loss = math_ops.reduce_sum(math_ops.square(output))
opt = gradient_descent.GradientDescentOptimizer(
0.0005).minimize(loss)
return loss, opt
def extract_image_patches(image, ksizes, strides, padding, name=None):
"""Extracts image patches for an N-dimensional convolution.
This function is a compatibility wrapper over tf.extract_image_patches(), as
ExtractImagePatches isn't yet implemented in XLA.
Args:
image: Tensor of shape [batch, in_x, in_y, ..., in_channels]. Input images.
All dimensions except 'batch' must be defined.
ksizes: [filter_x, filter_y, ...]. Spatial shape of filter in each
dimension.
strides: [stride_x, stride_y, ...]. Spatial stride for filter in each
dimension.
padding: str. "VALID" or "SAME".
name: str or None. name of Op.
Returns:
result: [batch, out_x, out_y, ..., filter_x, filter_y, ..., in_channels].
Contains image patches to which conv kernel would be applied for each
output location. [out_x, out_y, ...] depends on padding.
"""
if not utils.on_tpu():
return array_ops.extract_image_patches(
image,
ksizes=([1] + list(ksizes) + [1]),
strides=([1] + list(strides) + [1]),
rates=[1, 1, 1, 1],
padding=padding,
name=name)
with tf_ops.name_scope(name, "extract_image_patches",
[image, ksizes, strides, padding]):
batch = image.shape.as_list()[0]
in_channels = image.shape.as_list()[-1]
# Map each input feature to a location in the output.
out_channels = np.prod(ksizes) * in_channels
filters = linalg_ops.eye(out_channels),
filters = array_ops.reshape(filters,
ksizes + [in_channels, out_channels])
result = nn.convolution(image, filters, padding, strides=strides)
out_spatial = result.shape.as_list()[1:-1]
result = array_ops.reshape(result, [batch or -1] + out_spatial +
ksizes + [in_channels])
return result
def call(self, inputs):
# quantize the weights, if there is an weight quantizer
if self.weight_quantizer is not None:
used_kernel = self.weight_quantizer.quantize(self.kernel)
else:
used_kernel = self.kernel
# if intrinsic quantization, apply intr. quantization to weights, too!
if self.quantizer is not None:
used_kernel = self.quantizer.quantize(used_kernel)
if self.rank == 2:
if self.quantizer is None:
outputs = nn.convolution(
input=inputs,
filter=used_kernel,
dilation_rate=self.dilation_rate,
strides=self.strides,
padding=self.padding.upper(),
data_format=utils.convert_data_format(self.data_format, self.rank + 2))
else: # with quantization
outputs = q2dconvolution(input=inputs, filter=used_kernel, quantizer=self.quantizer,
padding=self.padding.upper(), strides=self.strides, dilation_rate=self.dilation_rate,
data_format=utils.convert_data_format(self.data_format, self.rank + 2))
else:
raise ValueError("quantized convolution not supported for input rank %d" % (self.rank))
if self.bias is not None:
if self.rank != 2 and self.data_format == 'channels_first':
# bias_add does not support channels_first for non-4D inputs.
if self.rank == 1:
bias = array_ops.reshape(self.bias, (1, self.filters, 1))
if self.rank == 3:
bias = array_ops.reshape(self.bias, (1, self.filters, 1, 1))
outputs += bias
else:
outputs = nn.bias_add(
outputs,
self.bias,
data_format=utils.convert_data_format(self.data_format, 4))
# Note that we passed rank=4 because bias_add will only accept
# NHWC and NCWH even if the rank of the inputs is 3 or 5.
if self.quantizer is not None: # quantize after activation
outputs = self.quantizer.quantize(outputs)
if self.activation is not None:
outputs = self.activation(outputs)
return outputs
def _MyNet():
with variable_scope.variable_scope("vs", use_resource=True):
with ops.device("cpu"):
inp = array_ops.placeholder(np.float32,
[1] + [24] * ndims + [M * K],
name="input")
bias = array_ops.placeholder(np.float32, [N * K], name="bias")
with ops.device("/device:IPU:0"):
weights = variable_scope.get_variable("weights",
[8] * ndims + [M, N * K])
output = nn.convolution(inp,
weights,
strides=[1] + [4] * ndims + [1],
padding="VALID",
name='cnv')
output = nn.bias_add(output, bias, name='bias_add')
loss = math_ops.reduce_sum(math_ops.square(output))
optimizer = gradient_descent.GradientDescentOptimizer(0.0005)
train = optimizer.minimize(loss)
return train, loss, inp, bias
def call(self, inputs):
inputs = ops.convert_to_tensor(inputs, dtype=self.dtype)
ndim = self._input_rank
if self.rectify:
inputs = nn.relu(inputs)
# Compute normalization pool.
if ndim == 2:
norm_pool = math_ops.matmul(math_ops.square(inputs), self.gamma)
norm_pool = nn.bias_add(norm_pool, self.beta)
elif self.data_format == "channels_last" and ndim <= 5:
shape = self.gamma.shape.as_list()
gamma = array_ops.reshape(self.gamma, (ndim - 2) * [1] + shape)
norm_pool = nn.convolution(math_ops.square(inputs), gamma, "VALID")
norm_pool = nn.bias_add(norm_pool, self.beta)
else: # generic implementation
# This puts channels in the last dimension regardless of input.
norm_pool = math_ops.tensordot(math_ops.square(inputs), self.gamma,
[[self._channel_axis()], [0]])
norm_pool += self.beta
if self.data_format == "channels_first":
# Return to channels_first format if necessary.
axes = range(ndim - 1)
axes.insert(1, ndim - 1)
norm_pool = array_ops.transpose(norm_pool, axes)
if self.inverse:
norm_pool = math_ops.sqrt(norm_pool)
else:
norm_pool = math_ops.rsqrt(norm_pool)
outputs = inputs * norm_pool
if not context.executing_eagerly():
outputs.set_shape(self.compute_output_shape(inputs.shape))
return outputs
def convolution(inputs,
num_outputs,
kernel_size,
stride=1,
padding='SAME',
data_format=None,
rate=1,
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_normalizer_fn=None,
weights_normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None):
# Be copied and modified from tensorflow-0.12.0.contrib.layer.convolution,
# add weights_nomalizer_* options.
"""Adds an N-D convolution followed by an optional batch_norm layer.
It is required that 1 <= N <= 3.
`convolution` creates a variable called `weights`, representing the
convolutional kernel, that is convolved (actually cross-correlated) with the
`inputs` to produce a `Tensor` of activations. If a `normalizer_fn` is
provided (such as `batch_norm`), it is then applied. Otherwise, if
`normalizer_fn` is None and a `biases_initializer` is provided then a `biases`
variable would be created and added the activations. Finally, if
`activation_fn` is not `None`, it is applied to the activations as well.
Performs a'trous convolution with input stride/dilation rate equal to `rate`
if a value > 1 for any dimension of `rate` is specified. In this case
`stride` values != 1 are not supported.
Args:
inputs: a Tensor of rank N+2 of shape
`[batch_size] + input_spatial_shape + [in_channels]` if data_format does
not start with "NC" (default), or
`[batch_size, in_channels] + input_spatial_shape` if data_format starts
with "NC".
num_outputs: integer, the number of output filters.
kernel_size: a sequence of N positive integers specifying the spatial
dimensions of of the filters. Can be a single integer to specify the same
value for all spatial dimensions.
stride: a sequence of N positive integers specifying the stride at which to
compute output. Can be a single integer to specify the same value for all
spatial dimensions. Specifying any `stride` value != 1 is incompatible
with specifying any `rate` value != 1.
padding: one of `"VALID"` or `"SAME"`.
data_format: A string or None. Specifies whether the channel dimension of
the `input` and output is the last dimension (default, or if `data_format`
does not start with "NC"), or the second dimension (if `data_format`
starts with "NC"). For N=1, the valid values are "NWC" (default) and
"NCW". For N=2, the valid values are "NHWC" (default) and "NCHW". For
N=3, currently the only valid value is "NDHWC".
rate: a sequence of N positive integers specifying the dilation rate to use
for a'trous convolution. Can be a single integer to specify the same
value for all spatial dimensions. Specifying any `rate` value != 1 is
incompatible with specifying any `stride` value != 1.
activation_fn: activation function, set to None to skip it and maintain
a linear activation.
normalizer_fn: normalization function to use instead of `biases`. If
`normalizer_fn` is provided then `biases_initializer` and
`biases_regularizer` are ignored and `biases` are not created nor added.
default set to None for no normalizer function
normalizer_params: normalization function parameters.
weights_normalizer_fn: weights normalization function.
weights_normalizer_params: weights normalization function parameters.
weights_initializer: An initializer for the weights.
weights_regularizer: Optional regularizer for the weights.
biases_initializer: An initializer for the biases. If None skip biases.
biases_regularizer: Optional regularizer for the biases.
reuse: whether or not the layer and its variables should be reused. To be
able to reuse the layer scope must be given.
variables_collections: optional list of collections for all the variables or
a dictionary containing a different list of collection per variable.
outputs_collections: collection to add the outputs.
trainable: If `True` also add variables to the graph collection
`GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
scope: Optional scope for `variable_scope`.
Returns:
a tensor representing the output of the operation.
Raises:
ValueError: if `data_format` is invalid.
ValueError: both 'rate' and `stride` are not uniformly 1.
"""
if data_format not in [None, 'NWC', 'NCW', 'NHWC', 'NCHW', 'NDHWC']:
raise ValueError('Invalid data_format: %r' % (data_format, ))
with variable_scope.variable_scope(scope, 'Conv', [inputs],
reuse=reuse) as sc:
inputs = ops.convert_to_tensor(inputs)
dtype = inputs.dtype.base_dtype
input_rank = inputs.get_shape().ndims
if input_rank is None:
raise ValueError('Rank of inputs must be known')
if input_rank < 3 or input_rank > 5:
raise ValueError(
'Rank of inputs is %d, which is not >= 3 and <= 5' %
input_rank)
conv_dims = input_rank - 2
kernel_size = utils.n_positive_integers(conv_dims, kernel_size)
stride = utils.n_positive_integers(conv_dims, stride)
rate = utils.n_positive_integers(conv_dims, rate)
if data_format is None or data_format.endswith('C'):
num_input_channels = inputs.get_shape()[input_rank - 1].value
elif data_format.startswith('NC'):
num_input_channels = inputs.get_shape()[1].value
else:
raise ValueError('Invalid data_format')
if num_input_channels is None:
raise ValueError('Number of in_channels must be known.')
weights_shape = (list(kernel_size) + [num_input_channels, num_outputs])
weights_collections = utils.get_variable_collections(
variables_collections, 'weights')
weights = variables.model_variable('weights',
shape=weights_shape,
dtype=dtype,
initializer=weights_initializer,
regularizer=weights_regularizer,
collections=weights_collections,
trainable=trainable)
if weights_normalizer_fn is not None:
weights_normalizer_params = weights_normalizer_params or {}
weights = weights_normalizer_fn(weights,
**weights_normalizer_params)
outputs = nn.convolution(input=inputs,
filter=weights,
dilation_rate=rate,
strides=stride,
padding=padding,
data_format=data_format)
if normalizer_fn is not None:
normalizer_params = normalizer_params or {}
outputs = normalizer_fn(outputs, **normalizer_params)
else:
if biases_initializer is not None:
biases_collections = utils.get_variable_collections(
variables_collections, 'biases')
biases = variables.model_variable(
'biases',
shape=[num_outputs],
dtype=dtype,
initializer=biases_initializer,
regularizer=biases_regularizer,
collections=biases_collections,
trainable=trainable)
outputs = nn.bias_add(outputs, biases, data_format=data_format)
if activation_fn is not None:
outputs = activation_fn(outputs)
return utils.collect_named_outputs(outputs_collections,
sc.original_name_scope, outputs)
def call(self, inputs):
inputs = ops.convert_to_tensor(inputs, dtype=self.dtype)
input_shape = array_ops.shape(inputs)
outputs = inputs
# First, perform any requested padding.
if self.padding in ("same_zeros", "same_reflect"):
padding = padding_ops.same_padding_for_kernel(
self.kernel_support, self.corr, self.strides_up)
if self.data_format == "channels_last":
padding = [[0, 0]] + list(padding) + [[0, 0]]
else:
padding = [[0, 0], [0, 0]] + list(padding)
outputs = array_ops.pad(outputs, padding, self._pad_mode)
# Now, perform valid convolutions/correlations.
# Not for all possible combinations of (`kernel_support`, `corr`,
# `strides_up`, `strides_down`) TF ops exist. We implement some additional
# combinations by manipulating the kernels and toggling `corr`.
kernel = self.kernel
corr = self.corr
# If a convolution with no upsampling is desired, we flip the kernels and
# use cross correlation to implement it, provided the kernels are odd-length
# in every dimension (with even-length kernels, the boundary handling
# would have to change, so we'll throw an error instead).
if (not corr and all(s == 1 for s in self.strides_up)
and all(s % 2 == 1 for s in self.kernel_support)):
corr = True
slices = self._rank * (slice(None, None,
-1), ) + 2 * (slice(None), )
kernel = kernel[slices]
# Similarly, we can implement a cross correlation with no downsampling using
# convolutions. However, we do this only if upsampling is requested, as we
# are wasting computation in the boundaries whenever we call the transpose
# convolution ops.
if (corr and all(s == 1 for s in self.strides_down)
and any(s != 1 for s in self.strides_up)
and all(s % 2 == 1 for s in self.kernel_support)):
corr = False
slices = self._rank * (slice(None, None,
-1), ) + 2 * (slice(None), )
kernel = kernel[slices]
data_format = utils.convert_data_format(self.data_format,
self._rank + 2)
if (corr and self.channel_separable and self._rank == 2
and all(s == 1 for s in self.strides_up)
and all(s == self.strides_down[0] for s in self.strides_down)):
# `nn.depthwise_conv2d_native` performs channel-separable correlations
# followed by optional downsampling.
outputs = nn.depthwise_conv2d_native(outputs,
kernel,
strides=self._pad_strides(
self.strides_down),
padding="VALID",
data_format=data_format)
elif (corr and all(s == 1 for s in self.strides_up)
and not self.channel_separable):
# `nn.convolution` performs correlations followed by optional
# downsampling.
outputs = nn.convolution(outputs,
kernel,
strides=self.strides_down,
padding="VALID",
data_format=data_format)
elif (not corr and all(s == 1 for s in self.strides_down)
and ((not self.channel_separable and 1 <= self._rank <= 3) or
(self.channel_separable and self.filters == 1
and self._rank == 2 and all(s == self.strides_up[0]
for s in self.strides_up)))):
# `nn.conv?d_transpose` perform convolutions, preceded by optional
# upsampling. Generally, they increase the spatial support of their
# inputs, so in order to implement 'valid', we need to crop their outputs.
# Transpose convolutions expect the output and input channels in reversed
# order. We implement this by swapping those dimensions of the kernel.
# For channel separable convolutions, we can't currently perform anything
# other than one filter per channel, so the last dimension needs to be of
# length one. Since this happens to be the format that the op expects it,
# we can skip the transpose in that case.
if not self.channel_separable:
kernel = array_ops.transpose(
kernel,
list(range(self._rank)) + [self._rank + 1, self._rank])
# Compute shape of temporary.
pad_shape = array_ops.shape(outputs)
temp_shape = [pad_shape[0]] + (self._rank + 1) * [None]
if self.data_format == "channels_last":
spatial_axes = range(1, self._rank + 1)
if self.channel_separable:
temp_shape[-1] = input_shape[-1]
else:
temp_shape[-1] = self.filters
else:
spatial_axes = range(2, self._rank + 2)
if self.channel_separable:
temp_shape[1] = input_shape[1]
else:
temp_shape[1] = self.filters
if self.extra_pad_end:
get_length = lambda l, s, k: l * s + (k - 1)
else:
get_length = lambda l, s, k: l * s + (k - s)
for i, a in enumerate(spatial_axes):
temp_shape[a] = get_length(pad_shape[a], self.strides_up[i],
self.kernel_support[i])
# Compute convolution.
if self._rank == 1 and not self.channel_separable:
# There's no 1D transpose convolution op, so we insert an extra
# dimension and use 2D.
extradim = {
"channels_first": 2,
"channels_last": 1
}[self.data_format]
strides = self._pad_strides(self.strides_up)
temp = array_ops.squeeze(
nn.conv2d_transpose(
array_ops.expand_dims(outputs, extradim),
array_ops.expand_dims(kernel, 0),
temp_shape[:extradim] + [1] + temp_shape[extradim:],
strides=strides[:extradim] + (1, ) +
strides[extradim:],
padding="VALID",
data_format=data_format.replace("W", "HW")),
[extradim])
elif self._rank == 2 and self.channel_separable:
temp = nn.depthwise_conv2d_native_backprop_input(
temp_shape,
kernel,
outputs,
strides=self._pad_strides(self.strides_up),
padding="VALID",
data_format=data_format)
elif self._rank == 2 and not self.channel_separable:
temp = nn.conv2d_transpose(outputs,
kernel,
temp_shape,
strides=self._pad_strides(
self.strides_up),
padding="VALID",
data_format=data_format)
elif self._rank == 3 and not self.channel_separable:
temp = nn.conv3d_transpose(outputs,
kernel,
temp_shape,
strides=self._pad_strides(
self.strides_up),
padding="VALID",
data_format=data_format)
else:
assert False # Should never reach this.
# Perform crop.
slices = [slice(None)] * (self._rank + 2)
if self.padding == "valid":
# Take `kernel_support - 1` samples away from both sides. This leaves
# just samples computed without padding.
for i, a in enumerate(spatial_axes):
slices[a] = slice(
self.kernel_support[i] - 1,
None if self.kernel_support[i] == 1 else 1 -
self.kernel_support[i])
else:
# Take `kernel_support // 2` plus the padding away from beginning, and
# crop end to input length multiplied by upsampling factor.
for i, a in enumerate(spatial_axes):
offset = padding[a][0] * self.strides_up[i]
offset += self.kernel_support[i] // 2
length = get_length(input_shape[a], self.strides_up[i],
offset + 1)
slices[a] = slice(offset, length)
outputs = temp[slices]
else:
raise NotImplementedError(
"The provided combination of SignalConv arguments is not currently "
"implemented (kernel_support={}, corr={}, strides_down={}, "
"strides_up={}, channel_separable={}, filters={}). "
"Try using odd-length kernels or turning off separability?".
format(self.kernel_support, self.corr, self.strides_down,
self.strides_up, self.channel_separable, self.filters))
# Now, add bias if requested.
if self.bias is not None:
if self.data_format == "channels_first":
# As of Mar 2017, direct addition is significantly slower than
# bias_add when computing gradients.
if self._rank == 1:
# nn.bias_add does not accept a 1D input tensor.
outputs = array_ops.expand_dims(outputs, 2)
outputs = nn.bias_add(outputs,
self.bias,
data_format="NCHW")
outputs = array_ops.squeeze(outputs, [2])
elif self._rank == 2:
outputs = nn.bias_add(outputs,
self.bias,
data_format="NCHW")
elif self._rank >= 3:
shape = array_ops.shape(outputs)
outputs = array_ops.reshape(outputs, shape[:3] + [-1])
outputs = nn.bias_add(outputs,
self.bias,
data_format="NCHW")
outputs = array_ops.reshape(outputs, shape)
else:
outputs = nn.bias_add(outputs, self.bias)
# Finally, pass through activation function if requested.
if self.activation is not None:
outputs = self.activation(outputs) # pylint:disable=not-callable
# Aid shape inference, for some reason shape info is not always available.
if not context.executing_eagerly():
outputs.set_shape(self.compute_output_shape(inputs.shape))
return outputs
def leaders_convolution2d(
inputs,
num_outputs,
kernel_size,
stride=1,
padding='SAME',
rates=[1, 2],
pooling_sizes=[3, 1],
pooling_type='MAX',
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
data_format='NHWC',
scope=None):
"""Adds a depthwise 2D convolution with optional batch_norm layer.
This op performs a depthwise convolution that acts separately on
channels, creating a variable called `depthwise_weights`. Then,
if `normalizer_fn` is None,
it adds bias to the result, creating a variable called 'biases', otherwise,
the `normalizer_fn` is applied. It finally applies an activation function
to produce the end result.
Args:
inputs: A tensor of size [batch_size, height, width, channels].
num_outputs: The number of pointwise convolution output filters. If is
None, then we skip the pointwise convolution stage.
kernel_size: A list of length 2: [kernel_height, kernel_width] of
of the filters. Can be an int if both values are the same.
depth_multiplier: The number of depthwise convolution output channels for
each input channel. The total number of depthwise convolution output
channels will be equal to `num_filters_in * depth_multiplier`.
stride: A list of length 2: [stride_height, stride_width], specifying the
depthwise convolution stride. Can be an int if both strides are the same.
padding: One of 'VALID' or 'SAME'.
rate: A list of length 2: [rate_height, rate_width], specifying the dilation
rates for atrous convolution. Can be an int if both rates are the same.
If any value is larger than one, then both stride values need to be one.
activation_fn: Activation function. The default value is a ReLU function.
Explicitly set it to None to skip it and maintain a linear activation.
normalizer_fn: Normalization function to use instead of `biases`. If
`normalizer_fn` is provided then `biases_initializer` and
`biases_regularizer` are ignored and `biases` are not created nor added.
default set to None for no normalizer function
normalizer_params: Normalization function parameters.
weights_initializer: An initializer for the weights.
weights_regularizer: Optional regularizer for the weights.
biases_initializer: An initializer for the biases. If None skip biases.
biases_regularizer: Optional regularizer for the biases.
reuse: Whether or not the layer and its variables should be reused. To be
able to reuse the layer scope must be given.
variables_collections: Optional list of collections for all the variables or
a dictionary containing a different list of collection per variable.
outputs_collections: Collection to add the outputs.
trainable: Whether or not the variables should be trainable or not.
scope: Optional scope for variable_scope.
Returns:
A `Tensor` representing the output of the operation.
"""
with variable_scope.variable_scope(scope,
'LeadersConv2d', [inputs],
reuse=reuse) as sc:
inputs = ops.convert_to_tensor(inputs)
dtype = inputs.dtype.base_dtype
kernel_h, kernel_w = utils.two_element_tuple(kernel_size)
stride_h, stride_w = utils.two_element_tuple(stride)
if data_format == 'NHWC':
num_filters_in = utils.last_dimension(inputs.get_shape(),
min_rank=4)
strides = [1, stride_h, stride_w, 1]
else:
num_filters_in = inputs.get_shape().as_list()[1]
strides = [1, 1, stride_h, stride_w]
# Conv weights + biases variables.
weights_collections = utils.get_variable_collections(
variables_collections, 'weights')
depthwise_shape = [kernel_h, kernel_w, num_filters_in, num_outputs]
depthwise_weights = variables.model_variable(
'leaders_weights',
shape=depthwise_shape,
dtype=dtype,
initializer=weights_initializer,
regularizer=weights_regularizer,
trainable=trainable,
collections=weights_collections)
biases_collections = utils.get_variable_collections(
variables_collections, 'biases')
biases = variables.model_variable('biases',
shape=[
num_outputs,
],
dtype=dtype,
initializer=biases_initializer,
regularizer=biases_regularizer,
trainable=trainable,
collections=biases_collections)
# Perform the convolution at different rates.
outputs = []
for i, rate in enumerate(rates):
# Depthwise conv.
net = nn.convolution(inputs,
depthwise_weights,
strides=[1, 1],
padding='SAME',
dilation_rate=utils.two_element_tuple(rate),
data_format=data_format)
# Add bias + abs. val.
net = tf.abs(nn.bias_add(net, biases, data_format=data_format))
# Pooling...
if pooling_sizes[i] > 1:
net = tf.nn.pool(net, [pooling_sizes[i], pooling_sizes[i]],
pooling_type,
padding='SAME',
data_format=data_format)
outputs.append(net)
# Fuse different rates/scales.
net = None
for i, o in enumerate(outputs):
if net is None:
# First in the list...
net = o
else:
# MAX or AVG pooling...
if pooling_type == 'MAX':
net = tf.maximum(net, o)
else:
net += o * pooling_sizes[i]**2
# Pooling => for stride > 1
if stride_h > 1 or stride_w > 1:
# net = tf.nn.pool(net,
# [1, 1],
# pooling_type,
# padding='SAME',
# strides=[stride_h, stride_w],
# data_format=data_format)
net = tf.nn.max_pool(net, [1, 1, 1, 1],
padding='SAME',
strides=strides,
data_format=data_format)
# (Batch normalization)...
normalizer_params = normalizer_params or {}
net = normalizer_fn(net, **normalizer_params)
# Split into two parts: positive and negative extreme.
net_p = slim.bias_add(net,
data_format=data_format,
scope='bias_positive')
net_p = activation_fn(net_p)
net_p = slim.dropout(net_p, 0.5, is_training=True, scope='dropout_p')
net_m = slim.bias_add(-net,
data_format=data_format,
scope='bias_negative')
net_m = activation_fn(net_m)
net_m = slim.dropout(net_m, 0.5, is_training=True, scope='dropout_m')
# Concat the final result...
outputs = concat_channels([net_p, net_m], data_format=data_format)
return utils.collect_named_outputs(outputs_collections,
sc.original_name_scope, outputs)
#filters_vals = np.ones((filter_width,filter_height,input_channels,output_channels)) # width, height, in_channels, out_channels
#filters_vals = np.arange(filter_width*filter_height*input_channels*output_channels).reshape(filter_width,filter_height,input_channels,output_channels)
filters_vals = np.random.normal(size=(filter_width, filter_height,
input_channels,
output_channels)) * testdata_scale // 1
inputs = tf.constant(inputs_vals, dtype=tf.float32)
filters = tf.constant(filters_vals, dtype=tf.float32)
#quantizer = Quantizers.NoQuantizer()
#output = Qnn_ops.convolution(inputs, filters, strides, padding, data_format,
# quantizer=quantizer)
gold_output = nn.convolution(input=inputs,
filter=filters,
dilation_rate=(1, 1),
strides=(stride, stride),
padding=padding)
with tf.Session() as sess:
gold_result = gold_output.eval().flatten()
result = output.eval().flatten()
# print(sess.run(gold_output))
# print('################################')
# print(sess.run(output))
pass
failed = False
for i in range(len(result)):
if result[i] != gold_result[i]:
failed = True
def testGroupedConvolutions(self):
"""
Check we can compile a graph with grouped convolutions, ie. where the
input and output features are divided into K groups, with the N outputs
in the kth group just depending on the M inputs in the kth group.
"""
ndims = 2
M = 3
N = 5
K = 7 # input features per group, output features per group, number of groups
with self.session() as sess:
with variable_scope.variable_scope("vs", use_resource=True):
with ops.device("cpu"):
inp = array_ops.placeholder(np.float32,
[1] + [24] * ndims + [M * K],
name="input")
bias = array_ops.placeholder(np.float32, [N * K],
name="bias")
with ops.device("/device:IPU:0"):
weights = variable_scope.get_variable(
"weights", [8] * ndims + [M, N * K])
output = nn.convolution(inp,
weights,
strides=[1] + [4] * ndims + [1],
padding="VALID",
name='cnv')
output = nn.bias_add(output, bias, name='bias_add')
loss = math_ops.reduce_sum(math_ops.square(output))
optimizer = gradient_descent.GradientDescentOptimizer(
0.0005)
train = optimizer.minimize(loss)
train = ipu.ipu_compiler.compile(lambda: (loss, train), [])
report = tu.ReportJSON(self, sess)
sess.run(variables.global_variables_initializer())
report.reset()
fd = {
inp: np.random.random_sample([1] + [24] * ndims + [M * K]),
bias: np.random.random_sample([N * K])
}
sess.run(train, fd)
report.parse_log()
ok = [
'__seed*',
'Copy_',
'host-exchange*',
'vs/cnv/convolution*/Conv_8x8_stride4x4/Convolve',
'vs/cnv/convolution*/Conv_8x8_stride4x4/Reduce',
'vs/gradients/vs/cnv_grad/Conv2DBackpropFilter/fusion*/Conv_5x5',
'vs/gradients/vs/cnv_grad/Conv2DBackpropFilter/fusion*/AddTo',
'vs/gradients/vs/Square_grad/Mul/fusion*/Op/Multiply',
'vs/bias_add/fusion/Op/Add',
'vs/Sum/reduce',
'vs/Square/multiply*/Op/Multiply',
]
report.assert_all_compute_sets_and_list(ok)
quantizer = Quantizers.NoQuantizer()
results = {} # dictionary with test results
test_name = 'conv2d'
results[test_name] = {}
results[test_name]['quant'] = QConv.q2dconvolution(input=inputs,
filter=filters,
quantizer=quantizer,
strides=stride,
dilation_rate=rate,
padding=padding,
data_format=data_format)
results[test_name]['gold'] = nn.convolution(input=inputs,
filter=filters,
dilation_rate=rate,
strides=stride,
padding=padding)
test_name = 'atrous_conv2d'
results[test_name] = {}
results[test_name]['quant'] = QConv.atrous_conv2d(inputs,
filters,
rate,
padding,
quantizer=quantizer)
results[test_name]['gold'] = nn.atrous_conv2d(inputs, filters, rate, padding)
with tf.Session() as sess:
for key in results.keys():
results[key]['quant'] = results[key]['quant'].eval().flatten()
