def _dilated_conv_layer(self, output_channels, dilation_rate, apply_relu,
name):
"""Create a dilated convolution layer.
Args:
output_channels: int. Number of output channels for each pixel.
dilation_rate: int. Represents how many pixels each stride offset will
move. A value of 1 indicates a standard convolution.
apply_relu: bool. If True, a ReLU non-linearlity is added.
name: string. Name for layer.
Returns:
a sonnet Module for a dilated convolution.
"""
layer_components = [
conv.Conv2D(output_channels, [3, 3],
initializers=self._initializers,
regularizers=self._regularizers,
rate=dilation_rate,
name="dilated_conv_" + name),
]
if apply_relu:
layer_components.append(
lambda net: tf.nn.relu(net, name="relu_" + name))
return sequential.Sequential(layer_components, name=name)
def testScopeRestore(self):
c1 = conv.Conv2D(16,
8,
4,
name='conv_2d_0',
padding=conv.VALID,
initializers={
'w':
initializers.restore_initializer(
_checkpoint(),
'w',
scope='agent/conv_net_2d/conv_2d_0'),
'b':
initializers.restore_initializer(
_checkpoint(),
'b',
scope='agent/conv_net_2d/conv_2d_0')
})
inputs = tf.constant(1 / 255.0, shape=[1, 86, 86, 3])
outputs = c1(inputs)
init = tf.global_variables_initializer()
tf.get_default_graph().finalize()
with self.test_session() as session:
session.run(init)
o = session.run(outputs)
self.assertAllClose(np.linalg.norm(o),
_ONE_CONV_LAYER,
atol=_TOLERANCE)
def testMultipleRestore(self):
g = tf.Graph()
restore_initializers = {
'w': initializers.restore_initializer(_checkpoint(), 'w'),
'b': initializers.restore_initializer(_checkpoint(), 'b')
}
with g.as_default():
with tf.variable_scope('agent/conv_net_2d'):
c1 = conv.Conv2D(16,
8,
4,
name='conv_2d_0',
padding=conv.VALID,
initializers=restore_initializers)
c2 = conv.Conv2D(32,
4,
2,
name='conv_2d_1',
padding=conv.VALID,
initializers=restore_initializers)
inputs = tf.constant(1 / 255.0, shape=[1, 86, 86, 3])
intermediate_1 = c1(inputs)
intermediate_2 = c2(tf.nn.relu(intermediate_1))
outputs = tf.nn.relu(intermediate_2)
init = tf.global_variables_initializer()
tf.get_default_graph().finalize()
with self.test_session() as session:
session.run(init)
i1, i2, o = session.run(
[intermediate_1, intermediate_2, outputs])
self.assertAllClose(np.linalg.norm(i1),
_ONE_CONV_LAYER,
rtol=0.001,
atol=0.001)
self.assertAllClose(np.linalg.norm(i2),
_TWO_CONV_LAYERS,
rtol=0.001,
atol=0.001)
self.assertAllClose(np.linalg.norm(o),
_TWO_CONV_LAYERS_RELU,
rtol=0.001,
atol=0.001)
def _instantiate_layers(self):
"""Instantiates all the convolutional modules used in the network."""
with self._enter_variable_scope():
self._layers = tuple(conv.Conv2D(name="conv_2d_{}".format(i),
output_channels=self._output_channels[i],
kernel_shape=self._kernel_shapes[i],
stride=self._strides[i],
padding=self._paddings[i],
use_bias=self._use_bias[i],
initializers=self._initializers,
partitioners=self._partitioners,
regularizers=self._regularizers)
for i in xrange(self._num_layers))
def _instantiate_layers(self):
"""Instantiates all the convolutional modules used in the network."""
# Here we are entering the module's variable scope to name our submodules
# correctly (not to create variables). As such it's safe to not check
# whether we're in the same graph. This is important if we're constructing
# the module in one graph and connecting it in another (e.g. with `defun`
# the module is created in some default graph, and connected to a capturing
# graph in order to turn it into a graph function).
with self._enter_variable_scope(check_same_graph=False):
self._layers = tuple(conv.Conv2D(name="conv_2d_{}".format(i),
output_channels=self._output_channels[i],
kernel_shape=self._kernel_shapes[i],
stride=self._strides[i],
rate=self._rates[i],
padding=self._paddings[i],
use_bias=self._use_bias[i],
initializers=self._initializers,
partitioners=self._partitioners,
regularizers=self._regularizers,
data_format=self._data_format)
for i in xrange(self._num_layers))
def _build(self,
inputs,
keep_prob=None,
is_training=True,
test_local_stats=True):
"""Connects the AlexNet module into the graph.
Args:
inputs: A Tensor of size [batch_size, input_height, input_width,
input_channels], representing a batch of input images.
keep_prob: A scalar Tensor representing the dropout keep probability.
is_training: Boolean to indicate to `snt.BatchNorm` if we are
currently training. By default `True`.
test_local_stats: Boolean to indicate to `snt.BatchNorm` if batch
normalization should use local batch statistics at test time.
By default `True`.
Returns:
A Tensor of size [batch_size, output_size], where `output_size` depends
on the mode the network was constructed in.
Raises:
base.IncompatibleShapeError: If any of the input image dimensions
(input_height, input_width) are too small for the given network mode.
"""
input_shape = inputs.get_shape().as_list()
if input_shape[1] < self._min_size or input_shape[2] < self._min_size:
raise base.IncompatibleShapeError(
"Image shape too small: ({:d}, {:d}) < {:d}".format(
input_shape[1], input_shape[2], self._min_size))
net = inputs
for i, params in enumerate(self._conv_layers):
output_channels, conv_params, max_pooling = params
kernel_size, stride = conv_params
conv_mod = conv.Conv2D(name="conv_{}".format(i),
output_channels=output_channels,
kernel_shape=kernel_size,
stride=stride,
padding=conv.VALID,
initializers=self._initializers,
partitioners=self._partitioners,
regularizers=self._regularizers)
if not self.is_connected:
self._conv_modules.append(conv_mod)
net = conv_mod(net)
if self._use_batch_norm:
bn = batch_norm.BatchNorm(**self._batch_norm_config)
net = bn(net, is_training, test_local_stats)
net = tf.nn.relu(net)
if max_pooling is not None:
pooling_kernel_size, pooling_stride = max_pooling
net = tf.nn.max_pool(
net,
ksize=[1, pooling_kernel_size, pooling_kernel_size, 1],
strides=[1, pooling_stride, pooling_stride, 1],
padding=conv.VALID)
net = basic.BatchFlatten(name="flatten")(net)
for i, output_size in enumerate(self._fc_layers):
linear_mod = basic.Linear(name="fc_{}".format(i),
output_size=output_size,
partitioners=self._partitioners)
if not self.is_connected:
self._linear_modules.append(linear_mod)
net = linear_mod(net)
if self._use_batch_norm:
bn = batch_norm.BatchNorm(**self._batch_norm_config)
net = bn(net, is_training, test_local_stats)
net = tf.nn.relu(net)
if keep_prob is not None:
net = tf.nn.dropout(net, keep_prob=keep_prob)
return net
def _build(self,
inputs,
keep_prob=None,
is_training=None,
test_local_stats=True):
"""Connects the AlexNet module into the graph.
The is_training flag only controls the batch norm settings, if `False` it
does not force no dropout by overriding any input `keep_prob`. To avoid any
confusion this may cause, if `is_training=False` and `keep_prob` would cause
dropout to be applied, an error is thrown.
Args:
inputs: A Tensor of size [batch_size, input_height, input_width,
input_channels], representing a batch of input images.
keep_prob: A scalar Tensor representing the dropout keep probability.
When `is_training=False` this must be None or 1 to give no dropout.
is_training: Boolean to indicate if we are currently training. Must be
specified if batch normalization or dropout is used.
test_local_stats: Boolean to indicate to `snt.BatchNorm` if batch
normalization should use local batch statistics at test time.
By default `True`.
Returns:
A Tensor of size [batch_size, output_size], where `output_size` depends
on the mode the network was constructed in.
Raises:
base.IncompatibleShapeError: If any of the input image dimensions
(input_height, input_width) are too small for the given network mode.
ValueError: If `keep_prob` is not None or 1 when `is_training=False`.
ValueError: If `is_training` is not explicitly specified when using
batch normalization.
"""
# Check input shape
if (self._use_batch_norm
or keep_prob is not None) and is_training is None:
raise ValueError(
"Boolean is_training flag must be explicitly specified "
"when using batch normalization or dropout.")
input_shape = inputs.get_shape().as_list()
if input_shape[1] < self._min_size or input_shape[2] < self._min_size:
raise base.IncompatibleShapeError(
"Image shape too small: ({:d}, {:d}) < {:d}".format(
input_shape[1], input_shape[2], self._min_size))
net = inputs
# Check keep prob
if keep_prob is not None:
valid_inputs = tf.logical_or(is_training, tf.equal(keep_prob, 1.))
keep_prob_check = tf.assert_equal(
valid_inputs,
True,
message=
"Input `keep_prob` must be None or 1 if `is_training=False`.")
with tf.control_dependencies([keep_prob_check]):
net = tf.identity(net)
for i, params in enumerate(self._conv_layers):
output_channels, conv_params, max_pooling = params
kernel_size, stride = conv_params
conv_mod = conv.Conv2D(name="conv_{}".format(i),
output_channels=output_channels,
kernel_shape=kernel_size,
stride=stride,
padding=conv.VALID,
initializers=self._initializers,
partitioners=self._partitioners,
regularizers=self._regularizers)
if not self.is_connected:
self._conv_modules.append(conv_mod)
net = conv_mod(net)
if self._use_batch_norm:
bn = batch_norm.BatchNorm(**self._batch_norm_config)
net = bn(net, is_training, test_local_stats)
net = tf.nn.relu(net)
if max_pooling is not None:
pooling_kernel_size, pooling_stride = max_pooling
net = tf.nn.max_pool(
net,
ksize=[1, pooling_kernel_size, pooling_kernel_size, 1],
strides=[1, pooling_stride, pooling_stride, 1],
padding=conv.VALID)
net = basic.BatchFlatten(name="flatten")(net)
for i, output_size in enumerate(self._fc_layers):
linear_mod = basic.Linear(name="fc_{}".format(i),
output_size=output_size,
initializers=self._initializers,
partitioners=self._partitioners)
if not self.is_connected:
self._linear_modules.append(linear_mod)
net = linear_mod(net)
if self._use_batch_norm and self._bn_on_fc_layers:
bn = batch_norm.BatchNorm(**self._batch_norm_config)
net = bn(net, is_training, test_local_stats)
net = tf.nn.relu(net)
if keep_prob is not None:
net = tf.nn.dropout(net, keep_prob=keep_prob)
return net
