def _dilated_conv_layer(self, output_channels, dilation_rate, apply_relu, name): """Create a dilated convolution layer. Args: output_channels: int. Number of output channels for each pixel. dilation_rate: int. Represents how many pixels each stride offset will move. A value of 1 indicates a standard convolution. apply_relu: bool. If True, a ReLU non-linearlity is added. name: string. Name for layer. Returns: a sonnet Module for a dilated convolution. """ layer_components = [ conv.Conv2D(output_channels, [3, 3], initializers=self._initializers, regularizers=self._regularizers, rate=dilation_rate, name="dilated_conv_" + name), ] if apply_relu: layer_components.append( lambda net: tf.nn.relu(net, name="relu_" + name)) return sequential.Sequential(layer_components, name=name)
def testScopeRestore(self): c1 = conv.Conv2D(16, 8, 4, name='conv_2d_0', padding=conv.VALID, initializers={ 'w': initializers.restore_initializer( _checkpoint(), 'w', scope='agent/conv_net_2d/conv_2d_0'), 'b': initializers.restore_initializer( _checkpoint(), 'b', scope='agent/conv_net_2d/conv_2d_0') }) inputs = tf.constant(1 / 255.0, shape=[1, 86, 86, 3]) outputs = c1(inputs) init = tf.global_variables_initializer() tf.get_default_graph().finalize() with self.test_session() as session: session.run(init) o = session.run(outputs) self.assertAllClose(np.linalg.norm(o), _ONE_CONV_LAYER, atol=_TOLERANCE)
def testMultipleRestore(self): g = tf.Graph() restore_initializers = { 'w': initializers.restore_initializer(_checkpoint(), 'w'), 'b': initializers.restore_initializer(_checkpoint(), 'b') } with g.as_default(): with tf.variable_scope('agent/conv_net_2d'): c1 = conv.Conv2D(16, 8, 4, name='conv_2d_0', padding=conv.VALID, initializers=restore_initializers) c2 = conv.Conv2D(32, 4, 2, name='conv_2d_1', padding=conv.VALID, initializers=restore_initializers) inputs = tf.constant(1 / 255.0, shape=[1, 86, 86, 3]) intermediate_1 = c1(inputs) intermediate_2 = c2(tf.nn.relu(intermediate_1)) outputs = tf.nn.relu(intermediate_2) init = tf.global_variables_initializer() tf.get_default_graph().finalize() with self.test_session() as session: session.run(init) i1, i2, o = session.run( [intermediate_1, intermediate_2, outputs]) self.assertAllClose(np.linalg.norm(i1), _ONE_CONV_LAYER, rtol=0.001, atol=0.001) self.assertAllClose(np.linalg.norm(i2), _TWO_CONV_LAYERS, rtol=0.001, atol=0.001) self.assertAllClose(np.linalg.norm(o), _TWO_CONV_LAYERS_RELU, rtol=0.001, atol=0.001)
def _instantiate_layers(self): """Instantiates all the convolutional modules used in the network.""" with self._enter_variable_scope(): self._layers = tuple(conv.Conv2D(name="conv_2d_{}".format(i), output_channels=self._output_channels[i], kernel_shape=self._kernel_shapes[i], stride=self._strides[i], padding=self._paddings[i], use_bias=self._use_bias[i], initializers=self._initializers, partitioners=self._partitioners, regularizers=self._regularizers) for i in xrange(self._num_layers))
def _instantiate_layers(self): """Instantiates all the convolutional modules used in the network.""" # Here we are entering the module's variable scope to name our submodules # correctly (not to create variables). As such it's safe to not check # whether we're in the same graph. This is important if we're constructing # the module in one graph and connecting it in another (e.g. with `defun` # the module is created in some default graph, and connected to a capturing # graph in order to turn it into a graph function). with self._enter_variable_scope(check_same_graph=False): self._layers = tuple(conv.Conv2D(name="conv_2d_{}".format(i), output_channels=self._output_channels[i], kernel_shape=self._kernel_shapes[i], stride=self._strides[i], rate=self._rates[i], padding=self._paddings[i], use_bias=self._use_bias[i], initializers=self._initializers, partitioners=self._partitioners, regularizers=self._regularizers, data_format=self._data_format) for i in xrange(self._num_layers))
def _build(self, inputs, keep_prob=None, is_training=True, test_local_stats=True): """Connects the AlexNet module into the graph. Args: inputs: A Tensor of size [batch_size, input_height, input_width, input_channels], representing a batch of input images. keep_prob: A scalar Tensor representing the dropout keep probability. is_training: Boolean to indicate to `snt.BatchNorm` if we are currently training. By default `True`. test_local_stats: Boolean to indicate to `snt.BatchNorm` if batch normalization should use local batch statistics at test time. By default `True`. Returns: A Tensor of size [batch_size, output_size], where `output_size` depends on the mode the network was constructed in. Raises: base.IncompatibleShapeError: If any of the input image dimensions (input_height, input_width) are too small for the given network mode. """ input_shape = inputs.get_shape().as_list() if input_shape[1] < self._min_size or input_shape[2] < self._min_size: raise base.IncompatibleShapeError( "Image shape too small: ({:d}, {:d}) < {:d}".format( input_shape[1], input_shape[2], self._min_size)) net = inputs for i, params in enumerate(self._conv_layers): output_channels, conv_params, max_pooling = params kernel_size, stride = conv_params conv_mod = conv.Conv2D(name="conv_{}".format(i), output_channels=output_channels, kernel_shape=kernel_size, stride=stride, padding=conv.VALID, initializers=self._initializers, partitioners=self._partitioners, regularizers=self._regularizers) if not self.is_connected: self._conv_modules.append(conv_mod) net = conv_mod(net) if self._use_batch_norm: bn = batch_norm.BatchNorm(**self._batch_norm_config) net = bn(net, is_training, test_local_stats) net = tf.nn.relu(net) if max_pooling is not None: pooling_kernel_size, pooling_stride = max_pooling net = tf.nn.max_pool( net, ksize=[1, pooling_kernel_size, pooling_kernel_size, 1], strides=[1, pooling_stride, pooling_stride, 1], padding=conv.VALID) net = basic.BatchFlatten(name="flatten")(net) for i, output_size in enumerate(self._fc_layers): linear_mod = basic.Linear(name="fc_{}".format(i), output_size=output_size, partitioners=self._partitioners) if not self.is_connected: self._linear_modules.append(linear_mod) net = linear_mod(net) if self._use_batch_norm: bn = batch_norm.BatchNorm(**self._batch_norm_config) net = bn(net, is_training, test_local_stats) net = tf.nn.relu(net) if keep_prob is not None: net = tf.nn.dropout(net, keep_prob=keep_prob) return net
def _build(self, inputs, keep_prob=None, is_training=None, test_local_stats=True): """Connects the AlexNet module into the graph. The is_training flag only controls the batch norm settings, if `False` it does not force no dropout by overriding any input `keep_prob`. To avoid any confusion this may cause, if `is_training=False` and `keep_prob` would cause dropout to be applied, an error is thrown. Args: inputs: A Tensor of size [batch_size, input_height, input_width, input_channels], representing a batch of input images. keep_prob: A scalar Tensor representing the dropout keep probability. When `is_training=False` this must be None or 1 to give no dropout. is_training: Boolean to indicate if we are currently training. Must be specified if batch normalization or dropout is used. test_local_stats: Boolean to indicate to `snt.BatchNorm` if batch normalization should use local batch statistics at test time. By default `True`. Returns: A Tensor of size [batch_size, output_size], where `output_size` depends on the mode the network was constructed in. Raises: base.IncompatibleShapeError: If any of the input image dimensions (input_height, input_width) are too small for the given network mode. ValueError: If `keep_prob` is not None or 1 when `is_training=False`. ValueError: If `is_training` is not explicitly specified when using batch normalization. """ # Check input shape if (self._use_batch_norm or keep_prob is not None) and is_training is None: raise ValueError( "Boolean is_training flag must be explicitly specified " "when using batch normalization or dropout.") input_shape = inputs.get_shape().as_list() if input_shape[1] < self._min_size or input_shape[2] < self._min_size: raise base.IncompatibleShapeError( "Image shape too small: ({:d}, {:d}) < {:d}".format( input_shape[1], input_shape[2], self._min_size)) net = inputs # Check keep prob if keep_prob is not None: valid_inputs = tf.logical_or(is_training, tf.equal(keep_prob, 1.)) keep_prob_check = tf.assert_equal( valid_inputs, True, message= "Input `keep_prob` must be None or 1 if `is_training=False`.") with tf.control_dependencies([keep_prob_check]): net = tf.identity(net) for i, params in enumerate(self._conv_layers): output_channels, conv_params, max_pooling = params kernel_size, stride = conv_params conv_mod = conv.Conv2D(name="conv_{}".format(i), output_channels=output_channels, kernel_shape=kernel_size, stride=stride, padding=conv.VALID, initializers=self._initializers, partitioners=self._partitioners, regularizers=self._regularizers) if not self.is_connected: self._conv_modules.append(conv_mod) net = conv_mod(net) if self._use_batch_norm: bn = batch_norm.BatchNorm(**self._batch_norm_config) net = bn(net, is_training, test_local_stats) net = tf.nn.relu(net) if max_pooling is not None: pooling_kernel_size, pooling_stride = max_pooling net = tf.nn.max_pool( net, ksize=[1, pooling_kernel_size, pooling_kernel_size, 1], strides=[1, pooling_stride, pooling_stride, 1], padding=conv.VALID) net = basic.BatchFlatten(name="flatten")(net) for i, output_size in enumerate(self._fc_layers): linear_mod = basic.Linear(name="fc_{}".format(i), output_size=output_size, initializers=self._initializers, partitioners=self._partitioners) if not self.is_connected: self._linear_modules.append(linear_mod) net = linear_mod(net) if self._use_batch_norm and self._bn_on_fc_layers: bn = batch_norm.BatchNorm(**self._batch_norm_config) net = bn(net, is_training, test_local_stats) net = tf.nn.relu(net) if keep_prob is not None: net = tf.nn.dropout(net, keep_prob=keep_prob) return net