def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, outputs_collections=None, scope=None): """Bottleneck residual unit variant with BN before convolutions.""" with variable_scope.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc: depth_in = utils.last_dimension(inputs.get_shape(), min_rank=3) preact = layers.batch_norm(inputs, activation_fn=nn_ops.relu, scope='preact') if depth == depth_in: shortcut = subsample(inputs, stride, 'shortcut') else: shortcut = layers.convolution(preact, depth, 1, stride=stride, normalizer_fn=None, activation_fn=None, scope='shortcut') residual = layers.convolution(preact, depth_bottleneck, 1, stride=1, scope='conv1') residual = conv1d_same(residual, depth_bottleneck, 3, stride, rate=rate, scope='conv2') residual = layers.convolution(residual, depth, 1, stride=1, normalizer_fn=None, activation_fn=None, scope='conv3') output = shortcut + residual return utils.collect_named_outputs(outputs_collections, sc.name, output)
def conv1d_same(inputs, num_outputs, kernel_size, stride, rate=1, scope=None): """Strided 1-D convolution with 'SAME' padding.""" if stride == 1: return layers.convolution(inputs, num_outputs, kernel_size, stride=1, rate=rate, padding='SAME', scope=scope) else: kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1) pad_total = kernel_size_effective - 1 pad_beg = pad_total // 2 pad_end = pad_total - pad_beg inputs = array_ops.pad(inputs, [[0, 0], [pad_beg, pad_end], [0, 0]]) return layers.convolution(inputs, num_outputs, kernel_size, stride=stride, rate=rate, padding='VALID', scope=scope)
def test_conv_layer(self): with compat.forward_compatibility_horizon(2019, 11, 11): g = ops.Graph() with g.as_default(): inputs = array_ops.placeholder(dtypes.float32, shape=[8, 5, 5, 3]) with contrib_ops.arg_scope([layers.batch_norm], fused=True, is_training=True, trainable=True): return layers.convolution( inputs, num_outputs=16, kernel_size=3, stride=1, padding='VALID', activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params={}, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, trainable=True, scope=None) inputs_pattern = graph_matcher.OpTypePattern('*', name='inputs') relu_pattern = graph_matcher.OpTypePattern( 'Relu', name='relu', inputs=[ graph_matcher.OpTypePattern( 'FusedBatchNormV3', inputs=[ graph_matcher.OpTypePattern( 'Conv2D', inputs=[inputs_pattern, '*']), '*', '*', '*', '*' ]) ]) matcher = graph_matcher.GraphMatcher(relu_pattern) match_results = list(matcher.match_graph(g)) self.assertEqual(1, len(match_results)) match_result = match_results[0] self.assertEqual(match_result.get_tensor(inputs_pattern), inputs) self.assertEqual(match_result.get_tensor('inputs'), inputs)
def test_conv_layer(self): g = ops.Graph() with g.as_default(): inputs = array_ops.placeholder(dtypes.float32, shape=[8, 5, 5, 3]) with contrib_ops.arg_scope( [layers.batch_norm], fused=True, is_training=True, trainable=True): return layers.convolution( inputs, num_outputs=16, kernel_size=3, stride=1, padding='VALID', activation_fn=nn_ops.relu, normalizer_fn=layers.batch_norm, normalizer_params={}, weights_initializer=initializers.xavier_initializer(), weights_regularizer=None, biases_initializer=init_ops.zeros_initializer(), biases_regularizer=None, reuse=None, trainable=True, scope=None) inputs_pattern = graph_matcher.OpTypePattern('*', name='inputs') relu_pattern = graph_matcher.OpTypePattern( 'Relu', name='relu', inputs=[ graph_matcher.OpTypePattern( 'FusedBatchNorm', inputs=[ graph_matcher.OpTypePattern( 'Conv2D', inputs=[inputs_pattern, '*']), '*', '*', '*', '*' ]) ]) matcher = graph_matcher.GraphMatcher(relu_pattern) match_results = list(matcher.match_graph(g)) self.assertEqual(1, len(match_results)) match_result = match_results[0] self.assertEqual(match_result.get_tensor(inputs_pattern), inputs) self.assertEqual(match_result.get_tensor('inputs'), inputs)
def rnn(self, sequence, sequence_length, max_length, dropout, batch_size, training, num_hidden=TC_MODEL_HIDDEN, num_layers=TC_MODEL_LAYERS, cnn_filters=TC_CNN_FILTERS, cnn_layers=TC_CNN_LAYERS): conv_sequence = sequence for _ in range(cnn_layers): conv_sequence = layers.convolution(conv_sequence, cnn_filters, [5], activation_fn=None) # Recurrent network. cells = [] for _ in range(num_layers): cell = tf.nn.rnn_cell.GRUCell(num_hidden) if training: cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=dropout) cells.append(cell) network = tf.nn.rnn_cell.MultiRNNCell(cells) type = sequence.dtype sequence_output, _ = tf.nn.dynamic_rnn( network, conv_sequence, dtype=tf.float32, sequence_length=sequence_length, initial_state=network.zero_state(batch_size, type)) # get last output of the dynamic_rnn sequence_output = tf.reshape(sequence_output, [batch_size * max_length, num_hidden]) indexes = tf.range(batch_size) * max_length + (sequence_length - 1) output = tf.gather(sequence_output, indexes) return output
def resnet_v2(inputs, blocks, num_classes=None, is_training=None, global_pool=True, output_stride=None, include_root_block=True, reuse=None, scope=None): with variable_scope.variable_scope(scope, 'resnet_v2', [inputs], reuse=reuse) as sc: end_points_collection = sc.original_name_scope + '_end_points' with arg_scope( [layers.convolution, bottleneck, resnet_utils.stack_blocks_dense], outputs_collections=end_points_collection): if is_training is not None: bn_scope = arg_scope([layers.batch_norm], is_training=is_training) else: bn_scope = arg_scope([]) with bn_scope: net = inputs if include_root_block: if output_stride is not None: if output_stride % 4 != 0: raise ValueError( 'The output_stride needs to be a multiple of 4.' ) output_stride /= 4 # We do not include batch normalization or activation functions in # conv1 because the first ResNet unit will perform these. Cf. # Appendix of [2]. with arg_scope([layers.convolution], activation_fn=None, normalizer_fn=None): net = conv1d_same(net, 64, 7, stride=2, scope='conv1') net = max_pool1d(net, 3, stride=2, scope='pool1') net = resnet_utils.stack_blocks_dense(net, blocks, output_stride) # This is needed because the pre-activation variant does not have batch # normalization or activation functions in the residual unit output. See # Appendix of [2]. net = layers.batch_norm(net, activation_fn=nn_ops.relu, scope='postnorm') if global_pool: # Global average pooling. net = math_ops.reduce_mean(net, [1], name='pool5', keep_dims=True) if num_classes is not None: net = layers.convolution(net, num_classes, 1, activation_fn=None, normalizer_fn=None, scope='logits') net = tf.squeeze(net) # Convert end_points_collection into a dictionary of end_points. end_points = utils.convert_collection_to_dict( end_points_collection) if num_classes is not None: end_points['predictions'] = layers.softmax( net, scope='predictions') return net, end_points