def bottleneck(inputs,
depth,
depth_bottleneck,
stride,
rate=1,
outputs_collections=None,
scope=None):
"""Bottleneck residual unit variant with BN before convolutions."""
with variable_scope.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc:
depth_in = utils.last_dimension(inputs.get_shape(), min_rank=3)
preact = layers.batch_norm(inputs,
activation_fn=nn_ops.relu,
scope='preact')
if depth == depth_in:
shortcut = subsample(inputs, stride, 'shortcut')
else:
shortcut = layers.convolution(preact,
depth,
1,
stride=stride,
normalizer_fn=None,
activation_fn=None,
scope='shortcut')
residual = layers.convolution(preact,
depth_bottleneck,
1,
stride=1,
scope='conv1')
residual = conv1d_same(residual,
depth_bottleneck,
3,
stride,
rate=rate,
scope='conv2')
residual = layers.convolution(residual,
depth,
1,
stride=1,
normalizer_fn=None,
activation_fn=None,
scope='conv3')
output = shortcut + residual
return utils.collect_named_outputs(outputs_collections, sc.name,
output)
def conv1d_same(inputs, num_outputs, kernel_size, stride, rate=1, scope=None):
"""Strided 1-D convolution with 'SAME' padding."""
if stride == 1:
return layers.convolution(inputs,
num_outputs,
kernel_size,
stride=1,
rate=rate,
padding='SAME',
scope=scope)
else:
kernel_size_effective = kernel_size + (kernel_size - 1) * (rate - 1)
pad_total = kernel_size_effective - 1
pad_beg = pad_total // 2
pad_end = pad_total - pad_beg
inputs = array_ops.pad(inputs, [[0, 0], [pad_beg, pad_end], [0, 0]])
return layers.convolution(inputs,
num_outputs,
kernel_size,
stride=stride,
rate=rate,
padding='VALID',
scope=scope)
def test_conv_layer(self):
with compat.forward_compatibility_horizon(2019, 11, 11):
g = ops.Graph()
with g.as_default():
inputs = array_ops.placeholder(dtypes.float32,
shape=[8, 5, 5, 3])
with contrib_ops.arg_scope([layers.batch_norm],
fused=True,
is_training=True,
trainable=True):
return layers.convolution(
inputs,
num_outputs=16,
kernel_size=3,
stride=1,
padding='VALID',
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params={},
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
trainable=True,
scope=None)
inputs_pattern = graph_matcher.OpTypePattern('*', name='inputs')
relu_pattern = graph_matcher.OpTypePattern(
'Relu',
name='relu',
inputs=[
graph_matcher.OpTypePattern(
'FusedBatchNormV3',
inputs=[
graph_matcher.OpTypePattern(
'Conv2D', inputs=[inputs_pattern, '*']), '*',
'*', '*', '*'
])
])
matcher = graph_matcher.GraphMatcher(relu_pattern)
match_results = list(matcher.match_graph(g))
self.assertEqual(1, len(match_results))
match_result = match_results[0]
self.assertEqual(match_result.get_tensor(inputs_pattern), inputs)
self.assertEqual(match_result.get_tensor('inputs'), inputs)
def test_conv_layer(self):
g = ops.Graph()
with g.as_default():
inputs = array_ops.placeholder(dtypes.float32, shape=[8, 5, 5, 3])
with contrib_ops.arg_scope(
[layers.batch_norm], fused=True, is_training=True, trainable=True):
return layers.convolution(
inputs,
num_outputs=16,
kernel_size=3,
stride=1,
padding='VALID',
activation_fn=nn_ops.relu,
normalizer_fn=layers.batch_norm,
normalizer_params={},
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer(),
biases_regularizer=None,
reuse=None,
trainable=True,
scope=None)
inputs_pattern = graph_matcher.OpTypePattern('*', name='inputs')
relu_pattern = graph_matcher.OpTypePattern(
'Relu',
name='relu',
inputs=[
graph_matcher.OpTypePattern(
'FusedBatchNorm',
inputs=[
graph_matcher.OpTypePattern(
'Conv2D', inputs=[inputs_pattern, '*']), '*', '*', '*',
'*'
])
])
matcher = graph_matcher.GraphMatcher(relu_pattern)
match_results = list(matcher.match_graph(g))
self.assertEqual(1, len(match_results))
match_result = match_results[0]
self.assertEqual(match_result.get_tensor(inputs_pattern), inputs)
self.assertEqual(match_result.get_tensor('inputs'), inputs)
def rnn(self,
sequence,
sequence_length,
max_length,
dropout,
batch_size,
training,
num_hidden=TC_MODEL_HIDDEN,
num_layers=TC_MODEL_LAYERS,
cnn_filters=TC_CNN_FILTERS,
cnn_layers=TC_CNN_LAYERS):
conv_sequence = sequence
for _ in range(cnn_layers):
conv_sequence = layers.convolution(conv_sequence,
cnn_filters, [5],
activation_fn=None)
# Recurrent network.
cells = []
for _ in range(num_layers):
cell = tf.nn.rnn_cell.GRUCell(num_hidden)
if training:
cell = tf.nn.rnn_cell.DropoutWrapper(cell,
output_keep_prob=dropout)
cells.append(cell)
network = tf.nn.rnn_cell.MultiRNNCell(cells)
type = sequence.dtype
sequence_output, _ = tf.nn.dynamic_rnn(
network,
conv_sequence,
dtype=tf.float32,
sequence_length=sequence_length,
initial_state=network.zero_state(batch_size, type))
# get last output of the dynamic_rnn
sequence_output = tf.reshape(sequence_output,
[batch_size * max_length, num_hidden])
indexes = tf.range(batch_size) * max_length + (sequence_length - 1)
output = tf.gather(sequence_output, indexes)
return output
def resnet_v2(inputs,
blocks,
num_classes=None,
is_training=None,
global_pool=True,
output_stride=None,
include_root_block=True,
reuse=None,
scope=None):
with variable_scope.variable_scope(scope,
'resnet_v2', [inputs],
reuse=reuse) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
with arg_scope(
[layers.convolution, bottleneck, resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
if is_training is not None:
bn_scope = arg_scope([layers.batch_norm],
is_training=is_training)
else:
bn_scope = arg_scope([])
with bn_scope:
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
# We do not include batch normalization or activation functions in
# conv1 because the first ResNet unit will perform these. Cf.
# Appendix of [2].
with arg_scope([layers.convolution],
activation_fn=None,
normalizer_fn=None):
net = conv1d_same(net, 64, 7, stride=2, scope='conv1')
net = max_pool1d(net, 3, stride=2, scope='pool1')
net = resnet_utils.stack_blocks_dense(net, blocks,
output_stride)
# This is needed because the pre-activation variant does not have batch
# normalization or activation functions in the residual unit output. See
# Appendix of [2].
net = layers.batch_norm(net,
activation_fn=nn_ops.relu,
scope='postnorm')
if global_pool:
# Global average pooling.
net = math_ops.reduce_mean(net, [1],
name='pool5',
keep_dims=True)
if num_classes is not None:
net = layers.convolution(net,
num_classes,
1,
activation_fn=None,
normalizer_fn=None,
scope='logits')
net = tf.squeeze(net)
# Convert end_points_collection into a dictionary of end_points.
end_points = utils.convert_collection_to_dict(
end_points_collection)
if num_classes is not None:
end_points['predictions'] = layers.softmax(
net, scope='predictions')
return net, end_points