def forward(inputs, num_outputs, is_training=True, scope=None):
with tf.variable_scope(scope, 'resnet_v2_50', [inputs], reuse=tf.AUTO_REUSE):
with slim.arg_scope([slim.batch_norm], is_training=is_training):
# root_block
with slim.arg_scope([slim.conv2d],
activation_fn=None, normalizer_fn=None):
net = conv2d_same(inputs, 64, 7, stride=2, scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = resnet_v2_block(
net, base_depth=64, num_units=3, stride=2, scope='block1')
net = resnet_v2_block(
net, base_depth=128, num_units=4, stride=2, scope='block2')
net = resnet_v2_block(
net, base_depth=256, num_units=6, stride=2, scope='block3')
net = resnet_v2_block(
net, base_depth=512, num_units=3, stride=1, scope='block4')
net = slim.batch_norm(
net, activation_fn=tf.nn.relu, scope='postnorm')
net = slim.conv2d(net, num_outputs, [1, 1],
activation_fn=None, normalizer_fn=None,
scope='_logits_')
return net
def _building_block(inputs):
depth_in = slim.utils.last_dimension(inputs.get_shape(),
min_rank=4)
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(inputs,
depth, [1, 1],
stride=stride,
activation_fn=tf.nn.relu6
if use_bounded_activations else None,
scope='shortcut')
residual = resnet_utils.conv2d_same(inputs,
depth,
3,
stride=1,
rate=rate,
scope='conv1')
residual = resnet_utils.conv2d_same_act(residual,
depth,
3,
stride=stride,
rate=rate,
scope='conv2')
if use_bounded_activations:
# Use clip_by_value to simulate bandpass activation.
residual = tf.clip_by_value(residual, -6.0, 6.0)
output = tf.nn.relu6(shortcut + residual)
else:
output = tf.nn.relu(shortcut + residual)
return output
def bottleneck(inputs,
depth,
depth_bottleneck,
stride,
rate=1,
outputs_collections=None,
scope=None):
"""Bottleneck residual unit variant with BN after convolutions.
This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for
its definition. Note that we use here the bottleneck variant which has an
extra bottleneck layer.
When putting together two consecutive ResNet blocks that use this unit, one
should use stride = 2 in the last unit of the first block.
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The depth of the ResNet unit output.
depth_bottleneck: The depth of the bottleneck layers.
stride: The ResNet unit's stride. Determines the amount of downsampling of
the units output compared to its input.
rate: An integer, rate for atrous convolution.
outputs_collections: Collection to add the ResNet unit output.
scope: Optional variable_scope.
Returns:
The ResNet unit's output.
"""
with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(inputs,
depth, [1, 1],
stride=stride,
activation_fn=None,
scope='shortcut')
residual = slim.conv2d(inputs,
depth_bottleneck, [1, 1],
stride=1,
scope='conv1')
residual = resnet_utils.conv2d_same(residual,
depth_bottleneck,
3,
stride,
rate=rate,
scope='conv2')
residual = slim.conv2d(residual,
depth, [1, 1],
stride=1,
activation_fn=None,
scope='conv3')
output = tf.nn.relu(shortcut + residual)
return slim.utils.collect_named_outputs(outputs_collections,
sc.original_name_scope, output)
def att_resnet_v1(inputs,
blocks,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
store_non_strided_activations=False,
reuse=None,
scope=None):
with tf.variable_scope(scope, 'att_resnet_v1', [inputs],
reuse=reuse) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
with slim.arg_scope(
[slim.conv2d, bottleneck, stack_attention_blocks_dense],
outputs_collections=end_points_collection):
with (slim.arg_scope([slim.batch_norm], is_training=is_training)
if is_training is not None else NoOpScope()):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = stack_attention_blocks_dense(
net, blocks, output_stride, store_non_strided_activations)
# Convert end_points_collection into a dictionary of end_points.
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if global_pool:
# Global average pooling.
net = tf.reduce_mean(net, [1, 2],
name='pool5',
keep_dims=True)
end_points['global_pool'] = net
if num_classes:
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='logits')
end_points[sc.name + '/logits'] = net
if spatial_squeeze:
net = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
end_points[sc.name + '/spatial_squeeze'] = net
end_points['predictions'] = slim.softmax(
net, scope='predictions')
return net, end_points
def h1():
temp = resnet_utils.conv2d_same(residual,
depth_bottleneck,
3,
stride,
rate=rate,
scope='conv2')
return temp
def bottleneck(inputs, depth, depth_bottleneck, stride, EPSILON=2.0, middle=False, num_classes=1001, rate=1,
outputs_collections=None, scope=None):
"""Bottleneck residual unit variant with BN before convolutions.
This is the full preactivation residual unit variant proposed in [2]. See
Fig. 1(b) of [2] for its definition. Note that we use here the bottleneck
variant which has an extra bottleneck layer.
When putting together two consecutive ResNet blocks that use this unit, one
should use stride = 2 in the last unit of the first block.
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The depth of the ResNet unit output.
depth_bottleneck: The depth of the bottleneck layers.
stride: The ResNet unit's stride. Determines the amount of downsampling of
the units output compared to its input.
rate: An integer, rate for atrous convolution.
outputs_collections: Collection to add the ResNet unit output.
scope: Optional variable_scope.
Returns:
The ResNet unit's output.
"""
global preds, side_output
with tf.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact')
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(preact, depth, [1, 1], stride=stride,
normalizer_fn=None, activation_fn=None,
scope='shortcut')
residual = slim.conv2d(preact, depth_bottleneck, [1, 1], stride=1,
scope='conv1')
residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride,
rate=rate, scope='conv2')
residual = slim.conv2d(residual, depth, [1, 1], stride=1,
normalizer_fn=None, activation_fn=None,
scope='conv3')
identity_w = strict_identity(residual, EPSILON)
is_discarded = tf.subtract(1.0, identity_w, 'is_discarded')
preds.append(is_discarded)
output = shortcut + residual*identity_w
if middle:
side_output = side_branch(output, num_classes)
return slim.utils.collect_named_outputs(outputs_collections,
sc.name,
output)
def bottleneck_normal(inputs,
depth,
depth_bottleneck,
stride,
nr_frames=None,
rate=1,
outputs_collections=None,
scope=None,
use_bounded_activations=False,
temporal=False):
with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(
inputs,
depth, [1, 1],
stride=stride,
activation_fn=tf.nn.relu6 if use_bounded_activations else None,
scope='shortcut')
residual = slim.conv2d(inputs,
depth_bottleneck, [1, 1],
stride=1,
scope='conv1')
residual = resnet_utils.conv2d_same(residual,
depth_bottleneck,
3,
stride,
rate=rate,
scope='conv2')
if temporal: # temporal conv
residual = resnet_utils.conv_temp2(residual,
nr_frames,
scope='conv_temp')
residual = slim.conv2d(residual,
depth, [1, 1],
stride=1,
activation_fn=None,
scope='conv3')
if use_bounded_activations:
# Use clip_by_value to simulate bandpass activation.
residual = tf.clip_by_value(residual, -6.0, 6.0)
output = tf.nn.relu6(shortcut + residual)
else:
output = tf.nn.relu(shortcut + residual)
return slim.utils.collect_named_outputs(outputs_collections,
sc.original_name_scope, output)
def resnet_v2_multiple(inputs,
blocks,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
reuse=None,
scope=None):
with tf.variable_scope(scope, 'resnet_v2', [inputs], reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope([slim.conv2d, bottleneck,
resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm], is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError('The output_stride needs to be a multiple of 4.')
output_stride /= 4
# We do not include batch normalization or activation functions in
# conv1 because the first ResNet unit will perform these. Cf.
# Appendix of [2].
with slim.arg_scope([slim.conv2d],
activation_fn=None, normalizer_fn=None):
net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = resnet_utils.stack_blocks_dense(net, blocks, output_stride)
# This is needed because the pre-activation variant does not have batch
# normalization or activation functions in the residual unit output. See
# Appendix of [2].
net = slim.batch_norm(net, activation_fn=tf.nn.relu, scope='postnorm')
if global_pool:
# Global average pooling.
net = tf.reduce_mean(net, [1, 2], name='pool5', keep_dims=True)
if num_classes is not None:
net = slim.conv2d(net, num_classes*3, [1, 1], activation_fn=None,
normalizer_fn=None, scope='logits')
if spatial_squeeze:
logits = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
else:
logits = net
# pdb.set_trace()
logits = tf.reshape(logits,(logits.get_shape().as_list()[0],num_classes,-1))
# Convert end_points_collection into a dictionary of end_points.
end_points = slim.utils.convert_collection_to_dict(end_points_collection)
if num_classes is not None:
end_points['predictions_0'] = slim.softmax(logits[:,:,0], scope='predictions')
end_points['predictions_1'] = slim.softmax(logits[:,:,1], scope='predictions')
end_points['predictions_2'] = slim.softmax(logits[:,:,2], scope='predictions')
return logits, end_points
def resnet_v1_pruned(inputs,
blocks,
num_classes=None,
prune_info = None,
is_training=True,
is_local_train = None,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
reuse=None,
scope=None):
"""Generator for v1 ResNet models.
Raises:
ValueError: If the target output_stride is not valid.
"""
# print('resnet_v1_pruned scope=', scope)
with tf.variable_scope(scope, 'resnet_v1', [inputs], reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope([slim.conv2d, bottleneck, slim.max_pool2d,
resnet_utils.stack_blocks_dense_pruned],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm], is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError('The output_stride needs to be a multiple of 4.')
output_stride /= 4
net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = resnet_utils.stack_blocks_dense_pruned(net, blocks,
prune_info=prune_info,
is_local_train=is_local_train,
output_stride = output_stride)
if global_pool:
# Global average pooling.
net = tf.reduce_mean(net, [1, 2], name='pool5', keep_dims=True)
if num_classes is not None:
net = slim.conv2d(net, num_classes, [1, 1], activation_fn=None,
normalizer_fn=None, scope='logits')
if spatial_squeeze:
net = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
# Convert end_points_collection into a dictionary of end_points.
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if num_classes is not None:
end_points['predictions'] = slim.softmax(net, scope='predictions')
return net, end_points
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1, scope=None):
"""Bottleneck residual unit variant with BN before convolutions.
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The depth of the ResNet unit output.
depth_bottleneck: The depth of the bottleneck layers.
stride: The ResNet unit's stride. Determines the amount of downsampling of
the units output compared to its input.
rate: An integer, rate for atrous convolution.
scope: Optional variable_scope.
Returns:
The ResNet unit's output.
"""
with tf.variable_scope(scope, 'bottleneck_v2', [inputs]):
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
preact = slim.batch_norm(inputs,
activation_fn=tf.nn.relu,
scope='preact')
if depth == depth_in:
shortcut = subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(preact,
depth, [1, 1],
stride=stride,
normalizer_fn=None,
activation_fn=None,
scope='shortcut')
residual = slim.conv2d(preact,
depth_bottleneck, [1, 1],
stride=1,
scope='conv1')
residual = conv2d_same(residual,
depth_bottleneck,
3,
stride=stride,
rate=rate,
scope='conv2')
residual = slim.conv2d(residual,
depth, [1, 1],
stride=1,
normalizer_fn=None,
activation_fn=None,
scope='conv3')
output = shortcut + residual
return output
def bottleneck(inputs, depth, depth_bottleneck, stride, rate=1,
outputs_collections=None, scope=None):
"""Bottleneck residual unit variant with BN before convolutions.
This is the full preactivation residual unit variant proposed in [2]. See
Fig. 1(b) of [2] for its definition. Note that we use here the bottleneck
variant which has an extra bottleneck layer.
When putting together two consecutive ResNet blocks that use this unit, one
should use stride = 2 in the last unit of the first block.
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The depth of the ResNet unit output.
depth_bottleneck: The depth of the bottleneck layers.
stride: The ResNet unit's stride. Determines the amount of downsampling of
the units output compared to its input.
rate: An integer, rate for atrous convolution.
outputs_collections: Collection to add the ResNet unit output.
scope: Optional variable_scope.
Returns:
The ResNet unit's output.
"""
with tf.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact')
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(preact, depth, [1, 1], stride=stride,
normalizer_fn=None, activation_fn=None,
scope='shortcut')
residual = slim.conv2d(preact, depth_bottleneck, [1, 1], stride=1,
scope='conv1')
residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride,
rate=rate, scope='conv2')
residual = slim.conv2d(residual, depth, [1, 1], stride=1,
normalizer_fn=None, activation_fn=None,
scope='conv3')
output = shortcut + residual
return slim.utils.collect_named_outputs(outputs_collections,
sc.original_name_scope,
output)
def resnet_v2(inputs,
blocks,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
reuse=None,
scope=None):
with tf.variable_scope(scope, 'resnet_v2', [inputs], reuse=reuse) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
with slim.arg_scope(
[slim.conv2d, bottleneck, resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm], is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
# We do not include batch normalization or activation functions in
# conv1 because the first ResNet unit will perform these. Cf.
# Appendix of [2].
with slim.arg_scope([slim.conv2d],
activation_fn=None,
normalizer_fn=None):
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = resnet_utils.stack_blocks_dense(net, blocks,
output_stride)
# This is needed because the pre-activation variant does not have batch
# normalization or activation functions in the residual unit output. See
# Appendix of [2].
net = slim.batch_norm(net,
activation_fn=tf.nn.relu,
scope='postnorm')
def bottleneck(inputs,
depth,
depth_bottleneck,
stride,
rate=1,
outputs_collections=None,
scope=None):
with tf.variable_scope(scope, 'bottleneck_v2', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
preact = slim.batch_norm(inputs,
activation_fn=tf.nn.relu,
scope='preact')
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(preact,
depth, [1, 1],
stride=stride,
normalizer_fn=None,
activation_fn=None,
scope='shortcut')
residual = slim.conv2d(preact,
depth_bottleneck, [1, 1],
stride=1,
scope='conv1')
residual = resnet_utils.conv2d_same(residual,
depth_bottleneck,
3,
stride,
rate=rate,
scope='conv2')
residual = slim.conv2d(residual,
depth, [1, 1],
stride=1,
normalizer_fn=None,
activation_fn=None,
scope='conv3')
output = shortcut + residual
return slim.utils.collect_named_outputs(outputs_collections, sc.name,
output)
def testConv2DSameEven(self):
n, n2 = 4, 2
# Input image.
x = create_test_input(1, n, n, 1)
# Convolution kernel.
w = create_test_input(1, 3, 3, 1)
w = tf.reshape(w, [3, 3, 1, 1])
tf.get_variable('Conv/weights', initializer=w)
tf.get_variable('Conv/biases', initializer=tf.zeros([1]))
tf.get_variable_scope().reuse_variables()
y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv')
y1_expected = tf.to_float([[14, 28, 43, 26],
[28, 48, 66, 37],
[43, 66, 84, 46],
[26, 37, 46, 22]])
y1_expected = tf.reshape(y1_expected, [1, n, n, 1])
y2 = resnet_utils.subsample(y1, 2)
y2_expected = tf.to_float([[14, 43],
[43, 84]])
y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1])
y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv')
y3_expected = y2_expected
y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv')
y4_expected = tf.to_float([[48, 37],
[37, 22]])
y4_expected = tf.reshape(y4_expected, [1, n2, n2, 1])
with self.test_session() as sess:
try:
sess.run(tf.global_variables_initializer())
except AttributeError:
sess.run(tf.initialize_all_variables())
self.assertAllClose(y1.eval(), y1_expected.eval())
self.assertAllClose(y2.eval(), y2_expected.eval())
self.assertAllClose(y3.eval(), y3_expected.eval())
self.assertAllClose(y4.eval(), y4_expected.eval())
def split(inputs, unit_depth, stride, rate=1):
"""
The split structure in Figure 3b of the paper. It takes an input tensor. Conv it by [1, 1,
64] filter, and then conv the result by [3, 3, 64]. Return the
final resulted tensor, which is in shape of [batch_size, input_height, input_width, 64]
:param inputs: 4D tensor in shape of [batch_size, input_height, input_width,
input_channel]
:param unit_depth: the depth of each split
:param stride: int. 1 or 2. If want to shrink the image size, then stride = 2
:return: 4D tensor in shape of [batch_size, input_height, input_width, input_channel/64]
"""
num_filter = unit_depth
with tf.variable_scope('bneck_reduce_size'):
conv = slim.conv2d(inputs, num_filter, [1, 1], stride=1)
with tf.variable_scope('bneck_conv'):
conv = resnet_utils.conv2d_same(conv, num_filter, 3, stride=stride, rate=rate)
return conv
def bottleneck_c(inputs, unit_depth, cardinality, stride, rate=1,
outputs_collections=None, scope=None):
with tf.variable_scope(scope, 'bottleneck_resnext_c', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
preact = slim.batch_norm(inputs, activation_fn=tf.nn.relu, scope='preact')
depth = unit_depth * cardinality * 2
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(preact, depth, [1, 1], stride=stride,
normalizer_fn=None, activation_fn=None,
scope='shortcut')
net = slim.conv2d(inputs, unit_depth*cardinality, [1, 1], stride=1, scope='conv1')
net = resnet_utils.conv2d_same(net, unit_depth*cardinality, 3, stride=stride, rate=rate, scope='grouped_conv2')
net = slim.conv2d(net, depth, [1, 1], stride=1, scope='conv3')
net = shortcut + net
output = tf.nn.relu(net)
return slim.utils.collect_named_outputs(outputs_collections,
sc.original_name_scope,
output)
def testConv2DSameEven(self):
n, n2 = 4, 2
# Input image.
x = create_test_input(1, n, n, 1)
# Convolution kernel.
w = create_test_input(1, 3, 3, 1)
w = tf.reshape(w, [3, 3, 1, 1])
tf.get_variable('Conv/weights', initializer=w)
tf.get_variable('Conv/biases', initializer=tf.zeros([1]))
tf.get_variable_scope().reuse_variables()
y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv')
y1_expected = tf.to_float([[14, 28, 43, 26],
[28, 48, 66, 37],
[43, 66, 84, 46],
[26, 37, 46, 22]])
y1_expected = tf.reshape(y1_expected, [1, n, n, 1])
y2 = resnet_utils.subsample(y1, 2)
y2_expected = tf.to_float([[14, 43],
[43, 84]])
y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1])
y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv')
y3_expected = y2_expected
y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv')
y4_expected = tf.to_float([[48, 37],
[37, 22]])
y4_expected = tf.reshape(y4_expected, [1, n2, n2, 1])
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
self.assertAllClose(y1.eval(), y1_expected.eval())
self.assertAllClose(y2.eval(), y2_expected.eval())
self.assertAllClose(y3.eval(), y3_expected.eval())
self.assertAllClose(y4.eval(), y4_expected.eval())
def testConv2DSameOdd(self):
n, n2 = 5, 3
# Input image.
x = create_test_input(1, n, n, 1)
# Convolution kernel.
w = create_test_input(1, 3, 3, 1)
w = tf.reshape(w, [3, 3, 1, 1])
tf.compat.v1.get_variable('Conv/weights', initializer=w)
tf.compat.v1.get_variable('Conv/biases', initializer=tf.zeros([1]))
tf.compat.v1.get_variable_scope().reuse_variables()
y1 = slim.conv2d(x, 1, [3, 3], stride=1, scope='Conv')
y1_expected = tf.cast([[14, 28, 43, 58, 34],
[28, 48, 66, 84, 46],
[43, 66, 84, 102, 55],
[58, 84, 102, 120, 64],
[34, 46, 55, 64, 30]], dtype=tf.float32)
y1_expected = tf.reshape(y1_expected, [1, n, n, 1])
y2 = resnet_utils.subsample(y1, 2)
y2_expected = tf.cast([[14, 43, 34],
[43, 84, 55],
[34, 55, 30]], dtype=tf.float32)
y2_expected = tf.reshape(y2_expected, [1, n2, n2, 1])
y3 = resnet_utils.conv2d_same(x, 1, 3, stride=2, scope='Conv')
y3_expected = y2_expected
y4 = slim.conv2d(x, 1, [3, 3], stride=2, scope='Conv')
y4_expected = y2_expected
with self.test_session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
self.assertAllClose(y1.eval(), y1_expected.eval())
self.assertAllClose(y2.eval(), y2_expected.eval())
self.assertAllClose(y3.eval(), y3_expected.eval())
self.assertAllClose(y4.eval(), y4_expected.eval())
def bottleneck_pruned(inputs,
depth,
depth_bottleneck,
stride,
rate=1,
outputs_collections=None,
scope=None,
use_bounded_activations=False,
prune_unit=None,
is_local_train = None):
"""Bottleneck residual unit variant with BN after convolutions.
"""
# if it is local train, choose the inputs as from the original graph's input
# inputs = tf.cond(is_local_train, lambda: prune_unit['inputs'], lambda: inputs)
if is_local_train and prune_unit['inputs']!=None:
inputs = prune_unit['inputs']
with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(
inputs,
depth, [1, 1],
stride=stride,
activation_fn=tf.nn.relu6 if use_bounded_activations else None,
scope='shortcut')
prune_layers = prune_unit['prune_layers']
# print('HG: prune_layers=', prune_layers)
# print('HG: depth_bottleneck=', depth_bottleneck)
# Trial 1: still use the depth of original network so that the other variables can easily restored from the
# pretrained graph. The variables related to the pruned layers could be reassigned values later with validate_shape = false
# then the shape of those variables could be changed to the shape of assigned values. This trial gives error since it does not
# really change the shape of the variables. Variables's shape are set once they are created.
# pruned_depth = depth_bottleneck
pruned_depth = int(depth_bottleneck*prune_layers[1]) if (1 in prune_layers) else depth_bottleneck
residual = slim.conv2d(inputs, pruned_depth, [1, 1], stride=1,
scope='conv1')
# print('HG: conv1=', residual)
pruned_depth = int(depth_bottleneck*prune_layers[2]) if (2 in prune_layers) else depth_bottleneck
residual = resnet_utils.conv2d_same(residual, pruned_depth, 3, stride,
rate=rate, scope='conv2')
# print('HG: conv2=', residual)
residual = slim.conv2d(residual, depth, [1, 1], stride=1,
activation_fn=None, scope='conv3')
# print('HG: conv3=', residual)
if use_bounded_activations:
# Use clip_by_value to simulate bandpass activation.
residual = tf.clip_by_value(residual, -6.0, 6.0)
output = tf.nn.relu6(shortcut + residual)
else:
output = tf.nn.relu(shortcut + residual)
return slim.utils.collect_named_outputs(outputs_collections,
sc.original_name_scope,
output)
def resnet_v1(inputs,
blocks,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
store_non_strided_activations=False,
reuse=None,
scope=None):
"""Generator for v1 ResNet models.
This function generates a family of ResNet v1 models. See the resnet_v1_*()
methods for specific model instantiations, obtained by selecting different
block instantiations that produce ResNets of various depths.
Training for image classification on Imagenet is usually done with [224, 224]
inputs, resulting in [7, 7] feature maps at the output of the last ResNet
block for the ResNets defined in [1] that have nominal stride equal to 32.
However, for dense prediction tasks we advise that one uses inputs with
spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In
this case the feature maps at the ResNet output will have spatial shape
[(height - 1) / output_stride + 1, (width - 1) / output_stride + 1]
and corners exactly aligned with the input image corners, which greatly
facilitates alignment of the features to the image. Using as input [225, 225]
images results in [8, 8] feature maps at the output of the last ResNet block.
For dense prediction tasks, the ResNet needs to run in fully-convolutional
(FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all
have nominal stride equal to 32 and a good choice in FCN mode is to use
output_stride=16 in order to increase the density of the computed features at
small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915.
Args:
inputs: A tensor of size [batch, height_in, width_in, channels].
blocks: A list of length equal to the number of ResNet blocks. Each element
is a resnet_utils.Block object describing the units in the block.
num_classes: Number of predicted classes for classification tasks.
If 0 or None, we return the features before the logit layer.
is_training: whether batch_norm layers are in training mode. If this is set
to None, the callers can specify slim.batch_norm's is_training parameter
from an outer slim.arg_scope.
global_pool: If True, we perform global average pooling before computing the
logits. Set to True for image classification, False for dense prediction.
output_stride: If None, then the output will be computed at the nominal
network stride. If output_stride is not None, it specifies the requested
ratio of input to output spatial resolution.
include_root_block: If True, include the initial convolution followed by
max-pooling, if False excludes it.
spatial_squeeze: if True, logits is of shape [B, C], if false logits is
of shape [B, 1, 1, C], where B is batch_size and C is number of classes.
To use this parameter, the input images must be smaller than 300x300
pixels, in which case the output logit layer does not contain spatial
information and can be removed.
store_non_strided_activations: If True, we compute non-strided (undecimated)
activations at the last unit of each block and store them in the
`outputs_collections` before subsampling them. This gives us access to
higher resolution intermediate activations which are useful in some
dense prediction problems but increases 4x the computation and memory cost
at the last unit of each block.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
Returns:
net: A rank-4 tensor of size [batch, height_out, width_out, channels_out].
If global_pool is False, then height_out and width_out are reduced by a
factor of output_stride compared to the respective height_in and width_in,
else both height_out and width_out equal one. If num_classes is 0 or None,
then net is the output of the last ResNet block, potentially after global
average pooling. If num_classes a non-zero integer, net contains the
pre-softmax activations.
end_points: A dictionary from components of the network to the corresponding
activation.
Raises:
ValueError: If the target output_stride is not valid.
"""
with tf.variable_scope(scope, 'resnet_v1', [inputs], reuse=reuse) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
with slim.arg_scope(
[slim.conv2d, bottleneck, resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with (slim.arg_scope([slim.batch_norm], is_training=is_training)
if is_training is not None else NoOpScope()):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = resnet_utils.stack_blocks_dense(
net, blocks, output_stride, store_non_strided_activations)
# Convert end_points_collection into a dictionary of end_points.
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
return net, end_points
def deeplabv3(inputs,
num_classes,
depth=101,
aspp=True,
reuse=None,
is_training=True):
"""DeepLabV3
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The number of layers of the ResNet.
aspp: Whether to use ASPP module, if True, will use 4 blocks with
multi_grid=(1,2,4), if False, will use 7 blocks with multi_grid=(1,2,1).
reuse: Whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
Returns:
net: A rank-4 tensor of size [batch, height_out, width_out, channels_out].
end_points: A dictionary from components of the network to the
corresponding activation.
"""
if aspp:
multi_grid = (1, 2, 4)
else:
multi_grid = (1, 2, 1)
scope = 'resnet_v1_{}'.format(depth)
with tf.variable_scope(scope, [inputs], reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope(
resnet_arg_scope(weight_decay=args.weight_decay,
batch_norm_decay=args.bn_weight_decay)):
with slim.arg_scope([slim.conv2d, bottleneck],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm],
is_training=is_training):
net = inputs
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
with tf.variable_scope('block1', [net]) as sc:
base_depth = 64
for i in range(2):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1)
with tf.variable_scope('unit_3', values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=2)
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
print(net)
with tf.variable_scope('block2', [net]) as sc:
base_depth = 128
for i in range(3):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1)
with tf.variable_scope('unit_4', values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=2)
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
with tf.variable_scope('block3', [net]) as sc:
base_depth = 256
num_units = 6
if depth == 101:
num_units = 23
elif depth == 152:
num_units = 36
for i in range(num_units):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1)
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
with tf.variable_scope('block4', [net]) as sc:
base_depth = 512
for i in range(3):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1,
rate=2 * multi_grid[i])
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
if aspp:
# parallel module with atrous convolution(ASPP)
with tf.variable_scope('aspp', [net]) as sc:
aspp_list = []
branch_1 = slim.conv2d(net,
256, [1, 1],
stride=1,
scope='1x1conv')
branch_1 = slim.utils.collect_named_outputs(
end_points_collection, sc.name, branch_1)
aspp_list.append(branch_1)
for i in range(3):
branch_2 = slim.conv2d(
net,
256, [3, 3],
stride=1,
rate=6 * (i + 1),
scope='3x3conv_rate{}'.format(6 * (i + 1)))
branch_2 = slim.utils.collect_named_outputs(
end_points_collection, sc.name, branch_2)
aspp_list.append(branch_2)
# aspp = tf.add_n(aspp_list)
# aspp = slim.utils.collect_named_outputs(end_points_collection, sc.name, aspp)
with tf.variable_scope('image-level', [net]) as sc:
"""Image Pooling
See ParseNet: Looking Wider to See Better
"""
pooled = tf.reduce_mean(net, [1, 2],
name='avg_pool',
keepdims=True)
pooled = slim.utils.collect_named_outputs(
end_points_collection, sc.name, pooled)
pooled = slim.conv2d(pooled,
256, [1, 1],
stride=1,
scope='1x1conv')
pooled = slim.utils.collect_named_outputs(
end_points_collection, sc.name, pooled)
pooled = tf.image.resize_bilinear(
pooled,
tf.shape(net)[1:3])
aspp_list.append(pooled)
pooled = slim.utils.collect_named_outputs(
end_points_collection, sc.name, pooled)
with tf.variable_scope('fusion',
[aspp_list, pooled]) as sc:
net = tf.concat(aspp_list, 3)
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
net = slim.conv2d(net,
256, [1, 1],
stride=1,
scope='1x1conv')
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
else:
# cascaded module with atrous convolution
with tf.variable_scope('block5', [net]) as sc:
base_depth = 512
for i in range(3):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(
net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1,
rate=4 * multi_grid[i])
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
with tf.variable_scope('block6', [net]) as sc:
base_depth = 512
for i in range(3):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(
net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1,
rate=8 * multi_grid[i])
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
with tf.variable_scope('block7', [net]) as sc:
base_depth = 512
for i in range(3):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(
net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1,
rate=16 * multi_grid[i])
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
# inputs_size = tf.shape(inputs)[1:3]
# with tf.variable_scope('upsampling_logits',[net]) as sc:
# net = slim.conv2d(net, num_classes, [1,1], stride=1,
# activation_fn=None, normalizer_fn=None)
# net = tf.image.resize_bilinear(net, inputs_size, name='upsample')
# net = slim.utils.collect_named_outputs(end_points_collection,
# sc.name, net)
#
# net = tf.identity(net,"semantic")
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
inputs_size = tf.shape(inputs)[1:3]
with tf.variable_scope("decoder"):
with slim.arg_scope(
resnet_arg_scope(
weight_decay=args.weight_decay,
batch_norm_decay=args.bn_weight_decay)):
with slim.arg_scope([slim.batch_norm],
is_training=is_training):
with tf.variable_scope("low_level_features"):
low_level_features = end_points[
scope +
'/block1/unit_3/bottleneck_v1/conv1']
low_level_features = slim.conv2d(
low_level_features,
48, [1, 1],
stride=1,
scope='conv_1x1')
low_level_features_size = tf.shape(
low_level_features)[1:3]
with tf.variable_scope("upsampling_logits"):
net = tf.image.resize_bilinear(
net,
low_level_features_size,
name='upsample_1')
net = tf.concat([net, low_level_features],
axis=3,
name='concat')
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv_3x3_1')
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv_3x3_2')
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='conv_1x1')
net = tf.image.resize_bilinear(
net, inputs_size, name='upsample_2')
net = tf.identity(net, "semantic")
return net, end_points
def _root_block(net):
if depthwise_convolution:
if sparse_dense_branch:
sparse_list = []
dense_list = []
for i, sparsity in enumerate(sparsity_type):
subnet = net[:, :, :, i:(i + 1)]
if sparsity:
sparse_list.append(subnet)
else:
dense_list.append(subnet)
sparse_net = tf.concat(sparse_list, axis=3)
dense_net = tf.concat(dense_list, axis=3)
sparse_net = resnet_utils.conv2d_same(
sparse_net,
32,
3,
stride=1,
scope='sparse_conv1')
dense_net = resnet_utils.conv2d_same(
dense_net,
16,
3,
stride=1,
scope='dense_conv1')
net = tf.concat([sparse_net, dense_net],
axis=3)
net = resnet_utils.conv2d_same(net,
64,
3,
stride=1,
scope='conv3')
net = resnet_utils.conv2d_same(
net,
64,
3,
stride=root_downsampling_rate,
scope='conv4')
else:
net = slim.separable_conv2d(net,
num_outputs=64,
kernel_size=3,
depth_multiplier=8,
scope='conv1_1')
net = resnet_utils.conv2d_same(
net,
64,
3,
stride=root_downsampling_rate,
scope='conv1_2')
else:
net = resnet_utils.conv2d_same(net,
32,
3,
stride=1,
scope='conv1_1')
net = resnet_utils.conv2d_same(
net,
64,
3,
stride=root_downsampling_rate,
scope='conv1_2')
return net
def bottleneck(inputs,
depth,
depth_bottleneck,
stride,
rate=1,
outputs_collections=None,
scope=None,
use_bounded_activations=False,
initializers=None,
insert_shift=False,
split_model=False):
"""Bottleneck residual unit variant with BN after convolutions.
This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for
its definition. Note that we use here the bottleneck variant which has an
extra bottleneck layer.
When putting together two consecutive ResNet blocks that use this unit, one
should use stride = 2 in the last unit of the first block.
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The depth of the ResNet unit output.
depth_bottleneck: The depth of the bottleneck layers.
stride: The ResNet unit's stride. Determines the amount of downsampling of
the units output compared to its input.
rate: An integer, rate for atrous convolution.
outputs_collections: Collection to add the ResNet unit output.
scope: Optional variable_scope.
use_bounded_activations: Whether or not to use bounded activations. Bounded
activations better lend themselves to quantized inference.
initializers <MODIFICATION>: Weight and bias initializers for the included conv blocks
insert_shift <MODIFICATION>: If true, inserts shift operation in front of every first conv1x1 within a block.
split_model <MODIFICATION>: (overrides "shift") If true, inserts placeholder in front of every first conv1x1 within a block. This allows for insertion of shift outside of TF.
Returns:
The ResNet unit's output.
"""
with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc:
fake_inputs = None
fake_shortcut = None
if split_model:
shift_input = tf.identity(inputs, 'prev_conv_output')
#shortcut_input = tf.identity(shortcut, 'prev_shortcut_output')
shortcut_input = tf.identity(inputs, 'prev_shortcut_output')
fake_inputs = tf.placeholder(tf.float32,
shape=inputs.get_shape(),
name='conv_input')
#fake_shortcut = tf.placeholder(tf.float32, shape=shortcut.get_shape(), name='shortcut_input')
fake_shortcut = tf.placeholder(tf.float32,
shape=inputs.get_shape(),
name='shortcut_input')
elif insert_shift:
shift_input = tf.identity(inputs, 'prev_conv_output')
#shortcut_input = tf.identity(shortcut, 'prev_shortcut_output')
shortcut_input = tf.identity(inputs, 'prev_shortcut_output')
fold = inputs.get_shape()[-1] // 8
shifted = tf.Variable(tf.zeros_like(inputs))
initial_print = tf.print("initial: ", shifted)
shift_left = tf.assign(shifted[:-1, :, :, :fold],
shift_input[1:, :, :, :fold])
shift_right = tf.assign(shifted[1:, :, :, fold:2 * fold],
shift_input[:-1, :, :, fold:2 * fold])
shift_copy = tf.assign(shifted[:, :, :, 2 * fold:],
shift_input[:, :, :, 2 * fold:])
final_print = tf.print("final: ", shifted)
with tf.control_dependencies([shift_left, shift_right,
shift_copy]):
fake_inputs = tf.identity(shifted, 'shifted_input')
fake_shortcut = shortcut_input
else:
fake_inputs = inputs
#fake_shortcut = shortcut
fake_shortcut = inputs
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
if depth == depth_in:
#shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
shortcut = resnet_utils.subsample(fake_shortcut, stride,
'shortcut')
else:
shortcut = slim.conv2d(
#inputs,
fake_shortcut,
depth,
[1, 1],
stride=stride,
activation_fn=tf.nn.relu6 if use_bounded_activations else None,
scope='shortcut',
**initializers["shortcut"])
residual = slim.conv2d(fake_inputs,
depth_bottleneck, [1, 1],
stride=1,
scope='conv1',
**initializers["conv1"])
residual = resnet_utils.conv2d_same(residual,
depth_bottleneck,
3,
stride,
rate=rate,
scope='conv2',
**initializers["conv2"])
residual = slim.conv2d(residual,
depth, [1, 1],
stride=1,
activation_fn=None,
scope='conv3',
**initializers["conv3"])
if use_bounded_activations:
# Use clip_by_value to simulate bandpass activation.
residual = tf.clip_by_value(residual, -6.0, 6.0)
#output = tf.nn.relu6(fake_shortcut + residual)
output = tf.nn.relu6(shortcut + residual)
else:
#output = tf.nn.relu(fake_shortcut + residual)
output = tf.nn.relu(shortcut + residual)
return slim.utils.collect_named_outputs(outputs_collections, sc.name,
output)
def bottleneck_pruned(inputs,
convDict,
stride,
rate=1,
outputs_collections=None,
scope=None,
use_bounded_activations=False):
"""Bottleneck residual unit variant with BN after convolutions.
This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for
its definition. Note that we use here the bottleneck variant which has an
extra bottleneck layer.
When putting together two consecutive ResNet blocks that use this unit, one
should use stride = 2 in the last unit of the first block.
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The depth of the ResNet unit output.
depth_bottleneck: The depth of the bottleneck layers.
stride: The ResNet unit's stride. Determines the amount of downsampling of
the units output compared to its input.
rate: An integer, rate for atrous convolution.
outputs_collections: Collection to add the ResNet unit output.
scope: Optional variable_scope.
use_bounded_activations: Whether or not to use bounded activations. Bounded
activations better lend themselves to quantized inference.
Returns:
The ResNet unit's output.
"""
with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
if convDict['shortcut'] == depth_in: #Used to be depth
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(
inputs,
convDict['shortcut'], #Used to be depth
[1, 1],
stride=stride,
activation_fn=tf.nn.relu6 if use_bounded_activations else None,
scope='shortcut')
residual = slim.conv2d(
inputs, convDict['conv1'], [1, 1], stride=1,
scope='conv1') #convDict['conv1'] used to be depth_bottleneck
residual = resnet_utils.conv2d_same(
residual, convDict['conv2'], 3, stride, rate=rate,
scope='conv2') #convDict['conv2'] used to be depth_bottleneck
residual = slim.conv2d(
residual,
convDict['conv3'], [1, 1],
stride=1,
activation_fn=None,
scope='conv3') #convDict['conv3'] used to be depth
if use_bounded_activations:
# Use clip_by_value to simulate bandpass activation.
residual = tf.clip_by_value(residual, -6.0, 6.0)
output = tf.nn.relu6(shortcut + residual)
else:
output = tf.nn.relu(shortcut + residual)
return slim.utils.collect_named_outputs(outputs_collections, sc.name,
output)
def resnet_v1(inputs,
blocks,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=False,
reuse=None,
scope=None):
"""Generator for v1 ResNet models.
This function generates a family of ResNet v1 models. See the resnet_v1_*()
methods for specific model instantiations, obtained by selecting different
block instantiations that produce ResNets of various depths.
Training for image classification on Imagenet is usually done with [224, 224]
inputs, resulting in [7, 7] feature maps at the output of the last ResNet
block for the ResNets defined in [1] that have nominal stride equal to 32.
However, for dense prediction tasks we advise that one uses inputs with
spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In
this case the feature maps at the ResNet output will have spatial shape
[(height - 1) / output_stride + 1, (width - 1) / output_stride + 1]
and corners exactly aligned with the input image corners, which greatly
facilitates alignment of the features to the image. Using as input [225, 225]
images results in [8, 8] feature maps at the output of the last ResNet block.
For dense prediction tasks, the ResNet needs to run in fully-convolutional
(FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all
have nominal stride equal to 32 and a good choice in FCN mode is to use
output_stride=16 in order to increase the density of the computed features at
small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915.
Args:
inputs: A tensor of size [batch, height_in, width_in, channels].
blocks: A list of length equal to the number of ResNet blocks. Each element
is a resnet_utils.Block object describing the units in the block.
num_classes: Number of predicted classes for classification tasks. If None
we return the features before the logit layer.
is_training: whether is training or not.
global_pool: If True, we perform global average pooling before computing the
logits. Set to True for image classification, False for dense prediction.
output_stride: If None, then the output will be computed at the nominal
network stride. If output_stride is not None, it specifies the requested
ratio of input to output spatial resolution.
include_root_block: If True, include the initial convolution followed by
max-pooling, if False excludes it.
spatial_squeeze: if True, logits is of shape [B, C], if false logits is
of shape [B, 1, 1, C], where B is batch_size and C is number of classes.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
Returns:
net: A rank-4 tensor of size [batch, height_out, width_out, channels_out].
If global_pool is False, then height_out and width_out are reduced by a
factor of output_stride compared to the respective height_in and width_in,
else both height_out and width_out equal one. If num_classes is None, then
net is the output of the last ResNet block, potentially after global
average pooling. If num_classes is not None, net contains the pre-softmax
activations.
end_points: A dictionary from components of the network to the corresponding
activation.
Raises:
ValueError: If the target output_stride is not valid.
"""
with tf.variable_scope(scope, 'resnet_v1', [inputs], reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope([slim.conv2d, bottleneck,
resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm], is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError('The output_stride needs to be a multiple of 4.')
output_stride /= 4
net = resnet_utils.conv2d_same(net, 64, 7, stride=2, scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = resnet_utils.stack_blocks_dense(net, blocks, output_stride)
if global_pool:
# Global average pooling.
net = tf.reduce_mean(net, [1, 2], name='pool5', keep_dims=True)
if num_classes is not None:
net = slim.conv2d(net, num_classes, [1, 1], activation_fn=None,
normalizer_fn=None, scope='logits')
if spatial_squeeze:
logits = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
else:
logits = net
# Convert end_points_collection into a dictionary of end_points.
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if num_classes is not None:
end_points['predictions'] = slim.softmax(logits, scope='predictions')
return logits, end_points
def resnet_twostream_inter(inputs_depth,
blocks_depth,
inputs_rgb,
blocks_rgb,
nr_frames,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
reuse=None,
scope_depth=None,
scope_rgb=None,
depth_training=True):
# depth / hallucination injects signal into rgb
# depth stream
with tf.device('/gpu:0'):
inputs = inputs_depth
scope = scope_depth
bottleneck = bottleneck_normal
blocks = blocks_depth
with tf.variable_scope(scope, 'resnet_v1', [inputs],
reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope(
[slim.conv2d, bottleneck, resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm],
is_training=depth_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3],
stride=2,
scope='pool1')
net = resnet_utils.stack_blocks_dense(
net, blocks, nr_frames, output_stride)
if global_pool:
net = tf.reduce_mean(net, [1, 2],
name='pool5',
keep_dims=True)
if num_classes is not None:
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='logits')
if spatial_squeeze:
net = tf.squeeze(net, [1, 2],
name='SpatialSqueeze')
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if num_classes is not None:
end_points['predictions'] = slim.softmax(
net, scope='predictions')
net_depth = net
end_points_depth = end_points
end_points_to_pass = {}
end_points_to_pass[scope_depth +
'/block1/unit_1/bottleneck_v1'] = end_points_depth[
scope_depth + '/block1/unit_1/bottleneck_v1']
end_points_to_pass[scope_depth +
'/block2/unit_1/bottleneck_v1'] = end_points_depth[
scope_depth + '/block2/unit_1/bottleneck_v1']
end_points_to_pass[scope_depth +
'/block3/unit_1/bottleneck_v1'] = end_points_depth[
scope_depth + '/block3/unit_1/bottleneck_v1']
end_points_to_pass[scope_depth +
'/block4/unit_1/bottleneck_v1'] = end_points_depth[
scope_depth + '/block4/unit_1/bottleneck_v1']
# rgb stream
with tf.device('/gpu:1'):
inputs = inputs_rgb
scope = scope_rgb
bottleneck = bottleneck_injected
blocks = blocks_rgb
with tf.variable_scope(scope, 'resnet_v1', [inputs],
reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope([
slim.conv2d, bottleneck,
resnet_utils.stack_blocks_dense_injected
],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm],
is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3],
stride=2,
scope='pool1')
net = resnet_utils.stack_blocks_dense_injected(
net, blocks, nr_frames, end_points_to_pass,
'resnet_v1_50_depth/', output_stride)
if global_pool:
net = tf.reduce_mean(net, [1, 2],
name='pool5',
keep_dims=True)
if num_classes is not None:
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='logits')
if spatial_squeeze:
net = tf.squeeze(net, [1, 2],
name='SpatialSqueeze')
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if num_classes is not None:
end_points['predictions'] = slim.softmax(
net, scope='predictions')
net_rgb = net
end_points_rgb = end_points
return net_depth, end_points_depth, net_rgb, end_points_rgb
def resnet_one_stream(inputs,
blocks,
nr_frames,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
reuse=None,
scope=None,
gpu_id='/gpu:0'):
bottleneck = bottleneck_normal
with tf.device(gpu_id):
with tf.variable_scope(scope, 'resnet_v1', [inputs],
reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope(
[slim.conv2d, bottleneck, resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm],
is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3],
stride=2,
scope='pool1')
net = resnet_utils.stack_blocks_dense(
net, blocks, nr_frames, output_stride)
if global_pool:
net = tf.reduce_mean(net, [1, 2],
name='pool5',
keep_dims=True)
last_pool = net
if num_classes is not None:
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='logits')
if spatial_squeeze:
net = tf.squeeze(net, [1, 2],
name='SpatialSqueeze')
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if num_classes is not None:
end_points['predictions'] = slim.softmax(
net, scope='predictions')
end_points['last_pool'] = last_pool
return net, end_points
def bottleneck_injected(inputs,
depth,
depth_bottleneck,
stride,
nr_frames=None,
rate=1,
outputs_collections=None,
scope=None,
use_bounded_activations=False,
temporal=False,
multiplier=None,
net_before_relu=None,
unit_id=-1):
with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(
inputs,
depth, [1, 1],
stride=stride,
activation_fn=tf.nn.relu6 if use_bounded_activations else None,
scope='shortcut')
if temporal:
# if unit x which coincides with temporal conv and depth / OF injection,
# multiply feature maps
residual_mult = tf.multiply(net_before_relu, multiplier)
inputs = inputs + residual_mult
residual = slim.conv2d(inputs,
depth_bottleneck, [1, 1],
stride=1,
scope='conv1')
residual = resnet_utils.conv2d_same(residual,
depth_bottleneck,
3,
stride,
rate=rate,
scope='conv2')
if temporal: # temporal conv
residual = resnet_utils.conv_temp2(residual,
nr_frames,
scope='conv_temp')
residual = slim.conv2d(residual,
depth, [1, 1],
stride=1,
activation_fn=None,
scope='conv3')
if use_bounded_activations:
# Use clip_by_value to simulate bandpass activation.
residual = tf.clip_by_value(residual, -6.0, 6.0)
output = tf.nn.relu6(shortcut + residual)
else:
if unit_id == 0:
output_before_relu = shortcut + residual
output = tf.nn.relu(shortcut + residual)
return slim.utils.collect_named_outputs(
outputs_collections, sc.original_name_scope,
output), output_before_relu
else:
output = tf.nn.relu(shortcut + residual)
return slim.utils.collect_named_outputs(
outputs_collections, sc.original_name_scope, output)
def bottleneck(inputs,
depth,
depth_bottleneck,
stride,
rate=1,
outputs_collections=None,
scope=None,
use_bounded_activations=False):
"""Bottleneck residual unit variant with BN after convolutions.
This is the original residual unit proposed in [1]. See Fig. 1(a) of [2] for
its definition. Note that we use here the bottleneck variant which has an
extra bottleneck layer.
When putting together two consecutive ResNet blocks that use this unit, one
should use stride = 2 in the last unit of the first block.
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The depth of the ResNet unit output.
depth_bottleneck: The depth of the bottleneck layers.
stride: The ResNet unit's stride. Determines the amount of downsampling of
the units output compared to its input.
rate: An integer, rate for atrous convolution.
outputs_collections: Collection to add the ResNet unit output.
scope: Optional variable_scope.
use_bounded_activations: Whether or not to use bounded activations. Bounded
activations better lend themselves to quantized inference.
Returns:
The ResNet unit's output.
"""
with tf.variable_scope(scope, 'bottleneck_v1', [inputs]) as sc:
depth_in = slim.utils.last_dimension(inputs.get_shape(), min_rank=4)
if depth == depth_in:
shortcut = resnet_utils.subsample(inputs, stride, 'shortcut')
else:
shortcut = slim.conv2d(
inputs,
depth, [1, 1],
stride=stride,
activation_fn=tf.nn.relu6 if use_bounded_activations else None,
scope='shortcut')
residual = slim.conv2d(inputs, depth_bottleneck, [1, 1], stride=1,
scope='conv1')
residual = resnet_utils.conv2d_same(residual, depth_bottleneck, 3, stride,
rate=rate, scope='conv2')
residual = slim.conv2d(residual, depth, [1, 1], stride=1,
activation_fn=None, scope='conv3')
if use_bounded_activations:
# Use clip_by_value to simulate bandpass activation.
residual = tf.clip_by_value(residual, -6.0, 6.0)
output = tf.nn.relu6(shortcut + residual)
else:
output = tf.nn.relu(shortcut + residual)
return slim.utils.collect_named_outputs(outputs_collections,
sc.name,
output)
def deeplabv2(inputs,
num_classes,
depth=50,
large_rate=True,
reuse=None,
is_training=True):
"""DeepLabV3
Args:
inputs: A tensor of size [batch, height, width, channels].
depth: The number of layers of the ResNet.
aspp: Whether to use ASPP module, if True, will use 4 blocks with
multi_grid=(1,2,4), if False, will use 7 blocks with multi_grid=(1,2,1).
reuse: Whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
Returns:
net: A rank-4 tensor of size [batch, height_out, width_out, channels_out].
end_points: A dictionary from components of the network to the
corresponding activation.
"""
if large_rate:
multi_grid = (6, 12, 18, 24)
else:
multi_grid = (2, 4, 8, 12)
scope = 'resnet_v1_{}'.format(depth)
with tf.variable_scope(scope, [inputs], reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope(
resnet_arg_scope(weight_decay=args.weight_decay,
batch_norm_decay=args.bn_weight_decay)):
with slim.arg_scope([slim.conv2d, bottleneck],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm],
is_training=is_training):
net = inputs
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
with tf.variable_scope('block1', [net]) as sc:
base_depth = 64
for i in range(2):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1)
with tf.variable_scope('unit_3', values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=2)
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
with tf.variable_scope('block2', [net]) as sc:
base_depth = 128
for i in range(3):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1)
with tf.variable_scope('unit_4', values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=2)
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
with tf.variable_scope('block3', [net]) as sc:
base_depth = 256
num_units = 6
if depth == 101:
num_units = 23
elif depth == 152:
num_units = 36
for i in range(num_units):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1,
rate=2)
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
with tf.variable_scope('block4', [net]) as sc:
base_depth = 512
for i in range(3):
with tf.variable_scope('unit_%d' % (i + 1),
values=[net]):
net = bottleneck(net,
depth=base_depth * 4,
depth_bottleneck=base_depth,
stride=1,
rate=4)
net = slim.utils.collect_named_outputs(
end_points_collection, sc.name, net)
with tf.variable_scope('aspp', [net]) as sc:
aspp_list = []
for i in range(4):
branch_2 = slim.conv2d(
net,
num_classes, [3, 3],
stride=1,
rate=multi_grid[i],
activation_fn=None,
normalizer_fn=None,
scope='3x3conv_rate{}'.format(multi_grid[i]))
branch_2 = slim.utils.collect_named_outputs(
end_points_collection, sc.name, branch_2)
aspp_list.append(branch_2)
aspp = tf.add_n(aspp_list, name='fc1_voc12')
aspp = slim.utils.collect_named_outputs(
end_points_collection, sc.name, aspp)
inputs_size = tf.shape(inputs)[1:3]
with tf.variable_scope('upsampling_logits', [net]) as sc:
net = tf.image.resize_bilinear(aspp,
inputs_size,
name='upsample')
net = tf.identity(net, "semantic")
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
return net, end_points
def resnet_v1_pathology(inputs,
blocks,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
store_non_strided_activations=False,
reuse=None,
scope=None):
with tf.variable_scope(scope, 'resnet_v1_pathology', [inputs],
reuse=reuse) as sc:
end_points_collection = sc.original_name_scope + '_end_points'
with slim.arg_scope(
[slim.conv2d, bottleneck, resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with (slim.arg_scope([slim.batch_norm], is_training=is_training)
if is_training is not None else NoOpScope()):
net = inputs
num_images = net.shape[0]
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
#tiling
net = _tile_images_2res(net, num_images)
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = resnet_utils.stack_blocks_dense(
net, blocks, output_stride, store_non_strided_activations)
# Convert end_points_collection into a dictionary of end_points.
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if global_pool:
# Global average pooling.
net = tf.reduce_mean(net, [1, 2],
name='pool5',
keep_dims=True)
end_points['global_pool'] = net
#max
net = _max_tile_2res(net, num_images)
if num_classes:
#first fully connected layer
net = slim.conv2d(net,
512, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='fc1')
end_points[sc.name + '/fc1'] = net
#dropout
net = slim.dropout(net,
keep_prob=0.8,
scope='dropout',
is_training=is_training)
end_points[sc.name + '/dropout'] = net
#final fully connected layer
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='logits')
end_points[sc.name + '/logits'] = net
if spatial_squeeze:
#remove dimensions of size 1
net = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
end_points[sc.name + '/spatial_squeeze'] = net
end_points['predictions'] = slim.softmax(
net, scope='predictions')
return net, end_points
def resnet_distributions_v1(inputs,
blocks,
num_classes=None,
is_training=True,
output_stride=None,
include_root_block=True,
reuse=None,
scope=None,
sample_number=1):
"""Generator for v1 ResNet models.
This function generates a family of ResNet v1 models. See the resnet_v1_*()
methods for specific model instantiations, obtained by selecting different
block instantiations that produce ResNets of various depths.
Training for image classification on Imagenet is usually done with [224, 224]
inputs, resulting in [7, 7] feature maps at the output of the last ResNet
block for the ResNets defined in [1] that have nominal stride equal to 32.
However, for dense prediction tasks we advise that one uses inputs with
spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In
this case the feature maps at the ResNet output will have spatial shape
[(height - 1) / output_stride + 1, (width - 1) / output_stride + 1]
and corners exactly aligned with the input image corners, which greatly
facilitates alignment of the features to the image. Using as input [225, 225]
images results in [8, 8] feature maps at the output of the last ResNet block.
For dense prediction tasks, the ResNet needs to run in fully-convolutional
(FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all
have nominal stride equal to 32 and a good choice in FCN mode is to use
output_stride=16 in order to increase the density of the computed features at
small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915.
Args:
inputs: A tensor of size [batch, height_in, width_in, channels].
blocks: A list of length equal to the number of ResNet blocks. Each element
is a resnet_utils.Block object describing the units in the block.
num_classes: Number of predicted classes for classification tasks. If None
we return the features before the logit layer.
is_training: whether is training or not.
global_pool: If True, we perform global average pooling before computing the
logits. Set to True for image classification, False for dense prediction.
output_stride: If None, then the output will be computed at the nominal
network stride. If output_stride is not None, it specifies the requested
ratio of input to output spatial resolution.
include_root_block: If True, include the initial convolution followed by
max-pooling, if False excludes it.
spatial_squeeze: if True, logits is of shape [B, C], if false logits is
of shape [B, 1, 1, C], where B is batch_size and C is number of classes.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
avr_cc: 0, default, average; 1, concatenate
Returns:
net: A rank-4 tensor of size [batch, height_out, width_out, channels_out].
If global_pool is False, then height_out and width_out are reduced by a
factor of output_stride compared to the respective height_in and width_in,
else both height_out and width_out equal one. If num_classes is None, then
net is the output of the last ResNet block, potentially after global
average pooling. If num_classes is not None, net contains the pre-softmax
activations.
end_points: A dictionary from components of the network to the corresponding
activation.
Raises:
ValueError: If the target output_stride is not valid.
"""
with tf.variable_scope(scope, 'resnet_v1', [inputs], reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope([
slim.conv2d, bottleneck, resnet_utils.stack_blocks_dense,
resnet_utils.extra_fc, resnet_utils.projecting_feats
],
outputs_collections=end_points_collection):
with slim.arg_scope(
[resnet_utils.extra_fc],
loss_collection=tf.GraphKeys.REGULARIZATION_LOSSES):
with slim.arg_scope([slim.batch_norm],
is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3],
stride=2,
scope='pool1')
net = resnet_utils.stack_blocks_dense(
net, blocks, output_stride)
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
with tf.variable_scope('Distributions'):
mu = tf.reduce_mean(net, [1, 2],
name='pool5',
keep_dims=True)
end_points['global_pool'] = mu
sig = slim.conv2d(
net,
net.shape[-1], [net.shape[1], net.shape[2]],
activation_fn=None,
normalizer_fn=None,
biases_initializer=tf.zeros_initializer(),
scope='sig',
padding='VALID')
sig += 1e-10
mu = slim.dropout(mu,
scope='Dropout',
is_training=is_training)
end_points['PreLogits_mean'] = tf.squeeze(
mu, [1, 2], name='PreLogits_mean')
end_points['PreLogits_sig'] = tf.squeeze(
sig, [1, 2], name='PreLogits_sig')
tfd = tf.contrib.distributions
#MultivariateNormalDiagWithSoftplusScale
sample_dist = tfd.MultivariateNormalDiagWithSoftplusScale(
loc=end_points['PreLogits_mean'],
scale_diag=end_points['PreLogits_sig'])
end_points['sample_dist'] = sample_dist
end_points['sample_dist_samples'] = sample_dist.sample(
100)
end_points[
'sample_dist_covariance'] = sample_dist.stddev()
if not num_classes:
return mu, end_points
logits = slim.conv2d(
mu,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
biases_initializer=tf.zeros_initializer(),
scope='logits')
logits = tf.squeeze(logits, [1, 2])
#with tf.variable_scope('Distributions'):
logits2 = []
for iii in range(sample_number):
z = sample_dist.sample(1)
z = tf.reshape(z, [-1, int(mu.shape[-1])])
#import pdb
#pdb.set_trace()
z = tf.expand_dims(z, 1)
z = tf.expand_dims(z, 1)
logits_tmp = slim.conv2d(
z,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
biases_initializer=tf.zeros_initializer(),
scope='logits',
reuse=True)
logits2.append(tf.squeeze(logits_tmp, [1, 2]))
logits = tf.identity(logits, name='output')
end_points['Logits'] = logits
end_points['Logits2'] = logits2
if sample_number == 1:
end_points['predictions'] = slim.softmax(
logits + 0.1 * logits2[0], scope='predictions')
else:
end_points['predictions'] = slim.softmax(
logits, scope='predictions')
return logits, logits2, end_points
def resnet_v2(inputs,
blocks,
num_classes=None,
is_training=True,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=False,
reuse=None,
scope=None):
"""Generator for v2 (preactivation) ResNet models.
This function generates a family of ResNet v2 models. See the resnet_v2_*()
methods for specific model instantiations, obtained by selecting different
block instantiations that produce ResNets of various depths.
Training for image classification on Imagenet is usually done with [224, 224]
inputs, resulting in [7, 7] feature maps at the output of the last ResNet
block for the ResNets defined in [1] that have nominal stride equal to 32.
However, for dense prediction tasks we advise that one uses inputs with
spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In
this case the feature maps at the ResNet output will have spatial shape
[(height - 1) / output_stride + 1, (width - 1) / output_stride + 1]
and corners exactly aligned with the input image corners, which greatly
facilitates alignment of the features to the image. Using as input [225, 225]
images results in [8, 8] feature maps at the output of the last ResNet block.
For dense prediction tasks, the ResNet needs to run in fully-convolutional
(FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all
have nominal stride equal to 32 and a good choice in FCN mode is to use
output_stride=16 in order to increase the density of the computed features at
small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915.
Args:
inputs: A tensor of size [batch, height_in, width_in, channels].
blocks: A list of length equal to the number of ResNet blocks. Each element
is a resnet_utils.Block object describing the units in the block.
num_classes: Number of predicted classes for classification tasks. If None
we return the features before the logit layer.
is_training: whether is training or not.
global_pool: If True, we perform global average pooling before computing the
logits. Set to True for image classification, False for dense prediction.
output_stride: If None, then the output will be computed at the nominal
network stride. If output_stride is not None, it specifies the requested
ratio of input to output spatial resolution.
include_root_block: If True, include the initial convolution followed by
max-pooling, if False excludes it. If excluded, `inputs` should be the
results of an activation-less convolution.
spatial_squeeze: if True, logits is of shape [B, C], if false logits is
of shape [B, 1, 1, C], where B is batch_size and C is number of classes.
To use this parameter, the input images must be smaller than 300x300
pixels, in which case the output logit layer does not contain spatial
information and can be removed.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
Returns:
net: A rank-4 tensor of size [batch, height_out, width_out, channels_out].
If global_pool is False, then height_out and width_out are reduced by a
factor of output_stride compared to the respective height_in and width_in,
else both height_out and width_out equal one. If num_classes is None, then
net is the output of the last ResNet block, potentially after global
average pooling. If num_classes is not None, net contains the pre-softmax
activations.
end_points: A dictionary from components of the network to the corresponding
activation.
Raises:
ValueError: If the target output_stride is not valid.
"""
with tf.variable_scope(scope, 'resnet_v2', [inputs], reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope(
[slim.conv2d, bottleneck, resnet_utils.stack_blocks_dense],
outputs_collections=end_points_collection):
with slim.arg_scope([slim.batch_norm], is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
# We do not include batch normalization or activation functions in
# conv1 because the first ResNet unit will perform these. Cf.
# Appendix of [2].
with slim.arg_scope([slim.conv2d],
activation_fn=None,
normalizer_fn=None):
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3], stride=2, scope='pool1')
net = resnet_utils.stack_blocks_dense(net, blocks,
output_stride)
# This is needed because the pre-activation variant does not have batch
# normalization or activation functions in the residual unit output. See
# Appendix of [2].
net = slim.batch_norm(net,
activation_fn=tf.nn.relu,
scope='postnorm')
if global_pool:
# Global average pooling.
net = tf.reduce_mean(net, [1, 2],
name='pool5',
keep_dims=True)
if num_classes is not None:
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='logits')
if spatial_squeeze:
net = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
# Convert end_points_collection into a dictionary of end_points.
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if num_classes is not None:
end_points['predictions'] = slim.softmax(
net, scope='predictions')
return net, end_points
def resnet_v1(inputs,
blocks,
num_classes=None,
is_training=True,
extra_fc_type=-1,
extra_fc_out_dim=0,
extra_fc_W_decay=0.0,
f_decorr_fr=-1.,
f_decorr_decay=0.0,
global_pool=True,
output_stride=None,
include_root_block=True,
spatial_squeeze=True,
avr_cc=0,
feat_proj_type=-1,
proj_dim=1024,
feat_prop_down=False,
reuse=None,
scope=None):
"""Generator for v1 ResNet models.
This function generates a family of ResNet v1 models. See the resnet_v1_*()
methods for specific model instantiations, obtained by selecting different
block instantiations that produce ResNets of various depths.
Training for image classification on Imagenet is usually done with [224, 224]
inputs, resulting in [7, 7] feature maps at the output of the last ResNet
block for the ResNets defined in [1] that have nominal stride equal to 32.
However, for dense prediction tasks we advise that one uses inputs with
spatial dimensions that are multiples of 32 plus 1, e.g., [321, 321]. In
this case the feature maps at the ResNet output will have spatial shape
[(height - 1) / output_stride + 1, (width - 1) / output_stride + 1]
and corners exactly aligned with the input image corners, which greatly
facilitates alignment of the features to the image. Using as input [225, 225]
images results in [8, 8] feature maps at the output of the last ResNet block.
For dense prediction tasks, the ResNet needs to run in fully-convolutional
(FCN) mode and global_pool needs to be set to False. The ResNets in [1, 2] all
have nominal stride equal to 32 and a good choice in FCN mode is to use
output_stride=16 in order to increase the density of the computed features at
small computational and memory overhead, cf. http://arxiv.org/abs/1606.00915.
Args:
inputs: A tensor of size [batch, height_in, width_in, channels].
blocks: A list of length equal to the number of ResNet blocks. Each element
is a resnet_utils.Block object describing the units in the block.
num_classes: Number of predicted classes for classification tasks. If None
we return the features before the logit layer.
is_training: whether is training or not.
global_pool: If True, we perform global average pooling before computing the
logits. Set to True for image classification, False for dense prediction.
output_stride: If None, then the output will be computed at the nominal
network stride. If output_stride is not None, it specifies the requested
ratio of input to output spatial resolution.
include_root_block: If True, include the initial convolution followed by
max-pooling, if False excludes it.
spatial_squeeze: if True, logits is of shape [B, C], if false logits is
of shape [B, 1, 1, C], where B is batch_size and C is number of classes.
reuse: whether or not the network and its variables should be reused. To be
able to reuse 'scope' must be given.
scope: Optional variable_scope.
avr_cc: 0, default, average; 1, concatenate
Returns:
net: A rank-4 tensor of size [batch, height_out, width_out, channels_out].
If global_pool is False, then height_out and width_out are reduced by a
factor of output_stride compared to the respective height_in and width_in,
else both height_out and width_out equal one. If num_classes is None, then
net is the output of the last ResNet block, potentially after global
average pooling. If num_classes is not None, net contains the pre-softmax
activations.
end_points: A dictionary from components of the network to the corresponding
activation.
Raises:
ValueError: If the target output_stride is not valid.
"""
with tf.variable_scope(scope, 'resnet_v1', [inputs], reuse=reuse) as sc:
end_points_collection = sc.name + '_end_points'
with slim.arg_scope([
slim.conv2d, bottleneck, resnet_utils.stack_blocks_dense,
resnet_utils.extra_fc, resnet_utils.projecting_feats
],
outputs_collections=end_points_collection):
with slim.arg_scope(
[resnet_utils.extra_fc],
loss_collection=tf.GraphKeys.REGULARIZATION_LOSSES):
with slim.arg_scope([slim.batch_norm],
is_training=is_training):
net = inputs
if include_root_block:
if output_stride is not None:
if output_stride % 4 != 0:
raise ValueError(
'The output_stride needs to be a multiple of 4.'
)
output_stride /= 4
net = resnet_utils.conv2d_same(net,
64,
7,
stride=2,
scope='conv1')
net = slim.max_pool2d(net, [3, 3],
stride=2,
scope='pool1')
net = resnet_utils.stack_blocks_dense(
net, blocks, output_stride)
if extra_fc_type >= 0:
# extra fc layer; keep its dimension as 4?
net, pre_pool5 = resnet_utils.extra_fc(
net, extra_fc_out_dim, extra_fc_W_decay,
extra_fc_type, f_decorr_fr, f_decorr_decay)
elif global_pool:
# Global average pooling.
net = tf.reduce_mean(net, [1, 2],
name='pool5',
keep_dims=True)
deep_branch_feat = net
# if (gate_proj_type != -1) and (gate_aug_type != -1):
# raise ValueError('Either gate_proj_type or gate_aug_type can be activated at a time.')
#
# if not gate_aug_type == -1:
# # Augmenting pool5 features with gates
# net = MoEL_utils.augmenting_gates(net, gate_aug_type, gate_prop_down, is_training,
# concat_gate_reg=concat_gate_reg,
# concat_gate_reg_type=concat_gate_reg_type)
if not feat_proj_type == -1:
# projecting hidden feats and/or deep fc features to the same dimension and fuse them up
net = resnet_utils.projecting_feats(net,
feat_proj_type,
proj_dim,
feat_prop_down,
is_training,
avr_cc=avr_cc)
if num_classes is not None:
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
normalizer_fn=None,
scope='logits')
if spatial_squeeze:
logits = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
# Convert end_points_collection into a dictionary of end_points.
end_points = slim.utils.convert_collection_to_dict(
end_points_collection)
if num_classes is not None:
end_points['predictions'] = slim.softmax(
net, scope='predictions')
if extra_fc_type >= 0:
end_points['pre_pool5'] = pre_pool5
elif global_pool:
end_points['deep_branch_feat'] = deep_branch_feat
end_points['PreLogits'] = tf.squeeze(deep_branch_feat,
[1, 2],
name='PreLogits')
end_points['Logits'] = logits
return logits, end_points
