def testEndPointsV2(self):
"""Test the end points of a tiny v2 bottleneck network."""
bottleneck = resnet_v2.bottleneck
blocks = [
resnet_utils.Block("block1", bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block("block2", bottleneck, [(8, 2, 1), (8, 2, 1)]),
]
inputs = create_test_input(2, 32, 16, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope="tiny")
expected = [
"tiny/block1/unit_1/bottleneck_v2/shortcut",
"tiny/block1/unit_1/bottleneck_v2/conv1",
"tiny/block1/unit_1/bottleneck_v2/conv2",
"tiny/block1/unit_1/bottleneck_v2/conv3",
"tiny/block1/unit_2/bottleneck_v2/conv1",
"tiny/block1/unit_2/bottleneck_v2/conv2",
"tiny/block1/unit_2/bottleneck_v2/conv3",
"tiny/block2/unit_1/bottleneck_v2/shortcut",
"tiny/block2/unit_1/bottleneck_v2/conv1",
"tiny/block2/unit_1/bottleneck_v2/conv2",
"tiny/block2/unit_1/bottleneck_v2/conv3",
"tiny/block2/unit_2/bottleneck_v2/conv1",
"tiny/block2/unit_2/bottleneck_v2/conv2",
"tiny/block2/unit_2/bottleneck_v2/conv3",
]
self.assertItemsEqual(expected, end_points)
def _extract_box_classifier_features(self, proposal_feature_maps, scope):
"""Extracts second stage box classifier features.
Args:
proposal_feature_maps: A 4-D float tensor with shape
[batch_size * self.max_num_proposals, crop_height, crop_width, depth]
representing the feature map cropped to each proposal.
scope: A scope name (unused).
Returns:
proposal_classifier_features: A 4-D float tensor with shape
[batch_size * self.max_num_proposals, height, width, depth]
representing box classifier features for each proposal.
"""
with tf.variable_scope(self._architecture, reuse=self._reuse_weights):
with slim.arg_scope(
resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=self._weight_decay)):
with slim.arg_scope([slim.batch_norm],
is_training=self._train_batch_norm):
blocks = [
resnet_utils.Block('block4', resnet_v1.bottleneck,
[{
'depth': 2048,
'depth_bottleneck': 512,
'stride': 1
}] * 3)
]
proposal_classifier_features = resnet_utils.stack_blocks_dense(
proposal_feature_maps, blocks)
return proposal_classifier_features
def testEndPointsV1(self):
"""Test the end points of a tiny v1 bottleneck network."""
bottleneck = resnet_v1.bottleneck
blocks = [
resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 1)])
]
inputs = create_test_input(2, 32, 16, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
expected = [
'tiny/block1/unit_1/bottleneck_v1/shortcut',
'tiny/block1/unit_1/bottleneck_v1/conv1',
'tiny/block1/unit_1/bottleneck_v1/conv2',
'tiny/block1/unit_1/bottleneck_v1/conv3',
'tiny/block1/unit_2/bottleneck_v1/conv1',
'tiny/block1/unit_2/bottleneck_v1/conv2',
'tiny/block1/unit_2/bottleneck_v1/conv3',
'tiny/block2/unit_1/bottleneck_v1/shortcut',
'tiny/block2/unit_1/bottleneck_v1/conv1',
'tiny/block2/unit_1/bottleneck_v1/conv2',
'tiny/block2/unit_1/bottleneck_v1/conv3',
'tiny/block2/unit_2/bottleneck_v1/conv1',
'tiny/block2/unit_2/bottleneck_v1/conv2',
'tiny/block2/unit_2/bottleneck_v1/conv3'
]
self.assertItemsEqual(expected, end_points)
def create_resnet_model(img_dim):
pre_image = tf.placeholder(tf.float32, [None, None, 3])
processed_image = cnn_preprocessing.preprocess_for_eval(pre_image/255.0, img_dim, img_dim)
images = tf.placeholder(tf.float32, [None, img_dim, img_dim, 3])
# mean = tf.constant([123.68, 116.779, 103.939], dtype=tf.float32, shape=[1, 1, 1, 3], name='img_mean')
# processed_images = images - mean
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
probs, endpoints = resnet_v2.resnet_v2_152(images, num_classes=1001, is_training = False)
print(endpoints['resnet_v2_152/block4'])
init_fn = slim.assign_from_checkpoint_fn(
'Data/CNNModels/resnet_v2_152.ckpt',
slim.get_model_variables('resnet_v2_152'))
sess = tf.Session()
init_fn(sess)
return {
'images_placeholder' : images,
'block4' : endpoints['resnet_v2_152/block4'],
'session' : sess,
'processed_image' : processed_image,
'pre_image' : pre_image,
'probs' : probs
}
def _build_tail(self, inputs, is_training=False):
if not self._use_tail:
return inputs
if self._architecture == 'resnet_v1_101':
train_batch_norm = (is_training
and self._config.get('train_batch_norm'))
with self._enter_variable_scope():
weight_decay = (self._config.get('arg_scope',
{}).get('weight_decay', 0))
with tf.variable_scope(self._architecture, reuse=True):
resnet_arg_scope = resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=weight_decay)
with slim.arg_scope(resnet_arg_scope):
with slim.arg_scope([slim.batch_norm],
is_training=train_batch_norm):
blocks = [
resnet_utils.Block('block4',
resnet_v1.bottleneck,
[{
'depth': 2048,
'depth_bottleneck': 512,
'stride': 1
}] * 3)
]
proposal_classifier_features = (
resnet_utils.stack_blocks_dense(
inputs, blocks))
else:
proposal_classifier_features = inputs
return proposal_classifier_features
def testEndPointsV1(self):
"""Test the end points of a tiny v1 bottleneck network."""
bottleneck = resnet_v1.bottleneck
blocks = [resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 1)])]
inputs = create_test_input(2, 32, 16, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_plain(inputs, blocks, scope='tiny')
expected = [
'tiny/block1/unit_1/bottleneck_v1/shortcut',
'tiny/block1/unit_1/bottleneck_v1/shortcut/BatchNorm',
'tiny/block1/unit_1/bottleneck_v1/conv1',
'tiny/block1/unit_1/bottleneck_v1/conv2',
'tiny/block1/unit_1/bottleneck_v1/conv3',
'tiny/block1/unit_1/bottleneck_v1/conv3/BatchNorm',
'tiny/block1/unit_2/bottleneck_v1/conv1',
'tiny/block1/unit_2/bottleneck_v1/conv2',
'tiny/block1/unit_2/bottleneck_v1/conv3',
'tiny/block1/unit_2/bottleneck_v1/conv3/BatchNorm',
'tiny/block2/unit_1/bottleneck_v1/shortcut',
'tiny/block2/unit_1/bottleneck_v1/shortcut/BatchNorm',
'tiny/block2/unit_1/bottleneck_v1/conv1',
'tiny/block2/unit_1/bottleneck_v1/conv2',
'tiny/block2/unit_1/bottleneck_v1/conv3',
'tiny/block2/unit_1/bottleneck_v1/conv3/BatchNorm',
'tiny/block2/unit_2/bottleneck_v1/conv1',
'tiny/block2/unit_2/bottleneck_v1/conv2',
'tiny/block2/unit_2/bottleneck_v1/conv3',
'tiny/block2/unit_2/bottleneck_v1/conv3/BatchNorm']
self.assertItemsEqual(expected, end_points)
def testAtrousFullyConvolutionalValues(self):
"""Verify dense feature extraction with atrous convolution."""
nominal_stride = 32
for output_stride in [4, 8, 16, 32, None]:
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(2, 81, 81, 3)
# Dense feature extraction followed by subsampling.
output, _ = self._resnet_small(inputs,
None,
is_training=False,
global_pool=False,
output_stride=output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected, _ = self._resnet_small(inputs,
None,
is_training=False,
global_pool=False)
sess.run(tf.global_variables_initializer())
self.assertAllClose(output.eval(), expected.eval(),
atol=1e-4, rtol=1e-4)
def testAtrousFullyConvolutionalValues(self):
"""Verify dense feature extraction with atrous convolution."""
nominal_stride = 32
for output_stride in [4, 8, 16, 32, None]:
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(2, 81, 81, 3)
# Dense feature extraction followed by subsampling.
output, _ = self._resnet_small(
inputs,
None,
is_training=False,
global_pool=False,
output_stride=output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected, _ = self._resnet_small(inputs,
None,
is_training=False,
global_pool=False)
sess.run(tf.global_variables_initializer())
self.assertAllClose(output.eval(),
expected.eval(),
atol=1e-4,
rtol=1e-4)
def testClassificationEndPoints(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = self._resnet_small(inputs, num_classes, global_pool, scope="resnet")
self.assertTrue(logits.op.name.startswith("resnet/logits"))
self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes])
self.assertTrue("predictions" in end_points)
self.assertListEqual(end_points["predictions"].get_shape().as_list(), [2, 1, 1, num_classes])
def testClassificationEndPoints(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = self._resnet_small(inputs, num_classes, global_pool,
scope='resnet')
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes])
self.assertTrue('predictions' in end_points)
self.assertListEqual(end_points['predictions'].get_shape().as_list(),
[2, 1, 1, num_classes])
def resnet_v1_101_beta(images, is_training):
"""ResNet-101 v1 beta.
Replace first 7*7 conv layers with three 3*3 conv layers.
"""
fine_tune_batch_norm = False
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
feature, end_points = resnet_v1_beta.resnet_v1_101_beta(
images,
num_classes=NUMBER_OUTPUT,
is_training=(is_training and fine_tune_batch_norm),
global_pool=True)
return feature
def testFullyConvolutionalUnknownHeightWidth(self):
batch = 2
height, width = 65, 65
global_pool = False
inputs = create_test_input(batch, None, None, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
output, _ = self._resnet_small(inputs, None, global_pool)
self.assertListEqual(output.get_shape().as_list(), [batch, None, None, 32])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
output = sess.run(output, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 3, 3, 32))
def _extract_proposal_features(self, preprocessed_inputs, scope):
"""Extracts first stage RPN features.
Args:
preprocessed_inputs: A [batch, height, width, channels] float32 tensor
representing a batch of images.
scope: A scope name.
Returns:
rpn_feature_map: A tensor with shape [batch, height, width, depth]
activations: A dictionary mapping feature extractor tensor names to
tensors
Raises:
InvalidArgumentError: If the spatial size of `preprocessed_inputs`
(height or width) is less than 33.
ValueError: If the created network is missing the required activation.
"""
if len(preprocessed_inputs.get_shape().as_list()) != 4:
raise ValueError(
'`preprocessed_inputs` must be 4 dimensional, got a '
'tensor of shape %s' % preprocessed_inputs.get_shape())
shape_assert = tf.Assert(
tf.logical_and(
tf.greater_equal(tf.shape(preprocessed_inputs)[1], 33),
tf.greater_equal(tf.shape(preprocessed_inputs)[2], 33)),
['image size must at least be 33 in both height and width.'])
with tf.control_dependencies([shape_assert]):
# Disables batchnorm for fine-tuning with smaller batch sizes.
# TODO(chensun): Figure out if it is needed when image
# batch size is bigger.
with slim.arg_scope(
resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=self._weight_decay)):
with tf.variable_scope(self._architecture,
reuse=self._reuse_weights) as var_scope:
_, activations = self._resnet_model(
preprocessed_inputs,
num_classes=None,
is_training=self._train_batch_norm,
global_pool=False,
output_stride=self._first_stage_features_stride,
spatial_squeeze=False,
scope=var_scope)
handle = scope + '/%s/block3' % self._architecture
return activations[handle], activations
def testFullyConvolutionalUnknownHeightWidth(self):
batch = 2
height, width = 65, 65
global_pool = False
inputs = create_test_input(batch, None, None, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
output, _ = self._resnet_small(inputs, None, global_pool)
self.assertListEqual(output.get_shape().as_list(),
[batch, None, None, 32])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
output = sess.run(output, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 3, 3, 32))
def __call__(self, ens_x_input, vgg_x_input, inc_x_input, tcd_x_input):
"""Constructs model and return probabilities for given input."""
reuse = True if self.built else None
logits = None
aux_logits = None
weights = [[0.7, 0.1], [0.2, 0.1]]
all_inputs = [[ens_x_input, tcd_x_input], [inc_x_input, tcd_x_input]]
scopes = [
inception_resnet_v2.inception_resnet_v2_arg_scope(),
inception.inception_v3_arg_scope()
]
reuse_flags = [reuse, True]
for model_idx, model in enumerate(
[inception_resnet_v2.inception_resnet_v2, inception.inception_v3]):
with slim.arg_scope(scopes[model_idx]):
for idx, inputs in enumerate(all_inputs[model_idx]):
result = model(inputs,
num_classes=self.num_classes,
is_training=False,
reuse=reuse_flags[idx])
weight = weights[model_idx][idx]
# :1 is for slicing out the background class
if logits == None:
logits = result[0][:, 1:] * weight
aux_logits = result[1]['AuxLogits'][:, 1:] * weight
else:
logits += result[0][:, 1:] * weight
aux_logits += result[1]['AuxLogits'][:, 1:] * weight
with slim.arg_scope(vgg.vgg_arg_scope()):
weight = 0.1
result = vgg.vgg_16(vgg_x_input,
num_classes=1000,
is_training=False)
logits += result[0] * weight
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
weight = 0.05
result = resnet_v2.resnet_v2_152(vgg_x_input,
num_classes=self.num_classes,
reuse=reuse)
logits += tf.squeeze(result[0])[:, 1:] * weight
self.built = True
aux_weight = 0.8
logits += aux_logits * aux_weight
predictions = layers_lib.softmax(logits)
return predictions
def testClassificationShapes(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes, global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 28, 28, 4],
'resnet/block2': [2, 14, 14, 8],
'resnet/block3': [2, 7, 7, 16],
'resnet/block4': [2, 7, 7, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 321, 321, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes, global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 41, 41, 4],
'resnet/block2': [2, 21, 21, 8],
'resnet/block3': [2, 11, 11, 16],
'resnet/block4': [2, 11, 11, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testClassificationShapes(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes, global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 28, 28, 4],
'resnet/block2': [2, 14, 14, 8],
'resnet/block3': [2, 7, 7, 16],
'resnet/block4': [2, 7, 7, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 321, 321, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs, num_classes, global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/block1': [2, 41, 41, 4],
'resnet/block2': [2, 21, 21, 8],
'resnet/block3': [2, 11, 11, 16],
'resnet/block4': [2, 11, 11, 32]}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testUnknownBatchSize(self):
batch = 2
height, width = 65, 65
global_pool = True
num_classes = 10
inputs = create_test_input(None, height, width, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, _ = self._resnet_small(inputs, num_classes, global_pool, scope="resnet")
self.assertTrue(logits.op.name.startswith("resnet/logits"))
self.assertListEqual(logits.get_shape().as_list(), [None, 1, 1, num_classes])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
output = sess.run(logits, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 1, 1, num_classes))
def testClassificationEndPoints(self):
globals_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = self._resnet_small(inputs,
num_classes,
global_pool=globals_pool,
scope='resnet')
#assert the expression is true.in this environment has a name
# of scape starting with 'resnet/logits'
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(),
[2, 1, 1, num_classes])
self.assertTrue('prediction' in end_points)
self.assertListEqual(end_points['predictions'.get_shape().as_list(),
[2, 1, 1, num_classes]])
def testClassificationEndPointsWithMultigrid(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
multi_grid = [1, 2, 4]
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = self._resnet_small(inputs,
num_classes,
global_pool=global_pool,
multi_grid=multi_grid,
scope='resnet')
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes])
self.assertTrue('predictions' in end_points)
self.assertListEqual(end_points['predictions'].get_shape().as_list(),
[2, 1, 1, num_classes])
def testRootlessFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 128, 128, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(
inputs, num_classes, global_pool, include_root_block=False, scope="resnet"
)
endpoint_to_shape = {
"resnet/block1": [2, 64, 64, 4],
"resnet/block2": [2, 32, 32, 8],
"resnet/block3": [2, 16, 16, 16],
"resnet/block4": [2, 16, 16, 32],
}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def encode_with_resnet(self, images, global_pool=False, output_stride=8):
self.global_pool = global_pool # needed for artus convolution
self.output_stride = output_stride # needed for artus convolution
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = resnet_v1.resnet_v1_101(
images,
global_pool=self.global_pool,
output_stride=self.output_stride)
#size = tf.slice(tf.shape(images), [1], [2]) #TODO: multiply by 0.5
size = [FLAGS.output_height,
FLAGS.output_width] #TODO: chenged fixed size
resized_logits = tf.image.resize_images(
logits,
size,
method=tf.image.ResizeMethod.BILINEAR,
align_corners=False)
return resized_logits
def testUnknownBatchSize(self):
batch = 2
height, width = 65, 65
global_pool = True
num_classes = 10
inputs = create_test_input(None, height, width, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, _ = self._resnet_small(inputs, num_classes, global_pool,
scope='resnet')
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(),
[None, 1, 1, num_classes])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
output = sess.run(logits, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 1, 1, num_classes))
def _atrousValues(self, bottleneck):
"""Verify the values of dense feature extraction by atrous convolution.
Make sure that dense feature extraction by stack_blocks_dense() followed by
subsampling gives identical results to feature extraction at the nominal
network output stride using the simple self._stack_blocks_nondense() above.
Args:
bottleneck: The bottleneck function.
"""
blocks = [
resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 2)]),
resnet_utils.Block('block3', bottleneck, [(16, 4, 1), (16, 4, 2)]),
resnet_utils.Block('block4', bottleneck, [(32, 8, 1), (32, 8, 1)])
]
nominal_stride = 8
# Test both odd and even input dimensions.
height = 30
width = 31
with slim.arg_scope(resnet_utils.resnet_arg_scope(is_training=False)):
for output_stride in [1, 2, 4, 8, None]:
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(1, height, width, 3)
# Dense feature extraction followed by subsampling.
output = resnet_utils.stack_blocks_dense(
inputs, blocks, output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected = self._stack_blocks_nondense(inputs, blocks)
sess.run(tf.initialize_all_variables())
output, expected = sess.run([output, expected])
self.assertAllClose(output,
expected,
atol=1e-4,
rtol=1e-4)
def testClassificationEndPointsWithMultigridAndLiteBottleneck(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
multi_grid = [1, 2]
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
logits, end_points = self._resnet_small_lite_bottleneck(
inputs,
num_classes,
global_pool=global_pool,
multi_grid=multi_grid,
scope='resnet')
self.assertTrue(logits.op.name.startswith('resnet/logits'))
self.assertListEqual(logits.get_shape().as_list(), [2, 1, 1, num_classes])
self.assertIn('predictions', end_points)
self.assertListEqual(end_points['predictions'].get_shape().as_list(),
[2, 1, 1, num_classes])
def testAtrousFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
output_stride = 8
inputs = create_test_input(2, 321, 321, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(
inputs, num_classes, global_pool, output_stride=output_stride, scope="resnet"
)
endpoint_to_shape = {
"resnet/block1": [2, 41, 41, 4],
"resnet/block2": [2, 41, 41, 8],
"resnet/block3": [2, 41, 41, 16],
"resnet/block4": [2, 41, 41, 32],
}
for endpoint in endpoint_to_shape:
shape = endpoint_to_shape[endpoint]
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testAtrousFullyConvolutionValue(self):
"""Verify dense feature extraction with atrous convolution."""
#从这个测试我们可以看出在带孔卷积加入后，输出了在相对自己solution
#的位置的pix 是不变的，只是分辨率变大了
nominal_stride = 32
for output_stride in [4, 8, 16, 32, None]:
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(2, 81, 81, 3)
# Dense feature extraction followed by subsampling.
output, _ = self._resnet_small(
inputs,
None,
is_training=False,
global_pool=False,
output_stride=output_stride)
if output_stride is None:
factor = 1
else:
#采用带孔卷积我们get a dense output 而另外
#使用normal conv output 要小很多
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Featrue extraction at the nominal network rate.
expected, _ = self._resnet_small(inputs,
None,
is_training=False,
global_pool=False)
# make the variable has their own value in the model
sess.run(tf.global_variable_initializer())
#如果你有一个Tensor t，在使用t.eval()时，等价于：tf.get_default_session().run(t).
'''Calling this method will execute all preceding operations that produce the inputs needed for
the operation that produces this tensor. N.B. Before invoking Tensor.eval(), its graph must have
been launched in a session,and either a default session must be available, or session must be
specified explicitly.'''
self.assertAllClose(output.eval(),
expected.eval(),
atol=1e-4,
rtol=1e-4)
def _atrousValues(self, bottleneck):
"""Verify the values of dense feature extraction by atrous convolution.
Make sure that dense feature extraction by stack_blocks_dense() followed by
subsampling gives identical results to feature extraction at the nominal
network output stride using the simple self._stack_blocks_nondense() above.
Args:
bottleneck: The bottleneck function.
"""
blocks = [
resnet_utils.Block('block1', bottleneck, [(4, 1, 1), (4, 1, 2)]),
resnet_utils.Block('block2', bottleneck, [(8, 2, 1), (8, 2, 2)]),
resnet_utils.Block('block3', bottleneck, [(16, 4, 1), (16, 4, 2)]),
resnet_utils.Block('block4', bottleneck, [(32, 8, 1), (32, 8, 1)])
]
nominal_stride = 8
# Test both odd and even input dimensions.
height = 30
width = 31
with slim.arg_scope(resnet_utils.resnet_arg_scope(is_training=False)):
for output_stride in [1, 2, 4, 8, None]:
with tf.Graph().as_default():
with self.test_session() as sess:
tf.set_random_seed(0)
inputs = create_test_input(1, height, width, 3)
# Dense feature extraction followed by subsampling.
output = resnet_utils.stack_blocks_dense(inputs,
blocks,
output_stride)
if output_stride is None:
factor = 1
else:
factor = nominal_stride // output_stride
output = resnet_utils.subsample(output, factor)
# Make the two networks use the same weights.
tf.get_variable_scope().reuse_variables()
# Feature extraction at the nominal network rate.
expected = self._stack_blocks_nondense(inputs, blocks)
sess.run(tf.global_variables_initializer())
output, expected = sess.run([output, expected])
self.assertAllClose(output, expected, atol=1e-4, rtol=1e-4)
def testAtrousFullyConvolutionalUnknownHeightWidthWithLiteBottleneck(self):
batch = 2
height, width = 65, 65
global_pool = False
output_stride = 8
inputs = create_test_input(batch, None, None, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
output, _ = self._resnet_small_lite_bottleneck(
inputs,
None,
global_pool=global_pool,
output_stride=output_stride)
self.assertListEqual(output.get_shape().as_list(),
[batch, None, None, 8])
images = create_test_input(batch, height, width, 3)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
output = sess.run(output, {inputs: images.eval()})
self.assertEqual(output.shape, (batch, 9, 9, 8))
def resnet_v1_50_beta_lstm(images, is_training):
"""ResNet-50 v1 beta with lstm head.
Replace first 7*7 conv layers with three 3*3 conv layers.
"""
fine_tune_batch_norm = False
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
feature, end_points = resnet_v1_beta.resnet_v1_50_beta(
images,
num_classes=None,
is_training=(is_training and fine_tune_batch_norm),
global_pool=True)
with tf.variable_scope('lstm'):
# feature has shape (batch_size, feature_size)
feature = tf.squeeze(feature, axis=(1, 2))
# feature has shape (1, batch_size, feature_size)
feature = tf.expand_dims(feature, axis=0)
# feature has shape (time_step, batch_size, feature_size)
feature = tf.tile(feature, (2, 1, 1))
lstm = tf.contrib.cudnn_rnn.CudnnLSTM(num_layers=1, num_units=128)
# lstm_output has shape (time_step, batch_size, 128)
lstm_output, state = lstm(feature)
# each element in lstm_output_list has shape (batch_size, 128)
lstm_output_list = tf.unstack(lstm_output, axis=0)
output_list = []
with tf.variable_scope('logits'):
with slim.arg_scope(
[slim.fully_connected],
activation_fn=None,
weights_regularizer=slim.l2_regularizer(WEIGHT_DECAY)):
for element in lstm_output_list:
feature = slim.fully_connected(element,
NUMBER_OUTPUT // 2,
activation_fn=None,
normalizer_fn=None)
output_list.append(feature)
# output has shape (batch_size, NUMBER_OUTPUT)
output = tf.concat(output_list, axis=1)
return output
def testFullyConvolutionalEndpointShapesWithLiteBottleneck(self):
global_pool = False
num_classes = 10
inputs = create_test_input(2, 321, 321, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small_lite_bottleneck(
inputs,
num_classes,
global_pool=global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/conv1_1': [2, 161, 161, 16],
'resnet/conv1_2': [2, 161, 161, 16],
'resnet/conv1_3': [2, 161, 161, 32],
'resnet/block1': [2, 41, 41, 1],
'resnet/block2': [2, 21, 21, 2],
'resnet/block3': [2, 11, 11, 4],
'resnet/block4': [2, 11, 11, 8]}
for endpoint, shape in six.iteritems(endpoint_to_shape):
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testClassificationShapesWithLiteBottleneck(self):
global_pool = True
num_classes = 10
inputs = create_test_input(2, 224, 224, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small_lite_bottleneck(
inputs,
num_classes,
global_pool=global_pool,
scope='resnet')
endpoint_to_shape = {
'resnet/conv1_1': [2, 112, 112, 16],
'resnet/conv1_2': [2, 112, 112, 16],
'resnet/conv1_3': [2, 112, 112, 32],
'resnet/block1': [2, 28, 28, 1],
'resnet/block2': [2, 14, 14, 2],
'resnet/block3': [2, 7, 7, 4],
'resnet/block4': [2, 7, 7, 8]}
for endpoint, shape in six.iteritems(endpoint_to_shape):
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def testAtrousFullyConvolutionalEndpointShapes(self):
global_pool = False
num_classes = 10
output_stride = 8
inputs = create_test_input(2, 321, 321, 3)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
_, end_points = self._resnet_small(inputs,
num_classes,
global_pool=global_pool,
output_stride=output_stride,
scope='resnet')
endpoint_to_shape = {
'resnet/conv1_1': [2, 161, 161, 64],
'resnet/conv1_2': [2, 161, 161, 64],
'resnet/conv1_3': [2, 161, 161, 128],
'resnet/block1': [2, 41, 41, 4],
'resnet/block2': [2, 41, 41, 8],
'resnet/block3': [2, 41, 41, 16],
'resnet/block4': [2, 41, 41, 32]}
for endpoint, shape in endpoint_to_shape.iteritems():
self.assertListEqual(end_points[endpoint].get_shape().as_list(), shape)
def _build_tail(self, inputs, is_training=False):
if not self._use_tail:
return inputs
if self._architecture == 'resnet_v1_101':
train_batch_norm = (
is_training and self._config.get('train_batch_norm')
)
with self._enter_variable_scope():
weight_decay = (
self._config.get('arg_scope', {}).get('weight_decay', 0)
)
with tf.variable_scope(self._architecture, reuse=True):
resnet_arg_scope = resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=weight_decay
)
with slim.arg_scope(resnet_arg_scope):
with slim.arg_scope(
[slim.batch_norm], is_training=train_batch_norm
):
blocks = [
resnet_utils.Block(
'block4',
resnet_v1.bottleneck,
[{
'depth': 2048,
'depth_bottleneck': 512,
'stride': 1
}] * 3
)
]
proposal_classifier_features = (
resnet_utils.stack_blocks_dense(inputs, blocks)
)
else:
proposal_classifier_features = inputs
return proposal_classifier_features
def _extract_features(self, preprocessed_inputs, scope):
half_res_scope = scope + '/%s/block1' % self._architecture
quarter_res_scope = scope + '/%s/block4' % self._architecture
psp_aux_scope = scope + '/%s/block2' % self._architecture
with slim.arg_scope(
resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=self._weight_decay)):
_, activations = self._resnet_model(
preprocessed_inputs,
filter_scale=self._filter_scale,
#mid_downsample=self._mid_downsample, #removed for resnet_beta
num_classes=None,
is_training=(self._is_training and self._train_batch_norm),
global_pool=False,
output_stride=self._features_stride)
half_res_features = activations[half_res_scope]
quarter_res_features = activations[quarter_res_scope]
psp_aux_features = activations[psp_aux_scope]
return half_res_features, quarter_res_features, psp_aux_features
def resnet_rcnn(features, num_classes):
resnet_arg_scope = resnet_utils.resnet_arg_scope(batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=5e-4)
with slim.arg_scope(resnet_arg_scope):
with slim.arg_scope([slim.batch_norm], is_training=True):
blocks = [
resnet_utils.Block('block4', resnet_v1.bottleneck,
[{
'depth': 2048,
'depth_bottleneck': 512,
'stride': 1
}] * 3)
]
net = resnet_utils.stack_blocks_dense(features, blocks)
flat = tf.layers.flatten(net)
cls_score = tf.layers.dense(flat, units=(num_classes + 1))
reg_score = tf.layers.dense(flat, units=(num_classes * 4))
cls_probs = tf.nn.softmax(cls_score)
return cls_score, cls_probs, reg_score
def _build_tail(self, inputs, is_training=False):
if not self._use_tail:
return inputs
if self._architecture == "resnet_v1_101":
train_batch_norm = is_training and self._config.get(
"train_batch_norm")
with self._enter_variable_scope():
weight_decay = self._config.get("arg_scope",
{}).get("weight_decay", 0)
with tf.variable_scope(self._architecture, reuse=True):
resnet_arg_scope = resnet_utils.resnet_arg_scope(
batch_norm_epsilon=1e-5,
batch_norm_scale=True,
weight_decay=weight_decay,
)
with slim.arg_scope(resnet_arg_scope):
with slim.arg_scope([slim.batch_norm],
is_training=train_batch_norm):
blocks = [
resnet_utils.Block(
"block4",
resnet_v1.bottleneck,
[{
"depth": 2048,
"depth_bottleneck": 512,
"stride": 1,
}] * 3,
)
]
proposal_classifier_features = (
resnet_utils.stack_blocks_dense(
inputs, blocks))
else:
proposal_classifier_features = inputs
return proposal_classifier_features
def __restore_model(self):
with tf.device(self._Graph__device):
with self.__sess.as_default():
with self._Graph__graph.as_default():
self.images = tf.placeholder(
shape=[None, config.IMAGE_H, config.IMAGE_W, 1],
dtype=tf.float32)
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
self.logits, _ = resnet_v2.resnet_v2_200(
self.images, config.NUM_CLASSES)
self.net = tf.squeeze(self.logits)
self.probs = tf.nn.softmax(self.net)
self.top_5_values, self.top_5_idx = tf.nn.top_k(
self.probs, 5)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
config.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore(
)
self.__saver_cls = tf.train.Saver(variables_to_restore)
print("All variables are loaded!")
# print("Meta-Graph Restored.")
ckpt = tf.train.get_checkpoint_state(
self._Graph__model_ckpt)
if ckpt and ckpt.model_checkpoint_path:
# Restores from checkpoint
self.__saver_cls.restore(self.__sess,
ckpt.model_checkpoint_path)
gbl_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
print("Model loaded successfully")
self._Graph__graph = tf.get_default_graph()
def build_model(self):
with tf.name_scope('input'):
input = self.input_dict['X']
targets = self.input_dict['Y_softmax']
targets_one_hot = tf.one_hot(targets, self.nb_classes + 1)
# with slim.arg_scope(inception.inception_v3_arg_scope()):
# net, endpoints = inception.inception_v3(input, is_training=self.is_train)
# zzz = 0
with slim.arg_scope(
resnet_utils.resnet_arg_scope(is_training=self.is_train)):
net, end_points = resnet_v2.resnet_v2_101(input,
num_classes=None,
global_pool=False,
output_stride=16)
zzz = 0
with tf.name_scope('upsamplig'):
# upsampling
# first, upsample x2
net = slim.conv2d_transpose(net,
256, [4, 4],
stride=2,
scope='upsample_1')
block1_scored = slim.conv2d(end_points['resnet_v2_101/block1'],
256, [1, 1],
scope='upsample_1_scored',
activation_fn=None)
net = net + block1_scored
# second, upsample x2
net = slim.conv2d_transpose(net,
128, [4, 4],
stride=2,
scope='upsample_2')
block2_scored = slim.conv2d(
end_points['resnet_v2_101/block1/unit_1/bottleneck_v2'],
128, [1, 1],
scope='upsample_2_scored',
activation_fn=None)
net = net + block2_scored
# finally, upsample x4
net = slim.conv2d_transpose(net,
64, [16, 16],
stride=4,
scope='upsample_3')
# add a few conv layers as the output
net = slim.conv2d(net, 32, [3, 3], scope='conv8_1')
net = slim.conv2d(net, 32, [3, 3], scope='conv8_2')
########
# Logits
with slim.arg_scope([
slim.conv2d,
],
normalizer_fn=slim.batch_norm,
weights_regularizer=slim.l2_regularizer(
self.regularization)):
logits = slim.conv2d(net,
self.nb_classes + 1, [1, 1],
scope='conv_final_classes',
activation_fn=None)
with tf.name_scope('prediction'):
classes_probs = tf.nn.softmax(logits)
self.op_predict = classes_probs
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=targets_one_hot, logits=logits)
loss_ce = tf.reduce_mean(tf.reduce_sum(cross_entropy, axis=[1, 2]))
tf.losses.add_loss(loss_ce)
total_loss = tf.losses.get_total_loss(
add_regularization_losses=True)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if update_ops:
updates = tf.group(*update_ops)
with tf.control_dependencies([updates]):
total_loss = tf.identity(total_loss)
self.op_loss = total_loss
sys.path.append(TF_API)
from deeplab.core import resnet_v1_beta
from tensorflow.contrib.slim.nets import resnet_utils
slim = tf.contrib.slim
if __name__ == '__main__':
inputs = tf.random_normal([1, 224, 224, 3])
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
net, end_points = resnet_v1_beta.resnet_v1_101(inputs,
num_classes=100,
is_training=False,
global_pool=True,
output_stride=None,
multi_grid=None,
reuse=None,
scope='resnet_v1_101')
writer = tf.summary.FileWriter("./logs", graph=tf.get_default_graph())
print("Layers")
for k, v in end_points.items():
def early_fusion_cnn(observation, element_count, element_length, model_config):
"""A cnn that maps 1+ images with 1+ chanels to a feature vector."""
inputs = observation.tensor
if model_config.hparams.cnna == 'cnn':
embedding = net_util.inputs_to_feature_vector(inputs, 1024, model_config)
elif model_config.hparams.cnna in ['r18', 'r50', 'h50', 'k50', 's50']:
batch_size, image_count, height, width, channel_count = (
inputs.get_shape().as_list())
log.verbose('Input shape to early-fusion cnn: %s' %
str(inputs.get_shape().as_list()))
if image_count == 1:
im = tf.reshape(inputs, [batch_size, height, width, channel_count])
else:
im = tf.reshape(
tf.transpose(inputs, perm=[0, 2, 3, 1, 4]),
[batch_size, height, width, image_count * channel_count])
if model_config.hparams.cnna == 'r18':
raise ValueError('ResNet18 is no longer supported.')
elif model_config.hparams.cnna == 'r50':
raise ValueError('r50 network is no longer supported.')
elif model_config.hparams.cnna == 'k50':
log.warning('Using a keras based model.')
resnet = tf.compat.v1.keras.applications.ResNet50V2(
include_top=False,
weights=None,
input_tensor=None,
input_shape=(height, width, image_count * channel_count),
pooling=None)
embedding = resnet(im)
elif model_config.hparams.cnna == 's50':
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
embedding_prenorm, _ = contrib_slim_resnet_v2.resnet_v2_50(
inputs=im,
num_classes=2048,
is_training=model_config.train,
global_pool=True,
reuse=tf.AUTO_REUSE,
scope='resnet_v2_50')
embedding_prenorm = tf.reshape(embedding_prenorm,
[model_config.hparams.bs, 2048])
embedding = tf.nn.l2_normalize(embedding_prenorm, axis=1)
elif model_config.hparams.cnna == 'h50':
log.warning('TF Hub not tested externally.')
resnet = hub.Module(
'https://tfhub.dev/google/imagenet/resnet_v2_50/feature_vector/1',
trainable=True)
expected_height, expected_width = hub.get_expected_image_size(resnet)
if channel_count == 1:
im = tf.tile(im, [1, 1, 1, 3])
if height != expected_height or width != expected_width:
raise ValueError(
('The input tensor has shape %s, but this tf.Hub()'
' r50 expects [%i, %i, 3].') %
(repr(im.get_shape().as_list()), expected_height, expected_width))
embedding = resnet(im)
log.verbose('Embedding shape: %s' % repr(embedding.get_shape().as_list()))
current_total_dimensionality = functools.reduce(
lambda x, y: x * y,
embedding.get_shape().as_list()[1:])
embedding = tf.reshape(
embedding, [model_config.hparams.bs, current_total_dimensionality])
net = embedding
normalizer, normalizer_params = net_util.get_normalizer_and_mode(model_config)
for _ in range(2):
net = contrib_layers.fully_connected(
inputs=net,
num_outputs=2048,
activation_fn=tf.nn.leaky_relu,
normalizer_fn=normalizer,
normalizer_params=normalizer_params)
prediction = contrib_layers.fully_connected(
inputs=net,
num_outputs=element_count * element_length,
activation_fn=None,
normalizer_fn=None)
batch_size = inputs.get_shape().as_list()[0]
prediction = tf.reshape(prediction,
[batch_size, element_count, element_length])
return prediction, embedding
