def testAnchorReference(self):
# Test simple case with one aspect ratio and one scale.
base_size = 256
aspect_ratios = [1.]
scales = [1.]
anchor_reference = generate_anchors_reference(
base_size=base_size, aspect_ratios=aspect_ratios, scales=scales)
# Should return a single anchor.
self.assertEqual(anchor_reference.shape, (1, 4))
self.assertAllEqual(anchor_reference[0], [
-(base_size - 1) / 2.0, -(base_size - 1) / 2.0,
(base_size - 1) / 2.0, (base_size - 1) / 2.0
])
# Test with fixed ratio and different scales.
scales = np.array([0.5, 1., 2., 4.])
anchor_reference = generate_anchors_reference(
base_size=base_size, aspect_ratios=aspect_ratios, scales=scales)
# Check that we have the correct number of anchors.
self.assertEqual(anchor_reference.shape, (4, 4))
width_heights = self._get_widths_heights(anchor_reference)
# Check that anchors are squares (aspect_ratio = [1.0]).
self.assertTrue((width_heights[:, 0] == width_heights[:, 1]).all())
# Check that widths are consistent with scales times base_size.
self.assertAllEqual(width_heights[:, 0], base_size * scales)
# Check exact values.
self.assertAllEqual(
anchor_reference,
np.array([[-63.5, -63.5, 63.5, 63.5],
[-127.5, -127.5, 127.5, 127.5],
[-255.5, -255.5, 255.5, 255.5],
[-511.5, -511.5, 511.5, 511.5]]))
# Test with different ratios and scales.
scales = np.array([0.5, 1., 2.])
aspect_ratios = np.array([0.5, 1., 2.])
anchor_reference = generate_anchors_reference(
base_size=base_size, aspect_ratios=aspect_ratios, scales=scales)
# Check we have the correct number of anchors.
self.assertEqual(anchor_reference.shape,
(len(scales) * len(aspect_ratios), 4))
width_heights = self._get_widths_heights(anchor_reference)
# Check ratios of height / widths
anchor_ratios = width_heights[:, 1] / width_heights[:, 0]
# Check scales (applied to )
anchor_scales = np.sqrt(
(width_heights[:, 1] * width_heights[:, 0]) / (base_size**2))
# Test that all ratios are used in the correct order.
self.assertAllClose(anchor_ratios,
[0.5, 0.5, 0.5, 1., 1., 1., 2., 2., 2.])
# Test that all scales are used in the correct order.
self.assertAllClose(anchor_scales,
[0.5, 1., 2., 0.5, 1., 2., 0.5, 1., 2.])
def testInvalidValues(self):
# Should fail because base_size is too small to for that scale and
# ratio.
base_size = 1
aspect_ratios = [0.5]
scales = [0.5]
try:
generate_anchors_reference(base_size=base_size,
aspect_ratios=aspect_ratios,
scales=scales)
except ValueError:
return
self.fail('Should have thrown an exception.')
def __init__(self, config, name='fasterrcnn'):
super(FasterRCNN, self).__init__(name=name)
self._config = config
self._num_classes = config.model.network.num_classes
# Generate network with RCNN
self._with_rcnn = config.model.network.with_rcnn
self._debug = config.train.debug
self._seed = config.train.seed
self._anchor_base_size = config.model.anchors.base_size
self._anchor_scales = np.array(config.model.anchors.scales)
self._anchor_ratios = np.array(config.model.anchors.ratios)
self._anchor_stride = config.model.anchors.stride
self._anchor_reference = generate_anchors_reference(
self._anchor_base_size, self._anchor_ratios, self._anchor_scales
)
self._num_anchors = self._anchor_reference.shape[0]
# Weights used to sum each of the losses of the submodules
self._rpn_cls_loss_weight = config.model.loss.rpn_cls_loss_weight
self._rpn_reg_loss_weight = config.model.loss.rpn_reg_loss_weights
self._rcnn_cls_loss_weight = config.model.loss.rcnn_cls_loss_weight
self._rcnn_reg_loss_weight = config.model.loss.rcnn_reg_loss_weights
self._losses_collections = ['fastercnn_losses']
self.base_network = TruncatedBaseNetwork(config.model.base_network)
def __init__(self, config, name='fasterrcnn'):
super(FasterRCNN, self).__init__(name=name)
# Main configuration object, it holds not only the necessary
# information for this module but also configuration for each of the
# different submodules.
self._config = config
# Total number of classes to classify. If not using RCNN then it is not
# used. TODO: Make it *more* optional.
self._num_classes = config.model.network.num_classes
# Generate network with RCNN thus allowing for classification of
# objects and not just finding them.
self._with_rcnn = config.model.network.with_rcnn
# Turn on debug mode with returns more Tensors which can be used for
# better visualization and (of course) debugging.
self._debug = config.train.debug
self._seed = config.train.seed
# Anchor config, check out the docs of base_config.yml for a better
# understanding of how anchors work.
self._anchor_base_size = config.model.anchors.base_size
self._anchor_scales = np.array(config.model.anchors.scales)
self._anchor_ratios = np.array(config.model.anchors.ratios)
self._anchor_stride = config.model.anchors.stride
####
# Anchor reference for building dynamic anchors for each image in the
# computation graph.
self._anchor_reference = generate_anchors_reference(
self._anchor_base_size, self._anchor_ratios, self._anchor_scales
)
# Total number of anchors per point.
self._num_anchors = self._anchor_reference.shape[0]
# Weights used to sum each of the losses of the submodules
self._rpn_cls_loss_weight = config.model.loss.rpn_cls_loss_weight
self._rpn_reg_loss_weight = config.model.loss.rpn_reg_loss_weights
self._rcnn_cls_loss_weight = config.model.loss.rcnn_cls_loss_weight
self._rcnn_reg_loss_weight = config.model.loss.rcnn_reg_loss_weights
self._losses_collections = ['fastercnn_losses']
# We want the pretrained model to be outside the FasterRCNN name scope.
self.base_network = TruncatedBaseNetwork(config.model.base_network)
def testBasic(self):
"""Tests shapes are consistent with anchor generation.
"""
model = RPN(
self.num_anchors, self.config, debug=True
)
# (plus the batch number)
pretrained_output_shape = (1, 32, 32, 512)
pretrained_output = tf.placeholder(
tf.float32, shape=pretrained_output_shape)
# Estimate image shape from the pretrained output and the anchor stride
image_shape_val = (
int(pretrained_output_shape[1] * self.stride),
int(pretrained_output_shape[2] * self.stride),
)
# Use 4 ground truth boxes.
gt_boxes_shape = (4, 4)
gt_boxes = tf.placeholder(tf.float32, shape=gt_boxes_shape)
image_shape_shape = (2,)
image_shape = tf.placeholder(tf.float32, shape=image_shape_shape)
# Total anchors depends on the pretrained output shape and the total
# number of anchors per point.
total_anchors = (
pretrained_output_shape[1] * pretrained_output_shape[2] *
self.num_anchors
)
all_anchors_shape = (total_anchors, 4)
all_anchors = tf.placeholder(tf.float32, shape=all_anchors_shape)
layers = model(
pretrained_output, image_shape, all_anchors, gt_boxes=gt_boxes
)
with self.test_session() as sess:
# As in the case of a real session we need to initialize the
# variables.
sess.run(tf.global_variables_initializer())
layers_inst = sess.run(layers, feed_dict={
# We don't really care about the value of the pretrained output
# only that has the correct shape.
pretrained_output: np.random.rand(
*pretrained_output_shape
),
# Generate random but valid ground truth boxes.
gt_boxes: generate_gt_boxes(
gt_boxes_shape[0], image_shape_val
),
# Generate anchors from a reference and the shape of the
# pretrained_output.
all_anchors: generate_anchors(
generate_anchors_reference(
self.base_size, self.ratios, self.scales
),
16,
pretrained_output_shape[1:3]
),
image_shape: image_shape_val,
})
# Class score generates 2 values per anchor.
rpn_cls_score_shape = layers_inst['rpn_cls_score'].shape
rpn_cls_score_true_shape = (total_anchors, 2)
self.assertEqual(rpn_cls_score_shape, rpn_cls_score_true_shape)
# Probs have the same shape as cls scores.
rpn_cls_prob_shape = layers_inst['rpn_cls_prob'].shape
self.assertEqual(rpn_cls_prob_shape, rpn_cls_score_true_shape)
# We check softmax with the sum of the output.
rpn_cls_prob_sum = layers_inst['rpn_cls_prob'].sum(axis=1)
self.assertAllClose(rpn_cls_prob_sum, np.ones(total_anchors))
# Proposals and scores are related to the output of the NMS with
# limits.
total_proposals = layers_inst['proposals'].shape[0]
total_scores = layers_inst['scores'].shape[0]
# Check we don't get more than top_n proposals.
self.assertGreaterEqual(
self.config.proposals.post_nms_top_n, total_proposals
)
# Check we get a score for each proposal.
self.assertEqual(total_proposals, total_scores)
# Check that we get a regression for each anchor.
self.assertEqual(
layers_inst['rpn_bbox_pred'].shape,
(total_anchors, 4)
)
# Check that we get a target for each regression for each anchor.
self.assertEqual(
layers_inst['rpn_bbox_target'].shape,
(total_anchors, 4)
)
# Check that we get a target class for each anchor.
self.assertEqual(
layers_inst['rpn_cls_target'].shape,
(total_anchors,)
)
# Check that targets are composed of [-1, 0, 1] only.
rpn_cls_target = layers_inst['rpn_cls_target']
self.assertEqual(
tuple(np.sort(np.unique(rpn_cls_target))),
(-1, 0., 1.)
)
batch_cls_target = rpn_cls_target[
(rpn_cls_target == 0.) | (rpn_cls_target == 1.)
]
# Check that the non negative target class are exactly the size
# as the minibatch
self.assertEqual(
batch_cls_target.shape,
(self.config.target.minibatch_size, )
)
# Check that we get upto foreground_fraction of positive anchors.
self.assertLessEqual(
batch_cls_target[batch_cls_target == 1.].shape[0] /
batch_cls_target.shape[0],
self.config.target.foreground_fraction
)
def testTypes(self):
"""Tests that return types are the expected ones.
"""
# We repeat testBasic's setup.
model = RPN(
self.num_anchors, self.config, debug=True
)
pretrained_output_shape = (1, 32, 32, 512)
pretrained_output = tf.placeholder(
tf.float32, shape=pretrained_output_shape)
image_shape_val = (
int(pretrained_output_shape[1] * self.stride),
int(pretrained_output_shape[2] * self.stride),
)
gt_boxes_shape = (4, 4)
gt_boxes = tf.placeholder(tf.float32, shape=gt_boxes_shape)
image_shape_shape = (2,)
image_shape = tf.placeholder(tf.float32, shape=image_shape_shape)
total_anchors = (
pretrained_output_shape[1] * pretrained_output_shape[2] *
self.num_anchors
)
all_anchors_shape = (total_anchors, 4)
all_anchors = tf.placeholder(tf.float32, shape=all_anchors_shape)
layers = model(
pretrained_output, image_shape, all_anchors, gt_boxes=gt_boxes
)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
layers_inst = sess.run(layers, feed_dict={
pretrained_output: np.random.rand(
*pretrained_output_shape
),
gt_boxes: generate_gt_boxes(
gt_boxes_shape[0], image_shape_val
),
all_anchors: generate_anchors(
generate_anchors_reference(
self.base_size, self.ratios, self.scales
),
16,
pretrained_output_shape[1:3]
),
image_shape: image_shape_val,
})
# Assertions
proposals = layers_inst['proposals']
scores = layers_inst['scores']
rpn_cls_prob = layers_inst['rpn_cls_prob']
rpn_cls_score = layers_inst['rpn_cls_score']
rpn_bbox_pred = layers_inst['rpn_bbox_pred']
rpn_cls_target = layers_inst['rpn_cls_target']
rpn_bbox_target = layers_inst['rpn_bbox_target']
# Everything should have dtype=tf.float32
self.assertAllEqual(
# We have 7 values we want to compare to tf.float32.
[tf.float32] * 7,
[
proposals.dtype, scores.dtype, rpn_cls_prob.dtype,
rpn_cls_score.dtype, rpn_bbox_pred.dtype,
rpn_cls_target.dtype, rpn_bbox_target.dtype,
]
)
def testBasic(self):
"""Tests shapes are consistent with anchor generation.
"""
model = RPN(self.num_anchors, self.config, debug=True)
# (plus the batch number)
pretrained_output_shape = (1, 32, 32, 512)
pretrained_output = tf.placeholder(tf.float32,
shape=pretrained_output_shape)
# Estimate image shape from the pretrained output and the anchor stride
image_shape_val = (
int(pretrained_output_shape[1] * self.stride),
int(pretrained_output_shape[2] * self.stride),
)
# Use 4 ground truth boxes.
gt_boxes_shape = (4, 4)
gt_boxes = tf.placeholder(tf.float32, shape=gt_boxes_shape)
image_shape_shape = (2, )
image_shape = tf.placeholder(tf.float32, shape=image_shape_shape)
# Total anchors depends on the pretrained output shape and the total
# number of anchors per point.
total_anchors = (pretrained_output_shape[1] *
pretrained_output_shape[2] * self.num_anchors)
all_anchors_shape = (total_anchors, 4)
all_anchors = tf.placeholder(tf.float32, shape=all_anchors_shape)
layers = model(pretrained_output,
image_shape,
all_anchors,
gt_boxes=gt_boxes)
with self.test_session() as sess:
# As in the case of a real session we need to initialize the
# variables.
sess.run(tf.global_variables_initializer())
layers_inst = sess.run(
layers,
feed_dict={
# We don't really care about the value of the pretrained output
# only that has the correct shape.
pretrained_output:
np.random.rand(*pretrained_output_shape),
# Generate random but valid ground truth boxes.
gt_boxes:
generate_gt_boxes(gt_boxes_shape[0], image_shape_val),
# Generate anchors from a reference and the shape of the
# pretrained_output.
all_anchors:
generate_anchors(
generate_anchors_reference(self.base_size, self.ratios,
self.scales), 16,
pretrained_output_shape[1:3]),
image_shape:
image_shape_val,
})
# Class score generates 2 values per anchor.
rpn_cls_score_shape = layers_inst['rpn_cls_score'].shape
rpn_cls_score_true_shape = (total_anchors, 2)
self.assertEqual(rpn_cls_score_shape, rpn_cls_score_true_shape)
# Probs have the same shape as cls scores.
rpn_cls_prob_shape = layers_inst['rpn_cls_prob'].shape
self.assertEqual(rpn_cls_prob_shape, rpn_cls_score_true_shape)
# We check softmax with the sum of the output.
rpn_cls_prob_sum = layers_inst['rpn_cls_prob'].sum(axis=1)
self.assertAllClose(rpn_cls_prob_sum, np.ones(total_anchors))
# Proposals and scores are related to the output of the NMS with
# limits.
total_proposals = layers_inst['proposals'].shape[0]
total_scores = layers_inst['scores'].shape[0]
# Check we don't get more than top_n proposals.
self.assertGreaterEqual(self.config.proposals.post_nms_top_n,
total_proposals)
# Check we get a score for each proposal.
self.assertEqual(total_proposals, total_scores)
# Check that we get a regression for each anchor.
self.assertEqual(layers_inst['rpn_bbox_pred'].shape,
(total_anchors, 4))
# Check that we get a target for each regression for each anchor.
self.assertEqual(layers_inst['rpn_bbox_target'].shape,
(total_anchors, 4))
# Check that we get a target class for each anchor.
self.assertEqual(layers_inst['rpn_cls_target'].shape,
(total_anchors, ))
# Check that targets are composed of [-1, 0, 1] only.
rpn_cls_target = layers_inst['rpn_cls_target']
self.assertEqual(tuple(np.sort(np.unique(rpn_cls_target))),
(-1, 0., 1.))
batch_cls_target = rpn_cls_target[(rpn_cls_target == 0.) |
(rpn_cls_target == 1.)]
# Check that the non negative target class are exactly the size
# as the minibatch
self.assertEqual(batch_cls_target.shape,
(self.config.target.minibatch_size, ))
# Check that we get upto foreground_fraction of positive anchors.
self.assertLessEqual(
batch_cls_target[batch_cls_target == 1.].shape[0] /
batch_cls_target.shape[0], self.config.target.foreground_fraction)
def testTypes(self):
"""Tests that return types are the expected ones.
"""
# We repeat testBasic's setup.
model = RPN(self.num_anchors, self.config, debug=True)
pretrained_output_shape = (1, 32, 32, 512)
pretrained_output = tf.placeholder(tf.float32,
shape=pretrained_output_shape)
image_shape_val = (
int(pretrained_output_shape[1] * self.stride),
int(pretrained_output_shape[2] * self.stride),
)
gt_boxes_shape = (4, 4)
gt_boxes = tf.placeholder(tf.float32, shape=gt_boxes_shape)
image_shape_shape = (2, )
image_shape = tf.placeholder(tf.float32, shape=image_shape_shape)
total_anchors = (pretrained_output_shape[1] *
pretrained_output_shape[2] * self.num_anchors)
all_anchors_shape = (total_anchors, 4)
all_anchors = tf.placeholder(tf.float32, shape=all_anchors_shape)
layers = model(pretrained_output,
image_shape,
all_anchors,
gt_boxes=gt_boxes)
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
layers_inst = sess.run(
layers,
feed_dict={
pretrained_output:
np.random.rand(*pretrained_output_shape),
gt_boxes:
generate_gt_boxes(gt_boxes_shape[0], image_shape_val),
all_anchors:
generate_anchors(
generate_anchors_reference(self.base_size, self.ratios,
self.scales), 16,
pretrained_output_shape[1:3]),
image_shape:
image_shape_val,
})
# Assertions
proposals = layers_inst['proposals']
scores = layers_inst['scores']
rpn_cls_prob = layers_inst['rpn_cls_prob']
rpn_cls_score = layers_inst['rpn_cls_score']
rpn_bbox_pred = layers_inst['rpn_bbox_pred']
rpn_cls_target = layers_inst['rpn_cls_target']
rpn_bbox_target = layers_inst['rpn_bbox_target']
# Everything should have dtype=tf.float32
self.assertAllEqual(
# We have 7 values we want to compare to tf.float32.
[tf.float32] * 7,
[
proposals.dtype,
scores.dtype,
rpn_cls_prob.dtype,
rpn_cls_score.dtype,
rpn_bbox_pred.dtype,
rpn_cls_target.dtype,
rpn_bbox_target.dtype,
])
[shift_x, shift_y, shift_x, shift_y],
axis=0
)
shifts = shifts.T
num_anchors = anchors_reference.shape[0]
num_anchor_points = shifts.shape[0]
all_anchors = (
anchors_reference.reshape((1, num_anchors, 4)) +
np.transpose(
shifts.reshape((1, num_anchor_points, 4)),
axes=(1, 0, 2)
)
)
all_anchors = np.reshape(
all_anchors, (num_anchors * num_anchor_points, 4)
)
return all_anchors
if __name__ == '__main__':
from luminoth.utils.anchors import generate_anchors_reference
ref = generate_anchors_reference(
base_size=16, ratios=[0.5, 1, 2], scales=2**np.arange(3, 6)
)
