def testAnchorReference(self): # Test simple case with one aspect ratio and one scale. base_size = 256 aspect_ratios = [1.] scales = [1.] anchor_reference = generate_anchors_reference( base_size=base_size, aspect_ratios=aspect_ratios, scales=scales) # Should return a single anchor. self.assertEqual(anchor_reference.shape, (1, 4)) self.assertAllEqual(anchor_reference[0], [ -(base_size - 1) / 2.0, -(base_size - 1) / 2.0, (base_size - 1) / 2.0, (base_size - 1) / 2.0 ]) # Test with fixed ratio and different scales. scales = np.array([0.5, 1., 2., 4.]) anchor_reference = generate_anchors_reference( base_size=base_size, aspect_ratios=aspect_ratios, scales=scales) # Check that we have the correct number of anchors. self.assertEqual(anchor_reference.shape, (4, 4)) width_heights = self._get_widths_heights(anchor_reference) # Check that anchors are squares (aspect_ratio = [1.0]). self.assertTrue((width_heights[:, 0] == width_heights[:, 1]).all()) # Check that widths are consistent with scales times base_size. self.assertAllEqual(width_heights[:, 0], base_size * scales) # Check exact values. self.assertAllEqual( anchor_reference, np.array([[-63.5, -63.5, 63.5, 63.5], [-127.5, -127.5, 127.5, 127.5], [-255.5, -255.5, 255.5, 255.5], [-511.5, -511.5, 511.5, 511.5]])) # Test with different ratios and scales. scales = np.array([0.5, 1., 2.]) aspect_ratios = np.array([0.5, 1., 2.]) anchor_reference = generate_anchors_reference( base_size=base_size, aspect_ratios=aspect_ratios, scales=scales) # Check we have the correct number of anchors. self.assertEqual(anchor_reference.shape, (len(scales) * len(aspect_ratios), 4)) width_heights = self._get_widths_heights(anchor_reference) # Check ratios of height / widths anchor_ratios = width_heights[:, 1] / width_heights[:, 0] # Check scales (applied to ) anchor_scales = np.sqrt( (width_heights[:, 1] * width_heights[:, 0]) / (base_size**2)) # Test that all ratios are used in the correct order. self.assertAllClose(anchor_ratios, [0.5, 0.5, 0.5, 1., 1., 1., 2., 2., 2.]) # Test that all scales are used in the correct order. self.assertAllClose(anchor_scales, [0.5, 1., 2., 0.5, 1., 2., 0.5, 1., 2.])
def testInvalidValues(self): # Should fail because base_size is too small to for that scale and # ratio. base_size = 1 aspect_ratios = [0.5] scales = [0.5] try: generate_anchors_reference(base_size=base_size, aspect_ratios=aspect_ratios, scales=scales) except ValueError: return self.fail('Should have thrown an exception.')
def __init__(self, config, name='fasterrcnn'): super(FasterRCNN, self).__init__(name=name) self._config = config self._num_classes = config.model.network.num_classes # Generate network with RCNN self._with_rcnn = config.model.network.with_rcnn self._debug = config.train.debug self._seed = config.train.seed self._anchor_base_size = config.model.anchors.base_size self._anchor_scales = np.array(config.model.anchors.scales) self._anchor_ratios = np.array(config.model.anchors.ratios) self._anchor_stride = config.model.anchors.stride self._anchor_reference = generate_anchors_reference( self._anchor_base_size, self._anchor_ratios, self._anchor_scales ) self._num_anchors = self._anchor_reference.shape[0] # Weights used to sum each of the losses of the submodules self._rpn_cls_loss_weight = config.model.loss.rpn_cls_loss_weight self._rpn_reg_loss_weight = config.model.loss.rpn_reg_loss_weights self._rcnn_cls_loss_weight = config.model.loss.rcnn_cls_loss_weight self._rcnn_reg_loss_weight = config.model.loss.rcnn_reg_loss_weights self._losses_collections = ['fastercnn_losses'] self.base_network = TruncatedBaseNetwork(config.model.base_network)
def __init__(self, config, name='fasterrcnn'): super(FasterRCNN, self).__init__(name=name) # Main configuration object, it holds not only the necessary # information for this module but also configuration for each of the # different submodules. self._config = config # Total number of classes to classify. If not using RCNN then it is not # used. TODO: Make it *more* optional. self._num_classes = config.model.network.num_classes # Generate network with RCNN thus allowing for classification of # objects and not just finding them. self._with_rcnn = config.model.network.with_rcnn # Turn on debug mode with returns more Tensors which can be used for # better visualization and (of course) debugging. self._debug = config.train.debug self._seed = config.train.seed # Anchor config, check out the docs of base_config.yml for a better # understanding of how anchors work. self._anchor_base_size = config.model.anchors.base_size self._anchor_scales = np.array(config.model.anchors.scales) self._anchor_ratios = np.array(config.model.anchors.ratios) self._anchor_stride = config.model.anchors.stride #### # Anchor reference for building dynamic anchors for each image in the # computation graph. self._anchor_reference = generate_anchors_reference( self._anchor_base_size, self._anchor_ratios, self._anchor_scales ) # Total number of anchors per point. self._num_anchors = self._anchor_reference.shape[0] # Weights used to sum each of the losses of the submodules self._rpn_cls_loss_weight = config.model.loss.rpn_cls_loss_weight self._rpn_reg_loss_weight = config.model.loss.rpn_reg_loss_weights self._rcnn_cls_loss_weight = config.model.loss.rcnn_cls_loss_weight self._rcnn_reg_loss_weight = config.model.loss.rcnn_reg_loss_weights self._losses_collections = ['fastercnn_losses'] # We want the pretrained model to be outside the FasterRCNN name scope. self.base_network = TruncatedBaseNetwork(config.model.base_network)
def testBasic(self): """Tests shapes are consistent with anchor generation. """ model = RPN( self.num_anchors, self.config, debug=True ) # (plus the batch number) pretrained_output_shape = (1, 32, 32, 512) pretrained_output = tf.placeholder( tf.float32, shape=pretrained_output_shape) # Estimate image shape from the pretrained output and the anchor stride image_shape_val = ( int(pretrained_output_shape[1] * self.stride), int(pretrained_output_shape[2] * self.stride), ) # Use 4 ground truth boxes. gt_boxes_shape = (4, 4) gt_boxes = tf.placeholder(tf.float32, shape=gt_boxes_shape) image_shape_shape = (2,) image_shape = tf.placeholder(tf.float32, shape=image_shape_shape) # Total anchors depends on the pretrained output shape and the total # number of anchors per point. total_anchors = ( pretrained_output_shape[1] * pretrained_output_shape[2] * self.num_anchors ) all_anchors_shape = (total_anchors, 4) all_anchors = tf.placeholder(tf.float32, shape=all_anchors_shape) layers = model( pretrained_output, image_shape, all_anchors, gt_boxes=gt_boxes ) with self.test_session() as sess: # As in the case of a real session we need to initialize the # variables. sess.run(tf.global_variables_initializer()) layers_inst = sess.run(layers, feed_dict={ # We don't really care about the value of the pretrained output # only that has the correct shape. pretrained_output: np.random.rand( *pretrained_output_shape ), # Generate random but valid ground truth boxes. gt_boxes: generate_gt_boxes( gt_boxes_shape[0], image_shape_val ), # Generate anchors from a reference and the shape of the # pretrained_output. all_anchors: generate_anchors( generate_anchors_reference( self.base_size, self.ratios, self.scales ), 16, pretrained_output_shape[1:3] ), image_shape: image_shape_val, }) # Class score generates 2 values per anchor. rpn_cls_score_shape = layers_inst['rpn_cls_score'].shape rpn_cls_score_true_shape = (total_anchors, 2) self.assertEqual(rpn_cls_score_shape, rpn_cls_score_true_shape) # Probs have the same shape as cls scores. rpn_cls_prob_shape = layers_inst['rpn_cls_prob'].shape self.assertEqual(rpn_cls_prob_shape, rpn_cls_score_true_shape) # We check softmax with the sum of the output. rpn_cls_prob_sum = layers_inst['rpn_cls_prob'].sum(axis=1) self.assertAllClose(rpn_cls_prob_sum, np.ones(total_anchors)) # Proposals and scores are related to the output of the NMS with # limits. total_proposals = layers_inst['proposals'].shape[0] total_scores = layers_inst['scores'].shape[0] # Check we don't get more than top_n proposals. self.assertGreaterEqual( self.config.proposals.post_nms_top_n, total_proposals ) # Check we get a score for each proposal. self.assertEqual(total_proposals, total_scores) # Check that we get a regression for each anchor. self.assertEqual( layers_inst['rpn_bbox_pred'].shape, (total_anchors, 4) ) # Check that we get a target for each regression for each anchor. self.assertEqual( layers_inst['rpn_bbox_target'].shape, (total_anchors, 4) ) # Check that we get a target class for each anchor. self.assertEqual( layers_inst['rpn_cls_target'].shape, (total_anchors,) ) # Check that targets are composed of [-1, 0, 1] only. rpn_cls_target = layers_inst['rpn_cls_target'] self.assertEqual( tuple(np.sort(np.unique(rpn_cls_target))), (-1, 0., 1.) ) batch_cls_target = rpn_cls_target[ (rpn_cls_target == 0.) | (rpn_cls_target == 1.) ] # Check that the non negative target class are exactly the size # as the minibatch self.assertEqual( batch_cls_target.shape, (self.config.target.minibatch_size, ) ) # Check that we get upto foreground_fraction of positive anchors. self.assertLessEqual( batch_cls_target[batch_cls_target == 1.].shape[0] / batch_cls_target.shape[0], self.config.target.foreground_fraction )
def testTypes(self): """Tests that return types are the expected ones. """ # We repeat testBasic's setup. model = RPN( self.num_anchors, self.config, debug=True ) pretrained_output_shape = (1, 32, 32, 512) pretrained_output = tf.placeholder( tf.float32, shape=pretrained_output_shape) image_shape_val = ( int(pretrained_output_shape[1] * self.stride), int(pretrained_output_shape[2] * self.stride), ) gt_boxes_shape = (4, 4) gt_boxes = tf.placeholder(tf.float32, shape=gt_boxes_shape) image_shape_shape = (2,) image_shape = tf.placeholder(tf.float32, shape=image_shape_shape) total_anchors = ( pretrained_output_shape[1] * pretrained_output_shape[2] * self.num_anchors ) all_anchors_shape = (total_anchors, 4) all_anchors = tf.placeholder(tf.float32, shape=all_anchors_shape) layers = model( pretrained_output, image_shape, all_anchors, gt_boxes=gt_boxes ) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) layers_inst = sess.run(layers, feed_dict={ pretrained_output: np.random.rand( *pretrained_output_shape ), gt_boxes: generate_gt_boxes( gt_boxes_shape[0], image_shape_val ), all_anchors: generate_anchors( generate_anchors_reference( self.base_size, self.ratios, self.scales ), 16, pretrained_output_shape[1:3] ), image_shape: image_shape_val, }) # Assertions proposals = layers_inst['proposals'] scores = layers_inst['scores'] rpn_cls_prob = layers_inst['rpn_cls_prob'] rpn_cls_score = layers_inst['rpn_cls_score'] rpn_bbox_pred = layers_inst['rpn_bbox_pred'] rpn_cls_target = layers_inst['rpn_cls_target'] rpn_bbox_target = layers_inst['rpn_bbox_target'] # Everything should have dtype=tf.float32 self.assertAllEqual( # We have 7 values we want to compare to tf.float32. [tf.float32] * 7, [ proposals.dtype, scores.dtype, rpn_cls_prob.dtype, rpn_cls_score.dtype, rpn_bbox_pred.dtype, rpn_cls_target.dtype, rpn_bbox_target.dtype, ] )
def testBasic(self): """Tests shapes are consistent with anchor generation. """ model = RPN(self.num_anchors, self.config, debug=True) # (plus the batch number) pretrained_output_shape = (1, 32, 32, 512) pretrained_output = tf.placeholder(tf.float32, shape=pretrained_output_shape) # Estimate image shape from the pretrained output and the anchor stride image_shape_val = ( int(pretrained_output_shape[1] * self.stride), int(pretrained_output_shape[2] * self.stride), ) # Use 4 ground truth boxes. gt_boxes_shape = (4, 4) gt_boxes = tf.placeholder(tf.float32, shape=gt_boxes_shape) image_shape_shape = (2, ) image_shape = tf.placeholder(tf.float32, shape=image_shape_shape) # Total anchors depends on the pretrained output shape and the total # number of anchors per point. total_anchors = (pretrained_output_shape[1] * pretrained_output_shape[2] * self.num_anchors) all_anchors_shape = (total_anchors, 4) all_anchors = tf.placeholder(tf.float32, shape=all_anchors_shape) layers = model(pretrained_output, image_shape, all_anchors, gt_boxes=gt_boxes) with self.test_session() as sess: # As in the case of a real session we need to initialize the # variables. sess.run(tf.global_variables_initializer()) layers_inst = sess.run( layers, feed_dict={ # We don't really care about the value of the pretrained output # only that has the correct shape. pretrained_output: np.random.rand(*pretrained_output_shape), # Generate random but valid ground truth boxes. gt_boxes: generate_gt_boxes(gt_boxes_shape[0], image_shape_val), # Generate anchors from a reference and the shape of the # pretrained_output. all_anchors: generate_anchors( generate_anchors_reference(self.base_size, self.ratios, self.scales), 16, pretrained_output_shape[1:3]), image_shape: image_shape_val, }) # Class score generates 2 values per anchor. rpn_cls_score_shape = layers_inst['rpn_cls_score'].shape rpn_cls_score_true_shape = (total_anchors, 2) self.assertEqual(rpn_cls_score_shape, rpn_cls_score_true_shape) # Probs have the same shape as cls scores. rpn_cls_prob_shape = layers_inst['rpn_cls_prob'].shape self.assertEqual(rpn_cls_prob_shape, rpn_cls_score_true_shape) # We check softmax with the sum of the output. rpn_cls_prob_sum = layers_inst['rpn_cls_prob'].sum(axis=1) self.assertAllClose(rpn_cls_prob_sum, np.ones(total_anchors)) # Proposals and scores are related to the output of the NMS with # limits. total_proposals = layers_inst['proposals'].shape[0] total_scores = layers_inst['scores'].shape[0] # Check we don't get more than top_n proposals. self.assertGreaterEqual(self.config.proposals.post_nms_top_n, total_proposals) # Check we get a score for each proposal. self.assertEqual(total_proposals, total_scores) # Check that we get a regression for each anchor. self.assertEqual(layers_inst['rpn_bbox_pred'].shape, (total_anchors, 4)) # Check that we get a target for each regression for each anchor. self.assertEqual(layers_inst['rpn_bbox_target'].shape, (total_anchors, 4)) # Check that we get a target class for each anchor. self.assertEqual(layers_inst['rpn_cls_target'].shape, (total_anchors, )) # Check that targets are composed of [-1, 0, 1] only. rpn_cls_target = layers_inst['rpn_cls_target'] self.assertEqual(tuple(np.sort(np.unique(rpn_cls_target))), (-1, 0., 1.)) batch_cls_target = rpn_cls_target[(rpn_cls_target == 0.) | (rpn_cls_target == 1.)] # Check that the non negative target class are exactly the size # as the minibatch self.assertEqual(batch_cls_target.shape, (self.config.target.minibatch_size, )) # Check that we get upto foreground_fraction of positive anchors. self.assertLessEqual( batch_cls_target[batch_cls_target == 1.].shape[0] / batch_cls_target.shape[0], self.config.target.foreground_fraction)
def testTypes(self): """Tests that return types are the expected ones. """ # We repeat testBasic's setup. model = RPN(self.num_anchors, self.config, debug=True) pretrained_output_shape = (1, 32, 32, 512) pretrained_output = tf.placeholder(tf.float32, shape=pretrained_output_shape) image_shape_val = ( int(pretrained_output_shape[1] * self.stride), int(pretrained_output_shape[2] * self.stride), ) gt_boxes_shape = (4, 4) gt_boxes = tf.placeholder(tf.float32, shape=gt_boxes_shape) image_shape_shape = (2, ) image_shape = tf.placeholder(tf.float32, shape=image_shape_shape) total_anchors = (pretrained_output_shape[1] * pretrained_output_shape[2] * self.num_anchors) all_anchors_shape = (total_anchors, 4) all_anchors = tf.placeholder(tf.float32, shape=all_anchors_shape) layers = model(pretrained_output, image_shape, all_anchors, gt_boxes=gt_boxes) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) layers_inst = sess.run( layers, feed_dict={ pretrained_output: np.random.rand(*pretrained_output_shape), gt_boxes: generate_gt_boxes(gt_boxes_shape[0], image_shape_val), all_anchors: generate_anchors( generate_anchors_reference(self.base_size, self.ratios, self.scales), 16, pretrained_output_shape[1:3]), image_shape: image_shape_val, }) # Assertions proposals = layers_inst['proposals'] scores = layers_inst['scores'] rpn_cls_prob = layers_inst['rpn_cls_prob'] rpn_cls_score = layers_inst['rpn_cls_score'] rpn_bbox_pred = layers_inst['rpn_bbox_pred'] rpn_cls_target = layers_inst['rpn_cls_target'] rpn_bbox_target = layers_inst['rpn_bbox_target'] # Everything should have dtype=tf.float32 self.assertAllEqual( # We have 7 values we want to compare to tf.float32. [tf.float32] * 7, [ proposals.dtype, scores.dtype, rpn_cls_prob.dtype, rpn_cls_score.dtype, rpn_bbox_pred.dtype, rpn_cls_target.dtype, rpn_bbox_target.dtype, ])
[shift_x, shift_y, shift_x, shift_y], axis=0 ) shifts = shifts.T num_anchors = anchors_reference.shape[0] num_anchor_points = shifts.shape[0] all_anchors = ( anchors_reference.reshape((1, num_anchors, 4)) + np.transpose( shifts.reshape((1, num_anchor_points, 4)), axes=(1, 0, 2) ) ) all_anchors = np.reshape( all_anchors, (num_anchors * num_anchor_points, 4) ) return all_anchors if __name__ == '__main__': from luminoth.utils.anchors import generate_anchors_reference ref = generate_anchors_reference( base_size=16, ratios=[0.5, 1, 2], scales=2**np.arange(3, 6) )