def test_softmax_score_converter(self): num_class_slots = 10 batch_size = 2 height = 17 width = 19 num_predictions_per_location = 2 assert num_predictions_per_location != 1 conv_hyperparams = self._build_conv_hyperparams() class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=num_class_slots, conv_hyperparams=conv_hyperparams, num_predictions_per_location=num_predictions_per_location, score_converter_fn=tf.nn.softmax) def graph_fn(): image_feature = tf.random_uniform([batch_size, height, width, 1024], minval=-10.0, maxval=10.0, dtype=tf.float32) class_predictions = class_prediction_head(image_feature) return class_predictions class_predictions_out = self.execute(graph_fn, []) class_predictions_sum = np.sum(class_predictions_out, axis=-1) num_anchors = height * width * num_predictions_per_location exp_class_predictions_sum = np.ones((batch_size, num_anchors), dtype=np.float32) self.assertAllEqual((batch_size, num_anchors, num_class_slots), class_predictions_out.shape) self.assertAllClose(class_predictions_sum, exp_class_predictions_sum)
def test_variable_count_depth_wise_False(self): conv_hyperparams = self._build_conv_hyperparams() class_prediction_head = ( keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=20, conv_hyperparams=conv_hyperparams, num_predictions_per_location=1, use_depthwise=False)) image_feature = tf.random_uniform( [64, 17, 19, 1024], minval=-10.0, maxval=10.0, dtype=tf.float32) class_prediction_head(image_feature) self.assertEqual(len(class_prediction_head.variables), 2)
def test_prediction_size_depthwise_true(self): conv_hyperparams = self._build_conv_hyperparams() class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=20, conv_hyperparams=conv_hyperparams, num_predictions_per_location=1, use_depthwise=True) image_feature = tf.random_uniform([64, 17, 19, 1024], minval=-10.0, maxval=10.0, dtype=tf.float32) class_predictions = class_prediction_head(image_feature) self.assertAllEqual([64, 323, 20], class_predictions.get_shape().as_list())
def test_prediction_size_depthwise_false(self): conv_hyperparams = self._build_conv_hyperparams() class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=20, conv_hyperparams=conv_hyperparams, num_predictions_per_location=1, use_depthwise=False) def graph_fn(): image_feature = tf.random_uniform( [64, 17, 19, 1024], minval=-10.0, maxval=10.0, dtype=tf.float32) class_predictions = class_prediction_head(image_feature) return class_predictions class_predictions = self.execute(graph_fn, []) self.assertAllEqual([64, 323, 20], class_predictions.shape)
def test_variable_count_depth_wise_False(self): g = tf.Graph() with g.as_default(): conv_hyperparams = self._build_conv_hyperparams() class_prediction_head = ( keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=20, conv_hyperparams=conv_hyperparams, num_predictions_per_location=1, use_depthwise=False)) image_feature = tf.random_uniform([64, 17, 19, 1024], minval=-10.0, maxval=10.0, dtype=tf.float32) _ = class_prediction_head(image_feature) variables = g.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) self.assertEqual(len(variables), 2)
def graph_fn(image_features): box_prediction_head = keras_box_head.WeightSharedConvolutionalBoxHead( box_code_size=box_code_size, conv_hyperparams=self._build_conv_hyperparams(), num_predictions_per_location=num_predictions_per_location) class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=num_classes_without_background + 1, conv_hyperparams=self._build_conv_hyperparams(), num_predictions_per_location=num_predictions_per_location) other_heads = { other_head_name: keras_mask_head.WeightSharedConvolutionalMaskHead( num_classes=num_classes_without_background, conv_hyperparams=self._build_conv_hyperparams(), num_predictions_per_location=num_predictions_per_location, mask_height=mask_height, mask_width=mask_width) } conv_box_predictor = box_predictor.WeightSharedConvolutionalBoxPredictor( is_training=False, num_classes=num_classes_without_background, box_prediction_head=box_prediction_head, class_prediction_head=class_prediction_head, other_heads=other_heads, conv_hyperparams=self._build_conv_hyperparams(), freeze_batchnorm=False, inplace_batchnorm_update=False, depth=32, num_layers_before_predictor=2) box_predictions = conv_box_predictor([image_features]) for key, value in box_predictions.items(): box_predictions[key] = tf.concat(value, axis=1) assert len(box_predictions) == 3 return (box_predictions[box_predictor.BOX_ENCODINGS], box_predictions[ box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND], box_predictions[other_head_name])
def build_weight_shared_convolutional_keras_box_predictor( is_training, num_classes, conv_hyperparams, freeze_batchnorm, inplace_batchnorm_update, num_predictions_per_location_list, depth, num_layers_before_predictor, box_code_size, kernel_size=3, add_background_class=True, class_prediction_bias_init=0.0, use_dropout=False, dropout_keep_prob=0.8, share_prediction_tower=False, apply_batch_norm=True, use_depthwise=False, score_converter_fn=tf.identity, box_encodings_clip_range=None, name='WeightSharedConvolutionalBoxPredictor'): """Builds the Keras WeightSharedConvolutionalBoxPredictor from the arguments. Args: is_training: Indicates whether the BoxPredictor is in training mode. num_classes: number of classes. Note that num_classes *does not* include the background category, so if groundtruth labels take values in {0, 1, .., K-1}, num_classes=K (and not K+1, even though the assigned classification targets can range from {0,... K}). conv_hyperparams: A `hyperparams_builder.KerasLayerHyperparams` object containing hyperparameters for convolution ops. freeze_batchnorm: Whether to freeze batch norm parameters during training or not. When training with a small batch size (e.g. 1), it is desirable to freeze batch norm update and use pretrained batch norm params. inplace_batchnorm_update: Whether to update batch norm moving average values inplace. When this is false train op must add a control dependency on tf.graphkeys.UPDATE_OPS collection in order to update batch norm statistics. num_predictions_per_location_list: A list of integers representing the number of box predictions to be made per spatial location for each feature map. depth: depth of conv layers. num_layers_before_predictor: Number of the additional conv layers before the predictor. box_code_size: Size of encoding for each box. kernel_size: Size of final convolution kernel. add_background_class: Whether to add an implicit background class. class_prediction_bias_init: constant value to initialize bias of the last conv2d layer before class prediction. use_dropout: Whether to apply dropout to class prediction head. dropout_keep_prob: Probability of keeping activiations. share_prediction_tower: Whether to share the multi-layer tower between box prediction and class prediction heads. apply_batch_norm: Whether to apply batch normalization to conv layers in this predictor. use_depthwise: Whether to use depthwise separable conv2d instead of conv2d. score_converter_fn: Callable score converter to perform elementwise op on class scores. box_encodings_clip_range: Min and max values for clipping the box_encodings. name: A string name scope to assign to the box predictor. If `None`, Keras will auto-generate one from the class name. Returns: A Keras WeightSharedConvolutionalBoxPredictor class. """ if len(set(num_predictions_per_location_list)) > 1: raise ValueError('num predictions per location must be same for all' 'feature maps, found: {}'.format( num_predictions_per_location_list)) num_predictions_per_location = num_predictions_per_location_list[0] box_prediction_head = keras_box_head.WeightSharedConvolutionalBoxHead( box_code_size=box_code_size, kernel_size=kernel_size, conv_hyperparams=conv_hyperparams, num_predictions_per_location=num_predictions_per_location, use_depthwise=use_depthwise, box_encodings_clip_range=box_encodings_clip_range, name='WeightSharedConvolutionalBoxHead') class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=(num_classes + 1 if add_background_class else num_classes), use_dropout=use_dropout, dropout_keep_prob=dropout_keep_prob, kernel_size=kernel_size, conv_hyperparams=conv_hyperparams, num_predictions_per_location=num_predictions_per_location, class_prediction_bias_init=class_prediction_bias_init, use_depthwise=use_depthwise, score_converter_fn=score_converter_fn, name='WeightSharedConvolutionalClassHead') other_heads = {} return (convolutional_keras_box_predictor. WeightSharedConvolutionalBoxPredictor( is_training=is_training, num_classes=num_classes, box_prediction_head=box_prediction_head, class_prediction_head=class_prediction_head, other_heads=other_heads, conv_hyperparams=conv_hyperparams, depth=depth, num_layers_before_predictor=num_layers_before_predictor, freeze_batchnorm=freeze_batchnorm, inplace_batchnorm_update=inplace_batchnorm_update, kernel_size=kernel_size, apply_batch_norm=apply_batch_norm, share_prediction_tower=share_prediction_tower, use_depthwise=use_depthwise, name=name))
def test_other_heads_predictions(self): box_code_size = 4 num_classes_without_background = 3 other_head_name = 'Mask' mask_height = 5 mask_width = 5 num_predictions_per_location = 5 box_prediction_head = keras_box_head.WeightSharedConvolutionalBoxHead( box_code_size=box_code_size, conv_hyperparams=self._build_conv_hyperparams(), num_predictions_per_location=num_predictions_per_location) class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=num_classes_without_background + 1, conv_hyperparams=self._build_conv_hyperparams(), num_predictions_per_location=num_predictions_per_location) other_heads = { other_head_name: keras_mask_head.WeightSharedConvolutionalMaskHead( num_classes=num_classes_without_background, conv_hyperparams=self._build_conv_hyperparams(), num_predictions_per_location=num_predictions_per_location, mask_height=mask_height, mask_width=mask_width) } conv_box_predictor = box_predictor.WeightSharedConvolutionalBoxPredictor( is_training=False, num_classes=num_classes_without_background, box_prediction_head=box_prediction_head, class_prediction_head=class_prediction_head, other_heads=other_heads, conv_hyperparams=self._build_conv_hyperparams(), freeze_batchnorm=False, inplace_batchnorm_update=False, depth=32, num_layers_before_predictor=2) def graph_fn(image_features): box_predictions = conv_box_predictor([image_features]) for key, value in box_predictions.items(): box_predictions[key] = tf.concat(value, axis=1) assert len(box_predictions) == 3 return (box_predictions[box_predictor.BOX_ENCODINGS], box_predictions[box_predictor.CLASS_PREDICTIONS_WITH_BACKGROUND], box_predictions[other_head_name]) batch_size = 4 feature_ht = 8 feature_wt = 8 image_features = np.random.rand(batch_size, feature_ht, feature_wt, 64).astype(np.float32) (box_encodings, class_predictions, other_head_predictions) = self.execute( graph_fn, [image_features]) num_anchors = feature_ht * feature_wt * num_predictions_per_location self.assertAllEqual(box_encodings.shape, [batch_size, num_anchors, box_code_size]) self.assertAllEqual( class_predictions.shape, [batch_size, num_anchors, num_classes_without_background + 1]) self.assertAllEqual(other_head_predictions.shape, [ batch_size, num_anchors, num_classes_without_background, mask_height, mask_width ])
def build_weight_shared_convolutional_keras_box_predictor( is_training, num_classes, conv_hyperparams, freeze_batchnorm, inplace_batchnorm_update, num_predictions_per_location_list, depth, num_layers_before_predictor, box_code_size, kernel_size=3, add_background_class=True, class_prediction_bias_init=0.0, use_dropout=False, dropout_keep_prob=0.8, share_prediction_tower=False, apply_batch_norm=True, use_depthwise=False, score_converter_fn=tf.identity, box_encodings_clip_range=None, name='WeightSharedConvolutionalBoxPredictor'): if len(set(num_predictions_per_location_list)) > 1: raise ValueError('num predictions per location must be same for all' 'feature maps, found: {}'.format( num_predictions_per_location_list)) num_predictions_per_location = num_predictions_per_location_list[0] box_prediction_head = keras_box_head.WeightSharedConvolutionalBoxHead( box_code_size=box_code_size, kernel_size=kernel_size, conv_hyperparams=conv_hyperparams, num_predictions_per_location=num_predictions_per_location, use_depthwise=use_depthwise, box_encodings_clip_range=box_encodings_clip_range, name='WeightSharedConvolutionalBoxHead') class_prediction_head = keras_class_head.WeightSharedConvolutionalClassHead( num_class_slots=( num_classes + 1 if add_background_class else num_classes), use_dropout=use_dropout, dropout_keep_prob=dropout_keep_prob, kernel_size=kernel_size, conv_hyperparams=conv_hyperparams, num_predictions_per_location=num_predictions_per_location, class_prediction_bias_init=class_prediction_bias_init, use_depthwise=use_depthwise, score_converter_fn=score_converter_fn, name='WeightSharedConvolutionalClassHead') other_heads = {} return ( convolutional_keras_box_predictor.WeightSharedConvolutionalBoxPredictor( is_training=is_training, num_classes=num_classes, box_prediction_head=box_prediction_head, class_prediction_head=class_prediction_head, other_heads=other_heads, conv_hyperparams=conv_hyperparams, depth=depth, num_layers_before_predictor=num_layers_before_predictor, freeze_batchnorm=freeze_batchnorm, inplace_batchnorm_update=inplace_batchnorm_update, kernel_size=kernel_size, apply_batch_norm=apply_batch_norm, share_prediction_tower=share_prediction_tower, use_depthwise=use_depthwise, name=name))