def test_raises_error_on_invalid_groundtruth_labels(self): similarity_calc = region_similarity_calculator.NegSqDistSimilarity() matcher = bipartite_matcher.GreedyBipartiteMatcher() box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() unmatched_cls_target = tf.constant([[0, 0], [0, 0], [0, 0]], tf.float32) target_assigner = targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, unmatched_cls_target=unmatched_cls_target) prior_means = tf.constant([[0.0, 0.0, 0.5, 0.5]]) prior_stddevs = tf.constant([[1.0, 1.0, 1.0, 1.0]]) priors = box_list.BoxList(prior_means) priors.add_field('stddev', prior_stddevs) box_corners = [[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 0.9, 0.9], [.75, 0, .95, .27]] boxes = box_list.BoxList(tf.constant(box_corners)) groundtruth_labels = tf.constant([[[0, 1], [1, 0]]], tf.float32) with self.assertRaises(ValueError): target_assigner.assign(priors, boxes, groundtruth_labels, num_valid_rows=3)
def _get_agnostic_target_assigner(self): similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher(matched_threshold=0.5, unmatched_threshold=0.5) box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() return targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, unmatched_cls_target=None)
def _get_multi_class_target_assigner(self, num_classes): similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher(matched_threshold=0.5, unmatched_threshold=0.5) box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() unmatched_cls_target = tf.constant([1] + num_classes * [0], tf.float32) return targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, unmatched_cls_target=unmatched_cls_target)
def test_assign_multidimensional_class_targets(self): similarity_calc = region_similarity_calculator.NegSqDistSimilarity() matcher = bipartite_matcher.GreedyBipartiteMatcher() box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() unmatched_cls_target = tf.constant([[0, 0], [0, 0]], tf.float32) target_assigner = targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, unmatched_cls_target=unmatched_cls_target) prior_means = tf.constant([[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 1.0, 0.8], [0, 0.5, .5, 1.0], [.75, 0, 1.0, .25]]) prior_stddevs = tf.constant(4 * [4 * [.1]]) priors = box_list.BoxList(prior_means) priors.add_field('stddev', prior_stddevs) box_corners = [[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 0.9, 0.9], [.75, 0, .95, .27]] boxes = box_list.BoxList(tf.constant(box_corners)) groundtruth_labels = tf.constant( [[[0, 1], [1, 0]], [[1, 0], [0, 1]], [[0, 1], [1, .5]]], tf.float32) exp_cls_targets = [[[0, 1], [1, 0]], [[1, 0], [0, 1]], [[0, 0], [0, 0]], [[0, 1], [1, .5]]] exp_cls_weights = [1, 1, 1, 1] exp_reg_targets = [[0, 0, 0, 0], [0, 0, -1, 1], [0, 0, 0, 0], [0, 0, -.5, .2]] exp_reg_weights = [1, 1, 0, 1] exp_matching_anchors = [0, 1, 3] result = target_assigner.assign(priors, boxes, groundtruth_labels, num_valid_rows=3) (cls_targets, cls_weights, reg_targets, reg_weights, match) = result with self.test_session() as sess: (cls_targets_out, cls_weights_out, reg_targets_out, reg_weights_out, matching_anchors_out) = sess.run([ cls_targets, cls_weights, reg_targets, reg_weights, match.matched_column_indices() ]) self.assertAllClose(cls_targets_out, exp_cls_targets) self.assertAllClose(cls_weights_out, exp_cls_weights) self.assertAllClose(reg_targets_out, exp_reg_targets) self.assertAllClose(reg_weights_out, exp_reg_weights) self.assertAllClose(matching_anchors_out, exp_matching_anchors) self.assertEquals(cls_targets_out.dtype, np.float32) self.assertEquals(cls_weights_out.dtype, np.float32) self.assertEquals(reg_targets_out.dtype, np.float32) self.assertEquals(reg_weights_out.dtype, np.float32) self.assertEquals(matching_anchors_out.dtype, np.int32)
def _get_multi_dimensional_target_assigner(self, target_dimensions): similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher(matched_threshold=0.5, unmatched_threshold=0.5) box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() unmatched_cls_target = tf.constant(np.zeros(target_dimensions), tf.float32) return targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, unmatched_cls_target=unmatched_cls_target)
def _get_agnostic_target_assigner(self): similarity_calc = region_similarity_calculator.NegSqDistSimilarity() matcher = bipartite_matcher.GreedyBipartiteMatcher() box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() return targetassigner.TargetAssigner(similarity_calc, matcher, box_coder, positive_class_weight=1.0, negative_class_weight=1.0, unmatched_cls_target=None)
def _get_multi_class_target_assigner(self, num_classes): similarity_calc = region_similarity_calculator.NegSqDistSimilarity() matcher = bipartite_matcher.GreedyBipartiteMatcher() box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() unmatched_cls_target = tf.constant([1] + num_classes * [0], tf.float32) return targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, positive_class_weight=1.0, negative_class_weight=1.0, unmatched_cls_target=unmatched_cls_target)
def test_assign_multidimensional_class_targets_rbox(self): similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher(matched_threshold=0.5, unmatched_threshold=0.3) box_coder = faster_rcnn_rbox_coder.FasterRcnnRBoxCoder() unmatched_cls_target = tf.constant([[0, 0], [0, 0]], tf.float32) target_assigner = targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, unmatched_cls_target=unmatched_cls_target) anchors = tf.constant([[0.0, 0.0, 0.5, 0.5, 0.0], [0.5, 0.5, 1.0, 0.8, 0.0], [0, 0.5, .5, 1.0, 0.0], [.75, 0, 1.0, .25, 0.0]]) anchors_rbox = rbox_list.RBoxList(anchors) box_corners = [[0.0, 0.0, 0.5, 0.5, 0.0], [0.5, 0.5, 0.9, 0.9, 0.0], [.75, 0, .95, .27, 0.0]] rboxes = rbox_list.RBoxList(tf.constant(box_corners)) groundtruth_labels = tf.constant( [[[0, 1], [1, 0]], [[1, 0], [0, 1]], [[0, 1], [1, .5]]], tf.float32) exp_cls_targets = [[[0, 1], [1, 0]], [[1, 0], [0, 1]], [[0, 0], [0, 0]], [[0, 1], [1, .5]]] exp_cls_weights = [1, 1, 1, 1] exp_reg_targets = [[0, 0, 0, 0, 0], [0, 0, -0.105361, 0.117783, 0], [0, 0, 0, 0, 0], [0, 0, -0.0512933, .0769611, 0]] exp_reg_weights = [1, 1, 0, 1] exp_matching_anchors = [0, 1, 3] result = target_assigner.assign(anchors_rbox, rboxes, groundtruth_labels) (cls_targets, cls_weights, reg_targets, reg_weights, match) = result with self.test_session() as sess: (cls_targets_out, cls_weights_out, reg_targets_out, reg_weights_out, matching_anchors_out) = \ sess.run([cls_targets, cls_weights, reg_targets, reg_weights, match.matched_column_indices()]) self.assertAllClose(cls_targets_out, exp_cls_targets) self.assertAllClose(cls_weights_out, exp_cls_weights) self.assertAllClose(reg_targets_out, exp_reg_targets) self.assertAllClose(reg_weights_out, exp_reg_weights) self.assertAllClose(matching_anchors_out, exp_matching_anchors) self.assertEquals(cls_targets_out.dtype, np.float32) self.assertEquals(cls_weights_out.dtype, np.float32) self.assertEquals(reg_targets_out.dtype, np.float32) self.assertEquals(reg_weights_out.dtype, np.float32) self.assertEquals(matching_anchors_out.dtype, np.int32)
def graph_fn(anchor_means, anchor_stddevs, groundtruth_box_corners): similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher(matched_threshold=0.5, unmatched_threshold=0.3) box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() target_assigner = targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, unmatched_cls_target=None) anchors_boxlist = box_list.BoxList(anchor_means) anchors_boxlist.add_field('stddev', anchor_stddevs) groundtruth_boxlist = box_list.BoxList(groundtruth_box_corners) result = target_assigner.assign(anchors_boxlist, groundtruth_boxlist) (cls_targets, cls_weights, reg_targets, reg_weights, _) = result return (cls_targets, cls_weights, reg_targets, reg_weights)
def _get_multi_dimensional_target_assigner_rbox(self, target_dimensions): similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher(matched_threshold=0.5, unmatched_threshold=0.3) box_coder = faster_rcnn_rbox_coder.FasterRcnnRBoxCoder() unmatched_cls_target = tf.constant(np.zeros(target_dimensions), tf.float32) return targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, positive_class_weight=1.0, negative_class_weight=1.0, unmatched_cls_target=unmatched_cls_target)
def graph_fn(anchor_means, groundtruth_box_corners, groundtruth_keypoints): similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher(matched_threshold=0.5, unmatched_threshold=0.5) box_coder = keypoint_box_coder.KeypointBoxCoder( num_keypoints=6, scale_factors=[10.0, 10.0, 5.0, 5.0]) target_assigner = targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, unmatched_cls_target=None) anchors_boxlist = box_list.BoxList(anchor_means) groundtruth_boxlist = box_list.BoxList(groundtruth_box_corners) groundtruth_boxlist.add_field(fields.BoxListFields.keypoints, groundtruth_keypoints) result = target_assigner.assign(anchors_boxlist, groundtruth_boxlist) (cls_targets, cls_weights, reg_targets, reg_weights, _) = result return (cls_targets, cls_weights, reg_targets, reg_weights)
def test_assign_with_ignored_matches(self): # Note: test is very similar to above. The third box matched with an IOU # of 0.35, which is between the matched and unmatched threshold. This means # That like above the expected classification targets are [1, 1, 0]. # Unlike above, the third target is ignored and therefore expected # classification weights are [1, 1, 0]. similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher(matched_threshold=0.5, unmatched_threshold=0.3) box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() target_assigner = targetassigner.TargetAssigner( similarity_calc, matcher, box_coder) prior_means = tf.constant([[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 1.0, 0.8], [0.0, 0.5, .9, 1.0]]) prior_stddevs = tf.constant(3 * [4 * [.1]]) priors = box_list.BoxList(prior_means) priors.add_field('stddev', prior_stddevs) box_corners = [[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 0.9, 0.9]] boxes = box_list.BoxList(tf.constant(box_corners)) exp_cls_targets = [[1], [1], [0]] exp_cls_weights = [1, 1, 0] exp_reg_targets = [[0, 0, 0, 0], [0, 0, -1, 1], [0, 0, 0, 0]] exp_reg_weights = [1, 1, 0] exp_matching_anchors = [0, 1] result = target_assigner.assign(priors, boxes) (cls_targets, cls_weights, reg_targets, reg_weights, match) = result with self.test_session() as sess: (cls_targets_out, cls_weights_out, reg_targets_out, reg_weights_out, matching_anchors_out) = sess.run([ cls_targets, cls_weights, reg_targets, reg_weights, match.matched_column_indices() ]) self.assertAllClose(cls_targets_out, exp_cls_targets) self.assertAllClose(cls_weights_out, exp_cls_weights) self.assertAllClose(reg_targets_out, exp_reg_targets) self.assertAllClose(reg_weights_out, exp_reg_weights) self.assertAllClose(matching_anchors_out, exp_matching_anchors) self.assertEquals(cls_targets_out.dtype, np.float32) self.assertEquals(cls_weights_out.dtype, np.float32) self.assertEquals(reg_targets_out.dtype, np.float32) self.assertEquals(reg_weights_out.dtype, np.float32) self.assertEquals(matching_anchors_out.dtype, np.int32)
def __init__(self, categories, iou_threshold=0.5): """Constructor. Args: categories: A list of dicts, each of which has the following keys - 'id': (required) an integer id uniquely identifying this category. 'name': (required) string representing category name e.g., 'cat', 'dog'. iou_threshold: Threshold above which to consider a box as matched during evaluation. """ super(CalibrationDetectionEvaluator, self).__init__(categories) # Constructing target_assigner to match detections to groundtruth. similarity_calc = region_similarity_calculator.IouSimilarity() matcher = argmax_matcher.ArgMaxMatcher( matched_threshold=iou_threshold, unmatched_threshold=iou_threshold) box_coder = mean_stddev_box_coder.MeanStddevBoxCoder(stddev=0.1) self._target_assigner = target_assigner.TargetAssigner( similarity_calc, matcher, box_coder)
def test_assign_multiclass_unequal_class_weights(self): similarity_calc = region_similarity_calculator.NegSqDistSimilarity() matcher = bipartite_matcher.GreedyBipartiteMatcher() box_coder = mean_stddev_box_coder.MeanStddevBoxCoder() unmatched_cls_target = tf.constant([1, 0, 0, 0, 0, 0, 0], tf.float32) target_assigner = targetassigner.TargetAssigner( similarity_calc, matcher, box_coder, positive_class_weight=1.0, negative_class_weight=0.5, unmatched_cls_target=unmatched_cls_target) prior_means = tf.constant([[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 1.0, 0.8], [0, 0.5, .5, 1.0], [.75, 0, 1.0, .25]]) prior_stddevs = tf.constant(4 * [4 * [.1]]) priors = box_list.BoxList(prior_means) priors.add_field('stddev', prior_stddevs) box_corners = [[0.0, 0.0, 0.5, 0.5], [0.5, 0.5, 0.9, 0.9], [.75, 0, .95, .27]] boxes = box_list.BoxList(tf.constant(box_corners)) groundtruth_labels = tf.constant( [[0, 1, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 1, 0], [0, 0, 0, 1, 0, 0, 0]], tf.float32) exp_cls_weights = [1, 1, .5, 1] result = target_assigner.assign(priors, boxes, groundtruth_labels, num_valid_rows=3) (_, cls_weights, _, _, _) = result with self.test_session() as sess: cls_weights_out = sess.run(cls_weights) self.assertAllClose(cls_weights_out, exp_cls_weights)
def __init__(self, is_training, anchor_generator, box_predictor, box_coder, feature_extractor, matcher, region_similarity_calculator, image_resizer_fn, non_max_suppression_fn, score_conversion_fn, classification_loss, localization_loss, classification_loss_weight, localization_loss_weight, normalize_loss_by_num_matches, hard_example_miner, add_summaries=True): """SSDMetaArch Constructor. TODO: group NMS parameters + score converter into a class and loss parameters into a class and write config protos for postprocessing and losses. Args: is_training: A boolean indicating whether the training version of the computation graph should be constructed. anchor_generator: an anchor_generator.AnchorGenerator object. box_predictor: a box_predictor.BoxPredictor object. box_coder: a box_coder.BoxCoder object. feature_extractor: a SSDFeatureExtractor object. matcher: a matcher.Matcher object. region_similarity_calculator: a region_similarity_calculator.RegionSimilarityCalculator object. image_resizer_fn: a callable for image resizing. This callable always takes a rank-3 image tensor (corresponding to a single image) and returns a rank-3 image tensor, possibly with new spatial dimensions. See builders/image_resizer_builder.py. non_max_suppression_fn: batch_multiclass_non_max_suppression callable that takes `boxes`, `scores` and optional `clip_window` inputs (with all other inputs already set) and returns a dictionary hold tensors with keys: `detection_boxes`, `detection_scores`, `detection_classes` and `num_detections`. See `post_processing. batch_multiclass_non_max_suppression` for the type and shape of these tensors. score_conversion_fn: callable elementwise nonlinearity (that takes tensors as inputs and returns tensors). This is usually used to convert logits to probabilities. classification_loss: an core.losses.Loss object. localization_loss: a core.losses.Loss object. classification_loss_weight: float localization_loss_weight: float normalize_loss_by_num_matches: boolean hard_example_miner: a losses.HardExampleMiner object (can be None) add_summaries: boolean (default: True) controlling whether summary ops should be added to tensorflow graph. """ super(SSDMetaArch, self).__init__(num_classes=box_predictor.num_classes) self._is_training = is_training # Needed for fine-tuning from classification checkpoints whose # variables do not have the feature extractor scope. self._extract_features_scope = 'FeatureExtractor' self._anchor_generator = anchor_generator self._box_predictor = box_predictor self._box_coder = box_coder self._feature_extractor = feature_extractor self._matcher = matcher self._region_similarity_calculator = region_similarity_calculator # TODO: handle agnostic mode and positive/negative class weights unmatched_cls_target = None unmatched_cls_target = tf.constant([1] + self.num_classes * [0], tf.float32) self._target_assigner = target_assigner.TargetAssigner( self._region_similarity_calculator, self._matcher, self._box_coder, positive_class_weight=1.0, negative_class_weight=1.0, unmatched_cls_target=unmatched_cls_target) self._classification_loss = classification_loss self._localization_loss = localization_loss self._classification_loss_weight = classification_loss_weight self._localization_loss_weight = localization_loss_weight self._normalize_loss_by_num_matches = normalize_loss_by_num_matches self._hard_example_miner = hard_example_miner self._image_resizer_fn = image_resizer_fn self._non_max_suppression_fn = non_max_suppression_fn self._score_conversion_fn = score_conversion_fn self._anchors = None self._add_summaries = add_summaries
def _build_ssd_model(ssd_config, is_training, add_summaries): """Builds an SSD detection model based on the model config. Args: ssd_config: A ssd.proto object containing the config for the desired SSDMetaArch. is_training: True if this model is being built for training purposes. add_summaries: Whether to add tf summaries in the model. Returns: SSDMetaArch based on the config. Raises: ValueError: If ssd_config.type is not recognized (i.e. not registered in model_class_map). """ num_classes = ssd_config.num_classes # Feature extractor feature_extractor = _build_ssd_feature_extractor( feature_extractor_config=ssd_config.feature_extractor, freeze_batchnorm=ssd_config.freeze_batchnorm, is_training=is_training) box_coder = box_coder_builder.build(ssd_config.box_coder) matcher = matcher_builder.build(ssd_config.matcher) region_similarity_calculator = sim_calc.build( ssd_config.similarity_calculator) encode_background_as_zeros = ssd_config.encode_background_as_zeros negative_class_weight = ssd_config.negative_class_weight anchor_generator = anchor_generator_builder.build( ssd_config.anchor_generator) if feature_extractor.is_keras_model: ssd_box_predictor = box_predictor_builder.build_keras( conv_hyperparams_fn=hyperparams_builder.KerasLayerHyperparams, freeze_batchnorm=ssd_config.freeze_batchnorm, inplace_batchnorm_update=False, num_predictions_per_location_list=anchor_generator. num_anchors_per_location(), box_predictor_config=ssd_config.box_predictor, is_training=is_training, num_classes=num_classes, add_background_class=ssd_config.add_background_class) else: ssd_box_predictor = box_predictor_builder.build( hyperparams_builder.build, ssd_config.box_predictor, is_training, num_classes, ssd_config.add_background_class) image_resizer_fn = image_resizer_builder.build(ssd_config.image_resizer) non_max_suppression_fn, score_conversion_fn = post_processing_builder.build( ssd_config.post_processing) (classification_loss, localization_loss, classification_weight, localization_weight, hard_example_miner, random_example_sampler, expected_loss_weights_fn) = losses_builder.build(ssd_config.loss) normalize_loss_by_num_matches = ssd_config.normalize_loss_by_num_matches normalize_loc_loss_by_codesize = ssd_config.normalize_loc_loss_by_codesize equalization_loss_config = ops.EqualizationLossConfig( weight=ssd_config.loss.equalization_loss.weight, exclude_prefixes=ssd_config.loss.equalization_loss.exclude_prefixes) target_assigner_instance = target_assigner.TargetAssigner( region_similarity_calculator, matcher, box_coder, negative_class_weight=negative_class_weight) ssd_meta_arch_fn = ssd_meta_arch.SSDMetaArch kwargs = {} return ssd_meta_arch_fn( is_training=is_training, anchor_generator=anchor_generator, box_predictor=ssd_box_predictor, box_coder=box_coder, feature_extractor=feature_extractor, encode_background_as_zeros=encode_background_as_zeros, image_resizer_fn=image_resizer_fn, non_max_suppression_fn=non_max_suppression_fn, score_conversion_fn=score_conversion_fn, classification_loss=classification_loss, localization_loss=localization_loss, classification_loss_weight=classification_weight, localization_loss_weight=localization_weight, normalize_loss_by_num_matches=normalize_loss_by_num_matches, hard_example_miner=hard_example_miner, target_assigner_instance=target_assigner_instance, add_summaries=add_summaries, normalize_loc_loss_by_codesize=normalize_loc_loss_by_codesize, freeze_batchnorm=ssd_config.freeze_batchnorm, inplace_batchnorm_update=ssd_config.inplace_batchnorm_update, add_background_class=ssd_config.add_background_class, explicit_background_class=ssd_config.explicit_background_class, random_example_sampler=random_example_sampler, expected_loss_weights_fn=expected_loss_weights_fn, use_confidences_as_targets=ssd_config.use_confidences_as_targets, implicit_example_weight=ssd_config.implicit_example_weight, equalization_loss_config=equalization_loss_config, **kwargs)