def loss(self, prediction_dict): # Fetch the ground truth values location_gt = prediction_dict[self.PRED_LOCATION_GT] orientation_gt = prediction_dict[self.PRED_ORIENT_GT] orientation_vec_gt = orientation_encoder.tf_orientation_to_angle_vector( orientation_gt) size_gt = prediction_dict[self.PRED_SIZE_GT] # Fetch the prediction values with tf.variable_scope('epbrm_prediction_batch'): location_pred = prediction_dict[self.PRED_LOCATION] orientation_vec_pred = prediction_dict[self.PRED_ORIENT] size_pred = prediction_dict[self.PRED_SIZE] # Calculate the losses with tf.variable_scope('epbrm_losses'): # Location loss with tf.variable_scope('location'): # Fetch the loss function reg_loss = losses.WeightedSmoothL1Loss() location_loss = reg_loss(location_pred, location_gt, weight=5.0) # Size loss with tf.variable_scope('size'): # Fetch the loss function reg_loss = losses.WeightedSmoothL1Loss() size_loss = reg_loss(size_pred, size_gt, weight=5.0) # Orientation loss with tf.variable_scope('orientation'): # Fetch the loss function reg_loss = losses.WeightedSmoothL1Loss() orientation_loss = reg_loss(orientation_vec_pred, orientation_vec_gt, weight=1.0) with tf.variable_scope('epBRM_loss'): total_loss = tf.reduce_sum(location_loss) + tf.reduce_sum( size_loss) + tf.reduce_sum(orientation_loss) return total_loss
def loss(self, prediction_dict): # these should include mini-batch values only objectness_gt = prediction_dict[self.PRED_MB_OBJECTNESS_GT] offsets_gt = prediction_dict[self.PRED_MB_OFFSETS_GT] # Predictions with tf.variable_scope('rpn_prediction_mini_batch'): objectness = prediction_dict[self.PRED_MB_OBJECTNESS] offsets = prediction_dict[self.PRED_MB_OFFSETS] with tf.variable_scope('rpn_losses'): with tf.variable_scope('objectness'): cls_loss = losses.WeightedSoftmaxLoss() cls_loss_weight = self._config.loss_config.cls_loss_weight objectness_loss = cls_loss(objectness, objectness_gt, weight=cls_loss_weight) with tf.variable_scope('obj_norm'): # normalize by the number of anchor mini-batches objectness_loss = objectness_loss / tf.cast( tf.shape(objectness_gt)[0], dtype=tf.float32) tf.summary.scalar('objectness', objectness_loss) with tf.variable_scope('regression'): reg_loss = losses.WeightedSmoothL1Loss() reg_loss_weight = self._config.loss_config.reg_loss_weight anchorwise_localization_loss = reg_loss(offsets, offsets_gt, weight=reg_loss_weight) masked_localization_loss = \ anchorwise_localization_loss * objectness_gt[:, 1] localization_loss = tf.reduce_sum(masked_localization_loss) with tf.variable_scope('reg_norm'): # normalize by the number of positive objects num_positives = tf.reduce_sum(objectness_gt[:, 1]) # Assert the condition `num_positives > 0` with tf.control_dependencies( [tf.assert_positive(num_positives)]): localization_loss = localization_loss / num_positives tf.summary.scalar('regression', localization_loss) with tf.variable_scope('total_loss'): total_loss = objectness_loss + localization_loss loss_dict = { self.LOSS_RPN_OBJECTNESS: objectness_loss, self.LOSS_RPN_REGRESSION: localization_loss, } return loss_dict, total_loss
def _get_offset_only_loss(model, offsets, offsets_gt, cls_softmax, cls_gt): """Calculates the smooth L1 combined offset and angle loss, normalized by the number of positives Args: model: network model offsets: prediction offsets offsets_gt: ground truth offsets cls_softmax: prediction classification softmax scores cls_gt: classification ground truth one-hot vector Returns: final_reg_loss: normalized offset loss offset_loss_norm: normalized offset loss """ weighted_smooth_l1_loss = losses.WeightedSmoothL1Loss() reg_loss_weight = model._config.loss_config.reg_loss_weight anchorwise_localization_loss = weighted_smooth_l1_loss( offsets, offsets_gt, weight=reg_loss_weight) classification_argmax = tf.argmax(cls_softmax, axis=1) # Get the ground truth class indices back from one_hot vector class_indices_gt = tf.argmax(cls_gt, axis=1) # Mask for which predictions are not background not_background_mask = tf.greater(class_indices_gt, 0) # Combine the masks if model._positive_selection == 'corr_cls': # Which prediction classifications match ground truth correct_classifications_mask = tf.equal(classification_argmax, class_indices_gt) pos_classification_mask = tf.logical_and(correct_classifications_mask, not_background_mask) elif model._positive_selection == 'not_bkg': pos_classification_mask = not_background_mask else: raise ValueError('Invalid positive selection', model._positive_selection) # Cast to float to get number of positives pos_classification_floats = tf.cast(pos_classification_mask, tf.float32) # Apply mask to only keep regression loss for positive predictions pos_localization_loss = tf.reduce_sum( tf.boolean_mask(anchorwise_localization_loss, pos_classification_mask)) with tf.variable_scope('reg_norm'): # normalize by the number of positive/desired classes # only if we have any positives num_positives = tf.reduce_sum(pos_classification_floats) pos_div_cond = tf.not_equal(num_positives, 0) offset_loss_norm = tf.cond( pos_div_cond, lambda: pos_localization_loss / num_positives, lambda: tf.constant(0.0)) reg_loss = tf.cond(pos_div_cond, lambda: pos_localization_loss / num_positives, lambda: tf.constant(0.0)) if model._train_val_test == 'train': tf.summary.scalar('localization', offset_loss_norm) tf.summary.scalar('regression_total', reg_loss) tf.summary.scalar('mb_num_positives', num_positives) return reg_loss, offset_loss_norm
def _get_off_ang_loss(model, offsets, offsets_gt, angle_vectors, angle_vectors_gt, cls_softmax, cls_gt): """Calculates the smooth L1 combined offset and angle loss, normalized by the number of positives Args: model: network model offsets: prediction offsets offsets_gt: ground truth offsets angle_vectors: prediction angle vectors angle_vectors_gt: ground truth angle vectors cls_softmax: prediction classification softmax scores cls_gt: classification ground truth one-hot vector Returns: final_reg_loss: combined offset and angle vector loss offset_loss_norm: normalized offset loss ang_loss_norm: normalized angle vector loss """ weighted_smooth_l1_localization_loss = losses.WeightedSmoothL1Loss() reg_loss_weight = model._config.loss_config.reg_loss_weight ang_loss_weight = model._config.loss_config.ang_loss_weight anchorwise_localization_loss = weighted_smooth_l1_localization_loss( offsets, offsets_gt, weight=reg_loss_weight) anchorwise_orientation_loss = weighted_smooth_l1_localization_loss( angle_vectors, angle_vectors_gt, weight=ang_loss_weight) positive_mask = _get_positive_mask(model._positive_selection, cls_softmax, cls_gt) # Cast to float to get number of positives pos_classification_floats = tf.cast(positive_mask, tf.float32) # Apply mask to only keep regression loss for positive predictions pos_localization_loss = tf.reduce_sum( tf.boolean_mask(anchorwise_localization_loss, positive_mask)) pos_orientation_loss = tf.reduce_sum( tf.boolean_mask(anchorwise_orientation_loss, positive_mask)) # Combine regression losses combined_reg_loss = pos_localization_loss + pos_orientation_loss with tf.variable_scope('reg_norm'): # Normalize by the number of positive/desired classes # only if we have any positives num_positives = tf.reduce_sum(pos_classification_floats) pos_div_cond = tf.not_equal(num_positives, 0) offset_loss_norm = tf.cond( pos_div_cond, lambda: pos_localization_loss / num_positives, lambda: tf.constant(0.0)) ang_loss_norm = tf.cond(pos_div_cond, lambda: pos_orientation_loss / num_positives, lambda: tf.constant(0.0)) final_reg_loss = tf.cond(pos_div_cond, lambda: combined_reg_loss / num_positives, lambda: tf.constant(0.0)) # Add summary scalars if model._train_val_test == 'train': tf.summary.scalar('localization', offset_loss_norm) tf.summary.scalar('orientation', ang_loss_norm) tf.summary.scalar('regression_total', final_reg_loss) tf.summary.scalar('mb_num_positives', num_positives) return final_reg_loss, offset_loss_norm, ang_loss_norm
def test_bool_pos_mask(self): # Check that applying a boolean mask gives the same output as # multiplying with a float mask pos_classification_mask = np.asarray([True, False, True], dtype=np.bool) offsets = np.asarray( [[1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2, 2], [3, 3, 3, 3, 3, 3]], dtype=np.float32) offsets_gt = offsets + [[1], [2], [3]] angle_vectors = np.asarray( [[0.9, 0.0361], [0.0361, 0.9], [0.7071, 0.7071]], dtype=np.float32) angle_vectors_gt = np.asarray([[1, 0], [0.8268, 0.5625], [0, 1]], dtype=np.float32) # Convert to tensors tf_pos_classification_floats = tf.cast(pos_classification_mask, dtype=tf.float32) tf_offsets = tf.convert_to_tensor(offsets, dtype=tf.float32) tf_offsets_gt = tf.convert_to_tensor(offsets_gt, dtype=tf.float32) tf_angle_vectors = tf.convert_to_tensor(angle_vectors, dtype=tf.float32) tf_angle_vectors_gt = tf.convert_to_tensor(angle_vectors_gt, dtype=tf.float32) reg_loss = losses.WeightedSmoothL1Loss() anchorwise_loc_loss = reg_loss(tf_offsets, tf_offsets_gt, weight=1.0) anchorwise_ang_loss = reg_loss(tf_angle_vectors, tf_angle_vectors_gt, weight=1.0) # Masking by multiplying with mask floats anchorwise_combined_reg_loss = (anchorwise_loc_loss + anchorwise_ang_loss) * \ tf_pos_classification_floats # Masking with tf.boolean_mask pos_localization_loss = tf.reduce_sum( tf.boolean_mask(anchorwise_loc_loss, pos_classification_mask)) pos_orientation_loss = tf.reduce_sum( tf.boolean_mask(anchorwise_ang_loss, pos_classification_mask)) combined_reg_loss = pos_localization_loss + pos_orientation_loss with self.test_session() as sess: anchorwise_loc_loss_out = sess.run(anchorwise_loc_loss) anchorwise_ang_loss = sess.run(anchorwise_ang_loss) # Masked with floats mulitplication anchorwise_combined_reg_loss_out = sess.run( anchorwise_combined_reg_loss) # Masked with tf.boolean_mask pos_loc_loss_out, pos_ang_loss_out, combined_reg_loss = \ sess.run([pos_localization_loss, pos_orientation_loss, combined_reg_loss]) pos_classification_floats_out = sess.run( tf_pos_classification_floats) expected_pos_loc_loss = np.sum(anchorwise_loc_loss_out * pos_classification_floats_out) expected_pos_ang_loss = np.sum(anchorwise_ang_loss * pos_classification_floats_out) expected_combined_reg_loss = expected_pos_loc_loss + \ expected_pos_ang_loss np.testing.assert_allclose(pos_loc_loss_out, expected_pos_loc_loss) np.testing.assert_allclose(pos_ang_loss_out, expected_pos_ang_loss) # Check that floats multiplication is the same as tf.boolean_mask np.testing.assert_almost_equal( np.sum(anchorwise_combined_reg_loss_out), combined_reg_loss) # Check that combined regression loss is as expected np.testing.assert_almost_equal(combined_reg_loss, expected_combined_reg_loss)