def loss(self, prediction_dict):
# these should include mini-batch values only
objectness_gt = prediction_dict[self.PRED_MB_OBJECTNESS_GT]
offsets_gt = prediction_dict[self.PRED_MB_OFFSETS_GT]
# Predictions
with tf.variable_scope('rpn_prediction_mini_batch'):
objectness = prediction_dict[self.PRED_MB_OBJECTNESS]
offsets = prediction_dict[self.PRED_MB_OFFSETS]
with tf.variable_scope('rpn_losses'):
with tf.variable_scope('objectness'):
cls_loss = losses.WeightedSoftmaxLoss()
cls_loss_weight = self._config.loss_config.cls_loss_weight
objectness_loss = cls_loss(objectness,
objectness_gt,
weight=cls_loss_weight)
with tf.variable_scope('obj_norm'):
# normalize by the number of anchor mini-batches
objectness_loss = objectness_loss / tf.cast(
tf.shape(objectness_gt)[0], dtype=tf.float32)
tf.summary.scalar('objectness', objectness_loss)
with tf.variable_scope('regression'):
reg_loss = losses.WeightedSmoothL1Loss()
reg_loss_weight = self._config.loss_config.reg_loss_weight
anchorwise_localization_loss = reg_loss(offsets,
offsets_gt,
weight=reg_loss_weight)
masked_localization_loss = \
anchorwise_localization_loss * objectness_gt[:, 1]
localization_loss = tf.reduce_sum(masked_localization_loss)
with tf.variable_scope('reg_norm'):
# normalize by the number of positive objects
num_positives = tf.reduce_sum(objectness_gt[:, 1])
# Assert the condition `num_positives > 0`
with tf.control_dependencies(
[tf.assert_positive(num_positives)]):
localization_loss = localization_loss / num_positives
tf.summary.scalar('regression', localization_loss)
with tf.variable_scope('total_loss'):
total_loss = objectness_loss + localization_loss
loss_dict = {
self.LOSS_RPN_OBJECTNESS: objectness_loss,
self.LOSS_RPN_REGRESSION: localization_loss,
}
return loss_dict, total_loss
def test_bool_pos_mask(self):
# Check that applying a boolean mask gives the same output as
# multiplying with a float mask
pos_classification_mask = np.asarray([True, False, True],
dtype=np.bool)
offsets = np.asarray(
[[1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2, 2], [3, 3, 3, 3, 3, 3]],
dtype=np.float32)
offsets_gt = offsets + [[1], [2], [3]]
angle_vectors = np.asarray(
[[0.9, 0.0361], [0.0361, 0.9], [0.7071, 0.7071]], dtype=np.float32)
angle_vectors_gt = np.asarray([[1, 0], [0.8268, 0.5625], [0, 1]],
dtype=np.float32)
# Convert to tensors
tf_pos_classification_floats = tf.cast(pos_classification_mask,
dtype=tf.float32)
tf_offsets = tf.convert_to_tensor(offsets, dtype=tf.float32)
tf_offsets_gt = tf.convert_to_tensor(offsets_gt, dtype=tf.float32)
tf_angle_vectors = tf.convert_to_tensor(angle_vectors,
dtype=tf.float32)
tf_angle_vectors_gt = tf.convert_to_tensor(angle_vectors_gt,
dtype=tf.float32)
reg_loss = losses.WeightedSmoothL1Loss()
anchorwise_loc_loss = reg_loss(tf_offsets, tf_offsets_gt, weight=1.0)
anchorwise_ang_loss = reg_loss(tf_angle_vectors,
tf_angle_vectors_gt,
weight=1.0)
# Masking by multiplying with mask floats
anchorwise_combined_reg_loss = (anchorwise_loc_loss +
anchorwise_ang_loss) * \
tf_pos_classification_floats
# Masking with tf.boolean_mask
pos_localization_loss = tf.reduce_sum(
tf.boolean_mask(anchorwise_loc_loss, pos_classification_mask))
pos_orientation_loss = tf.reduce_sum(
tf.boolean_mask(anchorwise_ang_loss, pos_classification_mask))
combined_reg_loss = pos_localization_loss + pos_orientation_loss
with self.test_session() as sess:
anchorwise_loc_loss_out = sess.run(anchorwise_loc_loss)
anchorwise_ang_loss = sess.run(anchorwise_ang_loss)
# Masked with floats mulitplication
anchorwise_combined_reg_loss_out = sess.run(
anchorwise_combined_reg_loss)
# Masked with tf.boolean_mask
pos_loc_loss_out, pos_ang_loss_out, combined_reg_loss = \
sess.run([pos_localization_loss,
pos_orientation_loss,
combined_reg_loss])
pos_classification_floats_out = sess.run(
tf_pos_classification_floats)
expected_pos_loc_loss = np.sum(anchorwise_loc_loss_out *
pos_classification_floats_out)
expected_pos_ang_loss = np.sum(anchorwise_ang_loss *
pos_classification_floats_out)
expected_combined_reg_loss = expected_pos_loc_loss + \
expected_pos_ang_loss
np.testing.assert_allclose(pos_loc_loss_out, expected_pos_loc_loss)
np.testing.assert_allclose(pos_ang_loss_out, expected_pos_ang_loss)
# Check that floats multiplication is the same as tf.boolean_mask
np.testing.assert_almost_equal(
np.sum(anchorwise_combined_reg_loss_out), combined_reg_loss)
# Check that combined regression loss is as expected
np.testing.assert_almost_equal(combined_reg_loss,
expected_combined_reg_loss)
def _get_off_ang_logits_loss(model, offsets, offsets_gt, angle_logits,
angle_logits_gt, cls_softmax, cls_gt):
"""Calculates the smooth L1 combined offset and angle loss, normalized by
the number of positives
Args:
model: network model
offsets: prediction offsets
offsets_gt: ground truth offsets
angle_logits: prediction angle logits shape=(?,16)
angle_logits_gt: ground truth angle logits shape=(?,16)
cls_softmax: prediction classification softmax scores
cls_gt: classification ground truth one-hot vector
Returns:
final_reg_loss: combined offset and angle vector loss
offset_loss_norm: normalized offset loss
ang_loss_norm: normalized angle vector loss
"""
weighted_smooth_l1_localization_loss = losses.WeightedSmoothL1Loss()
weighted_softmax_loss = losses.WeightedSoftmaxLoss()
reg_loss_weight = model._config.loss_config.reg_loss_weight
ang_loss_weight = model._config.loss_config.ang_loss_weight
anchorwise_localization_loss = weighted_smooth_l1_localization_loss( # shape=(?,)
offsets, offsets_gt, weight=reg_loss_weight)
# cls_gt = tf.Print(cls_gt, ['^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^line 144(pplp loss) : model._positive_selection =', model._positive_selection], summarize=1000)
# cls_softmax = tf.Print(cls_softmax, ['^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^line 145(pplp loss) : cls_softmax =', cls_softmax], summarize=1000)
# cls_gt = tf.Print(cls_gt, ['^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^line 146(pplp loss) : cls_gt =', cls_gt], summarize=1000)
positive_mask = _get_positive_mask(model._positive_selection, cls_softmax,
cls_gt)
# positive_mask = tf.Print(positive_mask, ['^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^line 149(pplp loss) : positive_mask =', positive_mask], summarize=1000)
# Cast to float to get number of positives
pos_classification_floats = tf.cast(positive_mask, tf.float32)
# Apply mask to only keep regression loss for positive predictions
pos_localization_loss = tf.reduce_sum(
tf.boolean_mask(anchorwise_localization_loss, positive_mask))
# Apply mask to angle_logits before we calculate final loss
pos_angle_logits = tf.boolean_mask(angle_logits, positive_mask)
pos_angle_logits_gt = tf.boolean_mask(angle_logits_gt, positive_mask)
# pos_angle_logits = tf.Print(pos_angle_logits, ['^^^^^^^^^^^^^^^line 247(pplp loss) : pos_angle_logits =', pos_angle_logits], summarize=1000)
# pos_angle_logits_gt = tf.Print(pos_angle_logits_gt, ['^^^^^^^^^^^^^^^line 248(pplp loss) : pos_angle_logits_gt =', pos_angle_logits_gt], summarize=1000)
pos_orientation_loss = weighted_softmax_loss(pos_angle_logits,
pos_angle_logits_gt,
weight=ang_loss_weight)
# pos_orientation_loss = tf.Print(pos_orientation_loss, ['^^^^^^^^^^^^^^^line 251(pplp loss) : pos_orientation_loss =', pos_orientation_loss], summarize=1000)
# Combine regression losses
combined_reg_loss = pos_localization_loss + pos_orientation_loss
with tf.variable_scope('reg_norm'):
# Normalize by the number of positive/desired classes
# only if we have any positives
num_positives = tf.reduce_sum(pos_classification_floats)
pos_div_cond = tf.not_equal(num_positives, 0)
offset_loss_norm = tf.cond(
pos_div_cond, lambda: pos_localization_loss / num_positives,
lambda: tf.constant(0.0))
ang_loss_norm = tf.cond(
pos_div_cond, lambda: pos_orientation_loss / tf.cast(
tf.shape(pos_angle_logits_gt)[0], dtype=tf.float32),
lambda: tf.constant(0.0))
final_reg_loss = tf.cond(pos_div_cond,
lambda: combined_reg_loss / num_positives,
lambda: tf.constant(0.0))
# Add summary scalars
if model._train_val_test == 'train':
tf.summary.scalar('localization', offset_loss_norm)
tf.summary.scalar('orientation', ang_loss_norm)
tf.summary.scalar('regression_total', final_reg_loss)
tf.summary.scalar('mb_num_positives', num_positives)
return final_reg_loss, offset_loss_norm, ang_loss_norm
def _get_offset_only_loss(model, offsets, offsets_gt,
cls_softmax, cls_gt):
"""Calculates the smooth L1 combined offset and angle loss, normalized by
the number of positives
Args:
model: network model
offsets: prediction offsets
offsets_gt: ground truth offsets
cls_softmax: prediction classification softmax scores
cls_gt: classification ground truth one-hot vector
Returns:
final_reg_loss: normalized offset loss
offset_loss_norm: normalized offset loss
"""
weighted_smooth_l1_loss = losses.WeightedSmoothL1Loss()
reg_loss_weight = model._config.loss_config.reg_loss_weight
anchorwise_localization_loss = weighted_smooth_l1_loss(
offsets, offsets_gt, weight=reg_loss_weight)
classification_argmax = tf.argmax(cls_softmax, axis=1)
# Get the ground truth class indices back from one_hot vector
class_indices_gt = tf.argmax(cls_gt, axis=1)
# Mask for which predictions are not background
not_background_mask = tf.greater(class_indices_gt, 0)
# Combine the masks
if model._positive_selection == 'corr_cls':
# Which prediction classifications match ground truth
correct_classifications_mask = tf.equal(
classification_argmax,
class_indices_gt)
pos_classification_mask = tf.logical_and(
correct_classifications_mask, not_background_mask)
elif model._positive_selection == 'not_bkg':
pos_classification_mask = not_background_mask
else:
raise ValueError('Invalid positive selection',
model._positive_selection)
# Cast to float to get number of positives
pos_classification_floats = tf.cast(
pos_classification_mask, tf.float32)
# Apply mask to only keep regression loss for positive predictions
pos_localization_loss = tf.reduce_sum(tf.boolean_mask(
anchorwise_localization_loss, pos_classification_mask))
with tf.variable_scope('reg_norm'):
# normalize by the number of positive/desired classes
# only if we have any positives
num_positives = tf.reduce_sum(pos_classification_floats)
pos_div_cond = tf.not_equal(num_positives, 0)
offset_loss_norm = tf.cond(
pos_div_cond,
lambda: pos_localization_loss / num_positives,
lambda: tf.constant(0.0))
reg_loss = tf.cond(
pos_div_cond,
lambda: pos_localization_loss / num_positives,
lambda: tf.constant(0.0))
if model._train_val_test == 'train':
tf.summary.scalar('localization', offset_loss_norm)
tf.summary.scalar('regression_total', reg_loss)
tf.summary.scalar('mb_num_positives', num_positives)
return reg_loss, offset_loss_norm
def _get_off_ang_loss(model, offsets, offsets_gt,
angle_vectors, angle_vectors_gt,
cls_softmax, cls_gt):
"""Calculates the smooth L1 combined offset and angle loss, normalized by
the number of positives
Args:
model: network model
offsets: prediction offsets
offsets_gt: ground truth offsets
angle_vectors: prediction angle vectors
angle_vectors_gt: ground truth angle vectors
cls_softmax: prediction classification softmax scores
cls_gt: classification ground truth one-hot vector
Returns:
final_reg_loss: combined offset and angle vector loss
offset_loss_norm: normalized offset loss
ang_loss_norm: normalized angle vector loss
"""
weighted_smooth_l1_localization_loss = losses.WeightedSmoothL1Loss()
reg_loss_weight = model._config.loss_config.reg_loss_weight
ang_loss_weight = model._config.loss_config.ang_loss_weight
anchorwise_localization_loss = weighted_smooth_l1_localization_loss(
offsets, offsets_gt, weight=reg_loss_weight)
anchorwise_orientation_loss = weighted_smooth_l1_localization_loss(
angle_vectors, angle_vectors_gt, weight=ang_loss_weight)
positive_mask = _get_positive_mask(model._positive_selection,
cls_softmax, cls_gt)
# Cast to float to get number of positives
pos_classification_floats = tf.cast(
positive_mask, tf.float32)
# Apply mask to only keep regression loss for positive predictions
pos_localization_loss = tf.reduce_sum(tf.boolean_mask(
anchorwise_localization_loss, positive_mask))
pos_orientation_loss = tf.reduce_sum(tf.boolean_mask(
anchorwise_orientation_loss, positive_mask))
# Combine regression losses
combined_reg_loss = pos_localization_loss + pos_orientation_loss
with tf.variable_scope('reg_norm'):
# Normalize by the number of positive/desired classes
# only if we have any positives
num_positives = tf.reduce_sum(pos_classification_floats)
pos_div_cond = tf.not_equal(num_positives, 0)
offset_loss_norm = tf.cond(
pos_div_cond,
lambda: pos_localization_loss / num_positives,
lambda: tf.constant(0.0))
ang_loss_norm = tf.cond(
pos_div_cond,
lambda: pos_orientation_loss / num_positives,
lambda: tf.constant(0.0))
final_reg_loss = tf.cond(
pos_div_cond,
lambda: combined_reg_loss / num_positives,
lambda: tf.constant(0.0))
# Add summary scalars
if model._train_val_test == 'train':
tf.summary.scalar('localization', offset_loss_norm)
tf.summary.scalar('orientation', ang_loss_norm)
tf.summary.scalar('regression_total', final_reg_loss)
tf.summary.scalar('mb_num_positives', num_positives)
return final_reg_loss, offset_loss_norm, ang_loss_norm