def rpn_losses(self):
with tf.variable_scope('rpn_losses'):
minibatch_indices, minibatch_anchor_matched_gtboxes, object_mask, minibatch_labels_one_hot = \
self.make_minibatch(self.anchors)
minibatch_anchors = tf.gather(self.anchors, minibatch_indices)
minibatch_encode_boxes = tf.gather(self.rpn_encode_boxes,
minibatch_indices)
minibatch_boxes_scores = tf.gather(self.rpn_scores,
minibatch_indices)
# encode gtboxes
minibatch_encode_gtboxes = encode_and_decode.encode_boxes(
unencode_boxes=minibatch_anchor_matched_gtboxes,
reference_boxes=minibatch_anchors,
scale_factors=self.scale_factors)
positive_anchors_in_img = draw_box_with_color(
self.img_batch,
minibatch_anchors * tf.expand_dims(object_mask, 1),
text=tf.shape(tf.where(tf.equal(object_mask, 1.0)))[0])
negative_mask = tf.cast(
tf.logical_not(tf.cast(object_mask, tf.bool)), tf.float32)
negative_anchors_in_img = draw_box_with_color(
self.img_batch,
minibatch_anchors * tf.expand_dims(negative_mask, 1),
text=tf.shape(tf.where(tf.equal(object_mask, 0.0)))[0])
minibatch_decode_boxes = encode_and_decode.decode_boxes(
encode_boxes=minibatch_encode_boxes,
reference_boxes=minibatch_anchors,
scale_factors=self.scale_factors)
tf.summary.image('/positive_anchors', positive_anchors_in_img)
tf.summary.image('/negative_anchors', negative_anchors_in_img)
top_k_scores, top_k_indices = tf.nn.top_k(
minibatch_boxes_scores[:, 1], k=5)
top_detections_in_img = draw_box_with_color(
self.img_batch,
tf.gather(minibatch_decode_boxes, top_k_indices),
text=tf.shape(top_k_scores)[0])
tf.summary.image('/top_5', top_detections_in_img)
# losses
with tf.variable_scope('rpn_location_loss'):
location_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=object_mask)
slim.losses.add_loss(
location_loss) # add smooth l1 loss to losses collection
with tf.variable_scope('rpn_classification_loss'):
classification_loss = slim.losses.softmax_cross_entropy(
logits=minibatch_boxes_scores,
onehot_labels=minibatch_labels_one_hot)
return location_loss, classification_loss
def rpn_losses(self):
with tf.variable_scope('rpn_losses'):
minibatch_indices, minibatch_anchor_matched_gtboxes, \
object_mask, minibatch_labels_one_hot = self.make_minibatch(self.anchors)
minibatch_anchors = tf.gather(self.anchors, minibatch_indices)
minibatch_encode_boxes = tf.gather(self.rpn_encode_boxes, minibatch_indices)
minibatch_boxes_scores = tf.gather(self.rpn_scores, minibatch_indices)
# encode gtboxes
minibatch_encode_gtboxes = encode_and_decode.encode_boxes(unencode_boxes=minibatch_anchor_matched_gtboxes,
reference_boxes=minibatch_anchors,
scale_factors=self.scale_factors)
positive_anchors_in_img = draw_box_with_color(self.img_batch,
minibatch_anchors * tf.expand_dims(object_mask, 1),
text=tf.shape(tf.where(tf.equal(object_mask, 1.0)))[0])
negative_mask = tf.cast(tf.logical_not(tf.cast(object_mask, tf.bool)), tf.float32)
negative_anchors_in_img = draw_box_with_color(self.img_batch,
minibatch_anchors * tf.expand_dims(negative_mask, 1),
text=tf.shape(tf.where(tf.equal(object_mask, 0.0)))[0])
minibatch_decode_boxes = encode_and_decode.decode_boxes(encode_boxes=minibatch_encode_boxes,
reference_boxes=minibatch_anchors,
scale_factors=self.scale_factors)
tf.summary.image('/positive_anchors', positive_anchors_in_img)
tf.summary.image('/negative_anchors', negative_anchors_in_img)
minibatch_boxes_softmax_scores = tf.gather(slim.softmax(self.rpn_scores), minibatch_indices)
top_k_scores, top_k_indices = tf.nn.top_k(minibatch_boxes_softmax_scores[:, 1], k=20)
top_k_boxes = tf.gather(minibatch_decode_boxes, top_k_indices)
top_detections_in_img = draw_boxes_with_scores(self.img_batch,
boxes=top_k_boxes,
scores=top_k_scores)
tf.summary.image('/top_20', top_detections_in_img)
temp_indices = tf.reshape(tf.where(tf.greater(top_k_scores, cfgs.FINAL_SCORE_THRESHOLD)), [-1])
rpn_predict_boxes = tf.gather(top_k_boxes, temp_indices)
rpn_predict_scores = tf.gather(top_k_scores, temp_indices)
# losses
with tf.variable_scope('rpn_location_loss'):
location_loss = losses.l1_smooth_losses(predict_boxes=minibatch_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=object_mask)
slim.losses.add_loss(location_loss) # add smooth l1 loss to losses collection
with tf.variable_scope('rpn_classification_loss'):
classification_loss = slim.losses.softmax_cross_entropy(logits=minibatch_boxes_scores,
onehot_labels=minibatch_labels_one_hot)
return location_loss, classification_loss, rpn_predict_boxes, rpn_predict_scores
def fast_rcnn_loss(self):
with tf.variable_scope('fast_rcnn_loss'):
minibatch_indices, minibatch_reference_boxes_mattached_gtboxes, minibatch_object_mask, \
minibatch_label_one_hot = self.fast_rcnn_minibatch(self.fast_rcnn_all_level_proposals)
minibatch_reference_boxes = tf.gather(
self.fast_rcnn_all_level_proposals, minibatch_indices)
minibatch_encode_boxes = tf.gather(
self.fast_rcnn_encode_boxes,
minibatch_indices) # [minibatch_size, num_classes*4]
minibatch_scores = tf.gather(self.fast_rcnn_scores,
minibatch_indices)
# encode gtboxes
minibatch_encode_gtboxes = \
encode_and_decode.encode_boxes(
unencode_boxes=minibatch_reference_boxes_mattached_gtboxes,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors
)
# [minibatch_size, num_classes*4]
minibatch_encode_gtboxes = tf.tile(minibatch_encode_gtboxes,
[1, self.num_classes])
class_weights_list = []
category_list = tf.unstack(minibatch_label_one_hot, axis=1)
for i in range(1, self.num_classes + 1):
tmp_class_weights = tf.ones(
shape=[tf.shape(minibatch_encode_boxes)[0], 4],
dtype=tf.float32)
tmp_class_weights = tmp_class_weights * tf.expand_dims(
category_list[i], axis=1)
class_weights_list.append(tmp_class_weights)
class_weights = tf.concat(
class_weights_list, axis=1) # [minibatch_size, num_classes*4]
# loss
with tf.variable_scope('fast_rcnn_classification_loss'):
fast_rcnn_classification_loss = slim.losses.softmax_cross_entropy(
logits=minibatch_scores,
onehot_labels=minibatch_label_one_hot)
with tf.variable_scope('fast_rcnn_location_loss'):
fast_rcnn_location_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=minibatch_object_mask,
classes_weights=class_weights)
slim.losses.add_loss(fast_rcnn_location_loss)
return fast_rcnn_location_loss, fast_rcnn_classification_loss
def head_loss(self):
minibatch_reference_proboxes, minibatch_encode_gtboxes,\
object_mask, gt_class_ids = self.build_head_train_sample()
pooled_feature = self.get_rois_feature(minibatch_reference_proboxes)
fast_rcnn_predict_boxes, fast_rcnn_predict_scores = self.head_net(
pooled_feature, self.IS_TRAINING)
with tf.variable_scope("head_loss"):
# from fast_rcnn_predict_boxes choose corresponding encode
row_index = tf.range(0, tf.shape(gt_class_ids)[1])
row_index = tf.expand_dims(row_index, 0)
multi_row_index = tf.tile(row_index, [cfgs.BATCH_SIZE, 1])
multi_row_index = tf.expand_dims(multi_row_index, axis=-1)
expand_gt_class_ids = tf.expand_dims(gt_class_ids, axis=-1)
index = tf.concat([multi_row_index, expand_gt_class_ids],
axis=-1) # 匹配对应gt那个类别的预测框吗？
fast_rcnn_predict_boxes = boxes_utils.batch_slice(
[fast_rcnn_predict_boxes, index],
lambda x, y: tf.gather_nd(x, y), cfgs.BATCH_SIZE)
# loss
with tf.variable_scope('head_class_loss'):
fast_rcnn_classification_loss = tf.losses.sparse_softmax_cross_entropy(
labels=gt_class_ids, # [batch_size,num]
logits=fast_rcnn_predict_scores) # [batch_size,num,2]
fast_rcnn_classification_loss = tf.cond(
tf.is_nan(fast_rcnn_classification_loss), lambda: 0.0,
lambda: fast_rcnn_classification_loss)
with tf.variable_scope('head_location_loss'):
fast_rcnn_location_loss = losses.l1_smooth_losses(
predict_boxes=fast_rcnn_predict_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=object_mask)
return fast_rcnn_location_loss, fast_rcnn_classification_loss
def fast_rcnn_loss(self):
with tf.variable_scope('fast_rcnn_loss'):
minibatch_indices, minibatch_reference_boxes_mattached_gtboxes, minibatch_object_mask, \
minibatch_label_one_hot = self.fast_rcnn_minibatch(self.fast_rcnn_all_level_proposals)
minibatch_reference_boxes = tf.gather(
self.fast_rcnn_all_level_proposals, minibatch_indices)
minibatch_encode_boxes = tf.gather(
self.fast_rcnn_encode_boxes,
minibatch_indices) # [minibatch_size, num_classes*4]
minibatch_scores = tf.gather(self.fast_rcnn_scores,
minibatch_indices)
positive_proposals_in_img = draw_box_with_color(
self.img_batch,
minibatch_reference_boxes *
tf.expand_dims(minibatch_object_mask, 1),
text=tf.shape(tf.where(tf.equal(minibatch_object_mask,
1.0)))[0])
negative_mask = tf.cast(
tf.logical_not(tf.cast(minibatch_object_mask, tf.bool)),
tf.float32)
negative_proposals_in_img = draw_box_with_color(
self.img_batch,
minibatch_reference_boxes * tf.expand_dims(negative_mask, 1),
text=tf.shape(tf.where(tf.equal(minibatch_object_mask,
0.0)))[0])
tf.summary.image('/positive_proposals', positive_proposals_in_img)
tf.summary.image('/negative_proposals', negative_proposals_in_img)
if cfgs.CLASS_NUM == 1:
minibatch_decode_boxes = encode_and_decode.decode_boxes(
encode_boxes=minibatch_encode_boxes,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors)
minibatch_softmax_scores = tf.gather(
slim.softmax(self.fast_rcnn_scores), minibatch_indices)
top_k_scores, top_k_indices = tf.nn.top_k(
minibatch_softmax_scores[:, 1], k=5)
top_detections_in_img = draw_boxes_with_scores(
self.img_batch,
boxes=tf.gather(minibatch_decode_boxes, top_k_indices),
scores=top_k_scores)
tf.summary.image('/top_5', top_detections_in_img)
# encode gtboxes
minibatch_encode_gtboxes = \
encode_and_decode.encode_boxes(
unencode_boxes=minibatch_reference_boxes_mattached_gtboxes,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors
)
# [minibatch_size, num_classes*4]
minibatch_encode_gtboxes = tf.tile(minibatch_encode_gtboxes,
[1, self.num_classes])
class_weights_list = []
category_list = tf.unstack(minibatch_label_one_hot, axis=1)
for i in range(1, self.num_classes + 1):
tmp_class_weights = tf.ones(
shape=[tf.shape(minibatch_encode_boxes)[0], 4],
dtype=tf.float32)
tmp_class_weights = tmp_class_weights * tf.expand_dims(
category_list[i], axis=1)
class_weights_list.append(tmp_class_weights)
class_weights = tf.concat(
class_weights_list, axis=1) # [minibatch_size, num_classes*4]
# loss
with tf.variable_scope('fast_rcnn_classification_loss'):
fast_rcnn_classification_loss = tf.losses.softmax_cross_entropy(
logits=minibatch_scores,
onehot_labels=minibatch_label_one_hot)
with tf.variable_scope('fast_rcnn_location_loss'):
fast_rcnn_location_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=minibatch_object_mask,
classes_weights=class_weights)
tf.losses.add_loss(fast_rcnn_location_loss)
return fast_rcnn_location_loss, fast_rcnn_classification_loss
def fast_rcnn_loss(self):
with tf.variable_scope('fast_rcnn_loss'):
minibatch_indices, minibatch_reference_boxes_mattached_gtboxes, minibatch_object_mask, \
minibatch_label_one_hot = self.fast_rcnn_minibatch(self.fast_rcnn_all_level_rotate_proposals) #######################
# minibatch_reference_boxes = tf.gather(self.fast_rcnn_all_level_horizontal_proposals, minibatch_indices)
minibatch_reference_boxes = tf.gather(self.fast_rcnn_all_level_rotate_proposals, minibatch_indices)
minibatch_encode_boxes = tf.gather(self.fast_rcnn_encode_boxes,
minibatch_indices) # [minibatch_size, num_classes*5]
minibatch_scores = tf.gather(self.fast_rcnn_scores, minibatch_indices)
positive_proposals_in_img = draw_box_with_color(self.img_batch,
minibatch_reference_boxes * tf.expand_dims(
minibatch_object_mask, 1),
text=tf.shape(tf.where(tf.equal(minibatch_object_mask, 1.0)))[0])
negative_mask = tf.cast(tf.logical_not(tf.cast(minibatch_object_mask, tf.bool)), tf.float32)
negative_proposals_in_img = draw_box_with_color(self.img_batch,
minibatch_reference_boxes * tf.expand_dims(negative_mask, 1),
text=tf.shape(tf.where(tf.equal(minibatch_object_mask, 0.0)))[0])
tf.summary.image('/positive_proposals', positive_proposals_in_img)
tf.summary.image('/negative_proposals', negative_proposals_in_img)
minibatch_decode_boxes = encode_and_decode.decode_boxes(encode_boxes=minibatch_encode_boxes,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors)
minibatch_softmax_scores = tf.gather(slim.softmax(self.fast_rcnn_scores), minibatch_indices)
top_k_scores, top_k_indices = tf.nn.top_k(minibatch_softmax_scores[:, 1], k=5)
top_detections_in_img = draw_boxes_with_scores(self.img_batch,
boxes=tf.gather(minibatch_decode_boxes, top_k_indices),
scores=top_k_scores)
tf.summary.image('/top_5', top_detections_in_img)
# encode gtboxes
minibatch_encode_gtboxes = \
encode_and_decode.encode_boxes(
unencode_boxes=minibatch_reference_boxes_mattached_gtboxes,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors)
# [minibatch_size, num_classes*5]
minibatch_encode_gtboxes = tf.tile(minibatch_encode_gtboxes, [1, self.num_classes])
class_weights_list = []
category_list = tf.unstack(minibatch_label_one_hot, axis=1)
for i in range(1, self.num_classes+1):
tmp_class_weights = tf.ones(shape=[tf.shape(minibatch_encode_boxes)[0], 5], dtype=tf.float32)
tmp_class_weights = tmp_class_weights * tf.expand_dims(category_list[i], axis=1)
class_weights_list.append(tmp_class_weights)
class_weights = tf.concat(class_weights_list, axis=1) # [minibatch_size, num_classes*5]
# loss
with tf.variable_scope('fast_rcnn_classification_loss'):
fast_rcnn_classification_loss = slim.losses.softmax_cross_entropy(logits=minibatch_scores,
onehot_labels=minibatch_label_one_hot)
# if DEBUG:
# print_tensors(minibatch_scores, 'minibatch_scores')
# print_tensors(classification_loss, '2nd_cls_loss')
with tf.variable_scope('fast_rcnn_location_loss'):
fast_rcnn_location_loss = losses.l1_smooth_losses(predict_boxes=minibatch_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=minibatch_object_mask,
classes_weights=class_weights)
slim.losses.add_loss(fast_rcnn_location_loss)
return fast_rcnn_location_loss, fast_rcnn_classification_loss
def rpn_loss(self):
'''
:param: self.gtboxes_and_label: [n, 5]->[ymin, xmin, ymax, xmax, cls]
:param: self.anchors: [m, 4]-> [ymin, xmin, ymax, xmax]
:param:self.rpn_encode_boxes: [m, 4]->[ycenter, xcenter, h, w]
:return:
'''
with tf.variable_scope('rpn_loss'):
minibatch_indices,\
minibatch_anchor_matched_gtboxes,\
object_mask,\
minibatch_label_onehot = self.make_minibatch()
minibatch_anchors = tf.gather(self.anchors, minibatch_indices)
minibatch_rpn_encode_boxes = tf.gather(self.rpn_encode_boxes,
minibatch_indices)
minibatch_rpn_scores = tf.gather(self.rpn_scores,
minibatch_indices)
minibatch_encode_boxes_label = encode_and_decode.encode_boxes(
minibatch_anchors, minibatch_anchor_matched_gtboxes,
self.scale_factors)
# summary
positive_anchors_in_img = draw_box_with_tensor(
img_batch=self.img_batch,
boxes=minibatch_anchors * tf.expand_dims(object_mask, 1),
text=tf.shape(tf.where(tf.equal(object_mask, 1)))[0])
negative_mask = tf.cast(
tf.logical_not(tf.cast(object_mask, tf.bool)), tf.float32)
negative_anchors_in_img = draw_box_with_tensor(
img_batch=self.img_batch,
boxes=minibatch_anchors * tf.expand_dims(negative_mask, 1),
text=tf.shape(tf.where(tf.equal(negative_mask, 1)))[0])
minibatch_decode_anchors = encode_and_decode.decode_boxes(
encode_boxes=minibatch_rpn_encode_boxes,
reference_boxes=minibatch_anchors,
scale_factors=self.scale_factors)
# clip boxes into image shape
minibatch_decode_anchors = boxes_utils.clip_boxes_to_img_boundaries(
minibatch_decode_anchors, tf.shape(self.img_batch))
positive_decode_anchor_in_img = \
draw_box_with_tensor(img_batch=self.img_batch,
boxes=minibatch_decode_anchors*tf.expand_dims(object_mask, 1),
text=tf.shape(tf.where(tf.equal(object_mask, 1)))[0]
)
tf.summary.image('images/rpn/losses/anchors_positive_minibatch',
positive_anchors_in_img)
tf.summary.image('images/rpn/losses/anchors_negative_minibatch',
negative_anchors_in_img)
tf.summary.image('images/rpn/losses/decode_anchor_positive',
positive_decode_anchor_in_img)
# losses
with tf.variable_scope('rpn_localization_losses'):
classify_loss = slim.losses.softmax_cross_entropy(
logits=minibatch_rpn_scores,
onehot_labels=minibatch_label_onehot)
location_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_rpn_encode_boxes,
gtboxes=minibatch_encode_boxes_label,
object_weights=object_mask)
slim.losses.add_loss(
location_loss) # add location loss to losses collections
return location_loss, classify_loss
def rpn_losses(self):
with tf.variable_scope('rpn_losses'):
minibatch_indices, minibatch_anchor_matched_gtboxes, object_mask, minibatch_labels_one_hot = \
self.make_minibatch(self.anchors)
minibatch_anchors = tf.gather(self.anchors, minibatch_indices)
minibatch_encode_boxes = tf.gather(self.rpn_encode_boxes,
minibatch_indices)
minibatch_boxes_scores = tf.gather(self.rpn_scores,
minibatch_indices)
# encode gtboxes
minibatch_encode_gtboxes = encode_and_decode.encode_boxes(
unencode_boxes=minibatch_anchor_matched_gtboxes,
reference_boxes=minibatch_anchors,
scale_factors=self.scale_factors)
positive_anchors_in_img = draw_box_with_color(
self.img_batch,
minibatch_anchors * tf.expand_dims(object_mask, 1),
text=tf.shape(tf.where(tf.equal(object_mask, 1.0)))[0])
negative_mask = tf.cast(
tf.logical_not(tf.cast(object_mask, tf.bool)), tf.float32)
negative_anchors_in_img = draw_box_with_color(
self.img_batch,
minibatch_anchors * tf.expand_dims(negative_mask, 1),
text=tf.shape(tf.where(tf.equal(object_mask, 0.0)))[0])
minibatch_decode_boxes = encode_and_decode.decode_boxes(
encode_boxes=minibatch_encode_boxes,
reference_boxes=minibatch_anchors,
scale_factors=self.scale_factors)
tf.summary.image('/positive_anchors', positive_anchors_in_img)
tf.summary.image('/negative_anchors', negative_anchors_in_img)
top_k_scores, top_k_indices = tf.nn.top_k(
minibatch_boxes_scores[:, 1], k=1)
top_detections_in_img = draw_box_with_color(
self.img_batch,
tf.gather(minibatch_decode_boxes, top_k_indices),
text=tf.shape(top_k_scores)[0])
tf.summary.image('/top_1', top_detections_in_img)
# losses
with tf.variable_scope('rpn_location_loss'):
location_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=object_mask)
slim.losses.add_loss(
location_loss) # add smooth l1 loss to losses collection
with tf.variable_scope('rpn_classification_loss'):
# logits = tf.cast(minibatch_boxes_scores, tf.float32)
# onehot_labels = tf.cast(minibatch_labels_one_hot, tf.float32)
# one = tf.ones(shape=tf.shape(onehot_labels), dtype=tf.float32)
# predictions_pt = tf.where(tf.equal(onehot_labels, 1), logits, 1-logits)
#
# # add small value to avoid
# alpha_t = tf.scalar_mul(0.25, one)
# alpha_t = tf.where(tf.equal(onehot_labels, 1), alpha_t, 1 - alpha_t)
# gamma = tf.scalar_mul(2, one)
# new_gamma = tf.where(tf.less(predictions_pt, 0.5), -gamma, gamma)
# classification_loss = tf.multiply(tf.multiply(alpha_t, slim.losses.softmax_cross_entropy(logits=logits,
# onehot_labels=onehot_labels)), tf.pow((1-predictions_pt), 2))
# # classification_loss = tf.multiply(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits,
# # labels=onehot_labels), tf.pow((1-predictions_pt), 2))
# classification_loss = tf.reduce_sum(classification_loss[:,0]+classification_loss[:,1])
# # classification_loss = slim.losses.softmax_cross_entropy(logits=tf.clip_by_value(minibatch_boxes_scores,1e-8,tf.reduce_max(minibatch_boxes_scores)),
# # onehot_labels=minibatch_labels_one_hot)
classification_loss = slim.losses.softmax_cross_entropy(
logits=minibatch_boxes_scores,
onehot_labels=minibatch_labels_one_hot)
return location_loss, classification_loss
def fast_rcnn_loss(self):
with tf.variable_scope('fast_rcnn_loss'):
minibatch_indices, minibatch_reference_boxes_mattached_gtboxes, minibatch_object_mask, \
minibatch_label_one_hot = self.fast_rcnn_minibatch(self.fast_rcnn_all_level_proposals)
minibatch_reference_boxes = tf.gather(
self.fast_rcnn_all_level_proposals, minibatch_indices)
minibatch_encode_boxes = tf.gather(
self.fast_rcnn_encode_boxes,
minibatch_indices) # [minibatch_size, num_classes*4]
minibatch_scores = tf.gather(self.fast_rcnn_scores,
minibatch_indices)
positive_proposals_in_img = draw_box_with_color(
self.img_batch,
minibatch_reference_boxes *
tf.expand_dims(minibatch_object_mask, 1),
text=tf.shape(tf.where(tf.equal(minibatch_object_mask,
1.0)))[0])
negative_mask = tf.cast(
tf.logical_not(tf.cast(minibatch_object_mask, tf.bool)),
tf.float32)
negative_proposals_in_img = draw_box_with_color(
self.img_batch,
minibatch_reference_boxes * tf.expand_dims(negative_mask, 1),
text=tf.shape(tf.where(tf.equal(minibatch_object_mask,
0.0)))[0])
tf.summary.image('/positive_proposals', positive_proposals_in_img)
tf.summary.image('/negative_proposals', negative_proposals_in_img)
if cfgs.CLASS_NUM == 1:
minibatch_decode_boxes = encode_and_decode.decode_boxes(
encode_boxes=minibatch_encode_boxes,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors)
minibatch_softmax_scores = tf.gather(
slim.softmax(self.fast_rcnn_scores), minibatch_indices)
top_k_scores, top_k_indices = tf.nn.top_k(
minibatch_softmax_scores[:, 1], k=5)
top_detections_in_img = draw_boxes_with_scores(
self.img_batch,
boxes=tf.gather(minibatch_decode_boxes, top_k_indices),
scores=top_k_scores)
tf.summary.image('/top_5', top_detections_in_img)
# encode gtboxes
minibatch_encode_gtboxes = \
encode_and_decode.encode_boxes(
unencode_boxes=minibatch_reference_boxes_mattached_gtboxes,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors
)
# [minibatch_size, num_classes*4]
minibatch_encode_gtboxes = tf.tile(minibatch_encode_gtboxes,
[1, self.num_classes])
class_weights_list = []
category_list = tf.unstack(minibatch_label_one_hot, axis=1)
for i in range(1, self.num_classes + 1):
tmp_class_weights = tf.ones(
shape=[tf.shape(minibatch_encode_boxes)[0], 4],
dtype=tf.float32)
tmp_class_weights = tmp_class_weights * tf.expand_dims(
category_list[i], axis=1)
class_weights_list.append(tmp_class_weights)
class_weights = tf.concat(
class_weights_list, axis=1) # [minibatch_size, num_classes*4]
# loss
with tf.variable_scope('fast_rcnn_classification_loss'):
logits = tf.cast(minibatch_scores, tf.float32)
onehot_labels = tf.cast(minibatch_label_one_hot, tf.float32)
one = tf.ones(shape=tf.shape(onehot_labels), dtype=tf.float32)
predictions_pt = tf.where(tf.equal(onehot_labels, 1), logits,
1 - logits)
# add small value to avoid
alpha_t = tf.scalar_mul(0.25, one)
alpha_t = tf.where(tf.equal(onehot_labels, 1), alpha_t,
1 - alpha_t)
gamma = tf.scalar_mul(2, one)
new_gamma = tf.where(tf.less(predictions_pt, 0.5), -gamma,
gamma)
##PFL
fast_rcnn_classification_loss = tf.multiply(
tf.multiply(
alpha_t,
slim.losses.softmax_cross_entropy(
logits=logits, onehot_labels=onehot_labels)),
tf.pow(1 - predictions_pt, new_gamma))
##FL
# fast_rcnn_classification_loss = tf.multiply(tf.multiply(alpha_t, slim.losses.softmax_cross_entropy(logits=logits,
# onehot_labels=onehot_labels)), tf.pow(1-predictions_pt, 2))
#FL和PFL，不注释这句；CE注释这句
fast_rcnn_classification_loss = tf.reduce_sum(
fast_rcnn_classification_loss[:, 0] +
fast_rcnn_classification_loss[:, 1])
##CE
# fast_rcnn_classification_loss = slim.losses.softmax_cross_entropy(logits=tf.clip_by_value(minibatch_scores,1e-8,tf.reduce_max(minibatch_scores)),
# onehot_labels=minibatch_label_one_hot)
# fast_rcnn_classification_loss = slim.losses.softmax_cross_entropy(
# logits=minibatch_scores,
# onehot_labels=minibatch_label_one_hot)
with tf.variable_scope('fast_rcnn_location_loss'):
fast_rcnn_location_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=minibatch_object_mask,
classes_weights=class_weights)
slim.losses.add_loss(fast_rcnn_location_loss)
return fast_rcnn_location_loss, fast_rcnn_classification_loss
def fast_rcnn_loss(self):
with tf.variable_scope('fast_rcnn_loss'):
minibatch_indices, minibatch_reference_boxes_mattached_gtboxes, \
minibatch_reference_boxes_mattached_gtboxes_rotate, \
minibatch_reference_boxes_mattached_head_quadrant, minibatch_object_mask, \
minibatch_label_one_hot = self.fast_rcnn_minibatch(self.fast_rcnn_all_level_proposals)
minibatch_reference_boxes = tf.gather(
self.fast_rcnn_all_level_proposals, minibatch_indices)
minibatch_encode_boxes = tf.gather(
self.fast_rcnn_encode_boxes,
minibatch_indices) # [minibatch_size, num_classes*4]
minibatch_encode_boxes_rotate = tf.gather(
self.fast_rcnn_encode_boxes_rotate,
minibatch_indices) # [minibatch_size, num_classes*5]
minibatch_head_quadrant = tf.gather(self.fast_rcnn_head_quadrant,
minibatch_indices)
minibatch_scores = tf.gather(self.fast_rcnn_scores,
minibatch_indices)
minibatch_scores_rotate = tf.gather(self.fast_rcnn_scores_rotate,
minibatch_indices)
# encode gtboxes
minibatch_encode_gtboxes = \
encode_and_decode.encode_boxes(
unencode_boxes=minibatch_reference_boxes_mattached_gtboxes,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors
)
minibatch_encode_gtboxes_rotate = encode_and_decode.encode_boxes_rotate(
unencode_boxes=
minibatch_reference_boxes_mattached_gtboxes_rotate,
reference_boxes=minibatch_reference_boxes,
scale_factors=self.scale_factors)
############### Class-agnostic Without tile
# [minibatch_size, num_classes*4]
# minibatch_encode_gtboxes = tf.tile(minibatch_encode_gtboxes, [1, self.num_classes])
############### Class-agnostic Without tile
# [minibatch_size, num_classes*5]
# minibatch_encode_gtboxes_rotate = tf.tile(minibatch_encode_gtboxes_rotate, [1, self.num_classes])
############### Class-agnostic Without tile
# minibatch_gt_head_quadrant = tf.tile(minibatch_reference_boxes_mattached_head_quadrant, [1, self.num_classes])
minibatch_gt_head_quadrant = minibatch_reference_boxes_mattached_head_quadrant
class_weights_list = []
category_list = tf.unstack(minibatch_label_one_hot, axis=1)
for i in range(1, self.num_classes + 1):
tmp_class_weights = tf.ones(
shape=[tf.shape(minibatch_encode_boxes)[0], 4],
dtype=tf.float32)
tmp_class_weights = tmp_class_weights * tf.expand_dims(
category_list[i], axis=1)
class_weights_list.append(tmp_class_weights)
class_weights = tf.concat(
class_weights_list, axis=1) # [minibatch_size, num_classes*4]
class_weights_list_rotate = []
category_list_rotate = tf.unstack(minibatch_label_one_hot, axis=1)
for i in range(1, self.num_classes + 1):
tmp_class_weights_rotate = tf.ones(
shape=[tf.shape(minibatch_encode_boxes_rotate)[0], 5],
dtype=tf.float32)
tmp_class_weights_rotate = tmp_class_weights_rotate * tf.expand_dims(
category_list_rotate[i], axis=1)
class_weights_list_rotate.append(tmp_class_weights_rotate)
class_weights_rotate = tf.concat(
class_weights_list_rotate,
axis=1) # [minibatch_size, num_classes*5]
class_weights_list_head = []
category_list_head = tf.unstack(minibatch_label_one_hot, axis=1)
for i in range(1, self.num_classes + 1):
tmp_class_weights_head = tf.ones(
shape=[tf.shape(minibatch_head_quadrant)[0], 4],
dtype=tf.float32)
tmp_class_weights_head = tmp_class_weights_head * tf.expand_dims(
category_list_head[i], axis=1)
class_weights_list_head.append(tmp_class_weights_head)
class_weights_head = tf.concat(class_weights_list_head, axis=1)
# loss
with tf.variable_scope('fast_rcnn_classification_loss'):
# fast_rcnn_classification_loss = slim.losses.softmax_cross_entropy(logits=minibatch_scores,
# onehot_labels=minibatch_label_one_hot)
fast_rcnn_classification_loss = losses.focal_loss(
prediction_tensor=minibatch_scores,
target_tensor=minibatch_label_one_hot)
slim.losses.add_loss(fast_rcnn_classification_loss)
with tf.variable_scope('fast_rcnn_location_loss'):
# fast_rcnn_location_loss = losses.l1_smooth_losses(predict_boxes=minibatch_encode_boxes,
# gtboxes=minibatch_encode_gtboxes,
# object_weights=minibatch_object_mask,
# classes_weights=class_weights)
# Class-agnostic regression
fast_rcnn_location_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=minibatch_object_mask,
classes_weights=None)
slim.losses.add_loss(fast_rcnn_location_loss)
with tf.variable_scope('fast_rcnn_classification_rotate_loss'):
# fast_rcnn_classification_rotate_loss = slim.losses.softmax_cross_entropy(logits=minibatch_scores_rotate,
# onehot_labels=minibatch_label_one_hot)
fast_rcnn_classification_rotate_loss = losses.focal_loss(
prediction_tensor=minibatch_scores_rotate,
target_tensor=minibatch_label_one_hot)
slim.losses.add_loss(fast_rcnn_classification_rotate_loss)
with tf.variable_scope('fast_rcnn_location_rotate_loss'):
# fast_rcnn_location_rotate_loss = losses.l1_smooth_losses(predict_boxes=minibatch_encode_boxes_rotate,
# gtboxes=minibatch_encode_gtboxes_rotate,
# object_weights=minibatch_object_mask,
# classes_weights=class_weights_rotate)
# Class-agnostic regression
fast_rcnn_location_rotate_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_encode_boxes_rotate,
gtboxes=minibatch_encode_gtboxes_rotate,
object_weights=minibatch_object_mask,
classes_weights=None)
slim.losses.add_loss(fast_rcnn_location_rotate_loss)
with tf.variable_scope('fast_rcnn_head_quadrant_loss'):
# fast_rcnn_head_quadrant_loss = losses.l1_smooth_losses(predict_boxes=minibatch_head_quadrant,
# gtboxes=minibatch_gt_head_quadrant,
# object_weights=minibatch_object_mask,
# classes_weights=class_weights_head)
# Class-agnostic regression
fast_rcnn_head_quadrant_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_head_quadrant,
gtboxes=minibatch_gt_head_quadrant,
object_weights=minibatch_object_mask,
classes_weights=None)
slim.losses.add_loss(
fast_rcnn_head_quadrant_loss *
10) # More importance by the bigger weight
return fast_rcnn_location_loss, fast_rcnn_classification_loss, \
fast_rcnn_location_rotate_loss, fast_rcnn_classification_rotate_loss, fast_rcnn_head_quadrant_loss * 10
def fast_rcnn_loss(self):
'''
:return:
'''
with tf.variable_scope('fast_rcnn_loss'):
minibatch_indices, minibatch_gtboxes, minibatch_onehot_label, minibatch_object_mask = self.make_minibatch(
)
minibatch_proposal_boxes = tf.gather(self.rois_boxes,
minibatch_indices)
minibatch_predict_scores = tf.gather(self.fast_rcnn_cls_scores,
minibatch_indices)
minibatch_predict_encode_boxes = tf.gather(
self.fast_rcnn_encode_boxes, minibatch_indices)
# encode minibatch_gtboxes
minibatch_encode_gtboxes = encode_boxes(
anchors=minibatch_proposal_boxes,
gtboxes=minibatch_gtboxes,
scale_factors=self.scale_factors)
# [minibatch_size, 4]->[minibatch_size, num_cls*4]
minibatch_encode_gtboxes = tf.tile(minibatch_encode_gtboxes,
[1, self.num_cls])
# class_weight_mask [minibatch_size, num_cls*4]
class_weight_mask_list = []
category_list = tf.unstack(minibatch_onehot_label, axis=1)
for i in range(1, self.num_cls + 1):
class_weight = tf.ones([self.fast_rcnn_minibatch_size, 4],
dtype=tf.float32)
class_weight = class_weight * tf.expand_dims(category_list[i],
axis=1)
class_weight_mask_list.append(class_weight)
class_weight_mask = tf.concat(class_weight_mask_list, axis=1)
# cls loss
with tf.variable_scope('fast_rcnn_cls_losses'):
fast_rcnn_cls_loss = slim.losses.softmax_cross_entropy(
logits=minibatch_predict_scores,
onehot_labels=minibatch_onehot_label)
# boxes loss
with tf.variable_scope('fast_rcnn_boxes_losses'):
fast_rcnn_boxes_loss = losses.l1_smooth_losses(
predict_boxes=minibatch_predict_encode_boxes,
gtboxes=minibatch_encode_gtboxes,
object_weights=minibatch_object_mask,
classes_weights=class_weight_mask)
slim.losses.add_loss(fast_rcnn_boxes_loss)
# check loss and decode boxes
# summary positive proposals and negative proposals
minibatch_proposal_boxes = boxes_utils.clip_boxes_to_img_boundaries(
minibatch_proposal_boxes, self.img_shape)
minibatch_positive_proposals = \
draw_box_with_tensor(img_batch=self.img_batch,
boxes=minibatch_proposal_boxes*tf.expand_dims(tf.cast(minibatch_object_mask,
tf.float32),
1),
text=tf.shape(tf.where(tf.equal(minibatch_object_mask, 1)))[0]
)
minibatch_negative_mask = tf.cast(
tf.logical_not(tf.cast(minibatch_object_mask, tf.bool)),
tf.float32)
minibatch_negative_proposals = \
draw_box_with_tensor(img_batch=self.img_batch,
boxes=minibatch_proposal_boxes * tf.expand_dims(minibatch_negative_mask, 1),
text=tf.shape(tf.where(tf.equal(minibatch_negative_mask, 1)))[0]
)
tf.summary.image('minibatch_positive_proposals',
minibatch_positive_proposals)
tf.summary.image('minibatch_negative_proposal',
minibatch_negative_proposals)
# check the cls tensor part
tf.summary.tensor_summary('minibatch_object_mask',
minibatch_object_mask)
tf.summary.tensor_summary('class_weight_mask', class_weight_mask)
tf.summary.tensor_summary('minibatch_predict_encode_boxes',
minibatch_predict_encode_boxes)
tf.summary.tensor_summary('minibatch_encode_gtboxes',
minibatch_encode_gtboxes)
tf.summary.tensor_summary('location_loss', fast_rcnn_boxes_loss)
tf.summary.tensor_summary('logits', minibatch_predict_scores)
tf.summary.tensor_summary('one_hot', minibatch_onehot_label)
return fast_rcnn_boxes_loss, fast_rcnn_cls_loss