def build_whole_detection_network(self,
input_img_batch,
gtboxes_batch_h,
gtboxes_batch_r,
gpu_id=0):
if self.is_training:
gtboxes_batch_h = tf.reshape(gtboxes_batch_h, [-1, 5])
gtboxes_batch_h = tf.cast(gtboxes_batch_h, tf.float32)
gtboxes_batch_r = tf.reshape(gtboxes_batch_r, [-1, 6])
gtboxes_batch_r = tf.cast(gtboxes_batch_r, tf.float32)
img_shape = tf.shape(input_img_batch)
# 1. build base network
feature_pyramid = self.build_base_network(input_img_batch)
# 2. build rpn
rpn_box_pred_list, rpn_cls_score_list, rpn_cls_prob_list = self.rpn_net(
feature_pyramid, 'rpn_net')
# 3. generate_anchors
anchor_list = self.make_anchors(feature_pyramid)
rpn_box_pred = tf.concat(rpn_box_pred_list, axis=0)
rpn_cls_score = tf.concat(rpn_cls_score_list, axis=0)
# rpn_cls_prob = tf.concat(rpn_cls_prob_list, axis=0)
anchors = tf.concat(anchor_list, axis=0)
if self.is_training:
with tf.variable_scope('build_loss'):
labels, target_delta, anchor_states, target_boxes = tf.py_func(
func=anchor_target_layer,
inp=[gtboxes_batch_h, gtboxes_batch_r, anchors],
Tout=[tf.float32, tf.float32, tf.float32, tf.float32])
if self.method == 'H':
self.add_anchor_img_smry(input_img_batch, anchors,
anchor_states, 0)
else:
self.add_anchor_img_smry(input_img_batch, anchors,
anchor_states, 1)
cls_loss = losses.focal_loss(labels, rpn_cls_score,
anchor_states)
if cfgs.USE_IOU_FACTOR:
reg_loss = losses.iou_smooth_l1_loss_(
target_delta, rpn_box_pred, anchor_states,
target_boxes, anchors)
# reg_loss = losses.adiou_smooth_l1_loss(target_delta, rpn_box_pred, anchor_states, target_boxes, anchors)
else:
reg_loss = losses.smooth_l1_loss(target_delta,
rpn_box_pred,
anchor_states)
self.losses_dict['cls_loss'] = cls_loss * cfgs.CLS_WEIGHT
self.losses_dict['reg_loss'] = reg_loss * cfgs.REG_WEIGHT
box_pred_list, cls_prob_list, proposal_list = rpn_box_pred_list, rpn_cls_prob_list, anchor_list
all_box_pred_list, all_cls_prob_list, all_proposal_list = [], [], []
for i in range(cfgs.NUM_REFINE_STAGE):
box_pred_list, cls_prob_list, proposal_list = self.refine_stage(
input_img_batch,
gtboxes_batch_r,
box_pred_list,
cls_prob_list,
proposal_list,
feature_pyramid,
gpu_id,
pos_threshold=cfgs.REFINE_IOU_POSITIVE_THRESHOLD[i],
neg_threshold=cfgs.REFINE_IOU_NEGATIVE_THRESHOLD[i],
stage='' if i == 0 else '_stage{}'.format(i + 2),
proposal_filter=True if i == 0 else False)
if not self.is_training:
all_box_pred_list.extend(box_pred_list)
all_cls_prob_list.extend(cls_prob_list)
all_proposal_list.extend(proposal_list)
else:
all_box_pred_list, all_cls_prob_list, all_proposal_list = box_pred_list, cls_prob_list, proposal_list
with tf.variable_scope('postprocess_detctions'):
box_pred = tf.concat(all_box_pred_list, axis=0)
cls_prob = tf.concat(all_cls_prob_list, axis=0)
proposal = tf.concat(all_proposal_list, axis=0)
boxes, scores, category = postprocess_detctions(
refine_bbox_pred=box_pred,
refine_cls_prob=cls_prob,
anchors=proposal,
is_training=self.is_training)
boxes = tf.stop_gradient(boxes)
scores = tf.stop_gradient(scores)
category = tf.stop_gradient(category)
if self.is_training:
return boxes, scores, category, self.losses_dict
else:
return boxes, scores, category
def build_whole_detection_network(self,
input_img_batch,
gtboxes_batch_h,
gtboxes_batch_r,
gpu_id=0):
if self.is_training:
gtboxes_batch_h = tf.reshape(gtboxes_batch_h, [-1, 5])
gtboxes_batch_h = tf.cast(gtboxes_batch_h, tf.float32)
gtboxes_batch_r = tf.reshape(gtboxes_batch_r, [-1, 6])
gtboxes_batch_r = tf.cast(gtboxes_batch_r, tf.float32)
# 1. build base network
feature_pyramid = self.build_base_network(input_img_batch)
# 2. build rpn
rpn_box_pred_list, rpn_cls_score_list, rpn_cls_prob_list = self.rpn_net(
feature_pyramid, 'rpn_net')
# 3. generate_anchors
anchor_list = self.make_anchors(feature_pyramid)
rpn_box_pred = tf.concat(rpn_box_pred_list, axis=0)
rpn_cls_score = tf.concat(rpn_cls_score_list, axis=0)
# rpn_cls_prob = tf.concat(rpn_cls_prob_list, axis=0)
anchors = tf.concat(anchor_list, axis=0)
if self.is_training:
with tf.variable_scope('build_loss'):
labels, target_delta, anchor_states, target_boxes = tf.py_func(
func=anchor_target_layer,
inp=[gtboxes_batch_h, gtboxes_batch_r, anchors],
Tout=[tf.float32, tf.float32, tf.float32, tf.float32])
if self.method == 'H':
self.add_anchor_img_smry(input_img_batch, anchors,
anchor_states, 0)
else:
self.add_anchor_img_smry(input_img_batch, anchors,
anchor_states, 1)
cls_loss = losses.focal_loss(labels, rpn_cls_score,
anchor_states)
if cfgs.USE_IOU_FACTOR:
reg_loss = losses.iou_smooth_l1_loss(
target_delta, rpn_box_pred, anchor_states,
target_boxes, anchors)
else:
reg_loss = losses.smooth_l1_loss(target_delta,
rpn_box_pred,
anchor_states)
self.losses_dict['cls_loss'] = cls_loss * cfgs.CLS_WEIGHT
self.losses_dict['reg_loss'] = reg_loss * cfgs.REG_WEIGHT
with tf.variable_scope('refine_feature_pyramid'):
refine_feature_pyramid = {}
for level in cfgs.LEVEL:
feature_1x5 = slim.conv2d(
inputs=feature_pyramid[level],
num_outputs=256,
kernel_size=[1, 5],
weights_initializer=cfgs.SUBNETS_WEIGHTS_INITIALIZER,
biases_initializer=cfgs.SUBNETS_BIAS_INITIALIZER,
stride=1,
activation_fn=None,
scope='refine_1x5_{}'.format(level))
feature5x1 = slim.conv2d(
inputs=feature_1x5,
num_outputs=256,
kernel_size=[5, 1],
weights_initializer=cfgs.SUBNETS_WEIGHTS_INITIALIZER,
biases_initializer=cfgs.SUBNETS_BIAS_INITIALIZER,
stride=1,
activation_fn=None,
scope='refine_5x1_{}'.format(level))
feature_1x1 = slim.conv2d(
inputs=feature_pyramid[level],
num_outputs=256,
kernel_size=[1, 1],
weights_initializer=cfgs.SUBNETS_WEIGHTS_INITIALIZER,
biases_initializer=cfgs.SUBNETS_BIAS_INITIALIZER,
stride=1,
activation_fn=None,
scope='refine_1x1_{}'.format(level))
refine_feature_pyramid[level] = feature5x1 + feature_1x1
refine_box_pred_list, refine_cls_score_list, refine_cls_prob_list = self.refine_net(
refine_feature_pyramid, 'refine_net')
# refine_box_pred_list, refine_cls_score_list, refine_cls_prob_list = self.refine_net(feature_pyramid, 'refine_net')
refine_box_pred = tf.concat(refine_box_pred_list, axis=0)
refine_cls_score = tf.concat(refine_cls_score_list, axis=0)
refine_cls_prob = tf.concat(refine_cls_prob_list, axis=0)
# refine_boxes = tf.concat(refine_boxes_list, axis=0)
if cfgs.METHOD == 'H':
x_c = (anchors[:, 2] + anchors[:, 0]) / 2
y_c = (anchors[:, 3] + anchors[:, 1]) / 2
h = anchors[:, 2] - anchors[:, 0] + 1
w = anchors[:, 3] - anchors[:, 1] + 1
theta = -90 * tf.ones_like(x_c)
anchors = tf.transpose(tf.stack([x_c, y_c, w, h, theta]))
refine_boxes = bbox_transform.rbbox_transform_inv(boxes=anchors,
deltas=rpn_box_pred)
# 4. postprocess rpn proposals. such as: decode, clip, filter
if not self.is_training:
with tf.variable_scope('postprocess_detctions'):
boxes, scores, category = postprocess_detctions(
refine_bbox_pred=refine_box_pred,
refine_cls_prob=refine_cls_prob,
anchors=refine_boxes,
is_training=self.is_training)
return boxes, scores, category
# 5. build loss
else:
with tf.variable_scope('build_refine_loss'):
refine_labels, refine_target_delta, refine_box_states, refine_target_boxes = tf.py_func(
func=refinebox_target_layer,
inp=[
gtboxes_batch_r, refine_boxes,
cfgs.REFINE_IOU_POSITIVE_THRESHOLD[0],
cfgs.REFINE_IOU_NEGATIVE_THRESHOLD[0], gpu_id
],
Tout=[tf.float32, tf.float32, tf.float32, tf.float32])
self.add_anchor_img_smry(input_img_batch, refine_boxes,
refine_box_states, 1)
refine_cls_loss = losses.focal_loss(refine_labels,
refine_cls_score,
refine_box_states)
if cfgs.USE_IOU_FACTOR:
refine_reg_loss = losses.iou_smooth_l1_loss(
refine_target_delta, refine_box_pred,
refine_box_states, refine_target_boxes, refine_boxes)
else:
refine_reg_loss = losses.smooth_l1_loss(
refine_target_delta, refine_box_pred,
refine_box_states)
self.losses_dict[
'refine_cls_loss'] = refine_cls_loss * cfgs.CLS_WEIGHT
self.losses_dict[
'refine_reg_loss'] = refine_reg_loss * cfgs.REG_WEIGHT
with tf.variable_scope('postprocess_detctions'):
boxes, scores, category = postprocess_detctions(
refine_bbox_pred=refine_box_pred,
refine_cls_prob=refine_cls_prob,
anchors=refine_boxes,
is_training=self.is_training)
boxes = tf.stop_gradient(boxes)
scores = tf.stop_gradient(scores)
category = tf.stop_gradient(category)
return boxes, scores, category, self.losses_dict
def build_whole_detection_network(self,
input_img_batch,
gtboxes_batch_h,
gtboxes_batch_r,
gpu_id=0):
if self.is_training:
gtboxes_batch_h = tf.reshape(gtboxes_batch_h, [-1, 5])
gtboxes_batch_h = tf.cast(gtboxes_batch_h, tf.float32)
gtboxes_batch_r = tf.reshape(gtboxes_batch_r, [-1, 6])
gtboxes_batch_r = tf.cast(gtboxes_batch_r, tf.float32)
img_shape = tf.shape(input_img_batch)
# 1. build base network
if cfgs.USE_SUPERVISED_MASK:
feature_pyramid, mask_list, dot_layer_list = self.build_base_network(
input_img_batch)
else:
feature_pyramid = self.build_base_network(input_img_batch)
dot_layer_list = None
mask_list = []
# 2. build rpn
# if cfgs.USE_SUPERVISED_MASK:
# for i, d in enumerate(dot_layer_list):
# feature_pyramid['P{}'.format(i + 3)] *= d
rpn_box_pred_list, rpn_cls_score_list, rpn_cls_prob_list = self.rpn_net(
feature_pyramid, 'rpn_net')
# 3. generate anchors and mask
anchor_list = self.make_anchors(feature_pyramid)
if cfgs.USE_SUPERVISED_MASK:
mask_gt_list = self.generate_mask(mask_list, img_shape,
gtboxes_batch_h, gtboxes_batch_r,
feature_pyramid)
rpn_box_pred = tf.concat(rpn_box_pred_list, axis=0)
rpn_cls_score = tf.concat(rpn_cls_score_list, axis=0)
# rpn_cls_prob = tf.concat(rpn_cls_prob_list, axis=0)
anchors = tf.concat(anchor_list, axis=0)
if self.is_training:
with tf.variable_scope('build_loss'):
labels, target_delta, anchor_states, target_boxes = tf.py_func(
func=anchor_target_layer,
inp=[gtboxes_batch_h, gtboxes_batch_r, anchors],
Tout=[tf.float32, tf.float32, tf.float32, tf.float32])
if self.method == 'H':
self.add_anchor_img_smry(input_img_batch, anchors,
anchor_states, 0)
else:
self.add_anchor_img_smry(input_img_batch, anchors,
anchor_states, 1)
cls_loss = losses.focal_loss(labels, rpn_cls_score,
anchor_states)
if cfgs.USE_IOU_FACTOR:
reg_loss = losses.iou_smooth_l1_loss_(
target_delta, rpn_box_pred, anchor_states,
target_boxes, anchors)
else:
reg_loss = losses.smooth_l1_loss(target_delta,
rpn_box_pred,
anchor_states)
if cfgs.USE_SUPERVISED_MASK:
with tf.variable_scope("supervised_mask_loss"):
mask_loss = 0.0
for i in range(len(mask_list)):
a_mask, a_mask_gt = mask_list[i], mask_gt_list[i]
# b, h, w, c = a_mask.shape
last_dim = 2 if cfgs.BINARY_MASK else cfgs.CLASS_NUM + 1
a_mask = tf.reshape(a_mask, shape=[-1, last_dim])
a_mask_gt = tf.reshape(a_mask_gt, shape=[-1])
a_mask_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=a_mask, labels=a_mask_gt))
mask_loss += a_mask_loss
self.losses_dict[
'mask_loss'] = mask_loss * cfgs.SUPERVISED_MASK_LOSS_WEIGHT / float(
len(mask_list))
self.losses_dict['cls_loss'] = cls_loss * cfgs.CLS_WEIGHT
self.losses_dict['reg_loss'] = reg_loss * cfgs.REG_WEIGHT
box_pred_list, cls_prob_list, proposal_list = rpn_box_pred_list, rpn_cls_prob_list, anchor_list
all_box_pred_list, all_cls_prob_list, all_proposal_list = [], [], []
for i in range(cfgs.NUM_REFINE_STAGE):
box_pred_list, cls_prob_list, proposal_list = self.refine_stage(
input_img_batch,
gtboxes_batch_r,
box_pred_list,
cls_prob_list,
proposal_list,
feature_pyramid,
dot_layer_list,
gpu_id,
pos_threshold=cfgs.REFINE_IOU_POSITIVE_THRESHOLD[i],
neg_threshold=cfgs.REFINE_IOU_NEGATIVE_THRESHOLD[i],
stage='' if i == 0 else '_stage{}'.format(i + 2),
proposal_filter=True if i == 0 else False)
if not self.is_training:
all_box_pred_list.extend(box_pred_list)
all_cls_prob_list.extend(cls_prob_list)
all_proposal_list.extend(proposal_list)
else:
all_box_pred_list, all_cls_prob_list, all_proposal_list = box_pred_list, cls_prob_list, proposal_list
with tf.variable_scope('postprocess_detctions'):
box_pred = tf.concat(all_box_pred_list, axis=0)
cls_prob = tf.concat(all_cls_prob_list, axis=0)
proposal = tf.concat(all_proposal_list, axis=0)
boxes, scores, category = postprocess_detctions(
refine_bbox_pred=box_pred,
refine_cls_prob=cls_prob,
anchors=proposal,
is_training=self.is_training,
gpu_id=gpu_id)
boxes = tf.stop_gradient(boxes)
scores = tf.stop_gradient(scores)
category = tf.stop_gradient(category)
if self.is_training:
return boxes, scores, category, self.losses_dict
else:
return boxes, scores, category