def build_graph(self, *inputs):
inputs = dict(zip(self.input_names, inputs))
num_fpn_level = len(cfg.FPN.ANCHOR_STRIDES)
assert len(cfg.RPN.ANCHOR_SIZES) == num_fpn_level
is_training = get_current_tower_context().is_training
all_anchors_fpn = get_all_anchors_fpn()
multilevel_anchors = [
RPNAnchors(all_anchors_fpn[i],
inputs['anchor_labels_lvl{}'.format(i + 2)],
inputs['anchor_boxes_lvl{}'.format(i + 2)])
for i in range(len(all_anchors_fpn))
]
image = self.preprocess(inputs['image']) # 1CHW
image_shape2d = tf.shape(image)[2:] # h,w
c2345 = resnet_fpn_backbone(image, cfg.BACKBONE.RESNET_NUM_BLOCK)
p23456 = fpn_model('fpn', c2345)
self.slice_feature_and_anchors(image_shape2d, p23456,
multilevel_anchors)
# Multi-Level RPN Proposals
rpn_outputs = [
rpn_head('rpn', pi, cfg.FPN.NUM_CHANNEL,
len(cfg.RPN.ANCHOR_RATIOS)) for pi in p23456
]
multilevel_label_logits = [k[0] for k in rpn_outputs]
multilevel_box_logits = [k[1] for k in rpn_outputs]
proposal_boxes, proposal_scores = generate_fpn_proposals(
multilevel_anchors, multilevel_label_logits, multilevel_box_logits,
image_shape2d)
gt_boxes, gt_labels = inputs['gt_boxes'], inputs['gt_labels']
if is_training:
proposals = sample_fast_rcnn_targets(proposal_boxes, gt_boxes,
gt_labels)
else:
proposals = BoxProposals(proposal_boxes)
fastrcnn_head_func = getattr(model_frcnn, cfg.FPN.FRCNN_HEAD_FUNC)
if not cfg.FPN.CASCADE:
roi_feature_fastrcnn = multilevel_roi_align(
p23456[:4], proposals.boxes, 7)
head_feature = fastrcnn_head_func('fastrcnn', roi_feature_fastrcnn)
fastrcnn_label_logits, fastrcnn_box_logits = fastrcnn_outputs(
'fastrcnn/outputs', head_feature, cfg.DATA.NUM_CLASS)
fastrcnn_head = FastRCNNHead(
proposals, fastrcnn_box_logits, fastrcnn_label_logits,
tf.constant(cfg.FRCNN.BBOX_REG_WEIGHTS, dtype=tf.float32))
else:
def roi_func(boxes):
return multilevel_roi_align(p23456[:4], boxes, 7)
fastrcnn_head = CascadeRCNNHead(proposals, roi_func,
fastrcnn_head_func, image_shape2d,
cfg.DATA.NUM_CLASS)
if is_training:
all_losses = []
all_losses.extend(
multilevel_rpn_losses(multilevel_anchors,
multilevel_label_logits,
multilevel_box_logits))
all_losses.extend(fastrcnn_head.losses())
if cfg.MODE_MASK:
# maskrcnn loss
roi_feature_maskrcnn = multilevel_roi_align(
p23456[:4],
proposals.fg_boxes(),
14,
name_scope='multilevel_roi_align_mask')
maskrcnn_head_func = getattr(model_mrcnn,
cfg.FPN.MRCNN_HEAD_FUNC)
mask_logits = maskrcnn_head_func(
'maskrcnn', roi_feature_maskrcnn,
cfg.DATA.NUM_CATEGORY) # #fg x #cat x 28 x 28
target_masks_for_fg = crop_and_resize(
tf.expand_dims(inputs['gt_masks'], 1),
proposals.fg_boxes(),
proposals.fg_inds_wrt_gt,
28,
pad_border=False) # fg x 1x28x28
target_masks_for_fg = tf.squeeze(target_masks_for_fg, 1,
'sampled_fg_mask_targets')
all_losses.append(
maskrcnn_loss(mask_logits, proposals.fg_labels(),
target_masks_for_fg))
wd_cost = regularize_cost('.*/W',
l2_regularizer(cfg.TRAIN.WEIGHT_DECAY),
name='wd_cost')
all_losses.append(wd_cost)
total_cost = tf.add_n(all_losses, 'total_cost')
add_moving_summary(total_cost, wd_cost)
return total_cost
else:
decoded_boxes = fastrcnn_head.decoded_output_boxes()
decoded_boxes = clip_boxes(decoded_boxes,
image_shape2d,
name='fastrcnn_all_boxes')
label_scores = fastrcnn_head.output_scores(
name='fastrcnn_all_scores')
final_boxes, final_scores, final_labels = fastrcnn_predictions(
decoded_boxes, label_scores, name_scope='output')
if cfg.MODE_MASK:
# Cascade inference needs roi transform with refined boxes.
roi_feature_maskrcnn = multilevel_roi_align(
p23456[:4], final_boxes, 14)
maskrcnn_head_func = getattr(model_mrcnn,
cfg.FPN.MRCNN_HEAD_FUNC)
mask_logits = maskrcnn_head_func(
'maskrcnn', roi_feature_maskrcnn,
cfg.DATA.NUM_CATEGORY) # #fg x #cat x 28 x 28
indices = tf.stack([
tf.range(tf.size(final_labels)),
tf.to_int32(final_labels) - 1
],
axis=1)
final_mask_logits = tf.gather_nd(mask_logits,
indices) # #resultx28x28
tf.sigmoid(final_mask_logits, name='output/masks')
def roi_heads(self, image, features, proposals, targets):
image_shape2d = tf.shape(image)[2:] # h,w
assert len(features) == 5, "Features have to be P23456!"
gt_boxes, gt_labels, *_ = targets
if self.training:
proposals = sample_fast_rcnn_targets(proposals.boxes, gt_boxes,
gt_labels)
fastrcnn_head_func = getattr(model_frcnn, cfg.FPN.FRCNN_HEAD_FUNC)
if not cfg.FPN.CASCADE:
roi_feature_fastrcnn = multilevel_roi_align(
features[:4], proposals.boxes, 7)
head_feature = fastrcnn_head_func('fastrcnn', roi_feature_fastrcnn)
fastrcnn_label_logits, fastrcnn_box_logits = fastrcnn_outputs(
'fastrcnn/outputs', head_feature, cfg.DATA.NUM_CLASS)
fastrcnn_head = FastRCNNHead(
proposals, fastrcnn_box_logits, fastrcnn_label_logits,
gt_boxes,
tf.constant(cfg.FRCNN.BBOX_REG_WEIGHTS, dtype=tf.float32))
else:
def roi_func(boxes):
return multilevel_roi_align(features[:4], boxes, 7)
fastrcnn_head = CascadeRCNNHead(proposals, roi_func,
fastrcnn_head_func,
(gt_boxes, gt_labels),
image_shape2d, cfg.DATA.NUM_CLASS)
if self.training:
all_losses = fastrcnn_head.losses()
if cfg.MODE_MASK:
gt_masks = targets[2]
# maskrcnn loss
roi_feature_maskrcnn = multilevel_roi_align(
features[:4],
proposals.fg_boxes(),
14,
name_scope='multilevel_roi_align_mask')
maskrcnn_head_func = getattr(model_mrcnn,
cfg.FPN.MRCNN_HEAD_FUNC)
mask_logits = maskrcnn_head_func(
'maskrcnn', roi_feature_maskrcnn,
cfg.DATA.NUM_CATEGORY) # #fg x #cat x 28 x 28
target_masks_for_fg = crop_and_resize(
tf.expand_dims(gt_masks, 1),
proposals.fg_boxes(),
proposals.fg_inds_wrt_gt,
28,
pad_border=False) # fg x 1x28x28
target_masks_for_fg = tf.squeeze(target_masks_for_fg, 1,
'sampled_fg_mask_targets')
all_losses.append(
maskrcnn_loss(mask_logits, proposals.fg_labels(),
target_masks_for_fg))
return all_losses
else:
decoded_boxes = fastrcnn_head.decoded_output_boxes()
decoded_boxes = clip_boxes(decoded_boxes,
image_shape2d,
name='fastrcnn_all_boxes')
label_scores = fastrcnn_head.output_scores(
name='fastrcnn_all_scores')
final_boxes, final_scores, final_labels = fastrcnn_predictions(
decoded_boxes, label_scores, name_scope='output')
if cfg.MODE_MASK:
# Cascade inference needs roi transform with refined boxes.
roi_feature_maskrcnn = multilevel_roi_align(
features[:4], final_boxes, 14)
maskrcnn_head_func = getattr(model_mrcnn,
cfg.FPN.MRCNN_HEAD_FUNC)
mask_logits = maskrcnn_head_func(
'maskrcnn', roi_feature_maskrcnn,
cfg.DATA.NUM_CATEGORY) # #fg x #cat x 28 x 28
indices = tf.stack([
tf.range(tf.size(final_labels)),
tf.cast(final_labels, tf.int32) - 1
],
axis=1)
final_mask_logits = tf.gather_nd(mask_logits,
indices) # #resultx28x28
tf.sigmoid(final_mask_logits, name='output/masks')
return []
def roi_heads(self,
image,
ref_features,
ref_box,
features,
proposals,
targets,
hard_negative_features=None,
hard_positive_features=None,
hard_positive_ious=None,
hard_positive_gt_boxes=None,
hard_positive_jitter_boxes=None,
precomputed_ref_features=None,
extra_feats=None):
image_shape2d = tf.shape(image)[2:] # h,w
assert len(features) == 5, "Features have to be P23456!"
gt_boxes, gt_labels, *_ = targets
if self.training:
proposals = sample_fast_rcnn_targets(proposals.boxes, gt_boxes,
gt_labels)
fastrcnn_head_func = getattr(model_frcnn, cfg.FPN.FRCNN_HEAD_FUNC)
if precomputed_ref_features is None:
roi_aligned_ref_features = multilevel_roi_align(
ref_features[:4], ref_box[tf.newaxis], 7)
else:
roi_aligned_ref_features = precomputed_ref_features[tf.newaxis]
# //////////
roi_aligned_extra_features = extra_feats[tf.newaxis]
# //////////
if cfg.MODE_SHARED_CONV_REDUCE:
scope = tf.get_variable_scope()
else:
scope = ""
assert cfg.FPN.CASCADE
def roi_func(boxes, already_aligned_features=None):
if already_aligned_features is None:
aligned_features = multilevel_roi_align(features[:4], boxes, 7)
else:
# for hard example mining
aligned_features = already_aligned_features
tiled = tf.tile(roi_aligned_ref_features,
[tf.shape(aligned_features)[0], 1, 1, 1])
concat_features = tf.concat((tiled, aligned_features), axis=1)
with argscope(Conv2D,
data_format='channels_first',
kernel_initializer=tf.variance_scaling_initializer(
scale=2.0,
mode='fan_out',
distribution='untruncated_normal'
if get_tf_version_tuple() >=
(1, 12) else 'normal')):
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
reduced_features = Conv2D('conv_reduce',
concat_features,
256,
1,
activation=None)
return reduced_features
def roi_func_extra(boxes, already_aligned_features=None):
if already_aligned_features is None:
aligned_features = multilevel_roi_align(features[:4], boxes, 7)
else:
# for hard example mining
aligned_features = already_aligned_features
tiled = tf.tile(roi_aligned_extra_features,
[tf.shape(aligned_features)[0], 1, 1, 1])
concat_features = tf.concat((tiled, aligned_features), axis=1)
with argscope(Conv2D,
data_format='channels_first',
kernel_initializer=tf.variance_scaling_initializer(
scale=2.0,
mode='fan_out',
distribution='untruncated_normal'
if get_tf_version_tuple() >=
(1, 12) else 'normal')):
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
reduced_features = Conv2D('conv_reduce',
concat_features,
256,
1,
activation=None)
return reduced_features
if cfg.MODE_HARD_MINING and self.training:
fastrcnn_head = CascadeRCNNHeadWithHardExamples(
proposals, roi_func, fastrcnn_head_func, (gt_boxes, gt_labels),
image_shape2d, cfg.DATA.NUM_CLASS, hard_negative_features,
hard_positive_features, cfg.HARD_NEGATIVE_LOSS_SCALING_FACTOR,
cfg.HARD_POSITIVE_LOSS_SCALING_FACTOR, hard_positive_ious,
hard_positive_gt_boxes, hard_positive_jitter_boxes)
else:
if cfg.MODE_EXTRA_FEATURES:
fastrcnn_head = CascadeRCNNHead(proposals, roi_func,
fastrcnn_head_func,
(gt_boxes, gt_labels),
image_shape2d,
cfg.DATA.NUM_CLASS,
roi_func_extra)
else:
fastrcnn_head = CascadeRCNNHead(proposals, roi_func,
fastrcnn_head_func,
(gt_boxes, gt_labels),
image_shape2d,
cfg.DATA.NUM_CLASS)
if cfg.EXTRACT_GT_FEATURES:
# get boxes and features for each of the three cascade stages!
b0 = proposals.boxes
b1, b2, _ = fastrcnn_head._cascade_boxes
f0 = multilevel_roi_align(features[:4], b0, 7)
f1 = multilevel_roi_align(features[:4], b1, 7)
f2 = multilevel_roi_align(features[:4], b2, 7)
tf.concat([b0, b1, b2], axis=0, name="boxes_for_extraction")
tf.concat([f0, f1, f2], axis=0, name="features_for_extraction")
if self.training:
all_losses = fastrcnn_head.losses()
if cfg.MODE_MASK:
gt_masks = targets[2]
# maskrcnn loss
roi_feature_maskrcnn = multilevel_roi_align(
features[:4],
proposals.fg_boxes(),
14,
name_scope='multilevel_roi_align_mask')
maskrcnn_head_func = getattr(model_mrcnn,
cfg.FPN.MRCNN_HEAD_FUNC)
mask_logits = maskrcnn_head_func(
'maskrcnn', roi_feature_maskrcnn,
cfg.DATA.NUM_CATEGORY) # #fg x #cat x 28 x 28
target_masks_for_fg = crop_and_resize(
tf.expand_dims(gt_masks, 1),
proposals.fg_boxes(),
proposals.fg_inds_wrt_gt,
28,
pad_border=False) # fg x 1x28x28
target_masks_for_fg = tf.squeeze(target_masks_for_fg, 1,
'sampled_fg_mask_targets')
all_losses.append(
maskrcnn_loss(mask_logits, proposals.fg_labels(),
target_masks_for_fg))
if cfg.MEASURE_IOU_DURING_TRAINING:
decoded_boxes = fastrcnn_head.decoded_output_boxes()
decoded_boxes = clip_boxes(decoded_boxes,
image_shape2d,
name='fastrcnn_all_boxes')
label_scores = fastrcnn_head.output_scores(
name='fastrcnn_all_scores')
final_boxes, final_scores, final_labels = fastrcnn_predictions(
decoded_boxes, label_scores, name_scope='output_train')
# if predictions are empty, this might break...
# to prevent, stack dummy box
boxes_for_iou = tf.concat([
final_boxes[:1],
tf.constant([[0.0, 0.0, 1.0, 1.0]], dtype=tf.float32)
],
axis=0)
from examples.FasterRCNN.utils.box_ops import pairwise_iou
iou_at_1 = tf.identity(pairwise_iou(gt_boxes[:1],
boxes_for_iou)[0, 0],
name="train_iou_at_1")
add_moving_summary(iou_at_1)
return all_losses
else:
decoded_boxes = fastrcnn_head.decoded_output_boxes()
decoded_boxes = clip_boxes(decoded_boxes,
image_shape2d,
name='fastrcnn_all_boxes')
label_scores = fastrcnn_head.output_scores(
name='fastrcnn_all_scores')
final_boxes, final_scores, final_labels = fastrcnn_predictions(
decoded_boxes, label_scores, name_scope='output')
if cfg.MODE_MASK:
# Cascade inference needs roi transform with refined boxes.
roi_feature_maskrcnn = multilevel_roi_align(
features[:4], final_boxes, 14)
maskrcnn_head_func = getattr(model_mrcnn,
cfg.FPN.MRCNN_HEAD_FUNC)
mask_logits = maskrcnn_head_func(
'maskrcnn', roi_feature_maskrcnn,
cfg.DATA.NUM_CATEGORY) # #fg x #cat x 28 x 28
indices = tf.stack([
tf.range(tf.size(final_labels)),
tf.cast(final_labels, tf.int32) - 1
],
axis=1)
final_mask_logits = tf.gather_nd(mask_logits,
indices) # #resultx28x28
tf.sigmoid(final_mask_logits, name='output/masks')
return []