def __init__(self, cfg):
super(RPNModule, self).__init__()
self.cfg = cfg.clone()
anchor_generator = make_anchor_generator(cfg)
in_channels = cfg.SPARSE3D.nPlaneMap
rpn_head = registry.RPN_HEADS[cfg.MODEL.RPN.RPN_HEAD]
head = rpn_head(cfg, in_channels,
anchor_generator.num_anchors_per_location())
rpn_box_coder = BoxCoder3D(is_corner_roi=False, weights=None)
box_selector_train = make_rpn_postprocessor(cfg,
rpn_box_coder,
is_train=True)
box_selector_test = make_rpn_postprocessor(cfg,
rpn_box_coder,
is_train=False)
loss_evaluator = make_rpn_loss_evaluator(cfg, rpn_box_coder)
class_specific = cfg.MODEL.CLASS_SPECIFIC
self.seperate_classifier = SeperateClassifier(
cfg.MODEL.SEPARATE_CLASSES_ID, len(cfg.INPUT.CLASSES),
class_specific, 'RPN')
self.box_coder = rpn_box_coder
self.anchor_generator = anchor_generator
self.head = head
self.box_selector_train = box_selector_train
self.box_selector_test = box_selector_test
self.loss_evaluator = loss_evaluator
self.add_gt_proposals = cfg.MODEL.RPN.ADD_GT_PROPOSALS
def __init__(self, cfg):
super(RPNModule, self).__init__()
self.cfg = cfg.clone()
anchor_generator = make_anchor_generator(cfg)
in_channels = cfg.MODEL.BACKBONE.OUT_CHANNELS
rpn_head = registry.RPN_HEADS[cfg.MODEL.RPN.RPN_HEAD]
head = rpn_head(cfg, in_channels,
anchor_generator.num_anchors_per_location())
rpn_box_coder = BoxCoder3D()
box_selector_train = make_rpn_postprocessor(cfg,
rpn_box_coder,
is_train=True)
box_selector_test = make_rpn_postprocessor(cfg,
rpn_box_coder,
is_train=False)
loss_evaluator = make_rpn_loss_evaluator(cfg, rpn_box_coder)
self.seperate_classifier = SeperateClassifier(
cfg.MODEL.SEPARATE_CLASSES_ID, len(cfg.INPUT.CLASSES))
self.box_coder = rpn_box_coder
self.anchor_generator = anchor_generator
self.head = head
self.box_selector_train = box_selector_train
self.box_selector_test = box_selector_test
self.loss_evaluator = loss_evaluator
self.add_gt_proposals = cfg.MODEL.RPN.ADD_GT_PROPOSALS
def __init__(self, cfg):
super(SparseRCNN, self).__init__()
self.backbone = build_backbone(cfg)
self.rpn = build_rpn(cfg)
self.roi_heads = build_roi_heads(cfg)
self.add_gt_proposals = cfg.MODEL.RPN.ADD_GT_PROPOSALS
class_specific = cfg.MODEL.CLASS_SPECIFIC
self.seperate_classifier = SeperateClassifier(cfg.MODEL.SEPARATE_CLASSES_ID, len(cfg.INPUT.CLASSES), class_specific, 'Detector_ROI_Heads')
def __init__(self, cfg):
super(SparseRCNN, self).__init__()
self.backbone = build_backbone(cfg)
self.rpn = build_rpn(cfg)
self.roi_heads = build_roi_heads(cfg)
self.add_gt_proposals = cfg.MODEL.RPN.ADD_GT_PROPOSALS
self.seperate_classifier = SeperateClassifier(
cfg.MODEL.SEPARATE_CLASSES_ID, len(cfg.INPUT.CLASSES))
class SparseRCNN(nn.Module):
"""
Main class for Generalized R-CNN. Currently supports boxes and masks.
It consists of three main parts:
- backbone
= rpn
- heads: takes the features + the proposals from the RPN and computes
detections / masks from it.
"""
def __init__(self, cfg):
super(SparseRCNN, self).__init__()
self.backbone = build_backbone(cfg)
self.rpn = build_rpn(cfg)
self.roi_heads = build_roi_heads(cfg)
self.add_gt_proposals = cfg.MODEL.RPN.ADD_GT_PROPOSALS
class_specific = cfg.MODEL.CLASS_SPECIFIC
self.seperate_classifier = SeperateClassifier(cfg.MODEL.SEPARATE_CLASSES_ID, len(cfg.INPUT.CLASSES), class_specific, 'Detector_ROI_Heads')
def forward(self, points, targets=None):
"""
Arguments:
points (list[Tensor] or ImageList): points to be processed
targets (list[BoxList]): ground-truth boxes present in the image (optional)
Returns:
result (list[BoxList] or dict[Tensor]): the output from the model.
During training, it returns a dict[Tensor] which contains the losses.
During testing, it returns list[BoxList] contains additional fields
like `scores`, `labels` and `mask` (for Mask R-CNN models).
"""
if self.training and targets is None:
raise ValueError("In training mode, targets should be passed")
rpn_features, roi_features = self.backbone(points)
proposals, proposal_losses = self.rpn(points, rpn_features, targets)
if isinstance(proposals, list):
proposals[0].clamp_size()
proposals[1].clamp_size()
else:
proposals.clamp_size()
if self.roi_heads:
if not self.seperate_classifier.need_seperate:
x, result, detector_losses = self.roi_heads(roi_features, proposals, targets)
else:
x, result, detector_losses = self.seperate_classifier.sep_roi_heads( self.roi_heads, roi_features, proposals, targets, points=points)
else:
# RPN-only models don't have roi_heads
x = rpn_features
result = proposals.seperate_examples()
detector_losses = {}
if self.training:
losses = {}
losses.update(detector_losses)
losses.update(proposal_losses)
return losses, result
return result
def make_roi_box_loss_evaluator(cfg):
matcher = Matcher(
cfg.MODEL.ROI_HEADS.FG_IOU_THRESHOLD,
cfg.MODEL.ROI_HEADS.BG_IOU_THRESHOLD,
allow_low_quality_matches=False,
)
bbox_reg_weights = cfg.MODEL.ROI_HEADS.BBOX_REG_WEIGHTS
box_coder = BoxCoder3D(is_corner_roi=cfg.MODEL.CORNER_ROI,
weights=bbox_reg_weights)
fg_bg_sampler = BalancedPositiveNegativeSampler(
cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE,
cfg.MODEL.ROI_HEADS.POSITIVE_FRACTION)
yaw_loss_mode = cfg.MODEL.LOSS.YAW_MODE
add_gt_proposals = cfg.MODEL.RPN.ADD_GT_PROPOSALS
ay = cfg.MODEL.ROI_HEADS.LABEL_AUG_THICKNESS_Y_TAR_ANC
az = cfg.MODEL.ROI_HEADS.LABEL_AUG_THICKNESS_Z_TAR_ANC
aug_thickness = {
'target_Y': ay[0],
'anchor_Y': ay[1],
'target_Z': az[0],
'anchor_Z': az[1]
}
in_classes = cfg.INPUT.CLASSES
num_input_classes = len(in_classes)
seperate_classes = cfg.MODEL.SEPARATE_CLASSES_ID
class_specific = cfg.MODEL.CLASS_SPECIFIC
seperate_classifier = SeperateClassifier(seperate_classes,
num_input_classes, class_specific,
'ROI_LOSS')
loss_evaluator = FastRCNNLossComputation(matcher,
fg_bg_sampler,
box_coder,
yaw_loss_mode,
add_gt_proposals,
aug_thickness,
seperate_classifier,
class_specific=class_specific)
return loss_evaluator, seperate_classifier
class RPNModule(torch.nn.Module):
"""
Module for RPN computation. Takes feature maps from the backbone and RPN
proposals and losses. Works for both FPN and non-FPN.
"""
def __init__(self, cfg):
super(RPNModule, self).__init__()
self.cfg = cfg.clone()
anchor_generator = make_anchor_generator(cfg)
in_channels = cfg.MODEL.BACKBONE.OUT_CHANNELS
rpn_head = registry.RPN_HEADS[cfg.MODEL.RPN.RPN_HEAD]
head = rpn_head(cfg, in_channels,
anchor_generator.num_anchors_per_location())
rpn_box_coder = BoxCoder3D()
box_selector_train = make_rpn_postprocessor(cfg,
rpn_box_coder,
is_train=True)
box_selector_test = make_rpn_postprocessor(cfg,
rpn_box_coder,
is_train=False)
loss_evaluator = make_rpn_loss_evaluator(cfg, rpn_box_coder)
self.seperate_classifier = SeperateClassifier(
cfg.MODEL.SEPARATE_CLASSES_ID, len(cfg.INPUT.CLASSES))
self.box_coder = rpn_box_coder
self.anchor_generator = anchor_generator
self.head = head
self.box_selector_train = box_selector_train
self.box_selector_test = box_selector_test
self.loss_evaluator = loss_evaluator
self.add_gt_proposals = cfg.MODEL.RPN.ADD_GT_PROPOSALS
def forward(self, inputs_sparse, features_sparse, targets=None):
"""
Arguments:
inputs_sparse (ImageList): inputs_sparse for which we want to compute the predictions
features (list[Tensor]): features computed from the inputs_sparse that are
used for computing the predictions. Each tensor in the list
correspond to different feature levels
targets (list[BoxList): ground-truth boxes present in the image (optional)
Returns:
boxes (list[BoxList]): the predicted boxes from the RPN, one BoxList per
image.
losses (dict[Tensor]): the losses for the model during training. During
testing, it is an empty dict.
"""
def reshape(f):
return f.t().unsqueeze(0).unsqueeze(3)
features = [fs.features for fs in features_sparse]
features = [reshape(f) for f in features]
#[print(f.shape) for f in features]
# len(features) == levels_num
# features[l]: [1,channels_num, n, 1]
# n is a flatten of all the locations of all examples in a batch.
# Because the feature map size in a batch may be diff for each example
objectness, rpn_box_regression = self.head(features)
anchors = self.anchor_generator(inputs_sparse, features_sparse,
targets)
objectness, rpn_box_regression = cat_scales_obj_reg(
objectness, rpn_box_regression, anchors)
scale_num = len(anchors)
anchors = cat_scales_anchor(anchors)
anchors.constants['scale_num'] = scale_num
anchors.constants[
'num_anchors_per_location'] = self.head.num_anchors_per_location
device = features_sparse[0].features.device
anchors = anchors.to(device)
if SHOW_ANCHORS_PER_LOC:
anchors.show_anchors_per_loc()
if SHOW_TARGETS_ANCHORS:
import numpy as np
batch_size = len(targets)
examples_scope = examples_bidx_2_sizes(inputs_sparse[0][:, -1])
for bi in range(batch_size):
se = examples_scope[bi]
points = inputs_sparse[1][se[0]:se[1], 0:3].cpu().data.numpy()
print(f'\n targets')
targets[bi].show(points=points)
anchor_num = len(anchors)
for i in np.random.choice(anchor_num, 5):
anchor_i = anchors[int(i)].example(bi)
print(f'\n anchor {i} / {anchor_num}')
anchor_i.show__together(targets[bi], 200, points=points)
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
if SHOW_PRED_GT:
self.show_pred_gt(0.95, rpn_box_regression, anchors, objectness,
targets,
inputs_sparse[1][:, :6].cpu().data.numpy())
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
if self.training:
return self._forward_train(anchors,
objectness,
rpn_box_regression,
targets,
debugs={'inputs_sparse': inputs_sparse})
else:
return self._forward_test(anchors, objectness, rpn_box_regression,
targets)
def show_pred_gt(self, thres, rpn_box_regression, anchors, objectness,
targets, points):
pred_boxes_3d = self.box_coder.decode(rpn_box_regression,
anchors.bbox3d)
objectness_normed = objectness.sigmoid()
pred_boxes = anchors.copy()
pred_boxes.bbox3d = pred_boxes_3d.reshape([-1, 7])
pred_boxes.add_field('objectness', objectness_normed.reshape([-1]))
for bi, pred in enumerate(pred_boxes.seperate_examples()):
pred.show_by_objectness(0.8, points=points)
#pred.show_by_objectness(0.97, targets[bi], points=points)
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
def _forward_train(self,
anchors,
objectness,
rpn_box_regression,
targets,
debugs={}):
if self.cfg.MODEL.RPN_ONLY:
# When training an RPN-only model, the loss is determined by the
# predicted objectness and rpn_box_regression values and there is
# no need to transform the anchors into predicted boxes; this is an
# optimization that avoids the unnecessary transformation.
boxes = anchors
else:
# For end-to-end models, anchors must be transformed into boxes and
# sampled into a training batch.
with torch.no_grad():
if not self.seperate_classifier.need_seperate:
boxes = self.box_selector_train(anchors,
objectness.squeeze(1),
rpn_box_regression,
targets,
self.add_gt_proposals)
else:
boxes = self.seperate_classifier.seperate_rpn_selector(
self.box_selector_train, anchors, objectness,
rpn_box_regression, targets, self.add_gt_proposals)
if not self.seperate_classifier.need_seperate:
loss_objectness, loss_rpn_box_reg = self.loss_evaluator(
anchors, objectness.squeeze(1), rpn_box_regression, targets,
debugs)
boxes.set_as_prediction()
losses = {
"loss_objectness": loss_objectness,
"loss_rpn_box_reg": loss_rpn_box_reg,
}
else:
loss_objectness, loss_rpn_box_reg = self.seperate_classifier.seperate_rpn_loss_evaluator(
self.loss_evaluator,
anchors,
objectness,
rpn_box_regression,
targets,
debugs=debugs)
gn = len(boxes)
losses = {}
for gi in range(gn):
boxes[gi].set_as_prediction()
losses[f"loss_objectness_{gi}"] = loss_objectness[gi]
losses[f"loss_rpn_box_reg_{gi}"] = loss_rpn_box_reg[gi]
return boxes, losses
def _forward_test(self,
anchors,
objectness,
rpn_box_regression,
targets=None):
if not self.seperate_classifier.need_seperate:
boxes = self.box_selector_test(anchors, objectness.squeeze(1),
rpn_box_regression, targets)
boxes.set_as_prediction()
else:
boxes = self.seperate_classifier.seperate_rpn_selector(
self.box_selector_test, anchors, objectness,
rpn_box_regression, targets, self.add_gt_proposals)
boxes[0].set_as_prediction()
boxes[1].set_as_prediction()
if self.cfg.MODEL.RPN_ONLY:
# For end-to-end models, the RPN proposals are an intermediate state
# and don't bother to sort them in decreasing score order. For RPN-only
# models, the proposals are the final output and we return them in
# high-to-low confidence order.
boxes = boxes.seperate_examples()
inds = [
box.get_field("objectness").sort(descending=True)[1]
for box in boxes
]
boxes = [box[ind] for box, ind in zip(boxes, inds)]
#boxes = cat_boxlist_3d(boxes, per_example=True)
return boxes, {}