def forward(self, fpn_outputs, anchor_coord, anchor_feat, num_proposals, generate_proposal=True):
# Forward network for each layer of the pyramid.
rpn_class_logits, rpn_probs, rpn_bbox, rpn_rotation = zip(
*[self._forward_stride(p) for p in fpn_outputs])
if generate_proposal:
# Get bounding boxes from detection output.
anchors = [(c, torch.from_numpy(utils.normalize_boxes(f.numpy(), self.config.max_ptc_size)))
for c, f in zip(anchor_coord, anchor_feat)]
rpn_rois, rpn_rois_rotation, rpn_scores, rpn2anchor_maps = self.get_proposal(
rpn_probs, rpn_bbox, rpn_rotation, anchors, num_proposals)
else:
rpn_rois, rpn_rois_rotation, rpn_scores, rpn2anchor_maps = None, None, None, None
return (rpn_class_logits, rpn_bbox, rpn_rotation, rpn_rois, rpn_rois_rotation, rpn_scores,
rpn2anchor_maps)
def forward(self, fpn_outputs, anchors, num_proposals, generate_proposal=True):
# Forward network for each layer of the pyramid.
rpn_outputs = [torch.cat(o, 1) for o in zip(*[self._forward_stride(p) for p in fpn_outputs])]
if self.is_rotation_bbox:
rpn_class_logits, rpn_probs, rpn_bbox, rpn_rotation = rpn_outputs
else:
rpn_class_logits, rpn_probs, rpn_bbox = rpn_outputs
rpn_rotation = None
if generate_proposal:
# Get bounding boxes from detection output.
anchors = utils.normalize_boxes(anchors, self.config.max_ptc_size)
rpn_rois, rpn_rois_rotation, rpn_scores = self.get_proposal(
rpn_probs, rpn_bbox, rpn_rotation, anchors, num_proposals)
else:
rpn_rois, rpn_rois_rotation, rpn_scores = None, None, None
return rpn_class_logits, rpn_bbox, rpn_rotation, rpn_rois, rpn_rois_rotation, rpn_scores
def visualize_groundtruth(self, datum, iteration):
coords = datum['coords'].numpy()
batch_size = coords[:, 0].max() + 1
output_path = pathlib.Path(self.config.visualize_path)
output_path.mkdir(exist_ok=True)
for i in range(batch_size):
# Visualize ground-truth positive anchors.
anchors_gt = datum['anchors'][torch.where(datum['rpn_match'].cpu() == 1)[1]]
anchors_gt_ptc = pc_utils.visualize_bboxes(anchors_gt)
anchors_gt_ply_dest = output_path / ('visualize_%04d_anchors_gt.ply' % iteration)
pc_utils.save_point_cloud(anchors_gt_ptc, anchors_gt_ply_dest)
# Visualize center location of all ground-truth anchors.
anchors_all = np.unique((datum['anchors'][:, 3:] + datum['anchors'][:, :3]) / 2, axis=0)
anchors_all_ply_dest = output_path / ('visualize_%04d_all_anchors_centers.ply' % iteration)
pc_utils.save_point_cloud(anchors_all, anchors_all_ply_dest)
# Visualize ground-truth positive anchors.
if datum.get('bboxes_rotations') is None:
bboxes_gt = pc_utils.visualize_bboxes(datum['bboxes_coords'][i], datum['bboxes_cls'][i])
else:
bboxes_gt = np.hstack((datum['bboxes_coords'][i], datum['bboxes_rotations'][i][:, None]))
bboxes_gt = pc_utils.visualize_bboxes(bboxes_gt, datum['bboxes_cls'][i],
bbox_param='xyzxyzr')
bboxes_gt_ply_dest = output_path / ('visualize_%04d_bboxes_gt.ply' % iteration)
pc_utils.save_point_cloud(bboxes_gt, bboxes_gt_ply_dest)
# Visualize reconstructed ground-truth rpn targets.
rpn_bbox_anchors = datum['anchors'][(datum['rpn_match'].flatten() == 1).cpu().numpy()]
rpn_bbox_anchors = detection_utils.normalize_boxes(rpn_bbox_anchors, self.config.max_ptc_size)
rpn_bbox_target = datum['rpn_bbox'].reshape(-1, 6)
rpn_bbox_mask = ~torch.all(rpn_bbox_target == 0, 1)
rpn_bbox_target = rpn_bbox_target[rpn_bbox_mask].cpu().numpy()
rpn_bbox_target *= np.reshape(self.config.rpn_bbox_std, (1, len(self.config.rpn_bbox_std)))
rpn_bbox_target = detection_utils.apply_box_deltas(torch.from_numpy(rpn_bbox_anchors),
torch.from_numpy(rpn_bbox_target),
self.config.normalize_bbox)
rpn_bbox_target = detection_utils.unnormalize_boxes(rpn_bbox_target.numpy(),
self.config.max_ptc_size)
if datum.get('rpn_rotation') is None:
bboxes_gt_recon = pc_utils.visualize_bboxes(rpn_bbox_target)
else:
rpn_rot_target = datum['rpn_rotation'][i][rpn_bbox_mask].cpu().numpy()
bboxes_gt_recon = np.hstack((rpn_bbox_target, rpn_rot_target[:, None]))
bboxes_gt_recon = pc_utils.visualize_bboxes(bboxes_gt_recon, bbox_param='xyzxyzr')
bboxes_gt_recon_ply_dest = output_path / ('visualize_%04d_bboxes_gt_recon.ply' % iteration)
pc_utils.save_point_cloud(bboxes_gt_recon, bboxes_gt_recon_ply_dest)
def __call__(self, list_data):
coords, feats, point_labels, bboxes = list(zip(*list_data))
coords_batch, feats_batch, labels_batch = [], [], []
bboxes_coords, bboxes_rotations, bboxes_cls = [], [], []
batch_num_points = 0
last_batch_id = 0
for batch_id in range(len(list_data)):
num_points = coords[batch_id].shape[0]
batch_num_points += num_points
if self.limit_numpoints and batch_num_points > self.limit_numpoints:
num_full_points = sum(len(c) for c in coords)
num_full_batch_size = len(coords)
logging.warning(
f'\t\tCannot fit {num_full_points} points into {self.limit_numpoints} points '
f'limit. Truncating batch size at {batch_id} out of {num_full_batch_size} with '
f'{batch_num_points - num_points}.'
)
break
coords_batch.append(coords[batch_id])
feats_batch.append(feats[batch_id])
labels_batch.append(point_labels[batch_id])
bboxes_cls.append(bboxes[batch_id][:, -1].astype(int))
if self.is_rotation_bbox:
bboxes_coords.append(bboxes[batch_id][:, :-2])
bboxes_rotations.append(bboxes[batch_id][:, -2])
else:
bboxes_coords.append(bboxes[batch_id][:, :-1])
last_batch_id = batch_id
# Concatenate all lists
bboxes_rotations = bboxes_rotations if self.is_rotation_bbox else None
coords_sbatch, feats_sbatch, labels_sbatch = \
ME.utils.sparse_collate(coords_batch, feats_batch, labels_batch)
datum = {
'coords': coords_sbatch,
'input': feats_sbatch.float(),
'target': labels_sbatch,
'bboxes_coords': bboxes_coords,
'bboxes_rotations': bboxes_rotations,
'bboxes_cls': bboxes_cls,
'last_batch_id': last_batch_id
}
# Precompute detection targets
if self.config.preload_anchor_data:
backbone_shapes = self._get_backbone_shapes(datum['coords'])
anchors, anchor_coords = detection_utils.generate_pyramid_anchors(
self.config.rpn_anchor_scales, self.anchor_ratios, backbone_shapes,
self.config.rpn_strides, get_negative_anchors=self.config.load_sparse_gt_data)
bboxes_normalized = [detection_utils.normalize_boxes(b, self.config.max_ptc_size)
for b in bboxes_coords]
datum.update({
'backbone_shapes': backbone_shapes,
'bboxes_normalized': bboxes_normalized,
})
if self.config.load_sparse_gt_data:
anchor_match_coords, sparse_anchor_centers, sparse_anchor_coords, sparse_rpn_match, \
sparse_rpn_bbox, sparse_rpn_rotation, sparse_rpn_cls = self._get_sparse_rpn_targets(
coords_batch, anchors, anchor_coords, bboxes_coords, bboxes_rotations, bboxes_cls,
self.config.rpn_strides)
datum.update({
'anchor_match_coords': anchor_match_coords,
'sparse_anchor_centers': sparse_anchor_centers,
'sparse_anchor_coords': sparse_anchor_coords,
'sparse_rpn_match': sparse_rpn_match,
'sparse_rpn_bbox': sparse_rpn_bbox,
'sparse_rpn_rotation': sparse_rpn_rotation,
'sparse_rpn_cls': sparse_rpn_cls,
})
else:
rpn_match, rpn_bbox, rpn_rotation, rpn_cls = self._get_rpn_targets(
anchors, bboxes_coords, bboxes_rotations, bboxes_cls)
datum.update({
'anchors': anchors,
'rpn_match': rpn_match,
'rpn_bbox': rpn_bbox,
'rpn_rotation': rpn_rotation,
'rpn_cls': rpn_cls,
})
return datum
def get_proposal(self, lrpn_probs, lrpn_sem, ldeltas, lrotation, num_proposals):
with torch.no_grad():
assert len(lrpn_probs) == len(lrpn_sem) == len(ldeltas) == len(lrotation)
rpn_cls = []
rpn_scores = []
rpn_return_scores = []
rpn_boxes = []
rpn_rotations = []
rpn_batch_idxs = []
for rpn_probs, rpn_semantic, deltas, rotation, anchor_size in zip(
lrpn_probs, lrpn_sem, ldeltas, lrotation, self.anchor_sizes):
if rpn_probs is None:
continue
num_anchors = rpn_probs.F.shape[1] / 2
assert rpn_probs.coords_key == deltas.coords_key
assert num_anchors == deltas.F.shape[1] / 6
rpn_batch_idxs.append(deltas.coords[:, 0])
rpn_semantic = rpn_semantic.reshape(-1, self.num_class)
rpn_semantic_prob, rpn_semantic_cls = rpn_semantic.max(1)
rpn_cls.append(rpn_semantic_cls)
rpn_prob = rpn_probs.F.reshape(-1, 2)[:, 1]
if self.config.detection_nms_score == 'obj':
rpn_score = rpn_prob
elif self.config.detection_nms_score == 'sem':
rpn_score = rpn_semantic_prob
elif self.config.detection_nms_score == 'objsem':
rpn_score = rpn_prob * rpn_semantic_prob
if self.config.detection_ap_score == 'obj':
ap_score = rpn_prob
elif self.config.detection_ap_score == 'sem':
ap_score = rpn_semantic_prob
elif self.config.detection_ap_score == 'objsem':
ap_score = rpn_prob * rpn_semantic_prob
rpn_scores.append(rpn_score)
rpn_return_scores.append(ap_score)
rpn_bbox_std = torch.from_numpy(np.expand_dims(self.config.rpn_bbox_std, 0)).to(deltas.F)
anchor_centers = deltas.coords[:, 1:] + deltas.tensor_stride[0] / 2
anchor_center = np.tile(anchor_centers, (1, int(num_anchors)))
anchors = np.hstack(((anchor_center - anchor_size).reshape(-1, 3),
(anchor_center + anchor_size).reshape(-1, 3)))
deltas = deltas.F.reshape(-1, 6) * rpn_bbox_std
anchors = torch.from_numpy(utils.normalize_boxes(anchors, self.config.max_ptc_size))
rpn_boxes.append(
utils.apply_box_deltas(anchors.to(deltas), deltas, self.config.normalize_bbox))
if rotation is not None:
num_rot_output = self.rotation_criterion.NUM_OUTPUT
assert rpn_probs.coords_key == rotation.coords_key
assert rpn_probs.F.shape[1] / 2 == rotation.F.shape[1] / num_rot_output
rpn_rotations.append(
self.rotation_criterion.pred(rotation.F.reshape(-1, num_rot_output)))
if not rpn_scores:
return None, None, None
all_scores = torch.cat(rpn_scores)
all_return_scores = torch.cat(rpn_return_scores)
all_cls = torch.cat(rpn_cls)
all_boxes = torch.cat(rpn_boxes)
all_batch_idxs = torch.cat(rpn_batch_idxs).repeat_interleave(int(num_anchors))
rotations = None
if rpn_rotations:
all_rotations = torch.cat(rpn_rotations)
rotations = []
boxes = []
scores = []
return_scores = []
classes = []
for i in range(all_batch_idxs.max().item() + 1):
batch_mask = all_batch_idxs == i
batch_scores = all_scores[batch_mask]
confidence_mask = batch_scores > self.config.rpn_pre_nms_min_confidence
pre_nms_limit = min(self.config.rpn_pre_nms_limit, confidence_mask.sum())
batch_scores, ix = torch.topk(batch_scores[confidence_mask], pre_nms_limit, sorted=True)
scores.append(batch_scores)
return_scores.append(all_return_scores[batch_mask][confidence_mask][ix])
boxes.append(all_boxes[batch_mask][confidence_mask][ix])
classes.append(all_cls[batch_mask][confidence_mask][ix])
if rotations is not None:
rotations.append(all_rotations[batch_mask][confidence_mask][ix])
rpn_proposal, rotation, rpn_scores = self.batch_non_maximum_suppression(
boxes, rotations, classes, scores, return_scores, num_proposals)
return rpn_proposal, rotation, rpn_scores