ty += h/2.0
return h,w,l,tx,ty,tz,ry
if __name__=='__main__':
import mayavi.mlab as mlab
sys.path.append(os.path.join(ROOT_DIR, 'mayavi'))
from viz_util import draw_lidar, draw_gt_boxes3d
median_list = []
dataset = FrustumDataset(1024, split='val',
rotate_to_center=True, random_flip=True, random_shift=True)
for i in range(len(dataset)):
data = dataset[i]
print(('Center: ', data[2], \
'angle_class: ', data[3], 'angle_res:', data[4], \
'size_class: ', data[5], 'size_residual:', data[6], \
'real_size:', g_type_mean_size[g_class2type[data[5]]]+data[6]))
print(('Frustum angle: ', dataset.frustum_angle_list[i]))
median_list.append(np.median(data[0][:,0]))
print((data[2], dataset.box3d_list[i], median_list[-1]))
box3d_from_label = get_3d_box(class2size(data[5],data[6]), class2angle(data[3], data[4],12), data[2])
ps = data[0]
seg = data[1]
fig = mlab.figure(figure=None, bgcolor=(0.4,0.4,0.4), fgcolor=None, engine=None, size=(1000, 500))
mlab.points3d(ps[:,0], ps[:,1], ps[:,2], seg, mode='point', colormap='gnuplot', scale_factor=1, figure=fig)
mlab.points3d(0, 0, 0, color=(1,1,1), mode='sphere', scale_factor=0.2, figure=fig)
draw_gt_boxes3d([box3d_from_label], fig, color=(1,0,0))
mlab.orientation_axes()
raw_input()
print(np.mean(np.abs(median_list)))
def parse_crop_predictions(end_points, point_cloud, DC):
pred_center = end_points['center'].clone() # B,num_proposal,3
pred_heading_class = torch.argmax(end_points['heading_scores'].clone(),
-1) # B,num_proposal
pred_heading_residual = torch.gather(
end_points['heading_residuals'].clone(), 2,
pred_heading_class.unsqueeze(-1)) # B,num_proposal,1
pred_heading_residual.squeeze_(2)
pred_size_class = torch.argmax(end_points['size_scores'].clone(),
-1) # B,num_proposal
pred_size_residual = torch.gather(
end_points['size_residuals'].clone(), 2,
pred_size_class.unsqueeze(-1).unsqueeze(-1).repeat(
1, 1, 1, 3)) # B,num_proposal,1,3
pred_size_residual.squeeze_(2)
pred_sem_cls = torch.argmax(end_points['sem_cls_scores'].clone(),
-1) # B,num_proposal
sem_cls_probs = softmax(end_points['sem_cls_scores'].detach().cpu().clone(
).numpy()) # B,num_proposal,10
pred_sem_cls_prob = np.max(sem_cls_probs, -1) # B,num_proposal
bsize = pred_center.shape[0]
assert bsize == 1
num_proposal = pred_center.shape[1]
# Since we operate in upright_depth coord for points, while util functions
# assume upright_camera coord.
pred_corners_3d_upright_camera = np.zeros((bsize, num_proposal, 8, 3))
pred_center_upright_camera = flip_axis_to_camera(
pred_center.detach().cpu().numpy())
for i in range(bsize):
for j in range(num_proposal):
heading_angle = DC.class2angle(
pred_heading_class[i, j].detach().cpu().numpy(),
pred_heading_residual[i, j].detach().cpu().numpy())
box_size = DC.class2size(
int(pred_size_class[i, j].detach().cpu().numpy()),
pred_size_residual[i, j].detach().cpu().numpy())
corners_3d_upright_camera = get_3d_box(
box_size, heading_angle, pred_center_upright_camera[i, j, :])
pred_corners_3d_upright_camera[i, j] = corners_3d_upright_camera
K = pred_center.shape[1] # K==num_proposal
nonempty_box_mask = np.ones((bsize, K))
# -------------------------------------
# Remove predicted boxes without any point within them..
batch_pc = point_cloud.copy()[:, 0:3] # B,N,3
i = 0
pc = batch_pc[:, :] # (N,3)
for j in range(K):
box3d = pred_corners_3d_upright_camera[i, j, :, :] # (8,3)
# -- OG version of getting in-box points
box3d = flip_axis_to_depth(box3d)
pc_in_box, inds = extract_pc_in_box3d(pc, box3d)
if len(pc_in_box) < 5:
nonempty_box_mask[i, j] = 0
# -- new version
#min_bound_box = np.min(box3d, axis=0)
#max_bound_box = np.max(box3d, axis=0)
#in_bound_pc = np.all(pc > min_bound_box, axis=1) *\
# np.all(pc < max_bound_box, axis=1)
#if np.sum(in_bound_pc) < 5:
# nonempty_box_mask[i,j] = 0
end_points['center'] = end_points['center'][i][nonempty_box_mask[i, :], :]
end_points['heading_scores'] = end_points['heading_scores'][i][
nonempty_box_mask[i, :], :]
end_points['heading_residuals'] = end_points['heading_residuals'][i][
nonempty_box_mask[i, :], :]
end_points['heading_residuals_normalized'] = end_points[
'heading_residuals_normalized'][i][nonempty_box_mask[i, :], :]
end_points['size_scores'] = end_points['size_scores'][i][nonempty_box_mask[
i, :], :]
end_points['size_residuals'] = end_points['size_residuals'][i][
nonempty_box_mask[i, :], :]
end_points['size_residuals_normalized'] = end_points[
'size_residuals_normalized'][i][nonempty_box_mask[i, :], :]
end_points['sem_cls_scores'] = end_points['sem_cls_scores'][i][
nonempty_box_mask[i, :], :]
end_points['objectness_scores'] = end_points['objectness_scores'][i][
nonempty_box_mask[i, :], :]
end_points['center'] = end_points['center'].unsqueeze(0)
end_points['heading_scores'] = end_points['heading_scores'].unsqueeze(0)
end_points['heading_residuals'] = end_points[
'heading_residuals'].unsqueeze(0)
end_points['heading_residuals_normalized'] = end_points[
'heading_residuals_normalized'].unsqueeze(0)
end_points['size_scores'] = end_points['size_scores'].unsqueeze(0)
end_points['size_residuals'] = end_points['size_residuals'].unsqueeze(0)
end_points['size_residuals_normalized'] = end_points[
'size_residuals_normalized'].unsqueeze(0)
end_points['sem_cls_scores'] = end_points['sem_cls_scores'].unsqueeze(0)
end_points['objectness_scores'] = end_points[
'objectness_scores'].unsqueeze(0)
return end_points
def parse_predictions(end_points, config_dict):
""" Parse predictions to OBB parameters and suppress overlapping boxes
Args:
end_points: dict
{point_clouds, center, heading_scores, heading_residuals,
size_scores, size_residuals, sem_cls_scores}
config_dict: dict
{dataset_config, remove_empty_box, use_3d_nms, nms_iou,
use_old_type_nms, conf_thresh, per_class_proposal}
Returns:
batch_pred_map_cls: a list of len == batch size (BS)
[pred_list_i], i = 0, 1, ..., BS-1
where pred_list_i = [(pred_sem_cls, box_params, box_score)_j]
where j = 0, ..., num of valid detections - 1 from sample input i
"""
pred_center = end_points['center'] # B,num_proposal,3
pred_heading_class = torch.argmax(end_points['heading_scores'],
-1) # B,num_proposal
pred_heading_residual = torch.gather(
end_points['heading_residuals'], 2,
pred_heading_class.unsqueeze(-1)) # B,num_proposal,1
pred_heading_residual.squeeze_(2)
pred_size_class = torch.argmax(end_points['size_scores'],
-1) # B,num_proposal
pred_size_residual = torch.gather(
end_points['size_residuals'], 2,
pred_size_class.unsqueeze(-1).unsqueeze(-1).repeat(
1, 1, 1, 3)) # B,num_proposal,1,3
pred_size_residual.squeeze_(2)
pred_sem_cls = torch.argmax(end_points['sem_cls_scores'],
-1) # B,num_proposal
sem_cls_probs = softmax(end_points['sem_cls_scores'].detach().cpu().numpy(
)) # B,num_proposal,10
pred_sem_cls_prob = np.max(sem_cls_probs, -1) # B,num_proposal
num_proposal = pred_center.shape[1]
# Since we operate in upright_depth coord for points, while util functions
# assume upright_camera coord.
bsize = pred_center.shape[0]
pred_corners_3d_upright_camera = np.zeros((bsize, num_proposal, 8, 3))
pred_center_upright_camera = flip_axis_to_camera(
pred_center.detach().cpu().numpy())
for i in range(bsize):
for j in range(num_proposal):
heading_angle = config_dict['dataset_config'].class2angle(\
pred_heading_class[i,j].detach().cpu().numpy(), pred_heading_residual[i,j].detach().cpu().numpy())
box_size = config_dict['dataset_config'].class2size(\
int(pred_size_class[i,j].detach().cpu().numpy()), pred_size_residual[i,j].detach().cpu().numpy())
corners_3d_upright_camera = get_3d_box(
box_size, heading_angle, pred_center_upright_camera[i, j, :])
pred_corners_3d_upright_camera[i, j] = corners_3d_upright_camera
K = pred_center.shape[1] # K==num_proposal
nonempty_box_mask = np.ones((bsize, K))
if config_dict['remove_empty_box']:
# -------------------------------------
# Remove predicted boxes without any point within them..
batch_pc = end_points['point_clouds'].cpu().numpy()[:, :, 0:3] # B,N,3
for i in range(bsize):
pc = batch_pc[i, :, :] # (N,3)
for j in range(K):
box3d = pred_corners_3d_upright_camera[i, j, :, :] # (8,3)
box3d = flip_axis_to_depth(box3d)
pc_in_box, inds = extract_pc_in_box3d(pc, box3d)
if len(pc_in_box) < 5:
nonempty_box_mask[i, j] = 0
# -------------------------------------
obj_logits = end_points['objectness_scores'].detach().cpu().numpy()
obj_prob = softmax(obj_logits)[:, :, 1] # (B,K)
if not config_dict['use_3d_nms']:
# ---------- NMS input: pred_with_prob in (B,K,7) -----------
pred_mask = np.zeros((bsize, K))
for i in range(bsize):
boxes_2d_with_prob = np.zeros((K, 5))
for j in range(K):
boxes_2d_with_prob[j, 0] = np.min(
pred_corners_3d_upright_camera[i, j, :, 0])
boxes_2d_with_prob[j, 2] = np.max(
pred_corners_3d_upright_camera[i, j, :, 0])
boxes_2d_with_prob[j, 1] = np.min(
pred_corners_3d_upright_camera[i, j, :, 2])
boxes_2d_with_prob[j, 3] = np.max(
pred_corners_3d_upright_camera[i, j, :, 2])
boxes_2d_with_prob[j, 4] = obj_prob[i, j]
nonempty_box_inds = np.where(nonempty_box_mask[i, :] == 1)[0]
pick = nms_2d_faster(
boxes_2d_with_prob[nonempty_box_mask[i, :] == 1, :],
config_dict['nms_iou'], config_dict['use_old_type_nms'])
assert (len(pick) > 0)
pred_mask[i, nonempty_box_inds[pick]] = 1
end_points['pred_mask'] = pred_mask
# ---------- NMS output: pred_mask in (B,K) -----------
elif config_dict['use_3d_nms'] and (not config_dict['cls_nms']):
# ---------- NMS input: pred_with_prob in (B,K,7) -----------
pred_mask = np.zeros((bsize, K))
for i in range(bsize):
boxes_3d_with_prob = np.zeros((K, 7))
for j in range(K):
boxes_3d_with_prob[j, 0] = np.min(
pred_corners_3d_upright_camera[i, j, :, 0])
boxes_3d_with_prob[j, 1] = np.min(
pred_corners_3d_upright_camera[i, j, :, 1])
boxes_3d_with_prob[j, 2] = np.min(
pred_corners_3d_upright_camera[i, j, :, 2])
boxes_3d_with_prob[j, 3] = np.max(
pred_corners_3d_upright_camera[i, j, :, 0])
boxes_3d_with_prob[j, 4] = np.max(
pred_corners_3d_upright_camera[i, j, :, 1])
boxes_3d_with_prob[j, 5] = np.max(
pred_corners_3d_upright_camera[i, j, :, 2])
boxes_3d_with_prob[j, 6] = obj_prob[i, j]
nonempty_box_inds = np.where(nonempty_box_mask[i, :] == 1)[0]
pick = nms_3d_faster(
boxes_3d_with_prob[nonempty_box_mask[i, :] == 1, :],
config_dict['nms_iou'], config_dict['use_old_type_nms'])
assert (len(pick) > 0)
pred_mask[i, nonempty_box_inds[pick]] = 1
end_points['pred_mask'] = pred_mask
# ---------- NMS output: pred_mask in (B,K) -----------
elif config_dict['use_3d_nms'] and config_dict['cls_nms']:
# ---------- NMS input: pred_with_prob in (B,K,8) -----------
pred_mask = np.zeros((bsize, K))
for i in range(bsize):
boxes_3d_with_prob = np.zeros((K, 8))
for j in range(K):
boxes_3d_with_prob[j, 0] = np.min(
pred_corners_3d_upright_camera[i, j, :, 0])
boxes_3d_with_prob[j, 1] = np.min(
pred_corners_3d_upright_camera[i, j, :, 1])
boxes_3d_with_prob[j, 2] = np.min(
pred_corners_3d_upright_camera[i, j, :, 2])
boxes_3d_with_prob[j, 3] = np.max(
pred_corners_3d_upright_camera[i, j, :, 0])
boxes_3d_with_prob[j, 4] = np.max(
pred_corners_3d_upright_camera[i, j, :, 1])
boxes_3d_with_prob[j, 5] = np.max(
pred_corners_3d_upright_camera[i, j, :, 2])
boxes_3d_with_prob[j, 6] = obj_prob[i, j]
boxes_3d_with_prob[j, 7] = pred_sem_cls[
i,
j] # only suppress if the two boxes are of the same class!!
nonempty_box_inds = np.where(nonempty_box_mask[i, :] == 1)[0]
pick = nms_3d_faster_samecls(
boxes_3d_with_prob[nonempty_box_mask[i, :] == 1, :],
config_dict['nms_iou'], config_dict['use_old_type_nms'])
assert (len(pick) > 0)
pred_mask[i, nonempty_box_inds[pick]] = 1
end_points['pred_mask'] = pred_mask
# ---------- NMS output: pred_mask in (B,K) -----------
batch_pred_map_cls = [
] # a list (len: batch_size) of list (len: num of predictions per sample) of tuples of pred_cls, pred_box and conf (0-1)
for i in range(bsize):
if config_dict['per_class_proposal']:
cur_list = []
for ii in range(config_dict['dataset_config'].num_class):
cur_list += [(ii, pred_corners_3d_upright_camera[i,j], sem_cls_probs[i,j,ii]*obj_prob[i,j]) \
for j in range(pred_center.shape[1]) if pred_mask[i,j]==1 and obj_prob[i,j]>config_dict['conf_thresh']]
batch_pred_map_cls.append(cur_list)
else:
batch_pred_map_cls.append([(pred_sem_cls[i,j].item(), pred_corners_3d_upright_camera[i,j], obj_prob[i,j]) \
for j in range(pred_center.shape[1]) if pred_mask[i,j]==1 and obj_prob[i,j]>config_dict['conf_thresh']])
end_points['batch_pred_map_cls'] = batch_pred_map_cls
return batch_pred_map_cls
def evaluate_one_epoch():
stat_dict = {}
ap_calculator_list = [APCalculator(iou_thresh, DATASET_CONFIG.class2type) \
for iou_thresh in AP_IOU_THRESHOLDS]
net.eval() # set model to eval mode (for bn and dp)
for batch_idx, batch_data_label in enumerate(TEST_DATALOADER):
scan_name_list = batch_data_label['scan_name']
del batch_data_label['scan_name']
if batch_idx % 10 == 0:
print('Eval batch: %d' % (batch_idx))
for key in batch_data_label:
batch_data_label[key] = batch_data_label[key].to(device)
# Forward pass
inputs = {'point_clouds': batch_data_label['point_clouds']}
with torch.no_grad():
end_points = net(inputs)
# Compute loss
for key in batch_data_label:
assert (key not in end_points)
end_points[key] = batch_data_label[key]
loss, end_points = criterion(end_points, DATASET_CONFIG)
# Accumulate statistics and print out
for key in end_points:
if 'loss' in key or 'acc' in key or 'ratio' in key:
if key not in stat_dict: stat_dict[key] = 0
stat_dict[key] += end_points[key].item()
batch_pred_map_cls = parse_predictions(end_points, CONFIG_DICT)
batch_gt_map_cls = parse_groundtruths(end_points, CONFIG_DICT)
for ap_calculator in ap_calculator_list:
ap_calculator.step(batch_pred_map_cls, batch_gt_map_cls)
######## Saving data ########
save_dir = '/home/sirdome/katefgroup/language_grounding/mlcvnet_dump'
# INPUT
point_clouds = end_points['point_clouds'].cpu().numpy()
batch_size = point_clouds.shape[0]
# NETWORK OUTPUTS
seed_xyz = end_points['seed_xyz'].detach().cpu().numpy(
) # (B,num_seed,3)
if 'vote_xyz' in end_points:
aggregated_vote_xyz = end_points['aggregated_vote_xyz'].detach(
).cpu().numpy()
vote_xyz = end_points['vote_xyz'].detach().cpu().numpy(
) # (B,num_seed,3)
aggregated_vote_xyz = end_points['aggregated_vote_xyz'].detach(
).cpu().numpy()
objectness_scores = end_points['objectness_scores'].detach().cpu(
).numpy() # (B,K,2)
pred_center = end_points['center'].detach().cpu().numpy() # (B,K,3)
pred_heading_class = torch.argmax(end_points['heading_scores'],
-1) # B,num_proposal
pred_heading_residual = torch.gather(
end_points['heading_residuals'], 2,
pred_heading_class.unsqueeze(-1)) # B,num_proposal,1
pred_heading_class = pred_heading_class.detach().cpu().numpy(
) # B,num_proposal
pred_heading_residual = pred_heading_residual.squeeze(
2).detach().cpu().numpy() # B,num_proposal
pred_size_class = torch.argmax(end_points['size_scores'],
-1) # B,num_proposal
pred_size_residual = torch.gather(
end_points['size_residuals'], 2,
pred_size_class.unsqueeze(-1).unsqueeze(-1).repeat(
1, 1, 1, 3)) # B,num_proposal,1,3
pred_size_residual = pred_size_residual.squeeze(
2).detach().cpu().numpy() # B,num_proposal,3
# OTHERS
pred_mask = end_points['pred_mask'] # B,num_proposal
idx_beg = 0
pred_center_upright_camera = flip_axis_to_camera(pred_center)
for i in range(batch_size):
objectness_prob = softmax(objectness_scores[i, :, :])[:, 1] # (K,)
# Dump predicted bounding boxes
if np.sum(objectness_prob > 0.5) > 0:
num_proposal = pred_center.shape[1]
sr3d_boxes = []
for j in range(num_proposal):
heading_angle = CONFIG_DICT['dataset_config'].class2angle(\
pred_heading_class[i,j], pred_heading_residual[i,j])
box_size = CONFIG_DICT['dataset_config'].class2size(\
int(pred_size_class[i,j]), pred_size_residual[i,j])
corners_3d_upright_camera = get_3d_box(
box_size, heading_angle,
pred_center_upright_camera[i, j, :])
box3d = corners_3d_upright_camera
box3d = flip_axis_to_depth(box3d)
sr3d_box = _convert_all_corners_to_end_points(box3d)
# sr3d_box = convert_mlcvnetbox_to_sr3d(DATASET_CONFIG, pred_center[i,j,0:3],
# pred_size_class[i,j], pred_size_residual[i,j])
sr3d_boxes.append(sr3d_box)
if len(sr3d_boxes) > 0:
sr3d_boxes = np.vstack(
tuple(sr3d_boxes)) # (num_proposal, 6)
# Output boxes according to their semantic labels
pred_sem_cls = torch.argmax(end_points['sem_cls_scores'],
-1) # B,num_proposal
pred_sem_cls = pred_sem_cls.detach().cpu().numpy()
mask = np.logical_and(objectness_prob > 0.5,
pred_mask[i, :] == 1)
sr3d_boxes = sr3d_boxes[mask, :]
sr3d_boxes = list(sr3d_boxes)
scan_name = scan_name_list[i]
class_label_list = [
DATASET_CONFIG.class2type[p[0]] for p in batch_pred_map_cls[i]
]
print(len(class_label_list))
assert (len(sr3d_boxes) == len(class_label_list))
data_dict = {
"class": class_label_list,
"box": sr3d_boxes,
"pc": point_clouds[i]
}
np.save(f'{save_dir}/{scan_name}.npy', data_dict)
# Log statistics
for key in sorted(stat_dict.keys()):
log_string('eval mean %s: %f' % (key, stat_dict[key] /
(float(batch_idx + 1))))
# Evaluate average precision
for i, ap_calculator in enumerate(ap_calculator_list):
print('-' * 10, 'iou_thresh: %f' % (AP_IOU_THRESHOLDS[i]), '-' * 10)
metrics_dict = ap_calculator.compute_metrics()
for key in metrics_dict:
log_string('eval %s: %f' % (key, metrics_dict[key]))
mean_loss = stat_dict['loss'] / float(batch_idx + 1)
return mean_loss