def main(args):
nocs_eval = NOCSEvaluation(args)
images_out = os.path.join(nocs_eval.out_path, 'images')
if not os.path.exists(images_out):
os.mkdir(images_out)
pcl_out = os.path.join(nocs_eval.out_path, 'pcl')
if not os.path.exists(pcl_out):
os.mkdir(pcl_out)
for i in range(10): #dpc_eval.dpc_results.length):
nocs_id = nocs_eval.nocs_results.get_model_info(i)[0][1]
# get GT point cloud (union of all available views)
gt_pc_frames = nocs_eval.nocs_results.get_pc_gt(i)
#print(len(gt_pc_frames))
gt_pc = np.concatenate(gt_pc_frames, axis=0)
if gt_pc.shape[0] > nocs_eval.max_pts:
gt_pc = prune_pc(gt_pc, nocs_eval.max_pts)
gt_pc -= np.mean(gt_pc, axis=0)
viz_pcs([gt_pc], os.path.join(images_out, nocs_id + '_gt_pc.png'))
np.save(os.path.join(pcl_out, nocs_id + '_gt_pc'), gt_pc)
nocs00_maps = nocs_eval.nocs_results.get_nox00_pred(i)
nocs01_maps = nocs_eval.nocs_results.get_nox01_pred(i)
nocs00_pcs = nocs_eval.nocs_results.get_pc_pred(-1,
nocs_maps=nocs00_maps)
nocs01_pcs = nocs_eval.nocs_results.get_pc_pred(-1,
nocs_maps=nocs01_maps)
#print(len(nocs00_pcs))
#print(len(nocs01_pcs))
# NOCS
if nocs_eval.nocs_view_type == 'single':
for j in range(nocs00_maps.shape[0]):
single_pc = np.concatenate([nocs00_pcs[j], nocs01_pcs[j]],
axis=0)
if single_pc.shape[0] > nocs_eval.max_pts:
single_pc = prune_pc(single_pc, nocs_eval.max_pts)
single_pc -= np.mean(single_pc, axis=0)
viz_pcs([single_pc],
os.path.join(
images_out,
nocs_id + '_nocs_pc_view' + str(j) + '.png'))
np.save(
os.path.join(pcl_out, nocs_id + '_nocs_pc_view' + str(j)),
single_pc)
elif nocs_eval.nocs_view_type == 'multi':
pc_list = nocs00_pcs + nocs01_pcs
multi_pc = np.concatenate(pc_list, axis=0)
if multi_pc.shape[0] > nocs_eval.max_pts:
multi_pc = prune_pc(multi_pc, nocs_eval.max_pts)
multi_pc -= np.mean(multi_pc, axis=0)
viz_pcs([multi_pc],
os.path.join(images_out, nocs_id + '_nocs_pc.png'))
np.save(os.path.join(pcl_out, nocs_id + '_nocs_pc'), multi_pc)
def main(args):
r2n2_eval = R2N2Evaluation(args)
images_out = os.path.join(r2n2_eval.out_path, 'images')
if not os.path.exists(images_out):
os.mkdir(images_out)
pcl_out = os.path.join(r2n2_eval.out_path, 'pcl')
if not os.path.exists(pcl_out):
os.mkdir(pcl_out)
for i in range(100): #dpc_eval.dpc_results.length):
r2n2_id = r2n2_eval.r2n2_results.meta_info[i]
nocs_id = r2n2_eval.nocs_results.get_model_info(i)[0][1]
if r2n2_id != nocs_id:
print(
'ERROR EVALUATION DATA IS NOT CONSISTENT BETWEEN R2N2 AND NOCS!!'
)
quit()
# get GT point cloud (union of all available views)
gt_pc_frames = r2n2_eval.nocs_results.get_pc_gt(i)
#print(len(gt_pc_frames))
gt_pc = np.concatenate(gt_pc_frames, axis=0)
if gt_pc.shape[0] > r2n2_eval.max_pts:
gt_pc = prune_pc(gt_pc, r2n2_eval.max_pts)
gt_pc -= np.mean(gt_pc, axis=0)
viz_pcs([gt_pc], os.path.join(images_out, r2n2_id + '_gt_pc.png'))
np.save(os.path.join(pcl_out, r2n2_id + '_gt_pc'), gt_pc)
nocs00_maps = r2n2_eval.nocs_results.get_nox00_pred(i)
nocs01_maps = r2n2_eval.nocs_results.get_nox01_pred(i)
nocs00_pcs = r2n2_eval.nocs_results.get_pc_pred(-1,
nocs_maps=nocs00_maps)
nocs01_pcs = r2n2_eval.nocs_results.get_pc_pred(-1,
nocs_maps=nocs01_maps)
#print(len(nocs00_pcs))
#print(len(nocs01_pcs))
# R2N2
multi_pc = r2n2_eval.r2n2_results.pcs[i]
if multi_pc.shape[0] > r2n2_eval.max_pts:
multi_pc = prune_pc(multi_pc, r2n2_eval.max_pts)
viz_pcs([multi_pc], os.path.join(images_out, r2n2_id + '_r2n2_pc.png'))
np.save(os.path.join(pcl_out, r2n2_id + '_r2n2_pc'), multi_pc)
# NOCS
pc_list = nocs00_pcs + nocs01_pcs
multi_pc = np.concatenate(pc_list, axis=0)
if multi_pc.shape[0] > r2n2_eval.max_pts:
multi_pc = prune_pc(multi_pc, r2n2_eval.max_pts)
multi_pc -= np.mean(multi_pc, axis=0)
viz_pcs([multi_pc], os.path.join(images_out, r2n2_id + '_nocs_pc.png'))
np.save(os.path.join(pcl_out, r2n2_id + '_nocs_pc'), multi_pc)
def main(args):
dpc_eval = DPCEvaluation(args)
images_out = os.path.join(dpc_eval.out_path, 'images')
if not os.path.exists(images_out):
os.mkdir(images_out)
pcl_out = os.path.join(dpc_eval.out_path, 'pcl')
if not os.path.exists(pcl_out):
os.mkdir(pcl_out)
# need to first do an alignment between the predicted and ground truth reference frames
default_num_align_models = 20
num_align_models = min(
[default_num_align_models, dpc_eval.dpc_results.length])
dpc_eval.log('Performing alignment...')
final_align_quat = dpc_eval.find_global_alignment(num_align_models)
# final_align_quat = np.array([ 0.02164055, 0.08669936, 0.02188158, -0.99575906]) # chair GT for debug
final_align_quat_conj = quaternion_conjugate(final_align_quat)
print(final_align_quat)
final_align_R = as_rotation_matrix(final_align_quat)
for i in range(20): #dpc_eval.dpc_results.length):
dpc_id = dpc_eval.dpc_results.meta_info[i]
nocs_id = dpc_eval.nocs_results.get_model_info(i)[0][1]
if dpc_id != nocs_id:
print(
'ERROR EVALUATION DATA IS NOT CONSISTENT BETWEEN DPC AND NOCS!!'
)
quit()
# get GT point cloud (union of all available views)
gt_pc_frames = dpc_eval.nocs_results.get_pc_gt(i)
gt_pc = np.concatenate(gt_pc_frames, axis=0)
if gt_pc.shape[0] > dpc_eval.max_pts:
gt_pc = prune_pc(gt_pc, dpc_eval.max_pts)
gt_pc -= np.mean(gt_pc, axis=0)
viz_pcs([gt_pc], os.path.join(images_out, dpc_id + '_gt_pc.png'))
np.save(os.path.join(pcl_out, dpc_id + '_gt_pc'), gt_pc)
nocs00_maps = dpc_eval.nocs_results.get_nox00_pred(i)
nocs01_maps = dpc_eval.nocs_results.get_nox01_pred(i)
nocs00_pcs = dpc_eval.nocs_results.get_pc_pred(-1,
nocs_maps=nocs00_maps)
nocs01_pcs = dpc_eval.nocs_results.get_pc_pred(-1,
nocs_maps=nocs01_maps)
pred_pcs = dpc_eval.dpc_results.pcs[i]
for j in range(len(pred_pcs)):
# DPC
single_pc = pred_pcs[j]
if single_pc.shape[0] > dpc_eval.max_pts:
single_pc = prune_pc(single_pc, dpc_eval.max_pts)
# center and align
single_pc -= np.mean(single_pc, axis=0)
single_pc = np.dot(final_align_R, single_pc.T).T
single_pc = normalize_pc(single_pc)
viz_pcs([single_pc],
os.path.join(images_out,
dpc_id + '_dpc_pc_' + str(j) + '.png'))
np.save(os.path.join(pcl_out, dpc_id + '_dpc_pc_' + str(j)),
single_pc)
# NOCS
single_pc = np.concatenate([nocs00_pcs[j], nocs01_pcs[j]], axis=0)
if single_pc.shape[0] > dpc_eval.max_pts:
single_pc = prune_pc(single_pc, dpc_eval.max_pts)
single_pc -= np.mean(single_pc, axis=0)
viz_pcs([single_pc],
os.path.join(images_out,
dpc_id + '_nocs_pc_' + str(j) + '.png'))
np.save(os.path.join(pcl_out, dpc_id + '_nocs_pc_' + str(j)),
single_pc)
def run(self):
''' Run through each model and evaluate point cloud prediction and camera pose. '''
cam_pos_err = []
cam_rot_err = []
dpc_rot_err = []
if self.nocs_view_type == 'single':
single_view_err = []
single_nview_errs = []
for n in self.num_views:
single_nview_errs.append([])
elif self.nocs_view_type == 'multi':
multi_view_err = []
for i in range(self.nocs_results.length):
# will need these for both
nocs00_maps = self.nocs_results.get_nox00_pred(i)
nocs01_maps = self.nocs_results.get_nox01_pred(i)
nocs00_pcs = self.nocs_results.get_pc_pred(-1,
nocs_maps=nocs00_maps)
nocs01_pcs = self.nocs_results.get_pc_pred(-1,
nocs_maps=nocs01_maps)
if self.pred_filter is not None and self.save_filtered:
# frame_971_view_00_nox00_gt.png
# frame is just i (model index)
# view is model_info[2]
info = self.nocs_results.get_model_info(i)
frame_idx = i
if self.nocs_view_type == 'multi':
for view_idx in range(len(info)):
# save front and back pred
cur_front_path = 'frame_%03d_view_%02d_nox00_pred_filtered.png' % (
frame_idx, view_idx)
cur_front_path = os.path.join(self.nocs_results.root,
cur_front_path)
cur_back_path = 'frame_%03d_view_%02d_nox01_pred_filtered.png' % (
frame_idx, view_idx)
cur_back_path = os.path.join(self.nocs_results.root,
cur_back_path)
# print(cur_front_path)
# print(cur_back_path)
write_nocs_map(nocs00_maps[view_idx], cur_front_path)
write_nocs_map(nocs01_maps[view_idx], cur_back_path)
elif self.nocs_view_type == 'single':
views_per_model = len(info)
for view_idx in range(len(info)):
global_frame_idx = i * views_per_model + view_idx
# save front and back pred
cur_front_path = 'frame_%03d_nox00_pred_filtered.png' % (
global_frame_idx)
cur_front_path = os.path.join(self.nocs_results.root,
cur_front_path)
cur_back_path = 'frame_%03d_nox01_pred_filtered.png' % (
global_frame_idx)
cur_back_path = os.path.join(self.nocs_results.root,
cur_back_path)
# print(cur_front_path)
# print(cur_back_path)
write_nocs_map(nocs00_maps[view_idx], cur_front_path)
write_nocs_map(nocs01_maps[view_idx], cur_back_path)
if not self.no_camera_eval:
# first evaluate camera pose, do this for every frame
# for both single-view and multi-view evaluation
# solve for camera pose using nocs maps and point clouds
cam_poses = self.nocs_results.get_camera_pose(i)
for j in range(nocs00_maps.shape[0]):
nocs00_map = nocs00_maps[j]
nocs01_map = nocs01_maps[j]
nocs00_pc = nocs00_pcs[j]
nocs01_pc = nocs01_pcs[j]
# solve using both layers jointly
_, _, R_pred, P_pred, flip = self.estimateCameraPoseFromNM(
[nocs00_map, nocs01_map], [nocs00_pc, nocs01_pc],
N=NUM_CAM_SOLVE_PTS)
# estimated position is w.r.t to (0.0, 0.0, 0.0) not NOCS center and in right-handed coords
P_pred += np.array([-0.5, -0.5, -0.5])
P_pred[0] *= -1.0
# change coordinates depending on if prediction is flipped
R_flip = axangle2mat(np.array([1.0, 0.0, 0.0]),
np.radians(180),
is_normalized=True)
R_noflip = axangle2mat(np.array([0.0, 1.0, 0.0]),
np.radians(180),
is_normalized=True)
# get GT pose
P_gt, gt_quat = cam_poses[j]
gt_quat = np.array(
[gt_quat[0], gt_quat[1], gt_quat[2], gt_quat[3]])
R_gt = quaternions.quat2mat(gt_quat).T
# compare (transform unit x-axis)
unit_dir = np.array([1.0, 0.0, 0.0])
# predicted tranform
R_pred = np.dot(R_pred, R_flip)
if not flip:
R_pred = np.dot(R_noflip, R_pred)
pred_dir = np.dot(R_pred, unit_dir)
pred_pos = P_pred
# ground truth transform
gt_dir = np.dot(R_gt, unit_dir)
gt_pos = P_gt
cur_pos_err = np.linalg.norm(gt_pos - pred_pos)
cam_pos_err.append(cur_pos_err)
cur_angle_err = np.degrees(
np.arccos(np.dot(gt_dir, pred_dir)))
cam_rot_err.append(cur_angle_err)
# print('Model %d Frame %d:' % (i, j))
# print('Flip?:', flip)
# print('Pred Pos:', pred_pos)
# print('Pred X-dir:', pred_dir)
# print('GT Pos:', gt_pos)
# print('GT X-dir:', gt_dir)
# print('=======================================================')
# print('Pos err:', cur_pos_err)
# print('rot err:', cur_angle_err)
# print('=======================================================')
# if the option is turned on, also evaluate camera pose for comparison with DPC
# this method uses quaternion differences and only measures angle difference
if self.dpc_camera_eval:
gt_pos_unity, _ = cam_poses[j]
gt_pos_blender = np.array([
-gt_pos_unity[0], -gt_pos_unity[2], gt_pos_unity[1]
])
gt_quat = quaternion_from_campos(gt_pos_blender)
gt_quat /= np.linalg.norm(gt_quat)
pred_pos_unity = pred_pos
pred_pos_blender = np.array([
-pred_pos_unity[0], -pred_pos_unity[2],
pred_pos_unity[1]
])
pred_quat = quaternion_from_campos(pred_pos_blender)
pred_quat /= np.linalg.norm(pred_quat)
# look at the quaternion difference between GT and pred
gt_conj = quaternion_conjugate(gt_quat)
q_diff = quaternion_multiply(gt_conj, pred_quat)
ang_diff = 2 * np.arccos(q_diff[0])
if ang_diff > np.pi:
ang_diff -= 2 * np.pi
dpc_rot_err.append(np.degrees(np.fabs(ang_diff)))
if not self.no_pc_eval:
# now we evaluate the estimated point cloud
# get GT point cloud (union of all available views)
gt_pc_frames = self.nocs_results.get_pc_gt(i)
gt_pc = np.concatenate(gt_pc_frames, axis=0)
if gt_pc.shape[0] > self.max_pts:
gt_pc = prune_pc(gt_pc, self.max_pts)
chamfer = ChamferMetric(device='cuda')
if self.nocs_view_type == 'single':
# for single-view we do evaluation for point cloud from every view
# and then evaluate any additional passed in nviews by taking the union
# of point clouds from the first N predicted views.
for j in range(nocs00_maps.shape[0]):
single_pc = np.concatenate(
[nocs00_pcs[j], nocs01_pcs[j]], axis=0)
if single_pc.shape[0] > self.max_pts:
single_pc = prune_pc(single_pc, self.max_pts)
# calculate chamfer distance
cur_err = chamfer.chamfer_dist(single_pc, gt_pc)
single_view_err.append(cur_err)
# print('SingleChamferErr (1 view):', cur_err)
if not self.no_save_pc:
cur_inf = self.nocs_results.get_model_info(i)[j]
save_pc_single(single_pc, self.pc_out_path,
cur_inf)
# then multi-views (using the first n views)
for k, n in enumerate(self.num_views):
if n > nocs00_maps.shape[0]:
continue
# aggregate point cloud
pc_list = []
for j in range(n):
pc_list += [nocs00_pcs[j], nocs01_pcs[j]]
multi_pc = np.concatenate(pc_list, axis=0)
if multi_pc.shape[0] > self.max_pts:
multi_pc = prune_pc(multi_pc, self.max_pts)
cur_err = chamfer.chamfer_dist(multi_pc, gt_pc)
single_nview_errs[k].append(cur_err)
# print('SingleChamferErr (%d views): %f' % (n, cur_err))
elif self.nocs_view_type == 'multi':
# for multi-view we only do a single evaluation on the point cloud
# from all views combined
pc_list = nocs00_pcs + nocs01_pcs
multi_pc = np.concatenate(pc_list, axis=0)
if multi_pc.shape[0] > self.max_pts:
multi_pc = prune_pc(multi_pc, self.max_pts)
# print(multi_pc.shape)
cur_err = chamfer.chamfer_dist(multi_pc, gt_pc)
multi_view_err.append(cur_err)
# print('MultiChamferErr (%d views): %f' % (len(nocs00_pcs), cur_err))
if not self.no_save_pc:
cur_inf = self.nocs_results.get_model_info(i)[0]
save_pc_multi(multi_pc, self.pc_out_path, cur_inf)
if i % 20 == 0:
self.log('Finished evaluating model ' + str(i) + '...')
# save evaluation statistics and do any aggregation
with open(os.path.join(self.out_path, 'cam_pose_results.csv'),
'w') as cam_out:
err_writer = csv.writer(cam_out, delimiter=',')
err_writer.writerow(['cat', 'model', 'frame', 'pos', 'rot'])
for i, (pos_err,
rot_err) in enumerate(zip(cam_pos_err, cam_rot_err)):
info = self.nocs_results.meta_info[i]
err_writer.writerow(
[info[0], info[1], info[2], pos_err, rot_err])
if not self.no_camera_eval:
self.log('Mean cam pos err: %06f' %
(np.mean(np.array(cam_pos_err))))
self.log('Mean cam rot err: %06f' %
(np.mean(np.array(cam_rot_err))))
if self.dpc_camera_eval:
with open(os.path.join(self.out_path, 'dpc_cam_rot_results.csv'),
'w') as cam_out:
err_writer = csv.writer(cam_out, delimiter=',')
err_writer.writerow(['cat', 'model', 'frame', 'rot'])
for i, rot_err in enumerate(dpc_rot_err):
info = self.nocs_results.meta_info[i]
err_writer.writerow([info[0], info[1], info[2], rot_err])
self.log('Mean dpc cam rot err: %06f' %
(np.mean(np.array(dpc_rot_err))))
all_errors = np.array(dpc_rot_err)
correct = all_errors < 30.0
num_predictions = correct.shape[0]
accuracy = np.count_nonzero(correct) / num_predictions
median_error = np.sort(all_errors)[num_predictions // 2]
self.log("accuracy: %f, median angular error: %f" %
(accuracy, median_error))
if self.nocs_view_type == 'single':
with open(os.path.join(self.out_path, 'single_view_results.csv'),
'w') as res_out:
err_writer = csv.writer(res_out, delimiter=',')
err_writer.writerow(['cat', 'model', 'frame', 'chamfer'])
for i, err in enumerate(single_view_err):
info = self.nocs_results.meta_info[i]
err_writer.writerow([info[0], info[1], info[2], err])
self.log('Mean single view err: %06f' %
(np.mean(np.array(single_view_err))))
if len(self.num_views) != 0:
with open(
os.path.join(self.out_path,
'single_nview_results.csv'),
'w') as res_out:
err_writer = csv.writer(res_out, delimiter=',')
err_writer.writerow(['cat', 'model', 'nviews', 'chamfer'])
for i in range(self.nocs_results.length):
info = self.nocs_results.get_model_info(i)
cat, model_id = info[0][0:2]
for j in range(len(self.num_views)):
if len(single_nview_errs[j]) != 0:
err = single_nview_errs[j][i]
err_writer.writerow(
[cat, model_id, self.num_views[j], err])
for i, err_list in enumerate(single_nview_errs):
if len(single_nview_errs[i]) != 0:
self.log(
'Mean %d-view err: %06f' %
(self.num_views[i], np.mean(np.array(err_list))))
elif self.nocs_view_type == 'multi':
with open(os.path.join(self.out_path, 'multi_view_results.csv'),
'w') as res_out:
err_writer = csv.writer(res_out, delimiter=',')
err_writer.writerow(['cat', 'model', 'nviews', 'chamfer'])
for i, err in enumerate(multi_view_err):
info = self.nocs_results.get_model_info(i)
cat, model_id = info[0][0:2]
nviews = len(info)
err_writer.writerow([cat, model_id, nviews, err])
self.log('Mean multi view err: %06f' %
(np.mean(np.array(multi_view_err))))
def run(self):
''' Run through each model and evaluate point cloud prediction and camera pose. '''
# need to first do an alignment between the predicted and ground truth reference frames
default_num_align_models = 20
num_align_models = min(
[default_num_align_models, self.dpc_results.length])
self.log('Performing alignment...')
final_align_quat = self.find_global_alignment(num_align_models)
# final_align_quat = np.array([ 0.02164055, 0.08669936, 0.02188158, -0.99575906]) # chair GT for debug
final_align_quat_conj = quaternion_conjugate(final_align_quat)
print(final_align_quat)
final_align_R = as_rotation_matrix(final_align_quat)
# print(final_align_R)
# now do the actual evaluation
cam_rot_errs = []
single_view_err = []
single_nview_errs = []
for n in self.num_views:
single_nview_errs.append([])
chamfer = ChamferMetric(device='cuda')
for i in range(self.dpc_results.length):
if not self.no_camera_eval:
# convert GT camera pose to blender quaternions
# compare angle of difference quaternion to outuput
dpc_id = self.dpc_results.meta_info[i]
nocs_id = self.nocs_results.get_model_info(i)[0][1]
if dpc_id != nocs_id:
print(
'ERROR EVALUATION DATA IS NOT CONSISTENT BETWEEN DPC AND NOCS!!'
)
quit()
cam_poses = self.dpc_results.camera_poses[i]
for j in range(cam_poses.shape[0]):
gt_pos_unity = self.nocs_results.get_camera_pose(i)[j][0]
gt_pos_blender = np.array(
[-gt_pos_unity[0], -gt_pos_unity[2], gt_pos_unity[1]])
gt_quat = quaternion_from_campos(gt_pos_blender)
gt_quat /= np.linalg.norm(gt_quat)
pred_quat = cam_poses[j]
pred_quat /= np.linalg.norm(pred_quat)
aligned_pred_quat = quaternion_multiply(
pred_quat, final_align_quat_conj)
# look at the quaternion difference between GT and pred
gt_conj = quaternion_conjugate(gt_quat)
q_diff = quaternion_multiply(gt_conj, aligned_pred_quat)
ang_diff = 2 * np.arccos(q_diff[0])
if ang_diff > np.pi:
ang_diff -= 2 * np.pi
cam_rot_errs.append(np.degrees(np.fabs(ang_diff)))
if not self.no_pc_eval:
# now we evaluate the estimated point cloud
# get GT point cloud (union of all available views)
gt_pc_frames = self.nocs_results.get_pc_gt(i)
gt_pc = np.concatenate(gt_pc_frames, axis=0)
if gt_pc.shape[0] > self.max_pts:
gt_pc = prune_pc(gt_pc, self.max_pts)
gt_pc -= np.mean(gt_pc, axis=0)
# for single-view we do evaluation for point cloud from every view
# and then evaluate any additional passed in nviews by taking the union
# of point clouds from the first N predicted views.
pred_pcs = self.dpc_results.pcs[i]
aligned_pcs = [
] #np.zeros((pred_pcs.shape[0], min(pred_pcs.shape[1], self.max_pts), 3))
for j in range(len(pred_pcs)):
single_pc = pred_pcs[j]
if single_pc.shape[0] > self.max_pts:
single_pc = prune_pc(single_pc, self.max_pts)
# center and align
single_pc -= np.mean(single_pc, axis=0)
single_pc = np.dot(final_align_R, single_pc.T).T
single_pc = normalize_pc(single_pc)
aligned_pcs.append(single_pc)
# calculate chamfer distance
cur_err = chamfer.chamfer_dist(single_pc, gt_pc)
single_view_err.append(cur_err)
# print('SingleChamferErr (1 view):', cur_err)
if not self.no_save_pc:
cur_inf = self.nocs_results.get_model_info(i)[j]
save_pc_single(single_pc, self.pc_out_path, cur_inf)
# then multi-views (using the first n views)
for k, n in enumerate(self.num_views):
if n > len(pred_pcs):
continue
# aggregate point cloud
multi_pc = np.concatenate(aligned_pcs[:n], axis=0)
# multi_pc = aligned_pcs[:n].reshape((n*aligned_pcs.shape[1], 3))
if multi_pc.shape[0] > self.max_pts:
multi_pc = prune_pc(multi_pc, self.max_pts)
cur_err = chamfer.chamfer_dist(multi_pc, gt_pc)
single_nview_errs[k].append(cur_err)
# print('SingleChamferErr (%d views): %f' % (n, cur_err))
if i % 20 == 0:
self.log('Finished evaluating model ' + str(i) + '...')
# calculate accuracy (% under 30 degrees) and median degree error
all_errors = np.array(cam_rot_errs)
correct = all_errors < 30.0
num_predictions = correct.shape[0]
accuracy = np.count_nonzero(correct) / num_predictions
median_error = np.sort(all_errors)[num_predictions // 2]
self.log("accuracy: %f, median angular error: %f" %
(accuracy, median_error))
# save evaluation statistics and do any aggregation
with open(os.path.join(self.out_path, 'cam_pose_results.csv'),
'w') as cam_out:
err_writer = csv.writer(cam_out, delimiter=',')
err_writer.writerow(['cat', 'model', 'frame', 'rot'])
for i, rot_err in enumerate(cam_rot_errs):
info = self.nocs_results.meta_info[
i] # can use meta info from here b/c in same order
err_writer.writerow([info[0], info[1], info[2], rot_err])
self.log('Mean cam rot err: %06f' % (np.mean(np.array(cam_rot_errs))))
with open(os.path.join(self.out_path, 'single_view_results.csv'),
'w') as res_out:
err_writer = csv.writer(res_out, delimiter=',')
err_writer.writerow(['cat', 'model', 'frame', 'chamfer'])
for i, err in enumerate(single_view_err):
info = self.nocs_results.meta_info[i]
err_writer.writerow([info[0], info[1], info[2], err])
self.log('Mean single view err: %06f' %
(np.mean(np.array(single_view_err))))
if len(self.num_views) != 0:
with open(os.path.join(self.out_path, 'single_nview_results.csv'),
'w') as res_out:
err_writer = csv.writer(res_out, delimiter=',')
err_writer.writerow(['cat', 'model', 'nviews', 'chamfer'])
for i in range(self.dpc_results.length):
info = self.nocs_results.get_model_info(i)
cat, model_id = info[0][0:2]
for j in range(len(self.num_views)):
err = single_nview_errs[j][i]
err_writer.writerow(
[cat, model_id, self.num_views[j], err])
for i, err_list in enumerate(single_nview_errs):
self.log('Mean %d-view err: %06f' %
(self.num_views[i], np.mean(np.array(err_list))))
def find_global_alignment(self, num_align_models):
''' Finds a rigid rotation between the network output and the ground truth coordinate system. '''
# use first N models or how every many are available
# heavily based on differentiable point clouds code
views_per_model = len(self.dpc_results.pcs[0])
rmse_vals = np.zeros(
(num_align_models, views_per_model)) # 5 views per model
align_quats = np.zeros((num_align_models, views_per_model, 4))
for i in range(num_align_models):
# same GT for every point cloud in this model
gt_pc_frames = self.nocs_results.get_pc_gt(i)
gt_pc = np.concatenate(gt_pc_frames, axis=0)
if gt_pc.shape[0] > self.max_pts:
gt_pc = prune_pc(gt_pc, self.max_pts)
gt_pc -= np.mean(gt_pc, axis=0)
# viz_pcs([gt_pc], os.path.join(self.out_path, 'gt_pc' + str(i) + '.png'))
pred_pcs = self.dpc_results.pcs[i]
cam_poses = self.dpc_results.camera_poses[i]
for j in range(len(pred_pcs)):
pc_pred = pred_pcs[j]
if pc_pred.shape[0] > self.max_pts:
pc_pred = prune_pc(pc_pred, self.max_pts)
pc_pred -= np.mean(pc_pred, axis=0)
pc_pred = normalize_pc(pc_pred)
# first rough align with camera quats
gt_pos_unity = self.nocs_results.get_camera_pose(i)[j][0]
gt_pos_blender = np.array(
[-gt_pos_unity[0], -gt_pos_unity[2], gt_pos_unity[1]])
gt_quat = quaternion_from_campos(gt_pos_blender)
gt_quat /= np.linalg.norm(gt_quat)
pred_quat = cam_poses[j]
pred_quat /= np.linalg.norm(pred_quat)
quat_unrot = quaternion_multiply(quaternion_conjugate(gt_quat),
pred_quat)
R_init = as_rotation_matrix(quat_unrot)
# then refine with ICP
reg_p2p = open3d_icp(pc_pred, gt_pc, init_rotation=R_init)
# print(reg_p2p.inlier_rmse)
rmse_vals[i, j] = reg_p2p.inlier_rmse
T = np.array(reg_p2p.transformation)
R = T[:3, :3]
assert (np.fabs(np.linalg.det(R) - 1.0) <= 0.0001)
align_quat = quaternions.mat2quat(R)
align_quats[i, j] = align_quat
# chair GT for debugging
# R = np.array([[-0.98402981, 0.04689179, -0.1717163 ],
# [-0.03930331, -0.99810577, -0.04733001],
# [-0.17361041, -0.03982512, 0.98400883]])
# aligned_pc_pred = np.dot(R, pc_pred.T).T
# viz_pcs([aligned_pc_pred], os.path.join(self.out_path, 'aligned_pred_pc' + str(i) + str(j) + '.png'))
# viz_pcs([gt_pc, aligned_pc_pred], os.path.join(self.out_path, 'overlay' + str(i) + str(j) + '.png'))
num_to_estimate = int(num_align_models * 0.75)
num_to_keep = int(views_per_model * 0.6)
self.log('Using ' + str(num_to_keep) + ' views from the ' +
str(num_to_estimate) + ' best models.')
# remove outliers from each model based on rmse
filtered_rmse = np.zeros((num_align_models, num_to_keep))
filtered_quats = np.zeros((num_align_models, num_to_keep, 4))
for i in range(num_align_models):
sort_inds = np.argsort(rmse_vals[i, :])
sort_inds = sort_inds[:num_to_keep]
filtered_rmse[i] = rmse_vals[i, sort_inds]
filtered_quats[i] = align_quats[i, sort_inds, :]
# now remove worst performing models
mean_rmse = np.mean(filtered_rmse, axis=1)
sort_inds = np.argsort(mean_rmse)
sort_inds = sort_inds[:num_to_estimate]
ref_rots = filtered_quats[sort_inds, :, :]
ref_rots = ref_rots.reshape((-1, 4))
print(ref_rots)
# find the mean rotation to get our final alignment rotation
final_align_quat = quatWAvgMarkley(ref_rots)
return final_align_quat