def main(args): nocs_eval = NOCSEvaluation(args) images_out = os.path.join(nocs_eval.out_path, 'images') if not os.path.exists(images_out): os.mkdir(images_out) pcl_out = os.path.join(nocs_eval.out_path, 'pcl') if not os.path.exists(pcl_out): os.mkdir(pcl_out) for i in range(10): #dpc_eval.dpc_results.length): nocs_id = nocs_eval.nocs_results.get_model_info(i)[0][1] # get GT point cloud (union of all available views) gt_pc_frames = nocs_eval.nocs_results.get_pc_gt(i) #print(len(gt_pc_frames)) gt_pc = np.concatenate(gt_pc_frames, axis=0) if gt_pc.shape[0] > nocs_eval.max_pts: gt_pc = prune_pc(gt_pc, nocs_eval.max_pts) gt_pc -= np.mean(gt_pc, axis=0) viz_pcs([gt_pc], os.path.join(images_out, nocs_id + '_gt_pc.png')) np.save(os.path.join(pcl_out, nocs_id + '_gt_pc'), gt_pc) nocs00_maps = nocs_eval.nocs_results.get_nox00_pred(i) nocs01_maps = nocs_eval.nocs_results.get_nox01_pred(i) nocs00_pcs = nocs_eval.nocs_results.get_pc_pred(-1, nocs_maps=nocs00_maps) nocs01_pcs = nocs_eval.nocs_results.get_pc_pred(-1, nocs_maps=nocs01_maps) #print(len(nocs00_pcs)) #print(len(nocs01_pcs)) # NOCS if nocs_eval.nocs_view_type == 'single': for j in range(nocs00_maps.shape[0]): single_pc = np.concatenate([nocs00_pcs[j], nocs01_pcs[j]], axis=0) if single_pc.shape[0] > nocs_eval.max_pts: single_pc = prune_pc(single_pc, nocs_eval.max_pts) single_pc -= np.mean(single_pc, axis=0) viz_pcs([single_pc], os.path.join( images_out, nocs_id + '_nocs_pc_view' + str(j) + '.png')) np.save( os.path.join(pcl_out, nocs_id + '_nocs_pc_view' + str(j)), single_pc) elif nocs_eval.nocs_view_type == 'multi': pc_list = nocs00_pcs + nocs01_pcs multi_pc = np.concatenate(pc_list, axis=0) if multi_pc.shape[0] > nocs_eval.max_pts: multi_pc = prune_pc(multi_pc, nocs_eval.max_pts) multi_pc -= np.mean(multi_pc, axis=0) viz_pcs([multi_pc], os.path.join(images_out, nocs_id + '_nocs_pc.png')) np.save(os.path.join(pcl_out, nocs_id + '_nocs_pc'), multi_pc)
def main(args): r2n2_eval = R2N2Evaluation(args) images_out = os.path.join(r2n2_eval.out_path, 'images') if not os.path.exists(images_out): os.mkdir(images_out) pcl_out = os.path.join(r2n2_eval.out_path, 'pcl') if not os.path.exists(pcl_out): os.mkdir(pcl_out) for i in range(100): #dpc_eval.dpc_results.length): r2n2_id = r2n2_eval.r2n2_results.meta_info[i] nocs_id = r2n2_eval.nocs_results.get_model_info(i)[0][1] if r2n2_id != nocs_id: print( 'ERROR EVALUATION DATA IS NOT CONSISTENT BETWEEN R2N2 AND NOCS!!' ) quit() # get GT point cloud (union of all available views) gt_pc_frames = r2n2_eval.nocs_results.get_pc_gt(i) #print(len(gt_pc_frames)) gt_pc = np.concatenate(gt_pc_frames, axis=0) if gt_pc.shape[0] > r2n2_eval.max_pts: gt_pc = prune_pc(gt_pc, r2n2_eval.max_pts) gt_pc -= np.mean(gt_pc, axis=0) viz_pcs([gt_pc], os.path.join(images_out, r2n2_id + '_gt_pc.png')) np.save(os.path.join(pcl_out, r2n2_id + '_gt_pc'), gt_pc) nocs00_maps = r2n2_eval.nocs_results.get_nox00_pred(i) nocs01_maps = r2n2_eval.nocs_results.get_nox01_pred(i) nocs00_pcs = r2n2_eval.nocs_results.get_pc_pred(-1, nocs_maps=nocs00_maps) nocs01_pcs = r2n2_eval.nocs_results.get_pc_pred(-1, nocs_maps=nocs01_maps) #print(len(nocs00_pcs)) #print(len(nocs01_pcs)) # R2N2 multi_pc = r2n2_eval.r2n2_results.pcs[i] if multi_pc.shape[0] > r2n2_eval.max_pts: multi_pc = prune_pc(multi_pc, r2n2_eval.max_pts) viz_pcs([multi_pc], os.path.join(images_out, r2n2_id + '_r2n2_pc.png')) np.save(os.path.join(pcl_out, r2n2_id + '_r2n2_pc'), multi_pc) # NOCS pc_list = nocs00_pcs + nocs01_pcs multi_pc = np.concatenate(pc_list, axis=0) if multi_pc.shape[0] > r2n2_eval.max_pts: multi_pc = prune_pc(multi_pc, r2n2_eval.max_pts) multi_pc -= np.mean(multi_pc, axis=0) viz_pcs([multi_pc], os.path.join(images_out, r2n2_id + '_nocs_pc.png')) np.save(os.path.join(pcl_out, r2n2_id + '_nocs_pc'), multi_pc)
def main(args): dpc_eval = DPCEvaluation(args) images_out = os.path.join(dpc_eval.out_path, 'images') if not os.path.exists(images_out): os.mkdir(images_out) pcl_out = os.path.join(dpc_eval.out_path, 'pcl') if not os.path.exists(pcl_out): os.mkdir(pcl_out) # need to first do an alignment between the predicted and ground truth reference frames default_num_align_models = 20 num_align_models = min( [default_num_align_models, dpc_eval.dpc_results.length]) dpc_eval.log('Performing alignment...') final_align_quat = dpc_eval.find_global_alignment(num_align_models) # final_align_quat = np.array([ 0.02164055, 0.08669936, 0.02188158, -0.99575906]) # chair GT for debug final_align_quat_conj = quaternion_conjugate(final_align_quat) print(final_align_quat) final_align_R = as_rotation_matrix(final_align_quat) for i in range(20): #dpc_eval.dpc_results.length): dpc_id = dpc_eval.dpc_results.meta_info[i] nocs_id = dpc_eval.nocs_results.get_model_info(i)[0][1] if dpc_id != nocs_id: print( 'ERROR EVALUATION DATA IS NOT CONSISTENT BETWEEN DPC AND NOCS!!' ) quit() # get GT point cloud (union of all available views) gt_pc_frames = dpc_eval.nocs_results.get_pc_gt(i) gt_pc = np.concatenate(gt_pc_frames, axis=0) if gt_pc.shape[0] > dpc_eval.max_pts: gt_pc = prune_pc(gt_pc, dpc_eval.max_pts) gt_pc -= np.mean(gt_pc, axis=0) viz_pcs([gt_pc], os.path.join(images_out, dpc_id + '_gt_pc.png')) np.save(os.path.join(pcl_out, dpc_id + '_gt_pc'), gt_pc) nocs00_maps = dpc_eval.nocs_results.get_nox00_pred(i) nocs01_maps = dpc_eval.nocs_results.get_nox01_pred(i) nocs00_pcs = dpc_eval.nocs_results.get_pc_pred(-1, nocs_maps=nocs00_maps) nocs01_pcs = dpc_eval.nocs_results.get_pc_pred(-1, nocs_maps=nocs01_maps) pred_pcs = dpc_eval.dpc_results.pcs[i] for j in range(len(pred_pcs)): # DPC single_pc = pred_pcs[j] if single_pc.shape[0] > dpc_eval.max_pts: single_pc = prune_pc(single_pc, dpc_eval.max_pts) # center and align single_pc -= np.mean(single_pc, axis=0) single_pc = np.dot(final_align_R, single_pc.T).T single_pc = normalize_pc(single_pc) viz_pcs([single_pc], os.path.join(images_out, dpc_id + '_dpc_pc_' + str(j) + '.png')) np.save(os.path.join(pcl_out, dpc_id + '_dpc_pc_' + str(j)), single_pc) # NOCS single_pc = np.concatenate([nocs00_pcs[j], nocs01_pcs[j]], axis=0) if single_pc.shape[0] > dpc_eval.max_pts: single_pc = prune_pc(single_pc, dpc_eval.max_pts) single_pc -= np.mean(single_pc, axis=0) viz_pcs([single_pc], os.path.join(images_out, dpc_id + '_nocs_pc_' + str(j) + '.png')) np.save(os.path.join(pcl_out, dpc_id + '_nocs_pc_' + str(j)), single_pc)
def run(self): ''' Run through each model and evaluate point cloud prediction and camera pose. ''' cam_pos_err = [] cam_rot_err = [] dpc_rot_err = [] if self.nocs_view_type == 'single': single_view_err = [] single_nview_errs = [] for n in self.num_views: single_nview_errs.append([]) elif self.nocs_view_type == 'multi': multi_view_err = [] for i in range(self.nocs_results.length): # will need these for both nocs00_maps = self.nocs_results.get_nox00_pred(i) nocs01_maps = self.nocs_results.get_nox01_pred(i) nocs00_pcs = self.nocs_results.get_pc_pred(-1, nocs_maps=nocs00_maps) nocs01_pcs = self.nocs_results.get_pc_pred(-1, nocs_maps=nocs01_maps) if self.pred_filter is not None and self.save_filtered: # frame_971_view_00_nox00_gt.png # frame is just i (model index) # view is model_info[2] info = self.nocs_results.get_model_info(i) frame_idx = i if self.nocs_view_type == 'multi': for view_idx in range(len(info)): # save front and back pred cur_front_path = 'frame_%03d_view_%02d_nox00_pred_filtered.png' % ( frame_idx, view_idx) cur_front_path = os.path.join(self.nocs_results.root, cur_front_path) cur_back_path = 'frame_%03d_view_%02d_nox01_pred_filtered.png' % ( frame_idx, view_idx) cur_back_path = os.path.join(self.nocs_results.root, cur_back_path) # print(cur_front_path) # print(cur_back_path) write_nocs_map(nocs00_maps[view_idx], cur_front_path) write_nocs_map(nocs01_maps[view_idx], cur_back_path) elif self.nocs_view_type == 'single': views_per_model = len(info) for view_idx in range(len(info)): global_frame_idx = i * views_per_model + view_idx # save front and back pred cur_front_path = 'frame_%03d_nox00_pred_filtered.png' % ( global_frame_idx) cur_front_path = os.path.join(self.nocs_results.root, cur_front_path) cur_back_path = 'frame_%03d_nox01_pred_filtered.png' % ( global_frame_idx) cur_back_path = os.path.join(self.nocs_results.root, cur_back_path) # print(cur_front_path) # print(cur_back_path) write_nocs_map(nocs00_maps[view_idx], cur_front_path) write_nocs_map(nocs01_maps[view_idx], cur_back_path) if not self.no_camera_eval: # first evaluate camera pose, do this for every frame # for both single-view and multi-view evaluation # solve for camera pose using nocs maps and point clouds cam_poses = self.nocs_results.get_camera_pose(i) for j in range(nocs00_maps.shape[0]): nocs00_map = nocs00_maps[j] nocs01_map = nocs01_maps[j] nocs00_pc = nocs00_pcs[j] nocs01_pc = nocs01_pcs[j] # solve using both layers jointly _, _, R_pred, P_pred, flip = self.estimateCameraPoseFromNM( [nocs00_map, nocs01_map], [nocs00_pc, nocs01_pc], N=NUM_CAM_SOLVE_PTS) # estimated position is w.r.t to (0.0, 0.0, 0.0) not NOCS center and in right-handed coords P_pred += np.array([-0.5, -0.5, -0.5]) P_pred[0] *= -1.0 # change coordinates depending on if prediction is flipped R_flip = axangle2mat(np.array([1.0, 0.0, 0.0]), np.radians(180), is_normalized=True) R_noflip = axangle2mat(np.array([0.0, 1.0, 0.0]), np.radians(180), is_normalized=True) # get GT pose P_gt, gt_quat = cam_poses[j] gt_quat = np.array( [gt_quat[0], gt_quat[1], gt_quat[2], gt_quat[3]]) R_gt = quaternions.quat2mat(gt_quat).T # compare (transform unit x-axis) unit_dir = np.array([1.0, 0.0, 0.0]) # predicted tranform R_pred = np.dot(R_pred, R_flip) if not flip: R_pred = np.dot(R_noflip, R_pred) pred_dir = np.dot(R_pred, unit_dir) pred_pos = P_pred # ground truth transform gt_dir = np.dot(R_gt, unit_dir) gt_pos = P_gt cur_pos_err = np.linalg.norm(gt_pos - pred_pos) cam_pos_err.append(cur_pos_err) cur_angle_err = np.degrees( np.arccos(np.dot(gt_dir, pred_dir))) cam_rot_err.append(cur_angle_err) # print('Model %d Frame %d:' % (i, j)) # print('Flip?:', flip) # print('Pred Pos:', pred_pos) # print('Pred X-dir:', pred_dir) # print('GT Pos:', gt_pos) # print('GT X-dir:', gt_dir) # print('=======================================================') # print('Pos err:', cur_pos_err) # print('rot err:', cur_angle_err) # print('=======================================================') # if the option is turned on, also evaluate camera pose for comparison with DPC # this method uses quaternion differences and only measures angle difference if self.dpc_camera_eval: gt_pos_unity, _ = cam_poses[j] gt_pos_blender = np.array([ -gt_pos_unity[0], -gt_pos_unity[2], gt_pos_unity[1] ]) gt_quat = quaternion_from_campos(gt_pos_blender) gt_quat /= np.linalg.norm(gt_quat) pred_pos_unity = pred_pos pred_pos_blender = np.array([ -pred_pos_unity[0], -pred_pos_unity[2], pred_pos_unity[1] ]) pred_quat = quaternion_from_campos(pred_pos_blender) pred_quat /= np.linalg.norm(pred_quat) # look at the quaternion difference between GT and pred gt_conj = quaternion_conjugate(gt_quat) q_diff = quaternion_multiply(gt_conj, pred_quat) ang_diff = 2 * np.arccos(q_diff[0]) if ang_diff > np.pi: ang_diff -= 2 * np.pi dpc_rot_err.append(np.degrees(np.fabs(ang_diff))) if not self.no_pc_eval: # now we evaluate the estimated point cloud # get GT point cloud (union of all available views) gt_pc_frames = self.nocs_results.get_pc_gt(i) gt_pc = np.concatenate(gt_pc_frames, axis=0) if gt_pc.shape[0] > self.max_pts: gt_pc = prune_pc(gt_pc, self.max_pts) chamfer = ChamferMetric(device='cuda') if self.nocs_view_type == 'single': # for single-view we do evaluation for point cloud from every view # and then evaluate any additional passed in nviews by taking the union # of point clouds from the first N predicted views. for j in range(nocs00_maps.shape[0]): single_pc = np.concatenate( [nocs00_pcs[j], nocs01_pcs[j]], axis=0) if single_pc.shape[0] > self.max_pts: single_pc = prune_pc(single_pc, self.max_pts) # calculate chamfer distance cur_err = chamfer.chamfer_dist(single_pc, gt_pc) single_view_err.append(cur_err) # print('SingleChamferErr (1 view):', cur_err) if not self.no_save_pc: cur_inf = self.nocs_results.get_model_info(i)[j] save_pc_single(single_pc, self.pc_out_path, cur_inf) # then multi-views (using the first n views) for k, n in enumerate(self.num_views): if n > nocs00_maps.shape[0]: continue # aggregate point cloud pc_list = [] for j in range(n): pc_list += [nocs00_pcs[j], nocs01_pcs[j]] multi_pc = np.concatenate(pc_list, axis=0) if multi_pc.shape[0] > self.max_pts: multi_pc = prune_pc(multi_pc, self.max_pts) cur_err = chamfer.chamfer_dist(multi_pc, gt_pc) single_nview_errs[k].append(cur_err) # print('SingleChamferErr (%d views): %f' % (n, cur_err)) elif self.nocs_view_type == 'multi': # for multi-view we only do a single evaluation on the point cloud # from all views combined pc_list = nocs00_pcs + nocs01_pcs multi_pc = np.concatenate(pc_list, axis=0) if multi_pc.shape[0] > self.max_pts: multi_pc = prune_pc(multi_pc, self.max_pts) # print(multi_pc.shape) cur_err = chamfer.chamfer_dist(multi_pc, gt_pc) multi_view_err.append(cur_err) # print('MultiChamferErr (%d views): %f' % (len(nocs00_pcs), cur_err)) if not self.no_save_pc: cur_inf = self.nocs_results.get_model_info(i)[0] save_pc_multi(multi_pc, self.pc_out_path, cur_inf) if i % 20 == 0: self.log('Finished evaluating model ' + str(i) + '...') # save evaluation statistics and do any aggregation with open(os.path.join(self.out_path, 'cam_pose_results.csv'), 'w') as cam_out: err_writer = csv.writer(cam_out, delimiter=',') err_writer.writerow(['cat', 'model', 'frame', 'pos', 'rot']) for i, (pos_err, rot_err) in enumerate(zip(cam_pos_err, cam_rot_err)): info = self.nocs_results.meta_info[i] err_writer.writerow( [info[0], info[1], info[2], pos_err, rot_err]) if not self.no_camera_eval: self.log('Mean cam pos err: %06f' % (np.mean(np.array(cam_pos_err)))) self.log('Mean cam rot err: %06f' % (np.mean(np.array(cam_rot_err)))) if self.dpc_camera_eval: with open(os.path.join(self.out_path, 'dpc_cam_rot_results.csv'), 'w') as cam_out: err_writer = csv.writer(cam_out, delimiter=',') err_writer.writerow(['cat', 'model', 'frame', 'rot']) for i, rot_err in enumerate(dpc_rot_err): info = self.nocs_results.meta_info[i] err_writer.writerow([info[0], info[1], info[2], rot_err]) self.log('Mean dpc cam rot err: %06f' % (np.mean(np.array(dpc_rot_err)))) all_errors = np.array(dpc_rot_err) correct = all_errors < 30.0 num_predictions = correct.shape[0] accuracy = np.count_nonzero(correct) / num_predictions median_error = np.sort(all_errors)[num_predictions // 2] self.log("accuracy: %f, median angular error: %f" % (accuracy, median_error)) if self.nocs_view_type == 'single': with open(os.path.join(self.out_path, 'single_view_results.csv'), 'w') as res_out: err_writer = csv.writer(res_out, delimiter=',') err_writer.writerow(['cat', 'model', 'frame', 'chamfer']) for i, err in enumerate(single_view_err): info = self.nocs_results.meta_info[i] err_writer.writerow([info[0], info[1], info[2], err]) self.log('Mean single view err: %06f' % (np.mean(np.array(single_view_err)))) if len(self.num_views) != 0: with open( os.path.join(self.out_path, 'single_nview_results.csv'), 'w') as res_out: err_writer = csv.writer(res_out, delimiter=',') err_writer.writerow(['cat', 'model', 'nviews', 'chamfer']) for i in range(self.nocs_results.length): info = self.nocs_results.get_model_info(i) cat, model_id = info[0][0:2] for j in range(len(self.num_views)): if len(single_nview_errs[j]) != 0: err = single_nview_errs[j][i] err_writer.writerow( [cat, model_id, self.num_views[j], err]) for i, err_list in enumerate(single_nview_errs): if len(single_nview_errs[i]) != 0: self.log( 'Mean %d-view err: %06f' % (self.num_views[i], np.mean(np.array(err_list)))) elif self.nocs_view_type == 'multi': with open(os.path.join(self.out_path, 'multi_view_results.csv'), 'w') as res_out: err_writer = csv.writer(res_out, delimiter=',') err_writer.writerow(['cat', 'model', 'nviews', 'chamfer']) for i, err in enumerate(multi_view_err): info = self.nocs_results.get_model_info(i) cat, model_id = info[0][0:2] nviews = len(info) err_writer.writerow([cat, model_id, nviews, err]) self.log('Mean multi view err: %06f' % (np.mean(np.array(multi_view_err))))
def run(self): ''' Run through each model and evaluate point cloud prediction and camera pose. ''' # need to first do an alignment between the predicted and ground truth reference frames default_num_align_models = 20 num_align_models = min( [default_num_align_models, self.dpc_results.length]) self.log('Performing alignment...') final_align_quat = self.find_global_alignment(num_align_models) # final_align_quat = np.array([ 0.02164055, 0.08669936, 0.02188158, -0.99575906]) # chair GT for debug final_align_quat_conj = quaternion_conjugate(final_align_quat) print(final_align_quat) final_align_R = as_rotation_matrix(final_align_quat) # print(final_align_R) # now do the actual evaluation cam_rot_errs = [] single_view_err = [] single_nview_errs = [] for n in self.num_views: single_nview_errs.append([]) chamfer = ChamferMetric(device='cuda') for i in range(self.dpc_results.length): if not self.no_camera_eval: # convert GT camera pose to blender quaternions # compare angle of difference quaternion to outuput dpc_id = self.dpc_results.meta_info[i] nocs_id = self.nocs_results.get_model_info(i)[0][1] if dpc_id != nocs_id: print( 'ERROR EVALUATION DATA IS NOT CONSISTENT BETWEEN DPC AND NOCS!!' ) quit() cam_poses = self.dpc_results.camera_poses[i] for j in range(cam_poses.shape[0]): gt_pos_unity = self.nocs_results.get_camera_pose(i)[j][0] gt_pos_blender = np.array( [-gt_pos_unity[0], -gt_pos_unity[2], gt_pos_unity[1]]) gt_quat = quaternion_from_campos(gt_pos_blender) gt_quat /= np.linalg.norm(gt_quat) pred_quat = cam_poses[j] pred_quat /= np.linalg.norm(pred_quat) aligned_pred_quat = quaternion_multiply( pred_quat, final_align_quat_conj) # look at the quaternion difference between GT and pred gt_conj = quaternion_conjugate(gt_quat) q_diff = quaternion_multiply(gt_conj, aligned_pred_quat) ang_diff = 2 * np.arccos(q_diff[0]) if ang_diff > np.pi: ang_diff -= 2 * np.pi cam_rot_errs.append(np.degrees(np.fabs(ang_diff))) if not self.no_pc_eval: # now we evaluate the estimated point cloud # get GT point cloud (union of all available views) gt_pc_frames = self.nocs_results.get_pc_gt(i) gt_pc = np.concatenate(gt_pc_frames, axis=0) if gt_pc.shape[0] > self.max_pts: gt_pc = prune_pc(gt_pc, self.max_pts) gt_pc -= np.mean(gt_pc, axis=0) # for single-view we do evaluation for point cloud from every view # and then evaluate any additional passed in nviews by taking the union # of point clouds from the first N predicted views. pred_pcs = self.dpc_results.pcs[i] aligned_pcs = [ ] #np.zeros((pred_pcs.shape[0], min(pred_pcs.shape[1], self.max_pts), 3)) for j in range(len(pred_pcs)): single_pc = pred_pcs[j] if single_pc.shape[0] > self.max_pts: single_pc = prune_pc(single_pc, self.max_pts) # center and align single_pc -= np.mean(single_pc, axis=0) single_pc = np.dot(final_align_R, single_pc.T).T single_pc = normalize_pc(single_pc) aligned_pcs.append(single_pc) # calculate chamfer distance cur_err = chamfer.chamfer_dist(single_pc, gt_pc) single_view_err.append(cur_err) # print('SingleChamferErr (1 view):', cur_err) if not self.no_save_pc: cur_inf = self.nocs_results.get_model_info(i)[j] save_pc_single(single_pc, self.pc_out_path, cur_inf) # then multi-views (using the first n views) for k, n in enumerate(self.num_views): if n > len(pred_pcs): continue # aggregate point cloud multi_pc = np.concatenate(aligned_pcs[:n], axis=0) # multi_pc = aligned_pcs[:n].reshape((n*aligned_pcs.shape[1], 3)) if multi_pc.shape[0] > self.max_pts: multi_pc = prune_pc(multi_pc, self.max_pts) cur_err = chamfer.chamfer_dist(multi_pc, gt_pc) single_nview_errs[k].append(cur_err) # print('SingleChamferErr (%d views): %f' % (n, cur_err)) if i % 20 == 0: self.log('Finished evaluating model ' + str(i) + '...') # calculate accuracy (% under 30 degrees) and median degree error all_errors = np.array(cam_rot_errs) correct = all_errors < 30.0 num_predictions = correct.shape[0] accuracy = np.count_nonzero(correct) / num_predictions median_error = np.sort(all_errors)[num_predictions // 2] self.log("accuracy: %f, median angular error: %f" % (accuracy, median_error)) # save evaluation statistics and do any aggregation with open(os.path.join(self.out_path, 'cam_pose_results.csv'), 'w') as cam_out: err_writer = csv.writer(cam_out, delimiter=',') err_writer.writerow(['cat', 'model', 'frame', 'rot']) for i, rot_err in enumerate(cam_rot_errs): info = self.nocs_results.meta_info[ i] # can use meta info from here b/c in same order err_writer.writerow([info[0], info[1], info[2], rot_err]) self.log('Mean cam rot err: %06f' % (np.mean(np.array(cam_rot_errs)))) with open(os.path.join(self.out_path, 'single_view_results.csv'), 'w') as res_out: err_writer = csv.writer(res_out, delimiter=',') err_writer.writerow(['cat', 'model', 'frame', 'chamfer']) for i, err in enumerate(single_view_err): info = self.nocs_results.meta_info[i] err_writer.writerow([info[0], info[1], info[2], err]) self.log('Mean single view err: %06f' % (np.mean(np.array(single_view_err)))) if len(self.num_views) != 0: with open(os.path.join(self.out_path, 'single_nview_results.csv'), 'w') as res_out: err_writer = csv.writer(res_out, delimiter=',') err_writer.writerow(['cat', 'model', 'nviews', 'chamfer']) for i in range(self.dpc_results.length): info = self.nocs_results.get_model_info(i) cat, model_id = info[0][0:2] for j in range(len(self.num_views)): err = single_nview_errs[j][i] err_writer.writerow( [cat, model_id, self.num_views[j], err]) for i, err_list in enumerate(single_nview_errs): self.log('Mean %d-view err: %06f' % (self.num_views[i], np.mean(np.array(err_list))))
def find_global_alignment(self, num_align_models): ''' Finds a rigid rotation between the network output and the ground truth coordinate system. ''' # use first N models or how every many are available # heavily based on differentiable point clouds code views_per_model = len(self.dpc_results.pcs[0]) rmse_vals = np.zeros( (num_align_models, views_per_model)) # 5 views per model align_quats = np.zeros((num_align_models, views_per_model, 4)) for i in range(num_align_models): # same GT for every point cloud in this model gt_pc_frames = self.nocs_results.get_pc_gt(i) gt_pc = np.concatenate(gt_pc_frames, axis=0) if gt_pc.shape[0] > self.max_pts: gt_pc = prune_pc(gt_pc, self.max_pts) gt_pc -= np.mean(gt_pc, axis=0) # viz_pcs([gt_pc], os.path.join(self.out_path, 'gt_pc' + str(i) + '.png')) pred_pcs = self.dpc_results.pcs[i] cam_poses = self.dpc_results.camera_poses[i] for j in range(len(pred_pcs)): pc_pred = pred_pcs[j] if pc_pred.shape[0] > self.max_pts: pc_pred = prune_pc(pc_pred, self.max_pts) pc_pred -= np.mean(pc_pred, axis=0) pc_pred = normalize_pc(pc_pred) # first rough align with camera quats gt_pos_unity = self.nocs_results.get_camera_pose(i)[j][0] gt_pos_blender = np.array( [-gt_pos_unity[0], -gt_pos_unity[2], gt_pos_unity[1]]) gt_quat = quaternion_from_campos(gt_pos_blender) gt_quat /= np.linalg.norm(gt_quat) pred_quat = cam_poses[j] pred_quat /= np.linalg.norm(pred_quat) quat_unrot = quaternion_multiply(quaternion_conjugate(gt_quat), pred_quat) R_init = as_rotation_matrix(quat_unrot) # then refine with ICP reg_p2p = open3d_icp(pc_pred, gt_pc, init_rotation=R_init) # print(reg_p2p.inlier_rmse) rmse_vals[i, j] = reg_p2p.inlier_rmse T = np.array(reg_p2p.transformation) R = T[:3, :3] assert (np.fabs(np.linalg.det(R) - 1.0) <= 0.0001) align_quat = quaternions.mat2quat(R) align_quats[i, j] = align_quat # chair GT for debugging # R = np.array([[-0.98402981, 0.04689179, -0.1717163 ], # [-0.03930331, -0.99810577, -0.04733001], # [-0.17361041, -0.03982512, 0.98400883]]) # aligned_pc_pred = np.dot(R, pc_pred.T).T # viz_pcs([aligned_pc_pred], os.path.join(self.out_path, 'aligned_pred_pc' + str(i) + str(j) + '.png')) # viz_pcs([gt_pc, aligned_pc_pred], os.path.join(self.out_path, 'overlay' + str(i) + str(j) + '.png')) num_to_estimate = int(num_align_models * 0.75) num_to_keep = int(views_per_model * 0.6) self.log('Using ' + str(num_to_keep) + ' views from the ' + str(num_to_estimate) + ' best models.') # remove outliers from each model based on rmse filtered_rmse = np.zeros((num_align_models, num_to_keep)) filtered_quats = np.zeros((num_align_models, num_to_keep, 4)) for i in range(num_align_models): sort_inds = np.argsort(rmse_vals[i, :]) sort_inds = sort_inds[:num_to_keep] filtered_rmse[i] = rmse_vals[i, sort_inds] filtered_quats[i] = align_quats[i, sort_inds, :] # now remove worst performing models mean_rmse = np.mean(filtered_rmse, axis=1) sort_inds = np.argsort(mean_rmse) sort_inds = sort_inds[:num_to_estimate] ref_rots = filtered_quats[sort_inds, :, :] ref_rots = ref_rots.reshape((-1, 4)) print(ref_rots) # find the mean rotation to get our final alignment rotation final_align_quat = quatWAvgMarkley(ref_rots) return final_align_quat