def create_data(i): data_info = pd.read_csv('./upright_examples.txt') depth_dir = './examples/' rgb_dir = '/home/paulo/datasets/my3d60/' rgb = mpimg.imread(rgb_dir + data_info['input'][i] + '.png') random_r = R.from_euler( 'zyx', [0, data_info['rand_y'][i], data_info['rand_x'][i]], degrees=True) correction_r = R.from_euler( 'zyx', [0, data_info['pred_y'][i], data_info['pred_x'][i]], degrees=True) rgb_random_rot = synthesizeRotation(rgb, random_r.as_matrix()) rgb_upright = synthesizeRotation(rgb_random_rot, correction_r.inv().as_matrix()) pred_depth = np.load(depth_dir + str(i) + '_corrected_output.npy') pred_depth_rot = synthesizeRotation(pred_depth, correction_r.as_matrix()) gt = cv2.imread( rgb_dir + data_info['input'][i].replace('color', 'depth') + '.exr', cv2.IMREAD_ANYDEPTH) gt[gt > 40] = 0 mpimg.imsave('./upright_vr/' + str(i) + '_rgb_random_rot.png', rgb_random_rot) mpimg.imsave('./upright_vr/' + str(i) + '_rgb_upright.png', rgb_upright) mpimg.imsave('./upright_vr/' + str(i) + '_rgb_original.png', rgb) np.save('./upright_vr/' + str(i) + '_pred_depth_rot.npy', pred_depth_rot) np.save('./upright_vr/' + str(i) + '_pred_depth_upright', pred_depth) np.save('./upright_vr/' + str(i) + '_gt_depth.npy', gt)
def create_data(i): data_info = pd.read_csv('./upright_examples.txt') depth_dir = './examples/' rgb_dir = '/home/paulo/datasets/my3d60/' rgb = mpimg.imread(rgb_dir + data_info['input'][i] + '.png') random_r = R.from_euler( 'zxy', [0, data_info['rand_x'][i], data_info['rand_y'][i]], degrees=True) correction_r = R.from_euler( 'zxy', [0, data_info['pred_x'][i], data_info['pred_y'][i]], degrees=True) rgb_random_rot = synthesizeRotation(rgb, random_r.as_matrix()) rgb_upright = synthesizeRotation(rgb_random_rot, correction_r.inv().as_matrix()) upright_output = np.load(depth_dir + str(i) + '_corrected_output.npy') rot_output = np.load(depth_dir + str(i) + '_rot_output.npy') gt = cv2.imread( rgb_dir + data_info['input'][i].replace('color', 'depth') + '.exr', cv2.IMREAD_ANYDEPTH) gt[gt > 40] = 0 import matplotlib.pyplot as plt import matplotlib.gridspec as gridspec plt.figure(figsize=(16, 64)) gs1 = gridspec.GridSpec(1, 4) gs1.update(wspace=0.025, hspace=0.05) # set the spacing between axes. ax1 = plt.subplot(gs1[0]) plt.axis('off') ax1.imshow(rgb_random_rot) ax1 = plt.subplot(gs1[1]) plt.axis('off') ax1.imshow(synthesizeRotation(gt, random_r.as_matrix())) ax1 = plt.subplot(gs1[2]) plt.axis('off') ax1.imshow(rot_output) ax1 = plt.subplot(gs1[3]) plt.axis('off') ax1.imshow(synthesizeRotation(upright_output, correction_r.as_matrix())) plt.savefig('upright_vs_omnidepth.png', bbox_inches='tight') """
def create_data(i): data_info = pd.read_csv('./upright_examples.txt') depth_dir = './examples/' rgb_dir = '/home/paulo/datasets/my3d60/' rgb = mpimg.imread(rgb_dir + data_info['input'][i] + '.png') random_r = R.from_euler( 'zyx', [0, data_info['rand_y'][i], data_info['rand_x'][i]], degrees=True) correction_r = R.from_euler( 'zyx', [0, data_info['pred_y'][i], data_info['pred_x'][i]], degrees=True) rgb_random_rot = synthesizeRotation(rgb, random_r.as_matrix()) rgb_upright = synthesizeRotation(rgb_random_rot, correction_r.inv().as_matrix()) upright_output = np.load(depth_dir + str(i) + '_corrected_output.npy') rot_output = np.load(depth_dir + str(i) + '_rot_output.npy') input_output = np.load(depth_dir + str(i) + '_input_output.npy') gt = cv2.imread( rgb_dir + data_info['input'][i].replace('color', 'depth') + '.exr', cv2.IMREAD_ANYDEPTH) gt[gt > 40] = 0 plt.subplot(1, 3, 1) plt.axis('off') plt.imshow(rgb) plt.subplot(1, 3, 2) plt.axis('off') plt.imshow(gt) plt.subplot(1, 3, 3) plt.axis('off') plt.imshow(input_output) plt.savefig('rectified_vs_rotated_depth_1.png', bbox_inches='tight') plt.subplot(1, 3, 1) plt.axis('off') plt.imshow(rgb_random_rot) plt.subplot(1, 3, 2) plt.axis('off') plt.imshow(synthesizeRotation(gt, random_r.as_matrix())) plt.subplot(1, 3, 3) plt.axis('off') plt.imshow(rot_output) plt.savefig('rectified_vs_rotated_depth_2.png', bbox_inches='tight')
def create_data(i): data_info = pd.read_csv('./upright_examples.txt') depth_dir = './examples/' rgb_dir = '/home/paulo/datasets/my3d60/' rgb = mpimg.imread(rgb_dir + data_info['input'][i] + '.png') random_r = R.from_euler( 'zyx', [0, data_info['rand_y'][i], data_info['rand_x'][i]], degrees=True) correction_r = R.from_euler( 'zyx', [0, data_info['pred_y'][i], data_info['pred_x'][i]], degrees=True) rgb_random_rot = synthesizeRotation(rgb, random_r.as_matrix()) rgb_upright = synthesizeRotation(rgb_random_rot, correction_r.inv().as_matrix()) pred_depth = np.load(depth_dir + str(i) + '_corrected_output.npy') pred_depth_rot = synthesizeRotation(pred_depth, correction_r.as_matrix()) gt = cv2.imread( rgb_dir + data_info['input'][i].replace('color', 'depth') + '.exr', cv2.IMREAD_ANYDEPTH) gt[gt > 40] = 0 plt.subplot(2, 3, 1) plt.imshow(rgb_random_rot) plt.subplot(2, 3, 2) plt.imshow(rgb_upright) plt.subplot(2, 3, 3) plt.imshow(rgb) plt.subplot(2, 3, 4) plt.imshow(pred_depth_rot) plt.subplot(2, 3, 5) plt.imshow(pred_depth) plt.subplot(2, 3, 6) plt.imshow(gt) plt.show()
if __name__ == '__main__': samples = 100 pano = mpimg.imread( '/home/paulo/datasets/3d60/Matteport3D/92_fa5f164b48f043c6b2b0bb9e8631a4821_color_0_Left_Down_0.0.png' ) plt.imshow(pano) plt.show() panos = np.zeros((samples, ) + pano.shape) r = R.from_rotvec([0, 0, 0]) points = fibonacci_sphere(samples) for i in range(len(points)): az, el, _ = cart2sph(points[i][0], points[i][1], points[i][2]) panos[i] = synthesizeRotation(pano, R.from_rotvec([el, 0, az]).as_matrix()) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') for point in points: ax.scatter(point[0], point[1], point[2], c='r', marker='o') plt.show() results = pd.read_csv('rotations_results.txt') error = np.zeros(100) frequency = np.zeros(100) for index, row in results.iterrows(): error[int(row['point'])] += row['abs_rel_error'] frequency[int(row['point'])] += 1 error = error / frequency
def upright_examples(self, checkpoint_path, num_tests, rot_range, device, seed=42): print('Evaluating on rotations....') # Set seed np.random.seed(seed) torch.manual_seed(seed) # Load the checkpoint to evaluate self.load_checkpoint(checkpoint_path, True, True) # Put the model in eval mode self.network = self.network.eval() # print(self.network.module.input0_0.conv.bias) # print(self.network.module.input0_0.conv.bias.shape) # exit() # Reset meter self.reset_eval_metrics() # Reset output file with open('upright_examples.txt', 'w') as output: output.write('input,rand_x,rand_y,pred_x,pred_y\n') #set upright model upright = DenseNetUprightAdjustment() up_save_dict = torch.load('best90.pth') upright.to(device) upright.load_state_dict(up_save_dict['model']) aug = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # Load data s = time.time() with torch.no_grad(): for iteration in range(num_tests): print('Evaluating {}/{}'.format(iteration, num_tests), end='\r') data = iter(self.val_dataloader).next() # Parse the data inputs, gt, other = self.parse_data(data) # Rotate randomly rx, ry = np.random.uniform(rot_range[0], rot_range[1], 2) random_r = R.from_euler('zxy', [0, rx, ry], degrees=True) rot_inputs = synthesizeRotation( np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3), random_r.as_matrix()) rot_inputs = np.expand_dims(np.rollaxis(rot_inputs, 2, 0), axis=0) rot_inputs = [torch.from_numpy(rot_inputs).float().to(device)] img = np.rollaxis(rot_inputs[0].cpu().data.numpy()[0], 0, 3) * 255 img = cv2.resize(img, (442, 221)) data = {"image": img} img = aug(**data)["image"] img_pt = torch.from_numpy(img).permute(2, 0, 1) img_pt = img_pt.unsqueeze(0).to(device) with torch.no_grad(): rot = upright(img_pt) rot = (float(rot[0][0] * 90), float(rot[0][1] * 90)) correction_r = R.from_euler('zxy', [0, rot[0], rot[1]], degrees=True) corrected_inputs = synthesizeRotation( np.rollaxis(rot_inputs[0].cpu().data.numpy()[0], 0, 3), correction_r.inv().as_matrix()) corrected_inputs = np.expand_dims(np.rollaxis( corrected_inputs, 2, 0), axis=0) corrected_inputs = [ torch.from_numpy(corrected_inputs).float().to(device) ] """ plt.subplot(2,3,1) plt.imshow(np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3)) plt.subplot(2,3,2) plt.imshow(np.rollaxis(rot_inputs[0].cpu().data.numpy()[0], 0, 3)) plt.subplot(2,3,3) plt.imshow(np.rollaxis(corrected_inputs[0].cpu().data.numpy()[0], 0, 3)) """ # Run a forward pass input_output = self.forward_pass(inputs) rot_output = self.forward_pass(rot_inputs) corrected_output = self.forward_pass(corrected_inputs) """ plt.subplot(2,3,4) plt.imshow(np.squeeze(np.rollaxis(input_output[0].cpu().data.numpy()[0], 0, 3))) plt.subplot(2,3,5) plt.imshow(np.squeeze(np.rollaxis(rot_output[0].cpu().data.numpy()[0], 0, 3))) plt.subplot(2,3,6) plt.imshow(np.squeeze(np.rollaxis(corrected_output[0].cpu().data.numpy()[0], 0, 3))) plt.show() """ np.save( './examples/' + str(iteration) + '_input_output.npy', np.squeeze( np.rollaxis(input_output[0].cpu().data.numpy()[0], 0, 3))) np.save( './examples/' + str(iteration) + '_rot_output.npy', np.squeeze( np.rollaxis(rot_output[0].cpu().data.numpy()[0], 0, 3))) np.save( './examples/' + str(iteration) + '_corrected_output.npy', np.squeeze( np.rollaxis(corrected_output[0].cpu().data.numpy()[0], 0, 3))) with open('upright_examples.txt', 'a') as output: output.write('{},{},{},{},{}\n'.format( other[0], rx, ry, rot[0], rot[1])) # Print a report on the validation results print('Evaluation finished in {} seconds'.format(time.time() - s))
def evaluate_upright(self, checkpoint_path, num_tests, rot_range, device, seed=42): print('Evaluating on rotations....') # Set seed np.random.seed(seed) torch.manual_seed(seed) # Load the checkpoint to evaluate self.load_checkpoint(checkpoint_path, True, True) # Put the model in eval mode self.network = self.network.eval() # print(self.network.module.input0_0.conv.bias) # print(self.network.module.input0_0.conv.bias.shape) # exit() # Reset meter self.reset_eval_metrics() # Reset output file with open('upright_results.txt', 'w') as output: output.write( 'input,rand_x,rand_y,pred_x,pred_y,abs_rel,sq_rel,rms_sq_lin,rms_sq_log,d1,d2,d3\n' ) #set upright model upright = DenseNetUprightAdjustment() up_save_dict = torch.load('best90.pth') upright.to(device) upright.load_state_dict(up_save_dict['model']) aug = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # Load data s = time.time() with torch.no_grad(): for iteration in range(num_tests): print('Evaluating {}/{}'.format(iteration, num_tests), end='\r') data = iter(self.val_dataloader).next() # Parse the data inputs, gt, other = self.parse_data(data) # Rotate randomly rx, ry = np.random.uniform(rot_range[0], rot_range[1], 2) random_r = R.from_euler('zxy', [0, rx, ry], degrees=True) inputs = synthesizeRotation( np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3), random_r.as_matrix()) inputs = np.expand_dims(np.rollaxis(inputs, 2, 0), axis=0) inputs = [torch.from_numpy(inputs).float().to(device)] """ plt.subplot(1,2,1) plt.imshow(np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3)) """ img = np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3) * 255 img = cv2.resize(img, (442, 221)) data = {"image": img} img = aug(**data)["image"] img_pt = torch.from_numpy(img).permute(2, 0, 1) img_pt = img_pt.unsqueeze(0).to(device) with torch.no_grad(): rot = upright(img_pt) rot = (float(rot[0][0] * 90), float(rot[0][1] * 90)) ################################################################# #if rot[1] > 90: # print(rot[1]) ################################################################# correction_r = R.from_euler('zxy', [0, rot[0], rot[1]], degrees=True) inputs = synthesizeRotation( np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3), correction_r.inv().as_matrix()) inputs = np.expand_dims(np.rollaxis(inputs, 2, 0), axis=0) inputs = [torch.from_numpy(inputs).float().to(device)] """ print('expected: {}, {}'.format(rx, ry)) print('predicted: {}, {}'.format(rot[0], rot[1])) plt.subplot(1,2,2) plt.imshow(np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3)) plt.show() """ # Run a forward pass output = self.forward_pass(inputs) output = synthesizeRotation( np.rollaxis(output[0].cpu().data.numpy()[0], 0, 3), correction_r.as_matrix()) output = np.expand_dims(np.rollaxis(output, 2, 0), axis=0) output = [torch.from_numpy(output).float().to(device)] output = synthesizeRotation( np.rollaxis(output[0].cpu().data.numpy()[0], 0, 3), random_r.inv().as_matrix()) output = np.expand_dims(np.rollaxis(output, 2, 0), axis=0) output = [torch.from_numpy(output).float().to(device)] # Compute the evaluation metrics self.compute_eval_metrics(output, gt) with open('upright_results.txt', 'a') as output: output.write( '{},{},{},{},{},{},{},{},{},{},{},{}\n'.format( other[0], rx, ry, rot[0], rot[1], self.abs_rel_error_meter.avg, self.sq_rel_error_meter.avg, self.lin_rms_sq_error_meter.avg, self.log_rms_sq_error_meter.avg, self.d1_inlier_meter.avg, self.d2_inlier_meter.avg, self.d3_inlier_meter.avg)) self.reset_eval_metrics() # If trying to save intermediate outputs if self.validation_sample_freq >= 0: # Save the intermediate outputs if i % self.validation_sample_freq == 0: self.save_samples(inputs, gt, other, output) # Print a report on the validation results print('Evaluation finished in {} seconds'.format(time.time() - s))
def evaluate_rotations(self, checkpoint_path, num_tests, rot_range, device, seed=42): print('Evaluating on rotations....') # Set seed np.random.seed(seed) torch.manual_seed(seed) # Load the checkpoint to evaluate self.load_checkpoint(checkpoint_path, True, True) # Put the model in eval mode self.network = self.network.eval() # print(self.network.module.input0_0.conv.bias) # print(self.network.module.input0_0.conv.bias.shape) # exit() # Reset meter self.reset_eval_metrics() # Reset output file with open('rotations_results.txt', 'w') as output: output.write( 'input,rand_x,rand_y,abs_rel,sq_rel,rms_sq_lin,rms_sq_log,d1,d2,d3\n' ) # Load data s = time.time() with torch.no_grad(): for iteration in range(num_tests): print('Evaluating {}/{}'.format(iteration, num_tests), end='\r') data = iter(self.val_dataloader).next() # Parse the data inputs, gt, other = self.parse_data(data) # Rotate randomly rx, ry = np.random.uniform(rot_range[0], rot_range[1], 2) random_r = R.from_euler('zxy', [0, rx, ry], degrees=True) inputs = synthesizeRotation( np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3), random_r.as_matrix()) inputs = np.expand_dims(np.rollaxis(inputs, 2, 0), axis=0) inputs = [torch.from_numpy(inputs).float().to(device)] """ plt.subplot(1,2,1) plt.imshow(np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3)) plt.show() """ # Run a forward pass output = self.forward_pass(inputs) output = synthesizeRotation( np.rollaxis(output[0].cpu().data.numpy()[0], 0, 3), random_r.inv().as_matrix()) output = np.expand_dims(np.rollaxis(output, 2, 0), axis=0) output = [torch.from_numpy(output).float().to(device)] # Compute the evaluation metrics self.compute_eval_metrics(output, gt) with open('rotations_results.txt', 'a') as output: output.write('{},{},{},{},{},{},{},{},{},{}\n'.format( other[0], rx, ry, self.abs_rel_error_meter.avg, self.sq_rel_error_meter.avg, self.lin_rms_sq_error_meter.avg, self.log_rms_sq_error_meter.avg, self.d1_inlier_meter.avg, self.d2_inlier_meter.avg, self.d3_inlier_meter.avg)) #self.reset_eval_metrics() # If trying to save intermediate outputs if self.validation_sample_freq >= 0: # Save the intermediate outputs if i % self.validation_sample_freq == 0: self.save_samples(inputs, gt, other, output) # Print a report on the validation results print('Evaluation finished in {} seconds'.format(time.time() - s)) self.print_validation_report()
for j in range(pano.shape[1] // 2 - 5, pano.shape[1] // 2 + 5): pano[i, j, 0] = 255 pano[i, j, 1] = 0 pano[i, j, 2] = 0 plt.imshow(pano) plt.show() panos = np.zeros((samples, ) + pano.shape) r = R.from_rotvec([0, 0, 0]) points = fibonacci_sphere(samples) sph_points = [cart2sph(p[0], p[1], p[2]) for p in points] for i in range(len(points)): az, el, _ = cart2sph(points[i][0], points[i][1], points[i][2]) #panos[i] = synthesizeRotation(pano, R.from_euler('zyx',[az, 0, el]).as_matrix()) panos[i] = synthesizeRotation(pano, R.from_rotvec(points[i]).as_matrix()) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') for i in range(samples): ax.scatter(points[i][0], points[i][1], points[i][2], c='r', marker='o') ax.text(points[i][0] + 0.01, points[i][1] + 0.01, points[i][2] + 0.01, "{0:.4f},{1:.4f}".format(np.rad2deg(sph_points[i][0]), np.rad2deg(sph_points[i][1])), fontsize=9) ax.set_xlim3d(-1, 1) ax.set_ylim3d(-1, 1) ax.set_zlim3d(-1, 1)