def create_data(i):
data_info = pd.read_csv('./upright_examples.txt')
depth_dir = './examples/'
rgb_dir = '/home/paulo/datasets/my3d60/'
rgb = mpimg.imread(rgb_dir + data_info['input'][i] + '.png')
random_r = R.from_euler(
'zyx', [0, data_info['rand_y'][i], data_info['rand_x'][i]],
degrees=True)
correction_r = R.from_euler(
'zyx', [0, data_info['pred_y'][i], data_info['pred_x'][i]],
degrees=True)
rgb_random_rot = synthesizeRotation(rgb, random_r.as_matrix())
rgb_upright = synthesizeRotation(rgb_random_rot,
correction_r.inv().as_matrix())
pred_depth = np.load(depth_dir + str(i) + '_corrected_output.npy')
pred_depth_rot = synthesizeRotation(pred_depth, correction_r.as_matrix())
gt = cv2.imread(
rgb_dir + data_info['input'][i].replace('color', 'depth') + '.exr',
cv2.IMREAD_ANYDEPTH)
gt[gt > 40] = 0
mpimg.imsave('./upright_vr/' + str(i) + '_rgb_random_rot.png',
rgb_random_rot)
mpimg.imsave('./upright_vr/' + str(i) + '_rgb_upright.png', rgb_upright)
mpimg.imsave('./upright_vr/' + str(i) + '_rgb_original.png', rgb)
np.save('./upright_vr/' + str(i) + '_pred_depth_rot.npy', pred_depth_rot)
np.save('./upright_vr/' + str(i) + '_pred_depth_upright', pred_depth)
np.save('./upright_vr/' + str(i) + '_gt_depth.npy', gt)
def create_data(i):
data_info = pd.read_csv('./upright_examples.txt')
depth_dir = './examples/'
rgb_dir = '/home/paulo/datasets/my3d60/'
rgb = mpimg.imread(rgb_dir + data_info['input'][i] + '.png')
random_r = R.from_euler(
'zxy', [0, data_info['rand_x'][i], data_info['rand_y'][i]],
degrees=True)
correction_r = R.from_euler(
'zxy', [0, data_info['pred_x'][i], data_info['pred_y'][i]],
degrees=True)
rgb_random_rot = synthesizeRotation(rgb, random_r.as_matrix())
rgb_upright = synthesizeRotation(rgb_random_rot,
correction_r.inv().as_matrix())
upright_output = np.load(depth_dir + str(i) + '_corrected_output.npy')
rot_output = np.load(depth_dir + str(i) + '_rot_output.npy')
gt = cv2.imread(
rgb_dir + data_info['input'][i].replace('color', 'depth') + '.exr',
cv2.IMREAD_ANYDEPTH)
gt[gt > 40] = 0
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
plt.figure(figsize=(16, 64))
gs1 = gridspec.GridSpec(1, 4)
gs1.update(wspace=0.025, hspace=0.05) # set the spacing between axes.
ax1 = plt.subplot(gs1[0])
plt.axis('off')
ax1.imshow(rgb_random_rot)
ax1 = plt.subplot(gs1[1])
plt.axis('off')
ax1.imshow(synthesizeRotation(gt, random_r.as_matrix()))
ax1 = plt.subplot(gs1[2])
plt.axis('off')
ax1.imshow(rot_output)
ax1 = plt.subplot(gs1[3])
plt.axis('off')
ax1.imshow(synthesizeRotation(upright_output, correction_r.as_matrix()))
plt.savefig('upright_vs_omnidepth.png', bbox_inches='tight')
"""
def create_data(i):
data_info = pd.read_csv('./upright_examples.txt')
depth_dir = './examples/'
rgb_dir = '/home/paulo/datasets/my3d60/'
rgb = mpimg.imread(rgb_dir + data_info['input'][i] + '.png')
random_r = R.from_euler(
'zyx', [0, data_info['rand_y'][i], data_info['rand_x'][i]],
degrees=True)
correction_r = R.from_euler(
'zyx', [0, data_info['pred_y'][i], data_info['pred_x'][i]],
degrees=True)
rgb_random_rot = synthesizeRotation(rgb, random_r.as_matrix())
rgb_upright = synthesizeRotation(rgb_random_rot,
correction_r.inv().as_matrix())
upright_output = np.load(depth_dir + str(i) + '_corrected_output.npy')
rot_output = np.load(depth_dir + str(i) + '_rot_output.npy')
input_output = np.load(depth_dir + str(i) + '_input_output.npy')
gt = cv2.imread(
rgb_dir + data_info['input'][i].replace('color', 'depth') + '.exr',
cv2.IMREAD_ANYDEPTH)
gt[gt > 40] = 0
plt.subplot(1, 3, 1)
plt.axis('off')
plt.imshow(rgb)
plt.subplot(1, 3, 2)
plt.axis('off')
plt.imshow(gt)
plt.subplot(1, 3, 3)
plt.axis('off')
plt.imshow(input_output)
plt.savefig('rectified_vs_rotated_depth_1.png', bbox_inches='tight')
plt.subplot(1, 3, 1)
plt.axis('off')
plt.imshow(rgb_random_rot)
plt.subplot(1, 3, 2)
plt.axis('off')
plt.imshow(synthesizeRotation(gt, random_r.as_matrix()))
plt.subplot(1, 3, 3)
plt.axis('off')
plt.imshow(rot_output)
plt.savefig('rectified_vs_rotated_depth_2.png', bbox_inches='tight')
def create_data(i):
data_info = pd.read_csv('./upright_examples.txt')
depth_dir = './examples/'
rgb_dir = '/home/paulo/datasets/my3d60/'
rgb = mpimg.imread(rgb_dir + data_info['input'][i] + '.png')
random_r = R.from_euler(
'zyx', [0, data_info['rand_y'][i], data_info['rand_x'][i]],
degrees=True)
correction_r = R.from_euler(
'zyx', [0, data_info['pred_y'][i], data_info['pred_x'][i]],
degrees=True)
rgb_random_rot = synthesizeRotation(rgb, random_r.as_matrix())
rgb_upright = synthesizeRotation(rgb_random_rot,
correction_r.inv().as_matrix())
pred_depth = np.load(depth_dir + str(i) + '_corrected_output.npy')
pred_depth_rot = synthesizeRotation(pred_depth, correction_r.as_matrix())
gt = cv2.imread(
rgb_dir + data_info['input'][i].replace('color', 'depth') + '.exr',
cv2.IMREAD_ANYDEPTH)
gt[gt > 40] = 0
plt.subplot(2, 3, 1)
plt.imshow(rgb_random_rot)
plt.subplot(2, 3, 2)
plt.imshow(rgb_upright)
plt.subplot(2, 3, 3)
plt.imshow(rgb)
plt.subplot(2, 3, 4)
plt.imshow(pred_depth_rot)
plt.subplot(2, 3, 5)
plt.imshow(pred_depth)
plt.subplot(2, 3, 6)
plt.imshow(gt)
plt.show()
if __name__ == '__main__':
samples = 100
pano = mpimg.imread(
'/home/paulo/datasets/3d60/Matteport3D/92_fa5f164b48f043c6b2b0bb9e8631a4821_color_0_Left_Down_0.0.png'
)
plt.imshow(pano)
plt.show()
panos = np.zeros((samples, ) + pano.shape)
r = R.from_rotvec([0, 0, 0])
points = fibonacci_sphere(samples)
for i in range(len(points)):
az, el, _ = cart2sph(points[i][0], points[i][1], points[i][2])
panos[i] = synthesizeRotation(pano,
R.from_rotvec([el, 0, az]).as_matrix())
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
for point in points:
ax.scatter(point[0], point[1], point[2], c='r', marker='o')
plt.show()
results = pd.read_csv('rotations_results.txt')
error = np.zeros(100)
frequency = np.zeros(100)
for index, row in results.iterrows():
error[int(row['point'])] += row['abs_rel_error']
frequency[int(row['point'])] += 1
error = error / frequency
def upright_examples(self,
checkpoint_path,
num_tests,
rot_range,
device,
seed=42):
print('Evaluating on rotations....')
# Set seed
np.random.seed(seed)
torch.manual_seed(seed)
# Load the checkpoint to evaluate
self.load_checkpoint(checkpoint_path, True, True)
# Put the model in eval mode
self.network = self.network.eval()
# print(self.network.module.input0_0.conv.bias)
# print(self.network.module.input0_0.conv.bias.shape)
# exit()
# Reset meter
self.reset_eval_metrics()
# Reset output file
with open('upright_examples.txt', 'w') as output:
output.write('input,rand_x,rand_y,pred_x,pred_y\n')
#set upright model
upright = DenseNetUprightAdjustment()
up_save_dict = torch.load('best90.pth')
upright.to(device)
upright.load_state_dict(up_save_dict['model'])
aug = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
# Load data
s = time.time()
with torch.no_grad():
for iteration in range(num_tests):
print('Evaluating {}/{}'.format(iteration, num_tests),
end='\r')
data = iter(self.val_dataloader).next()
# Parse the data
inputs, gt, other = self.parse_data(data)
# Rotate randomly
rx, ry = np.random.uniform(rot_range[0], rot_range[1], 2)
random_r = R.from_euler('zxy', [0, rx, ry], degrees=True)
rot_inputs = synthesizeRotation(
np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3),
random_r.as_matrix())
rot_inputs = np.expand_dims(np.rollaxis(rot_inputs, 2, 0),
axis=0)
rot_inputs = [torch.from_numpy(rot_inputs).float().to(device)]
img = np.rollaxis(rot_inputs[0].cpu().data.numpy()[0], 0,
3) * 255
img = cv2.resize(img, (442, 221))
data = {"image": img}
img = aug(**data)["image"]
img_pt = torch.from_numpy(img).permute(2, 0, 1)
img_pt = img_pt.unsqueeze(0).to(device)
with torch.no_grad():
rot = upright(img_pt)
rot = (float(rot[0][0] * 90), float(rot[0][1] * 90))
correction_r = R.from_euler('zxy', [0, rot[0], rot[1]],
degrees=True)
corrected_inputs = synthesizeRotation(
np.rollaxis(rot_inputs[0].cpu().data.numpy()[0], 0, 3),
correction_r.inv().as_matrix())
corrected_inputs = np.expand_dims(np.rollaxis(
corrected_inputs, 2, 0),
axis=0)
corrected_inputs = [
torch.from_numpy(corrected_inputs).float().to(device)
]
"""
plt.subplot(2,3,1)
plt.imshow(np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3))
plt.subplot(2,3,2)
plt.imshow(np.rollaxis(rot_inputs[0].cpu().data.numpy()[0], 0, 3))
plt.subplot(2,3,3)
plt.imshow(np.rollaxis(corrected_inputs[0].cpu().data.numpy()[0], 0, 3))
"""
# Run a forward pass
input_output = self.forward_pass(inputs)
rot_output = self.forward_pass(rot_inputs)
corrected_output = self.forward_pass(corrected_inputs)
"""
plt.subplot(2,3,4)
plt.imshow(np.squeeze(np.rollaxis(input_output[0].cpu().data.numpy()[0], 0, 3)))
plt.subplot(2,3,5)
plt.imshow(np.squeeze(np.rollaxis(rot_output[0].cpu().data.numpy()[0], 0, 3)))
plt.subplot(2,3,6)
plt.imshow(np.squeeze(np.rollaxis(corrected_output[0].cpu().data.numpy()[0], 0, 3)))
plt.show()
"""
np.save(
'./examples/' + str(iteration) + '_input_output.npy',
np.squeeze(
np.rollaxis(input_output[0].cpu().data.numpy()[0], 0,
3)))
np.save(
'./examples/' + str(iteration) + '_rot_output.npy',
np.squeeze(
np.rollaxis(rot_output[0].cpu().data.numpy()[0], 0,
3)))
np.save(
'./examples/' + str(iteration) + '_corrected_output.npy',
np.squeeze(
np.rollaxis(corrected_output[0].cpu().data.numpy()[0],
0, 3)))
with open('upright_examples.txt', 'a') as output:
output.write('{},{},{},{},{}\n'.format(
other[0], rx, ry, rot[0], rot[1]))
# Print a report on the validation results
print('Evaluation finished in {} seconds'.format(time.time() - s))
def evaluate_upright(self,
checkpoint_path,
num_tests,
rot_range,
device,
seed=42):
print('Evaluating on rotations....')
# Set seed
np.random.seed(seed)
torch.manual_seed(seed)
# Load the checkpoint to evaluate
self.load_checkpoint(checkpoint_path, True, True)
# Put the model in eval mode
self.network = self.network.eval()
# print(self.network.module.input0_0.conv.bias)
# print(self.network.module.input0_0.conv.bias.shape)
# exit()
# Reset meter
self.reset_eval_metrics()
# Reset output file
with open('upright_results.txt', 'w') as output:
output.write(
'input,rand_x,rand_y,pred_x,pred_y,abs_rel,sq_rel,rms_sq_lin,rms_sq_log,d1,d2,d3\n'
)
#set upright model
upright = DenseNetUprightAdjustment()
up_save_dict = torch.load('best90.pth')
upright.to(device)
upright.load_state_dict(up_save_dict['model'])
aug = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
# Load data
s = time.time()
with torch.no_grad():
for iteration in range(num_tests):
print('Evaluating {}/{}'.format(iteration, num_tests),
end='\r')
data = iter(self.val_dataloader).next()
# Parse the data
inputs, gt, other = self.parse_data(data)
# Rotate randomly
rx, ry = np.random.uniform(rot_range[0], rot_range[1], 2)
random_r = R.from_euler('zxy', [0, rx, ry], degrees=True)
inputs = synthesizeRotation(
np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3),
random_r.as_matrix())
inputs = np.expand_dims(np.rollaxis(inputs, 2, 0), axis=0)
inputs = [torch.from_numpy(inputs).float().to(device)]
"""
plt.subplot(1,2,1)
plt.imshow(np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3))
"""
img = np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3) * 255
img = cv2.resize(img, (442, 221))
data = {"image": img}
img = aug(**data)["image"]
img_pt = torch.from_numpy(img).permute(2, 0, 1)
img_pt = img_pt.unsqueeze(0).to(device)
with torch.no_grad():
rot = upright(img_pt)
rot = (float(rot[0][0] * 90), float(rot[0][1] * 90))
#################################################################
#if rot[1] > 90:
# print(rot[1])
#################################################################
correction_r = R.from_euler('zxy', [0, rot[0], rot[1]],
degrees=True)
inputs = synthesizeRotation(
np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3),
correction_r.inv().as_matrix())
inputs = np.expand_dims(np.rollaxis(inputs, 2, 0), axis=0)
inputs = [torch.from_numpy(inputs).float().to(device)]
"""
print('expected: {}, {}'.format(rx, ry))
print('predicted: {}, {}'.format(rot[0], rot[1]))
plt.subplot(1,2,2)
plt.imshow(np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3))
plt.show()
"""
# Run a forward pass
output = self.forward_pass(inputs)
output = synthesizeRotation(
np.rollaxis(output[0].cpu().data.numpy()[0], 0, 3),
correction_r.as_matrix())
output = np.expand_dims(np.rollaxis(output, 2, 0), axis=0)
output = [torch.from_numpy(output).float().to(device)]
output = synthesizeRotation(
np.rollaxis(output[0].cpu().data.numpy()[0], 0, 3),
random_r.inv().as_matrix())
output = np.expand_dims(np.rollaxis(output, 2, 0), axis=0)
output = [torch.from_numpy(output).float().to(device)]
# Compute the evaluation metrics
self.compute_eval_metrics(output, gt)
with open('upright_results.txt', 'a') as output:
output.write(
'{},{},{},{},{},{},{},{},{},{},{},{}\n'.format(
other[0], rx, ry, rot[0], rot[1],
self.abs_rel_error_meter.avg,
self.sq_rel_error_meter.avg,
self.lin_rms_sq_error_meter.avg,
self.log_rms_sq_error_meter.avg,
self.d1_inlier_meter.avg, self.d2_inlier_meter.avg,
self.d3_inlier_meter.avg))
self.reset_eval_metrics()
# If trying to save intermediate outputs
if self.validation_sample_freq >= 0:
# Save the intermediate outputs
if i % self.validation_sample_freq == 0:
self.save_samples(inputs, gt, other, output)
# Print a report on the validation results
print('Evaluation finished in {} seconds'.format(time.time() - s))
def evaluate_rotations(self,
checkpoint_path,
num_tests,
rot_range,
device,
seed=42):
print('Evaluating on rotations....')
# Set seed
np.random.seed(seed)
torch.manual_seed(seed)
# Load the checkpoint to evaluate
self.load_checkpoint(checkpoint_path, True, True)
# Put the model in eval mode
self.network = self.network.eval()
# print(self.network.module.input0_0.conv.bias)
# print(self.network.module.input0_0.conv.bias.shape)
# exit()
# Reset meter
self.reset_eval_metrics()
# Reset output file
with open('rotations_results.txt', 'w') as output:
output.write(
'input,rand_x,rand_y,abs_rel,sq_rel,rms_sq_lin,rms_sq_log,d1,d2,d3\n'
)
# Load data
s = time.time()
with torch.no_grad():
for iteration in range(num_tests):
print('Evaluating {}/{}'.format(iteration, num_tests),
end='\r')
data = iter(self.val_dataloader).next()
# Parse the data
inputs, gt, other = self.parse_data(data)
# Rotate randomly
rx, ry = np.random.uniform(rot_range[0], rot_range[1], 2)
random_r = R.from_euler('zxy', [0, rx, ry], degrees=True)
inputs = synthesizeRotation(
np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3),
random_r.as_matrix())
inputs = np.expand_dims(np.rollaxis(inputs, 2, 0), axis=0)
inputs = [torch.from_numpy(inputs).float().to(device)]
"""
plt.subplot(1,2,1)
plt.imshow(np.rollaxis(inputs[0].cpu().data.numpy()[0], 0, 3))
plt.show()
"""
# Run a forward pass
output = self.forward_pass(inputs)
output = synthesizeRotation(
np.rollaxis(output[0].cpu().data.numpy()[0], 0, 3),
random_r.inv().as_matrix())
output = np.expand_dims(np.rollaxis(output, 2, 0), axis=0)
output = [torch.from_numpy(output).float().to(device)]
# Compute the evaluation metrics
self.compute_eval_metrics(output, gt)
with open('rotations_results.txt', 'a') as output:
output.write('{},{},{},{},{},{},{},{},{},{}\n'.format(
other[0], rx, ry, self.abs_rel_error_meter.avg,
self.sq_rel_error_meter.avg,
self.lin_rms_sq_error_meter.avg,
self.log_rms_sq_error_meter.avg,
self.d1_inlier_meter.avg, self.d2_inlier_meter.avg,
self.d3_inlier_meter.avg))
#self.reset_eval_metrics()
# If trying to save intermediate outputs
if self.validation_sample_freq >= 0:
# Save the intermediate outputs
if i % self.validation_sample_freq == 0:
self.save_samples(inputs, gt, other, output)
# Print a report on the validation results
print('Evaluation finished in {} seconds'.format(time.time() - s))
self.print_validation_report()
for j in range(pano.shape[1] // 2 - 5, pano.shape[1] // 2 + 5):
pano[i, j, 0] = 255
pano[i, j, 1] = 0
pano[i, j, 2] = 0
plt.imshow(pano)
plt.show()
panos = np.zeros((samples, ) + pano.shape)
r = R.from_rotvec([0, 0, 0])
points = fibonacci_sphere(samples)
sph_points = [cart2sph(p[0], p[1], p[2]) for p in points]
for i in range(len(points)):
az, el, _ = cart2sph(points[i][0], points[i][1], points[i][2])
#panos[i] = synthesizeRotation(pano, R.from_euler('zyx',[az, 0, el]).as_matrix())
panos[i] = synthesizeRotation(pano,
R.from_rotvec(points[i]).as_matrix())
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
for i in range(samples):
ax.scatter(points[i][0], points[i][1], points[i][2], c='r', marker='o')
ax.text(points[i][0] + 0.01,
points[i][1] + 0.01,
points[i][2] + 0.01,
"{0:.4f},{1:.4f}".format(np.rad2deg(sph_points[i][0]),
np.rad2deg(sph_points[i][1])),
fontsize=9)
ax.set_xlim3d(-1, 1)
ax.set_ylim3d(-1, 1)
ax.set_zlim3d(-1, 1)