def __getitem__(self, idx):
"""Get item."""
# Get object path
obj_path1 = os.path.join(self.opt.root_dir1, 'cube.obj')
obj_path2 = os.path.join(self.opt.root_dir2, 'sphere_halfbox.obj')
obj_path3 = os.path.join(self.opt.root_dir3, 'cone.obj')
# obj_path4 = os.path.join(self.opt.root_dir4, self.samples[idx])
if not self.loaded:
self.fg_obj1 = load_model(obj_path1)
self.fg_obj2 = load_model(obj_path2)
self.fg_obj3 = load_model(obj_path3)
# self.fg_obj4 = load_model(obj_path4)
self.bg_obj = load_model(self.opt.bg_model)
self.loaded = True
offset_id = np.random.permutation(4)
obj_model1 = self.fg_obj1
obj_model2 = self.fg_obj2
obj_model3 = self.fg_obj3
# obj_model4 = self.fg_obj4
obj2 = self.bg_obj
v11 = (obj_model1['v'] - obj_model1['v'].mean()) / (
obj_model1['v'].max() - obj_model1['v'].min())
v12 = (obj_model2['v'] - obj_model2['v'].mean()) / (
obj_model2['v'].max() - obj_model2['v'].min())
v13 = (obj_model3['v'] - obj_model3['v'].mean()) / (
obj_model3['v'].max() - obj_model3['v'].min())
# v14 = (obj_model4['v'] - obj_model4['v'].mean()) / (obj_model4['v'].max() - obj_model4['v'].min())
v2 = obj2['v'] # / (obj2['v'].max() - obj2['v'].min())
scale = (obj2['v'].max() - obj2['v'].min()) * 0.22
offset = np.array([[6.9, 6.9, 7.0], [20.4, 6.7,
6.7], [20.4, 6.7, 20.2],
[7.0, 6.7, 20.4]
]) #+ 2 * np.random.rand(3) #[8.0, 5.0, 18.0],
if self.opt.only_background:
v = v2
f = obj2['f']
elif self.opt.only_foreground:
v = v1
f = obj_model['f']
else:
if self.opt.random_rotation:
random_axis = np_normalize(np.random.rand(3))
random_angle = np.random.rand(1) * np.pi * 2
M = axis_angle_matrix(axis=random_axis, angle=random_angle)
M[:3, 3] = offset[offset_id[0]] + 1.5 * np.random.randn(3)
v11 = np.matmul(scale * v11,
M.transpose(1, 0)[:3, :3]) + M[:3, 3]
else:
# random_axis = np_normalize(np.random.rand(3))
# random_angle = np.random.rand(1) * np.pi * 2
# M = axis_angle_matrix(axis=random_axis, angle=random_angle)
# M[:3, 3] = offset[offset_id[0]]#+1.5*np.random.randn(3)
# v11 = np.matmul(scale * v11, M.transpose(1, 0)[:3, :3]) + M[:3, 3]
#
# random_axis2 = np_normalize(np.random.rand(3))
# random_angle2 = np.random.rand(1) * np.pi * 2
# M2 = axis_angle_matrix(axis=random_axis2, angle=random_angle2)
# M2[:3, 3] = offset[offset_id[2]]#+1.5*np.random.randn(3)
# v13 = np.matmul(scale * v13, M2.transpose(1, 0)[:3, :3]) + M2[:3, 3]
v11 = scale * v11 + offset[
offset_id[0]] + 1.5 * np.random.randn(3)
v12 = scale * v12 + offset[
offset_id[1]] + 1.5 * np.random.randn(3)
v13 = scale * v13 + offset[
offset_id[2]] + 1.5 * np.random.randn(3)
# v14 = scale * v14 + offset[offset_id[3]]
# v = np.concatenate((v11,v12,v13,v14, v2))
# f = np.concatenate((obj_model1['f'],obj_model2['f']+ v11.shape[0],obj_model3['f']+ v12.shape[0],obj_model4['f']+ v13.shape[0],obj2['f'] + v14.shape[0]))
v = np.concatenate((v11, v12, v13, v2))
#import ipdb; ipdb.set_trace()
f = np.concatenate(
(obj_model1['f'], obj_model2['f'] + v11.shape[0],
obj_model3['f'] + v12.shape[0] + v11.shape[0],
obj2['f'] + v13.shape[0] + v12.shape[0] + v11.shape[0]))
obj_model = {'v': v, 'f': f}
if self.opt.use_mesh:
# normalize the vertices
v = obj_model['v']
axis_range = np.max(v, axis=0) - np.min(v, axis=0)
v = (v - np.mean(v, axis=0)) / max(
axis_range) # Normalize to make the largest spread 1
obj_model['v'] = v
mesh = obj_to_triangle_spec(obj_model)
meshes = {
'face': mesh['face'].astype(np.float32),
'normal': mesh['normal'].astype(np.float32)
}
sample = {'synset': 0, 'mesh': meshes}
else:
# Sample points from the 3D mesh
v, vn = uniform_sample_mesh(obj_model,
num_samples=self.opt.n_splats)
# Normalize the vertices
v = (v - np.mean(v, axis=0)) / (v.max() - v.min())
# Save the splats
splats = {
'pos': v.astype(np.float32),
'normal': vn.astype(np.float32)
}
# Add model and synset to the output dictionary
sample = {'synset': 0, 'splats': splats}
# Transform
if self.transform:
sample = self.transform(sample)
return sample
def render_sphere_world(out_dir, cam_pos, radius, width, height, fovy, focal_length,
b_display=False):
"""
Generate z positions on a grid fixed inside the view frustum in the world coordinate system. Place the camera and
choose the camera's field of view so that the side of the square touches the frustum.
"""
import copy
print('render sphere')
sampling_time = []
rendering_time = []
num_samples = width * height
r = np.ones(num_samples) * radius
large_scene = copy.deepcopy(SCENE_TEST)
large_scene['camera']['viewport'] = [0, 0, width, height]
large_scene['camera']['fovy'] = np.deg2rad(fovy)
large_scene['camera']['focal_length'] = focal_length
large_scene['objects']['disk']['radius'] = tch_var_f(r)
large_scene['objects']['disk']['material_idx'] = tch_var_l(np.zeros(num_samples, dtype=int).tolist())
large_scene['materials']['albedo'] = tch_var_f([[0.6, 0.6, 0.6]])
large_scene['tonemap']['gamma'] = tch_var_f([1.0]) # Linear output
x, y = np.meshgrid(np.linspace(-1, 1, width), np.linspace(-1, 1, height))
#z = np.sqrt(1 - np.min(np.stack((x ** 2 + y ** 2, np.ones_like(x)), axis=-1), axis=-1))
unit_disk_mask = (x ** 2 + y ** 2) <= 1
z = np.sqrt(1 - unit_disk_mask * (x ** 2 + y ** 2))
# Make a hemi-sphere bulging out of the xy-plane scene
z[~unit_disk_mask] = 0
pos = np.stack((x.ravel(), y.ravel(), z.ravel()), axis=1)
# Normals outside the sphere should be [0, 0, 1]
x[~unit_disk_mask] = 0
y[~unit_disk_mask] = 0
z[~unit_disk_mask] = 1
normals = np_normalize(np.stack((x.ravel(), y.ravel(), z.ravel()), axis=1))
if b_display:
plt.ion()
plt.figure()
plt.imshow(pos[..., 2].reshape((height, width)))
plt.figure()
plt.imshow(normals[..., 2].reshape((height, width)))
large_scene['objects']['disk']['pos'] = tch_var_f(pos)
large_scene['objects']['disk']['normal'] = tch_var_f(normals)
large_scene['camera']['eye'] = tch_var_f(cam_pos)
# main render run
start_time = time()
res = render(large_scene)
rendering_time.append(time() - start_time)
im = get_data(res['image'])
im = np.uint8(255. * im)
depth = get_data(res['depth'])
depth[depth >= large_scene['camera']['far']] = depth.min()
im_depth = np.uint8(255. * (depth - depth.min()) / (depth.max() - depth.min()))
if b_display:
plt.figure()
plt.imshow(im, interpolation='none')
plt.title('Image')
plt.savefig(out_dir + '/fig_img_orig.png')
plt.figure()
plt.imshow(im_depth, interpolation='none')
plt.title('Depth Image')
plt.savefig(out_dir + '/fig_depth_orig.png')
imsave(out_dir + '/img_orig.png', im)
imsave(out_dir + '/depth_orig.png', im_depth)
# hold matplotlib figure
plt.ioff()
plt.show()
def test_sphere_splat_render_along_ray(out_dir, cam_pos, width, height, fovy, focal_length, use_quartic,
b_display=False):
"""
Create a sphere on a square as in render_sphere_world, and then convert to the camera's coordinate system
and then render using render_splats_along_ray.
"""
import copy
print('render sphere along ray')
sampling_time = []
rendering_time = []
num_samples = width * height
large_scene = copy.deepcopy(SCENE_TEST)
large_scene['camera']['viewport'] = [0, 0, width, height]
large_scene['camera']['eye'] = tch_var_f(cam_pos)
large_scene['camera']['fovy'] = np.deg2rad(fovy)
large_scene['camera']['focal_length'] = focal_length
large_scene['objects']['disk']['material_idx'] = tch_var_l(np.zeros(num_samples, dtype=int).tolist())
large_scene['materials']['albedo'] = tch_var_f([[0.6, 0.6, 0.6]])
large_scene['tonemap']['gamma'] = tch_var_f([1.0]) # Linear output
x, y = np.meshgrid(np.linspace(-1, 1, width), np.linspace(-1, 1, height))
#z = np.sqrt(1 - np.min(np.stack((x ** 2 + y ** 2, np.ones_like(x)), axis=-1), axis=-1))
unit_disk_mask = (x ** 2 + y ** 2) <= 1
z = np.sqrt(1 - unit_disk_mask * (x ** 2 + y ** 2))
# Make a hemi-sphere bulging out of the xy-plane scene
z[~unit_disk_mask] = 0
pos = np.stack((x.ravel(), y.ravel(), z.ravel() - 5, np.ones(num_samples)), axis=1)
# Normals outside the sphere should be [0, 0, 1]
x[~unit_disk_mask] = 0
y[~unit_disk_mask] = 0
z[~unit_disk_mask] = 1
normals = np_normalize(np.stack((x.ravel(), y.ravel(), z.ravel(), np.zeros(num_samples)), axis=1))
if b_display:
plt.ion()
plt.figure()
plt.subplot(131)
plt.imshow(pos[..., 0].reshape((height, width)))
plt.subplot(132)
plt.imshow(pos[..., 1].reshape((height, width)))
plt.subplot(133)
plt.imshow(pos[..., 2].reshape((height, width)))
plt.figure()
plt.imshow(normals[..., 2].reshape((height, width)))
## Convert to the camera's coordinate system
#Mcam = lookat(eye=large_scene['camera']['eye'], at=large_scene['camera']['at'], up=large_scene['camera']['up'])
pos_CC = tch_var_f(pos) #torch.matmul(tch_var_f(pos), Mcam.transpose(1, 0))
large_scene['objects']['disk']['pos'] = pos_CC
large_scene['objects']['disk']['normal'] = None # Estimate the normals tch_var_f(normals)
# large_scene['camera']['eye'] = tch_var_f([-10., 0., 10.])
# large_scene['camera']['eye'] = tch_var_f([2., 0., 10.])
large_scene['camera']['eye'] = tch_var_f([-5., 0., 0.])
# main render run
start_time = time()
res = render_splats_along_ray(large_scene, use_quartic=use_quartic)
rendering_time.append(time() - start_time)
# Test cam_to_world
res_world = cam_to_world(res['pos'].reshape(-1, 3), res['normal'].reshape(-1, 3), large_scene['camera'])
im = get_data(res['image'])
im = np.uint8(255. * im)
depth = get_data(res['depth'])
depth[depth >= large_scene['camera']['far']] = large_scene['camera']['far']
if b_display:
plt.figure()
plt.imshow(im, interpolation='none')
plt.title('Image')
plt.savefig(out_dir + '/fig_img_orig.png')
plt.figure()
plt.imshow(depth, interpolation='none')
plt.title('Depth Image')
#plt.savefig(out_dir + '/fig_depth_orig.png')
plt.figure()
pos_world = get_data(res_world['pos'])
posx_world = pos_world[:, 0].reshape((im.shape[0], im.shape[1]))
posy_world = pos_world[:, 1].reshape((im.shape[0], im.shape[1]))
posz_world = pos_world[:, 2].reshape((im.shape[0], im.shape[1]))
plt.subplot(131)
plt.imshow(posx_world)
plt.title('x_world')
plt.subplot(132)
plt.imshow(posy_world)
plt.title('y_world')
plt.subplot(133)
plt.imshow(posz_world)
plt.title('z_world')
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(pos_world[:, 0], pos_world[:, 1], pos_world[:, 2], s=1.3)
ax.set_xlabel('x')
ax.set_ylabel('y')
plt.figure()
pos_world = get_data(res['pos'].reshape(-1, 3))
posx_world = pos_world[:, 0].reshape((im.shape[0], im.shape[1]))
posy_world = pos_world[:, 1].reshape((im.shape[0], im.shape[1]))
posz_world = pos_world[:, 2].reshape((im.shape[0], im.shape[1]))
plt.subplot(131)
plt.imshow(posx_world)
plt.title('x_CC')
plt.subplot(132)
plt.imshow(posy_world)
plt.title('y_CC')
plt.subplot(133)
plt.imshow(posz_world)
plt.title('z_CC')
imsave(out_dir + '/img_orig.png', im)
#imsave(out_dir + '/depth_orig.png', im_depth)
# hold matplotlib figure
plt.ioff()
plt.show()
def test_sphere_splat_NDC(out_dir, cam_pos, width, height, fovy, focal_length, b_display=False):
"""
Create a sphere on a square as in render_sphere_world, and then convert to the camera's coordinate system and to
NDC and then render using render_splat_NDC.
"""
import copy
print('render sphere')
sampling_time = []
rendering_time = []
num_samples = width * height
large_scene = copy.deepcopy(SCENE_TEST)
large_scene['camera']['viewport'] = [0, 0, width, height]
large_scene['camera']['eye'] = tch_var_f(cam_pos)
large_scene['camera']['fovy'] = np.deg2rad(fovy)
large_scene['camera']['focal_length'] = focal_length
large_scene['objects']['disk']['material_idx'] = tch_var_l(np.zeros(num_samples, dtype=int).tolist())
large_scene['materials']['albedo'] = tch_var_f([[0.6, 0.6, 0.6]])
large_scene['tonemap']['gamma'] = tch_var_f([1.0]) # Linear output
x, y = np.meshgrid(np.linspace(-1, 1, width), np.linspace(-1, 1, height))
#z = np.sqrt(1 - np.min(np.stack((x ** 2 + y ** 2, np.ones_like(x)), axis=-1), axis=-1))
unit_disk_mask = (x ** 2 + y ** 2) <= 1
z = np.sqrt(1 - unit_disk_mask * (x ** 2 + y ** 2))
# Make a hemi-sphere bulging out of the xy-plane scene
z[~unit_disk_mask] = 0
pos = np.stack((x.ravel(), y.ravel(), z.ravel(), np.ones(num_samples)), axis=1)
# Normals outside the sphere should be [0, 0, 1]
x[~unit_disk_mask] = 0
y[~unit_disk_mask] = 0
z[~unit_disk_mask] = 1
normals = np_normalize(np.stack((x.ravel(), y.ravel(), z.ravel(), np.zeros(num_samples)), axis=1))
if b_display:
plt.ion()
plt.figure()
plt.imshow(pos[..., 2].reshape((height, width)))
plt.figure()
plt.imshow(normals[..., 2].reshape((height, width)))
# Convert to the camera's coordinate system
Mcam = lookat(eye=large_scene['camera']['eye'], at=large_scene['camera']['at'], up=large_scene['camera']['up'])
Mproj = perspective(fovy=large_scene['camera']['fovy'], aspect=width/height, near=large_scene['camera']['near'],
far=large_scene['camera']['far'])
pos_CC = torch.matmul(tch_var_f(pos), Mcam.transpose(1, 0))
pos_NDC = torch.matmul(pos_CC, Mproj.transpose(1, 0))
large_scene['objects']['disk']['pos'] = pos_NDC / pos_NDC[..., 3][:, np.newaxis]
large_scene['objects']['disk']['normal'] = tch_var_f(normals)
# main render run
start_time = time()
res = render_splats_NDC(large_scene)
rendering_time.append(time() - start_time)
im = get_data(res['image'])
im = np.uint8(255. * im)
depth = get_data(res['depth'])
depth[depth >= large_scene['camera']['far']] = depth.min()
im_depth = np.uint8(255. * (depth - depth.min()) / (depth.max() - depth.min()))
if b_display:
plt.figure()
plt.imshow(im, interpolation='none')
plt.title('Image')
plt.savefig(out_dir + '/fig_img_orig.png')
plt.figure()
plt.imshow(im_depth, interpolation='none')
plt.title('Depth Image')
plt.savefig(out_dir + '/fig_depth_orig.png')
imsave(out_dir + '/img_orig.png', im)
imsave(out_dir + '/depth_orig.png', im_depth)
# hold matplotlib figure
plt.ioff()
plt.show()
def __getitem__(self, idx):
"""Get item."""
# Get object path
obj_path = os.path.join(self.opt.root_dir, self.samples[idx])
if not self.loaded:
self.fg_obj = load_model(obj_path)
self.bg_obj = load_model(self.opt.bg_model)
self.loaded = True
obj_model = self.fg_obj
obj2 = self.bg_obj
v1 = (obj_model['v'] - obj_model['v'].mean()) / (obj_model['v'].max() -
obj_model['v'].min())
v2 = obj2['v'] # / (obj2['v'].max() - obj2['v'].min())
scale = (obj2['v'].max() - obj2['v'].min()) * 0.4
offset = np.array([14.0, 8.0, 12.0]) #+ 2 * np.abs(np.random.randn(3))
if self.opt.only_background:
v = v2
f = obj2['f']
elif self.opt.only_foreground:
v = v1
f = obj_model['f']
else:
if self.opt.random_rotation:
random_axis = np_normalize(self.opt.axis)
random_angle = np.random.rand(1) * np.pi * 2
M = axis_angle_matrix(axis=random_axis, angle=random_angle)
M[:3, 3] = offset
v1 = np.matmul(scale * v1,
M.transpose(1, 0)[:3, :3]) + M[:3, 3]
else:
v1 = scale * v1 + offset
v = np.concatenate((v1, v2))
f = np.concatenate((obj_model['f'], obj2['f'] + v1.shape[0]))
obj_model = {'v': v, 'f': f}
if self.opt.use_mesh:
# normalize the vertices
v = obj_model['v']
axis_range = np.max(v, axis=0) - np.min(v, axis=0)
v = (v - np.mean(v, axis=0)) / max(
axis_range) # Normalize to make the largest spread 1
obj_model['v'] = v
mesh = obj_to_triangle_spec(obj_model)
meshes = {
'face': mesh['face'].astype(np.float32),
'normal': mesh['normal'].astype(np.float32)
}
sample = {'synset': 0, 'mesh': meshes}
else:
# Sample points from the 3D mesh
v, vn = uniform_sample_mesh(obj_model,
num_samples=self.opt.n_splats)
# Normalize the vertices
v = (v - np.mean(v, axis=0)) / (v.max() - v.min())
# Save the splats
splats = {
'pos': v.astype(np.float32),
'normal': vn.astype(np.float32)
}
# Add model and synset to the output dictionary
sample = {'synset': 0, 'splats': splats}
# Transform
if self.transform:
sample = self.transform(sample)
return sample
def render_sphere(out_dir,
cam_pos,
radius,
width,
height,
fovy,
focal_length,
num_views,
std_z=0.01,
std_normal=0.01,
b_display=False):
"""
Randomly generate N samples on a surface and render them. The samples include position and normal, the radius is set
to a constant.
"""
print('render sphere')
sampling_time = []
rendering_time = []
num_samples = width * height
r = np.ones(num_samples) * radius
scene = copy.deepcopy(SCENE_BASIC)
scene['camera']['viewport'] = [0, 0, width, height]
scene['camera']['fovy'] = np.deg2rad(fovy)
scene['camera']['focal_length'] = focal_length
scene['objects']['disk']['radius'] = tch_var_f(r)
scene['objects']['disk']['material_idx'] = tch_var_l(
np.zeros(num_samples, dtype=int).tolist())
scene['materials']['albedo'] = tch_var_f([[0.6, 0.6, 0.6]])
scene['tonemap']['gamma'] = tch_var_f([1.0]) # Linear output
x, y = np.meshgrid(np.linspace(-1, 1, width), np.linspace(-1, 1, height))
#z = np.sqrt(1 - np.min(np.stack((x ** 2 + y ** 2, np.ones_like(x)), axis=-1), axis=-1))
unit_disk_mask = (x**2 + y**2) <= 1
z = np.sqrt(1 - unit_disk_mask * (x**2 + y**2))
# Make a hemi-sphere bulging out of the xy-plane scene
z[~unit_disk_mask] = 0
pos = np.stack((x.ravel(), y.ravel(), z.ravel()), axis=1)
# Normals outside the sphere should be [0, 0, 1]
x[~unit_disk_mask] = 0
y[~unit_disk_mask] = 0
z[~unit_disk_mask] = 1
normals = np_normalize(np.stack((x.ravel(), y.ravel(), z.ravel()), axis=1))
if b_display:
plt.ion()
plt.figure()
plt.imshow(pos[..., 2].reshape((height, width)))
plt.figure()
plt.imshow(normals[..., 2].reshape((height, width)))
scene['objects']['disk']['pos'] = tch_var_f(pos)
scene['objects']['disk']['normal'] = tch_var_f(normals)
scene['camera']['eye'] = tch_var_f(cam_pos)
# main render run
start_time = time()
res = render(scene)
rendering_time.append(time() - start_time)
im = get_data(res['image'])
depth = get_data(res['depth'])
depth[depth >= scene['camera']['far']] = depth.min()
im_depth = np.uint8(255. * (depth - depth.min()) /
(depth.max() - depth.min()))
if b_display:
plt.figure()
plt.imshow(im)
plt.title('Image')
plt.savefig(out_dir + '/fig_img_orig.png')
plt.figure()
plt.imshow(im_depth)
plt.title('Depth Image')
plt.savefig(out_dir + '/fig_depth_orig.png')
imsave(out_dir + '/img_orig.png', im)
imsave(out_dir + '/depth_orig.png', im_depth)
# generate noisy data
if b_display:
h1 = plt.figure()
h2 = plt.figure()
noisy_pos = pos
for view_idx in range(num_views):
noisy_pos[..., 2] = pos[..., 2] + std_z * np.random.randn(num_samples)
noisy_normals = np_normalize(normals + std_normal *
np.random.randn(num_samples, 3))
scene['objects']['disk']['pos'] = tch_var_f(noisy_pos)
scene['objects']['disk']['normal'] = tch_var_f(noisy_normals)
scene['camera']['eye'] = tch_var_f(cam_pos)
# main render run
start_time = time()
res = render(scene)
rendering_time.append(time() - start_time)
im = get_data(res['image'])
depth = get_data(res['depth'])
depth[depth >= scene['camera']['far']] = depth.min()
im_depth = np.uint8(255. * (depth - depth.min()) /
(depth.max() - depth.min()))
suffix_str = '{:05d}'.format(view_idx)
if b_display:
plt.figure(h1.number)
plt.imshow(im)
plt.title('Image')
plt.savefig(out_dir + '/fig_img_' + suffix_str + '.png')
plt.figure(h2.number)
plt.imshow(im_depth)
plt.title('Depth Image')
plt.savefig(out_dir + '/fig_depth_' + suffix_str + '.png')
imsave(out_dir + '/img_' + suffix_str + '.png', im)
imsave(out_dir + '/depth_' + suffix_str + '.png', im_depth)
# hold matplotlib figure
plt.ioff()
plt.show()