def __getitem__(self, idx): """Get item.""" # Get object path obj_path1 = os.path.join(self.opt.root_dir1, 'cube.obj') obj_path2 = os.path.join(self.opt.root_dir2, 'sphere_halfbox.obj') obj_path3 = os.path.join(self.opt.root_dir3, 'cone.obj') # obj_path4 = os.path.join(self.opt.root_dir4, self.samples[idx]) if not self.loaded: self.fg_obj1 = load_model(obj_path1) self.fg_obj2 = load_model(obj_path2) self.fg_obj3 = load_model(obj_path3) # self.fg_obj4 = load_model(obj_path4) self.bg_obj = load_model(self.opt.bg_model) self.loaded = True offset_id = np.random.permutation(4) obj_model1 = self.fg_obj1 obj_model2 = self.fg_obj2 obj_model3 = self.fg_obj3 # obj_model4 = self.fg_obj4 obj2 = self.bg_obj v11 = (obj_model1['v'] - obj_model1['v'].mean()) / ( obj_model1['v'].max() - obj_model1['v'].min()) v12 = (obj_model2['v'] - obj_model2['v'].mean()) / ( obj_model2['v'].max() - obj_model2['v'].min()) v13 = (obj_model3['v'] - obj_model3['v'].mean()) / ( obj_model3['v'].max() - obj_model3['v'].min()) # v14 = (obj_model4['v'] - obj_model4['v'].mean()) / (obj_model4['v'].max() - obj_model4['v'].min()) v2 = obj2['v'] # / (obj2['v'].max() - obj2['v'].min()) scale = (obj2['v'].max() - obj2['v'].min()) * 0.22 offset = np.array([[6.9, 6.9, 7.0], [20.4, 6.7, 6.7], [20.4, 6.7, 20.2], [7.0, 6.7, 20.4] ]) #+ 2 * np.random.rand(3) #[8.0, 5.0, 18.0], if self.opt.only_background: v = v2 f = obj2['f'] elif self.opt.only_foreground: v = v1 f = obj_model['f'] else: if self.opt.random_rotation: random_axis = np_normalize(np.random.rand(3)) random_angle = np.random.rand(1) * np.pi * 2 M = axis_angle_matrix(axis=random_axis, angle=random_angle) M[:3, 3] = offset[offset_id[0]] + 1.5 * np.random.randn(3) v11 = np.matmul(scale * v11, M.transpose(1, 0)[:3, :3]) + M[:3, 3] else: # random_axis = np_normalize(np.random.rand(3)) # random_angle = np.random.rand(1) * np.pi * 2 # M = axis_angle_matrix(axis=random_axis, angle=random_angle) # M[:3, 3] = offset[offset_id[0]]#+1.5*np.random.randn(3) # v11 = np.matmul(scale * v11, M.transpose(1, 0)[:3, :3]) + M[:3, 3] # # random_axis2 = np_normalize(np.random.rand(3)) # random_angle2 = np.random.rand(1) * np.pi * 2 # M2 = axis_angle_matrix(axis=random_axis2, angle=random_angle2) # M2[:3, 3] = offset[offset_id[2]]#+1.5*np.random.randn(3) # v13 = np.matmul(scale * v13, M2.transpose(1, 0)[:3, :3]) + M2[:3, 3] v11 = scale * v11 + offset[ offset_id[0]] + 1.5 * np.random.randn(3) v12 = scale * v12 + offset[ offset_id[1]] + 1.5 * np.random.randn(3) v13 = scale * v13 + offset[ offset_id[2]] + 1.5 * np.random.randn(3) # v14 = scale * v14 + offset[offset_id[3]] # v = np.concatenate((v11,v12,v13,v14, v2)) # f = np.concatenate((obj_model1['f'],obj_model2['f']+ v11.shape[0],obj_model3['f']+ v12.shape[0],obj_model4['f']+ v13.shape[0],obj2['f'] + v14.shape[0])) v = np.concatenate((v11, v12, v13, v2)) #import ipdb; ipdb.set_trace() f = np.concatenate( (obj_model1['f'], obj_model2['f'] + v11.shape[0], obj_model3['f'] + v12.shape[0] + v11.shape[0], obj2['f'] + v13.shape[0] + v12.shape[0] + v11.shape[0])) obj_model = {'v': v, 'f': f} if self.opt.use_mesh: # normalize the vertices v = obj_model['v'] axis_range = np.max(v, axis=0) - np.min(v, axis=0) v = (v - np.mean(v, axis=0)) / max( axis_range) # Normalize to make the largest spread 1 obj_model['v'] = v mesh = obj_to_triangle_spec(obj_model) meshes = { 'face': mesh['face'].astype(np.float32), 'normal': mesh['normal'].astype(np.float32) } sample = {'synset': 0, 'mesh': meshes} else: # Sample points from the 3D mesh v, vn = uniform_sample_mesh(obj_model, num_samples=self.opt.n_splats) # Normalize the vertices v = (v - np.mean(v, axis=0)) / (v.max() - v.min()) # Save the splats splats = { 'pos': v.astype(np.float32), 'normal': vn.astype(np.float32) } # Add model and synset to the output dictionary sample = {'synset': 0, 'splats': splats} # Transform if self.transform: sample = self.transform(sample) return sample
def render_sphere_world(out_dir, cam_pos, radius, width, height, fovy, focal_length, b_display=False): """ Generate z positions on a grid fixed inside the view frustum in the world coordinate system. Place the camera and choose the camera's field of view so that the side of the square touches the frustum. """ import copy print('render sphere') sampling_time = [] rendering_time = [] num_samples = width * height r = np.ones(num_samples) * radius large_scene = copy.deepcopy(SCENE_TEST) large_scene['camera']['viewport'] = [0, 0, width, height] large_scene['camera']['fovy'] = np.deg2rad(fovy) large_scene['camera']['focal_length'] = focal_length large_scene['objects']['disk']['radius'] = tch_var_f(r) large_scene['objects']['disk']['material_idx'] = tch_var_l(np.zeros(num_samples, dtype=int).tolist()) large_scene['materials']['albedo'] = tch_var_f([[0.6, 0.6, 0.6]]) large_scene['tonemap']['gamma'] = tch_var_f([1.0]) # Linear output x, y = np.meshgrid(np.linspace(-1, 1, width), np.linspace(-1, 1, height)) #z = np.sqrt(1 - np.min(np.stack((x ** 2 + y ** 2, np.ones_like(x)), axis=-1), axis=-1)) unit_disk_mask = (x ** 2 + y ** 2) <= 1 z = np.sqrt(1 - unit_disk_mask * (x ** 2 + y ** 2)) # Make a hemi-sphere bulging out of the xy-plane scene z[~unit_disk_mask] = 0 pos = np.stack((x.ravel(), y.ravel(), z.ravel()), axis=1) # Normals outside the sphere should be [0, 0, 1] x[~unit_disk_mask] = 0 y[~unit_disk_mask] = 0 z[~unit_disk_mask] = 1 normals = np_normalize(np.stack((x.ravel(), y.ravel(), z.ravel()), axis=1)) if b_display: plt.ion() plt.figure() plt.imshow(pos[..., 2].reshape((height, width))) plt.figure() plt.imshow(normals[..., 2].reshape((height, width))) large_scene['objects']['disk']['pos'] = tch_var_f(pos) large_scene['objects']['disk']['normal'] = tch_var_f(normals) large_scene['camera']['eye'] = tch_var_f(cam_pos) # main render run start_time = time() res = render(large_scene) rendering_time.append(time() - start_time) im = get_data(res['image']) im = np.uint8(255. * im) depth = get_data(res['depth']) depth[depth >= large_scene['camera']['far']] = depth.min() im_depth = np.uint8(255. * (depth - depth.min()) / (depth.max() - depth.min())) if b_display: plt.figure() plt.imshow(im, interpolation='none') plt.title('Image') plt.savefig(out_dir + '/fig_img_orig.png') plt.figure() plt.imshow(im_depth, interpolation='none') plt.title('Depth Image') plt.savefig(out_dir + '/fig_depth_orig.png') imsave(out_dir + '/img_orig.png', im) imsave(out_dir + '/depth_orig.png', im_depth) # hold matplotlib figure plt.ioff() plt.show()
def test_sphere_splat_render_along_ray(out_dir, cam_pos, width, height, fovy, focal_length, use_quartic, b_display=False): """ Create a sphere on a square as in render_sphere_world, and then convert to the camera's coordinate system and then render using render_splats_along_ray. """ import copy print('render sphere along ray') sampling_time = [] rendering_time = [] num_samples = width * height large_scene = copy.deepcopy(SCENE_TEST) large_scene['camera']['viewport'] = [0, 0, width, height] large_scene['camera']['eye'] = tch_var_f(cam_pos) large_scene['camera']['fovy'] = np.deg2rad(fovy) large_scene['camera']['focal_length'] = focal_length large_scene['objects']['disk']['material_idx'] = tch_var_l(np.zeros(num_samples, dtype=int).tolist()) large_scene['materials']['albedo'] = tch_var_f([[0.6, 0.6, 0.6]]) large_scene['tonemap']['gamma'] = tch_var_f([1.0]) # Linear output x, y = np.meshgrid(np.linspace(-1, 1, width), np.linspace(-1, 1, height)) #z = np.sqrt(1 - np.min(np.stack((x ** 2 + y ** 2, np.ones_like(x)), axis=-1), axis=-1)) unit_disk_mask = (x ** 2 + y ** 2) <= 1 z = np.sqrt(1 - unit_disk_mask * (x ** 2 + y ** 2)) # Make a hemi-sphere bulging out of the xy-plane scene z[~unit_disk_mask] = 0 pos = np.stack((x.ravel(), y.ravel(), z.ravel() - 5, np.ones(num_samples)), axis=1) # Normals outside the sphere should be [0, 0, 1] x[~unit_disk_mask] = 0 y[~unit_disk_mask] = 0 z[~unit_disk_mask] = 1 normals = np_normalize(np.stack((x.ravel(), y.ravel(), z.ravel(), np.zeros(num_samples)), axis=1)) if b_display: plt.ion() plt.figure() plt.subplot(131) plt.imshow(pos[..., 0].reshape((height, width))) plt.subplot(132) plt.imshow(pos[..., 1].reshape((height, width))) plt.subplot(133) plt.imshow(pos[..., 2].reshape((height, width))) plt.figure() plt.imshow(normals[..., 2].reshape((height, width))) ## Convert to the camera's coordinate system #Mcam = lookat(eye=large_scene['camera']['eye'], at=large_scene['camera']['at'], up=large_scene['camera']['up']) pos_CC = tch_var_f(pos) #torch.matmul(tch_var_f(pos), Mcam.transpose(1, 0)) large_scene['objects']['disk']['pos'] = pos_CC large_scene['objects']['disk']['normal'] = None # Estimate the normals tch_var_f(normals) # large_scene['camera']['eye'] = tch_var_f([-10., 0., 10.]) # large_scene['camera']['eye'] = tch_var_f([2., 0., 10.]) large_scene['camera']['eye'] = tch_var_f([-5., 0., 0.]) # main render run start_time = time() res = render_splats_along_ray(large_scene, use_quartic=use_quartic) rendering_time.append(time() - start_time) # Test cam_to_world res_world = cam_to_world(res['pos'].reshape(-1, 3), res['normal'].reshape(-1, 3), large_scene['camera']) im = get_data(res['image']) im = np.uint8(255. * im) depth = get_data(res['depth']) depth[depth >= large_scene['camera']['far']] = large_scene['camera']['far'] if b_display: plt.figure() plt.imshow(im, interpolation='none') plt.title('Image') plt.savefig(out_dir + '/fig_img_orig.png') plt.figure() plt.imshow(depth, interpolation='none') plt.title('Depth Image') #plt.savefig(out_dir + '/fig_depth_orig.png') plt.figure() pos_world = get_data(res_world['pos']) posx_world = pos_world[:, 0].reshape((im.shape[0], im.shape[1])) posy_world = pos_world[:, 1].reshape((im.shape[0], im.shape[1])) posz_world = pos_world[:, 2].reshape((im.shape[0], im.shape[1])) plt.subplot(131) plt.imshow(posx_world) plt.title('x_world') plt.subplot(132) plt.imshow(posy_world) plt.title('y_world') plt.subplot(133) plt.imshow(posz_world) plt.title('z_world') fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.scatter(pos_world[:, 0], pos_world[:, 1], pos_world[:, 2], s=1.3) ax.set_xlabel('x') ax.set_ylabel('y') plt.figure() pos_world = get_data(res['pos'].reshape(-1, 3)) posx_world = pos_world[:, 0].reshape((im.shape[0], im.shape[1])) posy_world = pos_world[:, 1].reshape((im.shape[0], im.shape[1])) posz_world = pos_world[:, 2].reshape((im.shape[0], im.shape[1])) plt.subplot(131) plt.imshow(posx_world) plt.title('x_CC') plt.subplot(132) plt.imshow(posy_world) plt.title('y_CC') plt.subplot(133) plt.imshow(posz_world) plt.title('z_CC') imsave(out_dir + '/img_orig.png', im) #imsave(out_dir + '/depth_orig.png', im_depth) # hold matplotlib figure plt.ioff() plt.show()
def test_sphere_splat_NDC(out_dir, cam_pos, width, height, fovy, focal_length, b_display=False): """ Create a sphere on a square as in render_sphere_world, and then convert to the camera's coordinate system and to NDC and then render using render_splat_NDC. """ import copy print('render sphere') sampling_time = [] rendering_time = [] num_samples = width * height large_scene = copy.deepcopy(SCENE_TEST) large_scene['camera']['viewport'] = [0, 0, width, height] large_scene['camera']['eye'] = tch_var_f(cam_pos) large_scene['camera']['fovy'] = np.deg2rad(fovy) large_scene['camera']['focal_length'] = focal_length large_scene['objects']['disk']['material_idx'] = tch_var_l(np.zeros(num_samples, dtype=int).tolist()) large_scene['materials']['albedo'] = tch_var_f([[0.6, 0.6, 0.6]]) large_scene['tonemap']['gamma'] = tch_var_f([1.0]) # Linear output x, y = np.meshgrid(np.linspace(-1, 1, width), np.linspace(-1, 1, height)) #z = np.sqrt(1 - np.min(np.stack((x ** 2 + y ** 2, np.ones_like(x)), axis=-1), axis=-1)) unit_disk_mask = (x ** 2 + y ** 2) <= 1 z = np.sqrt(1 - unit_disk_mask * (x ** 2 + y ** 2)) # Make a hemi-sphere bulging out of the xy-plane scene z[~unit_disk_mask] = 0 pos = np.stack((x.ravel(), y.ravel(), z.ravel(), np.ones(num_samples)), axis=1) # Normals outside the sphere should be [0, 0, 1] x[~unit_disk_mask] = 0 y[~unit_disk_mask] = 0 z[~unit_disk_mask] = 1 normals = np_normalize(np.stack((x.ravel(), y.ravel(), z.ravel(), np.zeros(num_samples)), axis=1)) if b_display: plt.ion() plt.figure() plt.imshow(pos[..., 2].reshape((height, width))) plt.figure() plt.imshow(normals[..., 2].reshape((height, width))) # Convert to the camera's coordinate system Mcam = lookat(eye=large_scene['camera']['eye'], at=large_scene['camera']['at'], up=large_scene['camera']['up']) Mproj = perspective(fovy=large_scene['camera']['fovy'], aspect=width/height, near=large_scene['camera']['near'], far=large_scene['camera']['far']) pos_CC = torch.matmul(tch_var_f(pos), Mcam.transpose(1, 0)) pos_NDC = torch.matmul(pos_CC, Mproj.transpose(1, 0)) large_scene['objects']['disk']['pos'] = pos_NDC / pos_NDC[..., 3][:, np.newaxis] large_scene['objects']['disk']['normal'] = tch_var_f(normals) # main render run start_time = time() res = render_splats_NDC(large_scene) rendering_time.append(time() - start_time) im = get_data(res['image']) im = np.uint8(255. * im) depth = get_data(res['depth']) depth[depth >= large_scene['camera']['far']] = depth.min() im_depth = np.uint8(255. * (depth - depth.min()) / (depth.max() - depth.min())) if b_display: plt.figure() plt.imshow(im, interpolation='none') plt.title('Image') plt.savefig(out_dir + '/fig_img_orig.png') plt.figure() plt.imshow(im_depth, interpolation='none') plt.title('Depth Image') plt.savefig(out_dir + '/fig_depth_orig.png') imsave(out_dir + '/img_orig.png', im) imsave(out_dir + '/depth_orig.png', im_depth) # hold matplotlib figure plt.ioff() plt.show()
def __getitem__(self, idx): """Get item.""" # Get object path obj_path = os.path.join(self.opt.root_dir, self.samples[idx]) if not self.loaded: self.fg_obj = load_model(obj_path) self.bg_obj = load_model(self.opt.bg_model) self.loaded = True obj_model = self.fg_obj obj2 = self.bg_obj v1 = (obj_model['v'] - obj_model['v'].mean()) / (obj_model['v'].max() - obj_model['v'].min()) v2 = obj2['v'] # / (obj2['v'].max() - obj2['v'].min()) scale = (obj2['v'].max() - obj2['v'].min()) * 0.4 offset = np.array([14.0, 8.0, 12.0]) #+ 2 * np.abs(np.random.randn(3)) if self.opt.only_background: v = v2 f = obj2['f'] elif self.opt.only_foreground: v = v1 f = obj_model['f'] else: if self.opt.random_rotation: random_axis = np_normalize(self.opt.axis) random_angle = np.random.rand(1) * np.pi * 2 M = axis_angle_matrix(axis=random_axis, angle=random_angle) M[:3, 3] = offset v1 = np.matmul(scale * v1, M.transpose(1, 0)[:3, :3]) + M[:3, 3] else: v1 = scale * v1 + offset v = np.concatenate((v1, v2)) f = np.concatenate((obj_model['f'], obj2['f'] + v1.shape[0])) obj_model = {'v': v, 'f': f} if self.opt.use_mesh: # normalize the vertices v = obj_model['v'] axis_range = np.max(v, axis=0) - np.min(v, axis=0) v = (v - np.mean(v, axis=0)) / max( axis_range) # Normalize to make the largest spread 1 obj_model['v'] = v mesh = obj_to_triangle_spec(obj_model) meshes = { 'face': mesh['face'].astype(np.float32), 'normal': mesh['normal'].astype(np.float32) } sample = {'synset': 0, 'mesh': meshes} else: # Sample points from the 3D mesh v, vn = uniform_sample_mesh(obj_model, num_samples=self.opt.n_splats) # Normalize the vertices v = (v - np.mean(v, axis=0)) / (v.max() - v.min()) # Save the splats splats = { 'pos': v.astype(np.float32), 'normal': vn.astype(np.float32) } # Add model and synset to the output dictionary sample = {'synset': 0, 'splats': splats} # Transform if self.transform: sample = self.transform(sample) return sample
def render_sphere(out_dir, cam_pos, radius, width, height, fovy, focal_length, num_views, std_z=0.01, std_normal=0.01, b_display=False): """ Randomly generate N samples on a surface and render them. The samples include position and normal, the radius is set to a constant. """ print('render sphere') sampling_time = [] rendering_time = [] num_samples = width * height r = np.ones(num_samples) * radius scene = copy.deepcopy(SCENE_BASIC) scene['camera']['viewport'] = [0, 0, width, height] scene['camera']['fovy'] = np.deg2rad(fovy) scene['camera']['focal_length'] = focal_length scene['objects']['disk']['radius'] = tch_var_f(r) scene['objects']['disk']['material_idx'] = tch_var_l( np.zeros(num_samples, dtype=int).tolist()) scene['materials']['albedo'] = tch_var_f([[0.6, 0.6, 0.6]]) scene['tonemap']['gamma'] = tch_var_f([1.0]) # Linear output x, y = np.meshgrid(np.linspace(-1, 1, width), np.linspace(-1, 1, height)) #z = np.sqrt(1 - np.min(np.stack((x ** 2 + y ** 2, np.ones_like(x)), axis=-1), axis=-1)) unit_disk_mask = (x**2 + y**2) <= 1 z = np.sqrt(1 - unit_disk_mask * (x**2 + y**2)) # Make a hemi-sphere bulging out of the xy-plane scene z[~unit_disk_mask] = 0 pos = np.stack((x.ravel(), y.ravel(), z.ravel()), axis=1) # Normals outside the sphere should be [0, 0, 1] x[~unit_disk_mask] = 0 y[~unit_disk_mask] = 0 z[~unit_disk_mask] = 1 normals = np_normalize(np.stack((x.ravel(), y.ravel(), z.ravel()), axis=1)) if b_display: plt.ion() plt.figure() plt.imshow(pos[..., 2].reshape((height, width))) plt.figure() plt.imshow(normals[..., 2].reshape((height, width))) scene['objects']['disk']['pos'] = tch_var_f(pos) scene['objects']['disk']['normal'] = tch_var_f(normals) scene['camera']['eye'] = tch_var_f(cam_pos) # main render run start_time = time() res = render(scene) rendering_time.append(time() - start_time) im = get_data(res['image']) depth = get_data(res['depth']) depth[depth >= scene['camera']['far']] = depth.min() im_depth = np.uint8(255. * (depth - depth.min()) / (depth.max() - depth.min())) if b_display: plt.figure() plt.imshow(im) plt.title('Image') plt.savefig(out_dir + '/fig_img_orig.png') plt.figure() plt.imshow(im_depth) plt.title('Depth Image') plt.savefig(out_dir + '/fig_depth_orig.png') imsave(out_dir + '/img_orig.png', im) imsave(out_dir + '/depth_orig.png', im_depth) # generate noisy data if b_display: h1 = plt.figure() h2 = plt.figure() noisy_pos = pos for view_idx in range(num_views): noisy_pos[..., 2] = pos[..., 2] + std_z * np.random.randn(num_samples) noisy_normals = np_normalize(normals + std_normal * np.random.randn(num_samples, 3)) scene['objects']['disk']['pos'] = tch_var_f(noisy_pos) scene['objects']['disk']['normal'] = tch_var_f(noisy_normals) scene['camera']['eye'] = tch_var_f(cam_pos) # main render run start_time = time() res = render(scene) rendering_time.append(time() - start_time) im = get_data(res['image']) depth = get_data(res['depth']) depth[depth >= scene['camera']['far']] = depth.min() im_depth = np.uint8(255. * (depth - depth.min()) / (depth.max() - depth.min())) suffix_str = '{:05d}'.format(view_idx) if b_display: plt.figure(h1.number) plt.imshow(im) plt.title('Image') plt.savefig(out_dir + '/fig_img_' + suffix_str + '.png') plt.figure(h2.number) plt.imshow(im_depth) plt.title('Depth Image') plt.savefig(out_dir + '/fig_depth_' + suffix_str + '.png') imsave(out_dir + '/img_' + suffix_str + '.png', im) imsave(out_dir + '/depth_' + suffix_str + '.png', im_depth) # hold matplotlib figure plt.ioff() plt.show()