def compute_projection(self, inputs, outputs, is_training): cfg = self.cfg() all_points = outputs['all_points'] all_rgb = outputs['all_rgb'] if cfg.predict_pose: camera_pose = outputs['poses'] else: if cfg.pose_quaternion: camera_pose = inputs['camera_quaternion'] else: camera_pose = inputs['matrices'] if is_training and cfg.pc_point_dropout != 1: dropout_prob = self.get_dropout_keep_prob() if is_training: tf.contrib.summary.scalar("meta/pc_point_dropout_prob", dropout_prob) all_points, all_rgb = pc_point_dropout(all_points, all_rgb, dropout_prob) if cfg.pc_fast: predicted_translation = outputs[ "predicted_translation"] if cfg.predict_translation else None proj_out = pointcloud_project_fast( cfg, all_points, camera_pose, predicted_translation, all_rgb, self.gauss_kernel(), scaling_factor=outputs['all_scaling_factors'], focal_length=outputs['all_focal_length']) proj = proj_out["proj"] outputs["projs_rgb"] = proj_out["proj_rgb"] outputs["drc_probs"] = proj_out["drc_probs"] outputs["projs_depth"] = proj_out["proj_depth"] else: proj, voxels = pointcloud_project(cfg, all_points, camera_pose, self.gauss_sigma()) outputs["projs_rgb"] = None outputs["projs_depth"] = None outputs['projs'] = proj batch_size = outputs['points_1'].shape[0] outputs['projs_1'] = proj[0:batch_size, :, :, :] return outputs
def model_student(inputs, model): cfg = model.cfg() outputs = model.model_predict(inputs, is_training=False, reuse=True, predict_for_all=False) points = outputs["points_1"] camera_pose = outputs["pose_student"] rgb = None transl = outputs[ "predicted_translation"] if cfg.predict_translation else None proj_out = pointcloud_project_fast(model.cfg(), points, camera_pose, transl, rgb, model.gauss_kernel()) proj = proj_out["proj_depth"] return proj, camera_pose
def main(_): cfg = app_config config = tf.ConfigProto(device_count={'GPU': 1}) sess = tf.Session(config=config) print("vox size", cfg.vox_size) tfrecords_filename = "/home/eldar/src/3dshape/ptn/train_data/03001627_val.tfrecords" synth_set = cfg.synth_set data_split = cfg.eval_split tfrecords_filename = "{}/{}_{}.tfrecords".format(cfg.inp_dir, synth_set, data_split) # tfrecords_filename = "/home/eldar/src/3dshape/ptn/train_data_drc_2/03001627_val.tfrecords" record_iterator = tf.python_io.tf_record_iterator( path=tfrecords_filename, options=tf_record_options(cfg)) src_data_dir = "/home/eldar/src/3dshape/cachedir/blenderRender_128x128_24/03001627/" shapenet_dir = "/home/eldar/data/ShapeNetCore.v1/03001627/" surface_gt_root = "/home/eldar/storage/3dshape/surface/" surface_gt_root = "/BS/eldar-3dshape/work/dataset/shapenet_surface_gt/orig_coord" vis_voxels = True vis_images = False vis_projections = True num_points = 8000 has_rgb = cfg.pc_rgb num_views = cfg.num_views image_size = cfg.image_size vox_size = cfg.vox_size vis_size = cfg.vis_size sigma_rel = cfg.pc_relative_sigma sigma = sigma_rel / vox_size surface_gt_path = os.path.join(surface_gt_root, cfg.synth_set) model = model_pc.ModelPointCloud(cfg) input_pc = tf.placeholder(dtype=tf.float32, shape=[1, num_points, 3]) input_rgb = tf.placeholder(dtype=tf.float32, shape=[1, num_points, 3 ]) if has_rgb else None pc_voxels = pointcloud2voxels(cfg, input_pc, sigma) pc_voxels_fast, _ = pointcloud2voxels3d_fast(cfg, input_pc, None) pc_voxels_fast = tf.transpose(pc_voxels_fast, [0, 2, 1, 3]) transform_matrix_pc = tf.placeholder(tf.float32, [1, 4, 4]) transform_quaternion = tf.placeholder(tf.float32, [1, 4]) if cfg.pose_quaternion: cam_transform = transform_quaternion else: cam_transform = transform_matrix_pc if not cfg.pc_fast: proj_of_pc, voxels_tr = pointcloud_project(cfg, input_pc, cam_transform, sigma) translation = tf.placeholder(shape=[1, 3], dtype=tf.float32) gauss_kernel = smoothing_kernel(cfg, sigma_rel) out = pointcloud_project_fast(cfg, input_pc, cam_transform, translation, input_rgb, gauss_kernel) proj_of_pc_fast = out["proj"] tr_pc_fast = out["tr_pc"] proj_rgb = out["proj_rgb"] proj_depth = out["proj_depth"] img_2d_input = tf.placeholder(tf.float32, [1, cfg.vis_size, cfg.vis_size, 1]) img_2d_smoothed = gauss_smoothen_image(cfg, img_2d_input, sigma_rel) # Initialize session sess.run(tf.initialize_all_variables()) k = 0 for string_record in record_iterator: example = tf.train.Example() example.ParseFromString(string_record) byte_list = example.features.feature['name'].bytes_list.value[0] model_name = str(byte_list) print("model_name", k, model_name) #if model_name != "a9e72050a2964ba31030665054ebb2a9": # continue float_list = example.features.feature['mask'].float_list.value masks_1d = np.array(float_list) masks = masks_1d.reshape((num_views, image_size, image_size, -1)) float_list = example.features.feature['image'].float_list.value images_1d = np.array(float_list) images = images_1d.reshape((num_views, image_size, image_size, -1)) stuff = scipy.io.loadmat("{}/{}.mat".format(surface_gt_path, model_name)) orig_points = stuff["points"] xyz_s = subsample_points(orig_points, num_points) rgb_s = pc_colors(xyz_s, color_axis=2) #vis_pc(xyz_s, color_axis=2) if False: xyz_s = xyz_s[:, [0, 2, 1]] # rgb_s = np.zeros_like(xyz_s) # rgb_s[:, 0] = 1.0 if 'depth' in list(example.features.feature.keys()): float_list = example.features.feature['depth'].float_list.value depths_1d = np.array(float_list) depths = depths_1d.reshape((num_views, image_size, image_size, -1)) if False: float_list = example.features.feature['vox'].float_list.value vox_1d = np.array(float_list) vox_data_size = int(round(vox_1d.shape[0]**(1.0 / 3.0))) voxels = vox_1d.reshape( (vox_data_size, vox_data_size, vox_data_size)) voxels1 = vox_1d.reshape( (1, vox_data_size, vox_data_size, vox_data_size, 1)) #xyz_s = sample_points_from_voxels(voxels, num_points=num_points) #xyz_s = xyz_s.astype(np.float32) if cfg.saved_camera: float_list = example.features.feature['extrinsic'].float_list.value cam_1d = np.array(float_list) cameras = cam_1d.reshape((num_views, 4, 4)) float_list = example.features.feature['cam_pos'].float_list.value cam_pos_1d = np.array(float_list) cam_pos = cam_pos_1d.reshape((num_views, 3)) #vis_pc(np.squeeze(xyz_s), 2) #continue xyz_s = np.expand_dims(xyz_s, axis=0) rgb_s = np.expand_dims(rgb_s, axis=0) if False: start = time.time() voxels_new = sess.run(pc_voxels, feed_dict={input_pc: xyz_s}) voxels_new_fast = sess.run(pc_voxels_fast, feed_dict={input_pc: xyz_s}) end = time.time() #print(end - start) #visualise_voxels(cfg, voxels) #vis_pc(xyz_s, 1) visualise_voxels(cfg, voxels_new, vis_axis=0) visualise_voxels(cfg, voxels_new_fast, vis_axis=0) if vis_projections: subplot_height = 4 subplot_width = 4 num_plots = subplot_width * subplot_height grid = np.empty((subplot_height, subplot_width), dtype=object) start = time.time() for j in range(num_plots): plot_j = j // subplot_width plot_i = j % subplot_width if j % 2 == 0: view_idx = j // 2 orig_img = images if has_rgb else masks curr_img = np.squeeze(orig_img[view_idx, :, :, :]) else: view_idx = j // 2 extrinsic = cameras[view_idx, :, :] extrinsic = camera_from_blender(extrinsic) intrinsic = intrinsic_matrix(cfg, dims=4) full_cam_matrix = np.matmul(intrinsic, extrinsic) full_cam_matrix = np.expand_dims(full_cam_matrix, axis=0) pos = cam_pos[view_idx, :] quat = quaternion_from_campos(pos) #matr = rot_matrix_from_q(quat) quat_np = np.expand_dims(quat, axis=0) trans_np = np.zeros((1, 3)) if view_idx % 4 == 0: trans_np[0, 1] = -0.1 elif view_idx % 4 == 1: trans_np[0, 1] = 0.1 elif view_idx % 4 == 2: trans_np[0, 2] = -0.1 else: trans_np[0, 2] = 0.1 trans_np[0, 0] = 0.2 trans_np = np.zeros((1, 3)) if cfg.pc_fast: if has_rgb: (proj_xyz_np, tr_pc_fast_np, proj_rgb_np) = sess.run( [proj_of_pc_fast, tr_pc_fast, proj_rgb], feed_dict={ input_pc: xyz_s, input_rgb: rgb_s, transform_matrix_pc: full_cam_matrix, transform_quaternion: quat_np }) else: (proj_xyz_np, tr_pc_fast_np, proj_depth_np, tr_voxels_np) = sess.run( [ proj_of_pc_fast, tr_pc_fast, proj_depth, out["voxels"] ], feed_dict={ input_pc: xyz_s, transform_matrix_pc: full_cam_matrix, transform_quaternion: quat_np, translation: trans_np }) else: (proj_xyz_np, tr_voxels_np) = sess.run([proj_of_pc, voxels_tr], feed_dict={ input_pc: xyz_s, transform_matrix_pc: full_cam_matrix }) print(tr_voxels_np.shape) print(np.histogram(tr_voxels_np.flatten())) print("sum", np.sum(tr_voxels_np.flatten())) #plt.hist(tr_voxels_np.flatten(), bins=10) #plt.show() proj = proj_xyz_np proj = np.squeeze(proj) if has_rgb: proj_rgb_np = np.squeeze(proj_rgb_np) curr_img = proj_rgb_np if has_rgb else proj curr_img = np.clip(curr_img, 0.0, 1.0) curr_img = imresize(curr_img, (vis_size, vis_size), order=3) if cfg.pc_gauss_filter_gt: if j % 2 == 0: img_prc = np.reshape(curr_img, (1, vis_size, vis_size, 1)) img_smoothed_np = sess.run( img_2d_smoothed, feed_dict={img_2d_input: img_prc}) curr_img = np.squeeze(img_smoothed_np) if curr_img.shape[-1] != 3: curr_img = 255 - np.clip(curr_img * 255, 0, 255).astype(dtype=np.uint8) grid[plot_j, plot_i] = curr_img end = time.time() print("time:", end - start) grid_merged = merge_grid(cfg, grid) plt.imshow(grid_merged) plt.show() k += 1