def mvsnet_to_gipuma_dmb(in_path, out_path): '''convert mvsnet .pfm output to Gipuma .dmb format''' image, _ = read_pfm(in_path) write_gipuma_dmb(out_path, image) return
def probability_filter(dense_folder, prob_threshold): image_folder = os.path.join(dense_folder, 'images') # convert cameras image_names = os.listdir(image_folder) for image_name in image_names: image_prefix = os.path.splitext(image_name)[0] init_depth_map_path = os.path.join(dense_folder, "depth_est", image_prefix + '.pfm') prob_map_path = os.path.join(dense_folder, "confidence", image_prefix + '.pfm') out_depth_map_path = os.path.join(dense_folder, "depth_est", image_prefix + '_prob_filtered.pfm') depth_map, _ = read_pfm(init_depth_map_path) prob_map, _ = read_pfm(prob_map_path) depth_map[prob_map < prob_threshold] = 0 save_pfm(out_depth_map_path, depth_map)
def filter_depth(pair_folder, scan_folder, out_folder, plyfilename): # the pair file pair_file = os.path.join(pair_folder, "pair.txt") # for the final point cloud vertexs = [] vertex_colors = [] pair_data = read_pair_file(pair_file) nviews = len(pair_data) # for each reference view and the corresponding source views for ref_view, src_views in pair_data: # src_views = src_views[:args.num_view] # load the camera parameters ref_intrinsics, ref_extrinsics = read_camera_parameters( os.path.join(scan_folder, 'cams/{:0>8}_cam.txt'.format(ref_view))) # load the reference image ref_img = read_img( os.path.join(scan_folder, 'images/{:0>8}.jpg'.format(ref_view))) # load the estimated depth of the reference view ref_depth_est = read_pfm( os.path.join(out_folder, 'depth_est/{:0>8}.pfm'.format(ref_view)))[0] # load the photometric mask of the reference view confidence = read_pfm( os.path.join(out_folder, 'confidence/{:0>8}.pfm'.format(ref_view)))[0] photo_mask = confidence > args.conf all_srcview_depth_ests = [] all_srcview_x = [] all_srcview_y = [] all_srcview_geomask = [] # compute the geometric mask geo_mask_sum = 0 for src_view in src_views: # camera parameters of the source view src_intrinsics, src_extrinsics = read_camera_parameters( os.path.join(scan_folder, 'cams/{:0>8}_cam.txt'.format(src_view))) # the estimated depth of the source view src_depth_est = read_pfm( os.path.join(out_folder, 'depth_est/{:0>8}.pfm'.format(src_view)))[0] geo_mask, depth_reprojected, x2d_src, y2d_src = check_geometric_consistency( ref_depth_est, ref_intrinsics, ref_extrinsics, src_depth_est, src_intrinsics, src_extrinsics) geo_mask_sum += geo_mask.astype(np.int32) all_srcview_depth_ests.append(depth_reprojected) all_srcview_x.append(x2d_src) all_srcview_y.append(y2d_src) all_srcview_geomask.append(geo_mask) depth_est_averaged = (sum(all_srcview_depth_ests) + ref_depth_est) / (geo_mask_sum + 1) # at least 3 source views matched geo_mask = geo_mask_sum >= args.thres_view final_mask = np.logical_and(photo_mask, geo_mask) os.makedirs(os.path.join(out_folder, "mask"), exist_ok=True) save_mask( os.path.join(out_folder, "mask/{:0>8}_photo.png".format(ref_view)), photo_mask) save_mask( os.path.join(out_folder, "mask/{:0>8}_geo.png".format(ref_view)), geo_mask) save_mask( os.path.join(out_folder, "mask/{:0>8}_final.png".format(ref_view)), final_mask) print("processing {}, ref-view{:0>2}, photo/geo/final-mask:{}/{}/{}". format(scan_folder, ref_view, photo_mask.mean(), geo_mask.mean(), final_mask.mean())) if args.display: import cv2 cv2.imshow('ref_img', ref_img[:, :, ::-1]) cv2.imshow('ref_depth', ref_depth_est / 800) cv2.imshow('ref_depth * photo_mask', ref_depth_est * photo_mask.astype(np.float32) / 800) cv2.imshow('ref_depth * geo_mask', ref_depth_est * geo_mask.astype(np.float32) / 800) cv2.imshow('ref_depth * mask', ref_depth_est * final_mask.astype(np.float32) / 800) cv2.waitKey(0) height, width = depth_est_averaged.shape[:2] x, y = np.meshgrid(np.arange(0, width), np.arange(0, height)) # valid_points = np.logical_and(final_mask, ~used_mask[ref_view]) valid_points = final_mask print("valid_points", valid_points.mean()) x, y, depth = x[valid_points], y[valid_points], depth_est_averaged[ valid_points] #color = ref_img[1:-16:4, 1::4, :][valid_points] # hardcoded for DTU dataset if num_stage == 1: color = ref_img[1::4, 1::4, :][valid_points] elif num_stage == 2: color = ref_img[1::2, 1::2, :][valid_points] elif num_stage == 3: color = ref_img[valid_points] xyz_ref = np.matmul(np.linalg.inv(ref_intrinsics), np.vstack((x, y, np.ones_like(x))) * depth) xyz_world = np.matmul(np.linalg.inv(ref_extrinsics), np.vstack((xyz_ref, np.ones_like(x))))[:3] vertexs.append(xyz_world.transpose((1, 0))) vertex_colors.append((color * 255).astype(np.uint8)) # # set used_mask[ref_view] # used_mask[ref_view][...] = True # for idx, src_view in enumerate(src_views): # src_mask = np.logical_and(final_mask, all_srcview_geomask[idx]) # src_y = all_srcview_y[idx].astype(np.int) # src_x = all_srcview_x[idx].astype(np.int) # used_mask[src_view][src_y[src_mask], src_x[src_mask]] = True vertexs = np.concatenate(vertexs, axis=0) vertex_colors = np.concatenate(vertex_colors, axis=0) vertexs = np.array([tuple(v) for v in vertexs], dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) vertex_colors = np.array([tuple(v) for v in vertex_colors], dtype=[('red', 'u1'), ('green', 'u1'), ('blue', 'u1')]) vertex_all = np.empty(len(vertexs), vertexs.dtype.descr + vertex_colors.dtype.descr) for prop in vertexs.dtype.names: vertex_all[prop] = vertexs[prop] for prop in vertex_colors.dtype.names: vertex_all[prop] = vertex_colors[prop] el = PlyElement.describe(vertex_all, 'vertex') PlyData([el]).write(plyfilename) print("saving the final model to", plyfilename)
import numpy as np import cv2 import argparse import matplotlib.pyplot as plt from datasets.data_io import read_pfm, save_pfm import pdb if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('depth_path') args = parser.parse_args() depth_path = args.depth_path if depth_path.endswith('npy'): depth_image = np.load(depth_path) depth_image = np.squeeze(depth_image) print('value range: ', depth_image.min(), depth_image.max()) plt.imshow(depth_image, 'rainbow') plt.show() elif depth_path.endswith('pfm'): depth_image = read_pfm(depth_path) ma = np.ma.masked_equal(depth_image[0], 0.0, copy=False) print('value range: ', ma.min(), ma.max()) plt.imshow(depth_image[0], 'rainbow') plt.show() else: depth_image = cv2.imread(depth_path) ma = np.ma.masked_equal(depth_image, 0.0, copy=False) print('value range: ', ma.min(), ma.max()) plt.imshow(depth_image) plt.show()
def filter_depth(scan_folder, out_folder, plyfilename, geo_pixel_thres, geo_depth_thres, photo_thres, img_wh): # the pair file pair_file = os.path.join(scan_folder, "pair.txt") # for the final point cloud vertexs = [] vertex_colors = [] pair_data = read_pair_file(pair_file) nviews = len(pair_data) original_w = 1600 original_h = 1200 # for each reference view and the corresponding source views for ref_view, src_views in pair_data: # load the camera parameters ref_intrinsics, ref_extrinsics = read_camera_parameters( os.path.join(scan_folder, 'cams_1/{:0>8}_cam.txt'.format(ref_view))) ref_intrinsics[0] *= img_wh[0]/original_w ref_intrinsics[1] *= img_wh[1]/original_h # load the reference image ref_img = read_img(os.path.join(scan_folder, 'images/{:0>8}.jpg'.format(ref_view)), img_wh) # load the estimated depth of the reference view ref_depth_est = read_pfm(os.path.join(out_folder, 'depth_est/{:0>8}.pfm'.format(ref_view)))[0] ref_depth_est = np.squeeze(ref_depth_est, 2) # load the photometric mask of the reference view confidence = read_pfm(os.path.join(out_folder, 'confidence/{:0>8}.pfm'.format(ref_view)))[0] photo_mask = confidence > photo_thres photo_mask = np.squeeze(photo_mask, 2) all_srcview_depth_ests = [] # compute the geometric mask geo_mask_sum = 0 for src_view in src_views: # camera parameters of the source view src_intrinsics, src_extrinsics = read_camera_parameters( os.path.join(scan_folder, 'cams_1/{:0>8}_cam.txt'.format(src_view))) src_intrinsics[0] *= img_wh[0]/original_w src_intrinsics[1] *= img_wh[1]/original_h # the estimated depth of the source view src_depth_est = read_pfm(os.path.join(out_folder, 'depth_est/{:0>8}.pfm'.format(src_view)))[0] geo_mask, depth_reprojected, x2d_src, y2d_src = check_geometric_consistency(ref_depth_est, ref_intrinsics, ref_extrinsics, src_depth_est, src_intrinsics, src_extrinsics, geo_pixel_thres, geo_depth_thres) geo_mask_sum += geo_mask.astype(np.int32) all_srcview_depth_ests.append(depth_reprojected) depth_est_averaged = (sum(all_srcview_depth_ests) + ref_depth_est) / (geo_mask_sum + 1) # at least 3 source views matched # large threshold, high accuracy, low completeness geo_mask = geo_mask_sum >= 3 final_mask = np.logical_and(photo_mask, geo_mask) os.makedirs(os.path.join(out_folder, "mask"), exist_ok=True) save_mask(os.path.join(out_folder, "mask/{:0>8}_photo.png".format(ref_view)), photo_mask) save_mask(os.path.join(out_folder, "mask/{:0>8}_geo.png".format(ref_view)), geo_mask) save_mask(os.path.join(out_folder, "mask/{:0>8}_final.png".format(ref_view)), final_mask) os.makedirs(os.path.join(out_folder, "depth_img"), exist_ok=True) print("processing {}, ref-view{:0>2}, geo_mask:{:3f} photo_mask:{:3f} final_mask: {:3f}".format(scan_folder, ref_view, geo_mask.mean(), photo_mask.mean(), final_mask.mean())) if args.display: import cv2 cv2.imshow('ref_img', ref_img[:, :, ::-1]) cv2.imshow('ref_depth', ref_depth_est / 800) cv2.imshow('ref_depth * photo_mask', ref_depth_est * photo_mask.astype(np.float32) / 800) cv2.imshow('ref_depth * geo_mask', ref_depth_est * geo_mask.astype(np.float32) / 800) cv2.imshow('ref_depth * mask', ref_depth_est * final_mask.astype(np.float32) / 800) cv2.waitKey(1) height, width = depth_est_averaged.shape[:2] x, y = np.meshgrid(np.arange(0, width), np.arange(0, height)) valid_points = final_mask # print("valid_points", valid_points.mean()) x, y, depth = x[valid_points], y[valid_points], depth_est_averaged[valid_points] color = ref_img[valid_points] xyz_ref = np.matmul( np.linalg.inv(ref_intrinsics), np.vstack((x, y, np.ones_like(x))) * depth) xyz_world = np.matmul(np.linalg.inv(ref_extrinsics), np.vstack((xyz_ref, np.ones_like(x))))[:3] vertexs.append(xyz_world.transpose((1, 0))) vertex_colors.append((color * 255).astype(np.uint8)) vertexs = np.concatenate(vertexs, axis=0) vertex_colors = np.concatenate(vertex_colors, axis=0) vertexs = np.array([tuple(v) for v in vertexs], dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) vertex_colors = np.array([tuple(v) for v in vertex_colors], dtype=[('red', 'u1'), ('green', 'u1'), ('blue', 'u1')]) vertex_all = np.empty(len(vertexs), vertexs.dtype.descr + vertex_colors.dtype.descr) for prop in vertexs.dtype.names: vertex_all[prop] = vertexs[prop] for prop in vertex_colors.dtype.names: vertex_all[prop] = vertex_colors[prop] el = PlyElement.describe(vertex_all, 'vertex') PlyData([el]).write(plyfilename) print("saving the final model to", plyfilename)
def filter_depth(scan_folder, out_folder, plyfilename): # the pair file pair_file = os.path.join(scan_folder, "pair.txt") # for the final point cloud vertexs = [] vertex_colors = [] pair_data = read_pair_file(pair_file) score_data = read_score_file(pair_file) nviews = len(pair_data) # TODO: hardcode size # used_mask = [np.zeros([296, 400], dtype=np.bool) for _ in range(nviews)] # for each reference view and the corresponding source views ct2 = -1 for ref_view, src_views in pair_data: ct2 += 1 # load the camera parameters ref_intrinsics, ref_extrinsics = read_camera_parameters( os.path.join(scan_folder, 'cams/{:0>8}.txt'.format(ref_view))) # load the reference image ref_img = read_img( os.path.join(scan_folder, 'images/{:0>8}.jpg'.format(ref_view))) # load the estimated depth of the reference view ref_depth_est = read_pfm( os.path.join(out_folder, 'depth_est_0/{:0>8}.pfm'.format(ref_view)))[0] import cv2 #ref_img=cv2.pyrDown(ref_img) #ref_depth_est=cv2.pyrUp(ref_depth_est) #ref_depth_est=cv2.pyrDown(ref_depth_est) # load the photometric mask of the reference view confidence = read_pfm( os.path.join(out_folder, 'confidence_0/{:0>8}.pfm'.format(ref_view)))[0] #confidence=cv2.pyrUp(confidence) #confidence=cv2.pyrDown(confidence) #ref_img=cv2.pyrDown(ref_img) #ref_depth_est=cv2.resize(ref_depth_est,(ref_img.shape[1],ref_img.shape[0])) #confidence=cv2.resize(confidence,(ref_img.shape[1],ref_img.shape[0])) photo_mask = confidence > 0.3 # photo_mask = confidence>=0 # photo_mask = confidence > confidence.mean() # ref_depth_est=ref_depth_est * photo_mask all_srcview_depth_ests = [] all_srcview_x = [] all_srcview_y = [] all_srcview_geomask = [] # compute the geometric mask geo_mask_sum = 0 geo_mask_sums = [] ct = 0 for src_view in src_views: ct = ct + 1 # camera parameters of the source view src_intrinsics, src_extrinsics = read_camera_parameters( os.path.join(scan_folder, 'cams/{:0>8}.txt'.format(src_view))) # the estimated depth of the source view src_depth_est = read_pfm( os.path.join(out_folder, 'depth_est_0/{:0>8}.pfm'.format(src_view)))[0] #src_depth_est=cv2.resize(src_depth_est,(ref_img.shape[1],ref_img.shape[0])) #src_depth_est=cv2.pyrUp(src_depth_est) #src_depth_est=cv2.pyrDown(src_depth_est) src_confidence = read_pfm( os.path.join(out_folder, 'confidence_0/{:0>8}.pfm'.format(src_view)))[0] # src_mask=src_confidence>0.1 # src_mask=src_confidence>src_confidence.mean() # src_depth_est=src_depth_est*src_mask masks, geo_mask, depth_reprojected, x2d_src, y2d_src = check_geometric_consistency( ref_depth_est, ref_intrinsics, ref_extrinsics, src_depth_est, src_intrinsics, src_extrinsics) if (ct == 1): for i in range(2, 11): geo_mask_sums.append(masks[i - 2].astype(np.int32)) else: for i in range(2, 11): geo_mask_sums[i - 2] += masks[i - 2].astype(np.int32) geo_mask_sum += geo_mask.astype(np.int32) all_srcview_depth_ests.append(depth_reprojected) # all_srcview_x.append(x2d_src) # all_srcview_y.append(y2d_src) # all_srcview_geomask.append(geo_mask) geo_mask = geo_mask_sum >= 10 for i in range(2, 11): geo_mask = np.logical_or(geo_mask, geo_mask_sums[i - 2] >= i) depth_est_averaged = (sum(all_srcview_depth_ests) + ref_depth_est) / (geo_mask_sum + 1) if (not isinstance(geo_mask, bool)): final_mask = np.logical_and(photo_mask, geo_mask) os.makedirs(os.path.join(out_folder, "mask"), exist_ok=True) save_mask( os.path.join(out_folder, "mask/{:0>8}_photo.png".format(ref_view)), photo_mask) save_mask( os.path.join(out_folder, "mask/{:0>8}_geo.png".format(ref_view)), geo_mask) save_mask( os.path.join(out_folder, "mask/{:0>8}_final.png".format(ref_view)), final_mask) print( "processing {}, ref-view{:0>2}, photo/geo/final-mask:{}/{}/{}". format(scan_folder, ref_view, photo_mask.mean(), geo_mask.mean(), final_mask.mean())) if args.display: import cv2 cv2.imshow('ref_img', ref_img[:, :, ::-1]) cv2.imshow('ref_depth', ref_depth_est / 800) cv2.imshow('ref_depth * photo_mask', ref_depth_est * photo_mask.astype(np.float32) / 800) cv2.imshow('ref_depth * geo_mask', ref_depth_est * geo_mask.astype(np.float32) / 800) cv2.imshow('ref_depth * mask', ref_depth_est * final_mask.astype(np.float32) / 800) cv2.waitKey(0) height, width = depth_est_averaged.shape[:2] x, y = np.meshgrid(np.arange(0, width), np.arange(0, height)) # valid_points = np.logical_and(final_mask, ~used_mask[ref_view]) valid_points = final_mask print("valid_points", valid_points.mean()) x, y, depth = x[valid_points], y[valid_points], depth_est_averaged[ valid_points] color = ref_img[:, :, :][valid_points] # hardcoded for DTU dataset xyz_ref = np.matmul(np.linalg.inv(ref_intrinsics), np.vstack((x, y, np.ones_like(x))) * depth) xyz_world = np.matmul(np.linalg.inv(ref_extrinsics), np.vstack((xyz_ref, np.ones_like(x))))[:3] vertexs.append(xyz_world.transpose((1, 0))) vertex_colors.append((color * 255).astype(np.uint8)) # # set used_mask[ref_view] # used_mask[ref_view][...] = True # for idx, src_view in enumerate(src_views): # src_mask = np.logical_and(final_mask, all_srcview_geomask[idx]) # src_y = all_srcview_y[idx].astype(np.int) # src_x = all_srcview_x[idx].astype(np.int) # used_mask[src_view][src_y[src_mask], src_x[src_mask]] = True vertexs = np.concatenate(vertexs, axis=0) vertex_colors = np.concatenate(vertex_colors, axis=0) vertexs = np.array([tuple(v) for v in vertexs], dtype=[('x', 'f4'), ('y', 'f4'), ('z', 'f4')]) vertex_colors = np.array([tuple(v) for v in vertex_colors], dtype=[('red', 'u1'), ('green', 'u1'), ('blue', 'u1')]) vertex_all = np.empty(len(vertexs), vertexs.dtype.descr + vertex_colors.dtype.descr) for prop in vertexs.dtype.names: vertex_all[prop] = vertexs[prop] for prop in vertex_colors.dtype.names: vertex_all[prop] = vertex_colors[prop] el = PlyElement.describe(vertex_all, 'vertex') PlyData([el]).write(plyfilename) print("saving the final model to", plyfilename)
def multi_scale(depth_folder_down4, depth_folder_down8, depth_folder_down16, final_folder, test_list): depth_folder = depth_folder_down4 conf_folder = depth_folder output_folder = depth_folder depth2_folder = depth_folder_down8 conf2_folder = depth2_folder mask2_folder = depth2_folder depth3_folder = depth_folder_down16 conf3_folder = depth3_folder mask3_folder = depth3_folder if not os.path.exists(final_folder): os.mkdir(final_folder) file = open(test_list) scans = file.readlines() scans = [line.rstrip() for line in scans] sum = 0 ct = 0 shape_all = 0 for scan in scans: ct = ct + 1 depth_t = os.path.join(depth_folder, scan) depth2_t = os.path.join(depth2_folder, scan) depth3_t = os.path.join(depth3_folder, scan) conf_t = os.path.join(conf_folder, scan) conf2_t = os.path.join(conf2_folder, scan) conf3_t = os.path.join(conf3_folder, scan) mask2_folder_t = os.path.join(mask2_folder, scan) mask2_folder_t = os.path.join(mask2_folder_t, 'mask') mask3_folder_t = os.path.join(mask3_folder, scan) mask3_folder_t = os.path.join(mask3_folder_t, 'mask') output_t = os.path.join(output_folder, 'output') final_t = os.path.join(final_folder, scan) if not os.path.exists(final_t): os.mkdir(final_t) if not os.path.exists(output_t): os.mkdir(output_t) output_t = os.path.join(output_t, scan) if not os.path.exists(output_t): os.mkdir(output_t) sum_t = 0 shape = 0 for i in range(49): print('process depth ' + str(i)) depth_s = os.path.join(depth_t, 'depth_est') final_d = os.path.join(final_t, 'depth_est') if not os.path.exists(final_d): os.mkdir(final_d) final_c = os.path.join(final_t, 'confidence') if not os.path.exists(final_c): os.mkdir(final_c) depth_s = os.path.join(depth_s, '%08d.pfm' % i) depth2_s = os.path.join(depth2_t, 'depth_est') depth2_s = os.path.join(depth2_s, '%08d.pfm' % i) mask2_s = os.path.join(mask2_folder_t, '%08d_final.png' % i) depth3_s = os.path.join(depth3_t, 'depth_est') depth3_s = os.path.join(depth3_s, '%08d.pfm' % i) mask3_s = os.path.join(mask3_folder_t, '%08d_final.png' % i) conf_s = os.path.join(conf_t, 'confidence') conf_s = os.path.join(conf_s, '%08d.pfm' % i) final_d_s = os.path.join(final_d, '%08d.pfm' % i) final_c_s = os.path.join(final_c, '%08d.pfm' % i) conf2_s = os.path.join(conf2_t, 'confidence') conf2_s = os.path.join(conf2_s, '%08d.pfm' % i) conf3_s = os.path.join(conf3_t, 'confidence') conf3_s = os.path.join(conf3_s, '%08d.pfm' % i) output_s = os.path.join(output_t, 'error_map_%04d.png' % i) mask2_img_t = cv2.imread(mask2_s) mask2_img_t = cv2.cvtColor(mask2_img_t, cv2.COLOR_BGR2GRAY) mask2_arr = np.array(mask2_img_t) mask2_t = mask2_arr > 0 mask3_img_t = cv2.imread(mask3_s) mask3_img_t = cv2.cvtColor(mask3_img_t, cv2.COLOR_BGR2GRAY) mask3_arr = np.array(mask3_img_t) mask3_arr = cv2.pyrUp(mask3_arr) depth = read_pfm(depth_s)[0] depth2 = read_pfm(depth2_s)[0] depth3 = read_pfm(depth3_s)[0] depth3 = cv2.pyrUp(depth3) confidence = read_pfm(conf_s)[0] confidence2 = read_pfm(conf2_s)[0] confidence3 = read_pfm(conf3_s)[0] confidence3 = cv2.pyrUp(confidence3) depth2 = cv2.pyrUp(depth2) zero_depth = np.zeros([8, 400]) depth2 = np.r_[depth2, zero_depth] confidence2 = cv2.pyrUp(confidence2) zero_conf = np.zeros([8, 400]) confidence2 = np.r_[confidence2, zero_conf] mask2_arr = cv2.pyrUp(mask2_arr) zero_depth = np.zeros([8, 400]) mask2_arr = np.r_[mask2_arr, zero_depth] mask3 = (0.9 < confidence2) & (confidence < 0.5) & (mask2_arr > 0) depth[mask3] = depth2[mask3] confidence[mask3] = confidence2[mask3] save_pfm(final_d_s, depth) save_pfm(final_c_s, confidence)