def test_general(dt_cor_id, gt_cor_id, w, h, losses): dt_floor_coor = dt_cor_id[1::2] dt_ceil_coor = dt_cor_id[0::2] gt_floor_coor = gt_cor_id[1::2] gt_ceil_coor = gt_cor_id[0::2] assert (dt_floor_coor[:, 0] != dt_ceil_coor[:, 0]).sum() == 0 assert (gt_floor_coor[:, 0] != gt_ceil_coor[:, 0]).sum() == 0 # Eval 3d IoU and height error(in meter) N = len(dt_floor_coor) ch = -1.6 dt_floor_xy = post_proc.np_coor2xy(dt_floor_coor, ch, 1024, 512, floorW=1, floorH=1) gt_floor_xy = post_proc.np_coor2xy(gt_floor_coor, ch, 1024, 512, floorW=1, floorH=1) dt_poly = Polygon(dt_floor_xy) gt_poly = Polygon(gt_floor_xy) if not gt_poly.is_valid: print('Skip ground truth invalid (%s)' % gt_path) return area_dt = dt_poly.area area_gt = gt_poly.area area_inter = dt_poly.intersection(gt_poly).area iou2d = area_inter / (area_gt + area_dt - area_inter) cch_dt = post_proc.get_z1(dt_floor_coor[:, 1], dt_ceil_coor[:, 1], ch, 512) cch_gt = post_proc.get_z1(gt_floor_coor[:, 1], gt_ceil_coor[:, 1], ch, 512) h_dt = abs(cch_dt.mean() - ch) h_gt = abs(cch_gt.mean() - ch) area3d_inter = area_inter * min(h_dt, h_gt) area3d_pred = area_dt * h_dt area3d_gt = area_gt * h_gt iou3d = area3d_inter / (area3d_pred + area3d_gt - area3d_inter) # Add a result n_corners = len(gt_floor_coor) if n_corners % 2 == 1: n_corners = 'odd' elif n_corners < 10: n_corners = str(n_corners) else: n_corners = '10+' losses[n_corners]['2DIoU'].append(iou2d) losses[n_corners]['3DIoU'].append(iou3d) losses['overall']['2DIoU'].append(iou2d) losses['overall']['3DIoU'].append(iou3d)
def test_general(dt_cor_id, gt_cor_id, w, h, losses): dt_floor_coor = dt_cor_id[1::2] dt_ceil_coor = dt_cor_id[0::2] gt_floor_coor = gt_cor_id[1::2] gt_ceil_coor = gt_cor_id[0::2] assert (dt_floor_coor[:, 0] != dt_ceil_coor[:, 0]).sum() == 0 assert (gt_floor_coor[:, 0] != gt_ceil_coor[:, 0]).sum() == 0 # Eval 3d IoU and height error(in meter) N = len(dt_floor_coor) ch = -1.6 dt_floor_xy = post_proc.np_coor2xy(dt_floor_coor, ch, 1024, 512, floorW=1, floorH=1) gt_floor_xy = post_proc.np_coor2xy(gt_floor_coor, ch, 1024, 512, floorW=1, floorH=1) dt_poly = Polygon(dt_floor_xy) gt_poly = Polygon(gt_floor_xy) area_dt = dt_poly.area area_gt = gt_poly.area area_inter = dt_poly.intersection(gt_poly).area iou2d = area_inter / (area_gt + area_dt - area_inter) cch_dt = post_proc.get_z1(dt_floor_coor[:, 1], dt_ceil_coor[:, 1], ch, 512) cch_gt = post_proc.get_z1(gt_floor_coor[:, 1], gt_ceil_coor[:, 1], ch, 512) h_dt = abs(cch_dt.mean() - ch) h_gt = abs(cch_gt.mean() - ch) iouH = min(h_dt, h_gt) / max(h_dt, h_gt) iou3d = iou2d * iouH # Add a result n_corners = len(gt_floor_coor) n_corners = str(n_corners) if n_corners < 10 else '10+' losses[n_corners]['2DIoU'].append(iou2d) losses[n_corners]['3DIoU'].append(iou3d) losses['overall']['2DIoU'].append(iou2d) losses['overall']['3DIoU'].append(iou3d)
def eval_3diou(dt_floor_coor, dt_ceil_coor, gt_floor_coor, gt_ceil_coor, ch=-1.6, coorW=1024, coorH=512, floorW=1024, floorH=512): ''' Evaluate 3D IoU using halfspace intersection ''' dt_floor_coor = np.array(dt_floor_coor) dt_ceil_coor = np.array(dt_ceil_coor) gt_floor_coor = np.array(gt_floor_coor) gt_ceil_coor = np.array(gt_ceil_coor) assert (dt_floor_coor[:, 0] != dt_ceil_coor[:, 0]).sum() == 0 assert (gt_floor_coor[:, 0] != gt_ceil_coor[:, 0]).sum() == 0 N = len(dt_floor_coor) dt_floor_xyz = np.hstack([ post_proc.np_coor2xy(dt_floor_coor, ch, coorW, coorH, floorW=1, floorH=1), np.zeros((N, 1)) + ch, ]) gt_floor_xyz = np.hstack([ post_proc.np_coor2xy(gt_floor_coor, ch, coorW, coorH, floorW=1, floorH=1), np.zeros((N, 1)) + ch, ]) dt_c = np.sqrt((dt_floor_xyz[:, :2] ** 2).sum(1)) gt_c = np.sqrt((gt_floor_xyz[:, :2] ** 2).sum(1)) dt_v2 = post_proc.np_coory2v(dt_ceil_coor[:, 1], coorH) gt_v2 = post_proc.np_coory2v(gt_ceil_coor[:, 1], coorH) dt_ceil_z = dt_c * np.tan(dt_v2) gt_ceil_z = gt_c * np.tan(gt_v2) dt_ceil_xyz = dt_floor_xyz.copy() dt_ceil_xyz[:, 2] = dt_ceil_z gt_ceil_xyz = gt_floor_xyz.copy() gt_ceil_xyz[:, 2] = gt_ceil_z dt_halfspaces = xyzlst2halfspaces(dt_floor_xyz, dt_ceil_xyz) gt_halfspaces = xyzlst2halfspaces(gt_floor_xyz, gt_ceil_xyz) in_halfspaces = HalfspaceIntersection(np.concatenate([dt_halfspaces, gt_halfspaces]), np.zeros(3)) dt_halfspaces = HalfspaceIntersection(dt_halfspaces, np.zeros(3)) gt_halfspaces = HalfspaceIntersection(gt_halfspaces, np.zeros(3)) in_volume = ConvexHull(in_halfspaces.intersections).volume dt_volume = ConvexHull(dt_halfspaces.intersections).volume gt_volume = ConvexHull(gt_halfspaces.intersections).volume un_volume = dt_volume + gt_volume - in_volume return 100 * in_volume / un_volume
H = int(H * args.scale) equirect_texture = equirect_texture.resize((W, H)) equirect_texture = np.array(equirect_texture) / 255.0 H, W = equirect_texture.shape[:2] with open(args.layout) as f: inferenced_result = json.load(f) cor_id = np.array(inferenced_result['uv'], np.float32) cor_id[:, 0] *= W cor_id[:, 1] *= H # Show wireframe if not args.ignore_wireframe: # Convert cor_id to 3d xyz N = len(cor_id) // 2 floor_z = -1.6 floor_xy = np_coor2xy(cor_id[1::2], floor_z, W, H, floorW=1, floorH=1) c = np.sqrt((floor_xy**2).sum(1)) v = np_coory2v(cor_id[0::2, 1], H) ceil_z = (c * np.tan(v)).mean() # Prepare wireframe in open3d assert N == len(floor_xy) wf_points = [[x, y, floor_z] for x, y in floor_xy] +\ [[x, y, ceil_z] for x, y in floor_xy] wf_lines = [[i, (i+1)%N] for i in range(N)] +\ [[i+N, (i+1)%N+N] for i in range(N)] +\ [[i, i+N] for i in range(N)] wf_colors = [[1, 0, 0] for i in range(len(wf_lines))] wf_line_set = open3d.geometry.LineSet() wf_line_set.points = open3d.utility.Vector3dVector(wf_points) wf_line_set.lines = open3d.utility.Vector2iVector(wf_lines)
def test_general(dt_cor_id, gt_cor_id, w, h, losses): dt_floor_coor = dt_cor_id[1::2] dt_ceil_coor = dt_cor_id[0::2] gt_floor_coor = gt_cor_id[1::2] gt_ceil_coor = gt_cor_id[0::2] assert (dt_floor_coor[:, 0] != dt_ceil_coor[:, 0]).sum() == 0 assert (gt_floor_coor[:, 0] != gt_ceil_coor[:, 0]).sum() == 0 # Eval 3d IoU and height error(in meter) N = len(dt_floor_coor) ch = -1.6 dt_floor_xy = post_proc.np_coor2xy(dt_floor_coor, ch, 1024, 512, floorW=1, floorH=1) gt_floor_xy = post_proc.np_coor2xy(gt_floor_coor, ch, 1024, 512, floorW=1, floorH=1) dt_poly = Polygon(dt_floor_xy) gt_poly = Polygon(gt_floor_xy) if not gt_poly.is_valid: print('Skip ground truth invalid (%s)' % gt_path) return # 2D IoU try: area_dt = dt_poly.area area_gt = gt_poly.area area_inter = dt_poly.intersection(gt_poly).area iou2d = area_inter / (area_gt + area_dt - area_inter) except: iou2d = 0 # 3D IoU try: # cch_dt = post_proc.get_z1(dt_floor_coor[:, 1], dt_ceil_coor[:, 1], ch, 512) # cch_gt = post_proc.get_z1(gt_floor_coor[:, 1], gt_ceil_coor[:, 1], ch, 512) # h_dt = abs(cch_dt.mean() - ch) # h_gt = abs(cch_gt.mean() - ch) # area3d_inter = area_inter * min(h_dt, h_gt) # area3d_pred = area_dt * h_dt # area3d_gt = area_gt * h_gt # iou3d = area3d_inter / (area3d_pred + area3d_gt - area3d_inter) iou3d = eval_3diou(dt_floor_coor, dt_ceil_coor, gt_floor_coor, gt_ceil_coor) except: iou3d = 0 # PE error, surface, surface_gt = eval_PE(dt_ceil_coor, dt_floor_coor, gt_ceil_coor, gt_floor_coor) pe = error # rmse & delta_1 gt_layout_depth = layout_2_depth(gt_cor_id, h, w) try: dt_layout_depth = layout_2_depth(dt_cor_id, h, w) except: dt_layout_depth = np.zeros_like(gt_layout_depth) rmse = ((gt_layout_depth - dt_layout_depth)**2).mean() ** 0.5 thres = np.maximum(gt_layout_depth/dt_layout_depth, dt_layout_depth/gt_layout_depth) delta_1 = (thres < 1.25).mean() # Add a result n_corners = len(gt_floor_coor) if n_corners % 2 == 1: n_corners = 'odd' elif n_corners < 10: n_corners = str(n_corners) else: n_corners = '10+' losses[n_corners]['2DIoU'].append(iou2d) losses[n_corners]['3DIoU'].append(iou3d) losses[n_corners]['rmse'].append(rmse) losses[n_corners]['delta_1'].append(delta_1) losses[n_corners]['PE'].append(pe) losses['overall']['2DIoU'].append(iou2d) losses['overall']['3DIoU'].append(iou3d) losses['overall']['rmse'].append(rmse) losses['overall']['delta_1'].append(delta_1) losses['overall']['PE'].append(pe)
def make_3d_files(cor_id, original_equirect_img, output_dir, img_path, ppm=80, camera_height=1.6, ignore_wireframe=False, ignore_floor=False, ignore_ceiling=True, write_obj_files=True, write_point_cloud=True): # preprocessed_img = preprocess(original_equirect_img) # print(preprocessed_img.shape) # equirect_texture = preprocessed_img / 255.0 # equirect_texture = np.array(Image.open(original_equirect_img)) / 255.0 equirect_texture = original_equirect_img H, W = equirect_texture.shape[:2] cor_id[:, 0] *= W cor_id[:, 1] *= H ceil_floor_mask = create_ceiling_floor_mask(cor_id, H, W) # Convert cor_id to 3d xyz N = len(cor_id) // 2 floor_z = -camera_height floor_xy = np_coor2xy(cor_id[1::2], floor_z, W, H, floorW=1, floorH=1) c = np.sqrt((floor_xy**2).sum(1)) v = np_coory2v(cor_id[0::2, 1], H) ceil_z = (c * np.tan(v)).mean() # Prepare if not ignore_wireframe: assert N == len(floor_xy) wf_points = [[x, y, floor_z] for x, y in floor_xy] +\ [[x, y, ceil_z] for x, y in floor_xy] wf_lines = [[i, (i+1)%N] for i in range(N)] +\ [[i+N, (i+1)%N+N] for i in range(N)] +\ [[i, i+N] for i in range(N)] wf_colors = [[1, 0, 0] for i in range(len(wf_lines))] wf_line_set = open3d.geometry.LineSet() wf_line_set.points = open3d.utility.Vector3dVector(wf_points) wf_line_set.lines = open3d.utility.Vector2iVector(wf_lines) wf_line_set.colors = open3d.utility.Vector3dVector(wf_colors) # Warp each wall all_rgb, all_xyz, texture_info = warp_walls(equirect_texture, floor_xy, floor_z, ceil_z, ppm) # Warp floor and ceiling if not ignore_floor or not ignore_ceiling: fi, fp, ci, cp = warp_floor_ceiling(equirect_texture, ceil_floor_mask, floor_xy, floor_z, ceil_z, ppm=ppm, texture_info=texture_info) if not ignore_floor: all_rgb.extend(fi) all_xyz.extend(fp) if not ignore_ceiling: all_rgb.extend(ci) all_xyz.extend(cp) all_xyz = np.array(all_xyz) all_rgb = np.array(all_rgb) texture_info["floor"]["corner_xyz_coords"] = np.array( [[x, y, floor_z + camera_height] for x, y in floor_xy]) texture_info["ceiling"]["corner_xyz_coords"] = np.array( [[x, y, ceil_z + camera_height] for x, y in floor_xy]) if write_obj_files: write_textures(all_rgb, texture_info, output_dir, camera_height) # Filter occluded points occlusion_mask, reord_idx = create_occlusion_mask(all_xyz, H, W) all_xyz = all_xyz[reord_idx][~occlusion_mask] all_rgb = all_rgb[reord_idx][~occlusion_mask] pcd = open3d.PointCloud() pcd.points = open3d.Vector3dVector(all_xyz) pcd.colors = open3d.Vector3dVector(all_rgb) if write_point_cloud: write_point_cloud_files(pcd, str(img_path.stem), output_dir)
def make_3D_json_file(cor_id, original_equirect_img, boundary, output_dir, img_path, camera_height=1.0, set_floor_0=True): # cor_id = np.vstack((cor_id, np.array([0.5,cor_id[:, 1].mean()]))) # cor_id = np.vstack((cor_id, np.array([0.5,0.5]))) equirect_texture = original_equirect_img H, W = equirect_texture.shape[:2] cor_id[:, 0] *= W cor_id[:, 1] *= H # ceil_floor_mask = create_ceiling_floor_mask(cor_id, H, W) # Convert cor_id to 3d xyz N = len(cor_id) // 2 floor_z = -camera_height # print(cor_id[1::2]) floor_xy = np_coor2xy(cor_id[1::2], floor_z, W, H, floorW=1, floorH=1) c = np.sqrt((floor_xy**2).sum(1)) v = np_coory2v(cor_id[0::2, 1], H) ceil_z = (c * np.tan(v)).mean().item() cor_id[:, 0] /= W cor_id[:, 1] /= H # convert to list because numpy float32 cannot be seriarized by json uv_coords = cor_id[0::2].tolist() + cor_id[1::2].tolist() floor_xy = floor_xy.tolist() # print(cor_id[1::2]) # print(floor_xy) tmp = np_coor2xy(np.array([0.5 * W, cor_id[:, 1].mean() * H]).reshape(1, 2), floor_z, W, H, floorW=1, floorH=1) # print(tmp) data = {} data["panoramic_photo_filename"] = img_path.name data["ProjectPanelID"] = int(img_path.stem.split("_")[1]) # data["xyz_coords"] = [[x, y, ceil_z] for x, y in floor_xy] + [[x, y, floor_z] for x, y in floor_xy] data["xyz_coords"] = [] if set_floor_0: xyz_coords = [[x, y, ceil_z + camera_height] for x, y in floor_xy] + [[x, y, floor_z + camera_height] for x, y in floor_xy] else: xyz_coords = [[x, y, ceil_z] for x, y in floor_xy] + [[x, y, floor_z] for x, y in floor_xy] for x, y, z in xyz_coords: data["xyz_coords"].append({"x": x, "y": y, "z": z}) # data["uv_coords"] = uv_coords data["uv_coords"] = [] for u, v in uv_coords: data["uv_coords"].append({"u": u, "v": v}) data["n_vertices"] = len(floor_xy) * 2 data["n_walls"] = len(floor_xy) data["room_height"] = ceil_z + camera_height data["center_of_panorama_vector"] = {"x": 0, "y": -1, "z": 0} data["planes"] = [] # ceiling l = [i for i in range(len(floor_xy))] ceiling_plain = { "type": "ceiling", "vertex_ids": [i for i in range(len(floor_xy))], "uv_ids": [i for i in range(len(floor_xy))], } normal_vector, area = cal_normal_and_area(xyz_coords, ceiling_plain["vertex_ids"]) # ceiling_plain["normal_vector"] = normal_vector ceiling_plain["normal_vector"] = { "x": normal_vector[0], "y": normal_vector[1], "z": normal_vector[2] } ceiling_plain["area"] = area data["planes"].append(ceiling_plain) # floor floor_plain = { "type": "floor", "vertex_ids": [i for i in range(len(floor_xy), len(floor_xy) * 2)], "uv_ids": [i for i in range(len(floor_xy), len(floor_xy) * 2)], } normal_vector, area = cal_normal_and_area(xyz_coords, floor_plain["vertex_ids"]) # floor_plain["normal_vector"] = normal_vector floor_plain["normal_vector"] = { "x": normal_vector[0], "y": normal_vector[1], "z": normal_vector[2] } floor_plain["area"] = area data["planes"].append(floor_plain) # each wall for i in range(data["n_walls"]): vc_i = ceiling_plain["vertex_ids"][i] vc_j = ceiling_plain["vertex_ids"][(i + 1) % data["n_walls"]] cuurent_wall = { "type": "wall", "vertex_ids": [vc_i, vc_j, vc_j + data["n_walls"], vc_i + data["n_walls"]], "uv_ids": [vc_i, vc_j, vc_j + data["n_walls"], vc_i + data["n_walls"]], } normal_vector, area = cal_normal_and_area(xyz_coords, cuurent_wall["vertex_ids"]) # cuurent_wall["normal_vector"] = normal_vector cuurent_wall["normal_vector"] = { "x": normal_vector[0], "y": normal_vector[1], "z": normal_vector[2] } cuurent_wall["area"] = area data["planes"].append(cuurent_wall) # jsonified_data = json.dumps(data, indent=2) with open(Path(output_dir) / (img_path.stem + ".json"), "w") as f: json.dump(data, f, indent=4)