def test_general(dt_cor_id, gt_cor_id, w, h, losses):
dt_floor_coor = dt_cor_id[1::2]
dt_ceil_coor = dt_cor_id[0::2]
gt_floor_coor = gt_cor_id[1::2]
gt_ceil_coor = gt_cor_id[0::2]
assert (dt_floor_coor[:, 0] != dt_ceil_coor[:, 0]).sum() == 0
assert (gt_floor_coor[:, 0] != gt_ceil_coor[:, 0]).sum() == 0
# Eval 3d IoU and height error(in meter)
N = len(dt_floor_coor)
ch = -1.6
dt_floor_xy = post_proc.np_coor2xy(dt_floor_coor,
ch,
1024,
512,
floorW=1,
floorH=1)
gt_floor_xy = post_proc.np_coor2xy(gt_floor_coor,
ch,
1024,
512,
floorW=1,
floorH=1)
dt_poly = Polygon(dt_floor_xy)
gt_poly = Polygon(gt_floor_xy)
if not gt_poly.is_valid:
print('Skip ground truth invalid (%s)' % gt_path)
return
area_dt = dt_poly.area
area_gt = gt_poly.area
area_inter = dt_poly.intersection(gt_poly).area
iou2d = area_inter / (area_gt + area_dt - area_inter)
cch_dt = post_proc.get_z1(dt_floor_coor[:, 1], dt_ceil_coor[:, 1], ch, 512)
cch_gt = post_proc.get_z1(gt_floor_coor[:, 1], gt_ceil_coor[:, 1], ch, 512)
h_dt = abs(cch_dt.mean() - ch)
h_gt = abs(cch_gt.mean() - ch)
area3d_inter = area_inter * min(h_dt, h_gt)
area3d_pred = area_dt * h_dt
area3d_gt = area_gt * h_gt
iou3d = area3d_inter / (area3d_pred + area3d_gt - area3d_inter)
# Add a result
n_corners = len(gt_floor_coor)
if n_corners % 2 == 1:
n_corners = 'odd'
elif n_corners < 10:
n_corners = str(n_corners)
else:
n_corners = '10+'
losses[n_corners]['2DIoU'].append(iou2d)
losses[n_corners]['3DIoU'].append(iou3d)
losses['overall']['2DIoU'].append(iou2d)
losses['overall']['3DIoU'].append(iou3d)
def test_general(dt_cor_id, gt_cor_id, w, h, losses):
dt_floor_coor = dt_cor_id[1::2]
dt_ceil_coor = dt_cor_id[0::2]
gt_floor_coor = gt_cor_id[1::2]
gt_ceil_coor = gt_cor_id[0::2]
assert (dt_floor_coor[:, 0] != dt_ceil_coor[:, 0]).sum() == 0
assert (gt_floor_coor[:, 0] != gt_ceil_coor[:, 0]).sum() == 0
# Eval 3d IoU and height error(in meter)
N = len(dt_floor_coor)
ch = -1.6
dt_floor_xy = post_proc.np_coor2xy(dt_floor_coor,
ch,
1024,
512,
floorW=1,
floorH=1)
gt_floor_xy = post_proc.np_coor2xy(gt_floor_coor,
ch,
1024,
512,
floorW=1,
floorH=1)
dt_poly = Polygon(dt_floor_xy)
gt_poly = Polygon(gt_floor_xy)
area_dt = dt_poly.area
area_gt = gt_poly.area
area_inter = dt_poly.intersection(gt_poly).area
iou2d = area_inter / (area_gt + area_dt - area_inter)
cch_dt = post_proc.get_z1(dt_floor_coor[:, 1], dt_ceil_coor[:, 1], ch, 512)
cch_gt = post_proc.get_z1(gt_floor_coor[:, 1], gt_ceil_coor[:, 1], ch, 512)
h_dt = abs(cch_dt.mean() - ch)
h_gt = abs(cch_gt.mean() - ch)
iouH = min(h_dt, h_gt) / max(h_dt, h_gt)
iou3d = iou2d * iouH
# Add a result
n_corners = len(gt_floor_coor)
n_corners = str(n_corners) if n_corners < 10 else '10+'
losses[n_corners]['2DIoU'].append(iou2d)
losses[n_corners]['3DIoU'].append(iou3d)
losses['overall']['2DIoU'].append(iou2d)
losses['overall']['3DIoU'].append(iou3d)
def eval_3diou(dt_floor_coor, dt_ceil_coor, gt_floor_coor, gt_ceil_coor, ch=-1.6,
coorW=1024, coorH=512, floorW=1024, floorH=512):
''' Evaluate 3D IoU using halfspace intersection '''
dt_floor_coor = np.array(dt_floor_coor)
dt_ceil_coor = np.array(dt_ceil_coor)
gt_floor_coor = np.array(gt_floor_coor)
gt_ceil_coor = np.array(gt_ceil_coor)
assert (dt_floor_coor[:, 0] != dt_ceil_coor[:, 0]).sum() == 0
assert (gt_floor_coor[:, 0] != gt_ceil_coor[:, 0]).sum() == 0
N = len(dt_floor_coor)
dt_floor_xyz = np.hstack([
post_proc.np_coor2xy(dt_floor_coor, ch, coorW, coorH, floorW=1, floorH=1),
np.zeros((N, 1)) + ch,
])
gt_floor_xyz = np.hstack([
post_proc.np_coor2xy(gt_floor_coor, ch, coorW, coorH, floorW=1, floorH=1),
np.zeros((N, 1)) + ch,
])
dt_c = np.sqrt((dt_floor_xyz[:, :2] ** 2).sum(1))
gt_c = np.sqrt((gt_floor_xyz[:, :2] ** 2).sum(1))
dt_v2 = post_proc.np_coory2v(dt_ceil_coor[:, 1], coorH)
gt_v2 = post_proc.np_coory2v(gt_ceil_coor[:, 1], coorH)
dt_ceil_z = dt_c * np.tan(dt_v2)
gt_ceil_z = gt_c * np.tan(gt_v2)
dt_ceil_xyz = dt_floor_xyz.copy()
dt_ceil_xyz[:, 2] = dt_ceil_z
gt_ceil_xyz = gt_floor_xyz.copy()
gt_ceil_xyz[:, 2] = gt_ceil_z
dt_halfspaces = xyzlst2halfspaces(dt_floor_xyz, dt_ceil_xyz)
gt_halfspaces = xyzlst2halfspaces(gt_floor_xyz, gt_ceil_xyz)
in_halfspaces = HalfspaceIntersection(np.concatenate([dt_halfspaces, gt_halfspaces]),
np.zeros(3))
dt_halfspaces = HalfspaceIntersection(dt_halfspaces, np.zeros(3))
gt_halfspaces = HalfspaceIntersection(gt_halfspaces, np.zeros(3))
in_volume = ConvexHull(in_halfspaces.intersections).volume
dt_volume = ConvexHull(dt_halfspaces.intersections).volume
gt_volume = ConvexHull(gt_halfspaces.intersections).volume
un_volume = dt_volume + gt_volume - in_volume
return 100 * in_volume / un_volume
H = int(H * args.scale)
equirect_texture = equirect_texture.resize((W, H))
equirect_texture = np.array(equirect_texture) / 255.0
H, W = equirect_texture.shape[:2]
with open(args.layout) as f:
inferenced_result = json.load(f)
cor_id = np.array(inferenced_result['uv'], np.float32)
cor_id[:, 0] *= W
cor_id[:, 1] *= H
# Show wireframe
if not args.ignore_wireframe:
# Convert cor_id to 3d xyz
N = len(cor_id) // 2
floor_z = -1.6
floor_xy = np_coor2xy(cor_id[1::2], floor_z, W, H, floorW=1, floorH=1)
c = np.sqrt((floor_xy**2).sum(1))
v = np_coory2v(cor_id[0::2, 1], H)
ceil_z = (c * np.tan(v)).mean()
# Prepare wireframe in open3d
assert N == len(floor_xy)
wf_points = [[x, y, floor_z] for x, y in floor_xy] +\
[[x, y, ceil_z] for x, y in floor_xy]
wf_lines = [[i, (i+1)%N] for i in range(N)] +\
[[i+N, (i+1)%N+N] for i in range(N)] +\
[[i, i+N] for i in range(N)]
wf_colors = [[1, 0, 0] for i in range(len(wf_lines))]
wf_line_set = open3d.geometry.LineSet()
wf_line_set.points = open3d.utility.Vector3dVector(wf_points)
wf_line_set.lines = open3d.utility.Vector2iVector(wf_lines)
def test_general(dt_cor_id, gt_cor_id, w, h, losses):
dt_floor_coor = dt_cor_id[1::2]
dt_ceil_coor = dt_cor_id[0::2]
gt_floor_coor = gt_cor_id[1::2]
gt_ceil_coor = gt_cor_id[0::2]
assert (dt_floor_coor[:, 0] != dt_ceil_coor[:, 0]).sum() == 0
assert (gt_floor_coor[:, 0] != gt_ceil_coor[:, 0]).sum() == 0
# Eval 3d IoU and height error(in meter)
N = len(dt_floor_coor)
ch = -1.6
dt_floor_xy = post_proc.np_coor2xy(dt_floor_coor, ch, 1024, 512, floorW=1, floorH=1)
gt_floor_xy = post_proc.np_coor2xy(gt_floor_coor, ch, 1024, 512, floorW=1, floorH=1)
dt_poly = Polygon(dt_floor_xy)
gt_poly = Polygon(gt_floor_xy)
if not gt_poly.is_valid:
print('Skip ground truth invalid (%s)' % gt_path)
return
# 2D IoU
try:
area_dt = dt_poly.area
area_gt = gt_poly.area
area_inter = dt_poly.intersection(gt_poly).area
iou2d = area_inter / (area_gt + area_dt - area_inter)
except:
iou2d = 0
# 3D IoU
try:
# cch_dt = post_proc.get_z1(dt_floor_coor[:, 1], dt_ceil_coor[:, 1], ch, 512)
# cch_gt = post_proc.get_z1(gt_floor_coor[:, 1], gt_ceil_coor[:, 1], ch, 512)
# h_dt = abs(cch_dt.mean() - ch)
# h_gt = abs(cch_gt.mean() - ch)
# area3d_inter = area_inter * min(h_dt, h_gt)
# area3d_pred = area_dt * h_dt
# area3d_gt = area_gt * h_gt
# iou3d = area3d_inter / (area3d_pred + area3d_gt - area3d_inter)
iou3d = eval_3diou(dt_floor_coor, dt_ceil_coor, gt_floor_coor,
gt_ceil_coor)
except:
iou3d = 0
# PE
error, surface, surface_gt = eval_PE(dt_ceil_coor, dt_floor_coor, gt_ceil_coor, gt_floor_coor)
pe = error
# rmse & delta_1
gt_layout_depth = layout_2_depth(gt_cor_id, h, w)
try:
dt_layout_depth = layout_2_depth(dt_cor_id, h, w)
except:
dt_layout_depth = np.zeros_like(gt_layout_depth)
rmse = ((gt_layout_depth - dt_layout_depth)**2).mean() ** 0.5
thres = np.maximum(gt_layout_depth/dt_layout_depth, dt_layout_depth/gt_layout_depth)
delta_1 = (thres < 1.25).mean()
# Add a result
n_corners = len(gt_floor_coor)
if n_corners % 2 == 1:
n_corners = 'odd'
elif n_corners < 10:
n_corners = str(n_corners)
else:
n_corners = '10+'
losses[n_corners]['2DIoU'].append(iou2d)
losses[n_corners]['3DIoU'].append(iou3d)
losses[n_corners]['rmse'].append(rmse)
losses[n_corners]['delta_1'].append(delta_1)
losses[n_corners]['PE'].append(pe)
losses['overall']['2DIoU'].append(iou2d)
losses['overall']['3DIoU'].append(iou3d)
losses['overall']['rmse'].append(rmse)
losses['overall']['delta_1'].append(delta_1)
losses['overall']['PE'].append(pe)
def make_3d_files(cor_id,
original_equirect_img,
output_dir,
img_path,
ppm=80,
camera_height=1.6,
ignore_wireframe=False,
ignore_floor=False,
ignore_ceiling=True,
write_obj_files=True,
write_point_cloud=True):
# preprocessed_img = preprocess(original_equirect_img)
# print(preprocessed_img.shape)
# equirect_texture = preprocessed_img / 255.0
# equirect_texture = np.array(Image.open(original_equirect_img)) / 255.0
equirect_texture = original_equirect_img
H, W = equirect_texture.shape[:2]
cor_id[:, 0] *= W
cor_id[:, 1] *= H
ceil_floor_mask = create_ceiling_floor_mask(cor_id, H, W)
# Convert cor_id to 3d xyz
N = len(cor_id) // 2
floor_z = -camera_height
floor_xy = np_coor2xy(cor_id[1::2], floor_z, W, H, floorW=1, floorH=1)
c = np.sqrt((floor_xy**2).sum(1))
v = np_coory2v(cor_id[0::2, 1], H)
ceil_z = (c * np.tan(v)).mean()
# Prepare
if not ignore_wireframe:
assert N == len(floor_xy)
wf_points = [[x, y, floor_z] for x, y in floor_xy] +\
[[x, y, ceil_z] for x, y in floor_xy]
wf_lines = [[i, (i+1)%N] for i in range(N)] +\
[[i+N, (i+1)%N+N] for i in range(N)] +\
[[i, i+N] for i in range(N)]
wf_colors = [[1, 0, 0] for i in range(len(wf_lines))]
wf_line_set = open3d.geometry.LineSet()
wf_line_set.points = open3d.utility.Vector3dVector(wf_points)
wf_line_set.lines = open3d.utility.Vector2iVector(wf_lines)
wf_line_set.colors = open3d.utility.Vector3dVector(wf_colors)
# Warp each wall
all_rgb, all_xyz, texture_info = warp_walls(equirect_texture, floor_xy,
floor_z, ceil_z, ppm)
# Warp floor and ceiling
if not ignore_floor or not ignore_ceiling:
fi, fp, ci, cp = warp_floor_ceiling(equirect_texture,
ceil_floor_mask,
floor_xy,
floor_z,
ceil_z,
ppm=ppm,
texture_info=texture_info)
if not ignore_floor:
all_rgb.extend(fi)
all_xyz.extend(fp)
if not ignore_ceiling:
all_rgb.extend(ci)
all_xyz.extend(cp)
all_xyz = np.array(all_xyz)
all_rgb = np.array(all_rgb)
texture_info["floor"]["corner_xyz_coords"] = np.array(
[[x, y, floor_z + camera_height] for x, y in floor_xy])
texture_info["ceiling"]["corner_xyz_coords"] = np.array(
[[x, y, ceil_z + camera_height] for x, y in floor_xy])
if write_obj_files:
write_textures(all_rgb, texture_info, output_dir, camera_height)
# Filter occluded points
occlusion_mask, reord_idx = create_occlusion_mask(all_xyz, H, W)
all_xyz = all_xyz[reord_idx][~occlusion_mask]
all_rgb = all_rgb[reord_idx][~occlusion_mask]
pcd = open3d.PointCloud()
pcd.points = open3d.Vector3dVector(all_xyz)
pcd.colors = open3d.Vector3dVector(all_rgb)
if write_point_cloud:
write_point_cloud_files(pcd, str(img_path.stem), output_dir)
def make_3D_json_file(cor_id,
original_equirect_img,
boundary,
output_dir,
img_path,
camera_height=1.0,
set_floor_0=True):
# cor_id = np.vstack((cor_id, np.array([0.5,cor_id[:, 1].mean()])))
# cor_id = np.vstack((cor_id, np.array([0.5,0.5])))
equirect_texture = original_equirect_img
H, W = equirect_texture.shape[:2]
cor_id[:, 0] *= W
cor_id[:, 1] *= H
# ceil_floor_mask = create_ceiling_floor_mask(cor_id, H, W)
# Convert cor_id to 3d xyz
N = len(cor_id) // 2
floor_z = -camera_height
# print(cor_id[1::2])
floor_xy = np_coor2xy(cor_id[1::2], floor_z, W, H, floorW=1, floorH=1)
c = np.sqrt((floor_xy**2).sum(1))
v = np_coory2v(cor_id[0::2, 1], H)
ceil_z = (c * np.tan(v)).mean().item()
cor_id[:, 0] /= W
cor_id[:, 1] /= H
# convert to list because numpy float32 cannot be seriarized by json
uv_coords = cor_id[0::2].tolist() + cor_id[1::2].tolist()
floor_xy = floor_xy.tolist()
# print(cor_id[1::2])
# print(floor_xy)
tmp = np_coor2xy(np.array([0.5 * W,
cor_id[:, 1].mean() * H]).reshape(1, 2),
floor_z,
W,
H,
floorW=1,
floorH=1)
# print(tmp)
data = {}
data["panoramic_photo_filename"] = img_path.name
data["ProjectPanelID"] = int(img_path.stem.split("_")[1])
# data["xyz_coords"] = [[x, y, ceil_z] for x, y in floor_xy] + [[x, y, floor_z] for x, y in floor_xy]
data["xyz_coords"] = []
if set_floor_0:
xyz_coords = [[x, y, ceil_z + camera_height]
for x, y in floor_xy] + [[x, y, floor_z + camera_height]
for x, y in floor_xy]
else:
xyz_coords = [[x, y, ceil_z]
for x, y in floor_xy] + [[x, y, floor_z]
for x, y in floor_xy]
for x, y, z in xyz_coords:
data["xyz_coords"].append({"x": x, "y": y, "z": z})
# data["uv_coords"] = uv_coords
data["uv_coords"] = []
for u, v in uv_coords:
data["uv_coords"].append({"u": u, "v": v})
data["n_vertices"] = len(floor_xy) * 2
data["n_walls"] = len(floor_xy)
data["room_height"] = ceil_z + camera_height
data["center_of_panorama_vector"] = {"x": 0, "y": -1, "z": 0}
data["planes"] = []
# ceiling
l = [i for i in range(len(floor_xy))]
ceiling_plain = {
"type": "ceiling",
"vertex_ids": [i for i in range(len(floor_xy))],
"uv_ids": [i for i in range(len(floor_xy))],
}
normal_vector, area = cal_normal_and_area(xyz_coords,
ceiling_plain["vertex_ids"])
# ceiling_plain["normal_vector"] = normal_vector
ceiling_plain["normal_vector"] = {
"x": normal_vector[0],
"y": normal_vector[1],
"z": normal_vector[2]
}
ceiling_plain["area"] = area
data["planes"].append(ceiling_plain)
# floor
floor_plain = {
"type": "floor",
"vertex_ids": [i for i in range(len(floor_xy),
len(floor_xy) * 2)],
"uv_ids": [i for i in range(len(floor_xy),
len(floor_xy) * 2)],
}
normal_vector, area = cal_normal_and_area(xyz_coords,
floor_plain["vertex_ids"])
# floor_plain["normal_vector"] = normal_vector
floor_plain["normal_vector"] = {
"x": normal_vector[0],
"y": normal_vector[1],
"z": normal_vector[2]
}
floor_plain["area"] = area
data["planes"].append(floor_plain)
# each wall
for i in range(data["n_walls"]):
vc_i = ceiling_plain["vertex_ids"][i]
vc_j = ceiling_plain["vertex_ids"][(i + 1) % data["n_walls"]]
cuurent_wall = {
"type":
"wall",
"vertex_ids":
[vc_i, vc_j, vc_j + data["n_walls"], vc_i + data["n_walls"]],
"uv_ids":
[vc_i, vc_j, vc_j + data["n_walls"], vc_i + data["n_walls"]],
}
normal_vector, area = cal_normal_and_area(xyz_coords,
cuurent_wall["vertex_ids"])
# cuurent_wall["normal_vector"] = normal_vector
cuurent_wall["normal_vector"] = {
"x": normal_vector[0],
"y": normal_vector[1],
"z": normal_vector[2]
}
cuurent_wall["area"] = area
data["planes"].append(cuurent_wall)
# jsonified_data = json.dumps(data, indent=2)
with open(Path(output_dir) / (img_path.stem + ".json"), "w") as f:
json.dump(data, f, indent=4)
