def show_final_preds(pred_boxes, points, gt_boxes):
pred_boxes = pred_boxes.data.cpu().numpy()
boxes_show = np.concatenate([pred_boxes, gt_boxes], 0)
pnum = pred_boxes.shape[0]
gnum = gt_boxes.shape[0]
labels_show = np.array([0]*pnum + [1]*gnum)
Bbox3D.draw_points_bboxes(points, boxes_show, 'Z', is_yx_zb=True, labels=labels_show)
def show_pcl_corners(self, pcl):
import numpy as np
from utils3d.bbox3d_ops import Bbox3D
corners, _ = self.get_2top_corners_offseted()
corners = corners.view([-1, 3]).cpu().data.numpy()
corners_bot = corners.copy()
tmp = self.bbox3d[:, 5].cpu().data.numpy()
tmp = np.tile(np.expand_dims(tmp, 1), [1, 2]).reshape(-1)
corners_bot[:, 2] = corners_bot[:, 2] - tmp
corners = np.concatenate([corners, corners_bot], 0)
n = corners.shape[0]
boxes = np.zeros([n, 7])
boxes[:, 0:3] = corners
boxes[:, 3:6] = 0.2
#points = pcl.cpu().data.numpy()
points = pcl
Bbox3D.draw_points_bboxes(points,
boxes,
'Z',
is_yx_zb=self.mode == 'standard',
random_color=True)
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
def show(self, max_num=-1, points=None, with_centroids=False, boxes_show_together=None, points_keep_rate=POINTS_KEEP_RATE, points_sample_rate=POINTS_SAMPLE_RATE, colors=None):
import numpy as np
from utils3d.bbox3d_ops import Bbox3D
boxes = self.bbox3d.cpu().data.numpy()
if with_centroids:
centroids = boxes.copy()
if self.mode == 'yx_zb':
centroids[:,2] += centroids[:,5]*0.5
centroids[:,3:6] = 0.03
if max_num > 0 and max_num < boxes.shape[0]:
step = 4
ids0 = np.random.choice(boxes.shape[0]//step-1, max_num, replace=False).reshape(-1,1)*step
tmp = np.arange(step).reshape(1,step)
ids = (ids0 + tmp).reshape(-1)
print(ids)
boxes = boxes[ids]
if with_centroids:
boxes = np.concatenate([boxes, centroids], 0)
if boxes_show_together:
boxes_show_together = boxes_show_together.bbox3d.cpu().data.numpy()
labels = np.array( [0]*boxes.shape[0] + [1]*boxes_show_together.shape[0])
boxes = np.concatenate([boxes, boxes_show_together], 0)
else:
if 'labels' in self.fields():
labels = self.get_field('labels').cpu().data.numpy().astype(np.int32)
else:
labels = None
if points is None:
Bbox3D.draw_bboxes(boxes, 'Z', is_yx_zb=self.mode=='yx_zb', \
labels = labels, random_color=False)
else:
Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb',\
labels = labels, random_color=False, points_keep_rate=points_keep_rate, points_sample_rate=points_sample_rate, box_colors=colors)
def show_pcl_boxdic(pcl, bboxes_dic):
from utils3d.bbox3d_ops import Bbox3D
boxes = []
for obj in bboxes_dic:
print(f'{obj}: {len(bboxes_dic[obj])}')
boxes.append(bboxes_dic[obj])
boxes = np.concatenate(boxes, 0)
Bbox3D.draw_points_bboxes(pcl[:,0:6], boxes, 'Z', is_yx_zb=True)
Bbox3D.draw_points_bboxes(pcl[:,0:6], bboxes_dic['wall'], 'Z', is_yx_zb=True)
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
def show__together(self, boxlist_1, max_num=-1, max_num_1=-1, points=None, offset_x=None, twolabels=False,
mesh=False, points_keep_rate=POINTS_KEEP_RATE, points_sample_rate=POINTS_SAMPLE_RATE, random_color=False, colors=None):
import numpy as np
from utils3d.bbox3d_ops import Bbox3D
boxes = self.bbox3d.cpu().data.numpy().copy()
if max_num > 0 and max_num < boxes.shape[0]:
ids = np.random.choice(boxes.shape[0], max_num, replace=False)
boxes = boxes[ids]
boxes_1 = boxlist_1.bbox3d.cpu().data.numpy().copy()
if max_num_1 > 0 and max_num_1 < boxes_1.shape[0]:
ids = np.random.choice(boxes_1.shape[0], max_num_1, replace=False)
boxes_1 = boxes_1[ids]
if offset_x is not None:
boxes_1[:,0] += offset_x
if not twolabels and 'labels' in self.fields():
labels = self.get_field('labels').cpu().data.numpy().astype(np.int32)
labels_1 = boxlist_1.get_field('labels').cpu().data.numpy().astype(np.int32)
labels = np.concatenate([labels, labels_1], 0)
else:
labels = np.array([0]*boxes.shape[0] + [1]*boxes_1.shape[0])
boxes = np.concatenate([boxes, boxes_1], 0)
if colors is not None:
colors = np.concatenate(colors, 0)
assert colors.shape[0] == boxes.shape[0]
if points is None:
if mesh:
Bbox3D.draw_bboxes_mesh(boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels)
else:
Bbox3D.draw_bboxes(boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, random_color=False)
else:
if isinstance(points, torch.Tensor):
points = points.cpu().data.numpy()
if offset_x is not None:
tp = points.copy()
tp[:,0] += offset_x
points = np.concatenate([points, tp], 0)
#
tp = tp.copy()
tp[:,0] += offset_x
points = np.concatenate([points, tp], 0)
if mesh:
Bbox3D.draw_points_bboxes_mesh(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, points_keep_rate=points_keep_rate, points_sample_rate=points_sample_rate, random_color=random_color, box_colors=colors)
else:
Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, random_color=random_color,
points_keep_rate=points_keep_rate, points_sample_rate=points_sample_rate, box_colors=colors)
def show_with_corners(self, only_corner=0):
import numpy as np
from utils3d.bbox3d_ops import Bbox3D
corners, _ = self.get_2top_corners_offseted()
corners = corners.view([-1, 3])
points = corners.cpu().data.numpy()
boxes = self.bbox3d.cpu().data.numpy()
if only_corner:
boxes = boxes[0:0, :]
Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb',\
random_color=True)
pass
def show_highlight(self, ids, points=None):
from utils3d.bbox3d_ops import Bbox3D
ids = np.array(ids)
n = len(self)
labels = np.zeros([n]).astype(np.int)
labels[ids] = 1
boxes = self.bbox3d.cpu().data.numpy()
if points is None:
Bbox3D.draw_bboxes(boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, random_color=False)
else:
if isinstance(points, torch.Tensor):
points = points.cpu().data.numpy()
Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, random_color=False)
def is_edge_wall_of_ceiling(wall_cenlines, ceiling, walls_org):
c_corners = Bbox3D.bbox_corners(ceiling, 'Z')
c_lines = np.take(c_corners, Bbox3D._lines_z0_vids, axis=0)
c_lines[:, :, 2] = 0
wall_cenlines_ = np.concatenate(
[wall_cenlines, wall_cenlines.mean(1, keepdims=True)], 1)
wn = wall_cenlines.shape[0]
in_mask0 = points_in_lines(wall_cenlines_.reshape([-1, 3]),
c_lines,
threshold_dis=0.1) # [wn*3,4]
in_mask0 = in_mask0.reshape([wn, 3, 4]) # [wn,3,4]
# find all wdge walls
in_mask1 = in_mask0.any(2) # [wn,3] in any of four edge of a ceiling
in_state = in_mask1.sum(1)
winc_mask = in_state >= 2 # [wn] the number of points on edge: 0~3. two of three: two corners, one centroid
winc_ids = np.where(winc_mask)[0]
# clean repeat edge walls on same edge: only at most one edge wall on each
# edge allowed
if winc_ids.shape[0] == 0:
winc_ids_1 = winc_ids
winc_state = []
else:
keep_ids = clean_extend_lines(wall_cenlines[winc_ids])
winc_ids_1 = winc_ids[keep_ids]
winc_state = in_state[winc_ids_1]
edge_wall_num = winc_ids_1.shape[0]
if Debug and edge_wall_num < 2 and 0:
#if Debug:
print(f'edge_wall_num: {edge_wall_num}')
boxes = np.concatenate([ceiling.reshape([1, -1]), walls_org[winc_ids]],
0)
wo = walls_org.copy()
wo[:, 2] -= 2
#boxes = np.concatenate([boxes, wo],0)
#boxes = ceiling.reshape([1,-1])
points = np.concatenate(
[wall_cenlines_.reshape([-1, 3]),
c_lines.reshape([-1, 3])], 0)
Bbox3D.draw_points_bboxes(points, boxes, 'Z', False)
boxes = np.concatenate(
[ceiling.reshape([1, -1]), walls_org[winc_ids_1]], 0)
Bbox3D.draw_points_bboxes(points, boxes, 'Z', False)
#show_walls_1by1(walls_org)
return edge_wall_num, winc_state
def crop_walls(wall_bboxes):
'''
crop walls with intersections not on the corner
'''
#show_walls_1by1(wall_bboxes)
if wall_bboxes.shape[0] == 0:
return wall_bboxes
intersections = Bbox3D.all_intersections_by_cenline(
wall_bboxes, check_same_height=False, not_on_corners=True)
n = wall_bboxes.shape[0]
new_walls = []
keep_mask = np.array([True] * n)
for i in range(n):
inters_i = intersections[i]
if inters_i.shape[0] == 0:
continue
new_walls_i = Bbox3D.split_wall_by_centroid_intersections(
wall_bboxes[i], inters_i)
keep_mask[i] = False
new_walls.append(new_walls_i)
show = False
#tmp = np.concatenate(new_walls, 0)
#if tmp.shape[0] >= 7:
# show = True
if show:
#tmp = wall_bboxes.copy()
tmp = new_walls_i.copy()
tmp[:, 2] += 1
for ti in range(1, tmp.shape[0]):
tmp[ti:, 2] += 0.2
show_box = np.concatenate([tmp, wall_bboxes[i:i + 1]], 0)
Bbox3D.draw_points_bboxes(inters_i, show_box, 'Z', False)
#Bbox3D.draw_bboxes(show_box, 'Z', False)
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
num_croped = np.sum(keep_mask == False)
print(f'num croped:{num_croped}\nnum new:{len(new_walls)}')
wall_bboxes = wall_bboxes[keep_mask]
if len(new_walls) > 0:
new_walls = np.concatenate(new_walls, 0)
wall_bboxes = np.concatenate([wall_bboxes, new_walls], 0)
#show_walls_1by1(wall_bboxes)
#show_walls_offsetz(wall_bboxes)
return wall_bboxes
def show_anchors(points, anchors, gt_boxes, anchors_mask=None):
# show valid anchors
if anchors_mask is not None:
indices = np.where(anchors_mask)[0]
anchors = anchors[indices,:]
show_num = -1
if show_num>0 and show_num < anchors.shape[0]:
indices = np.random.choice(anchors.shape[0], show_num)
anchors = anchors[indices,:]
n0 = anchors.shape[0]
n1 = gt_boxes.shape[0]
anchors = np.concatenate([anchors, gt_boxes], 0)
labels = np.array([1]*n0 + [0]*n1)
Bbox3D.draw_points_bboxes(points, anchors, 'Z', is_yx_zb=True, labels=labels)
def forcely_crop_scene(pcl0, walls, ceilings):
scene_min = pcl0[:,:3].min(0)
scene_max = pcl0[:,:3].max(0)
scene_size = scene_max - scene_min
abandon = scene_size - MAX_SCENE_SIZE
masks = []
ref_boxes = [walls, walls, ceilings]
for i in range(3):
if abandon[i] > 0:
if ref_boxes[i].shape[0] == 0:
if i<2:
rate_min = 0.5
else:
rate_min = 0
else:
ref_min = ref_boxes[i][:,i].min()
ref_max = ref_boxes[i][:,i].max()
wmin = ref_min - scene_min[i]
wmax = scene_max[i] - ref_max
rate_min = wmin / (wmin+wmax)
new_mini = scene_min[i] + abandon[i] * rate_min
new_maxi = scene_max[i] - abandon[i] * (1-rate_min)
masks.append( (pcl0[:,i] > new_mini) * (pcl0[:,i] < new_maxi) )
if len(masks) > 0:
mask = masks[0]
for m in masks:
mask = mask * m
pcl1 = pcl0[mask]
else:
pcl1 = pcl0
show = 0
if show:
scene_min_new = pcl1[:,:3].min(0)
scene_max_new = pcl1[:,:3].max(0)
print(f'Org: {scene_min} - {scene_max}')
print(f'New: {scene_min_new} - {scene_max_new}')
pcl00 = pcl0.copy()
pcl00[:,2] -= 15
pcl_show = np.concatenate([pcl00, pcl1], 0)
Bbox3D.draw_points_bboxes(pcl_show, walls, 'Z', False, points_keep_rate=1)
return pcl1
def show_high_score_preds(score_threshold0, score_threshold1, total_scores, box_preds, anchors, gt_boxes, points):
pos_ids = np.where(np.logical_and(total_scores >= score_threshold0, total_scores < score_threshold1))[0]
box_preds_pos = box_preds[pos_ids]
anchors = anchors[pos_ids]
pnum = box_preds_pos.shape[0]
gnum = gt_boxes.shape[0]
#gt_boxes[:,2] -= 0.2
#points[:,2] -= 1
labels_show = np.array([0]*pnum + [1]*gnum)
boxes_show = np.concatenate([box_preds_pos, gt_boxes], 0)
#Bbox3D.draw_bboxes(boxes_show, 'Z', is_yx_zb=True, labels=labels_show)
print(f'pred boxes with scores in [{score_threshold0}, {score_threshold1}]')
Bbox3D.draw_points_bboxes(points, boxes_show, 'Z', is_yx_zb=True, labels=labels_show)
#print(f'anchors with scores in [{score_threshold0}, {score_threshold1}]')
#Bbox3D.draw_points_bboxes(points, anchors, 'Z', is_yx_zb=True)
pass
def unused_read_indoor_info():
info_path = f'{SPLITED_DIR}/sung_infos_train.pkl'
with open(info_path, 'rb') as f:
infos = pickle.load(f)
idx = 0
print(f'totally {len(infos)} blocks')
for idx in range(0,len(infos)):
info = infos[idx]
pcl_path = info['velodyne_path']
pointcloud_num_features = info['pointcloud_num_features']
points = np.fromfile( pcl_path, dtype=np.float32).reshape([-1, pointcloud_num_features])
annos = info['annos']
loc = annos['location']
dims = annos['dimensions']
rots = annos['rotation_y']
gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]], axis=1).astype(np.float32)
Bbox3D.draw_points_bboxes(points, gt_boxes, 'Z', is_yx_zb=True)
def pos_targets(labels, bbox_targets, anchors, is_show=False,\
gt_boxes=None, points=None):
anchors0 = anchors.copy()
anchors0[:,2] -= 5
if gt_boxes is None:
gt_boxes = np.zeros((0,7), dtype=np.float32)
else:
gt_boxes =gt_boxes.copy()
gt_boxes[:,2] -= 1
if points is None:
points = np.zeros((0,3), dtype=np.float32)
pos_ids = np.where(labels==1)[0]
neg_ids = np.where(labels==0)[0]
bbox_targets_pos = bbox_targets[pos_ids]
anchors_pos = anchors[pos_ids]
pos_num = pos_ids.shape[0]
gt_num = gt_boxes.shape[0]
anchor_num = anchors.shape[0]
if is_show:
print(f"gt_box_num:{gt_num} anchor_num: {anchor_num}")
print(f"all positive anchors: {pos_num}")
labels_show = np.array([0]*pos_num + [1]*gt_num)
boxes_show_0 = np.concatenate([anchors_pos, gt_boxes],0)
Bbox3D.draw_points_bboxes(points, boxes_show_0,
'Z', is_yx_zb=True, labels=labels_show)
# decoded anchors: shoud be exactly the gt_boxes
from second.core.box_np_ops import second_box_decode
bboxes_decoded = second_box_decode(bbox_targets_pos, anchors_pos, smooth_dim=True)
#Bbox3D.draw_bboxes(bboxes_decoded, 'Z', is_yx_zb=True)
#Bbox3D.draw_bboxes(gt_boxes, 'Z', is_yx_zb=True)
boxes_show = np.concatenate([bboxes_decoded, gt_boxes, anchors0], 0)
labels_show = np.array([0]*pos_num + [1]*gt_num + [2]*anchors0.shape[0])
if is_show:
print('gt boxes from targets')
Bbox3D.draw_points_bboxes(points, boxes_show, 'Z', is_yx_zb=True, labels=labels_show)
return bboxes_decoded
def show_pcl_boxes(pcl, boxes):
from utils3d.bbox3d_ops import Bbox3D
Bbox3D.draw_points_bboxes(pcl[:, 0:3], boxes, 'Z', is_yx_zb=True)
pass
def show_all(boxes_ls):
boxes = np.concatenate(boxes_ls, 0)
if boxes.shape[0] == 0:
return
Bbox3D.draw_points_bboxes(boxes[:, 0:3], boxes, 'Z', is_yx_zb=False)
def merge_pieces_of_same_walls_alongY(wall_bboxes):
'''
1) Find all the walls which are A) parallel along X, B) one of cen_corners is very close
C) centroid of one wall is inside of another
2) Merge along Y
'''
from utils3d.geometric_util import angle_with_x, vertical_dis_points_lines
if wall_bboxes.shape[0] == 0:
return wall_bboxes
show = False
if show:
wall_bboxes0 = wall_bboxes.copy()
wall_bboxes0[:, 2] -= 1
n = wall_bboxes.shape[0]
cen_lines = Bbox3D.bboxes_centroid_lines(wall_bboxes,
cen_axis='X',
up_axis='Z') # [n,2,3]
cen_direc = cen_lines[:, 1, :] - cen_lines[:, 0, :] # [n,3]
np.linalg.norm(cen_direc[:, 0:2], axis=1)
angles = angle_with_x(cen_direc[:, 0:2], 1)
boxes_merged_extra = []
remain_mask = np.array([True] * n)
merge_y_num = 0
split_and_merge_num = 0
for i in range(n - 1):
# 1) parallel along X
angles_dif = angles[i] - angles[i + 1:]
# make to [0,np.pi/2]
angles_dif = np.abs(limit_period(angles_dif, 0.5, np.pi))
angle_mask = angles_dif < 7 * np.pi / 180
# 2) one of centroid line corners is very close
cen_dis = cen_lines[i:i + 1].reshape(
[1, 2, 1, 3]) - cen_lines[i + 1:].reshape([-1, 1, 2, 3])
cen_dis = np.linalg.norm(cen_dis, axis=3)
cen_dis_min = cen_dis.min(axis=1).min(axis=1)
cen_dis_max = cen_dis.max(axis=1).max(axis=1)
thickness_sum = wall_bboxes[:, 4] * 0.5 + wall_bboxes[i, 4] * 0.5
thickness_sum = thickness_sum[i + 1:]
cendis_mask = cen_dis_min < thickness_sum
# 3) the other centroid line corner is inside of the long box
centroid_dis = np.linalg.norm(wall_bboxes[i:i + 1, 0:3] -
wall_bboxes[i + 1:, 0:3],
axis=1)
#tmp = np.maximum(wall_bboxes[i,3], wall_bboxes[i+1:,3])
tmp = (wall_bboxes[i, 3] + wall_bboxes[i + 1:, 3]) * 0.45 - 0.1
centroid_mask = centroid_dis < tmp
#if i==4:
# import pdb; pdb.set_trace() # XXX BREAKPOINT
# show_boxes = wall_bboxes[[4,12]]
# Bbox3D.draw_bboxes(show_boxes, 'Z', False)
# pass
mask_i = angle_mask * cendis_mask * centroid_mask
if mask_i.any():
ids = np.where(mask_i)[0] + i + 1
# 5) the centroid corner used to split the other
# 4) vertical dis
vertical_dis = vertical_dis_points_lines(cen_lines[i],
cen_lines[ids]).mean(0)
ave_thickness = (wall_bboxes[i, 4] + wall_bboxes[ids, 4]) * 0.5
thickness_gap_rate = vertical_dis / ave_thickness
verdis_mask = (thickness_gap_rate > 0.2) * (thickness_gap_rate <
1.2)
ids = ids[verdis_mask]
vertical_dis = vertical_dis[verdis_mask]
is_need_merge = ids.shape[0] > 0
if is_need_merge:
# split the longer box, and merge the shorter one along Y
k = ids.shape[0]
for j in range(k):
idx = ids[j]
size_x_i = wall_bboxes[i, 3]
size_x_j = wall_bboxes[idx, 3]
size_x_rate = size_x_i / size_x_j
print(f'size_x_rate: {size_x_rate}')
if np.abs(size_x_rate - 1) < 0.15:
merge_y_num += 1
box_merge = merge_2pieces_of_1wall(
wall_bboxes[i], wall_bboxes[idx], 'Y')
if show and False:
show_boxes = np.concatenate([
wall_bboxes[i:i + 1], wall_bboxes[idx:idx + 1]
], 0)
if box_merge is not None:
show_boxes = np.concatenate(
[show_boxes, box_merge], 0)
show_boxes[-1, 2] += 0.2
else:
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
show_boxes = np.concatenate(
[show_boxes, wall_bboxes0], 0)
Bbox3D.draw_bboxes(show_boxes, 'Z', False)
pass
if box_merge is not None:
wall_bboxes[idx] = box_merge.reshape([7])
remain_mask[i] = False
else:
print(
'The two walls cannot be merged along Y, this should not happen normally'
)
print(
'merge_pieces_of_same_walls_alongY again after crop_walls to solve some special case: Originally, a wall is broken by no intersection like 0058113bdc8bee5f387bb5ad316d7b28'
)
#show_boxes = np.concatenate([wall_bboxes[i:i+1], wall_bboxes[idx:idx+1]], 0)
#Bbox3D.draw_bboxes(wall_bboxes, 'Z', False, highlight_ids=[idx, i])
#Bbox3D.draw_bboxes(show_boxes, 'Z', False)
#import pdb; pdb.set_trace() # XXX BREAKPOINT
#pass
#raise NotImplementedError
else:
# the longer box need to be split along X before merging
split_and_merge_num += 1
cen_dis_ij = cen_dis[idx - i - 1]
if size_x_rate > 1:
longer_idx = i
short_idx = idx
cen_dis_ij = cen_dis_ij.min(axis=0)
else:
longer_idx = idx
short_idx = i
cen_dis_ij = cen_dis_ij.min(axis=1)
# find the intersection point on longer box
far_corner_id = int(cen_dis_ij[0] < cen_dis_ij[1])
cen_dir_longer = cen_lines[longer_idx,
1] - cen_lines[longer_idx,
0]
cen_dir_longer /= np.linalg.norm(cen_dir_longer)
sl_dir = cen_lines[
short_idx, far_corner_id] - cen_lines[longer_idx,
0]
intersection = np.sum(
sl_dir * cen_dir_longer
) * cen_dir_longer + cen_lines[longer_idx, 0]
# offset: half of the thickness
intersection += cen_dir_longer * wall_bboxes[i,
4] * 0.5
splited_boxes = Bbox3D.split_wall_by_centroid_intersections(
wall_bboxes[longer_idx],
intersection.reshape([1, 3])) # [2,7]
if splited_boxes.shape[0] == 1:
print(
'\n\n\t\tComplicated situation, not solved well yet.\n\n'
)
box_merge = None
if False and DEBUG and splited_boxes.shape[0] == 1:
box_tmp = np.array(
[[0, 0, 0, 0.5, 0.5, 0.5, 0]])
box_tmp[0, 0:3] = intersection.reshape([1, 3])
boxes_show = np.concatenate([
box_tmp, wall_bboxes[longer_idx].reshape(
[-1, 7])
], 0)
Bbox3D.draw_points_bboxes(
intersection.reshape([1, 3]), boxes_show,
'Z', False)
else:
tmp = wall_bboxes[short_idx,
0:3] - splited_boxes[:, 0:
3] # [2,3]
tmp = np.linalg.norm(tmp, axis=1) # [2]
merge_id = int(tmp[0] > tmp[1])
box_merge = merge_2pieces_of_1wall(
wall_bboxes[short_idx],
splited_boxes[merge_id], 'Y')
if show and False:
#if box_merge is None:
show_boxes = np.concatenate([
wall_bboxes[i:i + 1], wall_bboxes[idx:idx + 1]
], 0)
Bbox3D.draw_points_bboxes(
intersection.reshape([1, 3]), show_boxes, 'Z',
False)
show_boxes = np.concatenate(
[show_boxes, splited_boxes], 0)
show_boxes[-1, 2] += 0.5
show_boxes[-2, 2] += 0.7
if box_merge is not None:
show_boxes = np.concatenate(
[show_boxes, box_merge], 0)
show_boxes[-1, 2] += 1
show_boxes = np.concatenate(
[show_boxes, wall_bboxes0], 0)
Bbox3D.draw_bboxes(show_boxes, 'Z', False)
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
if box_merge is None:
# temperally solution, may because of a wall being changed several
# times
remain_mask[short_idx] = False
else:
wall_bboxes[longer_idx] = splited_boxes[1 -
merge_id]
wall_bboxes[short_idx] = box_merge.reshape([7])
boxes_merged_extra.append(box_merge.reshape([1,
7]))
wall_bboxes_new = wall_bboxes[remain_mask]
#boxes_merged_extra = np.concatenate(boxes_merged_extra, 0)
#wall_bboxes_new = np.concatenate([wall_bboxes, boxes_merged_extra], 0)
print(
f'merge_y_num:{merge_y_num} split_and_merge_num:{split_and_merge_num}'
)
if show:
show_walls_offsetz(wall_bboxes_new)
#show_walls_offsetz(wall_bboxes)
#show_walls_offsetz(wall_bboxes0[remain_mask])
show_walls_offsetz(wall_bboxes0[np.logical_not(remain_mask)])
#show_walls_offsetz(boxes_merged_extra)
return wall_bboxes_new
def render_parsed_house_walls(parsed_dir, show_pcl=SHOW_PCL, show_by_class=0):
print(f'parsed_dir:{parsed_dir}')
house_name = os.path.basename(parsed_dir)
bboxes = []
labels = []
for obj in CLASSES:
bbox_fn_ = f'{parsed_dir}/object_bbox/{obj}.txt'
bboxes_ = np.loadtxt(bbox_fn_).reshape([-1, 7])
bboxes.append(bboxes_)
label = DSET_METAS0.class_2_label[obj]
labels += [label] * bboxes_.shape[0]
bboxes = np.concatenate(bboxes, 0)
labels = np.array(labels).astype(np.int8)
if bboxes.shape[0] > 0:
scene_size = Bbox3D.boxes_size(bboxes)
print(f'scene wall size:{scene_size}')
#Bbox3D.draw_bboxes(bboxes, up_axis='Z', is_yx_zb=False, labels=labels)
#if not show_pcl:
Bbox3D.draw_bboxes_mesh(bboxes, up_axis='Z', is_yx_zb=False)
#Bbox3D.draw_bboxes_mesh(bboxes, up_axis='Z', is_yx_zb=False, labels=labels)
#show_walls_offsetz(bboxes)
if show_by_class:
for c in range(1, max(labels) + 1):
cs = DSET_METAS0.label_2_class[c]
print(cs)
if cs not in ['wall', 'window', 'door']:
#if cs not in ['wall']:
continue
mask = labels == c
bboxes_c = bboxes[mask]
show_walls_offsetz(bboxes_c)
if show_pcl:
pcl_fn = f'{parsed_dir}/pcl_camref.ply'
if not os.path.exists(pcl_fn):
return
pcd = open3d.read_point_cloud(pcl_fn)
points = np.asarray(pcd.points)
points = cam2world_pcl(points)
colors = np.asarray(pcd.colors)
pcl = np.concatenate([points, colors], 1)
scene_size = pcl_size(pcl)
print(f'scene pcl size:{scene_size}')
print(f'point num: {pcl.shape[0]}')
#pcl = cut_points_roof(pcl)
#Bbox3D.draw_points(pcl, points_keep_rate=POINTS_KEEP_RATE)
#Bbox3D.draw_points(pcl, points_keep_rate=POINTS_KEEP_RATE, animation_fn='points.mp4', ani_size=AniSizes[house_name])
bboxes[:, 2] += 0.1
Bbox3D.draw_points_bboxes(pcl,
bboxes,
up_axis='Z',
is_yx_zb=False,
points_keep_rate=POINTS_KEEP_RATE)
Bbox3D.draw_points_bboxes_mesh(pcl,
bboxes,
up_axis='Z',
is_yx_zb=False,
points_keep_rate=POINTS_KEEP_RATE)
def split_bbox(bbox_fn, points_splited):
'''
bbox in file bbox_fn: up_axis='Y' with always x_size > z_size
transform with cam2world_box:
up_axis == 'Z'
always: x_size > y_size
'''
obj = os.path.basename(bbox_fn).split('.')[0]
min_point_num_per1sm = 10
# thickness_aug for cropping x
thickness_aug = 0.3
assert IndoorData._block_size0[-1] == -1 # do not crop box along z
bboxes = np.loadtxt(bbox_fn).reshape([-1,7])
#if DEBUG:
# #show_walls_1by1(bboxes)
# bboxes = bboxes[3:4]
areas = bboxes[:,3] * bboxes[:,5]
min_point_num = np.minimum( min_point_num_per1sm * areas, 200 )
bboxes_aug = bboxes.copy()
#bboxes_aug[:,4] += thickness_aug
bboxes_aug[:,3:6] = np.clip(bboxes_aug[:,3:6],a_min= thickness_aug, a_max=None )
bn = bboxes.shape[0]
sn = len(points_splited)
bboxes_splited = []
for i in range(0, sn):
# Use to constrain size_x size_z
point_masks_aug_i = Bbox3D.points_in_bbox(points_splited[i][:,0:3].copy(), bboxes_aug.copy())
# Use to constrain size_y (the thickness)
bboxes_tc = IndoorData.adjust_box_for_thickness_crop(bboxes)
point_masks_i = Bbox3D.points_in_bbox(points_splited[i][:,0:3].copy(), bboxes_tc)
pn_in_box_aug_i = np.sum(point_masks_aug_i, 0)
pn_in_box_i = np.sum(point_masks_i, 0)
#print(f'no aug:{pn_in_box_i}\n auged:{pn_in_box_aug_i}')
# (1) The bboxes with no points with thickness_aug will be removed firstly
keep_box_aug_i = pn_in_box_aug_i > min_point_num
bboxes_i = bboxes[keep_box_aug_i]
if DEBUG and obj=='ceiling' and 0:
rm_box_aug_i = pn_in_box_aug_i <= min_point_num
print(rm_box_aug_i)
bboxes_no_points_i = bboxes[rm_box_aug_i].copy()
bboxes_no_points_i[:,0] += 30
bboxes_show = np.concatenate([bboxes_i, bboxes_no_points_i],0)
Bbox3D.draw_points_bboxes(points_splited[i], bboxes_show, up_axis='Z', is_yx_zb=False)
points_aug_i = [points_splited[i][point_masks_aug_i[:,j]] for j in range(bn)]
points_aug_i = [points_aug_i[j] for j in range(bn) if keep_box_aug_i[j]]
points_i = [points_splited[i][point_masks_i[:,j]] for j in range(bn)]
points_i = [points_i[j] for j in range(bn) if keep_box_aug_i[j]]
# (2) Crop all the boxes by points and intersec_corners seperately
bn_i = bboxes_i.shape[0]
croped_bboxes_i = []
keep_unseen_intersection = False
if keep_unseen_intersection:
intersec_corners_idx_i, intersec_corners_i = Bbox3D.detect_all_intersection_corners(bboxes_i, 'Z')
else:
intersec_corners_idx_i = [None]*bn_i
invalid_bn = 0
for k in range(bn_i):
croped_box_k = Bbox3D.crop_bbox_by_points(
bboxes_i[k], points_i[k], points_aug_i[k], 'Z', intersec_corners_idx_i[k])
if croped_box_k is not None:
croped_bboxes_i.append( croped_box_k )
else:
invalid_bn += 1
#Bbox3D.draw_points_bboxes(points_splited[i], bboxes_i[k:k+1], up_axis='Z', is_yx_zb=False, points_keep_rate=1.0)
pass
if len(croped_bboxes_i) > 0:
croped_bboxes_i = np.concatenate(croped_bboxes_i, 0)
else:
croped_bboxes_i = np.array([]).reshape([-1,7])
# (3) Refine x size of each bbox by thickness croped of intersected bbox
#croped_size = bboxes_i[:,3:6] - croped_bboxes_i[:,3:6]
refine_x_by_intersection = False # not correct yet
if refine_x_by_intersection:
for k in range(bn_i):
itsc0, itsc1 = intersec_corners_idx_i[k]
crop_value = [0,0]
if itsc0 >= 0:
crop_value[0] = Bbox3D.refine_intersection_x(croped_bboxes_i[k], 'neg', croped_bboxes_i[itsc0], 'Z')
if itsc1 >= 0:
crop_value[1] = Bbox3D.refine_intersection_x(croped_bboxes_i[k], 'pos', croped_bboxes_i[itsc1], 'Z')
if itsc0 >= 0 or itsc1 > 0:
croped_bboxes_i[k] = Bbox3D.crop_bbox_size(croped_bboxes_i[k], 'X', crop_value)
#crop_ysize_neg = croped_size[itsc0,1] if itsc0 >= 0 else 0
#crop_ysize_pos = croped_size[itsc1,1] if itsc1 >= 0 else 0
#croped_bboxes_i[k] = Bbox3D.crop_bbox_size(croped_bboxes_i[k], 'X', [crop_ysize_neg, crop_ysize_pos])
# (4) remove too small wall
min_wall_size_x = 0.2
sizex_mask = croped_bboxes_i[:,3] > min_wall_size_x
croped_bboxes_i = croped_bboxes_i[sizex_mask]
bboxes_splited.append(croped_bboxes_i)
show = False
if show and DEBUG and len(points_i) > 0 and obj=='ceiling':
print(croped_bboxes_i[:,3])
points = np.concatenate(points_i,0)
points = points_splited[i]
points1 = points.copy()
points1[:,0] += 30
points = np.concatenate([points, points1], 0)
bboxes_i[:,0] += 30
bboxes_i = np.concatenate([bboxes_i, croped_bboxes_i], 0)
#Bbox3D.draw_points_bboxes(points, bboxes_i, up_axis='Z', is_yx_zb=False)
Bbox3D.draw_points_bboxes_mesh(points, bboxes_i, up_axis='Z', is_yx_zb=False)
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
return bboxes_splited
def split_scene(scene_dir, splited_path):
from suncg_utils.suncg_preprocess import check_house_intact, read_summary, write_summary
scene_name = os.path.basename(scene_dir)
splited_path = os.path.join(splited_path, scene_name)
summary_0 = read_summary(splited_path)
always_update = ALWAYS_UPDATE
#house_intact, intacts = check_house_intact(scene_dir)
#if not house_intact:
# return
summary_raw = read_summary(scene_dir)
is_big_size = (summary_raw['scene_size'] > MAX_SCENE_SIZE).any()
always_update = always_update or ( ALWAYS_BIG_SIZE and is_big_size )
if (not always_update) and 'split_num' in summary_0:
sn = summary_0['split_num'].squeeze()
still_split = ALWAYS_UPDATE_MULTI_SPLITS and sn >1
if not still_split:
print(f'skip {splited_path}')
return
print(f'spliting {scene_dir}')
gen_ply = False
if 'level_num' in summary_raw and summary_raw['level_num'] != 1:
return
pcl_fn = os.path.join(scene_dir, 'pcl_camref.ply')
if not os.path.exists(pcl_fn):
print(f'pcl.ply not exist, skip {scene_dir}')
return
fns = glob.glob(os.path.join(splited_path, '*.pth'))
if ONLY_MODIFY_BOX and len(fns)>0:
print('Load points from pth directly')
scene_name = os.path.basename(os.path.dirname(fns[0]))
is_special_scene = IndoorData.is_a_special_scene(scene_name)
points_splited = []
for fni in fns:
pcl_i, boxes_i = torch.load(fni)
points_splited.append(pcl_i)
else:
points_splited, is_special_scene = IndoorData.split_pcl_plyf(pcl_fn)
n_block = len(points_splited)
bboxes_splited = {}
boxes_nums = {}
for obj in CLASSES_USED:
bbox_fn = os.path.join(scene_dir, f'object_bbox/{obj}.txt')
if os.path.exists(bbox_fn):
if is_special_scene or n_block > 1:
bboxes_splited[obj] = IndoorData.split_bbox(bbox_fn, points_splited)
else:
bboxes_splited[obj] = IndoorData.load_bboxes(bbox_fn)
boxes_nums[obj] = []
for ii in range(n_block):
boxes_nums[obj].append( len(bboxes_splited[obj][ii]) )
if not os.path.exists(splited_path):
os.makedirs(splited_path)
for i in range(n_block):
boxes_num_all_classes = sum([bn[i] for bn in boxes_nums.values()])
if n_block>1 and boxes_num_all_classes < MIN_BOXES_NUM:
continue
fni = splited_path + '/pcl_%d.pth'%(i)
pcl_i = points_splited[i].astype(np.float32)
#offset = pcl_i[:,0:3].mean(0)
#pcl_i[:,0:3] = pcl_i[:,0:3] - offset
pcl_i = np.ascontiguousarray(pcl_i)
boxes_i = {}
for obj in bboxes_splited:
boxes_i[obj] = bboxes_splited[obj][i].astype(np.float32)
if obj in ['ceiling', 'floor', 'room']:
boxes_i[obj] = Bbox3D.set_yaw_zero(boxes_i[obj])
#boxes_i[obj] = preprocess_cfr_standard(boxes_i[obj])
torch.save((pcl_i, boxes_i), fni)
if gen_ply:
Bbox3D.draw_points_bboxes(pcl_i, boxes_i['wall'], 'Z', False)
pclfn_i = splited_path + f'/pcl_{i}.ply'
points_ply(pcl_i[:,0:3], pclfn_i)
boxfn_i = splited_path + f'/wall_{i}.ply'
Bbox3D.save_bboxes_ply(boxfn_i, boxes_i['wall'], 'Z')
print(f'save {fni}')
write_summary(splited_path, 'split_num', n_block, 'w')
def show_all_valid_anchors(anchors, anchors_mask, points): anchors = anchors[anchors_mask] Bbox3D.draw_points_bboxes(points, anchors, 'Z', is_yx_zb=True)
def render_pth_file(pth_fn, show_by_class=0):
pcl, bboxes0 = torch.load(pth_fn)
pcl = down_sample_points(pcl)
if 'clutter' in bboxes0:
del bboxes0['clutter']
#points = pcl[:,0:3]
#colors = pcl[:,3:6]
#normals = pcl[:,6:9]
if CLASSES is None:
bboxes = bboxes0
else:
bboxes = {}
for c in CLASSES:
if c in bboxes0:
bboxes[c] = bboxes0[c]
scene_size = pcl_size(pcl)
print(f'scene pcl size:{scene_size}')
print(f'point num: {pcl.shape[0]}')
#pcl = cut_points_roof(pcl)
classes = [k for k in bboxes.keys()]
num_classes = {k: bboxes[k].shape[0] for k in bboxes.keys()}
print(f'\nclasses: {num_classes}\n\n')
all_bboxes = np.concatenate([boxes for boxes in bboxes.values()], 0)
nums = [boxes.shape[0] for boxes in bboxes.values()]
labels = []
for i, n in enumerate(nums):
labels += [i] * n
labels = np.array(labels)
#Bbox3D.draw_points(pcl, points_keep_rate=POINTS_KEEP_RATE)
#show_walls_offsetz(all_bboxes)
#Bbox3D.draw_bboxes_mesh(all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels)
#Bbox3D.draw_bboxes_mesh(all_bboxes, up_axis='Z', is_yx_zb=False)
#Bbox3D.draw_points_bboxes_mesh(pcl, all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels, points_keep_rate=POINTS_KEEP_RATE)
#Bbox3D.draw_points_bboxes_mesh(pcl, all_bboxes, up_axis='Z', is_yx_zb=False, points_keep_rate=POINTS_KEEP_RATE)
#Bbox3D.draw_points_bboxes(pcl, all_bboxes, up_axis='Z', is_yx_zb=False,points_keep_rate=POINTS_KEEP_RATE)
Bbox3D.draw_points_bboxes(pcl,
all_bboxes,
up_axis='Z',
is_yx_zb=False,
labels=labels,
points_keep_rate=POINTS_KEEP_RATE)
#Bbox3D.draw_points_bboxes(pcl, all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels, points_keep_rate=POINTS_KEEP_RATE, animation_fn='anima.mp4', ani_size=[280,700,550,1350])
#Bbox3D.draw_bboxes(all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels)
#boxlist = BoxList3D(all_bboxes, size3d=None, mode='standard', examples_idxscope=None, constants={})
#boxlist.show_with_corners()
#show_walls_offsetz(all_bboxes)
if show_by_class:
for clas in bboxes.keys():
#if clas not in ['wall']:
#if clas not in ['wall', 'window', 'door','ceiling', 'floor']:
# continue
#if clas not in CLASSES:
# continue
boxes = bboxes[clas]
bn = boxes.shape[0]
print(clas, f'num={bn}')
#Bbox3D.draw_points_bboxes(points, boxes, up_axis='Z', is_yx_zb=False)
#Bbox3D.draw_points_bboxes_mesh(pcl, boxes, up_axis='Z', is_yx_zb=False, points_keep_rate=POINTS_KEEP_RATE)
try:
Bbox3D.draw_points_bboxes(pcl,
boxes,
up_axis='Z',
is_yx_zb=False,
points_keep_rate=POINTS_KEEP_RATE)
except:
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
#show_walls_offsetz(boxes)
pass
def show_points_bbox(points, boxes, labels=None, names='', image_path=''):
from utils_3d.bbox3d_ops import Bbox3D
#print("\timage: ", image_path)
Bbox3D.draw_points_bboxes(points, boxes, up_axis='Z', labels=labels, names=names, is_yx_zb=True)
def get_part_bbox(vertices, triangle, triangle_norms, name=''):
'''
bbox: [xc, yc, zc, x_size, y_size, z_size, yaw]
'''
#show_mesh(vertices, triangle)
class_name = name.split('#')[0]
box_min = np.min(vertices, 0)
box_max = np.max(vertices, 0)
centroid = (box_min + box_max) / 2.0
y_size = box_max[1] - box_min[1]
## Find the 8 outside corners
distances = np.linalg.norm(vertices - np.expand_dims(centroid, 0), axis=1)
max_dis = max(distances)
out_corner_mask = (abs(distances - max_dis) < 1e-5)
n0 = vertices.shape[0]
#print([i for i in range(n0) if out_corner_mask[i]])
out_vertices = [
vertices[i:i + 1, :] for i in range(n0) if out_corner_mask[i]
]
if len(out_vertices) == 0:
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
return None
out_vertices = np.concatenate(out_vertices, 0)
if out_vertices.shape[0] != 8:
#Bbox3D.draw_points_open3d(out_vertices, show=True)
#Bbox3D.draw_points_open3d(vertices, show=True)
#show_mesh(vertices, triangle)
if class_name not in ENABLE_NO_RECTANGLE:
print(
f'\nFailed to find bbox, not rectangle, {class_name} \n {out_vertices.shape[0]} vertices\n'
)
assert False
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
return None
else:
print(
f'\nNot rectangle, use no yaw box, {class_name} \n {out_vertices.shape[0]} vertices\n'
)
min_max = {'min': box_min, 'max': box_max}
bbox_noyaw = Bbox3D.bbox_from_minmax(min_max)
#Bbox3D.draw_points_bboxes(vertices, bbox_noyaw, 'Y', is_yx_zb=False)
return bbox_noyaw
## Find the 4 corners on one side
x_right_mask = out_vertices[:, 0] - centroid[0] > 0
x_right_corners = np.concatenate(
[out_vertices[i:i + 1, :] for i in range(8) if x_right_mask[i]], 0)
assert x_right_corners.shape[0] == 4
x_right_cen = np.mean(x_right_corners, 0)
## get length
x_size = np.linalg.norm(x_right_cen - centroid) * 2
if not abs(x_right_cen[1] - centroid[1]) < 1e-7:
print("y should be same, pitch and rool should be 0")
## the angle between x_right_cen and x axis
x_direc = x_right_cen - centroid
x_direc = (x_direc) / np.linalg.norm(x_direc)
x_axis = np.array([1, 0, 0])
# [-90, 90]
yaw = np.arccos(np.sum(x_direc * x_axis))
if abs(yaw) < 0.01:
yaw = 0
else:
assert x_direc[0] > 0
assert x_direc[1] == 0
yaw *= np.sign(x_direc[2])
## Find the 2 corners at top within x_right_corners
top_mask = x_right_corners[:, 1] - centroid[1] > 0
x_right_top_corners = np.concatenate(
[x_right_corners[i:i + 1, :] for i in range(4) if top_mask[i]], 0)
assert x_right_top_corners.shape[0] == 2
# get z_size
z_size = np.linalg.norm(x_right_top_corners[0] - x_right_top_corners[1])
### got the bbox
xc, yc, zc = centroid
bbox = np.array([xc, yc, zc, x_size, y_size, z_size, yaw])
### Check
if yaw != 0 and False:
Bbox3D.draw_bboxes(bbox, 'Y', False)
if False:
min_max = {'min': box_min, 'max': box_max}
bbox_noyaw = Bbox3D.bbox_from_minmax(min_max)
bboxes_show = np.concatenate(
[bbox.reshape([1, 7]),
bbox_noyaw.reshape([1, 7])], 0)
Bbox3D.draw_points_bboxes(vertices, bboxes_show, 'Y', is_yx_zb=False)
return bbox
def show_all(boxes_ls):
boxes = np.concatenate(boxes_ls, 0)
Bbox3D.draw_points_bboxes(boxes[:, 0:3], boxes, 'Z', is_yx_zb=False)