def show_final_preds(pred_boxes, points, gt_boxes): pred_boxes = pred_boxes.data.cpu().numpy() boxes_show = np.concatenate([pred_boxes, gt_boxes], 0) pnum = pred_boxes.shape[0] gnum = gt_boxes.shape[0] labels_show = np.array([0]*pnum + [1]*gnum) Bbox3D.draw_points_bboxes(points, boxes_show, 'Z', is_yx_zb=True, labels=labels_show)
def show_pcl_corners(self, pcl): import numpy as np from utils3d.bbox3d_ops import Bbox3D corners, _ = self.get_2top_corners_offseted() corners = corners.view([-1, 3]).cpu().data.numpy() corners_bot = corners.copy() tmp = self.bbox3d[:, 5].cpu().data.numpy() tmp = np.tile(np.expand_dims(tmp, 1), [1, 2]).reshape(-1) corners_bot[:, 2] = corners_bot[:, 2] - tmp corners = np.concatenate([corners, corners_bot], 0) n = corners.shape[0] boxes = np.zeros([n, 7]) boxes[:, 0:3] = corners boxes[:, 3:6] = 0.2 #points = pcl.cpu().data.numpy() points = pcl Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode == 'standard', random_color=True) import pdb pdb.set_trace() # XXX BREAKPOINT pass
def show(self, max_num=-1, points=None, with_centroids=False, boxes_show_together=None, points_keep_rate=POINTS_KEEP_RATE, points_sample_rate=POINTS_SAMPLE_RATE, colors=None): import numpy as np from utils3d.bbox3d_ops import Bbox3D boxes = self.bbox3d.cpu().data.numpy() if with_centroids: centroids = boxes.copy() if self.mode == 'yx_zb': centroids[:,2] += centroids[:,5]*0.5 centroids[:,3:6] = 0.03 if max_num > 0 and max_num < boxes.shape[0]: step = 4 ids0 = np.random.choice(boxes.shape[0]//step-1, max_num, replace=False).reshape(-1,1)*step tmp = np.arange(step).reshape(1,step) ids = (ids0 + tmp).reshape(-1) print(ids) boxes = boxes[ids] if with_centroids: boxes = np.concatenate([boxes, centroids], 0) if boxes_show_together: boxes_show_together = boxes_show_together.bbox3d.cpu().data.numpy() labels = np.array( [0]*boxes.shape[0] + [1]*boxes_show_together.shape[0]) boxes = np.concatenate([boxes, boxes_show_together], 0) else: if 'labels' in self.fields(): labels = self.get_field('labels').cpu().data.numpy().astype(np.int32) else: labels = None if points is None: Bbox3D.draw_bboxes(boxes, 'Z', is_yx_zb=self.mode=='yx_zb', \ labels = labels, random_color=False) else: Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb',\ labels = labels, random_color=False, points_keep_rate=points_keep_rate, points_sample_rate=points_sample_rate, box_colors=colors)
def show_pcl_boxdic(pcl, bboxes_dic): from utils3d.bbox3d_ops import Bbox3D boxes = [] for obj in bboxes_dic: print(f'{obj}: {len(bboxes_dic[obj])}') boxes.append(bboxes_dic[obj]) boxes = np.concatenate(boxes, 0) Bbox3D.draw_points_bboxes(pcl[:,0:6], boxes, 'Z', is_yx_zb=True) Bbox3D.draw_points_bboxes(pcl[:,0:6], bboxes_dic['wall'], 'Z', is_yx_zb=True) import pdb; pdb.set_trace() # XXX BREAKPOINT pass
def show__together(self, boxlist_1, max_num=-1, max_num_1=-1, points=None, offset_x=None, twolabels=False, mesh=False, points_keep_rate=POINTS_KEEP_RATE, points_sample_rate=POINTS_SAMPLE_RATE, random_color=False, colors=None): import numpy as np from utils3d.bbox3d_ops import Bbox3D boxes = self.bbox3d.cpu().data.numpy().copy() if max_num > 0 and max_num < boxes.shape[0]: ids = np.random.choice(boxes.shape[0], max_num, replace=False) boxes = boxes[ids] boxes_1 = boxlist_1.bbox3d.cpu().data.numpy().copy() if max_num_1 > 0 and max_num_1 < boxes_1.shape[0]: ids = np.random.choice(boxes_1.shape[0], max_num_1, replace=False) boxes_1 = boxes_1[ids] if offset_x is not None: boxes_1[:,0] += offset_x if not twolabels and 'labels' in self.fields(): labels = self.get_field('labels').cpu().data.numpy().astype(np.int32) labels_1 = boxlist_1.get_field('labels').cpu().data.numpy().astype(np.int32) labels = np.concatenate([labels, labels_1], 0) else: labels = np.array([0]*boxes.shape[0] + [1]*boxes_1.shape[0]) boxes = np.concatenate([boxes, boxes_1], 0) if colors is not None: colors = np.concatenate(colors, 0) assert colors.shape[0] == boxes.shape[0] if points is None: if mesh: Bbox3D.draw_bboxes_mesh(boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels) else: Bbox3D.draw_bboxes(boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, random_color=False) else: if isinstance(points, torch.Tensor): points = points.cpu().data.numpy() if offset_x is not None: tp = points.copy() tp[:,0] += offset_x points = np.concatenate([points, tp], 0) # tp = tp.copy() tp[:,0] += offset_x points = np.concatenate([points, tp], 0) if mesh: Bbox3D.draw_points_bboxes_mesh(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, points_keep_rate=points_keep_rate, points_sample_rate=points_sample_rate, random_color=random_color, box_colors=colors) else: Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, random_color=random_color, points_keep_rate=points_keep_rate, points_sample_rate=points_sample_rate, box_colors=colors)
def show_with_corners(self, only_corner=0): import numpy as np from utils3d.bbox3d_ops import Bbox3D corners, _ = self.get_2top_corners_offseted() corners = corners.view([-1, 3]) points = corners.cpu().data.numpy() boxes = self.bbox3d.cpu().data.numpy() if only_corner: boxes = boxes[0:0, :] Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb',\ random_color=True) pass
def show_highlight(self, ids, points=None): from utils3d.bbox3d_ops import Bbox3D ids = np.array(ids) n = len(self) labels = np.zeros([n]).astype(np.int) labels[ids] = 1 boxes = self.bbox3d.cpu().data.numpy() if points is None: Bbox3D.draw_bboxes(boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, random_color=False) else: if isinstance(points, torch.Tensor): points = points.cpu().data.numpy() Bbox3D.draw_points_bboxes(points, boxes, 'Z', is_yx_zb=self.mode=='yx_zb', labels=labels, random_color=False)
def is_edge_wall_of_ceiling(wall_cenlines, ceiling, walls_org): c_corners = Bbox3D.bbox_corners(ceiling, 'Z') c_lines = np.take(c_corners, Bbox3D._lines_z0_vids, axis=0) c_lines[:, :, 2] = 0 wall_cenlines_ = np.concatenate( [wall_cenlines, wall_cenlines.mean(1, keepdims=True)], 1) wn = wall_cenlines.shape[0] in_mask0 = points_in_lines(wall_cenlines_.reshape([-1, 3]), c_lines, threshold_dis=0.1) # [wn*3,4] in_mask0 = in_mask0.reshape([wn, 3, 4]) # [wn,3,4] # find all wdge walls in_mask1 = in_mask0.any(2) # [wn,3] in any of four edge of a ceiling in_state = in_mask1.sum(1) winc_mask = in_state >= 2 # [wn] the number of points on edge: 0~3. two of three: two corners, one centroid winc_ids = np.where(winc_mask)[0] # clean repeat edge walls on same edge: only at most one edge wall on each # edge allowed if winc_ids.shape[0] == 0: winc_ids_1 = winc_ids winc_state = [] else: keep_ids = clean_extend_lines(wall_cenlines[winc_ids]) winc_ids_1 = winc_ids[keep_ids] winc_state = in_state[winc_ids_1] edge_wall_num = winc_ids_1.shape[0] if Debug and edge_wall_num < 2 and 0: #if Debug: print(f'edge_wall_num: {edge_wall_num}') boxes = np.concatenate([ceiling.reshape([1, -1]), walls_org[winc_ids]], 0) wo = walls_org.copy() wo[:, 2] -= 2 #boxes = np.concatenate([boxes, wo],0) #boxes = ceiling.reshape([1,-1]) points = np.concatenate( [wall_cenlines_.reshape([-1, 3]), c_lines.reshape([-1, 3])], 0) Bbox3D.draw_points_bboxes(points, boxes, 'Z', False) boxes = np.concatenate( [ceiling.reshape([1, -1]), walls_org[winc_ids_1]], 0) Bbox3D.draw_points_bboxes(points, boxes, 'Z', False) #show_walls_1by1(walls_org) return edge_wall_num, winc_state
def crop_walls(wall_bboxes): ''' crop walls with intersections not on the corner ''' #show_walls_1by1(wall_bboxes) if wall_bboxes.shape[0] == 0: return wall_bboxes intersections = Bbox3D.all_intersections_by_cenline( wall_bboxes, check_same_height=False, not_on_corners=True) n = wall_bboxes.shape[0] new_walls = [] keep_mask = np.array([True] * n) for i in range(n): inters_i = intersections[i] if inters_i.shape[0] == 0: continue new_walls_i = Bbox3D.split_wall_by_centroid_intersections( wall_bboxes[i], inters_i) keep_mask[i] = False new_walls.append(new_walls_i) show = False #tmp = np.concatenate(new_walls, 0) #if tmp.shape[0] >= 7: # show = True if show: #tmp = wall_bboxes.copy() tmp = new_walls_i.copy() tmp[:, 2] += 1 for ti in range(1, tmp.shape[0]): tmp[ti:, 2] += 0.2 show_box = np.concatenate([tmp, wall_bboxes[i:i + 1]], 0) Bbox3D.draw_points_bboxes(inters_i, show_box, 'Z', False) #Bbox3D.draw_bboxes(show_box, 'Z', False) import pdb pdb.set_trace() # XXX BREAKPOINT pass num_croped = np.sum(keep_mask == False) print(f'num croped:{num_croped}\nnum new:{len(new_walls)}') wall_bboxes = wall_bboxes[keep_mask] if len(new_walls) > 0: new_walls = np.concatenate(new_walls, 0) wall_bboxes = np.concatenate([wall_bboxes, new_walls], 0) #show_walls_1by1(wall_bboxes) #show_walls_offsetz(wall_bboxes) return wall_bboxes
def show_anchors(points, anchors, gt_boxes, anchors_mask=None): # show valid anchors if anchors_mask is not None: indices = np.where(anchors_mask)[0] anchors = anchors[indices,:] show_num = -1 if show_num>0 and show_num < anchors.shape[0]: indices = np.random.choice(anchors.shape[0], show_num) anchors = anchors[indices,:] n0 = anchors.shape[0] n1 = gt_boxes.shape[0] anchors = np.concatenate([anchors, gt_boxes], 0) labels = np.array([1]*n0 + [0]*n1) Bbox3D.draw_points_bboxes(points, anchors, 'Z', is_yx_zb=True, labels=labels)
def forcely_crop_scene(pcl0, walls, ceilings): scene_min = pcl0[:,:3].min(0) scene_max = pcl0[:,:3].max(0) scene_size = scene_max - scene_min abandon = scene_size - MAX_SCENE_SIZE masks = [] ref_boxes = [walls, walls, ceilings] for i in range(3): if abandon[i] > 0: if ref_boxes[i].shape[0] == 0: if i<2: rate_min = 0.5 else: rate_min = 0 else: ref_min = ref_boxes[i][:,i].min() ref_max = ref_boxes[i][:,i].max() wmin = ref_min - scene_min[i] wmax = scene_max[i] - ref_max rate_min = wmin / (wmin+wmax) new_mini = scene_min[i] + abandon[i] * rate_min new_maxi = scene_max[i] - abandon[i] * (1-rate_min) masks.append( (pcl0[:,i] > new_mini) * (pcl0[:,i] < new_maxi) ) if len(masks) > 0: mask = masks[0] for m in masks: mask = mask * m pcl1 = pcl0[mask] else: pcl1 = pcl0 show = 0 if show: scene_min_new = pcl1[:,:3].min(0) scene_max_new = pcl1[:,:3].max(0) print(f'Org: {scene_min} - {scene_max}') print(f'New: {scene_min_new} - {scene_max_new}') pcl00 = pcl0.copy() pcl00[:,2] -= 15 pcl_show = np.concatenate([pcl00, pcl1], 0) Bbox3D.draw_points_bboxes(pcl_show, walls, 'Z', False, points_keep_rate=1) return pcl1
def show_high_score_preds(score_threshold0, score_threshold1, total_scores, box_preds, anchors, gt_boxes, points): pos_ids = np.where(np.logical_and(total_scores >= score_threshold0, total_scores < score_threshold1))[0] box_preds_pos = box_preds[pos_ids] anchors = anchors[pos_ids] pnum = box_preds_pos.shape[0] gnum = gt_boxes.shape[0] #gt_boxes[:,2] -= 0.2 #points[:,2] -= 1 labels_show = np.array([0]*pnum + [1]*gnum) boxes_show = np.concatenate([box_preds_pos, gt_boxes], 0) #Bbox3D.draw_bboxes(boxes_show, 'Z', is_yx_zb=True, labels=labels_show) print(f'pred boxes with scores in [{score_threshold0}, {score_threshold1}]') Bbox3D.draw_points_bboxes(points, boxes_show, 'Z', is_yx_zb=True, labels=labels_show) #print(f'anchors with scores in [{score_threshold0}, {score_threshold1}]') #Bbox3D.draw_points_bboxes(points, anchors, 'Z', is_yx_zb=True) pass
def unused_read_indoor_info(): info_path = f'{SPLITED_DIR}/sung_infos_train.pkl' with open(info_path, 'rb') as f: infos = pickle.load(f) idx = 0 print(f'totally {len(infos)} blocks') for idx in range(0,len(infos)): info = infos[idx] pcl_path = info['velodyne_path'] pointcloud_num_features = info['pointcloud_num_features'] points = np.fromfile( pcl_path, dtype=np.float32).reshape([-1, pointcloud_num_features]) annos = info['annos'] loc = annos['location'] dims = annos['dimensions'] rots = annos['rotation_y'] gt_boxes = np.concatenate([loc, dims, rots[..., np.newaxis]], axis=1).astype(np.float32) Bbox3D.draw_points_bboxes(points, gt_boxes, 'Z', is_yx_zb=True)
def pos_targets(labels, bbox_targets, anchors, is_show=False,\ gt_boxes=None, points=None): anchors0 = anchors.copy() anchors0[:,2] -= 5 if gt_boxes is None: gt_boxes = np.zeros((0,7), dtype=np.float32) else: gt_boxes =gt_boxes.copy() gt_boxes[:,2] -= 1 if points is None: points = np.zeros((0,3), dtype=np.float32) pos_ids = np.where(labels==1)[0] neg_ids = np.where(labels==0)[0] bbox_targets_pos = bbox_targets[pos_ids] anchors_pos = anchors[pos_ids] pos_num = pos_ids.shape[0] gt_num = gt_boxes.shape[0] anchor_num = anchors.shape[0] if is_show: print(f"gt_box_num:{gt_num} anchor_num: {anchor_num}") print(f"all positive anchors: {pos_num}") labels_show = np.array([0]*pos_num + [1]*gt_num) boxes_show_0 = np.concatenate([anchors_pos, gt_boxes],0) Bbox3D.draw_points_bboxes(points, boxes_show_0, 'Z', is_yx_zb=True, labels=labels_show) # decoded anchors: shoud be exactly the gt_boxes from second.core.box_np_ops import second_box_decode bboxes_decoded = second_box_decode(bbox_targets_pos, anchors_pos, smooth_dim=True) #Bbox3D.draw_bboxes(bboxes_decoded, 'Z', is_yx_zb=True) #Bbox3D.draw_bboxes(gt_boxes, 'Z', is_yx_zb=True) boxes_show = np.concatenate([bboxes_decoded, gt_boxes, anchors0], 0) labels_show = np.array([0]*pos_num + [1]*gt_num + [2]*anchors0.shape[0]) if is_show: print('gt boxes from targets') Bbox3D.draw_points_bboxes(points, boxes_show, 'Z', is_yx_zb=True, labels=labels_show) return bboxes_decoded
def show_pcl_boxes(pcl, boxes): from utils3d.bbox3d_ops import Bbox3D Bbox3D.draw_points_bboxes(pcl[:, 0:3], boxes, 'Z', is_yx_zb=True) pass
def show_all(boxes_ls): boxes = np.concatenate(boxes_ls, 0) if boxes.shape[0] == 0: return Bbox3D.draw_points_bboxes(boxes[:, 0:3], boxes, 'Z', is_yx_zb=False)
def merge_pieces_of_same_walls_alongY(wall_bboxes): ''' 1) Find all the walls which are A) parallel along X, B) one of cen_corners is very close C) centroid of one wall is inside of another 2) Merge along Y ''' from utils3d.geometric_util import angle_with_x, vertical_dis_points_lines if wall_bboxes.shape[0] == 0: return wall_bboxes show = False if show: wall_bboxes0 = wall_bboxes.copy() wall_bboxes0[:, 2] -= 1 n = wall_bboxes.shape[0] cen_lines = Bbox3D.bboxes_centroid_lines(wall_bboxes, cen_axis='X', up_axis='Z') # [n,2,3] cen_direc = cen_lines[:, 1, :] - cen_lines[:, 0, :] # [n,3] np.linalg.norm(cen_direc[:, 0:2], axis=1) angles = angle_with_x(cen_direc[:, 0:2], 1) boxes_merged_extra = [] remain_mask = np.array([True] * n) merge_y_num = 0 split_and_merge_num = 0 for i in range(n - 1): # 1) parallel along X angles_dif = angles[i] - angles[i + 1:] # make to [0,np.pi/2] angles_dif = np.abs(limit_period(angles_dif, 0.5, np.pi)) angle_mask = angles_dif < 7 * np.pi / 180 # 2) one of centroid line corners is very close cen_dis = cen_lines[i:i + 1].reshape( [1, 2, 1, 3]) - cen_lines[i + 1:].reshape([-1, 1, 2, 3]) cen_dis = np.linalg.norm(cen_dis, axis=3) cen_dis_min = cen_dis.min(axis=1).min(axis=1) cen_dis_max = cen_dis.max(axis=1).max(axis=1) thickness_sum = wall_bboxes[:, 4] * 0.5 + wall_bboxes[i, 4] * 0.5 thickness_sum = thickness_sum[i + 1:] cendis_mask = cen_dis_min < thickness_sum # 3) the other centroid line corner is inside of the long box centroid_dis = np.linalg.norm(wall_bboxes[i:i + 1, 0:3] - wall_bboxes[i + 1:, 0:3], axis=1) #tmp = np.maximum(wall_bboxes[i,3], wall_bboxes[i+1:,3]) tmp = (wall_bboxes[i, 3] + wall_bboxes[i + 1:, 3]) * 0.45 - 0.1 centroid_mask = centroid_dis < tmp #if i==4: # import pdb; pdb.set_trace() # XXX BREAKPOINT # show_boxes = wall_bboxes[[4,12]] # Bbox3D.draw_bboxes(show_boxes, 'Z', False) # pass mask_i = angle_mask * cendis_mask * centroid_mask if mask_i.any(): ids = np.where(mask_i)[0] + i + 1 # 5) the centroid corner used to split the other # 4) vertical dis vertical_dis = vertical_dis_points_lines(cen_lines[i], cen_lines[ids]).mean(0) ave_thickness = (wall_bboxes[i, 4] + wall_bboxes[ids, 4]) * 0.5 thickness_gap_rate = vertical_dis / ave_thickness verdis_mask = (thickness_gap_rate > 0.2) * (thickness_gap_rate < 1.2) ids = ids[verdis_mask] vertical_dis = vertical_dis[verdis_mask] is_need_merge = ids.shape[0] > 0 if is_need_merge: # split the longer box, and merge the shorter one along Y k = ids.shape[0] for j in range(k): idx = ids[j] size_x_i = wall_bboxes[i, 3] size_x_j = wall_bboxes[idx, 3] size_x_rate = size_x_i / size_x_j print(f'size_x_rate: {size_x_rate}') if np.abs(size_x_rate - 1) < 0.15: merge_y_num += 1 box_merge = merge_2pieces_of_1wall( wall_bboxes[i], wall_bboxes[idx], 'Y') if show and False: show_boxes = np.concatenate([ wall_bboxes[i:i + 1], wall_bboxes[idx:idx + 1] ], 0) if box_merge is not None: show_boxes = np.concatenate( [show_boxes, box_merge], 0) show_boxes[-1, 2] += 0.2 else: import pdb pdb.set_trace() # XXX BREAKPOINT pass show_boxes = np.concatenate( [show_boxes, wall_bboxes0], 0) Bbox3D.draw_bboxes(show_boxes, 'Z', False) pass if box_merge is not None: wall_bboxes[idx] = box_merge.reshape([7]) remain_mask[i] = False else: print( 'The two walls cannot be merged along Y, this should not happen normally' ) print( 'merge_pieces_of_same_walls_alongY again after crop_walls to solve some special case: Originally, a wall is broken by no intersection like 0058113bdc8bee5f387bb5ad316d7b28' ) #show_boxes = np.concatenate([wall_bboxes[i:i+1], wall_bboxes[idx:idx+1]], 0) #Bbox3D.draw_bboxes(wall_bboxes, 'Z', False, highlight_ids=[idx, i]) #Bbox3D.draw_bboxes(show_boxes, 'Z', False) #import pdb; pdb.set_trace() # XXX BREAKPOINT #pass #raise NotImplementedError else: # the longer box need to be split along X before merging split_and_merge_num += 1 cen_dis_ij = cen_dis[idx - i - 1] if size_x_rate > 1: longer_idx = i short_idx = idx cen_dis_ij = cen_dis_ij.min(axis=0) else: longer_idx = idx short_idx = i cen_dis_ij = cen_dis_ij.min(axis=1) # find the intersection point on longer box far_corner_id = int(cen_dis_ij[0] < cen_dis_ij[1]) cen_dir_longer = cen_lines[longer_idx, 1] - cen_lines[longer_idx, 0] cen_dir_longer /= np.linalg.norm(cen_dir_longer) sl_dir = cen_lines[ short_idx, far_corner_id] - cen_lines[longer_idx, 0] intersection = np.sum( sl_dir * cen_dir_longer ) * cen_dir_longer + cen_lines[longer_idx, 0] # offset: half of the thickness intersection += cen_dir_longer * wall_bboxes[i, 4] * 0.5 splited_boxes = Bbox3D.split_wall_by_centroid_intersections( wall_bboxes[longer_idx], intersection.reshape([1, 3])) # [2,7] if splited_boxes.shape[0] == 1: print( '\n\n\t\tComplicated situation, not solved well yet.\n\n' ) box_merge = None if False and DEBUG and splited_boxes.shape[0] == 1: box_tmp = np.array( [[0, 0, 0, 0.5, 0.5, 0.5, 0]]) box_tmp[0, 0:3] = intersection.reshape([1, 3]) boxes_show = np.concatenate([ box_tmp, wall_bboxes[longer_idx].reshape( [-1, 7]) ], 0) Bbox3D.draw_points_bboxes( intersection.reshape([1, 3]), boxes_show, 'Z', False) else: tmp = wall_bboxes[short_idx, 0:3] - splited_boxes[:, 0: 3] # [2,3] tmp = np.linalg.norm(tmp, axis=1) # [2] merge_id = int(tmp[0] > tmp[1]) box_merge = merge_2pieces_of_1wall( wall_bboxes[short_idx], splited_boxes[merge_id], 'Y') if show and False: #if box_merge is None: show_boxes = np.concatenate([ wall_bboxes[i:i + 1], wall_bboxes[idx:idx + 1] ], 0) Bbox3D.draw_points_bboxes( intersection.reshape([1, 3]), show_boxes, 'Z', False) show_boxes = np.concatenate( [show_boxes, splited_boxes], 0) show_boxes[-1, 2] += 0.5 show_boxes[-2, 2] += 0.7 if box_merge is not None: show_boxes = np.concatenate( [show_boxes, box_merge], 0) show_boxes[-1, 2] += 1 show_boxes = np.concatenate( [show_boxes, wall_bboxes0], 0) Bbox3D.draw_bboxes(show_boxes, 'Z', False) import pdb pdb.set_trace() # XXX BREAKPOINT pass if box_merge is None: # temperally solution, may because of a wall being changed several # times remain_mask[short_idx] = False else: wall_bboxes[longer_idx] = splited_boxes[1 - merge_id] wall_bboxes[short_idx] = box_merge.reshape([7]) boxes_merged_extra.append(box_merge.reshape([1, 7])) wall_bboxes_new = wall_bboxes[remain_mask] #boxes_merged_extra = np.concatenate(boxes_merged_extra, 0) #wall_bboxes_new = np.concatenate([wall_bboxes, boxes_merged_extra], 0) print( f'merge_y_num:{merge_y_num} split_and_merge_num:{split_and_merge_num}' ) if show: show_walls_offsetz(wall_bboxes_new) #show_walls_offsetz(wall_bboxes) #show_walls_offsetz(wall_bboxes0[remain_mask]) show_walls_offsetz(wall_bboxes0[np.logical_not(remain_mask)]) #show_walls_offsetz(boxes_merged_extra) return wall_bboxes_new
def render_parsed_house_walls(parsed_dir, show_pcl=SHOW_PCL, show_by_class=0): print(f'parsed_dir:{parsed_dir}') house_name = os.path.basename(parsed_dir) bboxes = [] labels = [] for obj in CLASSES: bbox_fn_ = f'{parsed_dir}/object_bbox/{obj}.txt' bboxes_ = np.loadtxt(bbox_fn_).reshape([-1, 7]) bboxes.append(bboxes_) label = DSET_METAS0.class_2_label[obj] labels += [label] * bboxes_.shape[0] bboxes = np.concatenate(bboxes, 0) labels = np.array(labels).astype(np.int8) if bboxes.shape[0] > 0: scene_size = Bbox3D.boxes_size(bboxes) print(f'scene wall size:{scene_size}') #Bbox3D.draw_bboxes(bboxes, up_axis='Z', is_yx_zb=False, labels=labels) #if not show_pcl: Bbox3D.draw_bboxes_mesh(bboxes, up_axis='Z', is_yx_zb=False) #Bbox3D.draw_bboxes_mesh(bboxes, up_axis='Z', is_yx_zb=False, labels=labels) #show_walls_offsetz(bboxes) if show_by_class: for c in range(1, max(labels) + 1): cs = DSET_METAS0.label_2_class[c] print(cs) if cs not in ['wall', 'window', 'door']: #if cs not in ['wall']: continue mask = labels == c bboxes_c = bboxes[mask] show_walls_offsetz(bboxes_c) if show_pcl: pcl_fn = f'{parsed_dir}/pcl_camref.ply' if not os.path.exists(pcl_fn): return pcd = open3d.read_point_cloud(pcl_fn) points = np.asarray(pcd.points) points = cam2world_pcl(points) colors = np.asarray(pcd.colors) pcl = np.concatenate([points, colors], 1) scene_size = pcl_size(pcl) print(f'scene pcl size:{scene_size}') print(f'point num: {pcl.shape[0]}') #pcl = cut_points_roof(pcl) #Bbox3D.draw_points(pcl, points_keep_rate=POINTS_KEEP_RATE) #Bbox3D.draw_points(pcl, points_keep_rate=POINTS_KEEP_RATE, animation_fn='points.mp4', ani_size=AniSizes[house_name]) bboxes[:, 2] += 0.1 Bbox3D.draw_points_bboxes(pcl, bboxes, up_axis='Z', is_yx_zb=False, points_keep_rate=POINTS_KEEP_RATE) Bbox3D.draw_points_bboxes_mesh(pcl, bboxes, up_axis='Z', is_yx_zb=False, points_keep_rate=POINTS_KEEP_RATE)
def split_bbox(bbox_fn, points_splited): ''' bbox in file bbox_fn: up_axis='Y' with always x_size > z_size transform with cam2world_box: up_axis == 'Z' always: x_size > y_size ''' obj = os.path.basename(bbox_fn).split('.')[0] min_point_num_per1sm = 10 # thickness_aug for cropping x thickness_aug = 0.3 assert IndoorData._block_size0[-1] == -1 # do not crop box along z bboxes = np.loadtxt(bbox_fn).reshape([-1,7]) #if DEBUG: # #show_walls_1by1(bboxes) # bboxes = bboxes[3:4] areas = bboxes[:,3] * bboxes[:,5] min_point_num = np.minimum( min_point_num_per1sm * areas, 200 ) bboxes_aug = bboxes.copy() #bboxes_aug[:,4] += thickness_aug bboxes_aug[:,3:6] = np.clip(bboxes_aug[:,3:6],a_min= thickness_aug, a_max=None ) bn = bboxes.shape[0] sn = len(points_splited) bboxes_splited = [] for i in range(0, sn): # Use to constrain size_x size_z point_masks_aug_i = Bbox3D.points_in_bbox(points_splited[i][:,0:3].copy(), bboxes_aug.copy()) # Use to constrain size_y (the thickness) bboxes_tc = IndoorData.adjust_box_for_thickness_crop(bboxes) point_masks_i = Bbox3D.points_in_bbox(points_splited[i][:,0:3].copy(), bboxes_tc) pn_in_box_aug_i = np.sum(point_masks_aug_i, 0) pn_in_box_i = np.sum(point_masks_i, 0) #print(f'no aug:{pn_in_box_i}\n auged:{pn_in_box_aug_i}') # (1) The bboxes with no points with thickness_aug will be removed firstly keep_box_aug_i = pn_in_box_aug_i > min_point_num bboxes_i = bboxes[keep_box_aug_i] if DEBUG and obj=='ceiling' and 0: rm_box_aug_i = pn_in_box_aug_i <= min_point_num print(rm_box_aug_i) bboxes_no_points_i = bboxes[rm_box_aug_i].copy() bboxes_no_points_i[:,0] += 30 bboxes_show = np.concatenate([bboxes_i, bboxes_no_points_i],0) Bbox3D.draw_points_bboxes(points_splited[i], bboxes_show, up_axis='Z', is_yx_zb=False) points_aug_i = [points_splited[i][point_masks_aug_i[:,j]] for j in range(bn)] points_aug_i = [points_aug_i[j] for j in range(bn) if keep_box_aug_i[j]] points_i = [points_splited[i][point_masks_i[:,j]] for j in range(bn)] points_i = [points_i[j] for j in range(bn) if keep_box_aug_i[j]] # (2) Crop all the boxes by points and intersec_corners seperately bn_i = bboxes_i.shape[0] croped_bboxes_i = [] keep_unseen_intersection = False if keep_unseen_intersection: intersec_corners_idx_i, intersec_corners_i = Bbox3D.detect_all_intersection_corners(bboxes_i, 'Z') else: intersec_corners_idx_i = [None]*bn_i invalid_bn = 0 for k in range(bn_i): croped_box_k = Bbox3D.crop_bbox_by_points( bboxes_i[k], points_i[k], points_aug_i[k], 'Z', intersec_corners_idx_i[k]) if croped_box_k is not None: croped_bboxes_i.append( croped_box_k ) else: invalid_bn += 1 #Bbox3D.draw_points_bboxes(points_splited[i], bboxes_i[k:k+1], up_axis='Z', is_yx_zb=False, points_keep_rate=1.0) pass if len(croped_bboxes_i) > 0: croped_bboxes_i = np.concatenate(croped_bboxes_i, 0) else: croped_bboxes_i = np.array([]).reshape([-1,7]) # (3) Refine x size of each bbox by thickness croped of intersected bbox #croped_size = bboxes_i[:,3:6] - croped_bboxes_i[:,3:6] refine_x_by_intersection = False # not correct yet if refine_x_by_intersection: for k in range(bn_i): itsc0, itsc1 = intersec_corners_idx_i[k] crop_value = [0,0] if itsc0 >= 0: crop_value[0] = Bbox3D.refine_intersection_x(croped_bboxes_i[k], 'neg', croped_bboxes_i[itsc0], 'Z') if itsc1 >= 0: crop_value[1] = Bbox3D.refine_intersection_x(croped_bboxes_i[k], 'pos', croped_bboxes_i[itsc1], 'Z') if itsc0 >= 0 or itsc1 > 0: croped_bboxes_i[k] = Bbox3D.crop_bbox_size(croped_bboxes_i[k], 'X', crop_value) #crop_ysize_neg = croped_size[itsc0,1] if itsc0 >= 0 else 0 #crop_ysize_pos = croped_size[itsc1,1] if itsc1 >= 0 else 0 #croped_bboxes_i[k] = Bbox3D.crop_bbox_size(croped_bboxes_i[k], 'X', [crop_ysize_neg, crop_ysize_pos]) # (4) remove too small wall min_wall_size_x = 0.2 sizex_mask = croped_bboxes_i[:,3] > min_wall_size_x croped_bboxes_i = croped_bboxes_i[sizex_mask] bboxes_splited.append(croped_bboxes_i) show = False if show and DEBUG and len(points_i) > 0 and obj=='ceiling': print(croped_bboxes_i[:,3]) points = np.concatenate(points_i,0) points = points_splited[i] points1 = points.copy() points1[:,0] += 30 points = np.concatenate([points, points1], 0) bboxes_i[:,0] += 30 bboxes_i = np.concatenate([bboxes_i, croped_bboxes_i], 0) #Bbox3D.draw_points_bboxes(points, bboxes_i, up_axis='Z', is_yx_zb=False) Bbox3D.draw_points_bboxes_mesh(points, bboxes_i, up_axis='Z', is_yx_zb=False) import pdb; pdb.set_trace() # XXX BREAKPOINT pass return bboxes_splited
def split_scene(scene_dir, splited_path): from suncg_utils.suncg_preprocess import check_house_intact, read_summary, write_summary scene_name = os.path.basename(scene_dir) splited_path = os.path.join(splited_path, scene_name) summary_0 = read_summary(splited_path) always_update = ALWAYS_UPDATE #house_intact, intacts = check_house_intact(scene_dir) #if not house_intact: # return summary_raw = read_summary(scene_dir) is_big_size = (summary_raw['scene_size'] > MAX_SCENE_SIZE).any() always_update = always_update or ( ALWAYS_BIG_SIZE and is_big_size ) if (not always_update) and 'split_num' in summary_0: sn = summary_0['split_num'].squeeze() still_split = ALWAYS_UPDATE_MULTI_SPLITS and sn >1 if not still_split: print(f'skip {splited_path}') return print(f'spliting {scene_dir}') gen_ply = False if 'level_num' in summary_raw and summary_raw['level_num'] != 1: return pcl_fn = os.path.join(scene_dir, 'pcl_camref.ply') if not os.path.exists(pcl_fn): print(f'pcl.ply not exist, skip {scene_dir}') return fns = glob.glob(os.path.join(splited_path, '*.pth')) if ONLY_MODIFY_BOX and len(fns)>0: print('Load points from pth directly') scene_name = os.path.basename(os.path.dirname(fns[0])) is_special_scene = IndoorData.is_a_special_scene(scene_name) points_splited = [] for fni in fns: pcl_i, boxes_i = torch.load(fni) points_splited.append(pcl_i) else: points_splited, is_special_scene = IndoorData.split_pcl_plyf(pcl_fn) n_block = len(points_splited) bboxes_splited = {} boxes_nums = {} for obj in CLASSES_USED: bbox_fn = os.path.join(scene_dir, f'object_bbox/{obj}.txt') if os.path.exists(bbox_fn): if is_special_scene or n_block > 1: bboxes_splited[obj] = IndoorData.split_bbox(bbox_fn, points_splited) else: bboxes_splited[obj] = IndoorData.load_bboxes(bbox_fn) boxes_nums[obj] = [] for ii in range(n_block): boxes_nums[obj].append( len(bboxes_splited[obj][ii]) ) if not os.path.exists(splited_path): os.makedirs(splited_path) for i in range(n_block): boxes_num_all_classes = sum([bn[i] for bn in boxes_nums.values()]) if n_block>1 and boxes_num_all_classes < MIN_BOXES_NUM: continue fni = splited_path + '/pcl_%d.pth'%(i) pcl_i = points_splited[i].astype(np.float32) #offset = pcl_i[:,0:3].mean(0) #pcl_i[:,0:3] = pcl_i[:,0:3] - offset pcl_i = np.ascontiguousarray(pcl_i) boxes_i = {} for obj in bboxes_splited: boxes_i[obj] = bboxes_splited[obj][i].astype(np.float32) if obj in ['ceiling', 'floor', 'room']: boxes_i[obj] = Bbox3D.set_yaw_zero(boxes_i[obj]) #boxes_i[obj] = preprocess_cfr_standard(boxes_i[obj]) torch.save((pcl_i, boxes_i), fni) if gen_ply: Bbox3D.draw_points_bboxes(pcl_i, boxes_i['wall'], 'Z', False) pclfn_i = splited_path + f'/pcl_{i}.ply' points_ply(pcl_i[:,0:3], pclfn_i) boxfn_i = splited_path + f'/wall_{i}.ply' Bbox3D.save_bboxes_ply(boxfn_i, boxes_i['wall'], 'Z') print(f'save {fni}') write_summary(splited_path, 'split_num', n_block, 'w')
def show_all_valid_anchors(anchors, anchors_mask, points): anchors = anchors[anchors_mask] Bbox3D.draw_points_bboxes(points, anchors, 'Z', is_yx_zb=True)
def render_pth_file(pth_fn, show_by_class=0): pcl, bboxes0 = torch.load(pth_fn) pcl = down_sample_points(pcl) if 'clutter' in bboxes0: del bboxes0['clutter'] #points = pcl[:,0:3] #colors = pcl[:,3:6] #normals = pcl[:,6:9] if CLASSES is None: bboxes = bboxes0 else: bboxes = {} for c in CLASSES: if c in bboxes0: bboxes[c] = bboxes0[c] scene_size = pcl_size(pcl) print(f'scene pcl size:{scene_size}') print(f'point num: {pcl.shape[0]}') #pcl = cut_points_roof(pcl) classes = [k for k in bboxes.keys()] num_classes = {k: bboxes[k].shape[0] for k in bboxes.keys()} print(f'\nclasses: {num_classes}\n\n') all_bboxes = np.concatenate([boxes for boxes in bboxes.values()], 0) nums = [boxes.shape[0] for boxes in bboxes.values()] labels = [] for i, n in enumerate(nums): labels += [i] * n labels = np.array(labels) #Bbox3D.draw_points(pcl, points_keep_rate=POINTS_KEEP_RATE) #show_walls_offsetz(all_bboxes) #Bbox3D.draw_bboxes_mesh(all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels) #Bbox3D.draw_bboxes_mesh(all_bboxes, up_axis='Z', is_yx_zb=False) #Bbox3D.draw_points_bboxes_mesh(pcl, all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels, points_keep_rate=POINTS_KEEP_RATE) #Bbox3D.draw_points_bboxes_mesh(pcl, all_bboxes, up_axis='Z', is_yx_zb=False, points_keep_rate=POINTS_KEEP_RATE) #Bbox3D.draw_points_bboxes(pcl, all_bboxes, up_axis='Z', is_yx_zb=False,points_keep_rate=POINTS_KEEP_RATE) Bbox3D.draw_points_bboxes(pcl, all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels, points_keep_rate=POINTS_KEEP_RATE) #Bbox3D.draw_points_bboxes(pcl, all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels, points_keep_rate=POINTS_KEEP_RATE, animation_fn='anima.mp4', ani_size=[280,700,550,1350]) #Bbox3D.draw_bboxes(all_bboxes, up_axis='Z', is_yx_zb=False, labels=labels) #boxlist = BoxList3D(all_bboxes, size3d=None, mode='standard', examples_idxscope=None, constants={}) #boxlist.show_with_corners() #show_walls_offsetz(all_bboxes) if show_by_class: for clas in bboxes.keys(): #if clas not in ['wall']: #if clas not in ['wall', 'window', 'door','ceiling', 'floor']: # continue #if clas not in CLASSES: # continue boxes = bboxes[clas] bn = boxes.shape[0] print(clas, f'num={bn}') #Bbox3D.draw_points_bboxes(points, boxes, up_axis='Z', is_yx_zb=False) #Bbox3D.draw_points_bboxes_mesh(pcl, boxes, up_axis='Z', is_yx_zb=False, points_keep_rate=POINTS_KEEP_RATE) try: Bbox3D.draw_points_bboxes(pcl, boxes, up_axis='Z', is_yx_zb=False, points_keep_rate=POINTS_KEEP_RATE) except: import pdb pdb.set_trace() # XXX BREAKPOINT pass #show_walls_offsetz(boxes) pass
def show_points_bbox(points, boxes, labels=None, names='', image_path=''): from utils_3d.bbox3d_ops import Bbox3D #print("\timage: ", image_path) Bbox3D.draw_points_bboxes(points, boxes, up_axis='Z', labels=labels, names=names, is_yx_zb=True)
def get_part_bbox(vertices, triangle, triangle_norms, name=''): ''' bbox: [xc, yc, zc, x_size, y_size, z_size, yaw] ''' #show_mesh(vertices, triangle) class_name = name.split('#')[0] box_min = np.min(vertices, 0) box_max = np.max(vertices, 0) centroid = (box_min + box_max) / 2.0 y_size = box_max[1] - box_min[1] ## Find the 8 outside corners distances = np.linalg.norm(vertices - np.expand_dims(centroid, 0), axis=1) max_dis = max(distances) out_corner_mask = (abs(distances - max_dis) < 1e-5) n0 = vertices.shape[0] #print([i for i in range(n0) if out_corner_mask[i]]) out_vertices = [ vertices[i:i + 1, :] for i in range(n0) if out_corner_mask[i] ] if len(out_vertices) == 0: import pdb pdb.set_trace() # XXX BREAKPOINT pass return None out_vertices = np.concatenate(out_vertices, 0) if out_vertices.shape[0] != 8: #Bbox3D.draw_points_open3d(out_vertices, show=True) #Bbox3D.draw_points_open3d(vertices, show=True) #show_mesh(vertices, triangle) if class_name not in ENABLE_NO_RECTANGLE: print( f'\nFailed to find bbox, not rectangle, {class_name} \n {out_vertices.shape[0]} vertices\n' ) assert False import pdb pdb.set_trace() # XXX BREAKPOINT pass return None else: print( f'\nNot rectangle, use no yaw box, {class_name} \n {out_vertices.shape[0]} vertices\n' ) min_max = {'min': box_min, 'max': box_max} bbox_noyaw = Bbox3D.bbox_from_minmax(min_max) #Bbox3D.draw_points_bboxes(vertices, bbox_noyaw, 'Y', is_yx_zb=False) return bbox_noyaw ## Find the 4 corners on one side x_right_mask = out_vertices[:, 0] - centroid[0] > 0 x_right_corners = np.concatenate( [out_vertices[i:i + 1, :] for i in range(8) if x_right_mask[i]], 0) assert x_right_corners.shape[0] == 4 x_right_cen = np.mean(x_right_corners, 0) ## get length x_size = np.linalg.norm(x_right_cen - centroid) * 2 if not abs(x_right_cen[1] - centroid[1]) < 1e-7: print("y should be same, pitch and rool should be 0") ## the angle between x_right_cen and x axis x_direc = x_right_cen - centroid x_direc = (x_direc) / np.linalg.norm(x_direc) x_axis = np.array([1, 0, 0]) # [-90, 90] yaw = np.arccos(np.sum(x_direc * x_axis)) if abs(yaw) < 0.01: yaw = 0 else: assert x_direc[0] > 0 assert x_direc[1] == 0 yaw *= np.sign(x_direc[2]) ## Find the 2 corners at top within x_right_corners top_mask = x_right_corners[:, 1] - centroid[1] > 0 x_right_top_corners = np.concatenate( [x_right_corners[i:i + 1, :] for i in range(4) if top_mask[i]], 0) assert x_right_top_corners.shape[0] == 2 # get z_size z_size = np.linalg.norm(x_right_top_corners[0] - x_right_top_corners[1]) ### got the bbox xc, yc, zc = centroid bbox = np.array([xc, yc, zc, x_size, y_size, z_size, yaw]) ### Check if yaw != 0 and False: Bbox3D.draw_bboxes(bbox, 'Y', False) if False: min_max = {'min': box_min, 'max': box_max} bbox_noyaw = Bbox3D.bbox_from_minmax(min_max) bboxes_show = np.concatenate( [bbox.reshape([1, 7]), bbox_noyaw.reshape([1, 7])], 0) Bbox3D.draw_points_bboxes(vertices, bboxes_show, 'Y', is_yx_zb=False) return bbox
def show_all(boxes_ls): boxes = np.concatenate(boxes_ls, 0) Bbox3D.draw_points_bboxes(boxes[:, 0:3], boxes, 'Z', is_yx_zb=False)