def show_bboxes(bboxes_3d, points=None, feats=None): num_p = points.shape[0] choices = np.random.choice(num_p, 2 * 10000) #points = points[choices] #_show_3d_points_objs_ls(None, None, [bboxes_3d], obj_rep='RoBox3D_UpRight_xyxy_sin2a_thick_Z0Z1') bboxes_show = bboxes_3d.copy() voxel_size = 0.02 bboxes_show[:, :4] /= voxel_size bboxes_show[:, :4] += 10 points_ls = [points] if points is not None else None feats_ls = [feats] if feats is not None else None #_show_objs_ls_points_ls( (512,512), [bboxes_show[:,:5]], obj_rep='RoLine2D_UpRight_xyxy_sin2a') #_show_objs_ls_points_ls( (512,512), [bboxes_show[:,:6]], obj_rep='RoBox2D_UpRight_xyxy_sin2a_thick' ) _show_3d_points_objs_ls(points_ls, feats_ls, objs_ls=[bboxes_3d])
def show_bboxes(bboxes_3d, points=None, feats=None, obj_rep_in=None): n = points.shape[0] / 1000 print(f'num points: {n} K') points, feats = cut_roof(points, feats) num_p = points.shape[0] choices = np.random.choice(num_p, 2 * 10000) #points = points[choices] #_show_3d_points_objs_ls(None, None, [bboxes_3d], obj_rep='RoBox3D_UpRight_xyxy_sin2a_thick_Z0Z1') bboxes_show = bboxes_3d.copy() voxel_size = 0.02 bboxes_show[:, :4] /= voxel_size bboxes_show[:, :4] += 10 points_ls = [points] if points is not None else None feats_ls = [feats] if feats is not None else None #_show_objs_ls_points_ls( (512,512), [bboxes_show], obj_rep=obj_rep_in) #_show_objs_ls_points_ls( (512,512), [bboxes_show[:,:6]], obj_rep='RoBox2D_UpRight_xyxy_sin2a_thick' ) _show_3d_points_objs_ls(obj_rep=obj_rep_in, objs_ls=[bboxes_3d, bboxes_3d], obj_colors=['random', 'black'], box_types=['surface_mesh', 'line_mesh']) _show_3d_points_objs_ls(points_ls, feats_ls)
def rotated_bbox_overlaps(bboxes1, bboxes2, min_size=0.01, ref='union'): ''' XYLgWsA bbox: [cx, cy, size_x, size_y, angle] angle: from x_r to x_b, positive for clock-wise unit: degree ''' from detectron2 import _C assert bboxes1.shape[1] == 5 assert bboxes2.shape[1] == 5 bboxes1 = bboxes1.clone() bboxes2 = bboxes2.clone() n1 = bboxes1.shape[0] n2 = bboxes2.shape[0] if n1 == 0 or n2 == 0: return torch.zeros([n1, n2]).to(bboxes2.device) if not bboxes1[:, -1].abs().max() < np.pi * 2: import pdb pdb.set_trace() # XXX BREAKPOINT _show_objs_ls_points_ls((512, 512), [bboxes1.cpu().numpy()], obj_rep='XYLgWsA') pass if not bboxes2[:, -1].abs().max() < np.pi * 2: import pdb pdb.set_trace() # XXX BREAKPOINT _show_objs_ls_points_ls((512, 512), [bboxes1.cpu().numpy()], obj_rep='XYLgWsA') pass bboxes1[:, -1] *= 180 / np.pi bboxes2[:, -1] *= 180 / np.pi bboxes1[:, 2:4] = bboxes1[:, 2:4].clamp(min=min_size) bboxes2[:, 2:4] = bboxes2[:, 2:4].clamp(min=min_size) if bboxes1.shape[0] > 0: assert bboxes1[:, 2:4].min() >= 0.01 if bboxes2.shape[0] > 0: assert bboxes2[:, 2:4].min() >= 0.01 ious_2d = _C.box_iou_rotated(bboxes1, bboxes2) if torch.isnan(ious_2d).any(): print("nan iou from rotated_bbox_overlaps") import pdb pdb.set_trace() # XXX BREAKPOINT pass assert ref in ['union', 'bboxes1', 'bboxes2', 'min'] if ref == 'union': pass else: area1 = bboxes1[:, 2] * bboxes1[:, 3] area2 = bboxes2[:, 2] * bboxes2[:, 3] intersection = (area1[:, None] + area2[None, :]) / (1 / ious_2d + 1) if ref == 'bboxes1': ref_area = area1[:, None] elif ref == 'bboxes2': ref_area = area2[:, None] elif ref == 'min': ref_area = np.minimum(area1[:, None], area2[None, :]) ious_2d = intersection / ref_area if 0: from tools.visual_utils import _show_objs_ls_points_ls, _show_3d_points_objs_ls _show_3d_points_objs_ls(objs_ls=[bboxes1, bboxes2], obj_rep='XYLgWsA', obj_colors=['green', 'red']) import pdb pdb.set_trace() # XXX BREAKPOINT pass return ious_2d
def show_pcl_f(pcl_file): points = load_ply(pcl_file) points = cut_roof(points) _show_3d_points_objs_ls([points[:, :3]], [points[:, 3:6]])
def _show_2dlines_as_3d(lines_2d_ls, obj_rep, filename, height_method=['same','per_room'][1]): ''' lines_2d_ls: [walls, windows, doors] ''' from obj_geo_utils.geometry_utils import get_rooms_from_edges from beike_data_utils.beike_utils import load_ply wall_idx, door_idx, win_idx = 0, 1, 2 assert obj_rep == 'XYZLgWsHA' scene_name = os.path.splitext(os.path.basename(filename))[0] print(f'{scene_name}') m2p_file = '../'+filename.replace('TopView_VerD', 'meter_to_pix_rates').replace('npy','txt') m2p_rate = np.loadtxt(m2p_file) rooms_line_ids, room_ids_per_edge, num_walls_inside_room, rooms = get_rooms_from_edges(lines_2d_ls[0], obj_rep, gen_bbox=True, show_rooms=0) num_room = len(rooms_line_ids) scope_file = '../'+filename.replace('TopView_VerD', 'pcl_scopes').replace('npy','txt') pcl_scope = np.loadtxt(scope_file) max_height = pcl_scope[1,-1] - pcl_scope[0,-1] - 0.3 max_height = min(max_height, 4) #bottom = pcl_scope[0,-1] bottom = - max_height/2 + 0.1 if height_method == 'same': heights = [max_height for _ in range(num_room)] elif height_method == 'per_room': pcl_file = '../'+filename.replace('TopView_VerD', 'ply').replace('npy','ply') pcl = load_ply(pcl_file)[:,:3] pcl_min = pcl.min(0).reshape(1,-1) pcl = pcl - pcl_min ids = np.random.choice(int(pcl.shape[0]), 100*1000) pcl = pcl[ids] heights = [] point_nums = [] for i in range(num_room): rc = rooms[i:i+1,:2] / m2p_rate rs = rooms[i,3:5].max() / m2p_rate mask = np.linalg.norm( pcl[:,:2] - rc, axis=1) < rs * 0.7 p_n = mask.sum() point_nums.append(p_n) if p_n < 100: heights.append(max_height) continue r_points = pcl[mask] height_i = r_points[:,2].max() - bottom gap = (max_height - height_i)/ max_height if gap < 0.1: height_i = max_height heights.append(height_i) pass wall_thickness = 0.2 # 0.24 door_thickness = 0.23 door_height = 2.1 window_height = 1.5 for i,ls in enumerate( lines_2d_ls): ls[:,:6] /= m2p_rate if i==wall_idx: ls[:,4] = wall_thickness #ls[:,5] = max_height else: ls[:,4] = door_thickness if i==door_idx: ls[:,5] = door_height if i==win_idx: ls[:,5] = 1.5 for i in range(num_room): ids = rooms_line_ids[i] lines_2d_ls[0][ids, 5] = np.clip( lines_2d_ls[0][ids,5], a_min = heights[i], a_max=None) lines_2d_ls[wall_idx][:,2] = lines_2d_ls[wall_idx][:,2] + (lines_2d_ls[wall_idx][:,5] - max_height)/2 lines_2d_ls[wall_idx][:,3] += wall_thickness * 0.8 lines_2d_ls[door_idx][:,2] = lines_2d_ls[door_idx][:,2] + (lines_2d_ls[door_idx][:,5] - max_height)/2 lines_2d_ls[win_idx][:,2] = lines_2d_ls[win_idx][:,5]/2 + 0.85 - max_height/2 labels = [] cat_num = len(lines_2d_ls) for i in range(cat_num): labels += [i] * lines_2d_ls[i].shape[0] obj_colors = ['gray', 'green', 'blue'] colors = [obj_colors[l] for l in labels] # flip for ls in lines_2d_ls: ls[:,1] *= -1 ls[:,-1] *= -1 bboxes = np.concatenate(lines_2d_ls, 0) walls = lines_2d_ls[0] walls_2cor = OBJ_REPS_PARSE.encode_obj(walls, obj_rep, 'RoLine2D_2p') floors = gen_floor_mesh_from_walls(walls_2cor, rooms_line_ids, bottom-0.1) if 1: print('point_nums', point_nums) print('heights', heights) _show_3d_points_objs_ls(objs_ls = [bboxes, bboxes], obj_rep=obj_rep, obj_colors=[colors, 'black'], box_types=['surface_mesh','line_mesh'], polygons_ls=[floors], polygon_colors=['order'] ) pass
def __getitem__(self, index): if self.load_voxlized_sparse != False: return self.load_sparse_input(index, self.load_voxlized_sparse=='aug') debug = 0 img_info = self.img_infos[index].copy() is_include_gt_bboxes = 'gt_bboxes_2d_raw' in img_info if is_include_gt_bboxes: gt_bboxes = img_info['gt_bboxes_2d_raw'] else: gt_bboxes = None img_meta = img_info['img_meta'] coords, feats, labels, center = self.load_ply(index) # Downsample the pointcloud with finer voxel size before transformation for memory and speed if self.PREVOXELIZATION_VOXEL_SIZE is not None: inds = ME.utils.sparse_quantize( coords / self.PREVOXELIZATION_VOXEL_SIZE, return_index=True) coords = coords[inds] feats = feats[inds] labels = labels[inds] # Prevoxel transformations if self.prevoxel_transform is not None: coords, feats, labels = self.prevoxel_transform(coords, feats, labels) if DEBUG_CFG.VISUAL_SPARSE_3D_TRANSFORM and 0: scale = 1 print('\n\ncoords min:', coords.min(0)) print('coords max:', coords.max(0)) print('gt min :', gt_bboxes[:,:4].reshape(-1,2).min(0)) print('gt max :', gt_bboxes[:,:4].reshape(-1,2).max(0)) _show_3d_points_objs_ls([coords], objs_ls=[gt_bboxes], obj_rep=self.obj_rep) #coords_raw = coords.copy() coords, feats, labels, transformation, rotate_angles, scale_rate = self.voxelizer.voxelize( coords, feats, labels, center=center) #un_voxelization_matrix = np.eye(4) #np.fill_diagonal(un_voxelization_matrix[:3, :3], self.VOXEL_SIZE) #line_transformation = transformation @ un_voxelization_matrix line_transformation = transformation img_meta['data_aug']['transformation'] = transformation img_meta['data_aug']['rotate_angles'] = rotate_angles img_meta['data_aug']['scale_rate'] = scale_rate if is_include_gt_bboxes: gt_bboxes = m_transform_bboxes(gt_bboxes, line_transformation, self.obj_rep) if DEBUG_CFG.VISUAL_SPARSE_3D_TRANSFORM and 0: scale = 3 print('\n\ncoords min:', coords.min(0)) print('coords max:', coords.max(0)) print('gt min :', gt_bboxes[:,:4].reshape(-1,2).min(0)) print('gt max :', gt_bboxes[:,:4].reshape(-1,2).max(0)) print(gt_bboxes) _show_3d_points_objs_ls([coords], objs_ls=[gt_bboxes], obj_rep=self.obj_rep) # map labels not used for evaluation to ignore_label if self.input_transform is not None: coords, feats, labels, gt_bboxes, img_meta = self.input_transform( coords, feats, labels, gt_bboxes, img_meta) if self.target_transform is not None: coords, feats, labels = self.target_transform(coords, feats, labels) if self.IGNORE_LABELS is not None: labels = np.array([self.label_map[x] for x in labels], dtype=np.int) # Use coordinate features if config is set if self.AUGMENT_COORDS_TO_FEATS: coords, feats, labels = self._augment_coords_to_feats(coords, feats, labels) if self.NORMALIZATION: feats = self._normalization(feats) feats = self.select_data_types(feats) if is_include_gt_bboxes: gt_labels = img_info['gt_labels'] mask = gt_labels > 0 gt_bboxes = gt_bboxes[mask] gt_bboxes[:,:4] += self.bev_pad_pixels img_info['gt_bboxes'] = gt_bboxes.astype(np.float32) img_info['gt_labels'] = gt_labels[mask] img_meta['dynamic_vox_size_aug'] = coords.max(0)+1 img_info['img_meta'] = img_meta #print(img_meta['data_aug']) check_shape = 1 if check_shape: assert all(img_meta['dynamic_vox_size_aug'] == coords.max(0) +1) assert (coords.min(axis=0)==0).all() if DEBUG_CFG.VISUAL_SPARSE_3D_TRANSFORM and 1: scale = 3 print('\n\ndataset item') print('coords num:', coords.shape[0]) print('coords min:', coords.min(0)) print('coords max:', coords.max(0)) print('gt min :', gt_bboxes[:,:4].reshape(-1,2).min(0)) print('gt max :', gt_bboxes[:,:4].reshape(-1,2).max(0)) #print(gt_bboxes) _show_3d_points_objs_ls([coords], objs_ls=[gt_bboxes], obj_rep=self.obj_rep) pass return_args = [coords, feats, labels, img_info] #if self.return_transformation: # return_args.append(transformation.astype(np.float32)) if self.save_sparse_input_for_debug: self.save_sparse_input( coords, feats, labels, img_info ) return tuple(return_args)
def load_bboxes(pcl_file, classes, _category_ids_map, obj_rep, input_style): anno_file = pcl_file.replace('.ply', '.npy').replace('Area_', 'Boxes_Area_') scope_file = pcl_file.replace('.ply', '-scope.txt') bboxes_dict = np.load(anno_file, allow_pickle=True).tolist() if 'clutter' in bboxes_dict: bboxes_dict['background'] = bboxes_dict['clutter'] del bboxes_dict['clutter'] bboxes = [] bbox_cat_ids = [] for cat in bboxes_dict: if cat in _category_ids_map: bboxes.append( bboxes_dict[cat] ) num_box = bboxes_dict[cat].shape[0] cat_ids = _category_ids_map[cat] * np.ones([num_box], dtype=np.int64) bbox_cat_ids.append( cat_ids ) # the loaded format: XYXYSin2WZ0Z1 bboxes_XYXYSin2WZ0Z1 = np.concatenate(bboxes, axis=0) bbox_cat_ids = np.concatenate(bbox_cat_ids) scope = np.loadtxt(scope_file) bboxes_XYXYSin2WZ0Z1[:, :2] -= scope[0:1,:2] bboxes_XYXYSin2WZ0Z1[:, 2:4] -= scope[0:1,:2] bboxes_XYXYSin2WZ0Z1[:, 6] -= scope[0:1,2] bboxes_XYXYSin2WZ0Z1[:, 7] -= scope[0:1,2] #_show_3d_points_objs_ls(objs_ls=[bboxes_XYXYSin2WZ0Z1], obj_rep='XYXYSin2WZ0Z1') gt_bboxes = OBJ_REPS_PARSE.encode_obj(bboxes_XYXYSin2WZ0Z1, 'XYXYSin2WZ0Z1', obj_rep) anno = {} if input_style == 'bev': voxel_size_prj=0.01 if obj_rep == 'XYXYSin2' or obj_rep == 'XYXYSin2WZ0Z1': sin2 = gt_bboxes[:,4].copy() gt_bboxes /= voxel_size_prj gt_bboxes[:,4] = sin2 elif obj_rep == 'XYZLgWsHA': gt_bboxes[:,:6] /= voxel_size_prj elif obj_rep == 'XYLgWsAbsSin2Z0Z1': rotation = gt_bboxes[:,4:6].copy() gt_bboxes /= voxel_size_prj gt_bboxes[:,4:6] = rotation elif obj_rep == 'XYDAsinAsinSin2Z0Z1': rotation = gt_bboxes[:,3:6].copy() gt_bboxes /= voxel_size_prj gt_bboxes[:,3:6] = rotation elif obj_rep == 'Rect4CornersZ0Z1': gt_bboxes /= voxel_size_prj else: raise NotImplementedError anno['voxel_size_prj'] = voxel_size_prj filename = pcl_file # lines_2d: RoLine2D_UpRight_xyxy_sin2a anno['pcl_scope'] = scope anno['filename'] = filename anno['classes'] = [c for c in classes if c!='background'] anno['bboxes_XYXYSin2WZ0Z1'] = bboxes_XYXYSin2WZ0Z1 anno['gt_bboxes'] = gt_bboxes anno['labels'] = bbox_cat_ids show = 0 if show: print(f'\n\n{pcl_file}\n\n') _show_3d_points_objs_ls(objs_ls=[bboxes_XYXYSin2WZ0Z1], obj_rep='XYXYSin2WZ0Z1') _show_3d_points_objs_ls(objs_ls=[gt_bboxes], obj_rep=obj_rep) w,h = gt_bboxes[:,:4].reshape(-1,2).max(0).astype(np.int) + 10 _show_objs_ls_points_ls( (h,w), [gt_bboxes], obj_rep=obj_rep ) return anno
def show_gt_bboxes_1scene_3d(self, index): img_info = self.img_infos[index] scene_name = get_scene_name( img_info['img_meta']['filename'] ) print('\n\n', img_info['img_meta']['filename']) gt_bboxes_3d_raw = img_info['gt_bboxes_3d_raw'] gt_labels = img_info['gt_labels'] coords, colors_norms, point_labels, _ = self.load_ply(index) point_labels = raw_label_to_new_label(point_labels) min_xyz = coords.min(0,keepdims=True) coords -= min_xyz walls = gt_bboxes_3d_raw[gt_labels==self._category_ids_map['wall']] #ceilings, floors = self.load_polygons(index) floor_mask = gt_labels == self._category_ids_map['floor'] ceiling_mask = gt_labels == self._category_ids_map['ceiling'] floors = get_ceiling_floor_from_box_walls(gt_bboxes_3d_raw[floor_mask], walls, 'XYXYSin2WZ0Z1', 'floor') ceilings = get_ceiling_floor_from_box_walls(gt_bboxes_3d_raw[ceiling_mask], walls, 'XYXYSin2WZ0Z1', 'ceiling') kp_cats = ['wall', 'beam', 'column', 'door', 'window'] #kp_cats += ['floor'] #kp_cats += ['ceiling'] cat_2_ids = {kp_cats[i]:i for i in range(len(kp_cats))} #kp_cats += ['ceiling', 'floor'] points_rm_cats = ['ceiling'] # 3d points = np.concatenate([coords, colors_norms], axis=1) #points, point_labels = filter_categories('remove', points, point_labels, points_rm_cats, _raw_pcl_category_ids_map) points, point_labels = cut_top_points(points, 0.01, point_labels) #gt_bboxes_3d_raw, gt_labels = filter_categories('remove', gt_bboxes_3d_raw, gt_labels, ['ceiling', 'room', 'floor','door'], self._category_ids_map) gt_bboxes_3d_raw, gt_labels = filter_categories('keep', gt_bboxes_3d_raw, gt_labels, kp_cats, self._category_ids_map) gt_cats = [self._catid_2_cat[l+1] for l in gt_labels] print(gt_cats) #gt_bboxes_3d_raw, gt_labels = manual_rm_walls_for_vis(gt_bboxes_3d_raw, gt_labels, scene_name, 'XYXYSin2WZ0Z1') #gt_bboxes_3d_raw = gt_bboxes_3d_raw[7:8] #gt_labels = gt_labels[7:8] * 0 #add_noisy_box(gt_bboxes_3d_raw) corners = OBJ_REPS_PARSE.encode_obj(gt_bboxes_3d_raw, 'XYXYSin2WZ0Z1', 'Top_Corners').reshape(-1,3) #corners = add_noisy_corners(corners) _show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]]) #_show_3d_points_objs_ls([points[:,:3]], [point_labels-1]) #_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]], objs_ls=[gt_bboxes_3d_raw], obj_rep='XYXYSin2WZ0Z1', obj_colors=[gt_labels]) #_show_3d_points_objs_ls(objs_ls=[gt_bboxes_3d_raw, gt_bboxes_3d_raw], obj_rep='XYXYSin2WZ0Z1', obj_colors=[gt_labels, 'black'], box_types=['surface_mesh', 'line_mesh'], polygons_ls=[floors], polygon_colors=['yellow']) #_show_3d_points_objs_ls(objs_ls=[gt_bboxes_3d_raw, walls], obj_rep='XYXYSin2WZ0Z1', obj_colors=[gt_labels, 'black'], box_types=['line_mesh', 'line_mesh'], polygons_ls=[floors, ceilings], polygon_colors=['magenta', 'yellow']) #_show_3d_points_objs_ls(objs_ls=[gt_bboxes_3d_raw], obj_rep='XYXYSin2WZ0Z1', obj_colors='random') corner_boxes = points_to_bboxes(corners, 0.1) corner_labels = np.repeat( gt_labels[:,None], 4, 1 ).reshape(-1) gt_bboxes_3d = OBJ_REPS_PARSE.encode_obj( gt_bboxes_3d_raw, 'XYXYSin2WZ0Z1', 'XYZLgWsHA' ) #_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]], objs_ls=[ gt_bboxes_3d, ], obj_rep='XYZLgWsHA', obj_colors=[gt_labels], box_types=['line_mesh']) #_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]], objs_ls=[ gt_bboxes_3d, corner_boxes], obj_rep='XYZLgWsHA', obj_colors=[gt_labels, corner_labels], box_types=['line_mesh', 'surface_mesh']) #_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]], objs_ls=[corner_boxes], obj_rep='XYZLgWsHA', obj_colors=[corner_labels], box_types=['surface_mesh']) if self.obj_rep == 'XYXYSin2WZ0Z1': sin2 = gt_bboxes_3d_raw[:,4].copy() gt_bboxes_3d_raw /= 0.01 gt_bboxes_3d_raw[:,4] = sin2 elif self.obj_rep == 'XYZLgWsHA': theta = gt_bboxes_3d_raw[:,6].copy() gt_bboxes_3d_raw /= 0.01 gt_bboxes_3d_raw[:,6] = theta else: raise NotImplementedError corners /= 0.01 obj_cats = np.array(kp_cats)[gt_labels] #_show_objs_ls_points_ls((1024, 1024), [gt_bboxes_3d_raw], 'XYXYSin2WZ0Z1', points_ls=[corners], obj_scores_ls=[obj_cats]) import pdb; pdb.set_trace() # XXX BREAKPOINT pass
def gen_box_1_scene(coords, feats, categories, instances, scene_name_l, max_num_points=None, bbox_file=None): from obj_geo_utils.geometry_utils import get_cf_from_wall _show_pcl_per_cls = 1 min_points_per_object = 10 rm_large_thick_wall = 1 n, c = coords.shape assert c==3 assert feats.shape == (n,3) assert categories.shape == instances.shape == (n,) obj_rep_load = 'XYXYSin2WZ0Z1' if max_num_points is not None: coords, feats, categories, instances = sampling_points(max_num_points, coords, feats, categories, instances) #show_pcd(coords) cat_min = categories.min() cat_max = categories.max() cat_ids = [Stanford3DDatasetConverter.wall_id, ] + [i for i in range(cat_max+1) if i != Stanford3DDatasetConverter.wall_id] bboxes_wall = None abandon_num = 0 bboxes = defaultdict(list) polygons = {'ceiling':[], 'floor':[]} for cat in cat_ids: mask_cat = categories == cat cat_name = Stanford3DDatasetConverter.CLASSES[cat] if cat_name not in ['wall', 'beam', 'window', 'column', 'door', 'floor']: #if cat_name not in ['wall']: continue num_cat = mask_cat.sum() num_rate = num_cat / coords.shape[0] if num_cat == 0: continue coords_cat, instances_cat = coords[mask_cat], instances[mask_cat] num_objs = np.unique(instances_cat).shape ins_min = instances_cat.min() ins_max = instances_cat.max() print(f'\n{cat_name} has {num_cat} points, rate={num_rate}, {num_objs} instances') if num_cat == 0: continue if _show_pcl_per_cls and cat_name == 'wall': _show_3d_points_objs_ls([coords_cat], [instances_cat]) import pdb; pdb.set_trace() # XXX BREAKPOINT pass for ins in range(ins_min, ins_max+1): mask_ins = instances_cat == ins num_ins = mask_ins.sum() print(f'\n {cat_name} \t{ins}th instance has {num_ins} points\n') if num_ins < min_points_per_object: if num_ins > 0: print(f'\ntoo less points, abandon') continue coords_cat_ins = coords_cat[mask_ins] #_show_3d_points_objs_ls([coords_cat_ins]) if cat_name != 'clutter': bbox = points_to_bbox_sfd(coords_cat_ins, bboxes_wall, cat_name) if ASIS: if rm_large_thick_wall and cat_name == 'wall': w = OBJ_REPS_PARSE.encode_obj(bbox, 'XYXYSin2WZ0Z1', 'XYZLgWsHA') if w[0,4] > 3: continue if w[0,4] > 0.7: w[0,4] = 0.7 bbox = OBJ_REPS_PARSE.encode_obj(w, 'XYZLgWsHA', 'XYXYSin2WZ0Z1') if bbox is not None: bboxes[cat_name].append(bbox) else: abandon_num += 1 print(f'\n\nAbandon one {cat_name}\n\n') if cat_name in ['ceiling', 'floor']: polygons_i = points_to_polygon_ceiling_floor(coords_cat_ins, bboxes_wall, cat_name, obj_rep='XYXYSin2WZ0Z1') polygons[cat_name].append(polygons_i) pass if cat_name in bboxes: bboxes[cat_name] = np.concatenate(bboxes[cat_name], 0) if cat_name == 'wall': bboxes['wall'] = optimize_walls(bboxes['wall'], get_manual_merge_pairs(scene_name_l)) bboxes_wall = bboxes['wall'] pass # cal pcl scope min_pcl = coords.min(axis=0) min_pcl = np.floor(min_pcl*10)/10 max_pcl = coords.max(axis=0) max_pcl = np.ceil(max_pcl*10)/10 mean_pcl = (min_pcl + max_pcl) / 2 pcl_scope = np.concatenate([min_pcl, max_pcl], axis=0) room = np.concatenate([ mean_pcl, max_pcl-min_pcl, np.array([0]) ], axis=0) bboxes['room'] = OBJ_REPS_PARSE.encode_obj( room[None,:], 'XYZLgWsHA', 'XYXYSin2WZ0Z1' ) np.save(bbox_file, bboxes) print(f'\n save {bbox_file}') polygon_file = bbox_file.replace('Boxes', 'Polygons') np.save(polygon_file, polygons) print(f'\n save {polygon_file}') colors = instances.astype(np.int32) colors = feats if 0: _show_3d_points_objs_ls([coords], [colors], [bboxes['wall']], obj_rep='XYXYSin2WZ0Z1') if 1: print(f'abandon_num: {abandon_num}') all_bboxes = [] all_cats = [] all_labels = [] view_cats = ['wall', 'beam', 'window', 'column', 'door'] #view_cats += ['floor'] #view_cats += ['ceiling'] for l,cat in enumerate(view_cats): if cat not in bboxes: continue all_bboxes.append( bboxes[cat] ) all_cats += [cat] * bboxes[cat].shape[0] all_labels += [l]* bboxes[cat].shape[0] all_bboxes = np.concatenate(all_bboxes, 0) all_cats = np.array(all_cats) all_labels = np.array(all_labels) all_colors = [COLOR_MAP[c] for c in all_cats] floors_mesh = get_cf_from_wall( all_bboxes[all_cats=='floor'], all_bboxes[all_cats=='wall'], obj_rep_load, 'floor', check_valid=ASIS==False) if DEBUG: _show_3d_points_objs_ls(objs_ls= [all_bboxes, all_bboxes], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_colors, 'navy'], box_types= ['surface_mesh', 'line_mesh'], polygons_ls=[floors_mesh], polygon_colors='silver' ) if ASIS: all_bboxes_, all_colors_ = manual_rm_objs(all_bboxes, 'XYXYSin2WZ0Z1', all_colors, 7) _show_3d_points_objs_ls(objs_ls= [all_bboxes_, all_bboxes_], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_colors_, 'navy'], box_types= ['surface_mesh', 'line_mesh'], polygons_ls=[floors_mesh], polygon_colors='silver' ) import pdb; pdb.set_trace() # XXX BREAKPOINT all_bboxes_, all_colors_ = manual_rm_objs_ids(all_bboxes, all_colors, [6]) pass #_show_3d_points_objs_ls([coords], [feats], objs_ls= [all_bboxes], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_labels], box_types='line_mesh') scope = all_bboxes[:,:4].reshape(-1,2).max(0) - all_bboxes[:,:4].reshape(-1,2).min(0) voxel_size = max(scope) / 1000 all_bboxes_2d = all_bboxes[:,:6].copy() / voxel_size org = all_bboxes_2d[:,:4].reshape(-1,2).min(0)[None,:] - 50 all_bboxes_2d[:,:2] -= org all_bboxes_2d[:,2:4] -= org w, h = all_bboxes_2d[:,:4].reshape(-1,2).max(0).astype(np.int)+100 topview_file = bbox_file.replace('npy', 'png') _show_objs_ls_points_ls( (h,w), [all_bboxes_2d], obj_rep='XYXYSin2W', obj_scores_ls=[all_cats], out_file=topview_file, only_save=1) pass if 0: view_cats = ['column', 'beam'] view_cats = ['door',] for cat in view_cats: if cat not in bboxes: continue print(cat) _show_3d_points_objs_ls([coords], [colors], [bboxes[cat]], obj_rep='XYXYSin2WZ0Z1') pass return bboxes, polygons, abandon_num
def _gen_bboxes(max_num_points=1e6): from plyfile import PlyData from obj_geo_utils.geometry_utils import get_cf_from_wall obj_rep_load = 'XYXYSin2WZ0Z1' ply_files = glob.glob(STANFORD_3D_OUT_PATH + '/*/*.ply') #ply_files = [os.path.join(STANFORD_3D_OUT_PATH, 'Area_1/hallway_8.ply' )] UNALIGNED = ['Area_2/auditorium_1', 'Area_3/office_8', 'Area_4/hallway_14', 'Area_3/office_7'] DOOR_HAD = ['Area_1/hallway_4','Area_2/auditorium_1','Area_1/hallway_8'] ROTAE_WINDOW=['Area_1/office_11'] IntroSample = ['Area_3/office_4'] scenes = IntroSample ply_files = [os.path.join(STANFORD_3D_OUT_PATH, f'{s}.ply' ) for s in scenes] # The first 72 is checked for l, plyf in enumerate( ply_files ): scene_name_l = get_scene_name(plyf) bbox_file = plyf.replace('.ply', '.npy').replace('Area_', '__Boxes_Area_') topview_file = plyf.replace('.ply', '.png').replace('Area_', '__Boxes_Area_') bbox_dir = os.path.dirname(bbox_file) polygon_dir = bbox_dir.replace('Boxes', 'Polygons') if not os.path.exists(bbox_dir): os.makedirs(bbox_dir) os.makedirs(polygon_dir) print(f'\n\nStart processing \t{bbox_file} \n\t\t{l}\n') if os.path.exists(bbox_file): pass #continue plydata = PlyData.read(plyf) data = plydata.elements[0].data coords = np.array([data['x'], data['y'], data['z']], dtype=np.float32).T feats = np.array([data['red'], data['green'], data['blue']], dtype=np.float32).T categories = np.array(data['label'], dtype=np.int32) instances = np.array(data['instance'], dtype=np.int32) coords, feats, categories, instances = sampling_points(max_num_points, coords, feats, categories, instances) #show_pcd(coords) cat_min = categories.min() cat_max = categories.max() cat_ids = [Stanford3DDatasetConverter.wall_id, ] + [i for i in range(cat_max+1) if i != Stanford3DDatasetConverter.wall_id] bboxes_wall = None abandon_num = 0 bboxes = defaultdict(list) polygons = {'ceiling':[], 'floor':[]} for cat in cat_ids: mask_cat = categories == cat cat_name = Stanford3DDatasetConverter.CLASSES[cat] num_cat = mask_cat.sum() print(f'\n{cat_name} has {num_cat} points') if num_cat == 0: continue coords_cat, instances_cat = coords[mask_cat], instances[mask_cat] ins_min = instances_cat.min() ins_max = instances_cat.max() for ins in range(ins_min, ins_max+1): mask_ins = instances_cat == ins num_ins = mask_ins.sum() #print(f'\t{ins}th instance has {num_ins} points') if num_ins == 0: continue coords_cat_ins = coords_cat[mask_ins] if cat_name != 'clutter': #show_pcd(coords_cat_ins) bbox = points_to_bbox_sfd(coords_cat_ins, bboxes_wall, cat_name) if bbox is not None: bboxes[cat_name].append(bbox) else: abandon_num += 1 print(f'\n\nAbandon one {cat_name}\n\n') if cat_name in ['ceiling', 'floor']: polygons_i = points_to_polygon_ceiling_floor(coords_cat_ins, bboxes_wall, cat_name, obj_rep='XYXYSin2WZ0Z1') polygons[cat_name].append(polygons_i) pass if cat_name in bboxes: bboxes[cat_name] = np.concatenate(bboxes[cat_name], 0) if cat_name == 'wall': bboxes['wall'] = optimize_walls(bboxes['wall'], get_manual_merge_pairs(scene_name_l)) bboxes_wall = bboxes['wall'] pass # cal pcl scope min_pcl = coords.min(axis=0) min_pcl = np.floor(min_pcl*10)/10 max_pcl = coords.max(axis=0) max_pcl = np.ceil(max_pcl*10)/10 mean_pcl = (min_pcl + max_pcl) / 2 pcl_scope = np.concatenate([min_pcl, max_pcl], axis=0) room = np.concatenate([ mean_pcl, max_pcl-min_pcl, np.array([0]) ], axis=0) bboxes['room'] = OBJ_REPS_PARSE.encode_obj( room[None,:], 'XYZLgWsHA', 'XYXYSin2WZ0Z1' ) #bboxes['wall'] = optimize_walls(bboxes['wall'], get_manual_merge_pairs(scene_name_l)) np.save(bbox_file, bboxes) print(f'\n save {bbox_file}') polygon_file = bbox_file.replace('Boxes', 'Polygons') np.save(polygon_file, polygons) print(f'\n save {polygon_file}') colors = instances.astype(np.int32) colors = feats if 0: _show_3d_points_objs_ls([coords], [colors], [bboxes['wall']], obj_rep='XYXYSin2WZ0Z1') if 1: print(f'abandon_num: {abandon_num}') all_bboxes = [] all_cats = [] all_labels = [] view_cats = ['wall', 'beam', 'window', 'column', 'door'] view_cats += ['floor'] #view_cats += ['ceiling'] for l,cat in enumerate(view_cats): if cat not in bboxes: continue all_bboxes.append( bboxes[cat] ) all_cats += [cat] * bboxes[cat].shape[0] all_labels += [l]* bboxes[cat].shape[0] all_bboxes = np.concatenate(all_bboxes, 0) all_cats = np.array(all_cats) all_labels = np.array(all_labels) all_colors = [COLOR_MAP[c] for c in all_cats] floors_mesh = get_cf_from_wall( all_bboxes[all_cats=='floor'], all_bboxes[all_cats=='wall'], obj_rep_load, 'floor') if DEBUG: _show_3d_points_objs_ls(objs_ls= [all_bboxes, all_bboxes], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_colors, 'navy'], box_types= ['surface_mesh', 'line_mesh'], polygons_ls=[floors_mesh], polygon_colors='silver' ) #_show_3d_points_objs_ls([coords], [feats], objs_ls= [all_bboxes], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_labels], box_types='line_mesh') scope = all_bboxes[:,:4].reshape(-1,2).max(0) - all_bboxes[:,:4].reshape(-1,2).min(0) voxel_size = max(scope) / 1000 all_bboxes_2d = all_bboxes[:,:6].copy() / voxel_size org = all_bboxes_2d[:,:4].reshape(-1,2).min(0)[None,:] - 50 all_bboxes_2d[:,:2] -= org all_bboxes_2d[:,2:4] -= org w, h = all_bboxes_2d[:,:4].reshape(-1,2).max(0).astype(np.int)+100 _show_objs_ls_points_ls( (h,w), [all_bboxes_2d], obj_rep='XYXYSin2W', obj_scores_ls=[all_cats], out_file=topview_file, only_save=1) pass if 0: view_cats = ['column', 'beam'] view_cats = ['door',] for cat in view_cats: if cat not in bboxes: continue print(cat) _show_3d_points_objs_ls([coords], [colors], [bboxes[cat]], obj_rep='XYXYSin2WZ0Z1') pass pass
def prepare_sparse_input(img, img_meta=None, gt_bboxes=None, gt_labels=None, rescale=None): coords_batch, feats_batch = img ## For some networks, making the network invariant to even, odd coords is important #coord_base = (torch.rand(3) * 100).type_as(coords_batch) #coords_batch[:, 1:4] += coord_base sinput = SparseTensor(feats_batch, coords_batch) if DEBUG_CFG.SPARSE_BEV: sinput = get_pcl_topview(sinput, gt_bboxes) assert gt_labels[0].min() > 0 debug = 0 if DEBUG_CFG.VISUAL_SPARSE_3D_INPUT: obj_rep = img_meta[0]['obj_rep'] n = coords_batch.shape[0] print(f'batch voxe num: {n/1000}k') batch_size = coords_batch[:,0].max()+1 for i in range(batch_size): print(f'example {i}/{batch_size}') batch_mask = coords_batch[:,0] == i points = coords_batch[batch_mask][:, 1:].cpu().data.numpy() colors = feats_batch[batch_mask][:, :3].cpu().data.numpy() colors = colors+0.5 normals = feats_batch[batch_mask][:, 3:6].cpu().data.numpy() num_p = points.shape[0] img_meta_i = img_meta[i] voxel_size = img_meta_i['voxel_size'] raw_dynamic_vox_size = img_meta_i['raw_dynamic_vox_size'] mask_i = sinput.C[:,0] == i ci = sinput.C[mask_i] bboxes3d = gt_bboxes[i].cpu().data.numpy() min_points = points.min(axis=0) max_points = points.max(axis=0) gt_labels_i = gt_labels[i].cpu().data.numpy() data_aug = img_meta_i['data_aug'] dynamic_vox_size_aug = img_meta_i['dynamic_vox_size_aug'] print('\n\nfinal sparse input') footprint = dynamic_vox_size_aug[0] * dynamic_vox_size_aug[1] * (voxel_size*voxel_size) print(f'dynamic_vox_size_aug: {dynamic_vox_size_aug}, footprint: {footprint}') print(f'num voxel: {num_p/1000}K') print(img_meta[i]['filename']) print(f'points scope: {min_points} - {max_points}') print(f'data aug:\n {data_aug}\n') print(f'labels: {gt_labels}') scale = 1 #_show_lines_ls_points_ls((512,512), [lines2d*scale], [points*scale]) #_show_lines_ls_points_ls((512,512), [lines2d*scale]) #_show_objs_ls_points_ls((512,512), [lines2d*scale], 'RoLine2D_UpRight_xyxy_sin2a') #_show_objs_ls_points_ls((512,512), [lines2d*scale], 'RoLine2D_UpRight_xyxy_sin2a', [points*scale]) #_show_3d_points_objs_ls([points], None, [bboxes3d], obj_rep='RoBox3D_UpRight_xyxy_sin2a_thick_Z0Z1' , obj_colors=[gt_labels_i] ) _show_3d_points_objs_ls([points], [colors], [bboxes3d], obj_rep=obj_rep, obj_colors=[gt_labels_i]) import pdb; pdb.set_trace() # XXX BREAKPOINT pass if 0 and DEBUG_CFG.SPARSE_BEV: coords_batch = sinput.C feats_batch = sinput.F dense, _, _ = sinput.dense() dense = dense.permute(0,1,3,2,4) batch_size = coords_batch[:,0].max()+1 for i in range(batch_size): batch_mask = coords_batch[:,0] == i points = coords_batch[batch_mask][:, 1:].cpu().data.numpy() normals = feats_batch[batch_mask][:, 1:].cpu().data.numpy() num_p = points.shape[0] lines2d = gt_bboxes[i].cpu().data.numpy() density = dense[i,-1,:,:,0].cpu().data.numpy() min_points = points.min(axis=0) max_points = points.max(axis=0) min_lines = lines2d[:,:4].reshape(-1,2).min(axis=0) max_lines = lines2d[:,:4].reshape(-1,2).max(axis=0) img_meta_i = img_meta[i] data_aug = img_meta_i['data_aug'] dynamic_vox_size_aug = img_meta_i['dynamic_vox_size_aug'] print('\n\nfinal sparse input') footprint = dynamic_vox_size_aug[0] * dynamic_vox_size_aug[1] * (voxel_size*voxel_size) print(f'dynamic_vox_size_aug: {dynamic_vox_size_aug}, footprint: {footprint}') print(f'num voxel: {num_p/1000}K') print(img_meta[i]['filename']) print(f'points scope: {min_points} - {max_points}') print(f'lines scope: {min_lines} - {max_lines}') print(f'data aug:\n {data_aug}\n') #_show_lines_ls_points_ls(density, [lines2d]) from beike_data_utils.line_utils import lines2d_to_bboxes3d bboxes3d_pixel = lines2d_to_bboxes3d(lines2d, OBJ_REP, height=30, thickness=1) _show_3d_points_bboxes_ls([points], None, [ bboxes3d_pixel ], b_colors = 'red', box_oriented=True, point_normals=[normals]) import pdb; pdb.set_trace() # XXX BREAKPOINT pass if 0 and not DEBUG_CFG.SPARSE_BEV: n = coords_batch.shape[0] print(f'batch voxe num: {n/1000}k') batch_size = coords_batch[:,0].max()+1 for i in range(batch_size): print(f'example {i}/{batch_size}') batch_mask = coords_batch[:,0] == i points = coords_batch[batch_mask][:, 1:].cpu().data.numpy() colors = feats_batch[batch_mask][:, :3].cpu().data.numpy() colors = colors+0.5 normals = feats_batch[batch_mask][:, 3:6].cpu().data.numpy() num_p = points.shape[0] img_meta_i = img_meta[i] voxel_size = img_meta_i['voxel_size'] raw_dynamic_vox_size = img_meta_i['raw_dynamic_vox_size'] mask_i = sinput.c[:,0] == i ci = sinput.c[mask_i] lines2d = gt_bboxes[i].cpu().data.numpy() from beike_data_utils.line_utils import lines2d_to_bboxes3d bboxes3d_pixel = lines2d_to_bboxes3d(lines2d, OBJ_REP, height=50, thickness=1) min_points = points.min(axis=0) max_points = points.max(axis=0) min_lines = lines2d[:,:4].reshape(-1,2).min(axis=0) max_lines = lines2d[:,:4].reshape(-1,2).max(axis=0) data_aug = img_meta_i['data_aug'] dynamic_vox_size_aug = img_meta_i['dynamic_vox_size_aug'] print('\n\nfinal sparse input') footprint = dynamic_vox_size_aug[0] * dynamic_vox_size_aug[1] * (voxel_size*voxel_size) print(f'dynamic_vox_size_aug: {dynamic_vox_size_aug}, footprint: {footprint}') print(f'num voxel: {num_p/1000}K') print(img_meta[i]['filename']) print(f'points scope: {min_points} - {max_points}') print(f'lines scope: {min_lines} - {max_lines}') print(f'data aug:\n {data_aug}\n') scale = 1 #_show_lines_ls_points_ls((512,512), [lines2d*scale], [points*scale]) _show_3d_points_bboxes_ls([points], [colors], [ bboxes3d_pixel ], b_colors = 'red', box_oriented=True, point_normals=[normals]) import pdb; pdb.set_trace() # XXX BREAKPOINT pass return sinput