def show_bboxes(bboxes_3d, points=None, feats=None):
num_p = points.shape[0]
choices = np.random.choice(num_p, 2 * 10000)
#points = points[choices]
#_show_3d_points_objs_ls(None, None, [bboxes_3d], obj_rep='RoBox3D_UpRight_xyxy_sin2a_thick_Z0Z1')
bboxes_show = bboxes_3d.copy()
voxel_size = 0.02
bboxes_show[:, :4] /= voxel_size
bboxes_show[:, :4] += 10
points_ls = [points] if points is not None else None
feats_ls = [feats] if feats is not None else None
#_show_objs_ls_points_ls( (512,512), [bboxes_show[:,:5]], obj_rep='RoLine2D_UpRight_xyxy_sin2a')
#_show_objs_ls_points_ls( (512,512), [bboxes_show[:,:6]], obj_rep='RoBox2D_UpRight_xyxy_sin2a_thick' )
_show_3d_points_objs_ls(points_ls, feats_ls, objs_ls=[bboxes_3d])
def show_bboxes(bboxes_3d, points=None, feats=None, obj_rep_in=None):
n = points.shape[0] / 1000
print(f'num points: {n} K')
points, feats = cut_roof(points, feats)
num_p = points.shape[0]
choices = np.random.choice(num_p, 2 * 10000)
#points = points[choices]
#_show_3d_points_objs_ls(None, None, [bboxes_3d], obj_rep='RoBox3D_UpRight_xyxy_sin2a_thick_Z0Z1')
bboxes_show = bboxes_3d.copy()
voxel_size = 0.02
bboxes_show[:, :4] /= voxel_size
bboxes_show[:, :4] += 10
points_ls = [points] if points is not None else None
feats_ls = [feats] if feats is not None else None
#_show_objs_ls_points_ls( (512,512), [bboxes_show], obj_rep=obj_rep_in)
#_show_objs_ls_points_ls( (512,512), [bboxes_show[:,:6]], obj_rep='RoBox2D_UpRight_xyxy_sin2a_thick' )
_show_3d_points_objs_ls(obj_rep=obj_rep_in,
objs_ls=[bboxes_3d, bboxes_3d],
obj_colors=['random', 'black'],
box_types=['surface_mesh', 'line_mesh'])
_show_3d_points_objs_ls(points_ls, feats_ls)
def rotated_bbox_overlaps(bboxes1, bboxes2, min_size=0.01, ref='union'):
'''
XYLgWsA
bbox: [cx, cy, size_x, size_y, angle]
angle: from x_r to x_b, positive for clock-wise
unit: degree
'''
from detectron2 import _C
assert bboxes1.shape[1] == 5
assert bboxes2.shape[1] == 5
bboxes1 = bboxes1.clone()
bboxes2 = bboxes2.clone()
n1 = bboxes1.shape[0]
n2 = bboxes2.shape[0]
if n1 == 0 or n2 == 0:
return torch.zeros([n1, n2]).to(bboxes2.device)
if not bboxes1[:, -1].abs().max() < np.pi * 2:
import pdb
pdb.set_trace() # XXX BREAKPOINT
_show_objs_ls_points_ls((512, 512), [bboxes1.cpu().numpy()],
obj_rep='XYLgWsA')
pass
if not bboxes2[:, -1].abs().max() < np.pi * 2:
import pdb
pdb.set_trace() # XXX BREAKPOINT
_show_objs_ls_points_ls((512, 512), [bboxes1.cpu().numpy()],
obj_rep='XYLgWsA')
pass
bboxes1[:, -1] *= 180 / np.pi
bboxes2[:, -1] *= 180 / np.pi
bboxes1[:, 2:4] = bboxes1[:, 2:4].clamp(min=min_size)
bboxes2[:, 2:4] = bboxes2[:, 2:4].clamp(min=min_size)
if bboxes1.shape[0] > 0:
assert bboxes1[:, 2:4].min() >= 0.01
if bboxes2.shape[0] > 0:
assert bboxes2[:, 2:4].min() >= 0.01
ious_2d = _C.box_iou_rotated(bboxes1, bboxes2)
if torch.isnan(ious_2d).any():
print("nan iou from rotated_bbox_overlaps")
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
assert ref in ['union', 'bboxes1', 'bboxes2', 'min']
if ref == 'union':
pass
else:
area1 = bboxes1[:, 2] * bboxes1[:, 3]
area2 = bboxes2[:, 2] * bboxes2[:, 3]
intersection = (area1[:, None] + area2[None, :]) / (1 / ious_2d + 1)
if ref == 'bboxes1':
ref_area = area1[:, None]
elif ref == 'bboxes2':
ref_area = area2[:, None]
elif ref == 'min':
ref_area = np.minimum(area1[:, None], area2[None, :])
ious_2d = intersection / ref_area
if 0:
from tools.visual_utils import _show_objs_ls_points_ls, _show_3d_points_objs_ls
_show_3d_points_objs_ls(objs_ls=[bboxes1, bboxes2],
obj_rep='XYLgWsA',
obj_colors=['green', 'red'])
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
return ious_2d
def show_pcl_f(pcl_file):
points = load_ply(pcl_file)
points = cut_roof(points)
_show_3d_points_objs_ls([points[:, :3]], [points[:, 3:6]])
def _show_2dlines_as_3d(lines_2d_ls, obj_rep, filename, height_method=['same','per_room'][1]):
'''
lines_2d_ls: [walls, windows, doors]
'''
from obj_geo_utils.geometry_utils import get_rooms_from_edges
from beike_data_utils.beike_utils import load_ply
wall_idx, door_idx, win_idx = 0, 1, 2
assert obj_rep == 'XYZLgWsHA'
scene_name = os.path.splitext(os.path.basename(filename))[0]
print(f'{scene_name}')
m2p_file = '../'+filename.replace('TopView_VerD', 'meter_to_pix_rates').replace('npy','txt')
m2p_rate = np.loadtxt(m2p_file)
rooms_line_ids, room_ids_per_edge, num_walls_inside_room, rooms = get_rooms_from_edges(lines_2d_ls[0], obj_rep, gen_bbox=True, show_rooms=0)
num_room = len(rooms_line_ids)
scope_file = '../'+filename.replace('TopView_VerD', 'pcl_scopes').replace('npy','txt')
pcl_scope = np.loadtxt(scope_file)
max_height = pcl_scope[1,-1] - pcl_scope[0,-1] - 0.3
max_height = min(max_height, 4)
#bottom = pcl_scope[0,-1]
bottom = - max_height/2 + 0.1
if height_method == 'same':
heights = [max_height for _ in range(num_room)]
elif height_method == 'per_room':
pcl_file = '../'+filename.replace('TopView_VerD', 'ply').replace('npy','ply')
pcl = load_ply(pcl_file)[:,:3]
pcl_min = pcl.min(0).reshape(1,-1)
pcl = pcl - pcl_min
ids = np.random.choice(int(pcl.shape[0]), 100*1000)
pcl = pcl[ids]
heights = []
point_nums = []
for i in range(num_room):
rc = rooms[i:i+1,:2] / m2p_rate
rs = rooms[i,3:5].max() / m2p_rate
mask = np.linalg.norm( pcl[:,:2] - rc, axis=1) < rs * 0.7
p_n = mask.sum()
point_nums.append(p_n)
if p_n < 100:
heights.append(max_height)
continue
r_points = pcl[mask]
height_i = r_points[:,2].max() - bottom
gap = (max_height - height_i)/ max_height
if gap < 0.1:
height_i = max_height
heights.append(height_i)
pass
wall_thickness = 0.2 # 0.24
door_thickness = 0.23
door_height = 2.1
window_height = 1.5
for i,ls in enumerate( lines_2d_ls):
ls[:,:6] /= m2p_rate
if i==wall_idx:
ls[:,4] = wall_thickness
#ls[:,5] = max_height
else:
ls[:,4] = door_thickness
if i==door_idx:
ls[:,5] = door_height
if i==win_idx:
ls[:,5] = 1.5
for i in range(num_room):
ids = rooms_line_ids[i]
lines_2d_ls[0][ids, 5] = np.clip( lines_2d_ls[0][ids,5], a_min = heights[i], a_max=None)
lines_2d_ls[wall_idx][:,2] = lines_2d_ls[wall_idx][:,2] + (lines_2d_ls[wall_idx][:,5] - max_height)/2
lines_2d_ls[wall_idx][:,3] += wall_thickness * 0.8
lines_2d_ls[door_idx][:,2] = lines_2d_ls[door_idx][:,2] + (lines_2d_ls[door_idx][:,5] - max_height)/2
lines_2d_ls[win_idx][:,2] = lines_2d_ls[win_idx][:,5]/2 + 0.85 - max_height/2
labels = []
cat_num = len(lines_2d_ls)
for i in range(cat_num):
labels += [i] * lines_2d_ls[i].shape[0]
obj_colors = ['gray', 'green', 'blue']
colors = [obj_colors[l] for l in labels]
# flip
for ls in lines_2d_ls:
ls[:,1] *= -1
ls[:,-1] *= -1
bboxes = np.concatenate(lines_2d_ls, 0)
walls = lines_2d_ls[0]
walls_2cor = OBJ_REPS_PARSE.encode_obj(walls, obj_rep, 'RoLine2D_2p')
floors = gen_floor_mesh_from_walls(walls_2cor, rooms_line_ids, bottom-0.1)
if 1:
print('point_nums', point_nums)
print('heights', heights)
_show_3d_points_objs_ls(objs_ls = [bboxes, bboxes], obj_rep=obj_rep, obj_colors=[colors, 'black'], box_types=['surface_mesh','line_mesh'], polygons_ls=[floors], polygon_colors=['order'] )
pass
def __getitem__(self, index):
if self.load_voxlized_sparse != False:
return self.load_sparse_input(index, self.load_voxlized_sparse=='aug')
debug = 0
img_info = self.img_infos[index].copy()
is_include_gt_bboxes = 'gt_bboxes_2d_raw' in img_info
if is_include_gt_bboxes:
gt_bboxes = img_info['gt_bboxes_2d_raw']
else:
gt_bboxes = None
img_meta = img_info['img_meta']
coords, feats, labels, center = self.load_ply(index)
# Downsample the pointcloud with finer voxel size before transformation for memory and speed
if self.PREVOXELIZATION_VOXEL_SIZE is not None:
inds = ME.utils.sparse_quantize(
coords / self.PREVOXELIZATION_VOXEL_SIZE, return_index=True)
coords = coords[inds]
feats = feats[inds]
labels = labels[inds]
# Prevoxel transformations
if self.prevoxel_transform is not None:
coords, feats, labels = self.prevoxel_transform(coords, feats, labels)
if DEBUG_CFG.VISUAL_SPARSE_3D_TRANSFORM and 0:
scale = 1
print('\n\ncoords min:', coords.min(0))
print('coords max:', coords.max(0))
print('gt min :', gt_bboxes[:,:4].reshape(-1,2).min(0))
print('gt max :', gt_bboxes[:,:4].reshape(-1,2).max(0))
_show_3d_points_objs_ls([coords], objs_ls=[gt_bboxes], obj_rep=self.obj_rep)
#coords_raw = coords.copy()
coords, feats, labels, transformation, rotate_angles, scale_rate = self.voxelizer.voxelize(
coords, feats, labels, center=center)
#un_voxelization_matrix = np.eye(4)
#np.fill_diagonal(un_voxelization_matrix[:3, :3], self.VOXEL_SIZE)
#line_transformation = transformation @ un_voxelization_matrix
line_transformation = transformation
img_meta['data_aug']['transformation'] = transformation
img_meta['data_aug']['rotate_angles'] = rotate_angles
img_meta['data_aug']['scale_rate'] = scale_rate
if is_include_gt_bboxes:
gt_bboxes = m_transform_bboxes(gt_bboxes, line_transformation, self.obj_rep)
if DEBUG_CFG.VISUAL_SPARSE_3D_TRANSFORM and 0:
scale = 3
print('\n\ncoords min:', coords.min(0))
print('coords max:', coords.max(0))
print('gt min :', gt_bboxes[:,:4].reshape(-1,2).min(0))
print('gt max :', gt_bboxes[:,:4].reshape(-1,2).max(0))
print(gt_bboxes)
_show_3d_points_objs_ls([coords], objs_ls=[gt_bboxes], obj_rep=self.obj_rep)
# map labels not used for evaluation to ignore_label
if self.input_transform is not None:
coords, feats, labels, gt_bboxes, img_meta = self.input_transform(
coords, feats, labels, gt_bboxes, img_meta)
if self.target_transform is not None:
coords, feats, labels = self.target_transform(coords, feats, labels)
if self.IGNORE_LABELS is not None:
labels = np.array([self.label_map[x] for x in labels], dtype=np.int)
# Use coordinate features if config is set
if self.AUGMENT_COORDS_TO_FEATS:
coords, feats, labels = self._augment_coords_to_feats(coords, feats, labels)
if self.NORMALIZATION:
feats = self._normalization(feats)
feats = self.select_data_types(feats)
if is_include_gt_bboxes:
gt_labels = img_info['gt_labels']
mask = gt_labels > 0
gt_bboxes = gt_bboxes[mask]
gt_bboxes[:,:4] += self.bev_pad_pixels
img_info['gt_bboxes'] = gt_bboxes.astype(np.float32)
img_info['gt_labels'] = gt_labels[mask]
img_meta['dynamic_vox_size_aug'] = coords.max(0)+1
img_info['img_meta'] = img_meta
#print(img_meta['data_aug'])
check_shape = 1
if check_shape:
assert all(img_meta['dynamic_vox_size_aug'] == coords.max(0) +1)
assert (coords.min(axis=0)==0).all()
if DEBUG_CFG.VISUAL_SPARSE_3D_TRANSFORM and 1:
scale = 3
print('\n\ndataset item')
print('coords num:', coords.shape[0])
print('coords min:', coords.min(0))
print('coords max:', coords.max(0))
print('gt min :', gt_bboxes[:,:4].reshape(-1,2).min(0))
print('gt max :', gt_bboxes[:,:4].reshape(-1,2).max(0))
#print(gt_bboxes)
_show_3d_points_objs_ls([coords], objs_ls=[gt_bboxes], obj_rep=self.obj_rep)
pass
return_args = [coords, feats, labels, img_info]
#if self.return_transformation:
# return_args.append(transformation.astype(np.float32))
if self.save_sparse_input_for_debug:
self.save_sparse_input( coords, feats, labels, img_info )
return tuple(return_args)
def load_bboxes(pcl_file, classes, _category_ids_map, obj_rep, input_style):
anno_file = pcl_file.replace('.ply', '.npy').replace('Area_', 'Boxes_Area_')
scope_file = pcl_file.replace('.ply', '-scope.txt')
bboxes_dict = np.load(anno_file, allow_pickle=True).tolist()
if 'clutter' in bboxes_dict:
bboxes_dict['background'] = bboxes_dict['clutter']
del bboxes_dict['clutter']
bboxes = []
bbox_cat_ids = []
for cat in bboxes_dict:
if cat in _category_ids_map:
bboxes.append( bboxes_dict[cat] )
num_box = bboxes_dict[cat].shape[0]
cat_ids = _category_ids_map[cat] * np.ones([num_box], dtype=np.int64)
bbox_cat_ids.append( cat_ids )
# the loaded format: XYXYSin2WZ0Z1
bboxes_XYXYSin2WZ0Z1 = np.concatenate(bboxes, axis=0)
bbox_cat_ids = np.concatenate(bbox_cat_ids)
scope = np.loadtxt(scope_file)
bboxes_XYXYSin2WZ0Z1[:, :2] -= scope[0:1,:2]
bboxes_XYXYSin2WZ0Z1[:, 2:4] -= scope[0:1,:2]
bboxes_XYXYSin2WZ0Z1[:, 6] -= scope[0:1,2]
bboxes_XYXYSin2WZ0Z1[:, 7] -= scope[0:1,2]
#_show_3d_points_objs_ls(objs_ls=[bboxes_XYXYSin2WZ0Z1], obj_rep='XYXYSin2WZ0Z1')
gt_bboxes = OBJ_REPS_PARSE.encode_obj(bboxes_XYXYSin2WZ0Z1, 'XYXYSin2WZ0Z1', obj_rep)
anno = {}
if input_style == 'bev':
voxel_size_prj=0.01
if obj_rep == 'XYXYSin2' or obj_rep == 'XYXYSin2WZ0Z1':
sin2 = gt_bboxes[:,4].copy()
gt_bboxes /= voxel_size_prj
gt_bboxes[:,4] = sin2
elif obj_rep == 'XYZLgWsHA':
gt_bboxes[:,:6] /= voxel_size_prj
elif obj_rep == 'XYLgWsAbsSin2Z0Z1':
rotation = gt_bboxes[:,4:6].copy()
gt_bboxes /= voxel_size_prj
gt_bboxes[:,4:6] = rotation
elif obj_rep == 'XYDAsinAsinSin2Z0Z1':
rotation = gt_bboxes[:,3:6].copy()
gt_bboxes /= voxel_size_prj
gt_bboxes[:,3:6] = rotation
elif obj_rep == 'Rect4CornersZ0Z1':
gt_bboxes /= voxel_size_prj
else:
raise NotImplementedError
anno['voxel_size_prj'] = voxel_size_prj
filename = pcl_file
# lines_2d: RoLine2D_UpRight_xyxy_sin2a
anno['pcl_scope'] = scope
anno['filename'] = filename
anno['classes'] = [c for c in classes if c!='background']
anno['bboxes_XYXYSin2WZ0Z1'] = bboxes_XYXYSin2WZ0Z1
anno['gt_bboxes'] = gt_bboxes
anno['labels'] = bbox_cat_ids
show = 0
if show:
print(f'\n\n{pcl_file}\n\n')
_show_3d_points_objs_ls(objs_ls=[bboxes_XYXYSin2WZ0Z1], obj_rep='XYXYSin2WZ0Z1')
_show_3d_points_objs_ls(objs_ls=[gt_bboxes], obj_rep=obj_rep)
w,h = gt_bboxes[:,:4].reshape(-1,2).max(0).astype(np.int) + 10
_show_objs_ls_points_ls( (h,w), [gt_bboxes], obj_rep=obj_rep )
return anno
def show_gt_bboxes_1scene_3d(self, index):
img_info = self.img_infos[index]
scene_name = get_scene_name( img_info['img_meta']['filename'] )
print('\n\n', img_info['img_meta']['filename'])
gt_bboxes_3d_raw = img_info['gt_bboxes_3d_raw']
gt_labels = img_info['gt_labels']
coords, colors_norms, point_labels, _ = self.load_ply(index)
point_labels = raw_label_to_new_label(point_labels)
min_xyz = coords.min(0,keepdims=True)
coords -= min_xyz
walls = gt_bboxes_3d_raw[gt_labels==self._category_ids_map['wall']]
#ceilings, floors = self.load_polygons(index)
floor_mask = gt_labels == self._category_ids_map['floor']
ceiling_mask = gt_labels == self._category_ids_map['ceiling']
floors = get_ceiling_floor_from_box_walls(gt_bboxes_3d_raw[floor_mask], walls, 'XYXYSin2WZ0Z1', 'floor')
ceilings = get_ceiling_floor_from_box_walls(gt_bboxes_3d_raw[ceiling_mask], walls, 'XYXYSin2WZ0Z1', 'ceiling')
kp_cats = ['wall', 'beam', 'column', 'door', 'window']
#kp_cats += ['floor']
#kp_cats += ['ceiling']
cat_2_ids = {kp_cats[i]:i for i in range(len(kp_cats))}
#kp_cats += ['ceiling', 'floor']
points_rm_cats = ['ceiling']
# 3d
points = np.concatenate([coords, colors_norms], axis=1)
#points, point_labels = filter_categories('remove', points, point_labels, points_rm_cats, _raw_pcl_category_ids_map)
points, point_labels = cut_top_points(points, 0.01, point_labels)
#gt_bboxes_3d_raw, gt_labels = filter_categories('remove', gt_bboxes_3d_raw, gt_labels, ['ceiling', 'room', 'floor','door'], self._category_ids_map)
gt_bboxes_3d_raw, gt_labels = filter_categories('keep', gt_bboxes_3d_raw, gt_labels, kp_cats, self._category_ids_map)
gt_cats = [self._catid_2_cat[l+1] for l in gt_labels]
print(gt_cats)
#gt_bboxes_3d_raw, gt_labels = manual_rm_walls_for_vis(gt_bboxes_3d_raw, gt_labels, scene_name, 'XYXYSin2WZ0Z1')
#gt_bboxes_3d_raw = gt_bboxes_3d_raw[7:8]
#gt_labels = gt_labels[7:8] * 0
#add_noisy_box(gt_bboxes_3d_raw)
corners = OBJ_REPS_PARSE.encode_obj(gt_bboxes_3d_raw, 'XYXYSin2WZ0Z1', 'Top_Corners').reshape(-1,3)
#corners = add_noisy_corners(corners)
_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]])
#_show_3d_points_objs_ls([points[:,:3]], [point_labels-1])
#_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]], objs_ls=[gt_bboxes_3d_raw], obj_rep='XYXYSin2WZ0Z1', obj_colors=[gt_labels])
#_show_3d_points_objs_ls(objs_ls=[gt_bboxes_3d_raw, gt_bboxes_3d_raw], obj_rep='XYXYSin2WZ0Z1', obj_colors=[gt_labels, 'black'], box_types=['surface_mesh', 'line_mesh'], polygons_ls=[floors], polygon_colors=['yellow'])
#_show_3d_points_objs_ls(objs_ls=[gt_bboxes_3d_raw, walls], obj_rep='XYXYSin2WZ0Z1', obj_colors=[gt_labels, 'black'], box_types=['line_mesh', 'line_mesh'], polygons_ls=[floors, ceilings], polygon_colors=['magenta', 'yellow'])
#_show_3d_points_objs_ls(objs_ls=[gt_bboxes_3d_raw], obj_rep='XYXYSin2WZ0Z1', obj_colors='random')
corner_boxes = points_to_bboxes(corners, 0.1)
corner_labels = np.repeat( gt_labels[:,None], 4, 1 ).reshape(-1)
gt_bboxes_3d = OBJ_REPS_PARSE.encode_obj( gt_bboxes_3d_raw, 'XYXYSin2WZ0Z1', 'XYZLgWsHA' )
#_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]], objs_ls=[ gt_bboxes_3d, ], obj_rep='XYZLgWsHA', obj_colors=[gt_labels], box_types=['line_mesh'])
#_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]], objs_ls=[ gt_bboxes_3d, corner_boxes], obj_rep='XYZLgWsHA', obj_colors=[gt_labels, corner_labels], box_types=['line_mesh', 'surface_mesh'])
#_show_3d_points_objs_ls([points[:,:3]], [points[:,3:6]], objs_ls=[corner_boxes], obj_rep='XYZLgWsHA', obj_colors=[corner_labels], box_types=['surface_mesh'])
if self.obj_rep == 'XYXYSin2WZ0Z1':
sin2 = gt_bboxes_3d_raw[:,4].copy()
gt_bboxes_3d_raw /= 0.01
gt_bboxes_3d_raw[:,4] = sin2
elif self.obj_rep == 'XYZLgWsHA':
theta = gt_bboxes_3d_raw[:,6].copy()
gt_bboxes_3d_raw /= 0.01
gt_bboxes_3d_raw[:,6] = theta
else:
raise NotImplementedError
corners /= 0.01
obj_cats = np.array(kp_cats)[gt_labels]
#_show_objs_ls_points_ls((1024, 1024), [gt_bboxes_3d_raw], 'XYXYSin2WZ0Z1', points_ls=[corners], obj_scores_ls=[obj_cats])
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
def gen_box_1_scene(coords, feats, categories, instances, scene_name_l, max_num_points=None, bbox_file=None):
from obj_geo_utils.geometry_utils import get_cf_from_wall
_show_pcl_per_cls = 1
min_points_per_object = 10
rm_large_thick_wall = 1
n, c = coords.shape
assert c==3
assert feats.shape == (n,3)
assert categories.shape == instances.shape == (n,)
obj_rep_load = 'XYXYSin2WZ0Z1'
if max_num_points is not None:
coords, feats, categories, instances = sampling_points(max_num_points, coords, feats, categories, instances)
#show_pcd(coords)
cat_min = categories.min()
cat_max = categories.max()
cat_ids = [Stanford3DDatasetConverter.wall_id, ] + [i for i in range(cat_max+1) if i != Stanford3DDatasetConverter.wall_id]
bboxes_wall = None
abandon_num = 0
bboxes = defaultdict(list)
polygons = {'ceiling':[], 'floor':[]}
for cat in cat_ids:
mask_cat = categories == cat
cat_name = Stanford3DDatasetConverter.CLASSES[cat]
if cat_name not in ['wall', 'beam', 'window', 'column', 'door', 'floor']:
#if cat_name not in ['wall']:
continue
num_cat = mask_cat.sum()
num_rate = num_cat / coords.shape[0]
if num_cat == 0:
continue
coords_cat, instances_cat = coords[mask_cat], instances[mask_cat]
num_objs = np.unique(instances_cat).shape
ins_min = instances_cat.min()
ins_max = instances_cat.max()
print(f'\n{cat_name} has {num_cat} points, rate={num_rate}, {num_objs} instances')
if num_cat == 0:
continue
if _show_pcl_per_cls and cat_name == 'wall':
_show_3d_points_objs_ls([coords_cat], [instances_cat])
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
for ins in range(ins_min, ins_max+1):
mask_ins = instances_cat == ins
num_ins = mask_ins.sum()
print(f'\n {cat_name} \t{ins}th instance has {num_ins} points\n')
if num_ins < min_points_per_object:
if num_ins > 0:
print(f'\ntoo less points, abandon')
continue
coords_cat_ins = coords_cat[mask_ins]
#_show_3d_points_objs_ls([coords_cat_ins])
if cat_name != 'clutter':
bbox = points_to_bbox_sfd(coords_cat_ins, bboxes_wall, cat_name)
if ASIS:
if rm_large_thick_wall and cat_name == 'wall':
w = OBJ_REPS_PARSE.encode_obj(bbox, 'XYXYSin2WZ0Z1', 'XYZLgWsHA')
if w[0,4] > 3:
continue
if w[0,4] > 0.7:
w[0,4] = 0.7
bbox = OBJ_REPS_PARSE.encode_obj(w, 'XYZLgWsHA', 'XYXYSin2WZ0Z1')
if bbox is not None:
bboxes[cat_name].append(bbox)
else:
abandon_num += 1
print(f'\n\nAbandon one {cat_name}\n\n')
if cat_name in ['ceiling', 'floor']:
polygons_i = points_to_polygon_ceiling_floor(coords_cat_ins, bboxes_wall, cat_name, obj_rep='XYXYSin2WZ0Z1')
polygons[cat_name].append(polygons_i)
pass
if cat_name in bboxes:
bboxes[cat_name] = np.concatenate(bboxes[cat_name], 0)
if cat_name == 'wall':
bboxes['wall'] = optimize_walls(bboxes['wall'], get_manual_merge_pairs(scene_name_l))
bboxes_wall = bboxes['wall']
pass
# cal pcl scope
min_pcl = coords.min(axis=0)
min_pcl = np.floor(min_pcl*10)/10
max_pcl = coords.max(axis=0)
max_pcl = np.ceil(max_pcl*10)/10
mean_pcl = (min_pcl + max_pcl) / 2
pcl_scope = np.concatenate([min_pcl, max_pcl], axis=0)
room = np.concatenate([ mean_pcl, max_pcl-min_pcl, np.array([0]) ], axis=0)
bboxes['room'] = OBJ_REPS_PARSE.encode_obj( room[None,:], 'XYZLgWsHA', 'XYXYSin2WZ0Z1' )
np.save(bbox_file, bboxes)
print(f'\n save {bbox_file}')
polygon_file = bbox_file.replace('Boxes', 'Polygons')
np.save(polygon_file, polygons)
print(f'\n save {polygon_file}')
colors = instances.astype(np.int32)
colors = feats
if 0:
_show_3d_points_objs_ls([coords], [colors], [bboxes['wall']], obj_rep='XYXYSin2WZ0Z1')
if 1:
print(f'abandon_num: {abandon_num}')
all_bboxes = []
all_cats = []
all_labels = []
view_cats = ['wall', 'beam', 'window', 'column', 'door']
#view_cats += ['floor']
#view_cats += ['ceiling']
for l,cat in enumerate(view_cats):
if cat not in bboxes:
continue
all_bboxes.append( bboxes[cat] )
all_cats += [cat] * bboxes[cat].shape[0]
all_labels += [l]* bboxes[cat].shape[0]
all_bboxes = np.concatenate(all_bboxes, 0)
all_cats = np.array(all_cats)
all_labels = np.array(all_labels)
all_colors = [COLOR_MAP[c] for c in all_cats]
floors_mesh = get_cf_from_wall( all_bboxes[all_cats=='floor'], all_bboxes[all_cats=='wall'], obj_rep_load, 'floor', check_valid=ASIS==False)
if DEBUG:
_show_3d_points_objs_ls(objs_ls= [all_bboxes, all_bboxes], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_colors, 'navy'], box_types= ['surface_mesh', 'line_mesh'], polygons_ls=[floors_mesh], polygon_colors='silver' )
if ASIS:
all_bboxes_, all_colors_ = manual_rm_objs(all_bboxes, 'XYXYSin2WZ0Z1', all_colors, 7)
_show_3d_points_objs_ls(objs_ls= [all_bboxes_, all_bboxes_], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_colors_, 'navy'], box_types= ['surface_mesh', 'line_mesh'], polygons_ls=[floors_mesh], polygon_colors='silver' )
import pdb; pdb.set_trace() # XXX BREAKPOINT
all_bboxes_, all_colors_ = manual_rm_objs_ids(all_bboxes, all_colors, [6])
pass
#_show_3d_points_objs_ls([coords], [feats], objs_ls= [all_bboxes], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_labels], box_types='line_mesh')
scope = all_bboxes[:,:4].reshape(-1,2).max(0) - all_bboxes[:,:4].reshape(-1,2).min(0)
voxel_size = max(scope) / 1000
all_bboxes_2d = all_bboxes[:,:6].copy() / voxel_size
org = all_bboxes_2d[:,:4].reshape(-1,2).min(0)[None,:] - 50
all_bboxes_2d[:,:2] -= org
all_bboxes_2d[:,2:4] -= org
w, h = all_bboxes_2d[:,:4].reshape(-1,2).max(0).astype(np.int)+100
topview_file = bbox_file.replace('npy', 'png')
_show_objs_ls_points_ls( (h,w), [all_bboxes_2d], obj_rep='XYXYSin2W',
obj_scores_ls=[all_cats], out_file=topview_file,
only_save=1)
pass
if 0:
view_cats = ['column', 'beam']
view_cats = ['door',]
for cat in view_cats:
if cat not in bboxes:
continue
print(cat)
_show_3d_points_objs_ls([coords], [colors], [bboxes[cat]], obj_rep='XYXYSin2WZ0Z1')
pass
return bboxes, polygons, abandon_num
def _gen_bboxes(max_num_points=1e6):
from plyfile import PlyData
from obj_geo_utils.geometry_utils import get_cf_from_wall
obj_rep_load = 'XYXYSin2WZ0Z1'
ply_files = glob.glob(STANFORD_3D_OUT_PATH + '/*/*.ply')
#ply_files = [os.path.join(STANFORD_3D_OUT_PATH, 'Area_1/hallway_8.ply' )]
UNALIGNED = ['Area_2/auditorium_1', 'Area_3/office_8',
'Area_4/hallway_14', 'Area_3/office_7']
DOOR_HAD = ['Area_1/hallway_4','Area_2/auditorium_1','Area_1/hallway_8']
ROTAE_WINDOW=['Area_1/office_11']
IntroSample = ['Area_3/office_4']
scenes = IntroSample
ply_files = [os.path.join(STANFORD_3D_OUT_PATH, f'{s}.ply' ) for s in scenes]
# The first 72 is checked
for l, plyf in enumerate( ply_files ):
scene_name_l = get_scene_name(plyf)
bbox_file = plyf.replace('.ply', '.npy').replace('Area_', '__Boxes_Area_')
topview_file = plyf.replace('.ply', '.png').replace('Area_', '__Boxes_Area_')
bbox_dir = os.path.dirname(bbox_file)
polygon_dir = bbox_dir.replace('Boxes', 'Polygons')
if not os.path.exists(bbox_dir):
os.makedirs(bbox_dir)
os.makedirs(polygon_dir)
print(f'\n\nStart processing \t{bbox_file} \n\t\t{l}\n')
if os.path.exists(bbox_file):
pass
#continue
plydata = PlyData.read(plyf)
data = plydata.elements[0].data
coords = np.array([data['x'], data['y'], data['z']], dtype=np.float32).T
feats = np.array([data['red'], data['green'], data['blue']], dtype=np.float32).T
categories = np.array(data['label'], dtype=np.int32)
instances = np.array(data['instance'], dtype=np.int32)
coords, feats, categories, instances = sampling_points(max_num_points, coords, feats, categories, instances)
#show_pcd(coords)
cat_min = categories.min()
cat_max = categories.max()
cat_ids = [Stanford3DDatasetConverter.wall_id, ] + [i for i in range(cat_max+1) if i != Stanford3DDatasetConverter.wall_id]
bboxes_wall = None
abandon_num = 0
bboxes = defaultdict(list)
polygons = {'ceiling':[], 'floor':[]}
for cat in cat_ids:
mask_cat = categories == cat
cat_name = Stanford3DDatasetConverter.CLASSES[cat]
num_cat = mask_cat.sum()
print(f'\n{cat_name} has {num_cat} points')
if num_cat == 0:
continue
coords_cat, instances_cat = coords[mask_cat], instances[mask_cat]
ins_min = instances_cat.min()
ins_max = instances_cat.max()
for ins in range(ins_min, ins_max+1):
mask_ins = instances_cat == ins
num_ins = mask_ins.sum()
#print(f'\t{ins}th instance has {num_ins} points')
if num_ins == 0:
continue
coords_cat_ins = coords_cat[mask_ins]
if cat_name != 'clutter':
#show_pcd(coords_cat_ins)
bbox = points_to_bbox_sfd(coords_cat_ins, bboxes_wall, cat_name)
if bbox is not None:
bboxes[cat_name].append(bbox)
else:
abandon_num += 1
print(f'\n\nAbandon one {cat_name}\n\n')
if cat_name in ['ceiling', 'floor']:
polygons_i = points_to_polygon_ceiling_floor(coords_cat_ins, bboxes_wall, cat_name, obj_rep='XYXYSin2WZ0Z1')
polygons[cat_name].append(polygons_i)
pass
if cat_name in bboxes:
bboxes[cat_name] = np.concatenate(bboxes[cat_name], 0)
if cat_name == 'wall':
bboxes['wall'] = optimize_walls(bboxes['wall'], get_manual_merge_pairs(scene_name_l))
bboxes_wall = bboxes['wall']
pass
# cal pcl scope
min_pcl = coords.min(axis=0)
min_pcl = np.floor(min_pcl*10)/10
max_pcl = coords.max(axis=0)
max_pcl = np.ceil(max_pcl*10)/10
mean_pcl = (min_pcl + max_pcl) / 2
pcl_scope = np.concatenate([min_pcl, max_pcl], axis=0)
room = np.concatenate([ mean_pcl, max_pcl-min_pcl, np.array([0]) ], axis=0)
bboxes['room'] = OBJ_REPS_PARSE.encode_obj( room[None,:], 'XYZLgWsHA', 'XYXYSin2WZ0Z1' )
#bboxes['wall'] = optimize_walls(bboxes['wall'], get_manual_merge_pairs(scene_name_l))
np.save(bbox_file, bboxes)
print(f'\n save {bbox_file}')
polygon_file = bbox_file.replace('Boxes', 'Polygons')
np.save(polygon_file, polygons)
print(f'\n save {polygon_file}')
colors = instances.astype(np.int32)
colors = feats
if 0:
_show_3d_points_objs_ls([coords], [colors], [bboxes['wall']], obj_rep='XYXYSin2WZ0Z1')
if 1:
print(f'abandon_num: {abandon_num}')
all_bboxes = []
all_cats = []
all_labels = []
view_cats = ['wall', 'beam', 'window', 'column', 'door']
view_cats += ['floor']
#view_cats += ['ceiling']
for l,cat in enumerate(view_cats):
if cat not in bboxes:
continue
all_bboxes.append( bboxes[cat] )
all_cats += [cat] * bboxes[cat].shape[0]
all_labels += [l]* bboxes[cat].shape[0]
all_bboxes = np.concatenate(all_bboxes, 0)
all_cats = np.array(all_cats)
all_labels = np.array(all_labels)
all_colors = [COLOR_MAP[c] for c in all_cats]
floors_mesh = get_cf_from_wall( all_bboxes[all_cats=='floor'], all_bboxes[all_cats=='wall'], obj_rep_load, 'floor')
if DEBUG:
_show_3d_points_objs_ls(objs_ls= [all_bboxes, all_bboxes], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_colors, 'navy'], box_types= ['surface_mesh', 'line_mesh'], polygons_ls=[floors_mesh], polygon_colors='silver' )
#_show_3d_points_objs_ls([coords], [feats], objs_ls= [all_bboxes], obj_rep='XYXYSin2WZ0Z1', obj_colors=[all_labels], box_types='line_mesh')
scope = all_bboxes[:,:4].reshape(-1,2).max(0) - all_bboxes[:,:4].reshape(-1,2).min(0)
voxel_size = max(scope) / 1000
all_bboxes_2d = all_bboxes[:,:6].copy() / voxel_size
org = all_bboxes_2d[:,:4].reshape(-1,2).min(0)[None,:] - 50
all_bboxes_2d[:,:2] -= org
all_bboxes_2d[:,2:4] -= org
w, h = all_bboxes_2d[:,:4].reshape(-1,2).max(0).astype(np.int)+100
_show_objs_ls_points_ls( (h,w), [all_bboxes_2d], obj_rep='XYXYSin2W',
obj_scores_ls=[all_cats], out_file=topview_file,
only_save=1)
pass
if 0:
view_cats = ['column', 'beam']
view_cats = ['door',]
for cat in view_cats:
if cat not in bboxes:
continue
print(cat)
_show_3d_points_objs_ls([coords], [colors], [bboxes[cat]], obj_rep='XYXYSin2WZ0Z1')
pass
pass
def prepare_sparse_input(img, img_meta=None, gt_bboxes=None, gt_labels=None, rescale=None):
coords_batch, feats_batch = img
## For some networks, making the network invariant to even, odd coords is important
#coord_base = (torch.rand(3) * 100).type_as(coords_batch)
#coords_batch[:, 1:4] += coord_base
sinput = SparseTensor(feats_batch, coords_batch)
if DEBUG_CFG.SPARSE_BEV:
sinput = get_pcl_topview(sinput, gt_bboxes)
assert gt_labels[0].min() > 0
debug = 0
if DEBUG_CFG.VISUAL_SPARSE_3D_INPUT:
obj_rep = img_meta[0]['obj_rep']
n = coords_batch.shape[0]
print(f'batch voxe num: {n/1000}k')
batch_size = coords_batch[:,0].max()+1
for i in range(batch_size):
print(f'example {i}/{batch_size}')
batch_mask = coords_batch[:,0] == i
points = coords_batch[batch_mask][:, 1:].cpu().data.numpy()
colors = feats_batch[batch_mask][:, :3].cpu().data.numpy()
colors = colors+0.5
normals = feats_batch[batch_mask][:, 3:6].cpu().data.numpy()
num_p = points.shape[0]
img_meta_i = img_meta[i]
voxel_size = img_meta_i['voxel_size']
raw_dynamic_vox_size = img_meta_i['raw_dynamic_vox_size']
mask_i = sinput.C[:,0] == i
ci = sinput.C[mask_i]
bboxes3d = gt_bboxes[i].cpu().data.numpy()
min_points = points.min(axis=0)
max_points = points.max(axis=0)
gt_labels_i = gt_labels[i].cpu().data.numpy()
data_aug = img_meta_i['data_aug']
dynamic_vox_size_aug = img_meta_i['dynamic_vox_size_aug']
print('\n\nfinal sparse input')
footprint = dynamic_vox_size_aug[0] * dynamic_vox_size_aug[1] * (voxel_size*voxel_size)
print(f'dynamic_vox_size_aug: {dynamic_vox_size_aug}, footprint: {footprint}')
print(f'num voxel: {num_p/1000}K')
print(img_meta[i]['filename'])
print(f'points scope: {min_points} - {max_points}')
print(f'data aug:\n {data_aug}\n')
print(f'labels: {gt_labels}')
scale = 1
#_show_lines_ls_points_ls((512,512), [lines2d*scale], [points*scale])
#_show_lines_ls_points_ls((512,512), [lines2d*scale])
#_show_objs_ls_points_ls((512,512), [lines2d*scale], 'RoLine2D_UpRight_xyxy_sin2a')
#_show_objs_ls_points_ls((512,512), [lines2d*scale], 'RoLine2D_UpRight_xyxy_sin2a', [points*scale])
#_show_3d_points_objs_ls([points], None, [bboxes3d], obj_rep='RoBox3D_UpRight_xyxy_sin2a_thick_Z0Z1' , obj_colors=[gt_labels_i] )
_show_3d_points_objs_ls([points], [colors], [bboxes3d], obj_rep=obj_rep, obj_colors=[gt_labels_i])
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
if 0 and DEBUG_CFG.SPARSE_BEV:
coords_batch = sinput.C
feats_batch = sinput.F
dense, _, _ = sinput.dense()
dense = dense.permute(0,1,3,2,4)
batch_size = coords_batch[:,0].max()+1
for i in range(batch_size):
batch_mask = coords_batch[:,0] == i
points = coords_batch[batch_mask][:, 1:].cpu().data.numpy()
normals = feats_batch[batch_mask][:, 1:].cpu().data.numpy()
num_p = points.shape[0]
lines2d = gt_bboxes[i].cpu().data.numpy()
density = dense[i,-1,:,:,0].cpu().data.numpy()
min_points = points.min(axis=0)
max_points = points.max(axis=0)
min_lines = lines2d[:,:4].reshape(-1,2).min(axis=0)
max_lines = lines2d[:,:4].reshape(-1,2).max(axis=0)
img_meta_i = img_meta[i]
data_aug = img_meta_i['data_aug']
dynamic_vox_size_aug = img_meta_i['dynamic_vox_size_aug']
print('\n\nfinal sparse input')
footprint = dynamic_vox_size_aug[0] * dynamic_vox_size_aug[1] * (voxel_size*voxel_size)
print(f'dynamic_vox_size_aug: {dynamic_vox_size_aug}, footprint: {footprint}')
print(f'num voxel: {num_p/1000}K')
print(img_meta[i]['filename'])
print(f'points scope: {min_points} - {max_points}')
print(f'lines scope: {min_lines} - {max_lines}')
print(f'data aug:\n {data_aug}\n')
#_show_lines_ls_points_ls(density, [lines2d])
from beike_data_utils.line_utils import lines2d_to_bboxes3d
bboxes3d_pixel = lines2d_to_bboxes3d(lines2d, OBJ_REP, height=30, thickness=1)
_show_3d_points_bboxes_ls([points], None, [ bboxes3d_pixel ],
b_colors = 'red', box_oriented=True, point_normals=[normals])
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
if 0 and not DEBUG_CFG.SPARSE_BEV:
n = coords_batch.shape[0]
print(f'batch voxe num: {n/1000}k')
batch_size = coords_batch[:,0].max()+1
for i in range(batch_size):
print(f'example {i}/{batch_size}')
batch_mask = coords_batch[:,0] == i
points = coords_batch[batch_mask][:, 1:].cpu().data.numpy()
colors = feats_batch[batch_mask][:, :3].cpu().data.numpy()
colors = colors+0.5
normals = feats_batch[batch_mask][:, 3:6].cpu().data.numpy()
num_p = points.shape[0]
img_meta_i = img_meta[i]
voxel_size = img_meta_i['voxel_size']
raw_dynamic_vox_size = img_meta_i['raw_dynamic_vox_size']
mask_i = sinput.c[:,0] == i
ci = sinput.c[mask_i]
lines2d = gt_bboxes[i].cpu().data.numpy()
from beike_data_utils.line_utils import lines2d_to_bboxes3d
bboxes3d_pixel = lines2d_to_bboxes3d(lines2d, OBJ_REP, height=50, thickness=1)
min_points = points.min(axis=0)
max_points = points.max(axis=0)
min_lines = lines2d[:,:4].reshape(-1,2).min(axis=0)
max_lines = lines2d[:,:4].reshape(-1,2).max(axis=0)
data_aug = img_meta_i['data_aug']
dynamic_vox_size_aug = img_meta_i['dynamic_vox_size_aug']
print('\n\nfinal sparse input')
footprint = dynamic_vox_size_aug[0] * dynamic_vox_size_aug[1] * (voxel_size*voxel_size)
print(f'dynamic_vox_size_aug: {dynamic_vox_size_aug}, footprint: {footprint}')
print(f'num voxel: {num_p/1000}K')
print(img_meta[i]['filename'])
print(f'points scope: {min_points} - {max_points}')
print(f'lines scope: {min_lines} - {max_lines}')
print(f'data aug:\n {data_aug}\n')
scale = 1
#_show_lines_ls_points_ls((512,512), [lines2d*scale], [points*scale])
_show_3d_points_bboxes_ls([points], [colors], [ bboxes3d_pixel ],
b_colors = 'red', box_oriented=True, point_normals=[normals])
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
return sinput
