def optimize_walls(walls_3d_line, manual_merge_pairs=None):
'''
XYXYSin2WZ0Z1
'''
wall_bottom = walls_3d_line[:,-2].min()
walls_3d_line[:,-2] = wall_bottom
bottom_corners = OBJ_REPS_PARSE.encode_obj(walls_3d_line, 'XYXYSin2WZ0Z1', 'Bottom_Corners').reshape(-1,3)
#_show_3d_points_objs_ls([bottom_corners], objs_ls = [walls_3d_line], obj_rep='XYXYSin2WZ0Z1')
walls_2d_line = walls_3d_line[:,:5]
#_show_3d_points_objs_ls(objs_ls=[walls_2d_line], obj_rep='RoLine2D_UpRight_xyxy_sin2a')
walls_2d_line_new, _, valid_ids = GraphUtils.optimize_wall_graph(walls_2d_line, obj_rep_in='XYXYSin2', opt_graph_cor_dis_thr=0.15, min_out_length=0.22)
valid_ids = valid_ids.reshape(-1)
if manual_merge_pairs is not None:
walls_2d_line_new = GraphUtils.opti_wall_manually(walls_2d_line_new, 'XYXYSin2', manual_merge_pairs)
#_show_3d_points_objs_ls(objs_ls=[walls_2d_line_new], obj_rep='RoLine2D_UpRight_xyxy_sin2a')
try:
walls_3d_line_new = np.concatenate( [walls_2d_line_new, walls_3d_line[valid_ids][:,5:8]], axis=1 )
except:
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
bottom_corners_new = OBJ_REPS_PARSE.encode_obj(walls_3d_line_new, 'XYXYSin2WZ0Z1', 'Bottom_Corners').reshape(-1,3)
#_show_3d_points_objs_ls([bottom_corners_new], objs_ls = [walls_3d_line_new], obj_rep='XYXYSin2WZ0Z1')
return walls_3d_line_new
pass
def transfer_lines(lines, obj_rep, img_shape, angle, offset): ''' lines: [n,5] angle: clock-wise is positive offset: (2) ''' scale = 1 h, w = img_shape #assert h%2 == 0 #assert w%2 == 0 center = ((w - 0) * 0.5, (h - 0) * 0.5 ) matrix = cv2.getRotationMatrix2D(center, -angle, scale) n = lines.shape[0] assert lines.shape[1] == 4 assert obj_rep == 'RoLine2D_2p' #lines_2endpts = decode_line_rep(lines, obj_rep).reshape(n,2,2) lines_2endpts = OBJ_REPS_PARSE.encode_obj(lines, obj_rep, 'RoLine2D_2p').reshape(n,2,2) ones = np.ones([n,2,1], dtype=lines.dtype) tmp = np.concatenate([lines_2endpts, ones], axis=2).reshape([n*2, 3]) lines_2pts_r = np.matmul( tmp, matrix.T ).reshape([n,2,2]) lines_2pts_r[:,:,0] += offset[0] lines_2pts_r[:,:,1] += offset[1] lines_rotated = OBJ_REPS_PARSE.encode_obj(lines_2pts_r.reshape(n,4), 'RoLine2D_2p', obj_rep) return lines_rotated
def rotate_bboxes_img(bboxes, img, angle, obj_rep):
#angle = 30
if obj_rep != 'XYXYSin2':
XYXYSin2W = OBJ_REPS_PARSE.encode_obj(bboxes, obj_rep, 'XYXYSin2W')
lines = XYXYSin2W[:,:5]
else:
lines = bboxes
rotate_lines, new_img, scale = rotate_lines_img(lines, img, angle, 'XYXYSin2')
if obj_rep == 'XYXYSin2':
return rotate_lines, new_img
width = XYXYSin2W[:,5:6] * scale
r_XYXYSin2W = np.concatenate([ rotate_lines, width ], axis=1)
if obj_rep == 'XYXYSin2W':
return r_XYXYSin2W, new_img
elif obj_rep == 'XYXYSin2WZ0Z1':
XYXYSin2WZ0Z1 = np.concatenate([ r_XYXYSin2W, bboxes[:,[-2,-1]] ], axis=1)
return XYXYSin2WZ0Z1, new_img
elif obj_rep == 'XYZLgWsHA':
zc, H = bboxes[:,2:3], bboxes[:,5:6]
z0 = zc - H/2
z1 = zc + H/2
XYXYSin2WZ0Z1 = np.concatenate([ r_XYXYSin2W, z0, z1], axis=1)
elif obj_rep == 'Rect4CornersZ0Z1':
XYXYSin2WZ0Z1 = np.concatenate([ r_XYXYSin2W, bboxes[:,[-2,-1]] ], axis=1)
else:
raise NotImplementedError
rotated_bboxes = OBJ_REPS_PARSE.encode_obj(XYXYSin2WZ0Z1, 'XYXYSin2WZ0Z1', obj_rep)
rotated_bboxes = rotated_bboxes.astype(np.float32)
show = 0
if show:
_show_objs_ls_points_ls( new_img[:,:,0], [rotated_bboxes], obj_rep=obj_rep )
pass
return rotated_bboxes, new_img
def clean_outer_false_walls_of_1_room(room, walls, obj_rep):
from mmdet.core.bbox.geometry import dsiou_rotated_3d_bbox_np
assert obj_rep == 'XYZLgWsHA'
nw = walls.shape[0]
cen_room = room[None, :2]
w_lines = OBJ_REPS_PARSE.encode_obj(walls[:,:7], obj_rep, 'RoLine2D_2p').reshape(-1,2,2)
valid_ids = []
for i in range(nw):
cen_i = walls[i:i+1, :2]
box = np.concatenate([cen_i, cen_room], 1)
box = OBJ_REPS_PARSE.encode_obj(box, 'RoLine2D_2p', obj_rep)
box[:, 3:5] *= 0.95
line_i = OBJ_REPS_PARSE.encode_obj(box, obj_rep, 'RoLine2D_2p').reshape(1,2,2)
intsects = lines_intersection_2d( line_i, w_lines, True, True )
valid_i = np.isnan(intsects).all()
if valid_i:
valid_ids.append( i )
if not valid_i and 0:
print(valid_i)
print(intsects)
_show_objs_ls_points_ls( (512,512), [walls[:,:7], walls[i:i+1,:7], box], obj_rep, obj_colors=['white', 'lime', 'red'] )
#_show_objs_ls_points_ls( (512,512), [walls[:,:7], walls[valid_ids,:7], box], obj_rep, obj_colors=['white', 'lime', 'red'] )
return valid_ids
def intersect_2edges_by_modification(walls0, obj_rep, corners_bad=None):
'''
corners_bad: if this is not None, this is the corners that should be fixed.
'''
assert walls0.shape[0] == 2
#_show_objs_ls_points_ls( (512,512), [walls0], obj_rep, points_ls=[corners_bad] )
w0, w1 = walls0[0], walls0[1]
cor_0 = OBJ_REPS_PARSE.encode_obj(w0[None,:], obj_rep, 'RoLine2D_2p').reshape(2,2)
cor_1 = OBJ_REPS_PARSE.encode_obj(w1[None,:], obj_rep, 'RoLine2D_2p').reshape(2,2)
intersect = line_intersection_2d(cor_0, cor_1, min_angle=np.pi/8).reshape(1,2)
cor_0_new = replace_1cor_of_edge_to_midify(cor_0, intersect).reshape(1,4)
cor_1_new = replace_1cor_of_edge_to_midify(cor_1, intersect).reshape(1,4)
if corners_bad is not None:
dis_to_bad = np.linalg.norm( corners_bad - intersect, axis=1 )
dis0 = np.linalg.norm( cor_0 - intersect, axis=1 )
dis1 = np.linalg.norm( cor_1 - intersect, axis=1 )
check0 = abs(dis_to_bad[0] - dis0.min()) < 1
check1 = abs(dis_to_bad[1] - dis1.min()) < 1
check = check0 * check1
if not check:
# The corners_bad should be replaced. But it is still here. Invalid
# connection
return None
w_0_new = OBJ_REPS_PARSE.encode_obj(cor_0_new, 'RoLine2D_2p', obj_rep)[0]
w_1_new = OBJ_REPS_PARSE.encode_obj(cor_1_new, 'RoLine2D_2p', obj_rep)[0]
walls_new = np.concatenate([w_0_new[None], w_1_new[None]], 0)
return walls_new
def transfer_lines_points(lines, obj_rep, points=None, center=(0,0), angle=0, offset=(0,0)):
'''
lines: [n,5]
points: [m,2]
angle: clock-wise is positive, degree
offset: (2)
'''
assert lines.ndim == 2
n = lines.shape[0]
scale = 1
angle *= 180/np.pi
matrix = cv2.getRotationMatrix2D(center, -angle, scale)
lines_2endpts = OBJ_REPS_PARSE.encode_obj(lines, obj_rep, 'RoLine2D_2p').reshape(n,2,2)
ones = np.ones([n,2,1], dtype=lines.dtype)
tmp = np.concatenate([lines_2endpts, ones], axis=2).reshape([n*2, 3])
lines_2pts_r = np.matmul( tmp, matrix.T ).reshape([n,2,2])
lines_2pts_r[:,:,0] += offset[0]
lines_2pts_r[:,:,1] += offset[1]
lines_rotated = OBJ_REPS_PARSE.encode_obj(lines_2pts_r.reshape(n,4), 'RoLine2D_2p', obj_rep)
if obj_rep == 'XYLgWsA':
lines_rotated[:,3] = lines[:,3]
if obj_rep == 'XYZLgWsHA':
lines_rotated[:,[2,4,5]] = lines[:,[2,4,5]]
if points is not None:
assert points.ndim == 2
m,pc = points.shape
ones = np.ones([m,1], dtype=lines.dtype)
tmp = np.concatenate([points, ones], axis=1).reshape([m, pc+1])
points_r = np.matmul( tmp, matrix.T ).reshape([m,pc])
else:
points_r = None
return lines_rotated, points_r
def manual_rm_objs(bboxes0, obj_rep, all_colors, max_size=7): print(bboxes0[:,3] ) bboxes1 = OBJ_REPS_PARSE.encode_obj(bboxes0, obj_rep, 'XYZLgWsHA') #_show_3d_bboxes_ids(bboxes0, obj_rep) mask1 = bboxes1[:,3] < max_size angle_dif = np.abs(limit_period_np(bboxes1[:,6], 0.5, np.pi/2)) mask2 = angle_dif < 0.3 mask = mask1 * mask2 bboxes2 = bboxes1[mask] bboxes_out = OBJ_REPS_PARSE.encode_obj(bboxes2, 'XYZLgWsHA', obj_rep) all_colors = [all_colors[i] for i in range(len(mask)) if mask[i]] return bboxes_out, all_colors
def move_axis_aligned_box_to_wall(box2d, walls, cat_name, voxel_size):
assert box2d.shape == (1,5)
assert walls.shape[1] == 8
box2d0 = box2d.copy()
max_thick = MAX_THICK_MAP[cat_name] / voxel_size
thick_add_on_wall = int(THICK_GREATER_THAN_WALL[cat_name] / voxel_size)
n = walls.shape[0]
center = np.array(box2d[0,:2])
wall_lines = OBJ_REPS_PARSE.encode_obj( walls, 'XYXYSin2WZ0Z1', 'RoLine2D_2p' ).reshape(-1,2,2)
walls_ = OBJ_REPS_PARSE.encode_obj(walls, 'XYXYSin2WZ0Z1', 'XYLgWsA')
diss = vertical_dis_1point_lines(center, wall_lines, no_extend=False)
angles_dif = walls_[:,-1] - box2d[:,-1]
angles_dif = limit_period_np(angles_dif, 0.5, np.pi)
invalid_mask = np.abs(angles_dif) > np.pi/4
diss[invalid_mask] = 1000
diss *= voxel_size
if diss.min() > 0.3:
return box2d
the_wall_id = diss.argmin()
the_wall = walls_[the_wall_id]
thick_add_on_wall = int(THICK_GREATER_THAN_WALL[cat_name] / voxel_size)
new_thick = min(the_wall[3] + thick_add_on_wall, max_thick)
new_thick = max(new_thick, max_thick*0.7)
box2d[0,3] = new_thick
angle_dif = limit_period_np( the_wall[-1] - box2d[0,-1], 0.5, np.pi )
if abs(box2d[0,-1]) < np.pi/4:
# long axis is x, move y
move_axis = 1
else:
move_axis = 0
# if the center of box2d is not inside the wall, do not move
j = 1- move_axis
inside_wall = box2d0[0,j] < the_wall[j]+the_wall[2]/2 and box2d0[0,j] > the_wall[j]-the_wall[2]/2
if inside_wall:
move0 = the_wall[move_axis] - box2d0[0,move_axis]
if abs(move0 * voxel_size) < 0.1:
move = move0
else:
move = abs(new_thick - box2d0[0,3]) * np.sign(move0) / 2
box2d[0,move_axis] += move
print(f'move:{move}, move0:{move0}')
#_show_3d_points_objs_ls( objs_ls=[ box2d0, box2d, the_wall[None]], obj_rep='XYLgWsA', obj_colors=['blue', 'red', 'black'])
#_show_3d_points_objs_ls( objs_ls=[ box2d0, the_wall[None]], obj_rep='XYLgWsA', obj_colors=[ 'blue', 'black'])
#_show_3d_points_objs_ls( objs_ls=[ box2d, the_wall[None]], obj_rep='XYLgWsA', obj_colors=[ 'red', 'black'])
pass
return box2d
def intersect_2edges_by_add_new_wall(walls0, obj_rep, corners_bad=None): ''' corners_bad: if this is not None, this is the corners that should be fixed. ''' assert walls0.shape[0] == 2 #_show_objs_ls_points_ls( (512,512), [walls0], obj_rep, points_ls=[corners_bad] ) w0, w1 = walls0[0], walls0[1] cor_0 = OBJ_REPS_PARSE.encode_obj(w0[None,:], obj_rep, 'RoLine2D_2p').reshape(2,2) cor_1 = OBJ_REPS_PARSE.encode_obj(w1[None,:], obj_rep, 'RoLine2D_2p').reshape(2,2) intersect = line_intersection_2d(cor_0, cor_1, min_angle=np.pi/8).reshape(1,2) intersect = np.repeat( intersect, 2, 0 ) new_walls = np.concatenate( [intersect, corners_bad], 1 ) new_walls = OBJ_REPS_PARSE.encode_obj(new_walls, 'RoLine2D_2p', obj_rep) return new_walls
def draw_corners_to_box( point_thickness=25,
line_thickness=4,
box_thickness = 4):
corners = np.array( [
[100, 100],
[800, 300],
[900, 500],
[600, 800],
] )
corners = sort_four_corners_np(corners[None,:,:])[0].reshape(-1,2)
center = corners.mean(0,keepdims=True)
box_XYDAsinAsinSin2 = four_corners_to_box_np(corners[None,:,:])
box = OBJ_REPS_PARSE.encode_obj(box_XYDAsinAsinSin2, 'XYDAsinAsinSin2', 'XYLgWsA')
tmp = np.repeat( center, 4, 0)
mid_points = ( corners + corners[[1,2,3,0],:] ) /2
mid_lines = np.concatenate([mid_points[:,None,:], tmp[:,None,:]], 1)
lines = np.concatenate([corners[:,None,:], tmp[:,None,:]], 1)
h,w = 1024, 1024
img = np.zeros([h,w,3], dtype=np.uint8) + 255
img = _draw_points(img, corners, 'red', point_thickness)
img = _draw_points(img, center, 'blue', point_thickness)
img = _draw_points(img, mid_points, 'green', point_thickness)
img = _draw_lines(img, lines, 'red', line_thickness)
img = _draw_lines(img, mid_lines, 'blue', line_thickness)
img = draw_XYLgWsA(img, box, 'black', box_thickness)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
cv2.imshow('', img)
cv2.waitKey(0)
cv2.destroyAllWindows()
def _make_bbox_line_mesh(bbox, box_oriented, color):
assert box_oriented
assert bbox.shape == (7, )
radius = BOX_LINE_RADIUS
#radius = bbox[3:6].max()/200
lines = OBJ_REPS_PARSE.get_12_line_cors(bbox.reshape(1, 7),
'XYZLgWsHA')[0] # [12,2,3]
centroids = lines.mean(1)
directions = lines[:, 1, :] - lines[:, 0, :]
heights = np.linalg.norm(directions, axis=1)
heights = np.clip(heights, a_min=0.001, a_max=None)
directions = directions / heights.reshape([-1, 1])
mesh = []
for i in range(12):
cylinder_i = o3d.geometry.TriangleMesh.create_cylinder(
radius=radius, height=heights[i])
cylinder_i.paint_uniform_color(color)
transformation = np.identity(4)
transformation[:3, 3] = centroids[i]
transformation[:3, 2] = directions[i]
cylinder_i.transform(transformation)
mesh.append(cylinder_i)
cm = mesh[0]
for i in range(1, 12):
cm += mesh[i]
return cm
def draw_objs(img,
objs,
obj_rep,
color,
obj_thickness=1,
scores=None,
draw_rooms=False,
cats=None,
font_scale=FONT_SCALE,
text_color='green',
dash=False):
if obj_rep != 'XYLgWsA':
objs = OBJ_REPS_PARSE.encode_obj(objs, obj_rep, 'XYLgWsA')
if draw_rooms:
img = draw_rooms_from_walls(img.shape[:2], objs, 'XYLgWsA')
draw_XYLgWsA(img,
objs,
color,
obj_thickness=obj_thickness,
scores=scores,
cats=cats,
font_scale=font_scale,
text_color=text_color,
dash=dash)
return img
def m_transform_lines(lines, matrix, obj_rep): ''' lines: [n,5] matrix: [4,4] ''' assert matrix.shape == (4,4) assert lines.shape[1] == 5 assert obj_rep == 'XYXYSin2' n = lines.shape[0] lines_2endpts = OBJ_REPS_PARSE.encode_obj(lines, obj_rep, 'RoLine2D_2p').reshape(n,2,2) ones = np.ones([n,2,2], dtype=lines.dtype) tmp = np.concatenate([lines_2endpts, ones], axis=2).reshape([n*2, 4]) lines_2pts_r = (tmp @ matrix.T)[:,:2].reshape([n,2,2]) lines_rotated = OBJ_REPS_PARSE.encode_obj(lines_2pts_r.reshape(n,4), 'RoLine2D_2p', obj_rep) return lines_rotated.astype(np.float32)
def show_gt_bboxes_1scene_bev(self, index):
img_info = self.img_infos[index]
print('\n\n', img_info['filename'])
ann = img_info['ann']
gt_bboxes_3d_raw = ann['bboxes_XYXYSin2WZ0Z1']
gt_bboxes = ann['gt_bboxes']
gt_labels = ann['labels']
rm_cats = ['ceiling']
#rm_cats = ['beam']
kp_cats = ['wall', 'beam', 'column', 'door', 'window']
# 2d
gt_corners = OBJ_REPS_PARSE.encode_obj(gt_bboxes, self.obj_rep,
'RoLine2D_2p')
gt_corners, _ = filter_categories('keep', gt_corners, gt_labels,
kp_cats, self._category_ids_map)
gt_corners = gt_corners.reshape(-1, 2)
w, h = gt_corners.max(0).astype(np.int) + 10
gt_bboxes, gt_labels = filter_categories('keep', gt_bboxes, gt_labels,
kp_cats,
self._category_ids_map)
obj_cats = np.array(kp_cats)[gt_labels]
_show_objs_ls_points_ls((h, w), [gt_bboxes],
self.obj_rep,
points_ls=[gt_corners],
obj_scores_ls=[obj_cats])
def getOrientedLineRectSubPix(img, line, obj_rep, length_aug=-5, thickness_aug=40):
'''
img: [h,w] [h,w,1/3]
line: [5]
'''
debug = 0
assert line.shape == (5,)
assert img.shape[:2] == (512,512)
if debug:
img = _draw_lines_ls_points_ls(img, [line.reshape(-1,5)])
h, w = img.shape[:2]
#angle = line[-1]
#line = decode_line_rep(line[None, :], obj_rep)[0]
line = OBJ_REPS_PARSE.encode_obj(line[None,:], obj_rep, 'RoLine2D_2p').reshape(-1,2,2)[0]
points = line[:4].reshape(2,2)
x, y = points.mean(axis=0).astype(np.int)
# move to center first
xofs, yofs = w/2-x, h/2-y
M_mc = np.array([[1,0, xofs], [0,1,yofs]])
dir_v = points[1] - points[0]
angle = angle_with_x_np( dir_v.reshape(1,2), 1 )[0]
angle = -angle * 180 / np.pi
M = cv2.getRotationMatrix2D( (w/2,h/2), angle, 1 )
img_1 = cv2.warpAffine(img, M_mc, (w,h))
img_rot = cv2.warpAffine(img_1, M, (w,h))
tmp = np.concatenate([points, np.ones([2,1])], axis=1)
points_1 = tmp @ M_mc.T
tmp = np.concatenate([points_1, np.ones([2,1])], axis=1)
points_rot = tmp @ M.T
x,y = points_rot.mean(axis=0)
center = (x,y)
length = np.linalg.norm(points_rot[1] - points_rot[0]) + length_aug
length = max(length, 5)
thickness = thickness_aug
size = (int(length), int(thickness))
roi = cv2.getRectSubPix(img_rot, size, center)
if debug:
print(angle)
_show_lines_ls_points_ls(img, points_ls=[points])
_show_lines_ls_points_ls(img_rot, points_ls=[points_rot])
_show_lines_ls_points_ls(roi)
pass
return roi
def fulfil_a_rectangle_room(corners_sorted, walls, room, obj_rep, show_fix_process_per_room=0):
'''
There are only two walls for one room. Add these two walls are connected.
Add one extra corner and two extra walls to fulfil a rectangle.
'''
cors = corners_sorted
cor3 = cors[0] - cors[1] + cors[2]
tmp = np.repeat(cor3[None], 2, 0)
wall_cors_new = np.concatenate([tmp, cors[[0,2]]], 1).reshape(2,4)
walls_new = OBJ_REPS_PARSE.encode_obj(wall_cors_new, 'RoLine2D_2p', obj_rep)
walls_new = np.concatenate([walls_new, walls[:,7:8]], 1)
if show_fix_process_per_room:
_show_objs_ls_points_ls( (512,512), [ walls[:,:7], room[None,:7], walls_new[:,:7] ], obj_rep, obj_colors=['red', 'white', 'lime'])
return walls, walls_new
def nms_rotated(dets,
obj_rep,
iou_thr,
min_width_length_ratio=0.3,
device_id=None):
from detectron2 import _C
if isinstance(dets, torch.Tensor):
is_numpy = False
dets_th = dets
elif isinstance(dets, np.ndarray):
is_numpy = True
device = 'cpu' if device_id is None else 'cuda:{}'.format(device_id)
dets_th = torch.from_numpy(dets).to(device)
else:
raise TypeError(
'dets must be either a Tensor or numpy array, but got {}'.format(
type(dets)))
obj_dim = OBJ_REPS_PARSE._obj_dims[obj_rep]
assert dets.shape[1] >= obj_dim + 1
# execute cpu or cuda nms
if dets_th.shape[0] == 0:
inds = dets_th.new_zeros(0, dtype=torch.long)
else:
# patrse obj_rep to XYZHWA
scores = dets_th[:, -1].clone()
dets_th = OBJ_REPS_PARSE.encode_obj(dets_th[:, :obj_dim], obj_rep,
'XYLgWsA')
dets_th[:, 3] = torch.max(dets_th[:, 3],
dets_th[:, 2] * min_width_length_ratio)
#_show_objs_ls_points_ls_torch( (512,512), [dets_th[:,:obj_dim]], 'XYLgWsA' )
dets_th[:, 4] *= -180 / np.pi
inds = _C.nms_rotated(dets_th, scores, iou_thr)
if is_numpy:
inds = inds.cpu().numpy()
if DEBUG_CFG.SHOW_NMS_PROCESS:
n0 = scores.shape[0]
n1 = inds.shape[0]
print(f'\n\n{n0} -> {n1}')
_show_objs_ls_points_ls_torch((512, 512), [dets[:, :obj_dim]], obj_rep)
_show_objs_ls_points_ls_torch((512, 512), [dets[inds][:, :obj_dim]],
obj_rep)
import pdb
pdb.set_trace() # XXX BREAKPOINT
return dets[inds, :], inds
def rm_one_wall_for_vis(gt_bboxes_3d_raw, gt_labels, scene_name):
mask = gt_labels == 0
walls = gt_bboxes_3d_raw[mask]
walls = OBJ_REPS_PARSE.encode_obj(walls, 'XYXYSin2WZ0Z1', 'XYZLgWsHA')
max_xy = walls[:,:2].max(0)
min_xy = walls[:,:2].min(0)
if scene_name == 'Area_3/office_4':
del_axis=0
#i = np.where( walls[:,del_axis] == max_xy[del_axis] )[0]
i = np.where( walls[:,del_axis] == min_xy[del_axis] )[0]
else:
return gt_bboxes_3d_raw, gt_labels
gt_bboxes_3d_raw = np.delete(gt_bboxes_3d_raw, i, 0)
gt_labels = np.delete(gt_labels, i, 0)
return gt_bboxes_3d_raw, gt_labels
def test_rotation_order():
u = np.pi / 180
XYLgWsA = np.array([
[200, 200, 200, 10, 0],
[200, 200, 200, 30, 30 * u],
])
obj_rep = 'XYLgWsA'
img = np.zeros([512, 512, 3], dtype=np.uint8)
#img = draw_XYLgWsA( img, XYLgWsA, 'red')
#mmcv.imshow(img)
corners = OBJ_REPS_PARSE.encode_obj(XYLgWsA, 'XYLgWsA',
'RoLine2D_2p').reshape(-1, 2)
XYZLgWsHA = OBJ_REPS_PARSE.encode_obj(XYLgWsA, 'XYLgWsA', 'XYZLgWsHA')
XYZLgWsHA[:, 5] = 10
corners_3d = OBJ_REPS_PARSE.encode_obj(XYZLgWsHA, 'XYZLgWsHA',
'Bottom_Corners').reshape(-1, 3)
#_show_objs_ls_points_ls( (512,512), [XYLgWsA], obj_rep, points_ls=[corners])
_show_3d_points_objs_ls([corners_3d],
objs_ls=[XYZLgWsHA],
obj_rep='XYZLgWsHA')
pass
def _show_3d_points_objs_ls(points_ls=None,
point_feats=None,
objs_ls=None,
obj_rep=None,
obj_colors='random',
box_types=None,
polygons_ls=None,
polygon_colors='random'):
if objs_ls is not None:
if obj_rep == 'XYXYSin2':
if points_ls is not None:
for i in range(len(points_ls)):
if points_ls[i].shape[1] == 2:
n = points_ls[i].shape[0]
z = np.zeros([n, 1])
points_ls[i] = np.concatenate([points_ls[i], z],
axis=1)
if objs_ls is not None:
for i in range(len(objs_ls)):
n = objs_ls[i].shape[0]
tzz = np.ones([n, 3])
tzz[:, 0] = 0.01
tzz[:, 1] = 0
tzz[:, 2] = 0.01
objs_ls[i] = np.concatenate([objs_ls[i], tzz], axis=1)
obj_rep = 'XYXYSin2WZ0Z1'
bboxes_ls = [
OBJ_REPS_PARSE.encode_obj(o, obj_rep, 'XYZLgWsHA') for o in objs_ls
]
else:
bboxes_ls = None
mesh_ls = _show_3d_points_bboxes_ls(points_ls,
point_feats,
bboxes_ls,
obj_colors,
box_oriented=True,
box_types=box_types)
if polygons_ls is not None:
pls = _make_polygon_surface_ls(polygons_ls, polygon_colors)
mesh_ls += pls
#o3d.visualization.draw_geometries( show_ls )
custom_draw_geometry_with_key_callback(mesh_ls)
def draw_1_obj_dash(img, obj, color, obj_thickness, obj_rep):
assert obj_rep == 'XYLgWsA'
len_unit = 6
gap_unit = 8
corners = OBJ_REPS_PARSE.encode_obj(obj[None], obj_rep,
'RoLine2D_2p').reshape(2, 2)
length = obj[2]
n = length / (len_unit + gap_unit)
n = int(np.floor(n))
n = max(n, 2)
objs_dash = np.repeat(obj[None], n, 0)
objs_dash[:, 2] = len_unit
for i in range(n):
r = (i + 1) / n
center = corners[0] * r + corners[1] * (1 - r)
objs_dash[i, :2] = center
draw_1_obj(img, objs_dash[i], color, obj_thickness, obj_rep)
pass
def lines2d_to_bboxes3d(lines, line_obj_rep='lscope_istopleft', height=60, thickness=1):
'''
lines: [n,5]
bboxes: [n,9] [center, size, angle]
'''
import open3d as o3d
from beike_data_utils.geometric_utils import angle_with_x_np
assert line_obj_rep == 'lscope_istopleft'
assert lines.ndim == 2
assert lines.shape[1] == 5
n = lines.shape[0]
#lines_std = decode_line_rep(lines, line_obj_rep).reshape(n,2,2)
lines_std = OBJ_REPS_PARSE.encode_obj(lines, obj_rep, 'RoLine2D_2p').reshape(n,2,2)
center2d = lines_std.mean(axis=1)
vec_rotation = lines_std[:,0] - center2d
z_angles = -angle_with_x_np(vec_rotation, scope_id=0)
length = np.linalg.norm( lines_std[:,1] - lines_std[:,0] , axis=1)
center = np.zeros([lines.shape[0], 3], dtype=lines.dtype)
center[:,:2] = center2d
center[:,2] = height/2
extent = np.zeros([lines.shape[0], 3], dtype=lines.dtype)
extent[:,0] = length
extent[:,1] = thickness
extent[:,2] = height
angles = np.zeros([lines.shape[0], 3], dtype=lines.dtype)
angles[:,2] = z_angles
bboxes3d = np.concatenate([center, extent, angles], axis=1)
#_show_3d_points_bboxes_ls(bboxes_ls = [bboxes3d], box_oriented=True)
return bboxes3d
matrixes = []
for a in theta:
axis_angle = np.array([0,0,a]).reshape([3,1])
matrix = o3d.geometry.get_rotation_matrix_from_yxz(axis_angle)
matrixes.append( matrix )
matrixes = np.array(matrixes)
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
def draw_XYXYSin2(img,
objs,
color,
obj_thickness=1,
font_scale=0.5,
text_color='green'):
'''
img: [h,w,3]
objs: [n,5/6]
color: 'red'
'''
assert objs.ndim == 2
assert objs.shape[1] == 5 or objs.shape[1] == 6
if objs.shape[1] == 6:
scores = objs[:, 5]
objs = objs[:, :5]
else:
scores = None
rotations = objs[:, 4]
text_color = _get_color(text_color)
objs = decode_line_rep(objs, 'lscope_istopleft').reshape(-1, 2, 2)
objs = OBJ_REPS_PARSE.encode_obj(objs, 'XYXYSin2', 'RoLine2D_2p')
for i in range(objs.shape[0]):
s, e = np.round(objs[i]).astype(np.int32)
c = _get_color(color)
cv2.line(img, (s[0], s[1]), (e[0], e[1]), c, thickness=obj_thickness)
#label_text = class_names[label] if class_names is not None else 'cls {}'.format(label)
label_text = ''
if DEBUG_CFG.OBJ_LEGEND == 'score':
if scores is not None:
label_text += '{:.01f}'.format(scores[i]) # score
else:
label_text += '{:.01f}'.format(rotations[i]) # rotation
m = np.round(((s + e) / 2)).astype(np.int32)
if label_text != '':
cv2.putText(img, label_text, (m[0] - 4, m[1] - 4),
cv2.FONT_HERSHEY_COMPLEX, font_scale, text_color)
return img
def connect_two_corner(corners, walls0, obj_rep):
assert corners.shape == (2,2)
assert walls0.shape[0] == 2
show = 0
c = walls0.shape[1]
angle_dif = limit_period_np( walls0[0,6] - walls0[1,6], 0.5, np.pi)
angle_dif = abs(angle_dif)
if angle_dif > np.pi / 4:
new_wall = intersect_2edges_by_add_new_wall( walls0[:,:7], obj_rep, corners )
walls1 = walls0[:,:7]
else:
new_wall = OBJ_REPS_PARSE.encode_obj(corners.reshape(1,4), 'RoLine2D_2p', obj_rep)
walls1 = walls0[:,:7]
if c==8:
score = new_wall[:,0:1].copy()
score[:] =1
new_wall = np.concatenate([new_wall, score ], 1)
if show:
_show_objs_ls_points_ls( (512, 512), [walls0[:,:7], walls1[:,:7], new_wall[:,:7]], obj_rep,
obj_colors=['white', 'green', 'red'], obj_thickness=[4,1,1] )
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
return walls1, new_wall
def cal_wall_in_room_scores(walls, rooms_aug, rooms, obj_rep):
from mmdet.core.bbox.geometry import dsiou_rotated_3d_bbox_np
assert obj_rep == 'XYZLgWsHA'
ious_inner = dsiou_rotated_3d_bbox_np( walls, rooms_aug, iou_w=1.0, size_rate_thres=None, ref='bboxes1', only_2d=True )
# see if a wall is the edge of a room
num_walls = walls.shape[0]
num_rooms = rooms.shape[0]
border_4_lines_0 = OBJ_REPS_PARSE.get_border_4_lines( rooms, obj_rep )
border_4_lines = border_4_lines_0.reshape(num_rooms*4, 7)
#_show_objs_ls_points_ls( (512,512), [ border_4_lines_0.reshape(-1,7) ], obj_rep,)
ious_border_0 = dsiou_rotated_3d_bbox_np( walls, border_4_lines, iou_w=1, size_rate_thres=0.2, ref='bboxes1', only_2d=True )
ious_border_1 = ious_border_0.reshape(num_walls, num_rooms, 4)
ious_border = ious_border_1.max(2)
#angles_dif = walls[:, None, 6] - border_4_lines[None,:,6]
#angles_dif = np.abs( angle_dif_by_period_np( angles_dif, 0, 0 ))
#angles_dif = angles_dif.reshape(num_walls, num_rooms, 4).min(2)
ious_max = np.maximum( ious_border, ious_inner )
ious_min = np.minimum( ious_border, ious_inner )
ious = ious_max * 0.8 + ious_min * 0.2
if 0:
for i in range(num_rooms):
ious_b_i = ious_border[:,i]
ious_i = ious[:,i]
print(f'border iou')
#_show_objs_ls_points_ls( (512,512), [walls, rooms[i:i+1]], obj_rep, obj_colors=['random', 'white' ], obj_thickness=[1,2], obj_scores_ls=[ ious_b_i, None ] )
print(f'inner iou')
print(f'fuse iou')
_show_objs_ls_points_ls( (512,512), [walls, rooms_aug[i:i+1]], obj_rep, obj_colors=['random', 'white' ], obj_thickness=[1,2], obj_scores_ls=[ ious_i, None ] )
#print(ious)
#_show_objs_ls_points_ls( (512,512), [walls, rooms], obj_rep, obj_colors=['green', 'red' ], obj_thickness=[1,3] )
return ious
def bboxes_flip_scope_itl(bboxes_in, curr_ax, coord_max, obj_rep):
from obj_geo_utils.obj_utils import OBJ_REPS_PARSE
if obj_rep == 'XYXYSin2':
lines0 = bboxes_in
assert lines0.shape[1] == 5
assert curr_ax==0 or curr_ax==1
tmp = coord_max - lines0[:, curr_ax]
lines0[:, curr_ax] = coord_max - lines0[:, curr_ax + 2]
lines0[:, curr_ax + 2] = tmp
lines0[:,4] = -lines0[:,4]
return lines0
elif obj_rep == 'Rect4CornersZ0Z1':
assert bboxes_in.shape[1] == 10
assert curr_ax==0 or curr_ax==1
flipped = bboxes_in.copy()
if curr_ax == 0:
flipped[..., 0:8:2] = coord_max - bboxes_in[..., 0:8:2]
if curr_ax == 1:
flipped[..., 1:9:2] = coord_max - bboxes_in[..., 1:9:2]
flipped = OBJ_REPS_PARSE.update_corners_order( flipped, obj_rep )
return flipped
else:
raise NotImplementedError
def _show_2dlines_as_3d(lines_2d_ls, obj_rep, filename, height_method=['same','per_room'][1]):
'''
lines_2d_ls: [walls, windows, doors]
'''
from obj_geo_utils.geometry_utils import get_rooms_from_edges
from beike_data_utils.beike_utils import load_ply
wall_idx, door_idx, win_idx = 0, 1, 2
assert obj_rep == 'XYZLgWsHA'
scene_name = os.path.splitext(os.path.basename(filename))[0]
print(f'{scene_name}')
m2p_file = '../'+filename.replace('TopView_VerD', 'meter_to_pix_rates').replace('npy','txt')
m2p_rate = np.loadtxt(m2p_file)
rooms_line_ids, room_ids_per_edge, num_walls_inside_room, rooms = get_rooms_from_edges(lines_2d_ls[0], obj_rep, gen_bbox=True, show_rooms=0)
num_room = len(rooms_line_ids)
scope_file = '../'+filename.replace('TopView_VerD', 'pcl_scopes').replace('npy','txt')
pcl_scope = np.loadtxt(scope_file)
max_height = pcl_scope[1,-1] - pcl_scope[0,-1] - 0.3
max_height = min(max_height, 4)
#bottom = pcl_scope[0,-1]
bottom = - max_height/2 + 0.1
if height_method == 'same':
heights = [max_height for _ in range(num_room)]
elif height_method == 'per_room':
pcl_file = '../'+filename.replace('TopView_VerD', 'ply').replace('npy','ply')
pcl = load_ply(pcl_file)[:,:3]
pcl_min = pcl.min(0).reshape(1,-1)
pcl = pcl - pcl_min
ids = np.random.choice(int(pcl.shape[0]), 100*1000)
pcl = pcl[ids]
heights = []
point_nums = []
for i in range(num_room):
rc = rooms[i:i+1,:2] / m2p_rate
rs = rooms[i,3:5].max() / m2p_rate
mask = np.linalg.norm( pcl[:,:2] - rc, axis=1) < rs * 0.7
p_n = mask.sum()
point_nums.append(p_n)
if p_n < 100:
heights.append(max_height)
continue
r_points = pcl[mask]
height_i = r_points[:,2].max() - bottom
gap = (max_height - height_i)/ max_height
if gap < 0.1:
height_i = max_height
heights.append(height_i)
pass
wall_thickness = 0.2 # 0.24
door_thickness = 0.23
door_height = 2.1
window_height = 1.5
for i,ls in enumerate( lines_2d_ls):
ls[:,:6] /= m2p_rate
if i==wall_idx:
ls[:,4] = wall_thickness
#ls[:,5] = max_height
else:
ls[:,4] = door_thickness
if i==door_idx:
ls[:,5] = door_height
if i==win_idx:
ls[:,5] = 1.5
for i in range(num_room):
ids = rooms_line_ids[i]
lines_2d_ls[0][ids, 5] = np.clip( lines_2d_ls[0][ids,5], a_min = heights[i], a_max=None)
lines_2d_ls[wall_idx][:,2] = lines_2d_ls[wall_idx][:,2] + (lines_2d_ls[wall_idx][:,5] - max_height)/2
lines_2d_ls[wall_idx][:,3] += wall_thickness * 0.8
lines_2d_ls[door_idx][:,2] = lines_2d_ls[door_idx][:,2] + (lines_2d_ls[door_idx][:,5] - max_height)/2
lines_2d_ls[win_idx][:,2] = lines_2d_ls[win_idx][:,5]/2 + 0.85 - max_height/2
labels = []
cat_num = len(lines_2d_ls)
for i in range(cat_num):
labels += [i] * lines_2d_ls[i].shape[0]
obj_colors = ['gray', 'green', 'blue']
colors = [obj_colors[l] for l in labels]
# flip
for ls in lines_2d_ls:
ls[:,1] *= -1
ls[:,-1] *= -1
bboxes = np.concatenate(lines_2d_ls, 0)
walls = lines_2d_ls[0]
walls_2cor = OBJ_REPS_PARSE.encode_obj(walls, obj_rep, 'RoLine2D_2p')
floors = gen_floor_mesh_from_walls(walls_2cor, rooms_line_ids, bottom-0.1)
if 1:
print('point_nums', point_nums)
print('heights', heights)
_show_3d_points_objs_ls(objs_ls = [bboxes, bboxes], obj_rep=obj_rep, obj_colors=[colors, 'black'], box_types=['surface_mesh','line_mesh'], polygons_ls=[floors], polygon_colors=['order'] )
pass
def gen_corners_from_lines_np(lines, labels, obj_rep, min_cor_dis_thr, get_degree=False, flag=''):
'''
lines: [n,5]
labels: [n,1/2]: 1 for label only, 2 for label and score
corners: [m,2]
labels_cor: [m, 1/2]
corIds_per_line: [n,2]
num_cor_uq: m
'''
if lines.shape[0] == 0:
if labels is None:
labels_cor = None
else:
labels_cor = np.zeros([0,labels.shape[1]])
return np.zeros([0,2]), labels_cor, np.zeros([0,2], dtype=np.int), 0
#lines0 = decode_line_rep(lines, obj_rep)
lines0 = OBJ_REPS_PARSE.encode_obj(lines, obj_rep, 'RoLine2D_2p').reshape(-1,2,2)
if labels is not None:
num_line = lines.shape[0]
assert labels.shape[0] == num_line
labels = labels.reshape(num_line, -1)
lc = labels.shape[1]
labels_1 = labels.reshape(-1,lc)
lines1 = np.concatenate([lines0[:,0], labels_1, lines0[:,1], labels_1], axis=1)
corners1 = lines1.reshape(-1,2+lc)
else:
corners1 = lines0.reshape(-1,2)
corners1, _ = merge_close_corners(corners1, min_cor_dis_thr, labels)
corners1 = round_positions(corners1, 1000)
corners_uq, unique_indices, inds_inverse = np.unique(corners1, axis=0, return_index=True, return_inverse=True)
#print( abs(corners_uq[9] - corners_uq[10]).max() )
num_cor_uq = corners_uq.shape[0]
corners = corners_uq[:,:2]
if labels is not None:
labels_cor = corners_uq[:,2:].astype(labels.dtype)
else:
labels_cor = None
corIds_per_line = inds_inverse.reshape(-1,2)
lineIds_per_cor = get_lineIdsPerCor_from_corIdsPerLine(corIds_per_line, corners.shape[0])
if 0:
n_org = corners1.shape[0]
n_new = corners.shape[0]
print(f'{n_org} -> {n_new}')
_show_objs_ls_points_ls( (512,512), points_ls=[corners1], point_colors='red' )
_show_objs_ls_points_ls( (512,512), points_ls=[corners], point_colors='red' )
if find_duplicate_corners(corners, 1, 'B'):
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
cor_degrees = np.zeros([num_cor_uq], dtype=np.int32)-1
for i in corIds_per_line.reshape(-1):
cor_degrees[i] += 1
res = (corners, labels_cor, corIds_per_line, num_cor_uq)
if get_degree:
res = res + (cor_degrees,)
return res
def rotate_lines_img(lines, img, angle, obj_rep, debug_rotation=0):
'''
The img sizes of input and output are the same.
angle in degree
'''
assert obj_rep == 'XYXYSin2'
assert img.ndim == 3
assert lines.ndim == 2
assert lines.shape[1] == 5
assert img.shape[2] == 4
img_shape = img.shape[:2]
if debug_rotation:
add_cross = 1
if add_cross:
img[:,:,1:] = np.abs(img[:,:,1:])*255
lines = add_cross_in_lines(lines, img_shape)
#_show_lines_ls_points_ls(img[:,:,0], [lines])
#_show_lines_ls_points_ls(img[:,:,1:], [lines])
#_show_lines_ls_points_ls(img[:,:,:3], [lines])
pass
n = lines.shape[0]
if n == 0:
return lines, img
lines_2endpts = OBJ_REPS_PARSE.encode_obj(lines, obj_rep, 'RoLine2D_2p').reshape(n,2,2)
#lines_2endpts = decode_line_rep(lines, obj_rep).reshape(n,2,2)
h, w = img_shape
assert h%2 == 0
assert w%2 == 0
center = ((w - 1) * 0.5, (h - 1) * 0.5)
scale = 1.0
matrix = cv2.getRotationMatrix2D(center, -angle, scale)
ones = np.ones([n,2,1], dtype=lines.dtype)
tmp = np.concatenate([lines_2endpts, ones], axis=2)
lines_2pts_r = np.matmul( tmp, matrix.T )
# (1) rotate the lines
lines_rotated = OBJ_REPS_PARSE.encode_obj(lines_2pts_r.reshape(-1,4), 'RoLine2D_2p', obj_rep)
#lines_rotated = encode_line_rep(lines_2pts_r, obj_rep)
#_show_lines_ls_points_ls(img[:,:,0], [lines_rotated])
# (3) scale the lines to fit the image size
# Move before scaling can increase the scale ratio
x_min_ = lines_rotated[:,[0,2]].min()
x_max_ = lines_rotated[:,[0,2]].max()
y_min_ = lines_rotated[:,[1,3]].min()
y_max_ = lines_rotated[:,[1,3]].max()
border_pad = 4
gap_x0 = 0 - x_min_
gap_y0 = 0 - y_min_
gap_x1 = x_max_ - (w-1)
gap_y1 = y_max_ - (h-1)
gap_x = np.ceil(np.array([gap_x0, gap_x1, 0]).max())
gap_y = np.ceil(np.array([gap_y0, gap_y1, 0]).max())
scale_x = (w-border_pad*2) / (w+gap_x*2.0)
scale_y = (h-border_pad*2) / (h+gap_y*2.0)
scale = min(scale_x, scale_y)
scale = np.floor(scale * 100)/100.0
lines_rotated[:,:4] = ((lines_rotated[:,:4].reshape(-1,2) - center) * scale + center).reshape(-1,4)
# (4) rotate the image (do not scale at this stage)
if debug_rotation:
if add_cross:
add_cross_in_img(img)
#_show_lines_ls_points_ls(img[:,:,:3], [lines])
img_big = np.pad(img, ( (h//2,h//2), (w//2,w//2), (0,0) ), 'constant', constant_values=0)
img_r = mmcv.imrotate(img_big, angle, scale=scale)
# (5) Move the image
new_img = np.zeros([h,w], dtype=lines.dtype)
h1,w1 = img_r.shape[:2]
region = np.array([
w1/2-w/2,
h1/2-h/2,
w1/2+w/2-1,
h1/2+h/2-1,
])
region_int = region.astype(np.int32)
new_img = mmcv.imcrop(img_r, region_int)
assert new_img.shape[:2] == img_shape
# rotate the surface normal
new_img[:,:,[1,2]] = np.matmul( new_img[:,:,[1,2]], matrix[:,:2].T )
lines_rotated = lines_rotated.astype(np.float32)
new_img = new_img.astype(np.float32)
assert lines_rotated[:,:4].min() > 0
if debug_rotation:
print(f'\nscale: {scale}')
#_show_lines_ls_points_ls(img[:,:,:3], [lines])
_show_lines_ls_points_ls(new_img[:,:,0], [lines_rotated])
#_show_img_with_norm(img)
#_show_img_with_norm(new_img)
if add_cross:
lines_rotated = lines_rotated[:-4]
pass
return lines_rotated, new_img, scale
def assign(self, points, gt_bboxes, gt_bboxes_ignore=None, gt_labels=None,
img_meta=None):
"""Assign gt to points.
This method assign a gt bbox to every points set, each points set
will be assigned with 0, or a positive number.
0 means negative sample, positive number is the index (1-based) of
assigned gt.
The assignment is done in following steps, the order matters.
1. assign every points to 0
2. A point is assigned to some gt bbox if
(i) the point is within the k closest points to the gt bbox
(ii) the distance between this point and the gt is smaller than
other gt bboxes
Args:
points (Tensor): points to be assigned, shape(n, 3) while last
dimension stands for (x, y, stride).
gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4).
gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are
labelled as `ignored`, e.g., crowd boxes in COCO.
NOTE: currently unused.
gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ).
Returns:
:obj:`AssignResult`: The assign result.
"""
num_points = points.shape[0]
num_gts = gt_bboxes.shape[0]
if num_gts == 0 or num_points == 0:
# If no truth assign everything to the background
assigned_gt_inds = points.new_full((num_points, ),
0,
dtype=torch.long)
if gt_labels is None:
assigned_labels = None
else:
assigned_labels = points.new_zeros((num_points, ),
dtype=torch.long)
return AssignResult(
num_gts, assigned_gt_inds, None, labels=assigned_labels)
points_xy = points[:, :2]
points_stride = points[:, 2]
points_lvl = torch.log2(
points_stride).int() # [3...,4...,5...,6...,7...]
lvl_min, lvl_max = points_lvl.min(), points_lvl.max()
# assign gt box
if self.obj_rep == 'box_scope':
assert gt_bboxes.shape[1] == 4
if gt_bboxes_ignore is not None:
assert gt_bboxes_ignore.shape[1] == 4
gt_bboxes_wh = (gt_bboxes[:, 2:] - gt_bboxes[:, :2]).clamp(min=1e-6)
gt_bboxes_xy = (gt_bboxes[:, :2] + gt_bboxes[:, 2:]) / 2
elif self.obj_rep == 'XYXYSin2' or self.obj_rep == 'XYXYSin2WZ0Z1':
obj_dim = {'XYXYSin2':5, 'XYXYSin2WZ0Z1':8}[self.obj_rep]
assert gt_bboxes.shape[1] == obj_dim
if gt_bboxes_ignore is not None:
assert gt_bboxes_ignore.shape[1] == obj_dim
gt_bboxes_ignore = gt_bboxes_ignore[:,:8]
gt_bboxes_raw = gt_bboxes.clone()
gt_bboxes = gt_bboxes[:,:4]
gt_bboxes_wh = (gt_bboxes[:, 2:] - gt_bboxes[:, :2]).norm(dim=1)\
.clamp(min=1e-6)
gt_bboxes_wh = gt_bboxes_wh.unsqueeze(1).repeat(1,2) # [56, 2]
gt_bboxes_xy = (gt_bboxes[:, :2] + gt_bboxes[:, 2:]) / 2 # [56, 2]
elif self.obj_rep == 'Rect4CornersZ0Z1':
from obj_geo_utils.obj_utils import OBJ_REPS_PARSE
assert gt_bboxes.shape[1] == 10
if gt_bboxes_ignore is not None:
assert gt_bboxes_ignore.shape[1] == 10
gt_bboxes_raw = gt_bboxes.clone()
XYZLgWsHA = OBJ_REPS_PARSE.encode_obj(gt_bboxes, 'Rect4CornersZ0Z1', 'XYZLgWsHA')
gt_bboxes_xy = XYZLgWsHA[:,:2]
gt_bboxes_wh = XYZLgWsHA[:,3:4].repeat(1,2) * level_fac
elif self.obj_rep == 'XYDRSin2Cos2Z0Z1':
assert gt_bboxes.shape[1] == 8
if gt_bboxes_ignore is not None:
assert gt_bboxes_ignore.shape[1] == 8
gt_bboxes_ignore = gt_bboxes_ignore[:,:8]
gt_bboxes_raw = gt_bboxes.clone()
gt_bboxes_wh = gt_bboxes[:,2:3].repeat(1,2) * level_fac
gt_bboxes_xy = gt_bboxes[:,:2]
elif self.obj_rep == 'XYDAsinAsinSin2Z0Z1':
assert gt_bboxes.shape[1] == 8
if gt_bboxes_ignore is not None:
assert gt_bboxes_ignore.shape[1] == 8
gt_bboxes_ignore = gt_bboxes_ignore[:,:8]
gt_bboxes_raw = gt_bboxes.clone()
gt_bboxes_wh = gt_bboxes[:,2:3].repeat(1,2) * level_fac
gt_bboxes_xy = gt_bboxes[:,:2]
elif self.obj_rep in ['XYLgWsAsinSin2Z0Z1', 'XYLgWsAbsSin2Z0Z1', 'XYLgWsSin2Cos2Z0Z1']:
assert gt_bboxes.shape[1] == 8
if gt_bboxes_ignore is not None:
assert gt_bboxes_ignore.shape[1] == 8
gt_bboxes_ignore = gt_bboxes_ignore[:,:8]
gt_bboxes_raw = gt_bboxes.clone()
gt_bboxes_wh = gt_bboxes[:,2:3].repeat(1,2)
gt_bboxes_xy = gt_bboxes[:,:2]
elif self.obj_rep == 'corner':
raise NotImplemented
assert gt_bboxes.shape[1] == 2
assert lvl_min == lvl_max, "only use one level for corner heat map"
gt_bboxes_wh = gt_bboxes * 0 + 2**(lvl_min)*self.scale
gt_bboxes = gt_bboxes.repeat(1,2)
pass
else:
import pdb; pdb.set_trace() # XXX BREAKPOINT
raise NotImplemented
if DEBUG_CFG.CHECK_POINT_ASSIGN:
if not gt_bboxes_wh.min() > DEBUG_CFG.MIN_BOX_SIZE:
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
scale = self.scale
gt_bboxes_lvl = ((torch.log2(gt_bboxes_wh[:, 0] / scale) +
torch.log2(gt_bboxes_wh[:, 1] / scale)) / 2).int()
gt_bboxes_lvl = torch.clamp(gt_bboxes_lvl, min=lvl_min, max=lvl_max)
# stores the assigned gt index of each point
assigned_gt_inds = points.new_zeros((num_points, ), dtype=torch.long)
# stores the assigned gt dist (to this point) of each point
assigned_gt_dist = points.new_full((num_points, ), float('inf'))
points_range = torch.arange(points.shape[0])
for idx in range(num_gts):
gt_lvl = gt_bboxes_lvl[idx]
# get the index of points in this level
lvl_idx = gt_lvl == points_lvl
points_index = points_range[lvl_idx]
# get the points in this level
lvl_points = points_xy[lvl_idx, :]
# get the center point of gt
gt_point = gt_bboxes_xy[[idx], :]
# get width and height of gt
gt_wh = gt_bboxes_wh[[idx], :]
# compute the distance between gt center and
# all points in this level
points_gt_dist = ((lvl_points - gt_point) / gt_wh).norm(dim=1)
# find the nearest k points to gt center in this level
min_dist, min_dist_index = torch.topk(
points_gt_dist, self.pos_num, largest=False)
#print(f'min_dist: {min_dist}')
if DEBUG_CFG.CHECK_POINT_ASSIGN:
if min_dist.max() > 2:
print(f'\n\tmin_dist is too large: {min_dist}\n')
import pdb; pdb.set_trace() # XXX BREAKPOINT
#assert False
pass
# the index of nearest k points to gt center in this level
min_dist_points_index = points_index[min_dist_index]
# The less_than_recorded_index stores the index
# of min_dist that is less then the assigned_gt_dist. Where
# assigned_gt_dist stores the dist from previous assigned gt
# (if exist) to each point.
less_than_recorded_index = min_dist < assigned_gt_dist[
min_dist_points_index]
# The min_dist_points_index stores the index of points satisfy:
# (1) it is k nearest to current gt center in this level.
# (2) it is closer to current gt center than other gt center.
min_dist_points_index = min_dist_points_index[
less_than_recorded_index]
# assign the result
assigned_gt_inds[min_dist_points_index] = idx + 1
assigned_gt_dist[min_dist_points_index] = min_dist[
less_than_recorded_index]
if gt_labels is not None:
assigned_labels = assigned_gt_inds.new_zeros((num_points, ))
pos_inds = torch.nonzero(assigned_gt_inds > 0).squeeze()
if pos_inds.numel() > 0:
assigned_labels[pos_inds] = gt_labels[assigned_gt_inds[pos_inds] - 1]
if DEBUG_CFG.PRINT_POINT_ASSIGNER:
pos_dist = assigned_gt_dist[pos_inds]
# print(f'pos_dist: {pos_dist}')
else:
assigned_labels = None
assign_res = AssignResult(
num_gts, assigned_gt_inds, None, labels=assigned_labels,
env='PointAssigner', img_meta=img_meta)
if DEBUG_CFG.PRINT_POINT_ASSIGNER:
assign_res.dg_add_pos_dist(pos_dist)
print('\tPointAssigner\t' + str(assign_res))
if DEBUG_CFG.VISUALIZE_POINT_ASSIGNER:
#if (not assign_res.num_pos_inds == num_gts):
from tools.visual_utils import _show_objs_ls_points_ls
import numpy as np
filename = img_meta['filename']
points_scope = points.max(0)[0][:2].int()
print(f'\n\n{filename}\n')
print(f'lvl_min: {lvl_min}')
print(f'lvl_max: {lvl_max}')
print(f'gt_bboxes_lvl: {gt_bboxes_lvl}')
pos_inds = torch.nonzero(assigned_gt_inds).squeeze()
pos_gt_inds = assigned_gt_inds[pos_inds].cpu().data.numpy() - 1
pos_gt_inds = np.array(pos_gt_inds).reshape([-1])
neg_gt_inds = [i for i in range(num_gts) if i not in pos_gt_inds]
miss_gt_num = len(neg_gt_inds)
print(f'\n\nMiss {miss_gt_num} in point assigner\n')
gt_bboxes_raw = gt_bboxes_raw.cpu().data.numpy()
missed_gt_bboxes = gt_bboxes_raw[neg_gt_inds]
missed_gt_bboxes_center = missed_gt_bboxes[:,:4].reshape(-1,2,2).mean(1)
if miss_gt_num>0:
point_scope = points.max(0)[0][:2].cpu().data.numpy()
is_gt_out_scope = (missed_gt_bboxes_center > point_scope).any(-1)
if is_gt_out_scope.all():
print('\n\tthe missed gts are out of scope\n')
else:
import pdb; pdb.set_trace() # XXX BREAKPOINT
pass
#assert False, "miss gt"
if 1:
pos_points = points[:,:2][pos_inds].cpu().data.numpy().reshape(-1,2)
h = points_scope[1] + 100
w = points_scope[0] + 100
pos_levels = gt_bboxes_lvl[ pos_gt_inds ]
#_show_objs_ls_points_ls((h,w), [gt_bboxes_raw], obj_rep=self.obj_rep)
_show_objs_ls_points_ls((h,w), [gt_bboxes_raw[pos_gt_inds], missed_gt_bboxes], obj_rep=self.obj_rep, points_ls=[pos_points], obj_colors=['red', 'green'], obj_scores_ls=[pos_levels, None])
for i in range(len(pos_inds)):
l = gt_bboxes_lvl[ pos_gt_inds[i] ]
print(f'level: {l}')
pos_gt_i = gt_bboxes_raw[pos_gt_inds[i]].reshape(-1, gt_bboxes_raw.shape[1])
_show_objs_ls_points_ls((h,w), [gt_bboxes_raw, pos_gt_i], obj_rep=self.obj_rep, points_ls = [pos_points[i:i+1]], obj_colors=['red', 'green'], obj_scores_ls=[None, l])
pass
return assign_res