def pose_similarity(dt, gt, shape_sim_mat): """compute pose similarity in terms of shape, translation and rotation Input: dt: np.ndarray detection [N x 7] first 6 dims are roll, pitch, yaw, x, y, z gt: save with dt Output: sim_shape: similarity based on shape eval dis_trans: distance based on translation eval dis_rot: dis.. based on rotation eval """ dt_num = len(dt) gt_num = len(gt) car_num = shape_sim_mat.shape[0] dt_car_id = np.uint32(dt[:, -1]) gt_car_id = np.uint32(gt[:, -1]) idx = np.tile(dt_car_id[:, None], (1, gt_num)).flatten() * car_num + \ np.tile(gt_car_id[None, :], (dt_num, 1)).flatten() sims_shape = shape_sim_mat.flatten()[idx] sims_shape = np.reshape(sims_shape, [dt_num, gt_num]) # translation similarity dt_car_trans = dt[:, 3:-1] gt_car_trans = gt[:, 3:-1] dis_trans = np.linalg.norm( np.tile(dt_car_trans[:, None, :], [1, gt_num, 1]) - np.tile(gt_car_trans[None, :, :], [dt_num, 1, 1]), axis=2) # rotation similarity dt_car_rot = uts.euler_angles_to_quaternions(dt[:, :3]) gt_car_rot = uts.euler_angles_to_quaternions(gt[:, :3]) q1 = dt_car_rot / np.linalg.norm(dt_car_rot, axis=1)[:, None] q2 = gt_car_rot / np.linalg.norm(gt_car_rot, axis=1)[:, None] # diff = abs(np.matmul(q1, np.transpose(q2))) diff = abs(1 - np.sum(np.square( np.tile(q1[:, None, :], [1, gt_num, 1]) - np.tile(q2[None, :, :], [dt_num, 1, 1])), axis=2) / 2.0) dis_rot = 2 * np.arccos(diff) * 180 / np.pi if False: #cfg.eval.eval_on: if True: #cfg.eval.sims_shape: False True #sims_shape = np.ones(sims_shape.shape, dtype=np.float) sim = np.linspace(1, 0.95, sims_shape.shape[1], endpoint=True) sims_shape = np.tile(sim, [dt_num, 1]) if True: #cfg.eval.dis_trans: dis_trans = np.zeros(dis_trans.shape, dtype=np.float) if True: #cfg.eval.dis_rot: dis_rot = np.zeros(dis_rot.shape, dtype=np.float) return sims_shape, dis_trans, dis_rot
def _add_gt_annotations_BOP(self, entry): """Add ground truth annotation metadata to an roidb entry.""" entry_id = entry['entry_id'] model_num = entry_id.split('/')[-3] img_num = int(entry_id.split('/')[-1][:4]) entry['height'] = self.BOP.image_shape[0] entry['width'] = self.BOP.image_shape[1] cam_K = self.BOP.info_all[model_num][img_num]['cam_K'] # There is only one model in each image cam_R_m2c = self.BOP.gt_all[model_num][img_num][0]['cam_R_m2c'] cam_t_m2c = self.BOP.gt_all[model_num][img_num][0]['cam_t_m2c'] obj_bb = self.BOP.gt_all[model_num][img_num][0]['obj_bb'] # Sanitize bboxes -- some are invalid valid_objs = [] valid_segms = [] # We have the bounding box here x1, y1, x2, y2 = obj_bb[0], obj_bb[ 1], obj_bb[0] + obj_bb[2], obj_bb[1] + obj_bb[3] # We read the mask here: im = cv2.imread(entry_id) mask = im != 0 mask_binary = mask[:, :, 0] + mask[:, :, 1] + mask[:, :, 2] mask_f = np.array(mask_binary, order='F', dtype=np.uint8) rle = COCOmask.encode(mask_f) valid_segms.append(rle) rotation_matrix = np.array(cam_R_m2c).reshape((3, 3)) euler_angles = rotation_matrix_to_euler_angles(rotation_matrix) pose = np.concatenate((euler_angles, np.array(cam_t_m2c))) obj = { 'area': mask_f.sum(), 'clean_bbox': [x1, y1, x2, y2], 'model_num': model_num, 'img_num': img_num, 'pose': pose } valid_objs.append(obj) num_valid_objs = len(valid_objs) boxes = np.zeros((num_valid_objs, 4), dtype=np.float32) # this is a legacy network from WAD Mask-RCNN seg_areas = np.zeros((num_valid_objs), dtype=np.float32) box_to_gt_ind_map = np.zeros((num_valid_objs), dtype=np.int32) # newly added for 3d car poses = np.zeros((num_valid_objs, 6), dtype=np.float32) quaternions = np.zeros((num_valid_objs, 4), dtype=np.float32) for ix, obj in enumerate(valid_objs): boxes[ix, :] = obj['clean_bbox'] seg_areas[ix] = obj['area'] box_to_gt_ind_map[ix] = ix poses[ix] = obj['pose'] quaternions[ix] = euler_angles_to_quaternions( np.array([obj['pose'][:3]])) entry['boxes'] = np.append(entry['boxes'], boxes, axis=0) entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas) entry['box_to_gt_ind_map'] = np.append(entry['box_to_gt_ind_map'], box_to_gt_ind_map) entry['segms'].extend(valid_segms) # newly added for 3d car entry['poses'] = np.append(entry['poses'], poses, axis=0) entry['quaternions'] = np.append(entry['quaternions'], quaternions, axis=0)
def _add_gt_annotations_Car3d(self, entry): """Add ground truth annotation metadata to an roidb entry.""" entry_id = entry['entry_id'] # Make file_name an abs path car_pose_file = os.path.join(self.Car3D.data_dir, 'car_poses', entry_id + '.json') assert os.path.exists(car_pose_file), 'Label \'{}\' not found'.format( car_pose_file) with open(car_pose_file) as f: car_poses = json.load(f) entry['height'] = self.Car3D.image_shape[0] entry['width'] = self.Car3D.image_shape[1] intrinsic_mat = self.Car3D.get_intrinsic_mat() # Sanitize bboxes -- some are invalid valid_objs = [] for i, car_pose in enumerate(car_poses): car_name = self.Car3D.car_id2name[car_pose['car_id']].name car = self.car_models[car_name] pose = np.array(car_pose['pose']) # project 3D points to 2d image plane rot_mat = euler_angles_to_rotation_matrix(pose[:3]) rvect, _ = cv2.Rodrigues(rot_mat) imgpts, jac = cv2.projectPoints(np.float32(car['vertices']), rvect, pose[3:], intrinsic_mat, distCoeffs=None) imgpts = np.int32(imgpts).reshape(-1, 2) x1, y1, x2, y2 = imgpts[:, 0].min(), imgpts[:, 1].min( ), imgpts[:, 0].max(), imgpts[:, 1].max() x1, y1, x2, y2 = box_utils.clip_xyxy_to_image( x1, y1, x2, y2, entry['height'], entry['width']) # Require non-zero seg area and more than 1x1 box size\ obj = { 'area': car_pose['area'], 'clean_bbox': [x1, y1, x2, y2], 'category_id': 33, 'car_id': car_pose['car_id'], 'visible_rate': car_pose['visible_rate'], 'pose': car_pose['pose'] } valid_objs.append(obj) num_valid_objs = len(valid_objs) boxes = np.zeros((num_valid_objs, 4), dtype=np.float32) # this is a legacy network from WAD Mask-RCNN car_class = 4 gt_overlaps = np.zeros((num_valid_objs, 8), dtype=np.float32) seg_areas = np.zeros((num_valid_objs), dtype=np.float32) is_crowd = np.zeros((num_valid_objs), dtype=np.bool) box_to_gt_ind_map = np.zeros((num_valid_objs), dtype=np.int32) # newly added for 3d car visible_rate = np.zeros((num_valid_objs), dtype=np.float32) poses = np.zeros((num_valid_objs, 6), dtype=np.float32) quaternions = np.zeros((num_valid_objs, 4), dtype=np.float32) car_cat_classes = np.zeros((num_valid_objs), dtype=np.int32) for ix, obj in enumerate(valid_objs): cls = np.where(self.Car3D.unique_car_models == obj['car_id'])[0][0] boxes[ix, :] = obj['clean_bbox'] car_cat_classes[ix] = cls seg_areas[ix] = obj['area'] is_crowd[ix] = False # TODO: What's this flag for? box_to_gt_ind_map[ix] = ix gt_overlaps[ix, car_class] = 1.0 visible_rate[ix] = obj['visible_rate'] poses[ix] = obj['pose'] quaternions[ix] = euler_angles_to_quaternions( np.array([obj['pose'][:3]])) # ensure the quaternion is upper hemispher quaternions[ix] = quaternion_upper_hemispher(quaternions[ix]) entry['boxes'] = np.append(entry['boxes'], boxes, axis=0) entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas) entry['gt_overlaps'] = np.append(entry['gt_overlaps'].toarray(), gt_overlaps, axis=0) entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps']) entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd) entry['box_to_gt_ind_map'] = np.append(entry['box_to_gt_ind_map'], box_to_gt_ind_map) # newly added for 3d car entry['visible_rate'] = np.append(entry['visible_rate'], visible_rate) entry['poses'] = np.append(entry['poses'], poses, axis=0) entry['car_cat_classes'] = np.append(entry['car_cat_classes'], car_cat_classes) entry['quaternions'] = np.append(entry['quaternions'], quaternions, axis=0)