def target_to_cam(nusc: NuScenes,
pointsensor_token: str,
annotation_token: str,
pointsensor_channel: str = 'LIDAR_TOP'):
"""
Given an annotation token (3d detection in world coordinate frame) and pointsensor sample_data token,
transform the label from world-coordinate frame to LiDAR.
:param nusc: NuScenes instance.
:param pointsensor_token: Lidar/radar sample_data token.
:param annotation_token: Camera sample_annotation token.
:param pointsensor_channel: Laser channel name, e.g. 'LIDAR_TOP'.
:return box with the labels for the 3d detection task in the LiDAR channel frame.
"""
# Point LiDAR sample
point_data = nusc.get('sample_data',
pointsensor_token) # Sample LiDAR info
# From LiDAR to ego
cs_rec = nusc.get('calibrated_sensor',
point_data['calibrated_sensor_token'])
# Transformation metadata from ego to world coordinate frame
pose_rec = nusc.get('ego_pose', point_data['ego_pose_token'])
# Obtain the annotation from the token
annotation_metadata = nusc.get('sample_annotation', annotation_token)
# Obtain box parameters
box = nusc.get_box(annotation_metadata['token'])
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
return box
def get_2d_boxes(sample_data_token: str,
visibilities: List[str]) -> List[OrderedDict]:
"""
Get the 2D annotation records for a given `sample_data_token`.
:param sample_data_token: Sample data token belonging to a keyframe.
:param visibilities: Visibility filter.
:return: List of 2D annotation record that belongs to the input `sample_data_token`
"""
# Get the sample data and the sample corresponding to that sample data.
sd_rec = nusc.get('sample_data', sample_data_token)
if not sd_rec['is_key_frame']:
raise ValueError(
'The 2D re-projections are available only for keyframes.')
s_rec = nusc.get('sample', sd_rec['sample_token'])
# Get the calibrated sensor and ego pose record to get the transformation matrices.
cs_rec = nusc.get('calibrated_sensor', sd_rec['calibrated_sensor_token'])
pose_rec = nusc.get('ego_pose', sd_rec['ego_pose_token'])
camera_intrinsic = np.array(cs_rec['camera_intrinsic'])
# Get all the annotation with the specified visibilties.
ann_recs = [
nusc.get('sample_annotation', token) for token in s_rec['anns']
]
ann_recs = [
ann_rec for ann_rec in ann_recs
if (ann_rec['visibility_token'] in visibilities)
]
# Only for cars category
ann_recs = [
ann_rec for ann_rec in ann_recs
if (ann_rec['category_name'] == "vehicle.car")
]
#Determine token for parked attribute
for att in nusc.attribute:
if att['name'] == 'vehicle.parked':
parked_token = att['token']
if att['name'] == 'vehicle.stopped':
stopped_token = att['token']
# Only get the parked && stopped atrribute
ann_recs = [
ann_rec for ann_rec in ann_recs
if (parked_token in ann_rec['attribute_tokens']
or stopped_token in ann_rec['attribute_tokens'])
]
repro_recs = [sd_rec['filename']]
for ann_rec in ann_recs:
if parked_token in ann_rec['attribute_tokens']:
vehicle_state = 1
elif stopped_token in ann_rec['attribute_tokens']:
vehicle_state = 2
ann_rec['sample_annotation_token'] = ann_rec['token']
ann_rec['sample_data_token'] = sample_data_token
box = nusc.get_box(ann_rec['token'])
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
# Filter out the corners that are not in front of the calibrated sensor.
corners_3d = box.corners()
in_front = np.argwhere(corners_3d[2, :] > 0).flatten()
corners_3d = corners_3d[:, in_front]
# Project 3d box to 2d.
corner_coords = view_points(corners_3d, camera_intrinsic,
True).T[:, :2].tolist()
# Keep only corners that fall within the image.
final_coords = post_process_coords(corner_coords)
# Skip if the convex hull of the re-projected corners does not intersect the image canvas.
if final_coords is None:
continue
else:
min_x, min_y, max_x, max_y = final_coords
# Generate dictionary record to be included in the .json file.
#repro_rec = generate_record(ann_rec, min_x, min_y, max_x, max_y, sample_data_token, sd_rec['filename'])
repro_rec = [
vehicle_state, (max_x + min_x) / (2 * 1600),
(max_y + min_y) / (2 * 900), (max_x - min_x) / (1600),
(max_y - min_y) / (900)
]
repro_recs.append(repro_rec)
if (len(repro_recs) == 1):
return []
return repro_recs
def get_2d_boxes(nusc, sample_data_token: str,
visibilities: List[str]) -> List[OrderedDict]:
"""Get the 2D annotation records for a given `sample_data_token`.
Args:
sample_data_token: Sample data token belonging to a camera keyframe.
visibilities: Visibility filter.
Return:
list[dict]: List of 2D annotation record that belongs to the input
`sample_data_token`.
"""
# Get the sample data and the sample corresponding to that sample data.
sd_rec = nusc.get('sample_data', sample_data_token)
assert sd_rec[
'sensor_modality'] == 'camera', 'Error: get_2d_boxes only works' \
' for camera sample_data!'
if not sd_rec['is_key_frame']:
raise ValueError(
'The 2D re-projections are available only for keyframes.')
s_rec = nusc.get('sample', sd_rec['sample_token'])
# Get the calibrated sensor and ego pose
# record to get the transformation matrices.
cs_rec = nusc.get('calibrated_sensor', sd_rec['calibrated_sensor_token'])
pose_rec = nusc.get('ego_pose', sd_rec['ego_pose_token'])
camera_intrinsic = np.array(cs_rec['camera_intrinsic'])
# Get all the annotation with the specified visibilties.
ann_recs = [
nusc.get('sample_annotation', token) for token in s_rec['anns']
]
ann_recs = [
ann_rec for ann_rec in ann_recs
if (ann_rec['visibility_token'] in visibilities)
]
repro_recs = []
for ann_rec in ann_recs:
# Augment sample_annotation with token information.
ann_rec['sample_annotation_token'] = ann_rec['token']
ann_rec['sample_data_token'] = sample_data_token
# Get the box in global coordinates.
box = nusc.get_box(ann_rec['token'])
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
# Filter out the corners that are not in front of the calibrated
# sensor.
corners_3d = box.corners()
in_front = np.argwhere(corners_3d[2, :] > 0).flatten()
corners_3d = corners_3d[:, in_front]
# Project 3d box to 2d.
corner_coords = view_points(corners_3d, camera_intrinsic,
True).T[:, :2].tolist()
# Keep only corners that fall within the image.
final_coords = post_process_coords(corner_coords)
# Skip if the convex hull of the re-projected corners
# does not intersect the image canvas.
if final_coords is None:
continue
else:
min_x, min_y, max_x, max_y = final_coords
# Generate dictionary record to be included in the .json file.
repro_rec = generate_record(ann_rec, min_x, min_y, max_x, max_y,
sample_data_token, sd_rec['filename'])
repro_recs.append(repro_rec)
return repro_recs
def get_2d_boxes(sample_data_token: str,
visibilities: List[str]) -> List[OrderedDict]:
"""
Get the 2D annotation records for a given `sample_data_token`.
:param sample_data_token: Sample data token belonging to a keyframe.
:param visibilities: Visibility filter.
:return: List of 2D annotation record that belongs to the input `sample_data_token`
"""
# Get the sample data, and the sample corresponding to that sample data.
sd_rec = nusc.get('sample_data', sample_data_token)
if not sd_rec['is_key_frame']:
raise ValueError(
'The 2D re-projections are available only for keyframes.')
s_rec = nusc.get('sample', sd_rec['sample_token'])
# Get the calibrated sensor and ego pose record to get the transformation matrices.
cs_rec = nusc.get('calibrated_sensor', sd_rec['calibrated_sensor_token'])
pose_rec = nusc.get('ego_pose', sd_rec['ego_pose_token'])
camera_intrinsic = np.array(cs_rec['camera_intrinsic'])
# Get all the annotation that fulfills the visibilty values.
ann_recs = [
nusc.get('sample_annotation', token) for token in s_rec['anns']
]
ann_recs = [
ann_rec for ann_rec in ann_recs
if (ann_rec['visibility_token'] in visibilities)
]
repro_recs = []
for ann_rec in ann_recs:
ann_rec['sample_annotation_token'] = ann_rec['token']
ann_rec['sample_data_token'] = sample_data_token
box = nusc.get_box(ann_rec['token'])
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
# Filter out the corners that is not in front of the calibrated sensor.
corners_3d = box.corners()
in_front = np.argwhere(corners_3d[2, :] > 0).flatten()
corners_3d = corners_3d[:, in_front]
# Applying the re-projection algorithm post-processing step.
corner_coords = view_points(corners_3d, camera_intrinsic,
True).T[:, :2].tolist()
final_coords = post_process_coords(corner_coords)
# Skip if the convex hull of the re-projected corners does not intersect the image canvas.
if final_coords is None:
continue
else:
min_x, min_y, max_x, max_y = final_coords
# Generate dictionary record to be included in the .json file.
repro_rec = generate_record(ann_rec, min_x, min_y, max_x, max_y,
sample_data_token, sd_rec['filename'])
repro_recs.append(repro_rec)
return repro_recs
def get_2d_boxes(sample_data_token: str, visibilities: List[str], mode: str) -> List[OrderedDict]:
"""
Get the 2D annotation records for a given `sample_data_token`.
:param sample_data_token: Sample data token belonging to a camera keyframe.
:param visibilities: Visibility filter.
:param mode: 'xywh' or 'xyxy'
:return: List of 2D annotation record that belongs to the input `sample_data_token`
"""
# Get the sample data and the sample corresponding to that sample data.
sd_rec = nusc.get('sample_data', sample_data_token)
assert sd_rec['sensor_modality'] == 'camera', 'Error: get_2d_boxes only works for camera sample_data!'
if not sd_rec['is_key_frame']:
raise ValueError('The 2D re-projections are available only for keyframes.')
s_rec = nusc.get('sample', sd_rec['sample_token'])
# Get the calibrated sensor and ego pose record to get the transformation matrices.
cs_rec = nusc.get('calibrated_sensor', sd_rec['calibrated_sensor_token'])
pose_rec = nusc.get('ego_pose', sd_rec['ego_pose_token'])
camera_intrinsic = np.array(cs_rec['camera_intrinsic'])
# Get all the annotation with the specified visibilties.
ann_recs = [nusc.get('sample_annotation', token) for token in s_rec['anns']]
ann_recs = [ann_rec for ann_rec in ann_recs if (ann_rec['visibility_token'] in visibilities)]
sd_annotations = []
for ann_rec in ann_recs:
# Get the 3D box in global coordinates.
box = nusc.get_box(ann_rec['token'])
# Calculate distance from vehicle to box
ego_translation = (box.center[0] - pose_rec['translation'][0],
box.center[1] - pose_rec['translation'][1],
box.center[2] - pose_rec['translation'][2])
ego_dist = np.sqrt(np.sum(np.array(ego_translation[:2]) ** 2))
dist = round(ego_dist,2)
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
# Filter out the corners that are not in front of the calibrated sensor.
corners_3d = box.corners()
in_front = np.argwhere(corners_3d[2, :] > 0).flatten()
corners_3d = corners_3d[:, in_front]
# Project 3d box to 2d.
corner_coords = view_points(corners_3d, camera_intrinsic, True).T[:, :2].tolist()
# Keep only corners that fall within the image.
final_coords = post_process_coords(corner_coords)
# Skip if the convex hull of the re-projected corners does not intersect the image canvas.
if final_coords is None:
continue
min_x, min_y, max_x, max_y = final_coords
if mode == 'xywh':
bbox = [min_x, min_y, max_x-min_x, max_y-min_y]
else:
bbox = [min_x, min_y, max_x, max_y]
## Generate 2D record to be included in the .json file.
ann_2d = {}
ann_2d['sample_data_token'] = sample_data_token
ann_2d['bbox'] = np.around(bbox, 2).tolist()
ann_2d['distance'] = dist
ann_2d['category_name'] = ann_rec['category_name']
ann_2d['num_lidar_pts'] = ann_rec['num_lidar_pts']
ann_2d['num_radar_pts'] = ann_rec['num_radar_pts']
ann_2d['visibility_token'] = ann_rec['visibility_token']
sd_annotations.append(ann_2d)
return sd_annotations
def get_2d_boxes(self, index: int, visibilities: List[str]) -> List[List]:
"""
Get the 2D annotation records for a given `sample_data_token`.
:param sample_data_token: Sample data token belonging to a camera keyframe.
:param visibilities: Visibility filter.
:return: [catagory, min_x, min_y, max_x, max_y],format:xyxy,absolute_coor_value
List of 2D annotation record that belongs to the input `sample_data_token`
"""
# Gettign data from nuscenes database
sample_token = self.sample_tokens[index]
sample = self.nusc.get('sample', sample_token)
# Gettign sample_data_token from sensor['CAM_FRONT']
sample_data_token = sample['data'][self.camera_sensors[0]]
# Get the sample data and the sample corresponding to that sample data.
sd_rec = self.nusc.get('sample_data', sample_data_token)
assert sd_rec[
'sensor_modality'] == 'camera', 'Error: get_2d_boxes only works for camera sample_data!'
if not sd_rec['is_key_frame']:
raise ValueError(
'The 2D re-projections are available only for keyframes.')
s_rec = self.nusc.get('sample', sd_rec['sample_token'])
# Get the calibrated sensor and ego pose record to get the transformation matrices.
cs_rec = self.nusc.get('calibrated_sensor',
sd_rec['calibrated_sensor_token'])
pose_rec = self.nusc.get('ego_pose', sd_rec['ego_pose_token'])
camera_intrinsic = np.array(cs_rec['camera_intrinsic'])
# Get all the annotation with the specified visibilties.
ann_recs = [
self.nusc.get('sample_annotation', token)
for token in s_rec['anns']
]
ann_recs = [
ann_rec for ann_rec in ann_recs
if (ann_rec['visibility_token'] in visibilities)
]
repro_recs = []
bboxes = []
for ann_rec in ann_recs:
# Augment sample_annotation with token information.
ann_rec['sample_annotation_token'] = ann_rec['token']
ann_rec['sample_data_token'] = sample_data_token
# Get the box in global coordinates.
box = self.nusc.get_box(ann_rec['token'])
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
# Filter out the corners that are not in front of the calibrated sensor.
corners_3d = box.corners()
in_front = np.argwhere(corners_3d[2, :] > 0).flatten()
corners_3d = corners_3d[:, in_front]
# Project 3d box to 2d.
corner_coords = view_points(corners_3d, camera_intrinsic,
True).T[:, :2].tolist()
# Keep only corners that fall within the image.
final_coords = self.post_process_coords(corner_coords)
# Skip if the convex hull of the re-projected corners does not intersect the image canvas.
if final_coords is None:
continue
else:
min_x, min_y, max_x, max_y = final_coords
min_x = min_x / 1600 * self.image_max_side
max_x = max_x / 1600 * self.image_max_side
min_y = min_y / 900 * self.image_min_side
max_y = max_y / 900 * self.image_min_side
category = ann_rec['category_name']
#这里进行了类别的过滤！
if category in list(self.classes.keys()):
category_digit = self.classes[category]
bbox = [category_digit, min_x, min_y, max_x, max_y]
bboxes.append(bbox)
else:
continue
if bboxes == []:
bboxes.append([-1, 0, 0, 0, 0]) #添加-1为无anno,在可视化或者训练时要跳过处理
print(sample_token)
print("有一帧没有gt_box!!!")
return bboxes
return bboxes
def get_2d_boxes(nusc,
sample_data_token: str,
visibilities: List[str],
mono3d=True):
"""Get the 2D annotation records for a given `sample_data_token`.
Args:
sample_data_token (str): Sample data token belonging to a camera \
keyframe.
visibilities (list[str]): Visibility filter.
mono3d (bool): Whether to get boxes with mono3d annotation.
Return:
list[dict]: List of 2D annotation record that belongs to the input
`sample_data_token`.
"""
# Get the sample data and the sample corresponding to that sample data.
sd_rec = nusc.get('sample_data', sample_data_token)
assert sd_rec[
'sensor_modality'] == 'camera', 'Error: get_2d_boxes only works' \
' for camera sample_data!'
if not sd_rec['is_key_frame']:
raise ValueError(
'The 2D re-projections are available only for keyframes.')
s_rec = nusc.get('sample', sd_rec['sample_token'])
# Get the calibrated sensor and ego pose
# record to get the transformation matrices.
cs_rec = nusc.get('calibrated_sensor', sd_rec['calibrated_sensor_token'])
pose_rec = nusc.get('ego_pose', sd_rec['ego_pose_token'])
camera_intrinsic = np.array(cs_rec['camera_intrinsic'])
# Get all the annotation with the specified visibilties.
ann_recs = [
nusc.get('sample_annotation', token) for token in s_rec['anns']
]
ann_recs = [
ann_rec for ann_rec in ann_recs
if (ann_rec['visibility_token'] in visibilities)
]
repro_recs = []
for ann_rec in ann_recs:
# Augment sample_annotation with token information.
ann_rec['sample_annotation_token'] = ann_rec['token']
ann_rec['sample_data_token'] = sample_data_token
# Get the box in global coordinates.
box = nusc.get_box(ann_rec['token'])
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
# Filter out the corners that are not in front of the calibrated
# sensor.
corners_3d = box.corners()
in_front = np.argwhere(corners_3d[2, :] > 0).flatten()
corners_3d = corners_3d[:, in_front]
# Project 3d box to 2d.
corner_coords = view_points(corners_3d, camera_intrinsic,
True).T[:, :2].tolist()
# Keep only corners that fall within the image.
final_coords = post_process_coords(corner_coords)
# Skip if the convex hull of the re-projected corners
# does not intersect the image canvas.
if final_coords is None:
continue
else:
min_x, min_y, max_x, max_y = final_coords
# Generate dictionary record to be included in the .json file.
repro_rec = generate_record(ann_rec, min_x, min_y, max_x, max_y,
sample_data_token, sd_rec['filename'])
# If mono3d=True, add 3D annotations in camera coordinates
if mono3d and (repro_rec is not None):
loc = box.center.tolist()
dim = box.wlh.tolist()
rot = [box.orientation.yaw_pitch_roll[0]]
global_velo2d = nusc.box_velocity(box.token)[:2]
global_velo3d = np.array([*global_velo2d, 0.0])
e2g_r_mat = Quaternion(pose_rec['rotation']).rotation_matrix
c2e_r_mat = Quaternion(cs_rec['rotation']).rotation_matrix
cam_velo3d = global_velo3d @ np.linalg.inv(
e2g_r_mat).T @ np.linalg.inv(c2e_r_mat).T
velo = cam_velo3d[0::2].tolist()
repro_rec['bbox_cam3d'] = loc + dim + rot
repro_rec['velo_cam3d'] = velo
center3d = np.array(loc).reshape([1, 3])
center2d = points_cam2img(center3d,
camera_intrinsic,
with_depth=True)
repro_rec['center2d'] = center2d.squeeze().tolist()
# normalized center2D + depth
# if samples with depth < 0 will be removed
if repro_rec['center2d'][2] <= 0:
continue
ann_token = nusc.get('sample_annotation',
box.token)['attribute_tokens']
if len(ann_token) == 0:
attr_name = 'None'
else:
attr_name = nusc.get('attribute', ann_token[0])['name']
attr_id = nus_attributes.index(attr_name)
repro_rec['attribute_name'] = attr_name
repro_rec['attribute_id'] = attr_id
repro_recs.append(repro_rec)
return repro_recs
def bbox_3d_to_2d(nusc: NuScenes,
camera_token: str,
annotation_token: str,
visualize: bool = False) -> List:
"""
Get the 2D annotation bounding box for a given `sample_data_token`. return None if no
intersection (bounding box).
:param nusc: NuScenes instance.
:param camera_token: Camera sample_data token.
:param annotation_token: Sample data token belonging to a camera keyframe.
:param visualize: bool to plot the resulting bounding box.
:return: List of 2D annotation record that belongs to the input `sample_data_token`
"""
# Obtain camera sample_data
cam_data = nusc.get('sample_data', camera_token)
# Get the calibrated sensor and ego pose record to get the transformation matrices.
# From camera to ego
cs_rec = nusc.get('calibrated_sensor', cam_data['calibrated_sensor_token'])
# From ego to world coordinate frame
pose_rec = nusc.get('ego_pose', cam_data['ego_pose_token'])
# Camera intrinsic parameters
camera_intrinsic = np.array(cs_rec['camera_intrinsic'])
# Obtain the annotation from the token
annotation_metadata = nusc.get('sample_annotation', annotation_token)
# Get the box in global coordinates from sample ann token
box = nusc.get_box(annotation_metadata['token'])
# Mapping the box from world coordinate-frame to camera sensor
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
# Filter out the corners that are not in front of the calibrated sensor.
corners_3d = box.corners() # 8 corners of the 3d bounding box
in_front = np.argwhere(corners_3d[2, :] > 0).flatten(
) # corners that are behind the sensor are removed
corners_3d = corners_3d[:, in_front]
# Project 3d box to 2d.
corner_coords = view_points(corners_3d, camera_intrinsic,
True).T[:, :2].tolist()
# Filter points that are outside the image
final_coords = post_process_coords(corner_coords)
if final_coords is None:
return None
min_x, min_y, max_x, max_y = [int(coord) for coord in final_coords]
if visualize:
# Load image from dataroot
img_path = osp.join(nusc.dataroot, cam_data['filename'])
img = cv2.imread(img_path, 1)
# Draw rectangle on image with coords
img_r = cv2.rectangle(img, (min_x, min_y), (max_x, max_y),
(255, 165, 0), 3)
img_r = img_r[:, :, ::-1]
plt.figure(figsize=(12, 4), dpi=100)
plt.imshow(img_r)
plt.show()
return final_coords
def get_2d_boxes(sample_data_token: str, visibilities: List[str],
dataroot: str, box_image_dir: str) -> List[OrderedDict]:
"""
Get the 2D annotation records for a given `sample_data_token`.
:param sample_data_token: Sample data token belonging to a camera keyframe.
:param visibilities: Visibility filter.
:return: List of 2D annotation record that belongs to the input `sample_data_token`
"""
# Get the sample data and the sample corresponding to that sample data.
sd_rec = nusc.get('sample_data', sample_data_token)
s_rec = nusc.get('sample', sd_rec['sample_token'])
# Get the calibrated sensor and ego pose record to get the transformation matrices.
cs_rec = nusc.get('calibrated_sensor', sd_rec['calibrated_sensor_token'])
pose_rec = nusc.get('ego_pose', sd_rec['ego_pose_token'])
camera_intrinsic = np.array(cs_rec['camera_intrinsic'])
# Get all the annotation with the specified visibilties and in detection benchmark
ann_recs = [
nusc.get('sample_annotation', token) for token in s_rec['anns']
]
ann_recs = [ann_rec for ann_rec in ann_recs if (ann_rec['visibility_token'] in visibilities and \
ann_rec['category_name'] in category2class.keys())]
repro_recs = []
scene_image = Image.open(os.path.join(dataroot, sd_rec['filename']))
for i, ann_rec in enumerate(ann_recs):
# Augment sample_annotation with token information.
ann_rec['sample_annotation_token'] = ann_rec['token']
ann_rec['sample_data_token'] = sample_data_token
# Get the box in global coordinates.
box = nusc.get_box(ann_rec['token'])
# Move them to the ego-pose frame.
box.translate(-np.array(pose_rec['translation']))
box.rotate(Quaternion(pose_rec['rotation']).inverse)
# Move them to the calibrated sensor frame.
box.translate(-np.array(cs_rec['translation']))
box.rotate(Quaternion(cs_rec['rotation']).inverse)
# Filter out the corners that are not in front of the calibrated sensor.
corners_3d = box.corners()
in_front = np.argwhere(corners_3d[2, :] > 0).flatten()
corners_3d = corners_3d[:, in_front]
# Project 3d box to 2d.
corner_coords = view_points(corners_3d, camera_intrinsic,
True).T[:, :2].tolist()
# Keep only corners that fall within the image.
final_coords = post_process_coords(corner_coords)
# Skip if the convex hull of the re-projected corners does not intersect the image canvas.
if final_coords is None:
continue
else:
min_x, min_y, max_x, max_y = final_coords
if max_x - min_x < 25.0 or max_y - min_y < 25.0:
continue
# Generate dictionary record to be included in the .json file.
repro_rec = generate_record(ann_rec, min_x, min_y, max_x, max_y,
sample_data_token, sd_rec['filename'])
# add dimensions, location
repro_rec['dimensions'] = box.wlh.tolist()
repro_rec['location'] = box.center.tolist()
# add theta_l (approximate)
theta_ray = np.arctan2(repro_rec['location'][2],
repro_rec['location'][0])
front = box.orientation.rotate(np.array([1.0, 0.0, 0.0]))
ry = -np.arctan2(front[2], front[0])
theta_l = np.pi - theta_ray - ry
if theta_l > np.pi:
theta_l -= 2.0 * np.pi
if theta_l < -np.pi:
theta_l += 2.0 * np.pi
repro_rec['theta_l'] = theta_l
# save box image
box_image = scene_image.resize((224, 224), box=final_coords)
box_image_file = os.path.join(
box_image_dir, 'camera__{}__{}.png'.format(sample_data_token, i))
box_image.save(box_image_file)
repro_rec['box_image_file'] = os.path.join(
*box_image_file.split('/')[-4:])
repro_recs.append(repro_rec)
return repro_recs
