class LidarSegEval:
"""
This is the official nuScenes-lidarseg evaluation code.
Results are written to the provided output_dir.
nuScenes-lidarseg uses the following metrics:
- Mean Intersection-over-Union (mIOU): We use the well-known IOU metric, which is defined as TP / (TP + FP + FN).
The IOU score is calculated separately for each class, and then the mean is
computed across classes. Note that in the challenge, index 0 is ignored in
the calculation.
- Frequency-weighted IOU (FWIOU): Instead of taking the mean of the IOUs across all the classes, each IOU is
weighted by the point-level frequency of its class. Note that in the challenge,
index 0 is ignored in the calculation. FWIOU is not used for the challenge.
We assume that:
- For each pointcloud, the prediction for every point is present in a .bin file, in the same order as that of the
points stored in the corresponding .bin file.
- The naming convention of the .bin files containing the predictions for a single point cloud is:
<lidar_sample_data_token>_lidarseg.bin
- The predictions are between 1 and 16 (inclusive); 0 is the index of the ignored class.
Please see https://www.nuscenes.org/lidar-segmentation for more details.
"""
def __init__(self,
nusc: NuScenes,
results_folder: str,
eval_set: str,
verbose: bool = False):
"""
Initialize a LidarSegEval object.
:param nusc: A NuScenes object.
:param results_folder: Path to the folder.
:param eval_set: The dataset split to evaluate on, e.g. train, val or test.
:param verbose: Whether to print messages during the evaluation.
"""
# Check there are ground truth annotations.
assert len(nusc.lidarseg) > 0, 'Error: No ground truth annotations found in {}.'.format(nusc.version)
# Check results folder exists.
self.results_folder = results_folder
self.results_bin_folder = os.path.join(results_folder, 'lidarseg', eval_set)
assert os.path.exists(self.results_bin_folder), \
'Error: The folder containing the .bin files ({}) does not exist.'.format(self.results_bin_folder)
self.nusc = nusc
self.results_folder = results_folder
self.eval_set = eval_set
self.verbose = verbose
self.mapper = LidarsegClassMapper(nusc_)
self.ignore_idx = self.mapper.ignore_class['index']
self.id2name = {idx: name for name, idx in self.mapper.coarse_name_2_coarse_idx_mapping.items()}
self.num_classes = len(self.mapper.coarse_name_2_coarse_idx_mapping)
if self.verbose:
print('There are {} classes.'.format(self.num_classes))
self.global_cm = ConfusionMatrix(self.num_classes, self.ignore_idx)
self.sample_tokens = get_samples_in_eval_set(self.nusc, self.eval_set)
if self.verbose:
print('There are {} samples.'.format(len(self.sample_tokens)))
def evaluate(self) -> Dict:
"""
Performs the actual evaluation.
:return: A dictionary containing the evaluated metrics.
"""
for sample_token in tqdm(self.sample_tokens, disable=not self.verbose):
sample = self.nusc.get('sample', sample_token)
# Get the sample data token of the point cloud.
sd_token = sample['data']['LIDAR_TOP']
# Load the ground truth labels for the point cloud.
lidarseg_label_filename = os.path.join(self.nusc.dataroot,
self.nusc.get('lidarseg', sd_token)['filename'])
lidarseg_label = self.load_bin_file(lidarseg_label_filename)
lidarseg_label = self.mapper.convert_label(lidarseg_label)
# Load the predictions for the point cloud.
lidarseg_pred_filename = os.path.join(self.results_folder, 'lidarseg',
self.eval_set, sd_token + '_lidarseg.bin')
lidarseg_pred = self.load_bin_file(lidarseg_pred_filename)
# Get the confusion matrix between the ground truth and predictions.
# Update the confusion matrix for the sample data into the confusion matrix for the eval set.
self.global_cm.update(lidarseg_label, lidarseg_pred)
iou_per_class = self.global_cm.get_per_class_iou()
miou = self.global_cm.get_mean_iou()
freqweighted_iou = self.global_cm.get_freqweighted_iou()
# Put everything nicely into a dict.
results = {'iou_per_class': {self.id2name[i]: class_iou for i, class_iou in enumerate(iou_per_class)},
'miou': miou,
'freq_weighted_iou': freqweighted_iou}
# Print the results if desired.
if self.verbose:
print("======\nnuScenes-lidarseg evaluation for {}".format(self.eval_set))
print(json.dumps(results, indent=4, sort_keys=False))
print("======")
return results
@staticmethod
def load_bin_file(bin_path: str) -> np.ndarray:
"""
Loads a .bin file containing the labels.
:param bin_path: Path to the .bin file.
:return: An array containing the labels.
"""
assert os.path.exists(bin_path), 'Error: Unable to find {}.'.format(bin_path)
bin_content = np.fromfile(bin_path, dtype=np.uint8)
assert len(bin_content) > 0, 'Error: {} is empty.'.format(bin_path)
return bin_content
def visualize_semantic_differences_bev(nusc: NuScenes,
sample_token: str,
lidarseg_preds_folder: str = None,
axes_limit: float = 40,
dot_size: int = 5,
out_path: str = None) -> None:
"""
Visualize semantic difference of lidar segmentation results in bird's eye view.
:param nusc: A NuScenes object.
:param sample_token: Unique identifier.
:param lidarseg_preds_folder: A path to the folder which contains the user's lidar segmentation predictions for
the scene. The naming convention of each .bin file in the folder should be
named in this format: <lidar_sample_data_token>_lidarseg.bin.
:param axes_limit: Axes limit for plot (measured in meters).
:param dot_size: Scatter plot dot size.
:param out_path: Path to save visualization to (e.g. /save/to/here/bev_diff.png).
"""
mapper = LidarsegClassMapper(nusc)
sample = nusc.get('sample', sample_token)
# Get the sample data token of the point cloud.
sd_token = sample['data']['LIDAR_TOP']
pointsensor = nusc.get('sample_data', sd_token)
pcl_path = os.path.join(nusc.dataroot, pointsensor['filename'])
# Load the ground truth labels for the point cloud.
gt_path = os.path.join(nusc.dataroot,
nusc.get('lidarseg', sd_token)['filename'])
gt = LidarSegPointCloud(pcl_path, gt_path)
gt.labels = mapper.convert_label(gt.labels) # Map the labels as necessary.
# Load the predictions for the point cloud.
preds_path = os.path.join(lidarseg_preds_folder,
sd_token + '_lidarseg.bin')
preds = LidarSegPointCloud(pcl_path, preds_path)
# Do not compare points which are ignored.
ignored_points_idxs = np.where(
gt.labels != mapper.ignore_class['index'])[0]
gt.labels = gt.labels[ignored_points_idxs]
gt.points = gt.points[ignored_points_idxs]
preds.labels = preds.labels[ignored_points_idxs]
preds.points = preds.points[ignored_points_idxs]
# Init axes.
fig, axes = plt.subplots(1,
3,
figsize=(10 * 3, 10),
sharex='all',
sharey='all')
# Render ground truth and predictions.
gt.render(mapper.coarse_colormap,
mapper.coarse_name_2_coarse_idx_mapping,
ax=axes[0])
preds.render(mapper.coarse_colormap,
mapper.coarse_name_2_coarse_idx_mapping,
ax=axes[1])
# Render errors.
id2color_for_diff_bev = {
0: (191, 41, 0, 255), # red: wrong label
1: (50, 168, 82, 255)
} # green: correct label
colors_for_diff_bev = colormap_to_colors(id2color_for_diff_bev, {
0: 0,
1: 1
})
mask = np.array(gt.labels == preds.labels).astype(
int) # Convert array from bool to int.
axes[2].scatter(gt.points[:, 0],
gt.points[:, 1],
c=colors_for_diff_bev[mask],
s=dot_size)
axes[2].set_title('Errors (Correct: Green, Mislabeled: Red)')
# Limit visible range for all subplots.
plt.xlim(-axes_limit, axes_limit)
plt.ylim(-axes_limit, axes_limit)
plt.tight_layout()
if out_path:
plt.savefig(out_path, bbox_inches='tight', pad_inches=0)
plt.show()