class NuScenesEvaluator(Evaluator):
"""
NuScenes Model Evaluator
Evaluates models based on the nuScenes dataset. The evaluator logs per sample F1 scores as well as visualizations of the
validation data samples.
Arguments:
logdir: Folder path to store the evaluation log files, <str>.
batch_size: Batch size used for the model prediction, <int>.
verbose: Verbosity mode (0 = silent, 1 = progress bar, 2 = one line per epoch), <int>.
steps: Total number of epoch before stopping the evaluation, <int>.
callbacks: List of callbacks to apply during evaluation, <list>.
max_queue_size: Maximum size for the generator queue (used for generator input only), <int>.
workers: Maximum number of processes to spin up when using process-based threading (used for generator input only), <int>.
use_multiprocessing: Whether use process-based threading (used for generator input only), <bool>.
visualize: Whether to visualize the prediction or not, <bool>.
velocities: Whether to visualize the velocity vectors or not, <bool>.
version: Version of the nuScenes dataset, <str>.
dataroot: Path to the raw nuScenes dataset files, <str>.
class_matching_dict: Key-Value-Pairs of the original category names and the user configured class names, <dict>.
preprocessor_attributes: Keyword attributes for the nuScenes pre-processor, <dict>.
"""
def __init__(self,
logdir: str = None,
batch_size: int = 1,
verbose: int = 2,
steps: int = None,
callbacks: list = None,
max_queue_size: int = 10,
workers: int = 1,
use_multiprocessing: bool = False,
visualize: bool = True,
velocities: bool = True,
version: str = 'v1.0-mini',
dataroot: str = None,
class_matching_dict: dict = None,
preprocessor_attributes: dict = None):
# Initialize base class
super(NuScenesEvaluator,
self).__init__(logdir=logdir,
batch_size=batch_size,
steps=steps,
verbose=verbose,
callbacks=callbacks,
max_queue_size=max_queue_size,
workers=workers,
use_multiprocessing=use_multiprocessing)
# Initialize NuScenesEvaluator
self.visualize = visualize
self.velocities = velocities
self.version = version
self.dataroot = dataroot
self.class_matching_dict = class_matching_dict if class_matching_dict is not None else {}
self.preprocessor_attributes = preprocessor_attributes if preprocessor_attributes is not None else {}
# Load nuScenes dataset
self.nusc = NuScenes(version=self.version,
dataroot=self.dataroot,
verbose=False)
# Get nuScenes pre-processor
self.nusc_preprocessor = NuScenesPreProcessor(
self.class_matching_dict, **self.preprocessor_attributes)
def _write_to_csv(self, filename, data):
with open(filename, "w", newline="") as file:
writer = csv.writer(file)
writer.writerow(
["Scene token"] +
["Val F1 score sample " + str(i) for i in range(41)])
writer.writerows(data)
def _get_bounding_boxes(self, files):
"""
Returns the bounding boxes of the scenes specified in files.
Hint: The filenames must correspond to the nuScenes scene token.
Attributes:
files: Dataset of filenames of the nuScenes scenes, <tf.data.Dataset>.
Returns:
boxes: Nested list of bounding boxes (scenes, samples, boxes), <List>.
"""
# Define
boxes = []
# Get bounding boxes
for filename in files:
# Define
sample_boxes = []
# Get current scene
scene_token = os.path.basename(
os.path.splitext(filename.numpy())[0]).decode("utf-8")
scene = self.nusc.get('scene', scene_token)
sample = self.nusc.get('sample', scene['first_sample_token'])
# Skip first sample since it has less radar sweeps available
sample = self.nusc.get('sample', sample['next'])
# Iterate over all samples within the scene
while True:
# Get bounding boxes of the sample
sample_boxes.append(
self.nusc_preprocessor.get_annotations(dataset=self.nusc,
sample=sample))
# Check if sample is the last sample in the current scene
if sample['next']:
sample = self.nusc.get('sample', sample['next'])
else:
break
boxes.append(sample_boxes)
return boxes
def _get_f1_scores(self, confidence, dataset, num_classes):
"""
Returns sample-wise F1 scores for all scenes.
Arguments:
confidence: Confidance of the predicted label, <tf.Tensor>.
dataset: Dataset with features and labels, <tf.data.Dataset>.
num_classes: Number of classes, <int>.
Returns:
F1: List of F1 scores per sample, <List>.
"""
F1 = []
# Iterate over scenes
for (_, lables), score in zip(dataset, confidence):
sample_F1 = []
# Iterate over samples
for sample_labels, sample_scores in zip(tf.unstack(lables, axis=1),
tf.unstack(score, axis=1)):
sample_F1.append(
float(
metrics.F1Score(num_classes=num_classes,
average='macro',
top_k=1)(sample_labels,
sample_scores)))
F1.append(sample_F1)
return F1
def log_f1_scores(self, F1, files):
data = [[
os.path.basename(os.path.splitext(
token.numpy())[0]).decode("utf-8")
] + scores for token, scores in zip(files, F1)]
filename = os.path.join(self.logdir, 'F1Scores.csv')
self._write_to_csv(filename, data)
def visualize_results(self, dataset, prediction, confidence, boxes, files):
"""
Visualizes the model predictions and the ground truth data per sample.
Arguments:
dataset: Dataset with features and labels, <tf.data.Dataset>.
prediction: Predicted labels, <tf.Tensor>.
confidence: Confidance of the predicted labels, <tf.Tensor>.
boxes: Nested list of bounding boxes (scenes, samples, boxes), <List>.
files: Dataset of filenames of the nuScenes scenes, <tf.data.Dataset>.
"""
# Define
colors = utils.get_colors('blues')
i = 0
# Visualize samples
with progressbar.ProgressBar(max_value=len(prediction)) as bar:
for filename, (
features, _
), scene_predictions, scene_confidence, scene_boxes in zip(
files, dataset, prediction, confidence, boxes):
n_samples_dataset = features["x"].shape[1]
n_samples_script_loader = len(scene_boxes)
assert n_samples_dataset == n_samples_script_loader, f"""Number of scene samples in dataset: {n_samples_dataset}
is different than that from nuScenes loader {n_samples_script_loader}. Make sure to skip first sample in dataset creation."""
# Update progressbar
bar.update(i)
# Get current scene
scene_token = os.path.basename(
os.path.splitext(filename.numpy())[0]).decode("utf-8")
scene = self.nusc.get('scene', scene_token)
# Get map
log = self.nusc.get('log', scene['log_token'])
map_ = self.nusc.get('map', log['map_token'])
# Get sample
sample = self.nusc.get('sample', scene['first_sample_token'])
# Skip first sample since it has less radar sweeps available
sample = self.nusc.get('sample', sample['next'])
for j, sample_boxes in enumerate(scene_boxes):
# Get sample data
ref_data = self.nusc.get('sample_data',
sample['data']['LIDAR_TOP'])
pose = self.nusc.get('ego_pose',
ref_data['ego_pose_token'])
sample_points = tf.concat(
(features['x'][:, j, :], features['y'][:, j, :],
features['z'][:, j, :]),
axis=0).numpy()
sample_predictions = np.squeeze(scene_predictions[:, j, :])
sample_confidence = np.amax(np.squeeze(
scene_confidence[:, j, :]),
axis=-1)
# Get velocities
if self.velocities:
velocities = tf.concat((features['vx_comp'][:, j, :],
features['vy_comp'][:, j, :]),
axis=0).numpy()
else:
velocities = None
# Set image name
out_path = os.path.join(
self.logdir, 'images',
scene_token + '_' + str(sample['timestamp']) + '_' +
sample['token'] + '.svg')
# Render sample
render_sample.render_sample(sample_points,
sample_predictions,
sample_boxes,
score=sample_confidence,
velocities=velocities,
map_=map_,
pose=pose,
colors=colors,
out_path=out_path,
out_format='svg')
# Get next sample
if sample['next']:
sample = self.nusc.get('sample', sample['next'])
i += 1
def evaluate(self, model, dataset, files, num_classes):
# Make prediction
prediction, confidence = self.predict(model=model, dataset=dataset)
# Get bounding boxes
boxes = self._get_bounding_boxes(files)
# Log F1 scores
F1 = self._get_f1_scores(confidence, dataset, num_classes)
self.log_f1_scores(F1, files)
# Visualize prediction
if self.visualize:
self.visualize_results(dataset, prediction, confidence, boxes,
files)
def get_config(self):
# Get instance configuration
config = {
'visualize': self.visualize,
'version': self.version,
'dataroot': self.dataroot,
'class_matching_dict': self.class_matching_dict,
'preprocessor_attributes': self.preprocessor_attributes
}
# Get base class configuration
base_config = super(NuScenesEvaluator, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
class NuscenesVisualizer(Visualizer):
"""
NuScenes Model Visualizer
Visualizes the data set with its ground truth labels.
Arguments:
logdir: Folder path to store the evaluation log files, <str>.
batch_size: Batch size used for the model prediction, <int>.
verbose: Verbosity mode (0 = silent, 1 = progress bar, 2 = one line per epoch), <int>.
velocities: Whether to visualize the velocity vectors or not, <bool>.
version: Version of the nuScenes dataset, <str>.
dataroot: Path to the raw nuScenes dataset files, <str>.
class_matching_dict: Key-Value-Pairs of the original category names and the user configured class names, <dict>.
preprocessor_attributes: Keyword attributes for the nuScenes pre-processor, <dict>.
"""
def __init__(self,
logdir: str = None,
batch_size: int = 1,
velocities: bool = True,
version: str = 'v1.0-mini',
dataroot: str = None,
class_matching_dict: dict = None,
preprocessor_attributes: dict = None):
# Initialize base class
super(NuscenesVisualizer, self).__init__(logdir=logdir,
batch_size=batch_size)
# Initialize NuscenesVisualizer
self.velocities = velocities
self.version = version
self.dataroot = dataroot
self.class_matching_dict = class_matching_dict if class_matching_dict is not None else {}
self.preprocessor_attributes = preprocessor_attributes if preprocessor_attributes is not None else {}
# Load nuScenes dataset
file_ending = os.path.splitext(self.dataroot)[-1]
if file_ending == ".pkl":
with open(self.dataroot, 'rb') as f:
print("Loading NuScenes tables from pickle file...")
timer_start = time.time()
self.nusc = pickle.load(f)
timer_end = time.time()
print("Finished loading in %d seconds" %
(timer_end - timer_start))
else:
print("Loading NuScenes...")
timer_start = time.time()
self.nusc = NuScenes(version=self.version,
dataroot=self.dataroot,
verbose=False)
timer_end = time.time()
print("Finished loading in %d seconds" % (timer_end - timer_start))
# Get nuScenes pre-processor
self.nusc_preprocessor = NuScenesPreProcessor(
self.class_matching_dict, **self.preprocessor_attributes)
def _get_bounding_boxes(self, files):
"""
Returns the bounding boxes of the scenes specified in files.
Hint: The filenames must correspond to the nuScenes scene token.
Attributes:
files: Dataset of filenames of the nuScenes scenes, <tf.data.Dataset>.
Returns:
boxes: Nested list of bounding boxes (scenes, samples, boxes), <List>.
"""
# Define
boxes = []
# Get bounding boxes
for filename in files:
# Define
sample_boxes = []
# Get current scene
scene_token = os.path.basename(
os.path.splitext(filename.numpy())[0]).decode("utf-8")
scene = self.nusc.get('scene', scene_token)
sample = self.nusc.get('sample', scene['first_sample_token'])
# Skip first sample since it has less radar sweeps available
sample = self.nusc.get('sample', sample['next'])
# Iterate over all samples within the scene
while True:
# Get bounding boxes of the sample
sample_boxes.append(
self.nusc_preprocessor.get_annotations(dataset=self.nusc,
sample=sample))
# Check if sample is the last sample in the current scene
if sample['next']:
sample = self.nusc.get('sample', sample['next'])
else:
break
boxes.append(sample_boxes)
return boxes
def visualize_results(self, dataset, boxes, files):
"""
Visualizes the model predictions and the ground truth data per sample.
Arguments:
dataset: Dataset with features and labels, <tf.data.Dataset>.
prediction: Predicted labels, <tf.Tensor>.
confidence: Confidance of the predicted labels, <tf.Tensor>.
boxes: Nested list of bounding boxes (scenes, samples, boxes), <List>.
files: Dataset of filenames of the nuScenes scenes, <tf.data.Dataset>.
"""
# Define
colors = utils.get_colors('blues')
i = 0
# Visualize samples
with progressbar.ProgressBar(max_value=len(boxes)) as bar:
for filename, (features,
labels), scene_boxes in zip(files, dataset, boxes):
n_samples_dataset = features["x"].shape[1]
n_samples_script_loader = len(scene_boxes)
assert n_samples_dataset == n_samples_script_loader, f"""Number of scene samples in dataset: {n_samples_dataset}
is different than that from nuScenes loader {n_samples_script_loader}. Make sure to skip first sample in dataset creation."""
# Update progressbar
bar.update(i)
# Get current scene
scene_token = os.path.basename(
os.path.splitext(filename.numpy())[0]).decode("utf-8")
scene = self.nusc.get('scene', scene_token)
# Get map
log = self.nusc.get('log', scene['log_token'])
map_ = self.nusc.get('map', log['map_token'])
# Get sample
sample = self.nusc.get('sample', scene['first_sample_token'])
# Skip first sample since it has less radar sweeps available
sample = self.nusc.get('sample', sample['next'])
for j, sample_boxes in enumerate(scene_boxes):
# Get sample data
ref_data = self.nusc.get('sample_data',
sample['data']['LIDAR_TOP'])
pose = self.nusc.get('ego_pose',
ref_data['ego_pose_token'])
sample_points = tf.concat(
(features['x'][:, j, :], features['y'][:, j, :],
features['z'][:, j, :]),
axis=0).numpy()
sample_lls = labels[:, j, :, :].numpy()
sample_labels = np.squeeze(np.argmax(sample_lls, axis=2))
# Get velocities
if self.velocities and 'vx_comp' in features:
velocities = tf.concat((features['vx_comp'][:, j, :],
features['vy_comp'][:, j, :]),
axis=0).numpy()
else:
velocities = None
# Set image name
out_path = os.path.join(
self.logdir, 'images',
scene_token + '_' + str(sample['timestamp']) + '_' +
sample['token'] + '.svg')
# Render sample
render_sample.render_sample(points=sample_points,
labels=sample_labels,
annotations=sample_boxes,
velocities=velocities,
map_=map_,
pose=pose,
colors=colors,
out_path=out_path,
out_format='svg')
# Get next sample
if sample['next']:
sample = self.nusc.get('sample', sample['next'])
i += 1
def visualize(self, dataset, files, num_classes):
boxes = self._get_bounding_boxes(files)
self.visualize_results(dataset, boxes, files)
def get_config(self):
# Get instance configuration
config = {
'version': self.version,
'dataroot': self.dataroot,
'class_matching_dict': self.class_matching_dict,
'preprocessor_attributes': self.preprocessor_attributes
}
# Get base class configuration
base_config = super(NuscenesVisualizer, self).get_config()
return dict(list(base_config.items()) + list(config.items()))