def test_weighted_sum(self):
"""
This tests runs the full evaluation for an arbitrary random set of predictions.
"""
random.seed(42)
np.random.seed(42)
mdl = MetricDataList()
for class_name in self.cfg.class_names:
gt, pred = self._mock_results(30, 3, 25, class_name)
for dist_th in self.cfg.dist_ths:
mdl.set(class_name, dist_th, accumulate(gt, pred, class_name, 'center_distance', 2))
metrics = DetectionMetrics(self.cfg)
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
ap = calc_ap(mdl[(class_name, dist_th)], self.cfg.min_recall, self.cfg.min_precision)
metrics.add_label_ap(class_name, dist_th, ap)
for metric_name in TP_METRICS:
metric_data = mdl[(class_name, self.cfg.dist_th_tp)]
if class_name in ['traffic_cone'] and metric_name in ['attr_err', 'vel_err', 'orient_err']:
tp = np.nan
elif class_name in ['barrier'] and metric_name in ['attr_err', 'vel_err']:
tp = np.nan
else:
tp = calc_tp(metric_data, self.cfg.min_recall, metric_name)
metrics.add_label_tp(class_name, metric_name, tp)
self.assertEqual(0.08606662159639042, metrics.weighted_sum)
def run(self) -> Tuple[DetectionMetrics, MetricDataList]:
start_time = time.time()
# -----------------------------------
# Step 1: Accumulate metric data for all classes and distance thresholds
# -----------------------------------
metric_data_list = MetricDataList()
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
md = accumulate(self.gt_boxes, self.pred_boxes, class_name,
self.cfg.dist_fcn, dist_th)
metric_data_list.set(class_name, dist_th, md)
# -----------------------------------
# Step 2: Calculate metrics from the data
# -----------------------------------
metrics = DetectionMetrics(self.cfg)
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
metric_data = metric_data_list[(class_name, dist_th)]
ap = calc_ap(metric_data, self.cfg.min_recall,
self.cfg.min_precision)
metrics.add_label_ap(class_name, dist_th, ap)
for metric_name in TP_METRICS:
metric_data = metric_data_list[(class_name,
self.cfg.dist_th_tp)]
if class_name in ['traffic_cone'] and metric_name in [
'attr_err', 'vel_err', 'orient_err'
]:
tp = np.nan
elif class_name in ['barrier'] and metric_name in [
'attr_err', 'vel_err'
]:
tp = np.nan
else:
tp = calc_tp(metric_data, self.cfg.min_recall, metric_name)
metrics.add_label_tp(class_name, metric_name, tp)
metrics.add_runtime(time.time() - start_time)
# -----------------------------------
# Step 3: Dump the metric data and metrics to disk
# -----------------------------------
with open(os.path.join(self.output_dir, 'metrics.json'), 'w') as f:
json.dump(metrics.serialize(), f, indent=2)
with open(os.path.join(self.output_dir, 'metric_data_list.json'),
'w') as f:
json.dump(metric_data_list.serialize(), f, indent=2)
return metrics, metric_data_list
def evaluate(self) -> Tuple[DetectionMetrics, MetricDataList]:
"""
Performs the actual evaluation.
:return: A tuple of high-level and the raw metric data.
"""
start_time = time.time()
# -----------------------------------
# Step 1: Accumulate metric data for all classes and distance thresholds.
# -----------------------------------
if self.verbose:
print('Accumulating metric data')
metric_data_list = MetricDataList()
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
md = accumulate(self.gt_boxes, self.pred_boxes, class_name,
self.cfg.dist_fcn, dist_th)
metric_data_list.set(class_name, dist_th, md)
# -----------------------------------
# Step 2: Calculate metrics from the data.
# -----------------------------------
if self.verbose:
print('Calculating metrics')
metrics = DetectionMetrics(self.cfg)
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
metric_data = metric_data_list[(class_name, dist_th)]
ap = calc_ap(metric_data, self.cfg.min_recall,
self.cfg.min_precision)
metrics.add_label_ap(class_name, dist_th, ap)
for metric_name in TP_METRICS:
metric_data = metric_data_list[(class_name,
self.cfg.dist_th_tp)]
if class_name in ['traffic_cone'] and metric_name in [
'attr_err', 'vel_err', 'orient_err'
]:
tp = np.nan
elif class_name in ['barrier'] and metric_name in [
'attr_err', 'vel_err'
]:
tp = np.nan
else:
tp = calc_tp(metric_data, self.cfg.min_recall, metric_name)
metrics.add_label_tp(class_name, metric_name, tp)
metrics.add_runtime(time.time() - start_time)
return metrics, metric_data_list
def dist_pr_curve(md_list: MetricDataList,
metrics: DetectionMetrics,
dist_th: float,
min_precision: float,
min_recall: float,
savepath: str = None) -> None:
fig, (ax, lax) = plt.subplots(ncols=2,
gridspec_kw={"width_ratios": [4, 1]},
figsize=(7.5, 5))
ax = setup_axis(xlabel='Recall',
ylabel='Precision',
xlim=1,
ylim=1,
min_precision=min_precision,
min_recall=min_recall,
ax=ax)
# Plot the recall vs. precision curve for each detection class.
data = md_list.get_dist_data(dist_th)
for md, detection_name in data:
md = md_list[(detection_name, dist_th)]
ap = metrics.get_label_ap(detection_name, dist_th)
ax.plot(md.recall,
md.precision,
label='{}: {:.1f}%'.format(
PRETTY_DETECTION_NAMES[detection_name], ap * 100),
color=DETECTION_COLORS[detection_name])
hx, lx = ax.get_legend_handles_labels()
lax.legend(hx, lx, borderaxespad=0)
lax.axis("off")
plt.tight_layout()
if savepath is not None:
plt.savefig(savepath)
plt.close()
def class_pr_curve(md_list: MetricDataList,
metrics: DetectionMetrics,
detection_name: str,
min_precision: float,
min_recall: float,
savepath: str = None,
ax=None):
if ax is None:
ax = setup_axis(title=PRETTY_DETECTION_NAMES[detection_name],
xlabel='Recall',
ylabel='Precision',
xlim=1,
ylim=1,
min_precision=min_precision,
min_recall=min_recall)
# Get recall vs precision values of given class for each distance threshold.
data = md_list.get_class_data(detection_name)
# Plot the recall vs. precision curve for each distance threshold.
for md, dist_th in data:
ap = metrics.get_label_ap(detection_name, dist_th)
ax.plot(md.recall,
md.precision,
label='Dist. : {}, AP: {:.1f}'.format(dist_th, ap * 100))
ax.legend(loc='best')
if savepath is not None:
plt.savefig(savepath)
plt.close()
def test_serialization(self):
""" Test that instance serialization protocol works with json encoding. """
mdl = MetricDataList()
for i in range(10):
mdl.set('name', 0.1, MetricData.random_md())
recovered = MetricDataList.deserialize(
json.loads(json.dumps(mdl.serialize())))
self.assertEqual(mdl, recovered)
