def test_weighted_sum(self): """ This tests runs the full evaluation for an arbitrary random set of predictions. """ random.seed(42) np.random.seed(42) mdl = MetricDataList() for class_name in self.cfg.class_names: gt, pred = self._mock_results(30, 3, 25, class_name) for dist_th in self.cfg.dist_ths: mdl.set(class_name, dist_th, accumulate(gt, pred, class_name, 'center_distance', 2)) metrics = DetectionMetrics(self.cfg) for class_name in self.cfg.class_names: for dist_th in self.cfg.dist_ths: ap = calc_ap(mdl[(class_name, dist_th)], self.cfg.min_recall, self.cfg.min_precision) metrics.add_label_ap(class_name, dist_th, ap) for metric_name in TP_METRICS: metric_data = mdl[(class_name, self.cfg.dist_th_tp)] if class_name in ['traffic_cone'] and metric_name in ['attr_err', 'vel_err', 'orient_err']: tp = np.nan elif class_name in ['barrier'] and metric_name in ['attr_err', 'vel_err']: tp = np.nan else: tp = calc_tp(metric_data, self.cfg.min_recall, metric_name) metrics.add_label_tp(class_name, metric_name, tp) self.assertEqual(0.08606662159639042, metrics.weighted_sum)
def test_serialization(self): """ Test that instance serialization protocol works with json encoding. """ mdl = MetricDataList() for i in range(10): mdl.set('name', 0.1, MetricData.random_md()) recovered = MetricDataList.deserialize( json.loads(json.dumps(mdl.serialize()))) self.assertEqual(mdl, recovered)
def run(self) -> Tuple[DetectionMetrics, MetricDataList]: start_time = time.time() # ----------------------------------- # Step 1: Accumulate metric data for all classes and distance thresholds # ----------------------------------- metric_data_list = MetricDataList() for class_name in self.cfg.class_names: for dist_th in self.cfg.dist_ths: md = accumulate(self.gt_boxes, self.pred_boxes, class_name, self.cfg.dist_fcn, dist_th) metric_data_list.set(class_name, dist_th, md) # ----------------------------------- # Step 2: Calculate metrics from the data # ----------------------------------- metrics = DetectionMetrics(self.cfg) for class_name in self.cfg.class_names: for dist_th in self.cfg.dist_ths: metric_data = metric_data_list[(class_name, dist_th)] ap = calc_ap(metric_data, self.cfg.min_recall, self.cfg.min_precision) metrics.add_label_ap(class_name, dist_th, ap) for metric_name in TP_METRICS: metric_data = metric_data_list[(class_name, self.cfg.dist_th_tp)] if class_name in ['traffic_cone'] and metric_name in [ 'attr_err', 'vel_err', 'orient_err' ]: tp = np.nan elif class_name in ['barrier'] and metric_name in [ 'attr_err', 'vel_err' ]: tp = np.nan else: tp = calc_tp(metric_data, self.cfg.min_recall, metric_name) metrics.add_label_tp(class_name, metric_name, tp) metrics.add_runtime(time.time() - start_time) # ----------------------------------- # Step 3: Dump the metric data and metrics to disk # ----------------------------------- with open(os.path.join(self.output_dir, 'metrics.json'), 'w') as f: json.dump(metrics.serialize(), f, indent=2) with open(os.path.join(self.output_dir, 'metric_data_list.json'), 'w') as f: json.dump(metric_data_list.serialize(), f, indent=2) return metrics, metric_data_list
def evaluate(self) -> Tuple[DetectionMetrics, MetricDataList]: """ Performs the actual evaluation. :return: A tuple of high-level and the raw metric data. """ start_time = time.time() # ----------------------------------- # Step 1: Accumulate metric data for all classes and distance thresholds. # ----------------------------------- if self.verbose: print('Accumulating metric data') metric_data_list = MetricDataList() for class_name in self.cfg.class_names: for dist_th in self.cfg.dist_ths: md = accumulate(self.gt_boxes, self.pred_boxes, class_name, self.cfg.dist_fcn, dist_th) metric_data_list.set(class_name, dist_th, md) # ----------------------------------- # Step 2: Calculate metrics from the data. # ----------------------------------- if self.verbose: print('Calculating metrics') metrics = DetectionMetrics(self.cfg) for class_name in self.cfg.class_names: for dist_th in self.cfg.dist_ths: metric_data = metric_data_list[(class_name, dist_th)] ap = calc_ap(metric_data, self.cfg.min_recall, self.cfg.min_precision) metrics.add_label_ap(class_name, dist_th, ap) for metric_name in TP_METRICS: metric_data = metric_data_list[(class_name, self.cfg.dist_th_tp)] if class_name in ['traffic_cone'] and metric_name in [ 'attr_err', 'vel_err', 'orient_err' ]: tp = np.nan elif class_name in ['barrier'] and metric_name in [ 'attr_err', 'vel_err' ]: tp = np.nan else: tp = calc_tp(metric_data, self.cfg.min_recall, metric_name) metrics.add_label_tp(class_name, metric_name, tp) metrics.add_runtime(time.time() - start_time) return metrics, metric_data_list