Пример #1
0
def nusc_eval_kit(
    result_path:
    str = 'data/demo/vmetrics/submission2.json',  # The submission as a JSON file.
    groundtruth_path:
    str = 'data/demo/vmetrics/gt2.json',  # The ground truth as a JSON file.
    output_dir:
    str = 'data/eval_out/vmetrics/',  # Folder to store result metrics.
    config_path:
    str = '',  #'Path to the configuration file. If no path given, the CVPR 2019 configuration will be used.
    render_curves: bool = True,  # Whether to render PR and TP curves to disk.
    verbose: bool = True,  # Whether to print to stdout.
    is_average_delay=False
):  # Whether to calculate average delay metric. Keep it False for now.

    if config_path == '':
        cfg = config_factory('detection_cvpr_2019')
    else:
        with open(config_path, 'r') as _f:
            cfg = DetectionConfig.deserialize(json.load(_f))

    nusc_eval = DetectionEval(config=cfg,
                              result_path=result_path,
                              groundtruth_path=groundtruth_path,
                              output_dir=output_dir,
                              verbose=verbose,
                              is_average_delay=is_average_delay)

    nusc_eval.main(render_curves=render_curves)
Пример #2
0
    def test_DetectionConfig_serialization(self):
        """ Test serialization round trip """
        cfg = {
            'class_range': {
                'car': 1,
                'truck': 1,
                'bus': 1,
                'trailer': 1,
                'construction_vehicle': 1,
                'pedestrian': 1,
                'motorcycle': 1,
                'bicycle': 1,
                'traffic_cone': 1,
                'barrier': 1
            },
            'dist_fcn': 'distance',
            'dist_ths': [0.0, 1.0],
            'dist_th_tp': 1.0,
            'min_recall': 0.0,
            'min_precision': 0.0,
            'max_boxes_per_sample': 1,
            'mean_ap_weight': 1
        }

        detect_config = DetectionConfig.deserialize(cfg)
        cfg_output = detect_config.serialize()

        self.assertEqual(cfg, cfg_output)
Пример #3
0
def config_factory(
        configuration_name: str
) -> Union[DetectionConfig]:  #, TrackingConfig]:
    """
    Creates a *Config instance that can be used to initialize a *Eval instance, where * stands for Detection/Tracking.
    Note that this only works if the config file is located in the nuscenes/eval/common/configs folder.
    :param configuration_name: Name of desired configuration in eval_detection_configs.
    :return: *Config instance.
    """
    # Check if config exists.
    tokens = configuration_name.split('_')
    assert len(
        tokens
    ) > 1, 'Error: Configuration name must be have prefix "detection_"!'  # or "tracking_"!'
    task = tokens[0]
    this_dir = os.path.dirname(os.path.abspath(__file__))
    cfg_path = os.path.join(this_dir, '..', task, 'configs',
                            '%s.json' % configuration_name)
    assert os.path.exists(
        cfg_path), 'Requested unknown configuration {}'.format(
            configuration_name)

    # Load config file and deserialize it.
    with open(cfg_path, 'r') as f:
        data = json.load(f)
    if task == 'detection':
        cfg = DetectionConfig.deserialize(data)
    # elif task == 'tracking':
    #     cfg = TrackingConfig.deserialize(data)
    else:
        raise Exception('Error: Invalid config file name: %s' %
                        configuration_name)

    return cfg
Пример #4
0
    def test_serialization(self):
        """ test that instance serialization protocol works with json encoding """

        this_dir = os.path.dirname(os.path.abspath(__file__))
        cfg_name = 'cvpr_2019.json'
        config_path = os.path.join(this_dir, '..', 'configs', cfg_name)

        with open(config_path) as f:
            cfg = json.load(f)

        detect_cfg = DetectionConfig.deserialize(cfg)

        self.assertEqual(cfg, detect_cfg.serialize())

        recovered = DetectionConfig.deserialize(
            json.loads(json.dumps(detect_cfg.serialize())))
        self.assertEqual(detect_cfg, recovered)
Пример #5
0
def config_factory(configuration_name: str) -> DetectionConfig:
    """
    Creates a DetectionConfig instance that can be used to initialize a NuScenesEval instance.
    :param configuration_name: Name of desired configuration in eval_detection_configs.
    :return: DetectionConfig instance.
    """

    assert configuration_name in eval_detection_configs.keys(), \
        'Requested unknown configuration {}'.format(configuration_name)

    return DetectionConfig.deserialize(
        eval_detection_configs[configuration_name])
Пример #6
0
def config_factory() -> DetectionConfig:
    """
    Creates a DetectionConfig instance that can be used to initialize a NuScenesEval instance.
    Note that this only works if the config file is located in the nuscenes/eval/detection/configs folder.
    :param configuration_name: Name of desired configuration in eval_detection_configs.
    :return: DetectionConfig instance.
    """
    # Check if config exists.
    this_dir = os.path.dirname(os.path.abspath(__file__))
    cfg_path = os.path.join(this_dir, 'UDI_2020.json')
    assert os.path.exists(cfg_path), \
        'Wrong configuration file'
    # Load config file and deserialize it.
    with open(cfg_path, 'r') as f:
        data = json.load(f)
    cfg = DetectionConfig.deserialize(data)
    return cfg
Пример #7
0
    def test_serialization(self):
        """ Test that instance serialization protocol works with json encoding. """

        cfg = {
            'class_range': {
                'car': 1.0,
                'truck': 1.0,
                'bus': 1.0,
                'trailer': 1.0,
                'construction_vehicle': 1.0,
                'pedestrian': 1.0,
                'motorcycle': 1.0,
                'bicycle': 1.0,
                'traffic_cone': 1.0,
                'barrier': 1.0
            },
            'dist_fcn': 'distance',
            'dist_ths': [0.0, 1.0],
            'dist_th_tp': 1.0,
            'min_recall': 0.0,
            'min_precision': 0.0,
            'max_boxes_per_sample': 1,
            'mean_ap_weight': 1.0
        }
        detect_config = DetectionConfig.deserialize(cfg)

        metrics = DetectionMetrics(cfg=detect_config)

        for i, name in enumerate(cfg['class_range'].keys()):
            metrics.add_label_ap(name, 1.0, float(i))
            for j, tp_name in enumerate(TP_METRICS):
                metrics.add_label_tp(name, tp_name, float(j))

        serialized = json.dumps(metrics.serialize())
        deserialized = DetectionMetrics.deserialize(json.loads(serialized))

        self.assertEqual(metrics, deserialized)
Пример #8
0
    def test_delta(self):
        """
        This tests runs the evaluation for an arbitrary random set of predictions.
        This score is then captured in this very test such that if we change the eval code,
        this test will trigger if the results changed.
        """
        random.seed(42)
        np.random.seed(42)
        assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'

        nusc = NuScenes(version='v1.0-mini', dataroot=os.environ['NUSCENES'], verbose=False)

        with open(self.res_mockup, 'w') as f:
            json.dump(self._mock_submission(nusc, 'mini_val'), f, indent=2)

        this_dir = os.path.dirname(os.path.abspath(__file__))
        cfg_name = 'cvpr_2019.json'
        cfg_path = os.path.join(this_dir, '..', 'configs', cfg_name)
        with open(cfg_path, 'r') as f:
            cfg = DetectionConfig.deserialize(json.load(f))

        nusc_eval = NuScenesEval(nusc, cfg, self.res_mockup, eval_set='mini_val', output_dir=self.res_eval_folder,
                                 verbose=False)
        metrics, md_list = nusc_eval.evaluate()

        # 1. Score = 0.22082865720221012. Measured on the branch "release_v0.2" on March 7 2019.
        # 2. Score = 0.2199307290627096. Changed to measure center distance from the ego-vehicle.
        # 3. Score = 0.24954451673961747. Changed to 1.0-mini and cleaned up build script.
        # 4. Score = 0.20478832626986893. Updated treatment of cones, barriers, and other algo tunings.
        # 5. Score = 0.2043569666105005. AP calculation area is changed from >=min_recall to >min_recall.
        # 6. Score = 0.20636954644294506. After bike-rack filtering.
        # 7. Score = 0.20237925145690996. After TP reversion bug.
        # 8. Score = 0.24047129251302665. After bike racks bug.
        # 9. Score = 0.24104572227466886. After bug fix in calc_tp. Include the max recall and exclude the min recall.
        # 10. Score = 0.19449091580477748. Changed to use v1.0 mini_val split.
        self.assertAlmostEqual(metrics.nd_score, 0.19449091580477748)
Пример #9
0
    parser.add_argument('--verbose',
                        type=int,
                        default=1,
                        help='Whether to print to stdout.')
    args = parser.parse_args()

    result_path_ = os.path.expanduser(args.result_path)
    output_dir_ = os.path.expanduser(args.output_dir)
    eval_set_ = args.eval_set
    dataroot_ = args.dataroot
    version_ = args.version
    config_path = args.config_path
    plot_examples_ = args.plot_examples
    render_curves_ = bool(args.render_curves)
    verbose_ = bool(args.verbose)

    if config_path == '':
        cfg_ = config_factory('cvpr_2019')
    else:
        with open(config_path, 'r') as f:
            cfg_ = DetectionConfig.deserialize(json.load(f))

    nusc_ = NuScenes(version=version_, verbose=verbose_, dataroot=dataroot_)
    nusc_eval = NuScenesEval(nusc_,
                             config=cfg_,
                             result_path=result_path_,
                             eval_set=eval_set_,
                             output_dir=output_dir_,
                             verbose=verbose_)
    nusc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_)
Пример #10
0
    def test_filter_eval_boxes(self):
        """
        This tests runs the evaluation for an arbitrary random set of predictions.
        This score is then captured in this very test such that if we change the eval code,
        this test will trigger if the results changed.
        """

        # Get the maximum distance from the config
        this_dir = os.path.dirname(os.path.abspath(__file__))
        cfg_name = 'cvpr_2019.json'
        cfg_path = os.path.join(this_dir, '..', 'configs', cfg_name)
        with open(cfg_path, 'r') as f:
            cfg = DetectionConfig.deserialize(json.load(f))
        max_dist = cfg.class_range

        assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'

        nusc = NuScenes(version='v1.0-mini',
                        dataroot=os.environ['NUSCENES'],
                        verbose=False)

        sample_token = '0af0feb5b1394b928dd13d648de898f5'
        # This sample has a bike rack instance 'bfe685042aa34ab7b2b2f24ee0f1645f' with these parameters
        # 'translation': [683.681, 1592.002, 0.809],
        # 'size': [1.641, 14.465, 1.4],
        # 'rotation': [0.3473693995546558, 0.0, 0.0, 0.9377283723195315]

        # Test bicycle filtering by creating a box at the same position as the bike rack.
        box1 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='bicycle')

        eval_boxes = EvalBoxes()
        eval_boxes.add_boxes(sample_token, [box1])

        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)

        self.assertEqual(len(filtered_boxes.boxes[sample_token]),
                         0)  # box1 should be filtered.

        # Test motorcycle filtering by creating a box at the same position as the bike rack.
        box2 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='motorcycle')

        eval_boxes = EvalBoxes()
        eval_boxes.add_boxes(sample_token, [box1, box2])

        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)

        self.assertEqual(len(filtered_boxes.boxes[sample_token]),
                         0)  # both box1 and box2 should be filtered.

        # Now create a car at the same position as the bike rack.
        box3 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='car')

        eval_boxes = EvalBoxes()
        eval_boxes.add_boxes(sample_token, [box1, box2, box3])

        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)

        self.assertEqual(len(filtered_boxes.boxes[sample_token]),
                         1)  # box1 and box2 to be filtered. box3 to stay.
        self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
                         'car')

        # Now add a bike outside the bike rack.
        box4 = EvalBox(sample_token=sample_token,
                       translation=(68.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='bicycle')

        eval_boxes = EvalBoxes()
        eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4])

        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)

        self.assertEqual(len(filtered_boxes.boxes[sample_token]),
                         2)  # box1, box2 to be filtered. box3, box4 to stay.
        self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
                         'car')
        self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
                         'bicycle')
        self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
                         68.681)

        # Add another bike on the bike rack center but set the ego_dist higher than what's defined in max_dist
        box5 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='bicycle',
                       ego_dist=100.0)

        eval_boxes = EvalBoxes()
        eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4, box5])

        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
        self.assertEqual(len(filtered_boxes.boxes[sample_token]),
                         2)  # box1, box2, box5 filtered. box3, box4 to stay.
        self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
                         'car')
        self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
                         'bicycle')
        self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
                         68.681)

        # Add another bike on the bike rack center but set the num_pts to be zero so that it gets filtered.
        box6 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='bicycle',
                       num_pts=0)

        eval_boxes = EvalBoxes()
        eval_boxes.add_boxes(sample_token,
                             [box1, box2, box3, box4, box5, box6])

        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
        self.assertEqual(len(filtered_boxes.boxes[sample_token]),
                         2)  # box1, box2, box5, box6 filtered. box3, box4 stay
        self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
                         'car')
        self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
                         'bicycle')
        self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
                         68.681)

        # Check for a sample where there are no bike racks. Everything should be filtered correctly.
        sample_token = 'ca9a282c9e77460f8360f564131a8af5'  # This sample has no bike-racks.

        box1 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='bicycle',
                       ego_dist=25.0)

        box2 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='motorcycle',
                       ego_dist=45.0)

        box3 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='car',
                       ego_dist=45.0)

        box4 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='car',
                       ego_dist=55.0)

        box5 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='bicycle',
                       num_pts=1)

        box6 = EvalBox(sample_token=sample_token,
                       translation=(683.681, 1592.002, 0.809),
                       size=(1, 1, 1),
                       detection_name='bicycle',
                       num_pts=0)

        eval_boxes = EvalBoxes()
        eval_boxes.add_boxes(sample_token,
                             [box1, box2, box3, box4, box5, box6])

        filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
        self.assertEqual(len(filtered_boxes.boxes[sample_token]),
                         3)  # box2, box4, box6 filtered. box1, box3, box5 stay
        self.assertEqual(filtered_boxes.boxes[sample_token][0].ego_dist, 25.0)
        self.assertEqual(filtered_boxes.boxes[sample_token][1].ego_dist, 45.0)
        self.assertEqual(filtered_boxes.boxes[sample_token][2].num_pts, 1)
Пример #11
0
class TestAlgo(unittest.TestCase):

    this_dir = os.path.dirname(os.path.abspath(__file__))
    cfg_name = 'cvpr_2019.json'
    cfg_path = os.path.join(this_dir, '..', 'configs', cfg_name)
    with open(cfg_path, 'r') as f:
        cfg = DetectionConfig.deserialize(json.load(f))

    @staticmethod
    def _mock_results(nsamples, ngt, npred, detection_name):
        def random_attr():
            """
            This is the most straight-forward way to generate a random attribute.
            Not currently used b/c we want the test fixture to be back-wards compatible.
            """
            # Get relevant attributes.
            rel_attributes = detection_name_to_rel_attributes(detection_name)

            if len(rel_attributes) == 0:
                # Empty string for classes without attributes.
                return ''
            else:
                # Pick a random attribute otherwise.
                return rel_attributes[np.random.randint(
                    0, len(rel_attributes))]

        pred = EvalBoxes()
        gt = EvalBoxes()

        for sample_itt in range(nsamples):

            this_gt = []

            for box_itt in range(ngt):

                this_gt.append(
                    EvalBox(
                        sample_token=str(sample_itt),
                        translation=tuple(
                            list(np.random.rand(2) * 15) + [0.0]),
                        size=tuple(np.random.rand(3) * 4),
                        rotation=tuple(np.random.rand(4)),
                        velocity=tuple(np.random.rand(3)[:2] * 4),
                        detection_name=detection_name,
                        detection_score=random.random(),
                        attribute_name=random_attr(),
                        ego_dist=random.random() * 10,
                    ))
            gt.add_boxes(str(sample_itt), this_gt)

        for sample_itt in range(nsamples):
            this_pred = []

            for box_itt in range(npred):

                this_pred.append(
                    EvalBox(
                        sample_token=str(sample_itt),
                        translation=tuple(
                            list(np.random.rand(2) * 10) + [0.0]),
                        size=tuple(np.random.rand(3) * 4),
                        rotation=tuple(np.random.rand(4)),
                        velocity=tuple(np.random.rand(3)[:2] * 4),
                        detection_name=detection_name,
                        detection_score=random.random(),
                        attribute_name=random_attr(),
                        ego_dist=random.random() * 10,
                    ))

            pred.add_boxes(str(sample_itt), this_pred)

        return gt, pred

    def test_nd_score(self):
        """
        This tests runs the full evaluation for an arbitrary random set of predictions.
        """

        random.seed(42)
        np.random.seed(42)

        mdl = MetricDataList()
        for class_name in self.cfg.class_names:
            gt, pred = self._mock_results(30, 3, 25, class_name)
            for dist_th in self.cfg.dist_ths:
                mdl.set(class_name, dist_th,
                        accumulate(gt, pred, class_name, 'center_distance', 2))

        metrics = DetectionMetrics(self.cfg)
        for class_name in self.cfg.class_names:
            for dist_th in self.cfg.dist_ths:
                ap = calc_ap(mdl[(class_name, dist_th)], self.cfg.min_recall,
                             self.cfg.min_precision)
                metrics.add_label_ap(class_name, dist_th, ap)

            for metric_name in TP_METRICS:
                metric_data = mdl[(class_name, self.cfg.dist_th_tp)]
                if class_name in ['traffic_cone'] and metric_name in [
                        'attr_err', 'vel_err', 'orient_err'
                ]:
                    tp = np.nan
                elif class_name in ['barrier'] and metric_name in [
                        'attr_err', 'vel_err'
                ]:
                    tp = np.nan
                else:
                    tp = calc_tp(metric_data, self.cfg.min_recall, metric_name)
                metrics.add_label_tp(class_name, metric_name, tp)

        self.assertEqual(0.08606662159639042, metrics.nd_score)

    def test_calc_tp(self):
        """Test for calc_tp()."""

        random.seed(42)
        np.random.seed(42)

        md = MetricData.random_md()

        # min_recall greater than 1.
        self.assertEqual(1.0, calc_tp(md,
                                      min_recall=1,
                                      metric_name='trans_err'))

    def test_calc_ap(self):
        """Test for calc_ap()."""

        random.seed(42)
        np.random.seed(42)

        md = MetricData.random_md()

        # Negative min_recall and min_precision
        self.assertRaises(AssertionError, calc_ap, md, -0.5, 0.4)
        self.assertRaises(AssertionError, calc_ap, md, 0.5, -0.8)

        # More than 1 min_precision/min_recall
        self.assertRaises(AssertionError, calc_ap, md, 0.7, 1)
        self.assertRaises(AssertionError, calc_ap, md, 1.2, 0)