def test_mot(self) -> None:
"""Check mot evaluation correctness."""
cur_dir = os.path.dirname(os.path.abspath(__file__))
gts = group_and_sort(
load("{}/testcases/track_sample_anns.json".format(cur_dir)))
preds = group_and_sort(
load("{}/testcases/track_predictions.json".format(cur_dir)))
result = evaluate_track(acc_single_video_mot, gts, preds)
overall_reference = {
"IDF1": 0.7101073676416142,
"MOTA": 0.6420070762302992,
"MOTP": 0.871614396957838,
"FP": 126,
"FN": 942,
"IDSw": 45,
"MT": 62,
"PT": 47,
"ML": 33,
"FM": 66,
"mIDF1": 0.32247819436558384,
"mMOTA": 0.24324204637536687,
"mMOTP": 0.5001285135514636,
}
for key in result["OVERALL"]:
self.assertAlmostEqual(result["OVERALL"][key],
overall_reference[key])
def main() -> None:
"""Main."""
args = parse_args()
assert os.path.isdir(args.input)
dataset = load(args.input, args.nproc)
if args.config is not None:
bdd100k_config = load_bdd100k_config(args.config)
elif dataset.config is not None:
bdd100k_config = BDD100KConfig(config=dataset.config)
else:
bdd100k_config = load_bdd100k_config(args.mode)
categories = get_leaf_categories(bdd100k_config.scalabel.categories)
convert_funcs: Dict[str, ToRLEFunc] = dict(
ins_seg=insseg_to_rle,
sem_seg=semseg_to_rle,
drivable=semseg_to_rle,
seg_track=segtrack_to_rle,
)
if args.mode == "ins_seg":
assert args.score_file is not None
frames = load(args.score_file).frames
assert all(
os.path.exists(
os.path.join(args.input, frame.name.replace(".jpg", ".png")))
for frame in frames), "Missing some bitmasks."
elif args.mode in ("sem_seg", "drivable", "seg_track"):
files = list_files(args.input)
frames = []
for file in files:
if not file.endswith(".png") and not file.endswith(".jpg"):
continue
frame = Frame(name=file.replace(".png", ".jpg"), labels=[])
frames.append(frame)
else:
return
if args.nproc > 1:
with Pool(args.nproc) as pool:
frames = pool.map(
partial(
convert_funcs[args.mode],
input_dir=args.input,
categories=categories,
),
tqdm(frames),
)
else:
frames = [
convert_funcs[args.mode](frame, args.input, categories)
for frame in tqdm(frames)
]
save(args.output, frames)
def _load_frames(gt_base: str,
result_path: str,
config: BDD100KConfig,
nproc: int = NPROC) -> Tuple[List[Frame], List[Frame]]:
"""Load ground truth and prediction frames."""
gt_frames = bdd100k_to_scalabel(load(gt_base, nproc).frames, config)
result_frames = bdd100k_to_scalabel(
load(result_path, nproc).frames, config)
return gt_frames, result_frames
def evaluate_det(
ann_file: Union[str, List[Frame]],
pred_file: Union[str, List[Frame]],
cfg_path: str,
out_dir: str = "none",
nproc: int = 4,
) -> Dict[str, float]:
"""Load the ground truth and prediction results.
Args:
ann_file: path to the ground truth annotations. "*.json"
pred_file: path to the prediciton results in BDD format. "*.json"
cfg_path: path to the config file
out_dir: output_directory
nproc: processes number for loading jsons
Returns:
dict: detection metric scores
"""
# Convert the annotation file to COCO format
if isinstance(ann_file, str):
ann_frames = load(ann_file, nproc)
else:
ann_frames = ann_file
ann_frames = sorted(ann_frames, key=lambda frame: frame.name)
categories, name_mapping, ignore_mapping = load_coco_config(
mode="det",
filepath=cfg_path,
)
ann_coco = scalabel2coco_detection(SHAPE, ann_frames, categories,
name_mapping, ignore_mapping)
coco_gt = COCOV2(None, ann_coco)
# Load results and convert the predictions
if isinstance(pred_file, str):
pred_frames = load(pred_file, nproc)
else:
pred_frames = pred_file
pred_frames = sorted(pred_frames, key=lambda frame: frame.name)
pred_res = scalabel2coco_detection(SHAPE, pred_frames, categories,
name_mapping,
ignore_mapping)["annotations"]
coco_dt = coco_gt.loadRes(pred_res)
cat_ids = coco_dt.getCatIds()
cat_names = [cat["name"] for cat in coco_dt.loadCats(cat_ids)]
img_ids = sorted(coco_gt.getImgIds())
ann_type = "bbox"
coco_eval = COCOeval(coco_gt, coco_dt, ann_type)
coco_eval.params.imgIds = img_ids
return evaluate_workflow(coco_eval, cat_ids, cat_names, out_dir)
def test_reorder_preds(self) -> None:
"""Test reorder_preds function."""
pred_file = f"{self.cur_dir}/testcases/utils/preds.json"
pred_frames = load(pred_file).frames
gt_file = f"{self.cur_dir}/testcases/utils/gts.json"
gt_frames = load(gt_file).frames
pred_frames = reorder_preds(gt_frames, pred_frames)
self.assertEqual(len(pred_frames), len(gt_frames))
self.assertEqual(len(set(f.videoName for f in pred_frames)), 2)
for frame in pred_frames:
assert frame.labels is not None
self.assertGreater(len(frame.labels), 0)
def test_combine_stuff_masks(self) -> None:
"""Test combine_stuff_masks function."""
pred_file = f"{self.cur_dir}/testcases/utils/overlap_masks.json"
config = load_label_config(
f"{self.cur_dir}/testcases/pan_seg/pan_seg_configs.toml"
)
pred_frames = load(pred_file).frames
categories = config.categories
category_names = [category.name for category in categories]
rles, cids, iids = [], [], []
assert pred_frames[0].labels is not None
for label in pred_frames[0].labels:
assert label.category is not None
assert label.rle is not None
rles.append(label.rle)
cids.append(category_names.index(label.category))
iids.append(int(label.id))
out_rle, out_cid, out_iid = combine_stuff_masks(
rles, cids, iids, categories
)
self.assertEqual(len(out_rle), 3)
self.assertEqual(len(out_cid), 3)
self.assertEqual(len(out_iid), 3)
self.assertEqual(out_cid, [0, 34, 34])
self.assertEqual(len(out_iid), len(set(out_iid)))
def task_specific_test(
self,
task_name: str,
file_name: str,
output_name: str,
convert_func: Callable[[List[Frame], str, Config, int], None],
) -> None:
"""General test function for different tasks."""
cur_dir = os.path.dirname(os.path.abspath(__file__))
if task_name == "pan_seg":
json_path = (
f"{cur_dir}/testcases/panseg_bdd100k/panseg_bdd100k.json")
else:
json_path = f"{cur_dir}/testcases/example_annotation.json"
frames = load(json_path).frames
bdd100k_config = load_bdd100k_config(task_name)
convert_func(frames, self.test_out, bdd100k_config.scalabel, 1)
output_path = os.path.join(self.test_out, output_name)
mask: NDArrayU8 = np.asarray(Image.open(output_path), dtype=np.uint8)
gt_mask: NDArrayU8 = np.asarray(
Image.open(f"{cur_dir}/testcases/{file_name}"),
dtype=np.uint8,
)
self.assertTrue((mask == gt_mask).all())
def main() -> None:
"""Main function."""
args = parse_args()
if args.only_mask:
assert args.mode in ["ins_seg", "seg_track"]
convert_function = dict(
ins_seg=bitmask2coco_ins_seg,
seg_track=bitmask2coco_seg_track,
)[args.mode]
cfg_path = args.config if args.config is not None else args.mode
bdd100k_config = load_bdd100k_config(cfg_path)
logger.info("Start format converting...")
coco = convert_function(args.input, bdd100k_config.scalabel,
args.nproc)
else:
logger.info("Loading annotations...")
dataset = load(args.input, args.nproc)
if args.config is not None:
bdd100k_config = load_bdd100k_config(args.config)
elif dataset.config is not None:
bdd100k_config = BDD100KConfig(config=dataset.config)
else:
bdd100k_config = load_bdd100k_config(args.mode)
if args.mode in ["det", "box_track", "pose"]:
convert_func = dict(
det=scalabel2coco_detection,
box_track=scalabel2coco_box_track,
pose=scalabel2coco_pose,
)[args.mode]
else:
if args.mask_base is not None:
convert_func = partial(
dict(
ins_seg=bdd100k2coco_ins_seg,
seg_track=bdd100k2coco_seg_track,
)[args.mode],
mask_base=args.mask_base,
nproc=args.nproc,
)
else:
convert_func = partial(
dict(
ins_seg=scalabel2coco_ins_seg,
seg_track=scalabel2coco_seg_track,
)[args.mode],
nproc=args.nproc,
)
logger.info("Start format converting...")
frames = bdd100k_to_scalabel(dataset.frames, bdd100k_config)
coco = convert_func(frames=frames, config=bdd100k_config.scalabel)
logger.info("Saving converted annotations to disk...")
with open(args.output, "w", encoding="utf-8") as f:
json.dump(coco, f)
logger.info("Finished!")
def test_check_overlap(self) -> None:
"""Test check_overlap function."""
pred_file = f"{self.cur_dir}/testcases/utils/overlap_masks.json"
config = load_label_config(
f"{self.cur_dir}/testcases/pan_seg/pan_seg_configs.toml"
)
pred_frames = load(pred_file).frames
self.assertTrue(check_overlap(pred_frames, config, nproc=1))
self.assertFalse(check_overlap(pred_frames, config, nproc=1))
def start_converting(args: argparse.Namespace) -> List[Frame]:
"""Logs settings and load annoatations."""
logger.info(
"Mode: %s\nremove-ignore: %s\nignore-as-class: %s",
args.mode,
args.remove_ignore,
args.ignore_as_class,
)
logger.info("Loading annotations...")
labels = load(args.label, args.nproc)
logger.info("Start format converting...")
return labels
class TestRenderResults(unittest.TestCase):
"""Test cases for mot render results."""
cur_dir = os.path.dirname(os.path.abspath(__file__))
gts = load("{}/testcases/track_sample_anns.json".format(cur_dir))
preds = load("{}/testcases/track_predictions.json".format(cur_dir))
metrics = list(METRIC_MAPS.keys())
accs = [acc_single_video_mot(gts, preds)]
names, accs, items = aggregate_accs(accs)
summaries = [
evaluate_single_class(name, acc) for name, acc in zip(names, accs)
]
eval_results = render_results(summaries, items, metrics)
def test_categories(self) -> None:
"""Check the correctness of the 1st-level keys in eval_results."""
cate_names = ["OVERALL"]
for super_category, categories in SUPER_CLASSES.items():
cate_names.append(super_category)
cate_names.extend(categories)
self.assertEqual(len(self.eval_results), len(cate_names))
for key in self.eval_results:
self.assertIn(key, cate_names)
def test_metrics(self) -> None:
"""Check the correctness of the 2nd-level keys in eval_results."""
cate_metrics = list(METRIC_MAPS.values())
overall_metrics = cate_metrics + ["mIDF1", "mMOTA", "mMOTP"]
for cate, metrics in self.eval_results.items():
if cate == "OVERALL":
target_metrics = overall_metrics
else:
target_metrics = cate_metrics
self.assertEqual(len(metrics), len(target_metrics))
for metric in metrics:
self.assertIn(metric, target_metrics)
def test_label_ids_to_int(self) -> None:
"""Test label_ids_to_int function."""
gt_file = f"{self.cur_dir}/testcases/pan_seg/pan_seg_sample.json"
gt_frames = load(gt_file).frames[0]
assert gt_frames.labels is not None
old_ids = [label.id for label in gt_frames.labels]
label_ids_to_int([gt_frames])
new_ids = [label.id for label in gt_frames.labels]
self.assertEqual(len(old_ids), len(new_ids))
self.assertEqual(len(new_ids), len(set(new_ids)))
for new_id in new_ids:
self.assertTrue(isinstance(new_id, str))
self.assertTrue(new_id.isdigit())
def run() -> None:
"""Main."""
args = parse_args()
if args.task == "drivable":
evaluate_drivable(args.gt, args.result)
elif args.task == "lane_mark":
evaluate_lane_marking(args.gt, args.result)
elif args.task == "sem_seg":
evaluate_segmentation(args.gt, args.result)
elif args.task == "det":
evaluate_det(
args.gt, args.result, args.config, args.out_dir, args.nproc
)
elif args.task == "ins_seg":
evaluate_ins_seg(args.gt, args.result, args.score_file, args.out_dir)
elif args.task == "box_track":
evaluate_track(
acc_single_video_mot,
gts=group_and_sort(load(args.gt, args.nproc)),
results=group_and_sort(load(args.result, args.nproc)),
iou_thr=args.mot_iou_thr,
ignore_iof_thr=args.mot_ignore_iof_thr,
nproc=args.nproc,
)
elif args.task == "seg_track":
evaluate_track(
acc_single_video_mots,
gts=group_and_sort_files(
list_files(args.gt, ".png", with_prefix=True)
),
results=group_and_sort_files(
list_files(args.result, ".png", with_prefix=True)
),
iou_thr=args.mot_iou_thr,
ignore_iof_thr=args.mot_ignore_iof_thr,
nproc=args.nproc,
)
def test_summary(self) -> None:
"""Check evaluation scores' correctnes."""
gt_frames = load(self.gt_file).frames
pred_frames = load(self.pred_file).frames
result = evaluate_pan_seg(gt_frames, pred_frames, self.config, nproc=1)
summary = result.summary()
gt_summary = {
"PQ": 0.0,
"PQ/STUFF": 0.0,
"PQ/THING": 0.0,
"SQ": 0.0,
"SQ/STUFF": 0.0,
"SQ/THING": 0.0,
"RQ": 0.0,
"RQ/STUFF": 0.0,
"RQ/THING": 0.0,
"NUM": 15,
"NUM/STUFF": 12,
"NUM/THING": 3,
}
self.assertSetEqual(set(summary.keys()), set(gt_summary.keys()))
for name, score in gt_summary.items():
self.assertAlmostEqual(score, summary[name])
def test_summary(self) -> None:
"""Check evaluation scores' correctnes."""
gt_frames = load(self.gt_file).frames
pred_frames = load(self.pred_file).frames
result = evaluate_pan_seg(gt_frames, pred_frames, self.config, nproc=1)
summary = result.summary()
gt_summary = {
"PQ": 26.671487325762055,
"PQ/STUFF": 32.04431646856032,
"PQ/THING": 5.180170754569009,
"SQ": 30.815623929417264,
"SQ/STUFF": 32.04431646856032,
"SQ/THING": 25.90085377284505,
"RQ": 34.666666666666664,
"RQ/STUFF": 41.666666666666664,
"RQ/THING": 6.666666666666667,
"NUM": 15,
"NUM/STUFF": 12,
"NUM/THING": 3,
}
self.assertSetEqual(set(summary.keys()), set(gt_summary.keys()))
for name, score in gt_summary.items():
self.assertAlmostEqual(score, summary[name])
def test_summary(self) -> None:
"""Check evaluation scores' correctnes."""
gt_frames = load(self.gt_file).frames
pred_frames = load(self.pred_file).frames
result = evaluate_pan_seg(gt_frames, pred_frames, self.config, nproc=1)
summary = result.summary()
gt_summary = {
"PQ": 17.780991550508038,
"PQ/STUFF": 21.362877645706877,
"PQ/THING": 3.4534471697126734,
"SQ": 30.815623929417264,
"SQ/STUFF": 32.04431646856032,
"SQ/THING": 25.90085377284505,
"RQ": 23.111111111111104,
"RQ/STUFF": 27.77777777777777,
"RQ/THING": 4.444444444444445,
"NUM": 15,
"NUM/STUFF": 12,
"NUM/THING": 3,
}
self.assertSetEqual(set(summary.keys()), set(gt_summary.keys()))
for name, score in gt_summary.items():
self.assertAlmostEqual(score, summary[name])
def test_check_overlap_frame(self) -> None:
"""Test check_overlap_frame function."""
pred_files = [
f"{self.cur_dir}/testcases/utils/overlap_masks.json",
f"{self.cur_dir}/testcases/ins_seg/ins_seg_preds.json",
f"{self.cur_dir}/testcases/sem_seg/sem_seg_preds.json",
f"{self.cur_dir}/testcases/seg_track/seg_track_preds.json",
f"{self.cur_dir}/testcases/pan_seg/pan_seg_preds.json",
]
overlaps = [True, False, False, False, False]
for pred_file, is_overlap in zip(pred_files, overlaps):
pred_frames = load(pred_file).frames
self.assertEqual(
check_overlap_frame(pred_frames[0], ["car"]), is_overlap
)
def test_insseg_to_rle(self) -> None:
"""Test ins_seg to rle conversion."""
mask_dir = "./testcases/to_rle/ins_seg/masks"
score = "./testcases/to_rle/ins_seg/scores.json"
categories = get_leaf_categories(
load_bdd100k_config("ins_seg").scalabel.categories
)
frame = load(score).frames[0]
assert frame.labels is not None
new_frame = insseg_to_rle(frame, mask_dir, categories)
assert new_frame.labels is not None
self.assertEqual(len(new_frame.labels), 22)
for i, label in enumerate(new_frame.labels):
self.assertEqual(label.score, frame.labels[i].score)
def task_specific_test(
self,
file_name: str,
output_name: str,
convert_func: Callable[[List[Frame], str, bool, bool, int], None],
) -> None:
"""General test function for different tasks."""
cur_dir = os.path.dirname(os.path.abspath(__file__))
labels = load("{}/testcases/example_annotation.json".format(cur_dir))
convert_func(labels, self.test_out, False, False, 1)
output_path = os.path.join(self.test_out, output_name)
mask = np.asarray(Image.open(output_path))
gt_mask = np.asarray(
Image.open("{}/testcases/{}".format(cur_dir, file_name))
)
self.assertTrue((mask == gt_mask).all())
def main() -> None:
"""Main function."""
args = parse_args()
assert args.mode in [
"sem_seg",
"drivable",
"lane_mark",
"ins_seg",
"pan_seg",
"seg_track",
]
os.environ["QT_QPA_PLATFORM"] = "offscreen" # matplotlib offscreen render
convert_funcs: Dict[str, ToMasksFunc] = dict(
sem_seg=semseg_to_masks,
drivable=drivable_to_masks,
lane_mark=lanemark_to_masks,
pan_seg=panseg_to_bitmasks,
ins_seg=insseg_to_bitmasks,
seg_track=segtrack_to_bitmasks,
)
dataset = load(args.input, args.nproc)
if args.config is not None:
bdd100k_config = load_bdd100k_config(args.config)
elif dataset.config is not None:
bdd100k_config = BDD100KConfig(config=dataset.config)
else:
bdd100k_config = load_bdd100k_config(args.mode)
if args.mode in ["ins_seg", "seg_track"]:
frames = bdd100k_to_scalabel(dataset.frames, bdd100k_config)
else:
frames = dataset.frames
convert_funcs[args.mode](frames, args.output, bdd100k_config.scalabel,
args.nproc)
logger.info("Finished!")
def test_bdd100k_to_scalabel(self) -> None:
"""Test bdd100k_to_scalabel function."""
cur_dir = os.path.dirname(os.path.abspath(__file__))
dataset = load(f"{cur_dir}/testcases/example_ignore_annotation.json")
frames = dataset.frames
bdd100k_config = load_bdd100k_config("box_track")
new_frames = bdd100k_to_scalabel(copy.deepcopy(frames), bdd100k_config)
self.assertEqual(len(new_frames), 2)
labels = new_frames[0].labels
assert labels is not None
self.assertEqual(len(labels), 2)
self.assertEqual(labels[0].category, "pedestrian")
self.assertEqual(labels[1].category, "pedestrian")
assert labels[0].attributes is not None
self.assertTrue(labels[0].attributes[IGNORED])
self.assertEqual(new_frames[1].labels, None)
bdd100k_config.remove_ignored = True
new_frames = bdd100k_to_scalabel(frames, bdd100k_config)
labels = new_frames[0].labels
assert labels is not None
self.assertEqual(len(labels), 1)
self.assertEqual(labels[0].category, "pedestrian")
action="store_true",
help="ignore unknown categories for mots evaluation",
)
parser.add_argument(
"--nproc",
"-p",
type=int,
default=NPROC,
help="number of processes for mots evaluation",
)
return parser.parse_args()
if __name__ == "__main__":
args = parse_arguments()
dataset = load(args.gt)
gt_frames, cfg = dataset.frames, dataset.config
if args.config is not None:
cfg = load_label_config(args.config)
if cfg is None:
raise ValueError("Dataset config is not specified. Please use --config"
" to specify a config for this dataset.")
eval_result = evaluate_seg_track(
acc_single_video_mots,
group_and_sort(gt_frames),
group_and_sort(load(args.result).frames),
cfg,
args.iou_thr,
args.ignore_iof_thr,
args.ignore_unknown_cats,
args.nproc,
"--ignore-unknown-cats",
action="store_true",
help="ignore unknown categories for panseg evaluation",
)
parser.add_argument(
"--nproc",
"-p",
type=int,
default=NPROC,
help="number of processes for panseg evaluation",
)
return parser.parse_args()
if __name__ == "__main__":
args = parse_arguments()
dataset = load(args.gt, args.nproc)
gts, cfg = dataset.frames, dataset.config
preds = load(args.result).frames
if args.config is not None:
cfg = load_label_config(args.config)
assert cfg is not None
eval_result = evaluate_pan_seg(
gts, preds, cfg, args.ignore_unknown_cats, args.nproc
)
logger.info(eval_result)
logger.info(eval_result.summary())
if args.out_file:
with open_write_text(args.out_file) as fp:
json.dump(eval_result.dict(), fp, indent=2)
