def check_overlap(frames: List[Frame],
config: Config,
nproc: int = NPROC) -> bool:
"""Check overlap of segmentation masks.
Returns True if overlap found in masks of any frame.
"""
categories = get_leaf_categories(config.categories)
category_names = [category.name for category in categories]
if nproc > 1:
with Pool(nproc) as pool:
overlaps = pool.map(
partial(
check_overlap_frame,
categories=category_names,
image_size=config.imageSize,
),
tqdm(frames),
)
else:
overlaps = [
check_overlap_frame(frame, category_names, config.imageSize)
for frame in tqdm(frames)
]
for is_overlap, frame in zip(overlaps, frames):
if is_overlap:
# remove predictions with overlap
frame.labels = None
return any(overlaps)
def main() -> None:
"""Main."""
args = parse_args()
assert os.path.isdir(args.input)
dataset = load(args.input, args.nproc)
if args.config is not None:
bdd100k_config = load_bdd100k_config(args.config)
elif dataset.config is not None:
bdd100k_config = BDD100KConfig(config=dataset.config)
else:
bdd100k_config = load_bdd100k_config(args.mode)
categories = get_leaf_categories(bdd100k_config.scalabel.categories)
convert_funcs: Dict[str, ToRLEFunc] = dict(
ins_seg=insseg_to_rle,
sem_seg=semseg_to_rle,
drivable=semseg_to_rle,
seg_track=segtrack_to_rle,
)
if args.mode == "ins_seg":
assert args.score_file is not None
frames = load(args.score_file).frames
assert all(
os.path.exists(
os.path.join(args.input, frame.name.replace(".jpg", ".png")))
for frame in frames), "Missing some bitmasks."
elif args.mode in ("sem_seg", "drivable", "seg_track"):
files = list_files(args.input)
frames = []
for file in files:
if not file.endswith(".png") and not file.endswith(".jpg"):
continue
frame = Frame(name=file.replace(".png", ".jpg"), labels=[])
frames.append(frame)
else:
return
if args.nproc > 1:
with Pool(args.nproc) as pool:
frames = pool.map(
partial(
convert_funcs[args.mode],
input_dir=args.input,
categories=categories,
),
tqdm(frames),
)
else:
frames = [
convert_funcs[args.mode](frame, args.input, categories)
for frame in tqdm(frames)
]
save(args.output, frames)
def test_semseg_to_rle(self) -> None:
"""Test sem_seg to rle conversion."""
mask_dir = "./testcases/to_rle/sem_seg/masks"
mask = "0.jpg"
categories = get_leaf_categories(
load_bdd100k_config("sem_seg").scalabel.categories
)
frame = Frame(name=mask)
new_frame = semseg_to_rle(frame, mask_dir, categories)
assert new_frame.labels is not None
self.assertEqual(len(new_frame.labels), 11)
def bdd100k_to_scalabel(frames: List[Frame],
bdd100k_config: BDD100KConfig) -> List[Frame]:
"""Converting BDD100K to Scalabel format."""
categories = get_leaf_categories(bdd100k_config.scalabel.categories)
cat_name2id = {cat.name: i + 1 for i, cat in enumerate(categories)}
for image_anns in tqdm(frames):
if image_anns.labels is not None:
for i in reversed(range(len(image_anns.labels))):
label = deal_bdd100k_category(image_anns.labels[i],
bdd100k_config, cat_name2id)
if label is None:
image_anns.labels.pop(i)
return frames
def test_insseg_to_rle(self) -> None:
"""Test ins_seg to rle conversion."""
mask_dir = "./testcases/to_rle/ins_seg/masks"
score = "./testcases/to_rle/ins_seg/scores.json"
categories = get_leaf_categories(
load_bdd100k_config("ins_seg").scalabel.categories
)
frame = load(score).frames[0]
assert frame.labels is not None
new_frame = insseg_to_rle(frame, mask_dir, categories)
assert new_frame.labels is not None
self.assertEqual(len(new_frame.labels), 22)
for i, label in enumerate(new_frame.labels):
self.assertEqual(label.score, frame.labels[i].score)
def test_segtrack_to_rle(self) -> None:
"""Test seg_track to rle conversion."""
mask_dir = "./testcases/to_rle/seg_track/masks"
masks = ["0/0-1.jpg", "0/0-2.jpg"]
categories = get_leaf_categories(
load_bdd100k_config("seg_track").scalabel.categories
)
frames = [Frame(name=mask) for mask in masks]
new_frames = [
segtrack_to_rle(frame, mask_dir, categories) for frame in frames
]
for i in range(2):
labels = new_frames[i].labels
assert labels is not None
self.assertEqual(len(labels), 10)
self.assertEqual(new_frames[i].videoName, "0")
self.assertEqual(new_frames[i].frameIndex, i)
def evaluate_pan_seg(
ann_frames: List[Frame],
pred_frames: List[Frame],
config: Config,
ignore_unknown_cats: bool = False,
nproc: int = NPROC,
) -> PanSegResult:
"""Evaluate panoptic segmentation with Scalabel format.
Args:
ann_frames: the ground truth frames.
pred_frames: the prediction frames.
config: Metadata config.
ignore_unknown_cats: ignore unknown categories.
nproc: the number of process.
Returns:
PanSegResult: evaluation results.
"""
categories = get_leaf_categories(config.categories)
assert all(
category.isThing is not None for category in categories
), "isThing should be defined for all categories for PanSeg."
categories_stuff = [
category for category in categories if not category.isThing
]
categories_thing = [
category for category in categories if category.isThing
]
category_names = [category.name for category in categories]
pred_frames = reorder_preds(ann_frames, pred_frames)
label_ids_to_int(ann_frames)
# check overlap of masks
logger.info("checking for overlap of masks...")
if check_overlap(pred_frames, config, nproc):
logger.critical(
"Found overlap in prediction bitmasks, but panoptic segmentation "
"evaluation does not allow overlaps. Removing such predictions."
)
logger.info("evaluating...")
if nproc > 1:
with Pool(nproc) as pool:
pq_stats = pool.starmap(
partial(
pq_per_image,
categories=categories,
ignore_unknown_cats=ignore_unknown_cats,
image_size=config.imageSize,
),
tqdm(zip(ann_frames, pred_frames), total=len(ann_frames)),
)
else:
pq_stats = [
pq_per_image(
ann_frame,
pred_frame,
categories=categories,
ignore_unknown_cats=ignore_unknown_cats,
image_size=config.imageSize,
)
for ann_frame, pred_frame in tqdm(
zip(ann_frames, pred_frames), total=len(ann_frames)
)
]
pq_stat = PQStat(categories)
for pq_stat_ in pq_stats:
pq_stat += pq_stat_
logger.info("accumulating...")
res_dict: Dict[str, ScoresList] = {}
for category_name, category in zip(category_names, categories):
result = pq_stat.pq_average([category])
for metric, score in result.items():
if metric not in res_dict:
res_dict[metric] = [{}, {}, {}]
res_dict[metric][0][category_name] = score
result = pq_stat.pq_average(categories_stuff)
for metric, score in result.items():
res_dict[metric][1][STUFF] = score
result = pq_stat.pq_average(categories_thing)
for metric, score in result.items():
res_dict[metric][1][THING] = score
result = pq_stat.pq_average(categories)
for metric, score in result.items():
res_dict[metric][2][OVERALL] = score
return PanSegResult(**res_dict)
def evaluate_seg_track(
gts: FilesList,
results: FilesList,
config: Config,
iou_thr: float = 0.5,
ignore_iof_thr: float = 0.5,
nproc: int = NPROC,
) -> TrackResult:
"""Evaluate CLEAR MOT metrics for MOTS.
Args:
gts: the ground truth annotation files.
results: the prediction result files.
config: Config object
iou_thr: Minimum IoU for a bounding box to be considered a positive.
ignore_iof_thr: Min. Intersection over foreground with ignore regions.
nproc: processes number for loading files
Returns:
TrackResult: evaluation results.
"""
logger.info("Tracking evaluation with CLEAR MOT metrics.")
t = time.time()
assert len(gts) == len(results)
classes = get_leaf_categories(config.categories)
super_classes = get_parent_categories(config.categories)
logger.info("evaluating...")
class_names = [c.name for c in classes]
if nproc > 1:
with Pool(nproc) as pool:
video_accs = pool.starmap(
partial(
acc_single_video_mots,
classes=class_names,
ignore_iof_thr=ignore_iof_thr,
),
zip(gts, results),
)
else:
video_accs = [
acc_single_video_mots(
gt, result, class_names, iou_thr, ignore_iof_thr
)
for gt, result in zip(gts, results)
]
class_names, metric_names, class_accs = aggregate_accs(
video_accs, classes, super_classes
)
logger.info("accumulating...")
if nproc > 1:
with Pool(nproc) as pool:
flat_dicts = pool.starmap(
evaluate_single_class, zip(metric_names, class_accs)
)
else:
flat_dicts = [
evaluate_single_class(names, accs)
for names, accs in zip(metric_names, class_accs)
]
metrics = list(METRIC_MAPS.values())
result = generate_results(
flat_dicts, class_names, metrics, classes, super_classes
)
t = time.time() - t
logger.info("evaluation finishes with %.1f s.", t)
return result
def segtrack_to_bitmasks(frames: List[Frame],
out_base: str,
config: Config,
nproc: int = NPROC) -> None:
"""Converting segmentation tracking poly2d to bitmasks."""
frames_list = group_and_sort(frames)
img_shape = config.imageSize
out_paths: List[str] = []
shapes: List[ImageSize] = []
colors_list: List[List[NDArrayU8]] = []
poly2ds_list: List[List[List[Poly2D]]] = []
categories = get_leaf_categories(config.categories)
cat_name2id = {cat.name: i + 1 for i, cat in enumerate(categories)}
logger.info("Preparing annotations for SegTrack to Bitmasks")
for video_anns in tqdm(frames_list):
global_instance_id: int = 1
instance_id_maps: Dict[str, int] = {}
video_name = video_anns[0].videoName
out_dir = os.path.join(out_base, video_name)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
for image_anns in video_anns:
# Bitmask in .png format
image_name = image_anns.name.replace(".jpg", ".png")
image_name = os.path.split(image_name)[-1]
out_path = os.path.join(out_dir, image_name)
out_paths.append(out_path)
if img_shape is None:
if image_anns.size is not None:
img_shape = image_anns.size
else:
raise ValueError("Image shape not defined!")
shapes.append(img_shape)
colors: List[NDArrayU8] = []
poly2ds: List[List[Poly2D]] = []
colors_list.append(colors)
poly2ds_list.append(poly2ds)
labels_ = image_anns.labels
if labels_ is None or len(labels_) == 0:
continue
# Scores higher, rendering later
if labels_[0].score is not None:
labels_ = sorted(labels_, key=lambda label: float(label.score))
for label in labels_:
if label.poly2d is None:
continue
if label.category not in cat_name2id:
continue
instance_id, global_instance_id = get_bdd100k_instance_id(
instance_id_maps, global_instance_id, label.id)
category_id = cat_name2id[label.category]
color = set_instance_color(label, category_id, instance_id)
colors.append(color)
poly2ds.append(label.poly2d)
logger.info("Start Conversion for SegTrack to Bitmasks")
frames_to_masks(nproc, out_paths, shapes, colors_list, poly2ds_list)
def insseg_to_bitmasks(frames: List[Frame],
out_base: str,
config: Config,
nproc: int = NPROC) -> None:
"""Converting instance segmentation poly2d to bitmasks."""
os.makedirs(out_base, exist_ok=True)
img_shape = config.imageSize
out_paths: List[str] = []
shapes: List[ImageSize] = []
colors_list: List[List[NDArrayU8]] = []
poly2ds_list: List[List[List[Poly2D]]] = []
categories = get_leaf_categories(config.categories)
cat_name2id = {cat.name: i + 1 for i, cat in enumerate(categories)}
logger.info("Preparing annotations for InsSeg to Bitmasks")
for image_anns in tqdm(frames):
ann_id = 0
# Bitmask in .png format
image_name = image_anns.name.replace(".jpg", ".png")
image_name = os.path.split(image_name)[-1]
out_path = os.path.join(out_base, image_name)
out_paths.append(out_path)
if img_shape is None:
if image_anns.size is not None:
img_shape = image_anns.size
else:
raise ValueError("Image shape not defined!")
shapes.append(img_shape)
colors: List[NDArrayU8] = []
poly2ds: List[List[Poly2D]] = []
colors_list.append(colors)
poly2ds_list.append(poly2ds)
labels_ = image_anns.labels
if labels_ is None or len(labels_) == 0:
continue
# Scores higher, rendering later
if labels_[0].score is not None:
labels_ = sorted(labels_, key=lambda label: float(label.score))
for label in labels_:
if label.poly2d is None:
continue
if label.category not in cat_name2id:
continue
ann_id += 1
category_id = cat_name2id[label.category]
color = set_instance_color(label, category_id, ann_id)
colors.append(color)
poly2ds.append(label.poly2d)
logger.info("Start conversion for InsSeg to Bitmasks")
frames_to_masks(nproc, out_paths, shapes, colors_list, poly2ds_list)
def evaluate_sem_seg(
ann_frames: List[Frame],
pred_frames: List[Frame],
config: Config,
nproc: int = NPROC,
) -> SegResult:
"""Evaluate segmentation with Scalabel format.
Args:
ann_frames: the ground truth frames.
pred_frames: the prediction frames.
config: Metadata config.
nproc: the number of process.
Returns:
SegResult: evaluation results.
"""
categories = {
cat.name: id
for id, cat in enumerate(get_leaf_categories(config.categories))
}
ignore_label = 255
pred_frames = reorder_preds(ann_frames, pred_frames)
logger.info("evaluating...")
if nproc > 1:
with Pool(nproc) as pool:
hist_and_gt_id_sets = pool.starmap(
partial(
per_image_hist,
categories=categories,
image_size=config.imageSize,
ignore_label=ignore_label,
),
tqdm(zip(ann_frames, pred_frames), total=len(ann_frames)),
)
else:
hist_and_gt_id_sets = [
per_image_hist(
ann_frame,
pred_frame,
categories=categories,
image_size=config.imageSize,
ignore_label=ignore_label,
)
for ann_frame, pred_frame in tqdm(
zip(ann_frames, pred_frames), total=len(ann_frames)
)
]
logger.info("accumulating...")
num_classes = len(categories) + 1
hist: NDArrayI32 = np.zeros((num_classes, num_classes), dtype=np.int32)
gt_id_set = set()
for (hist_, gt_id_set_) in hist_and_gt_id_sets:
hist += hist_
gt_id_set.update(gt_id_set_)
ious = per_class_iou(hist)
accs = per_class_acc(hist)
IoUs = [ # pylint: disable=invalid-name
{cat_name: 100 * score for cat_name, score in zip(categories, ious)},
{AVERAGE: np.multiply(np.mean(ious[list(gt_id_set)]), 100)},
]
Accs = [ # pylint: disable=invalid-name
{cat_name: 100 * score for cat_name, score in zip(categories, accs)},
{AVERAGE: np.multiply(np.mean(accs[list(gt_id_set)]), 100)},
]
res_dict: Dict[str, Union[float, ScoresList]] = dict(
IoU=IoUs,
Acc=Accs,
fIoU=np.multiply(freq_iou(hist), 100), # pylint: disable=invalid-name
pAcc=np.multiply(whole_acc(hist), 100), # pylint: disable=invalid-name
)
logger.info("GT id set [%s]", ",".join(str(s) for s in gt_id_set))
return SegResult(**res_dict)
def bdd100k2coco_seg_track(mask_base: str,
frames: List[Frame],
config: Config,
nproc: int = NPROC) -> GtType:
"""Converting BDD100K Segmentation Tracking Set to COCO format."""
video_id, image_id, ann_id = 0, 0, 0
img_shape = config.imageSize
frames_list = group_and_sort(frames)
videos: List[VidType] = []
images: List[ImgType] = []
mask_names: List[str] = []
category_ids_list: List[List[int]] = []
instance_ids_list: List[List[int]] = []
annotations_list: List[List[AnnType]] = []
categories = get_leaf_categories(config.categories)
cat_name2id = {cat.name: i + 1 for i, cat in enumerate(categories)}
logger.info("Collecting annotations...")
for video_anns in tqdm(frames_list):
global_instance_id: int = 1
instance_id_maps: Dict[str, int] = {}
video_name = video_anns[0].videoName
video_id += 1
video = VidType(id=video_id, name=video_name)
videos.append(video)
for image_anns in video_anns:
image_id += 1
if img_shape is None:
if image_anns.size is not None:
img_shape = image_anns.size
else:
raise ValueError("Image shape not defined!")
image = ImgType(
video_id=video_id,
frame_id=image_anns.frameIndex,
id=image_id,
file_name=os.path.join(video_name, image_anns.name),
height=img_shape.height,
width=img_shape.width,
)
if image_anns.url is not None:
image["coco_url"] = image_anns.url
images.append(image)
mask_name = os.path.join(
mask_base,
video_name,
# Bitmask in .png format, image in .jpg format
image_anns.name.replace(".jpg", ".png"),
)
mask_names.append(mask_name)
category_ids: List[int] = []
instance_ids: List[int] = []
annotations: List[AnnType] = []
for label in image_anns.labels:
if label.poly2d is None:
continue
if label.category not in cat_name2id:
continue
ann_id += 1
instance_id, global_instance_id = get_bdd100k_instance_id(
instance_id_maps, global_instance_id, label.id)
category_id = cat_name2id[label.category]
annotation = AnnType(
id=ann_id,
image_id=image_id,
instance_id=instance_id,
category_id=category_id,
scalabel_id=label.id,
iscrowd=int(check_crowd(label) or check_ignored(label)),
ignore=0,
)
category_ids.append(category_id)
instance_ids.append(instance_id)
annotations.append(annotation)
category_ids_list.append(category_ids)
instance_ids_list.append(instance_ids)
annotations_list.append(annotations)
annotations = bitmask2coco_with_ids_parallel(
annotations_list,
mask_names,
category_ids_list,
instance_ids_list,
nproc,
)
return GtType(
type="instances",
categories=get_coco_categories(config),
videos=videos,
images=images,
annotations=annotations,
)
def evaluate_seg_track(
acc_single_video: VidFunc,
gts: List[List[Frame]],
results: List[List[Frame]],
config: Config,
iou_thr: float = 0.5,
ignore_iof_thr: float = 0.5,
ignore_unknown_cats: bool = False,
nproc: int = NPROC,
) -> TrackResult:
"""Evaluate CLEAR MOT metrics for a Scalabel format dataset.
Args:
acc_single_video: Function for calculating metrics over a single video.
gts: the ground truth annotations in Scalabel format
results: the prediction results in Scalabel format.
config: Config object
iou_thr: Minimum IoU for a mask to be considered a positive.
ignore_iof_thr: Min. Intersection over foreground with ignore regions.
ignore_unknown_cats: if False, raise KeyError when trying to evaluate
unknown categories.
nproc: processes number for loading files
Returns:
TrackResult: evaluation results.
"""
logger.info("Tracking evaluation with CLEAR MOT metrics.")
t = time.time()
assert len(gts) == len(results)
# check overlap of masks
logger.info("checking for overlap of masks...")
if check_overlap([frame for res in results for frame in res], config,
nproc):
logger.critical(
"Found overlap in prediction bitmasks, but segmentation tracking "
"evaluation does not allow overlaps. Removing such predictions.")
classes = get_leaf_categories(config.categories)
super_classes = get_parent_categories(config.categories)
logger.info("evaluating...")
image_size = config.imageSize
if nproc > 1:
with Pool(nproc) as pool:
video_accs = pool.starmap(
partial(
acc_single_video,
classes=classes,
ignore_iof_thr=ignore_iof_thr,
ignore_unknown_cats=ignore_unknown_cats,
image_size=image_size,
),
zip(gts, results),
)
else:
video_accs = [
acc_single_video(
gt,
result,
classes,
iou_thr,
ignore_iof_thr,
ignore_unknown_cats,
image_size,
) for gt, result in zip(gts, results)
]
class_names, metric_names, class_accs = aggregate_accs(
video_accs, classes, super_classes)
logger.info("accumulating...")
if nproc > 1:
with Pool(nproc) as pool:
flat_dicts = pool.starmap(evaluate_single_class,
zip(metric_names, class_accs))
else:
flat_dicts = [
evaluate_single_class(names, accs)
for names, accs in zip(metric_names, class_accs)
]
metrics = list(METRIC_MAPS.values())
result = generate_results(flat_dicts, class_names, metrics, classes,
super_classes)
t = time.time() - t
logger.info("evaluation finishes with %.1f s.", t)
return result