def check_overlap(frames: List[Frame], config: Config, nproc: int = NPROC) -> bool: """Check overlap of segmentation masks. Returns True if overlap found in masks of any frame. """ categories = get_leaf_categories(config.categories) category_names = [category.name for category in categories] if nproc > 1: with Pool(nproc) as pool: overlaps = pool.map( partial( check_overlap_frame, categories=category_names, image_size=config.imageSize, ), tqdm(frames), ) else: overlaps = [ check_overlap_frame(frame, category_names, config.imageSize) for frame in tqdm(frames) ] for is_overlap, frame in zip(overlaps, frames): if is_overlap: # remove predictions with overlap frame.labels = None return any(overlaps)
def main() -> None: """Main.""" args = parse_args() assert os.path.isdir(args.input) dataset = load(args.input, args.nproc) if args.config is not None: bdd100k_config = load_bdd100k_config(args.config) elif dataset.config is not None: bdd100k_config = BDD100KConfig(config=dataset.config) else: bdd100k_config = load_bdd100k_config(args.mode) categories = get_leaf_categories(bdd100k_config.scalabel.categories) convert_funcs: Dict[str, ToRLEFunc] = dict( ins_seg=insseg_to_rle, sem_seg=semseg_to_rle, drivable=semseg_to_rle, seg_track=segtrack_to_rle, ) if args.mode == "ins_seg": assert args.score_file is not None frames = load(args.score_file).frames assert all( os.path.exists( os.path.join(args.input, frame.name.replace(".jpg", ".png"))) for frame in frames), "Missing some bitmasks." elif args.mode in ("sem_seg", "drivable", "seg_track"): files = list_files(args.input) frames = [] for file in files: if not file.endswith(".png") and not file.endswith(".jpg"): continue frame = Frame(name=file.replace(".png", ".jpg"), labels=[]) frames.append(frame) else: return if args.nproc > 1: with Pool(args.nproc) as pool: frames = pool.map( partial( convert_funcs[args.mode], input_dir=args.input, categories=categories, ), tqdm(frames), ) else: frames = [ convert_funcs[args.mode](frame, args.input, categories) for frame in tqdm(frames) ] save(args.output, frames)
def test_semseg_to_rle(self) -> None: """Test sem_seg to rle conversion.""" mask_dir = "./testcases/to_rle/sem_seg/masks" mask = "0.jpg" categories = get_leaf_categories( load_bdd100k_config("sem_seg").scalabel.categories ) frame = Frame(name=mask) new_frame = semseg_to_rle(frame, mask_dir, categories) assert new_frame.labels is not None self.assertEqual(len(new_frame.labels), 11)
def bdd100k_to_scalabel(frames: List[Frame], bdd100k_config: BDD100KConfig) -> List[Frame]: """Converting BDD100K to Scalabel format.""" categories = get_leaf_categories(bdd100k_config.scalabel.categories) cat_name2id = {cat.name: i + 1 for i, cat in enumerate(categories)} for image_anns in tqdm(frames): if image_anns.labels is not None: for i in reversed(range(len(image_anns.labels))): label = deal_bdd100k_category(image_anns.labels[i], bdd100k_config, cat_name2id) if label is None: image_anns.labels.pop(i) return frames
def test_insseg_to_rle(self) -> None: """Test ins_seg to rle conversion.""" mask_dir = "./testcases/to_rle/ins_seg/masks" score = "./testcases/to_rle/ins_seg/scores.json" categories = get_leaf_categories( load_bdd100k_config("ins_seg").scalabel.categories ) frame = load(score).frames[0] assert frame.labels is not None new_frame = insseg_to_rle(frame, mask_dir, categories) assert new_frame.labels is not None self.assertEqual(len(new_frame.labels), 22) for i, label in enumerate(new_frame.labels): self.assertEqual(label.score, frame.labels[i].score)
def test_segtrack_to_rle(self) -> None: """Test seg_track to rle conversion.""" mask_dir = "./testcases/to_rle/seg_track/masks" masks = ["0/0-1.jpg", "0/0-2.jpg"] categories = get_leaf_categories( load_bdd100k_config("seg_track").scalabel.categories ) frames = [Frame(name=mask) for mask in masks] new_frames = [ segtrack_to_rle(frame, mask_dir, categories) for frame in frames ] for i in range(2): labels = new_frames[i].labels assert labels is not None self.assertEqual(len(labels), 10) self.assertEqual(new_frames[i].videoName, "0") self.assertEqual(new_frames[i].frameIndex, i)
def evaluate_pan_seg( ann_frames: List[Frame], pred_frames: List[Frame], config: Config, ignore_unknown_cats: bool = False, nproc: int = NPROC, ) -> PanSegResult: """Evaluate panoptic segmentation with Scalabel format. Args: ann_frames: the ground truth frames. pred_frames: the prediction frames. config: Metadata config. ignore_unknown_cats: ignore unknown categories. nproc: the number of process. Returns: PanSegResult: evaluation results. """ categories = get_leaf_categories(config.categories) assert all( category.isThing is not None for category in categories ), "isThing should be defined for all categories for PanSeg." categories_stuff = [ category for category in categories if not category.isThing ] categories_thing = [ category for category in categories if category.isThing ] category_names = [category.name for category in categories] pred_frames = reorder_preds(ann_frames, pred_frames) label_ids_to_int(ann_frames) # check overlap of masks logger.info("checking for overlap of masks...") if check_overlap(pred_frames, config, nproc): logger.critical( "Found overlap in prediction bitmasks, but panoptic segmentation " "evaluation does not allow overlaps. Removing such predictions." ) logger.info("evaluating...") if nproc > 1: with Pool(nproc) as pool: pq_stats = pool.starmap( partial( pq_per_image, categories=categories, ignore_unknown_cats=ignore_unknown_cats, image_size=config.imageSize, ), tqdm(zip(ann_frames, pred_frames), total=len(ann_frames)), ) else: pq_stats = [ pq_per_image( ann_frame, pred_frame, categories=categories, ignore_unknown_cats=ignore_unknown_cats, image_size=config.imageSize, ) for ann_frame, pred_frame in tqdm( zip(ann_frames, pred_frames), total=len(ann_frames) ) ] pq_stat = PQStat(categories) for pq_stat_ in pq_stats: pq_stat += pq_stat_ logger.info("accumulating...") res_dict: Dict[str, ScoresList] = {} for category_name, category in zip(category_names, categories): result = pq_stat.pq_average([category]) for metric, score in result.items(): if metric not in res_dict: res_dict[metric] = [{}, {}, {}] res_dict[metric][0][category_name] = score result = pq_stat.pq_average(categories_stuff) for metric, score in result.items(): res_dict[metric][1][STUFF] = score result = pq_stat.pq_average(categories_thing) for metric, score in result.items(): res_dict[metric][1][THING] = score result = pq_stat.pq_average(categories) for metric, score in result.items(): res_dict[metric][2][OVERALL] = score return PanSegResult(**res_dict)
def evaluate_seg_track( gts: FilesList, results: FilesList, config: Config, iou_thr: float = 0.5, ignore_iof_thr: float = 0.5, nproc: int = NPROC, ) -> TrackResult: """Evaluate CLEAR MOT metrics for MOTS. Args: gts: the ground truth annotation files. results: the prediction result files. config: Config object iou_thr: Minimum IoU for a bounding box to be considered a positive. ignore_iof_thr: Min. Intersection over foreground with ignore regions. nproc: processes number for loading files Returns: TrackResult: evaluation results. """ logger.info("Tracking evaluation with CLEAR MOT metrics.") t = time.time() assert len(gts) == len(results) classes = get_leaf_categories(config.categories) super_classes = get_parent_categories(config.categories) logger.info("evaluating...") class_names = [c.name for c in classes] if nproc > 1: with Pool(nproc) as pool: video_accs = pool.starmap( partial( acc_single_video_mots, classes=class_names, ignore_iof_thr=ignore_iof_thr, ), zip(gts, results), ) else: video_accs = [ acc_single_video_mots( gt, result, class_names, iou_thr, ignore_iof_thr ) for gt, result in zip(gts, results) ] class_names, metric_names, class_accs = aggregate_accs( video_accs, classes, super_classes ) logger.info("accumulating...") if nproc > 1: with Pool(nproc) as pool: flat_dicts = pool.starmap( evaluate_single_class, zip(metric_names, class_accs) ) else: flat_dicts = [ evaluate_single_class(names, accs) for names, accs in zip(metric_names, class_accs) ] metrics = list(METRIC_MAPS.values()) result = generate_results( flat_dicts, class_names, metrics, classes, super_classes ) t = time.time() - t logger.info("evaluation finishes with %.1f s.", t) return result
def segtrack_to_bitmasks(frames: List[Frame], out_base: str, config: Config, nproc: int = NPROC) -> None: """Converting segmentation tracking poly2d to bitmasks.""" frames_list = group_and_sort(frames) img_shape = config.imageSize out_paths: List[str] = [] shapes: List[ImageSize] = [] colors_list: List[List[NDArrayU8]] = [] poly2ds_list: List[List[List[Poly2D]]] = [] categories = get_leaf_categories(config.categories) cat_name2id = {cat.name: i + 1 for i, cat in enumerate(categories)} logger.info("Preparing annotations for SegTrack to Bitmasks") for video_anns in tqdm(frames_list): global_instance_id: int = 1 instance_id_maps: Dict[str, int] = {} video_name = video_anns[0].videoName out_dir = os.path.join(out_base, video_name) if not os.path.isdir(out_dir): os.makedirs(out_dir) for image_anns in video_anns: # Bitmask in .png format image_name = image_anns.name.replace(".jpg", ".png") image_name = os.path.split(image_name)[-1] out_path = os.path.join(out_dir, image_name) out_paths.append(out_path) if img_shape is None: if image_anns.size is not None: img_shape = image_anns.size else: raise ValueError("Image shape not defined!") shapes.append(img_shape) colors: List[NDArrayU8] = [] poly2ds: List[List[Poly2D]] = [] colors_list.append(colors) poly2ds_list.append(poly2ds) labels_ = image_anns.labels if labels_ is None or len(labels_) == 0: continue # Scores higher, rendering later if labels_[0].score is not None: labels_ = sorted(labels_, key=lambda label: float(label.score)) for label in labels_: if label.poly2d is None: continue if label.category not in cat_name2id: continue instance_id, global_instance_id = get_bdd100k_instance_id( instance_id_maps, global_instance_id, label.id) category_id = cat_name2id[label.category] color = set_instance_color(label, category_id, instance_id) colors.append(color) poly2ds.append(label.poly2d) logger.info("Start Conversion for SegTrack to Bitmasks") frames_to_masks(nproc, out_paths, shapes, colors_list, poly2ds_list)
def insseg_to_bitmasks(frames: List[Frame], out_base: str, config: Config, nproc: int = NPROC) -> None: """Converting instance segmentation poly2d to bitmasks.""" os.makedirs(out_base, exist_ok=True) img_shape = config.imageSize out_paths: List[str] = [] shapes: List[ImageSize] = [] colors_list: List[List[NDArrayU8]] = [] poly2ds_list: List[List[List[Poly2D]]] = [] categories = get_leaf_categories(config.categories) cat_name2id = {cat.name: i + 1 for i, cat in enumerate(categories)} logger.info("Preparing annotations for InsSeg to Bitmasks") for image_anns in tqdm(frames): ann_id = 0 # Bitmask in .png format image_name = image_anns.name.replace(".jpg", ".png") image_name = os.path.split(image_name)[-1] out_path = os.path.join(out_base, image_name) out_paths.append(out_path) if img_shape is None: if image_anns.size is not None: img_shape = image_anns.size else: raise ValueError("Image shape not defined!") shapes.append(img_shape) colors: List[NDArrayU8] = [] poly2ds: List[List[Poly2D]] = [] colors_list.append(colors) poly2ds_list.append(poly2ds) labels_ = image_anns.labels if labels_ is None or len(labels_) == 0: continue # Scores higher, rendering later if labels_[0].score is not None: labels_ = sorted(labels_, key=lambda label: float(label.score)) for label in labels_: if label.poly2d is None: continue if label.category not in cat_name2id: continue ann_id += 1 category_id = cat_name2id[label.category] color = set_instance_color(label, category_id, ann_id) colors.append(color) poly2ds.append(label.poly2d) logger.info("Start conversion for InsSeg to Bitmasks") frames_to_masks(nproc, out_paths, shapes, colors_list, poly2ds_list)
def evaluate_sem_seg( ann_frames: List[Frame], pred_frames: List[Frame], config: Config, nproc: int = NPROC, ) -> SegResult: """Evaluate segmentation with Scalabel format. Args: ann_frames: the ground truth frames. pred_frames: the prediction frames. config: Metadata config. nproc: the number of process. Returns: SegResult: evaluation results. """ categories = { cat.name: id for id, cat in enumerate(get_leaf_categories(config.categories)) } ignore_label = 255 pred_frames = reorder_preds(ann_frames, pred_frames) logger.info("evaluating...") if nproc > 1: with Pool(nproc) as pool: hist_and_gt_id_sets = pool.starmap( partial( per_image_hist, categories=categories, image_size=config.imageSize, ignore_label=ignore_label, ), tqdm(zip(ann_frames, pred_frames), total=len(ann_frames)), ) else: hist_and_gt_id_sets = [ per_image_hist( ann_frame, pred_frame, categories=categories, image_size=config.imageSize, ignore_label=ignore_label, ) for ann_frame, pred_frame in tqdm( zip(ann_frames, pred_frames), total=len(ann_frames) ) ] logger.info("accumulating...") num_classes = len(categories) + 1 hist: NDArrayI32 = np.zeros((num_classes, num_classes), dtype=np.int32) gt_id_set = set() for (hist_, gt_id_set_) in hist_and_gt_id_sets: hist += hist_ gt_id_set.update(gt_id_set_) ious = per_class_iou(hist) accs = per_class_acc(hist) IoUs = [ # pylint: disable=invalid-name {cat_name: 100 * score for cat_name, score in zip(categories, ious)}, {AVERAGE: np.multiply(np.mean(ious[list(gt_id_set)]), 100)}, ] Accs = [ # pylint: disable=invalid-name {cat_name: 100 * score for cat_name, score in zip(categories, accs)}, {AVERAGE: np.multiply(np.mean(accs[list(gt_id_set)]), 100)}, ] res_dict: Dict[str, Union[float, ScoresList]] = dict( IoU=IoUs, Acc=Accs, fIoU=np.multiply(freq_iou(hist), 100), # pylint: disable=invalid-name pAcc=np.multiply(whole_acc(hist), 100), # pylint: disable=invalid-name ) logger.info("GT id set [%s]", ",".join(str(s) for s in gt_id_set)) return SegResult(**res_dict)
def bdd100k2coco_seg_track(mask_base: str, frames: List[Frame], config: Config, nproc: int = NPROC) -> GtType: """Converting BDD100K Segmentation Tracking Set to COCO format.""" video_id, image_id, ann_id = 0, 0, 0 img_shape = config.imageSize frames_list = group_and_sort(frames) videos: List[VidType] = [] images: List[ImgType] = [] mask_names: List[str] = [] category_ids_list: List[List[int]] = [] instance_ids_list: List[List[int]] = [] annotations_list: List[List[AnnType]] = [] categories = get_leaf_categories(config.categories) cat_name2id = {cat.name: i + 1 for i, cat in enumerate(categories)} logger.info("Collecting annotations...") for video_anns in tqdm(frames_list): global_instance_id: int = 1 instance_id_maps: Dict[str, int] = {} video_name = video_anns[0].videoName video_id += 1 video = VidType(id=video_id, name=video_name) videos.append(video) for image_anns in video_anns: image_id += 1 if img_shape is None: if image_anns.size is not None: img_shape = image_anns.size else: raise ValueError("Image shape not defined!") image = ImgType( video_id=video_id, frame_id=image_anns.frameIndex, id=image_id, file_name=os.path.join(video_name, image_anns.name), height=img_shape.height, width=img_shape.width, ) if image_anns.url is not None: image["coco_url"] = image_anns.url images.append(image) mask_name = os.path.join( mask_base, video_name, # Bitmask in .png format, image in .jpg format image_anns.name.replace(".jpg", ".png"), ) mask_names.append(mask_name) category_ids: List[int] = [] instance_ids: List[int] = [] annotations: List[AnnType] = [] for label in image_anns.labels: if label.poly2d is None: continue if label.category not in cat_name2id: continue ann_id += 1 instance_id, global_instance_id = get_bdd100k_instance_id( instance_id_maps, global_instance_id, label.id) category_id = cat_name2id[label.category] annotation = AnnType( id=ann_id, image_id=image_id, instance_id=instance_id, category_id=category_id, scalabel_id=label.id, iscrowd=int(check_crowd(label) or check_ignored(label)), ignore=0, ) category_ids.append(category_id) instance_ids.append(instance_id) annotations.append(annotation) category_ids_list.append(category_ids) instance_ids_list.append(instance_ids) annotations_list.append(annotations) annotations = bitmask2coco_with_ids_parallel( annotations_list, mask_names, category_ids_list, instance_ids_list, nproc, ) return GtType( type="instances", categories=get_coco_categories(config), videos=videos, images=images, annotations=annotations, )
def evaluate_seg_track( acc_single_video: VidFunc, gts: List[List[Frame]], results: List[List[Frame]], config: Config, iou_thr: float = 0.5, ignore_iof_thr: float = 0.5, ignore_unknown_cats: bool = False, nproc: int = NPROC, ) -> TrackResult: """Evaluate CLEAR MOT metrics for a Scalabel format dataset. Args: acc_single_video: Function for calculating metrics over a single video. gts: the ground truth annotations in Scalabel format results: the prediction results in Scalabel format. config: Config object iou_thr: Minimum IoU for a mask to be considered a positive. ignore_iof_thr: Min. Intersection over foreground with ignore regions. ignore_unknown_cats: if False, raise KeyError when trying to evaluate unknown categories. nproc: processes number for loading files Returns: TrackResult: evaluation results. """ logger.info("Tracking evaluation with CLEAR MOT metrics.") t = time.time() assert len(gts) == len(results) # check overlap of masks logger.info("checking for overlap of masks...") if check_overlap([frame for res in results for frame in res], config, nproc): logger.critical( "Found overlap in prediction bitmasks, but segmentation tracking " "evaluation does not allow overlaps. Removing such predictions.") classes = get_leaf_categories(config.categories) super_classes = get_parent_categories(config.categories) logger.info("evaluating...") image_size = config.imageSize if nproc > 1: with Pool(nproc) as pool: video_accs = pool.starmap( partial( acc_single_video, classes=classes, ignore_iof_thr=ignore_iof_thr, ignore_unknown_cats=ignore_unknown_cats, image_size=image_size, ), zip(gts, results), ) else: video_accs = [ acc_single_video( gt, result, classes, iou_thr, ignore_iof_thr, ignore_unknown_cats, image_size, ) for gt, result in zip(gts, results) ] class_names, metric_names, class_accs = aggregate_accs( video_accs, classes, super_classes) logger.info("accumulating...") if nproc > 1: with Pool(nproc) as pool: flat_dicts = pool.starmap(evaluate_single_class, zip(metric_names, class_accs)) else: flat_dicts = [ evaluate_single_class(names, accs) for names, accs in zip(metric_names, class_accs) ] metrics = list(METRIC_MAPS.values()) result = generate_results(flat_dicts, class_names, metrics, classes, super_classes) t = time.time() - t logger.info("evaluation finishes with %.1f s.", t) return result