Exemple #1
0
 def run_eval_debug(self, results, save_dir):
     self.save_results(results, save_dir)
     save_path = "cache/fail/"
     coco_dets = self.coco.loadRes('{}/results.json'.format(save_dir))
     coco_eval = COCOeval(self.coco, coco_dets, "bbox")
     p = coco_eval.params
     p.imgIds = list(np.unique(p.imgIds))
     p.maxDets = sorted(p.maxDets)
     coco_eval.params = p
     coco_eval._prepare()
     catIds = [-1]
     computeIoU = coco_eval.computeIoU
     coco_eval.ious = {(imgId, catId): coco_eval.computeIoU(imgId, catId) \
                 for imgId in p.imgIds
                 for catId in catIds}
     maxDet = p.maxDets[-1]
     for imgId in p.imgIds:
         img_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/val2017/"
         fullImgId = str(imgId).zfill(11)
         img_path = os.path.join(img_path, fullImgId + '.jpg')
         img = cv2.imread(img_path)
         for catId in catIds:
             # for i, areaRng in enumerate(p.areaRng):
             areaRng = p.areaRng[0]
             print("areaRng: " + areaRng)
             result = coco_eval.getImageFailCase(imgId, catId, areaRng,
                                                 maxDet)
             gtIds = result['gtIds']
             dtIds = result['dtIds']
             gtFailIds = gtIds[result['gtfail'][0]]
             dtFailIds = dtIds[result['dtfail'][0]]
             gts = [
                 _ for cId in p.catIds for _ in coco_eval._gts[imgId, cId]
             ]
             dts = [
                 _ for cId in p.catIds for _ in coco_eval._dts[imgId, cId]
             ]
             gts = [g['bbox'] for g in gts]
             dts = [d['bbox'] for d in dts]
             pdb.set_trace()
             c = [255, 0, 0]
             for dt in dts:
                 bbox = np.array(dt, dtype=np.int32)
                 cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
                               c, 2)
             c = [0, 255, 0]
             for gt in gts:
                 bbox = np.array(gt, dtype=np.int32)
                 cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
                               c, 2)
     # saveImage
     save_path = os.path.join(save_path, fullImgId + "_" + str(i))
     cv2.imwrite(save_path, img)
Exemple #2
0
def evaluate(iou_type_evaluator: COCOeval) -> Tuple:
    """
Run per image evaluation on given images and store results (a list of dict) in self.evalImgs
:return: None
"""
    p = iou_type_evaluator.params
    # add backward compatibility if useSegm is specified in params
    if p.useSegm is not None:
        p.iouType = "segm" if p.useSegm == 1 else "bbox"
        print(
            f"useSegm (deprecated) is not None. Running {p.iouType} evaluation"
        )
    # print('Evaluate annotation type *{}*'.format(p.iouType))
    p.imgIds = list(numpy.unique(p.imgIds))
    if p.useCats:
        p.catIds = list(numpy.unique(p.catIds))
    p.maxDets = sorted(p.maxDets)
    iou_type_evaluator.params = p

    iou_type_evaluator._prepare()
    # loop through images, area range, max detection number
    cat_ids = p.catIds if p.useCats else [-1]

    compute_iou = None
    if p.iouType == "segm" or p.iouType == "bbox":
        compute_iou = iou_type_evaluator.computeIoU
    elif p.iouType == "keypoints":
        compute_iou = iou_type_evaluator.computeOks

    iou_type_evaluator.ious = {(imgId, catId): compute_iou(imgId, catId)
                               for imgId in p.imgIds for catId in cat_ids}

    evaluate_img = iou_type_evaluator.evaluateImg
    max_det = p.maxDets[-1]
    eval_imgs = [
        evaluate_img(img_id, cat_id, area_rng, max_det) for cat_id in cat_ids
        for area_rng in p.areaRng for img_id in p.imgIds
    ]

    eval_imgs = numpy.asarray(
        eval_imgs
    ).reshape(  # this is NOT in the pycocotools code, but could be done outside
        len(cat_ids), len(p.areaRng), len(p.imgIds))
    iou_type_evaluator._paramsEval = copy.deepcopy(iou_type_evaluator.params)

    return p.imgIds, eval_imgs
    def __init__(self,
                 root,
                 dataset,
                 data_format,
                 num_joints,
                 get_rescore_data,
                 transform=None,
                 target_transform=None,
                 bbox_file=None):
        from pycocotools.coco import COCO
        self.root = root
        self.dataset = dataset
        self.data_format = data_format
        self.coco = COCO(self._get_anno_file_name())
        self.ids = list(self.coco.imgs.keys())
        self.transform = transform
        self.target_transform = target_transform
        self.num_joints = num_joints
        self.bbox_preds = None
        self.get_rescore_data = get_rescore_data
        if bbox_file is not None:
            # 是否在结果评估中引入bbox的信息
            with open(bbox_file) as f:
                data = json.load(f)
            coco_dt = self.coco.loadRes(data)
            coco_eval = COCOeval(self.coco, coco_dt, 'keypoints')
            coco_eval._prepare()
            self.bbox_preds = coco_eval._dts

        # 读取COCO Dataset里面的所有类别并为接下来的分类做准备
        cats = [
            cat['name'] for cat in self.coco.loadCats(self.coco.getCatIds())
        ]
        self.classes = ['__background__'] + cats
        logger.info('=> classes: {}'.format(self.classes))
        self.num_classes = len(self.classes)
        self._class_to_ind = dict(zip(self.classes, range(self.num_classes)))
        self._class_to_coco_ind = dict(zip(cats, self.coco.getCatIds()))
        self._coco_ind_to_class_ind = dict([(self._class_to_coco_ind[cls],
                                             self._class_to_ind[cls])
                                            for cls in self.classes[1:]])
def plot(data,
         gt_file,
         img_path,
         save_path,
         link_pairs,
         ring_color,
         save=True):

    # joints
    coco = COCO(gt_file)
    coco_dt = coco.loadRes(data)
    coco_eval = COCOeval(coco, coco_dt, "keypoints")
    coco_eval._prepare()
    gts_ = coco_eval._gts
    dts_ = coco_eval._dts

    p = coco_eval.params
    p.imgIds = list(np.unique(p.imgIds))
    if p.useCats:
        p.catIds = list(np.unique(p.catIds))
    p.maxDets = sorted(p.maxDets)

    # loop through images, area range, max detection number
    catIds = p.catIds if p.useCats else [-1]
    threshold = 0.3
    joint_thres = 0.2
    for catId in catIds:
        for imgId in p.imgIds[:5000]:
            # dimention here should be Nxm
            gts = gts_[imgId, catId]
            dts = dts_[imgId, catId]
            inds = np.argsort([-d["score"] for d in dts], kind="mergesort")
            dts = [dts[i] for i in inds]
            if len(dts) > p.maxDets[-1]:
                dts = dts[0:p.maxDets[-1]]
            if len(gts) == 0 or len(dts) == 0:
                continue

            sum_score = 0
            num_box = 0
            img_name = str(imgId).zfill(12)

            # Read Images
            img_file = img_path + img_name + ".jpg"
            data_numpy = cv2.imread(
                img_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
            h = data_numpy.shape[0]
            w = data_numpy.shape[1]

            # Plot
            fig = plt.figure(figsize=(w / 100, h / 100), dpi=100)
            ax = plt.subplot(1, 1, 1)
            bk = plt.imshow(data_numpy[:, :, ::-1])
            bk.set_zorder(-1)
            print(img_name)
            for j, gt in enumerate(gts):
                # matching dt_box and gt_box
                bb = gt["bbox"]
                x0 = bb[0] - bb[2]
                x1 = bb[0] + bb[2] * 2
                y0 = bb[1] - bb[3]
                y1 = bb[1] + bb[3] * 2

                # create bounds for ignore regions(double the gt bbox)
                g = np.array(gt["keypoints"])
                # xg = g[0::3]; yg = g[1::3];
                vg = g[2::3]

                for i, dt in enumerate(dts):
                    # Calculate IoU
                    dt_bb = dt["bbox"]
                    dt_x0 = dt_bb[0] - dt_bb[2]
                    dt_x1 = dt_bb[0] + dt_bb[2] * 2
                    dt_y0 = dt_bb[1] - dt_bb[3]
                    dt_y1 = dt_bb[1] + dt_bb[3] * 2

                    ol_x = min(x1, dt_x1) - max(x0, dt_x0)
                    ol_y = min(y1, dt_y1) - max(y0, dt_y0)
                    ol_area = ol_x * ol_y
                    s_x = max(x1, dt_x1) - min(x0, dt_x0)
                    s_y = max(y1, dt_y1) - min(y0, dt_y0)
                    sum_area = s_x * s_y
                    iou = ol_area / (sum_area + np.spacing(1))
                    score = dt["score"]

                    if iou < 0.1 or score < threshold:
                        continue
                    else:
                        print("iou: ", iou)
                        dt_w = dt_x1 - dt_x0
                        dt_h = dt_y1 - dt_y0
                        ref = min(dt_w, dt_h)
                        num_box += 1
                        sum_score += dt["score"]
                        dt_joints = np.array(dt["keypoints"]).reshape(17, -1)
                        joints_dict = map_joint_dict(dt_joints)

                        # stick
                        for k, link_pair in enumerate(link_pairs):
                            if (link_pair[0] in joints_dict
                                    and link_pair[1] in joints_dict):
                                if (dt_joints[link_pair[0], 2] < joint_thres
                                        or dt_joints[link_pair[1], 2] <
                                        joint_thres or vg[link_pair[0]] == 0
                                        or vg[link_pair[1]] == 0):
                                    continue
                            if k in range(6, 11):
                                lw = 1
                            else:
                                lw = ref / 100.0
                            line = mlines.Line2D(
                                np.array([
                                    joints_dict[link_pair[0]][0],
                                    joints_dict[link_pair[1]][0],
                                ]),
                                np.array([
                                    joints_dict[link_pair[0]][1],
                                    joints_dict[link_pair[1]][1],
                                ]),
                                ls="-",
                                lw=lw,
                                alpha=1,
                                color=link_pair[2],
                            )
                            line.set_zorder(0)
                            ax.add_line(line)
                        # black ring
                        for k in range(dt_joints.shape[0]):
                            if (dt_joints[k, 2] < joint_thres
                                    or vg[link_pair[0]] == 0
                                    or vg[link_pair[1]] == 0):
                                continue
                            if dt_joints[k, 0] > w or dt_joints[k, 1] > h:
                                continue
                            if k in range(5):
                                radius = 1
                            else:
                                radius = ref / 100

                            circle = mpatches.Circle(
                                tuple(dt_joints[k, :2]),
                                radius=radius,
                                ec="black",
                                fc=ring_color[k],
                                alpha=1,
                                linewidth=1,
                            )
                            circle.set_zorder(1)
                            ax.add_patch(circle)

            avg_score = (sum_score / (num_box + np.spacing(1))) * 1000

            plt.gca().xaxis.set_major_locator(plt.NullLocator())
            plt.gca().yaxis.set_major_locator(plt.NullLocator())
            plt.axis("off")
            plt.subplots_adjust(top=1,
                                bottom=0,
                                left=0,
                                right=1,
                                hspace=0,
                                wspace=0)
            plt.margins(0, 0)
            if save:
                plt.savefig(
                    save_path + "score_" + str(np.int(avg_score)) + "_id_" +
                    str(imgId) + "_" + img_name + ".png",
                    format="png",
                    bbox_inckes="tight",
                    dpi=100,
                )
                plt.savefig(
                    save_path + "id_" + str(imgId) + ".pdf",
                    format="pdf",
                    bbox_inckes="tight",
                    dpi=100,
                )
            # plt.show()
            plt.close()
Exemple #5
0
    'hair drier', 'toothbrush'
]
annot_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/annotations/instances_val2017.json"
coco = coco.COCO(annot_path)
save_dir = "/home/user/home/user/Xinyuan/work/CenterNet-1/exp/ctdet/coco_hg/"
coco_dets = coco.loadRes('{}/results.json'.format(save_dir))
coco_eval = COCOeval(coco, coco_dets, "bbox")
print(dir(coco_eval))
save_path = "src/cache/fail/"
coco_dets = coco.loadRes('{}/results.json'.format(save_dir))
coco_eval = COCOeval(coco, coco_dets, "bbox")
p = coco_eval.params
p.imgIds = list(np.unique(p.imgIds))
p.maxDets = sorted(p.maxDets)
coco_eval.params = p
coco_eval._prepare()
cat_dict = coco.cats
# print(cat_dict)
catIds = coco_eval.params.catIds
computeIoU = coco_eval.computeIoU
coco_eval.ious = {(imgId, catId): coco_eval.computeIoU(imgId, catId) \
            for imgId in p.imgIds
            for catId in catIds}
maxDet = p.maxDets[-1]
for imgId in p.imgIds:
    img_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/val2017/"
    fullImgId = str(imgId).zfill(12)
    img_path = os.path.join(img_path, fullImgId + '.jpg')
    print(img_path)
    img = cv2.imread(img_path)
    for catId in catIds:
Exemple #6
0
def plot(data, gt_file, img_path, save_path,
         link_pairs, ring_color, save=True):
    # joints
    coco = COCO(gt_file)
    coco_dt = coco.loadRes(data)
    coco_eval = COCOeval(coco, coco_dt, 'keypoints')
    coco_eval._prepare()
    gts_ = coco_eval._gts
    dts_ = coco_eval._dts

    p = coco_eval.params
    p.imgIds = list(np.unique(p.imgIds))
    if p.useCats:
        p.catIds = list(np.unique(p.catIds))
    p.maxDets = sorted(p.maxDets)

    # loop through images, area range, max detection number
    catIds = p.catIds if p.useCats else [-1]
    threshold = 0
    joint_thres = 0.1

    imgs = coco.loadImgs(p.imgIds)
    mean_rmse_list = []
    mean_rmse_mask_list = []
    for catId in catIds:
        for imgId in imgs[:3]:
            # dimension here should be Nxm
            gts = gts_[imgId['id'], catId]
            dts = dts_[imgId['id'], catId]
            if len(gts) != 0 and len(dts) != 0:
                npgt = np.array(gts[0]["keypoints"])
                npdt = np.array(dts[0]["keypoints"])
                mask = npdt[2::3] >= joint_thres
                RMSE = np.sqrt((npgt[0::3] - npdt[0::3]) ** 2 + (npgt[1::3] - npdt[1::3]) ** 2)
                RMSE_mask = RMSE[mask]
                mean_rmse = np.round(np.nanmean(RMSE.flatten()), 2)
                mean_rmse_mask = np.round(np.nanmean(RMSE_mask.flatten()), 2)
                print(f"mean rmse: {mean_rmse}")
                print(f"mean rmse mask: {mean_rmse_mask}")
                mean_rmse_list.append(mean_rmse)
                mean_rmse_mask_list.append(mean_rmse_mask)

            inds = np.argsort([-d['score'] for d in dts], kind='mergesort')
            dts = [dts[i] for i in inds]
            if len(dts) > p.maxDets[-1]:
                dts = dts[0:p.maxDets[-1]]
            if len(gts) == 0 or len(dts) == 0:
                continue

            sum_score = 0
            num_box = 0
            # Read Images
            img_file = os.path.join(img_path, imgId["file_name"])
            #  img_file = img_path + img_name + '.jpg'
            data_numpy = cv2.imread(img_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
            h = data_numpy.shape[0]
            w = data_numpy.shape[1]

            # Plot
            fig = plt.figure(figsize=(w / 100, h / 100), dpi=100)
            ax = plt.subplot(1, 1, 1)
            bk = plt.imshow(data_numpy[:, :, ::-1])
            bk.set_zorder(-1)

            for j, gt in enumerate(gts):
                # matching dt_box and gt_box
                bb = gt['bbox']
                x0 = bb[0] - bb[2];
                x1 = bb[0] + bb[2] * 2
                y0 = bb[1] - bb[3];
                y1 = bb[1] + bb[3] * 2

                # create bounds for ignore regions(double the gt bbox)
                g = np.array(gt['keypoints'])
                # xg = g[0::3]; yg = g[1::3];
                vg = g[2::3]

                for i, dt in enumerate(dts):
                    # Calculate Bbox IoU
                    dt_bb = dt['bbox']
                    dt_x0 = dt_bb[0] - dt_bb[2];
                    dt_x1 = dt_bb[0] + dt_bb[2] * 2
                    dt_y0 = dt_bb[1] - dt_bb[3];
                    dt_y1 = dt_bb[1] + dt_bb[3] * 2

                    ol_x = min(x1, dt_x1) - max(x0, dt_x0)
                    ol_y = min(y1, dt_y1) - max(y0, dt_y0)
                    ol_area = ol_x * ol_y
                    s_x = max(x1, dt_x1) - min(x0, dt_x0)
                    s_y = max(y1, dt_y1) - min(y0, dt_y0)
                    sum_area = s_x * s_y
                    iou = np.round(ol_area / (sum_area + np.spacing(1)), 2)
                    score = np.round(dt['score'], 2)
                    print(f"score: {dt['score']}")
                    if iou < 0.1 or score < threshold:
                        continue
                    else:
                        print(f'iou: {iou}')
                        dt_w = dt_x1 - dt_x0
                        dt_h = dt_y1 - dt_y0
                        ref = min(dt_w, dt_h)
                        num_box += 1
                        sum_score += dt['score']
                        dt_joints = np.array(dt['keypoints']).reshape(20, -1)

                        joints_dict = map_joint_dict(dt_joints)
                        # print(joints_dict)
                        # print(link_pairs)
                        # print(dt_joints)
                        # stick
                        for k, link_pair in enumerate(link_pairs):
                            if link_pair[0] in joints_dict \
                                    and link_pair[1] in joints_dict:
                                # print(link_pair[0])
                                # print(vg)
                                if dt_joints[link_pair[0] - 1, 2] < joint_thres \
                                        or dt_joints[link_pair[1] - 1, 2] < joint_thres \
                                        or vg[link_pair[0] - 1] == 0 \
                                        or vg[link_pair[1] - 1] == 0:
                                    continue
                            # if k in range(6, 11):
                            #     lw = 1
                            # else:
                            lw = ref / 100.
                            line = mlines.Line2D(
                                np.array([joints_dict[link_pair[0]][0],
                                          joints_dict[link_pair[1]][0]]),
                                np.array([joints_dict[link_pair[0]][1],
                                          joints_dict[link_pair[1]][1]]),
                                ls='-', lw=lw, alpha=1, color=link_pair[2], )
                            line.set_zorder(0)
                            ax.add_line(line)
                        # black ring
                        for k in range(dt_joints.shape[0]):
                            if dt_joints[k, 2] < joint_thres \
                                    or vg[link_pair[0]] == 0 \
                                    or vg[link_pair[1]] == 0:
                                continue
                            if dt_joints[k, 0] > w or dt_joints[k, 1] > h:
                                continue
                            # if k in range(5):
                            #     radius = 1
                            # else:
                            radius = ref / 100

                            circle = mpatches.Circle(tuple(dt_joints[k, :2]),
                                                     radius=radius,
                                                     ec='black',
                                                     fc=ring_color[k],
                                                     alpha=1,
                                                     linewidth=1)
                            circle.set_zorder(1)
                            ax.add_patch(circle)

            avg_score = (sum_score / (num_box + np.spacing(1))) * 1000

            plt.gca().xaxis.set_major_locator(plt.NullLocator())
            plt.gca().yaxis.set_major_locator(plt.NullLocator())
            plt.axis('off')
            plt.subplots_adjust(top=1, bottom=0, left=0, right=1, hspace=0, wspace=0)
            plt.margins(0, 0)
            if save:
                plt.savefig(save_path + \
                            'score_' + str(np.int(avg_score)) + "_" +
                            imgId["file_name"].split(".")[0] + '.png',
                            format='png', bbox_inckes='tight', dpi=100)
                # plt.savefig(save_path + 'id_' + str(imgId) + '.pdf', format='pdf',
                #             bbox_inckes='tight', dpi=100)
            # plt.show()
            plt.close()
    print(f"total mean rmse: {np.mean(mean_rmse_list)}")
    print(f"total mean rmse mask: {np.mean(mean_rmse_mask_list)}")
Exemple #7
0
def validate(config,
             val_loader,
             val_dataset,
             model,
             output_dir,
             tb_log_dir,
             writer_dict=None):
    model.eval()
    if config.MODEL.NAME == 'pose_hourglass':
        transforms = torchvision.transforms.Compose([
            torchvision.transforms.ToTensor(),
        ])
    else:
        transforms = torchvision.transforms.Compose([
            torchvision.transforms.ToTensor(),
            torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                             std=[0.229, 0.224, 0.225])
        ])
    # rmse = AverageMeter()

    parser = HeatmapParser(config)
    all_preds = []
    all_scores = []

    pbar = tqdm(total=len(val_dataset)) if config.TEST.LOG_PROGRESS else None
    for i, (images, annos) in enumerate(val_loader):
        assert 1 == images.size(0), 'Test batch size should be 1'

        image = images[0].cpu().numpy()
        # size at scale 1.0
        base_size, center, scale = get_multi_scale_size(
            image, config.DATASET.INPUT_SIZE, 1.0,
            min(config.TEST.SCALE_FACTOR))

        with torch.no_grad():
            final_heatmaps = None
            tags_list = []
            for idx, s in enumerate(
                    sorted(config.TEST.SCALE_FACTOR, reverse=True)):
                input_size = config.DATASET.INPUT_SIZE
                image_resized, center, scale = resize_align_multi_scale(
                    image, input_size, s, min(config.TEST.SCALE_FACTOR))
                image_resized = transforms(image_resized)
                image_resized = image_resized.unsqueeze(0).cuda()

                outputs, heatmaps, tags = get_multi_stage_outputs(
                    config, model, image_resized, config.TEST.FLIP_TEST,
                    config.TEST.PROJECT2IMAGE, base_size)

                final_heatmaps, tags_list = aggregate_results(
                    config, s, final_heatmaps, tags_list, heatmaps, tags)

            final_heatmaps = final_heatmaps / float(
                len(config.TEST.SCALE_FACTOR))
            tags = torch.cat(tags_list, dim=4)
            grouped, scores = parser.parse(final_heatmaps, tags,
                                           config.TEST.ADJUST,
                                           config.TEST.REFINE)

            final_results = get_final_preds(
                grouped, center, scale,
                [final_heatmaps.size(3),
                 final_heatmaps.size(2)])

        if config.TEST.LOG_PROGRESS:
            pbar.update()

        # if i % config.PRINT_FREQ == 0:
        #     prefix = '{}_{}'.format(os.path.join(output_dir, 'result_valid'), i)
        #     # logger.info('=> write {}'.format(prefix))
        #     save_valid_image(image, final_results, '{}.jpg'.format(prefix), dataset=val_dataset.name)
        #     # save_debug_images(config, image_resized, None, None, outputs, prefix)

        all_preds.append(final_results)
        all_scores.append(scores)

    if config.TEST.LOG_PROGRESS:
        pbar.close()

    results, res_file = val_dataset.evaluate(config, all_preds, all_scores,
                                             output_dir)
    ##################################
    gt_file = val_dataset._get_anno_file_name()
    coco = COCO(gt_file)
    coco_dt = coco.loadRes(res_file)
    coco_eval = COCOeval(coco, coco_dt, 'keypoints')
    coco_eval._prepare()
    gts_ = coco_eval._gts
    dts_ = coco_eval._dts

    p = coco_eval.params
    p.imgIds = list(np.unique(p.imgIds))
    if p.useCats:
        p.catIds = list(np.unique(p.catIds))
    p.maxDets = sorted(p.maxDets)

    # loop through images, area range, max detection number
    catIds = p.catIds if p.useCats else [-1]
    pcutoff01 = 0.1
    pcutoff06 = 0.6
    mean_rmse_list = []
    mean_rmse_pcutoff01_list = []
    mean_rmse_pcutoff06_list = []
    for catId in catIds:
        for imgId in p.imgIds:
            # dimension here should be Nxm
            gts = gts_[imgId, catId]
            dts = dts_[imgId, catId]
            if len(gts) != 0 and len(dts) != 0:
                npgt = np.array(gts[0]["keypoints"])
                npdt = np.array(dts[0]["keypoints"])
                mask01 = npdt[2::3] >= pcutoff01
                mask06 = npdt[2::3] >= pcutoff06
                RMSE = np.sqrt((npgt[0::3] - npdt[0::3])**2 +
                               (npgt[1::3] - npdt[1::3])**2)
                RMSE_pcutoff01 = RMSE[mask01]
                RMSE_pcutoff06 = RMSE[mask06]
                mean_rmse = np.round(np.nanmean(RMSE.flatten()), 2)
                mean_rmse_pcutoff01 = np.nanmean(RMSE_pcutoff01.flatten())
                mean_rmse_pcutoff06 = np.nanmean(RMSE_pcutoff06.flatten())
                mean_rmse_list.append(mean_rmse)
                mean_rmse_pcutoff01_list.append(mean_rmse_pcutoff01)
                mean_rmse_pcutoff06_list.append(mean_rmse_pcutoff06)
    print(f"Mean RMSE: {np.mean(mean_rmse_list)}")
    print(
        f"Mean RMSE p-cutoff 0.1: {np.round(np.mean(mean_rmse_pcutoff01_list),2)}"
    )
    print(
        f"Mean RMSE p-cutoff 0.6: {np.round(np.mean(mean_rmse_pcutoff06_list),2)}"
    )
    global_steps = writer_dict['valid_global_steps']
    writer_dict["writer"].add_scalar("val_rmse", np.mean(mean_rmse_list),
                                     global_steps)
    writer_dict["writer"].add_scalar("val_rmse_pcutoff_0.1",
                                     np.mean(mean_rmse_pcutoff01_list),
                                     global_steps)
    writer_dict["writer"].add_scalar("val_rmse_pcutoff_0.6",
                                     np.mean(mean_rmse_pcutoff06_list),
                                     global_steps)
    writer_dict['valid_global_steps'] = global_steps + 1
    return np.mean(mean_rmse_list)
Exemple #8
0
    def evaluate(self,
                 results,
                 metric='bbox',
                 logger=None,
                 jsonfile_prefix=None,
                 classwise=False,
                 proposal_nums=(100, 300, 1000),
                 iou_thrs=None,
                 metric_items=None):
        """Evaluation in COCO protocol.

        Args:
            results (list[list | tuple]): Testing results of the dataset.
            metric (str | list[str]): Metrics to be evaluated. Options are
                'bbox', 'segm', 'proposal', 'proposal_fast'.
            logger (logging.Logger | str | None): Logger used for printing
                related information during evaluation. Default: None.
            jsonfile_prefix (str | None): The prefix of json files. It includes
                the file path and the prefix of filename, e.g., "a/b/prefix".
                If not specified, a temp file will be created. Default: None.
            classwise (bool): Whether to evaluating the AP for each class.
            proposal_nums (Sequence[int]): Proposal number used for evaluating
                recalls, such as recall@100, recall@1000.
                Default: (100, 300, 1000).
            iou_thrs (Sequence[float], optional): IoU threshold used for
                evaluating recalls/mAPs. If set to a list, the average of all
                IoUs will also be computed. If not specified, [0.50, 0.55,
                0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95] will be used.
                Default: None.
            metric_items (list[str] | str, optional): Metric items that will
                be returned. If not specified, ``['AR@100', 'AR@300',
                'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ]`` will be
                used when ``metric=='proposal'``, ``['mAP', 'mAP_50', 'mAP_75',
                'mAP_s', 'mAP_m', 'mAP_l']`` will be used when
                ``metric=='bbox' or metric=='segm'``.

        Returns:
            dict[str, float]: COCO style evaluation metric.
        """

        metrics = metric if isinstance(metric, list) else [metric]
        allowed_metrics = [
            'bbox', 'segm', 'proposal', 'proposal_fast', 'pixel'
        ]
        for metric in metrics:
            if metric not in allowed_metrics:
                raise KeyError(f'metric {metric} is not supported')
        if iou_thrs is None:
            iou_thrs = np.linspace(.5,
                                   0.95,
                                   int(np.round((0.95 - .5) / .05)) + 1,
                                   endpoint=True)
        if metric_items is not None:
            if not isinstance(metric_items, list):
                metric_items = [metric_items]

        result_files, tmp_dir = self.format_results(results, jsonfile_prefix)

        eval_results = OrderedDict()
        cocoGt = self.coco
        for metric in metrics:
            msg = f'Evaluating {metric}...'
            if logger is None:
                msg = '\n' + msg
            print_log(msg, logger=logger)

            if metric == 'proposal_fast':
                ar = self.fast_eval_recall(results,
                                           proposal_nums,
                                           iou_thrs,
                                           logger='silent')
                log_msg = []
                for i, num in enumerate(proposal_nums):
                    eval_results[f'AR@{num}'] = ar[i]
                    log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}')
                log_msg = ''.join(log_msg)
                print_log(log_msg, logger=logger)
                continue

            if metric not in result_files:
                raise KeyError(f'{metric} is not in results')
            try:
                cocoDt = cocoGt.loadRes(result_files[metric])
            except IndexError:
                print_log('The testing results of the whole dataset is empty.',
                          logger=logger,
                          level=logging.ERROR)
                break

            if metric == 'pixel':
                iou_type = 'segm'
            else:
                iou_type = 'bbox' if metric == 'proposal' else metric
            cocoEval = COCOeval(cocoGt, cocoDt, iou_type)
            cocoEval.params.catIds = self.cat_ids
            cocoEval.params.imgIds = self.img_ids
            cocoEval.params.maxDets = list(proposal_nums)
            cocoEval.params.iouThrs = iou_thrs
            # mapping of cocoEval.stats
            coco_metric_names = {
                'mAP': 0,
                'mAP_50': 1,
                'mAP_75': 2,
                'mAP_s': 3,
                'mAP_m': 4,
                'mAP_l': 5,
                'AR@100': 6,
                'AR@300': 7,
                'AR@1000': 8,
                'AR_s@1000': 9,
                'AR_m@1000': 10,
                'AR_l@1000': 11
            }
            if metric_items is not None:
                for metric_item in metric_items:
                    if metric_item not in coco_metric_names:
                        raise KeyError(
                            f'metric item {metric_item} is not supported')

            if metric == 'proposal':
                cocoEval.params.useCats = 0
                cocoEval.evaluate()
                cocoEval.accumulate()
                cocoEval.summarize()
                if metric_items is None:
                    metric_items = [
                        'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000',
                        'AR_m@1000', 'AR_l@1000'
                    ]

                for item in metric_items:
                    val = float(
                        f'{cocoEval.stats[coco_metric_names[item]]:.3f}')
                    eval_results[item] = val
            elif metric == 'pixel':
                cocoEval._prepare()
                full_gt_mask = []
                full_dt_mask = []
                for img_index in cocoGt.imgToAnns.keys():
                    gt_img_anns = cocoGt.imgToAnns[img_index]
                    gt_img_info = cocoGt.imgs[img_index]
                    dt_img_anns = cocoDt.imgToAnns[img_index]
                    dt_img_info = cocoDt.imgs[img_index]

                    gt_mask = anns_to_mask(gt_img_anns, gt_img_info['height'],
                                           gt_img_info['width'])
                    dt_mask = anns_to_mask(dt_img_anns, dt_img_info['height'],
                                           gt_img_info['width'])

                    full_gt_mask.append(gt_mask)
                    full_dt_mask.append(dt_mask)

                max_size = np.max([mask.shape[:2] for mask in full_gt_mask],
                                  axis=0)

                for i, _ in enumerate(full_gt_mask):
                    full_gt_mask[i] = pad_img_to_size(full_gt_mask[i],
                                                      max_size)
                    full_dt_mask[i] = pad_img_to_size(full_dt_mask[i],
                                                      max_size)

                full_gt_mask = np.stack(full_gt_mask)
                full_dt_mask = np.stack(full_dt_mask)

                metrics = calculate_metrics(full_dt_mask, full_gt_mask)

                table_data = [['Pixel Metric', 'Value']]
                # add the pixel-wise metrics to the output
                for name, value in metrics._asdict().items():
                    eval_results[f'{metric}_{name}'] = value
                    table_data.append([f'{metric}_{name}', '%.4f' % value])

                table = AsciiTable(table_data)
                print_log('\n' + table.table, logger=logger)
            else:
                cocoEval.evaluate()
                cocoEval.accumulate()
                cocoEval.summarize()
                if classwise:  # Compute per-category AP
                    # Compute per-category AP
                    # from https://github.com/facebookresearch/detectron2/
                    precisions = cocoEval.eval['precision']
                    # precision: (iou, recall, cls, area range, max dets)
                    assert len(self.cat_ids) == precisions.shape[2]

                    results_per_category = []
                    for idx, catId in enumerate(self.cat_ids):
                        # area range index 0: all area ranges
                        # max dets index -1: typically 100 per image
                        nm = self.coco.loadCats(catId)[0]
                        precision = precisions[:, :, idx, 0, -1]
                        precision = precision[precision > -1]
                        if precision.size:
                            ap = np.mean(precision)
                        else:
                            ap = float('nan')
                        results_per_category.append(
                            (f'{nm["name"]}', f'{float(ap):0.3f}'))

                    num_columns = min(6, len(results_per_category) * 2)
                    results_flatten = list(
                        itertools.chain(*results_per_category))
                    headers = ['category', 'AP'] * (num_columns // 2)
                    results_2d = itertools.zip_longest(*[
                        results_flatten[i::num_columns]
                        for i in range(num_columns)
                    ])
                    table_data = [headers]
                    table_data += [result for result in results_2d]
                    table = AsciiTable(table_data)
                    print_log('\n' + table.table, logger=logger)

                if metric_items is None:
                    metric_items = [
                        'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l'
                    ]

                for metric_item in metric_items:
                    key = f'{metric}_{metric_item}'
                    val = float(
                        f'{cocoEval.stats[coco_metric_names[metric_item]]:.3f}'
                    )
                    eval_results[key] = val
                ap = cocoEval.stats[:6]
                eval_results[f'{metric}_mAP_copypaste'] = (
                    f'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} '
                    f'{ap[4]:.3f} {ap[5]:.3f}')
        if tmp_dir is not None:
            tmp_dir.cleanup()
        return eval_results
class MapEvaluator:
    def __init__(self, gt_json, cat_ids):
        # Eval options
        self.iou_threshs = [.5]
        self.max_dets = [1, 10, 100]

        self.cat_ids = cat_ids

        self.coco_gt = COCO(gt_json)

    def _init_coco_eval(self, dt_json_data):
        coco_dt = self.coco_gt.loadRes(dt_json_data)

        self.coco_eval = COCOeval(self.coco_gt, coco_dt, 'bbox')
        self.coco_eval.params.iouThrs = np.array(self.iou_threshs)
        self.coco_eval.params.maxDets = self.max_dets
        self.coco_eval.params.catIds = self.cat_ids

    def do_eval(self, dt_json_data):
        self._init_coco_eval(dt_json_data)

        self.coco_eval.evaluate()
        self.coco_eval.accumulate()

    def do_eval_and_print(self, dt_json_data):
        self._init_coco_eval(dt_json_data)

        self.coco_eval.evaluate()
        self.coco_eval.accumulate()
        self.coco_eval.summarize()

    def get_avg_precision_recall(self, t=0, a=0, m=-1):
        p = self.coco_eval.eval['precision'][t, :, :, a, m]
        r = self.coco_eval.eval['recall'][t, :, a, m]

        # Removing -1 entries for classes that have no GT objects.
        p = p[:, np.where(r > -1)[0]]
        r = r[r > -1]
        assert (np.all(p >= 0.) and np.all(r >= 0.))

        ap = p.mean(axis=1)
        ar = r.mean()

        return ap, ar

    def evaluateImg(self, imgId, catId, aRng=(-np.inf, np.inf), maxDet=100):
        p = self.coco_eval.params
        # add backward compatibility if useSegm is specified in params
        p.iouType = 'bbox'
        p.imgIds = list(np.unique(p.imgIds))
        if p.useCats:
            p.catIds = list(np.unique(p.catIds))

        p.maxDets = sorted(p.maxDets)
        self.coco_eval.params = p

        self.coco_eval._prepare()

        self.coco_eval.ious = {
            (imgId, catId): self.coco_eval.computeIoU(imgId, catId)
        }
        img_eval = self.coco_eval.evaluateImg(imgId, catId, aRng, maxDet)
        inds = img_eval['dtIds']
        matches = img_eval['dtMatches']

        return inds, matches