def run_eval_debug(self, results, save_dir):
self.save_results(results, save_dir)
save_path = "cache/fail/"
coco_dets = self.coco.loadRes('{}/results.json'.format(save_dir))
coco_eval = COCOeval(self.coco, coco_dets, "bbox")
p = coco_eval.params
p.imgIds = list(np.unique(p.imgIds))
p.maxDets = sorted(p.maxDets)
coco_eval.params = p
coco_eval._prepare()
catIds = [-1]
computeIoU = coco_eval.computeIoU
coco_eval.ious = {(imgId, catId): coco_eval.computeIoU(imgId, catId) \
for imgId in p.imgIds
for catId in catIds}
maxDet = p.maxDets[-1]
for imgId in p.imgIds:
img_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/val2017/"
fullImgId = str(imgId).zfill(11)
img_path = os.path.join(img_path, fullImgId + '.jpg')
img = cv2.imread(img_path)
for catId in catIds:
# for i, areaRng in enumerate(p.areaRng):
areaRng = p.areaRng[0]
print("areaRng: " + areaRng)
result = coco_eval.getImageFailCase(imgId, catId, areaRng,
maxDet)
gtIds = result['gtIds']
dtIds = result['dtIds']
gtFailIds = gtIds[result['gtfail'][0]]
dtFailIds = dtIds[result['dtfail'][0]]
gts = [
_ for cId in p.catIds for _ in coco_eval._gts[imgId, cId]
]
dts = [
_ for cId in p.catIds for _ in coco_eval._dts[imgId, cId]
]
gts = [g['bbox'] for g in gts]
dts = [d['bbox'] for d in dts]
pdb.set_trace()
c = [255, 0, 0]
for dt in dts:
bbox = np.array(dt, dtype=np.int32)
cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
c, 2)
c = [0, 255, 0]
for gt in gts:
bbox = np.array(gt, dtype=np.int32)
cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
c, 2)
# saveImage
save_path = os.path.join(save_path, fullImgId + "_" + str(i))
cv2.imwrite(save_path, img)
def evaluate(iou_type_evaluator: COCOeval) -> Tuple:
"""
Run per image evaluation on given images and store results (a list of dict) in self.evalImgs
:return: None
"""
p = iou_type_evaluator.params
# add backward compatibility if useSegm is specified in params
if p.useSegm is not None:
p.iouType = "segm" if p.useSegm == 1 else "bbox"
print(
f"useSegm (deprecated) is not None. Running {p.iouType} evaluation"
)
# print('Evaluate annotation type *{}*'.format(p.iouType))
p.imgIds = list(numpy.unique(p.imgIds))
if p.useCats:
p.catIds = list(numpy.unique(p.catIds))
p.maxDets = sorted(p.maxDets)
iou_type_evaluator.params = p
iou_type_evaluator._prepare()
# loop through images, area range, max detection number
cat_ids = p.catIds if p.useCats else [-1]
compute_iou = None
if p.iouType == "segm" or p.iouType == "bbox":
compute_iou = iou_type_evaluator.computeIoU
elif p.iouType == "keypoints":
compute_iou = iou_type_evaluator.computeOks
iou_type_evaluator.ious = {(imgId, catId): compute_iou(imgId, catId)
for imgId in p.imgIds for catId in cat_ids}
evaluate_img = iou_type_evaluator.evaluateImg
max_det = p.maxDets[-1]
eval_imgs = [
evaluate_img(img_id, cat_id, area_rng, max_det) for cat_id in cat_ids
for area_rng in p.areaRng for img_id in p.imgIds
]
eval_imgs = numpy.asarray(
eval_imgs
).reshape( # this is NOT in the pycocotools code, but could be done outside
len(cat_ids), len(p.areaRng), len(p.imgIds))
iou_type_evaluator._paramsEval = copy.deepcopy(iou_type_evaluator.params)
return p.imgIds, eval_imgs
def __init__(self,
root,
dataset,
data_format,
num_joints,
get_rescore_data,
transform=None,
target_transform=None,
bbox_file=None):
from pycocotools.coco import COCO
self.root = root
self.dataset = dataset
self.data_format = data_format
self.coco = COCO(self._get_anno_file_name())
self.ids = list(self.coco.imgs.keys())
self.transform = transform
self.target_transform = target_transform
self.num_joints = num_joints
self.bbox_preds = None
self.get_rescore_data = get_rescore_data
if bbox_file is not None:
# 是否在结果评估中引入bbox的信息
with open(bbox_file) as f:
data = json.load(f)
coco_dt = self.coco.loadRes(data)
coco_eval = COCOeval(self.coco, coco_dt, 'keypoints')
coco_eval._prepare()
self.bbox_preds = coco_eval._dts
# 读取COCO Dataset里面的所有类别并为接下来的分类做准备
cats = [
cat['name'] for cat in self.coco.loadCats(self.coco.getCatIds())
]
self.classes = ['__background__'] + cats
logger.info('=> classes: {}'.format(self.classes))
self.num_classes = len(self.classes)
self._class_to_ind = dict(zip(self.classes, range(self.num_classes)))
self._class_to_coco_ind = dict(zip(cats, self.coco.getCatIds()))
self._coco_ind_to_class_ind = dict([(self._class_to_coco_ind[cls],
self._class_to_ind[cls])
for cls in self.classes[1:]])
def plot(data,
gt_file,
img_path,
save_path,
link_pairs,
ring_color,
save=True):
# joints
coco = COCO(gt_file)
coco_dt = coco.loadRes(data)
coco_eval = COCOeval(coco, coco_dt, "keypoints")
coco_eval._prepare()
gts_ = coco_eval._gts
dts_ = coco_eval._dts
p = coco_eval.params
p.imgIds = list(np.unique(p.imgIds))
if p.useCats:
p.catIds = list(np.unique(p.catIds))
p.maxDets = sorted(p.maxDets)
# loop through images, area range, max detection number
catIds = p.catIds if p.useCats else [-1]
threshold = 0.3
joint_thres = 0.2
for catId in catIds:
for imgId in p.imgIds[:5000]:
# dimention here should be Nxm
gts = gts_[imgId, catId]
dts = dts_[imgId, catId]
inds = np.argsort([-d["score"] for d in dts], kind="mergesort")
dts = [dts[i] for i in inds]
if len(dts) > p.maxDets[-1]:
dts = dts[0:p.maxDets[-1]]
if len(gts) == 0 or len(dts) == 0:
continue
sum_score = 0
num_box = 0
img_name = str(imgId).zfill(12)
# Read Images
img_file = img_path + img_name + ".jpg"
data_numpy = cv2.imread(
img_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
h = data_numpy.shape[0]
w = data_numpy.shape[1]
# Plot
fig = plt.figure(figsize=(w / 100, h / 100), dpi=100)
ax = plt.subplot(1, 1, 1)
bk = plt.imshow(data_numpy[:, :, ::-1])
bk.set_zorder(-1)
print(img_name)
for j, gt in enumerate(gts):
# matching dt_box and gt_box
bb = gt["bbox"]
x0 = bb[0] - bb[2]
x1 = bb[0] + bb[2] * 2
y0 = bb[1] - bb[3]
y1 = bb[1] + bb[3] * 2
# create bounds for ignore regions(double the gt bbox)
g = np.array(gt["keypoints"])
# xg = g[0::3]; yg = g[1::3];
vg = g[2::3]
for i, dt in enumerate(dts):
# Calculate IoU
dt_bb = dt["bbox"]
dt_x0 = dt_bb[0] - dt_bb[2]
dt_x1 = dt_bb[0] + dt_bb[2] * 2
dt_y0 = dt_bb[1] - dt_bb[3]
dt_y1 = dt_bb[1] + dt_bb[3] * 2
ol_x = min(x1, dt_x1) - max(x0, dt_x0)
ol_y = min(y1, dt_y1) - max(y0, dt_y0)
ol_area = ol_x * ol_y
s_x = max(x1, dt_x1) - min(x0, dt_x0)
s_y = max(y1, dt_y1) - min(y0, dt_y0)
sum_area = s_x * s_y
iou = ol_area / (sum_area + np.spacing(1))
score = dt["score"]
if iou < 0.1 or score < threshold:
continue
else:
print("iou: ", iou)
dt_w = dt_x1 - dt_x0
dt_h = dt_y1 - dt_y0
ref = min(dt_w, dt_h)
num_box += 1
sum_score += dt["score"]
dt_joints = np.array(dt["keypoints"]).reshape(17, -1)
joints_dict = map_joint_dict(dt_joints)
# stick
for k, link_pair in enumerate(link_pairs):
if (link_pair[0] in joints_dict
and link_pair[1] in joints_dict):
if (dt_joints[link_pair[0], 2] < joint_thres
or dt_joints[link_pair[1], 2] <
joint_thres or vg[link_pair[0]] == 0
or vg[link_pair[1]] == 0):
continue
if k in range(6, 11):
lw = 1
else:
lw = ref / 100.0
line = mlines.Line2D(
np.array([
joints_dict[link_pair[0]][0],
joints_dict[link_pair[1]][0],
]),
np.array([
joints_dict[link_pair[0]][1],
joints_dict[link_pair[1]][1],
]),
ls="-",
lw=lw,
alpha=1,
color=link_pair[2],
)
line.set_zorder(0)
ax.add_line(line)
# black ring
for k in range(dt_joints.shape[0]):
if (dt_joints[k, 2] < joint_thres
or vg[link_pair[0]] == 0
or vg[link_pair[1]] == 0):
continue
if dt_joints[k, 0] > w or dt_joints[k, 1] > h:
continue
if k in range(5):
radius = 1
else:
radius = ref / 100
circle = mpatches.Circle(
tuple(dt_joints[k, :2]),
radius=radius,
ec="black",
fc=ring_color[k],
alpha=1,
linewidth=1,
)
circle.set_zorder(1)
ax.add_patch(circle)
avg_score = (sum_score / (num_box + np.spacing(1))) * 1000
plt.gca().xaxis.set_major_locator(plt.NullLocator())
plt.gca().yaxis.set_major_locator(plt.NullLocator())
plt.axis("off")
plt.subplots_adjust(top=1,
bottom=0,
left=0,
right=1,
hspace=0,
wspace=0)
plt.margins(0, 0)
if save:
plt.savefig(
save_path + "score_" + str(np.int(avg_score)) + "_id_" +
str(imgId) + "_" + img_name + ".png",
format="png",
bbox_inckes="tight",
dpi=100,
)
plt.savefig(
save_path + "id_" + str(imgId) + ".pdf",
format="pdf",
bbox_inckes="tight",
dpi=100,
)
# plt.show()
plt.close()
'hair drier', 'toothbrush'
]
annot_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/annotations/instances_val2017.json"
coco = coco.COCO(annot_path)
save_dir = "/home/user/home/user/Xinyuan/work/CenterNet-1/exp/ctdet/coco_hg/"
coco_dets = coco.loadRes('{}/results.json'.format(save_dir))
coco_eval = COCOeval(coco, coco_dets, "bbox")
print(dir(coco_eval))
save_path = "src/cache/fail/"
coco_dets = coco.loadRes('{}/results.json'.format(save_dir))
coco_eval = COCOeval(coco, coco_dets, "bbox")
p = coco_eval.params
p.imgIds = list(np.unique(p.imgIds))
p.maxDets = sorted(p.maxDets)
coco_eval.params = p
coco_eval._prepare()
cat_dict = coco.cats
# print(cat_dict)
catIds = coco_eval.params.catIds
computeIoU = coco_eval.computeIoU
coco_eval.ious = {(imgId, catId): coco_eval.computeIoU(imgId, catId) \
for imgId in p.imgIds
for catId in catIds}
maxDet = p.maxDets[-1]
for imgId in p.imgIds:
img_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/val2017/"
fullImgId = str(imgId).zfill(12)
img_path = os.path.join(img_path, fullImgId + '.jpg')
print(img_path)
img = cv2.imread(img_path)
for catId in catIds:
def plot(data, gt_file, img_path, save_path,
link_pairs, ring_color, save=True):
# joints
coco = COCO(gt_file)
coco_dt = coco.loadRes(data)
coco_eval = COCOeval(coco, coco_dt, 'keypoints')
coco_eval._prepare()
gts_ = coco_eval._gts
dts_ = coco_eval._dts
p = coco_eval.params
p.imgIds = list(np.unique(p.imgIds))
if p.useCats:
p.catIds = list(np.unique(p.catIds))
p.maxDets = sorted(p.maxDets)
# loop through images, area range, max detection number
catIds = p.catIds if p.useCats else [-1]
threshold = 0
joint_thres = 0.1
imgs = coco.loadImgs(p.imgIds)
mean_rmse_list = []
mean_rmse_mask_list = []
for catId in catIds:
for imgId in imgs[:3]:
# dimension here should be Nxm
gts = gts_[imgId['id'], catId]
dts = dts_[imgId['id'], catId]
if len(gts) != 0 and len(dts) != 0:
npgt = np.array(gts[0]["keypoints"])
npdt = np.array(dts[0]["keypoints"])
mask = npdt[2::3] >= joint_thres
RMSE = np.sqrt((npgt[0::3] - npdt[0::3]) ** 2 + (npgt[1::3] - npdt[1::3]) ** 2)
RMSE_mask = RMSE[mask]
mean_rmse = np.round(np.nanmean(RMSE.flatten()), 2)
mean_rmse_mask = np.round(np.nanmean(RMSE_mask.flatten()), 2)
print(f"mean rmse: {mean_rmse}")
print(f"mean rmse mask: {mean_rmse_mask}")
mean_rmse_list.append(mean_rmse)
mean_rmse_mask_list.append(mean_rmse_mask)
inds = np.argsort([-d['score'] for d in dts], kind='mergesort')
dts = [dts[i] for i in inds]
if len(dts) > p.maxDets[-1]:
dts = dts[0:p.maxDets[-1]]
if len(gts) == 0 or len(dts) == 0:
continue
sum_score = 0
num_box = 0
# Read Images
img_file = os.path.join(img_path, imgId["file_name"])
# img_file = img_path + img_name + '.jpg'
data_numpy = cv2.imread(img_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
h = data_numpy.shape[0]
w = data_numpy.shape[1]
# Plot
fig = plt.figure(figsize=(w / 100, h / 100), dpi=100)
ax = plt.subplot(1, 1, 1)
bk = plt.imshow(data_numpy[:, :, ::-1])
bk.set_zorder(-1)
for j, gt in enumerate(gts):
# matching dt_box and gt_box
bb = gt['bbox']
x0 = bb[0] - bb[2];
x1 = bb[0] + bb[2] * 2
y0 = bb[1] - bb[3];
y1 = bb[1] + bb[3] * 2
# create bounds for ignore regions(double the gt bbox)
g = np.array(gt['keypoints'])
# xg = g[0::3]; yg = g[1::3];
vg = g[2::3]
for i, dt in enumerate(dts):
# Calculate Bbox IoU
dt_bb = dt['bbox']
dt_x0 = dt_bb[0] - dt_bb[2];
dt_x1 = dt_bb[0] + dt_bb[2] * 2
dt_y0 = dt_bb[1] - dt_bb[3];
dt_y1 = dt_bb[1] + dt_bb[3] * 2
ol_x = min(x1, dt_x1) - max(x0, dt_x0)
ol_y = min(y1, dt_y1) - max(y0, dt_y0)
ol_area = ol_x * ol_y
s_x = max(x1, dt_x1) - min(x0, dt_x0)
s_y = max(y1, dt_y1) - min(y0, dt_y0)
sum_area = s_x * s_y
iou = np.round(ol_area / (sum_area + np.spacing(1)), 2)
score = np.round(dt['score'], 2)
print(f"score: {dt['score']}")
if iou < 0.1 or score < threshold:
continue
else:
print(f'iou: {iou}')
dt_w = dt_x1 - dt_x0
dt_h = dt_y1 - dt_y0
ref = min(dt_w, dt_h)
num_box += 1
sum_score += dt['score']
dt_joints = np.array(dt['keypoints']).reshape(20, -1)
joints_dict = map_joint_dict(dt_joints)
# print(joints_dict)
# print(link_pairs)
# print(dt_joints)
# stick
for k, link_pair in enumerate(link_pairs):
if link_pair[0] in joints_dict \
and link_pair[1] in joints_dict:
# print(link_pair[0])
# print(vg)
if dt_joints[link_pair[0] - 1, 2] < joint_thres \
or dt_joints[link_pair[1] - 1, 2] < joint_thres \
or vg[link_pair[0] - 1] == 0 \
or vg[link_pair[1] - 1] == 0:
continue
# if k in range(6, 11):
# lw = 1
# else:
lw = ref / 100.
line = mlines.Line2D(
np.array([joints_dict[link_pair[0]][0],
joints_dict[link_pair[1]][0]]),
np.array([joints_dict[link_pair[0]][1],
joints_dict[link_pair[1]][1]]),
ls='-', lw=lw, alpha=1, color=link_pair[2], )
line.set_zorder(0)
ax.add_line(line)
# black ring
for k in range(dt_joints.shape[0]):
if dt_joints[k, 2] < joint_thres \
or vg[link_pair[0]] == 0 \
or vg[link_pair[1]] == 0:
continue
if dt_joints[k, 0] > w or dt_joints[k, 1] > h:
continue
# if k in range(5):
# radius = 1
# else:
radius = ref / 100
circle = mpatches.Circle(tuple(dt_joints[k, :2]),
radius=radius,
ec='black',
fc=ring_color[k],
alpha=1,
linewidth=1)
circle.set_zorder(1)
ax.add_patch(circle)
avg_score = (sum_score / (num_box + np.spacing(1))) * 1000
plt.gca().xaxis.set_major_locator(plt.NullLocator())
plt.gca().yaxis.set_major_locator(plt.NullLocator())
plt.axis('off')
plt.subplots_adjust(top=1, bottom=0, left=0, right=1, hspace=0, wspace=0)
plt.margins(0, 0)
if save:
plt.savefig(save_path + \
'score_' + str(np.int(avg_score)) + "_" +
imgId["file_name"].split(".")[0] + '.png',
format='png', bbox_inckes='tight', dpi=100)
# plt.savefig(save_path + 'id_' + str(imgId) + '.pdf', format='pdf',
# bbox_inckes='tight', dpi=100)
# plt.show()
plt.close()
print(f"total mean rmse: {np.mean(mean_rmse_list)}")
print(f"total mean rmse mask: {np.mean(mean_rmse_mask_list)}")
def validate(config,
val_loader,
val_dataset,
model,
output_dir,
tb_log_dir,
writer_dict=None):
model.eval()
if config.MODEL.NAME == 'pose_hourglass':
transforms = torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
])
else:
transforms = torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
# rmse = AverageMeter()
parser = HeatmapParser(config)
all_preds = []
all_scores = []
pbar = tqdm(total=len(val_dataset)) if config.TEST.LOG_PROGRESS else None
for i, (images, annos) in enumerate(val_loader):
assert 1 == images.size(0), 'Test batch size should be 1'
image = images[0].cpu().numpy()
# size at scale 1.0
base_size, center, scale = get_multi_scale_size(
image, config.DATASET.INPUT_SIZE, 1.0,
min(config.TEST.SCALE_FACTOR))
with torch.no_grad():
final_heatmaps = None
tags_list = []
for idx, s in enumerate(
sorted(config.TEST.SCALE_FACTOR, reverse=True)):
input_size = config.DATASET.INPUT_SIZE
image_resized, center, scale = resize_align_multi_scale(
image, input_size, s, min(config.TEST.SCALE_FACTOR))
image_resized = transforms(image_resized)
image_resized = image_resized.unsqueeze(0).cuda()
outputs, heatmaps, tags = get_multi_stage_outputs(
config, model, image_resized, config.TEST.FLIP_TEST,
config.TEST.PROJECT2IMAGE, base_size)
final_heatmaps, tags_list = aggregate_results(
config, s, final_heatmaps, tags_list, heatmaps, tags)
final_heatmaps = final_heatmaps / float(
len(config.TEST.SCALE_FACTOR))
tags = torch.cat(tags_list, dim=4)
grouped, scores = parser.parse(final_heatmaps, tags,
config.TEST.ADJUST,
config.TEST.REFINE)
final_results = get_final_preds(
grouped, center, scale,
[final_heatmaps.size(3),
final_heatmaps.size(2)])
if config.TEST.LOG_PROGRESS:
pbar.update()
# if i % config.PRINT_FREQ == 0:
# prefix = '{}_{}'.format(os.path.join(output_dir, 'result_valid'), i)
# # logger.info('=> write {}'.format(prefix))
# save_valid_image(image, final_results, '{}.jpg'.format(prefix), dataset=val_dataset.name)
# # save_debug_images(config, image_resized, None, None, outputs, prefix)
all_preds.append(final_results)
all_scores.append(scores)
if config.TEST.LOG_PROGRESS:
pbar.close()
results, res_file = val_dataset.evaluate(config, all_preds, all_scores,
output_dir)
##################################
gt_file = val_dataset._get_anno_file_name()
coco = COCO(gt_file)
coco_dt = coco.loadRes(res_file)
coco_eval = COCOeval(coco, coco_dt, 'keypoints')
coco_eval._prepare()
gts_ = coco_eval._gts
dts_ = coco_eval._dts
p = coco_eval.params
p.imgIds = list(np.unique(p.imgIds))
if p.useCats:
p.catIds = list(np.unique(p.catIds))
p.maxDets = sorted(p.maxDets)
# loop through images, area range, max detection number
catIds = p.catIds if p.useCats else [-1]
pcutoff01 = 0.1
pcutoff06 = 0.6
mean_rmse_list = []
mean_rmse_pcutoff01_list = []
mean_rmse_pcutoff06_list = []
for catId in catIds:
for imgId in p.imgIds:
# dimension here should be Nxm
gts = gts_[imgId, catId]
dts = dts_[imgId, catId]
if len(gts) != 0 and len(dts) != 0:
npgt = np.array(gts[0]["keypoints"])
npdt = np.array(dts[0]["keypoints"])
mask01 = npdt[2::3] >= pcutoff01
mask06 = npdt[2::3] >= pcutoff06
RMSE = np.sqrt((npgt[0::3] - npdt[0::3])**2 +
(npgt[1::3] - npdt[1::3])**2)
RMSE_pcutoff01 = RMSE[mask01]
RMSE_pcutoff06 = RMSE[mask06]
mean_rmse = np.round(np.nanmean(RMSE.flatten()), 2)
mean_rmse_pcutoff01 = np.nanmean(RMSE_pcutoff01.flatten())
mean_rmse_pcutoff06 = np.nanmean(RMSE_pcutoff06.flatten())
mean_rmse_list.append(mean_rmse)
mean_rmse_pcutoff01_list.append(mean_rmse_pcutoff01)
mean_rmse_pcutoff06_list.append(mean_rmse_pcutoff06)
print(f"Mean RMSE: {np.mean(mean_rmse_list)}")
print(
f"Mean RMSE p-cutoff 0.1: {np.round(np.mean(mean_rmse_pcutoff01_list),2)}"
)
print(
f"Mean RMSE p-cutoff 0.6: {np.round(np.mean(mean_rmse_pcutoff06_list),2)}"
)
global_steps = writer_dict['valid_global_steps']
writer_dict["writer"].add_scalar("val_rmse", np.mean(mean_rmse_list),
global_steps)
writer_dict["writer"].add_scalar("val_rmse_pcutoff_0.1",
np.mean(mean_rmse_pcutoff01_list),
global_steps)
writer_dict["writer"].add_scalar("val_rmse_pcutoff_0.6",
np.mean(mean_rmse_pcutoff06_list),
global_steps)
writer_dict['valid_global_steps'] = global_steps + 1
return np.mean(mean_rmse_list)
def evaluate(self,
results,
metric='bbox',
logger=None,
jsonfile_prefix=None,
classwise=False,
proposal_nums=(100, 300, 1000),
iou_thrs=None,
metric_items=None):
"""Evaluation in COCO protocol.
Args:
results (list[list | tuple]): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated. Options are
'bbox', 'segm', 'proposal', 'proposal_fast'.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
jsonfile_prefix (str | None): The prefix of json files. It includes
the file path and the prefix of filename, e.g., "a/b/prefix".
If not specified, a temp file will be created. Default: None.
classwise (bool): Whether to evaluating the AP for each class.
proposal_nums (Sequence[int]): Proposal number used for evaluating
recalls, such as recall@100, recall@1000.
Default: (100, 300, 1000).
iou_thrs (Sequence[float], optional): IoU threshold used for
evaluating recalls/mAPs. If set to a list, the average of all
IoUs will also be computed. If not specified, [0.50, 0.55,
0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95] will be used.
Default: None.
metric_items (list[str] | str, optional): Metric items that will
be returned. If not specified, ``['AR@100', 'AR@300',
'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ]`` will be
used when ``metric=='proposal'``, ``['mAP', 'mAP_50', 'mAP_75',
'mAP_s', 'mAP_m', 'mAP_l']`` will be used when
``metric=='bbox' or metric=='segm'``.
Returns:
dict[str, float]: COCO style evaluation metric.
"""
metrics = metric if isinstance(metric, list) else [metric]
allowed_metrics = [
'bbox', 'segm', 'proposal', 'proposal_fast', 'pixel'
]
for metric in metrics:
if metric not in allowed_metrics:
raise KeyError(f'metric {metric} is not supported')
if iou_thrs is None:
iou_thrs = np.linspace(.5,
0.95,
int(np.round((0.95 - .5) / .05)) + 1,
endpoint=True)
if metric_items is not None:
if not isinstance(metric_items, list):
metric_items = [metric_items]
result_files, tmp_dir = self.format_results(results, jsonfile_prefix)
eval_results = OrderedDict()
cocoGt = self.coco
for metric in metrics:
msg = f'Evaluating {metric}...'
if logger is None:
msg = '\n' + msg
print_log(msg, logger=logger)
if metric == 'proposal_fast':
ar = self.fast_eval_recall(results,
proposal_nums,
iou_thrs,
logger='silent')
log_msg = []
for i, num in enumerate(proposal_nums):
eval_results[f'AR@{num}'] = ar[i]
log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}')
log_msg = ''.join(log_msg)
print_log(log_msg, logger=logger)
continue
if metric not in result_files:
raise KeyError(f'{metric} is not in results')
try:
cocoDt = cocoGt.loadRes(result_files[metric])
except IndexError:
print_log('The testing results of the whole dataset is empty.',
logger=logger,
level=logging.ERROR)
break
if metric == 'pixel':
iou_type = 'segm'
else:
iou_type = 'bbox' if metric == 'proposal' else metric
cocoEval = COCOeval(cocoGt, cocoDt, iou_type)
cocoEval.params.catIds = self.cat_ids
cocoEval.params.imgIds = self.img_ids
cocoEval.params.maxDets = list(proposal_nums)
cocoEval.params.iouThrs = iou_thrs
# mapping of cocoEval.stats
coco_metric_names = {
'mAP': 0,
'mAP_50': 1,
'mAP_75': 2,
'mAP_s': 3,
'mAP_m': 4,
'mAP_l': 5,
'AR@100': 6,
'AR@300': 7,
'AR@1000': 8,
'AR_s@1000': 9,
'AR_m@1000': 10,
'AR_l@1000': 11
}
if metric_items is not None:
for metric_item in metric_items:
if metric_item not in coco_metric_names:
raise KeyError(
f'metric item {metric_item} is not supported')
if metric == 'proposal':
cocoEval.params.useCats = 0
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if metric_items is None:
metric_items = [
'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000',
'AR_m@1000', 'AR_l@1000'
]
for item in metric_items:
val = float(
f'{cocoEval.stats[coco_metric_names[item]]:.3f}')
eval_results[item] = val
elif metric == 'pixel':
cocoEval._prepare()
full_gt_mask = []
full_dt_mask = []
for img_index in cocoGt.imgToAnns.keys():
gt_img_anns = cocoGt.imgToAnns[img_index]
gt_img_info = cocoGt.imgs[img_index]
dt_img_anns = cocoDt.imgToAnns[img_index]
dt_img_info = cocoDt.imgs[img_index]
gt_mask = anns_to_mask(gt_img_anns, gt_img_info['height'],
gt_img_info['width'])
dt_mask = anns_to_mask(dt_img_anns, dt_img_info['height'],
gt_img_info['width'])
full_gt_mask.append(gt_mask)
full_dt_mask.append(dt_mask)
max_size = np.max([mask.shape[:2] for mask in full_gt_mask],
axis=0)
for i, _ in enumerate(full_gt_mask):
full_gt_mask[i] = pad_img_to_size(full_gt_mask[i],
max_size)
full_dt_mask[i] = pad_img_to_size(full_dt_mask[i],
max_size)
full_gt_mask = np.stack(full_gt_mask)
full_dt_mask = np.stack(full_dt_mask)
metrics = calculate_metrics(full_dt_mask, full_gt_mask)
table_data = [['Pixel Metric', 'Value']]
# add the pixel-wise metrics to the output
for name, value in metrics._asdict().items():
eval_results[f'{metric}_{name}'] = value
table_data.append([f'{metric}_{name}', '%.4f' % value])
table = AsciiTable(table_data)
print_log('\n' + table.table, logger=logger)
else:
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if classwise: # Compute per-category AP
# Compute per-category AP
# from https://github.com/facebookresearch/detectron2/
precisions = cocoEval.eval['precision']
# precision: (iou, recall, cls, area range, max dets)
assert len(self.cat_ids) == precisions.shape[2]
results_per_category = []
for idx, catId in enumerate(self.cat_ids):
# area range index 0: all area ranges
# max dets index -1: typically 100 per image
nm = self.coco.loadCats(catId)[0]
precision = precisions[:, :, idx, 0, -1]
precision = precision[precision > -1]
if precision.size:
ap = np.mean(precision)
else:
ap = float('nan')
results_per_category.append(
(f'{nm["name"]}', f'{float(ap):0.3f}'))
num_columns = min(6, len(results_per_category) * 2)
results_flatten = list(
itertools.chain(*results_per_category))
headers = ['category', 'AP'] * (num_columns // 2)
results_2d = itertools.zip_longest(*[
results_flatten[i::num_columns]
for i in range(num_columns)
])
table_data = [headers]
table_data += [result for result in results_2d]
table = AsciiTable(table_data)
print_log('\n' + table.table, logger=logger)
if metric_items is None:
metric_items = [
'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l'
]
for metric_item in metric_items:
key = f'{metric}_{metric_item}'
val = float(
f'{cocoEval.stats[coco_metric_names[metric_item]]:.3f}'
)
eval_results[key] = val
ap = cocoEval.stats[:6]
eval_results[f'{metric}_mAP_copypaste'] = (
f'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} '
f'{ap[4]:.3f} {ap[5]:.3f}')
if tmp_dir is not None:
tmp_dir.cleanup()
return eval_results
class MapEvaluator:
def __init__(self, gt_json, cat_ids):
# Eval options
self.iou_threshs = [.5]
self.max_dets = [1, 10, 100]
self.cat_ids = cat_ids
self.coco_gt = COCO(gt_json)
def _init_coco_eval(self, dt_json_data):
coco_dt = self.coco_gt.loadRes(dt_json_data)
self.coco_eval = COCOeval(self.coco_gt, coco_dt, 'bbox')
self.coco_eval.params.iouThrs = np.array(self.iou_threshs)
self.coco_eval.params.maxDets = self.max_dets
self.coco_eval.params.catIds = self.cat_ids
def do_eval(self, dt_json_data):
self._init_coco_eval(dt_json_data)
self.coco_eval.evaluate()
self.coco_eval.accumulate()
def do_eval_and_print(self, dt_json_data):
self._init_coco_eval(dt_json_data)
self.coco_eval.evaluate()
self.coco_eval.accumulate()
self.coco_eval.summarize()
def get_avg_precision_recall(self, t=0, a=0, m=-1):
p = self.coco_eval.eval['precision'][t, :, :, a, m]
r = self.coco_eval.eval['recall'][t, :, a, m]
# Removing -1 entries for classes that have no GT objects.
p = p[:, np.where(r > -1)[0]]
r = r[r > -1]
assert (np.all(p >= 0.) and np.all(r >= 0.))
ap = p.mean(axis=1)
ar = r.mean()
return ap, ar
def evaluateImg(self, imgId, catId, aRng=(-np.inf, np.inf), maxDet=100):
p = self.coco_eval.params
# add backward compatibility if useSegm is specified in params
p.iouType = 'bbox'
p.imgIds = list(np.unique(p.imgIds))
if p.useCats:
p.catIds = list(np.unique(p.catIds))
p.maxDets = sorted(p.maxDets)
self.coco_eval.params = p
self.coco_eval._prepare()
self.coco_eval.ious = {
(imgId, catId): self.coco_eval.computeIoU(imgId, catId)
}
img_eval = self.coco_eval.evaluateImg(imgId, catId, aRng, maxDet)
inds = img_eval['dtIds']
matches = img_eval['dtMatches']
return inds, matches