def evaluate_mAP_manufacturer(json_datasets,
roidbs,
all_boxes_list,
output_dir,
cleanup=False):
""" LJY
all_boxes: num_cls x num_images x [num_boxes x 5]
"""
mAP_folder = '/home/liujingyu/code/mAP'
json_dataset = json_datasets[0]
all_boxes = all_boxes_list[0]
roidb = roidbs[0]
for i, entry in enumerate(roidb): # for each image
print(i, entry['eva_id'], entry['file_name'])
file_name = entry['file_name'][:-4] + '.txt'
manufacturer, manufacturer_model = entry['manufacturer'], entry[
'manufacturer_model']
if manufacturer is not None:
# pdb.set_trace()
manufacturer, manufacturer_model = entry['manufacturer'].strip(
'\"'), entry['manufacturer_model'].strip('\"')
if not osp.exists(
osp.join(mAP_folder, 'ground-truth_' + manufacturer)):
os.makedirs(
osp.join(mAP_folder, 'ground-truth_' + manufacturer))
os.makedirs(osp.join(mAP_folder, 'predicted_' + manufacturer))
fgt = open(
osp.join(mAP_folder, 'ground-truth_' + manufacturer,
file_name), 'w')
fpred = open(
osp.join(mAP_folder, 'predicted_' + manufacturer, file_name),
'w')
for cls_ind, cls in enumerate(
json_dataset.classes): # for each cls
if cls == '__background__':
continue
if cls_ind >= len(all_boxes):
break
gt_classes = roidb[i]['gt_classes']
ind = np.where(gt_classes == cls_ind)
gt_boxes = roidb[i]['boxes'][ind]
dets = all_boxes[cls_ind][i]
# write gt files, format: tvmonitor 2 10 173 238
for k in range(gt_boxes.shape[0]):
s = '{} {:f} {:f} {:f} {:f}'.format(
cls, gt_boxes[k, 0], gt_boxes[k, 1], gt_boxes[k, 2],
gt_boxes[k, 3])
fgt.write(s)
fgt.write('\n')
# write pred files, format: tvmonitor 0.471781 0 13 174 244
for k in range(dets.shape[0]):
s = '{} {:f} {:f} {:f} {:f} {:f}'.format(
cls, dets[k, -1], dets[k, 0], dets[k, 1], dets[k, 2],
dets[k, 3])
fpred.write(s)
fpred.write('\n')
if gt_boxes.shape[0] > 0: # then we draw gt box and pred boxes
im = cv2.imread(entry['image'])
more_text = str(entry['eva_id']) + ' ' + entry['doc_name']
im = vis_boxes_ljy(im, gt_boxes, dets[:, :-1], more_text)
out_path = os.path.join('Outputs/vis', cls,
entry['file_name'])
cv2.imwrite(out_path, im)
pdb.set_trace()
def evaluate_boxes_recall_and_FP(json_dataset,
roidb,
all_boxes,
output_dir,
cleanup=False):
""" LJY
all_boxes: num_cls x num_images x [num_boxes x 5]
"""
res_file = os.path.join(output_dir,
'bbox_' + json_dataset.name + '_results')
res_file += '.json'
GTs = [0 for _ in json_dataset.classes]
TPs = [0 for _ in json_dataset.classes]
FPs = [0 for _ in json_dataset.classes]
NPreds = [0 for _ in json_dataset.classes]
FP_imgs = [0 for _ in json_dataset.classes] # num of images that have FPs
for i, entry in enumerate(roidb): # for each image
for cls_ind, cls in enumerate(json_dataset.classes): # for each cls
if cls == '__background__':
continue
if cls_ind >= len(all_boxes):
break
gt_classes = roidb[i]['gt_classes']
ind = np.where(gt_classes == cls_ind)
gt_boxes = roidb[i]['boxes'][ind]
dets = all_boxes[cls_ind][i]
scores = dets[:, -1]
pred_boxes = dets[scores > 0.3, :4] # xyxy
IOUs = bbox_overlaps(pred_boxes, gt_boxes) # N x 4, K x 4 -> N x K
npred, ngt = pred_boxes.shape[0], gt_boxes.shape[0]
hit_map = IOUs > 0.01
tp = hit_map.any(
axis=0).sum() # num of true positive of this cls, this image
fp = npred - hit_map.any(
axis=1).sum() # num of false positive of this cls, this image
GTs[cls_ind] += ngt
TPs[cls_ind] += tp
FPs[cls_ind] += fp
NPreds[cls_ind] += npred
if fp > 0:
FP_imgs[cls_ind] += 1
# pdb.set_trace()
if ngt > 0: # then we draw gt box and pred boxes
im = cv2.imread(entry['image'])
more_text = str(entry['eva_id']) + ' ' + entry['doc_name']
im = vis_boxes_ljy(im, gt_boxes, pred_boxes, more_text)
out_path = os.path.join(
'/data5/liujingyu/mask_rcnn_Outputs/vis', cls,
entry['file_name'])
cv2.imwrite(out_path, im)
print('# GT on instance level:')
for GT in GTs[1:]:
print(GT)
print('recall on instance level')
for TP, GT in zip(TPs[1:], GTs[1:]):
print("%.3f" % (TP / GT))
print('FPs per image')
for FP, NPred in zip(FPs[1:], NPreds[1:]):
print("%.3f" % (FP / len(roidb)))
pdb.set_trace()
def evaluate_mAP_combine(json_datasets,
roidbs,
all_boxes_list,
output_dir,
cleanup=False):
""" LJY
all_boxes: num_cls x num_images x [num_boxes x 5]
"""
json_dataset = json_datasets[0]
mAP_folder = '/home/liujingyu/code/mAP'
roidb, roidb_part = roidbs[0], roidbs[1]
all_boxes, all_boxes_part = all_boxes_list[0], all_boxes_list[1]
small_classes = ['结节', '肺实变', '膈面异常', '骨折']
for i, (entry, entry_part) in enumerate(zip(
roidb, roidb_part)): # for each pair of images
# print(i, entry['eva_id'], entry['file_name'])
assert entry['file_name'] == entry_part['file_name']
file_name = entry['file_name'][:-4] + '.txt'
fgt = open(osp.join(mAP_folder, 'ground-truth', file_name), 'w')
fpred = open(osp.join(mAP_folder, 'predicted', file_name), 'w')
for cls_ind, cls in enumerate(json_dataset.classes): # for each cls
if cls == '__background__':
continue
if cls_ind >= len(all_boxes):
break
gt_classes = roidb[i]['gt_classes']
ind = np.where(gt_classes == cls_ind)
gt_boxes = roidb[i]['boxes'][ind]
dets = all_boxes[cls_ind][i] # N x 5, [x1, y1, x2, y2, score]
dets_part = all_boxes_part[cls_ind][i] # N x 5
# offset the dets_part based on offset
dets_part[:, 0] += entry_part['offset_x']
dets_part[:, 2] += entry_part['offset_x']
dets_part[:, 1] += entry_part['offset_y']
dets_part[:, 3] += entry_part['offset_y']
# select
# if cls in small_classes:
# dets = dets_part
# merge
dets = np.vstack((dets, dets_part))
# NMS on dets and dets_part
if cfg.TEST.SOFT_NMS.ENABLED:
nms_dets, _ = box_utils.soft_nms(
dets,
sigma=cfg.TEST.SOFT_NMS.SIGMA,
overlap_thresh=cfg.TEST.NMS,
score_thresh=0.0001,
method=cfg.TEST.SOFT_NMS.METHOD)
else:
keep = box_utils.nms(dets, cfg.TEST.NMS)
nms_dets = dets[keep, :]
# Refine the post-NMS boxes using bounding-box voting
if cfg.TEST.BBOX_VOTE.ENABLED:
nms_dets = box_utils.box_voting(
nms_dets,
dets,
cfg.TEST.BBOX_VOTE.VOTE_TH,
scoring_method=cfg.TEST.BBOX_VOTE.SCORING_METHOD)
dets = nms_dets
# write gt boxes, format: tvmonitor 2 10 173 238
for k in range(gt_boxes.shape[0]):
s = '{} {:f} {:f} {:f} {:f}'.format(cls, gt_boxes[k, 0],
gt_boxes[k, 1],
gt_boxes[k,
2], gt_boxes[k,
3])
fgt.write(s)
fgt.write('\n')
if cls == '肿块' or cls == '结节' or cls == '钙化' or cls == '乳头影':
s = '{} {:f} {:f} {:f} {:f}'.format(
'肿块结节钙化', gt_boxes[k, 0], gt_boxes[k, 1],
gt_boxes[k, 2], gt_boxes[k, 3])
fgt.write(s)
fgt.write('\n')
if cls == '纤维化表现' or cls == '肺实变' or cls == '肺纹理增多' or cls == '肿块' or cls == '弥漫性结节':
s = '{} {:f} {:f} {:f} {:f}'.format(
'高密度影', gt_boxes[k, 0], gt_boxes[k, 1], gt_boxes[k, 2],
gt_boxes[k, 3])
fgt.write(s)
fgt.write('\n')
if cls == '气胸' or cls == '气肿':
s = '{} {:f} {:f} {:f} {:f}'.format(
'低密度影', gt_boxes[k, 0], gt_boxes[k, 1], gt_boxes[k, 2],
gt_boxes[k, 3])
fgt.write(s)
fgt.write('\n')
# write pred boxes, format: tvmonitor 0.471781 0 13 174 244
for k in range(dets.shape[0]):
s = '{} {:f} {:f} {:f} {:f} {:f}'.format(
cls, dets[k, -1], dets[k, 0], dets[k, 1], dets[k, 2],
dets[k, 3])
fpred.write(s)
fpred.write('\n')
if cls == '肿块' or cls == '结节' or cls == '钙化' or cls == '乳头影':
s = '{} {:f} {:f} {:f} {:f} {:f}'.format(
'肿块结节钙化', dets[k, -1], dets[k, 0], dets[k, 1],
dets[k, 2], dets[k, 3])
fpred.write(s)
fpred.write('\n')
if cls == '纤维化表现' or cls == '肺实变' or cls == '肺纹理增多' or cls == '肿块' or cls == '弥漫性结节':
s = '{} {:f} {:f} {:f} {:f} {:f}'.format(
'高密度影', dets[k, -1], dets[k, 0], dets[k, 1],
dets[k, 2], dets[k, 3])
fpred.write(s)
fpred.write('\n')
if cls == '气胸' or cls == '气肿':
s = '{} {:f} {:f} {:f} {:f} {:f}'.format(
'低密度影', dets[k, -1], dets[k, 0], dets[k, 1],
dets[k, 2], dets[k, 3])
fpred.write(s)
fpred.write('\n')
if gt_boxes.shape[0] > 0: # then we draw gt boxes and pred boxes
im = cv2.imread(entry['image'])
more_text = str(entry['eva_id']) + ' ' + entry['doc_name']
im = vis_boxes_ljy(im, gt_boxes, dets[:, :-1], more_text)
out_path = os.path.join(
'/data5/liujingyu/mask_rcnn_Outputs/vis', cls,
entry['file_name'])
cv2.imwrite(out_path, im)
pdb.set_trace()