def icdar_evaluate(model,
target_size,
test_path,
dataset):
if dataset == 'IC15':
output = './datasets/IC_eval/icdar15'
elif dataset == 'IC13':
output = './datasets/IC_eval/icdar13'
else:
raise NotImplementedError
ims_dir = test_path
out_dir = './temp'
if not os.path.exists(out_dir):
os.makedirs(out_dir)
ims_list = [x for x in os.listdir(ims_dir) if is_image(x)]
s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'Hmean')
nt = 0
for idx, im_name in enumerate(tqdm(ims_list, desc=s)):
im_path = os.path.join(ims_dir, im_name)
im = cv2.cvtColor(cv2.imread(im_path, cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)
dets = im_detect(model, im, target_sizes=target_size)
nt += len(dets)
out_file = os.path.join(out_dir, 'res_' + im_name[:im_name.rindex('.')] + '.txt')
with codecs.open(out_file, 'w', 'utf-8') as f:
if dets.shape[0] == 0:
continue
if dataset == 'IC15':
res = sort_corners(rbox_2_quad(dets[:, 2:]))
for k in range(dets.shape[0]):
f.write('{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f}\n'.format(
res[k, 0], res[k, 1], res[k, 2], res[k, 3],
res[k, 4], res[k, 5], res[k, 6], res[k, 7])
)
if dataset == 'IC13':
res = rbox_2_aabb(dets[:, 2:])
for k in range(dets.shape[0]):
f.write('{:.0f},{:.0f},{:.0f},{:.0f}\n'.format(
res[k, 0], res[k, 1], res[k, 2], res[k, 3])
)
zip_name = 'submit.zip'
make_zip(out_dir, zip_name)
shutil.move(os.path.join('./', zip_name), os.path.join(output, zip_name))
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
result = os.popen('cd {0} && python script.py -g=gt.zip -s=submit.zip '.format(output)).read()
sep = result.split(':')
precision = sep[1][:sep[1].find(',')].strip()
recall = sep[2][:sep[2].find(',')].strip()
f1 = sep[3][:sep[3].find(',')].strip()
map = 0
p = eval(precision)
r = eval(recall)
hmean = eval(f1)
# display result
pf = '%20s' + '%10.3g' * 6 # print format
print(pf % ('all', len(ims_list), nt, p, r, 0, hmean))
return p, r, map, hmean
def data_evaluate(model, target_size, test_path, conf=0.01, dataset=None):
root_dir = 'datasets/evaluate'
out_dir = os.path.join(root_dir, 'detection-results')
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
ds = DATASETS[dataset]()
with open(test_path, 'r') as f:
if dataset == 'VOC':
im_dir = test_path.replace('/ImageSets/Main/test.txt',
'/JPEGImages')
ims_list = [
os.path.join(im_dir,
x.strip('\n') + '.jpg') for x in f.readlines()
]
else:
ims_list = [
x.strip('\n') for x in f.readlines() if is_image(x.strip('\n'))
]
s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R',
'[email protected]', 'Hmean')
nt = 0
for idx, im_path in enumerate(tqdm(ims_list, desc=s)):
print(idx, im_path)
im_name = os.path.split(im_path)[1]
im = cv2.cvtColor(cv2.imread(im_path, cv2.IMREAD_COLOR),
cv2.COLOR_BGR2RGB)
dets = im_detect(model, im, target_sizes=target_size, conf=conf)
nt += len(dets)
out_file = os.path.join(out_dir,
im_name[:im_name.rindex('.')] + '.txt')
with codecs.open(out_file, 'w', 'utf-8') as f:
if dets.shape[0] == 0:
f.close()
continue
res = sort_corners(rbox_2_quad(dets[:, 2:]))
for k in range(dets.shape[0]):
f.write(
'{} {:.2f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f}\n'
.format(ds.return_class(dets[k, 0]), dets[k, 1], res[k, 0],
res[k, 1], res[k, 2], res[k, 3], res[k, 4],
res[k, 5], res[k, 6], res[k, 7]))
# assert len(os.listdir(os.path.join(root_dir,'ground-truth'))) != 0, 'No labels found in test/ground-truth!! '
assert len(os.listdir("./HRSC2016/Test/Annotations")
) != 0, 'No labels found in test/ground-truth!! '
mAP = eval_mAP(root_dir, use_07_metric=False)
# display result
pf = '%20s' + '%10.3g' * 6 # print format
print(pf % ('all', len(ims_list), nt, 0, 0, mAP, 0))
# thres = [0.5,0.6,0.7,0.8,0.9]
# for thre in thres:
# ap = eval_mAP(root_dir, use_07_metric=True, thres = thre)
# print('ap_{}: {}'.format(thre, ap))
return 0, 0, mAP, 0
def dota_evaluate(model,
target_size,
test_path,
conf = 0.01):
#
root_data, evaldata = os.path.split(test_path)
splitdata = evaldata + 'split'
ims_dir = os.path.join(root_data, splitdata + '/' + 'images')
root_dir = 'outputs'
res_dir = os.path.join(root_dir, 'detections') # 裁剪图像的检测结果
integrated_dir = os.path.join(root_dir, 'integrated') # 将裁剪图像整合后成15个txt的结果
merged_dir = os.path.join(root_dir, 'merged') # 将整合后的结果NMS
if os.path.exists(root_dir):
shutil.rmtree(root_dir)
os.makedirs(root_dir)
for f in [res_dir, integrated_dir, merged_dir]:
if os.path.exists(f):
shutil.rmtree(f)
os.makedirs(f)
ds = DOTADataset()
# loss = torch.zeros(3)
ims_list = [x for x in os.listdir(ims_dir) if is_image(x)]
s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', '[email protected]', 'Hmean')
nt = 0
for idx, im_name in enumerate(tqdm(ims_list, desc=s)):
im_path = os.path.join(ims_dir, im_name)
im = cv2.cvtColor(cv2.imread(im_path, cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)
dets = im_detect(model, im, target_sizes=target_size, conf = conf)
nt += len(dets)
out_file = os.path.join(res_dir, im_name[:im_name.rindex('.')] + '.txt')
with codecs.open(out_file, 'w', 'utf-8') as f:
if dets.shape[0] == 0:
f.close()
continue
res = sort_corners(rbox_2_quad(dets[:, 2:]))
for k in range(dets.shape[0]):
f.write('{:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {} {} {:.2f}\n'.format(
res[k, 0], res[k, 1], res[k, 2], res[k, 3],
res[k, 4], res[k, 5], res[k, 6], res[k, 7],
ds.return_class(dets[k, 0]), im_name[:-4], dets[k, 1],)
)
ResultMerge(res_dir, integrated_dir, merged_dir)
## calc mAP
mAP, classaps = task1_eval(merged_dir, test_path)
# # display result
pf = '%20s' + '%10.3g' * 6 # print format
print(pf % ('all', len(ims_list), nt, 0, 0, mAP, 0))
return 0, 0, mAP, 0
def evaluate(model, args):
#
if args.dataset == 'ICDAR 2013':
ims_dir = os.path.join(args.test_dir, args.dataset, 'Task 1/Test/IMS')
eval_dir = os.path.join(args.eval_dir, 'icdar13')
elif args.dataset == 'ICDAR 2015':
ims_dir = os.path.join(args.test_dir, args.dataset, 'Task 1/Test/IMS')
eval_dir = os.path.join(args.eval_dir, 'icdar15')
elif args.dataset == 'ICDAR 2017':
ims_dir = os.path.join(args.test_dir, args.dataset, 'Task 1/Test/IMS')
eval_dir = os.path.join(args.eval_dir, 'icdar17')
elif args.dataset == 'COCO':
ims_dir = os.path.join(args.test_dir, args.dataset, 'Test/IMS')
eval_dir = os.path.join(args.eval_dir, 'coco')
else:
raise NotImplementedError
#
out_dir = './temp'
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#
ims_list = [x for x in os.listdir(ims_dir) if is_image(x)]
for idx, im_name in enumerate(ims_list):
im_path = os.path.join(ims_dir, im_name)
im = cv2.cvtColor(cv2.imread(im_path, cv2.IMREAD_COLOR),
cv2.COLOR_BGR2RGB)
_t.tic()
dets = im_detect(model, im, target_sizes=args.target_size)
print('\rim_detect: {:d}/{:d}, time: {:.3f}, '.format(
idx + 1, len(ims_list), _t.toc()),
end='')
if args.dataset == 'ICDAR 2017':
out_file = os.path.join(out_dir,
im_name[:im_name.rindex('.')] + '.txt')
out_file = out_file.replace('ts', 'res')
elif args.dataset == 'COCO':
im_index = im_name[im_name.rindex('_') + 1:im_name.rindex('.')]
out_file = os.path.join(out_dir,
'res_' + str(int(im_index)) + '.txt')
else:
out_file = os.path.join(
out_dir, 'res_' + im_name[:im_name.rindex('.')] + '.txt')
with codecs.open(out_file, 'w', 'utf-8') as f:
if dets.shape[0] == 0:
continue
if args.dataset == 'ICDAR 2013':
res = rbox_2_aabb(dets[:, 2:])
for k in range(dets.shape[0]):
f.write('{:.0f},{:.0f},{:.0f},{:.0f}\n'.format(
res[k, 0], res[k, 1], res[k, 2], res[k, 3]))
elif args.dataset == 'ICDAR 2015':
res = sort_corners(rbox_2_quad(dets[:, 2:]))
for k in range(dets.shape[0]):
f.write(
'{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f}\n'
.format(res[k, 0], res[k, 1], res[k, 2], res[k, 3],
res[k, 4], res[k, 5], res[k, 6], res[k, 7]))
elif args.dataset == 'ICDAR 2017':
res = sort_corners(rbox_2_quad(dets[:, 2:]))
for k in range(dets.shape[0]):
f.write(
'{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.0f},{:.2f}\n'
.format(res[k, 0], res[k, 1], res[k, 2], res[k, 3],
res[k, 4], res[k, 5], res[k, 6], res[k, 7],
dets[k, 1]))
if args.dataset == 'COCO':
res = rbox_2_aabb(dets[:, 2:])
for k in range(dets.shape[0]):
f.write('{:.0f},{:.0f},{:.0f},{:.0f},{:.2f}\n'.format(
res[k, 0], res[k, 1], res[k, 2], res[k, 3], dets[k,
1]))
#
zip_name = 'submit.zip'
make_zip(out_dir, zip_name)
shutil.move(os.path.join('./', zip_name), os.path.join(eval_dir, zip_name))
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
if args.dataset == 'ICDAR 2013' or args.dataset == 'ICDAR 2015':
os.system(
'cd {0} && python2 script.py -g=gt.zip -s=submit.zip '.format(
eval_dir))
print()
else:
# evaluated online
raise NotImplementedError
