def cityperson_eval(src_pth,
annFile,
CUT_WH=None,
ignore_uncertain=False,
use_iod_for_ignore=False,
catIds=[],
use_citypersons_standard=True,
tiny_scale=1.0,
iou_ths=None,
setup_labels=None):
if os.path.isdir(src_pth):
resFile = src_pth + '/' + 'bbox.json'
else:
resFile = src_pth
Params.CITYPERSON_STANDARD = use_citypersons_standard
if use_citypersons_standard:
kwargs = {}
if CUT_WH is None: CUT_WH = (1, 1)
else:
kwargs = {'filter_type': 'size'}
if CUT_WH is None: CUT_WH = (1, 1)
Params.TINY_SCALE = tiny_scale
Params.IOU_THS = iou_ths
kwargs.update({
'use_iod_for_ignore': use_iod_for_ignore,
'ignore_uncertain': ignore_uncertain
})
kwargs['given_catIds'] = len(catIds) > 0
annType = 'bbox' # specify type here
print('Running demo for *%s* results.' % annType)
# running evaluation
print('CUT_WH:', CUT_WH)
print('use_citypersons_standard:', use_citypersons_standard)
print('tiny_scale:', tiny_scale)
print(kwargs)
res_file = open("results.txt", "w")
Params.CUT_WH = CUT_WH
setupLbl = Params().SetupLbl
for id_setup in range(len(setupLbl)):
if (setup_labels is None) or (setupLbl[id_setup] in setup_labels):
cocoGt = COCO(annFile)
cocoDt = cocoGt.loadRes(resFile)
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt, cocoDt, annType, **kwargs)
cocoEval.params.imgIds = imgIds
cocoEval.evaluate(id_setup)
cocoEval.accumulate()
cocoEval.summarize(id_setup, res_file)
res_file.close()
def validate(annFile, dt_path):
mean_MR = []
for id_setup in range(0, 4):
cocoGt = COCO(annFile)
cocoDt = cocoGt.loadRes(dt_path)
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds
cocoEval.evaluate(id_setup)
cocoEval.accumulate()
mean_MR.append(cocoEval.summarize_nofile(id_setup))
return mean_MR
def evaluation(annFile,resFile,outFile = "results.txt"):
from coco import COCO # IMPORT THEIR COCO, not pycocotools
from eval_MR_multisetup import COCOeval
# running evaluation
res_file = open("results.txt", "w")
for id_setup in range(0,4):
cocoGt = COCO(annFile)
cocoDt = cocoGt.loadRes(resFile)
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt,cocoDt,'bbox')
cocoEval.params.imgIds = imgIds
cocoEval.evaluate(id_setup)
cocoEval.accumulate()
cocoEval.summarize(id_setup,res_file)
res_file.close()
def mMR(resFile, annFile):
'''
:param resFile: json file detect result : list =[ dict, dict ...] dict = {'image_id':, 'bbox':, 'score':,
'category_id': }bbox = [x,y,w,h] image_id = ***(no .jpg) category_id = 1 for person score must be sort from high to low
:param annFile: json file format is same as mscoco dataset for example instances_val_{}2014.json
:return: None
'''
res_file = open("results.txt", "w")
for id_setup in range(3, 4):
cocoGt = COCO(annFile)
cocoDt = cocoGt.loadRes(resFile)
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt, cocoDt, annType)
cocoEval.params.imgIds = imgIds
cocoEval.evaluate(id_setup)
cocoEval.accumulate()
cocoEval.summarize(id_setup, res_file)
res_file.close()
def eval_json_reasonable(annFile, resFile):
dt_path = os.path.split(resFile)[0]
respath = os.path.join(dt_path, 'results.txt')
res_file = open(respath, "w")
mr_reasonable = None
for id_setup in range(6):
cocoGt = COCO(annFile)
cocoDt = cocoGt.loadRes(resFile)
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt, cocoDt, annType)
cocoEval.params.imgIds = imgIds
cocoEval.evaluate(id_setup)
cocoEval.accumulate()
mean_mr = cocoEval.summarize(id_setup, res_file)
if id_setup == 0:
mr_reasonable = mean_mr
print('')
res_file.close()
return mr_reasonable
if os.path.isfile(main_path):
print('Given file {} with detections'.format(main_path))
res_file = None
resFile_txt = os.path.join(main_path)
resFile = os.path.join(main_path.replace('.txt', '.json'))
for id_setup in range(6):
cocoGt = COCO(annFile)
img_lut = {img['id']: img for img in cocoGt.imgs.values()}
ann_lut = {ann['id']: ann for ann in cocoGt.anns.values()}
ann_lut_by_img = collections.defaultdict(list)
for ann in cocoGt.anns.values():
img_id = ann['image_id']
ann_lut_by_img[img_id].append(ann)
cocoDt = cocoGt.loadRes(resFile)
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt, cocoDt, annType)
setup_name = cocoEval.params.SetupLbl[id_setup]
setup_savedir = os.path.join(base_save_dir, setup_name)
cocoEval.params.imgIds = imgIds
cocoEval.evaluate(id_setup)
misses = cocoEval.accumulate(plot=True, return_misses=True)
mean_mr = cocoEval.summarize(id_setup, res_file)
missed_heights = []
missed_visibilities = []
for img_id, ms in misses.items():
if len(ms):
image_name = img_lut[img_id]['im_name']
city = image_name.split('_')[0]
image_path = os.path.join(img_base, city, image_name)
image = cv2.imread(image_path)
bbs_missed = [ann_lut[m]['bbox'] for m in ms]
main_path = '../../output/valresults/precarious/h/off{}_finetuned'.format(
exp_name)
for f in sorted(os.listdir(main_path)):
print('file: {}'.format(f))
# initialize COCO detections api
dt_path = os.path.join(main_path, f)
resFile = os.path.join(dt_path, 'val_dt.json')
print('Load results from {}'.format(resFile))
respath = os.path.join(dt_path, 'results.txt')
# if os.path.exists(respath):
# continue
## running evaluation
if not os.path.exists(resFile):
print("Skipping {} ... Doesn't exist yet.".format(resFile))
continue
res_file = open(respath, "w")
for iou_thr in ious:
for id_setup in range(1):
cocoGt = COCO(annFile)
cocoDt = cocoGt.loadRes(resFile)
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt, cocoDt, annType)
cocoEval.params.imgIds = imgIds
cocoEval.evaluate(id_setup)
cocoEval.accumulate()
cocoEval.summarize(id_setup, res_file, iou_thr=iou_thr)
print('')
res_file.close()
print('file: {}'.format(f))
if 'val' in f:
continue
# initialize COCO detections api
dt_path = os.path.join(main_path, f)
resFile = os.path.join(dt_path, 'val_dt.json')
respath = os.path.join(dt_path, 'results.txt')
# if os.path.exists(respath):
# continue
## running evaluation
res_file = open(respath, "w")
for id_setup in range(6):
cocoGt = COCO(annFile)
cocoDt = cocoGt.loadRes(resFile)
imgIds = sorted(cocoGt.getImgIds())
cocoEval = COCOeval(cocoGt, cocoDt, annType)
cocoEval.params.imgIds = imgIds
cocoEval.evaluate(id_setup)
cocoEval.accumulate()
mean_mr = cocoEval.summarize(id_setup, res_file)
if id_setup == 0:
mr_reasonable = mean_mr
if mr_reasonable < best_mr_reasonable:
print('New best validation MR with model {} : {} -> {}'.format(
f, best_mr_reasonable, mr_reasonable))
best_mr_reasonable = mr_reasonable
best_mr_name = f
print('')
res_file.close()
