def main():
random.seed(123)
dataDir='/home/gchrupala/repos/coco'
dataType='val2014'
cap = COCO('%s/annotations/captions_%s.json'%(dataDir,dataType))
coco = COCO('%s/annotations/instances_%s.json'%(dataDir,dataType))
imgCat = {}
for cat,imgs in coco.catToImgs.items():
for img in imgs:
if img in imgCat:
imgCat[img].add(cat)
else:
imgCat[img]=set([cat])
with open('hard2.csv','w') as file:
writer = csv.writer(file)
writer.writerow(["desc", "url_1", "url_2", "url_3", "url_4" ])
imgIds = random.sample(coco.getImgIds(), 1000)
for img in coco.loadImgs(imgIds):
if img['id'] not in imgCat:
continue
cats = imgCat[img['id']]
desc = random.sample(cap.imgToAnns[img['id']],1)[0]
imgs = coco.loadImgs(random.sample(sum([ coco.getImgIds(catIds=[cat])
for cat in cats ],[]),3))
urls = [ img['coco_url'] ] + [ img['coco_url'] for img in imgs ]
random.shuffle(urls)
writer.writerow([desc['caption']] + urls )
def _load_all(self, anno_file, shuffle):
"""
initialize all entries given annotation json file
Parameters:
----------
anno_file: str
annotation json file
shuffle: bool
whether to shuffle image list
"""
image_set_index = []
labels = []
coco = COCO(anno_file)
img_ids = coco.getImgIds()
for img_id in img_ids:
# filename
image_info = coco.loadImgs(img_id)[0]
filename = image_info["file_name"]
subdir = filename.split('_')[1]
height = image_info["height"]
width = image_info["width"]
# label
anno_ids = coco.getAnnIds(imgIds=img_id)
annos = coco.loadAnns(anno_ids)
label = []
for anno in annos:
cat_id = int(anno["category_id"])
bbox = anno["bbox"]
assert len(bbox) == 4
xmin = float(bbox[0]) / width
ymin = float(bbox[1]) / height
xmax = xmin + float(bbox[2]) / width
ymax = ymin + float(bbox[3]) / height
label.append([cat_id, xmin, ymin, xmax, ymax, 0])
if label:
labels.append(np.array(label))
image_set_index.append(os.path.join(subdir, filename))
if shuffle:
import random
indices = range(len(image_set_index))
random.shuffle(indices)
image_set_index = [image_set_index[i] for i in indices]
labels = [labels[i] for i in indices]
# store the results
self.image_set_index = image_set_index
self.labels = labels
def language_eval(dataset, preds):
import sys
if 'coco' in dataset:
sys.path.append("coco-caption")
annFile = 'coco-caption/annotations/captions_val2014.json'
else:
sys.path.append("f30k-caption")
annFile = 'f30k-caption/annotations/dataset_flickr30k.json'
from pycocotools.coco import COCO
from pycocoevalcap.eval import COCOEvalCap
encoder.FLOAT_REPR = lambda o: format(o, '.3f')
coco = COCO(annFile)
valids = coco.getImgIds()
# filter results to only those in MSCOCO validation set (will be about a third)
preds_filt = [p for p in preds if p['image_id'] in valids]
print 'using %d/%d predictions' % (len(preds_filt), len(preds))
json.dump(preds_filt, open('tmp.json', 'w')) # serialize to temporary json file. Sigh, COCO API...
resFile = 'tmp.json'
cocoRes = coco.loadRes(resFile)
cocoEval = COCOEvalCap(coco, cocoRes)
cocoEval.params['image_id'] = cocoRes.getImgIds()
cocoEval.evaluate()
# create output dictionary
out = {}
for metric, score in cocoEval.eval.items():
out[metric] = score
return out
def main():
HASH_IMG_NAME = True
pylab.rcParams['figure.figsize'] = (10.0, 8.0)
json.encoder.FLOAT_REPR = lambda o: format(o, '.3f')
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--inputfile", type=str, required=True,
help='File containing model-generated/hypothesis sentences.')
parser.add_argument("-r", "--references", type=str, required=True,
help='JSON File containing references/groundtruth sentences.')
args = parser.parse_args()
prediction_file = args.inputfile
reference_file = args.references
json_predictions_file = '{0}.json'.format(prediction_file)
crf = CocoResFormat()
crf.read_file(prediction_file, HASH_IMG_NAME)
crf.dump_json(json_predictions_file)
# create coco object and cocoRes object.
coco = COCO(reference_file)
cocoRes = coco.loadRes(json_predictions_file)
# create cocoEval object.
cocoEval = COCOEvalCap(coco, cocoRes)
# evaluate results
cocoEval.evaluate()
# print output evaluation scores
for metric, score in cocoEval.eval.items():
print '%s: %.3f'%(metric, score)
def main(argv):
input_json = 'results/' + sys.argv[1]
annFile = 'annotations/captions_val2014.json'
coco = COCO(annFile)
valids = coco.getImgIds()
checkpoint = json.load(open(input_json, 'r'))
preds = checkpoint['val_predictions']
# filter results to only those in MSCOCO validation set (will be about a third)
preds_filt = [p for p in preds if p['image_id'] in valids]
print 'using %d/%d predictions' % (len(preds_filt), len(preds))
json.dump(preds_filt, open('tmp.json', 'w')) # serialize to temporary json file. Sigh, COCO API...
resFile = 'tmp.json'
cocoRes = coco.loadRes(resFile)
cocoEval = COCOEvalCap(coco, cocoRes)
cocoEval.params['image_id'] = cocoRes.getImgIds()
cocoEval.evaluate()
# create output dictionary
out = {}
for metric, score in cocoEval.eval.items():
out[metric] = score
# serialize to file, to be read from Lua
json.dump(out, open(input_json + '_out.json', 'w'))
def evaluate():
cocoGt = COCO('annotations.json')
cocoDt = cocoGt.loadRes('detections.json')
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def ablate(imgIds = [], mode ='destroy', out_path="tmp", coco = coco, ct = None, **args):
"""[ablation entry point 2.0]
Created to accomodate background-destroying ablation. Will dispatch all
old ablations (gaussian, blackout, & median) to gen_ablation."""
if ct is None:
ct = coco_text.COCO_Text(os.path.join(CD, 'COCO_Text.json'))
if imgIds == []:
imgIds = ct.getImgIds(imgIds=ct.train, catIds=[('legibility','legible')])
imgIds = [imgIds[np.random.randint(0,len(imgIds))]]
#dispatch to old ablation entry point
if mode in ['gaussian', 'blackout', 'median']:
return gen_ablation(imgIds, mode, ct, out_path=out_path, **args)
#else do destroy_bg
if coco is None:
coco = COCO('%s/annotations/instances_%s.json'%(DATA_PATH,DATA_TYPE))
imgs = coco.loadImgs(imgIds)
results = []
for idx, img in enumerate(imgs):
print("Ablating image {}/{} with id {} ".format(idx+1, len(imgIds), img['id']))
ori_file_name = os.path.join(CD, DATA_PATH, DATA_TYPE, img['file_name'])
orig = io.imread(ori_file_name)
if mode == 'destroy':
ablt = destroy_bg(orig, img['id'], coco, **args)
elif mode == 'median_bg':
ablt = median_bg(orig, img['id'], coco, **args)
out_file_name = os.path.join(CD, "..", out_path, "%s_%s"%(mode, img['file_name']))
io.imsave(out_file_name, ablt)
results.append((img['id'], ori_file_name, out_file_name))
return results
def coco_eval(ann_fn, json_fn, save_fn):
coco = COCO(ann_fn)
coco_res = coco.loadRes(json_fn)
coco_evaluator = COCOEvalCap(coco, coco_res)
# comment below line to evaluate the full validation or testing set.
coco_evaluator.params['image_id'] = coco_res.getImgIds()
coco_evaluator.evaluate(save_fn)
def __init__(self, annotation_file=None):
"""
Constructor of SALICON helper class for reading and visualizing annotations.
:param annotation_file (str): location of annotation file
:return:
"""
COCO.__init__(self,annotation_file=annotation_file)
def language_eval(input_data, savedir, split):
if type(input_data) == str: # Filename given.
checkpoint = json.load(open(input_data, 'r'))
preds = checkpoint
elif type(input_data) == list: # Direct predictions give.
preds = input_data
annFile = 'third_party/coco-caption/annotations/captions_val2014.json'
coco = COCO(annFile)
valids = coco.getImgIds()
# Filter results to only those in MSCOCO validation set (will be about a third)
preds_filt = [p for p in preds if p['image_id'] in valids]
print 'Using %d/%d predictions' % (len(preds_filt), len(preds))
resFile = osp.join(savedir, 'result_%s.json' % (split))
json.dump(preds_filt, open(resFile, 'w')) # Serialize to temporary json file. Sigh, COCO API...
cocoRes = coco.loadRes(resFile)
cocoEval = COCOEvalCap(coco, cocoRes)
cocoEval.params['image_id'] = cocoRes.getImgIds()
cocoEval.evaluate()
# Create output dictionary.
out = {}
for metric, score in cocoEval.eval.items():
out[metric] = score
# Return aggregate and per image score.
return out, cocoEval.evalImgs
def main(argv):
## Parsing the command
in_path = ''
out_path = ''
ann_path = ''
try:
opts, args = getopt.getopt(argv,"hi:o:a:",["in=","out=","annotation="])
except getopt.GetoptError:
print 'test.py -i <inputfile> -o <outputfile> -a <annotationfile>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'test.py -i <inputfile> -o <outputfile> -a <annotationfile>'
sys.exit()
elif opt in ("-i", "--in"):
in_path = arg
elif opt in ("-o", "--out"):
out_path = arg
elif opt in ("-a", "--annotation"):
ann_path = arg
print('Performing evaluation using Coco Python API...')
_COCO = COCO(ann_path)
_cats = _COCO.loadCats(_COCO.getCatIds())
_classes = tuple(['__background__'] + [c['name'] for c in _cats])
_do_eval(in_path,out_path, _COCO, _classes)
def coco_eval(candidates_file, references_file):
"""
Given the candidates and references, the coco-caption module is
used to calculate various metrics. Returns a list of dictionaries containing:
-BLEU
-ROUGE
-METEOR
-CIDEr
"""
# This is used to suppress the output of coco-eval:
old_stdout = sys.stdout
sys.stdout = open(os.devnull, "w")
try:
# Derived from example code in coco-captions repo
coco = COCO( references_file )
cocoRes = coco.loadRes( candidates_file )
cocoEval = COCOEvalCap(coco, cocoRes)
cocoEval.evaluate()
finally:
# Change back to standard output
sys.stdout.close()
sys.stdout = old_stdout
return cocoEval.evalImgs
def __init__(self, root_dir, data_dir, anno_file):
coco = COCO(os.path.join(root_dir, anno_file))
anns = coco.loadAnns(coco.getAnnIds())
self.coco = coco
self.anns = anns
self.vocab = None # Later set from outside
self.coco_root = root_dir
self.coco_data = data_dir
def cocoval(detected_json):
eval_json = config.eval_json
eval_gt = COCO(eval_json)
eval_dt = eval_gt.loadRes(detected_json)
cocoEval = COCOeval(eval_gt, eval_dt, iouType='bbox')
# cocoEval.params.imgIds = eval_gt.getImgIds()
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
def score_generation(gt_filename=None, generation_result=None): coco_dict = read_json(generation_result) coco = COCO(gt_filename) generation_coco = coco.loadRes(generation_result) coco_evaluator = COCOEvalCap(coco, generation_coco) #coco_image_ids = [self.sg.image_path_to_id[image_path] # for image_path in self.images] coco_image_ids = [j['image_id'] for j in coco_dict] coco_evaluator.params['image_id'] = coco_image_ids results = coco_evaluator.evaluate(return_results=True) return results
def _load_gt_roidb(self):
_coco = COCO(self._anno_file)
# deal with class names
cats = [cat['name'] for cat in _coco.loadCats(_coco.getCatIds())]
class_to_coco_ind = dict(zip(cats, _coco.getCatIds()))
class_to_ind = dict(zip(self.classes, range(self.num_classes)))
coco_ind_to_class_ind = dict([(class_to_coco_ind[cls], class_to_ind[cls])
for cls in self.classes[1:]])
image_ids = _coco.getImgIds()
gt_roidb = [self._load_annotation(_coco, coco_ind_to_class_ind, index) for index in image_ids]
return gt_roidb
def __init__(self,dataType,usingSet,dataDir,savefileDir):
#setpath
self.dataType = dataType
self.usingSet = usingSet
self.dataDir = dataDir
self.savefileDir = savefileDir
self.InsFile='%s/annotations/instances_%s.json'%(dataDir,dataType)
self.CapFile='%s/annotations/captions_%s.json'%(dataDir,dataType)
self.SALICON = pickle.load(open('%s/%s.p'%(savefileDir,usingSet),'rb'))
self.Ins_ID = pickle.load(open('%s/Ins_ID_%s.p'%(savefileDir,usingSet),'rb'))
self.category = pickle.load(open('%s/category.p'%savefileDir,'rb'))
self.category_idx = pickle.load(open('%s/cat_dict_idx.p'%savefileDir,'rb'))#eg., person -- 1
self.category_supercategory_idx = pickle.load(open('%s/cat_dict_supercat.p'%savefileDir,'rb')) #eg., person--human
self.supercategory_idx = pickle.load(open('%s/supercate_id.p'%savefileDir,'rb'))#eg., food--1
self.imsal_dict = pickle.load(open('%s/imsal_dict_%s.p'%(savefileDir,usingSet),'rb'))
self.Ins_coco = COCO(self.InsFile)
self.Cap_coco = COCO(self.CapFile)
self.cat_list = self.Ins_coco.cats#category list (official)
wordmat = sio.loadmat('%s/word_mat_%s.mat'%(savefileDir,usingSet))
wordmat = wordmat['word_mat']
self.wordmat = wordmat[:,0]
self.correction_list = ['men','man','kid','boy','baby']
self.nounlist = []
self.nounID = []
self.Cardi_Noun = []
self.Seque_Noun = []
self.size_norm = float(640*480)
self.loc_norm = float(math.sqrt(640**2+480**2))
self.saliencydict_c = {}
self.saliencydict_s = {}
#******************10-03-2016 update***********************
self.saliencydict_i = {}
self.transformer = TfidfTransformer()
def language_eval(dataset, preds, model_id, split):
import sys
if 'coco' in dataset:
sys.path.append("coco-caption")
annFile = 'coco-caption/annotations/captions_val2014.json'
elif 'msvd' in dataset:
sys.path.append('coco-caption')
annFile = 'coco-caption/annotations/coco_ref_msvd.json'
elif 'kuaishou' in dataset:
sys.path.append('coco-caption')
annFile = 'coco-caption/annotations/coco_ref_kuaishou.json'
else:
sys.path.append("f30k-caption")
annFile = 'f30k-caption/annotations/dataset_flickr30k.json'
from pycocotools.coco import COCO
from pycocoevalcap.eval import COCOEvalCap
encoder.FLOAT_REPR = lambda o: format(o, '.3f')
if not os.path.isdir('eval_results'):
os.mkdir('eval_results')
cache_path = os.path.join('eval_results/', model_id + '_' + split + '.json')
coco = COCO(annFile)
valids = coco.getImgIds()
# filter results to only those in MSCOCO validation set (will be about a third)
preds_filt = [p for p in preds if p['image_id'] in valids]
print('using %d/%d predictions' % (len(preds_filt), len(preds)))
json.dump(preds_filt, open(cache_path, 'w')) # serialize to temporary json file. Sigh, COCO API...
cocoRes = coco.loadRes(cache_path)
cocoEval = COCOEvalCap(coco, cocoRes)
cocoEval.params['image_id'] = cocoRes.getImgIds()
cocoEval.evaluate()
# create output dictionary
out = {}
for metric, score in cocoEval.eval.items():
out[metric] = score
imgToEval = cocoEval.imgToEval
for p in preds_filt:
image_id, caption = p['image_id'], p['caption']
imgToEval[image_id]['caption'] = caption
with open(cache_path, 'w') as outfile:
json.dump({'overall': out, 'imgToEval': imgToEval}, outfile)
return out
def create_tokcap(data_folder=DATA_FOLDER):
cap = COCO(COCO_TRAIN_CAP_FILE)
listedCapMap = {}
for i in cap.loadAnns(cap.getAnnIds()):
listedCapMap[i['id']] = [dict([('caption',i['caption']), ('image_id', i['image_id'])])]
tokenizedListedCapMap = PTBTokenizer().tokenize(listedCapMap)
tokcap = [] #map caption ids to a map of its tokenized caption and image id
for i, j in tokenizedListedCapMap.iteritems():
tokcap += [(i, dict([('caption', j[0]), ('image_id', listedCapMap[i][0]['image_id'])]))]
f = open(data_folder + '/preprocessed/tokcap.json', 'w')
json.dump(tokcap, f)
f.close()
def split_valid(data_folder=DATA_FOLDER):
cap = COCO(COCO_VALID_CAP_FILE)
imgIds = cap.getImgIds()
random.seed(0)
random.shuffle(imgIds)
mid = len(imgIds)/2
vimgids, timgids = imgIds[:mid], imgIds[mid:]
f = open(data_folder + '/preprocessed/valimgids.json', 'w')
json.dump(vimgids, f)
f.close()
f = open(data_folder + '/preprocessed/tesimgids.json', 'w')
json.dump(timgids, f)
f.close()
class Resize_Image():
def __init__(self, imgeDir, resizeImageDir):
self.ImageDir = imgeDir
self.ResizeImageDir = resizeImageDir
self.dataDir = APP_ROOT + "/Data/"
self.dataType = 'val2014'
self.annFile = '%s/annotations/instances_%s.json'\
% (self.dataDir, self.dataType)
# initialize COCO api for instance annotations
self.coco = COCO(self.annFile)
# display COCO categories and supercategories
self.cats = self.coco.loadCats(self.coco.getCatIds())
self.names = [cat['name'] for cat in self.cats]
self.ids = [cat['id'] for cat in self.cats]
self.name_ids = {}
# get all images containing given categories, select one at random
self.img_dict = {}
def resize_image(self):
for i in range(len(self.names)):
if self.ids[i] not in self.name_ids:
self.name_ids.update({self.names[i]: self.ids[i]})
self.__image_dict_update()
def __image_dict_update(self):
for name in self.names:
catIds = self.coco.getCatIds(catNms=[name])
imgIds = self.coco.getImgIds(catIds=catIds)
for i in range(len(imgIds)):
img = self.coco.loadImgs(imgIds[i])[0]
if img["file_name"] not in self.img_dict:
self.img_dict.update({img["file_name"]: name})
self.__output_resize_images()
def __output_resize_images(self):
for k, v in sorted(self.img_dict.items(), key=lambda x: x[0]):
ImageFile = '%s/%s' % (self.ImageDir, k)
pil_im = Image.open(ImageFile)
out = pil_im.resize((255, 255))
save_image = '%s/%s' % (self.ResizeImageDir, k)
out.save(save_image)
print(save_image + " " + str(self.name_ids[v]))
def validate(val_loader, model, i, silence=False):
batch_time = AverageMeter()
coco_gt = val_loader.dataset.coco
coco_pred = COCO()
coco_pred.dataset['images'] = [img for img in coco_gt.datasets['images']]
coco_pred.dataset['categories'] = copy.deepcopy(coco_gt.dataset['categories'])
id = 0
# switch to evaluate mode
model.eval()
end = time.time()
for i, (inputs, anns) in enumerate(val_loader):
# forward images one by one (TODO: support batch mode later, or
# multiprocess)
for j, input in enumerate(inputs):
input_anns= anns[j] # anns of this input
gt_bbox= np.vstack([ann['bbox'] + [ann['ordered_id']] for ann in input_anns])
im_info= [[input.size(1), input.size(2),
input_anns[0]['scale_ratio']]]
input_var= Variable(input.unsqueeze(0),
requires_grad=False).cuda()
cls_prob, bbox_pred, rois = model(input_var, im_info)
scores, pred_boxes = model.interpret_outputs(cls_prob, bbox_pred, rois, im_info)
print(scores, pred_boxes)
# for i in range(scores.shape[0]):
# measure elapsed time
batch_time.update(time.time() - end)
end= time.time()
coco_pred.createIndex()
coco_eval = COCOeval(coco_gt, coco_pred, 'bbox')
coco_eval.params.imgIds= sorted(coco_gt.getImgIds())
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
print('iter: [{0}] '
'Time {batch_time.avg:.3f} '
'Val Stats: {1}'
.format(i, coco_eval.stats,
batch_time=batch_time))
return coco_eval.stats[0]
def __init__(self, image_set, year, devkit_path=None):
datasets.imdb.__init__(self, 'coco_' + image_set + year)
self._year = year
self._image_set = image_set
#_devkit_path points to MSCOCO
self._devkit_path = self._get_default_path() if devkit_path is None \
else devkit_path
#_data_path points to MSCOCO/images
self._data_path = os.path.join(self._devkit_path, 'images')
self._classes = ('__background__', 'object')
self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
self._image_ext = '.jpg'
# initialize COCO api for instance annotations
if image_set in ['train','val']:
self.coco=COCO('%s/annotations/instances_%s.json'%(self._devkit_path,self._image_set+self._year))
self._image_index = self._load_image_set_index()
# Default to roidb handler
self._roidb_handler = self.proposals_roidb
# Use top-k proposals to build imdb
self.config = {'top_k':1000}
assert os.path.exists(self._devkit_path), \
'COCO devkit path does not exist: {}'.format(self._devkit_path)
assert os.path.exists(self._data_path), \
'Path does not exist: {}'.format(self._data_path)
def __init__(self, name):
assert name in DATASETS.keys(), \
'Unknown dataset name: {}'.format(name)
assert os.path.exists(DATASETS[name][IM_DIR]), \
'Image directory \'{}\' not found'.format(DATASETS[name][IM_DIR])
assert os.path.exists(DATASETS[name][ANN_FN]), \
'Annotation file \'{}\' not found'.format(DATASETS[name][ANN_FN])
logger.debug('Creating: {}'.format(name))
self.name = name
self.image_directory = DATASETS[name][IM_DIR]
self.image_prefix = (
'' if IM_PREFIX not in DATASETS[name] else DATASETS[name][IM_PREFIX]
)
self.COCO = COCO(DATASETS[name][ANN_FN])
self.debug_timer = Timer()
# Set up dataset classes
category_ids = self.COCO.getCatIds()
categories = [c['name'] for c in self.COCO.loadCats(category_ids)]
self.category_to_id_map = dict(zip(categories, category_ids))
self.classes = ['__background__'] + categories
self.num_classes = len(self.classes)
self.json_category_id_to_contiguous_id = {
v: i + 1
for i, v in enumerate(self.COCO.getCatIds())
}
self.contiguous_category_id_to_json_id = {
v: k
for k, v in self.json_category_id_to_contiguous_id.items()
}
self._init_keypoints()
def __init__(self, root=os.path.expanduser('~/.mxnet/datasets/coco'),
split='train', mode=None, transform=None, **kwargs):
super(COCOSegmentation, self).__init__(root, split, mode, transform, **kwargs)
# lazy import pycocotools
try_import_pycocotools()
from pycocotools.coco import COCO
from pycocotools import mask
if split == 'train':
print('train set')
ann_file = os.path.join(root, 'annotations/instances_train2017.json')
ids_file = os.path.join(root, 'annotations/train_ids.mx')
self.root = os.path.join(root, 'train2017')
else:
print('val set')
ann_file = os.path.join(root, 'annotations/instances_val2017.json')
ids_file = os.path.join(root, 'annotations/val_ids.mx')
self.root = os.path.join(root, 'val2017')
self.coco = COCO(ann_file)
self.coco_mask = mask
if os.path.exists(ids_file):
with open(ids_file, 'rb') as f:
self.ids = pickle.load(f)
else:
ids = list(self.coco.imgs.keys())
self.ids = self._preprocess(ids, ids_file)
self.transform = transform
def __init__(self, image_set, year, devkit_path=None):
datasets.imdb.__init__(self, 'coco_' + year + '_' + image_set)
# COCO specific config options
self.config = {'top_k' : 2000,
'use_salt' : True,
'cleanup' : True,
'crowd_thresh' : 0.7,
'min_size' : 2}
# name, paths
self._year = year
self._image_set = image_set
self._data_path = self._get_default_path() if devkit_path is None \
else devkit_path
# load COCO API, classes, class <-> id mappings
self._COCO = COCO(self._get_ann_file())
cats = self._COCO.loadCats(self._COCO.getCatIds())
self._classes = tuple(['__background__'] + [c['name'] for c in cats])
self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
self._class_to_coco_cat_id = dict(zip([c['name'] for c in cats],
self._COCO.getCatIds()))
self._image_index = self._load_image_set_index()
# Default to roidb handler
self._roidb_handler = self.gt_roidb
self.competition_mode(False)
self._data_name = image_set + year # e.g., "val2014"
if self._data_name == 'test-dev2015':
self._data_name_path = 'test2015'
else:
self._data_name_path = self._data_name
def coco_val_eval(self, pred_path, result_path):
"""Evaluate the predicted sentences on MS COCO validation."""
sys.path.append('./external/coco-caption')
from pycocotools.coco import COCO
from pycocoevalcap.eval import COCOEvalCap
coco = COCO('./external/coco-caption/annotations/captions_val2014.json')
cocoRes = coco.loadRes(pred_path)
cocoEval = COCOEvalCap(coco, cocoRes)
cocoEval.params['image_id'] = cocoRes.getImgIds()
cocoEval.evaluate()
with open(result_path, 'w') as fout:
for metric, score in cocoEval.eval.items():
print('%s: %.3f' % (metric, score), file=fout)
def __init__(self, image_set, year):
imdb.__init__(self, 'coco_' + year + '_' + image_set)
# COCO specific config options
self.config = {'top_k' : 2000,
'use_salt' : True,
'cleanup' : True,
'crowd_thresh' : 0.7,
'min_size' : 2}
# name, paths
self._year = year
self._image_set = image_set
self._data_path = osp.join(cfg.DATA_DIR, 'coco')
# load COCO API, classes, class <-> id mappings
self._COCO = COCO(self._get_ann_file())
cats = self._COCO.loadCats(self._COCO.getCatIds())
self._classes = tuple(['__background__'] + [c['name'] for c in cats])
self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
self._class_to_coco_cat_id = dict(zip([c['name'] for c in cats],
self._COCO.getCatIds()))
self._image_index = self._load_image_set_index()
# Default to roidb handler
self.set_proposal_method('selective_search')
self.competition_mode(False)
# Some image sets are "views" (i.e. subsets) into others.
# For example, minival2014 is a random 5000 image subset of val2014.
# This mapping tells us where the view's images and proposals come from.
self._view_map = {'minival2014' : 'val2014'}
coco_name = image_set + year # e.g., "val2014"
self._data_name = (self._view_map[coco_name]
if self._view_map.has_key(coco_name)
else coco_name)
# Dataset splits that have ground-truth annotations (test splits
# do not have gt annotations)
self._gt_splits = ('train', 'val', 'minival')
def __init__(self, name):
assert dataset_catalog.contains(name), \
'Unknown dataset name: {}'.format(name)
assert os.path.exists(dataset_catalog.get_im_dir(name)), \
'Im dir \'{}\' not found'.format(dataset_catalog.get_im_dir(name))
assert os.path.exists(dataset_catalog.get_ann_fn(name)), \
'Ann fn \'{}\' not found'.format(dataset_catalog.get_ann_fn(name))
logger.debug('Creating: {}'.format(name))
self.name = name
self.image_directory = dataset_catalog.get_im_dir(name)
self.image_prefix = dataset_catalog.get_im_prefix(name)
self.COCO = COCO(dataset_catalog.get_ann_fn(name))
self.debug_timer = Timer()
# Set up dataset classes
category_ids = self.COCO.getCatIds()
categories = [c['name'] for c in self.COCO.loadCats(category_ids)]
self.category_to_id_map = dict(zip(categories, category_ids))
self.classes = ['__background__'] + categories
self.num_classes = len(self.classes)
self.json_category_id_to_contiguous_id = {
v: i + 1
for i, v in enumerate(self.COCO.getCatIds())
}
self.contiguous_category_id_to_json_id = {
v: k
for k, v in self.json_category_id_to_contiguous_id.items()
}
self._init_keypoints()
def load_dataset(self):
dataset = self.cfg.dataset
dataset_phase = self.cfg.dataset_phase
dataset_ann = self.cfg.dataset_ann
# initialize COCO api
annFile = '%s/annotations/%s_%s.json'%(dataset,dataset_ann,dataset_phase)
self.coco = COCO(annFile)
imgIds = self.coco.getImgIds()
data = []
# loop through each image
for imgId in imgIds:
item = DataItem()
img = self.coco.loadImgs(imgId)[0]
item.im_path = "%s/images/%s/%s"%(dataset, dataset_phase, img["file_name"])
item.im_size = [3, img["height"], img["width"]]
item.coco_id = imgId
annIds = self.coco.getAnnIds(imgIds=img['id'], iscrowd=False)
anns = self.coco.loadAnns(annIds)
all_person_keypoints = []
masked_persons_RLE = []
visible_persons_RLE = []
all_visibilities = []
# Consider only images with people
has_people = len(anns) > 0
if not has_people and self.cfg.coco_only_images_with_people:
continue
for ann in anns: # loop through each person
person_keypoints = []
visibilities = []
if ann["num_keypoints"] != 0:
for i in range(self.cfg.num_joints):
x_coord = ann["keypoints"][3 * i]
y_coord = ann["keypoints"][3 * i + 1]
visibility = ann["keypoints"][3 * i + 2]
visibilities.append(visibility)
if visibility != 0: # i.e. if labeled
person_keypoints.append([i, x_coord, y_coord])
all_person_keypoints.append(np.array(person_keypoints))
visible_persons_RLE.append(maskUtils.decode(self.coco.annToRLE(ann)))
all_visibilities.append(visibilities)
if ann["num_keypoints"] == 0:
masked_persons_RLE.append(self.coco.annToRLE(ann))
item.joints = all_person_keypoints
item.im_neg_mask = maskUtils.merge(masked_persons_RLE)
if self.cfg.use_gt_segm:
item.gt_segm = np.moveaxis(np.array(visible_persons_RLE), 0, -1)
item.visibilities = all_visibilities
data.append(item)
self.has_gt = self.cfg.dataset is not "image_info"
return data
class CocoDataset(Dataset):
def __init__(self,
root_dir,
coco_dir,
img_dir,
set_dir='train2017',
transform=None):
self.root_dir = root_dir
self.img_dir = img_dir
self.coco_dir = coco_dir
self.set_dir = set_dir
self.set_name = set_dir
self.transform = transform
self.coco = COCO(
os.path.join(self.root_dir, self.coco_dir, 'annotations',
'instances_' + self.set_name + '.json'))
self.image_ids = self.coco.getImgIds()
self.load_classes()
#print(self.classes)
#print(self.labels)
def load_classes(self):
# load class names (name -> label)
categories = self.coco.loadCats(self.coco.getCatIds())
categories.sort(key=lambda x: x['id'])
self.classes = {}
for c in categories:
self.classes[c['name']] = len(self.classes)
# also load the reverse (label -> name)
self.labels = {}
for key, value in self.classes.items():
self.labels[value] = key
def __len__(self):
return len(self.image_ids)
def __getitem__(self, idx):
img = self.load_image(idx)
annot = self.load_annotations(idx)
sample = {'img': img, 'annot': annot}
if self.transform:
sample = self.transform(sample)
return sample
def load_image(self, image_index):
image_info = self.coco.loadImgs(self.image_ids[image_index])[0]
path = os.path.join(self.root_dir, self.coco_dir, self.set_dir,
image_info['file_name'])
img = cv2.imread(path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
return img.astype(np.float32) / 255.
def load_annotations(self, image_index):
# get ground truth annotations
annotations_ids = self.coco.getAnnIds(
imgIds=self.image_ids[image_index], iscrowd=False)
annotations = np.zeros((0, 5))
# some images appear to miss annotations
if len(annotations_ids) == 0:
return annotations
# parse annotations
coco_annotations = self.coco.loadAnns(annotations_ids)
for idx, a in enumerate(coco_annotations):
# some annotations have basically no width / height, skip them
if a['bbox'][2] < 1 or a['bbox'][3] < 1:
continue
annotation = np.zeros((1, 5))
annotation[0, :4] = a['bbox']
#annotation[0, 4] = a['category_id'] - 1
annotation[0, 4] = a['category_id']
annotations = np.append(annotations, annotation, axis=0)
# transform from [x, y, w, h] to [x1, y1, x2, y2]
annotations[:, 2] = annotations[:, 0] + annotations[:, 2]
annotations[:, 3] = annotations[:, 1] + annotations[:, 3]
return annotations
def __init__(self, transform, mode):
self.mode = mode
self.root_path = '../data/STB/data'
self.rootnet_output_path = '../data/STB/rootnet_output/rootnet_stb_output.json'
self.original_img_shape = (480, 640) # height, width
self.transform = transform
self.joint_num = 21 # single hand
self.joint_type = {
'right': np.arange(self.joint_num, self.joint_num * 2),
'left': np.arange(0, self.joint_num)
}
self.root_joint_idx = {'right': self.joint_num, 'left': 0}
self.skeleton = load_skeleton(osp.join(self.root_path, 'skeleton.txt'),
self.joint_num * 2)
self.datalist = []
self.annot_path = osp.join(self.root_path,
'STB_' + self.mode + '.json')
db = COCO(self.annot_path)
if self.mode == 'test' and cfg.trans_test == 'rootnet':
print("Get bbox and root depth from " + self.rootnet_output_path)
rootnet_result = {}
with open(self.rootnet_output_path) as f:
annot = json.load(f)
for i in range(len(annot)):
rootnet_result[str(annot[i]['annot_id'])] = annot[i]
else:
print("Get bbox and root depth from groundtruth annotation")
for aid in db.anns.keys():
ann = db.anns[aid]
image_id = ann['image_id']
img = db.loadImgs(image_id)[0]
seq_name = img['seq_name']
img_path = osp.join(self.root_path, 'images', seq_name,
img['file_name'])
img_width, img_height = img['width'], img['height']
cam_param = img['cam_param']
focal, princpt = np.array(cam_param['focal'],
dtype=np.float32), np.array(
cam_param['princpt'],
dtype=np.float32)
joint_img = np.array(ann['joint_img'], dtype=np.float32)
joint_cam = np.array(ann['joint_cam'], dtype=np.float32)
joint_valid = np.array(ann['joint_valid'], dtype=np.float32)
# transform single hand data to double hand data structure
hand_type = ann['hand_type']
joint_img_dh = np.zeros((self.joint_num * 2, 2), dtype=np.float32)
joint_cam_dh = np.zeros((self.joint_num * 2, 3), dtype=np.float32)
joint_valid_dh = np.zeros((self.joint_num * 2), dtype=np.float32)
joint_img_dh[self.joint_type[hand_type]] = joint_img
joint_cam_dh[self.joint_type[hand_type]] = joint_cam
joint_valid_dh[self.joint_type[hand_type]] = joint_valid
joint_img = joint_img_dh
joint_cam = joint_cam_dh
joint_valid = joint_valid_dh
if self.mode == 'test' and cfg.trans_test == 'rootnet':
bbox = np.array(rootnet_result[str(aid)]['bbox'],
dtype=np.float32)
abs_depth = rootnet_result[str(aid)]['abs_depth']
else:
bbox = np.array(ann['bbox'], dtype=np.float32) # x,y,w,h
bbox = process_bbox(bbox, (img_height, img_width))
abs_depth = joint_cam[self.root_joint_idx[hand_type],
2] # single hand abs depth
cam_param = {'focal': focal, 'princpt': princpt}
joint = {
'cam_coord': joint_cam,
'img_coord': joint_img,
'valid': joint_valid
}
data = {
'img_path': img_path,
'bbox': bbox,
'cam_param': cam_param,
'joint': joint,
'hand_type': hand_type,
'abs_depth': abs_depth
}
self.datalist.append(data)
class CocoGenerator(Generator):
"""!@brief
Generate data from the COCO dataset.
@see https://github.com/cocodataset/cocoapi/tree/master/PythonAPI for more information.
"""
def __init__(self, data_dir, set_name, **kwargs):
"""!@brief
Initialize a COCO data generator.
@param data_dir: Path to where the COCO dataset is stored.
@param set_name: Name of the set to parse.
"""
self.data_dir = data_dir
self.set_name = set_name
self.coco = COCO(os.path.join(data_dir, 'annotations', 'instances_' + set_name + '.json'))
self.image_ids = self.coco.getImgIds()
self.load_classes()
super(CocoGenerator, self).__init__(**kwargs)
def load_classes(self):
"""!@brief
Loads the class to label mapping (and inverse) for COCO.
"""
# load class names (name -> label)
categories = self.coco.loadCats(self.coco.getCatIds())
categories.sort(key=lambda x: x['id'])
self.classes = {}
self.coco_labels = {}
self.coco_labels_inverse = {}
for c in categories:
self.coco_labels[len(self.classes)] = c['id']
self.coco_labels_inverse[c['id']] = len(self.classes)
self.classes[c['name']] = len(self.classes)
# also load the reverse (label -> name)
self.labels = {}
for key, value in self.classes.items():
self.labels[value] = key
def size(self):
"""!@brief
Size of the COCO dataset.
"""
return len(self.image_ids)
def num_classes(self):
"""!@brief
Number of classes in the dataset. For COCO this is 80.
"""
return len(self.classes)
def has_label(self, label):
"""!@brief
Return True if label is a known label.
"""
return label in self.labels
def has_name(self, name):
"""!@brief
Returns True if name is a known class.
"""
return name in self.classes
def name_to_label(self, name):
"""!@brief
Map name to label.
"""
return self.classes[name]
def label_to_name(self, label):
"""!@brief
Map label to name.
"""
return self.labels[label]
def coco_label_to_label(self, coco_label):
"""!@brief
Map COCO label to the label as used in the network.
COCO has some gaps in the order of labels. The highest label is 90,
but there are 80 classes.
"""
return self.coco_labels_inverse[coco_label]
def coco_label_to_name(self, coco_label):
"""!@brief
Map COCO label to name.
"""
return self.label_to_name(self.coco_label_to_label(coco_label))
def label_to_coco_label(self, label):
"""!@brief
Map label as used by the network to labels as used by COCO.
"""
return self.coco_labels[label]
def image_aspect_ratio(self, image_index):
"""!@brief
Compute the aspect ratio for an image with image_index.
"""
image = self.coco.loadImgs(self.image_ids[image_index])[0]
return float(image['width']) / float(image['height'])
def load_image(self, image_index):
"""!@brief
Load an image at the image_index.
"""
image_info = self.coco.loadImgs(self.image_ids[image_index])[0]
path = os.path.join(self.data_dir, 'images', self.set_name, image_info['file_name'])
return read_image_bgr(path)
def load_annotations(self, image_index):
"""!@brief
Load annotations for an image_index.
"""
# get ground truth annotations
annotations_ids = self.coco.getAnnIds(imgIds=self.image_ids[image_index], iscrowd=False)
annotations = {'labels': np.empty((0,)), 'bboxes': np.empty((0, 4))}
# some images appear to miss annotations (like image with id 257034)
if len(annotations_ids) == 0:
return annotations
# parse annotations
coco_annotations = self.coco.loadAnns(annotations_ids)
for idx, a in enumerate(coco_annotations):
# some annotations have basically no width / height, skip them
if a['bbox'][2] < 1 or a['bbox'][3] < 1:
continue
annotations['labels'] = np.concatenate([annotations['labels'], [self.coco_label_to_label(a['category_id'])]], axis=0)
annotations['bboxes'] = np.concatenate([annotations['bboxes'], [[
a['bbox'][0],
a['bbox'][1],
a['bbox'][0] + a['bbox'][2],
a['bbox'][1] + a['bbox'][3],
]]], axis=0)
return annotations
def initialize():
coco = COCO(ann_path)
imgIds = coco.getImgIds(catIds=[], imgIds=[])
dataset_size = len(imgIds)
return coco, imgIds, dataset_size
class CocoDataset(CustomDataset):
CLASSES = ('person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus',
'train', 'truck', 'boat', 'traffic_light', 'fire_hydrant',
'stop_sign', 'parking_meter', 'bench', 'bird', 'cat', 'dog',
'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',
'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
'skis', 'snowboard', 'sports_ball', 'kite', 'baseball_bat',
'baseball_glove', 'skateboard', 'surfboard', 'tennis_racket',
'bottle', 'wine_glass', 'cup', 'fork', 'knife', 'spoon', 'bowl',
'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
'hot_dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
'potted_plant', 'bed', 'dining_table', 'toilet', 'tv', 'laptop',
'mouse', 'remote', 'keyboard', 'cell_phone', 'microwave',
'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock',
'vase', 'scissors', 'teddy_bear', 'hair_drier', 'toothbrush')
def load_annotations(self, ann_file):
self.coco = COCO(ann_file)
self.cat_ids = self.coco.getCatIds()
self.cat2label = {
cat_id: i + 1
for i, cat_id in enumerate(self.cat_ids)
}
self.img_ids = self.coco.getImgIds()
img_infos = []
for i in self.img_ids:
info = self.coco.loadImgs([i])[0]
info['filename'] = info['file_name']
img_infos.append(info)
return img_infos
def get_ann_info(self, idx):
img_id = self.img_infos[idx]['id']
ann_ids = self.coco.getAnnIds(imgIds=[img_id])
ann_info = self.coco.loadAnns(ann_ids)
return self._parse_ann_info(self.img_infos[idx], ann_info)
def _filter_imgs(self, min_size=32):
"""Filter images too small or without ground truths."""
valid_inds = []
ids_with_ann = set(_['image_id'] for _ in self.coco.anns.values())
for i, img_info in enumerate(self.img_infos):
if self.filter_empty_gt and self.img_ids[i] not in ids_with_ann:
continue
if min(img_info['width'], img_info['height']) >= min_size:
valid_inds.append(i)
return valid_inds
def _parse_ann_info(self, img_info, ann_info):
"""Parse bbox and mask annotation.
Args:
ann_info (list[dict]): Annotation info of an image.
with_mask (bool): Whether to parse mask annotations.
Returns:
dict: A dict containing the following keys: bboxes, bboxes_ignore,
labels, masks, seg_map. "masks" are raw annotations and not
decoded into binary masks.
"""
gt_bboxes = []
gt_labels = []
gt_bboxes_ignore = []
gt_masks_ann = []
for i, ann in enumerate(ann_info):
if ann.get('ignore', False):
continue
x1, y1, w, h = ann['bbox']
if ann['area'] <= 0 or w < 1 or h < 1:
continue
bbox = [x1, y1, x1 + w - 1, y1 + h - 1]
if ann.get('iscrowd', False):
gt_bboxes_ignore.append(bbox)
else:
gt_bboxes.append(bbox)
gt_labels.append(self.cat2label[ann['category_id']])
gt_masks_ann.append(ann['segmentation'])
if gt_bboxes:
gt_bboxes = np.array(gt_bboxes, dtype=np.float32)
gt_labels = np.array(gt_labels, dtype=np.int64)
else:
gt_bboxes = np.zeros((0, 4), dtype=np.float32)
gt_labels = np.array([], dtype=np.int64)
if gt_bboxes_ignore:
gt_bboxes_ignore = np.array(gt_bboxes_ignore, dtype=np.float32)
else:
gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32)
seg_map = img_info['filename'].replace('jpg', 'png')
ann = dict(bboxes=gt_bboxes,
labels=gt_labels,
bboxes_ignore=gt_bboxes_ignore,
masks=gt_masks_ann,
seg_map=seg_map)
return ann
def xyxy2xywh(self, bbox):
_bbox = bbox.tolist()
return [
_bbox[0],
_bbox[1],
_bbox[2] - _bbox[0] + 1,
_bbox[3] - _bbox[1] + 1,
]
def _proposal2json(self, results):
json_results = []
for idx in range(len(self)):
img_id = self.img_ids[idx]
bboxes = results[idx]
for i in range(bboxes.shape[0]):
data = dict()
data['image_id'] = img_id
data['bbox'] = self.xyxy2xywh(bboxes[i])
data['score'] = float(bboxes[i][4])
data['category_id'] = 1
json_results.append(data)
return json_results
def _det2json(self, results):
json_results = []
for idx in range(len(self)):
img_id = self.img_ids[idx]
result = results[idx]
for label in range(len(result)):
bboxes = result[label]
for i in range(bboxes.shape[0]):
data = dict()
data['image_id'] = img_id
data['bbox'] = self.xyxy2xywh(bboxes[i])
data['score'] = float(bboxes[i][4])
data['category_id'] = self.cat_ids[label]
json_results.append(data)
return json_results
def _segm2json(self, results):
bbox_json_results = []
segm_json_results = []
for idx in range(len(self)):
img_id = self.img_ids[idx]
det, seg = results[idx]
for label in range(len(det)):
# bbox results
bboxes = det[label]
for i in range(bboxes.shape[0]):
data = dict()
data['image_id'] = img_id
data['bbox'] = self.xyxy2xywh(bboxes[i])
data['score'] = float(bboxes[i][4])
data['category_id'] = self.cat_ids[label]
bbox_json_results.append(data)
# segm results
# some detectors use different scores for bbox and mask
if isinstance(seg, tuple):
segms = seg[0][label]
mask_score = seg[1][label]
else:
segms = seg[label]
mask_score = [bbox[4] for bbox in bboxes]
for i in range(bboxes.shape[0]):
data = dict()
data['image_id'] = img_id
data['bbox'] = self.xyxy2xywh(bboxes[i])
data['score'] = float(mask_score[i])
data['category_id'] = self.cat_ids[label]
if isinstance(segms[i]['counts'], bytes):
segms[i]['counts'] = segms[i]['counts'].decode()
data['segmentation'] = segms[i]
segm_json_results.append(data)
return bbox_json_results, segm_json_results
def results2json(self, results, outfile_prefix):
"""Dump the detection results to a json file.
There are 3 types of results: proposals, bbox predictions, mask
predictions, and they have different data types. This method will
automatically recognize the type, and dump them to json files.
Args:
results (list[list | tuple | ndarray]): Testing results of the
dataset.
outfile_prefix (str): The filename prefix of the json files. If the
prefix is "somepath/xxx", the json files will be named
"somepath/xxx.bbox.json", "somepath/xxx.segm.json",
"somepath/xxx.proposal.json".
Returns:
dict[str: str]: Possible keys are "bbox", "segm", "proposal", and
values are corresponding filenames.
"""
result_files = dict()
if isinstance(results[0], list):
json_results = self._det2json(results)
result_files['bbox'] = '{}.{}.json'.format(outfile_prefix, 'bbox')
result_files['proposal'] = '{}.{}.json'.format(
outfile_prefix, 'bbox')
mmcv.dump(json_results, result_files['bbox'])
elif isinstance(results[0], tuple):
json_results = self._segm2json(results)
result_files['bbox'] = '{}.{}.json'.format(outfile_prefix, 'bbox')
result_files['proposal'] = '{}.{}.json'.format(
outfile_prefix, 'bbox')
result_files['segm'] = '{}.{}.json'.format(outfile_prefix, 'segm')
mmcv.dump(json_results[0], result_files['bbox'])
mmcv.dump(json_results[1], result_files['segm'])
elif isinstance(results[0], np.ndarray):
json_results = self._proposal2json(results)
result_files['proposal'] = '{}.{}.json'.format(
outfile_prefix, 'proposal')
mmcv.dump(json_results, result_files['proposal'])
else:
raise TypeError('invalid type of results')
return result_files
def fast_eval_recall(self, results, proposal_nums, iou_thrs, logger=None):
gt_bboxes = []
for i in range(len(self.img_ids)):
ann_ids = self.coco.getAnnIds(imgIds=self.img_ids[i])
ann_info = self.coco.loadAnns(ann_ids)
if len(ann_info) == 0:
gt_bboxes.append(np.zeros((0, 4)))
continue
bboxes = []
for ann in ann_info:
if ann.get('ignore', False) or ann['iscrowd']:
continue
x1, y1, w, h = ann['bbox']
bboxes.append([x1, y1, x1 + w - 1, y1 + h - 1])
bboxes = np.array(bboxes, dtype=np.float32)
if bboxes.shape[0] == 0:
bboxes = np.zeros((0, 4))
gt_bboxes.append(bboxes)
recalls = eval_recalls(gt_bboxes,
results,
proposal_nums,
iou_thrs,
logger=logger)
ar = recalls.mean(axis=1)
return ar
def evaluate(self,
results,
metric='bbox',
logger=None,
jsonfile_prefix=None,
classwise=False,
iou_thr_by_class=0.2,
proposal_nums=(100, 300, 1000),
iou_thrs=np.arange(0.5, 0.96, 0.05)):
"""Evaluation in COCO protocol.
Args:
results (list): Testing results of the dataset.
metric (str | list[str]): Metrics to be evaluated.
logger (logging.Logger | str | None): Logger used for printing
related information during evaluation. Default: None.
jsonfile_prefix (str | None):
classwise (bool): Whether to evaluating the AP for each class.
proposal_nums (Sequence[int]): Proposal number used for evaluating
recalls, such as recall@100, recall@1000.
Default: (100, 300, 1000).
iou_thrs (Sequence[float]): IoU threshold used for evaluating
recalls. If set to a list, the average recall of all IoUs will
also be computed. Default: 0.5.
Returns:
dict[str: float]
"""
assert isinstance(results, list), 'results must be a list'
assert len(results) == len(self), (
'The length of results is not equal to the dataset len: {} != {}'.
format(len(results), len(self)))
metrics = metric if isinstance(metric, list) else [metric]
allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast']
for metric in metrics:
if metric not in allowed_metrics:
raise KeyError('metric {} is not supported'.format(metric))
if jsonfile_prefix is None:
tmp_dir = tempfile.TemporaryDirectory()
jsonfile_prefix = osp.join(tmp_dir.name, 'results')
else:
tmp_dir = None
result_files = self.results2json(results, jsonfile_prefix)
eval_results = {}
cocoGt = self.coco
for metric in metrics:
msg = 'Evaluating {}...'.format(metric)
if logger is None:
msg = '\n' + msg
print_log(msg, logger=logger)
if metric == 'proposal_fast':
ar = self.fast_eval_recall(results,
proposal_nums,
iou_thrs,
logger='silent')
log_msg = []
for i, num in enumerate(proposal_nums):
eval_results['AR@{}'.format(num)] = ar[i]
log_msg.append('\nAR@{}\t{:.4f}'.format(num, ar[i]))
log_msg = ''.join(log_msg)
print_log(log_msg, logger=logger)
continue
if metric not in result_files:
raise KeyError('{} is not in results'.format(metric))
try:
cocoDt = cocoGt.loadRes(result_files[metric])
except IndexError:
print_log('The testing results of the whole dataset is empty.',
logger=logger,
level=logging.ERROR)
break
iou_type = 'bbox' if metric == 'proposal' else metric
cocoEval = COCOeval(cocoGt, cocoDt, iou_type)
cocoEval.params.imgIds = self.img_ids
if metric == 'proposal':
cocoEval.params.useCats = 0
cocoEval.params.maxDets = list(proposal_nums)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
metric_items = [
'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000',
'AR_l@1000'
]
for i, item in enumerate(metric_items):
val = float('{:.3f}'.format(cocoEval.stats[i + 6]))
eval_results[item] = val
else:
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if classwise: # Compute per-category AP
gt_lst = load_coco_bboxes(cocoGt, is_gt=True)
dt_lst = load_coco_bboxes(cocoDt, is_gt=False)
evaluator = Evaluator()
ret, mAP = evaluator.GetMAPbyClass(
gt_lst, dt_lst, iou_thr=iou_thr_by_class)
# Get metric values per each class
for metricsPerClass in ret:
cl = metricsPerClass['class']
ap = metricsPerClass['AP']
ap_str = '{0:.3f}'.format(ap)
eval_results['class_{}'.format(cl)] = float(ap_str)
print('AP: %s (%s)' % (ap_str, cl))
mAP_str = '{0:.3f}'.format(mAP)
eval_results['mAP'] = float(mAP_str)
print('mAP: {}\n'.format(mAP_str))
metric_items = [
'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l'
]
for i in range(len(metric_items)):
key = '{}_{}'.format(metric, metric_items[i])
val = float('{:.3f}'.format(cocoEval.stats[i]))
eval_results[key] = val
eval_results['{}_mAP_copypaste'.format(metric)] = (
'{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} '
'{ap[4]:.3f} {ap[5]:.3f}').format(ap=cocoEval.stats[:6])
if tmp_dir is not None:
tmp_dir.cleanup()
return eval_results
def __init__(self, image_path, info_file, transforms=None, is_train=False):
self.root = image_path
self.coco = COCO(info_file)
self.ids = list(self.coco.imgToAnns.keys()) # may be some imaegs don't have labels
self.transforms = transforms
self.is_train is_train
