def main(): random.seed(123) dataDir='/home/gchrupala/repos/coco' dataType='val2014' cap = COCO('%s/annotations/captions_%s.json'%(dataDir,dataType)) coco = COCO('%s/annotations/instances_%s.json'%(dataDir,dataType)) imgCat = {} for cat,imgs in coco.catToImgs.items(): for img in imgs: if img in imgCat: imgCat[img].add(cat) else: imgCat[img]=set([cat]) with open('hard2.csv','w') as file: writer = csv.writer(file) writer.writerow(["desc", "url_1", "url_2", "url_3", "url_4" ]) imgIds = random.sample(coco.getImgIds(), 1000) for img in coco.loadImgs(imgIds): if img['id'] not in imgCat: continue cats = imgCat[img['id']] desc = random.sample(cap.imgToAnns[img['id']],1)[0] imgs = coco.loadImgs(random.sample(sum([ coco.getImgIds(catIds=[cat]) for cat in cats ],[]),3)) urls = [ img['coco_url'] ] + [ img['coco_url'] for img in imgs ] random.shuffle(urls) writer.writerow([desc['caption']] + urls )
def _load_all(self, anno_file, shuffle): """ initialize all entries given annotation json file Parameters: ---------- anno_file: str annotation json file shuffle: bool whether to shuffle image list """ image_set_index = [] labels = [] coco = COCO(anno_file) img_ids = coco.getImgIds() for img_id in img_ids: # filename image_info = coco.loadImgs(img_id)[0] filename = image_info["file_name"] subdir = filename.split('_')[1] height = image_info["height"] width = image_info["width"] # label anno_ids = coco.getAnnIds(imgIds=img_id) annos = coco.loadAnns(anno_ids) label = [] for anno in annos: cat_id = int(anno["category_id"]) bbox = anno["bbox"] assert len(bbox) == 4 xmin = float(bbox[0]) / width ymin = float(bbox[1]) / height xmax = xmin + float(bbox[2]) / width ymax = ymin + float(bbox[3]) / height label.append([cat_id, xmin, ymin, xmax, ymax, 0]) if label: labels.append(np.array(label)) image_set_index.append(os.path.join(subdir, filename)) if shuffle: import random indices = range(len(image_set_index)) random.shuffle(indices) image_set_index = [image_set_index[i] for i in indices] labels = [labels[i] for i in indices] # store the results self.image_set_index = image_set_index self.labels = labels
def language_eval(dataset, preds): import sys if 'coco' in dataset: sys.path.append("coco-caption") annFile = 'coco-caption/annotations/captions_val2014.json' else: sys.path.append("f30k-caption") annFile = 'f30k-caption/annotations/dataset_flickr30k.json' from pycocotools.coco import COCO from pycocoevalcap.eval import COCOEvalCap encoder.FLOAT_REPR = lambda o: format(o, '.3f') coco = COCO(annFile) valids = coco.getImgIds() # filter results to only those in MSCOCO validation set (will be about a third) preds_filt = [p for p in preds if p['image_id'] in valids] print 'using %d/%d predictions' % (len(preds_filt), len(preds)) json.dump(preds_filt, open('tmp.json', 'w')) # serialize to temporary json file. Sigh, COCO API... resFile = 'tmp.json' cocoRes = coco.loadRes(resFile) cocoEval = COCOEvalCap(coco, cocoRes) cocoEval.params['image_id'] = cocoRes.getImgIds() cocoEval.evaluate() # create output dictionary out = {} for metric, score in cocoEval.eval.items(): out[metric] = score return out
def main(): HASH_IMG_NAME = True pylab.rcParams['figure.figsize'] = (10.0, 8.0) json.encoder.FLOAT_REPR = lambda o: format(o, '.3f') parser = argparse.ArgumentParser() parser.add_argument("-i", "--inputfile", type=str, required=True, help='File containing model-generated/hypothesis sentences.') parser.add_argument("-r", "--references", type=str, required=True, help='JSON File containing references/groundtruth sentences.') args = parser.parse_args() prediction_file = args.inputfile reference_file = args.references json_predictions_file = '{0}.json'.format(prediction_file) crf = CocoResFormat() crf.read_file(prediction_file, HASH_IMG_NAME) crf.dump_json(json_predictions_file) # create coco object and cocoRes object. coco = COCO(reference_file) cocoRes = coco.loadRes(json_predictions_file) # create cocoEval object. cocoEval = COCOEvalCap(coco, cocoRes) # evaluate results cocoEval.evaluate() # print output evaluation scores for metric, score in cocoEval.eval.items(): print '%s: %.3f'%(metric, score)
def main(argv): input_json = 'results/' + sys.argv[1] annFile = 'annotations/captions_val2014.json' coco = COCO(annFile) valids = coco.getImgIds() checkpoint = json.load(open(input_json, 'r')) preds = checkpoint['val_predictions'] # filter results to only those in MSCOCO validation set (will be about a third) preds_filt = [p for p in preds if p['image_id'] in valids] print 'using %d/%d predictions' % (len(preds_filt), len(preds)) json.dump(preds_filt, open('tmp.json', 'w')) # serialize to temporary json file. Sigh, COCO API... resFile = 'tmp.json' cocoRes = coco.loadRes(resFile) cocoEval = COCOEvalCap(coco, cocoRes) cocoEval.params['image_id'] = cocoRes.getImgIds() cocoEval.evaluate() # create output dictionary out = {} for metric, score in cocoEval.eval.items(): out[metric] = score # serialize to file, to be read from Lua json.dump(out, open(input_json + '_out.json', 'w'))
def evaluate(): cocoGt = COCO('annotations.json') cocoDt = cocoGt.loadRes('detections.json') cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def ablate(imgIds = [], mode ='destroy', out_path="tmp", coco = coco, ct = None, **args): """[ablation entry point 2.0] Created to accomodate background-destroying ablation. Will dispatch all old ablations (gaussian, blackout, & median) to gen_ablation.""" if ct is None: ct = coco_text.COCO_Text(os.path.join(CD, 'COCO_Text.json')) if imgIds == []: imgIds = ct.getImgIds(imgIds=ct.train, catIds=[('legibility','legible')]) imgIds = [imgIds[np.random.randint(0,len(imgIds))]] #dispatch to old ablation entry point if mode in ['gaussian', 'blackout', 'median']: return gen_ablation(imgIds, mode, ct, out_path=out_path, **args) #else do destroy_bg if coco is None: coco = COCO('%s/annotations/instances_%s.json'%(DATA_PATH,DATA_TYPE)) imgs = coco.loadImgs(imgIds) results = [] for idx, img in enumerate(imgs): print("Ablating image {}/{} with id {} ".format(idx+1, len(imgIds), img['id'])) ori_file_name = os.path.join(CD, DATA_PATH, DATA_TYPE, img['file_name']) orig = io.imread(ori_file_name) if mode == 'destroy': ablt = destroy_bg(orig, img['id'], coco, **args) elif mode == 'median_bg': ablt = median_bg(orig, img['id'], coco, **args) out_file_name = os.path.join(CD, "..", out_path, "%s_%s"%(mode, img['file_name'])) io.imsave(out_file_name, ablt) results.append((img['id'], ori_file_name, out_file_name)) return results
def coco_eval(ann_fn, json_fn, save_fn): coco = COCO(ann_fn) coco_res = coco.loadRes(json_fn) coco_evaluator = COCOEvalCap(coco, coco_res) # comment below line to evaluate the full validation or testing set. coco_evaluator.params['image_id'] = coco_res.getImgIds() coco_evaluator.evaluate(save_fn)
def __init__(self, annotation_file=None): """ Constructor of SALICON helper class for reading and visualizing annotations. :param annotation_file (str): location of annotation file :return: """ COCO.__init__(self,annotation_file=annotation_file)
def language_eval(input_data, savedir, split): if type(input_data) == str: # Filename given. checkpoint = json.load(open(input_data, 'r')) preds = checkpoint elif type(input_data) == list: # Direct predictions give. preds = input_data annFile = 'third_party/coco-caption/annotations/captions_val2014.json' coco = COCO(annFile) valids = coco.getImgIds() # Filter results to only those in MSCOCO validation set (will be about a third) preds_filt = [p for p in preds if p['image_id'] in valids] print 'Using %d/%d predictions' % (len(preds_filt), len(preds)) resFile = osp.join(savedir, 'result_%s.json' % (split)) json.dump(preds_filt, open(resFile, 'w')) # Serialize to temporary json file. Sigh, COCO API... cocoRes = coco.loadRes(resFile) cocoEval = COCOEvalCap(coco, cocoRes) cocoEval.params['image_id'] = cocoRes.getImgIds() cocoEval.evaluate() # Create output dictionary. out = {} for metric, score in cocoEval.eval.items(): out[metric] = score # Return aggregate and per image score. return out, cocoEval.evalImgs
def main(argv): ## Parsing the command in_path = '' out_path = '' ann_path = '' try: opts, args = getopt.getopt(argv,"hi:o:a:",["in=","out=","annotation="]) except getopt.GetoptError: print 'test.py -i <inputfile> -o <outputfile> -a <annotationfile>' sys.exit(2) for opt, arg in opts: if opt == '-h': print 'test.py -i <inputfile> -o <outputfile> -a <annotationfile>' sys.exit() elif opt in ("-i", "--in"): in_path = arg elif opt in ("-o", "--out"): out_path = arg elif opt in ("-a", "--annotation"): ann_path = arg print('Performing evaluation using Coco Python API...') _COCO = COCO(ann_path) _cats = _COCO.loadCats(_COCO.getCatIds()) _classes = tuple(['__background__'] + [c['name'] for c in _cats]) _do_eval(in_path,out_path, _COCO, _classes)
def coco_eval(candidates_file, references_file): """ Given the candidates and references, the coco-caption module is used to calculate various metrics. Returns a list of dictionaries containing: -BLEU -ROUGE -METEOR -CIDEr """ # This is used to suppress the output of coco-eval: old_stdout = sys.stdout sys.stdout = open(os.devnull, "w") try: # Derived from example code in coco-captions repo coco = COCO( references_file ) cocoRes = coco.loadRes( candidates_file ) cocoEval = COCOEvalCap(coco, cocoRes) cocoEval.evaluate() finally: # Change back to standard output sys.stdout.close() sys.stdout = old_stdout return cocoEval.evalImgs
def __init__(self, root_dir, data_dir, anno_file): coco = COCO(os.path.join(root_dir, anno_file)) anns = coco.loadAnns(coco.getAnnIds()) self.coco = coco self.anns = anns self.vocab = None # Later set from outside self.coco_root = root_dir self.coco_data = data_dir
def cocoval(detected_json): eval_json = config.eval_json eval_gt = COCO(eval_json) eval_dt = eval_gt.loadRes(detected_json) cocoEval = COCOeval(eval_gt, eval_dt, iouType='bbox') # cocoEval.params.imgIds = eval_gt.getImgIds() cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def score_generation(gt_filename=None, generation_result=None): coco_dict = read_json(generation_result) coco = COCO(gt_filename) generation_coco = coco.loadRes(generation_result) coco_evaluator = COCOEvalCap(coco, generation_coco) #coco_image_ids = [self.sg.image_path_to_id[image_path] # for image_path in self.images] coco_image_ids = [j['image_id'] for j in coco_dict] coco_evaluator.params['image_id'] = coco_image_ids results = coco_evaluator.evaluate(return_results=True) return results
def _load_gt_roidb(self): _coco = COCO(self._anno_file) # deal with class names cats = [cat['name'] for cat in _coco.loadCats(_coco.getCatIds())] class_to_coco_ind = dict(zip(cats, _coco.getCatIds())) class_to_ind = dict(zip(self.classes, range(self.num_classes))) coco_ind_to_class_ind = dict([(class_to_coco_ind[cls], class_to_ind[cls]) for cls in self.classes[1:]]) image_ids = _coco.getImgIds() gt_roidb = [self._load_annotation(_coco, coco_ind_to_class_ind, index) for index in image_ids] return gt_roidb
def __init__(self,dataType,usingSet,dataDir,savefileDir): #setpath self.dataType = dataType self.usingSet = usingSet self.dataDir = dataDir self.savefileDir = savefileDir self.InsFile='%s/annotations/instances_%s.json'%(dataDir,dataType) self.CapFile='%s/annotations/captions_%s.json'%(dataDir,dataType) self.SALICON = pickle.load(open('%s/%s.p'%(savefileDir,usingSet),'rb')) self.Ins_ID = pickle.load(open('%s/Ins_ID_%s.p'%(savefileDir,usingSet),'rb')) self.category = pickle.load(open('%s/category.p'%savefileDir,'rb')) self.category_idx = pickle.load(open('%s/cat_dict_idx.p'%savefileDir,'rb'))#eg., person -- 1 self.category_supercategory_idx = pickle.load(open('%s/cat_dict_supercat.p'%savefileDir,'rb')) #eg., person--human self.supercategory_idx = pickle.load(open('%s/supercate_id.p'%savefileDir,'rb'))#eg., food--1 self.imsal_dict = pickle.load(open('%s/imsal_dict_%s.p'%(savefileDir,usingSet),'rb')) self.Ins_coco = COCO(self.InsFile) self.Cap_coco = COCO(self.CapFile) self.cat_list = self.Ins_coco.cats#category list (official) wordmat = sio.loadmat('%s/word_mat_%s.mat'%(savefileDir,usingSet)) wordmat = wordmat['word_mat'] self.wordmat = wordmat[:,0] self.correction_list = ['men','man','kid','boy','baby'] self.nounlist = [] self.nounID = [] self.Cardi_Noun = [] self.Seque_Noun = [] self.size_norm = float(640*480) self.loc_norm = float(math.sqrt(640**2+480**2)) self.saliencydict_c = {} self.saliencydict_s = {} #******************10-03-2016 update*********************** self.saliencydict_i = {} self.transformer = TfidfTransformer()
def language_eval(dataset, preds, model_id, split): import sys if 'coco' in dataset: sys.path.append("coco-caption") annFile = 'coco-caption/annotations/captions_val2014.json' elif 'msvd' in dataset: sys.path.append('coco-caption') annFile = 'coco-caption/annotations/coco_ref_msvd.json' elif 'kuaishou' in dataset: sys.path.append('coco-caption') annFile = 'coco-caption/annotations/coco_ref_kuaishou.json' else: sys.path.append("f30k-caption") annFile = 'f30k-caption/annotations/dataset_flickr30k.json' from pycocotools.coco import COCO from pycocoevalcap.eval import COCOEvalCap encoder.FLOAT_REPR = lambda o: format(o, '.3f') if not os.path.isdir('eval_results'): os.mkdir('eval_results') cache_path = os.path.join('eval_results/', model_id + '_' + split + '.json') coco = COCO(annFile) valids = coco.getImgIds() # filter results to only those in MSCOCO validation set (will be about a third) preds_filt = [p for p in preds if p['image_id'] in valids] print('using %d/%d predictions' % (len(preds_filt), len(preds))) json.dump(preds_filt, open(cache_path, 'w')) # serialize to temporary json file. Sigh, COCO API... cocoRes = coco.loadRes(cache_path) cocoEval = COCOEvalCap(coco, cocoRes) cocoEval.params['image_id'] = cocoRes.getImgIds() cocoEval.evaluate() # create output dictionary out = {} for metric, score in cocoEval.eval.items(): out[metric] = score imgToEval = cocoEval.imgToEval for p in preds_filt: image_id, caption = p['image_id'], p['caption'] imgToEval[image_id]['caption'] = caption with open(cache_path, 'w') as outfile: json.dump({'overall': out, 'imgToEval': imgToEval}, outfile) return out
def create_tokcap(data_folder=DATA_FOLDER): cap = COCO(COCO_TRAIN_CAP_FILE) listedCapMap = {} for i in cap.loadAnns(cap.getAnnIds()): listedCapMap[i['id']] = [dict([('caption',i['caption']), ('image_id', i['image_id'])])] tokenizedListedCapMap = PTBTokenizer().tokenize(listedCapMap) tokcap = [] #map caption ids to a map of its tokenized caption and image id for i, j in tokenizedListedCapMap.iteritems(): tokcap += [(i, dict([('caption', j[0]), ('image_id', listedCapMap[i][0]['image_id'])]))] f = open(data_folder + '/preprocessed/tokcap.json', 'w') json.dump(tokcap, f) f.close()
def split_valid(data_folder=DATA_FOLDER): cap = COCO(COCO_VALID_CAP_FILE) imgIds = cap.getImgIds() random.seed(0) random.shuffle(imgIds) mid = len(imgIds)/2 vimgids, timgids = imgIds[:mid], imgIds[mid:] f = open(data_folder + '/preprocessed/valimgids.json', 'w') json.dump(vimgids, f) f.close() f = open(data_folder + '/preprocessed/tesimgids.json', 'w') json.dump(timgids, f) f.close()
class Resize_Image(): def __init__(self, imgeDir, resizeImageDir): self.ImageDir = imgeDir self.ResizeImageDir = resizeImageDir self.dataDir = APP_ROOT + "/Data/" self.dataType = 'val2014' self.annFile = '%s/annotations/instances_%s.json'\ % (self.dataDir, self.dataType) # initialize COCO api for instance annotations self.coco = COCO(self.annFile) # display COCO categories and supercategories self.cats = self.coco.loadCats(self.coco.getCatIds()) self.names = [cat['name'] for cat in self.cats] self.ids = [cat['id'] for cat in self.cats] self.name_ids = {} # get all images containing given categories, select one at random self.img_dict = {} def resize_image(self): for i in range(len(self.names)): if self.ids[i] not in self.name_ids: self.name_ids.update({self.names[i]: self.ids[i]}) self.__image_dict_update() def __image_dict_update(self): for name in self.names: catIds = self.coco.getCatIds(catNms=[name]) imgIds = self.coco.getImgIds(catIds=catIds) for i in range(len(imgIds)): img = self.coco.loadImgs(imgIds[i])[0] if img["file_name"] not in self.img_dict: self.img_dict.update({img["file_name"]: name}) self.__output_resize_images() def __output_resize_images(self): for k, v in sorted(self.img_dict.items(), key=lambda x: x[0]): ImageFile = '%s/%s' % (self.ImageDir, k) pil_im = Image.open(ImageFile) out = pil_im.resize((255, 255)) save_image = '%s/%s' % (self.ResizeImageDir, k) out.save(save_image) print(save_image + " " + str(self.name_ids[v]))
def validate(val_loader, model, i, silence=False): batch_time = AverageMeter() coco_gt = val_loader.dataset.coco coco_pred = COCO() coco_pred.dataset['images'] = [img for img in coco_gt.datasets['images']] coco_pred.dataset['categories'] = copy.deepcopy(coco_gt.dataset['categories']) id = 0 # switch to evaluate mode model.eval() end = time.time() for i, (inputs, anns) in enumerate(val_loader): # forward images one by one (TODO: support batch mode later, or # multiprocess) for j, input in enumerate(inputs): input_anns= anns[j] # anns of this input gt_bbox= np.vstack([ann['bbox'] + [ann['ordered_id']] for ann in input_anns]) im_info= [[input.size(1), input.size(2), input_anns[0]['scale_ratio']]] input_var= Variable(input.unsqueeze(0), requires_grad=False).cuda() cls_prob, bbox_pred, rois = model(input_var, im_info) scores, pred_boxes = model.interpret_outputs(cls_prob, bbox_pred, rois, im_info) print(scores, pred_boxes) # for i in range(scores.shape[0]): # measure elapsed time batch_time.update(time.time() - end) end= time.time() coco_pred.createIndex() coco_eval = COCOeval(coco_gt, coco_pred, 'bbox') coco_eval.params.imgIds= sorted(coco_gt.getImgIds()) coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() print('iter: [{0}] ' 'Time {batch_time.avg:.3f} ' 'Val Stats: {1}' .format(i, coco_eval.stats, batch_time=batch_time)) return coco_eval.stats[0]
def __init__(self, image_set, year, devkit_path=None): datasets.imdb.__init__(self, 'coco_' + image_set + year) self._year = year self._image_set = image_set #_devkit_path points to MSCOCO self._devkit_path = self._get_default_path() if devkit_path is None \ else devkit_path #_data_path points to MSCOCO/images self._data_path = os.path.join(self._devkit_path, 'images') self._classes = ('__background__', 'object') self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes))) self._image_ext = '.jpg' # initialize COCO api for instance annotations if image_set in ['train','val']: self.coco=COCO('%s/annotations/instances_%s.json'%(self._devkit_path,self._image_set+self._year)) self._image_index = self._load_image_set_index() # Default to roidb handler self._roidb_handler = self.proposals_roidb # Use top-k proposals to build imdb self.config = {'top_k':1000} assert os.path.exists(self._devkit_path), \ 'COCO devkit path does not exist: {}'.format(self._devkit_path) assert os.path.exists(self._data_path), \ 'Path does not exist: {}'.format(self._data_path)
def __init__(self, name): assert name in DATASETS.keys(), \ 'Unknown dataset name: {}'.format(name) assert os.path.exists(DATASETS[name][IM_DIR]), \ 'Image directory \'{}\' not found'.format(DATASETS[name][IM_DIR]) assert os.path.exists(DATASETS[name][ANN_FN]), \ 'Annotation file \'{}\' not found'.format(DATASETS[name][ANN_FN]) logger.debug('Creating: {}'.format(name)) self.name = name self.image_directory = DATASETS[name][IM_DIR] self.image_prefix = ( '' if IM_PREFIX not in DATASETS[name] else DATASETS[name][IM_PREFIX] ) self.COCO = COCO(DATASETS[name][ANN_FN]) self.debug_timer = Timer() # Set up dataset classes category_ids = self.COCO.getCatIds() categories = [c['name'] for c in self.COCO.loadCats(category_ids)] self.category_to_id_map = dict(zip(categories, category_ids)) self.classes = ['__background__'] + categories self.num_classes = len(self.classes) self.json_category_id_to_contiguous_id = { v: i + 1 for i, v in enumerate(self.COCO.getCatIds()) } self.contiguous_category_id_to_json_id = { v: k for k, v in self.json_category_id_to_contiguous_id.items() } self._init_keypoints()
def __init__(self, root=os.path.expanduser('~/.mxnet/datasets/coco'), split='train', mode=None, transform=None, **kwargs): super(COCOSegmentation, self).__init__(root, split, mode, transform, **kwargs) # lazy import pycocotools try_import_pycocotools() from pycocotools.coco import COCO from pycocotools import mask if split == 'train': print('train set') ann_file = os.path.join(root, 'annotations/instances_train2017.json') ids_file = os.path.join(root, 'annotations/train_ids.mx') self.root = os.path.join(root, 'train2017') else: print('val set') ann_file = os.path.join(root, 'annotations/instances_val2017.json') ids_file = os.path.join(root, 'annotations/val_ids.mx') self.root = os.path.join(root, 'val2017') self.coco = COCO(ann_file) self.coco_mask = mask if os.path.exists(ids_file): with open(ids_file, 'rb') as f: self.ids = pickle.load(f) else: ids = list(self.coco.imgs.keys()) self.ids = self._preprocess(ids, ids_file) self.transform = transform
def __init__(self, image_set, year, devkit_path=None): datasets.imdb.__init__(self, 'coco_' + year + '_' + image_set) # COCO specific config options self.config = {'top_k' : 2000, 'use_salt' : True, 'cleanup' : True, 'crowd_thresh' : 0.7, 'min_size' : 2} # name, paths self._year = year self._image_set = image_set self._data_path = self._get_default_path() if devkit_path is None \ else devkit_path # load COCO API, classes, class <-> id mappings self._COCO = COCO(self._get_ann_file()) cats = self._COCO.loadCats(self._COCO.getCatIds()) self._classes = tuple(['__background__'] + [c['name'] for c in cats]) self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes))) self._class_to_coco_cat_id = dict(zip([c['name'] for c in cats], self._COCO.getCatIds())) self._image_index = self._load_image_set_index() # Default to roidb handler self._roidb_handler = self.gt_roidb self.competition_mode(False) self._data_name = image_set + year # e.g., "val2014" if self._data_name == 'test-dev2015': self._data_name_path = 'test2015' else: self._data_name_path = self._data_name
def coco_val_eval(self, pred_path, result_path): """Evaluate the predicted sentences on MS COCO validation.""" sys.path.append('./external/coco-caption') from pycocotools.coco import COCO from pycocoevalcap.eval import COCOEvalCap coco = COCO('./external/coco-caption/annotations/captions_val2014.json') cocoRes = coco.loadRes(pred_path) cocoEval = COCOEvalCap(coco, cocoRes) cocoEval.params['image_id'] = cocoRes.getImgIds() cocoEval.evaluate() with open(result_path, 'w') as fout: for metric, score in cocoEval.eval.items(): print('%s: %.3f' % (metric, score), file=fout)
def __init__(self, image_set, year): imdb.__init__(self, 'coco_' + year + '_' + image_set) # COCO specific config options self.config = {'top_k' : 2000, 'use_salt' : True, 'cleanup' : True, 'crowd_thresh' : 0.7, 'min_size' : 2} # name, paths self._year = year self._image_set = image_set self._data_path = osp.join(cfg.DATA_DIR, 'coco') # load COCO API, classes, class <-> id mappings self._COCO = COCO(self._get_ann_file()) cats = self._COCO.loadCats(self._COCO.getCatIds()) self._classes = tuple(['__background__'] + [c['name'] for c in cats]) self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes))) self._class_to_coco_cat_id = dict(zip([c['name'] for c in cats], self._COCO.getCatIds())) self._image_index = self._load_image_set_index() # Default to roidb handler self.set_proposal_method('selective_search') self.competition_mode(False) # Some image sets are "views" (i.e. subsets) into others. # For example, minival2014 is a random 5000 image subset of val2014. # This mapping tells us where the view's images and proposals come from. self._view_map = {'minival2014' : 'val2014'} coco_name = image_set + year # e.g., "val2014" self._data_name = (self._view_map[coco_name] if self._view_map.has_key(coco_name) else coco_name) # Dataset splits that have ground-truth annotations (test splits # do not have gt annotations) self._gt_splits = ('train', 'val', 'minival')
def __init__(self, name): assert dataset_catalog.contains(name), \ 'Unknown dataset name: {}'.format(name) assert os.path.exists(dataset_catalog.get_im_dir(name)), \ 'Im dir \'{}\' not found'.format(dataset_catalog.get_im_dir(name)) assert os.path.exists(dataset_catalog.get_ann_fn(name)), \ 'Ann fn \'{}\' not found'.format(dataset_catalog.get_ann_fn(name)) logger.debug('Creating: {}'.format(name)) self.name = name self.image_directory = dataset_catalog.get_im_dir(name) self.image_prefix = dataset_catalog.get_im_prefix(name) self.COCO = COCO(dataset_catalog.get_ann_fn(name)) self.debug_timer = Timer() # Set up dataset classes category_ids = self.COCO.getCatIds() categories = [c['name'] for c in self.COCO.loadCats(category_ids)] self.category_to_id_map = dict(zip(categories, category_ids)) self.classes = ['__background__'] + categories self.num_classes = len(self.classes) self.json_category_id_to_contiguous_id = { v: i + 1 for i, v in enumerate(self.COCO.getCatIds()) } self.contiguous_category_id_to_json_id = { v: k for k, v in self.json_category_id_to_contiguous_id.items() } self._init_keypoints()
def load_dataset(self): dataset = self.cfg.dataset dataset_phase = self.cfg.dataset_phase dataset_ann = self.cfg.dataset_ann # initialize COCO api annFile = '%s/annotations/%s_%s.json'%(dataset,dataset_ann,dataset_phase) self.coco = COCO(annFile) imgIds = self.coco.getImgIds() data = [] # loop through each image for imgId in imgIds: item = DataItem() img = self.coco.loadImgs(imgId)[0] item.im_path = "%s/images/%s/%s"%(dataset, dataset_phase, img["file_name"]) item.im_size = [3, img["height"], img["width"]] item.coco_id = imgId annIds = self.coco.getAnnIds(imgIds=img['id'], iscrowd=False) anns = self.coco.loadAnns(annIds) all_person_keypoints = [] masked_persons_RLE = [] visible_persons_RLE = [] all_visibilities = [] # Consider only images with people has_people = len(anns) > 0 if not has_people and self.cfg.coco_only_images_with_people: continue for ann in anns: # loop through each person person_keypoints = [] visibilities = [] if ann["num_keypoints"] != 0: for i in range(self.cfg.num_joints): x_coord = ann["keypoints"][3 * i] y_coord = ann["keypoints"][3 * i + 1] visibility = ann["keypoints"][3 * i + 2] visibilities.append(visibility) if visibility != 0: # i.e. if labeled person_keypoints.append([i, x_coord, y_coord]) all_person_keypoints.append(np.array(person_keypoints)) visible_persons_RLE.append(maskUtils.decode(self.coco.annToRLE(ann))) all_visibilities.append(visibilities) if ann["num_keypoints"] == 0: masked_persons_RLE.append(self.coco.annToRLE(ann)) item.joints = all_person_keypoints item.im_neg_mask = maskUtils.merge(masked_persons_RLE) if self.cfg.use_gt_segm: item.gt_segm = np.moveaxis(np.array(visible_persons_RLE), 0, -1) item.visibilities = all_visibilities data.append(item) self.has_gt = self.cfg.dataset is not "image_info" return data
class CocoDataset(Dataset): def __init__(self, root_dir, coco_dir, img_dir, set_dir='train2017', transform=None): self.root_dir = root_dir self.img_dir = img_dir self.coco_dir = coco_dir self.set_dir = set_dir self.set_name = set_dir self.transform = transform self.coco = COCO( os.path.join(self.root_dir, self.coco_dir, 'annotations', 'instances_' + self.set_name + '.json')) self.image_ids = self.coco.getImgIds() self.load_classes() #print(self.classes) #print(self.labels) def load_classes(self): # load class names (name -> label) categories = self.coco.loadCats(self.coco.getCatIds()) categories.sort(key=lambda x: x['id']) self.classes = {} for c in categories: self.classes[c['name']] = len(self.classes) # also load the reverse (label -> name) self.labels = {} for key, value in self.classes.items(): self.labels[value] = key def __len__(self): return len(self.image_ids) def __getitem__(self, idx): img = self.load_image(idx) annot = self.load_annotations(idx) sample = {'img': img, 'annot': annot} if self.transform: sample = self.transform(sample) return sample def load_image(self, image_index): image_info = self.coco.loadImgs(self.image_ids[image_index])[0] path = os.path.join(self.root_dir, self.coco_dir, self.set_dir, image_info['file_name']) img = cv2.imread(path) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) return img.astype(np.float32) / 255. def load_annotations(self, image_index): # get ground truth annotations annotations_ids = self.coco.getAnnIds( imgIds=self.image_ids[image_index], iscrowd=False) annotations = np.zeros((0, 5)) # some images appear to miss annotations if len(annotations_ids) == 0: return annotations # parse annotations coco_annotations = self.coco.loadAnns(annotations_ids) for idx, a in enumerate(coco_annotations): # some annotations have basically no width / height, skip them if a['bbox'][2] < 1 or a['bbox'][3] < 1: continue annotation = np.zeros((1, 5)) annotation[0, :4] = a['bbox'] #annotation[0, 4] = a['category_id'] - 1 annotation[0, 4] = a['category_id'] annotations = np.append(annotations, annotation, axis=0) # transform from [x, y, w, h] to [x1, y1, x2, y2] annotations[:, 2] = annotations[:, 0] + annotations[:, 2] annotations[:, 3] = annotations[:, 1] + annotations[:, 3] return annotations
def __init__(self, transform, mode): self.mode = mode self.root_path = '../data/STB/data' self.rootnet_output_path = '../data/STB/rootnet_output/rootnet_stb_output.json' self.original_img_shape = (480, 640) # height, width self.transform = transform self.joint_num = 21 # single hand self.joint_type = { 'right': np.arange(self.joint_num, self.joint_num * 2), 'left': np.arange(0, self.joint_num) } self.root_joint_idx = {'right': self.joint_num, 'left': 0} self.skeleton = load_skeleton(osp.join(self.root_path, 'skeleton.txt'), self.joint_num * 2) self.datalist = [] self.annot_path = osp.join(self.root_path, 'STB_' + self.mode + '.json') db = COCO(self.annot_path) if self.mode == 'test' and cfg.trans_test == 'rootnet': print("Get bbox and root depth from " + self.rootnet_output_path) rootnet_result = {} with open(self.rootnet_output_path) as f: annot = json.load(f) for i in range(len(annot)): rootnet_result[str(annot[i]['annot_id'])] = annot[i] else: print("Get bbox and root depth from groundtruth annotation") for aid in db.anns.keys(): ann = db.anns[aid] image_id = ann['image_id'] img = db.loadImgs(image_id)[0] seq_name = img['seq_name'] img_path = osp.join(self.root_path, 'images', seq_name, img['file_name']) img_width, img_height = img['width'], img['height'] cam_param = img['cam_param'] focal, princpt = np.array(cam_param['focal'], dtype=np.float32), np.array( cam_param['princpt'], dtype=np.float32) joint_img = np.array(ann['joint_img'], dtype=np.float32) joint_cam = np.array(ann['joint_cam'], dtype=np.float32) joint_valid = np.array(ann['joint_valid'], dtype=np.float32) # transform single hand data to double hand data structure hand_type = ann['hand_type'] joint_img_dh = np.zeros((self.joint_num * 2, 2), dtype=np.float32) joint_cam_dh = np.zeros((self.joint_num * 2, 3), dtype=np.float32) joint_valid_dh = np.zeros((self.joint_num * 2), dtype=np.float32) joint_img_dh[self.joint_type[hand_type]] = joint_img joint_cam_dh[self.joint_type[hand_type]] = joint_cam joint_valid_dh[self.joint_type[hand_type]] = joint_valid joint_img = joint_img_dh joint_cam = joint_cam_dh joint_valid = joint_valid_dh if self.mode == 'test' and cfg.trans_test == 'rootnet': bbox = np.array(rootnet_result[str(aid)]['bbox'], dtype=np.float32) abs_depth = rootnet_result[str(aid)]['abs_depth'] else: bbox = np.array(ann['bbox'], dtype=np.float32) # x,y,w,h bbox = process_bbox(bbox, (img_height, img_width)) abs_depth = joint_cam[self.root_joint_idx[hand_type], 2] # single hand abs depth cam_param = {'focal': focal, 'princpt': princpt} joint = { 'cam_coord': joint_cam, 'img_coord': joint_img, 'valid': joint_valid } data = { 'img_path': img_path, 'bbox': bbox, 'cam_param': cam_param, 'joint': joint, 'hand_type': hand_type, 'abs_depth': abs_depth } self.datalist.append(data)
class CocoGenerator(Generator): """!@brief Generate data from the COCO dataset. @see https://github.com/cocodataset/cocoapi/tree/master/PythonAPI for more information. """ def __init__(self, data_dir, set_name, **kwargs): """!@brief Initialize a COCO data generator. @param data_dir: Path to where the COCO dataset is stored. @param set_name: Name of the set to parse. """ self.data_dir = data_dir self.set_name = set_name self.coco = COCO(os.path.join(data_dir, 'annotations', 'instances_' + set_name + '.json')) self.image_ids = self.coco.getImgIds() self.load_classes() super(CocoGenerator, self).__init__(**kwargs) def load_classes(self): """!@brief Loads the class to label mapping (and inverse) for COCO. """ # load class names (name -> label) categories = self.coco.loadCats(self.coco.getCatIds()) categories.sort(key=lambda x: x['id']) self.classes = {} self.coco_labels = {} self.coco_labels_inverse = {} for c in categories: self.coco_labels[len(self.classes)] = c['id'] self.coco_labels_inverse[c['id']] = len(self.classes) self.classes[c['name']] = len(self.classes) # also load the reverse (label -> name) self.labels = {} for key, value in self.classes.items(): self.labels[value] = key def size(self): """!@brief Size of the COCO dataset. """ return len(self.image_ids) def num_classes(self): """!@brief Number of classes in the dataset. For COCO this is 80. """ return len(self.classes) def has_label(self, label): """!@brief Return True if label is a known label. """ return label in self.labels def has_name(self, name): """!@brief Returns True if name is a known class. """ return name in self.classes def name_to_label(self, name): """!@brief Map name to label. """ return self.classes[name] def label_to_name(self, label): """!@brief Map label to name. """ return self.labels[label] def coco_label_to_label(self, coco_label): """!@brief Map COCO label to the label as used in the network. COCO has some gaps in the order of labels. The highest label is 90, but there are 80 classes. """ return self.coco_labels_inverse[coco_label] def coco_label_to_name(self, coco_label): """!@brief Map COCO label to name. """ return self.label_to_name(self.coco_label_to_label(coco_label)) def label_to_coco_label(self, label): """!@brief Map label as used by the network to labels as used by COCO. """ return self.coco_labels[label] def image_aspect_ratio(self, image_index): """!@brief Compute the aspect ratio for an image with image_index. """ image = self.coco.loadImgs(self.image_ids[image_index])[0] return float(image['width']) / float(image['height']) def load_image(self, image_index): """!@brief Load an image at the image_index. """ image_info = self.coco.loadImgs(self.image_ids[image_index])[0] path = os.path.join(self.data_dir, 'images', self.set_name, image_info['file_name']) return read_image_bgr(path) def load_annotations(self, image_index): """!@brief Load annotations for an image_index. """ # get ground truth annotations annotations_ids = self.coco.getAnnIds(imgIds=self.image_ids[image_index], iscrowd=False) annotations = {'labels': np.empty((0,)), 'bboxes': np.empty((0, 4))} # some images appear to miss annotations (like image with id 257034) if len(annotations_ids) == 0: return annotations # parse annotations coco_annotations = self.coco.loadAnns(annotations_ids) for idx, a in enumerate(coco_annotations): # some annotations have basically no width / height, skip them if a['bbox'][2] < 1 or a['bbox'][3] < 1: continue annotations['labels'] = np.concatenate([annotations['labels'], [self.coco_label_to_label(a['category_id'])]], axis=0) annotations['bboxes'] = np.concatenate([annotations['bboxes'], [[ a['bbox'][0], a['bbox'][1], a['bbox'][0] + a['bbox'][2], a['bbox'][1] + a['bbox'][3], ]]], axis=0) return annotations
def initialize(): coco = COCO(ann_path) imgIds = coco.getImgIds(catIds=[], imgIds=[]) dataset_size = len(imgIds) return coco, imgIds, dataset_size
class CocoDataset(CustomDataset): CLASSES = ('person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic_light', 'fire_hydrant', 'stop_sign', 'parking_meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports_ball', 'kite', 'baseball_bat', 'baseball_glove', 'skateboard', 'surfboard', 'tennis_racket', 'bottle', 'wine_glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot_dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted_plant', 'bed', 'dining_table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell_phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy_bear', 'hair_drier', 'toothbrush') def load_annotations(self, ann_file): self.coco = COCO(ann_file) self.cat_ids = self.coco.getCatIds() self.cat2label = { cat_id: i + 1 for i, cat_id in enumerate(self.cat_ids) } self.img_ids = self.coco.getImgIds() img_infos = [] for i in self.img_ids: info = self.coco.loadImgs([i])[0] info['filename'] = info['file_name'] img_infos.append(info) return img_infos def get_ann_info(self, idx): img_id = self.img_infos[idx]['id'] ann_ids = self.coco.getAnnIds(imgIds=[img_id]) ann_info = self.coco.loadAnns(ann_ids) return self._parse_ann_info(self.img_infos[idx], ann_info) def _filter_imgs(self, min_size=32): """Filter images too small or without ground truths.""" valid_inds = [] ids_with_ann = set(_['image_id'] for _ in self.coco.anns.values()) for i, img_info in enumerate(self.img_infos): if self.filter_empty_gt and self.img_ids[i] not in ids_with_ann: continue if min(img_info['width'], img_info['height']) >= min_size: valid_inds.append(i) return valid_inds def _parse_ann_info(self, img_info, ann_info): """Parse bbox and mask annotation. Args: ann_info (list[dict]): Annotation info of an image. with_mask (bool): Whether to parse mask annotations. Returns: dict: A dict containing the following keys: bboxes, bboxes_ignore, labels, masks, seg_map. "masks" are raw annotations and not decoded into binary masks. """ gt_bboxes = [] gt_labels = [] gt_bboxes_ignore = [] gt_masks_ann = [] for i, ann in enumerate(ann_info): if ann.get('ignore', False): continue x1, y1, w, h = ann['bbox'] if ann['area'] <= 0 or w < 1 or h < 1: continue bbox = [x1, y1, x1 + w - 1, y1 + h - 1] if ann.get('iscrowd', False): gt_bboxes_ignore.append(bbox) else: gt_bboxes.append(bbox) gt_labels.append(self.cat2label[ann['category_id']]) gt_masks_ann.append(ann['segmentation']) if gt_bboxes: gt_bboxes = np.array(gt_bboxes, dtype=np.float32) gt_labels = np.array(gt_labels, dtype=np.int64) else: gt_bboxes = np.zeros((0, 4), dtype=np.float32) gt_labels = np.array([], dtype=np.int64) if gt_bboxes_ignore: gt_bboxes_ignore = np.array(gt_bboxes_ignore, dtype=np.float32) else: gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32) seg_map = img_info['filename'].replace('jpg', 'png') ann = dict(bboxes=gt_bboxes, labels=gt_labels, bboxes_ignore=gt_bboxes_ignore, masks=gt_masks_ann, seg_map=seg_map) return ann def xyxy2xywh(self, bbox): _bbox = bbox.tolist() return [ _bbox[0], _bbox[1], _bbox[2] - _bbox[0] + 1, _bbox[3] - _bbox[1] + 1, ] def _proposal2json(self, results): json_results = [] for idx in range(len(self)): img_id = self.img_ids[idx] bboxes = results[idx] for i in range(bboxes.shape[0]): data = dict() data['image_id'] = img_id data['bbox'] = self.xyxy2xywh(bboxes[i]) data['score'] = float(bboxes[i][4]) data['category_id'] = 1 json_results.append(data) return json_results def _det2json(self, results): json_results = [] for idx in range(len(self)): img_id = self.img_ids[idx] result = results[idx] for label in range(len(result)): bboxes = result[label] for i in range(bboxes.shape[0]): data = dict() data['image_id'] = img_id data['bbox'] = self.xyxy2xywh(bboxes[i]) data['score'] = float(bboxes[i][4]) data['category_id'] = self.cat_ids[label] json_results.append(data) return json_results def _segm2json(self, results): bbox_json_results = [] segm_json_results = [] for idx in range(len(self)): img_id = self.img_ids[idx] det, seg = results[idx] for label in range(len(det)): # bbox results bboxes = det[label] for i in range(bboxes.shape[0]): data = dict() data['image_id'] = img_id data['bbox'] = self.xyxy2xywh(bboxes[i]) data['score'] = float(bboxes[i][4]) data['category_id'] = self.cat_ids[label] bbox_json_results.append(data) # segm results # some detectors use different scores for bbox and mask if isinstance(seg, tuple): segms = seg[0][label] mask_score = seg[1][label] else: segms = seg[label] mask_score = [bbox[4] for bbox in bboxes] for i in range(bboxes.shape[0]): data = dict() data['image_id'] = img_id data['bbox'] = self.xyxy2xywh(bboxes[i]) data['score'] = float(mask_score[i]) data['category_id'] = self.cat_ids[label] if isinstance(segms[i]['counts'], bytes): segms[i]['counts'] = segms[i]['counts'].decode() data['segmentation'] = segms[i] segm_json_results.append(data) return bbox_json_results, segm_json_results def results2json(self, results, outfile_prefix): """Dump the detection results to a json file. There are 3 types of results: proposals, bbox predictions, mask predictions, and they have different data types. This method will automatically recognize the type, and dump them to json files. Args: results (list[list | tuple | ndarray]): Testing results of the dataset. outfile_prefix (str): The filename prefix of the json files. If the prefix is "somepath/xxx", the json files will be named "somepath/xxx.bbox.json", "somepath/xxx.segm.json", "somepath/xxx.proposal.json". Returns: dict[str: str]: Possible keys are "bbox", "segm", "proposal", and values are corresponding filenames. """ result_files = dict() if isinstance(results[0], list): json_results = self._det2json(results) result_files['bbox'] = '{}.{}.json'.format(outfile_prefix, 'bbox') result_files['proposal'] = '{}.{}.json'.format( outfile_prefix, 'bbox') mmcv.dump(json_results, result_files['bbox']) elif isinstance(results[0], tuple): json_results = self._segm2json(results) result_files['bbox'] = '{}.{}.json'.format(outfile_prefix, 'bbox') result_files['proposal'] = '{}.{}.json'.format( outfile_prefix, 'bbox') result_files['segm'] = '{}.{}.json'.format(outfile_prefix, 'segm') mmcv.dump(json_results[0], result_files['bbox']) mmcv.dump(json_results[1], result_files['segm']) elif isinstance(results[0], np.ndarray): json_results = self._proposal2json(results) result_files['proposal'] = '{}.{}.json'.format( outfile_prefix, 'proposal') mmcv.dump(json_results, result_files['proposal']) else: raise TypeError('invalid type of results') return result_files def fast_eval_recall(self, results, proposal_nums, iou_thrs, logger=None): gt_bboxes = [] for i in range(len(self.img_ids)): ann_ids = self.coco.getAnnIds(imgIds=self.img_ids[i]) ann_info = self.coco.loadAnns(ann_ids) if len(ann_info) == 0: gt_bboxes.append(np.zeros((0, 4))) continue bboxes = [] for ann in ann_info: if ann.get('ignore', False) or ann['iscrowd']: continue x1, y1, w, h = ann['bbox'] bboxes.append([x1, y1, x1 + w - 1, y1 + h - 1]) bboxes = np.array(bboxes, dtype=np.float32) if bboxes.shape[0] == 0: bboxes = np.zeros((0, 4)) gt_bboxes.append(bboxes) recalls = eval_recalls(gt_bboxes, results, proposal_nums, iou_thrs, logger=logger) ar = recalls.mean(axis=1) return ar def evaluate(self, results, metric='bbox', logger=None, jsonfile_prefix=None, classwise=False, iou_thr_by_class=0.2, proposal_nums=(100, 300, 1000), iou_thrs=np.arange(0.5, 0.96, 0.05)): """Evaluation in COCO protocol. Args: results (list): Testing results of the dataset. metric (str | list[str]): Metrics to be evaluated. logger (logging.Logger | str | None): Logger used for printing related information during evaluation. Default: None. jsonfile_prefix (str | None): classwise (bool): Whether to evaluating the AP for each class. proposal_nums (Sequence[int]): Proposal number used for evaluating recalls, such as recall@100, recall@1000. Default: (100, 300, 1000). iou_thrs (Sequence[float]): IoU threshold used for evaluating recalls. If set to a list, the average recall of all IoUs will also be computed. Default: 0.5. Returns: dict[str: float] """ assert isinstance(results, list), 'results must be a list' assert len(results) == len(self), ( 'The length of results is not equal to the dataset len: {} != {}'. format(len(results), len(self))) metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] for metric in metrics: if metric not in allowed_metrics: raise KeyError('metric {} is not supported'.format(metric)) if jsonfile_prefix is None: tmp_dir = tempfile.TemporaryDirectory() jsonfile_prefix = osp.join(tmp_dir.name, 'results') else: tmp_dir = None result_files = self.results2json(results, jsonfile_prefix) eval_results = {} cocoGt = self.coco for metric in metrics: msg = 'Evaluating {}...'.format(metric) if logger is None: msg = '\n' + msg print_log(msg, logger=logger) if metric == 'proposal_fast': ar = self.fast_eval_recall(results, proposal_nums, iou_thrs, logger='silent') log_msg = [] for i, num in enumerate(proposal_nums): eval_results['AR@{}'.format(num)] = ar[i] log_msg.append('\nAR@{}\t{:.4f}'.format(num, ar[i])) log_msg = ''.join(log_msg) print_log(log_msg, logger=logger) continue if metric not in result_files: raise KeyError('{} is not in results'.format(metric)) try: cocoDt = cocoGt.loadRes(result_files[metric]) except IndexError: print_log('The testing results of the whole dataset is empty.', logger=logger, level=logging.ERROR) break iou_type = 'bbox' if metric == 'proposal' else metric cocoEval = COCOeval(cocoGt, cocoDt, iou_type) cocoEval.params.imgIds = self.img_ids if metric == 'proposal': cocoEval.params.useCats = 0 cocoEval.params.maxDets = list(proposal_nums) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() metric_items = [ 'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ] for i, item in enumerate(metric_items): val = float('{:.3f}'.format(cocoEval.stats[i + 6])) eval_results[item] = val else: cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() if classwise: # Compute per-category AP gt_lst = load_coco_bboxes(cocoGt, is_gt=True) dt_lst = load_coco_bboxes(cocoDt, is_gt=False) evaluator = Evaluator() ret, mAP = evaluator.GetMAPbyClass( gt_lst, dt_lst, iou_thr=iou_thr_by_class) # Get metric values per each class for metricsPerClass in ret: cl = metricsPerClass['class'] ap = metricsPerClass['AP'] ap_str = '{0:.3f}'.format(ap) eval_results['class_{}'.format(cl)] = float(ap_str) print('AP: %s (%s)' % (ap_str, cl)) mAP_str = '{0:.3f}'.format(mAP) eval_results['mAP'] = float(mAP_str) print('mAP: {}\n'.format(mAP_str)) metric_items = [ 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' ] for i in range(len(metric_items)): key = '{}_{}'.format(metric, metric_items[i]) val = float('{:.3f}'.format(cocoEval.stats[i])) eval_results[key] = val eval_results['{}_mAP_copypaste'.format(metric)] = ( '{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} ' '{ap[4]:.3f} {ap[5]:.3f}').format(ap=cocoEval.stats[:6]) if tmp_dir is not None: tmp_dir.cleanup() return eval_results
def __init__(self, image_path, info_file, transforms=None, is_train=False): self.root = image_path self.coco = COCO(info_file) self.ids = list(self.coco.imgToAnns.keys()) # may be some imaegs don't have labels self.transforms = transforms self.is_train is_train