def evaluate_class_agnostic(self): """ Treat all masks as one category. """ lvis_dt = self.lvis_dt lvis_gt = self.lvis_gt feats_ann = self.feats_ann cluster_to_coco = self.cluster_to_coco # by default, none of the predictions gets evaluated. for _, dt in lvis_dt.anns.items(): dt['category_id'] = -2 for _, dt in lvis_gt.anns.items(): dt['category_id'] = -2 print('Updating category ids') for i in tqdm(range(len(feats_ann))): cluster_id = self.clusters[i] ann_id = int(feats_ann[i]) if cluster_id in cluster_to_coco: lvis_dt.anns[ann_id]['category_id'] = -1 # the assigned ones are included in the eval cocoEval = LVISEval(lvis_gt, lvis_dt, 'segm') cocoEval.params.catIds = [-1] # only evaluate on the category -1. cocoEval.params.useCats = 0 cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def evaluate(self, results, metric=['track'], logger=None, resfile_path=None): if isinstance(metric, list): metrics = metric elif isinstance(metric, str): metrics = [metric] else: raise TypeError('metric must be a list or a str.') allowed_metrics = ['bbox', 'track'] for metric in metrics: if metric not in allowed_metrics: raise KeyError(f'metric {metric} is not supported.') result_files, tmp_dir = self.format_results(results, resfile_path) eval_results = dict() if 'track' in metrics: from tao.toolkit.tao import TaoEval print_log('Evaluating TAO results...', logger) tao_eval = TaoEval(self.ann_file, result_files['track']) tao_eval.params.img_ids = self.img_ids tao_eval.params.cat_ids = self.cat_ids tao_eval.params.iou_thrs = np.array([0.5, 0.75]) tao_eval.run() tao_eval.print_results() tao_results = tao_eval.get_results() for k, v in tao_results.items(): if isinstance(k, str) and k.startswith('AP'): key = 'track_{}'.format(k) val = float('{:.3f}'.format(float(v))) eval_results[key] = val if 'bbox' in metrics: print_log('Evaluating detection results...', logger) lvis_gt = LVIS(self.ann_file) lvis_dt = LVISResults(lvis_gt, result_files['bbox']) lvis_eval = LVISEval(lvis_gt, lvis_dt, 'bbox') lvis_eval.params.imgIds = self.img_ids lvis_eval.params.catIds = self.cat_ids lvis_eval.evaluate() lvis_eval.accumulate() lvis_eval.summarize() lvis_eval.print_results() lvis_results = lvis_eval.get_results() for k, v in lvis_results.items(): if k.startswith('AP'): key = '{}_{}'.format('bbox', k) val = float('{:.3f}'.format(float(v))) eval_results[key] = val ap_summary = ' '.join([ '{}:{:.3f}'.format(k, float(v)) for k, v in lvis_results.items() if k.startswith('AP') ]) eval_results['bbox_mAP_copypaste'] = ap_summary if tmp_dir is not None: tmp_dir.cleanup() return eval_results
def evaluate(self, results, metric='bbox', logger=None, jsonfile_prefix=None, classwise=False, proposal_nums=(100, 300, 1000), iou_thrs=np.arange(0.5, 0.96, 0.05)): """Evaluation in LVIS protocol. Args: results (list[list | tuple]): Testing results of the dataset. metric (str | list[str]): Metrics to be evaluated. Options are 'bbox', 'segm', 'proposal', 'proposal_fast'. logger (logging.Logger | str | None): Logger used for printing related information during evaluation. Default: None. jsonfile_prefix (str | None): classwise (bool): Whether to evaluating the AP for each class. proposal_nums (Sequence[int]): Proposal number used for evaluating recalls, such as recall@100, recall@1000. Default: (100, 300, 1000). iou_thrs (Sequence[float]): IoU threshold used for evaluating recalls. If set to a list, the average recall of all IoUs will also be computed. Default: 0.5. Returns: dict[str, float]: LVIS style metrics. """ try: import lvis assert lvis.__version__ >= '10.5.3' from lvis import LVISResults, LVISEval except AssertionError: raise AssertionError('Incompatible version of lvis is installed. ' 'Run pip uninstall lvis first. Then run pip ' 'install mmlvis to install open-mmlab forked ' 'lvis. ') except ImportError: raise ImportError('Package lvis is not installed. Please run pip ' 'install mmlvis to install open-mmlab forked ' 'lvis.') assert isinstance(results, list), 'results must be a list' assert len(results) == len(self), ( 'The length of results is not equal to the dataset len: {} != {}'. format(len(results), len(self))) metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] for metric in metrics: if metric not in allowed_metrics: raise KeyError('metric {} is not supported'.format(metric)) if jsonfile_prefix is None: tmp_dir = tempfile.TemporaryDirectory() jsonfile_prefix = osp.join(tmp_dir.name, 'results') else: tmp_dir = None result_files = self.results2json(results, jsonfile_prefix) eval_results = {} # get original api lvis_gt = self.coco for metric in metrics: msg = 'Evaluating {}...'.format(metric) if logger is None: msg = '\n' + msg print_log(msg, logger=logger) if metric == 'proposal_fast': ar = self.fast_eval_recall(results, proposal_nums, iou_thrs, logger='silent') log_msg = [] for i, num in enumerate(proposal_nums): eval_results['AR@{}'.format(num)] = ar[i] log_msg.append('\nAR@{}\t{:.4f}'.format(num, ar[i])) log_msg = ''.join(log_msg) print_log(log_msg, logger=logger) continue if metric not in result_files: raise KeyError('{} is not in results'.format(metric)) try: lvis_dt = LVISResults(lvis_gt, result_files[metric]) except IndexError: print_log('The testing results of the whole dataset is empty.', logger=logger, level=logging.ERROR) break iou_type = 'bbox' if metric == 'proposal' else metric lvis_eval = LVISEval(lvis_gt, lvis_dt, iou_type) lvis_eval.params.imgIds = self.img_ids if metric == 'proposal': lvis_eval.params.useCats = 0 lvis_eval.params.maxDets = list(proposal_nums) lvis_eval.evaluate() lvis_eval.accumulate() lvis_eval.summarize() for k, v in lvis_eval.get_results().items(): if k.startswith('AR'): val = float('{:.3f}'.format(float(v))) eval_results[k] = val else: lvis_eval.evaluate() lvis_eval.accumulate() lvis_eval.summarize() lvis_results = lvis_eval.get_results() if classwise: # Compute per-category AP # Compute per-category AP # from https://github.com/facebookresearch/detectron2/ precisions = lvis_eval.eval['precision'] # precision: (iou, recall, cls, area range, max dets) assert len(self.cat_ids) == precisions.shape[2] results_per_category = [] for idx, catId in enumerate(self.cat_ids): # area range index 0: all area ranges # max dets index -1: typically 100 per image nm = self.coco.load_cats(catId)[0] precision = precisions[:, :, idx, 0, -1] precision = precision[precision > -1] if precision.size: ap = np.mean(precision) else: ap = float('nan') results_per_category.append( (f'{nm["name"]}', f'{float(ap):0.3f}')) num_columns = min(6, len(results_per_category) * 2) results_flatten = list( itertools.chain(*results_per_category)) headers = ['category', 'AP'] * (num_columns // 2) results_2d = itertools.zip_longest(*[ results_flatten[i::num_columns] for i in range(num_columns) ]) table_data = [headers] table_data += [result for result in results_2d] table = AsciiTable(table_data) print_log('\n' + table.table, logger=logger) for k, v in lvis_results.items(): if k.startswith('AP'): key = '{}_{}'.format(metric, k) val = float('{:.3f}'.format(float(v))) eval_results[key] = val ap_summary = ' '.join([ '{}:{:.3f}'.format(k, float(v)) for k, v in lvis_results.items() if k.startswith('AP') ]) eval_results['{}_mAP_copypaste'.format(metric)] = ap_summary lvis_eval.print_results() if tmp_dir is not None: tmp_dir.cleanup() return eval_results
class LVISEvaluator(object): def __init__(self, run_path, model_ckpt): self.lvis_gt = LVIS(ANNOTATION_PATH) self.lvis_dt = LVISResults(self.lvis_gt, PREDICTION_PATH) self.run_path = run_path self.model_ckpt = model_ckpt self._build_coco_to_lvis_map() cocoEval = LVISEval(self.lvis_gt, self.lvis_dt, 'segm') self.freq_groups = cocoEval._prepare_freq_group() config_path = os.path.join(self.run_path, 'config_lvis.yaml') self.config = yaml.load(open(config_path, "r"), Loader=yaml.FullLoader) def _build_coco_to_lvis_map(self): coco_map = json.load(open(os.path.join(LVIS_API_PATH, 'data/coco_to_synset.json'))) synset_to_lvis = {cat['synset']: cat['id'] for cat in self.lvis_gt.cats.values()} synset_to_lvis['oven.n.01'] = synset_to_lvis['toaster_oven.n.01'] synset_to_lvis['frank.n.02'] = synset_to_lvis['sausage.n.01'] coco_to_lvis = {} lvis_to_coco = {} for item in coco_map.values(): coco_id, lvis_id = item['coco_cat_id'], synset_to_lvis[item['synset']] coco_to_lvis[coco_id] = lvis_id lvis_to_coco[lvis_id] = coco_id self.coco_to_lvis = coco_to_lvis self.lvis_to_coco = lvis_to_coco def reload_annotations(self): self.lvis_gt = LVIS(ANNOTATION_PATH) self.dt_path = PREDICTION_PATH self.lvis_dt = LVISResults(self.lvis_gt, self.dt_path) def _save_gt_features(self): # This takes a long time and shoud only be run once. from eval.feature_saver import LvisSaver import torch config = self.config if self.config['hyperbolic']: from models.hyperbolic_resnet import HResNetSimCLR model = HResNetSimCLR(config['model']['base_model'], config['model']['out_dim']) else: from models.resnet_simclr import ResNetSimCLR model = ResNetSimCLR(config['model']['base_model'], config['model']['out_dim']) state_dict = torch.load(os.path.join(self.run_path, self.model_ckpt)) # , map_location=device)i model.load_state_dict(state_dict) model.eval() saver = LvisSaver(model, self.lvis_gt, GT_FEATS) saver.save() def _save_dt_features(self): from eval.feature_saver import LvisSaver import torch config = self.config if self.config['hyperbolic']: from models.hyperbolic_resnet import HResNetSimCLR model = HResNetSimCLR(config['model']['base_model'], config['model']['out_dim']) else: from models.resnet_simclr import ResNetSimCLR model = ResNetSimCLR(config['model']['base_model'], config['model']['out_dim']) state_dict = torch.load(os.path.join(self.run_path, self.model_ckpt)) # , map_location=device)i model.load_state_dict(state_dict) model.eval() saver = LvisSaver(model, self.lvis_dt, GT_FEATS) saver.save() def load_gt_features(self, coco_only=False, k=100, freq_groups=None): """ Load gt features from GT_FEATS folder. :param coco_only: only load categories that are in COCO. :param k: only load k masks for each category. :param freq_groups: only load categories in the specified freq_groups. e.g. ['f', 'r'] """ feats = [] y = [] files = os.listdir(GT_FEATS) if len(files) == 0: self._save_gt_features() print('Found {} files.'.format(len(files))) for f in files: if f.endswith('_x.npy'): feats.append(np.load(os.path.join(GT_FEATS, f))) elif f.endswith('_y.npy'): y.extend(np.load(os.path.join(GT_FEATS, f))) feats = np.concatenate(feats) print(feats.shape) self.feats_gt = feats self.feats_gt_y = np.array(y) if freq_groups is not None: print('Filter by freq groups', freq_groups) freqs = (np.concatenate([self.freq_groups[i] for i in freq_groups]) + 1).astype(np.int) idx = np.isin(self.feats_gt_y, freqs) self.feats_gt_y = self.feats_gt_y[idx] self.feats_gt = self.feats_gt[idx] print('After:', self.feats_gt.shape) if coco_only: coco_cats = self.lvis_to_coco.keys() idx = np.array([y in coco_cats for y in self.feats_gt_y]) self.feats_gt = self.feats_gt[idx] self.feats_gt_y = self.feats_gt_y[idx] print('Keeping objects in COCO', self.feats_gt.shape) if k: print('Keeping only {} masks for each class'.format(k)) new_feats_gt = [] new_feats_gt_y = [] counts = Counter(self.feats_gt_y) for i, c in counts.items(): if c > k: idx = np.random.choice(np.arange(len(self.feats_gt_y))[self.feats_gt_y == i], k, replace=False) # print(self.feats_gt_y[idx]) new_feats_gt.append(self.feats_gt[idx]) new_feats_gt_y.extend([i] * k) else: new_feats_gt.append(self.feats_gt[self.feats_gt_y == i]) new_feats_gt_y.extend([i] * c) self.feats_gt = np.concatenate(new_feats_gt) self.feats_gt_y = new_feats_gt_y print(self.feats_gt.shape) def fit_knn(self, k=5, weights='distance'): """ Fit a KNN model on the ground truth mask features to see whether the embeddings makes sense. """ feats = self.feats_gt y = self.feats_gt_y if self.config['hyperbolic']: from hyperbolic_knn import HyperbolicKNN self.neigh = HyperbolicKNN(k, feats, y) pred_y = self.neigh.predict(feats) print('KNN accuracy', accuracy_score(y, pred_y)) else: from sklearn.neighbors import KNeighborsClassifier self.neigh = KNeighborsClassifier(n_neighbors=k, weights=weights) self.neigh.fit(feats, y) print('KNN accuracy', self.neigh.score(feats, y)) def load_dt_features(self): feats = [] feats_ann = [] files = os.listdir(DT_FEATS) if len(files) == 0: self._save_dt_features() print('Found {} files.'.format(len(files))) for f in files: if f.endswith('_x.npy'): feats.append(np.load(os.path.join(DT_FEATS, f))) elif f.endswith('_ann_id.npy'): feats_ann.extend(np.load(os.path.join(DT_FEATS, f))) self.feats = np.concatenate(feats) self.feats_ann = np.array(feats_ann) print(self.feats.shape, self.feats_ann.shape) def run_kmeans(self, C=1500): feats = self.feats if self.config['hyperbolic']: print('Running Hyperbolic KMeans...') from poincare_kmeans import PoincareKMeans as HKMeans assert self.feats.shape[1] == 2, 'only supports hkmeans in 2d.' kmeans = HKMeans(self.feats.shape[1], C) clusters = kmeans.fit_predict(self.feats) self.clusters = clusters else: print('Running Euclidean KMeans...') from sklearn.cluster import MiniBatchKMeans kmeans = MiniBatchKMeans(C) clusters = kmeans.fit_predict(feats) self.clusters = clusters def assign_labels(self): """ Take the clusters assigned by kmeans and assign labels to the clusters. :return: """ neigh = self.neigh clusters = self.clusters feats = self.feats C = len(set(clusters)) coco_clusters = {} cluster_to_coco = {} print('Assigning labels using KNN ...') for i in tqdm(range(C)): idx = np.where(clusters == i)[0] if len(idx) == 0: continue predicted = neigh.predict(feats[idx]) votes = sorted(Counter(predicted).items(), key=lambda tup: -tup[1]) best_ratio = votes[0][1] / len(predicted) if len(predicted) < 3: continue # ignore clusters with fewer than 5 if best_ratio < 0.95: continue cluster_to_coco[i] = (votes[0][0], best_ratio, len(predicted)) self.cluster_to_coco = cluster_to_coco self.coco_clusters = coco_clusters print('Number of assigned clusters:', len(cluster_to_coco)) def evaluate(self): cluster_to_coco = self.cluster_to_coco lvis_dt = self.lvis_dt clusters = self.clusters feats_ann = self.feats_ann # by default everything is -1. for _, dt in lvis_dt.anns.items(): dt['category_id'] = -1 print('Updating category ids') for i in tqdm(range(len(feats_ann))): ann_id = int(feats_ann[i]) cluster_id = clusters[i] if cluster_id in cluster_to_coco: lvis_dt.anns[ann_id]['category_id'] = cluster_to_coco[cluster_id][0] # print('assigned ', cluster_to_coco[cluster_id][0]) print('Finally, evaluate!!') self.lvisEval = LVISEval(self.lvis_gt, lvis_dt, 'segm') # img_ids = cocoDt.get_img_ids()[:100] # lvisEval.params.catIds = [1, 2, 3, 4]# 5, 6, 7, 8, 9, 10, 11, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 33, 34, 35, 37, 40, 41, 42, 43, 44, 46, 47, 49, 50, 51, 52, 54, 56, 57, 59, 60, 61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 77, 78, 79, 81, 82, 84, 85, 86, 87, 88, 90] # lvisEval.params.imgIds = img_ids # lvisEval.params.iouThrs = np.linspace(.25, 0.95, int(np.round((0.95 - .25) / .05)) + 1, endpoint=True) self.lvisEval.lvis_gt.cats[-1] = {'frequency': 'f', 'id': -1, 'synset': 'all', 'image_count': 0, 'instance_count': 0, 'synonyms': ['all'], 'def': 'dummy category', 'name': 'all'} self.lvisEval.evaluate() self.lvisEval.accumulate() self.lvisEval.summarize() def evaluate_class_agnostic(self): """ Treat all masks as one category. """ lvis_dt = self.lvis_dt lvis_gt = self.lvis_gt feats_ann = self.feats_ann cluster_to_coco = self.cluster_to_coco # by default, none of the predictions gets evaluated. for _, dt in lvis_dt.anns.items(): dt['category_id'] = -2 for _, dt in lvis_gt.anns.items(): dt['category_id'] = -2 print('Updating category ids') for i in tqdm(range(len(feats_ann))): cluster_id = self.clusters[i] ann_id = int(feats_ann[i]) if cluster_id in cluster_to_coco: lvis_dt.anns[ann_id]['category_id'] = -1 # the assigned ones are included in the eval cocoEval = LVISEval(lvis_gt, lvis_dt, 'segm') cocoEval.params.catIds = [-1] # only evaluate on the category -1. cocoEval.params.useCats = 0 cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
class Eval_KMeans(object): def __init__(self): self.lvis = LVIS('/scratch/users/zzweng/datasets/lvis/lvis_v0.5_val.json') self.dt_path = 'output/inference/lvis_instances_results.json' self.lvis_dt = LVISResults(self.lvis, self.dt_path) coco_map = json.load(open('lvis-api/data/coco_to_synset.json')) synset_to_lvis = {cat['synset']: cat['id'] for cat in self.lvis.cats.values()} synset_to_lvis['oven.n.01'] = synset_to_lvis['toaster_oven.n.01'] synset_to_lvis['frank.n.02'] = synset_to_lvis['sausage.n.01'] coco_to_lvis = {} lvis_to_coco = {} for item in coco_map.values(): coco_id, lvis_id = item['coco_cat_id'], synset_to_lvis[item['synset']] coco_to_lvis[coco_id] = lvis_id lvis_to_coco[lvis_id] = coco_id self.coco_to_lvis = coco_to_lvis self.lvis_to_coco = lvis_to_coco cocoEval = LVISEval(self.lvis, self.lvis_dt,'segm') self.freq_groups = cocoEval._prepare_freq_group() # just making sure I am not using any labels in DT annotations, let's wipe them all # for _, dt in self.cocoDt.anns.items(): dt['category_id'] = -1 def reload_annotations(self): self.lvis = LVIS('/scratch/users/zzweng/datasets/lvis/lvis_v0.5_val.json') self.dt_path = 'output/inference/lvis_instances_results.json' self.lvis_dt = LVISResults(self.lvis, self.dt_path) def load_gt_features(self, coco_only=False, k=100, freq_groups=None): feats = [] y = [] rng = np.linspace(0, 5000, 51, dtype=int) args = list(zip(rng[:-1], rng[1:])) # args = [(400, 500), (500, 600), (600, 700), (700, 800), # (800, 900), (900, 1000), (1000, 1100), (1100, 1200)] for rng in args: start, end = rng[0], rng[1] try: feats.append(np.load(r'{}/val_feats_{}_{}_x.npy'.format(val_feat_folder, start, end))) y.extend(np.load(r'{}/val_feats_{}_{}_y.npy'.format(val_feat_folder, start, end))) except FileNotFoundError: print('File {}/{}_{}.npy not found. Skipped.'.format(val_feat_folder, start, end)) feats = np.concatenate(feats) print(feats.shape) self.feats_gt = feats self.feats_gt_y = np.array(y) if freq_groups is not None: print('Filter by freq groups', freq_groups) freqs = (np.concatenate([self.freq_groups[i] for i in freq_groups])+1).astype(np.int) idx = np.isin(self.feats_gt_y, freqs) self.feats_gt_y = self.feats_gt_y[idx] self.feats_gt = self.feats_gt[idx] print('After:', self.feats_gt.shape) if coco_only: coco_cats = self.lvis_to_coco.keys() idx = np.array([y in coco_cats for y in self.feats_gt_y]) self.feats_gt = self.feats_gt[idx] self.feats_gt_y = self.feats_gt_y[idx] print('Keeping objects in COCO', self.feats_gt.shape) if k: print('Keeping only {} masks for each class'.format(k)) new_feats_gt = [] new_feats_gt_y = [] counts = Counter(self.feats_gt_y) for i, c in counts.items(): if c > k: idx = np.random.choice(np.arange(len(self.feats_gt_y))[self.feats_gt_y==i], k, replace=False) # print(self.feats_gt_y[idx]) new_feats_gt.append(self.feats_gt[idx]) new_feats_gt_y.extend([i]*k) else: new_feats_gt.append(self.feats_gt[self.feats_gt_y==i]) new_feats_gt_y.extend([i]*c) self.feats_gt = np.concatenate(new_feats_gt) self.feats_gt_y = new_feats_gt_y print(self.feats_gt.shape) def fit_knn(self, k=5, weights='distance'): feats = self.feats_gt y = self.feats_gt_y if 'hyperbolic' in dt_feat_folder: self.neigh = HyperbolicKNN(k, feats, y) pred_y = self.neigh.predict(feats[:50]) print('KNN accuracy', accuracy_score(y[:50], pred_y)) else: self.neigh = KNeighborsClassifier(n_neighbors=k, weights=weights) self.neigh.fit(feats, y) print('KNN accuracy', self.neigh.score(feats, y)) def load_dt_features(self): feats = [] feats_ann = [] rng = np.linspace(0, 5000, 51, dtype=int) args = list(zip(rng[:-1], rng[1:])) for rng in args: start, end = rng[0], rng[1] try: feats.append(np.load(r'{}/{}_{}.npy'.format(dt_feat_folder, start, end))) feats_ann.extend(np.load(r'{}/{}_{}_ann.npy'.format(dt_feat_folder, start, end))) except FileNotFoundError: print('File {}/{}_{}.npy not found. Skipped.'.format(dt_feat_folder, start, end)) self.feats = np.concatenate(feats) self.feats_ann = feats_ann print(self.feats.shape) def run_kmeans(self, C=1500): feats = self.feats feats_ann = self.feats_ann print('Running KMeans ...') kmeans = MiniBatchKMeans(C) clusters = kmeans.fit_predict(feats) self.clusters = clusters def run_hyperbolic_kmeans(self, C=200): from poincare_kmeans import PoincareKMeans as HKMeans kmeans = HKMeans(self.feats.shape[1], C) clusters = kmeans.fit_predict(self.feats) self.clusters = clusters def assign_labels(self): neigh = self.neigh cocoDt = self.lvis_dt clusters = self.clusters feats = self.feats feats_ann = self.feats_ann C = len(set(clusters)) # feats = self.pca.transform(feats) print(feats.shape) coco_clusters = {} cluster_to_coco = {} print('Assigning labels using KNN ...') for i in tqdm(range(C)): idx = np.where(clusters==i)[0] if len(idx) == 0: continue predicted = neigh.predict(feats[idx]) # neighbors = neigh.kneighbors(feats[np.where(clusters==i)])[1] # distances = neigh.kneighbors(feats[np.where(clusters==i)])[0] votes = sorted(Counter(predicted).items(), key=lambda tup:-tup[1]) best_ratio = votes[0][1] / len(predicted) # if len(predicted) < 3: continue # ignore clusters with fewer than 5 if best_ratio < 0.95: continue cluster_to_coco[i] = (votes[0][0], best_ratio, len(predicted)) # if votes[0][0] not in coco_clusters or coco_clusters[votes[0][0]][1] < best_ratio: # coco_clusters[votes[0][0]] = (i, best_ratio, len(predicted)) # cluster_to_coco[i] = (votes[0][0], best_ratio, len(predicted)) # for j in range(1, len(votes)): # if votes[j][0] not in coco_clusters or coco_clusters[votes[j][0]][1] < votes[j][1] / len(predicted): # coco_clusters[votes[j][0]] = (i, votes[j][1] / len(predicted), len(predicted)) # cluster_to_coco[i] = votes[j][0] self.cluster_to_coco = cluster_to_coco self.coco_clusters = coco_clusters print('Number of assigned clusters:', len(cluster_to_coco)) def evaluate_plain(self): cocoEval = COCOeval(self.cocoGt, self.cocoDt,'segm') img_ids = self.cocoDt.getImgIds()[:100] cocoEval.params.imgIds = img_ids cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() self.cocoEval = cocoEval def evaluate(self): # self.reload_annotation() cluster_to_coco = self.cluster_to_coco coco_clusters = self.coco_clusters cocoDt = self.lvis_dt clusters = self.clusters feats_ann = self.feats_ann # by default everything is -1. for _, dt in cocoDt.anns.items(): dt['category_id'] = -1 print('Updating category ids') for i in tqdm(range(len(feats_ann))): ann_id = int(feats_ann[i]) cluster_id = clusters[i] if cluster_id in cluster_to_coco: cocoDt.anns[ann_id]['category_id'] = cluster_to_coco[cluster_id][0] # print('assigned ', cluster_to_coco[cluster_id][0]) print('Finally, evaluate!!') self.cocoEval = LVISEval(self.lvis, cocoDt,'segm') img_ids = cocoDt.get_img_ids()[:100] # cocoEval.params.catIds = [1, 2, 3, 4]# 5, 6, 7, 8, 9, 10, 11, 13, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 33, 34, 35, 37, 40, 41, 42, 43, 44, 46, 47, 49, 50, 51, 52, 54, 56, 57, 59, 60, 61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 77, 78, 79, 81, 82, 84, 85, 86, 87, 88, 90] # cocoEval.params.imgIds = img_ids # cocoEval.params.iouThrs = np.linspace(.25, 0.95, int(np.round((0.95 - .25) / .05)) + 1, endpoint=True) self.cocoEval.lvis_gt.cats[-1] = {'frequency': 'f', 'id': -1, 'synset': 'all', 'image_count': 0, 'instance_count': 0, 'synonyms': ['all'], 'def': 'nut from an oak tree', 'name': 'all'} import pdb pdb.set_trace() self.cocoEval.evaluate() self.cocoEval.accumulate() self.cocoEval.summarize() def evaluate_class_agnostic_all(self): cocoDt = self.cocoDt cocoGt = self.cocoGt feats_ann = self.feats_ann clusters = self.clusters cluster_to_coco = self.cluster_to_coco for _, dt in cocoDt.anns.items(): dt['category_id'] = -1 for _, dt in cocoGt.anns.items(): dt['category_id'] = -1 cocoEval = COCOeval(cocoGt, cocoDt,'segm') img_ids = cocoDt.getImgIds()[:100] print(len(img_ids)) cocoEval.params.imgIds = img_ids cocoEval.params.catIds = [-1] cocoEval.params.useCats = 0 cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() def evaluate_class_agnostic(self): cocoDt = self.cocoDt cocoGt = self.cocoGt feats_ann = self.feats_ann clusters = self.clusters cluster_to_coco = self.cluster_to_coco for _, dt in cocoDt.anns.items(): dt['category_id'] = -2 for _, dt in cocoGt.anns.items(): dt['category_id'] = -1 for i in range(len(feats_ann)): ann_id = int(feats_ann[i]) cluster_id = clusters[i] ann = cocoDt.loadAnns(ann_id)[0] if cluster_id in cluster_to_coco: ann['category_id'] = -1 else: ann['category_id'] = -2 cocoEval = COCOeval(cocoGt, cocoDt,'segm') img_ids = cocoDt.getImgIds()[:100] print(len(img_ids)) cocoEval.params.imgIds = img_ids cocoEval.params.catIds = [-1] cocoEval.params.useCats = 0 cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def _evaluate_predictions_on_lvis_per_class(lvis_gt, lvis_results, iou_type, class_names=None): """ Args: iou_type (str): kpt_oks_sigmas (list[float]): class_names (None or list[str]): if provided, will use it to predict per-category AP. Returns: a dict of {metric name: score} """ metrics = { "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], }[iou_type] logger = logging.getLogger(__name__) if len(lvis_results) == 0: # TODO: check if needed logger.warn("No predictions from the model! Set scores to -1") return {metric: -1 for metric in metrics} if iou_type == "segm": lvis_results = copy.deepcopy(lvis_results) # When evaluating mask AP, if the results contain bbox, LVIS API will # use the box area as the area of the instance, instead of the mask area. # This leads to a different definition of small/medium/large. # We remove the bbox field to let mask AP use mask area. for c in lvis_results: c.pop("bbox", None) from lvis import LVISEval, LVISResults lvis_results = LVISResults(lvis_gt, lvis_results) lvis_eval = LVISEval(lvis_gt, lvis_results, iou_type) lvis_eval.evaluate() lvis_eval.accumulate() precisions = lvis_eval.eval["precision"] results_per_category = [] for idx, name in enumerate(class_names): # area range index 0: all area ranges # max dets index -1: typically 100 per image precision = precisions[:, :, idx, 0] precision = precision[precision > -1] ap = np.mean(precision) if precision.size else float("nan") results_per_category.append(("{}".format(name), float(ap * 100))) lvis_eval.summarize() lvis_eval.print_results() # Pull the standard metrics from the LVIS results results = lvis_eval.get_results() results = {metric: float(results[metric] * 100) for metric in metrics} logger.info("Evaluation results for {}: \n".format(iou_type) + create_small_table(results)) results.update({"AP-" + name: ap for name, ap in results_per_category}) return results
def eval_cocofied_lvis_result(self, gt_file, result_file, metric='segm'): def get_lvis_format_result(lvis_params, lvis_results): template = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} catIds={:>3s}] = {:0.3f}" result_list = [] for key, value in lvis_results.items(): max_dets = lvis_params.max_dets if "AP" in key: title = "Average Precision" _type = "(AP)" else: title = "Average Recall" _type = "(AR)" if len(key) > 2 and key[2].isdigit(): iou_thr = (float(key[2:]) / 100) iou = "{:0.2f}".format(iou_thr) else: iou = "{:0.2f}:{:0.2f}".format( lvis_params.iou_thrs[0], lvis_params.iou_thrs[-1] ) if len(key) > 2 and key[2] in ["r", "c", "f"]: cat_group_name = key[2] else: cat_group_name = "all" if len(key) > 2 and key[2] in ["s", "m", "l"]: area_rng = key[2] else: area_rng = "all" result_list.append(template.format(title, _type, iou, area_rng, max_dets, cat_group_name, value)) return result_list print('load gt json') lvis_gt = LVIS(gt_file) cat_ids = lvis_gt.get_cat_ids() print('load pred json') lvis_dt = LVISResults(lvis_gt, result_file) print('evaluating') lvis_eval = LVISEval(lvis_gt, lvis_dt, metric) lvis_eval.params.imgIds = lvis_gt.get_img_ids() lvis_eval.evaluate() lvis_eval.accumulate() lvis_eval.summarize() # Compute per-category AP precisions = lvis_eval.eval['precision'] assert len(cat_ids) == precisions.shape[2] results_per_category = [] for idx, catId in enumerate(cat_ids): nm = lvis_gt.load_cats([catId])[0] precision = precisions[:, :, idx, 0] precision = precision[precision > -1] if precision.size: ap = np.mean(precision) else: ap = float('nan') results_per_category.append( (f'{nm["name"]}', f'{float(ap):0.3f}')) num_columns = min(6, len(results_per_category) * 2) results_flatten = list( itertools.chain(*results_per_category)) headers = ['category', 'AP'] * (num_columns // 2) results_2d = itertools.zip_longest(*[ results_flatten[i::num_columns] for i in range(num_columns) ]) table_data = [headers] table_data += [result for result in results_2d] table = AsciiTable(table_data) print_log('\n' + table.table) format_summary_result_list = get_lvis_format_result(lvis_eval.params, lvis_eval.results) format_summary_result = "\n".join(format_summary_result_list) with open(f"cocofied_per-category-ap-{metric}.txt", 'w') as f: f.write(table.table + "\n" + format_summary_result) lvis_eval.print_results()