def evaluate(self, snapshot_name=''): current_device = cuda.get_device_from_id(self.args.gpu) with current_device: gt_data = [] pred_data = [] for i, batch in enumerate( tqdm(self.data_iterator, total=len(self.data_loader) // self.args.batchsize)): image, gt_bboxes, gt_labels = batch[0] gt_data.append((gt_bboxes, gt_labels)) # if self.args.gpu is not None: # image = cuda.to_gpu(image, current_device) with cuda.Device(self.args.gpu): with configuration.using_config('train', False): bboxes, labels, scores = self.model.predict( image.copy()[None, ...]) if len(bboxes[0]) == 0: bboxes = [np.zeros((1, 4), dtype=np.float32)] labels = [np.zeros((1, ), dtype=np.int32)] scores = [np.zeros((1, ), dtype=np.float32)] pred_data.append((bboxes[0], labels[0], scores[0])) # TODO handle empty predictions!! bboxes, labels, scores = zip(*pred_data) gt_bboxes, gt_labels = concat_examples(gt_data) result = eval_detection_voc(bboxes, labels, scores, gt_bboxes, gt_labels, None) map = result['map'] self.save_eval_results(snapshot_name, map)
def test_eval_detection_voc(self): pred_bboxes = [ np.array([[0., 4., 1., 5.], [0., 0., 1., 1.]]), np.array([[0., 0., 2., 2.], [2., 2., 3., 3.], [5., 5., 7., 7.]]) ] pred_labels = [np.array([0, 0]), np.array([0, 1, 1])] pred_scores = [np.array([1., 0.9]), np.array([0.7, 0.6, 0.8])] gt_bboxes = [ np.array([[0., 0., 1., 1.], [1., 0., 4., 4.]]), np.array([[2., 2., 3., 3.]]) ] gt_labels = [np.array([0, 0]), np.array([1])] results = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, iou_thresh=self.iou_thresh, use_07_metric=self.use_07_metric) np.testing.assert_equal(results[0]['recall'], self.rec0) np.testing.assert_equal(results[0]['precision'], self.prec0) np.testing.assert_almost_equal(results[0]['ap'], self.ap0) np.testing.assert_equal(results[1]['recall'], self.rec1) np.testing.assert_equal(results[1]['precision'], self.prec1) np.testing.assert_almost_equal(results[1]['ap'], self.ap1) np.testing.assert_almost_equal(results['map'], (self.ap0 + self.ap1) / 2)
def evaluate(self): iterator = self._iterators['main'] target = self._targets['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) pred_bboxes, pred_labels, pred_scores, gt_values =\ apply_detection_link(target, it) if len(gt_values) == 3: gt_bboxes, gt_labels, gt_difficults = gt_values elif len(gt_values) == 2: gt_bboxes, gt_labels = gt_values gt_difficults = None eval_ = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=self.use_07_metric) observation = {} with reporter.report_scope(observation): reporter.report({'map': eval_['map']}, target) return observation
def main(): config, args = parse_args() model = get_model(config["model"]) if args.gpu != -1: model.to_gpu(args.gpu) dataset = VOCBboxDataset( data_dir="../dataset/VOC_test/VOC2007_test", year='2007', split='test', use_difficult=True, return_difficult=True) iterator = iterators.SerialIterator( dataset, args.batchsize, repeat=False, shuffle=False) imgs, pred_values, gt_values = apply_prediction_to_iterator( model.predict, iterator, hook=ProgressHook(len(dataset))) # delete unused iterator explicitly del imgs pred_bboxes, pred_labels, pred_scores = pred_values gt_bboxes, gt_labels, gt_difficults = gt_values result = eval_detection_voc( pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=True) print() print('mAP: {:f}'.format(result['map'])) for l, name in enumerate(voc_bbox_label_names): if result['ap'][l]: print('{:s}: {:f}'.format(name, result['ap'][l])) else: print('{:s}: -'.format(name))
def main(): chainer.config.train = False parser = argparse.ArgumentParser() parser.add_argument('--root', required=True) parser.add_argument('--data_type', choices=opt.data_types, required=True) parser.add_argument('--det_type', choices=opt.detectors, required=True, default='ssd300') parser.add_argument('--load', help='load original trained model') parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--batchsize', type=int, default=32) args = parser.parse_args() model_args = { 'n_fg_class': len(voc_bbox_label_names), 'pretrained_model': 'voc0712' } model = helper.get_detector(args.det_type, model_args) if args.load: chainer.serializers.load_npz(args.load, model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() model.use_preset('evaluate') dataset = helper.get_detection_dataset(args.data_type, 'test', args.root) iterator = chainer.iterators.SerialIterator(dataset, args.batchsize, repeat=False, shuffle=False) imgs, pred_values, gt_values = apply_prediction_to_iterator( model.predict, iterator, hook=ProgressHook(len(dataset))) # delete unused iterator explicitly del imgs pred_bboxes, pred_labels, pred_scores = pred_values gt_bboxes, gt_labels = gt_values result = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, use_07_metric=True) aps = result['ap'] aps = aps[~np.isnan(aps)] print('') print('mAP: {:f}'.format(100.0 * result['map'])) print(aps)
def evaulate_single_model(self, logsdir): metrics_file = join([logsdir, 'metrics.list']) assert file_exists( metrics_file), "Metrics file {} doesn't exist!".format( metrics_file) with open(join([logsdir, 'metrics.list']), 'rb') as f: metrics = pickle.load(f) p_b, p_l, p_s, r_b, g_b, g_l, m_b = zip(*metrics.values()) pred_bboxes, pred_labels, pred_scores, refn_bboxes, gt_bboxes, gt_labels, mtsm_bboxes = self.get_list_of_metrics( [p_b, p_l, p_s, r_b, g_b, g_l, m_b]) for iou_thresh in np.linspace(0.5, 1.0, 11): detector_stats = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, iou_thresh=iou_thresh) bbox_align_stats = eval_detection_voc(refn_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, iou_thresh=iou_thresh) straddling_stats = eval_detection_voc(mtsm_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, iou_thresh=iou_thresh) sstr = self.pprint(detector_stats, bbox_align_stats, straddling_stats, iou_thresh) with open( join([logsdir, 'metrics_{:.2f}.table'.format(iou_thresh)]), 'w') as f: f.write(sstr)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', choices=('faster_rcnn', 'ssd300', 'ssd512'), default='ssd300') parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--batchsize', type=int, default=32) args = parser.parse_args() if args.model == 'faster_rcnn': model = FasterRCNNVGG16(pretrained_model='voc07') elif args.model == 'ssd300': model = SSD300(pretrained_model='voc0712') elif args.model == 'ssd512': model = SSD512(pretrained_model='voc0712') if args.gpu >= 0: chainer.cuda.get_device(args.gpu).use() model.to_gpu() model.use_preset('evaluate') dataset = VOCDetectionDataset(year='2007', split='test', use_difficult=True, return_difficult=True) iterator = iterators.SerialIterator(dataset, args.batchsize, repeat=False, shuffle=False) pred_bboxes, pred_labels, pred_scores, gt_values = \ apply_detection_link(model, iterator, hook=ProgressHook(len(dataset))) gt_bboxes, gt_labels, gt_difficults = gt_values eval_ = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=True) print() print('mAP: {:f}'.format(eval_['map'])) for l, name in enumerate(voc_detection_label_names): if l in eval_: print('{:s}: {:f}'.format(name, eval_[l]['ap'])) else: print('{:s}: -'.format(name))
def evaluate(self): target = self._targets['main'] iterator = self._iterators['main'] iterator.reset() ret = _apply_iterator(iterator, target) result = eval_detection_voc(*ret) report = {'map': result['map']} observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def eval_(out_values, rest_values): pred_bboxes, pred_labels, pred_scores = out_values gt_bboxes, gt_labels, gt_difficults = rest_values result = eval_detection_voc( pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=True) print() print('mAP: {:f}'.format(result['map'])) for l, name in enumerate(voc_bbox_label_names): if result['ap'][l]: print('{:s}: {:f}'.format(name, result['ap'][l])) else: print('{:s}: -'.format(name))
def evaluate(self): iterator = self._iterators['main'] target = self._targets['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) # set current device to devices['main'] with chainer.cuda.Device(self.device): print(self.device) imgs, pred_values, gt_values = apply_prediction_to_iterator( target.predict, it) # delete unused iterator explicitly del imgs pred_bboxes, pred_labels, pred_scores = pred_values if len(gt_values) == 3: gt_bboxes, gt_labels, gt_difficults = gt_values elif len(gt_values) == 2: gt_bboxes, gt_labels = gt_values gt_difficults = None result = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=self.use_07_metric) report = {'map': result['map']} if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['ap/{:s}'.format(label_name)] = result['ap'][l] except IndexError: report['ap/{:s}'.format(label_name)] = np.nan observation = dict() with reporter.report_scope(observation): reporter.report(report, target) return observation
def test_eval_detection_voc(self): pred_bboxes = self.result['bboxes'] pred_labels = self.result['labels'] pred_scores = self.result['scores'] gt_bboxes = self.dataset['bboxes'] gt_labels = self.dataset['labels'] gt_difficults = self.dataset['difficults'] result = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=True) # these scores were calculated by MATLAB code expected = [ 0.772727, 0.738780, 0.957576, 0.640153, 0.579473, 1.000000, 0.970030, 1.000000, 0.705931, 0.678719, 0.863636, 1.000000, 1.000000, 0.561364, 0.798813, 0.712121, 0.939394, 0.563636, 0.927273, 0.654545, ] np.testing.assert_almost_equal(result['ap'], expected, decimal=5) np.testing.assert_almost_equal(result['map'], np.nanmean(expected), decimal=5)
def evaluate(self): target = self._targets['main'] if self.comm is not None and self.comm.rank != 0: apply_to_iterator(target.predict, None, comm=self.comm) return {} iterator = self._iterators['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) in_values, out_values, rest_values = apply_to_iterator( target.predict, it, comm=self.comm) # delete unused iterators explicitly del in_values pred_bboxes, pred_labels, pred_scores = out_values if len(rest_values) == 3: gt_bboxes, gt_labels, gt_difficults = rest_values elif len(rest_values) == 2: gt_bboxes, gt_labels = rest_values gt_difficults = None result = eval_detection_voc( pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=self.use_07_metric) report = {'map': result['map']} if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['ap/{:s}'.format(label_name)] = result['ap'][l] except IndexError: report['ap/{:s}'.format(label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def test_eval_detection_voc_one_bbox(self): pred_bboxes = [ np.array([[0., 0., 1., 1.], [0., 0., 2., 2.], [0.3, 0.3, 0.5, 0.5]]) ] pred_labels = [np.array([0, 0, 0])] pred_scores = [np.array([0.8, 0.9, 1.])] gt_bboxes = [np.array([[0., 0., 1., 0.9]])] gt_labels = [np.array([0])] # iou is [0.95, 0.422, 0.3789] results = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, iou_thresh=self.iou_thresh) np.testing.assert_equal(results[0]['recall'], self.rec) np.testing.assert_equal(results[0]['precision'], self.prec)
def test_eval_detection_voc(self): pred_bboxes = self.result['bboxes'] pred_labels = self.result['labels'] pred_scores = self.result['scores'] gt_bboxes = self.dataset['bboxes'] gt_labels = self.dataset['labels'] gt_difficults = self.dataset['difficults'] result = eval_detection_voc( pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=True) # these scores were calculated by MATLAB code expected = [ 0.772727, 0.738780, 0.957576, 0.640153, 0.579473, 1.000000, 0.970030, 1.000000, 0.705931, 0.678719, 0.863636, 1.000000, 1.000000, 0.561364, 0.798813, 0.712121, 0.939394, 0.563636, 0.927273, 0.654545, ] np.testing.assert_almost_equal(result['ap'], expected, decimal=5) np.testing.assert_almost_equal( result['map'], np.nanmean(expected), decimal=5)
def evaluate(self): iterator = self._iterators['main'] target = self._targets['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) in_values, out_values, rest_values = apply_to_iterator( target.predict, it) # delete unused iterators explicitly del in_values pred_bboxes, pred_labels, pred_scores = out_values if len(rest_values) == 3: gt_bboxes, gt_labels, gt_difficults = rest_values elif len(rest_values) == 2: gt_bboxes, gt_labels = rest_values gt_difficults = None result = eval_detection_voc( pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=self.use_07_metric) report = {'map': result['map']} if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['ap/{:s}'.format(label_name)] = result['ap'][l] except IndexError: report['ap/{:s}'.format(label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def test_eval_detection_voc_difficult(self): pred_bboxes = [ np.array([[0., 0., 1., 1.], [0., 0., 2., 2.], [0.3, 0.3, 0.5, 0.5]]) ] pred_labels = [np.array([0, 0, 0])] pred_scores = [np.array([0.8, 0.9, 1.])] gt_bboxes = [np.array([[0., 0., 1., 0.9], [1., 1., 2., 2.]])] gt_labels = [np.array([0, 0])] gt_difficults = [np.array([False, True])] # iou is [0.95, 0.422, 0.3789] and [0.142, 0.444, 0.048] results = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults=gt_difficults, iou_thresh=self.iou_thresh) np.testing.assert_equal(results[0]['recall'], self.rec) np.testing.assert_equal(results[0]['precision'], self.prec)
def calc(self): pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels = self.data if self.metric == 'all' or self.metric == 'voc_detection': print('Calculating voc_detection ...') result = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, use_07_metric=True) print('mAP: {:f}'.format(result['map'])) print('person mAP: {:f}'.format( result['ap'][voc_utils.voc_bbox_label_names.index('person')])) if self.metric == 'all' or self.metric == 'pr_voc_detection': print('Calculating pr_voc_detection ...') prec, rec = calc_detection_voc_prec_rec(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults=None, iou_thresh=0.5) person_prec = prec[voc_utils.voc_bbox_label_names.index('person')] person_rec = rec[voc_utils.voc_bbox_label_names.index('person')] print('Avg person precision: {:f}'.format(np.average(person_prec))) print('Avg person recall: {:f}'.format(np.average(person_rec))) if self.plottings: self.plot(recall=person_rec, precision=person_prec, metric='pr_voc_detection') if self.metric == 'all' or self.metric == 'mot': print('Calculating mot_metrics ...') self.motmetrics()
def eval(Res,m=4952,F=True): pred_bboxes=[] pred_labels=[] pred_scores=[] gt_bboxes=[] gt_labels=[] names=GT.keys() names=sorted(names) for name in names[:m]: res = Res[name] res=res[res[:,4]>1e-10] p_bboxes=res[:,:4] p_labels=res[:,5] p_labels=p_labels.astype(np.int32) p_scores=res[:,4] pred_bboxes.append(p_bboxes) pred_labels.append(p_labels) pred_scores.append(p_scores) gt=GT[name] gt=gt[gt[:,-1]==0] g_bboxes = gt[:,:4] g_labels=gt[:,4]-1 g_labels=g_labels.astype(np.int32) gt_bboxes.append(g_bboxes) gt_labels.append(g_labels) result=eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels,use_07_metric=F) print(result)
def evaluate(self, predictions, targets, metric_list=['ap', 'map'], eval_cfg=None): result = eval_detection_voc( pred_bboxes=predictions['pred_bboxes'], pred_labels=predictions['pred_labels'], pred_scores=predictions['pred_scores'], gt_bboxes=targets['gt_bboxes'], gt_labels=targets['gt_labels'], iou_thresh=eval_cfg['iou_thresh'], use_07_metric=eval_cfg['use_07_metric'], ) result_selected = {} for metric in metric_list: result_selected[metric] = result[metric] if 'ap' in result_selected: ap_list = result_selected['ap'] ap_dict = {} for idx, item in enumerate(ap_list): ap_dict[self.classnames[idx]] = item result_selected['ap'] = ap_dict return result_selected
def evaluate(self): iterator = self._iterators['main'] target = self._targets['main'] if hasattr(target, 'prediction_detection'): target.prediction_detection = self.detection if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) in_values, out_values, rest_values = apply_to_iterator( target.predict, it) del in_values if self.detection: # TODO: evaluate the detection result pred_bboxes, pred_labels, pred_scores = out_values if len(rest_values) == 3: gt_bboxes, gt_labels, gt_difficults = rest_values elif len(rest_values) == 2: gt_bboxes, gt_labels = rest_values gt_difficults = None result = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=self.use_07_metric) report = {'map': result['map']} if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['ap/{:s}'.format(label_name)] = result['ap'][l] except IndexError: report['ap/{:s}'.format(label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation else: # TODO: evaluate the segmentation result pred_labels, = out_values gt_labels, = rest_values result = eval_semantic_segmentation(pred_labels, gt_labels) report = { 'miou': result['miou'], 'pixel_accuracy': result['pixel_accuracy'], 'mean_class_accuracy': result['mean_class_accuracy'] } if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['iou/{:s}'.format( label_name)] = result['iou'][l] report['class_accuracy/{:s}'.format(label_name)] = \ result['class_accuracy'][l] except IndexError: report['iou/{:s}'.format(label_name)] = np.nan report['class_accuracy/{:s}'.format( label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def main(): groundtruthpath = ARGS['groundtruthpath'] predictpath = ARGS['predictpath'] imagespath = ARGS['imagespath'] if (groundtruthpath == '' or predictpath == ''): raise Exception('missing pti database paths') #default yolo labeling format. groundtruths = glob.glob(os.path.join(groundtruthpath, '**/*.txt'), recursive=True) #darkflow prediction output. predictions = glob.glob(os.path.join(predictpath, '**/*.json'), recursive=True) gt_size = len(groundtruths) pred_size = len(predictions) print("{} prediction files | {} groundtruth files".format( pred_size, gt_size)) pred_bboxes = [] pred_labels = [] pred_scores = [] gt_bboxes = [] gt_labels = [] images = [] for pred_file_path in predictions: # print(pred_file_path) #remove heading base_pred_file_path = pred_file_path.replace(predictpath, '') # print(base_pred_file_path) gt_file_path = os.path.join(groundtruthpath, base_pred_file_path).replace( '.json', '.txt') if not os.path.exists(gt_file_path): print('GT file does not exist. Tried to open [{}]'.format( gt_file_path)) sys.exit() with open(gt_file_path) as gt_file: bboxes = [] labels = [] for index, line in enumerate(gt_file): if index == 0: continue gt = list(map(int, line.split())) #[min_x, min_y, max_x, max_y] #must be ('ymin', 'xmin', 'ymax', 'xmax') bboxes.append([gt[1], gt[0], gt[3], gt[2]]) labels.append(voc_utils.voc_bbox_label_names.index('person')) # print('bboxes',bboxes) if len(bboxes) > 0: bboxes = np.stack(bboxes).astype(np.float32) labels = np.stack(labels).astype(np.int32) else: bboxes = np.ndarray(shape=(0), dtype=np.float32) labels = np.ndarray(shape=(0), dtype=np.int32) if imagespath: images.append( os.path.join(imagespath, base_pred_file_path).replace('.json', '.jpg')) gt_bboxes.append(bboxes) gt_labels.append(labels) with open(pred_file_path) as pred_file: j_data = json.load(pred_file) # print(j_data) bboxes = [] labels = [] scores = [] for predicted_obj in j_data: #must be ('ymin', 'xmin', 'ymax', 'xmax') if predicted_obj['label'] != 'person': continue bboxes.append([ predicted_obj['topleft']['y'], predicted_obj['topleft']['x'], predicted_obj['bottomright']['y'], predicted_obj['bottomright']['x'] ]) labels.append(voc_utils.voc_bbox_label_names.index('person')) scores.append(predicted_obj['confidence']) if len(bboxes) > 0: bboxes = np.stack(bboxes).astype(np.float32) labels = np.stack(labels).astype(np.int32) scores = np.stack(scores).astype(np.float32) else: bboxes = np.ndarray(shape=(0), dtype=np.float32) labels = np.ndarray(shape=(0), dtype=np.int32) scores = np.ndarray(shape=(0), dtype=np.float32) pred_bboxes.append(bboxes) pred_labels.append(labels) pred_scores.append(scores) # pred_bboxes = np.stack(pred_bboxes).astype(np.float32) # pred_labels = np.stack(pred_labels).astype(np.int32) # pred_scores = np.stack(pred_scores).astype(np.float32) pred_bboxes = np.asarray(pred_bboxes) pred_labels = np.asarray(pred_labels) pred_scores = np.asarray(pred_scores) gt_bboxes = np.asarray(gt_bboxes) gt_labels = np.asarray(gt_labels) result = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, use_07_metric=True) if imagespath: print('Gerenrating plots...') for x in tqdm(range(100), total=100): target_image = random.randint(0, len(gt_bboxes) - 1) curr_img = images[target_image] im = np.array(Image.open(curr_img), dtype=np.uint8) fig, ax = plt.subplots(1) ax.imshow(im) for b in gt_bboxes[target_image]: rect = buildRect(b, 'g') ax.add_patch(rect) for b in pred_bboxes[target_image]: rect = buildRect(b, 'r') ax.add_patch(rect) plt.axis('off') fig_dir = 'plottings' os.makedirs(fig_dir, exist_ok=True) fig_path = os.path.join( fig_dir, curr_img.replace(imagespath, '').replace('/', '')) plt.savefig(fig_path, bbox_inches='tight') plt.show() plt.close() print('mAP: {:f}'.format(result['map'])) print('person: {:f}'.format( result['ap'][voc_utils.voc_bbox_label_names.index('person')]))
def evaluate(model, testdl, log, stop=100): total_pred_boxes = [] total_pred_labels = [] total_pred_scores = [] total_true_boxes = [] total_true_labels = [] scales = len(cfg.DATA.SCALES) n_i = 2 * scales k = cfg.K stop_flag = 0 with torch.no_grad(): model.eval() for n_imgs, gt, n_regions, region in tqdm(testdl, "Evaluation"): stop_flag += 1 if stop_flag > stop: break region = region.to(cfg.DEVICE) avg_scores = torch.zeros((len(region[0]), 20), dtype=torch.float32) for i in range(n_i): per_img = n_imgs[i].to(cfg.DEVICE) per_region = n_regions[i].to(cfg.DEVICE) ref_scores1, ref_scores2, ref_scores3, proposal_scores = model( per_img, per_region) avg_scores += (ref_scores1 + ref_scores2 + ref_scores3)[:, :-1].detach().cpu() / k avg_scores /= n_i gt = gt.numpy()[0] gt_boxex = gt[:, :4] gt_labels = gt[:, -1] per_pred_boxes = [] per_pred_scores = [] per_pred_labels = [] region = region[0].cpu() for i in range(20): cls_scores = avg_scores[:, i] cls_region = region nms_filter = nms(cls_region, cls_scores, 0.3) per_pred_boxes.append(cls_region[nms_filter].numpy()) per_pred_scores.append(cls_scores[nms_filter].numpy()) per_pred_labels.append( np.full(len(nms_filter), i, dtype=np.int32)) total_pred_boxes.append(np.concatenate(per_pred_boxes, axis=0)) total_pred_scores.append(np.concatenate(per_pred_scores, axis=0)) total_pred_labels.append(np.concatenate(per_pred_labels, axis=0)) total_true_boxes.append(gt_boxex) total_true_labels.append(gt_labels) result = eval_detection_voc( total_pred_boxes, total_pred_labels, total_pred_scores, total_true_boxes, total_true_labels, iou_thresh=0.5, use_07_metric=True, ) print(f"Avg AP: {result['ap']}") print(f"Avg mAP: {result['map']}") write_log(log, f"Avg AP: {result['ap']}") write_log(log, f"Avg mAP: {result['map']}") return result['map']
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', choices=('faster_rcnn', 'ssd300', 'ssd512', 'yolo_v3'), default='ssd300') parser.add_argument('--pretrained_model') parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--batchsize', type=int, default=32) args = parser.parse_args() if args.model == 'faster_rcnn': if args.pretrained_model: model = FasterRCNNVGG16(n_fg_class=len(voc_bbox_label_names), pretrained_model=args.pretrained_model) else: model = FasterRCNNVGG16(pretrained_model='voc07') elif args.model == 'ssd300': if args.pretrained_model: model = SSD300(n_fg_class=len(voc_bbox_label_names), pretrained_model=args.pretrained_model) else: model = SSD300(pretrained_model='voc0712') elif args.model == 'ssd512': if args.pretrained_model: model = SSD512(n_fg_class=len(voc_bbox_label_names), pretrained_model=args.pretrained_model) else: model = SSD512(pretrained_model='voc0712') elif args.model == 'yolo_v3': if args.pretrained_model: model = YOLOv3(n_fg_class=len(voc_bbox_label_names), pretrained_model=args.pretrained_model) else: model = YOLOv3(pretrained_model='voc0712') if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() model.use_preset('evaluate') dataset = VOCBboxDataset(year='2007', split='test', use_difficult=True, return_difficult=True) iterator = iterators.SerialIterator(dataset, args.batchsize, repeat=False, shuffle=False) in_values, out_values, rest_values = apply_to_iterator(model.predict, iterator, hook=ProgressHook( len(dataset))) # delete unused iterators explicitly del in_values pred_bboxes, pred_labels, pred_scores = out_values gt_bboxes, gt_labels, gt_difficults = rest_values result = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=True) print() print('mAP: {:f}'.format(result['map'])) for l, name in enumerate(voc_bbox_label_names): if result['ap'][l]: print('{:s}: {:f}'.format(name, result['ap'][l])) else: print('{:s}: -'.format(name))
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--model', choices=('faster_rcnn', 'ssd300', 'ssd512'), default='ssd300') parser.add_argument('--pretrained_model') parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--batchsize', type=int, default=32) args = parser.parse_args() if args.model == 'faster_rcnn': if args.pretrained_model: model = FasterRCNNVGG16( n_fg_class=20, pretrained_model=args.pretrained_model) else: model = FasterRCNNVGG16(pretrained_model='voc07') elif args.model == 'ssd300': if args.pretrained_model: model = SSD300( n_fg_class=20, pretrained_model=args.pretrained_model) else: model = SSD300(pretrained_model='voc0712') elif args.model == 'ssd512': if args.pretrained_model: model = SSD512( n_fg_class=20, pretrained_model=args.pretrained_model) else: model = SSD512(pretrained_model='voc0712') if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() model.use_preset('evaluate') dataset = VOCBboxDataset( year='2007', split='test', use_difficult=True, return_difficult=True) iterator = iterators.SerialIterator( dataset, args.batchsize, repeat=False, shuffle=False) in_values, out_values, rest_values = apply_to_iterator( model.predict, iterator, hook=ProgressHook(len(dataset))) # delete unused iterators explicitly del in_values pred_bboxes, pred_labels, pred_scores = out_values gt_bboxes, gt_labels, gt_difficults = rest_values result = eval_detection_voc( pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=True) print() print('mAP: {:f}'.format(result['map'])) for l, name in enumerate(voc_bbox_label_names): if result['ap'][l]: print('{:s}: {:f}'.format(name, result['ap'][l])) else: print('{:s}: -'.format(name))
gt_labels[i].append( voc_detection_label_names.index(anno_elem['name'])) gt_difficults[i].append(anno_elem['difficult']) gt_bboxes[i] = np.array(gt_bboxes[i]) gt_labels[i] = np.array(gt_labels[i]) gt_difficults[i] = np.array(gt_difficults[i], dtype=np.bool) return gt_bboxes, gt_labels, gt_difficults, ids if __name__ == '__main__': base_dir = 'Main' anno_fn = 'annotations_cache/annots.pkl' gt_bboxes, gt_labels, gt_difficults, ids = read_gt_annos(anno_fn) ids, bboxes, labels, confs = read_pascal_predictions(base_dir, ids) metric = eval_detection_voc(bboxes, labels, confs, gt_bboxes, gt_labels, gt_difficults, use_07_metric=True) labels = voc_detection_label_names[1:] for i, label in enumerate(labels): if i + 1 in metric: print(label, metric[i + 1]['ap']) print('>>>>>>>>>>>>>>>>>>>>>>>>>>>') print metric['map']
nms_filter = nms(cls_region, cls_scores, 0.4) per_pred_boxes.append(cls_region[nms_filter].numpy()) per_pred_scores.append(cls_scores[nms_filter].numpy()) per_pred_labels.append(np.full(len(nms_filter), i, dtype=np.int32)) total_pred_boxes.append(np.concatenate(per_pred_boxes, axis=0)) total_pred_scores.append(np.concatenate(per_pred_scores, axis=0)) total_pred_labels.append(np.concatenate(per_pred_labels, axis=0)) total_true_boxes.append(gt_boxex) total_true_labels.append(gt_labels) result = eval_detection_voc( total_pred_boxes, total_pred_labels, total_pred_scores, total_true_boxes, total_true_labels, iou_thresh=0.5, use_07_metric=True, ) write_log(log_file, f"Avg AP: {result['ap']}") write_log(log_file, f"Avg mAP: {result['map']}") cls_ap = [] y_pred = np.array(y_pred) y_true = np.array(y_true) for i in range(20): cls_ap.append(average_precision_score(y_true[:,i], y_pred[:,i])) write_log(log_file, f"Testset classify AP is {str(cls_ap)}") write_log(log_file, f"Testset classify mAP is {str(sum(cls_ap)/20)}") write_log(log_file, "-" * 30)
def evaluate(net, dataloader): """Evaluates network.""" with torch.no_grad(): net.eval() total_pred_boxes = [] total_pred_scores = [] total_pred_labels = [] total_gt_boxes = [] total_gt_labels = [] for ( img_id, img, # is it necessary boxes, scaled_imgs, scaled_boxes, scores, gt_boxes, gt_labels, ) in tqdm(dataloader, "Evaluation"): combined_scores = torch.zeros(len(boxes), 20, dtype=torch.float32) batch_scores = np2gpu(scores.numpy(), DEVICE) for i, scaled_img in enumerate(scaled_imgs): scaled_img = scaled_img.numpy() tmp_scaled_boxes = scaled_boxes[i].numpy() batch_imgs = np2gpu(scaled_img, DEVICE) batch_boxes = np2gpu(tmp_scaled_boxes, DEVICE) tmp_combined_scores = net(batch_imgs, batch_boxes, batch_scores) combined_scores += tmp_combined_scores.cpu() combined_scores /= 10 gt_boxes = gt_boxes.numpy() gt_labels = gt_labels.numpy() batch_gt_boxes = np2gpu(gt_boxes, DEVICE) batch_gt_labels = np2gpu(gt_labels, DEVICE) batch_pred_boxes = [] batch_pred_scores = [] batch_pred_labels = [] for i in range(20): region_scores = combined_scores[:, i] score_mask = region_scores > 0 selected_scores = region_scores[score_mask] selected_boxes = boxes[score_mask] nms_mask = nms(selected_boxes, selected_scores, 0.4) batch_pred_boxes.append(selected_boxes[nms_mask].cpu().numpy()) batch_pred_scores.append( selected_scores[nms_mask].cpu().numpy()) batch_pred_labels.append( np.full(len(nms_mask), i, dtype=np.int32)) total_pred_boxes.append(np.concatenate(batch_pred_boxes, axis=0)) total_pred_scores.append(np.concatenate(batch_pred_scores, axis=0)) total_pred_labels.append(np.concatenate(batch_pred_labels, axis=0)) total_gt_boxes.append(batch_gt_boxes[0].cpu().numpy()) total_gt_labels.append(batch_gt_labels[0].cpu().numpy()) result = eval_detection_voc( total_pred_boxes, total_pred_labels, total_pred_scores, total_gt_boxes, total_gt_labels, iou_thresh=0.5, use_07_metric=True, ) tqdm.write(f"Avg AP: {result['ap']}") tqdm.write(f"Avg mAP: {result['map']}") net.train()
def evaluate(net, dataloader, numADL, epoch): """Evaluates network.""" with torch.no_grad(): net.eval() total_pred_boxes = [] total_pred_scores = [] total_pred_labels = [] total_gt_boxes = [] total_gt_labels = [] count = 0 for ( img_id, img, # is it necessary boxes, scaled_imgs, scaled_boxes, scores, gt_boxes, gt_labels, ) in tqdm(dataloader, "Evaluation"): count += 1 # pdb.set_trace() # img_id = dataloader.dataset[336][0] # img = dataloader.dataset[336][1] # boxes = dataloader.dataset[336][2] # scaled_imgs = dataloader.dataset[336][3] # scaled_boxes = dataloader.dataset[336][4] # scores = dataloader.dataset[336][5] # gt_boxes = dataloader.dataset[336][0] # gt_labels = dataloader.dataset[336][0] combined_scores = torch.zeros(len(boxes), 20, dtype=torch.float32) batch_scores = np2gpu(scores.numpy(), DEVICE) # total_pred_boxes = [] # total_pred_scores = [] # total_pred_labels = [] # total_gt_boxes = [] # total_gt_labels = [] # if count >= 336: # print('stop') # pdb.set_trace() for i, scaled_img in enumerate(scaled_imgs): # print(i) # scaled_img = scaled_img.numpy() # tmp_scaled_boxes = scaled_boxes[i].numpy() # batch_imgs = np2gpu(scaled_img, DEVICE) # batch_boxes = np2gpu(tmp_scaled_boxes, DEVICE) batch_imgs = scaled_img.unsqueeze(0).cuda() batch_boxes = scaled_boxes[i].unsqueeze(0).cuda() # tmp_combined_scores = net(batch_imgs, batch_boxes, batch_scores, gt_labels) tmp_combined_scores = net.test(batch_imgs, batch_boxes, batch_scores, gt_labels) # tmp_combined_scores, _ = net(batch_imgs, batch_boxes, batch_scores, gt_labels, numADL, epoch) # tmp_combined_scores, _ = net.test(batch_imgs, batch_boxes, batch_scores, gt_labels, numADL, epoch) # tmp_combined_scores, _ = net.test_withD(batch_imgs, batch_boxes, batch_scores, gt_labels, numADL, epoch) # _, _, tmp_combined_scores = net(batch_imgs, batch_boxes, batch_scores, gt_labels.unsqueeze(0)) # pdb.set_trace() combined_scores += tmp_combined_scores.cpu() combined_scores /= 10 # gt_boxes = gt_boxes.numpy() # gt_labels = gt_labels.numpy() # batch_gt_boxes = np2gpu(gt_boxes, DEVICE) # batch_gt_labels = np2gpu(gt_labels, DEVICE) # pdb.set_trace() batch_gt_labels = gt_labels.unsqueeze(0).cuda() batch_gt_boxes = gt_boxes.unsqueeze(0).cuda() batch_pred_boxes = [] batch_pred_scores = [] batch_pred_labels = [] for i in range(20): # if i == 14 region_scores = combined_scores[:, i] score_mask = region_scores > 0 selected_scores = region_scores[score_mask] selected_boxes = boxes[score_mask] nms_mask = nms(selected_boxes, selected_scores, 0.4) # pdb.set_trace() batch_pred_boxes.append(selected_boxes[nms_mask].cpu().numpy()) batch_pred_scores.append( selected_scores[nms_mask].cpu().numpy()) batch_pred_labels.append( np.full(len(nms_mask), i, dtype=np.int32)) total_pred_boxes.append(np.concatenate(batch_pred_boxes, axis=0)) total_pred_scores.append(np.concatenate(batch_pred_scores, axis=0)) total_pred_labels.append(np.concatenate(batch_pred_labels, axis=0)) total_gt_boxes.append(batch_gt_boxes[0].cpu().numpy()) total_gt_labels.append(batch_gt_labels[0].cpu().numpy()) # pdb.set_trace() result = eval_detection_voc(total_pred_boxes, total_pred_labels, total_pred_scores, total_gt_boxes, total_gt_labels, iou_thresh=0.5, use_07_metric=True) # print(result['map']) # print(result['ap']) tqdm.write(f"Avg AP: {result['ap']}") tqdm.write(f"Avg mAP: {result['map']}") net.train()