def test_enet(): """Demo ENet.""" config, img_path = parse_args() test_data = load_dataset_test(config["dataset"]) test_iter = create_iterator_test(test_data, config['iterator']) model = get_model(config["model"]) devices = parse_devices(config['gpus']) if devices: model.to_gpu(devices['main']) imgs, pred_values, gt_values = apply_prediction_to_iterator( model.predict, test_iter) del imgs pred_labels, = pred_values gt_labels, = gt_values result = eval_semantic_segmentation(pred_labels, gt_labels) for iu, label_name in zip(result['iou'], cityscapes_label_names): print('{:>23} : {:.4f}'.format(label_name, iu)) print('=' * 34) print('{:>23} : {:.4f}'.format('mean IoU', result['miou'])) print('{:>23} : {:.4f}'.format('Class average accuracy', result['mean_class_accuracy'])) print('{:>23} : {:.4f}'.format('Global average accuracy', result['pixel_accuracy']))
def evaluate(self, predictions, groundtruths): result = eval_semantic_segmentation(predictions, groundtruths) result_str = 'IoUs: \n' for idx, item in enumerate(result['iou']): result_str += '%s %s\n' % (self.classnames[idx], item) result_str += 'MIoU: %s' % result['miou'] return result_str
def evaluate(self, predictions, groundtruths, metric_list=['iou', 'miou'], num_classes=None, ignore_index=-1, **kwargs): eval_client = None result = eval_semantic_segmentation(predictions, groundtruths) result_selected = {} for metric in metric_list: if metric in result: result_selected[metric] = result[metric] else: if eval_client is None: eval_client = Evaluation(predictions, groundtruths, num_classes, ignore_index, **kwargs) assert metric in eval_client.all_metric_results, 'unsupport %s as the metric...' % metric result_selected[metric] = eval_client.all_metric_results[ metric] if 'iou' in result_selected: iou_list = result_selected['iou'] iou_dict = {} for idx, item in enumerate(iou_list): iou_dict[self.classnames[idx]] = item result_selected['iou'] = iou_dict if 'dice' in result_selected: dice_list = result_selected['dice'] dice_dict = {} for idx, item in enumerate(dice_list): dice_dict[self.classnames[idx]] = item result_selected['dice'] = dice_dict return result_selected
def __call__(self, x, t): """Computes the loss value for an image and label pair. It also computes accuracy and stores it to the attribute. Args: x (~chainer.Variable): A variable with a batch of images. t (~chainer.Variable): A variable with the ground truth image-wise label. Returns: ~chainer.Variable: Loss value. """ self.y = self.predictor(x) self.loss = F.softmax_cross_entropy(self.y, t, class_weight=self.class_weight, ignore_label=self.ignore_label) reporter.report({'loss': self.loss}, self) self.accuracy = None if self.compute_accuracy: label = self.xp.argmax(self.y.data, axis=1) self.accuracy = eval_semantic_segmentation(label, t.data, self.n_class) reporter.report( { 'pixel_accuracy': self.xp.mean(self.accuracy[0]), 'mean_accuracy': self.xp.mean(self.accuracy[1]), 'mean_iou': self.xp.mean(self.accuracy[2]), 'fw_iou': self.xp.mean(self.accuracy[3]) }, self) return self.loss
def __call__(self, x, t, dataset, train=True): # Create variables x = Variable(x) x.to_gpu(self.gpu_id) t = Variable(t) t.to_gpu(self.gpu_id) # Config mode if len(t.shape) == 3: config_mode = 'segmentation' elif len(t.shape) == 2: config_mode = 'recognition' else: raise ValueError('label format is not supported') # Forward with chainer.using_config('train', train): with chainer.using_config('enable_backprop', train): # InceptionV3 backbone x = self.predictor(x) # Classifiers classifier_indx = self.args.dataset.split('+').index(dataset) y = self.classifiers[classifier_indx](x, train) # Loss if config_mode == 'segmentation': self.y = F.resize_images(y, t.shape[-2:]) # Upsampling logits self.loss = F.softmax_cross_entropy(self.y, t) elif config_mode == 'recognition': self.y = F.squeeze(F.average_pooling_2d( y, ksize=y.shape[-2:]), axis=(2, 3)) # Global Average Pooling self.loss = F.sigmoid_cross_entropy(self.y, t) # Backward if train: # Clear grads for uninitialized params self.cleargrads() # Backwards self.loss.backward() # Reporter if config_mode == 'segmentation': self.y = F.argmax(self.y, axis=1) self.y.to_cpu() t.to_cpu() result = eval_semantic_segmentation(list(self.y.data), list(t.data)) del result['iou'], result['class_accuracy'] result.update({'loss': self.loss.data.tolist()}) self.reporter.update({dataset: result}) elif config_mode == 'recognition': self.reporter.update({ dataset: { 'loss': self.loss.data.tolist(), 'prediction': F.sigmoid(self.y).data.tolist(), 'groundtruth': t.data.tolist() } })
def test_eval_semantic_segmentation(self): result = eval_semantic_segmentation(self.pred_labels, self.gt_labels) np.testing.assert_equal(result['iou'], self.iou) np.testing.assert_equal(result['pixel_accuracy'], self.pixel_accuracy) np.testing.assert_equal(result['class_accuracy'], self.class_accuracy) np.testing.assert_equal(result['miou'], np.nanmean(self.iou)) np.testing.assert_equal(result['mean_class_accuracy'], np.nanmean(self.class_accuracy))
def test_eval_semantic_segmentation(self): result = eval_semantic_segmentation( self.pred_labels, self.gt_labels) np.testing.assert_equal(result['iou'], self.iou) np.testing.assert_equal(result['pixel_accuracy'], self.pixel_accuracy) np.testing.assert_equal(result['class_accuracy'], self.class_accuracy) np.testing.assert_equal(result['miou'], np.nanmean(self.iou)) np.testing.assert_equal( result['mean_class_accuracy'], np.nanmean(self.class_accuracy))
def evaluate(self): iterator = self._iterators['main'] eval_func = self.eval_func or self._targets['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) summary = reporter_module.DictSummary() pred_labels = [] gt_labels = [] for batch in it: observation = {} with reporter_module.report_scope(observation): in_arrays = self.converter(batch, self.device) with function.no_backprop_mode(): if isinstance(in_arrays, tuple): eval_func(*in_arrays) elif isinstance(in_arrays, dict): eval_func(**in_arrays) else: eval_func(in_arrays) if eval_func.predictions is not None: pred_labels.extend(cuda.to_cpu(eval_func.predictions)) gt_labels.extend(cuda.to_cpu(eval_func.gt)) summary.add(observation) observation = summary.compute_mean() if self.label_names is not None and len(pred_labels) > 0: pred_labels = np.array(pred_labels) gt_labels = np.array(gt_labels) result = eval_semantic_segmentation(pred_labels, gt_labels) report = { 'miou': result['miou'], 'pixel_acc': result['pixel_accuracy'], 'mean_class_acc': result['mean_class_accuracy'] } for l, label_name in enumerate(self.label_names): try: report['iou/{:s}'.format(label_name)] = result['iou'][l] report['class_acc/{:s}'.format( label_name)] = result['class_accuracy'][l] except IndexError: report['iou/{:s}'.format(label_name)] = np.nan report['class_acc/{:s}'.format(label_name)] = np.nan with reporter_module.report_scope(observation): reporter_module.report(report, eval_func) return observation
def main(): parser = argparse.ArgumentParser() parser.add_argument('--dataset', choices=('cityscapes', 'ade20k', 'camvid', 'voc')) parser.add_argument('--model', choices=('pspnet_resnet101', 'segnet', 'deeplab_v3plus_xception65')) parser.add_argument('--pretrained-model') parser.add_argument('--input-size', type=int, default=None) args = parser.parse_args() comm = chainermn.create_communicator('pure_nccl') device = comm.intra_rank if args.input_size is None: input_size = None else: input_size = (args.input_size, args.input_size) dataset, label_names, model = get_dataset_and_model( args.dataset, args.model, args.pretrained_model, input_size) chainer.cuda.get_device_from_id(device).use() model.to_gpu() if not comm.rank == 0: apply_to_iterator(model.predict, None, comm=comm) return it = iterators.MultithreadIterator(dataset, comm.size, repeat=False, shuffle=False) in_values, out_values, rest_values = apply_to_iterator(model.predict, it, hook=ProgressHook( len(dataset)), comm=comm) # Delete an iterator of images to save memory usage. del in_values pred_labels, = out_values gt_labels, = rest_values result = eval_semantic_segmentation(pred_labels, gt_labels) for iu, label_name in zip(result['iou'], label_names): print('{:>23} : {:.4f}'.format(label_name, iu)) print('=' * 34) print('{:>23} : {:.4f}'.format('mean IoU', result['miou'])) print('{:>23} : {:.4f}'.format('Class average accuracy', result['mean_class_accuracy'])) print('{:>23} : {:.4f}'.format('Global average accuracy', result['pixel_accuracy']))
def evaluate(self, predictions, groundtruths, metric_list=['iou', 'miou']): result = eval_semantic_segmentation(predictions, groundtruths) result_selected = {} for metric in metric_list: result_selected[metric] = result[metric] if 'iou' in result_selected: iou_list = result_selected['iou'] iou_dict = {} for idx, item in enumerate(iou_list): iou_dict[self.classnames[idx]] = item result_selected['iou'] = iou_dict return result_selected
def eval_(out_values, rest_values): pred_labels, = out_values gt_labels, = rest_values result = eval_semantic_segmentation(pred_labels, gt_labels) for iu, label_name in zip(result['iou'], label_names): print('{:>23} : {:.4f}'.format(label_name, iu)) print('=' * 34) print('{:>23} : {:.4f}'.format('mean IoU', result['miou'])) print('{:>23} : {:.4f}'.format('Class average accuracy', result['mean_class_accuracy'])) print('{:>23} : {:.4f}'.format('Global average accuracy', result['pixel_accuracy']))
def evaluate(self): iterator = self._iterators['main'] target = self._targets['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) in_values, out_values, rest_values = apply_to_iterator( target.predict, it) # delete unused iterators explicitly del in_values pred_labels, pred_labels_occ = out_values gt_labels, gt_labels_occ = rest_values result_vis = eval_semantic_segmentation(pred_labels, gt_labels) result_occ = eval_occlusion_segmentation(pred_labels_occ, gt_labels_occ) report = { 'miou': (result_vis['miou'] + result_occ['miou']) / 2., 'miou/vis': result_vis['miou'], 'miou/occ': result_occ['miou'], } # if self.label_names is not None: # for l, label_name in enumerate(self.label_names): # try: # report['iou/{:s}'.format(label_name)] = result['iou'][l] # except IndexError: # report['iou/{:s}'.format(label_name)] = np.nan # # if l == 0: # continue # # try: # report['iou_occ/{:s}'.format(label_name)] = \ # result_occ['iou'][l - 1] # except IndexError: # result_occ['iou_occ/{:s}'.format(label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def main(): parser = argparse.ArgumentParser() parser.add_argument( '--model', choices=('pspnet_resnet101',), default='pspnet_resnet101') parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--pretrained-model') args = parser.parse_args() if args.model == 'pspnet_resnet101': if args.pretrained_model: model = PSPNetResNet101( n_class=len(cityscapes_semantic_segmentation_label_names), pretrained_model=args.pretrained_model, input_size=(713, 713) ) else: model = PSPNetResNet101(pretrained_model='cityscapes') if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() dataset = CityscapesSemanticSegmentationDataset( split='val', label_resolution='fine') it = iterators.SerialIterator( dataset, 1, repeat=False, shuffle=False) in_values, out_values, rest_values = apply_to_iterator( model.predict, it, hook=ProgressHook(len(dataset))) # Delete an iterator of images to save memory usage. del in_values pred_labels, = out_values gt_labels, = rest_values result = eval_semantic_segmentation(pred_labels, gt_labels) for iu, label_name in zip( result['iou'], cityscapes_semantic_segmentation_label_names): print('{:>23} : {:.4f}'.format(label_name, iu)) print('=' * 34) print('{:>23} : {:.4f}'.format('mean IoU', result['miou'])) print('{:>23} : {:.4f}'.format( 'Class average accuracy', result['mean_class_accuracy'])) print('{:>23} : {:.4f}'.format( 'Global average accuracy', result['pixel_accuracy']))
def evaluate(self): target = self._targets['main'] if self.comm is not None and self.comm.rank != 0: apply_to_iterator(target.predict, None, comm=self.comm) return {} iterator = self._iterators['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) in_values, out_values, rest_values = apply_to_iterator(target.predict, it, comm=self.comm) # delete unused iterators explicitly del in_values pred_labels, = out_values gt_labels, = rest_values result = eval_semantic_segmentation(pred_labels, gt_labels) report = { 'miou': result['miou'], 'pixel_accuracy': result['pixel_accuracy'], 'mean_class_accuracy': result['mean_class_accuracy'] } if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['iou/{:s}'.format(label_name)] = result['iou'][l] report['class_accuracy/{:s}'.format(label_name)] =\ result['class_accuracy'][l] except IndexError: report['iou/{:s}'.format(label_name)] = np.nan report['class_accuracy/{:s}'.format(label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def check_eval_semantic_segmentation(self, pred_labels, gt_labels, acc, acc_cls, mean_iou, fwavacc, n_class): with warnings.catch_warnings(record=True) as w: acc_o, acc_cls_o, mean_iou_o, fwavacc_o =\ eval_semantic_segmentation( pred_labels, gt_labels, n_class=n_class) self.assertIsInstance(acc_o, type(acc)) self.assertIsInstance(acc_cls_o, type(acc_cls)) self.assertIsInstance(mean_iou_o, type(mean_iou)) self.assertIsInstance(fwavacc_o, type(fwavacc)) np.testing.assert_equal(cuda.to_cpu(acc_o), cuda.to_cpu(acc)) np.testing.assert_equal(cuda.to_cpu(acc_cls_o), cuda.to_cpu(acc_cls)) np.testing.assert_equal(cuda.to_cpu(mean_iou_o), cuda.to_cpu(mean_iou)) np.testing.assert_equal(cuda.to_cpu(fwavacc_o), cuda.to_cpu(fwavacc)) # test that no warning has been created self.assertEqual(len(w), 0)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--gpu", type=int, default=-1) parser.add_argument("--pretrained_model", type=str, default="camvid") parser.add_argument("-batchsize", type=int, default=24) args = parser.parse_args() model = SegNetBasic(n_class=len(camvid_label_names), pretrained_model=args.pretrained_model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() model = calc_bn_statistics(model, args.batchsize) test = CamVidDataset(split="test") it = chainer.iterators.SerialIterator(test, batch_size=args.batchsize, repeat=False, shuffle=False) imgs, pred_values, gt_values = apply_prediction_to_iterator( model.predict, it) # Delete an iterator of iamges to save memory usage. del imgs pred_labels, = pred_values gt_labels, = gt_values result = eval_semantic_segmentation(pred_labels, gt_labels) for iu, label_name in zip(result["iou"], camvid_label_names): print("{:>23} : {:.4f}".format(label_name, iu)) print("=" * 34) print("{:>23} : {:.4f}".format("mean IoU", result["miou"])) print("{:>23} : {:.4f}".format("Class average accuracy", result["mean_calss_accuracy"])) print("{:>23} : {:.4f}".format("Global average accuracy", result["pixe;_accuracy"]))
def _evaluate_one(self, target, iterator): if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) if self._show_progress: it = tqdm.tqdm(it, total=len(it.dataset), leave=False) in_values, out_values, rest_values = apply_to_iterator( target.predict, it) imgs, = in_values pred_bboxes, pred_masks, pred_labels, pred_scores, \ pred_lbls_vis, pred_lbls_occ = out_values gt_bboxes, gt_labels, gt_masks, gt_lbls_vis, gt_lbls_occ = rest_values # evaluate result = eval_instseg_voc( pred_masks, pred_labels, pred_scores, gt_masks, gt_labels, None, use_07_metric=self.use_07_metric, ) result_lbl_vis = eval_semantic_segmentation(pred_lbls_vis, gt_lbls_vis) result['miou/vis'] = result_lbl_vis['miou'] result_lbl_occ = eval_occlusion_segmentation(pred_lbls_occ, gt_lbls_occ) result['miou/occ'] = result_lbl_occ['miou'] result['miou'] = (result['miou/vis'] + result['miou/occ']) / 2. return result
def main(): parser = argparse.ArgumentParser() parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--pretrained_model', type=str, default='camvid') parser.add_argument('--batchsize', type=int, default=24) args = parser.parse_args() model = SegNetBasic(n_class=len(camvid_label_names), pretrained_model=args.pretrained_model) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() model = calc_bn_statistics(model, args.batchsize) chainer.config.train = False test = CamVidDataset(split='test') it = chainer.iterators.SerialIterator(test, batch_size=args.batchsize, repeat=False, shuffle=False) imgs, pred_values, gt_values = apply_prediction_to_iterator( model.predict, it) # Delete an iterator of images to save memory usage. del imgs pred_labels, = pred_values gt_labels, = gt_values result = eval_semantic_segmentation(pred_labels, gt_labels) for iu, label_name in zip(result['iou'], camvid_label_names): print('{:>23} : {:.4f}'.format(label_name, iu)) print('=' * 34) print('{:>23} : {:.4f}'.format('mean IoU', result['miou'])) print('{:>23} : {:.4f}'.format('Class average accuracy', result['mean_class_accuracy'])) print('{:>23} : {:.4f}'.format('Global average accuracy', result['pixel_accuracy']))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--dataset', choices=('cityscapes', 'ade20k', 'camvid')) parser.add_argument('--model', choices=('pspnet_resnet101', 'segnet')) parser.add_argument('--gpu', type=int, default=-1) parser.add_argument('--pretrained-model') parser.add_argument('--input-size', type=int, default=None) args = parser.parse_args() dataset, label_names, model = get_dataset_and_model( args.dataset, args.model, args.pretrained_model, (args.input_size, args.input_size)) if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() it = iterators.SerialIterator(dataset, 1, repeat=False, shuffle=False) in_values, out_values, rest_values = apply_to_iterator(model.predict, it, hook=ProgressHook( len(dataset))) # Delete an iterator of images to save memory usage. del in_values pred_labels, = out_values gt_labels, = rest_values result = eval_semantic_segmentation(pred_labels, gt_labels) for iu, label_name in zip(result['iou'], label_names): print('{:>23} : {:.4f}'.format(label_name, iu)) print('=' * 34) print('{:>23} : {:.4f}'.format('mean IoU', result['miou'])) print('{:>23} : {:.4f}'.format('Class average accuracy', result['mean_class_accuracy'])) print('{:>23} : {:.4f}'.format('Global average accuracy', result['pixel_accuracy']))
def evaluate(self): iterator = self._iterators['main'] target = self._targets['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) in_values, out_values, rest_values = apply_to_iterator( target.predict, it) # delete unused iterators explicitly del in_values pred_labels, = out_values gt_labels, = rest_values result = eval_semantic_segmentation(pred_labels, gt_labels) report = {'miou': result['miou'], 'pixel_accuracy': result['pixel_accuracy'], 'mean_class_accuracy': result['mean_class_accuracy']} if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['iou/{:s}'.format(label_name)] = result['iou'][l] report['class_accuracy/{:s}'.format(label_name)] =\ result['class_accuracy'][l] except IndexError: report['iou/{:s}'.format(label_name)] = np.nan report['class_accuracy/{:s}'.format(label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def evaluate(self): iterator = self._iterators['main'] target = self._targets['main'] if hasattr(target, 'prediction_detection'): target.prediction_detection = self.detection if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) in_values, out_values, rest_values = apply_to_iterator( target.predict, it) del in_values if self.detection: # TODO: evaluate the detection result pred_bboxes, pred_labels, pred_scores = out_values if len(rest_values) == 3: gt_bboxes, gt_labels, gt_difficults = rest_values elif len(rest_values) == 2: gt_bboxes, gt_labels = rest_values gt_difficults = None result = eval_detection_voc(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_difficults, use_07_metric=self.use_07_metric) report = {'map': result['map']} if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['ap/{:s}'.format(label_name)] = result['ap'][l] except IndexError: report['ap/{:s}'.format(label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation else: # TODO: evaluate the segmentation result pred_labels, = out_values gt_labels, = rest_values result = eval_semantic_segmentation(pred_labels, gt_labels) report = { 'miou': result['miou'], 'pixel_accuracy': result['pixel_accuracy'], 'mean_class_accuracy': result['mean_class_accuracy'] } if self.label_names is not None: for l, label_name in enumerate(self.label_names): try: report['iou/{:s}'.format( label_name)] = result['iou'][l] report['class_accuracy/{:s}'.format(label_name)] = \ result['class_accuracy'][l] except IndexError: report['iou/{:s}'.format(label_name)] = np.nan report['class_accuracy/{:s}'.format( label_name)] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def evaluate(self): iterator = self._iterators['main'] model = self._targets['main'] eval_func = self.eval_func or model if self.eval_hook: self.eval_hook(self) if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) summary = reporter_module.DictSummary() labels_all = [] preds_all = [] for batch in it: observation = {} with reporter_module.report_scope(observation): in_arrays = self.converter(batch, self.device) with function.no_backprop_mode(): if isinstance(in_arrays, tuple): eval_func(*in_arrays) elif isinstance(in_arrays, dict): eval_func(**in_arrays) else: eval_func(in_arrays) _, labels = in_arrays if self.device_id >= 0: labels = chainer.cuda.to_cpu(labels) # Exclude pixels with ignore value from the evaluation labels[labels == self.ignore_value] = -1 y = model.y.data if self.device_id >= 0: y = chainer.cuda.to_cpu(y) preds = y.argmax(axis=1) for label, pred in zip(labels, preds): labels_all.append(label) preds_all.append(pred) # print(observation) summary.add(observation) ss_eval = eval_semantic_segmentation(preds_all, labels_all) iou = ss_eval['iou'][1:] # Assuming label '0' is assigned for background iou_observation = {} iou_observation['iou'] = np.nanmean(iou) for i, label_name in enumerate(self.label_names): iou_observation['iou/{:s}'.format(label_name)] = iou[i] summary.add(iou_observation) return summary.compute_mean()