def test_precision_recall(backend_default): be = NervanaObject.be be.bsz = 4 preds = np.array([[0, 1, 0, 1], [1, 0, 0, 0], [0, 0, 1, 0]]) targets = np.array([[0, 1, 0, 1], [1, 0, 1, 0], [0, 0, 0, 0]]) expected_result = np.array([1 + 1 + 0, 1 + 0.5 + 0]) / 3. compare_metric(PrecisionRecall(3), preds, targets, expected_result, tol=1e-6)
def test_precision_recall_binarize(backend_default): be = NervanaObject.be be.bsz = 4 preds = np.array([[0.2, 0.9, 0.01, 1], [0.75, 0.05, 0.44, 0], [0.05, 0.05, 0.55, 0]]) targets = np.array([[0, 1, 0, 1], [1, 0, 1, 0], [0, 0, 0, 0]]) expected_result = np.array([1 + 1 + 0, 1 + 0.5 + 0]) / 3. compare_metric(PrecisionRecall(3, binarize=True), preds, targets, expected_result, tol=1e-6)
def eval(self, test_set): """ Evaluate the model's test_set on error_rate, test_accuracy_rate and precision_recall_rate Args: test_set (ArrayIterator): The test set Returns: tuple(int): error_rate, test_accuracy_rate and precision_recall_rate """ error_rate = self.model.eval(test_set, metric=Misclassification()) test_accuracy_rate = self.model.eval(test_set, metric=Accuracy()) precision_recall_rate = self.model.eval(test_set, metric=PrecisionRecall(2)) return error_rate, test_accuracy_rate, precision_recall_rate
lunaModel = Model(layers=vgg_layers) if args.model_file: import os assert os.path.exists(args.model_file), '%s not found' % args.model_file lunaModel.load_params(args.model_file) # configure callbacks #callbacks = Callbacks(lunaModel, eval_set=valid_set, **args.callback_args) callbacks = Callbacks(lunaModel, eval_set=valid_set, metric=Misclassification(), **args.callback_args) if args.deconv: callbacks.add_deconv_callback(train_set, valid_set) lunaModel.fit(train_set, optimizer=opt, num_epochs=num_epochs, cost=cost, callbacks=callbacks) lunaModel.save_params('LUNA16_VGG_model.prm') neon_logger.display('Finished training. Calculating error on the validation set...') neon_logger.display('Misclassification error (validation) = {:.2f}%'.format(lunaModel.eval(valid_set, metric=Misclassification())[0] * 100)) neon_logger.display('Precision/recall (validation) = {}'.format(lunaModel.eval(valid_set, metric=PrecisionRecall(num_classes=2)))) neon_logger.display('Calculating metrics on the test set. This could take a while...') neon_logger.display('Misclassification error (test) = {:.2f}%'.format(lunaModel.eval(test_set, metric=Misclassification())[0] * 100)) neon_logger.display('Precision/recall (test) = {}'.format(lunaModel.eval(test_set, metric=PrecisionRecall(num_classes=2))))
lunaModel.load_params(args.model_file) # configure callbacks if args.callback_args['eval_freq'] is None: args.callback_args['eval_freq'] = 1 # configure callbacks callbacks = Callbacks(lunaModel, eval_set=valid_set, **args.callback_args) # add a callback that saves the best model state callbacks.add_save_best_state_callback( 'LUNA16_VGG_model_no_batch_sigmoid_pretrained.prm') if args.deconv: callbacks.add_deconv_callback(train_set, valid_set) lunaModel.fit(train_set, optimizer=opt, num_epochs=num_epochs, cost=cost, callbacks=callbacks) lunaModel.save_params('LUNA16_VGG_model_no_batch_sigmoid_pretrained.prm') neon_logger.display( 'Calculating metrics on the test set. This could take a while...') neon_logger.display('Misclassification error (test) = {:.2f}%'.format( lunaModel.eval(test_set, metric=Misclassification())[0] * 100)) neon_logger.display('Precision/recall (test) = {}'.format( lunaModel.eval(test_set, metric=PrecisionRecall(num_classes=2))))