def test_precision_recall(backend_default):
be = NervanaObject.be
be.bsz = 4
preds = np.array([[0, 1, 0, 1], [1, 0, 0, 0], [0, 0, 1, 0]])
targets = np.array([[0, 1, 0, 1], [1, 0, 1, 0], [0, 0, 0, 0]])
expected_result = np.array([1 + 1 + 0, 1 + 0.5 + 0]) / 3.
compare_metric(PrecisionRecall(3), preds, targets, expected_result, tol=1e-6)
def test_precision_recall_binarize(backend_default):
be = NervanaObject.be
be.bsz = 4
preds = np.array([[0.2, 0.9, 0.01, 1],
[0.75, 0.05, 0.44, 0],
[0.05, 0.05, 0.55, 0]])
targets = np.array([[0, 1, 0, 1],
[1, 0, 1, 0],
[0, 0, 0, 0]])
expected_result = np.array([1 + 1 + 0, 1 + 0.5 + 0]) / 3.
compare_metric(PrecisionRecall(3, binarize=True), preds, targets,
expected_result, tol=1e-6)
def eval(self, test_set):
"""
Evaluate the model's test_set on error_rate, test_accuracy_rate and precision_recall_rate
Args:
test_set (ArrayIterator): The test set
Returns:
tuple(int): error_rate, test_accuracy_rate and precision_recall_rate
"""
error_rate = self.model.eval(test_set, metric=Misclassification())
test_accuracy_rate = self.model.eval(test_set, metric=Accuracy())
precision_recall_rate = self.model.eval(test_set,
metric=PrecisionRecall(2))
return error_rate, test_accuracy_rate, precision_recall_rate
lunaModel = Model(layers=vgg_layers)
if args.model_file:
import os
assert os.path.exists(args.model_file), '%s not found' % args.model_file
lunaModel.load_params(args.model_file)
# configure callbacks
#callbacks = Callbacks(lunaModel, eval_set=valid_set, **args.callback_args)
callbacks = Callbacks(lunaModel, eval_set=valid_set, metric=Misclassification(), **args.callback_args)
if args.deconv:
callbacks.add_deconv_callback(train_set, valid_set)
lunaModel.fit(train_set, optimizer=opt, num_epochs=num_epochs,
cost=cost, callbacks=callbacks)
lunaModel.save_params('LUNA16_VGG_model.prm')
neon_logger.display('Finished training. Calculating error on the validation set...')
neon_logger.display('Misclassification error (validation) = {:.2f}%'.format(lunaModel.eval(valid_set, metric=Misclassification())[0] * 100))
neon_logger.display('Precision/recall (validation) = {}'.format(lunaModel.eval(valid_set, metric=PrecisionRecall(num_classes=2))))
neon_logger.display('Calculating metrics on the test set. This could take a while...')
neon_logger.display('Misclassification error (test) = {:.2f}%'.format(lunaModel.eval(test_set, metric=Misclassification())[0] * 100))
neon_logger.display('Precision/recall (test) = {}'.format(lunaModel.eval(test_set, metric=PrecisionRecall(num_classes=2))))
lunaModel.load_params(args.model_file)
# configure callbacks
if args.callback_args['eval_freq'] is None:
args.callback_args['eval_freq'] = 1
# configure callbacks
callbacks = Callbacks(lunaModel, eval_set=valid_set, **args.callback_args)
# add a callback that saves the best model state
callbacks.add_save_best_state_callback(
'LUNA16_VGG_model_no_batch_sigmoid_pretrained.prm')
if args.deconv:
callbacks.add_deconv_callback(train_set, valid_set)
lunaModel.fit(train_set,
optimizer=opt,
num_epochs=num_epochs,
cost=cost,
callbacks=callbacks)
lunaModel.save_params('LUNA16_VGG_model_no_batch_sigmoid_pretrained.prm')
neon_logger.display(
'Calculating metrics on the test set. This could take a while...')
neon_logger.display('Misclassification error (test) = {:.2f}%'.format(
lunaModel.eval(test_set, metric=Misclassification())[0] * 100))
neon_logger.display('Precision/recall (test) = {}'.format(
lunaModel.eval(test_set, metric=PrecisionRecall(num_classes=2))))
