def test_validate_before_predict(self):
# Test invalid inputs
neural_network_repurposer = NeuralNetworkRepurposer(
source_model=self.mxnet_model)
# Target model is neither created through repurpose nor explicitly assigned
self.assertRaisesRegex(
TypeError,
"Cannot predict because target_model is not an `mxnet.mod.Module` object",
neural_network_repurposer._validate_before_predict)
neural_network_repurposer.target_model = {}
self.assertRaisesRegex(
TypeError,
"Cannot predict because target_model is not an `mxnet.mod.Module` object",
neural_network_repurposer._validate_before_predict)
# Assert validate raises error for mxnet module that is not trained yet
neural_network_repurposer.target_model = self.mxnet_model
self.assertRaisesRegex(
ValueError,
"target_model params aren't initialized. Ensure model is trained before calling predict",
neural_network_repurposer._validate_before_predict)
# Test valid input
neural_network_repurposer.target_model = mx.module.Module.load(
prefix=self.dropout_model_path_prefix, epoch=0)
neural_network_repurposer._validate_before_predict()
def _test_predict(self, mock_validate_method, test_predict_probability):
neural_network_repurposer = NeuralNetworkRepurposer(source_model=None)
neural_network_repurposer.target_model = mx.module.Module.load(
prefix=RepurposerTestUtils.MNIST_MODEL_PATH_PREFIX, epoch=10)
test_iterator = RepurposerTestUtils.create_mnist_test_iterator()
mock_validate_method.reset_mock()
if test_predict_probability:
labels = np.argmax(
neural_network_repurposer.predict_probability(test_iterator),
axis=1)
else:
labels = neural_network_repurposer.predict_label(test_iterator)
# Check if predict called validate
self.assertTrue(
mock_validate_method.call_count == 1,
"Predict expected to called {} once. Found {} calls".format(
RepurposerTestUtils.VALIDATE_PREDICT_METHOD_NAME,
mock_validate_method.call_count))
expected_accuracy = 0.96985
accuracy = np.mean(
labels == RepurposerTestUtils.get_labels(test_iterator))
self.assertTrue(
np.isclose(accuracy, expected_accuracy, rtol=1e-3),
"Prediction accuracy is incorrect. Expected:{}. Got:{}".format(
expected_accuracy, accuracy))
def test_repurpose_calls_validate(self, mock_create_target_module,
mock_validate_method):
neural_network_repurposer = NeuralNetworkRepurposer(
source_model=self.mxnet_model)
neural_network_repurposer.target_model = mx.module.Module.load(
prefix=self.dropout_model_path_prefix, epoch=0)
mock_validate_method.reset_mock()
neural_network_repurposer.repurpose(Mock())
self.assertTrue(
mock_validate_method.call_count == 1,
"Repurpose expected to called {} once. Found {} calls".format(
RepurposerTestUtils.VALIDATE_REPURPOSE_METHOD_NAME,
mock_validate_method.call_count))
def test_prediction_consistency(self):
""" Test if predict method returns consistent predictions using the same model and test data """
if self.repurposer_class != NeuralNetworkRepurposer:
return
# Create test data iterator to run predictions on
test_iterator = RepurposerTestUtils.get_image_iterator()
# Load a pre-trained model to predict. The model has a dropout layer used for training.
# This test is to ensure that dropout doesn't happen during prediction.
target_model = mx.module.Module.load(
prefix=self.dropout_model_path_prefix, epoch=0, label_names=None)
# Create repurposer and set the target model loaded from file
repurposer = NeuralNetworkRepurposer(source_model=None)
repurposer.target_model = target_model
# Ensure prediction results are consistent
self._predict_and_compare_results(repurposer,
test_iterator,
test_predict_probability=True)
self._predict_and_compare_results(repurposer,
test_iterator,
test_predict_probability=False)