def test_finish_up_model_predicts_NaN(self):
'''Tests by handing in predictions which contain NaNs'''
rs = np.random.RandomState(1)
D = get_multiclass_classification_datamanager()
backend_api = unittest.mock.Mock()
backend_api.load_datamanager.return_value = D
queue_mock = unittest.mock.Mock()
ae = AbstractEvaluator(backend=backend_api,
output_y_hat_optimization=False,
queue=queue_mock,
metric=accuracy)
ae.Y_optimization = rs.rand(33, 3)
predictions_ensemble = rs.rand(33, 3)
predictions_test = rs.rand(25, 3)
predictions_valid = rs.rand(25, 3)
# NaNs in prediction ensemble
predictions_ensemble[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(0.1,
predictions_ensemble,
predictions_valid,
predictions_test)
self.assertEqual(loss, 1.0)
self.assertEqual(
additional_run_info, {
'error': 'Model predictions for optimization set '
'contains NaNs.'
})
# NaNs in prediction validation
predictions_ensemble[5, 2] = 0.5
predictions_valid[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(0.1,
predictions_ensemble,
predictions_valid,
predictions_test)
self.assertEqual(loss, 1.0)
self.assertEqual(
additional_run_info, {
'error': 'Model predictions for validation set '
'contains NaNs.'
})
# NaNs in prediction test
predictions_valid[5, 2] = 0.5
predictions_test[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(0.1,
predictions_ensemble,
predictions_valid,
predictions_test)
self.assertEqual(loss, 1.0)
self.assertEqual(
additional_run_info,
{'error': 'Model predictions for test set contains '
'NaNs.'})
self.assertEqual(backend_api.save_predictions_as_npy.call_count, 0)
def test_finish_up_model_predicts_NaN(self):
'''Tests by handing in predictions which contain NaNs'''
rs = np.random.RandomState(1)
queue_mock = unittest.mock.Mock()
ae = AbstractEvaluator(backend=self.backend_mock,
port=self.port,
output_y_hat_optimization=False,
queue=queue_mock,
metric=accuracy)
ae.Y_optimization = rs.rand(33, 3)
predictions_ensemble = rs.rand(33, 3)
predictions_test = rs.rand(25, 3)
predictions_valid = rs.rand(25, 3)
# NaNs in prediction ensemble
predictions_ensemble[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(
loss=0.1,
train_loss=0.1,
opt_pred=predictions_ensemble,
valid_pred=predictions_valid,
test_pred=predictions_test,
additional_run_info=None,
final_call=True,
file_output=True,
status=StatusType.SUCCESS,
)
self.assertEqual(loss, 1.0)
self.assertEqual(
additional_run_info, {
'error': 'Model predictions for optimization set '
'contains NaNs.'
})
# NaNs in prediction validation
predictions_ensemble[5, 2] = 0.5
predictions_valid[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(
loss=0.1,
train_loss=0.1,
opt_pred=predictions_ensemble,
valid_pred=predictions_valid,
test_pred=predictions_test,
additional_run_info=None,
final_call=True,
file_output=True,
status=StatusType.SUCCESS,
)
self.assertEqual(loss, 1.0)
self.assertEqual(
additional_run_info, {
'error': 'Model predictions for validation set '
'contains NaNs.'
})
# NaNs in prediction test
predictions_valid[5, 2] = 0.5
predictions_test[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(
loss=0.1,
train_loss=0.1,
opt_pred=predictions_ensemble,
valid_pred=predictions_valid,
test_pred=predictions_test,
additional_run_info=None,
final_call=True,
file_output=True,
status=StatusType.SUCCESS,
)
self.assertEqual(loss, 1.0)
self.assertEqual(
additional_run_info,
{'error': 'Model predictions for test set contains '
'NaNs.'})
self.assertEqual(self.backend_mock.save_predictions_as_npy.call_count,
0)
def test_finish_up_model_predicts_NaN(self):
'''Tests by handing in predictions which contain NaNs'''
rs = np.random.RandomState(1)
D = get_multiclass_classification_datamanager()
backend_api = unittest.mock.Mock()
backend_api.load_datamanager.return_value = D
queue_mock = unittest.mock.Mock()
ae = AbstractEvaluator(backend=backend_api,
output_y_hat_optimization=False,
queue=queue_mock, metric=accuracy)
ae.Y_optimization = rs.rand(33, 3)
predictions_train = rs.rand(66, 3)
predictions_ensemble = rs.rand(33, 3)
predictions_test = rs.rand(25, 3)
predictions_valid = rs.rand(25, 3)
# NaNs in prediction ensemble
predictions_ensemble[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(
loss=0.1,
train_pred=predictions_train,
opt_pred=predictions_ensemble,
valid_pred=predictions_valid,
test_pred=predictions_test,
additional_run_info=None,
final_call=True,
file_output=True,
)
self.assertEqual(loss, 1.0)
self.assertEqual(additional_run_info,
{'error': 'Model predictions for optimization set '
'contains NaNs.'})
# NaNs in prediction validation
predictions_ensemble[5, 2] = 0.5
predictions_valid[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(
loss=0.1,
train_pred=predictions_train,
opt_pred=predictions_ensemble,
valid_pred=predictions_valid,
test_pred=predictions_test,
additional_run_info=None,
final_call=True,
file_output=True,
)
self.assertEqual(loss, 1.0)
self.assertEqual(additional_run_info,
{'error': 'Model predictions for validation set '
'contains NaNs.'})
# NaNs in prediction test
predictions_valid[5, 2] = 0.5
predictions_test[5, 2] = np.NaN
_, loss, _, additional_run_info = ae.finish_up(
loss=0.1,
train_pred=predictions_train,
opt_pred=predictions_ensemble,
valid_pred=predictions_valid,
test_pred=predictions_test,
additional_run_info=None,
final_call=True,
file_output=True,
)
self.assertEqual(loss, 1.0)
self.assertEqual(additional_run_info,
{'error': 'Model predictions for test set contains '
'NaNs.'})
self.assertEqual(backend_api.save_predictions_as_npy.call_count, 0)