def test_load_best_individual_model(metric, backend, dask_client):
X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
automl = autosklearn.automl.AutoML(
backend=backend,
time_left_for_this_task=30,
per_run_time_limit=5,
metric=metric,
dask_client=dask_client,
)
# We cannot easily mock a function sent to dask
# so for this test we create the whole set of models/ensembles
# but prevent it to be loaded
automl.fit(
X_train,
Y_train,
task=MULTICLASS_CLASSIFICATION,
)
automl._backend.load_ensemble = unittest.mock.MagicMock(return_value=None)
# A memory error occurs in the ensemble construction
assert automl._backend.load_ensemble(automl._seed) is None
# The load model is robust to this and loads the best model
automl._load_models()
assert automl.ensemble_ is not None
# Just 1 model is there for ensemble and all weight must be on it
get_models_with_weights = automl.get_models_with_weights()
assert len(get_models_with_weights) == 1
assert get_models_with_weights[0][0] == 1.0
# Match a toy dataset
if metric.name == 'balanced_accuracy':
assert automl.score(X_test, Y_test) > 0.9
elif metric.name == 'log_loss':
# Seen values in github actions of 0.6978304740364537
assert automl.score(X_test, Y_test) < 0.7
else:
raise ValueError(metric.name)
del automl
def test_load_best_individual_model(self):
backend_api = self._create_backend('test_fit')
for metric in [log_loss, balanced_accuracy]:
X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
automl = autosklearn.automl.AutoML(
backend=backend_api,
time_left_for_this_task=20,
per_run_time_limit=5,
metric=metric,
)
with unittest.mock.patch(
'autosklearn.ensemble_builder.EnsembleBuilder.run'
) as mock_ensemble_run:
mock_ensemble_run.side_effect = MemoryError
automl.fit(
X_train, Y_train, task=MULTICLASS_CLASSIFICATION,
)
# A memory error occurs in the ensemble construction
self.assertIsNone(automl._backend.load_ensemble(automl._seed))
# The load model is robust to this and loads the best model
automl._load_models()
self.assertIsNotNone(automl.ensemble_)
# Just 1 model is there for ensemble and all weight must be on it
get_models_with_weights = automl.get_models_with_weights()
self.assertEqual(len(get_models_with_weights), 1)
self.assertEqual(get_models_with_weights[0][0], 1.0)
# Match a toy dataset
if metric._sign < 0:
self.assertLessEqual(automl.score(X_test, Y_test), 0.2)
else:
self.assertGreaterEqual(automl.score(X_test, Y_test), 0.8)
del automl
self._tearDown(backend_api.temporary_directory)
self._tearDown(backend_api.output_directory)