コード例 #1
0
ファイル: test_automl.py プロジェクト: PGijsbers/auto-sklearn
def test_load_best_individual_model(metric, backend, dask_client):

    X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
    automl = autosklearn.automl.AutoML(
        backend=backend,
        time_left_for_this_task=30,
        per_run_time_limit=5,
        metric=metric,
        dask_client=dask_client,
    )

    # We cannot easily mock a function sent to dask
    # so for this test we create the whole set of models/ensembles
    # but prevent it to be loaded
    automl.fit(
        X_train,
        Y_train,
        task=MULTICLASS_CLASSIFICATION,
    )
    automl._backend.load_ensemble = unittest.mock.MagicMock(return_value=None)

    # A memory error occurs in the ensemble construction
    assert automl._backend.load_ensemble(automl._seed) is None

    # The load model is robust to this and loads the best model
    automl._load_models()
    assert automl.ensemble_ is not None

    # Just 1 model is there for ensemble and all weight must be on it
    get_models_with_weights = automl.get_models_with_weights()
    assert len(get_models_with_weights) == 1
    assert get_models_with_weights[0][0] == 1.0

    # Match a toy dataset
    if metric.name == 'balanced_accuracy':
        assert automl.score(X_test, Y_test) > 0.9
    elif metric.name == 'log_loss':
        # Seen values in github actions of 0.6978304740364537
        assert automl.score(X_test, Y_test) < 0.7
    else:
        raise ValueError(metric.name)

    del automl
コード例 #2
0
ファイル: test_automl.py プロジェクト: rovedream/auto-sklearn
    def test_load_best_individual_model(self):

        backend_api = self._create_backend('test_fit')

        for metric in [log_loss, balanced_accuracy]:
            X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
            automl = autosklearn.automl.AutoML(
                backend=backend_api,
                time_left_for_this_task=20,
                per_run_time_limit=5,
                metric=metric,
            )

            with unittest.mock.patch(
                'autosklearn.ensemble_builder.EnsembleBuilder.run'
            ) as mock_ensemble_run:
                mock_ensemble_run.side_effect = MemoryError
                automl.fit(
                    X_train, Y_train, task=MULTICLASS_CLASSIFICATION,
                )

            # A memory error occurs in the ensemble construction
            self.assertIsNone(automl._backend.load_ensemble(automl._seed))

            # The load model is robust to this and loads the best model
            automl._load_models()
            self.assertIsNotNone(automl.ensemble_)

            # Just 1 model is there for ensemble and all weight must be on it
            get_models_with_weights = automl.get_models_with_weights()
            self.assertEqual(len(get_models_with_weights), 1)
            self.assertEqual(get_models_with_weights[0][0], 1.0)

            # Match a toy dataset
            if metric._sign < 0:
                self.assertLessEqual(automl.score(X_test, Y_test), 0.2)
            else:
                self.assertGreaterEqual(automl.score(X_test, Y_test), 0.8)

            del automl
            self._tearDown(backend_api.temporary_directory)
            self._tearDown(backend_api.output_directory)