def test_refit_shuffle_on_fail(self): backend_api = self._create_backend('test_refit_shuffle_on_fail') failing_model = unittest.mock.Mock() failing_model.fit.side_effect = [ValueError(), ValueError(), None] failing_model.fit_transformer.side_effect = [ ValueError(), ValueError(), (None, {})] failing_model.get_max_iter.return_value = 100 auto = AutoML(backend_api, 20, 5) ensemble_mock = unittest.mock.Mock() ensemble_mock.get_selected_model_identifiers.return_value = [(1, 1, 50.0)] auto.ensemble_ = ensemble_mock for budget_type in [None, 'iterations']: auto._budget_type = budget_type auto.models_ = {(1, 1, 50.0): failing_model} # Make sure a valid 2D array is given to automl X = np.array([1, 2, 3]).reshape(-1, 1) y = np.array([1, 2, 3]) auto.refit(X, y) self.assertEqual(failing_model.fit.call_count, 3) self.assertEqual(failing_model.fit_transformer.call_count, 3) del auto self._tearDown(backend_api.temporary_directory) self._tearDown(backend_api.output_directory)
def test_refit_shuffle_on_fail(backend, dask_client): failing_model = unittest.mock.Mock() failing_model.fit.side_effect = [ValueError(), ValueError(), None] failing_model.fit_transformer.side_effect = [ ValueError(), ValueError(), (None, {})] failing_model.get_max_iter.return_value = 100 auto = AutoML(backend, 30, 5, dask_client=dask_client) ensemble_mock = unittest.mock.Mock() ensemble_mock.get_selected_model_identifiers.return_value = [(1, 1, 50.0)] auto.ensemble_ = ensemble_mock auto.InputValidator = InputValidator() for budget_type in [None, 'iterations']: auto._budget_type = budget_type auto.models_ = {(1, 1, 50.0): failing_model} # Make sure a valid 2D array is given to automl X = np.array([1, 2, 3]).reshape(-1, 1) y = np.array([1, 2, 3]) auto.InputValidator.fit(X, y) auto.refit(X, y) assert failing_model.fit.call_count == 3 assert failing_model.fit_transformer.call_count == 3 del auto
def test_smbo_metalearning_configurations(backend, context, dask_client): # Get the inputs to the optimizer X_train, Y_train, X_test, Y_test = putil.get_dataset('iris') config_space = AutoML(backend=backend, metric=autosklearn.metrics.accuracy, time_left_for_this_task=20, per_run_time_limit=5).fit( X_train, Y_train, task=BINARY_CLASSIFICATION, only_return_configuration_space=True) watcher = StopWatch() # Create an optimizer smbo = AutoMLSMBO( config_space=config_space, dataset_name='iris', backend=backend, total_walltime_limit=10, func_eval_time_limit=5, memory_limit=4096, metric=autosklearn.metrics.accuracy, watcher=watcher, n_jobs=1, dask_client=dask_client, port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, start_num_run=1, data_memory_limit=None, num_metalearning_cfgs=25, pynisher_context=context, ) assert smbo.pynisher_context == context # Create the inputs to metalearning datamanager = XYDataManager( X_train, Y_train, X_test, Y_test, task=BINARY_CLASSIFICATION, dataset_name='iris', feat_type={i: 'numerical' for i in range(X_train.shape[1])}, ) backend.save_datamanager(datamanager) smbo.task = BINARY_CLASSIFICATION smbo.reset_data_manager() metalearning_configurations = smbo.get_metalearning_suggestions() # We should have 25 metalearning configurations assert len(metalearning_configurations) == 25 assert [ isinstance(config, Configuration) for config in metalearning_configurations ]
def test_refit_shuffle_on_fail(self): output = os.path.join(self.test_dir, '..', '.tmp_refit_shuffle_on_fail') context = BackendContext(output, output, False, False) backend = Backend(context) failing_model = unittest.mock.Mock() failing_model.fit.side_effect = [ValueError(), ValueError(), None] auto = AutoML(backend, 20, 5) ensemble_mock = unittest.mock.Mock() auto.ensemble_ = ensemble_mock ensemble_mock.get_model_identifiers.return_value = [1] auto.models_ = {1: failing_model} X = np.array([1, 2, 3]) y = np.array([1, 2, 3]) auto.refit(X, y) self.assertEqual(failing_model.fit.call_count, 3)
def test_refit_shuffle_on_fail(self): backend_api = self._create_backend('test_refit_shuffle_on_fail') failing_model = unittest.mock.Mock() failing_model.fit.side_effect = [ValueError(), ValueError(), None] auto = AutoML(backend_api, 20, 5) ensemble_mock = unittest.mock.Mock() auto.ensemble_ = ensemble_mock ensemble_mock.get_selected_model_identifiers.return_value = [1] auto.models_ = {1: failing_model} X = np.array([1, 2, 3]) y = np.array([1, 2, 3]) auto.refit(X, y) self.assertEqual(failing_model.fit.call_count, 3) del auto self._tearDown(backend_api.temporary_directory) self._tearDown(backend_api.output_directory)